cachepc-linux

Fork of AMDESE/linux with modifications for CachePC side-channel attack
git clone https://git.sinitax.com/sinitax/cachepc-linux
Log | Files | Refs | README | LICENSE | sfeed.txt

radeon_clocks.c (26886B)


      1/*
      2 * Copyright 2008 Advanced Micro Devices, Inc.
      3 * Copyright 2008 Red Hat Inc.
      4 * Copyright 2009 Jerome Glisse.
      5 *
      6 * Permission is hereby granted, free of charge, to any person obtaining a
      7 * copy of this software and associated documentation files (the "Software"),
      8 * to deal in the Software without restriction, including without limitation
      9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
     10 * and/or sell copies of the Software, and to permit persons to whom the
     11 * Software is furnished to do so, subject to the following conditions:
     12 *
     13 * The above copyright notice and this permission notice shall be included in
     14 * all copies or substantial portions of the Software.
     15 *
     16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
     19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
     20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     22 * OTHER DEALINGS IN THE SOFTWARE.
     23 *
     24 * Authors: Dave Airlie
     25 *          Alex Deucher
     26 *          Jerome Glisse
     27 */
     28
     29#include <linux/pci.h>
     30
     31#include <drm/drm_device.h>
     32#include <drm/radeon_drm.h>
     33
     34#include "atom.h"
     35#include "radeon.h"
     36#include "radeon_asic.h"
     37#include "radeon_reg.h"
     38
     39/* 10 khz */
     40uint32_t radeon_legacy_get_engine_clock(struct radeon_device *rdev)
     41{
     42	struct radeon_pll *spll = &rdev->clock.spll;
     43	uint32_t fb_div, ref_div, post_div, sclk;
     44
     45	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
     46	fb_div = (fb_div >> RADEON_SPLL_FB_DIV_SHIFT) & RADEON_SPLL_FB_DIV_MASK;
     47	fb_div <<= 1;
     48	fb_div *= spll->reference_freq;
     49
     50	ref_div =
     51	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
     52
     53	if (ref_div == 0)
     54		return 0;
     55
     56	sclk = fb_div / ref_div;
     57
     58	post_div = RREG32_PLL(RADEON_SCLK_CNTL) & RADEON_SCLK_SRC_SEL_MASK;
     59	if (post_div == 2)
     60		sclk >>= 1;
     61	else if (post_div == 3)
     62		sclk >>= 2;
     63	else if (post_div == 4)
     64		sclk >>= 3;
     65
     66	return sclk;
     67}
     68
     69/* 10 khz */
     70uint32_t radeon_legacy_get_memory_clock(struct radeon_device *rdev)
     71{
     72	struct radeon_pll *mpll = &rdev->clock.mpll;
     73	uint32_t fb_div, ref_div, post_div, mclk;
     74
     75	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
     76	fb_div = (fb_div >> RADEON_MPLL_FB_DIV_SHIFT) & RADEON_MPLL_FB_DIV_MASK;
     77	fb_div <<= 1;
     78	fb_div *= mpll->reference_freq;
     79
     80	ref_div =
     81	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
     82
     83	if (ref_div == 0)
     84		return 0;
     85
     86	mclk = fb_div / ref_div;
     87
     88	post_div = RREG32_PLL(RADEON_MCLK_CNTL) & 0x7;
     89	if (post_div == 2)
     90		mclk >>= 1;
     91	else if (post_div == 3)
     92		mclk >>= 2;
     93	else if (post_div == 4)
     94		mclk >>= 3;
     95
     96	return mclk;
     97}
     98
     99#ifdef CONFIG_OF
    100/*
    101 * Read XTAL (ref clock), SCLK and MCLK from Open Firmware device
    102 * tree. Hopefully, ATI OF driver is kind enough to fill these
    103 */
    104static bool radeon_read_clocks_OF(struct drm_device *dev)
    105{
    106	struct radeon_device *rdev = dev->dev_private;
    107	struct device_node *dp = rdev->pdev->dev.of_node;
    108	const u32 *val;
    109	struct radeon_pll *p1pll = &rdev->clock.p1pll;
    110	struct radeon_pll *p2pll = &rdev->clock.p2pll;
    111	struct radeon_pll *spll = &rdev->clock.spll;
    112	struct radeon_pll *mpll = &rdev->clock.mpll;
    113
    114	if (dp == NULL)
    115		return false;
    116	val = of_get_property(dp, "ATY,RefCLK", NULL);
    117	if (!val || !*val) {
    118		pr_warn("radeonfb: No ATY,RefCLK property !\n");
    119		return false;
    120	}
    121	p1pll->reference_freq = p2pll->reference_freq = (*val) / 10;
    122	p1pll->reference_div = RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
    123	if (p1pll->reference_div < 2)
    124		p1pll->reference_div = 12;
    125	p2pll->reference_div = p1pll->reference_div;
    126
    127	/* These aren't in the device-tree */
    128	if (rdev->family >= CHIP_R420) {
    129		p1pll->pll_in_min = 100;
    130		p1pll->pll_in_max = 1350;
    131		p1pll->pll_out_min = 20000;
    132		p1pll->pll_out_max = 50000;
    133		p2pll->pll_in_min = 100;
    134		p2pll->pll_in_max = 1350;
    135		p2pll->pll_out_min = 20000;
    136		p2pll->pll_out_max = 50000;
    137	} else {
    138		p1pll->pll_in_min = 40;
    139		p1pll->pll_in_max = 500;
    140		p1pll->pll_out_min = 12500;
    141		p1pll->pll_out_max = 35000;
    142		p2pll->pll_in_min = 40;
    143		p2pll->pll_in_max = 500;
    144		p2pll->pll_out_min = 12500;
    145		p2pll->pll_out_max = 35000;
    146	}
    147	/* not sure what the max should be in all cases */
    148	rdev->clock.max_pixel_clock = 35000;
    149
    150	spll->reference_freq = mpll->reference_freq = p1pll->reference_freq;
    151	spll->reference_div = mpll->reference_div =
    152		RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
    153			    RADEON_M_SPLL_REF_DIV_MASK;
    154
    155	val = of_get_property(dp, "ATY,SCLK", NULL);
    156	if (val && *val)
    157		rdev->clock.default_sclk = (*val) / 10;
    158	else
    159		rdev->clock.default_sclk =
    160			radeon_legacy_get_engine_clock(rdev);
    161
    162	val = of_get_property(dp, "ATY,MCLK", NULL);
    163	if (val && *val)
    164		rdev->clock.default_mclk = (*val) / 10;
    165	else
    166		rdev->clock.default_mclk =
    167			radeon_legacy_get_memory_clock(rdev);
    168
    169	DRM_INFO("Using device-tree clock info\n");
    170
    171	return true;
    172}
    173#else
    174static bool radeon_read_clocks_OF(struct drm_device *dev)
    175{
    176	return false;
    177}
    178#endif /* CONFIG_OF */
    179
    180void radeon_get_clock_info(struct drm_device *dev)
    181{
    182	struct radeon_device *rdev = dev->dev_private;
    183	struct radeon_pll *p1pll = &rdev->clock.p1pll;
    184	struct radeon_pll *p2pll = &rdev->clock.p2pll;
    185	struct radeon_pll *dcpll = &rdev->clock.dcpll;
    186	struct radeon_pll *spll = &rdev->clock.spll;
    187	struct radeon_pll *mpll = &rdev->clock.mpll;
    188	int ret;
    189
    190	if (rdev->is_atom_bios)
    191		ret = radeon_atom_get_clock_info(dev);
    192	else
    193		ret = radeon_combios_get_clock_info(dev);
    194	if (!ret)
    195		ret = radeon_read_clocks_OF(dev);
    196
    197	if (ret) {
    198		if (p1pll->reference_div < 2) {
    199			if (!ASIC_IS_AVIVO(rdev)) {
    200				u32 tmp = RREG32_PLL(RADEON_PPLL_REF_DIV);
    201				if (ASIC_IS_R300(rdev))
    202					p1pll->reference_div =
    203						(tmp & R300_PPLL_REF_DIV_ACC_MASK) >> R300_PPLL_REF_DIV_ACC_SHIFT;
    204				else
    205					p1pll->reference_div = tmp & RADEON_PPLL_REF_DIV_MASK;
    206				if (p1pll->reference_div < 2)
    207					p1pll->reference_div = 12;
    208			} else
    209				p1pll->reference_div = 12;
    210		}
    211		if (p2pll->reference_div < 2)
    212			p2pll->reference_div = 12;
    213		if (rdev->family < CHIP_RS600) {
    214			if (spll->reference_div < 2)
    215				spll->reference_div =
    216					RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
    217					RADEON_M_SPLL_REF_DIV_MASK;
    218		}
    219		if (mpll->reference_div < 2)
    220			mpll->reference_div = spll->reference_div;
    221	} else {
    222		if (ASIC_IS_AVIVO(rdev)) {
    223			/* TODO FALLBACK */
    224		} else {
    225			DRM_INFO("Using generic clock info\n");
    226
    227			/* may need to be per card */
    228			rdev->clock.max_pixel_clock = 35000;
    229
    230			if (rdev->flags & RADEON_IS_IGP) {
    231				p1pll->reference_freq = 1432;
    232				p2pll->reference_freq = 1432;
    233				spll->reference_freq = 1432;
    234				mpll->reference_freq = 1432;
    235			} else {
    236				p1pll->reference_freq = 2700;
    237				p2pll->reference_freq = 2700;
    238				spll->reference_freq = 2700;
    239				mpll->reference_freq = 2700;
    240			}
    241			p1pll->reference_div =
    242			    RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
    243			if (p1pll->reference_div < 2)
    244				p1pll->reference_div = 12;
    245			p2pll->reference_div = p1pll->reference_div;
    246
    247			if (rdev->family >= CHIP_R420) {
    248				p1pll->pll_in_min = 100;
    249				p1pll->pll_in_max = 1350;
    250				p1pll->pll_out_min = 20000;
    251				p1pll->pll_out_max = 50000;
    252				p2pll->pll_in_min = 100;
    253				p2pll->pll_in_max = 1350;
    254				p2pll->pll_out_min = 20000;
    255				p2pll->pll_out_max = 50000;
    256			} else {
    257				p1pll->pll_in_min = 40;
    258				p1pll->pll_in_max = 500;
    259				p1pll->pll_out_min = 12500;
    260				p1pll->pll_out_max = 35000;
    261				p2pll->pll_in_min = 40;
    262				p2pll->pll_in_max = 500;
    263				p2pll->pll_out_min = 12500;
    264				p2pll->pll_out_max = 35000;
    265			}
    266
    267			spll->reference_div =
    268			    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
    269			    RADEON_M_SPLL_REF_DIV_MASK;
    270			mpll->reference_div = spll->reference_div;
    271			rdev->clock.default_sclk =
    272			    radeon_legacy_get_engine_clock(rdev);
    273			rdev->clock.default_mclk =
    274			    radeon_legacy_get_memory_clock(rdev);
    275		}
    276	}
    277
    278	/* pixel clocks */
    279	if (ASIC_IS_AVIVO(rdev)) {
    280		p1pll->min_post_div = 2;
    281		p1pll->max_post_div = 0x7f;
    282		p1pll->min_frac_feedback_div = 0;
    283		p1pll->max_frac_feedback_div = 9;
    284		p2pll->min_post_div = 2;
    285		p2pll->max_post_div = 0x7f;
    286		p2pll->min_frac_feedback_div = 0;
    287		p2pll->max_frac_feedback_div = 9;
    288	} else {
    289		p1pll->min_post_div = 1;
    290		p1pll->max_post_div = 16;
    291		p1pll->min_frac_feedback_div = 0;
    292		p1pll->max_frac_feedback_div = 0;
    293		p2pll->min_post_div = 1;
    294		p2pll->max_post_div = 12;
    295		p2pll->min_frac_feedback_div = 0;
    296		p2pll->max_frac_feedback_div = 0;
    297	}
    298
    299	/* dcpll is DCE4 only */
    300	dcpll->min_post_div = 2;
    301	dcpll->max_post_div = 0x7f;
    302	dcpll->min_frac_feedback_div = 0;
    303	dcpll->max_frac_feedback_div = 9;
    304	dcpll->min_ref_div = 2;
    305	dcpll->max_ref_div = 0x3ff;
    306	dcpll->min_feedback_div = 4;
    307	dcpll->max_feedback_div = 0xfff;
    308	dcpll->best_vco = 0;
    309
    310	p1pll->min_ref_div = 2;
    311	p1pll->max_ref_div = 0x3ff;
    312	p1pll->min_feedback_div = 4;
    313	p1pll->max_feedback_div = 0x7ff;
    314	p1pll->best_vco = 0;
    315
    316	p2pll->min_ref_div = 2;
    317	p2pll->max_ref_div = 0x3ff;
    318	p2pll->min_feedback_div = 4;
    319	p2pll->max_feedback_div = 0x7ff;
    320	p2pll->best_vco = 0;
    321
    322	/* system clock */
    323	spll->min_post_div = 1;
    324	spll->max_post_div = 1;
    325	spll->min_ref_div = 2;
    326	spll->max_ref_div = 0xff;
    327	spll->min_feedback_div = 4;
    328	spll->max_feedback_div = 0xff;
    329	spll->best_vco = 0;
    330
    331	/* memory clock */
    332	mpll->min_post_div = 1;
    333	mpll->max_post_div = 1;
    334	mpll->min_ref_div = 2;
    335	mpll->max_ref_div = 0xff;
    336	mpll->min_feedback_div = 4;
    337	mpll->max_feedback_div = 0xff;
    338	mpll->best_vco = 0;
    339
    340	if (!rdev->clock.default_sclk)
    341		rdev->clock.default_sclk = radeon_get_engine_clock(rdev);
    342	if ((!rdev->clock.default_mclk) && rdev->asic->pm.get_memory_clock)
    343		rdev->clock.default_mclk = radeon_get_memory_clock(rdev);
    344
    345	rdev->pm.current_sclk = rdev->clock.default_sclk;
    346	rdev->pm.current_mclk = rdev->clock.default_mclk;
    347
    348}
    349
    350/* 10 khz */
    351static uint32_t calc_eng_mem_clock(struct radeon_device *rdev,
    352				   uint32_t req_clock,
    353				   int *fb_div, int *post_div)
    354{
    355	struct radeon_pll *spll = &rdev->clock.spll;
    356	int ref_div = spll->reference_div;
    357
    358	if (!ref_div)
    359		ref_div =
    360		    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
    361		    RADEON_M_SPLL_REF_DIV_MASK;
    362
    363	if (req_clock < 15000) {
    364		*post_div = 8;
    365		req_clock *= 8;
    366	} else if (req_clock < 30000) {
    367		*post_div = 4;
    368		req_clock *= 4;
    369	} else if (req_clock < 60000) {
    370		*post_div = 2;
    371		req_clock *= 2;
    372	} else
    373		*post_div = 1;
    374
    375	req_clock *= ref_div;
    376	req_clock += spll->reference_freq;
    377	req_clock /= (2 * spll->reference_freq);
    378
    379	*fb_div = req_clock & 0xff;
    380
    381	req_clock = (req_clock & 0xffff) << 1;
    382	req_clock *= spll->reference_freq;
    383	req_clock /= ref_div;
    384	req_clock /= *post_div;
    385
    386	return req_clock;
    387}
    388
    389/* 10 khz */
    390void radeon_legacy_set_engine_clock(struct radeon_device *rdev,
    391				    uint32_t eng_clock)
    392{
    393	uint32_t tmp;
    394	int fb_div, post_div;
    395
    396	/* XXX: wait for idle */
    397
    398	eng_clock = calc_eng_mem_clock(rdev, eng_clock, &fb_div, &post_div);
    399
    400	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
    401	tmp &= ~RADEON_DONT_USE_XTALIN;
    402	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
    403
    404	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    405	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
    406	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    407
    408	udelay(10);
    409
    410	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
    411	tmp |= RADEON_SPLL_SLEEP;
    412	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
    413
    414	udelay(2);
    415
    416	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
    417	tmp |= RADEON_SPLL_RESET;
    418	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
    419
    420	udelay(200);
    421
    422	tmp = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
    423	tmp &= ~(RADEON_SPLL_FB_DIV_MASK << RADEON_SPLL_FB_DIV_SHIFT);
    424	tmp |= (fb_div & RADEON_SPLL_FB_DIV_MASK) << RADEON_SPLL_FB_DIV_SHIFT;
    425	WREG32_PLL(RADEON_M_SPLL_REF_FB_DIV, tmp);
    426
    427	/* XXX: verify on different asics */
    428	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
    429	tmp &= ~RADEON_SPLL_PVG_MASK;
    430	if ((eng_clock * post_div) >= 90000)
    431		tmp |= (0x7 << RADEON_SPLL_PVG_SHIFT);
    432	else
    433		tmp |= (0x4 << RADEON_SPLL_PVG_SHIFT);
    434	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
    435
    436	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
    437	tmp &= ~RADEON_SPLL_SLEEP;
    438	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
    439
    440	udelay(2);
    441
    442	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
    443	tmp &= ~RADEON_SPLL_RESET;
    444	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
    445
    446	udelay(200);
    447
    448	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    449	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
    450	switch (post_div) {
    451	case 1:
    452	default:
    453		tmp |= 1;
    454		break;
    455	case 2:
    456		tmp |= 2;
    457		break;
    458	case 4:
    459		tmp |= 3;
    460		break;
    461	case 8:
    462		tmp |= 4;
    463		break;
    464	}
    465	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    466
    467	udelay(20);
    468
    469	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
    470	tmp |= RADEON_DONT_USE_XTALIN;
    471	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
    472
    473	udelay(10);
    474}
    475
    476void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable)
    477{
    478	uint32_t tmp;
    479
    480	if (enable) {
    481		if (rdev->flags & RADEON_SINGLE_CRTC) {
    482			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    483			if ((RREG32(RADEON_CONFIG_CNTL) &
    484			     RADEON_CFG_ATI_REV_ID_MASK) >
    485			    RADEON_CFG_ATI_REV_A13) {
    486				tmp &=
    487				    ~(RADEON_SCLK_FORCE_CP |
    488				      RADEON_SCLK_FORCE_RB);
    489			}
    490			tmp &=
    491			    ~(RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1 |
    492			      RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_SE |
    493			      RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_RE |
    494			      RADEON_SCLK_FORCE_PB | RADEON_SCLK_FORCE_TAM |
    495			      RADEON_SCLK_FORCE_TDM);
    496			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    497		} else if (ASIC_IS_R300(rdev)) {
    498			if ((rdev->family == CHIP_RS400) ||
    499			    (rdev->family == CHIP_RS480)) {
    500				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    501				tmp &=
    502				    ~(RADEON_SCLK_FORCE_DISP2 |
    503				      RADEON_SCLK_FORCE_CP |
    504				      RADEON_SCLK_FORCE_HDP |
    505				      RADEON_SCLK_FORCE_DISP1 |
    506				      RADEON_SCLK_FORCE_TOP |
    507				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
    508				      | RADEON_SCLK_FORCE_IDCT |
    509				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
    510				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
    511				      | R300_SCLK_FORCE_US |
    512				      RADEON_SCLK_FORCE_TV_SCLK |
    513				      R300_SCLK_FORCE_SU |
    514				      RADEON_SCLK_FORCE_OV0);
    515				tmp |= RADEON_DYN_STOP_LAT_MASK;
    516				tmp |=
    517				    RADEON_SCLK_FORCE_TOP |
    518				    RADEON_SCLK_FORCE_VIP;
    519				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    520
    521				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
    522				tmp &= ~RADEON_SCLK_MORE_FORCEON;
    523				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
    524				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
    525
    526				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
    527				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
    528					RADEON_PIXCLK_DAC_ALWAYS_ONb);
    529				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
    530
    531				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
    532				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
    533					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
    534					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
    535					R300_DVOCLK_ALWAYS_ONb |
    536					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
    537					RADEON_PIXCLK_GV_ALWAYS_ONb |
    538					R300_PIXCLK_DVO_ALWAYS_ONb |
    539					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
    540					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
    541					R300_PIXCLK_TRANS_ALWAYS_ONb |
    542					R300_PIXCLK_TVO_ALWAYS_ONb |
    543					R300_P2G2CLK_ALWAYS_ONb |
    544					R300_P2G2CLK_DAC_ALWAYS_ONb);
    545				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
    546			} else if (rdev->family >= CHIP_RV350) {
    547				tmp = RREG32_PLL(R300_SCLK_CNTL2);
    548				tmp &= ~(R300_SCLK_FORCE_TCL |
    549					 R300_SCLK_FORCE_GA |
    550					 R300_SCLK_FORCE_CBA);
    551				tmp |= (R300_SCLK_TCL_MAX_DYN_STOP_LAT |
    552					R300_SCLK_GA_MAX_DYN_STOP_LAT |
    553					R300_SCLK_CBA_MAX_DYN_STOP_LAT);
    554				WREG32_PLL(R300_SCLK_CNTL2, tmp);
    555
    556				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    557				tmp &=
    558				    ~(RADEON_SCLK_FORCE_DISP2 |
    559				      RADEON_SCLK_FORCE_CP |
    560				      RADEON_SCLK_FORCE_HDP |
    561				      RADEON_SCLK_FORCE_DISP1 |
    562				      RADEON_SCLK_FORCE_TOP |
    563				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
    564				      | RADEON_SCLK_FORCE_IDCT |
    565				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
    566				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
    567				      | R300_SCLK_FORCE_US |
    568				      RADEON_SCLK_FORCE_TV_SCLK |
    569				      R300_SCLK_FORCE_SU |
    570				      RADEON_SCLK_FORCE_OV0);
    571				tmp |= RADEON_DYN_STOP_LAT_MASK;
    572				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    573
    574				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
    575				tmp &= ~RADEON_SCLK_MORE_FORCEON;
    576				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
    577				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
    578
    579				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
    580				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
    581					RADEON_PIXCLK_DAC_ALWAYS_ONb);
    582				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
    583
    584				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
    585				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
    586					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
    587					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
    588					R300_DVOCLK_ALWAYS_ONb |
    589					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
    590					RADEON_PIXCLK_GV_ALWAYS_ONb |
    591					R300_PIXCLK_DVO_ALWAYS_ONb |
    592					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
    593					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
    594					R300_PIXCLK_TRANS_ALWAYS_ONb |
    595					R300_PIXCLK_TVO_ALWAYS_ONb |
    596					R300_P2G2CLK_ALWAYS_ONb |
    597					R300_P2G2CLK_DAC_ALWAYS_ONb);
    598				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
    599
    600				tmp = RREG32_PLL(RADEON_MCLK_MISC);
    601				tmp |= (RADEON_MC_MCLK_DYN_ENABLE |
    602					RADEON_IO_MCLK_DYN_ENABLE);
    603				WREG32_PLL(RADEON_MCLK_MISC, tmp);
    604
    605				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
    606				tmp |= (RADEON_FORCEON_MCLKA |
    607					RADEON_FORCEON_MCLKB);
    608
    609				tmp &= ~(RADEON_FORCEON_YCLKA |
    610					 RADEON_FORCEON_YCLKB |
    611					 RADEON_FORCEON_MC);
    612
    613				/* Some releases of vbios have set DISABLE_MC_MCLKA
    614				   and DISABLE_MC_MCLKB bits in the vbios table.  Setting these
    615				   bits will cause H/W hang when reading video memory with dynamic clocking
    616				   enabled. */
    617				if ((tmp & R300_DISABLE_MC_MCLKA) &&
    618				    (tmp & R300_DISABLE_MC_MCLKB)) {
    619					/* If both bits are set, then check the active channels */
    620					tmp = RREG32_PLL(RADEON_MCLK_CNTL);
    621					if (rdev->mc.vram_width == 64) {
    622						if (RREG32(RADEON_MEM_CNTL) &
    623						    R300_MEM_USE_CD_CH_ONLY)
    624							tmp &=
    625							    ~R300_DISABLE_MC_MCLKB;
    626						else
    627							tmp &=
    628							    ~R300_DISABLE_MC_MCLKA;
    629					} else {
    630						tmp &= ~(R300_DISABLE_MC_MCLKA |
    631							 R300_DISABLE_MC_MCLKB);
    632					}
    633				}
    634
    635				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
    636			} else {
    637				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    638				tmp &= ~(R300_SCLK_FORCE_VAP);
    639				tmp |= RADEON_SCLK_FORCE_CP;
    640				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    641				mdelay(15);
    642
    643				tmp = RREG32_PLL(R300_SCLK_CNTL2);
    644				tmp &= ~(R300_SCLK_FORCE_TCL |
    645					 R300_SCLK_FORCE_GA |
    646					 R300_SCLK_FORCE_CBA);
    647				WREG32_PLL(R300_SCLK_CNTL2, tmp);
    648			}
    649		} else {
    650			tmp = RREG32_PLL(RADEON_CLK_PWRMGT_CNTL);
    651
    652			tmp &= ~(RADEON_ACTIVE_HILO_LAT_MASK |
    653				 RADEON_DISP_DYN_STOP_LAT_MASK |
    654				 RADEON_DYN_STOP_MODE_MASK);
    655
    656			tmp |= (RADEON_ENGIN_DYNCLK_MODE |
    657				(0x01 << RADEON_ACTIVE_HILO_LAT_SHIFT));
    658			WREG32_PLL(RADEON_CLK_PWRMGT_CNTL, tmp);
    659			mdelay(15);
    660
    661			tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
    662			tmp |= RADEON_SCLK_DYN_START_CNTL;
    663			WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
    664			mdelay(15);
    665
    666			/* When DRI is enabled, setting DYN_STOP_LAT to zero can cause some R200
    667			   to lockup randomly, leave them as set by BIOS.
    668			 */
    669			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    670			/*tmp &= RADEON_SCLK_SRC_SEL_MASK; */
    671			tmp &= ~RADEON_SCLK_FORCEON_MASK;
    672
    673			/*RAGE_6::A11 A12 A12N1 A13, RV250::A11 A12, R300 */
    674			if (((rdev->family == CHIP_RV250) &&
    675			     ((RREG32(RADEON_CONFIG_CNTL) &
    676			       RADEON_CFG_ATI_REV_ID_MASK) <
    677			      RADEON_CFG_ATI_REV_A13))
    678			    || ((rdev->family == CHIP_RV100)
    679				&&
    680				((RREG32(RADEON_CONFIG_CNTL) &
    681				  RADEON_CFG_ATI_REV_ID_MASK) <=
    682				 RADEON_CFG_ATI_REV_A13))) {
    683				tmp |= RADEON_SCLK_FORCE_CP;
    684				tmp |= RADEON_SCLK_FORCE_VIP;
    685			}
    686
    687			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    688
    689			if ((rdev->family == CHIP_RV200) ||
    690			    (rdev->family == CHIP_RV250) ||
    691			    (rdev->family == CHIP_RV280)) {
    692				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
    693				tmp &= ~RADEON_SCLK_MORE_FORCEON;
    694
    695				/* RV200::A11 A12 RV250::A11 A12 */
    696				if (((rdev->family == CHIP_RV200) ||
    697				     (rdev->family == CHIP_RV250)) &&
    698				    ((RREG32(RADEON_CONFIG_CNTL) &
    699				      RADEON_CFG_ATI_REV_ID_MASK) <
    700				     RADEON_CFG_ATI_REV_A13)) {
    701					tmp |= RADEON_SCLK_MORE_FORCEON;
    702				}
    703				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
    704				mdelay(15);
    705			}
    706
    707			/* RV200::A11 A12, RV250::A11 A12 */
    708			if (((rdev->family == CHIP_RV200) ||
    709			     (rdev->family == CHIP_RV250)) &&
    710			    ((RREG32(RADEON_CONFIG_CNTL) &
    711			      RADEON_CFG_ATI_REV_ID_MASK) <
    712			     RADEON_CFG_ATI_REV_A13)) {
    713				tmp = RREG32_PLL(RADEON_PLL_PWRMGT_CNTL);
    714				tmp |= RADEON_TCL_BYPASS_DISABLE;
    715				WREG32_PLL(RADEON_PLL_PWRMGT_CNTL, tmp);
    716			}
    717			mdelay(15);
    718
    719			/*enable dynamic mode for display clocks (PIXCLK and PIX2CLK) */
    720			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
    721			tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
    722				RADEON_PIX2CLK_DAC_ALWAYS_ONb |
    723				RADEON_PIXCLK_BLEND_ALWAYS_ONb |
    724				RADEON_PIXCLK_GV_ALWAYS_ONb |
    725				RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
    726				RADEON_PIXCLK_LVDS_ALWAYS_ONb |
    727				RADEON_PIXCLK_TMDS_ALWAYS_ONb);
    728
    729			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
    730			mdelay(15);
    731
    732			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
    733			tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
    734				RADEON_PIXCLK_DAC_ALWAYS_ONb);
    735
    736			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
    737			mdelay(15);
    738		}
    739	} else {
    740		/* Turn everything OFF (ForceON to everything) */
    741		if (rdev->flags & RADEON_SINGLE_CRTC) {
    742			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    743			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_HDP |
    744				RADEON_SCLK_FORCE_DISP1 | RADEON_SCLK_FORCE_TOP
    745				| RADEON_SCLK_FORCE_E2 | RADEON_SCLK_FORCE_SE |
    746				RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_VIP |
    747				RADEON_SCLK_FORCE_RE | RADEON_SCLK_FORCE_PB |
    748				RADEON_SCLK_FORCE_TAM | RADEON_SCLK_FORCE_TDM |
    749				RADEON_SCLK_FORCE_RB);
    750			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    751		} else if ((rdev->family == CHIP_RS400) ||
    752			   (rdev->family == CHIP_RS480)) {
    753			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    754			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
    755				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
    756				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
    757				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
    758				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
    759				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
    760				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
    761				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
    762			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    763
    764			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
    765			tmp |= RADEON_SCLK_MORE_FORCEON;
    766			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
    767
    768			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
    769			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
    770				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
    771				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
    772			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
    773
    774			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
    775			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
    776				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
    777				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
    778				 R300_DVOCLK_ALWAYS_ONb |
    779				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
    780				 RADEON_PIXCLK_GV_ALWAYS_ONb |
    781				 R300_PIXCLK_DVO_ALWAYS_ONb |
    782				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
    783				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
    784				 R300_PIXCLK_TRANS_ALWAYS_ONb |
    785				 R300_PIXCLK_TVO_ALWAYS_ONb |
    786				 R300_P2G2CLK_ALWAYS_ONb |
    787				 R300_P2G2CLK_DAC_ALWAYS_ONb |
    788				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
    789			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
    790		} else if (rdev->family >= CHIP_RV350) {
    791			/* for RV350/M10, no delays are required. */
    792			tmp = RREG32_PLL(R300_SCLK_CNTL2);
    793			tmp |= (R300_SCLK_FORCE_TCL |
    794				R300_SCLK_FORCE_GA | R300_SCLK_FORCE_CBA);
    795			WREG32_PLL(R300_SCLK_CNTL2, tmp);
    796
    797			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    798			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
    799				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
    800				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
    801				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
    802				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
    803				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
    804				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
    805				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
    806			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    807
    808			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
    809			tmp |= RADEON_SCLK_MORE_FORCEON;
    810			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
    811
    812			tmp = RREG32_PLL(RADEON_MCLK_CNTL);
    813			tmp |= (RADEON_FORCEON_MCLKA |
    814				RADEON_FORCEON_MCLKB |
    815				RADEON_FORCEON_YCLKA |
    816				RADEON_FORCEON_YCLKB | RADEON_FORCEON_MC);
    817			WREG32_PLL(RADEON_MCLK_CNTL, tmp);
    818
    819			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
    820			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
    821				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
    822				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
    823			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
    824
    825			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
    826			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
    827				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
    828				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
    829				 R300_DVOCLK_ALWAYS_ONb |
    830				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
    831				 RADEON_PIXCLK_GV_ALWAYS_ONb |
    832				 R300_PIXCLK_DVO_ALWAYS_ONb |
    833				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
    834				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
    835				 R300_PIXCLK_TRANS_ALWAYS_ONb |
    836				 R300_PIXCLK_TVO_ALWAYS_ONb |
    837				 R300_P2G2CLK_ALWAYS_ONb |
    838				 R300_P2G2CLK_DAC_ALWAYS_ONb |
    839				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
    840			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
    841		} else {
    842			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
    843			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_E2);
    844			tmp |= RADEON_SCLK_FORCE_SE;
    845
    846			if (rdev->flags & RADEON_SINGLE_CRTC) {
    847				tmp |= (RADEON_SCLK_FORCE_RB |
    848					RADEON_SCLK_FORCE_TDM |
    849					RADEON_SCLK_FORCE_TAM |
    850					RADEON_SCLK_FORCE_PB |
    851					RADEON_SCLK_FORCE_RE |
    852					RADEON_SCLK_FORCE_VIP |
    853					RADEON_SCLK_FORCE_IDCT |
    854					RADEON_SCLK_FORCE_TOP |
    855					RADEON_SCLK_FORCE_DISP1 |
    856					RADEON_SCLK_FORCE_DISP2 |
    857					RADEON_SCLK_FORCE_HDP);
    858			} else if ((rdev->family == CHIP_R300) ||
    859				   (rdev->family == CHIP_R350)) {
    860				tmp |= (RADEON_SCLK_FORCE_HDP |
    861					RADEON_SCLK_FORCE_DISP1 |
    862					RADEON_SCLK_FORCE_DISP2 |
    863					RADEON_SCLK_FORCE_TOP |
    864					RADEON_SCLK_FORCE_IDCT |
    865					RADEON_SCLK_FORCE_VIP);
    866			}
    867			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
    868
    869			mdelay(16);
    870
    871			if ((rdev->family == CHIP_R300) ||
    872			    (rdev->family == CHIP_R350)) {
    873				tmp = RREG32_PLL(R300_SCLK_CNTL2);
    874				tmp |= (R300_SCLK_FORCE_TCL |
    875					R300_SCLK_FORCE_GA |
    876					R300_SCLK_FORCE_CBA);
    877				WREG32_PLL(R300_SCLK_CNTL2, tmp);
    878				mdelay(16);
    879			}
    880
    881			if (rdev->flags & RADEON_IS_IGP) {
    882				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
    883				tmp &= ~(RADEON_FORCEON_MCLKA |
    884					 RADEON_FORCEON_YCLKA);
    885				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
    886				mdelay(16);
    887			}
    888
    889			if ((rdev->family == CHIP_RV200) ||
    890			    (rdev->family == CHIP_RV250) ||
    891			    (rdev->family == CHIP_RV280)) {
    892				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
    893				tmp |= RADEON_SCLK_MORE_FORCEON;
    894				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
    895				mdelay(16);
    896			}
    897
    898			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
    899			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
    900				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
    901				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
    902				 RADEON_PIXCLK_GV_ALWAYS_ONb |
    903				 RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
    904				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
    905				 RADEON_PIXCLK_TMDS_ALWAYS_ONb);
    906
    907			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
    908			mdelay(16);
    909
    910			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
    911			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
    912				 RADEON_PIXCLK_DAC_ALWAYS_ONb);
    913			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
    914		}
    915	}
    916}
    917