1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <drm/drmP.h>
29 #include <drm/radeon_drm.h>
30 #include "radeon_reg.h"
31 #include "radeon.h"
32 #include "radeon_asic.h"
33 #include "atom.h"
34 
35 /* 10 khz */
36 uint32_t radeon_legacy_get_engine_clock(struct radeon_device *rdev)
37 {
38 	struct radeon_pll *spll = &rdev->clock.spll;
39 	uint32_t fb_div, ref_div, post_div, sclk;
40 
41 	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
42 	fb_div = (fb_div >> RADEON_SPLL_FB_DIV_SHIFT) & RADEON_SPLL_FB_DIV_MASK;
43 	fb_div <<= 1;
44 	fb_div *= spll->reference_freq;
45 
46 	ref_div =
47 	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
48 
49 	if (ref_div == 0)
50 		return 0;
51 
52 	sclk = fb_div / ref_div;
53 
54 	post_div = RREG32_PLL(RADEON_SCLK_CNTL) & RADEON_SCLK_SRC_SEL_MASK;
55 	if (post_div == 2)
56 		sclk >>= 1;
57 	else if (post_div == 3)
58 		sclk >>= 2;
59 	else if (post_div == 4)
60 		sclk >>= 3;
61 
62 	return sclk;
63 }
64 
65 /* 10 khz */
66 uint32_t radeon_legacy_get_memory_clock(struct radeon_device *rdev)
67 {
68 	struct radeon_pll *mpll = &rdev->clock.mpll;
69 	uint32_t fb_div, ref_div, post_div, mclk;
70 
71 	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
72 	fb_div = (fb_div >> RADEON_MPLL_FB_DIV_SHIFT) & RADEON_MPLL_FB_DIV_MASK;
73 	fb_div <<= 1;
74 	fb_div *= mpll->reference_freq;
75 
76 	ref_div =
77 	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
78 
79 	if (ref_div == 0)
80 		return 0;
81 
82 	mclk = fb_div / ref_div;
83 
84 	post_div = RREG32_PLL(RADEON_MCLK_CNTL) & 0x7;
85 	if (post_div == 2)
86 		mclk >>= 1;
87 	else if (post_div == 3)
88 		mclk >>= 2;
89 	else if (post_div == 4)
90 		mclk >>= 3;
91 
92 	return mclk;
93 }
94 
95 #ifdef CONFIG_OF
96 /*
97  * Read XTAL (ref clock), SCLK and MCLK from Open Firmware device
98  * tree. Hopefully, ATI OF driver is kind enough to fill these
99  */
100 static bool radeon_read_clocks_OF(struct drm_device *dev)
101 {
102 	struct radeon_device *rdev = dev->dev_private;
103 	struct device_node *dp = rdev->pdev->dev.of_node;
104 	const u32 *val;
105 	struct radeon_pll *p1pll = &rdev->clock.p1pll;
106 	struct radeon_pll *p2pll = &rdev->clock.p2pll;
107 	struct radeon_pll *spll = &rdev->clock.spll;
108 	struct radeon_pll *mpll = &rdev->clock.mpll;
109 
110 	if (dp == NULL)
111 		return false;
112 	val = of_get_property(dp, "ATY,RefCLK", NULL);
113 	if (!val || !*val) {
114 		pr_warn("radeonfb: No ATY,RefCLK property !\n");
115 		return false;
116 	}
117 	p1pll->reference_freq = p2pll->reference_freq = (*val) / 10;
118 	p1pll->reference_div = RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
119 	if (p1pll->reference_div < 2)
120 		p1pll->reference_div = 12;
121 	p2pll->reference_div = p1pll->reference_div;
122 
123 	/* These aren't in the device-tree */
124 	if (rdev->family >= CHIP_R420) {
125 		p1pll->pll_in_min = 100;
126 		p1pll->pll_in_max = 1350;
127 		p1pll->pll_out_min = 20000;
128 		p1pll->pll_out_max = 50000;
129 		p2pll->pll_in_min = 100;
130 		p2pll->pll_in_max = 1350;
131 		p2pll->pll_out_min = 20000;
132 		p2pll->pll_out_max = 50000;
133 	} else {
134 		p1pll->pll_in_min = 40;
135 		p1pll->pll_in_max = 500;
136 		p1pll->pll_out_min = 12500;
137 		p1pll->pll_out_max = 35000;
138 		p2pll->pll_in_min = 40;
139 		p2pll->pll_in_max = 500;
140 		p2pll->pll_out_min = 12500;
141 		p2pll->pll_out_max = 35000;
142 	}
143 	/* not sure what the max should be in all cases */
144 	rdev->clock.max_pixel_clock = 35000;
145 
146 	spll->reference_freq = mpll->reference_freq = p1pll->reference_freq;
147 	spll->reference_div = mpll->reference_div =
148 		RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
149 			    RADEON_M_SPLL_REF_DIV_MASK;
150 
151 	val = of_get_property(dp, "ATY,SCLK", NULL);
152 	if (val && *val)
153 		rdev->clock.default_sclk = (*val) / 10;
154 	else
155 		rdev->clock.default_sclk =
156 			radeon_legacy_get_engine_clock(rdev);
157 
158 	val = of_get_property(dp, "ATY,MCLK", NULL);
159 	if (val && *val)
160 		rdev->clock.default_mclk = (*val) / 10;
161 	else
162 		rdev->clock.default_mclk =
163 			radeon_legacy_get_memory_clock(rdev);
164 
165 	DRM_INFO("Using device-tree clock info\n");
166 
167 	return true;
168 }
169 #else
170 static bool radeon_read_clocks_OF(struct drm_device *dev)
171 {
172 	return false;
173 }
174 #endif /* CONFIG_OF */
175 
176 void radeon_get_clock_info(struct drm_device *dev)
177 {
178 	struct radeon_device *rdev = dev->dev_private;
179 	struct radeon_pll *p1pll = &rdev->clock.p1pll;
180 	struct radeon_pll *p2pll = &rdev->clock.p2pll;
181 	struct radeon_pll *dcpll = &rdev->clock.dcpll;
182 	struct radeon_pll *spll = &rdev->clock.spll;
183 	struct radeon_pll *mpll = &rdev->clock.mpll;
184 	int ret;
185 
186 	if (rdev->is_atom_bios)
187 		ret = radeon_atom_get_clock_info(dev);
188 	else
189 		ret = radeon_combios_get_clock_info(dev);
190 	if (!ret)
191 		ret = radeon_read_clocks_OF(dev);
192 
193 	if (ret) {
194 		if (p1pll->reference_div < 2) {
195 			if (!ASIC_IS_AVIVO(rdev)) {
196 				u32 tmp = RREG32_PLL(RADEON_PPLL_REF_DIV);
197 				if (ASIC_IS_R300(rdev))
198 					p1pll->reference_div =
199 						(tmp & R300_PPLL_REF_DIV_ACC_MASK) >> R300_PPLL_REF_DIV_ACC_SHIFT;
200 				else
201 					p1pll->reference_div = tmp & RADEON_PPLL_REF_DIV_MASK;
202 				if (p1pll->reference_div < 2)
203 					p1pll->reference_div = 12;
204 			} else
205 				p1pll->reference_div = 12;
206 		}
207 		if (p2pll->reference_div < 2)
208 			p2pll->reference_div = 12;
209 		if (rdev->family < CHIP_RS600) {
210 			if (spll->reference_div < 2)
211 				spll->reference_div =
212 					RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
213 					RADEON_M_SPLL_REF_DIV_MASK;
214 		}
215 		if (mpll->reference_div < 2)
216 			mpll->reference_div = spll->reference_div;
217 	} else {
218 		if (ASIC_IS_AVIVO(rdev)) {
219 			/* TODO FALLBACK */
220 		} else {
221 			DRM_INFO("Using generic clock info\n");
222 
223 			/* may need to be per card */
224 			rdev->clock.max_pixel_clock = 35000;
225 
226 			if (rdev->flags & RADEON_IS_IGP) {
227 				p1pll->reference_freq = 1432;
228 				p2pll->reference_freq = 1432;
229 				spll->reference_freq = 1432;
230 				mpll->reference_freq = 1432;
231 			} else {
232 				p1pll->reference_freq = 2700;
233 				p2pll->reference_freq = 2700;
234 				spll->reference_freq = 2700;
235 				mpll->reference_freq = 2700;
236 			}
237 			p1pll->reference_div =
238 			    RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
239 			if (p1pll->reference_div < 2)
240 				p1pll->reference_div = 12;
241 			p2pll->reference_div = p1pll->reference_div;
242 
243 			if (rdev->family >= CHIP_R420) {
244 				p1pll->pll_in_min = 100;
245 				p1pll->pll_in_max = 1350;
246 				p1pll->pll_out_min = 20000;
247 				p1pll->pll_out_max = 50000;
248 				p2pll->pll_in_min = 100;
249 				p2pll->pll_in_max = 1350;
250 				p2pll->pll_out_min = 20000;
251 				p2pll->pll_out_max = 50000;
252 			} else {
253 				p1pll->pll_in_min = 40;
254 				p1pll->pll_in_max = 500;
255 				p1pll->pll_out_min = 12500;
256 				p1pll->pll_out_max = 35000;
257 				p2pll->pll_in_min = 40;
258 				p2pll->pll_in_max = 500;
259 				p2pll->pll_out_min = 12500;
260 				p2pll->pll_out_max = 35000;
261 			}
262 
263 			spll->reference_div =
264 			    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
265 			    RADEON_M_SPLL_REF_DIV_MASK;
266 			mpll->reference_div = spll->reference_div;
267 			rdev->clock.default_sclk =
268 			    radeon_legacy_get_engine_clock(rdev);
269 			rdev->clock.default_mclk =
270 			    radeon_legacy_get_memory_clock(rdev);
271 		}
272 	}
273 
274 	/* pixel clocks */
275 	if (ASIC_IS_AVIVO(rdev)) {
276 		p1pll->min_post_div = 2;
277 		p1pll->max_post_div = 0x7f;
278 		p1pll->min_frac_feedback_div = 0;
279 		p1pll->max_frac_feedback_div = 9;
280 		p2pll->min_post_div = 2;
281 		p2pll->max_post_div = 0x7f;
282 		p2pll->min_frac_feedback_div = 0;
283 		p2pll->max_frac_feedback_div = 9;
284 	} else {
285 		p1pll->min_post_div = 1;
286 		p1pll->max_post_div = 16;
287 		p1pll->min_frac_feedback_div = 0;
288 		p1pll->max_frac_feedback_div = 0;
289 		p2pll->min_post_div = 1;
290 		p2pll->max_post_div = 12;
291 		p2pll->min_frac_feedback_div = 0;
292 		p2pll->max_frac_feedback_div = 0;
293 	}
294 
295 	/* dcpll is DCE4 only */
296 	dcpll->min_post_div = 2;
297 	dcpll->max_post_div = 0x7f;
298 	dcpll->min_frac_feedback_div = 0;
299 	dcpll->max_frac_feedback_div = 9;
300 	dcpll->min_ref_div = 2;
301 	dcpll->max_ref_div = 0x3ff;
302 	dcpll->min_feedback_div = 4;
303 	dcpll->max_feedback_div = 0xfff;
304 	dcpll->best_vco = 0;
305 
306 	p1pll->min_ref_div = 2;
307 	p1pll->max_ref_div = 0x3ff;
308 	p1pll->min_feedback_div = 4;
309 	p1pll->max_feedback_div = 0x7ff;
310 	p1pll->best_vco = 0;
311 
312 	p2pll->min_ref_div = 2;
313 	p2pll->max_ref_div = 0x3ff;
314 	p2pll->min_feedback_div = 4;
315 	p2pll->max_feedback_div = 0x7ff;
316 	p2pll->best_vco = 0;
317 
318 	/* system clock */
319 	spll->min_post_div = 1;
320 	spll->max_post_div = 1;
321 	spll->min_ref_div = 2;
322 	spll->max_ref_div = 0xff;
323 	spll->min_feedback_div = 4;
324 	spll->max_feedback_div = 0xff;
325 	spll->best_vco = 0;
326 
327 	/* memory clock */
328 	mpll->min_post_div = 1;
329 	mpll->max_post_div = 1;
330 	mpll->min_ref_div = 2;
331 	mpll->max_ref_div = 0xff;
332 	mpll->min_feedback_div = 4;
333 	mpll->max_feedback_div = 0xff;
334 	mpll->best_vco = 0;
335 
336 	if (!rdev->clock.default_sclk)
337 		rdev->clock.default_sclk = radeon_get_engine_clock(rdev);
338 	if ((!rdev->clock.default_mclk) && rdev->asic->pm.get_memory_clock)
339 		rdev->clock.default_mclk = radeon_get_memory_clock(rdev);
340 
341 	rdev->pm.current_sclk = rdev->clock.default_sclk;
342 	rdev->pm.current_mclk = rdev->clock.default_mclk;
343 
344 }
345 
346 /* 10 khz */
347 static uint32_t calc_eng_mem_clock(struct radeon_device *rdev,
348 				   uint32_t req_clock,
349 				   int *fb_div, int *post_div)
350 {
351 	struct radeon_pll *spll = &rdev->clock.spll;
352 	int ref_div = spll->reference_div;
353 
354 	if (!ref_div)
355 		ref_div =
356 		    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
357 		    RADEON_M_SPLL_REF_DIV_MASK;
358 
359 	if (req_clock < 15000) {
360 		*post_div = 8;
361 		req_clock *= 8;
362 	} else if (req_clock < 30000) {
363 		*post_div = 4;
364 		req_clock *= 4;
365 	} else if (req_clock < 60000) {
366 		*post_div = 2;
367 		req_clock *= 2;
368 	} else
369 		*post_div = 1;
370 
371 	req_clock *= ref_div;
372 	req_clock += spll->reference_freq;
373 	req_clock /= (2 * spll->reference_freq);
374 
375 	*fb_div = req_clock & 0xff;
376 
377 	req_clock = (req_clock & 0xffff) << 1;
378 	req_clock *= spll->reference_freq;
379 	req_clock /= ref_div;
380 	req_clock /= *post_div;
381 
382 	return req_clock;
383 }
384 
385 /* 10 khz */
386 void radeon_legacy_set_engine_clock(struct radeon_device *rdev,
387 				    uint32_t eng_clock)
388 {
389 	uint32_t tmp;
390 	int fb_div, post_div;
391 
392 	/* XXX: wait for idle */
393 
394 	eng_clock = calc_eng_mem_clock(rdev, eng_clock, &fb_div, &post_div);
395 
396 	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
397 	tmp &= ~RADEON_DONT_USE_XTALIN;
398 	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
399 
400 	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
401 	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
402 	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
403 
404 	udelay(10);
405 
406 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
407 	tmp |= RADEON_SPLL_SLEEP;
408 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
409 
410 	udelay(2);
411 
412 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
413 	tmp |= RADEON_SPLL_RESET;
414 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
415 
416 	udelay(200);
417 
418 	tmp = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
419 	tmp &= ~(RADEON_SPLL_FB_DIV_MASK << RADEON_SPLL_FB_DIV_SHIFT);
420 	tmp |= (fb_div & RADEON_SPLL_FB_DIV_MASK) << RADEON_SPLL_FB_DIV_SHIFT;
421 	WREG32_PLL(RADEON_M_SPLL_REF_FB_DIV, tmp);
422 
423 	/* XXX: verify on different asics */
424 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
425 	tmp &= ~RADEON_SPLL_PVG_MASK;
426 	if ((eng_clock * post_div) >= 90000)
427 		tmp |= (0x7 << RADEON_SPLL_PVG_SHIFT);
428 	else
429 		tmp |= (0x4 << RADEON_SPLL_PVG_SHIFT);
430 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
431 
432 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
433 	tmp &= ~RADEON_SPLL_SLEEP;
434 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
435 
436 	udelay(2);
437 
438 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
439 	tmp &= ~RADEON_SPLL_RESET;
440 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
441 
442 	udelay(200);
443 
444 	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
445 	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
446 	switch (post_div) {
447 	case 1:
448 	default:
449 		tmp |= 1;
450 		break;
451 	case 2:
452 		tmp |= 2;
453 		break;
454 	case 4:
455 		tmp |= 3;
456 		break;
457 	case 8:
458 		tmp |= 4;
459 		break;
460 	}
461 	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
462 
463 	udelay(20);
464 
465 	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
466 	tmp |= RADEON_DONT_USE_XTALIN;
467 	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
468 
469 	udelay(10);
470 }
471 
472 void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable)
473 {
474 	uint32_t tmp;
475 
476 	if (enable) {
477 		if (rdev->flags & RADEON_SINGLE_CRTC) {
478 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
479 			if ((RREG32(RADEON_CONFIG_CNTL) &
480 			     RADEON_CFG_ATI_REV_ID_MASK) >
481 			    RADEON_CFG_ATI_REV_A13) {
482 				tmp &=
483 				    ~(RADEON_SCLK_FORCE_CP |
484 				      RADEON_SCLK_FORCE_RB);
485 			}
486 			tmp &=
487 			    ~(RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1 |
488 			      RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_SE |
489 			      RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_RE |
490 			      RADEON_SCLK_FORCE_PB | RADEON_SCLK_FORCE_TAM |
491 			      RADEON_SCLK_FORCE_TDM);
492 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
493 		} else if (ASIC_IS_R300(rdev)) {
494 			if ((rdev->family == CHIP_RS400) ||
495 			    (rdev->family == CHIP_RS480)) {
496 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
497 				tmp &=
498 				    ~(RADEON_SCLK_FORCE_DISP2 |
499 				      RADEON_SCLK_FORCE_CP |
500 				      RADEON_SCLK_FORCE_HDP |
501 				      RADEON_SCLK_FORCE_DISP1 |
502 				      RADEON_SCLK_FORCE_TOP |
503 				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
504 				      | RADEON_SCLK_FORCE_IDCT |
505 				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
506 				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
507 				      | R300_SCLK_FORCE_US |
508 				      RADEON_SCLK_FORCE_TV_SCLK |
509 				      R300_SCLK_FORCE_SU |
510 				      RADEON_SCLK_FORCE_OV0);
511 				tmp |= RADEON_DYN_STOP_LAT_MASK;
512 				tmp |=
513 				    RADEON_SCLK_FORCE_TOP |
514 				    RADEON_SCLK_FORCE_VIP;
515 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
516 
517 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
518 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
519 				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
520 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
521 
522 				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
523 				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
524 					RADEON_PIXCLK_DAC_ALWAYS_ONb);
525 				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
526 
527 				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
528 				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
529 					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
530 					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
531 					R300_DVOCLK_ALWAYS_ONb |
532 					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
533 					RADEON_PIXCLK_GV_ALWAYS_ONb |
534 					R300_PIXCLK_DVO_ALWAYS_ONb |
535 					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
536 					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
537 					R300_PIXCLK_TRANS_ALWAYS_ONb |
538 					R300_PIXCLK_TVO_ALWAYS_ONb |
539 					R300_P2G2CLK_ALWAYS_ONb |
540 					R300_P2G2CLK_DAC_ALWAYS_ONb);
541 				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
542 			} else if (rdev->family >= CHIP_RV350) {
543 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
544 				tmp &= ~(R300_SCLK_FORCE_TCL |
545 					 R300_SCLK_FORCE_GA |
546 					 R300_SCLK_FORCE_CBA);
547 				tmp |= (R300_SCLK_TCL_MAX_DYN_STOP_LAT |
548 					R300_SCLK_GA_MAX_DYN_STOP_LAT |
549 					R300_SCLK_CBA_MAX_DYN_STOP_LAT);
550 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
551 
552 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
553 				tmp &=
554 				    ~(RADEON_SCLK_FORCE_DISP2 |
555 				      RADEON_SCLK_FORCE_CP |
556 				      RADEON_SCLK_FORCE_HDP |
557 				      RADEON_SCLK_FORCE_DISP1 |
558 				      RADEON_SCLK_FORCE_TOP |
559 				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
560 				      | RADEON_SCLK_FORCE_IDCT |
561 				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
562 				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
563 				      | R300_SCLK_FORCE_US |
564 				      RADEON_SCLK_FORCE_TV_SCLK |
565 				      R300_SCLK_FORCE_SU |
566 				      RADEON_SCLK_FORCE_OV0);
567 				tmp |= RADEON_DYN_STOP_LAT_MASK;
568 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
569 
570 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
571 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
572 				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
573 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
574 
575 				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
576 				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
577 					RADEON_PIXCLK_DAC_ALWAYS_ONb);
578 				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
579 
580 				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
581 				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
582 					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
583 					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
584 					R300_DVOCLK_ALWAYS_ONb |
585 					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
586 					RADEON_PIXCLK_GV_ALWAYS_ONb |
587 					R300_PIXCLK_DVO_ALWAYS_ONb |
588 					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
589 					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
590 					R300_PIXCLK_TRANS_ALWAYS_ONb |
591 					R300_PIXCLK_TVO_ALWAYS_ONb |
592 					R300_P2G2CLK_ALWAYS_ONb |
593 					R300_P2G2CLK_DAC_ALWAYS_ONb);
594 				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
595 
596 				tmp = RREG32_PLL(RADEON_MCLK_MISC);
597 				tmp |= (RADEON_MC_MCLK_DYN_ENABLE |
598 					RADEON_IO_MCLK_DYN_ENABLE);
599 				WREG32_PLL(RADEON_MCLK_MISC, tmp);
600 
601 				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
602 				tmp |= (RADEON_FORCEON_MCLKA |
603 					RADEON_FORCEON_MCLKB);
604 
605 				tmp &= ~(RADEON_FORCEON_YCLKA |
606 					 RADEON_FORCEON_YCLKB |
607 					 RADEON_FORCEON_MC);
608 
609 				/* Some releases of vbios have set DISABLE_MC_MCLKA
610 				   and DISABLE_MC_MCLKB bits in the vbios table.  Setting these
611 				   bits will cause H/W hang when reading video memory with dynamic clocking
612 				   enabled. */
613 				if ((tmp & R300_DISABLE_MC_MCLKA) &&
614 				    (tmp & R300_DISABLE_MC_MCLKB)) {
615 					/* If both bits are set, then check the active channels */
616 					tmp = RREG32_PLL(RADEON_MCLK_CNTL);
617 					if (rdev->mc.vram_width == 64) {
618 						if (RREG32(RADEON_MEM_CNTL) &
619 						    R300_MEM_USE_CD_CH_ONLY)
620 							tmp &=
621 							    ~R300_DISABLE_MC_MCLKB;
622 						else
623 							tmp &=
624 							    ~R300_DISABLE_MC_MCLKA;
625 					} else {
626 						tmp &= ~(R300_DISABLE_MC_MCLKA |
627 							 R300_DISABLE_MC_MCLKB);
628 					}
629 				}
630 
631 				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
632 			} else {
633 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
634 				tmp &= ~(R300_SCLK_FORCE_VAP);
635 				tmp |= RADEON_SCLK_FORCE_CP;
636 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
637 				mdelay(15);
638 
639 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
640 				tmp &= ~(R300_SCLK_FORCE_TCL |
641 					 R300_SCLK_FORCE_GA |
642 					 R300_SCLK_FORCE_CBA);
643 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
644 			}
645 		} else {
646 			tmp = RREG32_PLL(RADEON_CLK_PWRMGT_CNTL);
647 
648 			tmp &= ~(RADEON_ACTIVE_HILO_LAT_MASK |
649 				 RADEON_DISP_DYN_STOP_LAT_MASK |
650 				 RADEON_DYN_STOP_MODE_MASK);
651 
652 			tmp |= (RADEON_ENGIN_DYNCLK_MODE |
653 				(0x01 << RADEON_ACTIVE_HILO_LAT_SHIFT));
654 			WREG32_PLL(RADEON_CLK_PWRMGT_CNTL, tmp);
655 			mdelay(15);
656 
657 			tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
658 			tmp |= RADEON_SCLK_DYN_START_CNTL;
659 			WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
660 			mdelay(15);
661 
662 			/* When DRI is enabled, setting DYN_STOP_LAT to zero can cause some R200
663 			   to lockup randomly, leave them as set by BIOS.
664 			 */
665 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
666 			/*tmp &= RADEON_SCLK_SRC_SEL_MASK; */
667 			tmp &= ~RADEON_SCLK_FORCEON_MASK;
668 
669 			/*RAGE_6::A11 A12 A12N1 A13, RV250::A11 A12, R300 */
670 			if (((rdev->family == CHIP_RV250) &&
671 			     ((RREG32(RADEON_CONFIG_CNTL) &
672 			       RADEON_CFG_ATI_REV_ID_MASK) <
673 			      RADEON_CFG_ATI_REV_A13))
674 			    || ((rdev->family == CHIP_RV100)
675 				&&
676 				((RREG32(RADEON_CONFIG_CNTL) &
677 				  RADEON_CFG_ATI_REV_ID_MASK) <=
678 				 RADEON_CFG_ATI_REV_A13))) {
679 				tmp |= RADEON_SCLK_FORCE_CP;
680 				tmp |= RADEON_SCLK_FORCE_VIP;
681 			}
682 
683 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
684 
685 			if ((rdev->family == CHIP_RV200) ||
686 			    (rdev->family == CHIP_RV250) ||
687 			    (rdev->family == CHIP_RV280)) {
688 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
689 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
690 
691 				/* RV200::A11 A12 RV250::A11 A12 */
692 				if (((rdev->family == CHIP_RV200) ||
693 				     (rdev->family == CHIP_RV250)) &&
694 				    ((RREG32(RADEON_CONFIG_CNTL) &
695 				      RADEON_CFG_ATI_REV_ID_MASK) <
696 				     RADEON_CFG_ATI_REV_A13)) {
697 					tmp |= RADEON_SCLK_MORE_FORCEON;
698 				}
699 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
700 				mdelay(15);
701 			}
702 
703 			/* RV200::A11 A12, RV250::A11 A12 */
704 			if (((rdev->family == CHIP_RV200) ||
705 			     (rdev->family == CHIP_RV250)) &&
706 			    ((RREG32(RADEON_CONFIG_CNTL) &
707 			      RADEON_CFG_ATI_REV_ID_MASK) <
708 			     RADEON_CFG_ATI_REV_A13)) {
709 				tmp = RREG32_PLL(RADEON_PLL_PWRMGT_CNTL);
710 				tmp |= RADEON_TCL_BYPASS_DISABLE;
711 				WREG32_PLL(RADEON_PLL_PWRMGT_CNTL, tmp);
712 			}
713 			mdelay(15);
714 
715 			/*enable dynamic mode for display clocks (PIXCLK and PIX2CLK) */
716 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
717 			tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
718 				RADEON_PIX2CLK_DAC_ALWAYS_ONb |
719 				RADEON_PIXCLK_BLEND_ALWAYS_ONb |
720 				RADEON_PIXCLK_GV_ALWAYS_ONb |
721 				RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
722 				RADEON_PIXCLK_LVDS_ALWAYS_ONb |
723 				RADEON_PIXCLK_TMDS_ALWAYS_ONb);
724 
725 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
726 			mdelay(15);
727 
728 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
729 			tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
730 				RADEON_PIXCLK_DAC_ALWAYS_ONb);
731 
732 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
733 			mdelay(15);
734 		}
735 	} else {
736 		/* Turn everything OFF (ForceON to everything) */
737 		if (rdev->flags & RADEON_SINGLE_CRTC) {
738 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
739 			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_HDP |
740 				RADEON_SCLK_FORCE_DISP1 | RADEON_SCLK_FORCE_TOP
741 				| RADEON_SCLK_FORCE_E2 | RADEON_SCLK_FORCE_SE |
742 				RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_VIP |
743 				RADEON_SCLK_FORCE_RE | RADEON_SCLK_FORCE_PB |
744 				RADEON_SCLK_FORCE_TAM | RADEON_SCLK_FORCE_TDM |
745 				RADEON_SCLK_FORCE_RB);
746 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
747 		} else if ((rdev->family == CHIP_RS400) ||
748 			   (rdev->family == CHIP_RS480)) {
749 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
750 			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
751 				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
752 				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
753 				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
754 				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
755 				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
756 				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
757 				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
758 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
759 
760 			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
761 			tmp |= RADEON_SCLK_MORE_FORCEON;
762 			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
763 
764 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
765 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
766 				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
767 				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
768 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
769 
770 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
771 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
772 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
773 				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
774 				 R300_DVOCLK_ALWAYS_ONb |
775 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
776 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
777 				 R300_PIXCLK_DVO_ALWAYS_ONb |
778 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
779 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
780 				 R300_PIXCLK_TRANS_ALWAYS_ONb |
781 				 R300_PIXCLK_TVO_ALWAYS_ONb |
782 				 R300_P2G2CLK_ALWAYS_ONb |
783 				 R300_P2G2CLK_DAC_ALWAYS_ONb |
784 				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
785 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
786 		} else if (rdev->family >= CHIP_RV350) {
787 			/* for RV350/M10, no delays are required. */
788 			tmp = RREG32_PLL(R300_SCLK_CNTL2);
789 			tmp |= (R300_SCLK_FORCE_TCL |
790 				R300_SCLK_FORCE_GA | R300_SCLK_FORCE_CBA);
791 			WREG32_PLL(R300_SCLK_CNTL2, tmp);
792 
793 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
794 			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
795 				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
796 				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
797 				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
798 				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
799 				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
800 				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
801 				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
802 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
803 
804 			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
805 			tmp |= RADEON_SCLK_MORE_FORCEON;
806 			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
807 
808 			tmp = RREG32_PLL(RADEON_MCLK_CNTL);
809 			tmp |= (RADEON_FORCEON_MCLKA |
810 				RADEON_FORCEON_MCLKB |
811 				RADEON_FORCEON_YCLKA |
812 				RADEON_FORCEON_YCLKB | RADEON_FORCEON_MC);
813 			WREG32_PLL(RADEON_MCLK_CNTL, tmp);
814 
815 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
816 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
817 				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
818 				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
819 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
820 
821 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
822 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
823 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
824 				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
825 				 R300_DVOCLK_ALWAYS_ONb |
826 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
827 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
828 				 R300_PIXCLK_DVO_ALWAYS_ONb |
829 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
830 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
831 				 R300_PIXCLK_TRANS_ALWAYS_ONb |
832 				 R300_PIXCLK_TVO_ALWAYS_ONb |
833 				 R300_P2G2CLK_ALWAYS_ONb |
834 				 R300_P2G2CLK_DAC_ALWAYS_ONb |
835 				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
836 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
837 		} else {
838 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
839 			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_E2);
840 			tmp |= RADEON_SCLK_FORCE_SE;
841 
842 			if (rdev->flags & RADEON_SINGLE_CRTC) {
843 				tmp |= (RADEON_SCLK_FORCE_RB |
844 					RADEON_SCLK_FORCE_TDM |
845 					RADEON_SCLK_FORCE_TAM |
846 					RADEON_SCLK_FORCE_PB |
847 					RADEON_SCLK_FORCE_RE |
848 					RADEON_SCLK_FORCE_VIP |
849 					RADEON_SCLK_FORCE_IDCT |
850 					RADEON_SCLK_FORCE_TOP |
851 					RADEON_SCLK_FORCE_DISP1 |
852 					RADEON_SCLK_FORCE_DISP2 |
853 					RADEON_SCLK_FORCE_HDP);
854 			} else if ((rdev->family == CHIP_R300) ||
855 				   (rdev->family == CHIP_R350)) {
856 				tmp |= (RADEON_SCLK_FORCE_HDP |
857 					RADEON_SCLK_FORCE_DISP1 |
858 					RADEON_SCLK_FORCE_DISP2 |
859 					RADEON_SCLK_FORCE_TOP |
860 					RADEON_SCLK_FORCE_IDCT |
861 					RADEON_SCLK_FORCE_VIP);
862 			}
863 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
864 
865 			mdelay(16);
866 
867 			if ((rdev->family == CHIP_R300) ||
868 			    (rdev->family == CHIP_R350)) {
869 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
870 				tmp |= (R300_SCLK_FORCE_TCL |
871 					R300_SCLK_FORCE_GA |
872 					R300_SCLK_FORCE_CBA);
873 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
874 				mdelay(16);
875 			}
876 
877 			if (rdev->flags & RADEON_IS_IGP) {
878 				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
879 				tmp &= ~(RADEON_FORCEON_MCLKA |
880 					 RADEON_FORCEON_YCLKA);
881 				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
882 				mdelay(16);
883 			}
884 
885 			if ((rdev->family == CHIP_RV200) ||
886 			    (rdev->family == CHIP_RV250) ||
887 			    (rdev->family == CHIP_RV280)) {
888 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
889 				tmp |= RADEON_SCLK_MORE_FORCEON;
890 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
891 				mdelay(16);
892 			}
893 
894 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
895 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
896 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
897 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
898 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
899 				 RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
900 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
901 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb);
902 
903 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
904 			mdelay(16);
905 
906 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
907 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
908 				 RADEON_PIXCLK_DAC_ALWAYS_ONb);
909 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
910 		}
911 	}
912 }
913 
914