1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 
29 #include <drm/drm_device.h>
30 #include <drm/drm_pci.h>
31 #include <drm/radeon_drm.h>
32 
33 #include "atom.h"
34 #include "radeon.h"
35 #include "radeon_asic.h"
36 #include "radeon_reg.h"
37 
38 /* 10 khz */
39 uint32_t radeon_legacy_get_engine_clock(struct radeon_device *rdev)
40 {
41 	struct radeon_pll *spll = &rdev->clock.spll;
42 	uint32_t fb_div, ref_div, post_div, sclk;
43 
44 	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
45 	fb_div = (fb_div >> RADEON_SPLL_FB_DIV_SHIFT) & RADEON_SPLL_FB_DIV_MASK;
46 	fb_div <<= 1;
47 	fb_div *= spll->reference_freq;
48 
49 	ref_div =
50 	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
51 
52 	if (ref_div == 0)
53 		return 0;
54 
55 	sclk = fb_div / ref_div;
56 
57 	post_div = RREG32_PLL(RADEON_SCLK_CNTL) & RADEON_SCLK_SRC_SEL_MASK;
58 	if (post_div == 2)
59 		sclk >>= 1;
60 	else if (post_div == 3)
61 		sclk >>= 2;
62 	else if (post_div == 4)
63 		sclk >>= 3;
64 
65 	return sclk;
66 }
67 
68 /* 10 khz */
69 uint32_t radeon_legacy_get_memory_clock(struct radeon_device *rdev)
70 {
71 	struct radeon_pll *mpll = &rdev->clock.mpll;
72 	uint32_t fb_div, ref_div, post_div, mclk;
73 
74 	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
75 	fb_div = (fb_div >> RADEON_MPLL_FB_DIV_SHIFT) & RADEON_MPLL_FB_DIV_MASK;
76 	fb_div <<= 1;
77 	fb_div *= mpll->reference_freq;
78 
79 	ref_div =
80 	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
81 
82 	if (ref_div == 0)
83 		return 0;
84 
85 	mclk = fb_div / ref_div;
86 
87 	post_div = RREG32_PLL(RADEON_MCLK_CNTL) & 0x7;
88 	if (post_div == 2)
89 		mclk >>= 1;
90 	else if (post_div == 3)
91 		mclk >>= 2;
92 	else if (post_div == 4)
93 		mclk >>= 3;
94 
95 	return mclk;
96 }
97 
98 #ifdef CONFIG_OF
99 /*
100  * Read XTAL (ref clock), SCLK and MCLK from Open Firmware device
101  * tree. Hopefully, ATI OF driver is kind enough to fill these
102  */
103 static bool radeon_read_clocks_OF(struct drm_device *dev)
104 {
105 	struct radeon_device *rdev = dev->dev_private;
106 	struct device_node *dp = rdev->pdev->dev.of_node;
107 	const u32 *val;
108 	struct radeon_pll *p1pll = &rdev->clock.p1pll;
109 	struct radeon_pll *p2pll = &rdev->clock.p2pll;
110 	struct radeon_pll *spll = &rdev->clock.spll;
111 	struct radeon_pll *mpll = &rdev->clock.mpll;
112 
113 	if (dp == NULL)
114 		return false;
115 	val = of_get_property(dp, "ATY,RefCLK", NULL);
116 	if (!val || !*val) {
117 		pr_warn("radeonfb: No ATY,RefCLK property !\n");
118 		return false;
119 	}
120 	p1pll->reference_freq = p2pll->reference_freq = (*val) / 10;
121 	p1pll->reference_div = RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
122 	if (p1pll->reference_div < 2)
123 		p1pll->reference_div = 12;
124 	p2pll->reference_div = p1pll->reference_div;
125 
126 	/* These aren't in the device-tree */
127 	if (rdev->family >= CHIP_R420) {
128 		p1pll->pll_in_min = 100;
129 		p1pll->pll_in_max = 1350;
130 		p1pll->pll_out_min = 20000;
131 		p1pll->pll_out_max = 50000;
132 		p2pll->pll_in_min = 100;
133 		p2pll->pll_in_max = 1350;
134 		p2pll->pll_out_min = 20000;
135 		p2pll->pll_out_max = 50000;
136 	} else {
137 		p1pll->pll_in_min = 40;
138 		p1pll->pll_in_max = 500;
139 		p1pll->pll_out_min = 12500;
140 		p1pll->pll_out_max = 35000;
141 		p2pll->pll_in_min = 40;
142 		p2pll->pll_in_max = 500;
143 		p2pll->pll_out_min = 12500;
144 		p2pll->pll_out_max = 35000;
145 	}
146 	/* not sure what the max should be in all cases */
147 	rdev->clock.max_pixel_clock = 35000;
148 
149 	spll->reference_freq = mpll->reference_freq = p1pll->reference_freq;
150 	spll->reference_div = mpll->reference_div =
151 		RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
152 			    RADEON_M_SPLL_REF_DIV_MASK;
153 
154 	val = of_get_property(dp, "ATY,SCLK", NULL);
155 	if (val && *val)
156 		rdev->clock.default_sclk = (*val) / 10;
157 	else
158 		rdev->clock.default_sclk =
159 			radeon_legacy_get_engine_clock(rdev);
160 
161 	val = of_get_property(dp, "ATY,MCLK", NULL);
162 	if (val && *val)
163 		rdev->clock.default_mclk = (*val) / 10;
164 	else
165 		rdev->clock.default_mclk =
166 			radeon_legacy_get_memory_clock(rdev);
167 
168 	DRM_INFO("Using device-tree clock info\n");
169 
170 	return true;
171 }
172 #else
173 static bool radeon_read_clocks_OF(struct drm_device *dev)
174 {
175 	return false;
176 }
177 #endif /* CONFIG_OF */
178 
179 void radeon_get_clock_info(struct drm_device *dev)
180 {
181 	struct radeon_device *rdev = dev->dev_private;
182 	struct radeon_pll *p1pll = &rdev->clock.p1pll;
183 	struct radeon_pll *p2pll = &rdev->clock.p2pll;
184 	struct radeon_pll *dcpll = &rdev->clock.dcpll;
185 	struct radeon_pll *spll = &rdev->clock.spll;
186 	struct radeon_pll *mpll = &rdev->clock.mpll;
187 	int ret;
188 
189 	if (rdev->is_atom_bios)
190 		ret = radeon_atom_get_clock_info(dev);
191 	else
192 		ret = radeon_combios_get_clock_info(dev);
193 	if (!ret)
194 		ret = radeon_read_clocks_OF(dev);
195 
196 	if (ret) {
197 		if (p1pll->reference_div < 2) {
198 			if (!ASIC_IS_AVIVO(rdev)) {
199 				u32 tmp = RREG32_PLL(RADEON_PPLL_REF_DIV);
200 				if (ASIC_IS_R300(rdev))
201 					p1pll->reference_div =
202 						(tmp & R300_PPLL_REF_DIV_ACC_MASK) >> R300_PPLL_REF_DIV_ACC_SHIFT;
203 				else
204 					p1pll->reference_div = tmp & RADEON_PPLL_REF_DIV_MASK;
205 				if (p1pll->reference_div < 2)
206 					p1pll->reference_div = 12;
207 			} else
208 				p1pll->reference_div = 12;
209 		}
210 		if (p2pll->reference_div < 2)
211 			p2pll->reference_div = 12;
212 		if (rdev->family < CHIP_RS600) {
213 			if (spll->reference_div < 2)
214 				spll->reference_div =
215 					RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
216 					RADEON_M_SPLL_REF_DIV_MASK;
217 		}
218 		if (mpll->reference_div < 2)
219 			mpll->reference_div = spll->reference_div;
220 	} else {
221 		if (ASIC_IS_AVIVO(rdev)) {
222 			/* TODO FALLBACK */
223 		} else {
224 			DRM_INFO("Using generic clock info\n");
225 
226 			/* may need to be per card */
227 			rdev->clock.max_pixel_clock = 35000;
228 
229 			if (rdev->flags & RADEON_IS_IGP) {
230 				p1pll->reference_freq = 1432;
231 				p2pll->reference_freq = 1432;
232 				spll->reference_freq = 1432;
233 				mpll->reference_freq = 1432;
234 			} else {
235 				p1pll->reference_freq = 2700;
236 				p2pll->reference_freq = 2700;
237 				spll->reference_freq = 2700;
238 				mpll->reference_freq = 2700;
239 			}
240 			p1pll->reference_div =
241 			    RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
242 			if (p1pll->reference_div < 2)
243 				p1pll->reference_div = 12;
244 			p2pll->reference_div = p1pll->reference_div;
245 
246 			if (rdev->family >= CHIP_R420) {
247 				p1pll->pll_in_min = 100;
248 				p1pll->pll_in_max = 1350;
249 				p1pll->pll_out_min = 20000;
250 				p1pll->pll_out_max = 50000;
251 				p2pll->pll_in_min = 100;
252 				p2pll->pll_in_max = 1350;
253 				p2pll->pll_out_min = 20000;
254 				p2pll->pll_out_max = 50000;
255 			} else {
256 				p1pll->pll_in_min = 40;
257 				p1pll->pll_in_max = 500;
258 				p1pll->pll_out_min = 12500;
259 				p1pll->pll_out_max = 35000;
260 				p2pll->pll_in_min = 40;
261 				p2pll->pll_in_max = 500;
262 				p2pll->pll_out_min = 12500;
263 				p2pll->pll_out_max = 35000;
264 			}
265 
266 			spll->reference_div =
267 			    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
268 			    RADEON_M_SPLL_REF_DIV_MASK;
269 			mpll->reference_div = spll->reference_div;
270 			rdev->clock.default_sclk =
271 			    radeon_legacy_get_engine_clock(rdev);
272 			rdev->clock.default_mclk =
273 			    radeon_legacy_get_memory_clock(rdev);
274 		}
275 	}
276 
277 	/* pixel clocks */
278 	if (ASIC_IS_AVIVO(rdev)) {
279 		p1pll->min_post_div = 2;
280 		p1pll->max_post_div = 0x7f;
281 		p1pll->min_frac_feedback_div = 0;
282 		p1pll->max_frac_feedback_div = 9;
283 		p2pll->min_post_div = 2;
284 		p2pll->max_post_div = 0x7f;
285 		p2pll->min_frac_feedback_div = 0;
286 		p2pll->max_frac_feedback_div = 9;
287 	} else {
288 		p1pll->min_post_div = 1;
289 		p1pll->max_post_div = 16;
290 		p1pll->min_frac_feedback_div = 0;
291 		p1pll->max_frac_feedback_div = 0;
292 		p2pll->min_post_div = 1;
293 		p2pll->max_post_div = 12;
294 		p2pll->min_frac_feedback_div = 0;
295 		p2pll->max_frac_feedback_div = 0;
296 	}
297 
298 	/* dcpll is DCE4 only */
299 	dcpll->min_post_div = 2;
300 	dcpll->max_post_div = 0x7f;
301 	dcpll->min_frac_feedback_div = 0;
302 	dcpll->max_frac_feedback_div = 9;
303 	dcpll->min_ref_div = 2;
304 	dcpll->max_ref_div = 0x3ff;
305 	dcpll->min_feedback_div = 4;
306 	dcpll->max_feedback_div = 0xfff;
307 	dcpll->best_vco = 0;
308 
309 	p1pll->min_ref_div = 2;
310 	p1pll->max_ref_div = 0x3ff;
311 	p1pll->min_feedback_div = 4;
312 	p1pll->max_feedback_div = 0x7ff;
313 	p1pll->best_vco = 0;
314 
315 	p2pll->min_ref_div = 2;
316 	p2pll->max_ref_div = 0x3ff;
317 	p2pll->min_feedback_div = 4;
318 	p2pll->max_feedback_div = 0x7ff;
319 	p2pll->best_vco = 0;
320 
321 	/* system clock */
322 	spll->min_post_div = 1;
323 	spll->max_post_div = 1;
324 	spll->min_ref_div = 2;
325 	spll->max_ref_div = 0xff;
326 	spll->min_feedback_div = 4;
327 	spll->max_feedback_div = 0xff;
328 	spll->best_vco = 0;
329 
330 	/* memory clock */
331 	mpll->min_post_div = 1;
332 	mpll->max_post_div = 1;
333 	mpll->min_ref_div = 2;
334 	mpll->max_ref_div = 0xff;
335 	mpll->min_feedback_div = 4;
336 	mpll->max_feedback_div = 0xff;
337 	mpll->best_vco = 0;
338 
339 	if (!rdev->clock.default_sclk)
340 		rdev->clock.default_sclk = radeon_get_engine_clock(rdev);
341 	if ((!rdev->clock.default_mclk) && rdev->asic->pm.get_memory_clock)
342 		rdev->clock.default_mclk = radeon_get_memory_clock(rdev);
343 
344 	rdev->pm.current_sclk = rdev->clock.default_sclk;
345 	rdev->pm.current_mclk = rdev->clock.default_mclk;
346 
347 }
348 
349 /* 10 khz */
350 static uint32_t calc_eng_mem_clock(struct radeon_device *rdev,
351 				   uint32_t req_clock,
352 				   int *fb_div, int *post_div)
353 {
354 	struct radeon_pll *spll = &rdev->clock.spll;
355 	int ref_div = spll->reference_div;
356 
357 	if (!ref_div)
358 		ref_div =
359 		    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
360 		    RADEON_M_SPLL_REF_DIV_MASK;
361 
362 	if (req_clock < 15000) {
363 		*post_div = 8;
364 		req_clock *= 8;
365 	} else if (req_clock < 30000) {
366 		*post_div = 4;
367 		req_clock *= 4;
368 	} else if (req_clock < 60000) {
369 		*post_div = 2;
370 		req_clock *= 2;
371 	} else
372 		*post_div = 1;
373 
374 	req_clock *= ref_div;
375 	req_clock += spll->reference_freq;
376 	req_clock /= (2 * spll->reference_freq);
377 
378 	*fb_div = req_clock & 0xff;
379 
380 	req_clock = (req_clock & 0xffff) << 1;
381 	req_clock *= spll->reference_freq;
382 	req_clock /= ref_div;
383 	req_clock /= *post_div;
384 
385 	return req_clock;
386 }
387 
388 /* 10 khz */
389 void radeon_legacy_set_engine_clock(struct radeon_device *rdev,
390 				    uint32_t eng_clock)
391 {
392 	uint32_t tmp;
393 	int fb_div, post_div;
394 
395 	/* XXX: wait for idle */
396 
397 	eng_clock = calc_eng_mem_clock(rdev, eng_clock, &fb_div, &post_div);
398 
399 	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
400 	tmp &= ~RADEON_DONT_USE_XTALIN;
401 	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
402 
403 	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
404 	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
405 	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
406 
407 	udelay(10);
408 
409 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
410 	tmp |= RADEON_SPLL_SLEEP;
411 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
412 
413 	udelay(2);
414 
415 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
416 	tmp |= RADEON_SPLL_RESET;
417 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
418 
419 	udelay(200);
420 
421 	tmp = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
422 	tmp &= ~(RADEON_SPLL_FB_DIV_MASK << RADEON_SPLL_FB_DIV_SHIFT);
423 	tmp |= (fb_div & RADEON_SPLL_FB_DIV_MASK) << RADEON_SPLL_FB_DIV_SHIFT;
424 	WREG32_PLL(RADEON_M_SPLL_REF_FB_DIV, tmp);
425 
426 	/* XXX: verify on different asics */
427 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
428 	tmp &= ~RADEON_SPLL_PVG_MASK;
429 	if ((eng_clock * post_div) >= 90000)
430 		tmp |= (0x7 << RADEON_SPLL_PVG_SHIFT);
431 	else
432 		tmp |= (0x4 << RADEON_SPLL_PVG_SHIFT);
433 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
434 
435 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
436 	tmp &= ~RADEON_SPLL_SLEEP;
437 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
438 
439 	udelay(2);
440 
441 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
442 	tmp &= ~RADEON_SPLL_RESET;
443 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
444 
445 	udelay(200);
446 
447 	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
448 	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
449 	switch (post_div) {
450 	case 1:
451 	default:
452 		tmp |= 1;
453 		break;
454 	case 2:
455 		tmp |= 2;
456 		break;
457 	case 4:
458 		tmp |= 3;
459 		break;
460 	case 8:
461 		tmp |= 4;
462 		break;
463 	}
464 	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
465 
466 	udelay(20);
467 
468 	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
469 	tmp |= RADEON_DONT_USE_XTALIN;
470 	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
471 
472 	udelay(10);
473 }
474 
475 void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable)
476 {
477 	uint32_t tmp;
478 
479 	if (enable) {
480 		if (rdev->flags & RADEON_SINGLE_CRTC) {
481 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
482 			if ((RREG32(RADEON_CONFIG_CNTL) &
483 			     RADEON_CFG_ATI_REV_ID_MASK) >
484 			    RADEON_CFG_ATI_REV_A13) {
485 				tmp &=
486 				    ~(RADEON_SCLK_FORCE_CP |
487 				      RADEON_SCLK_FORCE_RB);
488 			}
489 			tmp &=
490 			    ~(RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1 |
491 			      RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_SE |
492 			      RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_RE |
493 			      RADEON_SCLK_FORCE_PB | RADEON_SCLK_FORCE_TAM |
494 			      RADEON_SCLK_FORCE_TDM);
495 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
496 		} else if (ASIC_IS_R300(rdev)) {
497 			if ((rdev->family == CHIP_RS400) ||
498 			    (rdev->family == CHIP_RS480)) {
499 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
500 				tmp &=
501 				    ~(RADEON_SCLK_FORCE_DISP2 |
502 				      RADEON_SCLK_FORCE_CP |
503 				      RADEON_SCLK_FORCE_HDP |
504 				      RADEON_SCLK_FORCE_DISP1 |
505 				      RADEON_SCLK_FORCE_TOP |
506 				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
507 				      | RADEON_SCLK_FORCE_IDCT |
508 				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
509 				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
510 				      | R300_SCLK_FORCE_US |
511 				      RADEON_SCLK_FORCE_TV_SCLK |
512 				      R300_SCLK_FORCE_SU |
513 				      RADEON_SCLK_FORCE_OV0);
514 				tmp |= RADEON_DYN_STOP_LAT_MASK;
515 				tmp |=
516 				    RADEON_SCLK_FORCE_TOP |
517 				    RADEON_SCLK_FORCE_VIP;
518 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
519 
520 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
521 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
522 				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
523 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
524 
525 				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
526 				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
527 					RADEON_PIXCLK_DAC_ALWAYS_ONb);
528 				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
529 
530 				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
531 				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
532 					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
533 					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
534 					R300_DVOCLK_ALWAYS_ONb |
535 					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
536 					RADEON_PIXCLK_GV_ALWAYS_ONb |
537 					R300_PIXCLK_DVO_ALWAYS_ONb |
538 					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
539 					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
540 					R300_PIXCLK_TRANS_ALWAYS_ONb |
541 					R300_PIXCLK_TVO_ALWAYS_ONb |
542 					R300_P2G2CLK_ALWAYS_ONb |
543 					R300_P2G2CLK_DAC_ALWAYS_ONb);
544 				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
545 			} else if (rdev->family >= CHIP_RV350) {
546 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
547 				tmp &= ~(R300_SCLK_FORCE_TCL |
548 					 R300_SCLK_FORCE_GA |
549 					 R300_SCLK_FORCE_CBA);
550 				tmp |= (R300_SCLK_TCL_MAX_DYN_STOP_LAT |
551 					R300_SCLK_GA_MAX_DYN_STOP_LAT |
552 					R300_SCLK_CBA_MAX_DYN_STOP_LAT);
553 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
554 
555 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
556 				tmp &=
557 				    ~(RADEON_SCLK_FORCE_DISP2 |
558 				      RADEON_SCLK_FORCE_CP |
559 				      RADEON_SCLK_FORCE_HDP |
560 				      RADEON_SCLK_FORCE_DISP1 |
561 				      RADEON_SCLK_FORCE_TOP |
562 				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
563 				      | RADEON_SCLK_FORCE_IDCT |
564 				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
565 				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
566 				      | R300_SCLK_FORCE_US |
567 				      RADEON_SCLK_FORCE_TV_SCLK |
568 				      R300_SCLK_FORCE_SU |
569 				      RADEON_SCLK_FORCE_OV0);
570 				tmp |= RADEON_DYN_STOP_LAT_MASK;
571 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
572 
573 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
574 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
575 				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
576 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
577 
578 				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
579 				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
580 					RADEON_PIXCLK_DAC_ALWAYS_ONb);
581 				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
582 
583 				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
584 				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
585 					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
586 					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
587 					R300_DVOCLK_ALWAYS_ONb |
588 					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
589 					RADEON_PIXCLK_GV_ALWAYS_ONb |
590 					R300_PIXCLK_DVO_ALWAYS_ONb |
591 					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
592 					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
593 					R300_PIXCLK_TRANS_ALWAYS_ONb |
594 					R300_PIXCLK_TVO_ALWAYS_ONb |
595 					R300_P2G2CLK_ALWAYS_ONb |
596 					R300_P2G2CLK_DAC_ALWAYS_ONb);
597 				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
598 
599 				tmp = RREG32_PLL(RADEON_MCLK_MISC);
600 				tmp |= (RADEON_MC_MCLK_DYN_ENABLE |
601 					RADEON_IO_MCLK_DYN_ENABLE);
602 				WREG32_PLL(RADEON_MCLK_MISC, tmp);
603 
604 				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
605 				tmp |= (RADEON_FORCEON_MCLKA |
606 					RADEON_FORCEON_MCLKB);
607 
608 				tmp &= ~(RADEON_FORCEON_YCLKA |
609 					 RADEON_FORCEON_YCLKB |
610 					 RADEON_FORCEON_MC);
611 
612 				/* Some releases of vbios have set DISABLE_MC_MCLKA
613 				   and DISABLE_MC_MCLKB bits in the vbios table.  Setting these
614 				   bits will cause H/W hang when reading video memory with dynamic clocking
615 				   enabled. */
616 				if ((tmp & R300_DISABLE_MC_MCLKA) &&
617 				    (tmp & R300_DISABLE_MC_MCLKB)) {
618 					/* If both bits are set, then check the active channels */
619 					tmp = RREG32_PLL(RADEON_MCLK_CNTL);
620 					if (rdev->mc.vram_width == 64) {
621 						if (RREG32(RADEON_MEM_CNTL) &
622 						    R300_MEM_USE_CD_CH_ONLY)
623 							tmp &=
624 							    ~R300_DISABLE_MC_MCLKB;
625 						else
626 							tmp &=
627 							    ~R300_DISABLE_MC_MCLKA;
628 					} else {
629 						tmp &= ~(R300_DISABLE_MC_MCLKA |
630 							 R300_DISABLE_MC_MCLKB);
631 					}
632 				}
633 
634 				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
635 			} else {
636 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
637 				tmp &= ~(R300_SCLK_FORCE_VAP);
638 				tmp |= RADEON_SCLK_FORCE_CP;
639 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
640 				mdelay(15);
641 
642 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
643 				tmp &= ~(R300_SCLK_FORCE_TCL |
644 					 R300_SCLK_FORCE_GA |
645 					 R300_SCLK_FORCE_CBA);
646 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
647 			}
648 		} else {
649 			tmp = RREG32_PLL(RADEON_CLK_PWRMGT_CNTL);
650 
651 			tmp &= ~(RADEON_ACTIVE_HILO_LAT_MASK |
652 				 RADEON_DISP_DYN_STOP_LAT_MASK |
653 				 RADEON_DYN_STOP_MODE_MASK);
654 
655 			tmp |= (RADEON_ENGIN_DYNCLK_MODE |
656 				(0x01 << RADEON_ACTIVE_HILO_LAT_SHIFT));
657 			WREG32_PLL(RADEON_CLK_PWRMGT_CNTL, tmp);
658 			mdelay(15);
659 
660 			tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
661 			tmp |= RADEON_SCLK_DYN_START_CNTL;
662 			WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
663 			mdelay(15);
664 
665 			/* When DRI is enabled, setting DYN_STOP_LAT to zero can cause some R200
666 			   to lockup randomly, leave them as set by BIOS.
667 			 */
668 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
669 			/*tmp &= RADEON_SCLK_SRC_SEL_MASK; */
670 			tmp &= ~RADEON_SCLK_FORCEON_MASK;
671 
672 			/*RAGE_6::A11 A12 A12N1 A13, RV250::A11 A12, R300 */
673 			if (((rdev->family == CHIP_RV250) &&
674 			     ((RREG32(RADEON_CONFIG_CNTL) &
675 			       RADEON_CFG_ATI_REV_ID_MASK) <
676 			      RADEON_CFG_ATI_REV_A13))
677 			    || ((rdev->family == CHIP_RV100)
678 				&&
679 				((RREG32(RADEON_CONFIG_CNTL) &
680 				  RADEON_CFG_ATI_REV_ID_MASK) <=
681 				 RADEON_CFG_ATI_REV_A13))) {
682 				tmp |= RADEON_SCLK_FORCE_CP;
683 				tmp |= RADEON_SCLK_FORCE_VIP;
684 			}
685 
686 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
687 
688 			if ((rdev->family == CHIP_RV200) ||
689 			    (rdev->family == CHIP_RV250) ||
690 			    (rdev->family == CHIP_RV280)) {
691 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
692 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
693 
694 				/* RV200::A11 A12 RV250::A11 A12 */
695 				if (((rdev->family == CHIP_RV200) ||
696 				     (rdev->family == CHIP_RV250)) &&
697 				    ((RREG32(RADEON_CONFIG_CNTL) &
698 				      RADEON_CFG_ATI_REV_ID_MASK) <
699 				     RADEON_CFG_ATI_REV_A13)) {
700 					tmp |= RADEON_SCLK_MORE_FORCEON;
701 				}
702 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
703 				mdelay(15);
704 			}
705 
706 			/* RV200::A11 A12, RV250::A11 A12 */
707 			if (((rdev->family == CHIP_RV200) ||
708 			     (rdev->family == CHIP_RV250)) &&
709 			    ((RREG32(RADEON_CONFIG_CNTL) &
710 			      RADEON_CFG_ATI_REV_ID_MASK) <
711 			     RADEON_CFG_ATI_REV_A13)) {
712 				tmp = RREG32_PLL(RADEON_PLL_PWRMGT_CNTL);
713 				tmp |= RADEON_TCL_BYPASS_DISABLE;
714 				WREG32_PLL(RADEON_PLL_PWRMGT_CNTL, tmp);
715 			}
716 			mdelay(15);
717 
718 			/*enable dynamic mode for display clocks (PIXCLK and PIX2CLK) */
719 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
720 			tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
721 				RADEON_PIX2CLK_DAC_ALWAYS_ONb |
722 				RADEON_PIXCLK_BLEND_ALWAYS_ONb |
723 				RADEON_PIXCLK_GV_ALWAYS_ONb |
724 				RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
725 				RADEON_PIXCLK_LVDS_ALWAYS_ONb |
726 				RADEON_PIXCLK_TMDS_ALWAYS_ONb);
727 
728 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
729 			mdelay(15);
730 
731 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
732 			tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
733 				RADEON_PIXCLK_DAC_ALWAYS_ONb);
734 
735 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
736 			mdelay(15);
737 		}
738 	} else {
739 		/* Turn everything OFF (ForceON to everything) */
740 		if (rdev->flags & RADEON_SINGLE_CRTC) {
741 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
742 			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_HDP |
743 				RADEON_SCLK_FORCE_DISP1 | RADEON_SCLK_FORCE_TOP
744 				| RADEON_SCLK_FORCE_E2 | RADEON_SCLK_FORCE_SE |
745 				RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_VIP |
746 				RADEON_SCLK_FORCE_RE | RADEON_SCLK_FORCE_PB |
747 				RADEON_SCLK_FORCE_TAM | RADEON_SCLK_FORCE_TDM |
748 				RADEON_SCLK_FORCE_RB);
749 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
750 		} else if ((rdev->family == CHIP_RS400) ||
751 			   (rdev->family == CHIP_RS480)) {
752 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
753 			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
754 				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
755 				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
756 				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
757 				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
758 				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
759 				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
760 				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
761 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
762 
763 			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
764 			tmp |= RADEON_SCLK_MORE_FORCEON;
765 			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
766 
767 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
768 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
769 				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
770 				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
771 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
772 
773 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
774 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
775 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
776 				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
777 				 R300_DVOCLK_ALWAYS_ONb |
778 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
779 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
780 				 R300_PIXCLK_DVO_ALWAYS_ONb |
781 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
782 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
783 				 R300_PIXCLK_TRANS_ALWAYS_ONb |
784 				 R300_PIXCLK_TVO_ALWAYS_ONb |
785 				 R300_P2G2CLK_ALWAYS_ONb |
786 				 R300_P2G2CLK_DAC_ALWAYS_ONb |
787 				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
788 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
789 		} else if (rdev->family >= CHIP_RV350) {
790 			/* for RV350/M10, no delays are required. */
791 			tmp = RREG32_PLL(R300_SCLK_CNTL2);
792 			tmp |= (R300_SCLK_FORCE_TCL |
793 				R300_SCLK_FORCE_GA | R300_SCLK_FORCE_CBA);
794 			WREG32_PLL(R300_SCLK_CNTL2, tmp);
795 
796 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
797 			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
798 				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
799 				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
800 				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
801 				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
802 				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
803 				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
804 				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
805 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
806 
807 			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
808 			tmp |= RADEON_SCLK_MORE_FORCEON;
809 			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
810 
811 			tmp = RREG32_PLL(RADEON_MCLK_CNTL);
812 			tmp |= (RADEON_FORCEON_MCLKA |
813 				RADEON_FORCEON_MCLKB |
814 				RADEON_FORCEON_YCLKA |
815 				RADEON_FORCEON_YCLKB | RADEON_FORCEON_MC);
816 			WREG32_PLL(RADEON_MCLK_CNTL, tmp);
817 
818 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
819 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
820 				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
821 				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
822 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
823 
824 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
825 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
826 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
827 				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
828 				 R300_DVOCLK_ALWAYS_ONb |
829 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
830 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
831 				 R300_PIXCLK_DVO_ALWAYS_ONb |
832 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
833 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
834 				 R300_PIXCLK_TRANS_ALWAYS_ONb |
835 				 R300_PIXCLK_TVO_ALWAYS_ONb |
836 				 R300_P2G2CLK_ALWAYS_ONb |
837 				 R300_P2G2CLK_DAC_ALWAYS_ONb |
838 				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
839 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
840 		} else {
841 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
842 			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_E2);
843 			tmp |= RADEON_SCLK_FORCE_SE;
844 
845 			if (rdev->flags & RADEON_SINGLE_CRTC) {
846 				tmp |= (RADEON_SCLK_FORCE_RB |
847 					RADEON_SCLK_FORCE_TDM |
848 					RADEON_SCLK_FORCE_TAM |
849 					RADEON_SCLK_FORCE_PB |
850 					RADEON_SCLK_FORCE_RE |
851 					RADEON_SCLK_FORCE_VIP |
852 					RADEON_SCLK_FORCE_IDCT |
853 					RADEON_SCLK_FORCE_TOP |
854 					RADEON_SCLK_FORCE_DISP1 |
855 					RADEON_SCLK_FORCE_DISP2 |
856 					RADEON_SCLK_FORCE_HDP);
857 			} else if ((rdev->family == CHIP_R300) ||
858 				   (rdev->family == CHIP_R350)) {
859 				tmp |= (RADEON_SCLK_FORCE_HDP |
860 					RADEON_SCLK_FORCE_DISP1 |
861 					RADEON_SCLK_FORCE_DISP2 |
862 					RADEON_SCLK_FORCE_TOP |
863 					RADEON_SCLK_FORCE_IDCT |
864 					RADEON_SCLK_FORCE_VIP);
865 			}
866 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
867 
868 			mdelay(16);
869 
870 			if ((rdev->family == CHIP_R300) ||
871 			    (rdev->family == CHIP_R350)) {
872 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
873 				tmp |= (R300_SCLK_FORCE_TCL |
874 					R300_SCLK_FORCE_GA |
875 					R300_SCLK_FORCE_CBA);
876 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
877 				mdelay(16);
878 			}
879 
880 			if (rdev->flags & RADEON_IS_IGP) {
881 				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
882 				tmp &= ~(RADEON_FORCEON_MCLKA |
883 					 RADEON_FORCEON_YCLKA);
884 				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
885 				mdelay(16);
886 			}
887 
888 			if ((rdev->family == CHIP_RV200) ||
889 			    (rdev->family == CHIP_RV250) ||
890 			    (rdev->family == CHIP_RV280)) {
891 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
892 				tmp |= RADEON_SCLK_MORE_FORCEON;
893 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
894 				mdelay(16);
895 			}
896 
897 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
898 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
899 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
900 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
901 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
902 				 RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
903 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
904 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb);
905 
906 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
907 			mdelay(16);
908 
909 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
910 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
911 				 RADEON_PIXCLK_DAC_ALWAYS_ONb);
912 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
913 		}
914 	}
915 }
916 
917