1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 
29 #include <linux/pci.h>
30 
31 #include <drm/drm_device.h>
32 #include <drm/radeon_drm.h>
33 
34 #include "atom.h"
35 #include "radeon.h"
36 #include "radeon_asic.h"
37 #include "radeon_reg.h"
38 
39 /* 10 khz */
40 uint32_t radeon_legacy_get_engine_clock(struct radeon_device *rdev)
41 {
42 	struct radeon_pll *spll = &rdev->clock.spll;
43 	uint32_t fb_div, ref_div, post_div, sclk;
44 
45 	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
46 	fb_div = (fb_div >> RADEON_SPLL_FB_DIV_SHIFT) & RADEON_SPLL_FB_DIV_MASK;
47 	fb_div <<= 1;
48 	fb_div *= spll->reference_freq;
49 
50 	ref_div =
51 	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
52 
53 	if (ref_div == 0)
54 		return 0;
55 
56 	sclk = fb_div / ref_div;
57 
58 	post_div = RREG32_PLL(RADEON_SCLK_CNTL) & RADEON_SCLK_SRC_SEL_MASK;
59 	if (post_div == 2)
60 		sclk >>= 1;
61 	else if (post_div == 3)
62 		sclk >>= 2;
63 	else if (post_div == 4)
64 		sclk >>= 3;
65 
66 	return sclk;
67 }
68 
69 /* 10 khz */
70 uint32_t radeon_legacy_get_memory_clock(struct radeon_device *rdev)
71 {
72 	struct radeon_pll *mpll = &rdev->clock.mpll;
73 	uint32_t fb_div, ref_div, post_div, mclk;
74 
75 	fb_div = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
76 	fb_div = (fb_div >> RADEON_MPLL_FB_DIV_SHIFT) & RADEON_MPLL_FB_DIV_MASK;
77 	fb_div <<= 1;
78 	fb_div *= mpll->reference_freq;
79 
80 	ref_div =
81 	    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) & RADEON_M_SPLL_REF_DIV_MASK;
82 
83 	if (ref_div == 0)
84 		return 0;
85 
86 	mclk = fb_div / ref_div;
87 
88 	post_div = RREG32_PLL(RADEON_MCLK_CNTL) & 0x7;
89 	if (post_div == 2)
90 		mclk >>= 1;
91 	else if (post_div == 3)
92 		mclk >>= 2;
93 	else if (post_div == 4)
94 		mclk >>= 3;
95 
96 	return mclk;
97 }
98 
99 #ifdef CONFIG_OF
100 /*
101  * Read XTAL (ref clock), SCLK and MCLK from Open Firmware device
102  * tree. Hopefully, ATI OF driver is kind enough to fill these
103  */
104 static bool radeon_read_clocks_OF(struct drm_device *dev)
105 {
106 	struct radeon_device *rdev = dev->dev_private;
107 	struct device_node *dp = rdev->pdev->dev.of_node;
108 	const u32 *val;
109 	struct radeon_pll *p1pll = &rdev->clock.p1pll;
110 	struct radeon_pll *p2pll = &rdev->clock.p2pll;
111 	struct radeon_pll *spll = &rdev->clock.spll;
112 	struct radeon_pll *mpll = &rdev->clock.mpll;
113 
114 	if (dp == NULL)
115 		return false;
116 	val = of_get_property(dp, "ATY,RefCLK", NULL);
117 	if (!val || !*val) {
118 		pr_warn("radeonfb: No ATY,RefCLK property !\n");
119 		return false;
120 	}
121 	p1pll->reference_freq = p2pll->reference_freq = (*val) / 10;
122 	p1pll->reference_div = RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
123 	if (p1pll->reference_div < 2)
124 		p1pll->reference_div = 12;
125 	p2pll->reference_div = p1pll->reference_div;
126 
127 	/* These aren't in the device-tree */
128 	if (rdev->family >= CHIP_R420) {
129 		p1pll->pll_in_min = 100;
130 		p1pll->pll_in_max = 1350;
131 		p1pll->pll_out_min = 20000;
132 		p1pll->pll_out_max = 50000;
133 		p2pll->pll_in_min = 100;
134 		p2pll->pll_in_max = 1350;
135 		p2pll->pll_out_min = 20000;
136 		p2pll->pll_out_max = 50000;
137 	} else {
138 		p1pll->pll_in_min = 40;
139 		p1pll->pll_in_max = 500;
140 		p1pll->pll_out_min = 12500;
141 		p1pll->pll_out_max = 35000;
142 		p2pll->pll_in_min = 40;
143 		p2pll->pll_in_max = 500;
144 		p2pll->pll_out_min = 12500;
145 		p2pll->pll_out_max = 35000;
146 	}
147 	/* not sure what the max should be in all cases */
148 	rdev->clock.max_pixel_clock = 35000;
149 
150 	spll->reference_freq = mpll->reference_freq = p1pll->reference_freq;
151 	spll->reference_div = mpll->reference_div =
152 		RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
153 			    RADEON_M_SPLL_REF_DIV_MASK;
154 
155 	val = of_get_property(dp, "ATY,SCLK", NULL);
156 	if (val && *val)
157 		rdev->clock.default_sclk = (*val) / 10;
158 	else
159 		rdev->clock.default_sclk =
160 			radeon_legacy_get_engine_clock(rdev);
161 
162 	val = of_get_property(dp, "ATY,MCLK", NULL);
163 	if (val && *val)
164 		rdev->clock.default_mclk = (*val) / 10;
165 	else
166 		rdev->clock.default_mclk =
167 			radeon_legacy_get_memory_clock(rdev);
168 
169 	DRM_INFO("Using device-tree clock info\n");
170 
171 	return true;
172 }
173 #else
174 static bool radeon_read_clocks_OF(struct drm_device *dev)
175 {
176 	return false;
177 }
178 #endif /* CONFIG_OF */
179 
180 void radeon_get_clock_info(struct drm_device *dev)
181 {
182 	struct radeon_device *rdev = dev->dev_private;
183 	struct radeon_pll *p1pll = &rdev->clock.p1pll;
184 	struct radeon_pll *p2pll = &rdev->clock.p2pll;
185 	struct radeon_pll *dcpll = &rdev->clock.dcpll;
186 	struct radeon_pll *spll = &rdev->clock.spll;
187 	struct radeon_pll *mpll = &rdev->clock.mpll;
188 	int ret;
189 
190 	if (rdev->is_atom_bios)
191 		ret = radeon_atom_get_clock_info(dev);
192 	else
193 		ret = radeon_combios_get_clock_info(dev);
194 	if (!ret)
195 		ret = radeon_read_clocks_OF(dev);
196 
197 	if (ret) {
198 		if (p1pll->reference_div < 2) {
199 			if (!ASIC_IS_AVIVO(rdev)) {
200 				u32 tmp = RREG32_PLL(RADEON_PPLL_REF_DIV);
201 				if (ASIC_IS_R300(rdev))
202 					p1pll->reference_div =
203 						(tmp & R300_PPLL_REF_DIV_ACC_MASK) >> R300_PPLL_REF_DIV_ACC_SHIFT;
204 				else
205 					p1pll->reference_div = tmp & RADEON_PPLL_REF_DIV_MASK;
206 				if (p1pll->reference_div < 2)
207 					p1pll->reference_div = 12;
208 			} else
209 				p1pll->reference_div = 12;
210 		}
211 		if (p2pll->reference_div < 2)
212 			p2pll->reference_div = 12;
213 		if (rdev->family < CHIP_RS600) {
214 			if (spll->reference_div < 2)
215 				spll->reference_div =
216 					RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
217 					RADEON_M_SPLL_REF_DIV_MASK;
218 		}
219 		if (mpll->reference_div < 2)
220 			mpll->reference_div = spll->reference_div;
221 	} else {
222 		if (ASIC_IS_AVIVO(rdev)) {
223 			/* TODO FALLBACK */
224 		} else {
225 			DRM_INFO("Using generic clock info\n");
226 
227 			/* may need to be per card */
228 			rdev->clock.max_pixel_clock = 35000;
229 
230 			if (rdev->flags & RADEON_IS_IGP) {
231 				p1pll->reference_freq = 1432;
232 				p2pll->reference_freq = 1432;
233 				spll->reference_freq = 1432;
234 				mpll->reference_freq = 1432;
235 			} else {
236 				p1pll->reference_freq = 2700;
237 				p2pll->reference_freq = 2700;
238 				spll->reference_freq = 2700;
239 				mpll->reference_freq = 2700;
240 			}
241 			p1pll->reference_div =
242 			    RREG32_PLL(RADEON_PPLL_REF_DIV) & 0x3ff;
243 			if (p1pll->reference_div < 2)
244 				p1pll->reference_div = 12;
245 			p2pll->reference_div = p1pll->reference_div;
246 
247 			if (rdev->family >= CHIP_R420) {
248 				p1pll->pll_in_min = 100;
249 				p1pll->pll_in_max = 1350;
250 				p1pll->pll_out_min = 20000;
251 				p1pll->pll_out_max = 50000;
252 				p2pll->pll_in_min = 100;
253 				p2pll->pll_in_max = 1350;
254 				p2pll->pll_out_min = 20000;
255 				p2pll->pll_out_max = 50000;
256 			} else {
257 				p1pll->pll_in_min = 40;
258 				p1pll->pll_in_max = 500;
259 				p1pll->pll_out_min = 12500;
260 				p1pll->pll_out_max = 35000;
261 				p2pll->pll_in_min = 40;
262 				p2pll->pll_in_max = 500;
263 				p2pll->pll_out_min = 12500;
264 				p2pll->pll_out_max = 35000;
265 			}
266 
267 			spll->reference_div =
268 			    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
269 			    RADEON_M_SPLL_REF_DIV_MASK;
270 			mpll->reference_div = spll->reference_div;
271 			rdev->clock.default_sclk =
272 			    radeon_legacy_get_engine_clock(rdev);
273 			rdev->clock.default_mclk =
274 			    radeon_legacy_get_memory_clock(rdev);
275 		}
276 	}
277 
278 	/* pixel clocks */
279 	if (ASIC_IS_AVIVO(rdev)) {
280 		p1pll->min_post_div = 2;
281 		p1pll->max_post_div = 0x7f;
282 		p1pll->min_frac_feedback_div = 0;
283 		p1pll->max_frac_feedback_div = 9;
284 		p2pll->min_post_div = 2;
285 		p2pll->max_post_div = 0x7f;
286 		p2pll->min_frac_feedback_div = 0;
287 		p2pll->max_frac_feedback_div = 9;
288 	} else {
289 		p1pll->min_post_div = 1;
290 		p1pll->max_post_div = 16;
291 		p1pll->min_frac_feedback_div = 0;
292 		p1pll->max_frac_feedback_div = 0;
293 		p2pll->min_post_div = 1;
294 		p2pll->max_post_div = 12;
295 		p2pll->min_frac_feedback_div = 0;
296 		p2pll->max_frac_feedback_div = 0;
297 	}
298 
299 	/* dcpll is DCE4 only */
300 	dcpll->min_post_div = 2;
301 	dcpll->max_post_div = 0x7f;
302 	dcpll->min_frac_feedback_div = 0;
303 	dcpll->max_frac_feedback_div = 9;
304 	dcpll->min_ref_div = 2;
305 	dcpll->max_ref_div = 0x3ff;
306 	dcpll->min_feedback_div = 4;
307 	dcpll->max_feedback_div = 0xfff;
308 	dcpll->best_vco = 0;
309 
310 	p1pll->min_ref_div = 2;
311 	p1pll->max_ref_div = 0x3ff;
312 	p1pll->min_feedback_div = 4;
313 	p1pll->max_feedback_div = 0x7ff;
314 	p1pll->best_vco = 0;
315 
316 	p2pll->min_ref_div = 2;
317 	p2pll->max_ref_div = 0x3ff;
318 	p2pll->min_feedback_div = 4;
319 	p2pll->max_feedback_div = 0x7ff;
320 	p2pll->best_vco = 0;
321 
322 	/* system clock */
323 	spll->min_post_div = 1;
324 	spll->max_post_div = 1;
325 	spll->min_ref_div = 2;
326 	spll->max_ref_div = 0xff;
327 	spll->min_feedback_div = 4;
328 	spll->max_feedback_div = 0xff;
329 	spll->best_vco = 0;
330 
331 	/* memory clock */
332 	mpll->min_post_div = 1;
333 	mpll->max_post_div = 1;
334 	mpll->min_ref_div = 2;
335 	mpll->max_ref_div = 0xff;
336 	mpll->min_feedback_div = 4;
337 	mpll->max_feedback_div = 0xff;
338 	mpll->best_vco = 0;
339 
340 	if (!rdev->clock.default_sclk)
341 		rdev->clock.default_sclk = radeon_get_engine_clock(rdev);
342 	if ((!rdev->clock.default_mclk) && rdev->asic->pm.get_memory_clock)
343 		rdev->clock.default_mclk = radeon_get_memory_clock(rdev);
344 
345 	rdev->pm.current_sclk = rdev->clock.default_sclk;
346 	rdev->pm.current_mclk = rdev->clock.default_mclk;
347 
348 }
349 
350 /* 10 khz */
351 static uint32_t calc_eng_mem_clock(struct radeon_device *rdev,
352 				   uint32_t req_clock,
353 				   int *fb_div, int *post_div)
354 {
355 	struct radeon_pll *spll = &rdev->clock.spll;
356 	int ref_div = spll->reference_div;
357 
358 	if (!ref_div)
359 		ref_div =
360 		    RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV) &
361 		    RADEON_M_SPLL_REF_DIV_MASK;
362 
363 	if (req_clock < 15000) {
364 		*post_div = 8;
365 		req_clock *= 8;
366 	} else if (req_clock < 30000) {
367 		*post_div = 4;
368 		req_clock *= 4;
369 	} else if (req_clock < 60000) {
370 		*post_div = 2;
371 		req_clock *= 2;
372 	} else
373 		*post_div = 1;
374 
375 	req_clock *= ref_div;
376 	req_clock += spll->reference_freq;
377 	req_clock /= (2 * spll->reference_freq);
378 
379 	*fb_div = req_clock & 0xff;
380 
381 	req_clock = (req_clock & 0xffff) << 1;
382 	req_clock *= spll->reference_freq;
383 	req_clock /= ref_div;
384 	req_clock /= *post_div;
385 
386 	return req_clock;
387 }
388 
389 /* 10 khz */
390 void radeon_legacy_set_engine_clock(struct radeon_device *rdev,
391 				    uint32_t eng_clock)
392 {
393 	uint32_t tmp;
394 	int fb_div, post_div;
395 
396 	/* XXX: wait for idle */
397 
398 	eng_clock = calc_eng_mem_clock(rdev, eng_clock, &fb_div, &post_div);
399 
400 	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
401 	tmp &= ~RADEON_DONT_USE_XTALIN;
402 	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
403 
404 	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
405 	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
406 	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
407 
408 	udelay(10);
409 
410 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
411 	tmp |= RADEON_SPLL_SLEEP;
412 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
413 
414 	udelay(2);
415 
416 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
417 	tmp |= RADEON_SPLL_RESET;
418 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
419 
420 	udelay(200);
421 
422 	tmp = RREG32_PLL(RADEON_M_SPLL_REF_FB_DIV);
423 	tmp &= ~(RADEON_SPLL_FB_DIV_MASK << RADEON_SPLL_FB_DIV_SHIFT);
424 	tmp |= (fb_div & RADEON_SPLL_FB_DIV_MASK) << RADEON_SPLL_FB_DIV_SHIFT;
425 	WREG32_PLL(RADEON_M_SPLL_REF_FB_DIV, tmp);
426 
427 	/* XXX: verify on different asics */
428 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
429 	tmp &= ~RADEON_SPLL_PVG_MASK;
430 	if ((eng_clock * post_div) >= 90000)
431 		tmp |= (0x7 << RADEON_SPLL_PVG_SHIFT);
432 	else
433 		tmp |= (0x4 << RADEON_SPLL_PVG_SHIFT);
434 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
435 
436 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
437 	tmp &= ~RADEON_SPLL_SLEEP;
438 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
439 
440 	udelay(2);
441 
442 	tmp = RREG32_PLL(RADEON_SPLL_CNTL);
443 	tmp &= ~RADEON_SPLL_RESET;
444 	WREG32_PLL(RADEON_SPLL_CNTL, tmp);
445 
446 	udelay(200);
447 
448 	tmp = RREG32_PLL(RADEON_SCLK_CNTL);
449 	tmp &= ~RADEON_SCLK_SRC_SEL_MASK;
450 	switch (post_div) {
451 	case 1:
452 	default:
453 		tmp |= 1;
454 		break;
455 	case 2:
456 		tmp |= 2;
457 		break;
458 	case 4:
459 		tmp |= 3;
460 		break;
461 	case 8:
462 		tmp |= 4;
463 		break;
464 	}
465 	WREG32_PLL(RADEON_SCLK_CNTL, tmp);
466 
467 	udelay(20);
468 
469 	tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
470 	tmp |= RADEON_DONT_USE_XTALIN;
471 	WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
472 
473 	udelay(10);
474 }
475 
476 void radeon_legacy_set_clock_gating(struct radeon_device *rdev, int enable)
477 {
478 	uint32_t tmp;
479 
480 	if (enable) {
481 		if (rdev->flags & RADEON_SINGLE_CRTC) {
482 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
483 			if ((RREG32(RADEON_CONFIG_CNTL) &
484 			     RADEON_CFG_ATI_REV_ID_MASK) >
485 			    RADEON_CFG_ATI_REV_A13) {
486 				tmp &=
487 				    ~(RADEON_SCLK_FORCE_CP |
488 				      RADEON_SCLK_FORCE_RB);
489 			}
490 			tmp &=
491 			    ~(RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1 |
492 			      RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_SE |
493 			      RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_RE |
494 			      RADEON_SCLK_FORCE_PB | RADEON_SCLK_FORCE_TAM |
495 			      RADEON_SCLK_FORCE_TDM);
496 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
497 		} else if (ASIC_IS_R300(rdev)) {
498 			if ((rdev->family == CHIP_RS400) ||
499 			    (rdev->family == CHIP_RS480)) {
500 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
501 				tmp &=
502 				    ~(RADEON_SCLK_FORCE_DISP2 |
503 				      RADEON_SCLK_FORCE_CP |
504 				      RADEON_SCLK_FORCE_HDP |
505 				      RADEON_SCLK_FORCE_DISP1 |
506 				      RADEON_SCLK_FORCE_TOP |
507 				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
508 				      | RADEON_SCLK_FORCE_IDCT |
509 				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
510 				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
511 				      | R300_SCLK_FORCE_US |
512 				      RADEON_SCLK_FORCE_TV_SCLK |
513 				      R300_SCLK_FORCE_SU |
514 				      RADEON_SCLK_FORCE_OV0);
515 				tmp |= RADEON_DYN_STOP_LAT_MASK;
516 				tmp |=
517 				    RADEON_SCLK_FORCE_TOP |
518 				    RADEON_SCLK_FORCE_VIP;
519 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
520 
521 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
522 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
523 				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
524 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
525 
526 				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
527 				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
528 					RADEON_PIXCLK_DAC_ALWAYS_ONb);
529 				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
530 
531 				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
532 				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
533 					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
534 					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
535 					R300_DVOCLK_ALWAYS_ONb |
536 					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
537 					RADEON_PIXCLK_GV_ALWAYS_ONb |
538 					R300_PIXCLK_DVO_ALWAYS_ONb |
539 					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
540 					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
541 					R300_PIXCLK_TRANS_ALWAYS_ONb |
542 					R300_PIXCLK_TVO_ALWAYS_ONb |
543 					R300_P2G2CLK_ALWAYS_ONb |
544 					R300_P2G2CLK_DAC_ALWAYS_ONb);
545 				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
546 			} else if (rdev->family >= CHIP_RV350) {
547 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
548 				tmp &= ~(R300_SCLK_FORCE_TCL |
549 					 R300_SCLK_FORCE_GA |
550 					 R300_SCLK_FORCE_CBA);
551 				tmp |= (R300_SCLK_TCL_MAX_DYN_STOP_LAT |
552 					R300_SCLK_GA_MAX_DYN_STOP_LAT |
553 					R300_SCLK_CBA_MAX_DYN_STOP_LAT);
554 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
555 
556 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
557 				tmp &=
558 				    ~(RADEON_SCLK_FORCE_DISP2 |
559 				      RADEON_SCLK_FORCE_CP |
560 				      RADEON_SCLK_FORCE_HDP |
561 				      RADEON_SCLK_FORCE_DISP1 |
562 				      RADEON_SCLK_FORCE_TOP |
563 				      RADEON_SCLK_FORCE_E2 | R300_SCLK_FORCE_VAP
564 				      | RADEON_SCLK_FORCE_IDCT |
565 				      RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR
566 				      | R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX
567 				      | R300_SCLK_FORCE_US |
568 				      RADEON_SCLK_FORCE_TV_SCLK |
569 				      R300_SCLK_FORCE_SU |
570 				      RADEON_SCLK_FORCE_OV0);
571 				tmp |= RADEON_DYN_STOP_LAT_MASK;
572 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
573 
574 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
575 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
576 				tmp |= RADEON_SCLK_MORE_MAX_DYN_STOP_LAT;
577 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
578 
579 				tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
580 				tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
581 					RADEON_PIXCLK_DAC_ALWAYS_ONb);
582 				WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
583 
584 				tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
585 				tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
586 					RADEON_PIX2CLK_DAC_ALWAYS_ONb |
587 					RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
588 					R300_DVOCLK_ALWAYS_ONb |
589 					RADEON_PIXCLK_BLEND_ALWAYS_ONb |
590 					RADEON_PIXCLK_GV_ALWAYS_ONb |
591 					R300_PIXCLK_DVO_ALWAYS_ONb |
592 					RADEON_PIXCLK_LVDS_ALWAYS_ONb |
593 					RADEON_PIXCLK_TMDS_ALWAYS_ONb |
594 					R300_PIXCLK_TRANS_ALWAYS_ONb |
595 					R300_PIXCLK_TVO_ALWAYS_ONb |
596 					R300_P2G2CLK_ALWAYS_ONb |
597 					R300_P2G2CLK_DAC_ALWAYS_ONb);
598 				WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
599 
600 				tmp = RREG32_PLL(RADEON_MCLK_MISC);
601 				tmp |= (RADEON_MC_MCLK_DYN_ENABLE |
602 					RADEON_IO_MCLK_DYN_ENABLE);
603 				WREG32_PLL(RADEON_MCLK_MISC, tmp);
604 
605 				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
606 				tmp |= (RADEON_FORCEON_MCLKA |
607 					RADEON_FORCEON_MCLKB);
608 
609 				tmp &= ~(RADEON_FORCEON_YCLKA |
610 					 RADEON_FORCEON_YCLKB |
611 					 RADEON_FORCEON_MC);
612 
613 				/* Some releases of vbios have set DISABLE_MC_MCLKA
614 				   and DISABLE_MC_MCLKB bits in the vbios table.  Setting these
615 				   bits will cause H/W hang when reading video memory with dynamic clocking
616 				   enabled. */
617 				if ((tmp & R300_DISABLE_MC_MCLKA) &&
618 				    (tmp & R300_DISABLE_MC_MCLKB)) {
619 					/* If both bits are set, then check the active channels */
620 					tmp = RREG32_PLL(RADEON_MCLK_CNTL);
621 					if (rdev->mc.vram_width == 64) {
622 						if (RREG32(RADEON_MEM_CNTL) &
623 						    R300_MEM_USE_CD_CH_ONLY)
624 							tmp &=
625 							    ~R300_DISABLE_MC_MCLKB;
626 						else
627 							tmp &=
628 							    ~R300_DISABLE_MC_MCLKA;
629 					} else {
630 						tmp &= ~(R300_DISABLE_MC_MCLKA |
631 							 R300_DISABLE_MC_MCLKB);
632 					}
633 				}
634 
635 				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
636 			} else {
637 				tmp = RREG32_PLL(RADEON_SCLK_CNTL);
638 				tmp &= ~(R300_SCLK_FORCE_VAP);
639 				tmp |= RADEON_SCLK_FORCE_CP;
640 				WREG32_PLL(RADEON_SCLK_CNTL, tmp);
641 				mdelay(15);
642 
643 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
644 				tmp &= ~(R300_SCLK_FORCE_TCL |
645 					 R300_SCLK_FORCE_GA |
646 					 R300_SCLK_FORCE_CBA);
647 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
648 			}
649 		} else {
650 			tmp = RREG32_PLL(RADEON_CLK_PWRMGT_CNTL);
651 
652 			tmp &= ~(RADEON_ACTIVE_HILO_LAT_MASK |
653 				 RADEON_DISP_DYN_STOP_LAT_MASK |
654 				 RADEON_DYN_STOP_MODE_MASK);
655 
656 			tmp |= (RADEON_ENGIN_DYNCLK_MODE |
657 				(0x01 << RADEON_ACTIVE_HILO_LAT_SHIFT));
658 			WREG32_PLL(RADEON_CLK_PWRMGT_CNTL, tmp);
659 			mdelay(15);
660 
661 			tmp = RREG32_PLL(RADEON_CLK_PIN_CNTL);
662 			tmp |= RADEON_SCLK_DYN_START_CNTL;
663 			WREG32_PLL(RADEON_CLK_PIN_CNTL, tmp);
664 			mdelay(15);
665 
666 			/* When DRI is enabled, setting DYN_STOP_LAT to zero can cause some R200
667 			   to lockup randomly, leave them as set by BIOS.
668 			 */
669 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
670 			/*tmp &= RADEON_SCLK_SRC_SEL_MASK; */
671 			tmp &= ~RADEON_SCLK_FORCEON_MASK;
672 
673 			/*RAGE_6::A11 A12 A12N1 A13, RV250::A11 A12, R300 */
674 			if (((rdev->family == CHIP_RV250) &&
675 			     ((RREG32(RADEON_CONFIG_CNTL) &
676 			       RADEON_CFG_ATI_REV_ID_MASK) <
677 			      RADEON_CFG_ATI_REV_A13))
678 			    || ((rdev->family == CHIP_RV100)
679 				&&
680 				((RREG32(RADEON_CONFIG_CNTL) &
681 				  RADEON_CFG_ATI_REV_ID_MASK) <=
682 				 RADEON_CFG_ATI_REV_A13))) {
683 				tmp |= RADEON_SCLK_FORCE_CP;
684 				tmp |= RADEON_SCLK_FORCE_VIP;
685 			}
686 
687 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
688 
689 			if ((rdev->family == CHIP_RV200) ||
690 			    (rdev->family == CHIP_RV250) ||
691 			    (rdev->family == CHIP_RV280)) {
692 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
693 				tmp &= ~RADEON_SCLK_MORE_FORCEON;
694 
695 				/* RV200::A11 A12 RV250::A11 A12 */
696 				if (((rdev->family == CHIP_RV200) ||
697 				     (rdev->family == CHIP_RV250)) &&
698 				    ((RREG32(RADEON_CONFIG_CNTL) &
699 				      RADEON_CFG_ATI_REV_ID_MASK) <
700 				     RADEON_CFG_ATI_REV_A13)) {
701 					tmp |= RADEON_SCLK_MORE_FORCEON;
702 				}
703 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
704 				mdelay(15);
705 			}
706 
707 			/* RV200::A11 A12, RV250::A11 A12 */
708 			if (((rdev->family == CHIP_RV200) ||
709 			     (rdev->family == CHIP_RV250)) &&
710 			    ((RREG32(RADEON_CONFIG_CNTL) &
711 			      RADEON_CFG_ATI_REV_ID_MASK) <
712 			     RADEON_CFG_ATI_REV_A13)) {
713 				tmp = RREG32_PLL(RADEON_PLL_PWRMGT_CNTL);
714 				tmp |= RADEON_TCL_BYPASS_DISABLE;
715 				WREG32_PLL(RADEON_PLL_PWRMGT_CNTL, tmp);
716 			}
717 			mdelay(15);
718 
719 			/*enable dynamic mode for display clocks (PIXCLK and PIX2CLK) */
720 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
721 			tmp |= (RADEON_PIX2CLK_ALWAYS_ONb |
722 				RADEON_PIX2CLK_DAC_ALWAYS_ONb |
723 				RADEON_PIXCLK_BLEND_ALWAYS_ONb |
724 				RADEON_PIXCLK_GV_ALWAYS_ONb |
725 				RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
726 				RADEON_PIXCLK_LVDS_ALWAYS_ONb |
727 				RADEON_PIXCLK_TMDS_ALWAYS_ONb);
728 
729 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
730 			mdelay(15);
731 
732 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
733 			tmp |= (RADEON_PIXCLK_ALWAYS_ONb |
734 				RADEON_PIXCLK_DAC_ALWAYS_ONb);
735 
736 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
737 			mdelay(15);
738 		}
739 	} else {
740 		/* Turn everything OFF (ForceON to everything) */
741 		if (rdev->flags & RADEON_SINGLE_CRTC) {
742 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
743 			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_HDP |
744 				RADEON_SCLK_FORCE_DISP1 | RADEON_SCLK_FORCE_TOP
745 				| RADEON_SCLK_FORCE_E2 | RADEON_SCLK_FORCE_SE |
746 				RADEON_SCLK_FORCE_IDCT | RADEON_SCLK_FORCE_VIP |
747 				RADEON_SCLK_FORCE_RE | RADEON_SCLK_FORCE_PB |
748 				RADEON_SCLK_FORCE_TAM | RADEON_SCLK_FORCE_TDM |
749 				RADEON_SCLK_FORCE_RB);
750 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
751 		} else if ((rdev->family == CHIP_RS400) ||
752 			   (rdev->family == CHIP_RS480)) {
753 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
754 			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
755 				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
756 				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
757 				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
758 				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
759 				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
760 				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
761 				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
762 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
763 
764 			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
765 			tmp |= RADEON_SCLK_MORE_FORCEON;
766 			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
767 
768 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
769 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
770 				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
771 				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
772 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
773 
774 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
775 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
776 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
777 				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
778 				 R300_DVOCLK_ALWAYS_ONb |
779 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
780 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
781 				 R300_PIXCLK_DVO_ALWAYS_ONb |
782 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
783 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
784 				 R300_PIXCLK_TRANS_ALWAYS_ONb |
785 				 R300_PIXCLK_TVO_ALWAYS_ONb |
786 				 R300_P2G2CLK_ALWAYS_ONb |
787 				 R300_P2G2CLK_DAC_ALWAYS_ONb |
788 				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
789 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
790 		} else if (rdev->family >= CHIP_RV350) {
791 			/* for RV350/M10, no delays are required. */
792 			tmp = RREG32_PLL(R300_SCLK_CNTL2);
793 			tmp |= (R300_SCLK_FORCE_TCL |
794 				R300_SCLK_FORCE_GA | R300_SCLK_FORCE_CBA);
795 			WREG32_PLL(R300_SCLK_CNTL2, tmp);
796 
797 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
798 			tmp |= (RADEON_SCLK_FORCE_DISP2 | RADEON_SCLK_FORCE_CP |
799 				RADEON_SCLK_FORCE_HDP | RADEON_SCLK_FORCE_DISP1
800 				| RADEON_SCLK_FORCE_TOP | RADEON_SCLK_FORCE_E2 |
801 				R300_SCLK_FORCE_VAP | RADEON_SCLK_FORCE_IDCT |
802 				RADEON_SCLK_FORCE_VIP | R300_SCLK_FORCE_SR |
803 				R300_SCLK_FORCE_PX | R300_SCLK_FORCE_TX |
804 				R300_SCLK_FORCE_US | RADEON_SCLK_FORCE_TV_SCLK |
805 				R300_SCLK_FORCE_SU | RADEON_SCLK_FORCE_OV0);
806 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
807 
808 			tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
809 			tmp |= RADEON_SCLK_MORE_FORCEON;
810 			WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
811 
812 			tmp = RREG32_PLL(RADEON_MCLK_CNTL);
813 			tmp |= (RADEON_FORCEON_MCLKA |
814 				RADEON_FORCEON_MCLKB |
815 				RADEON_FORCEON_YCLKA |
816 				RADEON_FORCEON_YCLKB | RADEON_FORCEON_MC);
817 			WREG32_PLL(RADEON_MCLK_CNTL, tmp);
818 
819 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
820 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
821 				 RADEON_PIXCLK_DAC_ALWAYS_ONb |
822 				 R300_DISP_DAC_PIXCLK_DAC_BLANK_OFF);
823 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
824 
825 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
826 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
827 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
828 				 RADEON_DISP_TVOUT_PIXCLK_TV_ALWAYS_ONb |
829 				 R300_DVOCLK_ALWAYS_ONb |
830 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
831 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
832 				 R300_PIXCLK_DVO_ALWAYS_ONb |
833 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
834 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb |
835 				 R300_PIXCLK_TRANS_ALWAYS_ONb |
836 				 R300_PIXCLK_TVO_ALWAYS_ONb |
837 				 R300_P2G2CLK_ALWAYS_ONb |
838 				 R300_P2G2CLK_DAC_ALWAYS_ONb |
839 				 R300_DISP_DAC_PIXCLK_DAC2_BLANK_OFF);
840 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
841 		} else {
842 			tmp = RREG32_PLL(RADEON_SCLK_CNTL);
843 			tmp |= (RADEON_SCLK_FORCE_CP | RADEON_SCLK_FORCE_E2);
844 			tmp |= RADEON_SCLK_FORCE_SE;
845 
846 			if (rdev->flags & RADEON_SINGLE_CRTC) {
847 				tmp |= (RADEON_SCLK_FORCE_RB |
848 					RADEON_SCLK_FORCE_TDM |
849 					RADEON_SCLK_FORCE_TAM |
850 					RADEON_SCLK_FORCE_PB |
851 					RADEON_SCLK_FORCE_RE |
852 					RADEON_SCLK_FORCE_VIP |
853 					RADEON_SCLK_FORCE_IDCT |
854 					RADEON_SCLK_FORCE_TOP |
855 					RADEON_SCLK_FORCE_DISP1 |
856 					RADEON_SCLK_FORCE_DISP2 |
857 					RADEON_SCLK_FORCE_HDP);
858 			} else if ((rdev->family == CHIP_R300) ||
859 				   (rdev->family == CHIP_R350)) {
860 				tmp |= (RADEON_SCLK_FORCE_HDP |
861 					RADEON_SCLK_FORCE_DISP1 |
862 					RADEON_SCLK_FORCE_DISP2 |
863 					RADEON_SCLK_FORCE_TOP |
864 					RADEON_SCLK_FORCE_IDCT |
865 					RADEON_SCLK_FORCE_VIP);
866 			}
867 			WREG32_PLL(RADEON_SCLK_CNTL, tmp);
868 
869 			mdelay(16);
870 
871 			if ((rdev->family == CHIP_R300) ||
872 			    (rdev->family == CHIP_R350)) {
873 				tmp = RREG32_PLL(R300_SCLK_CNTL2);
874 				tmp |= (R300_SCLK_FORCE_TCL |
875 					R300_SCLK_FORCE_GA |
876 					R300_SCLK_FORCE_CBA);
877 				WREG32_PLL(R300_SCLK_CNTL2, tmp);
878 				mdelay(16);
879 			}
880 
881 			if (rdev->flags & RADEON_IS_IGP) {
882 				tmp = RREG32_PLL(RADEON_MCLK_CNTL);
883 				tmp &= ~(RADEON_FORCEON_MCLKA |
884 					 RADEON_FORCEON_YCLKA);
885 				WREG32_PLL(RADEON_MCLK_CNTL, tmp);
886 				mdelay(16);
887 			}
888 
889 			if ((rdev->family == CHIP_RV200) ||
890 			    (rdev->family == CHIP_RV250) ||
891 			    (rdev->family == CHIP_RV280)) {
892 				tmp = RREG32_PLL(RADEON_SCLK_MORE_CNTL);
893 				tmp |= RADEON_SCLK_MORE_FORCEON;
894 				WREG32_PLL(RADEON_SCLK_MORE_CNTL, tmp);
895 				mdelay(16);
896 			}
897 
898 			tmp = RREG32_PLL(RADEON_PIXCLKS_CNTL);
899 			tmp &= ~(RADEON_PIX2CLK_ALWAYS_ONb |
900 				 RADEON_PIX2CLK_DAC_ALWAYS_ONb |
901 				 RADEON_PIXCLK_BLEND_ALWAYS_ONb |
902 				 RADEON_PIXCLK_GV_ALWAYS_ONb |
903 				 RADEON_PIXCLK_DIG_TMDS_ALWAYS_ONb |
904 				 RADEON_PIXCLK_LVDS_ALWAYS_ONb |
905 				 RADEON_PIXCLK_TMDS_ALWAYS_ONb);
906 
907 			WREG32_PLL(RADEON_PIXCLKS_CNTL, tmp);
908 			mdelay(16);
909 
910 			tmp = RREG32_PLL(RADEON_VCLK_ECP_CNTL);
911 			tmp &= ~(RADEON_PIXCLK_ALWAYS_ONb |
912 				 RADEON_PIXCLK_DAC_ALWAYS_ONb);
913 			WREG32_PLL(RADEON_VCLK_ECP_CNTL, tmp);
914 		}
915 	}
916 }
917 
918