xref: /openbmc/linux/drivers/gpu/drm/radeon/rv770_dpm.c (revision b34e08d5)
1 /*
2  * Copyright 2011 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 
25 #include "drmP.h"
26 #include "radeon.h"
27 #include "rv770d.h"
28 #include "r600_dpm.h"
29 #include "rv770_dpm.h"
30 #include "cypress_dpm.h"
31 #include "atom.h"
32 #include <linux/seq_file.h>
33 
34 #define MC_CG_ARB_FREQ_F0           0x0a
35 #define MC_CG_ARB_FREQ_F1           0x0b
36 #define MC_CG_ARB_FREQ_F2           0x0c
37 #define MC_CG_ARB_FREQ_F3           0x0d
38 
39 #define MC_CG_SEQ_DRAMCONF_S0       0x05
40 #define MC_CG_SEQ_DRAMCONF_S1       0x06
41 
42 #define PCIE_BUS_CLK                10000
43 #define TCLK                        (PCIE_BUS_CLK / 10)
44 
45 #define SMC_RAM_END 0xC000
46 
47 struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps)
48 {
49 	struct rv7xx_ps *ps = rps->ps_priv;
50 
51 	return ps;
52 }
53 
54 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev)
55 {
56 	struct rv7xx_power_info *pi = rdev->pm.dpm.priv;
57 
58 	return pi;
59 }
60 
61 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev)
62 {
63 	struct evergreen_power_info *pi = rdev->pm.dpm.priv;
64 
65 	return pi;
66 }
67 
68 static void rv770_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
69 					       bool enable)
70 {
71 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
72 	u32 tmp;
73 
74 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
75 	if (enable) {
76 		tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
77 		tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
78 		tmp |= LC_GEN2_EN_STRAP;
79 	} else {
80 		if (!pi->boot_in_gen2) {
81 			tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
82 			tmp &= ~LC_GEN2_EN_STRAP;
83 		}
84 	}
85 	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
86 	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
87 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
88 
89 }
90 
91 static void rv770_enable_l0s(struct radeon_device *rdev)
92 {
93 	u32 tmp;
94 
95 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK;
96 	tmp |= LC_L0S_INACTIVITY(3);
97 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
98 }
99 
100 static void rv770_enable_l1(struct radeon_device *rdev)
101 {
102 	u32 tmp;
103 
104 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL);
105 	tmp &= ~LC_L1_INACTIVITY_MASK;
106 	tmp |= LC_L1_INACTIVITY(4);
107 	tmp &= ~LC_PMI_TO_L1_DIS;
108 	tmp &= ~LC_ASPM_TO_L1_DIS;
109 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
110 }
111 
112 static void rv770_enable_pll_sleep_in_l1(struct radeon_device *rdev)
113 {
114 	u32 tmp;
115 
116 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK;
117 	tmp |= LC_L1_INACTIVITY(8);
118 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
119 
120 	/* NOTE, this is a PCIE indirect reg, not PCIE PORT */
121 	tmp = RREG32_PCIE(PCIE_P_CNTL);
122 	tmp |= P_PLL_PWRDN_IN_L1L23;
123 	tmp &= ~P_PLL_BUF_PDNB;
124 	tmp &= ~P_PLL_PDNB;
125 	tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF;
126 	WREG32_PCIE(PCIE_P_CNTL, tmp);
127 }
128 
129 static void rv770_gfx_clock_gating_enable(struct radeon_device *rdev,
130 					  bool enable)
131 {
132 	if (enable)
133 		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
134 	else {
135 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
136 		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
137 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
138 		RREG32(GB_TILING_CONFIG);
139 	}
140 }
141 
142 static void rv770_mg_clock_gating_enable(struct radeon_device *rdev,
143 					 bool enable)
144 {
145 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
146 
147 	if (enable) {
148 		u32 mgcg_cgtt_local0;
149 
150 		if (rdev->family == CHIP_RV770)
151 			mgcg_cgtt_local0 = RV770_MGCGTTLOCAL0_DFLT;
152 		else
153 			mgcg_cgtt_local0 = RV7XX_MGCGTTLOCAL0_DFLT;
154 
155 		WREG32(CG_CGTT_LOCAL_0, mgcg_cgtt_local0);
156 		WREG32(CG_CGTT_LOCAL_1, (RV770_MGCGTTLOCAL1_DFLT & 0xFFFFCFFF));
157 
158 		if (pi->mgcgtssm)
159 			WREG32(CGTS_SM_CTRL_REG, RV770_MGCGCGTSSMCTRL_DFLT);
160 	} else {
161 		WREG32(CG_CGTT_LOCAL_0, 0xFFFFFFFF);
162 		WREG32(CG_CGTT_LOCAL_1, 0xFFFFCFFF);
163 	}
164 }
165 
166 void rv770_restore_cgcg(struct radeon_device *rdev)
167 {
168 	bool dpm_en = false, cg_en = false;
169 
170 	if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
171 		dpm_en = true;
172 	if (RREG32(SCLK_PWRMGT_CNTL) & DYN_GFX_CLK_OFF_EN)
173 		cg_en = true;
174 
175 	if (dpm_en && !cg_en)
176 		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
177 }
178 
179 static void rv770_start_dpm(struct radeon_device *rdev)
180 {
181 	WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF);
182 
183 	WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF);
184 
185 	WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
186 }
187 
188 void rv770_stop_dpm(struct radeon_device *rdev)
189 {
190 	PPSMC_Result result;
191 
192 	result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_TwoLevelsDisabled);
193 
194 	if (result != PPSMC_Result_OK)
195 		DRM_ERROR("Could not force DPM to low.\n");
196 
197 	WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
198 
199 	WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
200 
201 	WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF);
202 }
203 
204 bool rv770_dpm_enabled(struct radeon_device *rdev)
205 {
206 	if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
207 		return true;
208 	else
209 		return false;
210 }
211 
212 void rv770_enable_thermal_protection(struct radeon_device *rdev,
213 				     bool enable)
214 {
215 	if (enable)
216 		WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
217 	else
218 		WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
219 }
220 
221 void rv770_enable_acpi_pm(struct radeon_device *rdev)
222 {
223 	WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN);
224 }
225 
226 u8 rv770_get_seq_value(struct radeon_device *rdev,
227 		       struct rv7xx_pl *pl)
228 {
229 	return (pl->flags & ATOM_PPLIB_R600_FLAGS_LOWPOWER) ?
230 		MC_CG_SEQ_DRAMCONF_S0 : MC_CG_SEQ_DRAMCONF_S1;
231 }
232 
233 int rv770_read_smc_soft_register(struct radeon_device *rdev,
234 				 u16 reg_offset, u32 *value)
235 {
236 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
237 
238 	return rv770_read_smc_sram_dword(rdev,
239 					 pi->soft_regs_start + reg_offset,
240 					 value, pi->sram_end);
241 }
242 
243 int rv770_write_smc_soft_register(struct radeon_device *rdev,
244 				  u16 reg_offset, u32 value)
245 {
246 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
247 
248 	return rv770_write_smc_sram_dword(rdev,
249 					  pi->soft_regs_start + reg_offset,
250 					  value, pi->sram_end);
251 }
252 
253 int rv770_populate_smc_t(struct radeon_device *rdev,
254 			 struct radeon_ps *radeon_state,
255 			 RV770_SMC_SWSTATE *smc_state)
256 {
257 	struct rv7xx_ps *state = rv770_get_ps(radeon_state);
258 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
259 	int i;
260 	int a_n;
261 	int a_d;
262 	u8 l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
263 	u8 r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
264 	u32 a_t;
265 
266 	l[0] = 0;
267 	r[2] = 100;
268 
269 	a_n = (int)state->medium.sclk * pi->lmp +
270 		(int)state->low.sclk * (R600_AH_DFLT - pi->rlp);
271 	a_d = (int)state->low.sclk * (100 - (int)pi->rlp) +
272 		(int)state->medium.sclk * pi->lmp;
273 
274 	l[1] = (u8)(pi->lmp - (int)pi->lmp * a_n / a_d);
275 	r[0] = (u8)(pi->rlp + (100 - (int)pi->rlp) * a_n / a_d);
276 
277 	a_n = (int)state->high.sclk * pi->lhp + (int)state->medium.sclk *
278 		(R600_AH_DFLT - pi->rmp);
279 	a_d = (int)state->medium.sclk * (100 - (int)pi->rmp) +
280 		(int)state->high.sclk * pi->lhp;
281 
282 	l[2] = (u8)(pi->lhp - (int)pi->lhp * a_n / a_d);
283 	r[1] = (u8)(pi->rmp + (100 - (int)pi->rmp) * a_n / a_d);
284 
285 	for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++) {
286 		a_t = CG_R(r[i] * pi->bsp / 200) | CG_L(l[i] * pi->bsp / 200);
287 		smc_state->levels[i].aT = cpu_to_be32(a_t);
288 	}
289 
290 	a_t = CG_R(r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200) |
291 		CG_L(l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200);
292 
293 	smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].aT =
294 		cpu_to_be32(a_t);
295 
296 	return 0;
297 }
298 
299 int rv770_populate_smc_sp(struct radeon_device *rdev,
300 			  struct radeon_ps *radeon_state,
301 			  RV770_SMC_SWSTATE *smc_state)
302 {
303 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
304 	int i;
305 
306 	for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++)
307 		smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
308 
309 	smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].bSP =
310 		cpu_to_be32(pi->psp);
311 
312 	return 0;
313 }
314 
315 static void rv770_calculate_fractional_mpll_feedback_divider(u32 memory_clock,
316 							     u32 reference_clock,
317 							     bool gddr5,
318 							     struct atom_clock_dividers *dividers,
319 							     u32 *clkf,
320 							     u32 *clkfrac)
321 {
322 	u32 post_divider, reference_divider, feedback_divider8;
323 	u32 fyclk;
324 
325 	if (gddr5)
326 		fyclk = (memory_clock * 8) / 2;
327 	else
328 		fyclk = (memory_clock * 4) / 2;
329 
330 	post_divider = dividers->post_div;
331 	reference_divider = dividers->ref_div;
332 
333 	feedback_divider8 =
334 		(8 * fyclk * reference_divider * post_divider) / reference_clock;
335 
336 	*clkf = feedback_divider8 / 8;
337 	*clkfrac = feedback_divider8 % 8;
338 }
339 
340 static int rv770_encode_yclk_post_div(u32 postdiv, u32 *encoded_postdiv)
341 {
342 	int ret = 0;
343 
344 	switch (postdiv) {
345         case 1:
346 		*encoded_postdiv = 0;
347 		break;
348         case 2:
349 		*encoded_postdiv = 1;
350 		break;
351         case 4:
352 		*encoded_postdiv = 2;
353 		break;
354         case 8:
355 		*encoded_postdiv = 3;
356 		break;
357         case 16:
358 		*encoded_postdiv = 4;
359 		break;
360         default:
361 		ret = -EINVAL;
362 		break;
363 	}
364 
365     return ret;
366 }
367 
368 u32 rv770_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf)
369 {
370 	if (clkf <= 0x10)
371 		return 0x4B;
372 	if (clkf <= 0x19)
373 		return 0x5B;
374 	if (clkf <= 0x21)
375 		return 0x2B;
376 	if (clkf <= 0x27)
377 		return 0x6C;
378 	if (clkf <= 0x31)
379 		return 0x9D;
380 	return 0xC6;
381 }
382 
383 static int rv770_populate_mclk_value(struct radeon_device *rdev,
384 				     u32 engine_clock, u32 memory_clock,
385 				     RV7XX_SMC_MCLK_VALUE *mclk)
386 {
387 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
388 	u8 encoded_reference_dividers[] = { 0, 16, 17, 20, 21 };
389 	u32 mpll_ad_func_cntl =
390 		pi->clk_regs.rv770.mpll_ad_func_cntl;
391 	u32 mpll_ad_func_cntl_2 =
392 		pi->clk_regs.rv770.mpll_ad_func_cntl_2;
393 	u32 mpll_dq_func_cntl =
394 		pi->clk_regs.rv770.mpll_dq_func_cntl;
395 	u32 mpll_dq_func_cntl_2 =
396 		pi->clk_regs.rv770.mpll_dq_func_cntl_2;
397 	u32 mclk_pwrmgt_cntl =
398 		pi->clk_regs.rv770.mclk_pwrmgt_cntl;
399 	u32 dll_cntl = pi->clk_regs.rv770.dll_cntl;
400 	struct atom_clock_dividers dividers;
401 	u32 reference_clock = rdev->clock.mpll.reference_freq;
402 	u32 clkf, clkfrac;
403 	u32 postdiv_yclk;
404 	u32 ibias;
405 	int ret;
406 
407 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
408 					     memory_clock, false, &dividers);
409 	if (ret)
410 		return ret;
411 
412 	if ((dividers.ref_div < 1) || (dividers.ref_div > 5))
413 		return -EINVAL;
414 
415 	rv770_calculate_fractional_mpll_feedback_divider(memory_clock, reference_clock,
416 							 pi->mem_gddr5,
417 							 &dividers, &clkf, &clkfrac);
418 
419 	ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk);
420 	if (ret)
421 		return ret;
422 
423 	ibias = rv770_map_clkf_to_ibias(rdev, clkf);
424 
425 	mpll_ad_func_cntl &= ~(CLKR_MASK |
426 			       YCLK_POST_DIV_MASK |
427 			       CLKF_MASK |
428 			       CLKFRAC_MASK |
429 			       IBIAS_MASK);
430 	mpll_ad_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]);
431 	mpll_ad_func_cntl |= YCLK_POST_DIV(postdiv_yclk);
432 	mpll_ad_func_cntl |= CLKF(clkf);
433 	mpll_ad_func_cntl |= CLKFRAC(clkfrac);
434 	mpll_ad_func_cntl |= IBIAS(ibias);
435 
436 	if (dividers.vco_mode)
437 		mpll_ad_func_cntl_2 |= VCO_MODE;
438 	else
439 		mpll_ad_func_cntl_2 &= ~VCO_MODE;
440 
441 	if (pi->mem_gddr5) {
442 		rv770_calculate_fractional_mpll_feedback_divider(memory_clock,
443 								 reference_clock,
444 								 pi->mem_gddr5,
445 								 &dividers, &clkf, &clkfrac);
446 
447 		ibias = rv770_map_clkf_to_ibias(rdev, clkf);
448 
449 		ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk);
450 		if (ret)
451 			return ret;
452 
453 		mpll_dq_func_cntl &= ~(CLKR_MASK |
454 				       YCLK_POST_DIV_MASK |
455 				       CLKF_MASK |
456 				       CLKFRAC_MASK |
457 				       IBIAS_MASK);
458 		mpll_dq_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]);
459 		mpll_dq_func_cntl |= YCLK_POST_DIV(postdiv_yclk);
460 		mpll_dq_func_cntl |= CLKF(clkf);
461 		mpll_dq_func_cntl |= CLKFRAC(clkfrac);
462 		mpll_dq_func_cntl |= IBIAS(ibias);
463 
464 		if (dividers.vco_mode)
465 			mpll_dq_func_cntl_2 |= VCO_MODE;
466 		else
467 			mpll_dq_func_cntl_2 &= ~VCO_MODE;
468 	}
469 
470 	mclk->mclk770.mclk_value = cpu_to_be32(memory_clock);
471 	mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
472 	mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
473 	mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
474 	mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
475 	mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
476 	mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
477 
478 	return 0;
479 }
480 
481 static int rv770_populate_sclk_value(struct radeon_device *rdev,
482 				     u32 engine_clock,
483 				     RV770_SMC_SCLK_VALUE *sclk)
484 {
485 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
486 	struct atom_clock_dividers dividers;
487 	u32 spll_func_cntl =
488 		pi->clk_regs.rv770.cg_spll_func_cntl;
489 	u32 spll_func_cntl_2 =
490 		pi->clk_regs.rv770.cg_spll_func_cntl_2;
491 	u32 spll_func_cntl_3 =
492 		pi->clk_regs.rv770.cg_spll_func_cntl_3;
493 	u32 cg_spll_spread_spectrum =
494 		pi->clk_regs.rv770.cg_spll_spread_spectrum;
495 	u32 cg_spll_spread_spectrum_2 =
496 		pi->clk_regs.rv770.cg_spll_spread_spectrum_2;
497 	u64 tmp;
498 	u32 reference_clock = rdev->clock.spll.reference_freq;
499 	u32 reference_divider, post_divider;
500 	u32 fbdiv;
501 	int ret;
502 
503 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
504 					     engine_clock, false, &dividers);
505 	if (ret)
506 		return ret;
507 
508 	reference_divider = 1 + dividers.ref_div;
509 
510 	if (dividers.enable_post_div)
511 		post_divider = (0x0f & (dividers.post_div >> 4)) + (0x0f & dividers.post_div) + 2;
512 	else
513 		post_divider = 1;
514 
515 	tmp = (u64) engine_clock * reference_divider * post_divider * 16384;
516 	do_div(tmp, reference_clock);
517 	fbdiv = (u32) tmp;
518 
519 	if (dividers.enable_post_div)
520 		spll_func_cntl |= SPLL_DIVEN;
521 	else
522 		spll_func_cntl &= ~SPLL_DIVEN;
523 	spll_func_cntl &= ~(SPLL_HILEN_MASK | SPLL_LOLEN_MASK | SPLL_REF_DIV_MASK);
524 	spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
525 	spll_func_cntl |= SPLL_HILEN((dividers.post_div >> 4) & 0xf);
526 	spll_func_cntl |= SPLL_LOLEN(dividers.post_div & 0xf);
527 
528 	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
529 	spll_func_cntl_2 |= SCLK_MUX_SEL(2);
530 
531 	spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
532 	spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
533 	spll_func_cntl_3 |= SPLL_DITHEN;
534 
535 	if (pi->sclk_ss) {
536 		struct radeon_atom_ss ss;
537 		u32 vco_freq = engine_clock * post_divider;
538 
539 		if (radeon_atombios_get_asic_ss_info(rdev, &ss,
540 						     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
541 			u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
542 			u32 clk_v = ss.percentage * fbdiv / (clk_s * 10000);
543 
544 			cg_spll_spread_spectrum &= ~CLKS_MASK;
545 			cg_spll_spread_spectrum |= CLKS(clk_s);
546 			cg_spll_spread_spectrum |= SSEN;
547 
548 			cg_spll_spread_spectrum_2 &= ~CLKV_MASK;
549 			cg_spll_spread_spectrum_2 |= CLKV(clk_v);
550 		}
551 	}
552 
553 	sclk->sclk_value = cpu_to_be32(engine_clock);
554 	sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
555 	sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
556 	sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
557 	sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum);
558 	sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2);
559 
560 	return 0;
561 }
562 
563 int rv770_populate_vddc_value(struct radeon_device *rdev, u16 vddc,
564 			      RV770_SMC_VOLTAGE_VALUE *voltage)
565 {
566 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
567 	int i;
568 
569 	if (!pi->voltage_control) {
570 		voltage->index = 0;
571 		voltage->value = 0;
572 		return 0;
573 	}
574 
575 	for (i = 0; i < pi->valid_vddc_entries; i++) {
576 		if (vddc <= pi->vddc_table[i].vddc) {
577 			voltage->index = pi->vddc_table[i].vddc_index;
578 			voltage->value = cpu_to_be16(vddc);
579 			break;
580 		}
581 	}
582 
583 	if (i == pi->valid_vddc_entries)
584 		return -EINVAL;
585 
586 	return 0;
587 }
588 
589 int rv770_populate_mvdd_value(struct radeon_device *rdev, u32 mclk,
590 			      RV770_SMC_VOLTAGE_VALUE *voltage)
591 {
592 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
593 
594 	if (!pi->mvdd_control) {
595 		voltage->index = MVDD_HIGH_INDEX;
596 		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
597 		return 0;
598 	}
599 
600 	if (mclk <= pi->mvdd_split_frequency) {
601 		voltage->index = MVDD_LOW_INDEX;
602 		voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
603 	} else {
604 		voltage->index = MVDD_HIGH_INDEX;
605 		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
606 	}
607 
608 	return 0;
609 }
610 
611 static int rv770_convert_power_level_to_smc(struct radeon_device *rdev,
612 					    struct rv7xx_pl *pl,
613 					    RV770_SMC_HW_PERFORMANCE_LEVEL *level,
614 					    u8 watermark_level)
615 {
616 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
617 	int ret;
618 
619 	level->gen2PCIE = pi->pcie_gen2 ?
620 		((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
621 	level->gen2XSP  = (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0;
622 	level->backbias = (pl->flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? 1 : 0;
623 	level->displayWatermark = watermark_level;
624 
625 	if (rdev->family == CHIP_RV740)
626 		ret = rv740_populate_sclk_value(rdev, pl->sclk,
627 						&level->sclk);
628 	else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
629 		ret = rv730_populate_sclk_value(rdev, pl->sclk,
630 						&level->sclk);
631 	else
632 		ret = rv770_populate_sclk_value(rdev, pl->sclk,
633 						&level->sclk);
634 	if (ret)
635 		return ret;
636 
637 	if (rdev->family == CHIP_RV740) {
638 		if (pi->mem_gddr5) {
639 			if (pl->mclk <= pi->mclk_strobe_mode_threshold)
640 				level->strobeMode =
641 					rv740_get_mclk_frequency_ratio(pl->mclk) | 0x10;
642 			else
643 				level->strobeMode = 0;
644 
645 			if (pl->mclk > pi->mclk_edc_enable_threshold)
646 				level->mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
647 			else
648 				level->mcFlags =  0;
649 		}
650 		ret = rv740_populate_mclk_value(rdev, pl->sclk,
651 						pl->mclk, &level->mclk);
652 	} else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
653 		ret = rv730_populate_mclk_value(rdev, pl->sclk,
654 						pl->mclk, &level->mclk);
655 	else
656 		ret = rv770_populate_mclk_value(rdev, pl->sclk,
657 						pl->mclk, &level->mclk);
658 	if (ret)
659 		return ret;
660 
661 	ret = rv770_populate_vddc_value(rdev, pl->vddc,
662 					&level->vddc);
663 	if (ret)
664 		return ret;
665 
666 	ret = rv770_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
667 
668 	return ret;
669 }
670 
671 static int rv770_convert_power_state_to_smc(struct radeon_device *rdev,
672 					    struct radeon_ps *radeon_state,
673 					    RV770_SMC_SWSTATE *smc_state)
674 {
675 	struct rv7xx_ps *state = rv770_get_ps(radeon_state);
676 	int ret;
677 
678 	if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
679 		smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
680 
681 	ret = rv770_convert_power_level_to_smc(rdev,
682 					       &state->low,
683 					       &smc_state->levels[0],
684 					       PPSMC_DISPLAY_WATERMARK_LOW);
685 	if (ret)
686 		return ret;
687 
688 	ret = rv770_convert_power_level_to_smc(rdev,
689 					       &state->medium,
690 					       &smc_state->levels[1],
691 					       PPSMC_DISPLAY_WATERMARK_LOW);
692 	if (ret)
693 		return ret;
694 
695 	ret = rv770_convert_power_level_to_smc(rdev,
696 					       &state->high,
697 					       &smc_state->levels[2],
698 					       PPSMC_DISPLAY_WATERMARK_HIGH);
699 	if (ret)
700 		return ret;
701 
702 	smc_state->levels[0].arbValue = MC_CG_ARB_FREQ_F1;
703 	smc_state->levels[1].arbValue = MC_CG_ARB_FREQ_F2;
704 	smc_state->levels[2].arbValue = MC_CG_ARB_FREQ_F3;
705 
706 	smc_state->levels[0].seqValue = rv770_get_seq_value(rdev,
707 							    &state->low);
708 	smc_state->levels[1].seqValue = rv770_get_seq_value(rdev,
709 							    &state->medium);
710 	smc_state->levels[2].seqValue = rv770_get_seq_value(rdev,
711 							    &state->high);
712 
713 	rv770_populate_smc_sp(rdev, radeon_state, smc_state);
714 
715 	return rv770_populate_smc_t(rdev, radeon_state, smc_state);
716 
717 }
718 
719 u32 rv770_calculate_memory_refresh_rate(struct radeon_device *rdev,
720 					u32 engine_clock)
721 {
722 	u32 dram_rows;
723 	u32 dram_refresh_rate;
724 	u32 mc_arb_rfsh_rate;
725 	u32 tmp;
726 
727 	tmp = (RREG32(MC_ARB_RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
728 	dram_rows = 1 << (tmp + 10);
729 	tmp = RREG32(MC_SEQ_MISC0) & 3;
730 	dram_refresh_rate = 1 << (tmp + 3);
731 	mc_arb_rfsh_rate = ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
732 
733 	return mc_arb_rfsh_rate;
734 }
735 
736 static void rv770_program_memory_timing_parameters(struct radeon_device *rdev,
737 						   struct radeon_ps *radeon_state)
738 {
739 	struct rv7xx_ps *state = rv770_get_ps(radeon_state);
740 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
741 	u32 sqm_ratio;
742 	u32 arb_refresh_rate;
743 	u32 high_clock;
744 
745 	if (state->high.sclk < (state->low.sclk * 0xFF / 0x40))
746 		high_clock = state->high.sclk;
747 	else
748 		high_clock = (state->low.sclk * 0xFF / 0x40);
749 
750 	radeon_atom_set_engine_dram_timings(rdev, high_clock,
751 					    state->high.mclk);
752 
753 	sqm_ratio =
754 		STATE0(64 * high_clock / pi->boot_sclk) |
755 		STATE1(64 * high_clock / state->low.sclk) |
756 		STATE2(64 * high_clock / state->medium.sclk) |
757 		STATE3(64 * high_clock / state->high.sclk);
758 	WREG32(MC_ARB_SQM_RATIO, sqm_ratio);
759 
760 	arb_refresh_rate =
761 		POWERMODE0(rv770_calculate_memory_refresh_rate(rdev, pi->boot_sclk)) |
762 		POWERMODE1(rv770_calculate_memory_refresh_rate(rdev, state->low.sclk)) |
763 		POWERMODE2(rv770_calculate_memory_refresh_rate(rdev, state->medium.sclk)) |
764 		POWERMODE3(rv770_calculate_memory_refresh_rate(rdev, state->high.sclk));
765 	WREG32(MC_ARB_RFSH_RATE, arb_refresh_rate);
766 }
767 
768 void rv770_enable_backbias(struct radeon_device *rdev,
769 			   bool enable)
770 {
771 	if (enable)
772 		WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN, ~BACKBIAS_PAD_EN);
773 	else
774 		WREG32_P(GENERAL_PWRMGT, 0, ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN));
775 }
776 
777 static void rv770_enable_spread_spectrum(struct radeon_device *rdev,
778 					 bool enable)
779 {
780 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
781 
782 	if (enable) {
783 		if (pi->sclk_ss)
784 			WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
785 
786 		if (pi->mclk_ss) {
787 			if (rdev->family == CHIP_RV740)
788 				rv740_enable_mclk_spread_spectrum(rdev, true);
789 		}
790 	} else {
791 		WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN);
792 
793 		WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
794 
795 		WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN);
796 
797 		if (rdev->family == CHIP_RV740)
798 			rv740_enable_mclk_spread_spectrum(rdev, false);
799 	}
800 }
801 
802 static void rv770_program_mpll_timing_parameters(struct radeon_device *rdev)
803 {
804 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
805 
806 	if ((rdev->family == CHIP_RV770) && !pi->mem_gddr5) {
807 		WREG32(MPLL_TIME,
808 		       (MPLL_LOCK_TIME(R600_MPLLLOCKTIME_DFLT * pi->ref_div) |
809 			MPLL_RESET_TIME(R600_MPLLRESETTIME_DFLT)));
810 	}
811 }
812 
813 void rv770_setup_bsp(struct radeon_device *rdev)
814 {
815 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
816 	u32 xclk = radeon_get_xclk(rdev);
817 
818 	r600_calculate_u_and_p(pi->asi,
819 			       xclk,
820 			       16,
821 			       &pi->bsp,
822 			       &pi->bsu);
823 
824 	r600_calculate_u_and_p(pi->pasi,
825 			       xclk,
826 			       16,
827 			       &pi->pbsp,
828 			       &pi->pbsu);
829 
830 	pi->dsp = BSP(pi->bsp) | BSU(pi->bsu);
831 	pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu);
832 
833 	WREG32(CG_BSP, pi->dsp);
834 
835 }
836 
837 void rv770_program_git(struct radeon_device *rdev)
838 {
839 	WREG32_P(CG_GIT, CG_GICST(R600_GICST_DFLT), ~CG_GICST_MASK);
840 }
841 
842 void rv770_program_tp(struct radeon_device *rdev)
843 {
844 	int i;
845 	enum r600_td td = R600_TD_DFLT;
846 
847 	for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
848 		WREG32(CG_FFCT_0 + (i * 4), (UTC_0(r600_utc[i]) | DTC_0(r600_dtc[i])));
849 
850 	if (td == R600_TD_AUTO)
851 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL);
852 	else
853 		WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL);
854 	if (td == R600_TD_UP)
855 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE);
856 	if (td == R600_TD_DOWN)
857 		WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE);
858 }
859 
860 void rv770_program_tpp(struct radeon_device *rdev)
861 {
862 	WREG32(CG_TPC, R600_TPC_DFLT);
863 }
864 
865 void rv770_program_sstp(struct radeon_device *rdev)
866 {
867 	WREG32(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT)));
868 }
869 
870 void rv770_program_engine_speed_parameters(struct radeon_device *rdev)
871 {
872 	WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC);
873 }
874 
875 static void rv770_enable_display_gap(struct radeon_device *rdev)
876 {
877 	u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
878 
879 	tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
880 	tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
881 		DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE));
882 	WREG32(CG_DISPLAY_GAP_CNTL, tmp);
883 }
884 
885 void rv770_program_vc(struct radeon_device *rdev)
886 {
887 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
888 
889 	WREG32(CG_FTV, pi->vrc);
890 }
891 
892 void rv770_clear_vc(struct radeon_device *rdev)
893 {
894 	WREG32(CG_FTV, 0);
895 }
896 
897 int rv770_upload_firmware(struct radeon_device *rdev)
898 {
899 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
900 	int ret;
901 
902 	rv770_reset_smc(rdev);
903 	rv770_stop_smc_clock(rdev);
904 
905 	ret = rv770_load_smc_ucode(rdev, pi->sram_end);
906 	if (ret)
907 		return ret;
908 
909 	return 0;
910 }
911 
912 static int rv770_populate_smc_acpi_state(struct radeon_device *rdev,
913 					 RV770_SMC_STATETABLE *table)
914 {
915 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
916 
917 	u32 mpll_ad_func_cntl =
918 		pi->clk_regs.rv770.mpll_ad_func_cntl;
919 	u32 mpll_ad_func_cntl_2 =
920 		pi->clk_regs.rv770.mpll_ad_func_cntl_2;
921 	u32 mpll_dq_func_cntl =
922 		pi->clk_regs.rv770.mpll_dq_func_cntl;
923 	u32 mpll_dq_func_cntl_2 =
924 		pi->clk_regs.rv770.mpll_dq_func_cntl_2;
925 	u32 spll_func_cntl =
926 		pi->clk_regs.rv770.cg_spll_func_cntl;
927 	u32 spll_func_cntl_2 =
928 		pi->clk_regs.rv770.cg_spll_func_cntl_2;
929 	u32 spll_func_cntl_3 =
930 		pi->clk_regs.rv770.cg_spll_func_cntl_3;
931 	u32 mclk_pwrmgt_cntl;
932 	u32 dll_cntl;
933 
934 	table->ACPIState = table->initialState;
935 
936 	table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
937 
938 	if (pi->acpi_vddc) {
939 		rv770_populate_vddc_value(rdev, pi->acpi_vddc,
940 					  &table->ACPIState.levels[0].vddc);
941 		if (pi->pcie_gen2) {
942 			if (pi->acpi_pcie_gen2)
943 				table->ACPIState.levels[0].gen2PCIE = 1;
944 			else
945 				table->ACPIState.levels[0].gen2PCIE = 0;
946 		} else
947 			table->ACPIState.levels[0].gen2PCIE = 0;
948 		if (pi->acpi_pcie_gen2)
949 			table->ACPIState.levels[0].gen2XSP = 1;
950 		else
951 			table->ACPIState.levels[0].gen2XSP = 0;
952 	} else {
953 		rv770_populate_vddc_value(rdev, pi->min_vddc_in_table,
954 					  &table->ACPIState.levels[0].vddc);
955 		table->ACPIState.levels[0].gen2PCIE = 0;
956 	}
957 
958 
959 	mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
960 
961 	mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
962 
963 	mclk_pwrmgt_cntl = (MRDCKA0_RESET |
964 			    MRDCKA1_RESET |
965 			    MRDCKB0_RESET |
966 			    MRDCKB1_RESET |
967 			    MRDCKC0_RESET |
968 			    MRDCKC1_RESET |
969 			    MRDCKD0_RESET |
970 			    MRDCKD1_RESET);
971 
972 	dll_cntl = 0xff000000;
973 
974 	spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN;
975 
976 	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
977 	spll_func_cntl_2 |= SCLK_MUX_SEL(4);
978 
979 	table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
980 	table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
981 	table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
982 	table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
983 
984 	table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
985 	table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
986 
987 	table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0;
988 
989 	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
990 	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
991 	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
992 
993 	table->ACPIState.levels[0].sclk.sclk_value = 0;
994 
995 	rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
996 
997 	table->ACPIState.levels[1] = table->ACPIState.levels[0];
998 	table->ACPIState.levels[2] = table->ACPIState.levels[0];
999 
1000 	return 0;
1001 }
1002 
1003 int rv770_populate_initial_mvdd_value(struct radeon_device *rdev,
1004 				      RV770_SMC_VOLTAGE_VALUE *voltage)
1005 {
1006 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1007 
1008 	if ((pi->s0_vid_lower_smio_cntl & pi->mvdd_mask_low) ==
1009 	     (pi->mvdd_low_smio[MVDD_LOW_INDEX] & pi->mvdd_mask_low) ) {
1010 		voltage->index = MVDD_LOW_INDEX;
1011 		voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1012 	} else {
1013 		voltage->index = MVDD_HIGH_INDEX;
1014 		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1015 	}
1016 
1017 	return 0;
1018 }
1019 
1020 static int rv770_populate_smc_initial_state(struct radeon_device *rdev,
1021 					    struct radeon_ps *radeon_state,
1022 					    RV770_SMC_STATETABLE *table)
1023 {
1024 	struct rv7xx_ps *initial_state = rv770_get_ps(radeon_state);
1025 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1026 	u32 a_t;
1027 
1028 	table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL =
1029 		cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl);
1030 	table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 =
1031 		cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl_2);
1032 	table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL =
1033 		cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl);
1034 	table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 =
1035 		cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl_2);
1036 	table->initialState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL =
1037 		cpu_to_be32(pi->clk_regs.rv770.mclk_pwrmgt_cntl);
1038 	table->initialState.levels[0].mclk.mclk770.vDLL_CNTL =
1039 		cpu_to_be32(pi->clk_regs.rv770.dll_cntl);
1040 
1041 	table->initialState.levels[0].mclk.mclk770.vMPLL_SS =
1042 		cpu_to_be32(pi->clk_regs.rv770.mpll_ss1);
1043 	table->initialState.levels[0].mclk.mclk770.vMPLL_SS2 =
1044 		cpu_to_be32(pi->clk_regs.rv770.mpll_ss2);
1045 
1046 	table->initialState.levels[0].mclk.mclk770.mclk_value =
1047 		cpu_to_be32(initial_state->low.mclk);
1048 
1049 	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1050 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl);
1051 	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1052 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_2);
1053 	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1054 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_3);
1055 	table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1056 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum);
1057 	table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1058 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum_2);
1059 
1060 	table->initialState.levels[0].sclk.sclk_value =
1061 		cpu_to_be32(initial_state->low.sclk);
1062 
1063 	table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0;
1064 
1065 	table->initialState.levels[0].seqValue =
1066 		rv770_get_seq_value(rdev, &initial_state->low);
1067 
1068 	rv770_populate_vddc_value(rdev,
1069 				  initial_state->low.vddc,
1070 				  &table->initialState.levels[0].vddc);
1071 	rv770_populate_initial_mvdd_value(rdev,
1072 					  &table->initialState.levels[0].mvdd);
1073 
1074 	a_t = CG_R(0xffff) | CG_L(0);
1075 	table->initialState.levels[0].aT = cpu_to_be32(a_t);
1076 
1077 	table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1078 
1079 	if (pi->boot_in_gen2)
1080 		table->initialState.levels[0].gen2PCIE = 1;
1081 	else
1082 		table->initialState.levels[0].gen2PCIE = 0;
1083 	if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
1084 		table->initialState.levels[0].gen2XSP = 1;
1085 	else
1086 		table->initialState.levels[0].gen2XSP = 0;
1087 
1088 	if (rdev->family == CHIP_RV740) {
1089 		if (pi->mem_gddr5) {
1090 			if (initial_state->low.mclk <= pi->mclk_strobe_mode_threshold)
1091 				table->initialState.levels[0].strobeMode =
1092 					rv740_get_mclk_frequency_ratio(initial_state->low.mclk) | 0x10;
1093 			else
1094 				table->initialState.levels[0].strobeMode = 0;
1095 
1096 			if (initial_state->low.mclk >= pi->mclk_edc_enable_threshold)
1097 				table->initialState.levels[0].mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
1098 			else
1099 				table->initialState.levels[0].mcFlags =  0;
1100 		}
1101 	}
1102 
1103 	table->initialState.levels[1] = table->initialState.levels[0];
1104 	table->initialState.levels[2] = table->initialState.levels[0];
1105 
1106 	table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1107 
1108 	return 0;
1109 }
1110 
1111 static int rv770_populate_smc_vddc_table(struct radeon_device *rdev,
1112 					 RV770_SMC_STATETABLE *table)
1113 {
1114 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1115 	int i;
1116 
1117 	for (i = 0; i < pi->valid_vddc_entries; i++) {
1118 		table->highSMIO[pi->vddc_table[i].vddc_index] =
1119 			pi->vddc_table[i].high_smio;
1120 		table->lowSMIO[pi->vddc_table[i].vddc_index] =
1121 			cpu_to_be32(pi->vddc_table[i].low_smio);
1122 	}
1123 
1124 	table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDC] = 0;
1125 	table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDC] =
1126 		cpu_to_be32(pi->vddc_mask_low);
1127 
1128 	for (i = 0;
1129 	     ((i < pi->valid_vddc_entries) &&
1130 	      (pi->max_vddc_in_table >
1131 	       pi->vddc_table[i].vddc));
1132 	     i++);
1133 
1134 	table->maxVDDCIndexInPPTable =
1135 		pi->vddc_table[i].vddc_index;
1136 
1137 	return 0;
1138 }
1139 
1140 static int rv770_populate_smc_mvdd_table(struct radeon_device *rdev,
1141 					 RV770_SMC_STATETABLE *table)
1142 {
1143 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1144 
1145 	if (pi->mvdd_control) {
1146 		table->lowSMIO[MVDD_HIGH_INDEX] |=
1147 			cpu_to_be32(pi->mvdd_low_smio[MVDD_HIGH_INDEX]);
1148 		table->lowSMIO[MVDD_LOW_INDEX] |=
1149 			cpu_to_be32(pi->mvdd_low_smio[MVDD_LOW_INDEX]);
1150 
1151 		table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_MVDD] = 0;
1152 		table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_MVDD] =
1153 			cpu_to_be32(pi->mvdd_mask_low);
1154 	}
1155 
1156 	return 0;
1157 }
1158 
1159 static int rv770_init_smc_table(struct radeon_device *rdev,
1160 				struct radeon_ps *radeon_boot_state)
1161 {
1162 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1163 	struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state);
1164 	RV770_SMC_STATETABLE *table = &pi->smc_statetable;
1165 	int ret;
1166 
1167 	memset(table, 0, sizeof(RV770_SMC_STATETABLE));
1168 
1169 	pi->boot_sclk = boot_state->low.sclk;
1170 
1171 	rv770_populate_smc_vddc_table(rdev, table);
1172 	rv770_populate_smc_mvdd_table(rdev, table);
1173 
1174 	switch (rdev->pm.int_thermal_type) {
1175         case THERMAL_TYPE_RV770:
1176         case THERMAL_TYPE_ADT7473_WITH_INTERNAL:
1177 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1178 		break;
1179         case THERMAL_TYPE_NONE:
1180 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1181 		break;
1182         case THERMAL_TYPE_EXTERNAL_GPIO:
1183         default:
1184 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1185 		break;
1186 	}
1187 
1188 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) {
1189 		table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1190 
1191 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_DONT_WAIT_FOR_VBLANK_ON_ALERT)
1192 			table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK;
1193 
1194 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_GOTO_BOOT_ON_ALERT)
1195 			table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE;
1196 	}
1197 
1198 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1199 		table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1200 
1201 	if (pi->mem_gddr5)
1202 		table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1203 
1204 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1205 		ret = rv730_populate_smc_initial_state(rdev, radeon_boot_state, table);
1206 	else
1207 		ret = rv770_populate_smc_initial_state(rdev, radeon_boot_state, table);
1208 	if (ret)
1209 		return ret;
1210 
1211 	if (rdev->family == CHIP_RV740)
1212 		ret = rv740_populate_smc_acpi_state(rdev, table);
1213 	else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1214 		ret = rv730_populate_smc_acpi_state(rdev, table);
1215 	else
1216 		ret = rv770_populate_smc_acpi_state(rdev, table);
1217 	if (ret)
1218 		return ret;
1219 
1220 	table->driverState = table->initialState;
1221 
1222 	return rv770_copy_bytes_to_smc(rdev,
1223 				       pi->state_table_start,
1224 				       (const u8 *)table,
1225 				       sizeof(RV770_SMC_STATETABLE),
1226 				       pi->sram_end);
1227 }
1228 
1229 static int rv770_construct_vddc_table(struct radeon_device *rdev)
1230 {
1231 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1232 	u16 min, max, step;
1233 	u32 steps = 0;
1234 	u8 vddc_index = 0;
1235 	u32 i;
1236 
1237 	radeon_atom_get_min_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &min);
1238 	radeon_atom_get_max_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &max);
1239 	radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &step);
1240 
1241 	steps = (max - min) / step + 1;
1242 
1243 	if (steps > MAX_NO_VREG_STEPS)
1244 		return -EINVAL;
1245 
1246 	for (i = 0; i < steps; i++) {
1247 		u32 gpio_pins, gpio_mask;
1248 
1249 		pi->vddc_table[i].vddc = (u16)(min + i * step);
1250 		radeon_atom_get_voltage_gpio_settings(rdev,
1251 						      pi->vddc_table[i].vddc,
1252 						      SET_VOLTAGE_TYPE_ASIC_VDDC,
1253 						      &gpio_pins, &gpio_mask);
1254 		pi->vddc_table[i].low_smio = gpio_pins & gpio_mask;
1255 		pi->vddc_table[i].high_smio = 0;
1256 		pi->vddc_mask_low = gpio_mask;
1257 		if (i > 0) {
1258 			if ((pi->vddc_table[i].low_smio !=
1259 			     pi->vddc_table[i - 1].low_smio ) ||
1260 			     (pi->vddc_table[i].high_smio !=
1261 			      pi->vddc_table[i - 1].high_smio))
1262 				vddc_index++;
1263 		}
1264 		pi->vddc_table[i].vddc_index = vddc_index;
1265 	}
1266 
1267 	pi->valid_vddc_entries = (u8)steps;
1268 
1269 	return 0;
1270 }
1271 
1272 static u32 rv770_get_mclk_split_point(struct atom_memory_info *memory_info)
1273 {
1274 	if (memory_info->mem_type == MEM_TYPE_GDDR3)
1275 		return 30000;
1276 
1277 	return 0;
1278 }
1279 
1280 static int rv770_get_mvdd_pin_configuration(struct radeon_device *rdev)
1281 {
1282 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1283 	u32 gpio_pins, gpio_mask;
1284 
1285 	radeon_atom_get_voltage_gpio_settings(rdev,
1286 					      MVDD_HIGH_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC,
1287 					      &gpio_pins, &gpio_mask);
1288 	pi->mvdd_mask_low = gpio_mask;
1289 	pi->mvdd_low_smio[MVDD_HIGH_INDEX] =
1290 		gpio_pins & gpio_mask;
1291 
1292 	radeon_atom_get_voltage_gpio_settings(rdev,
1293 					      MVDD_LOW_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC,
1294 					      &gpio_pins, &gpio_mask);
1295 	pi->mvdd_low_smio[MVDD_LOW_INDEX] =
1296 		gpio_pins & gpio_mask;
1297 
1298 	return 0;
1299 }
1300 
1301 u8 rv770_get_memory_module_index(struct radeon_device *rdev)
1302 {
1303 	return (u8) ((RREG32(BIOS_SCRATCH_4) >> 16) & 0xff);
1304 }
1305 
1306 static int rv770_get_mvdd_configuration(struct radeon_device *rdev)
1307 {
1308 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1309 	u8 memory_module_index;
1310 	struct atom_memory_info memory_info;
1311 
1312 	memory_module_index = rv770_get_memory_module_index(rdev);
1313 
1314 	if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info)) {
1315 		pi->mvdd_control = false;
1316 		return 0;
1317 	}
1318 
1319 	pi->mvdd_split_frequency =
1320 		rv770_get_mclk_split_point(&memory_info);
1321 
1322 	if (pi->mvdd_split_frequency == 0) {
1323 		pi->mvdd_control = false;
1324 		return 0;
1325 	}
1326 
1327 	return rv770_get_mvdd_pin_configuration(rdev);
1328 }
1329 
1330 void rv770_enable_voltage_control(struct radeon_device *rdev,
1331 				  bool enable)
1332 {
1333 	if (enable)
1334 		WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
1335 	else
1336 		WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
1337 }
1338 
1339 static void rv770_program_display_gap(struct radeon_device *rdev)
1340 {
1341 	u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
1342 
1343 	tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
1344 	if (rdev->pm.dpm.new_active_crtcs & 1) {
1345 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
1346 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1347 	} else if (rdev->pm.dpm.new_active_crtcs & 2) {
1348 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1349 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
1350 	} else {
1351 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1352 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1353 	}
1354 	WREG32(CG_DISPLAY_GAP_CNTL, tmp);
1355 }
1356 
1357 static void rv770_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
1358 					   bool enable)
1359 {
1360 	rv770_enable_bif_dynamic_pcie_gen2(rdev, enable);
1361 
1362 	if (enable)
1363 		WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
1364 	else
1365 		WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
1366 }
1367 
1368 static void r7xx_program_memory_timing_parameters(struct radeon_device *rdev,
1369 						  struct radeon_ps *radeon_new_state)
1370 {
1371 	if ((rdev->family == CHIP_RV730) ||
1372 	    (rdev->family == CHIP_RV710) ||
1373 	    (rdev->family == CHIP_RV740))
1374 		rv730_program_memory_timing_parameters(rdev, radeon_new_state);
1375 	else
1376 		rv770_program_memory_timing_parameters(rdev, radeon_new_state);
1377 }
1378 
1379 static int rv770_upload_sw_state(struct radeon_device *rdev,
1380 				 struct radeon_ps *radeon_new_state)
1381 {
1382 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1383 	u16 address = pi->state_table_start +
1384 		offsetof(RV770_SMC_STATETABLE, driverState);
1385 	RV770_SMC_SWSTATE state = { 0 };
1386 	int ret;
1387 
1388 	ret = rv770_convert_power_state_to_smc(rdev, radeon_new_state, &state);
1389 	if (ret)
1390 		return ret;
1391 
1392 	return rv770_copy_bytes_to_smc(rdev, address, (const u8 *)&state,
1393 				       sizeof(RV770_SMC_SWSTATE),
1394 				       pi->sram_end);
1395 }
1396 
1397 int rv770_halt_smc(struct radeon_device *rdev)
1398 {
1399 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Halt) != PPSMC_Result_OK)
1400 		return -EINVAL;
1401 
1402 	if (rv770_wait_for_smc_inactive(rdev) != PPSMC_Result_OK)
1403 		return -EINVAL;
1404 
1405 	return 0;
1406 }
1407 
1408 int rv770_resume_smc(struct radeon_device *rdev)
1409 {
1410 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Resume) != PPSMC_Result_OK)
1411 		return -EINVAL;
1412 	return 0;
1413 }
1414 
1415 int rv770_set_sw_state(struct radeon_device *rdev)
1416 {
1417 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToSwState) != PPSMC_Result_OK)
1418 		return -EINVAL;
1419 	return 0;
1420 }
1421 
1422 int rv770_set_boot_state(struct radeon_device *rdev)
1423 {
1424 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToInitialState) != PPSMC_Result_OK)
1425 		return -EINVAL;
1426 	return 0;
1427 }
1428 
1429 void rv770_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
1430 					      struct radeon_ps *new_ps,
1431 					      struct radeon_ps *old_ps)
1432 {
1433 	struct rv7xx_ps *new_state = rv770_get_ps(new_ps);
1434 	struct rv7xx_ps *current_state = rv770_get_ps(old_ps);
1435 
1436 	if ((new_ps->vclk == old_ps->vclk) &&
1437 	    (new_ps->dclk == old_ps->dclk))
1438 		return;
1439 
1440 	if (new_state->high.sclk >= current_state->high.sclk)
1441 		return;
1442 
1443 	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
1444 }
1445 
1446 void rv770_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
1447 					     struct radeon_ps *new_ps,
1448 					     struct radeon_ps *old_ps)
1449 {
1450 	struct rv7xx_ps *new_state = rv770_get_ps(new_ps);
1451 	struct rv7xx_ps *current_state = rv770_get_ps(old_ps);
1452 
1453 	if ((new_ps->vclk == old_ps->vclk) &&
1454 	    (new_ps->dclk == old_ps->dclk))
1455 		return;
1456 
1457 	if (new_state->high.sclk < current_state->high.sclk)
1458 		return;
1459 
1460 	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
1461 }
1462 
1463 int rv770_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1464 {
1465 	if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_NoForcedLevel)) != PPSMC_Result_OK)
1466 		return -EINVAL;
1467 
1468 	if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled)) != PPSMC_Result_OK)
1469 		return -EINVAL;
1470 
1471 	return 0;
1472 }
1473 
1474 int rv770_dpm_force_performance_level(struct radeon_device *rdev,
1475 				      enum radeon_dpm_forced_level level)
1476 {
1477 	PPSMC_Msg msg;
1478 
1479 	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1480 		if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_ZeroLevelsDisabled) != PPSMC_Result_OK)
1481 			return -EINVAL;
1482 		msg = PPSMC_MSG_ForceHigh;
1483 	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1484 		if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1485 			return -EINVAL;
1486 		msg = (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled);
1487 	} else {
1488 		if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1489 			return -EINVAL;
1490 		msg = (PPSMC_Msg)(PPSMC_MSG_ZeroLevelsDisabled);
1491 	}
1492 
1493 	if (rv770_send_msg_to_smc(rdev, msg) != PPSMC_Result_OK)
1494 		return -EINVAL;
1495 
1496 	rdev->pm.dpm.forced_level = level;
1497 
1498 	return 0;
1499 }
1500 
1501 void r7xx_start_smc(struct radeon_device *rdev)
1502 {
1503 	rv770_start_smc(rdev);
1504 	rv770_start_smc_clock(rdev);
1505 }
1506 
1507 
1508 void r7xx_stop_smc(struct radeon_device *rdev)
1509 {
1510 	rv770_reset_smc(rdev);
1511 	rv770_stop_smc_clock(rdev);
1512 }
1513 
1514 static void rv770_read_clock_registers(struct radeon_device *rdev)
1515 {
1516 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1517 
1518 	pi->clk_regs.rv770.cg_spll_func_cntl =
1519 		RREG32(CG_SPLL_FUNC_CNTL);
1520 	pi->clk_regs.rv770.cg_spll_func_cntl_2 =
1521 		RREG32(CG_SPLL_FUNC_CNTL_2);
1522 	pi->clk_regs.rv770.cg_spll_func_cntl_3 =
1523 		RREG32(CG_SPLL_FUNC_CNTL_3);
1524 	pi->clk_regs.rv770.cg_spll_spread_spectrum =
1525 		RREG32(CG_SPLL_SPREAD_SPECTRUM);
1526 	pi->clk_regs.rv770.cg_spll_spread_spectrum_2 =
1527 		RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1528 	pi->clk_regs.rv770.mpll_ad_func_cntl =
1529 		RREG32(MPLL_AD_FUNC_CNTL);
1530 	pi->clk_regs.rv770.mpll_ad_func_cntl_2 =
1531 		RREG32(MPLL_AD_FUNC_CNTL_2);
1532 	pi->clk_regs.rv770.mpll_dq_func_cntl =
1533 		RREG32(MPLL_DQ_FUNC_CNTL);
1534 	pi->clk_regs.rv770.mpll_dq_func_cntl_2 =
1535 		RREG32(MPLL_DQ_FUNC_CNTL_2);
1536 	pi->clk_regs.rv770.mclk_pwrmgt_cntl =
1537 		RREG32(MCLK_PWRMGT_CNTL);
1538 	pi->clk_regs.rv770.dll_cntl = RREG32(DLL_CNTL);
1539 }
1540 
1541 static void r7xx_read_clock_registers(struct radeon_device *rdev)
1542 {
1543 	if (rdev->family == CHIP_RV740)
1544 		rv740_read_clock_registers(rdev);
1545 	else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1546 		rv730_read_clock_registers(rdev);
1547 	else
1548 		rv770_read_clock_registers(rdev);
1549 }
1550 
1551 void rv770_read_voltage_smio_registers(struct radeon_device *rdev)
1552 {
1553 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1554 
1555 	pi->s0_vid_lower_smio_cntl =
1556 		RREG32(S0_VID_LOWER_SMIO_CNTL);
1557 }
1558 
1559 void rv770_reset_smio_status(struct radeon_device *rdev)
1560 {
1561 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1562 	u32 sw_smio_index, vid_smio_cntl;
1563 
1564 	sw_smio_index =
1565 		(RREG32(GENERAL_PWRMGT) & SW_SMIO_INDEX_MASK) >> SW_SMIO_INDEX_SHIFT;
1566 	switch (sw_smio_index) {
1567         case 3:
1568 		vid_smio_cntl = RREG32(S3_VID_LOWER_SMIO_CNTL);
1569 		break;
1570         case 2:
1571 		vid_smio_cntl = RREG32(S2_VID_LOWER_SMIO_CNTL);
1572 		break;
1573         case 1:
1574 		vid_smio_cntl = RREG32(S1_VID_LOWER_SMIO_CNTL);
1575 		break;
1576         case 0:
1577 		return;
1578         default:
1579 		vid_smio_cntl = pi->s0_vid_lower_smio_cntl;
1580 		break;
1581 	}
1582 
1583 	WREG32(S0_VID_LOWER_SMIO_CNTL, vid_smio_cntl);
1584 	WREG32_P(GENERAL_PWRMGT, SW_SMIO_INDEX(0), ~SW_SMIO_INDEX_MASK);
1585 }
1586 
1587 void rv770_get_memory_type(struct radeon_device *rdev)
1588 {
1589 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1590 	u32 tmp;
1591 
1592 	tmp = RREG32(MC_SEQ_MISC0);
1593 
1594 	if (((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT) ==
1595 	    MC_SEQ_MISC0_GDDR5_VALUE)
1596 		pi->mem_gddr5 = true;
1597 	else
1598 		pi->mem_gddr5 = false;
1599 
1600 }
1601 
1602 void rv770_get_pcie_gen2_status(struct radeon_device *rdev)
1603 {
1604 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1605 	u32 tmp;
1606 
1607 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1608 
1609 	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
1610 	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
1611 		pi->pcie_gen2 = true;
1612 	else
1613 		pi->pcie_gen2 = false;
1614 
1615 	if (pi->pcie_gen2) {
1616 		if (tmp & LC_CURRENT_DATA_RATE)
1617 			pi->boot_in_gen2 = true;
1618 		else
1619 			pi->boot_in_gen2 = false;
1620 	} else
1621 		pi->boot_in_gen2 = false;
1622 }
1623 
1624 #if 0
1625 static int rv770_enter_ulp_state(struct radeon_device *rdev)
1626 {
1627 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1628 
1629 	if (pi->gfx_clock_gating) {
1630 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1631 		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1632 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1633 		RREG32(GB_TILING_CONFIG);
1634 	}
1635 
1636 	WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1637 		 ~HOST_SMC_MSG_MASK);
1638 
1639 	udelay(7000);
1640 
1641 	return 0;
1642 }
1643 
1644 static int rv770_exit_ulp_state(struct radeon_device *rdev)
1645 {
1646 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1647 	int i;
1648 
1649 	WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_ResumeFromMinimumPower),
1650 		 ~HOST_SMC_MSG_MASK);
1651 
1652 	udelay(7000);
1653 
1654 	for (i = 0; i < rdev->usec_timeout; i++) {
1655 		if (((RREG32(SMC_MSG) & HOST_SMC_RESP_MASK) >> HOST_SMC_RESP_SHIFT) == 1)
1656 			break;
1657 		udelay(1000);
1658 	}
1659 
1660 	if (pi->gfx_clock_gating)
1661 		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
1662 
1663 	return 0;
1664 }
1665 #endif
1666 
1667 static void rv770_get_mclk_odt_threshold(struct radeon_device *rdev)
1668 {
1669 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1670 	u8 memory_module_index;
1671 	struct atom_memory_info memory_info;
1672 
1673 	pi->mclk_odt_threshold = 0;
1674 
1675 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) {
1676 		memory_module_index = rv770_get_memory_module_index(rdev);
1677 
1678 		if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info))
1679 			return;
1680 
1681 		if (memory_info.mem_type == MEM_TYPE_DDR2 ||
1682 		    memory_info.mem_type == MEM_TYPE_DDR3)
1683 			pi->mclk_odt_threshold = 30000;
1684 	}
1685 }
1686 
1687 void rv770_get_max_vddc(struct radeon_device *rdev)
1688 {
1689 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1690 	u16 vddc;
1691 
1692 	if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc))
1693 		pi->max_vddc = 0;
1694 	else
1695 		pi->max_vddc = vddc;
1696 }
1697 
1698 void rv770_program_response_times(struct radeon_device *rdev)
1699 {
1700 	u32 voltage_response_time, backbias_response_time;
1701 	u32 acpi_delay_time, vbi_time_out;
1702 	u32 vddc_dly, bb_dly, acpi_dly, vbi_dly;
1703 	u32 reference_clock;
1704 
1705 	voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1706 	backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1707 
1708 	if (voltage_response_time == 0)
1709 		voltage_response_time = 1000;
1710 
1711 	if (backbias_response_time == 0)
1712 		backbias_response_time = 1000;
1713 
1714 	acpi_delay_time = 15000;
1715 	vbi_time_out = 100000;
1716 
1717 	reference_clock = radeon_get_xclk(rdev);
1718 
1719 	vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1720 	bb_dly = (backbias_response_time * reference_clock) / 1600;
1721 	acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1722 	vbi_dly = (vbi_time_out * reference_clock) / 1600;
1723 
1724 	rv770_write_smc_soft_register(rdev,
1725 				      RV770_SMC_SOFT_REGISTER_delay_vreg, vddc_dly);
1726 	rv770_write_smc_soft_register(rdev,
1727 				      RV770_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1728 	rv770_write_smc_soft_register(rdev,
1729 				      RV770_SMC_SOFT_REGISTER_delay_acpi, acpi_dly);
1730 	rv770_write_smc_soft_register(rdev,
1731 				      RV770_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1732 #if 0
1733 	/* XXX look up hw revision */
1734 	if (WEKIVA_A21)
1735 		rv770_write_smc_soft_register(rdev,
1736 					      RV770_SMC_SOFT_REGISTER_baby_step_timer,
1737 					      0x10);
1738 #endif
1739 }
1740 
1741 static void rv770_program_dcodt_before_state_switch(struct radeon_device *rdev,
1742 						    struct radeon_ps *radeon_new_state,
1743 						    struct radeon_ps *radeon_current_state)
1744 {
1745 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1746 	struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
1747 	struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state);
1748 	bool current_use_dc = false;
1749 	bool new_use_dc = false;
1750 
1751 	if (pi->mclk_odt_threshold == 0)
1752 		return;
1753 
1754 	if (current_state->high.mclk <= pi->mclk_odt_threshold)
1755 		current_use_dc = true;
1756 
1757 	if (new_state->high.mclk <= pi->mclk_odt_threshold)
1758 		new_use_dc = true;
1759 
1760 	if (current_use_dc == new_use_dc)
1761 		return;
1762 
1763 	if (!current_use_dc && new_use_dc)
1764 		return;
1765 
1766 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1767 		rv730_program_dcodt(rdev, new_use_dc);
1768 }
1769 
1770 static void rv770_program_dcodt_after_state_switch(struct radeon_device *rdev,
1771 						   struct radeon_ps *radeon_new_state,
1772 						   struct radeon_ps *radeon_current_state)
1773 {
1774 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1775 	struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
1776 	struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state);
1777 	bool current_use_dc = false;
1778 	bool new_use_dc = false;
1779 
1780 	if (pi->mclk_odt_threshold == 0)
1781 		return;
1782 
1783 	if (current_state->high.mclk <= pi->mclk_odt_threshold)
1784 		current_use_dc = true;
1785 
1786 	if (new_state->high.mclk <= pi->mclk_odt_threshold)
1787 		new_use_dc = true;
1788 
1789 	if (current_use_dc == new_use_dc)
1790 		return;
1791 
1792 	if (current_use_dc && !new_use_dc)
1793 		return;
1794 
1795 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1796 		rv730_program_dcodt(rdev, new_use_dc);
1797 }
1798 
1799 static void rv770_retrieve_odt_values(struct radeon_device *rdev)
1800 {
1801 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1802 
1803 	if (pi->mclk_odt_threshold == 0)
1804 		return;
1805 
1806 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1807 		rv730_get_odt_values(rdev);
1808 }
1809 
1810 static void rv770_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
1811 {
1812 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1813 	bool want_thermal_protection;
1814 	enum radeon_dpm_event_src dpm_event_src;
1815 
1816 	switch (sources) {
1817         case 0:
1818         default:
1819 		want_thermal_protection = false;
1820 		break;
1821         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
1822 		want_thermal_protection = true;
1823 		dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL;
1824 		break;
1825 
1826         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
1827 		want_thermal_protection = true;
1828 		dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL;
1829 		break;
1830 
1831         case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
1832 	      (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
1833 		want_thermal_protection = true;
1834 		dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
1835 		break;
1836 	}
1837 
1838 	if (want_thermal_protection) {
1839 		WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
1840 		if (pi->thermal_protection)
1841 			WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
1842 	} else {
1843 		WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
1844 	}
1845 }
1846 
1847 void rv770_enable_auto_throttle_source(struct radeon_device *rdev,
1848 				       enum radeon_dpm_auto_throttle_src source,
1849 				       bool enable)
1850 {
1851 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1852 
1853 	if (enable) {
1854 		if (!(pi->active_auto_throttle_sources & (1 << source))) {
1855 			pi->active_auto_throttle_sources |= 1 << source;
1856 			rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1857 		}
1858 	} else {
1859 		if (pi->active_auto_throttle_sources & (1 << source)) {
1860 			pi->active_auto_throttle_sources &= ~(1 << source);
1861 			rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1862 		}
1863 	}
1864 }
1865 
1866 static int rv770_set_thermal_temperature_range(struct radeon_device *rdev,
1867 					       int min_temp, int max_temp)
1868 {
1869 	int low_temp = 0 * 1000;
1870 	int high_temp = 255 * 1000;
1871 
1872 	if (low_temp < min_temp)
1873 		low_temp = min_temp;
1874 	if (high_temp > max_temp)
1875 		high_temp = max_temp;
1876 	if (high_temp < low_temp) {
1877 		DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1878 		return -EINVAL;
1879 	}
1880 
1881 	WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK);
1882 	WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK);
1883 	WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK);
1884 
1885 	rdev->pm.dpm.thermal.min_temp = low_temp;
1886 	rdev->pm.dpm.thermal.max_temp = high_temp;
1887 
1888 	return 0;
1889 }
1890 
1891 int rv770_dpm_enable(struct radeon_device *rdev)
1892 {
1893 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1894 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
1895 	int ret;
1896 
1897 	if (pi->gfx_clock_gating)
1898 		rv770_restore_cgcg(rdev);
1899 
1900 	if (rv770_dpm_enabled(rdev))
1901 		return -EINVAL;
1902 
1903 	if (pi->voltage_control) {
1904 		rv770_enable_voltage_control(rdev, true);
1905 		ret = rv770_construct_vddc_table(rdev);
1906 		if (ret) {
1907 			DRM_ERROR("rv770_construct_vddc_table failed\n");
1908 			return ret;
1909 		}
1910 	}
1911 
1912 	if (pi->dcodt)
1913 		rv770_retrieve_odt_values(rdev);
1914 
1915 	if (pi->mvdd_control) {
1916 		ret = rv770_get_mvdd_configuration(rdev);
1917 		if (ret) {
1918 			DRM_ERROR("rv770_get_mvdd_configuration failed\n");
1919 			return ret;
1920 		}
1921 	}
1922 
1923 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
1924 		rv770_enable_backbias(rdev, true);
1925 
1926 	rv770_enable_spread_spectrum(rdev, true);
1927 
1928 	if (pi->thermal_protection)
1929 		rv770_enable_thermal_protection(rdev, true);
1930 
1931 	rv770_program_mpll_timing_parameters(rdev);
1932 	rv770_setup_bsp(rdev);
1933 	rv770_program_git(rdev);
1934 	rv770_program_tp(rdev);
1935 	rv770_program_tpp(rdev);
1936 	rv770_program_sstp(rdev);
1937 	rv770_program_engine_speed_parameters(rdev);
1938 	rv770_enable_display_gap(rdev);
1939 	rv770_program_vc(rdev);
1940 
1941 	if (pi->dynamic_pcie_gen2)
1942 		rv770_enable_dynamic_pcie_gen2(rdev, true);
1943 
1944 	ret = rv770_upload_firmware(rdev);
1945 	if (ret) {
1946 		DRM_ERROR("rv770_upload_firmware failed\n");
1947 		return ret;
1948 	}
1949 	ret = rv770_init_smc_table(rdev, boot_ps);
1950 	if (ret) {
1951 		DRM_ERROR("rv770_init_smc_table failed\n");
1952 		return ret;
1953 	}
1954 
1955 	rv770_program_response_times(rdev);
1956 	r7xx_start_smc(rdev);
1957 
1958 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1959 		rv730_start_dpm(rdev);
1960 	else
1961 		rv770_start_dpm(rdev);
1962 
1963 	if (pi->gfx_clock_gating)
1964 		rv770_gfx_clock_gating_enable(rdev, true);
1965 
1966 	if (pi->mg_clock_gating)
1967 		rv770_mg_clock_gating_enable(rdev, true);
1968 
1969 	rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
1970 
1971 	return 0;
1972 }
1973 
1974 int rv770_dpm_late_enable(struct radeon_device *rdev)
1975 {
1976 	int ret;
1977 
1978 	if (rdev->irq.installed &&
1979 	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1980 		PPSMC_Result result;
1981 
1982 		ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1983 		if (ret)
1984 			return ret;
1985 		rdev->irq.dpm_thermal = true;
1986 		radeon_irq_set(rdev);
1987 		result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
1988 
1989 		if (result != PPSMC_Result_OK)
1990 			DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
1991 	}
1992 
1993 	return 0;
1994 }
1995 
1996 void rv770_dpm_disable(struct radeon_device *rdev)
1997 {
1998 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1999 
2000 	if (!rv770_dpm_enabled(rdev))
2001 		return;
2002 
2003 	rv770_clear_vc(rdev);
2004 
2005 	if (pi->thermal_protection)
2006 		rv770_enable_thermal_protection(rdev, false);
2007 
2008 	rv770_enable_spread_spectrum(rdev, false);
2009 
2010 	if (pi->dynamic_pcie_gen2)
2011 		rv770_enable_dynamic_pcie_gen2(rdev, false);
2012 
2013 	if (rdev->irq.installed &&
2014 	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
2015 		rdev->irq.dpm_thermal = false;
2016 		radeon_irq_set(rdev);
2017 	}
2018 
2019 	if (pi->gfx_clock_gating)
2020 		rv770_gfx_clock_gating_enable(rdev, false);
2021 
2022 	if (pi->mg_clock_gating)
2023 		rv770_mg_clock_gating_enable(rdev, false);
2024 
2025 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
2026 		rv730_stop_dpm(rdev);
2027 	else
2028 		rv770_stop_dpm(rdev);
2029 
2030 	r7xx_stop_smc(rdev);
2031 	rv770_reset_smio_status(rdev);
2032 }
2033 
2034 int rv770_dpm_set_power_state(struct radeon_device *rdev)
2035 {
2036 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2037 	struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
2038 	struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
2039 	int ret;
2040 
2041 	ret = rv770_restrict_performance_levels_before_switch(rdev);
2042 	if (ret) {
2043 		DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n");
2044 		return ret;
2045 	}
2046 	rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
2047 	ret = rv770_halt_smc(rdev);
2048 	if (ret) {
2049 		DRM_ERROR("rv770_halt_smc failed\n");
2050 		return ret;
2051 	}
2052 	ret = rv770_upload_sw_state(rdev, new_ps);
2053 	if (ret) {
2054 		DRM_ERROR("rv770_upload_sw_state failed\n");
2055 		return ret;
2056 	}
2057 	r7xx_program_memory_timing_parameters(rdev, new_ps);
2058 	if (pi->dcodt)
2059 		rv770_program_dcodt_before_state_switch(rdev, new_ps, old_ps);
2060 	ret = rv770_resume_smc(rdev);
2061 	if (ret) {
2062 		DRM_ERROR("rv770_resume_smc failed\n");
2063 		return ret;
2064 	}
2065 	ret = rv770_set_sw_state(rdev);
2066 	if (ret) {
2067 		DRM_ERROR("rv770_set_sw_state failed\n");
2068 		return ret;
2069 	}
2070 	if (pi->dcodt)
2071 		rv770_program_dcodt_after_state_switch(rdev, new_ps, old_ps);
2072 	rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
2073 
2074 	return 0;
2075 }
2076 
2077 void rv770_dpm_reset_asic(struct radeon_device *rdev)
2078 {
2079 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2080 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
2081 
2082 	rv770_restrict_performance_levels_before_switch(rdev);
2083 	if (pi->dcodt)
2084 		rv770_program_dcodt_before_state_switch(rdev, boot_ps, boot_ps);
2085 	rv770_set_boot_state(rdev);
2086 	if (pi->dcodt)
2087 		rv770_program_dcodt_after_state_switch(rdev, boot_ps, boot_ps);
2088 }
2089 
2090 void rv770_dpm_setup_asic(struct radeon_device *rdev)
2091 {
2092 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2093 
2094 	r7xx_read_clock_registers(rdev);
2095 	rv770_read_voltage_smio_registers(rdev);
2096 	rv770_get_memory_type(rdev);
2097 	if (pi->dcodt)
2098 		rv770_get_mclk_odt_threshold(rdev);
2099 	rv770_get_pcie_gen2_status(rdev);
2100 
2101 	rv770_enable_acpi_pm(rdev);
2102 
2103 	if (radeon_aspm != 0) {
2104 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s)
2105 			rv770_enable_l0s(rdev);
2106 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1)
2107 			rv770_enable_l1(rdev);
2108 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1)
2109 			rv770_enable_pll_sleep_in_l1(rdev);
2110 	}
2111 }
2112 
2113 void rv770_dpm_display_configuration_changed(struct radeon_device *rdev)
2114 {
2115 	rv770_program_display_gap(rdev);
2116 }
2117 
2118 union power_info {
2119 	struct _ATOM_POWERPLAY_INFO info;
2120 	struct _ATOM_POWERPLAY_INFO_V2 info_2;
2121 	struct _ATOM_POWERPLAY_INFO_V3 info_3;
2122 	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
2123 	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
2124 	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
2125 };
2126 
2127 union pplib_clock_info {
2128 	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
2129 	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
2130 	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
2131 	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
2132 };
2133 
2134 union pplib_power_state {
2135 	struct _ATOM_PPLIB_STATE v1;
2136 	struct _ATOM_PPLIB_STATE_V2 v2;
2137 };
2138 
2139 static void rv7xx_parse_pplib_non_clock_info(struct radeon_device *rdev,
2140 					     struct radeon_ps *rps,
2141 					     struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
2142 					     u8 table_rev)
2143 {
2144 	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
2145 	rps->class = le16_to_cpu(non_clock_info->usClassification);
2146 	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
2147 
2148 	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
2149 		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
2150 		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
2151 	} else {
2152 		rps->vclk = 0;
2153 		rps->dclk = 0;
2154 	}
2155 
2156 	if (r600_is_uvd_state(rps->class, rps->class2)) {
2157 		if ((rps->vclk == 0) || (rps->dclk == 0)) {
2158 			rps->vclk = RV770_DEFAULT_VCLK_FREQ;
2159 			rps->dclk = RV770_DEFAULT_DCLK_FREQ;
2160 		}
2161 	}
2162 
2163 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
2164 		rdev->pm.dpm.boot_ps = rps;
2165 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
2166 		rdev->pm.dpm.uvd_ps = rps;
2167 }
2168 
2169 static void rv7xx_parse_pplib_clock_info(struct radeon_device *rdev,
2170 					 struct radeon_ps *rps, int index,
2171 					 union pplib_clock_info *clock_info)
2172 {
2173 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2174 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2175 	struct rv7xx_ps *ps = rv770_get_ps(rps);
2176 	u32 sclk, mclk;
2177 	struct rv7xx_pl *pl;
2178 
2179 	switch (index) {
2180 	case 0:
2181 		pl = &ps->low;
2182 		break;
2183 	case 1:
2184 		pl = &ps->medium;
2185 		break;
2186 	case 2:
2187 	default:
2188 		pl = &ps->high;
2189 		break;
2190 	}
2191 
2192 	if (rdev->family >= CHIP_CEDAR) {
2193 		sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
2194 		sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
2195 		mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
2196 		mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
2197 
2198 		pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
2199 		pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
2200 		pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
2201 	} else {
2202 		sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
2203 		sclk |= clock_info->r600.ucEngineClockHigh << 16;
2204 		mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
2205 		mclk |= clock_info->r600.ucMemoryClockHigh << 16;
2206 
2207 		pl->vddc = le16_to_cpu(clock_info->r600.usVDDC);
2208 		pl->flags = le32_to_cpu(clock_info->r600.ulFlags);
2209 	}
2210 
2211 	pl->mclk = mclk;
2212 	pl->sclk = sclk;
2213 
2214 	/* patch up vddc if necessary */
2215 	if (pl->vddc == 0xff01) {
2216 		if (pi->max_vddc)
2217 			pl->vddc = pi->max_vddc;
2218 	}
2219 
2220 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
2221 		pi->acpi_vddc = pl->vddc;
2222 		if (rdev->family >= CHIP_CEDAR)
2223 			eg_pi->acpi_vddci = pl->vddci;
2224 		if (ps->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
2225 			pi->acpi_pcie_gen2 = true;
2226 		else
2227 			pi->acpi_pcie_gen2 = false;
2228 	}
2229 
2230 	if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
2231 		if (rdev->family >= CHIP_BARTS) {
2232 			eg_pi->ulv.supported = true;
2233 			eg_pi->ulv.pl = pl;
2234 		}
2235 	}
2236 
2237 	if (pi->min_vddc_in_table > pl->vddc)
2238 		pi->min_vddc_in_table = pl->vddc;
2239 
2240 	if (pi->max_vddc_in_table < pl->vddc)
2241 		pi->max_vddc_in_table = pl->vddc;
2242 
2243 	/* patch up boot state */
2244 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
2245 		u16 vddc, vddci, mvdd;
2246 		radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
2247 		pl->mclk = rdev->clock.default_mclk;
2248 		pl->sclk = rdev->clock.default_sclk;
2249 		pl->vddc = vddc;
2250 		pl->vddci = vddci;
2251 	}
2252 
2253 	if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
2254 	    ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
2255 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
2256 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
2257 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
2258 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
2259 	}
2260 }
2261 
2262 int rv7xx_parse_power_table(struct radeon_device *rdev)
2263 {
2264 	struct radeon_mode_info *mode_info = &rdev->mode_info;
2265 	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
2266 	union pplib_power_state *power_state;
2267 	int i, j;
2268 	union pplib_clock_info *clock_info;
2269 	union power_info *power_info;
2270 	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
2271         u16 data_offset;
2272 	u8 frev, crev;
2273 	struct rv7xx_ps *ps;
2274 
2275 	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
2276 				   &frev, &crev, &data_offset))
2277 		return -EINVAL;
2278 	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
2279 
2280 	rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
2281 				  power_info->pplib.ucNumStates, GFP_KERNEL);
2282 	if (!rdev->pm.dpm.ps)
2283 		return -ENOMEM;
2284 
2285 	for (i = 0; i < power_info->pplib.ucNumStates; i++) {
2286 		power_state = (union pplib_power_state *)
2287 			(mode_info->atom_context->bios + data_offset +
2288 			 le16_to_cpu(power_info->pplib.usStateArrayOffset) +
2289 			 i * power_info->pplib.ucStateEntrySize);
2290 		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
2291 			(mode_info->atom_context->bios + data_offset +
2292 			 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
2293 			 (power_state->v1.ucNonClockStateIndex *
2294 			  power_info->pplib.ucNonClockSize));
2295 		if (power_info->pplib.ucStateEntrySize - 1) {
2296 			u8 *idx;
2297 			ps = kzalloc(sizeof(struct rv7xx_ps), GFP_KERNEL);
2298 			if (ps == NULL) {
2299 				kfree(rdev->pm.dpm.ps);
2300 				return -ENOMEM;
2301 			}
2302 			rdev->pm.dpm.ps[i].ps_priv = ps;
2303 			rv7xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
2304 							 non_clock_info,
2305 							 power_info->pplib.ucNonClockSize);
2306 			idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
2307 			for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
2308 				clock_info = (union pplib_clock_info *)
2309 					(mode_info->atom_context->bios + data_offset +
2310 					 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
2311 					 (idx[j] * power_info->pplib.ucClockInfoSize));
2312 				rv7xx_parse_pplib_clock_info(rdev,
2313 							     &rdev->pm.dpm.ps[i], j,
2314 							     clock_info);
2315 			}
2316 		}
2317 	}
2318 	rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
2319 	return 0;
2320 }
2321 
2322 void rv770_get_engine_memory_ss(struct radeon_device *rdev)
2323 {
2324 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2325 	struct radeon_atom_ss ss;
2326 
2327 	pi->sclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
2328 						       ASIC_INTERNAL_ENGINE_SS, 0);
2329 	pi->mclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
2330 						       ASIC_INTERNAL_MEMORY_SS, 0);
2331 
2332 	/* disable ss, causes hangs on some cayman boards */
2333 	if (rdev->family == CHIP_CAYMAN) {
2334 		pi->sclk_ss = false;
2335 		pi->mclk_ss = false;
2336 	}
2337 
2338 	if (pi->sclk_ss || pi->mclk_ss)
2339 		pi->dynamic_ss = true;
2340 	else
2341 		pi->dynamic_ss = false;
2342 }
2343 
2344 int rv770_dpm_init(struct radeon_device *rdev)
2345 {
2346 	struct rv7xx_power_info *pi;
2347 	struct atom_clock_dividers dividers;
2348 	int ret;
2349 
2350 	pi = kzalloc(sizeof(struct rv7xx_power_info), GFP_KERNEL);
2351 	if (pi == NULL)
2352 		return -ENOMEM;
2353 	rdev->pm.dpm.priv = pi;
2354 
2355 	rv770_get_max_vddc(rdev);
2356 
2357 	pi->acpi_vddc = 0;
2358 	pi->min_vddc_in_table = 0;
2359 	pi->max_vddc_in_table = 0;
2360 
2361 	ret = r600_get_platform_caps(rdev);
2362 	if (ret)
2363 		return ret;
2364 
2365 	ret = rv7xx_parse_power_table(rdev);
2366 	if (ret)
2367 		return ret;
2368 
2369 	if (rdev->pm.dpm.voltage_response_time == 0)
2370 		rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
2371 	if (rdev->pm.dpm.backbias_response_time == 0)
2372 		rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
2373 
2374 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2375 					     0, false, &dividers);
2376 	if (ret)
2377 		pi->ref_div = dividers.ref_div + 1;
2378 	else
2379 		pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
2380 
2381 	pi->mclk_strobe_mode_threshold = 30000;
2382 	pi->mclk_edc_enable_threshold = 30000;
2383 
2384 	pi->rlp = RV770_RLP_DFLT;
2385 	pi->rmp = RV770_RMP_DFLT;
2386 	pi->lhp = RV770_LHP_DFLT;
2387 	pi->lmp = RV770_LMP_DFLT;
2388 
2389 	pi->voltage_control =
2390 		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
2391 
2392 	pi->mvdd_control =
2393 		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
2394 
2395 	rv770_get_engine_memory_ss(rdev);
2396 
2397 	pi->asi = RV770_ASI_DFLT;
2398 	pi->pasi = RV770_HASI_DFLT;
2399 	pi->vrc = RV770_VRC_DFLT;
2400 
2401 	pi->power_gating = false;
2402 
2403 	pi->gfx_clock_gating = true;
2404 
2405 	pi->mg_clock_gating = true;
2406 	pi->mgcgtssm = true;
2407 
2408 	pi->dynamic_pcie_gen2 = true;
2409 
2410 	if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
2411 		pi->thermal_protection = true;
2412 	else
2413 		pi->thermal_protection = false;
2414 
2415 	pi->display_gap = true;
2416 
2417 	if (rdev->flags & RADEON_IS_MOBILITY)
2418 		pi->dcodt = true;
2419 	else
2420 		pi->dcodt = false;
2421 
2422 	pi->ulps = true;
2423 
2424 	pi->mclk_stutter_mode_threshold = 0;
2425 
2426 	pi->sram_end = SMC_RAM_END;
2427 	pi->state_table_start = RV770_SMC_TABLE_ADDRESS;
2428 	pi->soft_regs_start = RV770_SMC_SOFT_REGISTERS_START;
2429 
2430 	return 0;
2431 }
2432 
2433 void rv770_dpm_print_power_state(struct radeon_device *rdev,
2434 				 struct radeon_ps *rps)
2435 {
2436 	struct rv7xx_ps *ps = rv770_get_ps(rps);
2437 	struct rv7xx_pl *pl;
2438 
2439 	r600_dpm_print_class_info(rps->class, rps->class2);
2440 	r600_dpm_print_cap_info(rps->caps);
2441 	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2442 	if (rdev->family >= CHIP_CEDAR) {
2443 		pl = &ps->low;
2444 		printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2445 		       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2446 		pl = &ps->medium;
2447 		printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2448 		       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2449 		pl = &ps->high;
2450 		printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2451 		       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2452 	} else {
2453 		pl = &ps->low;
2454 		printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u\n",
2455 		       pl->sclk, pl->mclk, pl->vddc);
2456 		pl = &ps->medium;
2457 		printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u\n",
2458 		       pl->sclk, pl->mclk, pl->vddc);
2459 		pl = &ps->high;
2460 		printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u\n",
2461 		       pl->sclk, pl->mclk, pl->vddc);
2462 	}
2463 	r600_dpm_print_ps_status(rdev, rps);
2464 }
2465 
2466 void rv770_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
2467 						       struct seq_file *m)
2468 {
2469 	struct radeon_ps *rps = rdev->pm.dpm.current_ps;
2470 	struct rv7xx_ps *ps = rv770_get_ps(rps);
2471 	struct rv7xx_pl *pl;
2472 	u32 current_index =
2473 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
2474 		CURRENT_PROFILE_INDEX_SHIFT;
2475 
2476 	if (current_index > 2) {
2477 		seq_printf(m, "invalid dpm profile %d\n", current_index);
2478 	} else {
2479 		if (current_index == 0)
2480 			pl = &ps->low;
2481 		else if (current_index == 1)
2482 			pl = &ps->medium;
2483 		else /* current_index == 2 */
2484 			pl = &ps->high;
2485 		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2486 		if (rdev->family >= CHIP_CEDAR) {
2487 			seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2488 				   current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2489 		} else {
2490 			seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u\n",
2491 				   current_index, pl->sclk, pl->mclk, pl->vddc);
2492 		}
2493 	}
2494 }
2495 
2496 void rv770_dpm_fini(struct radeon_device *rdev)
2497 {
2498 	int i;
2499 
2500 	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2501 		kfree(rdev->pm.dpm.ps[i].ps_priv);
2502 	}
2503 	kfree(rdev->pm.dpm.ps);
2504 	kfree(rdev->pm.dpm.priv);
2505 }
2506 
2507 u32 rv770_dpm_get_sclk(struct radeon_device *rdev, bool low)
2508 {
2509 	struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps);
2510 
2511 	if (low)
2512 		return requested_state->low.sclk;
2513 	else
2514 		return requested_state->high.sclk;
2515 }
2516 
2517 u32 rv770_dpm_get_mclk(struct radeon_device *rdev, bool low)
2518 {
2519 	struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps);
2520 
2521 	if (low)
2522 		return requested_state->low.mclk;
2523 	else
2524 		return requested_state->high.mclk;
2525 }
2526 
2527 bool rv770_dpm_vblank_too_short(struct radeon_device *rdev)
2528 {
2529 	u32 vblank_time = r600_dpm_get_vblank_time(rdev);
2530 	u32 switch_limit = 200; /* 300 */
2531 
2532 	/* RV770 */
2533 	/* mclk switching doesn't seem to work reliably on desktop RV770s */
2534 	if ((rdev->family == CHIP_RV770) &&
2535 	    !(rdev->flags & RADEON_IS_MOBILITY))
2536 		switch_limit = 0xffffffff; /* disable mclk switching */
2537 
2538 	if (vblank_time < switch_limit)
2539 		return true;
2540 	else
2541 		return false;
2542 
2543 }
2544