xref: /openbmc/linux/drivers/gpu/drm/radeon/rv770_dpm.c (revision c819e2cf)
1 /*
2  * Copyright 2011 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 
25 #include "drmP.h"
26 #include "radeon.h"
27 #include "radeon_asic.h"
28 #include "rv770d.h"
29 #include "r600_dpm.h"
30 #include "rv770_dpm.h"
31 #include "cypress_dpm.h"
32 #include "atom.h"
33 #include <linux/seq_file.h>
34 
35 #define MC_CG_ARB_FREQ_F0           0x0a
36 #define MC_CG_ARB_FREQ_F1           0x0b
37 #define MC_CG_ARB_FREQ_F2           0x0c
38 #define MC_CG_ARB_FREQ_F3           0x0d
39 
40 #define MC_CG_SEQ_DRAMCONF_S0       0x05
41 #define MC_CG_SEQ_DRAMCONF_S1       0x06
42 
43 #define PCIE_BUS_CLK                10000
44 #define TCLK                        (PCIE_BUS_CLK / 10)
45 
46 #define SMC_RAM_END 0xC000
47 
48 struct rv7xx_ps *rv770_get_ps(struct radeon_ps *rps)
49 {
50 	struct rv7xx_ps *ps = rps->ps_priv;
51 
52 	return ps;
53 }
54 
55 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev)
56 {
57 	struct rv7xx_power_info *pi = rdev->pm.dpm.priv;
58 
59 	return pi;
60 }
61 
62 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev)
63 {
64 	struct evergreen_power_info *pi = rdev->pm.dpm.priv;
65 
66 	return pi;
67 }
68 
69 static void rv770_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
70 					       bool enable)
71 {
72 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
73 	u32 tmp;
74 
75 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
76 	if (enable) {
77 		tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
78 		tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
79 		tmp |= LC_GEN2_EN_STRAP;
80 	} else {
81 		if (!pi->boot_in_gen2) {
82 			tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
83 			tmp &= ~LC_GEN2_EN_STRAP;
84 		}
85 	}
86 	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
87 	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
88 		WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
89 
90 }
91 
92 static void rv770_enable_l0s(struct radeon_device *rdev)
93 {
94 	u32 tmp;
95 
96 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L0S_INACTIVITY_MASK;
97 	tmp |= LC_L0S_INACTIVITY(3);
98 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
99 }
100 
101 static void rv770_enable_l1(struct radeon_device *rdev)
102 {
103 	u32 tmp;
104 
105 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL);
106 	tmp &= ~LC_L1_INACTIVITY_MASK;
107 	tmp |= LC_L1_INACTIVITY(4);
108 	tmp &= ~LC_PMI_TO_L1_DIS;
109 	tmp &= ~LC_ASPM_TO_L1_DIS;
110 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
111 }
112 
113 static void rv770_enable_pll_sleep_in_l1(struct radeon_device *rdev)
114 {
115 	u32 tmp;
116 
117 	tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL) & ~LC_L1_INACTIVITY_MASK;
118 	tmp |= LC_L1_INACTIVITY(8);
119 	WREG32_PCIE_PORT(PCIE_LC_CNTL, tmp);
120 
121 	/* NOTE, this is a PCIE indirect reg, not PCIE PORT */
122 	tmp = RREG32_PCIE(PCIE_P_CNTL);
123 	tmp |= P_PLL_PWRDN_IN_L1L23;
124 	tmp &= ~P_PLL_BUF_PDNB;
125 	tmp &= ~P_PLL_PDNB;
126 	tmp |= P_ALLOW_PRX_FRONTEND_SHUTOFF;
127 	WREG32_PCIE(PCIE_P_CNTL, tmp);
128 }
129 
130 static void rv770_gfx_clock_gating_enable(struct radeon_device *rdev,
131 					  bool enable)
132 {
133 	if (enable)
134 		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
135 	else {
136 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
137 		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
138 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
139 		RREG32(GB_TILING_CONFIG);
140 	}
141 }
142 
143 static void rv770_mg_clock_gating_enable(struct radeon_device *rdev,
144 					 bool enable)
145 {
146 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
147 
148 	if (enable) {
149 		u32 mgcg_cgtt_local0;
150 
151 		if (rdev->family == CHIP_RV770)
152 			mgcg_cgtt_local0 = RV770_MGCGTTLOCAL0_DFLT;
153 		else
154 			mgcg_cgtt_local0 = RV7XX_MGCGTTLOCAL0_DFLT;
155 
156 		WREG32(CG_CGTT_LOCAL_0, mgcg_cgtt_local0);
157 		WREG32(CG_CGTT_LOCAL_1, (RV770_MGCGTTLOCAL1_DFLT & 0xFFFFCFFF));
158 
159 		if (pi->mgcgtssm)
160 			WREG32(CGTS_SM_CTRL_REG, RV770_MGCGCGTSSMCTRL_DFLT);
161 	} else {
162 		WREG32(CG_CGTT_LOCAL_0, 0xFFFFFFFF);
163 		WREG32(CG_CGTT_LOCAL_1, 0xFFFFCFFF);
164 	}
165 }
166 
167 void rv770_restore_cgcg(struct radeon_device *rdev)
168 {
169 	bool dpm_en = false, cg_en = false;
170 
171 	if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
172 		dpm_en = true;
173 	if (RREG32(SCLK_PWRMGT_CNTL) & DYN_GFX_CLK_OFF_EN)
174 		cg_en = true;
175 
176 	if (dpm_en && !cg_en)
177 		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
178 }
179 
180 static void rv770_start_dpm(struct radeon_device *rdev)
181 {
182 	WREG32_P(SCLK_PWRMGT_CNTL, 0, ~SCLK_PWRMGT_OFF);
183 
184 	WREG32_P(MCLK_PWRMGT_CNTL, 0, ~MPLL_PWRMGT_OFF);
185 
186 	WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
187 }
188 
189 void rv770_stop_dpm(struct radeon_device *rdev)
190 {
191 	PPSMC_Result result;
192 
193 	result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_TwoLevelsDisabled);
194 
195 	if (result != PPSMC_Result_OK)
196 		DRM_ERROR("Could not force DPM to low.\n");
197 
198 	WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
199 
200 	WREG32_P(SCLK_PWRMGT_CNTL, SCLK_PWRMGT_OFF, ~SCLK_PWRMGT_OFF);
201 
202 	WREG32_P(MCLK_PWRMGT_CNTL, MPLL_PWRMGT_OFF, ~MPLL_PWRMGT_OFF);
203 }
204 
205 bool rv770_dpm_enabled(struct radeon_device *rdev)
206 {
207 	if (RREG32(GENERAL_PWRMGT) & GLOBAL_PWRMGT_EN)
208 		return true;
209 	else
210 		return false;
211 }
212 
213 void rv770_enable_thermal_protection(struct radeon_device *rdev,
214 				     bool enable)
215 {
216 	if (enable)
217 		WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
218 	else
219 		WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
220 }
221 
222 void rv770_enable_acpi_pm(struct radeon_device *rdev)
223 {
224 	WREG32_P(GENERAL_PWRMGT, STATIC_PM_EN, ~STATIC_PM_EN);
225 }
226 
227 u8 rv770_get_seq_value(struct radeon_device *rdev,
228 		       struct rv7xx_pl *pl)
229 {
230 	return (pl->flags & ATOM_PPLIB_R600_FLAGS_LOWPOWER) ?
231 		MC_CG_SEQ_DRAMCONF_S0 : MC_CG_SEQ_DRAMCONF_S1;
232 }
233 
234 int rv770_read_smc_soft_register(struct radeon_device *rdev,
235 				 u16 reg_offset, u32 *value)
236 {
237 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
238 
239 	return rv770_read_smc_sram_dword(rdev,
240 					 pi->soft_regs_start + reg_offset,
241 					 value, pi->sram_end);
242 }
243 
244 int rv770_write_smc_soft_register(struct radeon_device *rdev,
245 				  u16 reg_offset, u32 value)
246 {
247 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
248 
249 	return rv770_write_smc_sram_dword(rdev,
250 					  pi->soft_regs_start + reg_offset,
251 					  value, pi->sram_end);
252 }
253 
254 int rv770_populate_smc_t(struct radeon_device *rdev,
255 			 struct radeon_ps *radeon_state,
256 			 RV770_SMC_SWSTATE *smc_state)
257 {
258 	struct rv7xx_ps *state = rv770_get_ps(radeon_state);
259 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
260 	int i;
261 	int a_n;
262 	int a_d;
263 	u8 l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
264 	u8 r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE];
265 	u32 a_t;
266 
267 	l[0] = 0;
268 	r[2] = 100;
269 
270 	a_n = (int)state->medium.sclk * pi->lmp +
271 		(int)state->low.sclk * (R600_AH_DFLT - pi->rlp);
272 	a_d = (int)state->low.sclk * (100 - (int)pi->rlp) +
273 		(int)state->medium.sclk * pi->lmp;
274 
275 	l[1] = (u8)(pi->lmp - (int)pi->lmp * a_n / a_d);
276 	r[0] = (u8)(pi->rlp + (100 - (int)pi->rlp) * a_n / a_d);
277 
278 	a_n = (int)state->high.sclk * pi->lhp + (int)state->medium.sclk *
279 		(R600_AH_DFLT - pi->rmp);
280 	a_d = (int)state->medium.sclk * (100 - (int)pi->rmp) +
281 		(int)state->high.sclk * pi->lhp;
282 
283 	l[2] = (u8)(pi->lhp - (int)pi->lhp * a_n / a_d);
284 	r[1] = (u8)(pi->rmp + (100 - (int)pi->rmp) * a_n / a_d);
285 
286 	for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++) {
287 		a_t = CG_R(r[i] * pi->bsp / 200) | CG_L(l[i] * pi->bsp / 200);
288 		smc_state->levels[i].aT = cpu_to_be32(a_t);
289 	}
290 
291 	a_t = CG_R(r[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200) |
292 		CG_L(l[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1] * pi->pbsp / 200);
293 
294 	smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].aT =
295 		cpu_to_be32(a_t);
296 
297 	return 0;
298 }
299 
300 int rv770_populate_smc_sp(struct radeon_device *rdev,
301 			  struct radeon_ps *radeon_state,
302 			  RV770_SMC_SWSTATE *smc_state)
303 {
304 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
305 	int i;
306 
307 	for (i = 0; i < (RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1); i++)
308 		smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
309 
310 	smc_state->levels[RV770_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1].bSP =
311 		cpu_to_be32(pi->psp);
312 
313 	return 0;
314 }
315 
316 static void rv770_calculate_fractional_mpll_feedback_divider(u32 memory_clock,
317 							     u32 reference_clock,
318 							     bool gddr5,
319 							     struct atom_clock_dividers *dividers,
320 							     u32 *clkf,
321 							     u32 *clkfrac)
322 {
323 	u32 post_divider, reference_divider, feedback_divider8;
324 	u32 fyclk;
325 
326 	if (gddr5)
327 		fyclk = (memory_clock * 8) / 2;
328 	else
329 		fyclk = (memory_clock * 4) / 2;
330 
331 	post_divider = dividers->post_div;
332 	reference_divider = dividers->ref_div;
333 
334 	feedback_divider8 =
335 		(8 * fyclk * reference_divider * post_divider) / reference_clock;
336 
337 	*clkf = feedback_divider8 / 8;
338 	*clkfrac = feedback_divider8 % 8;
339 }
340 
341 static int rv770_encode_yclk_post_div(u32 postdiv, u32 *encoded_postdiv)
342 {
343 	int ret = 0;
344 
345 	switch (postdiv) {
346         case 1:
347 		*encoded_postdiv = 0;
348 		break;
349         case 2:
350 		*encoded_postdiv = 1;
351 		break;
352         case 4:
353 		*encoded_postdiv = 2;
354 		break;
355         case 8:
356 		*encoded_postdiv = 3;
357 		break;
358         case 16:
359 		*encoded_postdiv = 4;
360 		break;
361         default:
362 		ret = -EINVAL;
363 		break;
364 	}
365 
366     return ret;
367 }
368 
369 u32 rv770_map_clkf_to_ibias(struct radeon_device *rdev, u32 clkf)
370 {
371 	if (clkf <= 0x10)
372 		return 0x4B;
373 	if (clkf <= 0x19)
374 		return 0x5B;
375 	if (clkf <= 0x21)
376 		return 0x2B;
377 	if (clkf <= 0x27)
378 		return 0x6C;
379 	if (clkf <= 0x31)
380 		return 0x9D;
381 	return 0xC6;
382 }
383 
384 static int rv770_populate_mclk_value(struct radeon_device *rdev,
385 				     u32 engine_clock, u32 memory_clock,
386 				     RV7XX_SMC_MCLK_VALUE *mclk)
387 {
388 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
389 	u8 encoded_reference_dividers[] = { 0, 16, 17, 20, 21 };
390 	u32 mpll_ad_func_cntl =
391 		pi->clk_regs.rv770.mpll_ad_func_cntl;
392 	u32 mpll_ad_func_cntl_2 =
393 		pi->clk_regs.rv770.mpll_ad_func_cntl_2;
394 	u32 mpll_dq_func_cntl =
395 		pi->clk_regs.rv770.mpll_dq_func_cntl;
396 	u32 mpll_dq_func_cntl_2 =
397 		pi->clk_regs.rv770.mpll_dq_func_cntl_2;
398 	u32 mclk_pwrmgt_cntl =
399 		pi->clk_regs.rv770.mclk_pwrmgt_cntl;
400 	u32 dll_cntl = pi->clk_regs.rv770.dll_cntl;
401 	struct atom_clock_dividers dividers;
402 	u32 reference_clock = rdev->clock.mpll.reference_freq;
403 	u32 clkf, clkfrac;
404 	u32 postdiv_yclk;
405 	u32 ibias;
406 	int ret;
407 
408 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
409 					     memory_clock, false, &dividers);
410 	if (ret)
411 		return ret;
412 
413 	if ((dividers.ref_div < 1) || (dividers.ref_div > 5))
414 		return -EINVAL;
415 
416 	rv770_calculate_fractional_mpll_feedback_divider(memory_clock, reference_clock,
417 							 pi->mem_gddr5,
418 							 &dividers, &clkf, &clkfrac);
419 
420 	ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk);
421 	if (ret)
422 		return ret;
423 
424 	ibias = rv770_map_clkf_to_ibias(rdev, clkf);
425 
426 	mpll_ad_func_cntl &= ~(CLKR_MASK |
427 			       YCLK_POST_DIV_MASK |
428 			       CLKF_MASK |
429 			       CLKFRAC_MASK |
430 			       IBIAS_MASK);
431 	mpll_ad_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]);
432 	mpll_ad_func_cntl |= YCLK_POST_DIV(postdiv_yclk);
433 	mpll_ad_func_cntl |= CLKF(clkf);
434 	mpll_ad_func_cntl |= CLKFRAC(clkfrac);
435 	mpll_ad_func_cntl |= IBIAS(ibias);
436 
437 	if (dividers.vco_mode)
438 		mpll_ad_func_cntl_2 |= VCO_MODE;
439 	else
440 		mpll_ad_func_cntl_2 &= ~VCO_MODE;
441 
442 	if (pi->mem_gddr5) {
443 		rv770_calculate_fractional_mpll_feedback_divider(memory_clock,
444 								 reference_clock,
445 								 pi->mem_gddr5,
446 								 &dividers, &clkf, &clkfrac);
447 
448 		ibias = rv770_map_clkf_to_ibias(rdev, clkf);
449 
450 		ret = rv770_encode_yclk_post_div(dividers.post_div, &postdiv_yclk);
451 		if (ret)
452 			return ret;
453 
454 		mpll_dq_func_cntl &= ~(CLKR_MASK |
455 				       YCLK_POST_DIV_MASK |
456 				       CLKF_MASK |
457 				       CLKFRAC_MASK |
458 				       IBIAS_MASK);
459 		mpll_dq_func_cntl |= CLKR(encoded_reference_dividers[dividers.ref_div - 1]);
460 		mpll_dq_func_cntl |= YCLK_POST_DIV(postdiv_yclk);
461 		mpll_dq_func_cntl |= CLKF(clkf);
462 		mpll_dq_func_cntl |= CLKFRAC(clkfrac);
463 		mpll_dq_func_cntl |= IBIAS(ibias);
464 
465 		if (dividers.vco_mode)
466 			mpll_dq_func_cntl_2 |= VCO_MODE;
467 		else
468 			mpll_dq_func_cntl_2 &= ~VCO_MODE;
469 	}
470 
471 	mclk->mclk770.mclk_value = cpu_to_be32(memory_clock);
472 	mclk->mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
473 	mclk->mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
474 	mclk->mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
475 	mclk->mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
476 	mclk->mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
477 	mclk->mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
478 
479 	return 0;
480 }
481 
482 static int rv770_populate_sclk_value(struct radeon_device *rdev,
483 				     u32 engine_clock,
484 				     RV770_SMC_SCLK_VALUE *sclk)
485 {
486 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
487 	struct atom_clock_dividers dividers;
488 	u32 spll_func_cntl =
489 		pi->clk_regs.rv770.cg_spll_func_cntl;
490 	u32 spll_func_cntl_2 =
491 		pi->clk_regs.rv770.cg_spll_func_cntl_2;
492 	u32 spll_func_cntl_3 =
493 		pi->clk_regs.rv770.cg_spll_func_cntl_3;
494 	u32 cg_spll_spread_spectrum =
495 		pi->clk_regs.rv770.cg_spll_spread_spectrum;
496 	u32 cg_spll_spread_spectrum_2 =
497 		pi->clk_regs.rv770.cg_spll_spread_spectrum_2;
498 	u64 tmp;
499 	u32 reference_clock = rdev->clock.spll.reference_freq;
500 	u32 reference_divider, post_divider;
501 	u32 fbdiv;
502 	int ret;
503 
504 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
505 					     engine_clock, false, &dividers);
506 	if (ret)
507 		return ret;
508 
509 	reference_divider = 1 + dividers.ref_div;
510 
511 	if (dividers.enable_post_div)
512 		post_divider = (0x0f & (dividers.post_div >> 4)) + (0x0f & dividers.post_div) + 2;
513 	else
514 		post_divider = 1;
515 
516 	tmp = (u64) engine_clock * reference_divider * post_divider * 16384;
517 	do_div(tmp, reference_clock);
518 	fbdiv = (u32) tmp;
519 
520 	if (dividers.enable_post_div)
521 		spll_func_cntl |= SPLL_DIVEN;
522 	else
523 		spll_func_cntl &= ~SPLL_DIVEN;
524 	spll_func_cntl &= ~(SPLL_HILEN_MASK | SPLL_LOLEN_MASK | SPLL_REF_DIV_MASK);
525 	spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
526 	spll_func_cntl |= SPLL_HILEN((dividers.post_div >> 4) & 0xf);
527 	spll_func_cntl |= SPLL_LOLEN(dividers.post_div & 0xf);
528 
529 	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
530 	spll_func_cntl_2 |= SCLK_MUX_SEL(2);
531 
532 	spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
533 	spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
534 	spll_func_cntl_3 |= SPLL_DITHEN;
535 
536 	if (pi->sclk_ss) {
537 		struct radeon_atom_ss ss;
538 		u32 vco_freq = engine_clock * post_divider;
539 
540 		if (radeon_atombios_get_asic_ss_info(rdev, &ss,
541 						     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
542 			u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
543 			u32 clk_v = ss.percentage * fbdiv / (clk_s * 10000);
544 
545 			cg_spll_spread_spectrum &= ~CLKS_MASK;
546 			cg_spll_spread_spectrum |= CLKS(clk_s);
547 			cg_spll_spread_spectrum |= SSEN;
548 
549 			cg_spll_spread_spectrum_2 &= ~CLKV_MASK;
550 			cg_spll_spread_spectrum_2 |= CLKV(clk_v);
551 		}
552 	}
553 
554 	sclk->sclk_value = cpu_to_be32(engine_clock);
555 	sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
556 	sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
557 	sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
558 	sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(cg_spll_spread_spectrum);
559 	sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(cg_spll_spread_spectrum_2);
560 
561 	return 0;
562 }
563 
564 int rv770_populate_vddc_value(struct radeon_device *rdev, u16 vddc,
565 			      RV770_SMC_VOLTAGE_VALUE *voltage)
566 {
567 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
568 	int i;
569 
570 	if (!pi->voltage_control) {
571 		voltage->index = 0;
572 		voltage->value = 0;
573 		return 0;
574 	}
575 
576 	for (i = 0; i < pi->valid_vddc_entries; i++) {
577 		if (vddc <= pi->vddc_table[i].vddc) {
578 			voltage->index = pi->vddc_table[i].vddc_index;
579 			voltage->value = cpu_to_be16(vddc);
580 			break;
581 		}
582 	}
583 
584 	if (i == pi->valid_vddc_entries)
585 		return -EINVAL;
586 
587 	return 0;
588 }
589 
590 int rv770_populate_mvdd_value(struct radeon_device *rdev, u32 mclk,
591 			      RV770_SMC_VOLTAGE_VALUE *voltage)
592 {
593 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
594 
595 	if (!pi->mvdd_control) {
596 		voltage->index = MVDD_HIGH_INDEX;
597 		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
598 		return 0;
599 	}
600 
601 	if (mclk <= pi->mvdd_split_frequency) {
602 		voltage->index = MVDD_LOW_INDEX;
603 		voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
604 	} else {
605 		voltage->index = MVDD_HIGH_INDEX;
606 		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
607 	}
608 
609 	return 0;
610 }
611 
612 static int rv770_convert_power_level_to_smc(struct radeon_device *rdev,
613 					    struct rv7xx_pl *pl,
614 					    RV770_SMC_HW_PERFORMANCE_LEVEL *level,
615 					    u8 watermark_level)
616 {
617 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
618 	int ret;
619 
620 	level->gen2PCIE = pi->pcie_gen2 ?
621 		((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
622 	level->gen2XSP  = (pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0;
623 	level->backbias = (pl->flags & ATOM_PPLIB_R600_FLAGS_BACKBIASENABLE) ? 1 : 0;
624 	level->displayWatermark = watermark_level;
625 
626 	if (rdev->family == CHIP_RV740)
627 		ret = rv740_populate_sclk_value(rdev, pl->sclk,
628 						&level->sclk);
629 	else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
630 		ret = rv730_populate_sclk_value(rdev, pl->sclk,
631 						&level->sclk);
632 	else
633 		ret = rv770_populate_sclk_value(rdev, pl->sclk,
634 						&level->sclk);
635 	if (ret)
636 		return ret;
637 
638 	if (rdev->family == CHIP_RV740) {
639 		if (pi->mem_gddr5) {
640 			if (pl->mclk <= pi->mclk_strobe_mode_threshold)
641 				level->strobeMode =
642 					rv740_get_mclk_frequency_ratio(pl->mclk) | 0x10;
643 			else
644 				level->strobeMode = 0;
645 
646 			if (pl->mclk > pi->mclk_edc_enable_threshold)
647 				level->mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
648 			else
649 				level->mcFlags =  0;
650 		}
651 		ret = rv740_populate_mclk_value(rdev, pl->sclk,
652 						pl->mclk, &level->mclk);
653 	} else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
654 		ret = rv730_populate_mclk_value(rdev, pl->sclk,
655 						pl->mclk, &level->mclk);
656 	else
657 		ret = rv770_populate_mclk_value(rdev, pl->sclk,
658 						pl->mclk, &level->mclk);
659 	if (ret)
660 		return ret;
661 
662 	ret = rv770_populate_vddc_value(rdev, pl->vddc,
663 					&level->vddc);
664 	if (ret)
665 		return ret;
666 
667 	ret = rv770_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
668 
669 	return ret;
670 }
671 
672 static int rv770_convert_power_state_to_smc(struct radeon_device *rdev,
673 					    struct radeon_ps *radeon_state,
674 					    RV770_SMC_SWSTATE *smc_state)
675 {
676 	struct rv7xx_ps *state = rv770_get_ps(radeon_state);
677 	int ret;
678 
679 	if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
680 		smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
681 
682 	ret = rv770_convert_power_level_to_smc(rdev,
683 					       &state->low,
684 					       &smc_state->levels[0],
685 					       PPSMC_DISPLAY_WATERMARK_LOW);
686 	if (ret)
687 		return ret;
688 
689 	ret = rv770_convert_power_level_to_smc(rdev,
690 					       &state->medium,
691 					       &smc_state->levels[1],
692 					       PPSMC_DISPLAY_WATERMARK_LOW);
693 	if (ret)
694 		return ret;
695 
696 	ret = rv770_convert_power_level_to_smc(rdev,
697 					       &state->high,
698 					       &smc_state->levels[2],
699 					       PPSMC_DISPLAY_WATERMARK_HIGH);
700 	if (ret)
701 		return ret;
702 
703 	smc_state->levels[0].arbValue = MC_CG_ARB_FREQ_F1;
704 	smc_state->levels[1].arbValue = MC_CG_ARB_FREQ_F2;
705 	smc_state->levels[2].arbValue = MC_CG_ARB_FREQ_F3;
706 
707 	smc_state->levels[0].seqValue = rv770_get_seq_value(rdev,
708 							    &state->low);
709 	smc_state->levels[1].seqValue = rv770_get_seq_value(rdev,
710 							    &state->medium);
711 	smc_state->levels[2].seqValue = rv770_get_seq_value(rdev,
712 							    &state->high);
713 
714 	rv770_populate_smc_sp(rdev, radeon_state, smc_state);
715 
716 	return rv770_populate_smc_t(rdev, radeon_state, smc_state);
717 
718 }
719 
720 u32 rv770_calculate_memory_refresh_rate(struct radeon_device *rdev,
721 					u32 engine_clock)
722 {
723 	u32 dram_rows;
724 	u32 dram_refresh_rate;
725 	u32 mc_arb_rfsh_rate;
726 	u32 tmp;
727 
728 	tmp = (RREG32(MC_ARB_RAMCFG) & NOOFROWS_MASK) >> NOOFROWS_SHIFT;
729 	dram_rows = 1 << (tmp + 10);
730 	tmp = RREG32(MC_SEQ_MISC0) & 3;
731 	dram_refresh_rate = 1 << (tmp + 3);
732 	mc_arb_rfsh_rate = ((engine_clock * 10) * dram_refresh_rate / dram_rows - 32) / 64;
733 
734 	return mc_arb_rfsh_rate;
735 }
736 
737 static void rv770_program_memory_timing_parameters(struct radeon_device *rdev,
738 						   struct radeon_ps *radeon_state)
739 {
740 	struct rv7xx_ps *state = rv770_get_ps(radeon_state);
741 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
742 	u32 sqm_ratio;
743 	u32 arb_refresh_rate;
744 	u32 high_clock;
745 
746 	if (state->high.sclk < (state->low.sclk * 0xFF / 0x40))
747 		high_clock = state->high.sclk;
748 	else
749 		high_clock = (state->low.sclk * 0xFF / 0x40);
750 
751 	radeon_atom_set_engine_dram_timings(rdev, high_clock,
752 					    state->high.mclk);
753 
754 	sqm_ratio =
755 		STATE0(64 * high_clock / pi->boot_sclk) |
756 		STATE1(64 * high_clock / state->low.sclk) |
757 		STATE2(64 * high_clock / state->medium.sclk) |
758 		STATE3(64 * high_clock / state->high.sclk);
759 	WREG32(MC_ARB_SQM_RATIO, sqm_ratio);
760 
761 	arb_refresh_rate =
762 		POWERMODE0(rv770_calculate_memory_refresh_rate(rdev, pi->boot_sclk)) |
763 		POWERMODE1(rv770_calculate_memory_refresh_rate(rdev, state->low.sclk)) |
764 		POWERMODE2(rv770_calculate_memory_refresh_rate(rdev, state->medium.sclk)) |
765 		POWERMODE3(rv770_calculate_memory_refresh_rate(rdev, state->high.sclk));
766 	WREG32(MC_ARB_RFSH_RATE, arb_refresh_rate);
767 }
768 
769 void rv770_enable_backbias(struct radeon_device *rdev,
770 			   bool enable)
771 {
772 	if (enable)
773 		WREG32_P(GENERAL_PWRMGT, BACKBIAS_PAD_EN, ~BACKBIAS_PAD_EN);
774 	else
775 		WREG32_P(GENERAL_PWRMGT, 0, ~(BACKBIAS_VALUE | BACKBIAS_PAD_EN));
776 }
777 
778 static void rv770_enable_spread_spectrum(struct radeon_device *rdev,
779 					 bool enable)
780 {
781 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
782 
783 	if (enable) {
784 		if (pi->sclk_ss)
785 			WREG32_P(GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, ~DYN_SPREAD_SPECTRUM_EN);
786 
787 		if (pi->mclk_ss) {
788 			if (rdev->family == CHIP_RV740)
789 				rv740_enable_mclk_spread_spectrum(rdev, true);
790 		}
791 	} else {
792 		WREG32_P(CG_SPLL_SPREAD_SPECTRUM, 0, ~SSEN);
793 
794 		WREG32_P(GENERAL_PWRMGT, 0, ~DYN_SPREAD_SPECTRUM_EN);
795 
796 		WREG32_P(CG_MPLL_SPREAD_SPECTRUM, 0, ~SSEN);
797 
798 		if (rdev->family == CHIP_RV740)
799 			rv740_enable_mclk_spread_spectrum(rdev, false);
800 	}
801 }
802 
803 static void rv770_program_mpll_timing_parameters(struct radeon_device *rdev)
804 {
805 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
806 
807 	if ((rdev->family == CHIP_RV770) && !pi->mem_gddr5) {
808 		WREG32(MPLL_TIME,
809 		       (MPLL_LOCK_TIME(R600_MPLLLOCKTIME_DFLT * pi->ref_div) |
810 			MPLL_RESET_TIME(R600_MPLLRESETTIME_DFLT)));
811 	}
812 }
813 
814 void rv770_setup_bsp(struct radeon_device *rdev)
815 {
816 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
817 	u32 xclk = radeon_get_xclk(rdev);
818 
819 	r600_calculate_u_and_p(pi->asi,
820 			       xclk,
821 			       16,
822 			       &pi->bsp,
823 			       &pi->bsu);
824 
825 	r600_calculate_u_and_p(pi->pasi,
826 			       xclk,
827 			       16,
828 			       &pi->pbsp,
829 			       &pi->pbsu);
830 
831 	pi->dsp = BSP(pi->bsp) | BSU(pi->bsu);
832 	pi->psp = BSP(pi->pbsp) | BSU(pi->pbsu);
833 
834 	WREG32(CG_BSP, pi->dsp);
835 
836 }
837 
838 void rv770_program_git(struct radeon_device *rdev)
839 {
840 	WREG32_P(CG_GIT, CG_GICST(R600_GICST_DFLT), ~CG_GICST_MASK);
841 }
842 
843 void rv770_program_tp(struct radeon_device *rdev)
844 {
845 	int i;
846 	enum r600_td td = R600_TD_DFLT;
847 
848 	for (i = 0; i < R600_PM_NUMBER_OF_TC; i++)
849 		WREG32(CG_FFCT_0 + (i * 4), (UTC_0(r600_utc[i]) | DTC_0(r600_dtc[i])));
850 
851 	if (td == R600_TD_AUTO)
852 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_FORCE_TREND_SEL);
853 	else
854 		WREG32_P(SCLK_PWRMGT_CNTL, FIR_FORCE_TREND_SEL, ~FIR_FORCE_TREND_SEL);
855 	if (td == R600_TD_UP)
856 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~FIR_TREND_MODE);
857 	if (td == R600_TD_DOWN)
858 		WREG32_P(SCLK_PWRMGT_CNTL, FIR_TREND_MODE, ~FIR_TREND_MODE);
859 }
860 
861 void rv770_program_tpp(struct radeon_device *rdev)
862 {
863 	WREG32(CG_TPC, R600_TPC_DFLT);
864 }
865 
866 void rv770_program_sstp(struct radeon_device *rdev)
867 {
868 	WREG32(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT)));
869 }
870 
871 void rv770_program_engine_speed_parameters(struct radeon_device *rdev)
872 {
873 	WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC);
874 }
875 
876 static void rv770_enable_display_gap(struct radeon_device *rdev)
877 {
878 	u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
879 
880 	tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
881 	tmp |= (DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE) |
882 		DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE));
883 	WREG32(CG_DISPLAY_GAP_CNTL, tmp);
884 }
885 
886 void rv770_program_vc(struct radeon_device *rdev)
887 {
888 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
889 
890 	WREG32(CG_FTV, pi->vrc);
891 }
892 
893 void rv770_clear_vc(struct radeon_device *rdev)
894 {
895 	WREG32(CG_FTV, 0);
896 }
897 
898 int rv770_upload_firmware(struct radeon_device *rdev)
899 {
900 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
901 	int ret;
902 
903 	rv770_reset_smc(rdev);
904 	rv770_stop_smc_clock(rdev);
905 
906 	ret = rv770_load_smc_ucode(rdev, pi->sram_end);
907 	if (ret)
908 		return ret;
909 
910 	return 0;
911 }
912 
913 static int rv770_populate_smc_acpi_state(struct radeon_device *rdev,
914 					 RV770_SMC_STATETABLE *table)
915 {
916 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
917 
918 	u32 mpll_ad_func_cntl =
919 		pi->clk_regs.rv770.mpll_ad_func_cntl;
920 	u32 mpll_ad_func_cntl_2 =
921 		pi->clk_regs.rv770.mpll_ad_func_cntl_2;
922 	u32 mpll_dq_func_cntl =
923 		pi->clk_regs.rv770.mpll_dq_func_cntl;
924 	u32 mpll_dq_func_cntl_2 =
925 		pi->clk_regs.rv770.mpll_dq_func_cntl_2;
926 	u32 spll_func_cntl =
927 		pi->clk_regs.rv770.cg_spll_func_cntl;
928 	u32 spll_func_cntl_2 =
929 		pi->clk_regs.rv770.cg_spll_func_cntl_2;
930 	u32 spll_func_cntl_3 =
931 		pi->clk_regs.rv770.cg_spll_func_cntl_3;
932 	u32 mclk_pwrmgt_cntl;
933 	u32 dll_cntl;
934 
935 	table->ACPIState = table->initialState;
936 
937 	table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
938 
939 	if (pi->acpi_vddc) {
940 		rv770_populate_vddc_value(rdev, pi->acpi_vddc,
941 					  &table->ACPIState.levels[0].vddc);
942 		if (pi->pcie_gen2) {
943 			if (pi->acpi_pcie_gen2)
944 				table->ACPIState.levels[0].gen2PCIE = 1;
945 			else
946 				table->ACPIState.levels[0].gen2PCIE = 0;
947 		} else
948 			table->ACPIState.levels[0].gen2PCIE = 0;
949 		if (pi->acpi_pcie_gen2)
950 			table->ACPIState.levels[0].gen2XSP = 1;
951 		else
952 			table->ACPIState.levels[0].gen2XSP = 0;
953 	} else {
954 		rv770_populate_vddc_value(rdev, pi->min_vddc_in_table,
955 					  &table->ACPIState.levels[0].vddc);
956 		table->ACPIState.levels[0].gen2PCIE = 0;
957 	}
958 
959 
960 	mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
961 
962 	mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
963 
964 	mclk_pwrmgt_cntl = (MRDCKA0_RESET |
965 			    MRDCKA1_RESET |
966 			    MRDCKB0_RESET |
967 			    MRDCKB1_RESET |
968 			    MRDCKC0_RESET |
969 			    MRDCKC1_RESET |
970 			    MRDCKD0_RESET |
971 			    MRDCKD1_RESET);
972 
973 	dll_cntl = 0xff000000;
974 
975 	spll_func_cntl |= SPLL_RESET | SPLL_SLEEP | SPLL_BYPASS_EN;
976 
977 	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
978 	spll_func_cntl_2 |= SCLK_MUX_SEL(4);
979 
980 	table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
981 	table->ACPIState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
982 	table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
983 	table->ACPIState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
984 
985 	table->ACPIState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
986 	table->ACPIState.levels[0].mclk.mclk770.vDLL_CNTL = cpu_to_be32(dll_cntl);
987 
988 	table->ACPIState.levels[0].mclk.mclk770.mclk_value = 0;
989 
990 	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
991 	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
992 	table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
993 
994 	table->ACPIState.levels[0].sclk.sclk_value = 0;
995 
996 	rv770_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
997 
998 	table->ACPIState.levels[1] = table->ACPIState.levels[0];
999 	table->ACPIState.levels[2] = table->ACPIState.levels[0];
1000 
1001 	return 0;
1002 }
1003 
1004 int rv770_populate_initial_mvdd_value(struct radeon_device *rdev,
1005 				      RV770_SMC_VOLTAGE_VALUE *voltage)
1006 {
1007 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1008 
1009 	if ((pi->s0_vid_lower_smio_cntl & pi->mvdd_mask_low) ==
1010 	     (pi->mvdd_low_smio[MVDD_LOW_INDEX] & pi->mvdd_mask_low) ) {
1011 		voltage->index = MVDD_LOW_INDEX;
1012 		voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1013 	} else {
1014 		voltage->index = MVDD_HIGH_INDEX;
1015 		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1016 	}
1017 
1018 	return 0;
1019 }
1020 
1021 static int rv770_populate_smc_initial_state(struct radeon_device *rdev,
1022 					    struct radeon_ps *radeon_state,
1023 					    RV770_SMC_STATETABLE *table)
1024 {
1025 	struct rv7xx_ps *initial_state = rv770_get_ps(radeon_state);
1026 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1027 	u32 a_t;
1028 
1029 	table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL =
1030 		cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl);
1031 	table->initialState.levels[0].mclk.mclk770.vMPLL_AD_FUNC_CNTL_2 =
1032 		cpu_to_be32(pi->clk_regs.rv770.mpll_ad_func_cntl_2);
1033 	table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL =
1034 		cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl);
1035 	table->initialState.levels[0].mclk.mclk770.vMPLL_DQ_FUNC_CNTL_2 =
1036 		cpu_to_be32(pi->clk_regs.rv770.mpll_dq_func_cntl_2);
1037 	table->initialState.levels[0].mclk.mclk770.vMCLK_PWRMGT_CNTL =
1038 		cpu_to_be32(pi->clk_regs.rv770.mclk_pwrmgt_cntl);
1039 	table->initialState.levels[0].mclk.mclk770.vDLL_CNTL =
1040 		cpu_to_be32(pi->clk_regs.rv770.dll_cntl);
1041 
1042 	table->initialState.levels[0].mclk.mclk770.vMPLL_SS =
1043 		cpu_to_be32(pi->clk_regs.rv770.mpll_ss1);
1044 	table->initialState.levels[0].mclk.mclk770.vMPLL_SS2 =
1045 		cpu_to_be32(pi->clk_regs.rv770.mpll_ss2);
1046 
1047 	table->initialState.levels[0].mclk.mclk770.mclk_value =
1048 		cpu_to_be32(initial_state->low.mclk);
1049 
1050 	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1051 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl);
1052 	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1053 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_2);
1054 	table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1055 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_func_cntl_3);
1056 	table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1057 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum);
1058 	table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1059 		cpu_to_be32(pi->clk_regs.rv770.cg_spll_spread_spectrum_2);
1060 
1061 	table->initialState.levels[0].sclk.sclk_value =
1062 		cpu_to_be32(initial_state->low.sclk);
1063 
1064 	table->initialState.levels[0].arbValue = MC_CG_ARB_FREQ_F0;
1065 
1066 	table->initialState.levels[0].seqValue =
1067 		rv770_get_seq_value(rdev, &initial_state->low);
1068 
1069 	rv770_populate_vddc_value(rdev,
1070 				  initial_state->low.vddc,
1071 				  &table->initialState.levels[0].vddc);
1072 	rv770_populate_initial_mvdd_value(rdev,
1073 					  &table->initialState.levels[0].mvdd);
1074 
1075 	a_t = CG_R(0xffff) | CG_L(0);
1076 	table->initialState.levels[0].aT = cpu_to_be32(a_t);
1077 
1078 	table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1079 
1080 	if (pi->boot_in_gen2)
1081 		table->initialState.levels[0].gen2PCIE = 1;
1082 	else
1083 		table->initialState.levels[0].gen2PCIE = 0;
1084 	if (initial_state->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
1085 		table->initialState.levels[0].gen2XSP = 1;
1086 	else
1087 		table->initialState.levels[0].gen2XSP = 0;
1088 
1089 	if (rdev->family == CHIP_RV740) {
1090 		if (pi->mem_gddr5) {
1091 			if (initial_state->low.mclk <= pi->mclk_strobe_mode_threshold)
1092 				table->initialState.levels[0].strobeMode =
1093 					rv740_get_mclk_frequency_ratio(initial_state->low.mclk) | 0x10;
1094 			else
1095 				table->initialState.levels[0].strobeMode = 0;
1096 
1097 			if (initial_state->low.mclk >= pi->mclk_edc_enable_threshold)
1098 				table->initialState.levels[0].mcFlags = SMC_MC_EDC_RD_FLAG | SMC_MC_EDC_WR_FLAG;
1099 			else
1100 				table->initialState.levels[0].mcFlags =  0;
1101 		}
1102 	}
1103 
1104 	table->initialState.levels[1] = table->initialState.levels[0];
1105 	table->initialState.levels[2] = table->initialState.levels[0];
1106 
1107 	table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1108 
1109 	return 0;
1110 }
1111 
1112 static int rv770_populate_smc_vddc_table(struct radeon_device *rdev,
1113 					 RV770_SMC_STATETABLE *table)
1114 {
1115 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1116 	int i;
1117 
1118 	for (i = 0; i < pi->valid_vddc_entries; i++) {
1119 		table->highSMIO[pi->vddc_table[i].vddc_index] =
1120 			pi->vddc_table[i].high_smio;
1121 		table->lowSMIO[pi->vddc_table[i].vddc_index] =
1122 			cpu_to_be32(pi->vddc_table[i].low_smio);
1123 	}
1124 
1125 	table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_VDDC] = 0;
1126 	table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_VDDC] =
1127 		cpu_to_be32(pi->vddc_mask_low);
1128 
1129 	for (i = 0;
1130 	     ((i < pi->valid_vddc_entries) &&
1131 	      (pi->max_vddc_in_table >
1132 	       pi->vddc_table[i].vddc));
1133 	     i++);
1134 
1135 	table->maxVDDCIndexInPPTable =
1136 		pi->vddc_table[i].vddc_index;
1137 
1138 	return 0;
1139 }
1140 
1141 static int rv770_populate_smc_mvdd_table(struct radeon_device *rdev,
1142 					 RV770_SMC_STATETABLE *table)
1143 {
1144 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1145 
1146 	if (pi->mvdd_control) {
1147 		table->lowSMIO[MVDD_HIGH_INDEX] |=
1148 			cpu_to_be32(pi->mvdd_low_smio[MVDD_HIGH_INDEX]);
1149 		table->lowSMIO[MVDD_LOW_INDEX] |=
1150 			cpu_to_be32(pi->mvdd_low_smio[MVDD_LOW_INDEX]);
1151 
1152 		table->voltageMaskTable.highMask[RV770_SMC_VOLTAGEMASK_MVDD] = 0;
1153 		table->voltageMaskTable.lowMask[RV770_SMC_VOLTAGEMASK_MVDD] =
1154 			cpu_to_be32(pi->mvdd_mask_low);
1155 	}
1156 
1157 	return 0;
1158 }
1159 
1160 static int rv770_init_smc_table(struct radeon_device *rdev,
1161 				struct radeon_ps *radeon_boot_state)
1162 {
1163 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1164 	struct rv7xx_ps *boot_state = rv770_get_ps(radeon_boot_state);
1165 	RV770_SMC_STATETABLE *table = &pi->smc_statetable;
1166 	int ret;
1167 
1168 	memset(table, 0, sizeof(RV770_SMC_STATETABLE));
1169 
1170 	pi->boot_sclk = boot_state->low.sclk;
1171 
1172 	rv770_populate_smc_vddc_table(rdev, table);
1173 	rv770_populate_smc_mvdd_table(rdev, table);
1174 
1175 	switch (rdev->pm.int_thermal_type) {
1176         case THERMAL_TYPE_RV770:
1177         case THERMAL_TYPE_ADT7473_WITH_INTERNAL:
1178 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1179 		break;
1180         case THERMAL_TYPE_NONE:
1181 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1182 		break;
1183         case THERMAL_TYPE_EXTERNAL_GPIO:
1184         default:
1185 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1186 		break;
1187 	}
1188 
1189 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC) {
1190 		table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1191 
1192 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_DONT_WAIT_FOR_VBLANK_ON_ALERT)
1193 			table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_DONT_WAIT_FOR_VBLANK;
1194 
1195 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_GOTO_BOOT_ON_ALERT)
1196 			table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_ACTION_GOTOINITIALSTATE;
1197 	}
1198 
1199 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1200 		table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1201 
1202 	if (pi->mem_gddr5)
1203 		table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1204 
1205 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1206 		ret = rv730_populate_smc_initial_state(rdev, radeon_boot_state, table);
1207 	else
1208 		ret = rv770_populate_smc_initial_state(rdev, radeon_boot_state, table);
1209 	if (ret)
1210 		return ret;
1211 
1212 	if (rdev->family == CHIP_RV740)
1213 		ret = rv740_populate_smc_acpi_state(rdev, table);
1214 	else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1215 		ret = rv730_populate_smc_acpi_state(rdev, table);
1216 	else
1217 		ret = rv770_populate_smc_acpi_state(rdev, table);
1218 	if (ret)
1219 		return ret;
1220 
1221 	table->driverState = table->initialState;
1222 
1223 	return rv770_copy_bytes_to_smc(rdev,
1224 				       pi->state_table_start,
1225 				       (const u8 *)table,
1226 				       sizeof(RV770_SMC_STATETABLE),
1227 				       pi->sram_end);
1228 }
1229 
1230 static int rv770_construct_vddc_table(struct radeon_device *rdev)
1231 {
1232 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1233 	u16 min, max, step;
1234 	u32 steps = 0;
1235 	u8 vddc_index = 0;
1236 	u32 i;
1237 
1238 	radeon_atom_get_min_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &min);
1239 	radeon_atom_get_max_voltage(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &max);
1240 	radeon_atom_get_voltage_step(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, &step);
1241 
1242 	steps = (max - min) / step + 1;
1243 
1244 	if (steps > MAX_NO_VREG_STEPS)
1245 		return -EINVAL;
1246 
1247 	for (i = 0; i < steps; i++) {
1248 		u32 gpio_pins, gpio_mask;
1249 
1250 		pi->vddc_table[i].vddc = (u16)(min + i * step);
1251 		radeon_atom_get_voltage_gpio_settings(rdev,
1252 						      pi->vddc_table[i].vddc,
1253 						      SET_VOLTAGE_TYPE_ASIC_VDDC,
1254 						      &gpio_pins, &gpio_mask);
1255 		pi->vddc_table[i].low_smio = gpio_pins & gpio_mask;
1256 		pi->vddc_table[i].high_smio = 0;
1257 		pi->vddc_mask_low = gpio_mask;
1258 		if (i > 0) {
1259 			if ((pi->vddc_table[i].low_smio !=
1260 			     pi->vddc_table[i - 1].low_smio ) ||
1261 			     (pi->vddc_table[i].high_smio !=
1262 			      pi->vddc_table[i - 1].high_smio))
1263 				vddc_index++;
1264 		}
1265 		pi->vddc_table[i].vddc_index = vddc_index;
1266 	}
1267 
1268 	pi->valid_vddc_entries = (u8)steps;
1269 
1270 	return 0;
1271 }
1272 
1273 static u32 rv770_get_mclk_split_point(struct atom_memory_info *memory_info)
1274 {
1275 	if (memory_info->mem_type == MEM_TYPE_GDDR3)
1276 		return 30000;
1277 
1278 	return 0;
1279 }
1280 
1281 static int rv770_get_mvdd_pin_configuration(struct radeon_device *rdev)
1282 {
1283 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1284 	u32 gpio_pins, gpio_mask;
1285 
1286 	radeon_atom_get_voltage_gpio_settings(rdev,
1287 					      MVDD_HIGH_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC,
1288 					      &gpio_pins, &gpio_mask);
1289 	pi->mvdd_mask_low = gpio_mask;
1290 	pi->mvdd_low_smio[MVDD_HIGH_INDEX] =
1291 		gpio_pins & gpio_mask;
1292 
1293 	radeon_atom_get_voltage_gpio_settings(rdev,
1294 					      MVDD_LOW_VALUE, SET_VOLTAGE_TYPE_ASIC_MVDDC,
1295 					      &gpio_pins, &gpio_mask);
1296 	pi->mvdd_low_smio[MVDD_LOW_INDEX] =
1297 		gpio_pins & gpio_mask;
1298 
1299 	return 0;
1300 }
1301 
1302 u8 rv770_get_memory_module_index(struct radeon_device *rdev)
1303 {
1304 	return (u8) ((RREG32(BIOS_SCRATCH_4) >> 16) & 0xff);
1305 }
1306 
1307 static int rv770_get_mvdd_configuration(struct radeon_device *rdev)
1308 {
1309 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1310 	u8 memory_module_index;
1311 	struct atom_memory_info memory_info;
1312 
1313 	memory_module_index = rv770_get_memory_module_index(rdev);
1314 
1315 	if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info)) {
1316 		pi->mvdd_control = false;
1317 		return 0;
1318 	}
1319 
1320 	pi->mvdd_split_frequency =
1321 		rv770_get_mclk_split_point(&memory_info);
1322 
1323 	if (pi->mvdd_split_frequency == 0) {
1324 		pi->mvdd_control = false;
1325 		return 0;
1326 	}
1327 
1328 	return rv770_get_mvdd_pin_configuration(rdev);
1329 }
1330 
1331 void rv770_enable_voltage_control(struct radeon_device *rdev,
1332 				  bool enable)
1333 {
1334 	if (enable)
1335 		WREG32_P(GENERAL_PWRMGT, VOLT_PWRMGT_EN, ~VOLT_PWRMGT_EN);
1336 	else
1337 		WREG32_P(GENERAL_PWRMGT, 0, ~VOLT_PWRMGT_EN);
1338 }
1339 
1340 static void rv770_program_display_gap(struct radeon_device *rdev)
1341 {
1342 	u32 tmp = RREG32(CG_DISPLAY_GAP_CNTL);
1343 
1344 	tmp &= ~(DISP1_GAP_MCHG_MASK | DISP2_GAP_MCHG_MASK);
1345 	if (rdev->pm.dpm.new_active_crtcs & 1) {
1346 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
1347 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1348 	} else if (rdev->pm.dpm.new_active_crtcs & 2) {
1349 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1350 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK);
1351 	} else {
1352 		tmp |= DISP1_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1353 		tmp |= DISP2_GAP_MCHG(R600_PM_DISPLAY_GAP_IGNORE);
1354 	}
1355 	WREG32(CG_DISPLAY_GAP_CNTL, tmp);
1356 }
1357 
1358 static void rv770_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
1359 					   bool enable)
1360 {
1361 	rv770_enable_bif_dynamic_pcie_gen2(rdev, enable);
1362 
1363 	if (enable)
1364 		WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
1365 	else
1366 		WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
1367 }
1368 
1369 static void r7xx_program_memory_timing_parameters(struct radeon_device *rdev,
1370 						  struct radeon_ps *radeon_new_state)
1371 {
1372 	if ((rdev->family == CHIP_RV730) ||
1373 	    (rdev->family == CHIP_RV710) ||
1374 	    (rdev->family == CHIP_RV740))
1375 		rv730_program_memory_timing_parameters(rdev, radeon_new_state);
1376 	else
1377 		rv770_program_memory_timing_parameters(rdev, radeon_new_state);
1378 }
1379 
1380 static int rv770_upload_sw_state(struct radeon_device *rdev,
1381 				 struct radeon_ps *radeon_new_state)
1382 {
1383 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1384 	u16 address = pi->state_table_start +
1385 		offsetof(RV770_SMC_STATETABLE, driverState);
1386 	RV770_SMC_SWSTATE state = { 0 };
1387 	int ret;
1388 
1389 	ret = rv770_convert_power_state_to_smc(rdev, radeon_new_state, &state);
1390 	if (ret)
1391 		return ret;
1392 
1393 	return rv770_copy_bytes_to_smc(rdev, address, (const u8 *)&state,
1394 				       sizeof(RV770_SMC_SWSTATE),
1395 				       pi->sram_end);
1396 }
1397 
1398 int rv770_halt_smc(struct radeon_device *rdev)
1399 {
1400 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Halt) != PPSMC_Result_OK)
1401 		return -EINVAL;
1402 
1403 	if (rv770_wait_for_smc_inactive(rdev) != PPSMC_Result_OK)
1404 		return -EINVAL;
1405 
1406 	return 0;
1407 }
1408 
1409 int rv770_resume_smc(struct radeon_device *rdev)
1410 {
1411 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_Resume) != PPSMC_Result_OK)
1412 		return -EINVAL;
1413 	return 0;
1414 }
1415 
1416 int rv770_set_sw_state(struct radeon_device *rdev)
1417 {
1418 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToSwState) != PPSMC_Result_OK)
1419 		return -EINVAL;
1420 	return 0;
1421 }
1422 
1423 int rv770_set_boot_state(struct radeon_device *rdev)
1424 {
1425 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_SwitchToInitialState) != PPSMC_Result_OK)
1426 		return -EINVAL;
1427 	return 0;
1428 }
1429 
1430 void rv770_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
1431 					      struct radeon_ps *new_ps,
1432 					      struct radeon_ps *old_ps)
1433 {
1434 	struct rv7xx_ps *new_state = rv770_get_ps(new_ps);
1435 	struct rv7xx_ps *current_state = rv770_get_ps(old_ps);
1436 
1437 	if ((new_ps->vclk == old_ps->vclk) &&
1438 	    (new_ps->dclk == old_ps->dclk))
1439 		return;
1440 
1441 	if (new_state->high.sclk >= current_state->high.sclk)
1442 		return;
1443 
1444 	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
1445 }
1446 
1447 void rv770_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
1448 					     struct radeon_ps *new_ps,
1449 					     struct radeon_ps *old_ps)
1450 {
1451 	struct rv7xx_ps *new_state = rv770_get_ps(new_ps);
1452 	struct rv7xx_ps *current_state = rv770_get_ps(old_ps);
1453 
1454 	if ((new_ps->vclk == old_ps->vclk) &&
1455 	    (new_ps->dclk == old_ps->dclk))
1456 		return;
1457 
1458 	if (new_state->high.sclk < current_state->high.sclk)
1459 		return;
1460 
1461 	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
1462 }
1463 
1464 int rv770_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1465 {
1466 	if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_NoForcedLevel)) != PPSMC_Result_OK)
1467 		return -EINVAL;
1468 
1469 	if (rv770_send_msg_to_smc(rdev, (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled)) != PPSMC_Result_OK)
1470 		return -EINVAL;
1471 
1472 	return 0;
1473 }
1474 
1475 int rv770_dpm_force_performance_level(struct radeon_device *rdev,
1476 				      enum radeon_dpm_forced_level level)
1477 {
1478 	PPSMC_Msg msg;
1479 
1480 	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1481 		if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_ZeroLevelsDisabled) != PPSMC_Result_OK)
1482 			return -EINVAL;
1483 		msg = PPSMC_MSG_ForceHigh;
1484 	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1485 		if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1486 			return -EINVAL;
1487 		msg = (PPSMC_Msg)(PPSMC_MSG_TwoLevelsDisabled);
1488 	} else {
1489 		if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1490 			return -EINVAL;
1491 		msg = (PPSMC_Msg)(PPSMC_MSG_ZeroLevelsDisabled);
1492 	}
1493 
1494 	if (rv770_send_msg_to_smc(rdev, msg) != PPSMC_Result_OK)
1495 		return -EINVAL;
1496 
1497 	rdev->pm.dpm.forced_level = level;
1498 
1499 	return 0;
1500 }
1501 
1502 void r7xx_start_smc(struct radeon_device *rdev)
1503 {
1504 	rv770_start_smc(rdev);
1505 	rv770_start_smc_clock(rdev);
1506 }
1507 
1508 
1509 void r7xx_stop_smc(struct radeon_device *rdev)
1510 {
1511 	rv770_reset_smc(rdev);
1512 	rv770_stop_smc_clock(rdev);
1513 }
1514 
1515 static void rv770_read_clock_registers(struct radeon_device *rdev)
1516 {
1517 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1518 
1519 	pi->clk_regs.rv770.cg_spll_func_cntl =
1520 		RREG32(CG_SPLL_FUNC_CNTL);
1521 	pi->clk_regs.rv770.cg_spll_func_cntl_2 =
1522 		RREG32(CG_SPLL_FUNC_CNTL_2);
1523 	pi->clk_regs.rv770.cg_spll_func_cntl_3 =
1524 		RREG32(CG_SPLL_FUNC_CNTL_3);
1525 	pi->clk_regs.rv770.cg_spll_spread_spectrum =
1526 		RREG32(CG_SPLL_SPREAD_SPECTRUM);
1527 	pi->clk_regs.rv770.cg_spll_spread_spectrum_2 =
1528 		RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1529 	pi->clk_regs.rv770.mpll_ad_func_cntl =
1530 		RREG32(MPLL_AD_FUNC_CNTL);
1531 	pi->clk_regs.rv770.mpll_ad_func_cntl_2 =
1532 		RREG32(MPLL_AD_FUNC_CNTL_2);
1533 	pi->clk_regs.rv770.mpll_dq_func_cntl =
1534 		RREG32(MPLL_DQ_FUNC_CNTL);
1535 	pi->clk_regs.rv770.mpll_dq_func_cntl_2 =
1536 		RREG32(MPLL_DQ_FUNC_CNTL_2);
1537 	pi->clk_regs.rv770.mclk_pwrmgt_cntl =
1538 		RREG32(MCLK_PWRMGT_CNTL);
1539 	pi->clk_regs.rv770.dll_cntl = RREG32(DLL_CNTL);
1540 }
1541 
1542 static void r7xx_read_clock_registers(struct radeon_device *rdev)
1543 {
1544 	if (rdev->family == CHIP_RV740)
1545 		rv740_read_clock_registers(rdev);
1546 	else if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1547 		rv730_read_clock_registers(rdev);
1548 	else
1549 		rv770_read_clock_registers(rdev);
1550 }
1551 
1552 void rv770_read_voltage_smio_registers(struct radeon_device *rdev)
1553 {
1554 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1555 
1556 	pi->s0_vid_lower_smio_cntl =
1557 		RREG32(S0_VID_LOWER_SMIO_CNTL);
1558 }
1559 
1560 void rv770_reset_smio_status(struct radeon_device *rdev)
1561 {
1562 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1563 	u32 sw_smio_index, vid_smio_cntl;
1564 
1565 	sw_smio_index =
1566 		(RREG32(GENERAL_PWRMGT) & SW_SMIO_INDEX_MASK) >> SW_SMIO_INDEX_SHIFT;
1567 	switch (sw_smio_index) {
1568         case 3:
1569 		vid_smio_cntl = RREG32(S3_VID_LOWER_SMIO_CNTL);
1570 		break;
1571         case 2:
1572 		vid_smio_cntl = RREG32(S2_VID_LOWER_SMIO_CNTL);
1573 		break;
1574         case 1:
1575 		vid_smio_cntl = RREG32(S1_VID_LOWER_SMIO_CNTL);
1576 		break;
1577         case 0:
1578 		return;
1579         default:
1580 		vid_smio_cntl = pi->s0_vid_lower_smio_cntl;
1581 		break;
1582 	}
1583 
1584 	WREG32(S0_VID_LOWER_SMIO_CNTL, vid_smio_cntl);
1585 	WREG32_P(GENERAL_PWRMGT, SW_SMIO_INDEX(0), ~SW_SMIO_INDEX_MASK);
1586 }
1587 
1588 void rv770_get_memory_type(struct radeon_device *rdev)
1589 {
1590 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1591 	u32 tmp;
1592 
1593 	tmp = RREG32(MC_SEQ_MISC0);
1594 
1595 	if (((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT) ==
1596 	    MC_SEQ_MISC0_GDDR5_VALUE)
1597 		pi->mem_gddr5 = true;
1598 	else
1599 		pi->mem_gddr5 = false;
1600 
1601 }
1602 
1603 void rv770_get_pcie_gen2_status(struct radeon_device *rdev)
1604 {
1605 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1606 	u32 tmp;
1607 
1608 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
1609 
1610 	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
1611 	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
1612 		pi->pcie_gen2 = true;
1613 	else
1614 		pi->pcie_gen2 = false;
1615 
1616 	if (pi->pcie_gen2) {
1617 		if (tmp & LC_CURRENT_DATA_RATE)
1618 			pi->boot_in_gen2 = true;
1619 		else
1620 			pi->boot_in_gen2 = false;
1621 	} else
1622 		pi->boot_in_gen2 = false;
1623 }
1624 
1625 #if 0
1626 static int rv770_enter_ulp_state(struct radeon_device *rdev)
1627 {
1628 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1629 
1630 	if (pi->gfx_clock_gating) {
1631 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1632 		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1633 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1634 		RREG32(GB_TILING_CONFIG);
1635 	}
1636 
1637 	WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1638 		 ~HOST_SMC_MSG_MASK);
1639 
1640 	udelay(7000);
1641 
1642 	return 0;
1643 }
1644 
1645 static int rv770_exit_ulp_state(struct radeon_device *rdev)
1646 {
1647 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1648 	int i;
1649 
1650 	WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_ResumeFromMinimumPower),
1651 		 ~HOST_SMC_MSG_MASK);
1652 
1653 	udelay(7000);
1654 
1655 	for (i = 0; i < rdev->usec_timeout; i++) {
1656 		if (((RREG32(SMC_MSG) & HOST_SMC_RESP_MASK) >> HOST_SMC_RESP_SHIFT) == 1)
1657 			break;
1658 		udelay(1000);
1659 	}
1660 
1661 	if (pi->gfx_clock_gating)
1662 		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
1663 
1664 	return 0;
1665 }
1666 #endif
1667 
1668 static void rv770_get_mclk_odt_threshold(struct radeon_device *rdev)
1669 {
1670 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1671 	u8 memory_module_index;
1672 	struct atom_memory_info memory_info;
1673 
1674 	pi->mclk_odt_threshold = 0;
1675 
1676 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710)) {
1677 		memory_module_index = rv770_get_memory_module_index(rdev);
1678 
1679 		if (radeon_atom_get_memory_info(rdev, memory_module_index, &memory_info))
1680 			return;
1681 
1682 		if (memory_info.mem_type == MEM_TYPE_DDR2 ||
1683 		    memory_info.mem_type == MEM_TYPE_DDR3)
1684 			pi->mclk_odt_threshold = 30000;
1685 	}
1686 }
1687 
1688 void rv770_get_max_vddc(struct radeon_device *rdev)
1689 {
1690 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1691 	u16 vddc;
1692 
1693 	if (radeon_atom_get_max_vddc(rdev, 0, 0, &vddc))
1694 		pi->max_vddc = 0;
1695 	else
1696 		pi->max_vddc = vddc;
1697 }
1698 
1699 void rv770_program_response_times(struct radeon_device *rdev)
1700 {
1701 	u32 voltage_response_time, backbias_response_time;
1702 	u32 acpi_delay_time, vbi_time_out;
1703 	u32 vddc_dly, bb_dly, acpi_dly, vbi_dly;
1704 	u32 reference_clock;
1705 
1706 	voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1707 	backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1708 
1709 	if (voltage_response_time == 0)
1710 		voltage_response_time = 1000;
1711 
1712 	if (backbias_response_time == 0)
1713 		backbias_response_time = 1000;
1714 
1715 	acpi_delay_time = 15000;
1716 	vbi_time_out = 100000;
1717 
1718 	reference_clock = radeon_get_xclk(rdev);
1719 
1720 	vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1721 	bb_dly = (backbias_response_time * reference_clock) / 1600;
1722 	acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1723 	vbi_dly = (vbi_time_out * reference_clock) / 1600;
1724 
1725 	rv770_write_smc_soft_register(rdev,
1726 				      RV770_SMC_SOFT_REGISTER_delay_vreg, vddc_dly);
1727 	rv770_write_smc_soft_register(rdev,
1728 				      RV770_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1729 	rv770_write_smc_soft_register(rdev,
1730 				      RV770_SMC_SOFT_REGISTER_delay_acpi, acpi_dly);
1731 	rv770_write_smc_soft_register(rdev,
1732 				      RV770_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1733 #if 0
1734 	/* XXX look up hw revision */
1735 	if (WEKIVA_A21)
1736 		rv770_write_smc_soft_register(rdev,
1737 					      RV770_SMC_SOFT_REGISTER_baby_step_timer,
1738 					      0x10);
1739 #endif
1740 }
1741 
1742 static void rv770_program_dcodt_before_state_switch(struct radeon_device *rdev,
1743 						    struct radeon_ps *radeon_new_state,
1744 						    struct radeon_ps *radeon_current_state)
1745 {
1746 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1747 	struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
1748 	struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state);
1749 	bool current_use_dc = false;
1750 	bool new_use_dc = false;
1751 
1752 	if (pi->mclk_odt_threshold == 0)
1753 		return;
1754 
1755 	if (current_state->high.mclk <= pi->mclk_odt_threshold)
1756 		current_use_dc = true;
1757 
1758 	if (new_state->high.mclk <= pi->mclk_odt_threshold)
1759 		new_use_dc = true;
1760 
1761 	if (current_use_dc == new_use_dc)
1762 		return;
1763 
1764 	if (!current_use_dc && new_use_dc)
1765 		return;
1766 
1767 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1768 		rv730_program_dcodt(rdev, new_use_dc);
1769 }
1770 
1771 static void rv770_program_dcodt_after_state_switch(struct radeon_device *rdev,
1772 						   struct radeon_ps *radeon_new_state,
1773 						   struct radeon_ps *radeon_current_state)
1774 {
1775 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1776 	struct rv7xx_ps *new_state = rv770_get_ps(radeon_new_state);
1777 	struct rv7xx_ps *current_state = rv770_get_ps(radeon_current_state);
1778 	bool current_use_dc = false;
1779 	bool new_use_dc = false;
1780 
1781 	if (pi->mclk_odt_threshold == 0)
1782 		return;
1783 
1784 	if (current_state->high.mclk <= pi->mclk_odt_threshold)
1785 		current_use_dc = true;
1786 
1787 	if (new_state->high.mclk <= pi->mclk_odt_threshold)
1788 		new_use_dc = true;
1789 
1790 	if (current_use_dc == new_use_dc)
1791 		return;
1792 
1793 	if (current_use_dc && !new_use_dc)
1794 		return;
1795 
1796 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1797 		rv730_program_dcodt(rdev, new_use_dc);
1798 }
1799 
1800 static void rv770_retrieve_odt_values(struct radeon_device *rdev)
1801 {
1802 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1803 
1804 	if (pi->mclk_odt_threshold == 0)
1805 		return;
1806 
1807 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1808 		rv730_get_odt_values(rdev);
1809 }
1810 
1811 static void rv770_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
1812 {
1813 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1814 	bool want_thermal_protection;
1815 	enum radeon_dpm_event_src dpm_event_src;
1816 
1817 	switch (sources) {
1818         case 0:
1819         default:
1820 		want_thermal_protection = false;
1821 		break;
1822         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
1823 		want_thermal_protection = true;
1824 		dpm_event_src = RADEON_DPM_EVENT_SRC_DIGITAL;
1825 		break;
1826 
1827         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
1828 		want_thermal_protection = true;
1829 		dpm_event_src = RADEON_DPM_EVENT_SRC_EXTERNAL;
1830 		break;
1831 
1832         case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
1833 	      (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
1834 		want_thermal_protection = true;
1835 		dpm_event_src = RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL;
1836 		break;
1837 	}
1838 
1839 	if (want_thermal_protection) {
1840 		WREG32_P(CG_THERMAL_CTRL, DPM_EVENT_SRC(dpm_event_src), ~DPM_EVENT_SRC_MASK);
1841 		if (pi->thermal_protection)
1842 			WREG32_P(GENERAL_PWRMGT, 0, ~THERMAL_PROTECTION_DIS);
1843 	} else {
1844 		WREG32_P(GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, ~THERMAL_PROTECTION_DIS);
1845 	}
1846 }
1847 
1848 void rv770_enable_auto_throttle_source(struct radeon_device *rdev,
1849 				       enum radeon_dpm_auto_throttle_src source,
1850 				       bool enable)
1851 {
1852 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1853 
1854 	if (enable) {
1855 		if (!(pi->active_auto_throttle_sources & (1 << source))) {
1856 			pi->active_auto_throttle_sources |= 1 << source;
1857 			rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1858 		}
1859 	} else {
1860 		if (pi->active_auto_throttle_sources & (1 << source)) {
1861 			pi->active_auto_throttle_sources &= ~(1 << source);
1862 			rv770_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1863 		}
1864 	}
1865 }
1866 
1867 static int rv770_set_thermal_temperature_range(struct radeon_device *rdev,
1868 					       int min_temp, int max_temp)
1869 {
1870 	int low_temp = 0 * 1000;
1871 	int high_temp = 255 * 1000;
1872 
1873 	if (low_temp < min_temp)
1874 		low_temp = min_temp;
1875 	if (high_temp > max_temp)
1876 		high_temp = max_temp;
1877 	if (high_temp < low_temp) {
1878 		DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1879 		return -EINVAL;
1880 	}
1881 
1882 	WREG32_P(CG_THERMAL_INT, DIG_THERM_INTH(high_temp / 1000), ~DIG_THERM_INTH_MASK);
1883 	WREG32_P(CG_THERMAL_INT, DIG_THERM_INTL(low_temp / 1000), ~DIG_THERM_INTL_MASK);
1884 	WREG32_P(CG_THERMAL_CTRL, DIG_THERM_DPM(high_temp / 1000), ~DIG_THERM_DPM_MASK);
1885 
1886 	rdev->pm.dpm.thermal.min_temp = low_temp;
1887 	rdev->pm.dpm.thermal.max_temp = high_temp;
1888 
1889 	return 0;
1890 }
1891 
1892 int rv770_dpm_enable(struct radeon_device *rdev)
1893 {
1894 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1895 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
1896 	int ret;
1897 
1898 	if (pi->gfx_clock_gating)
1899 		rv770_restore_cgcg(rdev);
1900 
1901 	if (rv770_dpm_enabled(rdev))
1902 		return -EINVAL;
1903 
1904 	if (pi->voltage_control) {
1905 		rv770_enable_voltage_control(rdev, true);
1906 		ret = rv770_construct_vddc_table(rdev);
1907 		if (ret) {
1908 			DRM_ERROR("rv770_construct_vddc_table failed\n");
1909 			return ret;
1910 		}
1911 	}
1912 
1913 	if (pi->dcodt)
1914 		rv770_retrieve_odt_values(rdev);
1915 
1916 	if (pi->mvdd_control) {
1917 		ret = rv770_get_mvdd_configuration(rdev);
1918 		if (ret) {
1919 			DRM_ERROR("rv770_get_mvdd_configuration failed\n");
1920 			return ret;
1921 		}
1922 	}
1923 
1924 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_BACKBIAS)
1925 		rv770_enable_backbias(rdev, true);
1926 
1927 	rv770_enable_spread_spectrum(rdev, true);
1928 
1929 	if (pi->thermal_protection)
1930 		rv770_enable_thermal_protection(rdev, true);
1931 
1932 	rv770_program_mpll_timing_parameters(rdev);
1933 	rv770_setup_bsp(rdev);
1934 	rv770_program_git(rdev);
1935 	rv770_program_tp(rdev);
1936 	rv770_program_tpp(rdev);
1937 	rv770_program_sstp(rdev);
1938 	rv770_program_engine_speed_parameters(rdev);
1939 	rv770_enable_display_gap(rdev);
1940 	rv770_program_vc(rdev);
1941 
1942 	if (pi->dynamic_pcie_gen2)
1943 		rv770_enable_dynamic_pcie_gen2(rdev, true);
1944 
1945 	ret = rv770_upload_firmware(rdev);
1946 	if (ret) {
1947 		DRM_ERROR("rv770_upload_firmware failed\n");
1948 		return ret;
1949 	}
1950 	ret = rv770_init_smc_table(rdev, boot_ps);
1951 	if (ret) {
1952 		DRM_ERROR("rv770_init_smc_table failed\n");
1953 		return ret;
1954 	}
1955 
1956 	rv770_program_response_times(rdev);
1957 	r7xx_start_smc(rdev);
1958 
1959 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
1960 		rv730_start_dpm(rdev);
1961 	else
1962 		rv770_start_dpm(rdev);
1963 
1964 	if (pi->gfx_clock_gating)
1965 		rv770_gfx_clock_gating_enable(rdev, true);
1966 
1967 	if (pi->mg_clock_gating)
1968 		rv770_mg_clock_gating_enable(rdev, true);
1969 
1970 	rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
1971 
1972 	return 0;
1973 }
1974 
1975 int rv770_dpm_late_enable(struct radeon_device *rdev)
1976 {
1977 	int ret;
1978 
1979 	if (rdev->irq.installed &&
1980 	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1981 		PPSMC_Result result;
1982 
1983 		ret = rv770_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1984 		if (ret)
1985 			return ret;
1986 		rdev->irq.dpm_thermal = true;
1987 		radeon_irq_set(rdev);
1988 		result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableThermalInterrupt);
1989 
1990 		if (result != PPSMC_Result_OK)
1991 			DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
1992 	}
1993 
1994 	return 0;
1995 }
1996 
1997 void rv770_dpm_disable(struct radeon_device *rdev)
1998 {
1999 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2000 
2001 	if (!rv770_dpm_enabled(rdev))
2002 		return;
2003 
2004 	rv770_clear_vc(rdev);
2005 
2006 	if (pi->thermal_protection)
2007 		rv770_enable_thermal_protection(rdev, false);
2008 
2009 	rv770_enable_spread_spectrum(rdev, false);
2010 
2011 	if (pi->dynamic_pcie_gen2)
2012 		rv770_enable_dynamic_pcie_gen2(rdev, false);
2013 
2014 	if (rdev->irq.installed &&
2015 	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
2016 		rdev->irq.dpm_thermal = false;
2017 		radeon_irq_set(rdev);
2018 	}
2019 
2020 	if (pi->gfx_clock_gating)
2021 		rv770_gfx_clock_gating_enable(rdev, false);
2022 
2023 	if (pi->mg_clock_gating)
2024 		rv770_mg_clock_gating_enable(rdev, false);
2025 
2026 	if ((rdev->family == CHIP_RV730) || (rdev->family == CHIP_RV710))
2027 		rv730_stop_dpm(rdev);
2028 	else
2029 		rv770_stop_dpm(rdev);
2030 
2031 	r7xx_stop_smc(rdev);
2032 	rv770_reset_smio_status(rdev);
2033 }
2034 
2035 int rv770_dpm_set_power_state(struct radeon_device *rdev)
2036 {
2037 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2038 	struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
2039 	struct radeon_ps *old_ps = rdev->pm.dpm.current_ps;
2040 	int ret;
2041 
2042 	ret = rv770_restrict_performance_levels_before_switch(rdev);
2043 	if (ret) {
2044 		DRM_ERROR("rv770_restrict_performance_levels_before_switch failed\n");
2045 		return ret;
2046 	}
2047 	rv770_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
2048 	ret = rv770_halt_smc(rdev);
2049 	if (ret) {
2050 		DRM_ERROR("rv770_halt_smc failed\n");
2051 		return ret;
2052 	}
2053 	ret = rv770_upload_sw_state(rdev, new_ps);
2054 	if (ret) {
2055 		DRM_ERROR("rv770_upload_sw_state failed\n");
2056 		return ret;
2057 	}
2058 	r7xx_program_memory_timing_parameters(rdev, new_ps);
2059 	if (pi->dcodt)
2060 		rv770_program_dcodt_before_state_switch(rdev, new_ps, old_ps);
2061 	ret = rv770_resume_smc(rdev);
2062 	if (ret) {
2063 		DRM_ERROR("rv770_resume_smc failed\n");
2064 		return ret;
2065 	}
2066 	ret = rv770_set_sw_state(rdev);
2067 	if (ret) {
2068 		DRM_ERROR("rv770_set_sw_state failed\n");
2069 		return ret;
2070 	}
2071 	if (pi->dcodt)
2072 		rv770_program_dcodt_after_state_switch(rdev, new_ps, old_ps);
2073 	rv770_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
2074 
2075 	return 0;
2076 }
2077 
2078 void rv770_dpm_reset_asic(struct radeon_device *rdev)
2079 {
2080 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2081 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
2082 
2083 	rv770_restrict_performance_levels_before_switch(rdev);
2084 	if (pi->dcodt)
2085 		rv770_program_dcodt_before_state_switch(rdev, boot_ps, boot_ps);
2086 	rv770_set_boot_state(rdev);
2087 	if (pi->dcodt)
2088 		rv770_program_dcodt_after_state_switch(rdev, boot_ps, boot_ps);
2089 }
2090 
2091 void rv770_dpm_setup_asic(struct radeon_device *rdev)
2092 {
2093 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2094 
2095 	r7xx_read_clock_registers(rdev);
2096 	rv770_read_voltage_smio_registers(rdev);
2097 	rv770_get_memory_type(rdev);
2098 	if (pi->dcodt)
2099 		rv770_get_mclk_odt_threshold(rdev);
2100 	rv770_get_pcie_gen2_status(rdev);
2101 
2102 	rv770_enable_acpi_pm(rdev);
2103 
2104 	if (radeon_aspm != 0) {
2105 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L0s)
2106 			rv770_enable_l0s(rdev);
2107 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_ASPM_L1)
2108 			rv770_enable_l1(rdev);
2109 		if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_TURNOFFPLL_ASPML1)
2110 			rv770_enable_pll_sleep_in_l1(rdev);
2111 	}
2112 }
2113 
2114 void rv770_dpm_display_configuration_changed(struct radeon_device *rdev)
2115 {
2116 	rv770_program_display_gap(rdev);
2117 }
2118 
2119 union power_info {
2120 	struct _ATOM_POWERPLAY_INFO info;
2121 	struct _ATOM_POWERPLAY_INFO_V2 info_2;
2122 	struct _ATOM_POWERPLAY_INFO_V3 info_3;
2123 	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
2124 	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
2125 	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
2126 };
2127 
2128 union pplib_clock_info {
2129 	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
2130 	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
2131 	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
2132 	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
2133 };
2134 
2135 union pplib_power_state {
2136 	struct _ATOM_PPLIB_STATE v1;
2137 	struct _ATOM_PPLIB_STATE_V2 v2;
2138 };
2139 
2140 static void rv7xx_parse_pplib_non_clock_info(struct radeon_device *rdev,
2141 					     struct radeon_ps *rps,
2142 					     struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
2143 					     u8 table_rev)
2144 {
2145 	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
2146 	rps->class = le16_to_cpu(non_clock_info->usClassification);
2147 	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
2148 
2149 	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
2150 		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
2151 		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
2152 	} else {
2153 		rps->vclk = 0;
2154 		rps->dclk = 0;
2155 	}
2156 
2157 	if (r600_is_uvd_state(rps->class, rps->class2)) {
2158 		if ((rps->vclk == 0) || (rps->dclk == 0)) {
2159 			rps->vclk = RV770_DEFAULT_VCLK_FREQ;
2160 			rps->dclk = RV770_DEFAULT_DCLK_FREQ;
2161 		}
2162 	}
2163 
2164 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
2165 		rdev->pm.dpm.boot_ps = rps;
2166 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
2167 		rdev->pm.dpm.uvd_ps = rps;
2168 }
2169 
2170 static void rv7xx_parse_pplib_clock_info(struct radeon_device *rdev,
2171 					 struct radeon_ps *rps, int index,
2172 					 union pplib_clock_info *clock_info)
2173 {
2174 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2175 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2176 	struct rv7xx_ps *ps = rv770_get_ps(rps);
2177 	u32 sclk, mclk;
2178 	struct rv7xx_pl *pl;
2179 
2180 	switch (index) {
2181 	case 0:
2182 		pl = &ps->low;
2183 		break;
2184 	case 1:
2185 		pl = &ps->medium;
2186 		break;
2187 	case 2:
2188 	default:
2189 		pl = &ps->high;
2190 		break;
2191 	}
2192 
2193 	if (rdev->family >= CHIP_CEDAR) {
2194 		sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
2195 		sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
2196 		mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
2197 		mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
2198 
2199 		pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
2200 		pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
2201 		pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
2202 	} else {
2203 		sclk = le16_to_cpu(clock_info->r600.usEngineClockLow);
2204 		sclk |= clock_info->r600.ucEngineClockHigh << 16;
2205 		mclk = le16_to_cpu(clock_info->r600.usMemoryClockLow);
2206 		mclk |= clock_info->r600.ucMemoryClockHigh << 16;
2207 
2208 		pl->vddc = le16_to_cpu(clock_info->r600.usVDDC);
2209 		pl->flags = le32_to_cpu(clock_info->r600.ulFlags);
2210 	}
2211 
2212 	pl->mclk = mclk;
2213 	pl->sclk = sclk;
2214 
2215 	/* patch up vddc if necessary */
2216 	if (pl->vddc == 0xff01) {
2217 		if (pi->max_vddc)
2218 			pl->vddc = pi->max_vddc;
2219 	}
2220 
2221 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
2222 		pi->acpi_vddc = pl->vddc;
2223 		if (rdev->family >= CHIP_CEDAR)
2224 			eg_pi->acpi_vddci = pl->vddci;
2225 		if (ps->low.flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
2226 			pi->acpi_pcie_gen2 = true;
2227 		else
2228 			pi->acpi_pcie_gen2 = false;
2229 	}
2230 
2231 	if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
2232 		if (rdev->family >= CHIP_BARTS) {
2233 			eg_pi->ulv.supported = true;
2234 			eg_pi->ulv.pl = pl;
2235 		}
2236 	}
2237 
2238 	if (pi->min_vddc_in_table > pl->vddc)
2239 		pi->min_vddc_in_table = pl->vddc;
2240 
2241 	if (pi->max_vddc_in_table < pl->vddc)
2242 		pi->max_vddc_in_table = pl->vddc;
2243 
2244 	/* patch up boot state */
2245 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
2246 		u16 vddc, vddci, mvdd;
2247 		radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
2248 		pl->mclk = rdev->clock.default_mclk;
2249 		pl->sclk = rdev->clock.default_sclk;
2250 		pl->vddc = vddc;
2251 		pl->vddci = vddci;
2252 	}
2253 
2254 	if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
2255 	    ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
2256 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
2257 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
2258 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
2259 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
2260 	}
2261 }
2262 
2263 int rv7xx_parse_power_table(struct radeon_device *rdev)
2264 {
2265 	struct radeon_mode_info *mode_info = &rdev->mode_info;
2266 	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
2267 	union pplib_power_state *power_state;
2268 	int i, j;
2269 	union pplib_clock_info *clock_info;
2270 	union power_info *power_info;
2271 	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
2272         u16 data_offset;
2273 	u8 frev, crev;
2274 	struct rv7xx_ps *ps;
2275 
2276 	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
2277 				   &frev, &crev, &data_offset))
2278 		return -EINVAL;
2279 	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
2280 
2281 	rdev->pm.dpm.ps = kzalloc(sizeof(struct radeon_ps) *
2282 				  power_info->pplib.ucNumStates, GFP_KERNEL);
2283 	if (!rdev->pm.dpm.ps)
2284 		return -ENOMEM;
2285 
2286 	for (i = 0; i < power_info->pplib.ucNumStates; i++) {
2287 		power_state = (union pplib_power_state *)
2288 			(mode_info->atom_context->bios + data_offset +
2289 			 le16_to_cpu(power_info->pplib.usStateArrayOffset) +
2290 			 i * power_info->pplib.ucStateEntrySize);
2291 		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
2292 			(mode_info->atom_context->bios + data_offset +
2293 			 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
2294 			 (power_state->v1.ucNonClockStateIndex *
2295 			  power_info->pplib.ucNonClockSize));
2296 		if (power_info->pplib.ucStateEntrySize - 1) {
2297 			u8 *idx;
2298 			ps = kzalloc(sizeof(struct rv7xx_ps), GFP_KERNEL);
2299 			if (ps == NULL) {
2300 				kfree(rdev->pm.dpm.ps);
2301 				return -ENOMEM;
2302 			}
2303 			rdev->pm.dpm.ps[i].ps_priv = ps;
2304 			rv7xx_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
2305 							 non_clock_info,
2306 							 power_info->pplib.ucNonClockSize);
2307 			idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
2308 			for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
2309 				clock_info = (union pplib_clock_info *)
2310 					(mode_info->atom_context->bios + data_offset +
2311 					 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
2312 					 (idx[j] * power_info->pplib.ucClockInfoSize));
2313 				rv7xx_parse_pplib_clock_info(rdev,
2314 							     &rdev->pm.dpm.ps[i], j,
2315 							     clock_info);
2316 			}
2317 		}
2318 	}
2319 	rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
2320 	return 0;
2321 }
2322 
2323 void rv770_get_engine_memory_ss(struct radeon_device *rdev)
2324 {
2325 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2326 	struct radeon_atom_ss ss;
2327 
2328 	pi->sclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
2329 						       ASIC_INTERNAL_ENGINE_SS, 0);
2330 	pi->mclk_ss = radeon_atombios_get_asic_ss_info(rdev, &ss,
2331 						       ASIC_INTERNAL_MEMORY_SS, 0);
2332 
2333 	if (pi->sclk_ss || pi->mclk_ss)
2334 		pi->dynamic_ss = true;
2335 	else
2336 		pi->dynamic_ss = false;
2337 }
2338 
2339 int rv770_dpm_init(struct radeon_device *rdev)
2340 {
2341 	struct rv7xx_power_info *pi;
2342 	struct atom_clock_dividers dividers;
2343 	int ret;
2344 
2345 	pi = kzalloc(sizeof(struct rv7xx_power_info), GFP_KERNEL);
2346 	if (pi == NULL)
2347 		return -ENOMEM;
2348 	rdev->pm.dpm.priv = pi;
2349 
2350 	rv770_get_max_vddc(rdev);
2351 
2352 	pi->acpi_vddc = 0;
2353 	pi->min_vddc_in_table = 0;
2354 	pi->max_vddc_in_table = 0;
2355 
2356 	ret = r600_get_platform_caps(rdev);
2357 	if (ret)
2358 		return ret;
2359 
2360 	ret = rv7xx_parse_power_table(rdev);
2361 	if (ret)
2362 		return ret;
2363 
2364 	if (rdev->pm.dpm.voltage_response_time == 0)
2365 		rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
2366 	if (rdev->pm.dpm.backbias_response_time == 0)
2367 		rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
2368 
2369 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2370 					     0, false, &dividers);
2371 	if (ret)
2372 		pi->ref_div = dividers.ref_div + 1;
2373 	else
2374 		pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
2375 
2376 	pi->mclk_strobe_mode_threshold = 30000;
2377 	pi->mclk_edc_enable_threshold = 30000;
2378 
2379 	pi->rlp = RV770_RLP_DFLT;
2380 	pi->rmp = RV770_RMP_DFLT;
2381 	pi->lhp = RV770_LHP_DFLT;
2382 	pi->lmp = RV770_LMP_DFLT;
2383 
2384 	pi->voltage_control =
2385 		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
2386 
2387 	pi->mvdd_control =
2388 		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
2389 
2390 	rv770_get_engine_memory_ss(rdev);
2391 
2392 	pi->asi = RV770_ASI_DFLT;
2393 	pi->pasi = RV770_HASI_DFLT;
2394 	pi->vrc = RV770_VRC_DFLT;
2395 
2396 	pi->power_gating = false;
2397 
2398 	pi->gfx_clock_gating = true;
2399 
2400 	pi->mg_clock_gating = true;
2401 	pi->mgcgtssm = true;
2402 
2403 	pi->dynamic_pcie_gen2 = true;
2404 
2405 	if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
2406 		pi->thermal_protection = true;
2407 	else
2408 		pi->thermal_protection = false;
2409 
2410 	pi->display_gap = true;
2411 
2412 	if (rdev->flags & RADEON_IS_MOBILITY)
2413 		pi->dcodt = true;
2414 	else
2415 		pi->dcodt = false;
2416 
2417 	pi->ulps = true;
2418 
2419 	pi->mclk_stutter_mode_threshold = 0;
2420 
2421 	pi->sram_end = SMC_RAM_END;
2422 	pi->state_table_start = RV770_SMC_TABLE_ADDRESS;
2423 	pi->soft_regs_start = RV770_SMC_SOFT_REGISTERS_START;
2424 
2425 	return 0;
2426 }
2427 
2428 void rv770_dpm_print_power_state(struct radeon_device *rdev,
2429 				 struct radeon_ps *rps)
2430 {
2431 	struct rv7xx_ps *ps = rv770_get_ps(rps);
2432 	struct rv7xx_pl *pl;
2433 
2434 	r600_dpm_print_class_info(rps->class, rps->class2);
2435 	r600_dpm_print_cap_info(rps->caps);
2436 	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2437 	if (rdev->family >= CHIP_CEDAR) {
2438 		pl = &ps->low;
2439 		printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2440 		       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2441 		pl = &ps->medium;
2442 		printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2443 		       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2444 		pl = &ps->high;
2445 		printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2446 		       pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2447 	} else {
2448 		pl = &ps->low;
2449 		printk("\t\tpower level 0    sclk: %u mclk: %u vddc: %u\n",
2450 		       pl->sclk, pl->mclk, pl->vddc);
2451 		pl = &ps->medium;
2452 		printk("\t\tpower level 1    sclk: %u mclk: %u vddc: %u\n",
2453 		       pl->sclk, pl->mclk, pl->vddc);
2454 		pl = &ps->high;
2455 		printk("\t\tpower level 2    sclk: %u mclk: %u vddc: %u\n",
2456 		       pl->sclk, pl->mclk, pl->vddc);
2457 	}
2458 	r600_dpm_print_ps_status(rdev, rps);
2459 }
2460 
2461 void rv770_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
2462 						       struct seq_file *m)
2463 {
2464 	struct radeon_ps *rps = rdev->pm.dpm.current_ps;
2465 	struct rv7xx_ps *ps = rv770_get_ps(rps);
2466 	struct rv7xx_pl *pl;
2467 	u32 current_index =
2468 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_PROFILE_INDEX_MASK) >>
2469 		CURRENT_PROFILE_INDEX_SHIFT;
2470 
2471 	if (current_index > 2) {
2472 		seq_printf(m, "invalid dpm profile %d\n", current_index);
2473 	} else {
2474 		if (current_index == 0)
2475 			pl = &ps->low;
2476 		else if (current_index == 1)
2477 			pl = &ps->medium;
2478 		else /* current_index == 2 */
2479 			pl = &ps->high;
2480 		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2481 		if (rdev->family >= CHIP_CEDAR) {
2482 			seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
2483 				   current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
2484 		} else {
2485 			seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u\n",
2486 				   current_index, pl->sclk, pl->mclk, pl->vddc);
2487 		}
2488 	}
2489 }
2490 
2491 void rv770_dpm_fini(struct radeon_device *rdev)
2492 {
2493 	int i;
2494 
2495 	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2496 		kfree(rdev->pm.dpm.ps[i].ps_priv);
2497 	}
2498 	kfree(rdev->pm.dpm.ps);
2499 	kfree(rdev->pm.dpm.priv);
2500 }
2501 
2502 u32 rv770_dpm_get_sclk(struct radeon_device *rdev, bool low)
2503 {
2504 	struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps);
2505 
2506 	if (low)
2507 		return requested_state->low.sclk;
2508 	else
2509 		return requested_state->high.sclk;
2510 }
2511 
2512 u32 rv770_dpm_get_mclk(struct radeon_device *rdev, bool low)
2513 {
2514 	struct rv7xx_ps *requested_state = rv770_get_ps(rdev->pm.dpm.requested_ps);
2515 
2516 	if (low)
2517 		return requested_state->low.mclk;
2518 	else
2519 		return requested_state->high.mclk;
2520 }
2521 
2522 bool rv770_dpm_vblank_too_short(struct radeon_device *rdev)
2523 {
2524 	u32 vblank_time = r600_dpm_get_vblank_time(rdev);
2525 	u32 switch_limit = 200; /* 300 */
2526 
2527 	/* RV770 */
2528 	/* mclk switching doesn't seem to work reliably on desktop RV770s */
2529 	if ((rdev->family == CHIP_RV770) &&
2530 	    !(rdev->flags & RADEON_IS_MOBILITY))
2531 		switch_limit = 0xffffffff; /* disable mclk switching */
2532 
2533 	if (vblank_time < switch_limit)
2534 		return true;
2535 	else
2536 		return false;
2537 
2538 }
2539