1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/pci.h>
25 #include <linux/seq_file.h>
26 
27 #include "r600_dpm.h"
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "trinity_dpm.h"
31 #include "trinityd.h"
32 #include "vce.h"
33 
34 #define TRINITY_MAX_DEEPSLEEP_DIVIDER_ID 5
35 #define TRINITY_MINIMUM_ENGINE_CLOCK 800
36 #define SCLK_MIN_DIV_INTV_SHIFT     12
37 #define TRINITY_DISPCLK_BYPASS_THRESHOLD 10000
38 
39 #ifndef TRINITY_MGCG_SEQUENCE
40 #define TRINITY_MGCG_SEQUENCE  100
41 
42 static const u32 trinity_mgcg_shls_default[] =
43 {
44 	/* Register, Value, Mask */
45 	0x0000802c, 0xc0000000, 0xffffffff,
46 	0x00003fc4, 0xc0000000, 0xffffffff,
47 	0x00005448, 0x00000100, 0xffffffff,
48 	0x000055e4, 0x00000100, 0xffffffff,
49 	0x0000160c, 0x00000100, 0xffffffff,
50 	0x00008984, 0x06000100, 0xffffffff,
51 	0x0000c164, 0x00000100, 0xffffffff,
52 	0x00008a18, 0x00000100, 0xffffffff,
53 	0x0000897c, 0x06000100, 0xffffffff,
54 	0x00008b28, 0x00000100, 0xffffffff,
55 	0x00009144, 0x00800200, 0xffffffff,
56 	0x00009a60, 0x00000100, 0xffffffff,
57 	0x00009868, 0x00000100, 0xffffffff,
58 	0x00008d58, 0x00000100, 0xffffffff,
59 	0x00009510, 0x00000100, 0xffffffff,
60 	0x0000949c, 0x00000100, 0xffffffff,
61 	0x00009654, 0x00000100, 0xffffffff,
62 	0x00009030, 0x00000100, 0xffffffff,
63 	0x00009034, 0x00000100, 0xffffffff,
64 	0x00009038, 0x00000100, 0xffffffff,
65 	0x0000903c, 0x00000100, 0xffffffff,
66 	0x00009040, 0x00000100, 0xffffffff,
67 	0x0000a200, 0x00000100, 0xffffffff,
68 	0x0000a204, 0x00000100, 0xffffffff,
69 	0x0000a208, 0x00000100, 0xffffffff,
70 	0x0000a20c, 0x00000100, 0xffffffff,
71 	0x00009744, 0x00000100, 0xffffffff,
72 	0x00003f80, 0x00000100, 0xffffffff,
73 	0x0000a210, 0x00000100, 0xffffffff,
74 	0x0000a214, 0x00000100, 0xffffffff,
75 	0x000004d8, 0x00000100, 0xffffffff,
76 	0x00009664, 0x00000100, 0xffffffff,
77 	0x00009698, 0x00000100, 0xffffffff,
78 	0x000004d4, 0x00000200, 0xffffffff,
79 	0x000004d0, 0x00000000, 0xffffffff,
80 	0x000030cc, 0x00000104, 0xffffffff,
81 	0x0000d0c0, 0x00000100, 0xffffffff,
82 	0x0000d8c0, 0x00000100, 0xffffffff,
83 	0x0000951c, 0x00010000, 0xffffffff,
84 	0x00009160, 0x00030002, 0xffffffff,
85 	0x00009164, 0x00050004, 0xffffffff,
86 	0x00009168, 0x00070006, 0xffffffff,
87 	0x00009178, 0x00070000, 0xffffffff,
88 	0x0000917c, 0x00030002, 0xffffffff,
89 	0x00009180, 0x00050004, 0xffffffff,
90 	0x0000918c, 0x00010006, 0xffffffff,
91 	0x00009190, 0x00090008, 0xffffffff,
92 	0x00009194, 0x00070000, 0xffffffff,
93 	0x00009198, 0x00030002, 0xffffffff,
94 	0x0000919c, 0x00050004, 0xffffffff,
95 	0x000091a8, 0x00010006, 0xffffffff,
96 	0x000091ac, 0x00090008, 0xffffffff,
97 	0x000091b0, 0x00070000, 0xffffffff,
98 	0x000091b4, 0x00030002, 0xffffffff,
99 	0x000091b8, 0x00050004, 0xffffffff,
100 	0x000091c4, 0x00010006, 0xffffffff,
101 	0x000091c8, 0x00090008, 0xffffffff,
102 	0x000091cc, 0x00070000, 0xffffffff,
103 	0x000091d0, 0x00030002, 0xffffffff,
104 	0x000091d4, 0x00050004, 0xffffffff,
105 	0x000091e0, 0x00010006, 0xffffffff,
106 	0x000091e4, 0x00090008, 0xffffffff,
107 	0x000091e8, 0x00000000, 0xffffffff,
108 	0x000091ec, 0x00070000, 0xffffffff,
109 	0x000091f0, 0x00030002, 0xffffffff,
110 	0x000091f4, 0x00050004, 0xffffffff,
111 	0x00009200, 0x00010006, 0xffffffff,
112 	0x00009204, 0x00090008, 0xffffffff,
113 	0x00009208, 0x00070000, 0xffffffff,
114 	0x0000920c, 0x00030002, 0xffffffff,
115 	0x00009210, 0x00050004, 0xffffffff,
116 	0x0000921c, 0x00010006, 0xffffffff,
117 	0x00009220, 0x00090008, 0xffffffff,
118 	0x00009294, 0x00000000, 0xffffffff
119 };
120 #endif
121 
122 #ifndef TRINITY_SYSLS_SEQUENCE
123 #define TRINITY_SYSLS_SEQUENCE  100
124 
125 static const u32 trinity_sysls_disable[] =
126 {
127 	/* Register, Value, Mask */
128 	0x0000d0c0, 0x00000000, 0xffffffff,
129 	0x0000d8c0, 0x00000000, 0xffffffff,
130 	0x000055e8, 0x00000000, 0xffffffff,
131 	0x0000d0bc, 0x00000000, 0xffffffff,
132 	0x0000d8bc, 0x00000000, 0xffffffff,
133 	0x000015c0, 0x00041401, 0xffffffff,
134 	0x0000264c, 0x00040400, 0xffffffff,
135 	0x00002648, 0x00040400, 0xffffffff,
136 	0x00002650, 0x00040400, 0xffffffff,
137 	0x000020b8, 0x00040400, 0xffffffff,
138 	0x000020bc, 0x00040400, 0xffffffff,
139 	0x000020c0, 0x00040c80, 0xffffffff,
140 	0x0000f4a0, 0x000000c0, 0xffffffff,
141 	0x0000f4a4, 0x00680000, 0xffffffff,
142 	0x00002f50, 0x00000404, 0xffffffff,
143 	0x000004c8, 0x00000001, 0xffffffff,
144 	0x0000641c, 0x00007ffd, 0xffffffff,
145 	0x00000c7c, 0x0000ff00, 0xffffffff,
146 	0x00006dfc, 0x0000007f, 0xffffffff
147 };
148 
149 static const u32 trinity_sysls_enable[] =
150 {
151 	/* Register, Value, Mask */
152 	0x000055e8, 0x00000001, 0xffffffff,
153 	0x0000d0bc, 0x00000100, 0xffffffff,
154 	0x0000d8bc, 0x00000100, 0xffffffff,
155 	0x000015c0, 0x000c1401, 0xffffffff,
156 	0x0000264c, 0x000c0400, 0xffffffff,
157 	0x00002648, 0x000c0400, 0xffffffff,
158 	0x00002650, 0x000c0400, 0xffffffff,
159 	0x000020b8, 0x000c0400, 0xffffffff,
160 	0x000020bc, 0x000c0400, 0xffffffff,
161 	0x000020c0, 0x000c0c80, 0xffffffff,
162 	0x0000f4a0, 0x000000c0, 0xffffffff,
163 	0x0000f4a4, 0x00680fff, 0xffffffff,
164 	0x00002f50, 0x00000903, 0xffffffff,
165 	0x000004c8, 0x00000000, 0xffffffff,
166 	0x0000641c, 0x00000000, 0xffffffff,
167 	0x00000c7c, 0x00000000, 0xffffffff,
168 	0x00006dfc, 0x00000000, 0xffffffff
169 };
170 #endif
171 
172 static const u32 trinity_override_mgpg_sequences[] =
173 {
174 	/* Register, Value */
175 	0x00000200, 0xE030032C,
176 	0x00000204, 0x00000FFF,
177 	0x00000200, 0xE0300058,
178 	0x00000204, 0x00030301,
179 	0x00000200, 0xE0300054,
180 	0x00000204, 0x500010FF,
181 	0x00000200, 0xE0300074,
182 	0x00000204, 0x00030301,
183 	0x00000200, 0xE0300070,
184 	0x00000204, 0x500010FF,
185 	0x00000200, 0xE0300090,
186 	0x00000204, 0x00030301,
187 	0x00000200, 0xE030008C,
188 	0x00000204, 0x500010FF,
189 	0x00000200, 0xE03000AC,
190 	0x00000204, 0x00030301,
191 	0x00000200, 0xE03000A8,
192 	0x00000204, 0x500010FF,
193 	0x00000200, 0xE03000C8,
194 	0x00000204, 0x00030301,
195 	0x00000200, 0xE03000C4,
196 	0x00000204, 0x500010FF,
197 	0x00000200, 0xE03000E4,
198 	0x00000204, 0x00030301,
199 	0x00000200, 0xE03000E0,
200 	0x00000204, 0x500010FF,
201 	0x00000200, 0xE0300100,
202 	0x00000204, 0x00030301,
203 	0x00000200, 0xE03000FC,
204 	0x00000204, 0x500010FF,
205 	0x00000200, 0xE0300058,
206 	0x00000204, 0x00030303,
207 	0x00000200, 0xE0300054,
208 	0x00000204, 0x600010FF,
209 	0x00000200, 0xE0300074,
210 	0x00000204, 0x00030303,
211 	0x00000200, 0xE0300070,
212 	0x00000204, 0x600010FF,
213 	0x00000200, 0xE0300090,
214 	0x00000204, 0x00030303,
215 	0x00000200, 0xE030008C,
216 	0x00000204, 0x600010FF,
217 	0x00000200, 0xE03000AC,
218 	0x00000204, 0x00030303,
219 	0x00000200, 0xE03000A8,
220 	0x00000204, 0x600010FF,
221 	0x00000200, 0xE03000C8,
222 	0x00000204, 0x00030303,
223 	0x00000200, 0xE03000C4,
224 	0x00000204, 0x600010FF,
225 	0x00000200, 0xE03000E4,
226 	0x00000204, 0x00030303,
227 	0x00000200, 0xE03000E0,
228 	0x00000204, 0x600010FF,
229 	0x00000200, 0xE0300100,
230 	0x00000204, 0x00030303,
231 	0x00000200, 0xE03000FC,
232 	0x00000204, 0x600010FF,
233 	0x00000200, 0xE0300058,
234 	0x00000204, 0x00030303,
235 	0x00000200, 0xE0300054,
236 	0x00000204, 0x700010FF,
237 	0x00000200, 0xE0300074,
238 	0x00000204, 0x00030303,
239 	0x00000200, 0xE0300070,
240 	0x00000204, 0x700010FF,
241 	0x00000200, 0xE0300090,
242 	0x00000204, 0x00030303,
243 	0x00000200, 0xE030008C,
244 	0x00000204, 0x700010FF,
245 	0x00000200, 0xE03000AC,
246 	0x00000204, 0x00030303,
247 	0x00000200, 0xE03000A8,
248 	0x00000204, 0x700010FF,
249 	0x00000200, 0xE03000C8,
250 	0x00000204, 0x00030303,
251 	0x00000200, 0xE03000C4,
252 	0x00000204, 0x700010FF,
253 	0x00000200, 0xE03000E4,
254 	0x00000204, 0x00030303,
255 	0x00000200, 0xE03000E0,
256 	0x00000204, 0x700010FF,
257 	0x00000200, 0xE0300100,
258 	0x00000204, 0x00030303,
259 	0x00000200, 0xE03000FC,
260 	0x00000204, 0x700010FF,
261 	0x00000200, 0xE0300058,
262 	0x00000204, 0x00010303,
263 	0x00000200, 0xE0300054,
264 	0x00000204, 0x800010FF,
265 	0x00000200, 0xE0300074,
266 	0x00000204, 0x00010303,
267 	0x00000200, 0xE0300070,
268 	0x00000204, 0x800010FF,
269 	0x00000200, 0xE0300090,
270 	0x00000204, 0x00010303,
271 	0x00000200, 0xE030008C,
272 	0x00000204, 0x800010FF,
273 	0x00000200, 0xE03000AC,
274 	0x00000204, 0x00010303,
275 	0x00000200, 0xE03000A8,
276 	0x00000204, 0x800010FF,
277 	0x00000200, 0xE03000C4,
278 	0x00000204, 0x800010FF,
279 	0x00000200, 0xE03000C8,
280 	0x00000204, 0x00010303,
281 	0x00000200, 0xE03000E4,
282 	0x00000204, 0x00010303,
283 	0x00000200, 0xE03000E0,
284 	0x00000204, 0x800010FF,
285 	0x00000200, 0xE0300100,
286 	0x00000204, 0x00010303,
287 	0x00000200, 0xE03000FC,
288 	0x00000204, 0x800010FF,
289 	0x00000200, 0x0001f198,
290 	0x00000204, 0x0003ffff,
291 	0x00000200, 0x0001f19C,
292 	0x00000204, 0x3fffffff,
293 	0x00000200, 0xE030032C,
294 	0x00000204, 0x00000000,
295 };
296 
297 static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
298 						   const u32 *seq, u32 count);
299 static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev);
300 static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
301 					     struct radeon_ps *new_rps,
302 					     struct radeon_ps *old_rps);
303 
304 static struct trinity_ps *trinity_get_ps(struct radeon_ps *rps)
305 {
306 	struct trinity_ps *ps = rps->ps_priv;
307 
308 	return ps;
309 }
310 
311 static struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev)
312 {
313 	struct trinity_power_info *pi = rdev->pm.dpm.priv;
314 
315 	return pi;
316 }
317 
318 static void trinity_gfx_powergating_initialize(struct radeon_device *rdev)
319 {
320 	struct trinity_power_info *pi = trinity_get_pi(rdev);
321 	u32 p, u;
322 	u32 value;
323 	struct atom_clock_dividers dividers;
324 	u32 xclk = radeon_get_xclk(rdev);
325 	u32 sssd = 1;
326 	int ret;
327 	u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT;
328 
329 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
330 					     25000, false, &dividers);
331 	if (ret)
332 		return;
333 
334 	value = RREG32_SMC(GFX_POWER_GATING_CNTL);
335 	value &= ~(SSSD_MASK | PDS_DIV_MASK);
336 	if (sssd)
337 		value |= SSSD(1);
338 	value |= PDS_DIV(dividers.post_div);
339 	WREG32_SMC(GFX_POWER_GATING_CNTL, value);
340 
341 	r600_calculate_u_and_p(500, xclk, 16, &p, &u);
342 
343 	WREG32(CG_PG_CTRL, SP(p) | SU(u));
344 
345 	WREG32_P(CG_GIPOTS, CG_GIPOT(p), ~CG_GIPOT_MASK);
346 
347 	/* XXX double check hw_rev */
348 	if (pi->override_dynamic_mgpg && (hw_rev == 0))
349 		trinity_override_dynamic_mg_powergating(rdev);
350 
351 }
352 
353 #define CGCG_CGTT_LOCAL0_MASK       0xFFFF33FF
354 #define CGCG_CGTT_LOCAL1_MASK       0xFFFB0FFE
355 #define CGTS_SM_CTRL_REG_DISABLE    0x00600000
356 #define CGTS_SM_CTRL_REG_ENABLE     0x96944200
357 
358 static void trinity_mg_clockgating_enable(struct radeon_device *rdev,
359 					  bool enable)
360 {
361 	u32 local0;
362 	u32 local1;
363 
364 	if (enable) {
365 		local0 = RREG32_CG(CG_CGTT_LOCAL_0);
366 		local1 = RREG32_CG(CG_CGTT_LOCAL_1);
367 
368 		WREG32_CG(CG_CGTT_LOCAL_0,
369 			  (0x00380000 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
370 		WREG32_CG(CG_CGTT_LOCAL_1,
371 			  (0x0E000000 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
372 
373 		WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_ENABLE);
374 	} else {
375 		WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_DISABLE);
376 
377 		local0 = RREG32_CG(CG_CGTT_LOCAL_0);
378 		local1 = RREG32_CG(CG_CGTT_LOCAL_1);
379 
380 		WREG32_CG(CG_CGTT_LOCAL_0,
381 			  CGCG_CGTT_LOCAL0_MASK | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
382 		WREG32_CG(CG_CGTT_LOCAL_1,
383 			  CGCG_CGTT_LOCAL1_MASK | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
384 	}
385 }
386 
387 static void trinity_mg_clockgating_initialize(struct radeon_device *rdev)
388 {
389 	u32 count;
390 	const u32 *seq = NULL;
391 
392 	seq = &trinity_mgcg_shls_default[0];
393 	count = sizeof(trinity_mgcg_shls_default) / (3 * sizeof(u32));
394 
395 	trinity_program_clk_gating_hw_sequence(rdev, seq, count);
396 }
397 
398 static void trinity_gfx_clockgating_enable(struct radeon_device *rdev,
399 					   bool enable)
400 {
401 	if (enable) {
402 		WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
403 	} else {
404 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
405 		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
406 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
407 		RREG32(GB_ADDR_CONFIG);
408 	}
409 }
410 
411 static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
412 						   const u32 *seq, u32 count)
413 {
414 	u32 i, length = count * 3;
415 
416 	for (i = 0; i < length; i += 3)
417 		WREG32_P(seq[i], seq[i+1], ~seq[i+2]);
418 }
419 
420 static void trinity_program_override_mgpg_sequences(struct radeon_device *rdev,
421 						    const u32 *seq, u32 count)
422 {
423 	u32  i, length = count * 2;
424 
425 	for (i = 0; i < length; i += 2)
426 		WREG32(seq[i], seq[i+1]);
427 
428 }
429 
430 static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev)
431 {
432 	u32 count;
433 	const u32 *seq = NULL;
434 
435 	seq = &trinity_override_mgpg_sequences[0];
436 	count = sizeof(trinity_override_mgpg_sequences) / (2 * sizeof(u32));
437 
438 	trinity_program_override_mgpg_sequences(rdev, seq, count);
439 }
440 
441 static void trinity_ls_clockgating_enable(struct radeon_device *rdev,
442 					  bool enable)
443 {
444 	u32 count;
445 	const u32 *seq = NULL;
446 
447 	if (enable) {
448 		seq = &trinity_sysls_enable[0];
449 		count = sizeof(trinity_sysls_enable) / (3 * sizeof(u32));
450 	} else {
451 		seq = &trinity_sysls_disable[0];
452 		count = sizeof(trinity_sysls_disable) / (3 * sizeof(u32));
453 	}
454 
455 	trinity_program_clk_gating_hw_sequence(rdev, seq, count);
456 }
457 
458 static void trinity_gfx_powergating_enable(struct radeon_device *rdev,
459 					   bool enable)
460 {
461 	if (enable) {
462 		if (RREG32_SMC(CC_SMU_TST_EFUSE1_MISC) & RB_BACKEND_DISABLE_MASK)
463 			WREG32_SMC(SMU_SCRATCH_A, (RREG32_SMC(SMU_SCRATCH_A) | 0x01));
464 
465 		WREG32_P(SCLK_PWRMGT_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN);
466 	} else {
467 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_PWR_DOWN_EN);
468 		RREG32(GB_ADDR_CONFIG);
469 	}
470 }
471 
472 static void trinity_gfx_dynamic_mgpg_enable(struct radeon_device *rdev,
473 					    bool enable)
474 {
475 	u32 value;
476 
477 	if (enable) {
478 		value = RREG32_SMC(PM_I_CNTL_1);
479 		value &= ~DS_PG_CNTL_MASK;
480 		value |= DS_PG_CNTL(1);
481 		WREG32_SMC(PM_I_CNTL_1, value);
482 
483 		value = RREG32_SMC(SMU_S_PG_CNTL);
484 		value &= ~DS_PG_EN_MASK;
485 		value |= DS_PG_EN(1);
486 		WREG32_SMC(SMU_S_PG_CNTL, value);
487 	} else {
488 		value = RREG32_SMC(SMU_S_PG_CNTL);
489 		value &= ~DS_PG_EN_MASK;
490 		WREG32_SMC(SMU_S_PG_CNTL, value);
491 
492 		value = RREG32_SMC(PM_I_CNTL_1);
493 		value &= ~DS_PG_CNTL_MASK;
494 		WREG32_SMC(PM_I_CNTL_1, value);
495 	}
496 
497 	trinity_gfx_dynamic_mgpg_config(rdev);
498 
499 }
500 
501 static void trinity_enable_clock_power_gating(struct radeon_device *rdev)
502 {
503 	struct trinity_power_info *pi = trinity_get_pi(rdev);
504 
505 	if (pi->enable_gfx_clock_gating)
506 		sumo_gfx_clockgating_initialize(rdev);
507 	if (pi->enable_mg_clock_gating)
508 		trinity_mg_clockgating_initialize(rdev);
509 	if (pi->enable_gfx_power_gating)
510 		trinity_gfx_powergating_initialize(rdev);
511 	if (pi->enable_mg_clock_gating) {
512 		trinity_ls_clockgating_enable(rdev, true);
513 		trinity_mg_clockgating_enable(rdev, true);
514 	}
515 	if (pi->enable_gfx_clock_gating)
516 		trinity_gfx_clockgating_enable(rdev, true);
517 	if (pi->enable_gfx_dynamic_mgpg)
518 		trinity_gfx_dynamic_mgpg_enable(rdev, true);
519 	if (pi->enable_gfx_power_gating)
520 		trinity_gfx_powergating_enable(rdev, true);
521 }
522 
523 static void trinity_disable_clock_power_gating(struct radeon_device *rdev)
524 {
525 	struct trinity_power_info *pi = trinity_get_pi(rdev);
526 
527 	if (pi->enable_gfx_power_gating)
528 		trinity_gfx_powergating_enable(rdev, false);
529 	if (pi->enable_gfx_dynamic_mgpg)
530 		trinity_gfx_dynamic_mgpg_enable(rdev, false);
531 	if (pi->enable_gfx_clock_gating)
532 		trinity_gfx_clockgating_enable(rdev, false);
533 	if (pi->enable_mg_clock_gating) {
534 		trinity_mg_clockgating_enable(rdev, false);
535 		trinity_ls_clockgating_enable(rdev, false);
536 	}
537 }
538 
539 static void trinity_set_divider_value(struct radeon_device *rdev,
540 				      u32 index, u32 sclk)
541 {
542 	struct atom_clock_dividers  dividers;
543 	int ret;
544 	u32 value;
545 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
546 
547 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
548 					     sclk, false, &dividers);
549 	if (ret)
550 		return;
551 
552 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
553 	value &= ~CLK_DIVIDER_MASK;
554 	value |= CLK_DIVIDER(dividers.post_div);
555 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
556 
557 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
558 					     sclk/2, false, &dividers);
559 	if (ret)
560 		return;
561 
562 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix);
563 	value &= ~PD_SCLK_DIVIDER_MASK;
564 	value |= PD_SCLK_DIVIDER(dividers.post_div);
565 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix, value);
566 }
567 
568 static void trinity_set_ds_dividers(struct radeon_device *rdev,
569 				    u32 index, u32 divider)
570 {
571 	u32 value;
572 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
573 
574 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
575 	value &= ~DS_DIV_MASK;
576 	value |= DS_DIV(divider);
577 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
578 }
579 
580 static void trinity_set_ss_dividers(struct radeon_device *rdev,
581 				    u32 index, u32 divider)
582 {
583 	u32 value;
584 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
585 
586 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
587 	value &= ~DS_SH_DIV_MASK;
588 	value |= DS_SH_DIV(divider);
589 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
590 }
591 
592 static void trinity_set_vid(struct radeon_device *rdev, u32 index, u32 vid)
593 {
594 	struct trinity_power_info *pi = trinity_get_pi(rdev);
595 	u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid);
596 	u32 value;
597 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
598 
599 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
600 	value &= ~VID_MASK;
601 	value |= VID(vid_7bit);
602 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
603 
604 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
605 	value &= ~LVRT_MASK;
606 	value |= LVRT(0);
607 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
608 }
609 
610 static void trinity_set_allos_gnb_slow(struct radeon_device *rdev,
611 				       u32 index, u32 gnb_slow)
612 {
613 	u32 value;
614 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
615 
616 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
617 	value &= ~GNB_SLOW_MASK;
618 	value |= GNB_SLOW(gnb_slow);
619 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
620 }
621 
622 static void trinity_set_force_nbp_state(struct radeon_device *rdev,
623 					u32 index, u32 force_nbp_state)
624 {
625 	u32 value;
626 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
627 
628 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
629 	value &= ~FORCE_NBPS1_MASK;
630 	value |= FORCE_NBPS1(force_nbp_state);
631 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
632 }
633 
634 static void trinity_set_display_wm(struct radeon_device *rdev,
635 				   u32 index, u32 wm)
636 {
637 	u32 value;
638 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
639 
640 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
641 	value &= ~DISPLAY_WM_MASK;
642 	value |= DISPLAY_WM(wm);
643 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
644 }
645 
646 static void trinity_set_vce_wm(struct radeon_device *rdev,
647 			       u32 index, u32 wm)
648 {
649 	u32 value;
650 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
651 
652 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
653 	value &= ~VCE_WM_MASK;
654 	value |= VCE_WM(wm);
655 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
656 }
657 
658 static void trinity_set_at(struct radeon_device *rdev,
659 			   u32 index, u32 at)
660 {
661 	u32 value;
662 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
663 
664 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix);
665 	value &= ~AT_MASK;
666 	value |= AT(at);
667 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix, value);
668 }
669 
670 static void trinity_program_power_level(struct radeon_device *rdev,
671 					struct trinity_pl *pl, u32 index)
672 {
673 	struct trinity_power_info *pi = trinity_get_pi(rdev);
674 
675 	if (index >= SUMO_MAX_HARDWARE_POWERLEVELS)
676 		return;
677 
678 	trinity_set_divider_value(rdev, index, pl->sclk);
679 	trinity_set_vid(rdev, index, pl->vddc_index);
680 	trinity_set_ss_dividers(rdev, index, pl->ss_divider_index);
681 	trinity_set_ds_dividers(rdev, index, pl->ds_divider_index);
682 	trinity_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow);
683 	trinity_set_force_nbp_state(rdev, index, pl->force_nbp_state);
684 	trinity_set_display_wm(rdev, index, pl->display_wm);
685 	trinity_set_vce_wm(rdev, index, pl->vce_wm);
686 	trinity_set_at(rdev, index, pi->at[index]);
687 }
688 
689 static void trinity_power_level_enable_disable(struct radeon_device *rdev,
690 					       u32 index, bool enable)
691 {
692 	u32 value;
693 	u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
694 
695 	value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
696 	value &= ~STATE_VALID_MASK;
697 	if (enable)
698 		value |= STATE_VALID(1);
699 	WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
700 }
701 
702 static bool trinity_dpm_enabled(struct radeon_device *rdev)
703 {
704 	if (RREG32_SMC(SMU_SCLK_DPM_CNTL) & SCLK_DPM_EN(1))
705 		return true;
706 	else
707 		return false;
708 }
709 
710 static void trinity_start_dpm(struct radeon_device *rdev)
711 {
712 	u32 value = RREG32_SMC(SMU_SCLK_DPM_CNTL);
713 
714 	value &= ~(SCLK_DPM_EN_MASK | SCLK_DPM_BOOT_STATE_MASK | VOLTAGE_CHG_EN_MASK);
715 	value |= SCLK_DPM_EN(1) | SCLK_DPM_BOOT_STATE(0) | VOLTAGE_CHG_EN(1);
716 	WREG32_SMC(SMU_SCLK_DPM_CNTL, value);
717 
718 	WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
719 	WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~EN);
720 
721 	trinity_dpm_config(rdev, true);
722 }
723 
724 static void trinity_wait_for_dpm_enabled(struct radeon_device *rdev)
725 {
726 	int i;
727 
728 	for (i = 0; i < rdev->usec_timeout; i++) {
729 		if (RREG32(SCLK_PWRMGT_CNTL) & DYNAMIC_PM_EN)
730 			break;
731 		udelay(1);
732 	}
733 	for (i = 0; i < rdev->usec_timeout; i++) {
734 		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_STATE_MASK) == 0)
735 			break;
736 		udelay(1);
737 	}
738 	for (i = 0; i < rdev->usec_timeout; i++) {
739 		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
740 			break;
741 		udelay(1);
742 	}
743 }
744 
745 static void trinity_stop_dpm(struct radeon_device *rdev)
746 {
747 	u32 sclk_dpm_cntl;
748 
749 	WREG32_P(CG_CG_VOLTAGE_CNTL, EN, ~EN);
750 
751 	sclk_dpm_cntl = RREG32_SMC(SMU_SCLK_DPM_CNTL);
752 	sclk_dpm_cntl &= ~(SCLK_DPM_EN_MASK | VOLTAGE_CHG_EN_MASK);
753 	WREG32_SMC(SMU_SCLK_DPM_CNTL, sclk_dpm_cntl);
754 
755 	trinity_dpm_config(rdev, false);
756 }
757 
758 static void trinity_start_am(struct radeon_device *rdev)
759 {
760 	WREG32_P(SCLK_PWRMGT_CNTL, 0, ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
761 }
762 
763 static void trinity_reset_am(struct radeon_device *rdev)
764 {
765 	WREG32_P(SCLK_PWRMGT_CNTL, RESET_SCLK_CNT | RESET_BUSY_CNT,
766 		 ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
767 }
768 
769 static void trinity_wait_for_level_0(struct radeon_device *rdev)
770 {
771 	int i;
772 
773 	for (i = 0; i < rdev->usec_timeout; i++) {
774 		if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
775 			break;
776 		udelay(1);
777 	}
778 }
779 
780 static void trinity_enable_power_level_0(struct radeon_device *rdev)
781 {
782 	trinity_power_level_enable_disable(rdev, 0, true);
783 }
784 
785 static void trinity_force_level_0(struct radeon_device *rdev)
786 {
787 	trinity_dpm_force_state(rdev, 0);
788 }
789 
790 static void trinity_unforce_levels(struct radeon_device *rdev)
791 {
792 	trinity_dpm_no_forced_level(rdev);
793 }
794 
795 static void trinity_program_power_levels_0_to_n(struct radeon_device *rdev,
796 						struct radeon_ps *new_rps,
797 						struct radeon_ps *old_rps)
798 {
799 	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
800 	struct trinity_ps *old_ps = trinity_get_ps(old_rps);
801 	u32 i;
802 	u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels;
803 
804 	for (i = 0; i < new_ps->num_levels; i++) {
805 		trinity_program_power_level(rdev, &new_ps->levels[i], i);
806 		trinity_power_level_enable_disable(rdev, i, true);
807 	}
808 
809 	for (i = new_ps->num_levels; i < n_current_state_levels; i++)
810 		trinity_power_level_enable_disable(rdev, i, false);
811 }
812 
813 static void trinity_program_bootup_state(struct radeon_device *rdev)
814 {
815 	struct trinity_power_info *pi = trinity_get_pi(rdev);
816 	u32 i;
817 
818 	trinity_program_power_level(rdev, &pi->boot_pl, 0);
819 	trinity_power_level_enable_disable(rdev, 0, true);
820 
821 	for (i = 1; i < 8; i++)
822 		trinity_power_level_enable_disable(rdev, i, false);
823 }
824 
825 static void trinity_setup_uvd_clock_table(struct radeon_device *rdev,
826 					  struct radeon_ps *rps)
827 {
828 	struct trinity_ps *ps = trinity_get_ps(rps);
829 	u32 uvdstates = (ps->vclk_low_divider |
830 			 ps->vclk_high_divider << 8 |
831 			 ps->dclk_low_divider << 16 |
832 			 ps->dclk_high_divider << 24);
833 
834 	WREG32_SMC(SMU_UVD_DPM_STATES, uvdstates);
835 }
836 
837 static void trinity_setup_uvd_dpm_interval(struct radeon_device *rdev,
838 					   u32 interval)
839 {
840 	u32 p, u;
841 	u32 tp = RREG32_SMC(PM_TP);
842 	u32 val;
843 	u32 xclk = radeon_get_xclk(rdev);
844 
845 	r600_calculate_u_and_p(interval, xclk, 16, &p, &u);
846 
847 	val = (p + tp - 1) / tp;
848 
849 	WREG32_SMC(SMU_UVD_DPM_CNTL, val);
850 }
851 
852 static bool trinity_uvd_clocks_zero(struct radeon_ps *rps)
853 {
854 	if ((rps->vclk == 0) && (rps->dclk == 0))
855 		return true;
856 	else
857 		return false;
858 }
859 
860 static bool trinity_uvd_clocks_equal(struct radeon_ps *rps1,
861 				     struct radeon_ps *rps2)
862 {
863 	struct trinity_ps *ps1 = trinity_get_ps(rps1);
864 	struct trinity_ps *ps2 = trinity_get_ps(rps2);
865 
866 	if ((rps1->vclk == rps2->vclk) &&
867 	    (rps1->dclk == rps2->dclk) &&
868 	    (ps1->vclk_low_divider == ps2->vclk_low_divider) &&
869 	    (ps1->vclk_high_divider == ps2->vclk_high_divider) &&
870 	    (ps1->dclk_low_divider == ps2->dclk_low_divider) &&
871 	    (ps1->dclk_high_divider == ps2->dclk_high_divider))
872 		return true;
873 	else
874 		return false;
875 }
876 
877 static void trinity_setup_uvd_clocks(struct radeon_device *rdev,
878 				     struct radeon_ps *new_rps,
879 				     struct radeon_ps *old_rps)
880 {
881 	struct trinity_power_info *pi = trinity_get_pi(rdev);
882 
883 	if (pi->enable_gfx_power_gating) {
884 		trinity_gfx_powergating_enable(rdev, false);
885 	}
886 
887 	if (pi->uvd_dpm) {
888 		if (trinity_uvd_clocks_zero(new_rps) &&
889 		    !trinity_uvd_clocks_zero(old_rps)) {
890 			trinity_setup_uvd_dpm_interval(rdev, 0);
891 		} else if (!trinity_uvd_clocks_zero(new_rps)) {
892 			trinity_setup_uvd_clock_table(rdev, new_rps);
893 
894 			if (trinity_uvd_clocks_zero(old_rps)) {
895 				u32 tmp = RREG32(CG_MISC_REG);
896 				tmp &= 0xfffffffd;
897 				WREG32(CG_MISC_REG, tmp);
898 
899 				radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
900 
901 				trinity_setup_uvd_dpm_interval(rdev, 3000);
902 			}
903 		}
904 		trinity_uvd_dpm_config(rdev);
905 	} else {
906 		if (trinity_uvd_clocks_zero(new_rps) ||
907 		    trinity_uvd_clocks_equal(new_rps, old_rps))
908 			return;
909 
910 		radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
911 	}
912 
913 	if (pi->enable_gfx_power_gating) {
914 		trinity_gfx_powergating_enable(rdev, true);
915 	}
916 }
917 
918 static void trinity_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
919 						       struct radeon_ps *new_rps,
920 						       struct radeon_ps *old_rps)
921 {
922 	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
923 	struct trinity_ps *current_ps = trinity_get_ps(new_rps);
924 
925 	if (new_ps->levels[new_ps->num_levels - 1].sclk >=
926 	    current_ps->levels[current_ps->num_levels - 1].sclk)
927 		return;
928 
929 	trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
930 }
931 
932 static void trinity_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
933 						      struct radeon_ps *new_rps,
934 						      struct radeon_ps *old_rps)
935 {
936 	struct trinity_ps *new_ps = trinity_get_ps(new_rps);
937 	struct trinity_ps *current_ps = trinity_get_ps(old_rps);
938 
939 	if (new_ps->levels[new_ps->num_levels - 1].sclk <
940 	    current_ps->levels[current_ps->num_levels - 1].sclk)
941 		return;
942 
943 	trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
944 }
945 
946 static void trinity_set_vce_clock(struct radeon_device *rdev,
947 				  struct radeon_ps *new_rps,
948 				  struct radeon_ps *old_rps)
949 {
950 	if ((old_rps->evclk != new_rps->evclk) ||
951 	    (old_rps->ecclk != new_rps->ecclk)) {
952 		/* turn the clocks on when encoding, off otherwise */
953 		if (new_rps->evclk || new_rps->ecclk)
954 			vce_v1_0_enable_mgcg(rdev, false);
955 		else
956 			vce_v1_0_enable_mgcg(rdev, true);
957 		radeon_set_vce_clocks(rdev, new_rps->evclk, new_rps->ecclk);
958 	}
959 }
960 
961 static void trinity_program_ttt(struct radeon_device *rdev)
962 {
963 	struct trinity_power_info *pi = trinity_get_pi(rdev);
964 	u32 value = RREG32_SMC(SMU_SCLK_DPM_TTT);
965 
966 	value &= ~(HT_MASK | LT_MASK);
967 	value |= HT((pi->thermal_auto_throttling + 49) * 8);
968 	value |= LT((pi->thermal_auto_throttling + 49 - pi->sys_info.htc_hyst_lmt) * 8);
969 	WREG32_SMC(SMU_SCLK_DPM_TTT, value);
970 }
971 
972 static void trinity_enable_att(struct radeon_device *rdev)
973 {
974 	u32 value = RREG32_SMC(SMU_SCLK_DPM_TT_CNTL);
975 
976 	value &= ~SCLK_TT_EN_MASK;
977 	value |= SCLK_TT_EN(1);
978 	WREG32_SMC(SMU_SCLK_DPM_TT_CNTL, value);
979 }
980 
981 static void trinity_program_sclk_dpm(struct radeon_device *rdev)
982 {
983 	u32 p, u;
984 	u32 tp = RREG32_SMC(PM_TP);
985 	u32 ni;
986 	u32 xclk = radeon_get_xclk(rdev);
987 	u32 value;
988 
989 	r600_calculate_u_and_p(400, xclk, 16, &p, &u);
990 
991 	ni = (p + tp - 1) / tp;
992 
993 	value = RREG32_SMC(PM_I_CNTL_1);
994 	value &= ~SCLK_DPM_MASK;
995 	value |= SCLK_DPM(ni);
996 	WREG32_SMC(PM_I_CNTL_1, value);
997 }
998 
999 static int trinity_set_thermal_temperature_range(struct radeon_device *rdev,
1000 						 int min_temp, int max_temp)
1001 {
1002 	int low_temp = 0 * 1000;
1003 	int high_temp = 255 * 1000;
1004 
1005 	if (low_temp < min_temp)
1006 		low_temp = min_temp;
1007 	if (high_temp > max_temp)
1008 		high_temp = max_temp;
1009 	if (high_temp < low_temp) {
1010 		DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1011 		return -EINVAL;
1012 	}
1013 
1014 	WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK);
1015 	WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK);
1016 
1017 	rdev->pm.dpm.thermal.min_temp = low_temp;
1018 	rdev->pm.dpm.thermal.max_temp = high_temp;
1019 
1020 	return 0;
1021 }
1022 
1023 static void trinity_update_current_ps(struct radeon_device *rdev,
1024 				      struct radeon_ps *rps)
1025 {
1026 	struct trinity_ps *new_ps = trinity_get_ps(rps);
1027 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1028 
1029 	pi->current_rps = *rps;
1030 	pi->current_ps = *new_ps;
1031 	pi->current_rps.ps_priv = &pi->current_ps;
1032 }
1033 
1034 static void trinity_update_requested_ps(struct radeon_device *rdev,
1035 					struct radeon_ps *rps)
1036 {
1037 	struct trinity_ps *new_ps = trinity_get_ps(rps);
1038 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1039 
1040 	pi->requested_rps = *rps;
1041 	pi->requested_ps = *new_ps;
1042 	pi->requested_rps.ps_priv = &pi->requested_ps;
1043 }
1044 
1045 void trinity_dpm_enable_bapm(struct radeon_device *rdev, bool enable)
1046 {
1047 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1048 
1049 	if (pi->enable_bapm) {
1050 		trinity_acquire_mutex(rdev);
1051 		trinity_dpm_bapm_enable(rdev, enable);
1052 		trinity_release_mutex(rdev);
1053 	}
1054 }
1055 
1056 int trinity_dpm_enable(struct radeon_device *rdev)
1057 {
1058 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1059 
1060 	trinity_acquire_mutex(rdev);
1061 
1062 	if (trinity_dpm_enabled(rdev)) {
1063 		trinity_release_mutex(rdev);
1064 		return -EINVAL;
1065 	}
1066 
1067 	trinity_program_bootup_state(rdev);
1068 	sumo_program_vc(rdev, 0x00C00033);
1069 	trinity_start_am(rdev);
1070 	if (pi->enable_auto_thermal_throttling) {
1071 		trinity_program_ttt(rdev);
1072 		trinity_enable_att(rdev);
1073 	}
1074 	trinity_program_sclk_dpm(rdev);
1075 	trinity_start_dpm(rdev);
1076 	trinity_wait_for_dpm_enabled(rdev);
1077 	trinity_dpm_bapm_enable(rdev, false);
1078 	trinity_release_mutex(rdev);
1079 
1080 	trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1081 
1082 	return 0;
1083 }
1084 
1085 int trinity_dpm_late_enable(struct radeon_device *rdev)
1086 {
1087 	int ret;
1088 
1089 	trinity_acquire_mutex(rdev);
1090 	trinity_enable_clock_power_gating(rdev);
1091 
1092 	if (rdev->irq.installed &&
1093 	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1094 		ret = trinity_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1095 		if (ret) {
1096 			trinity_release_mutex(rdev);
1097 			return ret;
1098 		}
1099 		rdev->irq.dpm_thermal = true;
1100 		radeon_irq_set(rdev);
1101 	}
1102 	trinity_release_mutex(rdev);
1103 
1104 	return 0;
1105 }
1106 
1107 void trinity_dpm_disable(struct radeon_device *rdev)
1108 {
1109 	trinity_acquire_mutex(rdev);
1110 	if (!trinity_dpm_enabled(rdev)) {
1111 		trinity_release_mutex(rdev);
1112 		return;
1113 	}
1114 	trinity_dpm_bapm_enable(rdev, false);
1115 	trinity_disable_clock_power_gating(rdev);
1116 	sumo_clear_vc(rdev);
1117 	trinity_wait_for_level_0(rdev);
1118 	trinity_stop_dpm(rdev);
1119 	trinity_reset_am(rdev);
1120 	trinity_release_mutex(rdev);
1121 
1122 	if (rdev->irq.installed &&
1123 	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1124 		rdev->irq.dpm_thermal = false;
1125 		radeon_irq_set(rdev);
1126 	}
1127 
1128 	trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1129 }
1130 
1131 static void trinity_get_min_sclk_divider(struct radeon_device *rdev)
1132 {
1133 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1134 
1135 	pi->min_sclk_did =
1136 		(RREG32_SMC(CC_SMU_MISC_FUSES) & MinSClkDid_MASK) >> MinSClkDid_SHIFT;
1137 }
1138 
1139 static void trinity_setup_nbp_sim(struct radeon_device *rdev,
1140 				  struct radeon_ps *rps)
1141 {
1142 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1143 	struct trinity_ps *new_ps = trinity_get_ps(rps);
1144 	u32 nbpsconfig;
1145 
1146 	if (pi->sys_info.nb_dpm_enable) {
1147 		nbpsconfig = RREG32_SMC(NB_PSTATE_CONFIG);
1148 		nbpsconfig &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | DpmXNbPsLo_MASK | DpmXNbPsHi_MASK);
1149 		nbpsconfig |= (Dpm0PgNbPsLo(new_ps->Dpm0PgNbPsLo) |
1150 			       Dpm0PgNbPsHi(new_ps->Dpm0PgNbPsHi) |
1151 			       DpmXNbPsLo(new_ps->DpmXNbPsLo) |
1152 			       DpmXNbPsHi(new_ps->DpmXNbPsHi));
1153 		WREG32_SMC(NB_PSTATE_CONFIG, nbpsconfig);
1154 	}
1155 }
1156 
1157 int trinity_dpm_force_performance_level(struct radeon_device *rdev,
1158 					enum radeon_dpm_forced_level level)
1159 {
1160 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1161 	struct radeon_ps *rps = &pi->current_rps;
1162 	struct trinity_ps *ps = trinity_get_ps(rps);
1163 	int i, ret;
1164 
1165 	if (ps->num_levels <= 1)
1166 		return 0;
1167 
1168 	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1169 		/* not supported by the hw */
1170 		return -EINVAL;
1171 	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1172 		ret = trinity_dpm_n_levels_disabled(rdev, ps->num_levels - 1);
1173 		if (ret)
1174 			return ret;
1175 	} else {
1176 		for (i = 0; i < ps->num_levels; i++) {
1177 			ret = trinity_dpm_n_levels_disabled(rdev, 0);
1178 			if (ret)
1179 				return ret;
1180 		}
1181 	}
1182 
1183 	rdev->pm.dpm.forced_level = level;
1184 
1185 	return 0;
1186 }
1187 
1188 int trinity_dpm_pre_set_power_state(struct radeon_device *rdev)
1189 {
1190 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1191 	struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
1192 	struct radeon_ps *new_ps = &requested_ps;
1193 
1194 	trinity_update_requested_ps(rdev, new_ps);
1195 
1196 	trinity_apply_state_adjust_rules(rdev,
1197 					 &pi->requested_rps,
1198 					 &pi->current_rps);
1199 
1200 	return 0;
1201 }
1202 
1203 int trinity_dpm_set_power_state(struct radeon_device *rdev)
1204 {
1205 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1206 	struct radeon_ps *new_ps = &pi->requested_rps;
1207 	struct radeon_ps *old_ps = &pi->current_rps;
1208 
1209 	trinity_acquire_mutex(rdev);
1210 	if (pi->enable_dpm) {
1211 		if (pi->enable_bapm)
1212 			trinity_dpm_bapm_enable(rdev, rdev->pm.dpm.ac_power);
1213 		trinity_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
1214 		trinity_enable_power_level_0(rdev);
1215 		trinity_force_level_0(rdev);
1216 		trinity_wait_for_level_0(rdev);
1217 		trinity_setup_nbp_sim(rdev, new_ps);
1218 		trinity_program_power_levels_0_to_n(rdev, new_ps, old_ps);
1219 		trinity_force_level_0(rdev);
1220 		trinity_unforce_levels(rdev);
1221 		trinity_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
1222 		trinity_set_vce_clock(rdev, new_ps, old_ps);
1223 	}
1224 	trinity_release_mutex(rdev);
1225 
1226 	return 0;
1227 }
1228 
1229 void trinity_dpm_post_set_power_state(struct radeon_device *rdev)
1230 {
1231 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1232 	struct radeon_ps *new_ps = &pi->requested_rps;
1233 
1234 	trinity_update_current_ps(rdev, new_ps);
1235 }
1236 
1237 void trinity_dpm_setup_asic(struct radeon_device *rdev)
1238 {
1239 	trinity_acquire_mutex(rdev);
1240 	sumo_program_sstp(rdev);
1241 	sumo_take_smu_control(rdev, true);
1242 	trinity_get_min_sclk_divider(rdev);
1243 	trinity_release_mutex(rdev);
1244 }
1245 
1246 #if 0
1247 void trinity_dpm_reset_asic(struct radeon_device *rdev)
1248 {
1249 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1250 
1251 	trinity_acquire_mutex(rdev);
1252 	if (pi->enable_dpm) {
1253 		trinity_enable_power_level_0(rdev);
1254 		trinity_force_level_0(rdev);
1255 		trinity_wait_for_level_0(rdev);
1256 		trinity_program_bootup_state(rdev);
1257 		trinity_force_level_0(rdev);
1258 		trinity_unforce_levels(rdev);
1259 	}
1260 	trinity_release_mutex(rdev);
1261 }
1262 #endif
1263 
1264 static u16 trinity_convert_voltage_index_to_value(struct radeon_device *rdev,
1265 						  u32 vid_2bit)
1266 {
1267 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1268 	u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit);
1269 	u32 svi_mode = (RREG32_SMC(PM_CONFIG) & SVI_Mode) ? 1 : 0;
1270 	u32 step = (svi_mode == 0) ? 1250 : 625;
1271 	u32 delta = vid_7bit * step + 50;
1272 
1273 	if (delta > 155000)
1274 		return 0;
1275 
1276 	return (155000 - delta) / 100;
1277 }
1278 
1279 static void trinity_patch_boot_state(struct radeon_device *rdev,
1280 				     struct trinity_ps *ps)
1281 {
1282 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1283 
1284 	ps->num_levels = 1;
1285 	ps->nbps_flags = 0;
1286 	ps->bapm_flags = 0;
1287 	ps->levels[0] = pi->boot_pl;
1288 }
1289 
1290 static u8 trinity_calculate_vce_wm(struct radeon_device *rdev, u32 sclk)
1291 {
1292 	if (sclk < 20000)
1293 		return 1;
1294 	return 0;
1295 }
1296 
1297 static void trinity_construct_boot_state(struct radeon_device *rdev)
1298 {
1299 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1300 
1301 	pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
1302 	pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
1303 	pi->boot_pl.ds_divider_index = 0;
1304 	pi->boot_pl.ss_divider_index = 0;
1305 	pi->boot_pl.allow_gnb_slow = 1;
1306 	pi->boot_pl.force_nbp_state = 0;
1307 	pi->boot_pl.display_wm = 0;
1308 	pi->boot_pl.vce_wm = 0;
1309 	pi->current_ps.num_levels = 1;
1310 	pi->current_ps.levels[0] = pi->boot_pl;
1311 }
1312 
1313 static u8 trinity_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
1314 						  u32 sclk, u32 min_sclk_in_sr)
1315 {
1316 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1317 	u32 i;
1318 	u32 temp;
1319 	u32 min = (min_sclk_in_sr > TRINITY_MINIMUM_ENGINE_CLOCK) ?
1320 		min_sclk_in_sr : TRINITY_MINIMUM_ENGINE_CLOCK;
1321 
1322 	if (sclk < min)
1323 		return 0;
1324 
1325 	if (!pi->enable_sclk_ds)
1326 		return 0;
1327 
1328 	for (i = TRINITY_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
1329 		temp = sclk / sumo_get_sleep_divider_from_id(i);
1330 		if (temp >= min || i == 0)
1331 			break;
1332 	}
1333 
1334 	return (u8)i;
1335 }
1336 
1337 static u32 trinity_get_valid_engine_clock(struct radeon_device *rdev,
1338 					  u32 lower_limit)
1339 {
1340 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1341 	u32 i;
1342 
1343 	for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) {
1344 		if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit)
1345 			return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency;
1346 	}
1347 
1348 	if (i == pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries)
1349 		DRM_ERROR("engine clock out of range!");
1350 
1351 	return 0;
1352 }
1353 
1354 static void trinity_patch_thermal_state(struct radeon_device *rdev,
1355 					struct trinity_ps *ps,
1356 					struct trinity_ps *current_ps)
1357 {
1358 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1359 	u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1360 	u32 current_vddc;
1361 	u32 current_sclk;
1362 	u32 current_index = 0;
1363 
1364 	if (current_ps) {
1365 		current_vddc = current_ps->levels[current_index].vddc_index;
1366 		current_sclk = current_ps->levels[current_index].sclk;
1367 	} else {
1368 		current_vddc = pi->boot_pl.vddc_index;
1369 		current_sclk = pi->boot_pl.sclk;
1370 	}
1371 
1372 	ps->levels[0].vddc_index = current_vddc;
1373 
1374 	if (ps->levels[0].sclk > current_sclk)
1375 		ps->levels[0].sclk = current_sclk;
1376 
1377 	ps->levels[0].ds_divider_index =
1378 		trinity_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr);
1379 	ps->levels[0].ss_divider_index = ps->levels[0].ds_divider_index;
1380 	ps->levels[0].allow_gnb_slow = 1;
1381 	ps->levels[0].force_nbp_state = 0;
1382 	ps->levels[0].display_wm = 0;
1383 	ps->levels[0].vce_wm =
1384 		trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1385 }
1386 
1387 static u8 trinity_calculate_display_wm(struct radeon_device *rdev,
1388 				       struct trinity_ps *ps, u32 index)
1389 {
1390 	if (ps == NULL || ps->num_levels <= 1)
1391 		return 0;
1392 	else if (ps->num_levels == 2) {
1393 		if (index == 0)
1394 			return 0;
1395 		else
1396 			return 1;
1397 	} else {
1398 		if (index == 0)
1399 			return 0;
1400 		else if (ps->levels[index].sclk < 30000)
1401 			return 0;
1402 		else
1403 			return 1;
1404 	}
1405 }
1406 
1407 static u32 trinity_get_uvd_clock_index(struct radeon_device *rdev,
1408 				       struct radeon_ps *rps)
1409 {
1410 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1411 	u32 i = 0;
1412 
1413 	for (i = 0; i < 4; i++) {
1414 		if ((rps->vclk == pi->sys_info.uvd_clock_table_entries[i].vclk) &&
1415 		    (rps->dclk == pi->sys_info.uvd_clock_table_entries[i].dclk))
1416 		    break;
1417 	}
1418 
1419 	if (i >= 4) {
1420 		DRM_ERROR("UVD clock index not found!\n");
1421 		i = 3;
1422 	}
1423 	return i;
1424 }
1425 
1426 static void trinity_adjust_uvd_state(struct radeon_device *rdev,
1427 				     struct radeon_ps *rps)
1428 {
1429 	struct trinity_ps *ps = trinity_get_ps(rps);
1430 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1431 	u32 high_index = 0;
1432 	u32 low_index = 0;
1433 
1434 	if (pi->uvd_dpm && r600_is_uvd_state(rps->class, rps->class2)) {
1435 		high_index = trinity_get_uvd_clock_index(rdev, rps);
1436 
1437 		switch(high_index) {
1438 		case 3:
1439 		case 2:
1440 			low_index = 1;
1441 			break;
1442 		case 1:
1443 		case 0:
1444 		default:
1445 			low_index = 0;
1446 			break;
1447 		}
1448 
1449 		ps->vclk_low_divider =
1450 			pi->sys_info.uvd_clock_table_entries[high_index].vclk_did;
1451 		ps->dclk_low_divider =
1452 			pi->sys_info.uvd_clock_table_entries[high_index].dclk_did;
1453 		ps->vclk_high_divider =
1454 			pi->sys_info.uvd_clock_table_entries[low_index].vclk_did;
1455 		ps->dclk_high_divider =
1456 			pi->sys_info.uvd_clock_table_entries[low_index].dclk_did;
1457 	}
1458 }
1459 
1460 static int trinity_get_vce_clock_voltage(struct radeon_device *rdev,
1461 					 u32 evclk, u32 ecclk, u16 *voltage)
1462 {
1463 	u32 i;
1464 	int ret = -EINVAL;
1465 	struct radeon_vce_clock_voltage_dependency_table *table =
1466 		&rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
1467 
1468 	if (((evclk == 0) && (ecclk == 0)) ||
1469 	    (table && (table->count == 0))) {
1470 		*voltage = 0;
1471 		return 0;
1472 	}
1473 
1474 	for (i = 0; i < table->count; i++) {
1475 		if ((evclk <= table->entries[i].evclk) &&
1476 		    (ecclk <= table->entries[i].ecclk)) {
1477 			*voltage = table->entries[i].v;
1478 			ret = 0;
1479 			break;
1480 		}
1481 	}
1482 
1483 	/* if no match return the highest voltage */
1484 	if (ret)
1485 		*voltage = table->entries[table->count - 1].v;
1486 
1487 	return ret;
1488 }
1489 
1490 static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
1491 					     struct radeon_ps *new_rps,
1492 					     struct radeon_ps *old_rps)
1493 {
1494 	struct trinity_ps *ps = trinity_get_ps(new_rps);
1495 	struct trinity_ps *current_ps = trinity_get_ps(old_rps);
1496 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1497 	u32 min_voltage = 0; /* ??? */
1498 	u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */
1499 	u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1500 	u32 i;
1501 	u16 min_vce_voltage;
1502 	bool force_high;
1503 	u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1504 
1505 	if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
1506 		return trinity_patch_thermal_state(rdev, ps, current_ps);
1507 
1508 	trinity_adjust_uvd_state(rdev, new_rps);
1509 
1510 	if (new_rps->vce_active) {
1511 		new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
1512 		new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
1513 	} else {
1514 		new_rps->evclk = 0;
1515 		new_rps->ecclk = 0;
1516 	}
1517 
1518 	for (i = 0; i < ps->num_levels; i++) {
1519 		if (ps->levels[i].vddc_index < min_voltage)
1520 			ps->levels[i].vddc_index = min_voltage;
1521 
1522 		if (ps->levels[i].sclk < min_sclk)
1523 			ps->levels[i].sclk =
1524 				trinity_get_valid_engine_clock(rdev, min_sclk);
1525 
1526 		/* patch in vce limits */
1527 		if (new_rps->vce_active) {
1528 			/* sclk */
1529 			if (ps->levels[i].sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
1530 				ps->levels[i].sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
1531 			/* vddc */
1532 			trinity_get_vce_clock_voltage(rdev, new_rps->evclk, new_rps->ecclk, &min_vce_voltage);
1533 			if (ps->levels[i].vddc_index < min_vce_voltage)
1534 				ps->levels[i].vddc_index = min_vce_voltage;
1535 		}
1536 
1537 		ps->levels[i].ds_divider_index =
1538 			sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr);
1539 
1540 		ps->levels[i].ss_divider_index = ps->levels[i].ds_divider_index;
1541 
1542 		ps->levels[i].allow_gnb_slow = 1;
1543 		ps->levels[i].force_nbp_state = 0;
1544 		ps->levels[i].display_wm =
1545 			trinity_calculate_display_wm(rdev, ps, i);
1546 		ps->levels[i].vce_wm =
1547 			trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1548 	}
1549 
1550 	if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1551 	    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY))
1552 		ps->bapm_flags |= TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE;
1553 
1554 	if (pi->sys_info.nb_dpm_enable) {
1555 		ps->Dpm0PgNbPsLo = 0x1;
1556 		ps->Dpm0PgNbPsHi = 0x0;
1557 		ps->DpmXNbPsLo = 0x2;
1558 		ps->DpmXNbPsHi = 0x1;
1559 
1560 		if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1561 		    ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) {
1562 			force_high = ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) ||
1563 				      ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) &&
1564 				       (pi->sys_info.uma_channel_number == 1)));
1565 			force_high = (num_active_displays >= 3) || force_high;
1566 			ps->Dpm0PgNbPsLo = force_high ? 0x2 : 0x3;
1567 			ps->Dpm0PgNbPsHi = 0x1;
1568 			ps->DpmXNbPsLo = force_high ? 0x2 : 0x3;
1569 			ps->DpmXNbPsHi = 0x2;
1570 			ps->levels[ps->num_levels - 1].allow_gnb_slow = 0;
1571 		}
1572 	}
1573 }
1574 
1575 static void trinity_cleanup_asic(struct radeon_device *rdev)
1576 {
1577 	sumo_take_smu_control(rdev, false);
1578 }
1579 
1580 #if 0
1581 static void trinity_pre_display_configuration_change(struct radeon_device *rdev)
1582 {
1583 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1584 
1585 	if (pi->voltage_drop_in_dce)
1586 		trinity_dce_enable_voltage_adjustment(rdev, false);
1587 }
1588 #endif
1589 
1590 static void trinity_add_dccac_value(struct radeon_device *rdev)
1591 {
1592 	u32 gpu_cac_avrg_cntl_window_size;
1593 	u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1594 	u64 disp_clk = rdev->clock.default_dispclk / 100;
1595 	u32 dc_cac_value;
1596 
1597 	gpu_cac_avrg_cntl_window_size =
1598 		(RREG32_SMC(GPU_CAC_AVRG_CNTL) & WINDOW_SIZE_MASK) >> WINDOW_SIZE_SHIFT;
1599 
1600 	dc_cac_value = (u32)((14213 * disp_clk * disp_clk * (u64)num_active_displays) >>
1601 			     (32 - gpu_cac_avrg_cntl_window_size));
1602 
1603 	WREG32_SMC(DC_CAC_VALUE, dc_cac_value);
1604 }
1605 
1606 void trinity_dpm_display_configuration_changed(struct radeon_device *rdev)
1607 {
1608 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1609 
1610 	if (pi->voltage_drop_in_dce)
1611 		trinity_dce_enable_voltage_adjustment(rdev, true);
1612 	trinity_add_dccac_value(rdev);
1613 }
1614 
1615 union power_info {
1616 	struct _ATOM_POWERPLAY_INFO info;
1617 	struct _ATOM_POWERPLAY_INFO_V2 info_2;
1618 	struct _ATOM_POWERPLAY_INFO_V3 info_3;
1619 	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
1620 	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
1621 	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
1622 };
1623 
1624 union pplib_clock_info {
1625 	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
1626 	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
1627 	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
1628 	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
1629 };
1630 
1631 union pplib_power_state {
1632 	struct _ATOM_PPLIB_STATE v1;
1633 	struct _ATOM_PPLIB_STATE_V2 v2;
1634 };
1635 
1636 static void trinity_parse_pplib_non_clock_info(struct radeon_device *rdev,
1637 					       struct radeon_ps *rps,
1638 					       struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
1639 					       u8 table_rev)
1640 {
1641 	struct trinity_ps *ps = trinity_get_ps(rps);
1642 
1643 	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
1644 	rps->class = le16_to_cpu(non_clock_info->usClassification);
1645 	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
1646 
1647 	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
1648 		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
1649 		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
1650 	} else {
1651 		rps->vclk = 0;
1652 		rps->dclk = 0;
1653 	}
1654 
1655 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
1656 		rdev->pm.dpm.boot_ps = rps;
1657 		trinity_patch_boot_state(rdev, ps);
1658 	}
1659 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
1660 		rdev->pm.dpm.uvd_ps = rps;
1661 }
1662 
1663 static void trinity_parse_pplib_clock_info(struct radeon_device *rdev,
1664 					   struct radeon_ps *rps, int index,
1665 					   union pplib_clock_info *clock_info)
1666 {
1667 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1668 	struct trinity_ps *ps = trinity_get_ps(rps);
1669 	struct trinity_pl *pl = &ps->levels[index];
1670 	u32 sclk;
1671 
1672 	sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1673 	sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1674 	pl->sclk = sclk;
1675 	pl->vddc_index = clock_info->sumo.vddcIndex;
1676 
1677 	ps->num_levels = index + 1;
1678 
1679 	if (pi->enable_sclk_ds) {
1680 		pl->ds_divider_index = 5;
1681 		pl->ss_divider_index = 5;
1682 	}
1683 }
1684 
1685 static int trinity_parse_power_table(struct radeon_device *rdev)
1686 {
1687 	struct radeon_mode_info *mode_info = &rdev->mode_info;
1688 	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
1689 	union pplib_power_state *power_state;
1690 	int i, j, k, non_clock_array_index, clock_array_index;
1691 	union pplib_clock_info *clock_info;
1692 	struct _StateArray *state_array;
1693 	struct _ClockInfoArray *clock_info_array;
1694 	struct _NonClockInfoArray *non_clock_info_array;
1695 	union power_info *power_info;
1696 	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
1697 	u16 data_offset;
1698 	u8 frev, crev;
1699 	u8 *power_state_offset;
1700 	struct sumo_ps *ps;
1701 
1702 	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
1703 				   &frev, &crev, &data_offset))
1704 		return -EINVAL;
1705 	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
1706 
1707 	state_array = (struct _StateArray *)
1708 		(mode_info->atom_context->bios + data_offset +
1709 		 le16_to_cpu(power_info->pplib.usStateArrayOffset));
1710 	clock_info_array = (struct _ClockInfoArray *)
1711 		(mode_info->atom_context->bios + data_offset +
1712 		 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
1713 	non_clock_info_array = (struct _NonClockInfoArray *)
1714 		(mode_info->atom_context->bios + data_offset +
1715 		 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
1716 
1717 	rdev->pm.dpm.ps = kcalloc(state_array->ucNumEntries,
1718 				  sizeof(struct radeon_ps),
1719 				  GFP_KERNEL);
1720 	if (!rdev->pm.dpm.ps)
1721 		return -ENOMEM;
1722 	power_state_offset = (u8 *)state_array->states;
1723 	for (i = 0; i < state_array->ucNumEntries; i++) {
1724 		u8 *idx;
1725 		power_state = (union pplib_power_state *)power_state_offset;
1726 		non_clock_array_index = power_state->v2.nonClockInfoIndex;
1727 		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
1728 			&non_clock_info_array->nonClockInfo[non_clock_array_index];
1729 		if (!rdev->pm.power_state[i].clock_info)
1730 			return -EINVAL;
1731 		ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL);
1732 		if (ps == NULL) {
1733 			kfree(rdev->pm.dpm.ps);
1734 			return -ENOMEM;
1735 		}
1736 		rdev->pm.dpm.ps[i].ps_priv = ps;
1737 		k = 0;
1738 		idx = (u8 *)&power_state->v2.clockInfoIndex[0];
1739 		for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
1740 			clock_array_index = idx[j];
1741 			if (clock_array_index >= clock_info_array->ucNumEntries)
1742 				continue;
1743 			if (k >= SUMO_MAX_HARDWARE_POWERLEVELS)
1744 				break;
1745 			clock_info = (union pplib_clock_info *)
1746 				((u8 *)&clock_info_array->clockInfo[0] +
1747 				 (clock_array_index * clock_info_array->ucEntrySize));
1748 			trinity_parse_pplib_clock_info(rdev,
1749 						       &rdev->pm.dpm.ps[i], k,
1750 						       clock_info);
1751 			k++;
1752 		}
1753 		trinity_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
1754 						   non_clock_info,
1755 						   non_clock_info_array->ucEntrySize);
1756 		power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
1757 	}
1758 	rdev->pm.dpm.num_ps = state_array->ucNumEntries;
1759 
1760 	/* fill in the vce power states */
1761 	for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
1762 		u32 sclk;
1763 		clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
1764 		clock_info = (union pplib_clock_info *)
1765 			&clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
1766 		sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1767 		sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1768 		rdev->pm.dpm.vce_states[i].sclk = sclk;
1769 		rdev->pm.dpm.vce_states[i].mclk = 0;
1770 	}
1771 
1772 	return 0;
1773 }
1774 
1775 union igp_info {
1776 	struct _ATOM_INTEGRATED_SYSTEM_INFO info;
1777 	struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
1778 	struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5;
1779 	struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
1780 	struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
1781 };
1782 
1783 static u32 trinity_convert_did_to_freq(struct radeon_device *rdev, u8 did)
1784 {
1785 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1786 	u32 divider;
1787 
1788 	if (did >= 8 && did <= 0x3f)
1789 		divider = did * 25;
1790 	else if (did > 0x3f && did <= 0x5f)
1791 		divider = (did - 64) * 50 + 1600;
1792 	else if (did > 0x5f && did <= 0x7e)
1793 		divider = (did - 96) * 100 + 3200;
1794 	else if (did == 0x7f)
1795 		divider = 128 * 100;
1796 	else
1797 		return 10000;
1798 
1799 	return ((pi->sys_info.dentist_vco_freq * 100) + (divider - 1)) / divider;
1800 }
1801 
1802 static int trinity_parse_sys_info_table(struct radeon_device *rdev)
1803 {
1804 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1805 	struct radeon_mode_info *mode_info = &rdev->mode_info;
1806 	int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
1807 	union igp_info *igp_info;
1808 	u8 frev, crev;
1809 	u16 data_offset;
1810 	int i;
1811 
1812 	if (atom_parse_data_header(mode_info->atom_context, index, NULL,
1813 				   &frev, &crev, &data_offset)) {
1814 		igp_info = (union igp_info *)(mode_info->atom_context->bios +
1815 					      data_offset);
1816 
1817 		if (crev != 7) {
1818 			DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
1819 			return -EINVAL;
1820 		}
1821 		pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_7.ulBootUpEngineClock);
1822 		pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_7.ulMinEngineClock);
1823 		pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_7.ulBootUpUMAClock);
1824 		pi->sys_info.dentist_vco_freq = le32_to_cpu(igp_info->info_7.ulDentistVCOFreq);
1825 		pi->sys_info.bootup_nb_voltage_index =
1826 			le16_to_cpu(igp_info->info_7.usBootUpNBVoltage);
1827 		if (igp_info->info_7.ucHtcTmpLmt == 0)
1828 			pi->sys_info.htc_tmp_lmt = 203;
1829 		else
1830 			pi->sys_info.htc_tmp_lmt = igp_info->info_7.ucHtcTmpLmt;
1831 		if (igp_info->info_7.ucHtcHystLmt == 0)
1832 			pi->sys_info.htc_hyst_lmt = 5;
1833 		else
1834 			pi->sys_info.htc_hyst_lmt = igp_info->info_7.ucHtcHystLmt;
1835 		if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
1836 			DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
1837 		}
1838 
1839 		if (pi->enable_nbps_policy)
1840 			pi->sys_info.nb_dpm_enable = igp_info->info_7.ucNBDPMEnable;
1841 		else
1842 			pi->sys_info.nb_dpm_enable = 0;
1843 
1844 		for (i = 0; i < TRINITY_NUM_NBPSTATES; i++) {
1845 			pi->sys_info.nbp_mclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateMemclkFreq[i]);
1846 			pi->sys_info.nbp_nclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateNClkFreq[i]);
1847 		}
1848 
1849 		pi->sys_info.nbp_voltage_index[0] = le16_to_cpu(igp_info->info_7.usNBP0Voltage);
1850 		pi->sys_info.nbp_voltage_index[1] = le16_to_cpu(igp_info->info_7.usNBP1Voltage);
1851 		pi->sys_info.nbp_voltage_index[2] = le16_to_cpu(igp_info->info_7.usNBP2Voltage);
1852 		pi->sys_info.nbp_voltage_index[3] = le16_to_cpu(igp_info->info_7.usNBP3Voltage);
1853 
1854 		if (!pi->sys_info.nb_dpm_enable) {
1855 			for (i = 1; i < TRINITY_NUM_NBPSTATES; i++) {
1856 				pi->sys_info.nbp_mclk[i] = pi->sys_info.nbp_mclk[0];
1857 				pi->sys_info.nbp_nclk[i] = pi->sys_info.nbp_nclk[0];
1858 				pi->sys_info.nbp_voltage_index[i] = pi->sys_info.nbp_voltage_index[0];
1859 			}
1860 		}
1861 
1862 		pi->sys_info.uma_channel_number = igp_info->info_7.ucUMAChannelNumber;
1863 
1864 		sumo_construct_sclk_voltage_mapping_table(rdev,
1865 							  &pi->sys_info.sclk_voltage_mapping_table,
1866 							  igp_info->info_7.sAvail_SCLK);
1867 		sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table,
1868 						 igp_info->info_7.sAvail_SCLK);
1869 
1870 		pi->sys_info.uvd_clock_table_entries[0].vclk_did =
1871 			igp_info->info_7.ucDPMState0VclkFid;
1872 		pi->sys_info.uvd_clock_table_entries[1].vclk_did =
1873 			igp_info->info_7.ucDPMState1VclkFid;
1874 		pi->sys_info.uvd_clock_table_entries[2].vclk_did =
1875 			igp_info->info_7.ucDPMState2VclkFid;
1876 		pi->sys_info.uvd_clock_table_entries[3].vclk_did =
1877 			igp_info->info_7.ucDPMState3VclkFid;
1878 
1879 		pi->sys_info.uvd_clock_table_entries[0].dclk_did =
1880 			igp_info->info_7.ucDPMState0DclkFid;
1881 		pi->sys_info.uvd_clock_table_entries[1].dclk_did =
1882 			igp_info->info_7.ucDPMState1DclkFid;
1883 		pi->sys_info.uvd_clock_table_entries[2].dclk_did =
1884 			igp_info->info_7.ucDPMState2DclkFid;
1885 		pi->sys_info.uvd_clock_table_entries[3].dclk_did =
1886 			igp_info->info_7.ucDPMState3DclkFid;
1887 
1888 		for (i = 0; i < 4; i++) {
1889 			pi->sys_info.uvd_clock_table_entries[i].vclk =
1890 				trinity_convert_did_to_freq(rdev,
1891 							    pi->sys_info.uvd_clock_table_entries[i].vclk_did);
1892 			pi->sys_info.uvd_clock_table_entries[i].dclk =
1893 				trinity_convert_did_to_freq(rdev,
1894 							    pi->sys_info.uvd_clock_table_entries[i].dclk_did);
1895 		}
1896 
1897 
1898 
1899 	}
1900 	return 0;
1901 }
1902 
1903 int trinity_dpm_init(struct radeon_device *rdev)
1904 {
1905 	struct trinity_power_info *pi;
1906 	int ret, i;
1907 
1908 	pi = kzalloc(sizeof(struct trinity_power_info), GFP_KERNEL);
1909 	if (pi == NULL)
1910 		return -ENOMEM;
1911 	rdev->pm.dpm.priv = pi;
1912 
1913 	for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++)
1914 		pi->at[i] = TRINITY_AT_DFLT;
1915 
1916 	if (radeon_bapm == -1) {
1917 		/* There are stability issues reported on with
1918 		 * bapm enabled when switching between AC and battery
1919 		 * power.  At the same time, some MSI boards hang
1920 		 * if it's not enabled and dpm is enabled.  Just enable
1921 		 * it for MSI boards right now.
1922 		 */
1923 		if (rdev->pdev->subsystem_vendor == 0x1462)
1924 			pi->enable_bapm = true;
1925 		else
1926 			pi->enable_bapm = false;
1927 	} else if (radeon_bapm == 0) {
1928 		pi->enable_bapm = false;
1929 	} else {
1930 		pi->enable_bapm = true;
1931 	}
1932 	pi->enable_nbps_policy = true;
1933 	pi->enable_sclk_ds = true;
1934 	pi->enable_gfx_power_gating = true;
1935 	pi->enable_gfx_clock_gating = true;
1936 	pi->enable_mg_clock_gating = false;
1937 	pi->enable_gfx_dynamic_mgpg = false;
1938 	pi->override_dynamic_mgpg = false;
1939 	pi->enable_auto_thermal_throttling = true;
1940 	pi->voltage_drop_in_dce = false; /* need to restructure dpm/modeset interaction */
1941 	pi->uvd_dpm = true; /* ??? */
1942 
1943 	ret = trinity_parse_sys_info_table(rdev);
1944 	if (ret)
1945 		return ret;
1946 
1947 	trinity_construct_boot_state(rdev);
1948 
1949 	ret = r600_get_platform_caps(rdev);
1950 	if (ret)
1951 		return ret;
1952 
1953 	ret = r600_parse_extended_power_table(rdev);
1954 	if (ret)
1955 		return ret;
1956 
1957 	ret = trinity_parse_power_table(rdev);
1958 	if (ret)
1959 		return ret;
1960 
1961 	pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt;
1962 	pi->enable_dpm = true;
1963 
1964 	return 0;
1965 }
1966 
1967 void trinity_dpm_print_power_state(struct radeon_device *rdev,
1968 				   struct radeon_ps *rps)
1969 {
1970 	int i;
1971 	struct trinity_ps *ps = trinity_get_ps(rps);
1972 
1973 	r600_dpm_print_class_info(rps->class, rps->class2);
1974 	r600_dpm_print_cap_info(rps->caps);
1975 	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
1976 	for (i = 0; i < ps->num_levels; i++) {
1977 		struct trinity_pl *pl = &ps->levels[i];
1978 		printk("\t\tpower level %d    sclk: %u vddc: %u\n",
1979 		       i, pl->sclk,
1980 		       trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
1981 	}
1982 	r600_dpm_print_ps_status(rdev, rps);
1983 }
1984 
1985 void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
1986 							 struct seq_file *m)
1987 {
1988 	struct trinity_power_info *pi = trinity_get_pi(rdev);
1989 	struct radeon_ps *rps = &pi->current_rps;
1990 	struct trinity_ps *ps = trinity_get_ps(rps);
1991 	struct trinity_pl *pl;
1992 	u32 current_index =
1993 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
1994 		CURRENT_STATE_SHIFT;
1995 
1996 	if (current_index >= ps->num_levels) {
1997 		seq_printf(m, "invalid dpm profile %d\n", current_index);
1998 	} else {
1999 		pl = &ps->levels[current_index];
2000 		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2001 		seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
2002 			   current_index, pl->sclk,
2003 			   trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
2004 	}
2005 }
2006 
2007 u32 trinity_dpm_get_current_sclk(struct radeon_device *rdev)
2008 {
2009 	struct trinity_power_info *pi = trinity_get_pi(rdev);
2010 	struct radeon_ps *rps = &pi->current_rps;
2011 	struct trinity_ps *ps = trinity_get_ps(rps);
2012 	struct trinity_pl *pl;
2013 	u32 current_index =
2014 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
2015 		CURRENT_STATE_SHIFT;
2016 
2017 	if (current_index >= ps->num_levels) {
2018 		return 0;
2019 	} else {
2020 		pl = &ps->levels[current_index];
2021 		return pl->sclk;
2022 	}
2023 }
2024 
2025 u32 trinity_dpm_get_current_mclk(struct radeon_device *rdev)
2026 {
2027 	struct trinity_power_info *pi = trinity_get_pi(rdev);
2028 
2029 	return pi->sys_info.bootup_uma_clk;
2030 }
2031 
2032 void trinity_dpm_fini(struct radeon_device *rdev)
2033 {
2034 	int i;
2035 
2036 	trinity_cleanup_asic(rdev); /* ??? */
2037 
2038 	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2039 		kfree(rdev->pm.dpm.ps[i].ps_priv);
2040 	}
2041 	kfree(rdev->pm.dpm.ps);
2042 	kfree(rdev->pm.dpm.priv);
2043 	r600_free_extended_power_table(rdev);
2044 }
2045 
2046 u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low)
2047 {
2048 	struct trinity_power_info *pi = trinity_get_pi(rdev);
2049 	struct trinity_ps *requested_state = trinity_get_ps(&pi->requested_rps);
2050 
2051 	if (low)
2052 		return requested_state->levels[0].sclk;
2053 	else
2054 		return requested_state->levels[requested_state->num_levels - 1].sclk;
2055 }
2056 
2057 u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low)
2058 {
2059 	struct trinity_power_info *pi = trinity_get_pi(rdev);
2060 
2061 	return pi->sys_info.bootup_uma_clk;
2062 }
2063