xref: /openbmc/linux/drivers/gpu/drm/radeon/ni_dpm.c (revision f052febd)
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/math64.h>
25 #include <linux/pci.h>
26 #include <linux/seq_file.h>
27 
28 #include "atom.h"
29 #include "evergreen.h"
30 #include "ni_dpm.h"
31 #include "nid.h"
32 #include "r600_dpm.h"
33 #include "rv770.h"
34 #include "radeon.h"
35 #include "radeon_asic.h"
36 
37 #define MC_CG_ARB_FREQ_F0           0x0a
38 #define MC_CG_ARB_FREQ_F1           0x0b
39 #define MC_CG_ARB_FREQ_F2           0x0c
40 #define MC_CG_ARB_FREQ_F3           0x0d
41 
42 #define SMC_RAM_END 0xC000
43 
44 static const struct ni_cac_weights cac_weights_cayman_xt =
45 {
46 	0x15,
47 	0x2,
48 	0x19,
49 	0x2,
50 	0x8,
51 	0x14,
52 	0x2,
53 	0x16,
54 	0xE,
55 	0x17,
56 	0x13,
57 	0x2B,
58 	0x10,
59 	0x7,
60 	0x5,
61 	0x5,
62 	0x5,
63 	0x2,
64 	0x3,
65 	0x9,
66 	0x10,
67 	0x10,
68 	0x2B,
69 	0xA,
70 	0x9,
71 	0x4,
72 	0xD,
73 	0xD,
74 	0x3E,
75 	0x18,
76 	0x14,
77 	0,
78 	0x3,
79 	0x3,
80 	0x5,
81 	0,
82 	0x2,
83 	0,
84 	0,
85 	0,
86 	0,
87 	0,
88 	0,
89 	0,
90 	0,
91 	0,
92 	0x1CC,
93 	0,
94 	0x164,
95 	1,
96 	1,
97 	1,
98 	1,
99 	12,
100 	12,
101 	12,
102 	0x12,
103 	0x1F,
104 	132,
105 	5,
106 	7,
107 	0,
108 	{ 0, 0, 0, 0, 0, 0, 0, 0 },
109 	{ 0, 0, 0, 0 },
110 	true
111 };
112 
113 static const struct ni_cac_weights cac_weights_cayman_pro =
114 {
115 	0x16,
116 	0x4,
117 	0x10,
118 	0x2,
119 	0xA,
120 	0x16,
121 	0x2,
122 	0x18,
123 	0x10,
124 	0x1A,
125 	0x16,
126 	0x2D,
127 	0x12,
128 	0xA,
129 	0x6,
130 	0x6,
131 	0x6,
132 	0x2,
133 	0x4,
134 	0xB,
135 	0x11,
136 	0x11,
137 	0x2D,
138 	0xC,
139 	0xC,
140 	0x7,
141 	0x10,
142 	0x10,
143 	0x3F,
144 	0x1A,
145 	0x16,
146 	0,
147 	0x7,
148 	0x4,
149 	0x6,
150 	1,
151 	0x2,
152 	0x1,
153 	0,
154 	0,
155 	0,
156 	0,
157 	0,
158 	0,
159 	0x30,
160 	0,
161 	0x1CF,
162 	0,
163 	0x166,
164 	1,
165 	1,
166 	1,
167 	1,
168 	12,
169 	12,
170 	12,
171 	0x15,
172 	0x1F,
173 	132,
174 	6,
175 	6,
176 	0,
177 	{ 0, 0, 0, 0, 0, 0, 0, 0 },
178 	{ 0, 0, 0, 0 },
179 	true
180 };
181 
182 static const struct ni_cac_weights cac_weights_cayman_le =
183 {
184 	0x7,
185 	0xE,
186 	0x1,
187 	0xA,
188 	0x1,
189 	0x3F,
190 	0x2,
191 	0x18,
192 	0x10,
193 	0x1A,
194 	0x1,
195 	0x3F,
196 	0x1,
197 	0xE,
198 	0x6,
199 	0x6,
200 	0x6,
201 	0x2,
202 	0x4,
203 	0x9,
204 	0x1A,
205 	0x1A,
206 	0x2C,
207 	0xA,
208 	0x11,
209 	0x8,
210 	0x19,
211 	0x19,
212 	0x1,
213 	0x1,
214 	0x1A,
215 	0,
216 	0x8,
217 	0x5,
218 	0x8,
219 	0x1,
220 	0x3,
221 	0x1,
222 	0,
223 	0,
224 	0,
225 	0,
226 	0,
227 	0,
228 	0x38,
229 	0x38,
230 	0x239,
231 	0x3,
232 	0x18A,
233 	1,
234 	1,
235 	1,
236 	1,
237 	12,
238 	12,
239 	12,
240 	0x15,
241 	0x22,
242 	132,
243 	6,
244 	6,
245 	0,
246 	{ 0, 0, 0, 0, 0, 0, 0, 0 },
247 	{ 0, 0, 0, 0 },
248 	true
249 };
250 
251 #define NISLANDS_MGCG_SEQUENCE  300
252 
253 static const u32 cayman_cgcg_cgls_default[] =
254 {
255 	0x000008f8, 0x00000010, 0xffffffff,
256 	0x000008fc, 0x00000000, 0xffffffff,
257 	0x000008f8, 0x00000011, 0xffffffff,
258 	0x000008fc, 0x00000000, 0xffffffff,
259 	0x000008f8, 0x00000012, 0xffffffff,
260 	0x000008fc, 0x00000000, 0xffffffff,
261 	0x000008f8, 0x00000013, 0xffffffff,
262 	0x000008fc, 0x00000000, 0xffffffff,
263 	0x000008f8, 0x00000014, 0xffffffff,
264 	0x000008fc, 0x00000000, 0xffffffff,
265 	0x000008f8, 0x00000015, 0xffffffff,
266 	0x000008fc, 0x00000000, 0xffffffff,
267 	0x000008f8, 0x00000016, 0xffffffff,
268 	0x000008fc, 0x00000000, 0xffffffff,
269 	0x000008f8, 0x00000017, 0xffffffff,
270 	0x000008fc, 0x00000000, 0xffffffff,
271 	0x000008f8, 0x00000018, 0xffffffff,
272 	0x000008fc, 0x00000000, 0xffffffff,
273 	0x000008f8, 0x00000019, 0xffffffff,
274 	0x000008fc, 0x00000000, 0xffffffff,
275 	0x000008f8, 0x0000001a, 0xffffffff,
276 	0x000008fc, 0x00000000, 0xffffffff,
277 	0x000008f8, 0x0000001b, 0xffffffff,
278 	0x000008fc, 0x00000000, 0xffffffff,
279 	0x000008f8, 0x00000020, 0xffffffff,
280 	0x000008fc, 0x00000000, 0xffffffff,
281 	0x000008f8, 0x00000021, 0xffffffff,
282 	0x000008fc, 0x00000000, 0xffffffff,
283 	0x000008f8, 0x00000022, 0xffffffff,
284 	0x000008fc, 0x00000000, 0xffffffff,
285 	0x000008f8, 0x00000023, 0xffffffff,
286 	0x000008fc, 0x00000000, 0xffffffff,
287 	0x000008f8, 0x00000024, 0xffffffff,
288 	0x000008fc, 0x00000000, 0xffffffff,
289 	0x000008f8, 0x00000025, 0xffffffff,
290 	0x000008fc, 0x00000000, 0xffffffff,
291 	0x000008f8, 0x00000026, 0xffffffff,
292 	0x000008fc, 0x00000000, 0xffffffff,
293 	0x000008f8, 0x00000027, 0xffffffff,
294 	0x000008fc, 0x00000000, 0xffffffff,
295 	0x000008f8, 0x00000028, 0xffffffff,
296 	0x000008fc, 0x00000000, 0xffffffff,
297 	0x000008f8, 0x00000029, 0xffffffff,
298 	0x000008fc, 0x00000000, 0xffffffff,
299 	0x000008f8, 0x0000002a, 0xffffffff,
300 	0x000008fc, 0x00000000, 0xffffffff,
301 	0x000008f8, 0x0000002b, 0xffffffff,
302 	0x000008fc, 0x00000000, 0xffffffff
303 };
304 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
305 
306 static const u32 cayman_cgcg_cgls_disable[] =
307 {
308 	0x000008f8, 0x00000010, 0xffffffff,
309 	0x000008fc, 0xffffffff, 0xffffffff,
310 	0x000008f8, 0x00000011, 0xffffffff,
311 	0x000008fc, 0xffffffff, 0xffffffff,
312 	0x000008f8, 0x00000012, 0xffffffff,
313 	0x000008fc, 0xffffffff, 0xffffffff,
314 	0x000008f8, 0x00000013, 0xffffffff,
315 	0x000008fc, 0xffffffff, 0xffffffff,
316 	0x000008f8, 0x00000014, 0xffffffff,
317 	0x000008fc, 0xffffffff, 0xffffffff,
318 	0x000008f8, 0x00000015, 0xffffffff,
319 	0x000008fc, 0xffffffff, 0xffffffff,
320 	0x000008f8, 0x00000016, 0xffffffff,
321 	0x000008fc, 0xffffffff, 0xffffffff,
322 	0x000008f8, 0x00000017, 0xffffffff,
323 	0x000008fc, 0xffffffff, 0xffffffff,
324 	0x000008f8, 0x00000018, 0xffffffff,
325 	0x000008fc, 0xffffffff, 0xffffffff,
326 	0x000008f8, 0x00000019, 0xffffffff,
327 	0x000008fc, 0xffffffff, 0xffffffff,
328 	0x000008f8, 0x0000001a, 0xffffffff,
329 	0x000008fc, 0xffffffff, 0xffffffff,
330 	0x000008f8, 0x0000001b, 0xffffffff,
331 	0x000008fc, 0xffffffff, 0xffffffff,
332 	0x000008f8, 0x00000020, 0xffffffff,
333 	0x000008fc, 0x00000000, 0xffffffff,
334 	0x000008f8, 0x00000021, 0xffffffff,
335 	0x000008fc, 0x00000000, 0xffffffff,
336 	0x000008f8, 0x00000022, 0xffffffff,
337 	0x000008fc, 0x00000000, 0xffffffff,
338 	0x000008f8, 0x00000023, 0xffffffff,
339 	0x000008fc, 0x00000000, 0xffffffff,
340 	0x000008f8, 0x00000024, 0xffffffff,
341 	0x000008fc, 0x00000000, 0xffffffff,
342 	0x000008f8, 0x00000025, 0xffffffff,
343 	0x000008fc, 0x00000000, 0xffffffff,
344 	0x000008f8, 0x00000026, 0xffffffff,
345 	0x000008fc, 0x00000000, 0xffffffff,
346 	0x000008f8, 0x00000027, 0xffffffff,
347 	0x000008fc, 0x00000000, 0xffffffff,
348 	0x000008f8, 0x00000028, 0xffffffff,
349 	0x000008fc, 0x00000000, 0xffffffff,
350 	0x000008f8, 0x00000029, 0xffffffff,
351 	0x000008fc, 0x00000000, 0xffffffff,
352 	0x000008f8, 0x0000002a, 0xffffffff,
353 	0x000008fc, 0x00000000, 0xffffffff,
354 	0x000008f8, 0x0000002b, 0xffffffff,
355 	0x000008fc, 0x00000000, 0xffffffff,
356 	0x00000644, 0x000f7902, 0x001f4180,
357 	0x00000644, 0x000f3802, 0x001f4180
358 };
359 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
360 
361 static const u32 cayman_cgcg_cgls_enable[] =
362 {
363 	0x00000644, 0x000f7882, 0x001f4080,
364 	0x000008f8, 0x00000010, 0xffffffff,
365 	0x000008fc, 0x00000000, 0xffffffff,
366 	0x000008f8, 0x00000011, 0xffffffff,
367 	0x000008fc, 0x00000000, 0xffffffff,
368 	0x000008f8, 0x00000012, 0xffffffff,
369 	0x000008fc, 0x00000000, 0xffffffff,
370 	0x000008f8, 0x00000013, 0xffffffff,
371 	0x000008fc, 0x00000000, 0xffffffff,
372 	0x000008f8, 0x00000014, 0xffffffff,
373 	0x000008fc, 0x00000000, 0xffffffff,
374 	0x000008f8, 0x00000015, 0xffffffff,
375 	0x000008fc, 0x00000000, 0xffffffff,
376 	0x000008f8, 0x00000016, 0xffffffff,
377 	0x000008fc, 0x00000000, 0xffffffff,
378 	0x000008f8, 0x00000017, 0xffffffff,
379 	0x000008fc, 0x00000000, 0xffffffff,
380 	0x000008f8, 0x00000018, 0xffffffff,
381 	0x000008fc, 0x00000000, 0xffffffff,
382 	0x000008f8, 0x00000019, 0xffffffff,
383 	0x000008fc, 0x00000000, 0xffffffff,
384 	0x000008f8, 0x0000001a, 0xffffffff,
385 	0x000008fc, 0x00000000, 0xffffffff,
386 	0x000008f8, 0x0000001b, 0xffffffff,
387 	0x000008fc, 0x00000000, 0xffffffff,
388 	0x000008f8, 0x00000020, 0xffffffff,
389 	0x000008fc, 0xffffffff, 0xffffffff,
390 	0x000008f8, 0x00000021, 0xffffffff,
391 	0x000008fc, 0xffffffff, 0xffffffff,
392 	0x000008f8, 0x00000022, 0xffffffff,
393 	0x000008fc, 0xffffffff, 0xffffffff,
394 	0x000008f8, 0x00000023, 0xffffffff,
395 	0x000008fc, 0xffffffff, 0xffffffff,
396 	0x000008f8, 0x00000024, 0xffffffff,
397 	0x000008fc, 0xffffffff, 0xffffffff,
398 	0x000008f8, 0x00000025, 0xffffffff,
399 	0x000008fc, 0xffffffff, 0xffffffff,
400 	0x000008f8, 0x00000026, 0xffffffff,
401 	0x000008fc, 0xffffffff, 0xffffffff,
402 	0x000008f8, 0x00000027, 0xffffffff,
403 	0x000008fc, 0xffffffff, 0xffffffff,
404 	0x000008f8, 0x00000028, 0xffffffff,
405 	0x000008fc, 0xffffffff, 0xffffffff,
406 	0x000008f8, 0x00000029, 0xffffffff,
407 	0x000008fc, 0xffffffff, 0xffffffff,
408 	0x000008f8, 0x0000002a, 0xffffffff,
409 	0x000008fc, 0xffffffff, 0xffffffff,
410 	0x000008f8, 0x0000002b, 0xffffffff,
411 	0x000008fc, 0xffffffff, 0xffffffff
412 };
413 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
414 
415 static const u32 cayman_mgcg_default[] =
416 {
417 	0x0000802c, 0xc0000000, 0xffffffff,
418 	0x00003fc4, 0xc0000000, 0xffffffff,
419 	0x00005448, 0x00000100, 0xffffffff,
420 	0x000055e4, 0x00000100, 0xffffffff,
421 	0x0000160c, 0x00000100, 0xffffffff,
422 	0x00008984, 0x06000100, 0xffffffff,
423 	0x0000c164, 0x00000100, 0xffffffff,
424 	0x00008a18, 0x00000100, 0xffffffff,
425 	0x0000897c, 0x06000100, 0xffffffff,
426 	0x00008b28, 0x00000100, 0xffffffff,
427 	0x00009144, 0x00800200, 0xffffffff,
428 	0x00009a60, 0x00000100, 0xffffffff,
429 	0x00009868, 0x00000100, 0xffffffff,
430 	0x00008d58, 0x00000100, 0xffffffff,
431 	0x00009510, 0x00000100, 0xffffffff,
432 	0x0000949c, 0x00000100, 0xffffffff,
433 	0x00009654, 0x00000100, 0xffffffff,
434 	0x00009030, 0x00000100, 0xffffffff,
435 	0x00009034, 0x00000100, 0xffffffff,
436 	0x00009038, 0x00000100, 0xffffffff,
437 	0x0000903c, 0x00000100, 0xffffffff,
438 	0x00009040, 0x00000100, 0xffffffff,
439 	0x0000a200, 0x00000100, 0xffffffff,
440 	0x0000a204, 0x00000100, 0xffffffff,
441 	0x0000a208, 0x00000100, 0xffffffff,
442 	0x0000a20c, 0x00000100, 0xffffffff,
443 	0x00009744, 0x00000100, 0xffffffff,
444 	0x00003f80, 0x00000100, 0xffffffff,
445 	0x0000a210, 0x00000100, 0xffffffff,
446 	0x0000a214, 0x00000100, 0xffffffff,
447 	0x000004d8, 0x00000100, 0xffffffff,
448 	0x00009664, 0x00000100, 0xffffffff,
449 	0x00009698, 0x00000100, 0xffffffff,
450 	0x000004d4, 0x00000200, 0xffffffff,
451 	0x000004d0, 0x00000000, 0xffffffff,
452 	0x000030cc, 0x00000104, 0xffffffff,
453 	0x0000d0c0, 0x00000100, 0xffffffff,
454 	0x0000d8c0, 0x00000100, 0xffffffff,
455 	0x0000802c, 0x40000000, 0xffffffff,
456 	0x00003fc4, 0x40000000, 0xffffffff,
457 	0x0000915c, 0x00010000, 0xffffffff,
458 	0x00009160, 0x00030002, 0xffffffff,
459 	0x00009164, 0x00050004, 0xffffffff,
460 	0x00009168, 0x00070006, 0xffffffff,
461 	0x00009178, 0x00070000, 0xffffffff,
462 	0x0000917c, 0x00030002, 0xffffffff,
463 	0x00009180, 0x00050004, 0xffffffff,
464 	0x0000918c, 0x00010006, 0xffffffff,
465 	0x00009190, 0x00090008, 0xffffffff,
466 	0x00009194, 0x00070000, 0xffffffff,
467 	0x00009198, 0x00030002, 0xffffffff,
468 	0x0000919c, 0x00050004, 0xffffffff,
469 	0x000091a8, 0x00010006, 0xffffffff,
470 	0x000091ac, 0x00090008, 0xffffffff,
471 	0x000091b0, 0x00070000, 0xffffffff,
472 	0x000091b4, 0x00030002, 0xffffffff,
473 	0x000091b8, 0x00050004, 0xffffffff,
474 	0x000091c4, 0x00010006, 0xffffffff,
475 	0x000091c8, 0x00090008, 0xffffffff,
476 	0x000091cc, 0x00070000, 0xffffffff,
477 	0x000091d0, 0x00030002, 0xffffffff,
478 	0x000091d4, 0x00050004, 0xffffffff,
479 	0x000091e0, 0x00010006, 0xffffffff,
480 	0x000091e4, 0x00090008, 0xffffffff,
481 	0x000091e8, 0x00000000, 0xffffffff,
482 	0x000091ec, 0x00070000, 0xffffffff,
483 	0x000091f0, 0x00030002, 0xffffffff,
484 	0x000091f4, 0x00050004, 0xffffffff,
485 	0x00009200, 0x00010006, 0xffffffff,
486 	0x00009204, 0x00090008, 0xffffffff,
487 	0x00009208, 0x00070000, 0xffffffff,
488 	0x0000920c, 0x00030002, 0xffffffff,
489 	0x00009210, 0x00050004, 0xffffffff,
490 	0x0000921c, 0x00010006, 0xffffffff,
491 	0x00009220, 0x00090008, 0xffffffff,
492 	0x00009224, 0x00070000, 0xffffffff,
493 	0x00009228, 0x00030002, 0xffffffff,
494 	0x0000922c, 0x00050004, 0xffffffff,
495 	0x00009238, 0x00010006, 0xffffffff,
496 	0x0000923c, 0x00090008, 0xffffffff,
497 	0x00009240, 0x00070000, 0xffffffff,
498 	0x00009244, 0x00030002, 0xffffffff,
499 	0x00009248, 0x00050004, 0xffffffff,
500 	0x00009254, 0x00010006, 0xffffffff,
501 	0x00009258, 0x00090008, 0xffffffff,
502 	0x0000925c, 0x00070000, 0xffffffff,
503 	0x00009260, 0x00030002, 0xffffffff,
504 	0x00009264, 0x00050004, 0xffffffff,
505 	0x00009270, 0x00010006, 0xffffffff,
506 	0x00009274, 0x00090008, 0xffffffff,
507 	0x00009278, 0x00070000, 0xffffffff,
508 	0x0000927c, 0x00030002, 0xffffffff,
509 	0x00009280, 0x00050004, 0xffffffff,
510 	0x0000928c, 0x00010006, 0xffffffff,
511 	0x00009290, 0x00090008, 0xffffffff,
512 	0x000092a8, 0x00070000, 0xffffffff,
513 	0x000092ac, 0x00030002, 0xffffffff,
514 	0x000092b0, 0x00050004, 0xffffffff,
515 	0x000092bc, 0x00010006, 0xffffffff,
516 	0x000092c0, 0x00090008, 0xffffffff,
517 	0x000092c4, 0x00070000, 0xffffffff,
518 	0x000092c8, 0x00030002, 0xffffffff,
519 	0x000092cc, 0x00050004, 0xffffffff,
520 	0x000092d8, 0x00010006, 0xffffffff,
521 	0x000092dc, 0x00090008, 0xffffffff,
522 	0x00009294, 0x00000000, 0xffffffff,
523 	0x0000802c, 0x40010000, 0xffffffff,
524 	0x00003fc4, 0x40010000, 0xffffffff,
525 	0x0000915c, 0x00010000, 0xffffffff,
526 	0x00009160, 0x00030002, 0xffffffff,
527 	0x00009164, 0x00050004, 0xffffffff,
528 	0x00009168, 0x00070006, 0xffffffff,
529 	0x00009178, 0x00070000, 0xffffffff,
530 	0x0000917c, 0x00030002, 0xffffffff,
531 	0x00009180, 0x00050004, 0xffffffff,
532 	0x0000918c, 0x00010006, 0xffffffff,
533 	0x00009190, 0x00090008, 0xffffffff,
534 	0x00009194, 0x00070000, 0xffffffff,
535 	0x00009198, 0x00030002, 0xffffffff,
536 	0x0000919c, 0x00050004, 0xffffffff,
537 	0x000091a8, 0x00010006, 0xffffffff,
538 	0x000091ac, 0x00090008, 0xffffffff,
539 	0x000091b0, 0x00070000, 0xffffffff,
540 	0x000091b4, 0x00030002, 0xffffffff,
541 	0x000091b8, 0x00050004, 0xffffffff,
542 	0x000091c4, 0x00010006, 0xffffffff,
543 	0x000091c8, 0x00090008, 0xffffffff,
544 	0x000091cc, 0x00070000, 0xffffffff,
545 	0x000091d0, 0x00030002, 0xffffffff,
546 	0x000091d4, 0x00050004, 0xffffffff,
547 	0x000091e0, 0x00010006, 0xffffffff,
548 	0x000091e4, 0x00090008, 0xffffffff,
549 	0x000091e8, 0x00000000, 0xffffffff,
550 	0x000091ec, 0x00070000, 0xffffffff,
551 	0x000091f0, 0x00030002, 0xffffffff,
552 	0x000091f4, 0x00050004, 0xffffffff,
553 	0x00009200, 0x00010006, 0xffffffff,
554 	0x00009204, 0x00090008, 0xffffffff,
555 	0x00009208, 0x00070000, 0xffffffff,
556 	0x0000920c, 0x00030002, 0xffffffff,
557 	0x00009210, 0x00050004, 0xffffffff,
558 	0x0000921c, 0x00010006, 0xffffffff,
559 	0x00009220, 0x00090008, 0xffffffff,
560 	0x00009224, 0x00070000, 0xffffffff,
561 	0x00009228, 0x00030002, 0xffffffff,
562 	0x0000922c, 0x00050004, 0xffffffff,
563 	0x00009238, 0x00010006, 0xffffffff,
564 	0x0000923c, 0x00090008, 0xffffffff,
565 	0x00009240, 0x00070000, 0xffffffff,
566 	0x00009244, 0x00030002, 0xffffffff,
567 	0x00009248, 0x00050004, 0xffffffff,
568 	0x00009254, 0x00010006, 0xffffffff,
569 	0x00009258, 0x00090008, 0xffffffff,
570 	0x0000925c, 0x00070000, 0xffffffff,
571 	0x00009260, 0x00030002, 0xffffffff,
572 	0x00009264, 0x00050004, 0xffffffff,
573 	0x00009270, 0x00010006, 0xffffffff,
574 	0x00009274, 0x00090008, 0xffffffff,
575 	0x00009278, 0x00070000, 0xffffffff,
576 	0x0000927c, 0x00030002, 0xffffffff,
577 	0x00009280, 0x00050004, 0xffffffff,
578 	0x0000928c, 0x00010006, 0xffffffff,
579 	0x00009290, 0x00090008, 0xffffffff,
580 	0x000092a8, 0x00070000, 0xffffffff,
581 	0x000092ac, 0x00030002, 0xffffffff,
582 	0x000092b0, 0x00050004, 0xffffffff,
583 	0x000092bc, 0x00010006, 0xffffffff,
584 	0x000092c0, 0x00090008, 0xffffffff,
585 	0x000092c4, 0x00070000, 0xffffffff,
586 	0x000092c8, 0x00030002, 0xffffffff,
587 	0x000092cc, 0x00050004, 0xffffffff,
588 	0x000092d8, 0x00010006, 0xffffffff,
589 	0x000092dc, 0x00090008, 0xffffffff,
590 	0x00009294, 0x00000000, 0xffffffff,
591 	0x0000802c, 0xc0000000, 0xffffffff,
592 	0x00003fc4, 0xc0000000, 0xffffffff,
593 	0x000008f8, 0x00000010, 0xffffffff,
594 	0x000008fc, 0x00000000, 0xffffffff,
595 	0x000008f8, 0x00000011, 0xffffffff,
596 	0x000008fc, 0x00000000, 0xffffffff,
597 	0x000008f8, 0x00000012, 0xffffffff,
598 	0x000008fc, 0x00000000, 0xffffffff,
599 	0x000008f8, 0x00000013, 0xffffffff,
600 	0x000008fc, 0x00000000, 0xffffffff,
601 	0x000008f8, 0x00000014, 0xffffffff,
602 	0x000008fc, 0x00000000, 0xffffffff,
603 	0x000008f8, 0x00000015, 0xffffffff,
604 	0x000008fc, 0x00000000, 0xffffffff,
605 	0x000008f8, 0x00000016, 0xffffffff,
606 	0x000008fc, 0x00000000, 0xffffffff,
607 	0x000008f8, 0x00000017, 0xffffffff,
608 	0x000008fc, 0x00000000, 0xffffffff,
609 	0x000008f8, 0x00000018, 0xffffffff,
610 	0x000008fc, 0x00000000, 0xffffffff,
611 	0x000008f8, 0x00000019, 0xffffffff,
612 	0x000008fc, 0x00000000, 0xffffffff,
613 	0x000008f8, 0x0000001a, 0xffffffff,
614 	0x000008fc, 0x00000000, 0xffffffff,
615 	0x000008f8, 0x0000001b, 0xffffffff,
616 	0x000008fc, 0x00000000, 0xffffffff
617 };
618 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
619 
620 static const u32 cayman_mgcg_disable[] =
621 {
622 	0x0000802c, 0xc0000000, 0xffffffff,
623 	0x000008f8, 0x00000000, 0xffffffff,
624 	0x000008fc, 0xffffffff, 0xffffffff,
625 	0x000008f8, 0x00000001, 0xffffffff,
626 	0x000008fc, 0xffffffff, 0xffffffff,
627 	0x000008f8, 0x00000002, 0xffffffff,
628 	0x000008fc, 0xffffffff, 0xffffffff,
629 	0x000008f8, 0x00000003, 0xffffffff,
630 	0x000008fc, 0xffffffff, 0xffffffff,
631 	0x00009150, 0x00600000, 0xffffffff
632 };
633 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
634 
635 static const u32 cayman_mgcg_enable[] =
636 {
637 	0x0000802c, 0xc0000000, 0xffffffff,
638 	0x000008f8, 0x00000000, 0xffffffff,
639 	0x000008fc, 0x00000000, 0xffffffff,
640 	0x000008f8, 0x00000001, 0xffffffff,
641 	0x000008fc, 0x00000000, 0xffffffff,
642 	0x000008f8, 0x00000002, 0xffffffff,
643 	0x000008fc, 0x00600000, 0xffffffff,
644 	0x000008f8, 0x00000003, 0xffffffff,
645 	0x000008fc, 0x00000000, 0xffffffff,
646 	0x00009150, 0x96944200, 0xffffffff
647 };
648 
649 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
650 
651 #define NISLANDS_SYSLS_SEQUENCE  100
652 
653 static const u32 cayman_sysls_default[] =
654 {
655 	/* Register,   Value,     Mask bits */
656 	0x000055e8, 0x00000000, 0xffffffff,
657 	0x0000d0bc, 0x00000000, 0xffffffff,
658 	0x0000d8bc, 0x00000000, 0xffffffff,
659 	0x000015c0, 0x000c1401, 0xffffffff,
660 	0x0000264c, 0x000c0400, 0xffffffff,
661 	0x00002648, 0x000c0400, 0xffffffff,
662 	0x00002650, 0x000c0400, 0xffffffff,
663 	0x000020b8, 0x000c0400, 0xffffffff,
664 	0x000020bc, 0x000c0400, 0xffffffff,
665 	0x000020c0, 0x000c0c80, 0xffffffff,
666 	0x0000f4a0, 0x000000c0, 0xffffffff,
667 	0x0000f4a4, 0x00680fff, 0xffffffff,
668 	0x00002f50, 0x00000404, 0xffffffff,
669 	0x000004c8, 0x00000001, 0xffffffff,
670 	0x000064ec, 0x00000000, 0xffffffff,
671 	0x00000c7c, 0x00000000, 0xffffffff,
672 	0x00008dfc, 0x00000000, 0xffffffff
673 };
674 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
675 
676 static const u32 cayman_sysls_disable[] =
677 {
678 	/* Register,   Value,     Mask bits */
679 	0x0000d0c0, 0x00000000, 0xffffffff,
680 	0x0000d8c0, 0x00000000, 0xffffffff,
681 	0x000055e8, 0x00000000, 0xffffffff,
682 	0x0000d0bc, 0x00000000, 0xffffffff,
683 	0x0000d8bc, 0x00000000, 0xffffffff,
684 	0x000015c0, 0x00041401, 0xffffffff,
685 	0x0000264c, 0x00040400, 0xffffffff,
686 	0x00002648, 0x00040400, 0xffffffff,
687 	0x00002650, 0x00040400, 0xffffffff,
688 	0x000020b8, 0x00040400, 0xffffffff,
689 	0x000020bc, 0x00040400, 0xffffffff,
690 	0x000020c0, 0x00040c80, 0xffffffff,
691 	0x0000f4a0, 0x000000c0, 0xffffffff,
692 	0x0000f4a4, 0x00680000, 0xffffffff,
693 	0x00002f50, 0x00000404, 0xffffffff,
694 	0x000004c8, 0x00000001, 0xffffffff,
695 	0x000064ec, 0x00007ffd, 0xffffffff,
696 	0x00000c7c, 0x0000ff00, 0xffffffff,
697 	0x00008dfc, 0x0000007f, 0xffffffff
698 };
699 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
700 
701 static const u32 cayman_sysls_enable[] =
702 {
703 	/* Register,   Value,     Mask bits */
704 	0x000055e8, 0x00000001, 0xffffffff,
705 	0x0000d0bc, 0x00000100, 0xffffffff,
706 	0x0000d8bc, 0x00000100, 0xffffffff,
707 	0x000015c0, 0x000c1401, 0xffffffff,
708 	0x0000264c, 0x000c0400, 0xffffffff,
709 	0x00002648, 0x000c0400, 0xffffffff,
710 	0x00002650, 0x000c0400, 0xffffffff,
711 	0x000020b8, 0x000c0400, 0xffffffff,
712 	0x000020bc, 0x000c0400, 0xffffffff,
713 	0x000020c0, 0x000c0c80, 0xffffffff,
714 	0x0000f4a0, 0x000000c0, 0xffffffff,
715 	0x0000f4a4, 0x00680fff, 0xffffffff,
716 	0x00002f50, 0x00000903, 0xffffffff,
717 	0x000004c8, 0x00000000, 0xffffffff,
718 	0x000064ec, 0x00000000, 0xffffffff,
719 	0x00000c7c, 0x00000000, 0xffffffff,
720 	0x00008dfc, 0x00000000, 0xffffffff
721 };
722 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
723 
724 extern int ni_mc_load_microcode(struct radeon_device *rdev);
725 
726 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
727 {
728 	struct ni_power_info *pi = rdev->pm.dpm.priv;
729 
730 	return pi;
731 }
732 
733 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
734 {
735 	struct ni_ps *ps = rps->ps_priv;
736 
737 	return ps;
738 }
739 
740 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
741 						     u16 v, s32 t,
742 						     u32 ileakage,
743 						     u32 *leakage)
744 {
745 	s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
746 
747 	i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
748 	vddc = div64_s64(drm_int2fixp(v), 1000);
749 	temperature = div64_s64(drm_int2fixp(t), 1000);
750 
751 	kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
752 			  drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
753 	kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
754 			  drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
755 
756 	leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
757 
758 	*leakage = drm_fixp2int(leakage_w * 1000);
759 }
760 
761 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
762 					     const struct ni_leakage_coeffients *coeff,
763 					     u16 v,
764 					     s32 t,
765 					     u32 i_leakage,
766 					     u32 *leakage)
767 {
768 	ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
769 }
770 
771 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
772 {
773 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
774 	u32 vblank_time = r600_dpm_get_vblank_time(rdev);
775 	/* we never hit the non-gddr5 limit so disable it */
776 	u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
777 
778 	if (vblank_time < switch_limit)
779 		return true;
780 	else
781 		return false;
782 
783 }
784 
785 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
786 					struct radeon_ps *rps)
787 {
788 	struct ni_ps *ps = ni_get_ps(rps);
789 	struct radeon_clock_and_voltage_limits *max_limits;
790 	bool disable_mclk_switching;
791 	u32 mclk;
792 	u16 vddci;
793 	int i;
794 
795 	if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
796 	    ni_dpm_vblank_too_short(rdev))
797 		disable_mclk_switching = true;
798 	else
799 		disable_mclk_switching = false;
800 
801 	if (rdev->pm.dpm.ac_power)
802 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
803 	else
804 		max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
805 
806 	if (rdev->pm.dpm.ac_power == false) {
807 		for (i = 0; i < ps->performance_level_count; i++) {
808 			if (ps->performance_levels[i].mclk > max_limits->mclk)
809 				ps->performance_levels[i].mclk = max_limits->mclk;
810 			if (ps->performance_levels[i].sclk > max_limits->sclk)
811 				ps->performance_levels[i].sclk = max_limits->sclk;
812 			if (ps->performance_levels[i].vddc > max_limits->vddc)
813 				ps->performance_levels[i].vddc = max_limits->vddc;
814 			if (ps->performance_levels[i].vddci > max_limits->vddci)
815 				ps->performance_levels[i].vddci = max_limits->vddci;
816 		}
817 	}
818 
819 	/* XXX validate the min clocks required for display */
820 
821 	/* adjust low state */
822 	if (disable_mclk_switching) {
823 		ps->performance_levels[0].mclk =
824 			ps->performance_levels[ps->performance_level_count - 1].mclk;
825 		ps->performance_levels[0].vddci =
826 			ps->performance_levels[ps->performance_level_count - 1].vddci;
827 	}
828 
829 	btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
830 				  &ps->performance_levels[0].sclk,
831 				  &ps->performance_levels[0].mclk);
832 
833 	for (i = 1; i < ps->performance_level_count; i++) {
834 		if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
835 			ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
836 		if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
837 			ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
838 	}
839 
840 	/* adjust remaining states */
841 	if (disable_mclk_switching) {
842 		mclk = ps->performance_levels[0].mclk;
843 		vddci = ps->performance_levels[0].vddci;
844 		for (i = 1; i < ps->performance_level_count; i++) {
845 			if (mclk < ps->performance_levels[i].mclk)
846 				mclk = ps->performance_levels[i].mclk;
847 			if (vddci < ps->performance_levels[i].vddci)
848 				vddci = ps->performance_levels[i].vddci;
849 		}
850 		for (i = 0; i < ps->performance_level_count; i++) {
851 			ps->performance_levels[i].mclk = mclk;
852 			ps->performance_levels[i].vddci = vddci;
853 		}
854 	} else {
855 		for (i = 1; i < ps->performance_level_count; i++) {
856 			if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
857 				ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
858 			if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
859 				ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
860 		}
861 	}
862 
863 	for (i = 1; i < ps->performance_level_count; i++)
864 		btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
865 					  &ps->performance_levels[i].sclk,
866 					  &ps->performance_levels[i].mclk);
867 
868 	for (i = 0; i < ps->performance_level_count; i++)
869 		btc_adjust_clock_combinations(rdev, max_limits,
870 					      &ps->performance_levels[i]);
871 
872 	for (i = 0; i < ps->performance_level_count; i++) {
873 		btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
874 						   ps->performance_levels[i].sclk,
875 						   max_limits->vddc,  &ps->performance_levels[i].vddc);
876 		btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
877 						   ps->performance_levels[i].mclk,
878 						   max_limits->vddci, &ps->performance_levels[i].vddci);
879 		btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
880 						   ps->performance_levels[i].mclk,
881 						   max_limits->vddc,  &ps->performance_levels[i].vddc);
882 		btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
883 						   rdev->clock.current_dispclk,
884 						   max_limits->vddc,  &ps->performance_levels[i].vddc);
885 	}
886 
887 	for (i = 0; i < ps->performance_level_count; i++) {
888 		btc_apply_voltage_delta_rules(rdev,
889 					      max_limits->vddc, max_limits->vddci,
890 					      &ps->performance_levels[i].vddc,
891 					      &ps->performance_levels[i].vddci);
892 	}
893 
894 	ps->dc_compatible = true;
895 	for (i = 0; i < ps->performance_level_count; i++) {
896 		if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
897 			ps->dc_compatible = false;
898 
899 		if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
900 			ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
901 	}
902 }
903 
904 static void ni_cg_clockgating_default(struct radeon_device *rdev)
905 {
906 	u32 count;
907 	const u32 *ps = NULL;
908 
909 	ps = (const u32 *)&cayman_cgcg_cgls_default;
910 	count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
911 
912 	btc_program_mgcg_hw_sequence(rdev, ps, count);
913 }
914 
915 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
916 				      bool enable)
917 {
918 	u32 count;
919 	const u32 *ps = NULL;
920 
921 	if (enable) {
922 		ps = (const u32 *)&cayman_cgcg_cgls_enable;
923 		count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
924 	} else {
925 		ps = (const u32 *)&cayman_cgcg_cgls_disable;
926 		count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
927 	}
928 
929 	btc_program_mgcg_hw_sequence(rdev, ps, count);
930 }
931 
932 static void ni_mg_clockgating_default(struct radeon_device *rdev)
933 {
934 	u32 count;
935 	const u32 *ps = NULL;
936 
937 	ps = (const u32 *)&cayman_mgcg_default;
938 	count = CAYMAN_MGCG_DEFAULT_LENGTH;
939 
940 	btc_program_mgcg_hw_sequence(rdev, ps, count);
941 }
942 
943 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
944 				     bool enable)
945 {
946 	u32 count;
947 	const u32 *ps = NULL;
948 
949 	if (enable) {
950 		ps = (const u32 *)&cayman_mgcg_enable;
951 		count = CAYMAN_MGCG_ENABLE_LENGTH;
952 	} else {
953 		ps = (const u32 *)&cayman_mgcg_disable;
954 		count = CAYMAN_MGCG_DISABLE_LENGTH;
955 	}
956 
957 	btc_program_mgcg_hw_sequence(rdev, ps, count);
958 }
959 
960 static void ni_ls_clockgating_default(struct radeon_device *rdev)
961 {
962 	u32 count;
963 	const u32 *ps = NULL;
964 
965 	ps = (const u32 *)&cayman_sysls_default;
966 	count = CAYMAN_SYSLS_DEFAULT_LENGTH;
967 
968 	btc_program_mgcg_hw_sequence(rdev, ps, count);
969 }
970 
971 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
972 				     bool enable)
973 {
974 	u32 count;
975 	const u32 *ps = NULL;
976 
977 	if (enable) {
978 		ps = (const u32 *)&cayman_sysls_enable;
979 		count = CAYMAN_SYSLS_ENABLE_LENGTH;
980 	} else {
981 		ps = (const u32 *)&cayman_sysls_disable;
982 		count = CAYMAN_SYSLS_DISABLE_LENGTH;
983 	}
984 
985 	btc_program_mgcg_hw_sequence(rdev, ps, count);
986 
987 }
988 
989 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
990 							     struct radeon_clock_voltage_dependency_table *table)
991 {
992 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
993 	u32 i;
994 
995 	if (table) {
996 		for (i = 0; i < table->count; i++) {
997 			if (0xff01 == table->entries[i].v) {
998 				if (pi->max_vddc == 0)
999 					return -EINVAL;
1000 				table->entries[i].v = pi->max_vddc;
1001 			}
1002 		}
1003 	}
1004 	return 0;
1005 }
1006 
1007 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1008 {
1009 	int ret = 0;
1010 
1011 	ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1012 								&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1013 
1014 	ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1015 								&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1016 	return ret;
1017 }
1018 
1019 static void ni_stop_dpm(struct radeon_device *rdev)
1020 {
1021 	WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1022 }
1023 
1024 #if 0
1025 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1026 					bool ac_power)
1027 {
1028 	if (ac_power)
1029 		return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1030 			0 : -EINVAL;
1031 
1032 	return 0;
1033 }
1034 #endif
1035 
1036 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1037 						      PPSMC_Msg msg, u32 parameter)
1038 {
1039 	WREG32(SMC_SCRATCH0, parameter);
1040 	return rv770_send_msg_to_smc(rdev, msg);
1041 }
1042 
1043 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1044 {
1045 	if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1046 		return -EINVAL;
1047 
1048 	return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1049 		0 : -EINVAL;
1050 }
1051 
1052 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1053 				   enum radeon_dpm_forced_level level)
1054 {
1055 	if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1056 		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1057 			return -EINVAL;
1058 
1059 		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1060 			return -EINVAL;
1061 	} else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1062 		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1063 			return -EINVAL;
1064 
1065 		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1066 			return -EINVAL;
1067 	} else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1068 		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1069 			return -EINVAL;
1070 
1071 		if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1072 			return -EINVAL;
1073 	}
1074 
1075 	rdev->pm.dpm.forced_level = level;
1076 
1077 	return 0;
1078 }
1079 
1080 static void ni_stop_smc(struct radeon_device *rdev)
1081 {
1082 	u32 tmp;
1083 	int i;
1084 
1085 	for (i = 0; i < rdev->usec_timeout; i++) {
1086 		tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1087 		if (tmp != 1)
1088 			break;
1089 		udelay(1);
1090 	}
1091 
1092 	udelay(100);
1093 
1094 	r7xx_stop_smc(rdev);
1095 }
1096 
1097 static int ni_process_firmware_header(struct radeon_device *rdev)
1098 {
1099 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1100 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1101 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1102 	u32 tmp;
1103 	int ret;
1104 
1105 	ret = rv770_read_smc_sram_dword(rdev,
1106 					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1107 					NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1108 					&tmp, pi->sram_end);
1109 
1110 	if (ret)
1111 		return ret;
1112 
1113 	pi->state_table_start = (u16)tmp;
1114 
1115 	ret = rv770_read_smc_sram_dword(rdev,
1116 					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1117 					NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1118 					&tmp, pi->sram_end);
1119 
1120 	if (ret)
1121 		return ret;
1122 
1123 	pi->soft_regs_start = (u16)tmp;
1124 
1125 	ret = rv770_read_smc_sram_dword(rdev,
1126 					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1127 					NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1128 					&tmp, pi->sram_end);
1129 
1130 	if (ret)
1131 		return ret;
1132 
1133 	eg_pi->mc_reg_table_start = (u16)tmp;
1134 
1135 	ret = rv770_read_smc_sram_dword(rdev,
1136 					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1137 					NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1138 					&tmp, pi->sram_end);
1139 
1140 	if (ret)
1141 		return ret;
1142 
1143 	ni_pi->fan_table_start = (u16)tmp;
1144 
1145 	ret = rv770_read_smc_sram_dword(rdev,
1146 					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1147 					NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1148 					&tmp, pi->sram_end);
1149 
1150 	if (ret)
1151 		return ret;
1152 
1153 	ni_pi->arb_table_start = (u16)tmp;
1154 
1155 	ret = rv770_read_smc_sram_dword(rdev,
1156 					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1157 					NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1158 					&tmp, pi->sram_end);
1159 
1160 	if (ret)
1161 		return ret;
1162 
1163 	ni_pi->cac_table_start = (u16)tmp;
1164 
1165 	ret = rv770_read_smc_sram_dword(rdev,
1166 					NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1167 					NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1168 					&tmp, pi->sram_end);
1169 
1170 	if (ret)
1171 		return ret;
1172 
1173 	ni_pi->spll_table_start = (u16)tmp;
1174 
1175 
1176 	return ret;
1177 }
1178 
1179 static void ni_read_clock_registers(struct radeon_device *rdev)
1180 {
1181 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1182 
1183 	ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1184 	ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1185 	ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1186 	ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1187 	ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1188 	ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1189 	ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1190 	ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1191 	ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1192 	ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1193 	ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1194 	ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1195 	ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1196 	ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1197 }
1198 
1199 #if 0
1200 static int ni_enter_ulp_state(struct radeon_device *rdev)
1201 {
1202 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1203 
1204 	if (pi->gfx_clock_gating) {
1205 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1206 		WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1207 		WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1208 		RREG32(GB_ADDR_CONFIG);
1209 	}
1210 
1211 	WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1212 		 ~HOST_SMC_MSG_MASK);
1213 
1214 	udelay(25000);
1215 
1216 	return 0;
1217 }
1218 #endif
1219 
1220 static void ni_program_response_times(struct radeon_device *rdev)
1221 {
1222 	u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1223 	u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1224 	u32 reference_clock;
1225 
1226 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1227 
1228 	voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1229 	backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1230 
1231 	if (voltage_response_time == 0)
1232 		voltage_response_time = 1000;
1233 
1234 	if (backbias_response_time == 0)
1235 		backbias_response_time = 1000;
1236 
1237 	acpi_delay_time = 15000;
1238 	vbi_time_out = 100000;
1239 
1240 	reference_clock = radeon_get_xclk(rdev);
1241 
1242 	vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1243 	bb_dly   = (backbias_response_time * reference_clock) / 1600;
1244 	acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1245 	vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1246 
1247 	mclk_switch_limit = (460 * reference_clock) / 100;
1248 
1249 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1250 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1251 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1252 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1253 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1254 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1255 }
1256 
1257 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1258 					  struct atom_voltage_table *voltage_table,
1259 					  NISLANDS_SMC_STATETABLE *table)
1260 {
1261 	unsigned int i;
1262 
1263 	for (i = 0; i < voltage_table->count; i++) {
1264 		table->highSMIO[i] = 0;
1265 		table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1266 	}
1267 }
1268 
1269 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1270 					   NISLANDS_SMC_STATETABLE *table)
1271 {
1272 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1273 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1274 	unsigned char i;
1275 
1276 	if (eg_pi->vddc_voltage_table.count) {
1277 		ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1278 		table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1279 		table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1280 			cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1281 
1282 		for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1283 			if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1284 				table->maxVDDCIndexInPPTable = i;
1285 				break;
1286 			}
1287 		}
1288 	}
1289 
1290 	if (eg_pi->vddci_voltage_table.count) {
1291 		ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1292 
1293 		table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1294 		table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1295 			cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
1296 	}
1297 }
1298 
1299 static int ni_populate_voltage_value(struct radeon_device *rdev,
1300 				     struct atom_voltage_table *table,
1301 				     u16 value,
1302 				     NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1303 {
1304 	unsigned int i;
1305 
1306 	for (i = 0; i < table->count; i++) {
1307 		if (value <= table->entries[i].value) {
1308 			voltage->index = (u8)i;
1309 			voltage->value = cpu_to_be16(table->entries[i].value);
1310 			break;
1311 		}
1312 	}
1313 
1314 	if (i >= table->count)
1315 		return -EINVAL;
1316 
1317 	return 0;
1318 }
1319 
1320 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1321 				   u32 mclk,
1322 				   NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1323 {
1324 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1325 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1326 
1327 	if (!pi->mvdd_control) {
1328 		voltage->index = eg_pi->mvdd_high_index;
1329 		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1330 		return;
1331 	}
1332 
1333 	if (mclk <= pi->mvdd_split_frequency) {
1334 		voltage->index = eg_pi->mvdd_low_index;
1335 		voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1336 	} else {
1337 		voltage->index = eg_pi->mvdd_high_index;
1338 		voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1339 	}
1340 }
1341 
1342 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1343 				    NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1344 				    u16 *std_voltage)
1345 {
1346 	if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1347 	    ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1348 		*std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1349 	else
1350 		*std_voltage = be16_to_cpu(voltage->value);
1351 
1352 	return 0;
1353 }
1354 
1355 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1356 					  u16 value, u8 index,
1357 					  NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1358 {
1359 	voltage->index = index;
1360 	voltage->value = cpu_to_be16(value);
1361 }
1362 
1363 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1364 {
1365 	u32 xclk_period;
1366 	u32 xclk = radeon_get_xclk(rdev);
1367 	u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1368 
1369 	xclk_period = (1000000000UL / xclk);
1370 	xclk_period /= 10000UL;
1371 
1372 	return tmp * xclk_period;
1373 }
1374 
1375 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1376 {
1377 	return (power_in_watts * scaling_factor) << 2;
1378 }
1379 
1380 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1381 					  struct radeon_ps *radeon_state,
1382 					  u32 near_tdp_limit)
1383 {
1384 	struct ni_ps *state = ni_get_ps(radeon_state);
1385 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1386 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1387 	u32 power_boost_limit = 0;
1388 	int ret;
1389 
1390 	if (ni_pi->enable_power_containment &&
1391 	    ni_pi->use_power_boost_limit) {
1392 		NISLANDS_SMC_VOLTAGE_VALUE vddc;
1393 		u16 std_vddc_med;
1394 		u16 std_vddc_high;
1395 		u64 tmp, n, d;
1396 
1397 		if (state->performance_level_count < 3)
1398 			return 0;
1399 
1400 		ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1401 						state->performance_levels[state->performance_level_count - 2].vddc,
1402 						&vddc);
1403 		if (ret)
1404 			return 0;
1405 
1406 		ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1407 		if (ret)
1408 			return 0;
1409 
1410 		ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1411 						state->performance_levels[state->performance_level_count - 1].vddc,
1412 						&vddc);
1413 		if (ret)
1414 			return 0;
1415 
1416 		ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1417 		if (ret)
1418 			return 0;
1419 
1420 		n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1421 		d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1422 		tmp = div64_u64(n, d);
1423 
1424 		if (tmp >> 32)
1425 			return 0;
1426 		power_boost_limit = (u32)tmp;
1427 	}
1428 
1429 	return power_boost_limit;
1430 }
1431 
1432 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1433 					    bool adjust_polarity,
1434 					    u32 tdp_adjustment,
1435 					    u32 *tdp_limit,
1436 					    u32 *near_tdp_limit)
1437 {
1438 	if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1439 		return -EINVAL;
1440 
1441 	if (adjust_polarity) {
1442 		*tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1443 		*near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1444 	} else {
1445 		*tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1446 		*near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1447 	}
1448 
1449 	return 0;
1450 }
1451 
1452 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1453 				      struct radeon_ps *radeon_state)
1454 {
1455 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1456 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1457 
1458 	if (ni_pi->enable_power_containment) {
1459 		NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1460 		u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1461 		u32 tdp_limit;
1462 		u32 near_tdp_limit;
1463 		u32 power_boost_limit;
1464 		int ret;
1465 
1466 		if (scaling_factor == 0)
1467 			return -EINVAL;
1468 
1469 		memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1470 
1471 		ret = ni_calculate_adjusted_tdp_limits(rdev,
1472 						       false, /* ??? */
1473 						       rdev->pm.dpm.tdp_adjustment,
1474 						       &tdp_limit,
1475 						       &near_tdp_limit);
1476 		if (ret)
1477 			return ret;
1478 
1479 		power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1480 								   near_tdp_limit);
1481 
1482 		smc_table->dpm2Params.TDPLimit =
1483 			cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1484 		smc_table->dpm2Params.NearTDPLimit =
1485 			cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1486 		smc_table->dpm2Params.SafePowerLimit =
1487 			cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1488 							   scaling_factor));
1489 		smc_table->dpm2Params.PowerBoostLimit =
1490 			cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1491 
1492 		ret = rv770_copy_bytes_to_smc(rdev,
1493 					      (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1494 						    offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1495 					      (u8 *)(&smc_table->dpm2Params.TDPLimit),
1496 					      sizeof(u32) * 4, pi->sram_end);
1497 		if (ret)
1498 			return ret;
1499 	}
1500 
1501 	return 0;
1502 }
1503 
1504 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1505 				u32 arb_freq_src, u32 arb_freq_dest)
1506 {
1507 	u32 mc_arb_dram_timing;
1508 	u32 mc_arb_dram_timing2;
1509 	u32 burst_time;
1510 	u32 mc_cg_config;
1511 
1512 	switch (arb_freq_src) {
1513 	case MC_CG_ARB_FREQ_F0:
1514 		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1515 		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1516 		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1517 		break;
1518 	case MC_CG_ARB_FREQ_F1:
1519 		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1520 		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1521 		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1522 		break;
1523 	case MC_CG_ARB_FREQ_F2:
1524 		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1525 		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1526 		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1527 		break;
1528 	case MC_CG_ARB_FREQ_F3:
1529 		mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1530 		mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1531 		burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1532 		break;
1533 	default:
1534 		return -EINVAL;
1535 	}
1536 
1537 	switch (arb_freq_dest) {
1538 	case MC_CG_ARB_FREQ_F0:
1539 		WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1540 		WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1541 		WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1542 		break;
1543 	case MC_CG_ARB_FREQ_F1:
1544 		WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1545 		WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1546 		WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1547 		break;
1548 	case MC_CG_ARB_FREQ_F2:
1549 		WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1550 		WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1551 		WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1552 		break;
1553 	case MC_CG_ARB_FREQ_F3:
1554 		WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1555 		WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1556 		WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1557 		break;
1558 	default:
1559 		return -EINVAL;
1560 	}
1561 
1562 	mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1563 	WREG32(MC_CG_CONFIG, mc_cg_config);
1564 	WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1565 
1566 	return 0;
1567 }
1568 
1569 static int ni_init_arb_table_index(struct radeon_device *rdev)
1570 {
1571 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1572 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1573 	u32 tmp;
1574 	int ret;
1575 
1576 	ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1577 					&tmp, pi->sram_end);
1578 	if (ret)
1579 		return ret;
1580 
1581 	tmp &= 0x00FFFFFF;
1582 	tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1583 
1584 	return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1585 					  tmp, pi->sram_end);
1586 }
1587 
1588 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1589 {
1590 	return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1591 }
1592 
1593 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1594 {
1595 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1596 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1597 	u32 tmp;
1598 	int ret;
1599 
1600 	ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1601 					&tmp, pi->sram_end);
1602 	if (ret)
1603 		return ret;
1604 
1605 	tmp = (tmp >> 24) & 0xff;
1606 
1607 	if (tmp == MC_CG_ARB_FREQ_F0)
1608 		return 0;
1609 
1610 	return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1611 }
1612 
1613 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1614 						struct rv7xx_pl *pl,
1615 						SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1616 {
1617 	u32 dram_timing;
1618 	u32 dram_timing2;
1619 
1620 	arb_regs->mc_arb_rfsh_rate =
1621 		(u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1622 
1623 
1624 	radeon_atom_set_engine_dram_timings(rdev, pl->sclk, pl->mclk);
1625 
1626 	dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1627 	dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1628 
1629 	arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1630 	arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1631 
1632 	return 0;
1633 }
1634 
1635 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1636 						  struct radeon_ps *radeon_state,
1637 						  unsigned int first_arb_set)
1638 {
1639 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1640 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1641 	struct ni_ps *state = ni_get_ps(radeon_state);
1642 	SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1643 	int i, ret = 0;
1644 
1645 	for (i = 0; i < state->performance_level_count; i++) {
1646 		ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1647 		if (ret)
1648 			break;
1649 
1650 		ret = rv770_copy_bytes_to_smc(rdev,
1651 					      (u16)(ni_pi->arb_table_start +
1652 						    offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1653 						    sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1654 					      (u8 *)&arb_regs,
1655 					      (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1656 					      pi->sram_end);
1657 		if (ret)
1658 			break;
1659 	}
1660 	return ret;
1661 }
1662 
1663 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1664 					       struct radeon_ps *radeon_new_state)
1665 {
1666 	return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1667 						      NISLANDS_DRIVER_STATE_ARB_INDEX);
1668 }
1669 
1670 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1671 					   struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1672 {
1673 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1674 
1675 	voltage->index = eg_pi->mvdd_high_index;
1676 	voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1677 }
1678 
1679 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1680 					 struct radeon_ps *radeon_initial_state,
1681 					 NISLANDS_SMC_STATETABLE *table)
1682 {
1683 	struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1684 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1685 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1686 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1687 	u32 reg;
1688 	int ret;
1689 
1690 	table->initialState.level.mclk.vMPLL_AD_FUNC_CNTL =
1691 		cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1692 	table->initialState.level.mclk.vMPLL_AD_FUNC_CNTL_2 =
1693 		cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1694 	table->initialState.level.mclk.vMPLL_DQ_FUNC_CNTL =
1695 		cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1696 	table->initialState.level.mclk.vMPLL_DQ_FUNC_CNTL_2 =
1697 		cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1698 	table->initialState.level.mclk.vMCLK_PWRMGT_CNTL =
1699 		cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1700 	table->initialState.level.mclk.vDLL_CNTL =
1701 		cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1702 	table->initialState.level.mclk.vMPLL_SS =
1703 		cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1704 	table->initialState.level.mclk.vMPLL_SS2 =
1705 		cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1706 	table->initialState.level.mclk.mclk_value =
1707 		cpu_to_be32(initial_state->performance_levels[0].mclk);
1708 
1709 	table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL =
1710 		cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1711 	table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_2 =
1712 		cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1713 	table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_3 =
1714 		cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1715 	table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_4 =
1716 		cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1717 	table->initialState.level.sclk.vCG_SPLL_SPREAD_SPECTRUM =
1718 		cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1719 	table->initialState.level.sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1720 		cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1721 	table->initialState.level.sclk.sclk_value =
1722 		cpu_to_be32(initial_state->performance_levels[0].sclk);
1723 	table->initialState.level.arbRefreshState =
1724 		NISLANDS_INITIAL_STATE_ARB_INDEX;
1725 
1726 	table->initialState.level.ACIndex = 0;
1727 
1728 	ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1729 					initial_state->performance_levels[0].vddc,
1730 					&table->initialState.level.vddc);
1731 	if (!ret) {
1732 		u16 std_vddc;
1733 
1734 		ret = ni_get_std_voltage_value(rdev,
1735 					       &table->initialState.level.vddc,
1736 					       &std_vddc);
1737 		if (!ret)
1738 			ni_populate_std_voltage_value(rdev, std_vddc,
1739 						      table->initialState.level.vddc.index,
1740 						      &table->initialState.level.std_vddc);
1741 	}
1742 
1743 	if (eg_pi->vddci_control)
1744 		ni_populate_voltage_value(rdev,
1745 					  &eg_pi->vddci_voltage_table,
1746 					  initial_state->performance_levels[0].vddci,
1747 					  &table->initialState.level.vddci);
1748 
1749 	ni_populate_initial_mvdd_value(rdev, &table->initialState.level.mvdd);
1750 
1751 	reg = CG_R(0xffff) | CG_L(0);
1752 	table->initialState.level.aT = cpu_to_be32(reg);
1753 
1754 	table->initialState.level.bSP = cpu_to_be32(pi->dsp);
1755 
1756 	if (pi->boot_in_gen2)
1757 		table->initialState.level.gen2PCIE = 1;
1758 	else
1759 		table->initialState.level.gen2PCIE = 0;
1760 
1761 	if (pi->mem_gddr5) {
1762 		table->initialState.level.strobeMode =
1763 			cypress_get_strobe_mode_settings(rdev,
1764 							 initial_state->performance_levels[0].mclk);
1765 
1766 		if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1767 			table->initialState.level.mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1768 		else
1769 			table->initialState.level.mcFlags =  0;
1770 	}
1771 
1772 	table->initialState.levelCount = 1;
1773 
1774 	table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1775 
1776 	table->initialState.level.dpm2.MaxPS = 0;
1777 	table->initialState.level.dpm2.NearTDPDec = 0;
1778 	table->initialState.level.dpm2.AboveSafeInc = 0;
1779 	table->initialState.level.dpm2.BelowSafeInc = 0;
1780 
1781 	reg = MIN_POWER_MASK | MAX_POWER_MASK;
1782 	table->initialState.level.SQPowerThrottle = cpu_to_be32(reg);
1783 
1784 	reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1785 	table->initialState.level.SQPowerThrottle_2 = cpu_to_be32(reg);
1786 
1787 	return 0;
1788 }
1789 
1790 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1791 				      NISLANDS_SMC_STATETABLE *table)
1792 {
1793 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1794 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1795 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1796 	u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1797 	u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1798 	u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1799 	u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1800 	u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1801 	u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1802 	u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1803 	u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1804 	u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1805 	u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1806 	u32 reg;
1807 	int ret;
1808 
1809 	table->ACPIState = table->initialState;
1810 
1811 	table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1812 
1813 	if (pi->acpi_vddc) {
1814 		ret = ni_populate_voltage_value(rdev,
1815 						&eg_pi->vddc_voltage_table,
1816 						pi->acpi_vddc, &table->ACPIState.level.vddc);
1817 		if (!ret) {
1818 			u16 std_vddc;
1819 
1820 			ret = ni_get_std_voltage_value(rdev,
1821 						       &table->ACPIState.level.vddc, &std_vddc);
1822 			if (!ret)
1823 				ni_populate_std_voltage_value(rdev, std_vddc,
1824 							      table->ACPIState.level.vddc.index,
1825 							      &table->ACPIState.level.std_vddc);
1826 		}
1827 
1828 		if (pi->pcie_gen2) {
1829 			if (pi->acpi_pcie_gen2)
1830 				table->ACPIState.level.gen2PCIE = 1;
1831 			else
1832 				table->ACPIState.level.gen2PCIE = 0;
1833 		} else {
1834 			table->ACPIState.level.gen2PCIE = 0;
1835 		}
1836 	} else {
1837 		ret = ni_populate_voltage_value(rdev,
1838 						&eg_pi->vddc_voltage_table,
1839 						pi->min_vddc_in_table,
1840 						&table->ACPIState.level.vddc);
1841 		if (!ret) {
1842 			u16 std_vddc;
1843 
1844 			ret = ni_get_std_voltage_value(rdev,
1845 						       &table->ACPIState.level.vddc,
1846 						       &std_vddc);
1847 			if (!ret)
1848 				ni_populate_std_voltage_value(rdev, std_vddc,
1849 							      table->ACPIState.level.vddc.index,
1850 							      &table->ACPIState.level.std_vddc);
1851 		}
1852 		table->ACPIState.level.gen2PCIE = 0;
1853 	}
1854 
1855 	if (eg_pi->acpi_vddci) {
1856 		if (eg_pi->vddci_control)
1857 			ni_populate_voltage_value(rdev,
1858 						  &eg_pi->vddci_voltage_table,
1859 						  eg_pi->acpi_vddci,
1860 						  &table->ACPIState.level.vddci);
1861 	}
1862 
1863 
1864 	mpll_ad_func_cntl &= ~PDNB;
1865 
1866 	mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1867 
1868 	if (pi->mem_gddr5)
1869 		mpll_dq_func_cntl &= ~PDNB;
1870 	mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1871 
1872 
1873 	mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1874 			     MRDCKA1_RESET |
1875 			     MRDCKB0_RESET |
1876 			     MRDCKB1_RESET |
1877 			     MRDCKC0_RESET |
1878 			     MRDCKC1_RESET |
1879 			     MRDCKD0_RESET |
1880 			     MRDCKD1_RESET);
1881 
1882 	mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1883 			      MRDCKA1_PDNB |
1884 			      MRDCKB0_PDNB |
1885 			      MRDCKB1_PDNB |
1886 			      MRDCKC0_PDNB |
1887 			      MRDCKC1_PDNB |
1888 			      MRDCKD0_PDNB |
1889 			      MRDCKD1_PDNB);
1890 
1891 	dll_cntl |= (MRDCKA0_BYPASS |
1892 		     MRDCKA1_BYPASS |
1893 		     MRDCKB0_BYPASS |
1894 		     MRDCKB1_BYPASS |
1895 		     MRDCKC0_BYPASS |
1896 		     MRDCKC1_BYPASS |
1897 		     MRDCKD0_BYPASS |
1898 		     MRDCKD1_BYPASS);
1899 
1900 	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1901 	spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1902 
1903 	table->ACPIState.level.mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1904 	table->ACPIState.level.mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1905 	table->ACPIState.level.mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1906 	table->ACPIState.level.mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1907 	table->ACPIState.level.mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1908 	table->ACPIState.level.mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1909 
1910 	table->ACPIState.level.mclk.mclk_value = 0;
1911 
1912 	table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1913 	table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1914 	table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1915 	table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1916 
1917 	table->ACPIState.level.sclk.sclk_value = 0;
1918 
1919 	ni_populate_mvdd_value(rdev, 0, &table->ACPIState.level.mvdd);
1920 
1921 	if (eg_pi->dynamic_ac_timing)
1922 		table->ACPIState.level.ACIndex = 1;
1923 
1924 	table->ACPIState.level.dpm2.MaxPS = 0;
1925 	table->ACPIState.level.dpm2.NearTDPDec = 0;
1926 	table->ACPIState.level.dpm2.AboveSafeInc = 0;
1927 	table->ACPIState.level.dpm2.BelowSafeInc = 0;
1928 
1929 	reg = MIN_POWER_MASK | MAX_POWER_MASK;
1930 	table->ACPIState.level.SQPowerThrottle = cpu_to_be32(reg);
1931 
1932 	reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1933 	table->ACPIState.level.SQPowerThrottle_2 = cpu_to_be32(reg);
1934 
1935 	return 0;
1936 }
1937 
1938 static int ni_init_smc_table(struct radeon_device *rdev)
1939 {
1940 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1941 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
1942 	int ret;
1943 	struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1944 	NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1945 
1946 	memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1947 
1948 	ni_populate_smc_voltage_tables(rdev, table);
1949 
1950 	switch (rdev->pm.int_thermal_type) {
1951 	case THERMAL_TYPE_NI:
1952 	case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1953 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1954 		break;
1955 	case THERMAL_TYPE_NONE:
1956 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1957 		break;
1958 	default:
1959 		table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1960 		break;
1961 	}
1962 
1963 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1964 		table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1965 
1966 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1967 		table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1968 
1969 	if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1970 		table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1971 
1972 	if (pi->mem_gddr5)
1973 		table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1974 
1975 	ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1976 	if (ret)
1977 		return ret;
1978 
1979 	ret = ni_populate_smc_acpi_state(rdev, table);
1980 	if (ret)
1981 		return ret;
1982 
1983 	table->driverState.flags = table->initialState.flags;
1984 	table->driverState.levelCount = table->initialState.levelCount;
1985 	table->driverState.levels[0] = table->initialState.level;
1986 
1987 	table->ULVState = table->initialState;
1988 
1989 	ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1990 						     NISLANDS_INITIAL_STATE_ARB_INDEX);
1991 	if (ret)
1992 		return ret;
1993 
1994 	return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1995 				       sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1996 }
1997 
1998 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1999 				    u32 engine_clock,
2000 				    NISLANDS_SMC_SCLK_VALUE *sclk)
2001 {
2002 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2003 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2004 	struct atom_clock_dividers dividers;
2005 	u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2006 	u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2007 	u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2008 	u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2009 	u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2010 	u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2011 	u64 tmp;
2012 	u32 reference_clock = rdev->clock.spll.reference_freq;
2013 	u32 reference_divider;
2014 	u32 fbdiv;
2015 	int ret;
2016 
2017 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2018 					     engine_clock, false, &dividers);
2019 	if (ret)
2020 		return ret;
2021 
2022 	reference_divider = 1 + dividers.ref_div;
2023 
2024 
2025 	tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2026 	do_div(tmp, reference_clock);
2027 	fbdiv = (u32) tmp;
2028 
2029 	spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2030 	spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2031 	spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2032 
2033 	spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2034 	spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2035 
2036 	spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2037 	spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2038 	spll_func_cntl_3 |= SPLL_DITHEN;
2039 
2040 	if (pi->sclk_ss) {
2041 		struct radeon_atom_ss ss;
2042 		u32 vco_freq = engine_clock * dividers.post_div;
2043 
2044 		if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2045 						     ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2046 			u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2047 			u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2048 
2049 			cg_spll_spread_spectrum &= ~CLK_S_MASK;
2050 			cg_spll_spread_spectrum |= CLK_S(clk_s);
2051 			cg_spll_spread_spectrum |= SSEN;
2052 
2053 			cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2054 			cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2055 		}
2056 	}
2057 
2058 	sclk->sclk_value = engine_clock;
2059 	sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2060 	sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2061 	sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2062 	sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2063 	sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2064 	sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2065 
2066 	return 0;
2067 }
2068 
2069 static int ni_populate_sclk_value(struct radeon_device *rdev,
2070 				  u32 engine_clock,
2071 				  NISLANDS_SMC_SCLK_VALUE *sclk)
2072 {
2073 	NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2074 	int ret;
2075 
2076 	ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2077 	if (!ret) {
2078 		sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2079 		sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2080 		sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2081 		sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2082 		sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2083 		sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2084 		sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2085 	}
2086 
2087 	return ret;
2088 }
2089 
2090 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2091 {
2092 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2093 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2094 	SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2095 	NISLANDS_SMC_SCLK_VALUE sclk_params;
2096 	u32 fb_div;
2097 	u32 p_div;
2098 	u32 clk_s;
2099 	u32 clk_v;
2100 	u32 sclk = 0;
2101 	int i, ret;
2102 	u32 tmp;
2103 
2104 	if (ni_pi->spll_table_start == 0)
2105 		return -EINVAL;
2106 
2107 	spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2108 	if (spll_table == NULL)
2109 		return -ENOMEM;
2110 
2111 	for (i = 0; i < 256; i++) {
2112 		ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2113 		if (ret)
2114 			break;
2115 
2116 		p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2117 		fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2118 		clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2119 		clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2120 
2121 		fb_div &= ~0x00001FFF;
2122 		fb_div >>= 1;
2123 		clk_v >>= 6;
2124 
2125 		if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2126 			ret = -EINVAL;
2127 
2128 		if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2129 			ret = -EINVAL;
2130 
2131 		if (fb_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT))
2132 			ret = -EINVAL;
2133 
2134 		if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2135 			ret = -EINVAL;
2136 
2137 		if (ret)
2138 			break;
2139 
2140 		tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2141 			((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2142 		spll_table->freq[i] = cpu_to_be32(tmp);
2143 
2144 		tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2145 			((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2146 		spll_table->ss[i] = cpu_to_be32(tmp);
2147 
2148 		sclk += 512;
2149 	}
2150 
2151 	if (!ret)
2152 		ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2153 					      sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2154 
2155 	kfree(spll_table);
2156 
2157 	return ret;
2158 }
2159 
2160 static int ni_populate_mclk_value(struct radeon_device *rdev,
2161 				  u32 engine_clock,
2162 				  u32 memory_clock,
2163 				  NISLANDS_SMC_MCLK_VALUE *mclk,
2164 				  bool strobe_mode,
2165 				  bool dll_state_on)
2166 {
2167 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2168 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2169 	u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2170 	u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2171 	u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2172 	u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2173 	u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2174 	u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2175 	u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2176 	u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2177 	struct atom_clock_dividers dividers;
2178 	u32 ibias;
2179 	u32 dll_speed;
2180 	int ret;
2181 	u32 mc_seq_misc7;
2182 
2183 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2184 					     memory_clock, strobe_mode, &dividers);
2185 	if (ret)
2186 		return ret;
2187 
2188 	if (!strobe_mode) {
2189 		mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2190 
2191 		if (mc_seq_misc7 & 0x8000000)
2192 			dividers.post_div = 1;
2193 	}
2194 
2195 	ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2196 
2197 	mpll_ad_func_cntl &= ~(CLKR_MASK |
2198 			       YCLK_POST_DIV_MASK |
2199 			       CLKF_MASK |
2200 			       CLKFRAC_MASK |
2201 			       IBIAS_MASK);
2202 	mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2203 	mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2204 	mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2205 	mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2206 	mpll_ad_func_cntl |= IBIAS(ibias);
2207 
2208 	if (dividers.vco_mode)
2209 		mpll_ad_func_cntl_2 |= VCO_MODE;
2210 	else
2211 		mpll_ad_func_cntl_2 &= ~VCO_MODE;
2212 
2213 	if (pi->mem_gddr5) {
2214 		mpll_dq_func_cntl &= ~(CLKR_MASK |
2215 				       YCLK_POST_DIV_MASK |
2216 				       CLKF_MASK |
2217 				       CLKFRAC_MASK |
2218 				       IBIAS_MASK);
2219 		mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2220 		mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2221 		mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2222 		mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2223 		mpll_dq_func_cntl |= IBIAS(ibias);
2224 
2225 		if (strobe_mode)
2226 			mpll_dq_func_cntl &= ~PDNB;
2227 		else
2228 			mpll_dq_func_cntl |= PDNB;
2229 
2230 		if (dividers.vco_mode)
2231 			mpll_dq_func_cntl_2 |= VCO_MODE;
2232 		else
2233 			mpll_dq_func_cntl_2 &= ~VCO_MODE;
2234 	}
2235 
2236 	if (pi->mclk_ss) {
2237 		struct radeon_atom_ss ss;
2238 		u32 vco_freq = memory_clock * dividers.post_div;
2239 
2240 		if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2241 						     ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2242 			u32 reference_clock = rdev->clock.mpll.reference_freq;
2243 			u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2244 			u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2245 			u32 clk_v = ss.percentage *
2246 				(0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2247 
2248 			mpll_ss1 &= ~CLKV_MASK;
2249 			mpll_ss1 |= CLKV(clk_v);
2250 
2251 			mpll_ss2 &= ~CLKS_MASK;
2252 			mpll_ss2 |= CLKS(clk_s);
2253 		}
2254 	}
2255 
2256 	dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2257 					memory_clock);
2258 
2259 	mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2260 	mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2261 	if (dll_state_on)
2262 		mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2263 				     MRDCKA1_PDNB |
2264 				     MRDCKB0_PDNB |
2265 				     MRDCKB1_PDNB |
2266 				     MRDCKC0_PDNB |
2267 				     MRDCKC1_PDNB |
2268 				     MRDCKD0_PDNB |
2269 				     MRDCKD1_PDNB);
2270 	else
2271 		mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2272 				      MRDCKA1_PDNB |
2273 				      MRDCKB0_PDNB |
2274 				      MRDCKB1_PDNB |
2275 				      MRDCKC0_PDNB |
2276 				      MRDCKC1_PDNB |
2277 				      MRDCKD0_PDNB |
2278 				      MRDCKD1_PDNB);
2279 
2280 
2281 	mclk->mclk_value = cpu_to_be32(memory_clock);
2282 	mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2283 	mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2284 	mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2285 	mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2286 	mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2287 	mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2288 	mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2289 	mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2290 
2291 	return 0;
2292 }
2293 
2294 static void ni_populate_smc_sp(struct radeon_device *rdev,
2295 			       struct radeon_ps *radeon_state,
2296 			       NISLANDS_SMC_SWSTATE *smc_state)
2297 {
2298 	struct ni_ps *ps = ni_get_ps(radeon_state);
2299 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2300 	int i;
2301 
2302 	for (i = 0; i < ps->performance_level_count - 1; i++)
2303 		smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2304 
2305 	smc_state->levels[ps->performance_level_count - 1].bSP =
2306 		cpu_to_be32(pi->psp);
2307 }
2308 
2309 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2310 					 struct rv7xx_pl *pl,
2311 					 NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2312 {
2313 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2314 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2315 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2316 	int ret;
2317 	bool dll_state_on;
2318 	u16 std_vddc;
2319 	u32 tmp = RREG32(DC_STUTTER_CNTL);
2320 
2321 	level->gen2PCIE = pi->pcie_gen2 ?
2322 		((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2323 
2324 	ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2325 	if (ret)
2326 		return ret;
2327 
2328 	level->mcFlags =  0;
2329 	if (pi->mclk_stutter_mode_threshold &&
2330 	    (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2331 	    !eg_pi->uvd_enabled &&
2332 	    (tmp & DC_STUTTER_ENABLE_A) &&
2333 	    (tmp & DC_STUTTER_ENABLE_B))
2334 		level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2335 
2336 	if (pi->mem_gddr5) {
2337 		if (pl->mclk > pi->mclk_edc_enable_threshold)
2338 			level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2339 		if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2340 			level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2341 
2342 		level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2343 
2344 		if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2345 			if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2346 			    ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2347 				dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2348 			else
2349 				dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2350 		} else {
2351 			dll_state_on = false;
2352 			if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2353 				level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2354 		}
2355 
2356 		ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2357 					     &level->mclk,
2358 					     (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2359 					     dll_state_on);
2360 	} else
2361 		ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2362 
2363 	if (ret)
2364 		return ret;
2365 
2366 	ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2367 					pl->vddc, &level->vddc);
2368 	if (ret)
2369 		return ret;
2370 
2371 	ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2372 	if (ret)
2373 		return ret;
2374 
2375 	ni_populate_std_voltage_value(rdev, std_vddc,
2376 				      level->vddc.index, &level->std_vddc);
2377 
2378 	if (eg_pi->vddci_control) {
2379 		ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2380 						pl->vddci, &level->vddci);
2381 		if (ret)
2382 			return ret;
2383 	}
2384 
2385 	ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2386 
2387 	return ret;
2388 }
2389 
2390 static int ni_populate_smc_t(struct radeon_device *rdev,
2391 			     struct radeon_ps *radeon_state,
2392 			     NISLANDS_SMC_SWSTATE *smc_state)
2393 {
2394 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2395 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2396 	struct ni_ps *state = ni_get_ps(radeon_state);
2397 	u32 a_t;
2398 	u32 t_l, t_h;
2399 	u32 high_bsp;
2400 	int i, ret;
2401 
2402 	if (state->performance_level_count >= 9)
2403 		return -EINVAL;
2404 
2405 	if (state->performance_level_count < 2) {
2406 		a_t = CG_R(0xffff) | CG_L(0);
2407 		smc_state->levels[0].aT = cpu_to_be32(a_t);
2408 		return 0;
2409 	}
2410 
2411 	smc_state->levels[0].aT = cpu_to_be32(0);
2412 
2413 	for (i = 0; i <= state->performance_level_count - 2; i++) {
2414 		if (eg_pi->uvd_enabled)
2415 			ret = r600_calculate_at(
2416 				1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2417 				100 * R600_AH_DFLT,
2418 				state->performance_levels[i + 1].sclk,
2419 				state->performance_levels[i].sclk,
2420 				&t_l,
2421 				&t_h);
2422 		else
2423 			ret = r600_calculate_at(
2424 				1000 * (i + 1),
2425 				100 * R600_AH_DFLT,
2426 				state->performance_levels[i + 1].sclk,
2427 				state->performance_levels[i].sclk,
2428 				&t_l,
2429 				&t_h);
2430 
2431 		if (ret) {
2432 			t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2433 			t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2434 		}
2435 
2436 		a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2437 		a_t |= CG_R(t_l * pi->bsp / 20000);
2438 		smc_state->levels[i].aT = cpu_to_be32(a_t);
2439 
2440 		high_bsp = (i == state->performance_level_count - 2) ?
2441 			pi->pbsp : pi->bsp;
2442 
2443 		a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2444 		smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2445 	}
2446 
2447 	return 0;
2448 }
2449 
2450 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2451 						struct radeon_ps *radeon_state,
2452 						NISLANDS_SMC_SWSTATE *smc_state)
2453 {
2454 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2455 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2456 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2457 	struct ni_ps *state = ni_get_ps(radeon_state);
2458 	u32 prev_sclk;
2459 	u32 max_sclk;
2460 	u32 min_sclk;
2461 	int i, ret;
2462 	u32 tdp_limit;
2463 	u32 near_tdp_limit;
2464 	u32 power_boost_limit;
2465 	u8 max_ps_percent;
2466 
2467 	if (ni_pi->enable_power_containment == false)
2468 		return 0;
2469 
2470 	if (state->performance_level_count == 0)
2471 		return -EINVAL;
2472 
2473 	if (smc_state->levelCount != state->performance_level_count)
2474 		return -EINVAL;
2475 
2476 	ret = ni_calculate_adjusted_tdp_limits(rdev,
2477 					       false, /* ??? */
2478 					       rdev->pm.dpm.tdp_adjustment,
2479 					       &tdp_limit,
2480 					       &near_tdp_limit);
2481 	if (ret)
2482 		return ret;
2483 
2484 	power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2485 
2486 	ret = rv770_write_smc_sram_dword(rdev,
2487 					 pi->state_table_start +
2488 					 offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2489 					 offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2490 					 ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2491 					 pi->sram_end);
2492 	if (ret)
2493 		power_boost_limit = 0;
2494 
2495 	smc_state->levels[0].dpm2.MaxPS = 0;
2496 	smc_state->levels[0].dpm2.NearTDPDec = 0;
2497 	smc_state->levels[0].dpm2.AboveSafeInc = 0;
2498 	smc_state->levels[0].dpm2.BelowSafeInc = 0;
2499 	smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2500 
2501 	for (i = 1; i < state->performance_level_count; i++) {
2502 		prev_sclk = state->performance_levels[i-1].sclk;
2503 		max_sclk  = state->performance_levels[i].sclk;
2504 		max_ps_percent = (i != (state->performance_level_count - 1)) ?
2505 			NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2506 
2507 		if (max_sclk < prev_sclk)
2508 			return -EINVAL;
2509 
2510 		if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2511 			min_sclk = max_sclk;
2512 		else if (1 == i)
2513 			min_sclk = prev_sclk;
2514 		else
2515 			min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2516 
2517 		if (min_sclk < state->performance_levels[0].sclk)
2518 			min_sclk = state->performance_levels[0].sclk;
2519 
2520 		if (min_sclk == 0)
2521 			return -EINVAL;
2522 
2523 		smc_state->levels[i].dpm2.MaxPS =
2524 			(u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2525 		smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2526 		smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2527 		smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2528 		smc_state->levels[i].stateFlags |=
2529 			((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2530 			PPSMC_STATEFLAG_POWERBOOST : 0;
2531 	}
2532 
2533 	return 0;
2534 }
2535 
2536 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2537 					 struct radeon_ps *radeon_state,
2538 					 NISLANDS_SMC_SWSTATE *smc_state)
2539 {
2540 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2541 	struct ni_ps *state = ni_get_ps(radeon_state);
2542 	u32 sq_power_throttle;
2543 	u32 sq_power_throttle2;
2544 	bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2545 	int i;
2546 
2547 	if (state->performance_level_count == 0)
2548 		return -EINVAL;
2549 
2550 	if (smc_state->levelCount != state->performance_level_count)
2551 		return -EINVAL;
2552 
2553 	if (rdev->pm.dpm.sq_ramping_threshold == 0)
2554 		return -EINVAL;
2555 
2556 	if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2557 		enable_sq_ramping = false;
2558 
2559 	if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2560 		enable_sq_ramping = false;
2561 
2562 	if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2563 		enable_sq_ramping = false;
2564 
2565 	if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2566 		enable_sq_ramping = false;
2567 
2568 	if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2569 		enable_sq_ramping = false;
2570 
2571 	for (i = 0; i < state->performance_level_count; i++) {
2572 		sq_power_throttle  = 0;
2573 		sq_power_throttle2 = 0;
2574 
2575 		if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2576 		    enable_sq_ramping) {
2577 			sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2578 			sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2579 			sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2580 			sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2581 			sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2582 		} else {
2583 			sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2584 			sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2585 		}
2586 
2587 		smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2588 		smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2589 	}
2590 
2591 	return 0;
2592 }
2593 
2594 static int ni_enable_power_containment(struct radeon_device *rdev,
2595 				       struct radeon_ps *radeon_new_state,
2596 				       bool enable)
2597 {
2598 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2599 	PPSMC_Result smc_result;
2600 	int ret = 0;
2601 
2602 	if (ni_pi->enable_power_containment) {
2603 		if (enable) {
2604 			if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2605 				smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2606 				if (smc_result != PPSMC_Result_OK) {
2607 					ret = -EINVAL;
2608 					ni_pi->pc_enabled = false;
2609 				} else {
2610 					ni_pi->pc_enabled = true;
2611 				}
2612 			}
2613 		} else {
2614 			smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2615 			if (smc_result != PPSMC_Result_OK)
2616 				ret = -EINVAL;
2617 			ni_pi->pc_enabled = false;
2618 		}
2619 	}
2620 
2621 	return ret;
2622 }
2623 
2624 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2625 					 struct radeon_ps *radeon_state,
2626 					 NISLANDS_SMC_SWSTATE *smc_state)
2627 {
2628 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2629 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2630 	struct ni_ps *state = ni_get_ps(radeon_state);
2631 	int i, ret;
2632 	u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2633 
2634 	if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2635 		smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2636 
2637 	smc_state->levelCount = 0;
2638 
2639 	if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2640 		return -EINVAL;
2641 
2642 	for (i = 0; i < state->performance_level_count; i++) {
2643 		ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2644 						    &smc_state->levels[i]);
2645 		smc_state->levels[i].arbRefreshState =
2646 			(u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2647 
2648 		if (ret)
2649 			return ret;
2650 
2651 		if (ni_pi->enable_power_containment)
2652 			smc_state->levels[i].displayWatermark =
2653 				(state->performance_levels[i].sclk < threshold) ?
2654 				PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2655 		else
2656 			smc_state->levels[i].displayWatermark = (i < 2) ?
2657 				PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2658 
2659 		if (eg_pi->dynamic_ac_timing)
2660 			smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2661 		else
2662 			smc_state->levels[i].ACIndex = 0;
2663 
2664 		smc_state->levelCount++;
2665 	}
2666 
2667 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2668 				      cpu_to_be32(threshold / 512));
2669 
2670 	ni_populate_smc_sp(rdev, radeon_state, smc_state);
2671 
2672 	ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2673 	if (ret)
2674 		ni_pi->enable_power_containment = false;
2675 
2676 	ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2677 	if (ret)
2678 		ni_pi->enable_sq_ramping = false;
2679 
2680 	return ni_populate_smc_t(rdev, radeon_state, smc_state);
2681 }
2682 
2683 static int ni_upload_sw_state(struct radeon_device *rdev,
2684 			      struct radeon_ps *radeon_new_state)
2685 {
2686 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2687 	u16 address = pi->state_table_start +
2688 		offsetof(NISLANDS_SMC_STATETABLE, driverState);
2689 	NISLANDS_SMC_SWSTATE *smc_state;
2690 	size_t state_size = struct_size(smc_state, levels,
2691 			NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE);
2692 	int ret;
2693 
2694 	smc_state = kzalloc(state_size, GFP_KERNEL);
2695 	if (smc_state == NULL)
2696 		return -ENOMEM;
2697 
2698 	ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2699 	if (ret)
2700 		goto done;
2701 
2702 	ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2703 
2704 done:
2705 	kfree(smc_state);
2706 
2707 	return ret;
2708 }
2709 
2710 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2711 				       struct ni_mc_reg_table *table)
2712 {
2713 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2714 	u8 i, j, k;
2715 	u32 temp_reg;
2716 
2717 	for (i = 0, j = table->last; i < table->last; i++) {
2718 		switch (table->mc_reg_address[i].s1) {
2719 		case MC_SEQ_MISC1 >> 2:
2720 			if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2721 				return -EINVAL;
2722 			temp_reg = RREG32(MC_PMG_CMD_EMRS);
2723 			table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2724 			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2725 			for (k = 0; k < table->num_entries; k++)
2726 				table->mc_reg_table_entry[k].mc_data[j] =
2727 					((temp_reg & 0xffff0000)) |
2728 					((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2729 			j++;
2730 			if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2731 				return -EINVAL;
2732 
2733 			temp_reg = RREG32(MC_PMG_CMD_MRS);
2734 			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2735 			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2736 			for(k = 0; k < table->num_entries; k++) {
2737 				table->mc_reg_table_entry[k].mc_data[j] =
2738 					(temp_reg & 0xffff0000) |
2739 					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2740 				if (!pi->mem_gddr5)
2741 					table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2742 			}
2743 			j++;
2744 			break;
2745 		case MC_SEQ_RESERVE_M >> 2:
2746 			if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2747 				return -EINVAL;
2748 			temp_reg = RREG32(MC_PMG_CMD_MRS1);
2749 			table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2750 			table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2751 			for (k = 0; k < table->num_entries; k++)
2752 				table->mc_reg_table_entry[k].mc_data[j] =
2753 					(temp_reg & 0xffff0000) |
2754 					(table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2755 			j++;
2756 			break;
2757 		default:
2758 			break;
2759 		}
2760 	}
2761 
2762 	table->last = j;
2763 
2764 	return 0;
2765 }
2766 
2767 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2768 {
2769 	bool result = true;
2770 
2771 	switch (in_reg) {
2772 	case  MC_SEQ_RAS_TIMING >> 2:
2773 		*out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2774 		break;
2775 	case MC_SEQ_CAS_TIMING >> 2:
2776 		*out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2777 		break;
2778 	case MC_SEQ_MISC_TIMING >> 2:
2779 		*out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2780 		break;
2781 	case MC_SEQ_MISC_TIMING2 >> 2:
2782 		*out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2783 		break;
2784 	case MC_SEQ_RD_CTL_D0 >> 2:
2785 		*out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2786 		break;
2787 	case MC_SEQ_RD_CTL_D1 >> 2:
2788 		*out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2789 		break;
2790 	case MC_SEQ_WR_CTL_D0 >> 2:
2791 		*out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2792 		break;
2793 	case MC_SEQ_WR_CTL_D1 >> 2:
2794 		*out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2795 		break;
2796 	case MC_PMG_CMD_EMRS >> 2:
2797 		*out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2798 		break;
2799 	case MC_PMG_CMD_MRS >> 2:
2800 		*out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2801 		break;
2802 	case MC_PMG_CMD_MRS1 >> 2:
2803 		*out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2804 		break;
2805 	case MC_SEQ_PMG_TIMING >> 2:
2806 		*out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2807 		break;
2808 	case MC_PMG_CMD_MRS2 >> 2:
2809 		*out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2810 		break;
2811 	default:
2812 		result = false;
2813 		break;
2814 	}
2815 
2816 	return result;
2817 }
2818 
2819 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2820 {
2821 	u8 i, j;
2822 
2823 	for (i = 0; i < table->last; i++) {
2824 		for (j = 1; j < table->num_entries; j++) {
2825 			if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2826 				table->valid_flag |= 1 << i;
2827 				break;
2828 			}
2829 		}
2830 	}
2831 }
2832 
2833 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2834 {
2835 	u32 i;
2836 	u16 address;
2837 
2838 	for (i = 0; i < table->last; i++)
2839 		table->mc_reg_address[i].s0 =
2840 			ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2841 			address : table->mc_reg_address[i].s1;
2842 }
2843 
2844 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2845 				      struct ni_mc_reg_table *ni_table)
2846 {
2847 	u8 i, j;
2848 
2849 	if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2850 		return -EINVAL;
2851 	if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2852 		return -EINVAL;
2853 
2854 	for (i = 0; i < table->last; i++)
2855 		ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2856 	ni_table->last = table->last;
2857 
2858 	for (i = 0; i < table->num_entries; i++) {
2859 		ni_table->mc_reg_table_entry[i].mclk_max =
2860 			table->mc_reg_table_entry[i].mclk_max;
2861 		for (j = 0; j < table->last; j++)
2862 			ni_table->mc_reg_table_entry[i].mc_data[j] =
2863 				table->mc_reg_table_entry[i].mc_data[j];
2864 	}
2865 	ni_table->num_entries = table->num_entries;
2866 
2867 	return 0;
2868 }
2869 
2870 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2871 {
2872 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2873 	int ret;
2874 	struct atom_mc_reg_table *table;
2875 	struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2876 	u8 module_index = rv770_get_memory_module_index(rdev);
2877 
2878 	table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2879 	if (!table)
2880 		return -ENOMEM;
2881 
2882 	WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2883 	WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2884 	WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2885 	WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2886 	WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2887 	WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2888 	WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2889 	WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2890 	WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2891 	WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2892 	WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2893 	WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2894 	WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2895 
2896 	ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2897 
2898 	if (ret)
2899 		goto init_mc_done;
2900 
2901 	ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2902 
2903 	if (ret)
2904 		goto init_mc_done;
2905 
2906 	ni_set_s0_mc_reg_index(ni_table);
2907 
2908 	ret = ni_set_mc_special_registers(rdev, ni_table);
2909 
2910 	if (ret)
2911 		goto init_mc_done;
2912 
2913 	ni_set_valid_flag(ni_table);
2914 
2915 init_mc_done:
2916 	kfree(table);
2917 
2918 	return ret;
2919 }
2920 
2921 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2922 					 SMC_NIslands_MCRegisters *mc_reg_table)
2923 {
2924 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2925 	u32 i, j;
2926 
2927 	for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2928 		if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2929 			if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2930 				break;
2931 			mc_reg_table->address[i].s0 =
2932 				cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2933 			mc_reg_table->address[i].s1 =
2934 				cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2935 			i++;
2936 		}
2937 	}
2938 	mc_reg_table->last = (u8)i;
2939 }
2940 
2941 
2942 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2943 				    SMC_NIslands_MCRegisterSet *data,
2944 				    u32 num_entries, u32 valid_flag)
2945 {
2946 	u32 i, j;
2947 
2948 	for (i = 0, j = 0; j < num_entries; j++) {
2949 		if (valid_flag & (1 << j)) {
2950 			data->value[i] = cpu_to_be32(entry->mc_data[j]);
2951 			i++;
2952 		}
2953 	}
2954 }
2955 
2956 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2957 						 struct rv7xx_pl *pl,
2958 						 SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2959 {
2960 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2961 	u32 i = 0;
2962 
2963 	for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2964 		if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2965 			break;
2966 	}
2967 
2968 	if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2969 		--i;
2970 
2971 	ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2972 				mc_reg_table_data,
2973 				ni_pi->mc_reg_table.last,
2974 				ni_pi->mc_reg_table.valid_flag);
2975 }
2976 
2977 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2978 					   struct radeon_ps *radeon_state,
2979 					   SMC_NIslands_MCRegisters *mc_reg_table)
2980 {
2981 	struct ni_ps *state = ni_get_ps(radeon_state);
2982 	int i;
2983 
2984 	for (i = 0; i < state->performance_level_count; i++) {
2985 		ni_convert_mc_reg_table_entry_to_smc(rdev,
2986 						     &state->performance_levels[i],
2987 						     &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2988 	}
2989 }
2990 
2991 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2992 				    struct radeon_ps *radeon_boot_state)
2993 {
2994 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2995 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2996 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
2997 	struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
2998 	SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
2999 
3000 	memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3001 
3002 	rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3003 
3004 	ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3005 
3006 	ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3007 					     &mc_reg_table->data[0]);
3008 
3009 	ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3010 				&mc_reg_table->data[1],
3011 				ni_pi->mc_reg_table.last,
3012 				ni_pi->mc_reg_table.valid_flag);
3013 
3014 	ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3015 
3016 	return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3017 				       (u8 *)mc_reg_table,
3018 				       sizeof(SMC_NIslands_MCRegisters),
3019 				       pi->sram_end);
3020 }
3021 
3022 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3023 				  struct radeon_ps *radeon_new_state)
3024 {
3025 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3026 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3027 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
3028 	struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3029 	SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3030 	u16 address;
3031 
3032 	memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3033 
3034 	ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3035 
3036 	address = eg_pi->mc_reg_table_start +
3037 		(u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3038 
3039 	return rv770_copy_bytes_to_smc(rdev, address,
3040 				       (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3041 				       sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3042 				       pi->sram_end);
3043 }
3044 
3045 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3046 						   PP_NIslands_CACTABLES *cac_tables)
3047 {
3048 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
3049 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3050 	u32 leakage = 0;
3051 	unsigned int i, j, table_size;
3052 	s32 t;
3053 	u32 smc_leakage, max_leakage = 0;
3054 	u32 scaling_factor;
3055 
3056 	table_size = eg_pi->vddc_voltage_table.count;
3057 
3058 	if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3059 		table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3060 
3061 	scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3062 
3063 	for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3064 		for (j = 0; j < table_size; j++) {
3065 			t = (1000 * ((i + 1) * 8));
3066 
3067 			if (t < ni_pi->cac_data.leakage_minimum_temperature)
3068 				t = ni_pi->cac_data.leakage_minimum_temperature;
3069 
3070 			ni_calculate_leakage_for_v_and_t(rdev,
3071 							 &ni_pi->cac_data.leakage_coefficients,
3072 							 eg_pi->vddc_voltage_table.entries[j].value,
3073 							 t,
3074 							 ni_pi->cac_data.i_leakage,
3075 							 &leakage);
3076 
3077 			smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3078 			if (smc_leakage > max_leakage)
3079 				max_leakage = smc_leakage;
3080 
3081 			cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3082 		}
3083 	}
3084 
3085 	for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3086 		for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3087 			cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3088 	}
3089 	return 0;
3090 }
3091 
3092 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3093 					    PP_NIslands_CACTABLES *cac_tables)
3094 {
3095 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3096 	struct radeon_cac_leakage_table *leakage_table =
3097 		&rdev->pm.dpm.dyn_state.cac_leakage_table;
3098 	u32 i, j, table_size;
3099 	u32 smc_leakage, max_leakage = 0;
3100 	u32 scaling_factor;
3101 
3102 	if (!leakage_table)
3103 		return -EINVAL;
3104 
3105 	table_size = leakage_table->count;
3106 
3107 	if (eg_pi->vddc_voltage_table.count != table_size)
3108 		table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3109 			eg_pi->vddc_voltage_table.count : leakage_table->count;
3110 
3111 	if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3112 		table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3113 
3114 	if (table_size == 0)
3115 		return -EINVAL;
3116 
3117 	scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3118 
3119 	for (j = 0; j < table_size; j++) {
3120 		smc_leakage = leakage_table->entries[j].leakage;
3121 
3122 		if (smc_leakage > max_leakage)
3123 			max_leakage = smc_leakage;
3124 
3125 		for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3126 			cac_tables->cac_lkge_lut[i][j] =
3127 				cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3128 	}
3129 
3130 	for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3131 		for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3132 			cac_tables->cac_lkge_lut[i][j] =
3133 				cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3134 	}
3135 	return 0;
3136 }
3137 
3138 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3139 {
3140 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3141 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
3142 	PP_NIslands_CACTABLES *cac_tables = NULL;
3143 	int i, ret;
3144 	u32 reg;
3145 
3146 	if (ni_pi->enable_cac == false)
3147 		return 0;
3148 
3149 	cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3150 	if (!cac_tables)
3151 		return -ENOMEM;
3152 
3153 	reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3154 	reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3155 		TID_UNIT(ni_pi->cac_weights->tid_unit));
3156 	WREG32(CG_CAC_CTRL, reg);
3157 
3158 	for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3159 		ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3160 
3161 	for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3162 		cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3163 
3164 	ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3165 	ni_pi->cac_data.pwr_const = 0;
3166 	ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3167 	ni_pi->cac_data.bif_cac_value = 0;
3168 	ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3169 	ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3170 	ni_pi->cac_data.allow_ovrflw = 0;
3171 	ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3172 	ni_pi->cac_data.num_win_tdp = 0;
3173 	ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3174 
3175 	if (ni_pi->driver_calculate_cac_leakage)
3176 		ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3177 	else
3178 		ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3179 
3180 	if (ret)
3181 		goto done_free;
3182 
3183 	cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3184 	cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3185 	cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3186 	cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3187 	cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3188 	cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3189 	cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3190 	cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3191 	cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3192 
3193 	ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3194 				      sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3195 
3196 done_free:
3197 	if (ret) {
3198 		ni_pi->enable_cac = false;
3199 		ni_pi->enable_power_containment = false;
3200 	}
3201 
3202 	kfree(cac_tables);
3203 
3204 	return 0;
3205 }
3206 
3207 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3208 {
3209 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
3210 	u32 reg;
3211 
3212 	if (!ni_pi->enable_cac ||
3213 	    !ni_pi->cac_configuration_required)
3214 		return 0;
3215 
3216 	if (ni_pi->cac_weights == NULL)
3217 		return -EINVAL;
3218 
3219 	reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3220 						      WEIGHT_TCP_SIG1_MASK |
3221 						      WEIGHT_TA_SIG_MASK);
3222 	reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3223 		WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3224 		WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3225 	WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3226 
3227 	reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3228 						      WEIGHT_TCC_EN1_MASK |
3229 						      WEIGHT_TCC_EN2_MASK);
3230 	reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3231 		WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3232 		WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3233 	WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3234 
3235 	reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3236 						      WEIGHT_CB_EN1_MASK |
3237 						      WEIGHT_CB_EN2_MASK |
3238 						      WEIGHT_CB_EN3_MASK);
3239 	reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3240 		WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3241 		WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3242 		WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3243 	WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3244 
3245 	reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3246 						      WEIGHT_DB_SIG1_MASK |
3247 						      WEIGHT_DB_SIG2_MASK |
3248 						      WEIGHT_DB_SIG3_MASK);
3249 	reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3250 		WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3251 		WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3252 		WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3253 	WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3254 
3255 	reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3256 						      WEIGHT_SXM_SIG1_MASK |
3257 						      WEIGHT_SXM_SIG2_MASK |
3258 						      WEIGHT_SXS_SIG0_MASK |
3259 						      WEIGHT_SXS_SIG1_MASK);
3260 	reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3261 		WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3262 		WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3263 		WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3264 		WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3265 	WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3266 
3267 	reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3268 						      WEIGHT_XBR_1_MASK |
3269 						      WEIGHT_XBR_2_MASK |
3270 						      WEIGHT_SPI_SIG0_MASK);
3271 	reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3272 		WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3273 		WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3274 		WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3275 	WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3276 
3277 	reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3278 						      WEIGHT_SPI_SIG2_MASK |
3279 						      WEIGHT_SPI_SIG3_MASK |
3280 						      WEIGHT_SPI_SIG4_MASK |
3281 						      WEIGHT_SPI_SIG5_MASK);
3282 	reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3283 		WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3284 		WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3285 		WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3286 		WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3287 	WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3288 
3289 	reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3290 						      WEIGHT_LDS_SIG1_MASK |
3291 						      WEIGHT_SC_MASK);
3292 	reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3293 		WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3294 		WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3295 	WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3296 
3297 	reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3298 						      WEIGHT_CP_MASK |
3299 						      WEIGHT_PA_SIG0_MASK |
3300 						      WEIGHT_PA_SIG1_MASK |
3301 						      WEIGHT_VGT_SIG0_MASK);
3302 	reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3303 		WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3304 		WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3305 		WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3306 		WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3307 	WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3308 
3309 	reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3310 						      WEIGHT_VGT_SIG2_MASK |
3311 						      WEIGHT_DC_SIG0_MASK |
3312 						      WEIGHT_DC_SIG1_MASK |
3313 						      WEIGHT_DC_SIG2_MASK);
3314 	reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3315 		WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3316 		WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3317 		WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3318 		WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3319 	WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3320 
3321 	reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3322 						      WEIGHT_UVD_SIG0_MASK |
3323 						      WEIGHT_UVD_SIG1_MASK |
3324 						      WEIGHT_SPARE0_MASK |
3325 						      WEIGHT_SPARE1_MASK);
3326 	reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3327 		WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3328 		WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3329 		WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3330 		WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3331 	WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3332 
3333 	reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3334 						      WEIGHT_SQ_VSP0_MASK);
3335 	reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3336 		WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3337 	WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3338 
3339 	reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3340 	reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3341 	WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3342 
3343 	reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3344 							OVR_VAL_SPARE_0_MASK |
3345 							OVR_MODE_SPARE_1_MASK |
3346 							OVR_VAL_SPARE_1_MASK);
3347 	reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3348 		OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3349 		OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3350 		OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3351 	WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3352 
3353 	reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3354 					   VSP0_MASK |
3355 					   GPR_MASK);
3356 	reg |= (VSP(ni_pi->cac_weights->vsp) |
3357 		VSP0(ni_pi->cac_weights->vsp0) |
3358 		GPR(ni_pi->cac_weights->gpr));
3359 	WREG32(SQ_CAC_THRESHOLD, reg);
3360 
3361 	reg = (MCDW_WR_ENABLE |
3362 	       MCDX_WR_ENABLE |
3363 	       MCDY_WR_ENABLE |
3364 	       MCDZ_WR_ENABLE |
3365 	       INDEX(0x09D4));
3366 	WREG32(MC_CG_CONFIG, reg);
3367 
3368 	reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3369 	       WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3370 	       ALLOW_OVERFLOW);
3371 	WREG32(MC_CG_DATAPORT, reg);
3372 
3373 	return 0;
3374 }
3375 
3376 static int ni_enable_smc_cac(struct radeon_device *rdev,
3377 			     struct radeon_ps *radeon_new_state,
3378 			     bool enable)
3379 {
3380 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
3381 	int ret = 0;
3382 	PPSMC_Result smc_result;
3383 
3384 	if (ni_pi->enable_cac) {
3385 		if (enable) {
3386 			if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3387 				smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3388 
3389 				if (ni_pi->support_cac_long_term_average) {
3390 					smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3391 					if (PPSMC_Result_OK != smc_result)
3392 						ni_pi->support_cac_long_term_average = false;
3393 				}
3394 
3395 				smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3396 				if (PPSMC_Result_OK != smc_result)
3397 					ret = -EINVAL;
3398 
3399 				ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3400 			}
3401 		} else if (ni_pi->cac_enabled) {
3402 			smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3403 
3404 			ni_pi->cac_enabled = false;
3405 
3406 			if (ni_pi->support_cac_long_term_average) {
3407 				smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3408 				if (PPSMC_Result_OK != smc_result)
3409 					ni_pi->support_cac_long_term_average = false;
3410 			}
3411 		}
3412 	}
3413 
3414 	return ret;
3415 }
3416 
3417 static int ni_pcie_performance_request(struct radeon_device *rdev,
3418 				       u8 perf_req, bool advertise)
3419 {
3420 #if defined(CONFIG_ACPI)
3421 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3422 
3423 	if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3424 	    (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3425 		if (eg_pi->pcie_performance_request_registered == false)
3426 			radeon_acpi_pcie_notify_device_ready(rdev);
3427 		eg_pi->pcie_performance_request_registered = true;
3428 		return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3429 	} else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3430 		    eg_pi->pcie_performance_request_registered) {
3431 		eg_pi->pcie_performance_request_registered = false;
3432 		return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3433 	}
3434 #endif
3435 	return 0;
3436 }
3437 
3438 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3439 {
3440 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3441 	u32 tmp;
3442 
3443 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3444 
3445 	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3446 	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3447 		pi->pcie_gen2 = true;
3448 	else
3449 		pi->pcie_gen2 = false;
3450 
3451 	if (!pi->pcie_gen2)
3452 		ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3453 
3454 	return 0;
3455 }
3456 
3457 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3458 					    bool enable)
3459 {
3460 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3461 	u32 tmp, bif;
3462 
3463 	tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3464 
3465 	if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3466 	    (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3467 		if (enable) {
3468 			if (!pi->boot_in_gen2) {
3469 				bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3470 				bif |= CG_CLIENT_REQ(0xd);
3471 				WREG32(CG_BIF_REQ_AND_RSP, bif);
3472 			}
3473 			tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3474 			tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3475 			tmp |= LC_GEN2_EN_STRAP;
3476 
3477 			tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3478 			WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3479 			udelay(10);
3480 			tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3481 			WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3482 		} else {
3483 			if (!pi->boot_in_gen2) {
3484 				bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3485 				bif |= CG_CLIENT_REQ(0xd);
3486 				WREG32(CG_BIF_REQ_AND_RSP, bif);
3487 
3488 				tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3489 				tmp &= ~LC_GEN2_EN_STRAP;
3490 			}
3491 			WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3492 		}
3493 	}
3494 }
3495 
3496 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3497 					bool enable)
3498 {
3499 	ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3500 
3501 	if (enable)
3502 		WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3503 	else
3504 		WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3505 }
3506 
3507 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3508 					   struct radeon_ps *new_ps,
3509 					   struct radeon_ps *old_ps)
3510 {
3511 	struct ni_ps *new_state = ni_get_ps(new_ps);
3512 	struct ni_ps *current_state = ni_get_ps(old_ps);
3513 
3514 	if ((new_ps->vclk == old_ps->vclk) &&
3515 	    (new_ps->dclk == old_ps->dclk))
3516 		return;
3517 
3518 	if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3519 	    current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3520 		return;
3521 
3522 	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3523 }
3524 
3525 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3526 					  struct radeon_ps *new_ps,
3527 					  struct radeon_ps *old_ps)
3528 {
3529 	struct ni_ps *new_state = ni_get_ps(new_ps);
3530 	struct ni_ps *current_state = ni_get_ps(old_ps);
3531 
3532 	if ((new_ps->vclk == old_ps->vclk) &&
3533 	    (new_ps->dclk == old_ps->dclk))
3534 		return;
3535 
3536 	if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3537 	    current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3538 		return;
3539 
3540 	radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3541 }
3542 
3543 void ni_dpm_setup_asic(struct radeon_device *rdev)
3544 {
3545 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3546 	int r;
3547 
3548 	r = ni_mc_load_microcode(rdev);
3549 	if (r)
3550 		DRM_ERROR("Failed to load MC firmware!\n");
3551 	ni_read_clock_registers(rdev);
3552 	btc_read_arb_registers(rdev);
3553 	rv770_get_memory_type(rdev);
3554 	if (eg_pi->pcie_performance_request)
3555 		ni_advertise_gen2_capability(rdev);
3556 	rv770_get_pcie_gen2_status(rdev);
3557 	rv770_enable_acpi_pm(rdev);
3558 }
3559 
3560 void ni_update_current_ps(struct radeon_device *rdev,
3561 			  struct radeon_ps *rps)
3562 {
3563 	struct ni_ps *new_ps = ni_get_ps(rps);
3564 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3565 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
3566 
3567 	eg_pi->current_rps = *rps;
3568 	ni_pi->current_ps = *new_ps;
3569 	eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3570 }
3571 
3572 void ni_update_requested_ps(struct radeon_device *rdev,
3573 			    struct radeon_ps *rps)
3574 {
3575 	struct ni_ps *new_ps = ni_get_ps(rps);
3576 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3577 	struct ni_power_info *ni_pi = ni_get_pi(rdev);
3578 
3579 	eg_pi->requested_rps = *rps;
3580 	ni_pi->requested_ps = *new_ps;
3581 	eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3582 }
3583 
3584 int ni_dpm_enable(struct radeon_device *rdev)
3585 {
3586 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3587 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3588 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3589 	int ret;
3590 
3591 	if (pi->gfx_clock_gating)
3592 		ni_cg_clockgating_default(rdev);
3593 	if (btc_dpm_enabled(rdev))
3594 		return -EINVAL;
3595 	if (pi->mg_clock_gating)
3596 		ni_mg_clockgating_default(rdev);
3597 	if (eg_pi->ls_clock_gating)
3598 		ni_ls_clockgating_default(rdev);
3599 	if (pi->voltage_control) {
3600 		rv770_enable_voltage_control(rdev, true);
3601 		ret = cypress_construct_voltage_tables(rdev);
3602 		if (ret) {
3603 			DRM_ERROR("cypress_construct_voltage_tables failed\n");
3604 			return ret;
3605 		}
3606 	}
3607 	if (eg_pi->dynamic_ac_timing) {
3608 		ret = ni_initialize_mc_reg_table(rdev);
3609 		if (ret)
3610 			eg_pi->dynamic_ac_timing = false;
3611 	}
3612 	if (pi->dynamic_ss)
3613 		cypress_enable_spread_spectrum(rdev, true);
3614 	if (pi->thermal_protection)
3615 		rv770_enable_thermal_protection(rdev, true);
3616 	rv770_setup_bsp(rdev);
3617 	rv770_program_git(rdev);
3618 	rv770_program_tp(rdev);
3619 	rv770_program_tpp(rdev);
3620 	rv770_program_sstp(rdev);
3621 	cypress_enable_display_gap(rdev);
3622 	rv770_program_vc(rdev);
3623 	if (pi->dynamic_pcie_gen2)
3624 		ni_enable_dynamic_pcie_gen2(rdev, true);
3625 	ret = rv770_upload_firmware(rdev);
3626 	if (ret) {
3627 		DRM_ERROR("rv770_upload_firmware failed\n");
3628 		return ret;
3629 	}
3630 	ret = ni_process_firmware_header(rdev);
3631 	if (ret) {
3632 		DRM_ERROR("ni_process_firmware_header failed\n");
3633 		return ret;
3634 	}
3635 	ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3636 	if (ret) {
3637 		DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3638 		return ret;
3639 	}
3640 	ret = ni_init_smc_table(rdev);
3641 	if (ret) {
3642 		DRM_ERROR("ni_init_smc_table failed\n");
3643 		return ret;
3644 	}
3645 	ret = ni_init_smc_spll_table(rdev);
3646 	if (ret) {
3647 		DRM_ERROR("ni_init_smc_spll_table failed\n");
3648 		return ret;
3649 	}
3650 	ret = ni_init_arb_table_index(rdev);
3651 	if (ret) {
3652 		DRM_ERROR("ni_init_arb_table_index failed\n");
3653 		return ret;
3654 	}
3655 	if (eg_pi->dynamic_ac_timing) {
3656 		ret = ni_populate_mc_reg_table(rdev, boot_ps);
3657 		if (ret) {
3658 			DRM_ERROR("ni_populate_mc_reg_table failed\n");
3659 			return ret;
3660 		}
3661 	}
3662 	ret = ni_initialize_smc_cac_tables(rdev);
3663 	if (ret) {
3664 		DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3665 		return ret;
3666 	}
3667 	ret = ni_initialize_hardware_cac_manager(rdev);
3668 	if (ret) {
3669 		DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3670 		return ret;
3671 	}
3672 	ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3673 	if (ret) {
3674 		DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3675 		return ret;
3676 	}
3677 	ni_program_response_times(rdev);
3678 	r7xx_start_smc(rdev);
3679 	ret = cypress_notify_smc_display_change(rdev, false);
3680 	if (ret) {
3681 		DRM_ERROR("cypress_notify_smc_display_change failed\n");
3682 		return ret;
3683 	}
3684 	cypress_enable_sclk_control(rdev, true);
3685 	if (eg_pi->memory_transition)
3686 		cypress_enable_mclk_control(rdev, true);
3687 	cypress_start_dpm(rdev);
3688 	if (pi->gfx_clock_gating)
3689 		ni_gfx_clockgating_enable(rdev, true);
3690 	if (pi->mg_clock_gating)
3691 		ni_mg_clockgating_enable(rdev, true);
3692 	if (eg_pi->ls_clock_gating)
3693 		ni_ls_clockgating_enable(rdev, true);
3694 
3695 	rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3696 
3697 	ni_update_current_ps(rdev, boot_ps);
3698 
3699 	return 0;
3700 }
3701 
3702 void ni_dpm_disable(struct radeon_device *rdev)
3703 {
3704 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3705 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3706 	struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3707 
3708 	if (!btc_dpm_enabled(rdev))
3709 		return;
3710 	rv770_clear_vc(rdev);
3711 	if (pi->thermal_protection)
3712 		rv770_enable_thermal_protection(rdev, false);
3713 	ni_enable_power_containment(rdev, boot_ps, false);
3714 	ni_enable_smc_cac(rdev, boot_ps, false);
3715 	cypress_enable_spread_spectrum(rdev, false);
3716 	rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3717 	if (pi->dynamic_pcie_gen2)
3718 		ni_enable_dynamic_pcie_gen2(rdev, false);
3719 
3720 	if (rdev->irq.installed &&
3721 	    r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3722 		rdev->irq.dpm_thermal = false;
3723 		radeon_irq_set(rdev);
3724 	}
3725 
3726 	if (pi->gfx_clock_gating)
3727 		ni_gfx_clockgating_enable(rdev, false);
3728 	if (pi->mg_clock_gating)
3729 		ni_mg_clockgating_enable(rdev, false);
3730 	if (eg_pi->ls_clock_gating)
3731 		ni_ls_clockgating_enable(rdev, false);
3732 	ni_stop_dpm(rdev);
3733 	btc_reset_to_default(rdev);
3734 	ni_stop_smc(rdev);
3735 	ni_force_switch_to_arb_f0(rdev);
3736 
3737 	ni_update_current_ps(rdev, boot_ps);
3738 }
3739 
3740 static int ni_power_control_set_level(struct radeon_device *rdev)
3741 {
3742 	struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3743 	int ret;
3744 
3745 	ret = ni_restrict_performance_levels_before_switch(rdev);
3746 	if (ret)
3747 		return ret;
3748 	ret = rv770_halt_smc(rdev);
3749 	if (ret)
3750 		return ret;
3751 	ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3752 	if (ret)
3753 		return ret;
3754 	ret = rv770_resume_smc(rdev);
3755 	if (ret)
3756 		return ret;
3757 	ret = rv770_set_sw_state(rdev);
3758 	if (ret)
3759 		return ret;
3760 
3761 	return 0;
3762 }
3763 
3764 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3765 {
3766 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3767 	struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3768 	struct radeon_ps *new_ps = &requested_ps;
3769 
3770 	ni_update_requested_ps(rdev, new_ps);
3771 
3772 	ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3773 
3774 	return 0;
3775 }
3776 
3777 int ni_dpm_set_power_state(struct radeon_device *rdev)
3778 {
3779 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3780 	struct radeon_ps *new_ps = &eg_pi->requested_rps;
3781 	struct radeon_ps *old_ps = &eg_pi->current_rps;
3782 	int ret;
3783 
3784 	ret = ni_restrict_performance_levels_before_switch(rdev);
3785 	if (ret) {
3786 		DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3787 		return ret;
3788 	}
3789 	ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3790 	ret = ni_enable_power_containment(rdev, new_ps, false);
3791 	if (ret) {
3792 		DRM_ERROR("ni_enable_power_containment failed\n");
3793 		return ret;
3794 	}
3795 	ret = ni_enable_smc_cac(rdev, new_ps, false);
3796 	if (ret) {
3797 		DRM_ERROR("ni_enable_smc_cac failed\n");
3798 		return ret;
3799 	}
3800 	ret = rv770_halt_smc(rdev);
3801 	if (ret) {
3802 		DRM_ERROR("rv770_halt_smc failed\n");
3803 		return ret;
3804 	}
3805 	if (eg_pi->smu_uvd_hs)
3806 		btc_notify_uvd_to_smc(rdev, new_ps);
3807 	ret = ni_upload_sw_state(rdev, new_ps);
3808 	if (ret) {
3809 		DRM_ERROR("ni_upload_sw_state failed\n");
3810 		return ret;
3811 	}
3812 	if (eg_pi->dynamic_ac_timing) {
3813 		ret = ni_upload_mc_reg_table(rdev, new_ps);
3814 		if (ret) {
3815 			DRM_ERROR("ni_upload_mc_reg_table failed\n");
3816 			return ret;
3817 		}
3818 	}
3819 	ret = ni_program_memory_timing_parameters(rdev, new_ps);
3820 	if (ret) {
3821 		DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3822 		return ret;
3823 	}
3824 	ret = rv770_resume_smc(rdev);
3825 	if (ret) {
3826 		DRM_ERROR("rv770_resume_smc failed\n");
3827 		return ret;
3828 	}
3829 	ret = rv770_set_sw_state(rdev);
3830 	if (ret) {
3831 		DRM_ERROR("rv770_set_sw_state failed\n");
3832 		return ret;
3833 	}
3834 	ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3835 	ret = ni_enable_smc_cac(rdev, new_ps, true);
3836 	if (ret) {
3837 		DRM_ERROR("ni_enable_smc_cac failed\n");
3838 		return ret;
3839 	}
3840 	ret = ni_enable_power_containment(rdev, new_ps, true);
3841 	if (ret) {
3842 		DRM_ERROR("ni_enable_power_containment failed\n");
3843 		return ret;
3844 	}
3845 
3846 	/* update tdp */
3847 	ret = ni_power_control_set_level(rdev);
3848 	if (ret) {
3849 		DRM_ERROR("ni_power_control_set_level failed\n");
3850 		return ret;
3851 	}
3852 
3853 	return 0;
3854 }
3855 
3856 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3857 {
3858 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3859 	struct radeon_ps *new_ps = &eg_pi->requested_rps;
3860 
3861 	ni_update_current_ps(rdev, new_ps);
3862 }
3863 
3864 #if 0
3865 void ni_dpm_reset_asic(struct radeon_device *rdev)
3866 {
3867 	ni_restrict_performance_levels_before_switch(rdev);
3868 	rv770_set_boot_state(rdev);
3869 }
3870 #endif
3871 
3872 union power_info {
3873 	struct _ATOM_POWERPLAY_INFO info;
3874 	struct _ATOM_POWERPLAY_INFO_V2 info_2;
3875 	struct _ATOM_POWERPLAY_INFO_V3 info_3;
3876 	struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3877 	struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3878 	struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3879 };
3880 
3881 union pplib_clock_info {
3882 	struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3883 	struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3884 	struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3885 	struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3886 };
3887 
3888 union pplib_power_state {
3889 	struct _ATOM_PPLIB_STATE v1;
3890 	struct _ATOM_PPLIB_STATE_V2 v2;
3891 };
3892 
3893 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3894 					  struct radeon_ps *rps,
3895 					  struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3896 					  u8 table_rev)
3897 {
3898 	rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3899 	rps->class = le16_to_cpu(non_clock_info->usClassification);
3900 	rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3901 
3902 	if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3903 		rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3904 		rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3905 	} else if (r600_is_uvd_state(rps->class, rps->class2)) {
3906 		rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3907 		rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3908 	} else {
3909 		rps->vclk = 0;
3910 		rps->dclk = 0;
3911 	}
3912 
3913 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3914 		rdev->pm.dpm.boot_ps = rps;
3915 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3916 		rdev->pm.dpm.uvd_ps = rps;
3917 }
3918 
3919 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3920 				      struct radeon_ps *rps, int index,
3921 				      union pplib_clock_info *clock_info)
3922 {
3923 	struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3924 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3925 	struct ni_ps *ps = ni_get_ps(rps);
3926 	struct rv7xx_pl *pl = &ps->performance_levels[index];
3927 
3928 	ps->performance_level_count = index + 1;
3929 
3930 	pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3931 	pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3932 	pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3933 	pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3934 
3935 	pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3936 	pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3937 	pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3938 
3939 	/* patch up vddc if necessary */
3940 	if (pl->vddc == 0xff01) {
3941 		if (pi->max_vddc)
3942 			pl->vddc = pi->max_vddc;
3943 	}
3944 
3945 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3946 		pi->acpi_vddc = pl->vddc;
3947 		eg_pi->acpi_vddci = pl->vddci;
3948 		if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3949 			pi->acpi_pcie_gen2 = true;
3950 		else
3951 			pi->acpi_pcie_gen2 = false;
3952 	}
3953 
3954 	if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3955 		eg_pi->ulv.supported = true;
3956 		eg_pi->ulv.pl = pl;
3957 	}
3958 
3959 	if (pi->min_vddc_in_table > pl->vddc)
3960 		pi->min_vddc_in_table = pl->vddc;
3961 
3962 	if (pi->max_vddc_in_table < pl->vddc)
3963 		pi->max_vddc_in_table = pl->vddc;
3964 
3965 	/* patch up boot state */
3966 	if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3967 		u16 vddc, vddci, mvdd;
3968 		radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3969 		pl->mclk = rdev->clock.default_mclk;
3970 		pl->sclk = rdev->clock.default_sclk;
3971 		pl->vddc = vddc;
3972 		pl->vddci = vddci;
3973 	}
3974 
3975 	if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3976 	    ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3977 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3978 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3979 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3980 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3981 	}
3982 }
3983 
3984 static int ni_parse_power_table(struct radeon_device *rdev)
3985 {
3986 	struct radeon_mode_info *mode_info = &rdev->mode_info;
3987 	struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3988 	union pplib_power_state *power_state;
3989 	int i, j;
3990 	union pplib_clock_info *clock_info;
3991 	union power_info *power_info;
3992 	int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
3993 	u16 data_offset;
3994 	u8 frev, crev;
3995 	struct ni_ps *ps;
3996 
3997 	if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
3998 				   &frev, &crev, &data_offset))
3999 		return -EINVAL;
4000 	power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4001 
4002 	rdev->pm.dpm.ps = kcalloc(power_info->pplib.ucNumStates,
4003 				  sizeof(struct radeon_ps),
4004 				  GFP_KERNEL);
4005 	if (!rdev->pm.dpm.ps)
4006 		return -ENOMEM;
4007 
4008 	for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4009 		power_state = (union pplib_power_state *)
4010 			(mode_info->atom_context->bios + data_offset +
4011 			 le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4012 			 i * power_info->pplib.ucStateEntrySize);
4013 		non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4014 			(mode_info->atom_context->bios + data_offset +
4015 			 le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4016 			 (power_state->v1.ucNonClockStateIndex *
4017 			  power_info->pplib.ucNonClockSize));
4018 		if (power_info->pplib.ucStateEntrySize - 1) {
4019 			u8 *idx;
4020 			ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4021 			if (ps == NULL) {
4022 				kfree(rdev->pm.dpm.ps);
4023 				return -ENOMEM;
4024 			}
4025 			rdev->pm.dpm.ps[i].ps_priv = ps;
4026 			ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4027 							 non_clock_info,
4028 							 power_info->pplib.ucNonClockSize);
4029 			idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4030 			for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4031 				clock_info = (union pplib_clock_info *)
4032 					(mode_info->atom_context->bios + data_offset +
4033 					 le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4034 					 (idx[j] * power_info->pplib.ucClockInfoSize));
4035 				ni_parse_pplib_clock_info(rdev,
4036 							  &rdev->pm.dpm.ps[i], j,
4037 							  clock_info);
4038 			}
4039 		}
4040 	}
4041 	rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4042 	return 0;
4043 }
4044 
4045 int ni_dpm_init(struct radeon_device *rdev)
4046 {
4047 	struct rv7xx_power_info *pi;
4048 	struct evergreen_power_info *eg_pi;
4049 	struct ni_power_info *ni_pi;
4050 	struct atom_clock_dividers dividers;
4051 	int ret;
4052 
4053 	ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4054 	if (ni_pi == NULL)
4055 		return -ENOMEM;
4056 	rdev->pm.dpm.priv = ni_pi;
4057 	eg_pi = &ni_pi->eg;
4058 	pi = &eg_pi->rv7xx;
4059 
4060 	rv770_get_max_vddc(rdev);
4061 
4062 	eg_pi->ulv.supported = false;
4063 	pi->acpi_vddc = 0;
4064 	eg_pi->acpi_vddci = 0;
4065 	pi->min_vddc_in_table = 0;
4066 	pi->max_vddc_in_table = 0;
4067 
4068 	ret = r600_get_platform_caps(rdev);
4069 	if (ret)
4070 		return ret;
4071 
4072 	ret = ni_parse_power_table(rdev);
4073 	if (ret)
4074 		return ret;
4075 	ret = r600_parse_extended_power_table(rdev);
4076 	if (ret)
4077 		return ret;
4078 
4079 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4080 		kcalloc(4,
4081 			sizeof(struct radeon_clock_voltage_dependency_entry),
4082 			GFP_KERNEL);
4083 	if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4084 		r600_free_extended_power_table(rdev);
4085 		return -ENOMEM;
4086 	}
4087 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4088 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4089 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4090 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4091 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4092 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4093 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4094 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4095 	rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4096 
4097 	ni_patch_dependency_tables_based_on_leakage(rdev);
4098 
4099 	if (rdev->pm.dpm.voltage_response_time == 0)
4100 		rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4101 	if (rdev->pm.dpm.backbias_response_time == 0)
4102 		rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4103 
4104 	ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4105 					     0, false, &dividers);
4106 	if (ret)
4107 		pi->ref_div = dividers.ref_div + 1;
4108 	else
4109 		pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4110 
4111 	pi->rlp = RV770_RLP_DFLT;
4112 	pi->rmp = RV770_RMP_DFLT;
4113 	pi->lhp = RV770_LHP_DFLT;
4114 	pi->lmp = RV770_LMP_DFLT;
4115 
4116 	eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4117 	eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4118 	eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4119 	eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4120 
4121 	eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4122 	eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4123 	eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4124 	eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4125 
4126 	eg_pi->smu_uvd_hs = true;
4127 
4128 	if (rdev->pdev->device == 0x6707) {
4129 		pi->mclk_strobe_mode_threshold = 55000;
4130 		pi->mclk_edc_enable_threshold = 55000;
4131 		eg_pi->mclk_edc_wr_enable_threshold = 55000;
4132 	} else {
4133 		pi->mclk_strobe_mode_threshold = 40000;
4134 		pi->mclk_edc_enable_threshold = 40000;
4135 		eg_pi->mclk_edc_wr_enable_threshold = 40000;
4136 	}
4137 	ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4138 
4139 	pi->voltage_control =
4140 		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4141 
4142 	pi->mvdd_control =
4143 		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4144 
4145 	eg_pi->vddci_control =
4146 		radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4147 
4148 	rv770_get_engine_memory_ss(rdev);
4149 
4150 	pi->asi = RV770_ASI_DFLT;
4151 	pi->pasi = CYPRESS_HASI_DFLT;
4152 	pi->vrc = CYPRESS_VRC_DFLT;
4153 
4154 	pi->power_gating = false;
4155 
4156 	pi->gfx_clock_gating = true;
4157 
4158 	pi->mg_clock_gating = true;
4159 	pi->mgcgtssm = true;
4160 	eg_pi->ls_clock_gating = false;
4161 	eg_pi->sclk_deep_sleep = false;
4162 
4163 	pi->dynamic_pcie_gen2 = true;
4164 
4165 	if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4166 		pi->thermal_protection = true;
4167 	else
4168 		pi->thermal_protection = false;
4169 
4170 	pi->display_gap = true;
4171 
4172 	pi->dcodt = true;
4173 
4174 	pi->ulps = true;
4175 
4176 	eg_pi->dynamic_ac_timing = true;
4177 	eg_pi->abm = true;
4178 	eg_pi->mcls = true;
4179 	eg_pi->light_sleep = true;
4180 	eg_pi->memory_transition = true;
4181 #if defined(CONFIG_ACPI)
4182 	eg_pi->pcie_performance_request =
4183 		radeon_acpi_is_pcie_performance_request_supported(rdev);
4184 #else
4185 	eg_pi->pcie_performance_request = false;
4186 #endif
4187 
4188 	eg_pi->dll_default_on = false;
4189 
4190 	eg_pi->sclk_deep_sleep = false;
4191 
4192 	pi->mclk_stutter_mode_threshold = 0;
4193 
4194 	pi->sram_end = SMC_RAM_END;
4195 
4196 	rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4197 	rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4198 	rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4199 	rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4200 	rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4201 	rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4202 	rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4203 	rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4204 
4205 	ni_pi->cac_data.leakage_coefficients.at = 516;
4206 	ni_pi->cac_data.leakage_coefficients.bt = 18;
4207 	ni_pi->cac_data.leakage_coefficients.av = 51;
4208 	ni_pi->cac_data.leakage_coefficients.bv = 2957;
4209 
4210 	switch (rdev->pdev->device) {
4211 	case 0x6700:
4212 	case 0x6701:
4213 	case 0x6702:
4214 	case 0x6703:
4215 	case 0x6718:
4216 		ni_pi->cac_weights = &cac_weights_cayman_xt;
4217 		break;
4218 	case 0x6705:
4219 	case 0x6719:
4220 	case 0x671D:
4221 	case 0x671C:
4222 	default:
4223 		ni_pi->cac_weights = &cac_weights_cayman_pro;
4224 		break;
4225 	case 0x6704:
4226 	case 0x6706:
4227 	case 0x6707:
4228 	case 0x6708:
4229 	case 0x6709:
4230 		ni_pi->cac_weights = &cac_weights_cayman_le;
4231 		break;
4232 	}
4233 
4234 	if (ni_pi->cac_weights->enable_power_containment_by_default) {
4235 		ni_pi->enable_power_containment = true;
4236 		ni_pi->enable_cac = true;
4237 		ni_pi->enable_sq_ramping = true;
4238 	} else {
4239 		ni_pi->enable_power_containment = false;
4240 		ni_pi->enable_cac = false;
4241 		ni_pi->enable_sq_ramping = false;
4242 	}
4243 
4244 	ni_pi->driver_calculate_cac_leakage = false;
4245 	ni_pi->cac_configuration_required = true;
4246 
4247 	if (ni_pi->cac_configuration_required) {
4248 		ni_pi->support_cac_long_term_average = true;
4249 		ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4250 		ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4251 	} else {
4252 		ni_pi->support_cac_long_term_average = false;
4253 		ni_pi->lta_window_size = 0;
4254 		ni_pi->lts_truncate = 0;
4255 	}
4256 
4257 	ni_pi->use_power_boost_limit = true;
4258 
4259 	/* make sure dc limits are valid */
4260 	if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4261 	    (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4262 		rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4263 			rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4264 
4265 	return 0;
4266 }
4267 
4268 void ni_dpm_fini(struct radeon_device *rdev)
4269 {
4270 	int i;
4271 
4272 	for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4273 		kfree(rdev->pm.dpm.ps[i].ps_priv);
4274 	}
4275 	kfree(rdev->pm.dpm.ps);
4276 	kfree(rdev->pm.dpm.priv);
4277 	kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4278 	r600_free_extended_power_table(rdev);
4279 }
4280 
4281 void ni_dpm_print_power_state(struct radeon_device *rdev,
4282 			      struct radeon_ps *rps)
4283 {
4284 	struct ni_ps *ps = ni_get_ps(rps);
4285 	struct rv7xx_pl *pl;
4286 	int i;
4287 
4288 	r600_dpm_print_class_info(rps->class, rps->class2);
4289 	r600_dpm_print_cap_info(rps->caps);
4290 	printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4291 	for (i = 0; i < ps->performance_level_count; i++) {
4292 		pl = &ps->performance_levels[i];
4293 		if (rdev->family >= CHIP_TAHITI)
4294 			printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4295 			       i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4296 		else
4297 			printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4298 			       i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4299 	}
4300 	r600_dpm_print_ps_status(rdev, rps);
4301 }
4302 
4303 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4304 						    struct seq_file *m)
4305 {
4306 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4307 	struct radeon_ps *rps = &eg_pi->current_rps;
4308 	struct ni_ps *ps = ni_get_ps(rps);
4309 	struct rv7xx_pl *pl;
4310 	u32 current_index =
4311 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4312 		CURRENT_STATE_INDEX_SHIFT;
4313 
4314 	if (current_index >= ps->performance_level_count) {
4315 		seq_printf(m, "invalid dpm profile %d\n", current_index);
4316 	} else {
4317 		pl = &ps->performance_levels[current_index];
4318 		seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4319 		seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4320 			   current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4321 	}
4322 }
4323 
4324 u32 ni_dpm_get_current_sclk(struct radeon_device *rdev)
4325 {
4326 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4327 	struct radeon_ps *rps = &eg_pi->current_rps;
4328 	struct ni_ps *ps = ni_get_ps(rps);
4329 	struct rv7xx_pl *pl;
4330 	u32 current_index =
4331 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4332 		CURRENT_STATE_INDEX_SHIFT;
4333 
4334 	if (current_index >= ps->performance_level_count) {
4335 		return 0;
4336 	} else {
4337 		pl = &ps->performance_levels[current_index];
4338 		return pl->sclk;
4339 	}
4340 }
4341 
4342 u32 ni_dpm_get_current_mclk(struct radeon_device *rdev)
4343 {
4344 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4345 	struct radeon_ps *rps = &eg_pi->current_rps;
4346 	struct ni_ps *ps = ni_get_ps(rps);
4347 	struct rv7xx_pl *pl;
4348 	u32 current_index =
4349 		(RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4350 		CURRENT_STATE_INDEX_SHIFT;
4351 
4352 	if (current_index >= ps->performance_level_count) {
4353 		return 0;
4354 	} else {
4355 		pl = &ps->performance_levels[current_index];
4356 		return pl->mclk;
4357 	}
4358 }
4359 
4360 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4361 {
4362 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4363 	struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4364 
4365 	if (low)
4366 		return requested_state->performance_levels[0].sclk;
4367 	else
4368 		return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4369 }
4370 
4371 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4372 {
4373 	struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4374 	struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4375 
4376 	if (low)
4377 		return requested_state->performance_levels[0].mclk;
4378 	else
4379 		return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4380 }
4381 
4382