xref: /openbmc/linux/drivers/gpu/drm/radeon/si.c (revision a2cce7a9)
1 /*
2  * Copyright 2011 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include "radeon_audio.h"
31 #include <drm/radeon_drm.h>
32 #include "sid.h"
33 #include "atom.h"
34 #include "si_blit_shaders.h"
35 #include "clearstate_si.h"
36 #include "radeon_ucode.h"
37 
38 
39 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
40 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
41 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
42 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
43 MODULE_FIRMWARE("radeon/TAHITI_mc2.bin");
44 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
45 MODULE_FIRMWARE("radeon/TAHITI_smc.bin");
46 
47 MODULE_FIRMWARE("radeon/tahiti_pfp.bin");
48 MODULE_FIRMWARE("radeon/tahiti_me.bin");
49 MODULE_FIRMWARE("radeon/tahiti_ce.bin");
50 MODULE_FIRMWARE("radeon/tahiti_mc.bin");
51 MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
52 MODULE_FIRMWARE("radeon/tahiti_smc.bin");
53 
54 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
55 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
56 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
57 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
58 MODULE_FIRMWARE("radeon/PITCAIRN_mc2.bin");
59 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
60 MODULE_FIRMWARE("radeon/PITCAIRN_smc.bin");
61 
62 MODULE_FIRMWARE("radeon/pitcairn_pfp.bin");
63 MODULE_FIRMWARE("radeon/pitcairn_me.bin");
64 MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
65 MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
66 MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
67 MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
68 
69 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
70 MODULE_FIRMWARE("radeon/VERDE_me.bin");
71 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
72 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
73 MODULE_FIRMWARE("radeon/VERDE_mc2.bin");
74 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
75 MODULE_FIRMWARE("radeon/VERDE_smc.bin");
76 
77 MODULE_FIRMWARE("radeon/verde_pfp.bin");
78 MODULE_FIRMWARE("radeon/verde_me.bin");
79 MODULE_FIRMWARE("radeon/verde_ce.bin");
80 MODULE_FIRMWARE("radeon/verde_mc.bin");
81 MODULE_FIRMWARE("radeon/verde_rlc.bin");
82 MODULE_FIRMWARE("radeon/verde_smc.bin");
83 
84 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
85 MODULE_FIRMWARE("radeon/OLAND_me.bin");
86 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
87 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
88 MODULE_FIRMWARE("radeon/OLAND_mc2.bin");
89 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
90 MODULE_FIRMWARE("radeon/OLAND_smc.bin");
91 
92 MODULE_FIRMWARE("radeon/oland_pfp.bin");
93 MODULE_FIRMWARE("radeon/oland_me.bin");
94 MODULE_FIRMWARE("radeon/oland_ce.bin");
95 MODULE_FIRMWARE("radeon/oland_mc.bin");
96 MODULE_FIRMWARE("radeon/oland_rlc.bin");
97 MODULE_FIRMWARE("radeon/oland_smc.bin");
98 
99 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
100 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
101 MODULE_FIRMWARE("radeon/HAINAN_ce.bin");
102 MODULE_FIRMWARE("radeon/HAINAN_mc.bin");
103 MODULE_FIRMWARE("radeon/HAINAN_mc2.bin");
104 MODULE_FIRMWARE("radeon/HAINAN_rlc.bin");
105 MODULE_FIRMWARE("radeon/HAINAN_smc.bin");
106 
107 MODULE_FIRMWARE("radeon/hainan_pfp.bin");
108 MODULE_FIRMWARE("radeon/hainan_me.bin");
109 MODULE_FIRMWARE("radeon/hainan_ce.bin");
110 MODULE_FIRMWARE("radeon/hainan_mc.bin");
111 MODULE_FIRMWARE("radeon/hainan_rlc.bin");
112 MODULE_FIRMWARE("radeon/hainan_smc.bin");
113 
114 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
115 static void si_pcie_gen3_enable(struct radeon_device *rdev);
116 static void si_program_aspm(struct radeon_device *rdev);
117 extern void sumo_rlc_fini(struct radeon_device *rdev);
118 extern int sumo_rlc_init(struct radeon_device *rdev);
119 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
120 extern void r600_ih_ring_fini(struct radeon_device *rdev);
121 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
122 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
123 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
124 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
125 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
126 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
127 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
128 					 bool enable);
129 static void si_init_pg(struct radeon_device *rdev);
130 static void si_init_cg(struct radeon_device *rdev);
131 static void si_fini_pg(struct radeon_device *rdev);
132 static void si_fini_cg(struct radeon_device *rdev);
133 static void si_rlc_stop(struct radeon_device *rdev);
134 
135 static const u32 verde_rlc_save_restore_register_list[] =
136 {
137 	(0x8000 << 16) | (0x98f4 >> 2),
138 	0x00000000,
139 	(0x8040 << 16) | (0x98f4 >> 2),
140 	0x00000000,
141 	(0x8000 << 16) | (0xe80 >> 2),
142 	0x00000000,
143 	(0x8040 << 16) | (0xe80 >> 2),
144 	0x00000000,
145 	(0x8000 << 16) | (0x89bc >> 2),
146 	0x00000000,
147 	(0x8040 << 16) | (0x89bc >> 2),
148 	0x00000000,
149 	(0x8000 << 16) | (0x8c1c >> 2),
150 	0x00000000,
151 	(0x8040 << 16) | (0x8c1c >> 2),
152 	0x00000000,
153 	(0x9c00 << 16) | (0x98f0 >> 2),
154 	0x00000000,
155 	(0x9c00 << 16) | (0xe7c >> 2),
156 	0x00000000,
157 	(0x8000 << 16) | (0x9148 >> 2),
158 	0x00000000,
159 	(0x8040 << 16) | (0x9148 >> 2),
160 	0x00000000,
161 	(0x9c00 << 16) | (0x9150 >> 2),
162 	0x00000000,
163 	(0x9c00 << 16) | (0x897c >> 2),
164 	0x00000000,
165 	(0x9c00 << 16) | (0x8d8c >> 2),
166 	0x00000000,
167 	(0x9c00 << 16) | (0xac54 >> 2),
168 	0X00000000,
169 	0x3,
170 	(0x9c00 << 16) | (0x98f8 >> 2),
171 	0x00000000,
172 	(0x9c00 << 16) | (0x9910 >> 2),
173 	0x00000000,
174 	(0x9c00 << 16) | (0x9914 >> 2),
175 	0x00000000,
176 	(0x9c00 << 16) | (0x9918 >> 2),
177 	0x00000000,
178 	(0x9c00 << 16) | (0x991c >> 2),
179 	0x00000000,
180 	(0x9c00 << 16) | (0x9920 >> 2),
181 	0x00000000,
182 	(0x9c00 << 16) | (0x9924 >> 2),
183 	0x00000000,
184 	(0x9c00 << 16) | (0x9928 >> 2),
185 	0x00000000,
186 	(0x9c00 << 16) | (0x992c >> 2),
187 	0x00000000,
188 	(0x9c00 << 16) | (0x9930 >> 2),
189 	0x00000000,
190 	(0x9c00 << 16) | (0x9934 >> 2),
191 	0x00000000,
192 	(0x9c00 << 16) | (0x9938 >> 2),
193 	0x00000000,
194 	(0x9c00 << 16) | (0x993c >> 2),
195 	0x00000000,
196 	(0x9c00 << 16) | (0x9940 >> 2),
197 	0x00000000,
198 	(0x9c00 << 16) | (0x9944 >> 2),
199 	0x00000000,
200 	(0x9c00 << 16) | (0x9948 >> 2),
201 	0x00000000,
202 	(0x9c00 << 16) | (0x994c >> 2),
203 	0x00000000,
204 	(0x9c00 << 16) | (0x9950 >> 2),
205 	0x00000000,
206 	(0x9c00 << 16) | (0x9954 >> 2),
207 	0x00000000,
208 	(0x9c00 << 16) | (0x9958 >> 2),
209 	0x00000000,
210 	(0x9c00 << 16) | (0x995c >> 2),
211 	0x00000000,
212 	(0x9c00 << 16) | (0x9960 >> 2),
213 	0x00000000,
214 	(0x9c00 << 16) | (0x9964 >> 2),
215 	0x00000000,
216 	(0x9c00 << 16) | (0x9968 >> 2),
217 	0x00000000,
218 	(0x9c00 << 16) | (0x996c >> 2),
219 	0x00000000,
220 	(0x9c00 << 16) | (0x9970 >> 2),
221 	0x00000000,
222 	(0x9c00 << 16) | (0x9974 >> 2),
223 	0x00000000,
224 	(0x9c00 << 16) | (0x9978 >> 2),
225 	0x00000000,
226 	(0x9c00 << 16) | (0x997c >> 2),
227 	0x00000000,
228 	(0x9c00 << 16) | (0x9980 >> 2),
229 	0x00000000,
230 	(0x9c00 << 16) | (0x9984 >> 2),
231 	0x00000000,
232 	(0x9c00 << 16) | (0x9988 >> 2),
233 	0x00000000,
234 	(0x9c00 << 16) | (0x998c >> 2),
235 	0x00000000,
236 	(0x9c00 << 16) | (0x8c00 >> 2),
237 	0x00000000,
238 	(0x9c00 << 16) | (0x8c14 >> 2),
239 	0x00000000,
240 	(0x9c00 << 16) | (0x8c04 >> 2),
241 	0x00000000,
242 	(0x9c00 << 16) | (0x8c08 >> 2),
243 	0x00000000,
244 	(0x8000 << 16) | (0x9b7c >> 2),
245 	0x00000000,
246 	(0x8040 << 16) | (0x9b7c >> 2),
247 	0x00000000,
248 	(0x8000 << 16) | (0xe84 >> 2),
249 	0x00000000,
250 	(0x8040 << 16) | (0xe84 >> 2),
251 	0x00000000,
252 	(0x8000 << 16) | (0x89c0 >> 2),
253 	0x00000000,
254 	(0x8040 << 16) | (0x89c0 >> 2),
255 	0x00000000,
256 	(0x8000 << 16) | (0x914c >> 2),
257 	0x00000000,
258 	(0x8040 << 16) | (0x914c >> 2),
259 	0x00000000,
260 	(0x8000 << 16) | (0x8c20 >> 2),
261 	0x00000000,
262 	(0x8040 << 16) | (0x8c20 >> 2),
263 	0x00000000,
264 	(0x8000 << 16) | (0x9354 >> 2),
265 	0x00000000,
266 	(0x8040 << 16) | (0x9354 >> 2),
267 	0x00000000,
268 	(0x9c00 << 16) | (0x9060 >> 2),
269 	0x00000000,
270 	(0x9c00 << 16) | (0x9364 >> 2),
271 	0x00000000,
272 	(0x9c00 << 16) | (0x9100 >> 2),
273 	0x00000000,
274 	(0x9c00 << 16) | (0x913c >> 2),
275 	0x00000000,
276 	(0x8000 << 16) | (0x90e0 >> 2),
277 	0x00000000,
278 	(0x8000 << 16) | (0x90e4 >> 2),
279 	0x00000000,
280 	(0x8000 << 16) | (0x90e8 >> 2),
281 	0x00000000,
282 	(0x8040 << 16) | (0x90e0 >> 2),
283 	0x00000000,
284 	(0x8040 << 16) | (0x90e4 >> 2),
285 	0x00000000,
286 	(0x8040 << 16) | (0x90e8 >> 2),
287 	0x00000000,
288 	(0x9c00 << 16) | (0x8bcc >> 2),
289 	0x00000000,
290 	(0x9c00 << 16) | (0x8b24 >> 2),
291 	0x00000000,
292 	(0x9c00 << 16) | (0x88c4 >> 2),
293 	0x00000000,
294 	(0x9c00 << 16) | (0x8e50 >> 2),
295 	0x00000000,
296 	(0x9c00 << 16) | (0x8c0c >> 2),
297 	0x00000000,
298 	(0x9c00 << 16) | (0x8e58 >> 2),
299 	0x00000000,
300 	(0x9c00 << 16) | (0x8e5c >> 2),
301 	0x00000000,
302 	(0x9c00 << 16) | (0x9508 >> 2),
303 	0x00000000,
304 	(0x9c00 << 16) | (0x950c >> 2),
305 	0x00000000,
306 	(0x9c00 << 16) | (0x9494 >> 2),
307 	0x00000000,
308 	(0x9c00 << 16) | (0xac0c >> 2),
309 	0x00000000,
310 	(0x9c00 << 16) | (0xac10 >> 2),
311 	0x00000000,
312 	(0x9c00 << 16) | (0xac14 >> 2),
313 	0x00000000,
314 	(0x9c00 << 16) | (0xae00 >> 2),
315 	0x00000000,
316 	(0x9c00 << 16) | (0xac08 >> 2),
317 	0x00000000,
318 	(0x9c00 << 16) | (0x88d4 >> 2),
319 	0x00000000,
320 	(0x9c00 << 16) | (0x88c8 >> 2),
321 	0x00000000,
322 	(0x9c00 << 16) | (0x88cc >> 2),
323 	0x00000000,
324 	(0x9c00 << 16) | (0x89b0 >> 2),
325 	0x00000000,
326 	(0x9c00 << 16) | (0x8b10 >> 2),
327 	0x00000000,
328 	(0x9c00 << 16) | (0x8a14 >> 2),
329 	0x00000000,
330 	(0x9c00 << 16) | (0x9830 >> 2),
331 	0x00000000,
332 	(0x9c00 << 16) | (0x9834 >> 2),
333 	0x00000000,
334 	(0x9c00 << 16) | (0x9838 >> 2),
335 	0x00000000,
336 	(0x9c00 << 16) | (0x9a10 >> 2),
337 	0x00000000,
338 	(0x8000 << 16) | (0x9870 >> 2),
339 	0x00000000,
340 	(0x8000 << 16) | (0x9874 >> 2),
341 	0x00000000,
342 	(0x8001 << 16) | (0x9870 >> 2),
343 	0x00000000,
344 	(0x8001 << 16) | (0x9874 >> 2),
345 	0x00000000,
346 	(0x8040 << 16) | (0x9870 >> 2),
347 	0x00000000,
348 	(0x8040 << 16) | (0x9874 >> 2),
349 	0x00000000,
350 	(0x8041 << 16) | (0x9870 >> 2),
351 	0x00000000,
352 	(0x8041 << 16) | (0x9874 >> 2),
353 	0x00000000,
354 	0x00000000
355 };
356 
357 static const u32 tahiti_golden_rlc_registers[] =
358 {
359 	0xc424, 0xffffffff, 0x00601005,
360 	0xc47c, 0xffffffff, 0x10104040,
361 	0xc488, 0xffffffff, 0x0100000a,
362 	0xc314, 0xffffffff, 0x00000800,
363 	0xc30c, 0xffffffff, 0x800000f4,
364 	0xf4a8, 0xffffffff, 0x00000000
365 };
366 
367 static const u32 tahiti_golden_registers[] =
368 {
369 	0x9a10, 0x00010000, 0x00018208,
370 	0x9830, 0xffffffff, 0x00000000,
371 	0x9834, 0xf00fffff, 0x00000400,
372 	0x9838, 0x0002021c, 0x00020200,
373 	0xc78, 0x00000080, 0x00000000,
374 	0xd030, 0x000300c0, 0x00800040,
375 	0xd830, 0x000300c0, 0x00800040,
376 	0x5bb0, 0x000000f0, 0x00000070,
377 	0x5bc0, 0x00200000, 0x50100000,
378 	0x7030, 0x31000311, 0x00000011,
379 	0x277c, 0x00000003, 0x000007ff,
380 	0x240c, 0x000007ff, 0x00000000,
381 	0x8a14, 0xf000001f, 0x00000007,
382 	0x8b24, 0xffffffff, 0x00ffffff,
383 	0x8b10, 0x0000ff0f, 0x00000000,
384 	0x28a4c, 0x07ffffff, 0x4e000000,
385 	0x28350, 0x3f3f3fff, 0x2a00126a,
386 	0x30, 0x000000ff, 0x0040,
387 	0x34, 0x00000040, 0x00004040,
388 	0x9100, 0x07ffffff, 0x03000000,
389 	0x8e88, 0x01ff1f3f, 0x00000000,
390 	0x8e84, 0x01ff1f3f, 0x00000000,
391 	0x9060, 0x0000007f, 0x00000020,
392 	0x9508, 0x00010000, 0x00010000,
393 	0xac14, 0x00000200, 0x000002fb,
394 	0xac10, 0xffffffff, 0x0000543b,
395 	0xac0c, 0xffffffff, 0xa9210876,
396 	0x88d0, 0xffffffff, 0x000fff40,
397 	0x88d4, 0x0000001f, 0x00000010,
398 	0x1410, 0x20000000, 0x20fffed8,
399 	0x15c0, 0x000c0fc0, 0x000c0400
400 };
401 
402 static const u32 tahiti_golden_registers2[] =
403 {
404 	0xc64, 0x00000001, 0x00000001
405 };
406 
407 static const u32 pitcairn_golden_rlc_registers[] =
408 {
409 	0xc424, 0xffffffff, 0x00601004,
410 	0xc47c, 0xffffffff, 0x10102020,
411 	0xc488, 0xffffffff, 0x01000020,
412 	0xc314, 0xffffffff, 0x00000800,
413 	0xc30c, 0xffffffff, 0x800000a4
414 };
415 
416 static const u32 pitcairn_golden_registers[] =
417 {
418 	0x9a10, 0x00010000, 0x00018208,
419 	0x9830, 0xffffffff, 0x00000000,
420 	0x9834, 0xf00fffff, 0x00000400,
421 	0x9838, 0x0002021c, 0x00020200,
422 	0xc78, 0x00000080, 0x00000000,
423 	0xd030, 0x000300c0, 0x00800040,
424 	0xd830, 0x000300c0, 0x00800040,
425 	0x5bb0, 0x000000f0, 0x00000070,
426 	0x5bc0, 0x00200000, 0x50100000,
427 	0x7030, 0x31000311, 0x00000011,
428 	0x2ae4, 0x00073ffe, 0x000022a2,
429 	0x240c, 0x000007ff, 0x00000000,
430 	0x8a14, 0xf000001f, 0x00000007,
431 	0x8b24, 0xffffffff, 0x00ffffff,
432 	0x8b10, 0x0000ff0f, 0x00000000,
433 	0x28a4c, 0x07ffffff, 0x4e000000,
434 	0x28350, 0x3f3f3fff, 0x2a00126a,
435 	0x30, 0x000000ff, 0x0040,
436 	0x34, 0x00000040, 0x00004040,
437 	0x9100, 0x07ffffff, 0x03000000,
438 	0x9060, 0x0000007f, 0x00000020,
439 	0x9508, 0x00010000, 0x00010000,
440 	0xac14, 0x000003ff, 0x000000f7,
441 	0xac10, 0xffffffff, 0x00000000,
442 	0xac0c, 0xffffffff, 0x32761054,
443 	0x88d4, 0x0000001f, 0x00000010,
444 	0x15c0, 0x000c0fc0, 0x000c0400
445 };
446 
447 static const u32 verde_golden_rlc_registers[] =
448 {
449 	0xc424, 0xffffffff, 0x033f1005,
450 	0xc47c, 0xffffffff, 0x10808020,
451 	0xc488, 0xffffffff, 0x00800008,
452 	0xc314, 0xffffffff, 0x00001000,
453 	0xc30c, 0xffffffff, 0x80010014
454 };
455 
456 static const u32 verde_golden_registers[] =
457 {
458 	0x9a10, 0x00010000, 0x00018208,
459 	0x9830, 0xffffffff, 0x00000000,
460 	0x9834, 0xf00fffff, 0x00000400,
461 	0x9838, 0x0002021c, 0x00020200,
462 	0xc78, 0x00000080, 0x00000000,
463 	0xd030, 0x000300c0, 0x00800040,
464 	0xd030, 0x000300c0, 0x00800040,
465 	0xd830, 0x000300c0, 0x00800040,
466 	0xd830, 0x000300c0, 0x00800040,
467 	0x5bb0, 0x000000f0, 0x00000070,
468 	0x5bc0, 0x00200000, 0x50100000,
469 	0x7030, 0x31000311, 0x00000011,
470 	0x2ae4, 0x00073ffe, 0x000022a2,
471 	0x2ae4, 0x00073ffe, 0x000022a2,
472 	0x2ae4, 0x00073ffe, 0x000022a2,
473 	0x240c, 0x000007ff, 0x00000000,
474 	0x240c, 0x000007ff, 0x00000000,
475 	0x240c, 0x000007ff, 0x00000000,
476 	0x8a14, 0xf000001f, 0x00000007,
477 	0x8a14, 0xf000001f, 0x00000007,
478 	0x8a14, 0xf000001f, 0x00000007,
479 	0x8b24, 0xffffffff, 0x00ffffff,
480 	0x8b10, 0x0000ff0f, 0x00000000,
481 	0x28a4c, 0x07ffffff, 0x4e000000,
482 	0x28350, 0x3f3f3fff, 0x0000124a,
483 	0x28350, 0x3f3f3fff, 0x0000124a,
484 	0x28350, 0x3f3f3fff, 0x0000124a,
485 	0x30, 0x000000ff, 0x0040,
486 	0x34, 0x00000040, 0x00004040,
487 	0x9100, 0x07ffffff, 0x03000000,
488 	0x9100, 0x07ffffff, 0x03000000,
489 	0x8e88, 0x01ff1f3f, 0x00000000,
490 	0x8e88, 0x01ff1f3f, 0x00000000,
491 	0x8e88, 0x01ff1f3f, 0x00000000,
492 	0x8e84, 0x01ff1f3f, 0x00000000,
493 	0x8e84, 0x01ff1f3f, 0x00000000,
494 	0x8e84, 0x01ff1f3f, 0x00000000,
495 	0x9060, 0x0000007f, 0x00000020,
496 	0x9508, 0x00010000, 0x00010000,
497 	0xac14, 0x000003ff, 0x00000003,
498 	0xac14, 0x000003ff, 0x00000003,
499 	0xac14, 0x000003ff, 0x00000003,
500 	0xac10, 0xffffffff, 0x00000000,
501 	0xac10, 0xffffffff, 0x00000000,
502 	0xac10, 0xffffffff, 0x00000000,
503 	0xac0c, 0xffffffff, 0x00001032,
504 	0xac0c, 0xffffffff, 0x00001032,
505 	0xac0c, 0xffffffff, 0x00001032,
506 	0x88d4, 0x0000001f, 0x00000010,
507 	0x88d4, 0x0000001f, 0x00000010,
508 	0x88d4, 0x0000001f, 0x00000010,
509 	0x15c0, 0x000c0fc0, 0x000c0400
510 };
511 
512 static const u32 oland_golden_rlc_registers[] =
513 {
514 	0xc424, 0xffffffff, 0x00601005,
515 	0xc47c, 0xffffffff, 0x10104040,
516 	0xc488, 0xffffffff, 0x0100000a,
517 	0xc314, 0xffffffff, 0x00000800,
518 	0xc30c, 0xffffffff, 0x800000f4
519 };
520 
521 static const u32 oland_golden_registers[] =
522 {
523 	0x9a10, 0x00010000, 0x00018208,
524 	0x9830, 0xffffffff, 0x00000000,
525 	0x9834, 0xf00fffff, 0x00000400,
526 	0x9838, 0x0002021c, 0x00020200,
527 	0xc78, 0x00000080, 0x00000000,
528 	0xd030, 0x000300c0, 0x00800040,
529 	0xd830, 0x000300c0, 0x00800040,
530 	0x5bb0, 0x000000f0, 0x00000070,
531 	0x5bc0, 0x00200000, 0x50100000,
532 	0x7030, 0x31000311, 0x00000011,
533 	0x2ae4, 0x00073ffe, 0x000022a2,
534 	0x240c, 0x000007ff, 0x00000000,
535 	0x8a14, 0xf000001f, 0x00000007,
536 	0x8b24, 0xffffffff, 0x00ffffff,
537 	0x8b10, 0x0000ff0f, 0x00000000,
538 	0x28a4c, 0x07ffffff, 0x4e000000,
539 	0x28350, 0x3f3f3fff, 0x00000082,
540 	0x30, 0x000000ff, 0x0040,
541 	0x34, 0x00000040, 0x00004040,
542 	0x9100, 0x07ffffff, 0x03000000,
543 	0x9060, 0x0000007f, 0x00000020,
544 	0x9508, 0x00010000, 0x00010000,
545 	0xac14, 0x000003ff, 0x000000f3,
546 	0xac10, 0xffffffff, 0x00000000,
547 	0xac0c, 0xffffffff, 0x00003210,
548 	0x88d4, 0x0000001f, 0x00000010,
549 	0x15c0, 0x000c0fc0, 0x000c0400
550 };
551 
552 static const u32 hainan_golden_registers[] =
553 {
554 	0x9a10, 0x00010000, 0x00018208,
555 	0x9830, 0xffffffff, 0x00000000,
556 	0x9834, 0xf00fffff, 0x00000400,
557 	0x9838, 0x0002021c, 0x00020200,
558 	0xd0c0, 0xff000fff, 0x00000100,
559 	0xd030, 0x000300c0, 0x00800040,
560 	0xd8c0, 0xff000fff, 0x00000100,
561 	0xd830, 0x000300c0, 0x00800040,
562 	0x2ae4, 0x00073ffe, 0x000022a2,
563 	0x240c, 0x000007ff, 0x00000000,
564 	0x8a14, 0xf000001f, 0x00000007,
565 	0x8b24, 0xffffffff, 0x00ffffff,
566 	0x8b10, 0x0000ff0f, 0x00000000,
567 	0x28a4c, 0x07ffffff, 0x4e000000,
568 	0x28350, 0x3f3f3fff, 0x00000000,
569 	0x30, 0x000000ff, 0x0040,
570 	0x34, 0x00000040, 0x00004040,
571 	0x9100, 0x03e00000, 0x03600000,
572 	0x9060, 0x0000007f, 0x00000020,
573 	0x9508, 0x00010000, 0x00010000,
574 	0xac14, 0x000003ff, 0x000000f1,
575 	0xac10, 0xffffffff, 0x00000000,
576 	0xac0c, 0xffffffff, 0x00003210,
577 	0x88d4, 0x0000001f, 0x00000010,
578 	0x15c0, 0x000c0fc0, 0x000c0400
579 };
580 
581 static const u32 hainan_golden_registers2[] =
582 {
583 	0x98f8, 0xffffffff, 0x02010001
584 };
585 
586 static const u32 tahiti_mgcg_cgcg_init[] =
587 {
588 	0xc400, 0xffffffff, 0xfffffffc,
589 	0x802c, 0xffffffff, 0xe0000000,
590 	0x9a60, 0xffffffff, 0x00000100,
591 	0x92a4, 0xffffffff, 0x00000100,
592 	0xc164, 0xffffffff, 0x00000100,
593 	0x9774, 0xffffffff, 0x00000100,
594 	0x8984, 0xffffffff, 0x06000100,
595 	0x8a18, 0xffffffff, 0x00000100,
596 	0x92a0, 0xffffffff, 0x00000100,
597 	0xc380, 0xffffffff, 0x00000100,
598 	0x8b28, 0xffffffff, 0x00000100,
599 	0x9144, 0xffffffff, 0x00000100,
600 	0x8d88, 0xffffffff, 0x00000100,
601 	0x8d8c, 0xffffffff, 0x00000100,
602 	0x9030, 0xffffffff, 0x00000100,
603 	0x9034, 0xffffffff, 0x00000100,
604 	0x9038, 0xffffffff, 0x00000100,
605 	0x903c, 0xffffffff, 0x00000100,
606 	0xad80, 0xffffffff, 0x00000100,
607 	0xac54, 0xffffffff, 0x00000100,
608 	0x897c, 0xffffffff, 0x06000100,
609 	0x9868, 0xffffffff, 0x00000100,
610 	0x9510, 0xffffffff, 0x00000100,
611 	0xaf04, 0xffffffff, 0x00000100,
612 	0xae04, 0xffffffff, 0x00000100,
613 	0x949c, 0xffffffff, 0x00000100,
614 	0x802c, 0xffffffff, 0xe0000000,
615 	0x9160, 0xffffffff, 0x00010000,
616 	0x9164, 0xffffffff, 0x00030002,
617 	0x9168, 0xffffffff, 0x00040007,
618 	0x916c, 0xffffffff, 0x00060005,
619 	0x9170, 0xffffffff, 0x00090008,
620 	0x9174, 0xffffffff, 0x00020001,
621 	0x9178, 0xffffffff, 0x00040003,
622 	0x917c, 0xffffffff, 0x00000007,
623 	0x9180, 0xffffffff, 0x00060005,
624 	0x9184, 0xffffffff, 0x00090008,
625 	0x9188, 0xffffffff, 0x00030002,
626 	0x918c, 0xffffffff, 0x00050004,
627 	0x9190, 0xffffffff, 0x00000008,
628 	0x9194, 0xffffffff, 0x00070006,
629 	0x9198, 0xffffffff, 0x000a0009,
630 	0x919c, 0xffffffff, 0x00040003,
631 	0x91a0, 0xffffffff, 0x00060005,
632 	0x91a4, 0xffffffff, 0x00000009,
633 	0x91a8, 0xffffffff, 0x00080007,
634 	0x91ac, 0xffffffff, 0x000b000a,
635 	0x91b0, 0xffffffff, 0x00050004,
636 	0x91b4, 0xffffffff, 0x00070006,
637 	0x91b8, 0xffffffff, 0x0008000b,
638 	0x91bc, 0xffffffff, 0x000a0009,
639 	0x91c0, 0xffffffff, 0x000d000c,
640 	0x91c4, 0xffffffff, 0x00060005,
641 	0x91c8, 0xffffffff, 0x00080007,
642 	0x91cc, 0xffffffff, 0x0000000b,
643 	0x91d0, 0xffffffff, 0x000a0009,
644 	0x91d4, 0xffffffff, 0x000d000c,
645 	0x91d8, 0xffffffff, 0x00070006,
646 	0x91dc, 0xffffffff, 0x00090008,
647 	0x91e0, 0xffffffff, 0x0000000c,
648 	0x91e4, 0xffffffff, 0x000b000a,
649 	0x91e8, 0xffffffff, 0x000e000d,
650 	0x91ec, 0xffffffff, 0x00080007,
651 	0x91f0, 0xffffffff, 0x000a0009,
652 	0x91f4, 0xffffffff, 0x0000000d,
653 	0x91f8, 0xffffffff, 0x000c000b,
654 	0x91fc, 0xffffffff, 0x000f000e,
655 	0x9200, 0xffffffff, 0x00090008,
656 	0x9204, 0xffffffff, 0x000b000a,
657 	0x9208, 0xffffffff, 0x000c000f,
658 	0x920c, 0xffffffff, 0x000e000d,
659 	0x9210, 0xffffffff, 0x00110010,
660 	0x9214, 0xffffffff, 0x000a0009,
661 	0x9218, 0xffffffff, 0x000c000b,
662 	0x921c, 0xffffffff, 0x0000000f,
663 	0x9220, 0xffffffff, 0x000e000d,
664 	0x9224, 0xffffffff, 0x00110010,
665 	0x9228, 0xffffffff, 0x000b000a,
666 	0x922c, 0xffffffff, 0x000d000c,
667 	0x9230, 0xffffffff, 0x00000010,
668 	0x9234, 0xffffffff, 0x000f000e,
669 	0x9238, 0xffffffff, 0x00120011,
670 	0x923c, 0xffffffff, 0x000c000b,
671 	0x9240, 0xffffffff, 0x000e000d,
672 	0x9244, 0xffffffff, 0x00000011,
673 	0x9248, 0xffffffff, 0x0010000f,
674 	0x924c, 0xffffffff, 0x00130012,
675 	0x9250, 0xffffffff, 0x000d000c,
676 	0x9254, 0xffffffff, 0x000f000e,
677 	0x9258, 0xffffffff, 0x00100013,
678 	0x925c, 0xffffffff, 0x00120011,
679 	0x9260, 0xffffffff, 0x00150014,
680 	0x9264, 0xffffffff, 0x000e000d,
681 	0x9268, 0xffffffff, 0x0010000f,
682 	0x926c, 0xffffffff, 0x00000013,
683 	0x9270, 0xffffffff, 0x00120011,
684 	0x9274, 0xffffffff, 0x00150014,
685 	0x9278, 0xffffffff, 0x000f000e,
686 	0x927c, 0xffffffff, 0x00110010,
687 	0x9280, 0xffffffff, 0x00000014,
688 	0x9284, 0xffffffff, 0x00130012,
689 	0x9288, 0xffffffff, 0x00160015,
690 	0x928c, 0xffffffff, 0x0010000f,
691 	0x9290, 0xffffffff, 0x00120011,
692 	0x9294, 0xffffffff, 0x00000015,
693 	0x9298, 0xffffffff, 0x00140013,
694 	0x929c, 0xffffffff, 0x00170016,
695 	0x9150, 0xffffffff, 0x96940200,
696 	0x8708, 0xffffffff, 0x00900100,
697 	0xc478, 0xffffffff, 0x00000080,
698 	0xc404, 0xffffffff, 0x0020003f,
699 	0x30, 0xffffffff, 0x0000001c,
700 	0x34, 0x000f0000, 0x000f0000,
701 	0x160c, 0xffffffff, 0x00000100,
702 	0x1024, 0xffffffff, 0x00000100,
703 	0x102c, 0x00000101, 0x00000000,
704 	0x20a8, 0xffffffff, 0x00000104,
705 	0x264c, 0x000c0000, 0x000c0000,
706 	0x2648, 0x000c0000, 0x000c0000,
707 	0x55e4, 0xff000fff, 0x00000100,
708 	0x55e8, 0x00000001, 0x00000001,
709 	0x2f50, 0x00000001, 0x00000001,
710 	0x30cc, 0xc0000fff, 0x00000104,
711 	0xc1e4, 0x00000001, 0x00000001,
712 	0xd0c0, 0xfffffff0, 0x00000100,
713 	0xd8c0, 0xfffffff0, 0x00000100
714 };
715 
716 static const u32 pitcairn_mgcg_cgcg_init[] =
717 {
718 	0xc400, 0xffffffff, 0xfffffffc,
719 	0x802c, 0xffffffff, 0xe0000000,
720 	0x9a60, 0xffffffff, 0x00000100,
721 	0x92a4, 0xffffffff, 0x00000100,
722 	0xc164, 0xffffffff, 0x00000100,
723 	0x9774, 0xffffffff, 0x00000100,
724 	0x8984, 0xffffffff, 0x06000100,
725 	0x8a18, 0xffffffff, 0x00000100,
726 	0x92a0, 0xffffffff, 0x00000100,
727 	0xc380, 0xffffffff, 0x00000100,
728 	0x8b28, 0xffffffff, 0x00000100,
729 	0x9144, 0xffffffff, 0x00000100,
730 	0x8d88, 0xffffffff, 0x00000100,
731 	0x8d8c, 0xffffffff, 0x00000100,
732 	0x9030, 0xffffffff, 0x00000100,
733 	0x9034, 0xffffffff, 0x00000100,
734 	0x9038, 0xffffffff, 0x00000100,
735 	0x903c, 0xffffffff, 0x00000100,
736 	0xad80, 0xffffffff, 0x00000100,
737 	0xac54, 0xffffffff, 0x00000100,
738 	0x897c, 0xffffffff, 0x06000100,
739 	0x9868, 0xffffffff, 0x00000100,
740 	0x9510, 0xffffffff, 0x00000100,
741 	0xaf04, 0xffffffff, 0x00000100,
742 	0xae04, 0xffffffff, 0x00000100,
743 	0x949c, 0xffffffff, 0x00000100,
744 	0x802c, 0xffffffff, 0xe0000000,
745 	0x9160, 0xffffffff, 0x00010000,
746 	0x9164, 0xffffffff, 0x00030002,
747 	0x9168, 0xffffffff, 0x00040007,
748 	0x916c, 0xffffffff, 0x00060005,
749 	0x9170, 0xffffffff, 0x00090008,
750 	0x9174, 0xffffffff, 0x00020001,
751 	0x9178, 0xffffffff, 0x00040003,
752 	0x917c, 0xffffffff, 0x00000007,
753 	0x9180, 0xffffffff, 0x00060005,
754 	0x9184, 0xffffffff, 0x00090008,
755 	0x9188, 0xffffffff, 0x00030002,
756 	0x918c, 0xffffffff, 0x00050004,
757 	0x9190, 0xffffffff, 0x00000008,
758 	0x9194, 0xffffffff, 0x00070006,
759 	0x9198, 0xffffffff, 0x000a0009,
760 	0x919c, 0xffffffff, 0x00040003,
761 	0x91a0, 0xffffffff, 0x00060005,
762 	0x91a4, 0xffffffff, 0x00000009,
763 	0x91a8, 0xffffffff, 0x00080007,
764 	0x91ac, 0xffffffff, 0x000b000a,
765 	0x91b0, 0xffffffff, 0x00050004,
766 	0x91b4, 0xffffffff, 0x00070006,
767 	0x91b8, 0xffffffff, 0x0008000b,
768 	0x91bc, 0xffffffff, 0x000a0009,
769 	0x91c0, 0xffffffff, 0x000d000c,
770 	0x9200, 0xffffffff, 0x00090008,
771 	0x9204, 0xffffffff, 0x000b000a,
772 	0x9208, 0xffffffff, 0x000c000f,
773 	0x920c, 0xffffffff, 0x000e000d,
774 	0x9210, 0xffffffff, 0x00110010,
775 	0x9214, 0xffffffff, 0x000a0009,
776 	0x9218, 0xffffffff, 0x000c000b,
777 	0x921c, 0xffffffff, 0x0000000f,
778 	0x9220, 0xffffffff, 0x000e000d,
779 	0x9224, 0xffffffff, 0x00110010,
780 	0x9228, 0xffffffff, 0x000b000a,
781 	0x922c, 0xffffffff, 0x000d000c,
782 	0x9230, 0xffffffff, 0x00000010,
783 	0x9234, 0xffffffff, 0x000f000e,
784 	0x9238, 0xffffffff, 0x00120011,
785 	0x923c, 0xffffffff, 0x000c000b,
786 	0x9240, 0xffffffff, 0x000e000d,
787 	0x9244, 0xffffffff, 0x00000011,
788 	0x9248, 0xffffffff, 0x0010000f,
789 	0x924c, 0xffffffff, 0x00130012,
790 	0x9250, 0xffffffff, 0x000d000c,
791 	0x9254, 0xffffffff, 0x000f000e,
792 	0x9258, 0xffffffff, 0x00100013,
793 	0x925c, 0xffffffff, 0x00120011,
794 	0x9260, 0xffffffff, 0x00150014,
795 	0x9150, 0xffffffff, 0x96940200,
796 	0x8708, 0xffffffff, 0x00900100,
797 	0xc478, 0xffffffff, 0x00000080,
798 	0xc404, 0xffffffff, 0x0020003f,
799 	0x30, 0xffffffff, 0x0000001c,
800 	0x34, 0x000f0000, 0x000f0000,
801 	0x160c, 0xffffffff, 0x00000100,
802 	0x1024, 0xffffffff, 0x00000100,
803 	0x102c, 0x00000101, 0x00000000,
804 	0x20a8, 0xffffffff, 0x00000104,
805 	0x55e4, 0xff000fff, 0x00000100,
806 	0x55e8, 0x00000001, 0x00000001,
807 	0x2f50, 0x00000001, 0x00000001,
808 	0x30cc, 0xc0000fff, 0x00000104,
809 	0xc1e4, 0x00000001, 0x00000001,
810 	0xd0c0, 0xfffffff0, 0x00000100,
811 	0xd8c0, 0xfffffff0, 0x00000100
812 };
813 
814 static const u32 verde_mgcg_cgcg_init[] =
815 {
816 	0xc400, 0xffffffff, 0xfffffffc,
817 	0x802c, 0xffffffff, 0xe0000000,
818 	0x9a60, 0xffffffff, 0x00000100,
819 	0x92a4, 0xffffffff, 0x00000100,
820 	0xc164, 0xffffffff, 0x00000100,
821 	0x9774, 0xffffffff, 0x00000100,
822 	0x8984, 0xffffffff, 0x06000100,
823 	0x8a18, 0xffffffff, 0x00000100,
824 	0x92a0, 0xffffffff, 0x00000100,
825 	0xc380, 0xffffffff, 0x00000100,
826 	0x8b28, 0xffffffff, 0x00000100,
827 	0x9144, 0xffffffff, 0x00000100,
828 	0x8d88, 0xffffffff, 0x00000100,
829 	0x8d8c, 0xffffffff, 0x00000100,
830 	0x9030, 0xffffffff, 0x00000100,
831 	0x9034, 0xffffffff, 0x00000100,
832 	0x9038, 0xffffffff, 0x00000100,
833 	0x903c, 0xffffffff, 0x00000100,
834 	0xad80, 0xffffffff, 0x00000100,
835 	0xac54, 0xffffffff, 0x00000100,
836 	0x897c, 0xffffffff, 0x06000100,
837 	0x9868, 0xffffffff, 0x00000100,
838 	0x9510, 0xffffffff, 0x00000100,
839 	0xaf04, 0xffffffff, 0x00000100,
840 	0xae04, 0xffffffff, 0x00000100,
841 	0x949c, 0xffffffff, 0x00000100,
842 	0x802c, 0xffffffff, 0xe0000000,
843 	0x9160, 0xffffffff, 0x00010000,
844 	0x9164, 0xffffffff, 0x00030002,
845 	0x9168, 0xffffffff, 0x00040007,
846 	0x916c, 0xffffffff, 0x00060005,
847 	0x9170, 0xffffffff, 0x00090008,
848 	0x9174, 0xffffffff, 0x00020001,
849 	0x9178, 0xffffffff, 0x00040003,
850 	0x917c, 0xffffffff, 0x00000007,
851 	0x9180, 0xffffffff, 0x00060005,
852 	0x9184, 0xffffffff, 0x00090008,
853 	0x9188, 0xffffffff, 0x00030002,
854 	0x918c, 0xffffffff, 0x00050004,
855 	0x9190, 0xffffffff, 0x00000008,
856 	0x9194, 0xffffffff, 0x00070006,
857 	0x9198, 0xffffffff, 0x000a0009,
858 	0x919c, 0xffffffff, 0x00040003,
859 	0x91a0, 0xffffffff, 0x00060005,
860 	0x91a4, 0xffffffff, 0x00000009,
861 	0x91a8, 0xffffffff, 0x00080007,
862 	0x91ac, 0xffffffff, 0x000b000a,
863 	0x91b0, 0xffffffff, 0x00050004,
864 	0x91b4, 0xffffffff, 0x00070006,
865 	0x91b8, 0xffffffff, 0x0008000b,
866 	0x91bc, 0xffffffff, 0x000a0009,
867 	0x91c0, 0xffffffff, 0x000d000c,
868 	0x9200, 0xffffffff, 0x00090008,
869 	0x9204, 0xffffffff, 0x000b000a,
870 	0x9208, 0xffffffff, 0x000c000f,
871 	0x920c, 0xffffffff, 0x000e000d,
872 	0x9210, 0xffffffff, 0x00110010,
873 	0x9214, 0xffffffff, 0x000a0009,
874 	0x9218, 0xffffffff, 0x000c000b,
875 	0x921c, 0xffffffff, 0x0000000f,
876 	0x9220, 0xffffffff, 0x000e000d,
877 	0x9224, 0xffffffff, 0x00110010,
878 	0x9228, 0xffffffff, 0x000b000a,
879 	0x922c, 0xffffffff, 0x000d000c,
880 	0x9230, 0xffffffff, 0x00000010,
881 	0x9234, 0xffffffff, 0x000f000e,
882 	0x9238, 0xffffffff, 0x00120011,
883 	0x923c, 0xffffffff, 0x000c000b,
884 	0x9240, 0xffffffff, 0x000e000d,
885 	0x9244, 0xffffffff, 0x00000011,
886 	0x9248, 0xffffffff, 0x0010000f,
887 	0x924c, 0xffffffff, 0x00130012,
888 	0x9250, 0xffffffff, 0x000d000c,
889 	0x9254, 0xffffffff, 0x000f000e,
890 	0x9258, 0xffffffff, 0x00100013,
891 	0x925c, 0xffffffff, 0x00120011,
892 	0x9260, 0xffffffff, 0x00150014,
893 	0x9150, 0xffffffff, 0x96940200,
894 	0x8708, 0xffffffff, 0x00900100,
895 	0xc478, 0xffffffff, 0x00000080,
896 	0xc404, 0xffffffff, 0x0020003f,
897 	0x30, 0xffffffff, 0x0000001c,
898 	0x34, 0x000f0000, 0x000f0000,
899 	0x160c, 0xffffffff, 0x00000100,
900 	0x1024, 0xffffffff, 0x00000100,
901 	0x102c, 0x00000101, 0x00000000,
902 	0x20a8, 0xffffffff, 0x00000104,
903 	0x264c, 0x000c0000, 0x000c0000,
904 	0x2648, 0x000c0000, 0x000c0000,
905 	0x55e4, 0xff000fff, 0x00000100,
906 	0x55e8, 0x00000001, 0x00000001,
907 	0x2f50, 0x00000001, 0x00000001,
908 	0x30cc, 0xc0000fff, 0x00000104,
909 	0xc1e4, 0x00000001, 0x00000001,
910 	0xd0c0, 0xfffffff0, 0x00000100,
911 	0xd8c0, 0xfffffff0, 0x00000100
912 };
913 
914 static const u32 oland_mgcg_cgcg_init[] =
915 {
916 	0xc400, 0xffffffff, 0xfffffffc,
917 	0x802c, 0xffffffff, 0xe0000000,
918 	0x9a60, 0xffffffff, 0x00000100,
919 	0x92a4, 0xffffffff, 0x00000100,
920 	0xc164, 0xffffffff, 0x00000100,
921 	0x9774, 0xffffffff, 0x00000100,
922 	0x8984, 0xffffffff, 0x06000100,
923 	0x8a18, 0xffffffff, 0x00000100,
924 	0x92a0, 0xffffffff, 0x00000100,
925 	0xc380, 0xffffffff, 0x00000100,
926 	0x8b28, 0xffffffff, 0x00000100,
927 	0x9144, 0xffffffff, 0x00000100,
928 	0x8d88, 0xffffffff, 0x00000100,
929 	0x8d8c, 0xffffffff, 0x00000100,
930 	0x9030, 0xffffffff, 0x00000100,
931 	0x9034, 0xffffffff, 0x00000100,
932 	0x9038, 0xffffffff, 0x00000100,
933 	0x903c, 0xffffffff, 0x00000100,
934 	0xad80, 0xffffffff, 0x00000100,
935 	0xac54, 0xffffffff, 0x00000100,
936 	0x897c, 0xffffffff, 0x06000100,
937 	0x9868, 0xffffffff, 0x00000100,
938 	0x9510, 0xffffffff, 0x00000100,
939 	0xaf04, 0xffffffff, 0x00000100,
940 	0xae04, 0xffffffff, 0x00000100,
941 	0x949c, 0xffffffff, 0x00000100,
942 	0x802c, 0xffffffff, 0xe0000000,
943 	0x9160, 0xffffffff, 0x00010000,
944 	0x9164, 0xffffffff, 0x00030002,
945 	0x9168, 0xffffffff, 0x00040007,
946 	0x916c, 0xffffffff, 0x00060005,
947 	0x9170, 0xffffffff, 0x00090008,
948 	0x9174, 0xffffffff, 0x00020001,
949 	0x9178, 0xffffffff, 0x00040003,
950 	0x917c, 0xffffffff, 0x00000007,
951 	0x9180, 0xffffffff, 0x00060005,
952 	0x9184, 0xffffffff, 0x00090008,
953 	0x9188, 0xffffffff, 0x00030002,
954 	0x918c, 0xffffffff, 0x00050004,
955 	0x9190, 0xffffffff, 0x00000008,
956 	0x9194, 0xffffffff, 0x00070006,
957 	0x9198, 0xffffffff, 0x000a0009,
958 	0x919c, 0xffffffff, 0x00040003,
959 	0x91a0, 0xffffffff, 0x00060005,
960 	0x91a4, 0xffffffff, 0x00000009,
961 	0x91a8, 0xffffffff, 0x00080007,
962 	0x91ac, 0xffffffff, 0x000b000a,
963 	0x91b0, 0xffffffff, 0x00050004,
964 	0x91b4, 0xffffffff, 0x00070006,
965 	0x91b8, 0xffffffff, 0x0008000b,
966 	0x91bc, 0xffffffff, 0x000a0009,
967 	0x91c0, 0xffffffff, 0x000d000c,
968 	0x91c4, 0xffffffff, 0x00060005,
969 	0x91c8, 0xffffffff, 0x00080007,
970 	0x91cc, 0xffffffff, 0x0000000b,
971 	0x91d0, 0xffffffff, 0x000a0009,
972 	0x91d4, 0xffffffff, 0x000d000c,
973 	0x9150, 0xffffffff, 0x96940200,
974 	0x8708, 0xffffffff, 0x00900100,
975 	0xc478, 0xffffffff, 0x00000080,
976 	0xc404, 0xffffffff, 0x0020003f,
977 	0x30, 0xffffffff, 0x0000001c,
978 	0x34, 0x000f0000, 0x000f0000,
979 	0x160c, 0xffffffff, 0x00000100,
980 	0x1024, 0xffffffff, 0x00000100,
981 	0x102c, 0x00000101, 0x00000000,
982 	0x20a8, 0xffffffff, 0x00000104,
983 	0x264c, 0x000c0000, 0x000c0000,
984 	0x2648, 0x000c0000, 0x000c0000,
985 	0x55e4, 0xff000fff, 0x00000100,
986 	0x55e8, 0x00000001, 0x00000001,
987 	0x2f50, 0x00000001, 0x00000001,
988 	0x30cc, 0xc0000fff, 0x00000104,
989 	0xc1e4, 0x00000001, 0x00000001,
990 	0xd0c0, 0xfffffff0, 0x00000100,
991 	0xd8c0, 0xfffffff0, 0x00000100
992 };
993 
994 static const u32 hainan_mgcg_cgcg_init[] =
995 {
996 	0xc400, 0xffffffff, 0xfffffffc,
997 	0x802c, 0xffffffff, 0xe0000000,
998 	0x9a60, 0xffffffff, 0x00000100,
999 	0x92a4, 0xffffffff, 0x00000100,
1000 	0xc164, 0xffffffff, 0x00000100,
1001 	0x9774, 0xffffffff, 0x00000100,
1002 	0x8984, 0xffffffff, 0x06000100,
1003 	0x8a18, 0xffffffff, 0x00000100,
1004 	0x92a0, 0xffffffff, 0x00000100,
1005 	0xc380, 0xffffffff, 0x00000100,
1006 	0x8b28, 0xffffffff, 0x00000100,
1007 	0x9144, 0xffffffff, 0x00000100,
1008 	0x8d88, 0xffffffff, 0x00000100,
1009 	0x8d8c, 0xffffffff, 0x00000100,
1010 	0x9030, 0xffffffff, 0x00000100,
1011 	0x9034, 0xffffffff, 0x00000100,
1012 	0x9038, 0xffffffff, 0x00000100,
1013 	0x903c, 0xffffffff, 0x00000100,
1014 	0xad80, 0xffffffff, 0x00000100,
1015 	0xac54, 0xffffffff, 0x00000100,
1016 	0x897c, 0xffffffff, 0x06000100,
1017 	0x9868, 0xffffffff, 0x00000100,
1018 	0x9510, 0xffffffff, 0x00000100,
1019 	0xaf04, 0xffffffff, 0x00000100,
1020 	0xae04, 0xffffffff, 0x00000100,
1021 	0x949c, 0xffffffff, 0x00000100,
1022 	0x802c, 0xffffffff, 0xe0000000,
1023 	0x9160, 0xffffffff, 0x00010000,
1024 	0x9164, 0xffffffff, 0x00030002,
1025 	0x9168, 0xffffffff, 0x00040007,
1026 	0x916c, 0xffffffff, 0x00060005,
1027 	0x9170, 0xffffffff, 0x00090008,
1028 	0x9174, 0xffffffff, 0x00020001,
1029 	0x9178, 0xffffffff, 0x00040003,
1030 	0x917c, 0xffffffff, 0x00000007,
1031 	0x9180, 0xffffffff, 0x00060005,
1032 	0x9184, 0xffffffff, 0x00090008,
1033 	0x9188, 0xffffffff, 0x00030002,
1034 	0x918c, 0xffffffff, 0x00050004,
1035 	0x9190, 0xffffffff, 0x00000008,
1036 	0x9194, 0xffffffff, 0x00070006,
1037 	0x9198, 0xffffffff, 0x000a0009,
1038 	0x919c, 0xffffffff, 0x00040003,
1039 	0x91a0, 0xffffffff, 0x00060005,
1040 	0x91a4, 0xffffffff, 0x00000009,
1041 	0x91a8, 0xffffffff, 0x00080007,
1042 	0x91ac, 0xffffffff, 0x000b000a,
1043 	0x91b0, 0xffffffff, 0x00050004,
1044 	0x91b4, 0xffffffff, 0x00070006,
1045 	0x91b8, 0xffffffff, 0x0008000b,
1046 	0x91bc, 0xffffffff, 0x000a0009,
1047 	0x91c0, 0xffffffff, 0x000d000c,
1048 	0x91c4, 0xffffffff, 0x00060005,
1049 	0x91c8, 0xffffffff, 0x00080007,
1050 	0x91cc, 0xffffffff, 0x0000000b,
1051 	0x91d0, 0xffffffff, 0x000a0009,
1052 	0x91d4, 0xffffffff, 0x000d000c,
1053 	0x9150, 0xffffffff, 0x96940200,
1054 	0x8708, 0xffffffff, 0x00900100,
1055 	0xc478, 0xffffffff, 0x00000080,
1056 	0xc404, 0xffffffff, 0x0020003f,
1057 	0x30, 0xffffffff, 0x0000001c,
1058 	0x34, 0x000f0000, 0x000f0000,
1059 	0x160c, 0xffffffff, 0x00000100,
1060 	0x1024, 0xffffffff, 0x00000100,
1061 	0x20a8, 0xffffffff, 0x00000104,
1062 	0x264c, 0x000c0000, 0x000c0000,
1063 	0x2648, 0x000c0000, 0x000c0000,
1064 	0x2f50, 0x00000001, 0x00000001,
1065 	0x30cc, 0xc0000fff, 0x00000104,
1066 	0xc1e4, 0x00000001, 0x00000001,
1067 	0xd0c0, 0xfffffff0, 0x00000100,
1068 	0xd8c0, 0xfffffff0, 0x00000100
1069 };
1070 
1071 static u32 verde_pg_init[] =
1072 {
1073 	0x353c, 0xffffffff, 0x40000,
1074 	0x3538, 0xffffffff, 0x200010ff,
1075 	0x353c, 0xffffffff, 0x0,
1076 	0x353c, 0xffffffff, 0x0,
1077 	0x353c, 0xffffffff, 0x0,
1078 	0x353c, 0xffffffff, 0x0,
1079 	0x353c, 0xffffffff, 0x0,
1080 	0x353c, 0xffffffff, 0x7007,
1081 	0x3538, 0xffffffff, 0x300010ff,
1082 	0x353c, 0xffffffff, 0x0,
1083 	0x353c, 0xffffffff, 0x0,
1084 	0x353c, 0xffffffff, 0x0,
1085 	0x353c, 0xffffffff, 0x0,
1086 	0x353c, 0xffffffff, 0x0,
1087 	0x353c, 0xffffffff, 0x400000,
1088 	0x3538, 0xffffffff, 0x100010ff,
1089 	0x353c, 0xffffffff, 0x0,
1090 	0x353c, 0xffffffff, 0x0,
1091 	0x353c, 0xffffffff, 0x0,
1092 	0x353c, 0xffffffff, 0x0,
1093 	0x353c, 0xffffffff, 0x0,
1094 	0x353c, 0xffffffff, 0x120200,
1095 	0x3538, 0xffffffff, 0x500010ff,
1096 	0x353c, 0xffffffff, 0x0,
1097 	0x353c, 0xffffffff, 0x0,
1098 	0x353c, 0xffffffff, 0x0,
1099 	0x353c, 0xffffffff, 0x0,
1100 	0x353c, 0xffffffff, 0x0,
1101 	0x353c, 0xffffffff, 0x1e1e16,
1102 	0x3538, 0xffffffff, 0x600010ff,
1103 	0x353c, 0xffffffff, 0x0,
1104 	0x353c, 0xffffffff, 0x0,
1105 	0x353c, 0xffffffff, 0x0,
1106 	0x353c, 0xffffffff, 0x0,
1107 	0x353c, 0xffffffff, 0x0,
1108 	0x353c, 0xffffffff, 0x171f1e,
1109 	0x3538, 0xffffffff, 0x700010ff,
1110 	0x353c, 0xffffffff, 0x0,
1111 	0x353c, 0xffffffff, 0x0,
1112 	0x353c, 0xffffffff, 0x0,
1113 	0x353c, 0xffffffff, 0x0,
1114 	0x353c, 0xffffffff, 0x0,
1115 	0x353c, 0xffffffff, 0x0,
1116 	0x3538, 0xffffffff, 0x9ff,
1117 	0x3500, 0xffffffff, 0x0,
1118 	0x3504, 0xffffffff, 0x10000800,
1119 	0x3504, 0xffffffff, 0xf,
1120 	0x3504, 0xffffffff, 0xf,
1121 	0x3500, 0xffffffff, 0x4,
1122 	0x3504, 0xffffffff, 0x1000051e,
1123 	0x3504, 0xffffffff, 0xffff,
1124 	0x3504, 0xffffffff, 0xffff,
1125 	0x3500, 0xffffffff, 0x8,
1126 	0x3504, 0xffffffff, 0x80500,
1127 	0x3500, 0xffffffff, 0x12,
1128 	0x3504, 0xffffffff, 0x9050c,
1129 	0x3500, 0xffffffff, 0x1d,
1130 	0x3504, 0xffffffff, 0xb052c,
1131 	0x3500, 0xffffffff, 0x2a,
1132 	0x3504, 0xffffffff, 0x1053e,
1133 	0x3500, 0xffffffff, 0x2d,
1134 	0x3504, 0xffffffff, 0x10546,
1135 	0x3500, 0xffffffff, 0x30,
1136 	0x3504, 0xffffffff, 0xa054e,
1137 	0x3500, 0xffffffff, 0x3c,
1138 	0x3504, 0xffffffff, 0x1055f,
1139 	0x3500, 0xffffffff, 0x3f,
1140 	0x3504, 0xffffffff, 0x10567,
1141 	0x3500, 0xffffffff, 0x42,
1142 	0x3504, 0xffffffff, 0x1056f,
1143 	0x3500, 0xffffffff, 0x45,
1144 	0x3504, 0xffffffff, 0x10572,
1145 	0x3500, 0xffffffff, 0x48,
1146 	0x3504, 0xffffffff, 0x20575,
1147 	0x3500, 0xffffffff, 0x4c,
1148 	0x3504, 0xffffffff, 0x190801,
1149 	0x3500, 0xffffffff, 0x67,
1150 	0x3504, 0xffffffff, 0x1082a,
1151 	0x3500, 0xffffffff, 0x6a,
1152 	0x3504, 0xffffffff, 0x1b082d,
1153 	0x3500, 0xffffffff, 0x87,
1154 	0x3504, 0xffffffff, 0x310851,
1155 	0x3500, 0xffffffff, 0xba,
1156 	0x3504, 0xffffffff, 0x891,
1157 	0x3500, 0xffffffff, 0xbc,
1158 	0x3504, 0xffffffff, 0x893,
1159 	0x3500, 0xffffffff, 0xbe,
1160 	0x3504, 0xffffffff, 0x20895,
1161 	0x3500, 0xffffffff, 0xc2,
1162 	0x3504, 0xffffffff, 0x20899,
1163 	0x3500, 0xffffffff, 0xc6,
1164 	0x3504, 0xffffffff, 0x2089d,
1165 	0x3500, 0xffffffff, 0xca,
1166 	0x3504, 0xffffffff, 0x8a1,
1167 	0x3500, 0xffffffff, 0xcc,
1168 	0x3504, 0xffffffff, 0x8a3,
1169 	0x3500, 0xffffffff, 0xce,
1170 	0x3504, 0xffffffff, 0x308a5,
1171 	0x3500, 0xffffffff, 0xd3,
1172 	0x3504, 0xffffffff, 0x6d08cd,
1173 	0x3500, 0xffffffff, 0x142,
1174 	0x3504, 0xffffffff, 0x2000095a,
1175 	0x3504, 0xffffffff, 0x1,
1176 	0x3500, 0xffffffff, 0x144,
1177 	0x3504, 0xffffffff, 0x301f095b,
1178 	0x3500, 0xffffffff, 0x165,
1179 	0x3504, 0xffffffff, 0xc094d,
1180 	0x3500, 0xffffffff, 0x173,
1181 	0x3504, 0xffffffff, 0xf096d,
1182 	0x3500, 0xffffffff, 0x184,
1183 	0x3504, 0xffffffff, 0x15097f,
1184 	0x3500, 0xffffffff, 0x19b,
1185 	0x3504, 0xffffffff, 0xc0998,
1186 	0x3500, 0xffffffff, 0x1a9,
1187 	0x3504, 0xffffffff, 0x409a7,
1188 	0x3500, 0xffffffff, 0x1af,
1189 	0x3504, 0xffffffff, 0xcdc,
1190 	0x3500, 0xffffffff, 0x1b1,
1191 	0x3504, 0xffffffff, 0x800,
1192 	0x3508, 0xffffffff, 0x6c9b2000,
1193 	0x3510, 0xfc00, 0x2000,
1194 	0x3544, 0xffffffff, 0xfc0,
1195 	0x28d4, 0x00000100, 0x100
1196 };
1197 
1198 static void si_init_golden_registers(struct radeon_device *rdev)
1199 {
1200 	switch (rdev->family) {
1201 	case CHIP_TAHITI:
1202 		radeon_program_register_sequence(rdev,
1203 						 tahiti_golden_registers,
1204 						 (const u32)ARRAY_SIZE(tahiti_golden_registers));
1205 		radeon_program_register_sequence(rdev,
1206 						 tahiti_golden_rlc_registers,
1207 						 (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
1208 		radeon_program_register_sequence(rdev,
1209 						 tahiti_mgcg_cgcg_init,
1210 						 (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
1211 		radeon_program_register_sequence(rdev,
1212 						 tahiti_golden_registers2,
1213 						 (const u32)ARRAY_SIZE(tahiti_golden_registers2));
1214 		break;
1215 	case CHIP_PITCAIRN:
1216 		radeon_program_register_sequence(rdev,
1217 						 pitcairn_golden_registers,
1218 						 (const u32)ARRAY_SIZE(pitcairn_golden_registers));
1219 		radeon_program_register_sequence(rdev,
1220 						 pitcairn_golden_rlc_registers,
1221 						 (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
1222 		radeon_program_register_sequence(rdev,
1223 						 pitcairn_mgcg_cgcg_init,
1224 						 (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
1225 		break;
1226 	case CHIP_VERDE:
1227 		radeon_program_register_sequence(rdev,
1228 						 verde_golden_registers,
1229 						 (const u32)ARRAY_SIZE(verde_golden_registers));
1230 		radeon_program_register_sequence(rdev,
1231 						 verde_golden_rlc_registers,
1232 						 (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
1233 		radeon_program_register_sequence(rdev,
1234 						 verde_mgcg_cgcg_init,
1235 						 (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
1236 		radeon_program_register_sequence(rdev,
1237 						 verde_pg_init,
1238 						 (const u32)ARRAY_SIZE(verde_pg_init));
1239 		break;
1240 	case CHIP_OLAND:
1241 		radeon_program_register_sequence(rdev,
1242 						 oland_golden_registers,
1243 						 (const u32)ARRAY_SIZE(oland_golden_registers));
1244 		radeon_program_register_sequence(rdev,
1245 						 oland_golden_rlc_registers,
1246 						 (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
1247 		radeon_program_register_sequence(rdev,
1248 						 oland_mgcg_cgcg_init,
1249 						 (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
1250 		break;
1251 	case CHIP_HAINAN:
1252 		radeon_program_register_sequence(rdev,
1253 						 hainan_golden_registers,
1254 						 (const u32)ARRAY_SIZE(hainan_golden_registers));
1255 		radeon_program_register_sequence(rdev,
1256 						 hainan_golden_registers2,
1257 						 (const u32)ARRAY_SIZE(hainan_golden_registers2));
1258 		radeon_program_register_sequence(rdev,
1259 						 hainan_mgcg_cgcg_init,
1260 						 (const u32)ARRAY_SIZE(hainan_mgcg_cgcg_init));
1261 		break;
1262 	default:
1263 		break;
1264 	}
1265 }
1266 
1267 /**
1268  * si_get_allowed_info_register - fetch the register for the info ioctl
1269  *
1270  * @rdev: radeon_device pointer
1271  * @reg: register offset in bytes
1272  * @val: register value
1273  *
1274  * Returns 0 for success or -EINVAL for an invalid register
1275  *
1276  */
1277 int si_get_allowed_info_register(struct radeon_device *rdev,
1278 				 u32 reg, u32 *val)
1279 {
1280 	switch (reg) {
1281 	case GRBM_STATUS:
1282 	case GRBM_STATUS2:
1283 	case GRBM_STATUS_SE0:
1284 	case GRBM_STATUS_SE1:
1285 	case SRBM_STATUS:
1286 	case SRBM_STATUS2:
1287 	case (DMA_STATUS_REG + DMA0_REGISTER_OFFSET):
1288 	case (DMA_STATUS_REG + DMA1_REGISTER_OFFSET):
1289 	case UVD_STATUS:
1290 		*val = RREG32(reg);
1291 		return 0;
1292 	default:
1293 		return -EINVAL;
1294 	}
1295 }
1296 
1297 #define PCIE_BUS_CLK                10000
1298 #define TCLK                        (PCIE_BUS_CLK / 10)
1299 
1300 /**
1301  * si_get_xclk - get the xclk
1302  *
1303  * @rdev: radeon_device pointer
1304  *
1305  * Returns the reference clock used by the gfx engine
1306  * (SI).
1307  */
1308 u32 si_get_xclk(struct radeon_device *rdev)
1309 {
1310         u32 reference_clock = rdev->clock.spll.reference_freq;
1311 	u32 tmp;
1312 
1313 	tmp = RREG32(CG_CLKPIN_CNTL_2);
1314 	if (tmp & MUX_TCLK_TO_XCLK)
1315 		return TCLK;
1316 
1317 	tmp = RREG32(CG_CLKPIN_CNTL);
1318 	if (tmp & XTALIN_DIVIDE)
1319 		return reference_clock / 4;
1320 
1321 	return reference_clock;
1322 }
1323 
1324 /* get temperature in millidegrees */
1325 int si_get_temp(struct radeon_device *rdev)
1326 {
1327 	u32 temp;
1328 	int actual_temp = 0;
1329 
1330 	temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
1331 		CTF_TEMP_SHIFT;
1332 
1333 	if (temp & 0x200)
1334 		actual_temp = 255;
1335 	else
1336 		actual_temp = temp & 0x1ff;
1337 
1338 	actual_temp = (actual_temp * 1000);
1339 
1340 	return actual_temp;
1341 }
1342 
1343 #define TAHITI_IO_MC_REGS_SIZE 36
1344 
1345 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1346 	{0x0000006f, 0x03044000},
1347 	{0x00000070, 0x0480c018},
1348 	{0x00000071, 0x00000040},
1349 	{0x00000072, 0x01000000},
1350 	{0x00000074, 0x000000ff},
1351 	{0x00000075, 0x00143400},
1352 	{0x00000076, 0x08ec0800},
1353 	{0x00000077, 0x040000cc},
1354 	{0x00000079, 0x00000000},
1355 	{0x0000007a, 0x21000409},
1356 	{0x0000007c, 0x00000000},
1357 	{0x0000007d, 0xe8000000},
1358 	{0x0000007e, 0x044408a8},
1359 	{0x0000007f, 0x00000003},
1360 	{0x00000080, 0x00000000},
1361 	{0x00000081, 0x01000000},
1362 	{0x00000082, 0x02000000},
1363 	{0x00000083, 0x00000000},
1364 	{0x00000084, 0xe3f3e4f4},
1365 	{0x00000085, 0x00052024},
1366 	{0x00000087, 0x00000000},
1367 	{0x00000088, 0x66036603},
1368 	{0x00000089, 0x01000000},
1369 	{0x0000008b, 0x1c0a0000},
1370 	{0x0000008c, 0xff010000},
1371 	{0x0000008e, 0xffffefff},
1372 	{0x0000008f, 0xfff3efff},
1373 	{0x00000090, 0xfff3efbf},
1374 	{0x00000094, 0x00101101},
1375 	{0x00000095, 0x00000fff},
1376 	{0x00000096, 0x00116fff},
1377 	{0x00000097, 0x60010000},
1378 	{0x00000098, 0x10010000},
1379 	{0x00000099, 0x00006000},
1380 	{0x0000009a, 0x00001000},
1381 	{0x0000009f, 0x00a77400}
1382 };
1383 
1384 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1385 	{0x0000006f, 0x03044000},
1386 	{0x00000070, 0x0480c018},
1387 	{0x00000071, 0x00000040},
1388 	{0x00000072, 0x01000000},
1389 	{0x00000074, 0x000000ff},
1390 	{0x00000075, 0x00143400},
1391 	{0x00000076, 0x08ec0800},
1392 	{0x00000077, 0x040000cc},
1393 	{0x00000079, 0x00000000},
1394 	{0x0000007a, 0x21000409},
1395 	{0x0000007c, 0x00000000},
1396 	{0x0000007d, 0xe8000000},
1397 	{0x0000007e, 0x044408a8},
1398 	{0x0000007f, 0x00000003},
1399 	{0x00000080, 0x00000000},
1400 	{0x00000081, 0x01000000},
1401 	{0x00000082, 0x02000000},
1402 	{0x00000083, 0x00000000},
1403 	{0x00000084, 0xe3f3e4f4},
1404 	{0x00000085, 0x00052024},
1405 	{0x00000087, 0x00000000},
1406 	{0x00000088, 0x66036603},
1407 	{0x00000089, 0x01000000},
1408 	{0x0000008b, 0x1c0a0000},
1409 	{0x0000008c, 0xff010000},
1410 	{0x0000008e, 0xffffefff},
1411 	{0x0000008f, 0xfff3efff},
1412 	{0x00000090, 0xfff3efbf},
1413 	{0x00000094, 0x00101101},
1414 	{0x00000095, 0x00000fff},
1415 	{0x00000096, 0x00116fff},
1416 	{0x00000097, 0x60010000},
1417 	{0x00000098, 0x10010000},
1418 	{0x00000099, 0x00006000},
1419 	{0x0000009a, 0x00001000},
1420 	{0x0000009f, 0x00a47400}
1421 };
1422 
1423 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1424 	{0x0000006f, 0x03044000},
1425 	{0x00000070, 0x0480c018},
1426 	{0x00000071, 0x00000040},
1427 	{0x00000072, 0x01000000},
1428 	{0x00000074, 0x000000ff},
1429 	{0x00000075, 0x00143400},
1430 	{0x00000076, 0x08ec0800},
1431 	{0x00000077, 0x040000cc},
1432 	{0x00000079, 0x00000000},
1433 	{0x0000007a, 0x21000409},
1434 	{0x0000007c, 0x00000000},
1435 	{0x0000007d, 0xe8000000},
1436 	{0x0000007e, 0x044408a8},
1437 	{0x0000007f, 0x00000003},
1438 	{0x00000080, 0x00000000},
1439 	{0x00000081, 0x01000000},
1440 	{0x00000082, 0x02000000},
1441 	{0x00000083, 0x00000000},
1442 	{0x00000084, 0xe3f3e4f4},
1443 	{0x00000085, 0x00052024},
1444 	{0x00000087, 0x00000000},
1445 	{0x00000088, 0x66036603},
1446 	{0x00000089, 0x01000000},
1447 	{0x0000008b, 0x1c0a0000},
1448 	{0x0000008c, 0xff010000},
1449 	{0x0000008e, 0xffffefff},
1450 	{0x0000008f, 0xfff3efff},
1451 	{0x00000090, 0xfff3efbf},
1452 	{0x00000094, 0x00101101},
1453 	{0x00000095, 0x00000fff},
1454 	{0x00000096, 0x00116fff},
1455 	{0x00000097, 0x60010000},
1456 	{0x00000098, 0x10010000},
1457 	{0x00000099, 0x00006000},
1458 	{0x0000009a, 0x00001000},
1459 	{0x0000009f, 0x00a37400}
1460 };
1461 
1462 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1463 	{0x0000006f, 0x03044000},
1464 	{0x00000070, 0x0480c018},
1465 	{0x00000071, 0x00000040},
1466 	{0x00000072, 0x01000000},
1467 	{0x00000074, 0x000000ff},
1468 	{0x00000075, 0x00143400},
1469 	{0x00000076, 0x08ec0800},
1470 	{0x00000077, 0x040000cc},
1471 	{0x00000079, 0x00000000},
1472 	{0x0000007a, 0x21000409},
1473 	{0x0000007c, 0x00000000},
1474 	{0x0000007d, 0xe8000000},
1475 	{0x0000007e, 0x044408a8},
1476 	{0x0000007f, 0x00000003},
1477 	{0x00000080, 0x00000000},
1478 	{0x00000081, 0x01000000},
1479 	{0x00000082, 0x02000000},
1480 	{0x00000083, 0x00000000},
1481 	{0x00000084, 0xe3f3e4f4},
1482 	{0x00000085, 0x00052024},
1483 	{0x00000087, 0x00000000},
1484 	{0x00000088, 0x66036603},
1485 	{0x00000089, 0x01000000},
1486 	{0x0000008b, 0x1c0a0000},
1487 	{0x0000008c, 0xff010000},
1488 	{0x0000008e, 0xffffefff},
1489 	{0x0000008f, 0xfff3efff},
1490 	{0x00000090, 0xfff3efbf},
1491 	{0x00000094, 0x00101101},
1492 	{0x00000095, 0x00000fff},
1493 	{0x00000096, 0x00116fff},
1494 	{0x00000097, 0x60010000},
1495 	{0x00000098, 0x10010000},
1496 	{0x00000099, 0x00006000},
1497 	{0x0000009a, 0x00001000},
1498 	{0x0000009f, 0x00a17730}
1499 };
1500 
1501 static const u32 hainan_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1502 	{0x0000006f, 0x03044000},
1503 	{0x00000070, 0x0480c018},
1504 	{0x00000071, 0x00000040},
1505 	{0x00000072, 0x01000000},
1506 	{0x00000074, 0x000000ff},
1507 	{0x00000075, 0x00143400},
1508 	{0x00000076, 0x08ec0800},
1509 	{0x00000077, 0x040000cc},
1510 	{0x00000079, 0x00000000},
1511 	{0x0000007a, 0x21000409},
1512 	{0x0000007c, 0x00000000},
1513 	{0x0000007d, 0xe8000000},
1514 	{0x0000007e, 0x044408a8},
1515 	{0x0000007f, 0x00000003},
1516 	{0x00000080, 0x00000000},
1517 	{0x00000081, 0x01000000},
1518 	{0x00000082, 0x02000000},
1519 	{0x00000083, 0x00000000},
1520 	{0x00000084, 0xe3f3e4f4},
1521 	{0x00000085, 0x00052024},
1522 	{0x00000087, 0x00000000},
1523 	{0x00000088, 0x66036603},
1524 	{0x00000089, 0x01000000},
1525 	{0x0000008b, 0x1c0a0000},
1526 	{0x0000008c, 0xff010000},
1527 	{0x0000008e, 0xffffefff},
1528 	{0x0000008f, 0xfff3efff},
1529 	{0x00000090, 0xfff3efbf},
1530 	{0x00000094, 0x00101101},
1531 	{0x00000095, 0x00000fff},
1532 	{0x00000096, 0x00116fff},
1533 	{0x00000097, 0x60010000},
1534 	{0x00000098, 0x10010000},
1535 	{0x00000099, 0x00006000},
1536 	{0x0000009a, 0x00001000},
1537 	{0x0000009f, 0x00a07730}
1538 };
1539 
1540 /* ucode loading */
1541 int si_mc_load_microcode(struct radeon_device *rdev)
1542 {
1543 	const __be32 *fw_data = NULL;
1544 	const __le32 *new_fw_data = NULL;
1545 	u32 running, blackout = 0;
1546 	u32 *io_mc_regs = NULL;
1547 	const __le32 *new_io_mc_regs = NULL;
1548 	int i, regs_size, ucode_size;
1549 
1550 	if (!rdev->mc_fw)
1551 		return -EINVAL;
1552 
1553 	if (rdev->new_fw) {
1554 		const struct mc_firmware_header_v1_0 *hdr =
1555 			(const struct mc_firmware_header_v1_0 *)rdev->mc_fw->data;
1556 
1557 		radeon_ucode_print_mc_hdr(&hdr->header);
1558 		regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
1559 		new_io_mc_regs = (const __le32 *)
1560 			(rdev->mc_fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
1561 		ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
1562 		new_fw_data = (const __le32 *)
1563 			(rdev->mc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1564 	} else {
1565 		ucode_size = rdev->mc_fw->size / 4;
1566 
1567 		switch (rdev->family) {
1568 		case CHIP_TAHITI:
1569 			io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1570 			regs_size = TAHITI_IO_MC_REGS_SIZE;
1571 			break;
1572 		case CHIP_PITCAIRN:
1573 			io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1574 			regs_size = TAHITI_IO_MC_REGS_SIZE;
1575 			break;
1576 		case CHIP_VERDE:
1577 		default:
1578 			io_mc_regs = (u32 *)&verde_io_mc_regs;
1579 			regs_size = TAHITI_IO_MC_REGS_SIZE;
1580 			break;
1581 		case CHIP_OLAND:
1582 			io_mc_regs = (u32 *)&oland_io_mc_regs;
1583 			regs_size = TAHITI_IO_MC_REGS_SIZE;
1584 			break;
1585 		case CHIP_HAINAN:
1586 			io_mc_regs = (u32 *)&hainan_io_mc_regs;
1587 			regs_size = TAHITI_IO_MC_REGS_SIZE;
1588 			break;
1589 		}
1590 		fw_data = (const __be32 *)rdev->mc_fw->data;
1591 	}
1592 
1593 	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1594 
1595 	if (running == 0) {
1596 		if (running) {
1597 			blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1598 			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1599 		}
1600 
1601 		/* reset the engine and set to writable */
1602 		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1603 		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1604 
1605 		/* load mc io regs */
1606 		for (i = 0; i < regs_size; i++) {
1607 			if (rdev->new_fw) {
1608 				WREG32(MC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
1609 				WREG32(MC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
1610 			} else {
1611 				WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1612 				WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1613 			}
1614 		}
1615 		/* load the MC ucode */
1616 		for (i = 0; i < ucode_size; i++) {
1617 			if (rdev->new_fw)
1618 				WREG32(MC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
1619 			else
1620 				WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1621 		}
1622 
1623 		/* put the engine back into the active state */
1624 		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1625 		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1626 		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1627 
1628 		/* wait for training to complete */
1629 		for (i = 0; i < rdev->usec_timeout; i++) {
1630 			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1631 				break;
1632 			udelay(1);
1633 		}
1634 		for (i = 0; i < rdev->usec_timeout; i++) {
1635 			if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1636 				break;
1637 			udelay(1);
1638 		}
1639 
1640 		if (running)
1641 			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1642 	}
1643 
1644 	return 0;
1645 }
1646 
1647 static int si_init_microcode(struct radeon_device *rdev)
1648 {
1649 	const char *chip_name;
1650 	const char *new_chip_name;
1651 	size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1652 	size_t smc_req_size, mc2_req_size;
1653 	char fw_name[30];
1654 	int err;
1655 	int new_fw = 0;
1656 
1657 	DRM_DEBUG("\n");
1658 
1659 	switch (rdev->family) {
1660 	case CHIP_TAHITI:
1661 		chip_name = "TAHITI";
1662 		new_chip_name = "tahiti";
1663 		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1664 		me_req_size = SI_PM4_UCODE_SIZE * 4;
1665 		ce_req_size = SI_CE_UCODE_SIZE * 4;
1666 		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1667 		mc_req_size = SI_MC_UCODE_SIZE * 4;
1668 		mc2_req_size = TAHITI_MC_UCODE_SIZE * 4;
1669 		smc_req_size = ALIGN(TAHITI_SMC_UCODE_SIZE, 4);
1670 		break;
1671 	case CHIP_PITCAIRN:
1672 		chip_name = "PITCAIRN";
1673 		new_chip_name = "pitcairn";
1674 		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1675 		me_req_size = SI_PM4_UCODE_SIZE * 4;
1676 		ce_req_size = SI_CE_UCODE_SIZE * 4;
1677 		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1678 		mc_req_size = SI_MC_UCODE_SIZE * 4;
1679 		mc2_req_size = PITCAIRN_MC_UCODE_SIZE * 4;
1680 		smc_req_size = ALIGN(PITCAIRN_SMC_UCODE_SIZE, 4);
1681 		break;
1682 	case CHIP_VERDE:
1683 		chip_name = "VERDE";
1684 		new_chip_name = "verde";
1685 		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1686 		me_req_size = SI_PM4_UCODE_SIZE * 4;
1687 		ce_req_size = SI_CE_UCODE_SIZE * 4;
1688 		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1689 		mc_req_size = SI_MC_UCODE_SIZE * 4;
1690 		mc2_req_size = VERDE_MC_UCODE_SIZE * 4;
1691 		smc_req_size = ALIGN(VERDE_SMC_UCODE_SIZE, 4);
1692 		break;
1693 	case CHIP_OLAND:
1694 		chip_name = "OLAND";
1695 		new_chip_name = "oland";
1696 		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1697 		me_req_size = SI_PM4_UCODE_SIZE * 4;
1698 		ce_req_size = SI_CE_UCODE_SIZE * 4;
1699 		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1700 		mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1701 		smc_req_size = ALIGN(OLAND_SMC_UCODE_SIZE, 4);
1702 		break;
1703 	case CHIP_HAINAN:
1704 		chip_name = "HAINAN";
1705 		new_chip_name = "hainan";
1706 		pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1707 		me_req_size = SI_PM4_UCODE_SIZE * 4;
1708 		ce_req_size = SI_CE_UCODE_SIZE * 4;
1709 		rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1710 		mc_req_size = mc2_req_size = OLAND_MC_UCODE_SIZE * 4;
1711 		smc_req_size = ALIGN(HAINAN_SMC_UCODE_SIZE, 4);
1712 		break;
1713 	default: BUG();
1714 	}
1715 
1716 	DRM_INFO("Loading %s Microcode\n", new_chip_name);
1717 
1718 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", new_chip_name);
1719 	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1720 	if (err) {
1721 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1722 		err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
1723 		if (err)
1724 			goto out;
1725 		if (rdev->pfp_fw->size != pfp_req_size) {
1726 			printk(KERN_ERR
1727 			       "si_cp: Bogus length %zu in firmware \"%s\"\n",
1728 			       rdev->pfp_fw->size, fw_name);
1729 			err = -EINVAL;
1730 			goto out;
1731 		}
1732 	} else {
1733 		err = radeon_ucode_validate(rdev->pfp_fw);
1734 		if (err) {
1735 			printk(KERN_ERR
1736 			       "si_cp: validation failed for firmware \"%s\"\n",
1737 			       fw_name);
1738 			goto out;
1739 		} else {
1740 			new_fw++;
1741 		}
1742 	}
1743 
1744 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", new_chip_name);
1745 	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1746 	if (err) {
1747 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1748 		err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
1749 		if (err)
1750 			goto out;
1751 		if (rdev->me_fw->size != me_req_size) {
1752 			printk(KERN_ERR
1753 			       "si_cp: Bogus length %zu in firmware \"%s\"\n",
1754 			       rdev->me_fw->size, fw_name);
1755 			err = -EINVAL;
1756 		}
1757 	} else {
1758 		err = radeon_ucode_validate(rdev->me_fw);
1759 		if (err) {
1760 			printk(KERN_ERR
1761 			       "si_cp: validation failed for firmware \"%s\"\n",
1762 			       fw_name);
1763 			goto out;
1764 		} else {
1765 			new_fw++;
1766 		}
1767 	}
1768 
1769 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", new_chip_name);
1770 	err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1771 	if (err) {
1772 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1773 		err = request_firmware(&rdev->ce_fw, fw_name, rdev->dev);
1774 		if (err)
1775 			goto out;
1776 		if (rdev->ce_fw->size != ce_req_size) {
1777 			printk(KERN_ERR
1778 			       "si_cp: Bogus length %zu in firmware \"%s\"\n",
1779 			       rdev->ce_fw->size, fw_name);
1780 			err = -EINVAL;
1781 		}
1782 	} else {
1783 		err = radeon_ucode_validate(rdev->ce_fw);
1784 		if (err) {
1785 			printk(KERN_ERR
1786 			       "si_cp: validation failed for firmware \"%s\"\n",
1787 			       fw_name);
1788 			goto out;
1789 		} else {
1790 			new_fw++;
1791 		}
1792 	}
1793 
1794 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", new_chip_name);
1795 	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1796 	if (err) {
1797 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", chip_name);
1798 		err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
1799 		if (err)
1800 			goto out;
1801 		if (rdev->rlc_fw->size != rlc_req_size) {
1802 			printk(KERN_ERR
1803 			       "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1804 			       rdev->rlc_fw->size, fw_name);
1805 			err = -EINVAL;
1806 		}
1807 	} else {
1808 		err = radeon_ucode_validate(rdev->rlc_fw);
1809 		if (err) {
1810 			printk(KERN_ERR
1811 			       "si_cp: validation failed for firmware \"%s\"\n",
1812 			       fw_name);
1813 			goto out;
1814 		} else {
1815 			new_fw++;
1816 		}
1817 	}
1818 
1819 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", new_chip_name);
1820 	err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1821 	if (err) {
1822 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc2.bin", chip_name);
1823 		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1824 		if (err) {
1825 			snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1826 			err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
1827 			if (err)
1828 				goto out;
1829 		}
1830 		if ((rdev->mc_fw->size != mc_req_size) &&
1831 		    (rdev->mc_fw->size != mc2_req_size)) {
1832 			printk(KERN_ERR
1833 			       "si_mc: Bogus length %zu in firmware \"%s\"\n",
1834 			       rdev->mc_fw->size, fw_name);
1835 			err = -EINVAL;
1836 		}
1837 		DRM_INFO("%s: %zu bytes\n", fw_name, rdev->mc_fw->size);
1838 	} else {
1839 		err = radeon_ucode_validate(rdev->mc_fw);
1840 		if (err) {
1841 			printk(KERN_ERR
1842 			       "si_cp: validation failed for firmware \"%s\"\n",
1843 			       fw_name);
1844 			goto out;
1845 		} else {
1846 			new_fw++;
1847 		}
1848 	}
1849 
1850 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
1851 	err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1852 	if (err) {
1853 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
1854 		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
1855 		if (err) {
1856 			printk(KERN_ERR
1857 			       "smc: error loading firmware \"%s\"\n",
1858 			       fw_name);
1859 			release_firmware(rdev->smc_fw);
1860 			rdev->smc_fw = NULL;
1861 			err = 0;
1862 		} else if (rdev->smc_fw->size != smc_req_size) {
1863 			printk(KERN_ERR
1864 			       "si_smc: Bogus length %zu in firmware \"%s\"\n",
1865 			       rdev->smc_fw->size, fw_name);
1866 			err = -EINVAL;
1867 		}
1868 	} else {
1869 		err = radeon_ucode_validate(rdev->smc_fw);
1870 		if (err) {
1871 			printk(KERN_ERR
1872 			       "si_cp: validation failed for firmware \"%s\"\n",
1873 			       fw_name);
1874 			goto out;
1875 		} else {
1876 			new_fw++;
1877 		}
1878 	}
1879 
1880 	if (new_fw == 0) {
1881 		rdev->new_fw = false;
1882 	} else if (new_fw < 6) {
1883 		printk(KERN_ERR "si_fw: mixing new and old firmware!\n");
1884 		err = -EINVAL;
1885 	} else {
1886 		rdev->new_fw = true;
1887 	}
1888 out:
1889 	if (err) {
1890 		if (err != -EINVAL)
1891 			printk(KERN_ERR
1892 			       "si_cp: Failed to load firmware \"%s\"\n",
1893 			       fw_name);
1894 		release_firmware(rdev->pfp_fw);
1895 		rdev->pfp_fw = NULL;
1896 		release_firmware(rdev->me_fw);
1897 		rdev->me_fw = NULL;
1898 		release_firmware(rdev->ce_fw);
1899 		rdev->ce_fw = NULL;
1900 		release_firmware(rdev->rlc_fw);
1901 		rdev->rlc_fw = NULL;
1902 		release_firmware(rdev->mc_fw);
1903 		rdev->mc_fw = NULL;
1904 		release_firmware(rdev->smc_fw);
1905 		rdev->smc_fw = NULL;
1906 	}
1907 	return err;
1908 }
1909 
1910 /* watermark setup */
1911 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1912 				   struct radeon_crtc *radeon_crtc,
1913 				   struct drm_display_mode *mode,
1914 				   struct drm_display_mode *other_mode)
1915 {
1916 	u32 tmp, buffer_alloc, i;
1917 	u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1918 	/*
1919 	 * Line Buffer Setup
1920 	 * There are 3 line buffers, each one shared by 2 display controllers.
1921 	 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1922 	 * the display controllers.  The paritioning is done via one of four
1923 	 * preset allocations specified in bits 21:20:
1924 	 *  0 - half lb
1925 	 *  2 - whole lb, other crtc must be disabled
1926 	 */
1927 	/* this can get tricky if we have two large displays on a paired group
1928 	 * of crtcs.  Ideally for multiple large displays we'd assign them to
1929 	 * non-linked crtcs for maximum line buffer allocation.
1930 	 */
1931 	if (radeon_crtc->base.enabled && mode) {
1932 		if (other_mode) {
1933 			tmp = 0; /* 1/2 */
1934 			buffer_alloc = 1;
1935 		} else {
1936 			tmp = 2; /* whole */
1937 			buffer_alloc = 2;
1938 		}
1939 	} else {
1940 		tmp = 0;
1941 		buffer_alloc = 0;
1942 	}
1943 
1944 	WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1945 	       DC_LB_MEMORY_CONFIG(tmp));
1946 
1947 	WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1948 	       DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1949 	for (i = 0; i < rdev->usec_timeout; i++) {
1950 		if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1951 		    DMIF_BUFFERS_ALLOCATED_COMPLETED)
1952 			break;
1953 		udelay(1);
1954 	}
1955 
1956 	if (radeon_crtc->base.enabled && mode) {
1957 		switch (tmp) {
1958 		case 0:
1959 		default:
1960 			return 4096 * 2;
1961 		case 2:
1962 			return 8192 * 2;
1963 		}
1964 	}
1965 
1966 	/* controller not enabled, so no lb used */
1967 	return 0;
1968 }
1969 
1970 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1971 {
1972 	u32 tmp = RREG32(MC_SHARED_CHMAP);
1973 
1974 	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1975 	case 0:
1976 	default:
1977 		return 1;
1978 	case 1:
1979 		return 2;
1980 	case 2:
1981 		return 4;
1982 	case 3:
1983 		return 8;
1984 	case 4:
1985 		return 3;
1986 	case 5:
1987 		return 6;
1988 	case 6:
1989 		return 10;
1990 	case 7:
1991 		return 12;
1992 	case 8:
1993 		return 16;
1994 	}
1995 }
1996 
1997 struct dce6_wm_params {
1998 	u32 dram_channels; /* number of dram channels */
1999 	u32 yclk;          /* bandwidth per dram data pin in kHz */
2000 	u32 sclk;          /* engine clock in kHz */
2001 	u32 disp_clk;      /* display clock in kHz */
2002 	u32 src_width;     /* viewport width */
2003 	u32 active_time;   /* active display time in ns */
2004 	u32 blank_time;    /* blank time in ns */
2005 	bool interlaced;    /* mode is interlaced */
2006 	fixed20_12 vsc;    /* vertical scale ratio */
2007 	u32 num_heads;     /* number of active crtcs */
2008 	u32 bytes_per_pixel; /* bytes per pixel display + overlay */
2009 	u32 lb_size;       /* line buffer allocated to pipe */
2010 	u32 vtaps;         /* vertical scaler taps */
2011 };
2012 
2013 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
2014 {
2015 	/* Calculate raw DRAM Bandwidth */
2016 	fixed20_12 dram_efficiency; /* 0.7 */
2017 	fixed20_12 yclk, dram_channels, bandwidth;
2018 	fixed20_12 a;
2019 
2020 	a.full = dfixed_const(1000);
2021 	yclk.full = dfixed_const(wm->yclk);
2022 	yclk.full = dfixed_div(yclk, a);
2023 	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2024 	a.full = dfixed_const(10);
2025 	dram_efficiency.full = dfixed_const(7);
2026 	dram_efficiency.full = dfixed_div(dram_efficiency, a);
2027 	bandwidth.full = dfixed_mul(dram_channels, yclk);
2028 	bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
2029 
2030 	return dfixed_trunc(bandwidth);
2031 }
2032 
2033 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2034 {
2035 	/* Calculate DRAM Bandwidth and the part allocated to display. */
2036 	fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
2037 	fixed20_12 yclk, dram_channels, bandwidth;
2038 	fixed20_12 a;
2039 
2040 	a.full = dfixed_const(1000);
2041 	yclk.full = dfixed_const(wm->yclk);
2042 	yclk.full = dfixed_div(yclk, a);
2043 	dram_channels.full = dfixed_const(wm->dram_channels * 4);
2044 	a.full = dfixed_const(10);
2045 	disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
2046 	disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
2047 	bandwidth.full = dfixed_mul(dram_channels, yclk);
2048 	bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
2049 
2050 	return dfixed_trunc(bandwidth);
2051 }
2052 
2053 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
2054 {
2055 	/* Calculate the display Data return Bandwidth */
2056 	fixed20_12 return_efficiency; /* 0.8 */
2057 	fixed20_12 sclk, bandwidth;
2058 	fixed20_12 a;
2059 
2060 	a.full = dfixed_const(1000);
2061 	sclk.full = dfixed_const(wm->sclk);
2062 	sclk.full = dfixed_div(sclk, a);
2063 	a.full = dfixed_const(10);
2064 	return_efficiency.full = dfixed_const(8);
2065 	return_efficiency.full = dfixed_div(return_efficiency, a);
2066 	a.full = dfixed_const(32);
2067 	bandwidth.full = dfixed_mul(a, sclk);
2068 	bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2069 
2070 	return dfixed_trunc(bandwidth);
2071 }
2072 
2073 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
2074 {
2075 	return 32;
2076 }
2077 
2078 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
2079 {
2080 	/* Calculate the DMIF Request Bandwidth */
2081 	fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2082 	fixed20_12 disp_clk, sclk, bandwidth;
2083 	fixed20_12 a, b1, b2;
2084 	u32 min_bandwidth;
2085 
2086 	a.full = dfixed_const(1000);
2087 	disp_clk.full = dfixed_const(wm->disp_clk);
2088 	disp_clk.full = dfixed_div(disp_clk, a);
2089 	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
2090 	b1.full = dfixed_mul(a, disp_clk);
2091 
2092 	a.full = dfixed_const(1000);
2093 	sclk.full = dfixed_const(wm->sclk);
2094 	sclk.full = dfixed_div(sclk, a);
2095 	a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
2096 	b2.full = dfixed_mul(a, sclk);
2097 
2098 	a.full = dfixed_const(10);
2099 	disp_clk_request_efficiency.full = dfixed_const(8);
2100 	disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2101 
2102 	min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
2103 
2104 	a.full = dfixed_const(min_bandwidth);
2105 	bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
2106 
2107 	return dfixed_trunc(bandwidth);
2108 }
2109 
2110 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
2111 {
2112 	/* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2113 	u32 dram_bandwidth = dce6_dram_bandwidth(wm);
2114 	u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
2115 	u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
2116 
2117 	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2118 }
2119 
2120 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
2121 {
2122 	/* Calculate the display mode Average Bandwidth
2123 	 * DisplayMode should contain the source and destination dimensions,
2124 	 * timing, etc.
2125 	 */
2126 	fixed20_12 bpp;
2127 	fixed20_12 line_time;
2128 	fixed20_12 src_width;
2129 	fixed20_12 bandwidth;
2130 	fixed20_12 a;
2131 
2132 	a.full = dfixed_const(1000);
2133 	line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2134 	line_time.full = dfixed_div(line_time, a);
2135 	bpp.full = dfixed_const(wm->bytes_per_pixel);
2136 	src_width.full = dfixed_const(wm->src_width);
2137 	bandwidth.full = dfixed_mul(src_width, bpp);
2138 	bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2139 	bandwidth.full = dfixed_div(bandwidth, line_time);
2140 
2141 	return dfixed_trunc(bandwidth);
2142 }
2143 
2144 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
2145 {
2146 	/* First calcualte the latency in ns */
2147 	u32 mc_latency = 2000; /* 2000 ns. */
2148 	u32 available_bandwidth = dce6_available_bandwidth(wm);
2149 	u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2150 	u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2151 	u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2152 	u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2153 		(wm->num_heads * cursor_line_pair_return_time);
2154 	u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2155 	u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2156 	u32 tmp, dmif_size = 12288;
2157 	fixed20_12 a, b, c;
2158 
2159 	if (wm->num_heads == 0)
2160 		return 0;
2161 
2162 	a.full = dfixed_const(2);
2163 	b.full = dfixed_const(1);
2164 	if ((wm->vsc.full > a.full) ||
2165 	    ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2166 	    (wm->vtaps >= 5) ||
2167 	    ((wm->vsc.full >= a.full) && wm->interlaced))
2168 		max_src_lines_per_dst_line = 4;
2169 	else
2170 		max_src_lines_per_dst_line = 2;
2171 
2172 	a.full = dfixed_const(available_bandwidth);
2173 	b.full = dfixed_const(wm->num_heads);
2174 	a.full = dfixed_div(a, b);
2175 
2176 	b.full = dfixed_const(mc_latency + 512);
2177 	c.full = dfixed_const(wm->disp_clk);
2178 	b.full = dfixed_div(b, c);
2179 
2180 	c.full = dfixed_const(dmif_size);
2181 	b.full = dfixed_div(c, b);
2182 
2183 	tmp = min(dfixed_trunc(a), dfixed_trunc(b));
2184 
2185 	b.full = dfixed_const(1000);
2186 	c.full = dfixed_const(wm->disp_clk);
2187 	b.full = dfixed_div(c, b);
2188 	c.full = dfixed_const(wm->bytes_per_pixel);
2189 	b.full = dfixed_mul(b, c);
2190 
2191 	lb_fill_bw = min(tmp, dfixed_trunc(b));
2192 
2193 	a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2194 	b.full = dfixed_const(1000);
2195 	c.full = dfixed_const(lb_fill_bw);
2196 	b.full = dfixed_div(c, b);
2197 	a.full = dfixed_div(a, b);
2198 	line_fill_time = dfixed_trunc(a);
2199 
2200 	if (line_fill_time < wm->active_time)
2201 		return latency;
2202 	else
2203 		return latency + (line_fill_time - wm->active_time);
2204 
2205 }
2206 
2207 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
2208 {
2209 	if (dce6_average_bandwidth(wm) <=
2210 	    (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
2211 		return true;
2212 	else
2213 		return false;
2214 };
2215 
2216 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
2217 {
2218 	if (dce6_average_bandwidth(wm) <=
2219 	    (dce6_available_bandwidth(wm) / wm->num_heads))
2220 		return true;
2221 	else
2222 		return false;
2223 };
2224 
2225 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
2226 {
2227 	u32 lb_partitions = wm->lb_size / wm->src_width;
2228 	u32 line_time = wm->active_time + wm->blank_time;
2229 	u32 latency_tolerant_lines;
2230 	u32 latency_hiding;
2231 	fixed20_12 a;
2232 
2233 	a.full = dfixed_const(1);
2234 	if (wm->vsc.full > a.full)
2235 		latency_tolerant_lines = 1;
2236 	else {
2237 		if (lb_partitions <= (wm->vtaps + 1))
2238 			latency_tolerant_lines = 1;
2239 		else
2240 			latency_tolerant_lines = 2;
2241 	}
2242 
2243 	latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2244 
2245 	if (dce6_latency_watermark(wm) <= latency_hiding)
2246 		return true;
2247 	else
2248 		return false;
2249 }
2250 
2251 static void dce6_program_watermarks(struct radeon_device *rdev,
2252 					 struct radeon_crtc *radeon_crtc,
2253 					 u32 lb_size, u32 num_heads)
2254 {
2255 	struct drm_display_mode *mode = &radeon_crtc->base.mode;
2256 	struct dce6_wm_params wm_low, wm_high;
2257 	u32 dram_channels;
2258 	u32 pixel_period;
2259 	u32 line_time = 0;
2260 	u32 latency_watermark_a = 0, latency_watermark_b = 0;
2261 	u32 priority_a_mark = 0, priority_b_mark = 0;
2262 	u32 priority_a_cnt = PRIORITY_OFF;
2263 	u32 priority_b_cnt = PRIORITY_OFF;
2264 	u32 tmp, arb_control3;
2265 	fixed20_12 a, b, c;
2266 
2267 	if (radeon_crtc->base.enabled && num_heads && mode) {
2268 		pixel_period = 1000000 / (u32)mode->clock;
2269 		line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
2270 		priority_a_cnt = 0;
2271 		priority_b_cnt = 0;
2272 
2273 		if (rdev->family == CHIP_ARUBA)
2274 			dram_channels = evergreen_get_number_of_dram_channels(rdev);
2275 		else
2276 			dram_channels = si_get_number_of_dram_channels(rdev);
2277 
2278 		/* watermark for high clocks */
2279 		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2280 			wm_high.yclk =
2281 				radeon_dpm_get_mclk(rdev, false) * 10;
2282 			wm_high.sclk =
2283 				radeon_dpm_get_sclk(rdev, false) * 10;
2284 		} else {
2285 			wm_high.yclk = rdev->pm.current_mclk * 10;
2286 			wm_high.sclk = rdev->pm.current_sclk * 10;
2287 		}
2288 
2289 		wm_high.disp_clk = mode->clock;
2290 		wm_high.src_width = mode->crtc_hdisplay;
2291 		wm_high.active_time = mode->crtc_hdisplay * pixel_period;
2292 		wm_high.blank_time = line_time - wm_high.active_time;
2293 		wm_high.interlaced = false;
2294 		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2295 			wm_high.interlaced = true;
2296 		wm_high.vsc = radeon_crtc->vsc;
2297 		wm_high.vtaps = 1;
2298 		if (radeon_crtc->rmx_type != RMX_OFF)
2299 			wm_high.vtaps = 2;
2300 		wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2301 		wm_high.lb_size = lb_size;
2302 		wm_high.dram_channels = dram_channels;
2303 		wm_high.num_heads = num_heads;
2304 
2305 		/* watermark for low clocks */
2306 		if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2307 			wm_low.yclk =
2308 				radeon_dpm_get_mclk(rdev, true) * 10;
2309 			wm_low.sclk =
2310 				radeon_dpm_get_sclk(rdev, true) * 10;
2311 		} else {
2312 			wm_low.yclk = rdev->pm.current_mclk * 10;
2313 			wm_low.sclk = rdev->pm.current_sclk * 10;
2314 		}
2315 
2316 		wm_low.disp_clk = mode->clock;
2317 		wm_low.src_width = mode->crtc_hdisplay;
2318 		wm_low.active_time = mode->crtc_hdisplay * pixel_period;
2319 		wm_low.blank_time = line_time - wm_low.active_time;
2320 		wm_low.interlaced = false;
2321 		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2322 			wm_low.interlaced = true;
2323 		wm_low.vsc = radeon_crtc->vsc;
2324 		wm_low.vtaps = 1;
2325 		if (radeon_crtc->rmx_type != RMX_OFF)
2326 			wm_low.vtaps = 2;
2327 		wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2328 		wm_low.lb_size = lb_size;
2329 		wm_low.dram_channels = dram_channels;
2330 		wm_low.num_heads = num_heads;
2331 
2332 		/* set for high clocks */
2333 		latency_watermark_a = min(dce6_latency_watermark(&wm_high), (u32)65535);
2334 		/* set for low clocks */
2335 		latency_watermark_b = min(dce6_latency_watermark(&wm_low), (u32)65535);
2336 
2337 		/* possibly force display priority to high */
2338 		/* should really do this at mode validation time... */
2339 		if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2340 		    !dce6_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2341 		    !dce6_check_latency_hiding(&wm_high) ||
2342 		    (rdev->disp_priority == 2)) {
2343 			DRM_DEBUG_KMS("force priority to high\n");
2344 			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2345 			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2346 		}
2347 		if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2348 		    !dce6_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2349 		    !dce6_check_latency_hiding(&wm_low) ||
2350 		    (rdev->disp_priority == 2)) {
2351 			DRM_DEBUG_KMS("force priority to high\n");
2352 			priority_a_cnt |= PRIORITY_ALWAYS_ON;
2353 			priority_b_cnt |= PRIORITY_ALWAYS_ON;
2354 		}
2355 
2356 		a.full = dfixed_const(1000);
2357 		b.full = dfixed_const(mode->clock);
2358 		b.full = dfixed_div(b, a);
2359 		c.full = dfixed_const(latency_watermark_a);
2360 		c.full = dfixed_mul(c, b);
2361 		c.full = dfixed_mul(c, radeon_crtc->hsc);
2362 		c.full = dfixed_div(c, a);
2363 		a.full = dfixed_const(16);
2364 		c.full = dfixed_div(c, a);
2365 		priority_a_mark = dfixed_trunc(c);
2366 		priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2367 
2368 		a.full = dfixed_const(1000);
2369 		b.full = dfixed_const(mode->clock);
2370 		b.full = dfixed_div(b, a);
2371 		c.full = dfixed_const(latency_watermark_b);
2372 		c.full = dfixed_mul(c, b);
2373 		c.full = dfixed_mul(c, radeon_crtc->hsc);
2374 		c.full = dfixed_div(c, a);
2375 		a.full = dfixed_const(16);
2376 		c.full = dfixed_div(c, a);
2377 		priority_b_mark = dfixed_trunc(c);
2378 		priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2379 	}
2380 
2381 	/* select wm A */
2382 	arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2383 	tmp = arb_control3;
2384 	tmp &= ~LATENCY_WATERMARK_MASK(3);
2385 	tmp |= LATENCY_WATERMARK_MASK(1);
2386 	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2387 	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2388 	       (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2389 		LATENCY_HIGH_WATERMARK(line_time)));
2390 	/* select wm B */
2391 	tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
2392 	tmp &= ~LATENCY_WATERMARK_MASK(3);
2393 	tmp |= LATENCY_WATERMARK_MASK(2);
2394 	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
2395 	WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
2396 	       (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2397 		LATENCY_HIGH_WATERMARK(line_time)));
2398 	/* restore original selection */
2399 	WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
2400 
2401 	/* write the priority marks */
2402 	WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2403 	WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2404 
2405 	/* save values for DPM */
2406 	radeon_crtc->line_time = line_time;
2407 	radeon_crtc->wm_high = latency_watermark_a;
2408 	radeon_crtc->wm_low = latency_watermark_b;
2409 }
2410 
2411 void dce6_bandwidth_update(struct radeon_device *rdev)
2412 {
2413 	struct drm_display_mode *mode0 = NULL;
2414 	struct drm_display_mode *mode1 = NULL;
2415 	u32 num_heads = 0, lb_size;
2416 	int i;
2417 
2418 	if (!rdev->mode_info.mode_config_initialized)
2419 		return;
2420 
2421 	radeon_update_display_priority(rdev);
2422 
2423 	for (i = 0; i < rdev->num_crtc; i++) {
2424 		if (rdev->mode_info.crtcs[i]->base.enabled)
2425 			num_heads++;
2426 	}
2427 	for (i = 0; i < rdev->num_crtc; i += 2) {
2428 		mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2429 		mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2430 		lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2431 		dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2432 		lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2433 		dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2434 	}
2435 }
2436 
2437 /*
2438  * Core functions
2439  */
2440 static void si_tiling_mode_table_init(struct radeon_device *rdev)
2441 {
2442 	const u32 num_tile_mode_states = 32;
2443 	u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
2444 
2445 	switch (rdev->config.si.mem_row_size_in_kb) {
2446 	case 1:
2447 		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
2448 		break;
2449 	case 2:
2450 	default:
2451 		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
2452 		break;
2453 	case 4:
2454 		split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
2455 		break;
2456 	}
2457 
2458 	if ((rdev->family == CHIP_TAHITI) ||
2459 	    (rdev->family == CHIP_PITCAIRN)) {
2460 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2461 			switch (reg_offset) {
2462 			case 0:  /* non-AA compressed depth or any compressed stencil */
2463 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2464 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2465 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2466 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2467 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2468 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2469 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2470 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2471 				break;
2472 			case 1:  /* 2xAA/4xAA compressed depth only */
2473 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2474 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2475 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2476 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2477 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2478 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2479 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2480 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2481 				break;
2482 			case 2:  /* 8xAA compressed depth only */
2483 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2484 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2485 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2486 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2487 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2488 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2489 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2490 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2491 				break;
2492 			case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2493 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2494 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2495 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2496 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2497 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2498 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2499 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2500 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2501 				break;
2502 			case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2503 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2504 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2505 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2506 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2507 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2508 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2509 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2510 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2511 				break;
2512 			case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2513 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2514 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2515 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2516 						 TILE_SPLIT(split_equal_to_row_size) |
2517 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2518 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2519 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2520 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2521 				break;
2522 			case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2523 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2524 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2525 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2526 						 TILE_SPLIT(split_equal_to_row_size) |
2527 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2528 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2529 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2530 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2531 				break;
2532 			case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2533 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2534 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2535 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2536 						 TILE_SPLIT(split_equal_to_row_size) |
2537 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2538 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2539 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2540 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2541 				break;
2542 			case 8:  /* 1D and 1D Array Surfaces */
2543 				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2544 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2545 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2546 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2547 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2548 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2549 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2550 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2551 				break;
2552 			case 9:  /* Displayable maps. */
2553 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2554 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2555 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2556 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2557 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2558 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2559 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2560 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2561 				break;
2562 			case 10:  /* Display 8bpp. */
2563 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2564 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2565 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2566 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2567 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2568 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2569 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2570 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2571 				break;
2572 			case 11:  /* Display 16bpp. */
2573 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2574 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2575 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2576 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2577 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2578 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2579 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2580 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2581 				break;
2582 			case 12:  /* Display 32bpp. */
2583 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2584 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2585 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2586 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2587 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2588 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2589 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2590 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2591 				break;
2592 			case 13:  /* Thin. */
2593 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2594 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2595 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2596 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2597 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2598 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2599 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2600 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2601 				break;
2602 			case 14:  /* Thin 8 bpp. */
2603 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2604 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2605 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2606 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2607 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2608 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2609 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2610 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2611 				break;
2612 			case 15:  /* Thin 16 bpp. */
2613 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2614 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2615 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2616 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2617 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2618 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2619 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2620 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2621 				break;
2622 			case 16:  /* Thin 32 bpp. */
2623 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2624 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2625 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2626 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2627 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2628 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2629 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2630 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2631 				break;
2632 			case 17:  /* Thin 64 bpp. */
2633 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2634 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2635 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2636 						 TILE_SPLIT(split_equal_to_row_size) |
2637 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2638 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2639 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2640 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2641 				break;
2642 			case 21:  /* 8 bpp PRT. */
2643 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2644 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2645 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2646 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2647 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2648 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2649 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2650 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2651 				break;
2652 			case 22:  /* 16 bpp PRT */
2653 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2654 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2655 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2656 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2657 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2658 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2659 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2660 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2661 				break;
2662 			case 23:  /* 32 bpp PRT */
2663 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2664 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2665 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2666 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2667 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2668 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2669 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2670 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2671 				break;
2672 			case 24:  /* 64 bpp PRT */
2673 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2674 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2675 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2676 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2677 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2678 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2679 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2680 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2681 				break;
2682 			case 25:  /* 128 bpp PRT */
2683 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2684 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2685 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2686 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2687 						 NUM_BANKS(ADDR_SURF_8_BANK) |
2688 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2689 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2690 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2691 				break;
2692 			default:
2693 				gb_tile_moden = 0;
2694 				break;
2695 			}
2696 			rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2697 			WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2698 		}
2699 	} else if ((rdev->family == CHIP_VERDE) ||
2700 		   (rdev->family == CHIP_OLAND) ||
2701 		   (rdev->family == CHIP_HAINAN)) {
2702 		for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2703 			switch (reg_offset) {
2704 			case 0:  /* non-AA compressed depth or any compressed stencil */
2705 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2706 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2707 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2708 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2709 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2710 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2711 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2712 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2713 				break;
2714 			case 1:  /* 2xAA/4xAA compressed depth only */
2715 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2716 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2717 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2718 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2719 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2720 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2721 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2722 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2723 				break;
2724 			case 2:  /* 8xAA compressed depth only */
2725 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2726 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2727 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2728 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2729 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2730 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2731 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2732 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2733 				break;
2734 			case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2735 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2736 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2737 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2738 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2739 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2740 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2741 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2742 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2743 				break;
2744 			case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2745 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2746 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2747 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2748 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2749 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2750 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2751 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2752 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2753 				break;
2754 			case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2755 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2756 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2757 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2758 						 TILE_SPLIT(split_equal_to_row_size) |
2759 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2760 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2761 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2762 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2763 				break;
2764 			case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2765 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2766 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2767 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2768 						 TILE_SPLIT(split_equal_to_row_size) |
2769 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2770 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2771 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2772 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2773 				break;
2774 			case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2775 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2776 						 MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2777 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2778 						 TILE_SPLIT(split_equal_to_row_size) |
2779 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2780 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2781 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2782 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2783 				break;
2784 			case 8:  /* 1D and 1D Array Surfaces */
2785 				gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2786 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2787 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2788 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2789 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2790 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2791 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2792 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2793 				break;
2794 			case 9:  /* Displayable maps. */
2795 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2796 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2797 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2798 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2799 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2800 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2801 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2802 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2803 				break;
2804 			case 10:  /* Display 8bpp. */
2805 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2806 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2807 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2808 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2809 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2810 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2811 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2812 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2813 				break;
2814 			case 11:  /* Display 16bpp. */
2815 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2816 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2817 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2818 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2819 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2820 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2821 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2822 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2823 				break;
2824 			case 12:  /* Display 32bpp. */
2825 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2826 						 MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2827 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2828 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2829 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2830 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2831 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2832 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2833 				break;
2834 			case 13:  /* Thin. */
2835 				gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2836 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2837 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2838 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2839 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2840 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2841 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2842 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2843 				break;
2844 			case 14:  /* Thin 8 bpp. */
2845 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2846 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2847 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2848 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2849 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2850 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2851 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2852 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2853 				break;
2854 			case 15:  /* Thin 16 bpp. */
2855 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2856 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2857 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2858 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2859 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2860 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2861 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2862 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2863 				break;
2864 			case 16:  /* Thin 32 bpp. */
2865 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2866 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2867 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2868 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2869 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2870 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2871 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2872 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2873 				break;
2874 			case 17:  /* Thin 64 bpp. */
2875 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2876 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2877 						 PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2878 						 TILE_SPLIT(split_equal_to_row_size) |
2879 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2880 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2881 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2882 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2883 				break;
2884 			case 21:  /* 8 bpp PRT. */
2885 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2886 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2887 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2888 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2889 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2890 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2891 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2892 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2893 				break;
2894 			case 22:  /* 16 bpp PRT */
2895 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2896 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2897 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2898 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2899 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2900 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2901 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2902 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2903 				break;
2904 			case 23:  /* 32 bpp PRT */
2905 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2906 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2907 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2908 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2909 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2910 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2911 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2912 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2913 				break;
2914 			case 24:  /* 64 bpp PRT */
2915 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2916 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2917 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2918 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2919 						 NUM_BANKS(ADDR_SURF_16_BANK) |
2920 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2921 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2922 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2923 				break;
2924 			case 25:  /* 128 bpp PRT */
2925 				gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2926 						 MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2927 						 PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2928 						 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2929 						 NUM_BANKS(ADDR_SURF_8_BANK) |
2930 						 BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2931 						 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2932 						 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2933 				break;
2934 			default:
2935 				gb_tile_moden = 0;
2936 				break;
2937 			}
2938 			rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2939 			WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2940 		}
2941 	} else
2942 		DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2943 }
2944 
2945 static void si_select_se_sh(struct radeon_device *rdev,
2946 			    u32 se_num, u32 sh_num)
2947 {
2948 	u32 data = INSTANCE_BROADCAST_WRITES;
2949 
2950 	if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2951 		data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2952 	else if (se_num == 0xffffffff)
2953 		data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2954 	else if (sh_num == 0xffffffff)
2955 		data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2956 	else
2957 		data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2958 	WREG32(GRBM_GFX_INDEX, data);
2959 }
2960 
2961 static u32 si_create_bitmask(u32 bit_width)
2962 {
2963 	u32 i, mask = 0;
2964 
2965 	for (i = 0; i < bit_width; i++) {
2966 		mask <<= 1;
2967 		mask |= 1;
2968 	}
2969 	return mask;
2970 }
2971 
2972 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2973 {
2974 	u32 data, mask;
2975 
2976 	data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2977 	if (data & 1)
2978 		data &= INACTIVE_CUS_MASK;
2979 	else
2980 		data = 0;
2981 	data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2982 
2983 	data >>= INACTIVE_CUS_SHIFT;
2984 
2985 	mask = si_create_bitmask(cu_per_sh);
2986 
2987 	return ~data & mask;
2988 }
2989 
2990 static void si_setup_spi(struct radeon_device *rdev,
2991 			 u32 se_num, u32 sh_per_se,
2992 			 u32 cu_per_sh)
2993 {
2994 	int i, j, k;
2995 	u32 data, mask, active_cu;
2996 
2997 	for (i = 0; i < se_num; i++) {
2998 		for (j = 0; j < sh_per_se; j++) {
2999 			si_select_se_sh(rdev, i, j);
3000 			data = RREG32(SPI_STATIC_THREAD_MGMT_3);
3001 			active_cu = si_get_cu_enabled(rdev, cu_per_sh);
3002 
3003 			mask = 1;
3004 			for (k = 0; k < 16; k++) {
3005 				mask <<= k;
3006 				if (active_cu & mask) {
3007 					data &= ~mask;
3008 					WREG32(SPI_STATIC_THREAD_MGMT_3, data);
3009 					break;
3010 				}
3011 			}
3012 		}
3013 	}
3014 	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3015 }
3016 
3017 static u32 si_get_rb_disabled(struct radeon_device *rdev,
3018 			      u32 max_rb_num_per_se,
3019 			      u32 sh_per_se)
3020 {
3021 	u32 data, mask;
3022 
3023 	data = RREG32(CC_RB_BACKEND_DISABLE);
3024 	if (data & 1)
3025 		data &= BACKEND_DISABLE_MASK;
3026 	else
3027 		data = 0;
3028 	data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
3029 
3030 	data >>= BACKEND_DISABLE_SHIFT;
3031 
3032 	mask = si_create_bitmask(max_rb_num_per_se / sh_per_se);
3033 
3034 	return data & mask;
3035 }
3036 
3037 static void si_setup_rb(struct radeon_device *rdev,
3038 			u32 se_num, u32 sh_per_se,
3039 			u32 max_rb_num_per_se)
3040 {
3041 	int i, j;
3042 	u32 data, mask;
3043 	u32 disabled_rbs = 0;
3044 	u32 enabled_rbs = 0;
3045 
3046 	for (i = 0; i < se_num; i++) {
3047 		for (j = 0; j < sh_per_se; j++) {
3048 			si_select_se_sh(rdev, i, j);
3049 			data = si_get_rb_disabled(rdev, max_rb_num_per_se, sh_per_se);
3050 			disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
3051 		}
3052 	}
3053 	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3054 
3055 	mask = 1;
3056 	for (i = 0; i < max_rb_num_per_se * se_num; i++) {
3057 		if (!(disabled_rbs & mask))
3058 			enabled_rbs |= mask;
3059 		mask <<= 1;
3060 	}
3061 
3062 	rdev->config.si.backend_enable_mask = enabled_rbs;
3063 
3064 	for (i = 0; i < se_num; i++) {
3065 		si_select_se_sh(rdev, i, 0xffffffff);
3066 		data = 0;
3067 		for (j = 0; j < sh_per_se; j++) {
3068 			switch (enabled_rbs & 3) {
3069 			case 1:
3070 				data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
3071 				break;
3072 			case 2:
3073 				data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
3074 				break;
3075 			case 3:
3076 			default:
3077 				data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
3078 				break;
3079 			}
3080 			enabled_rbs >>= 2;
3081 		}
3082 		WREG32(PA_SC_RASTER_CONFIG, data);
3083 	}
3084 	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
3085 }
3086 
3087 static void si_gpu_init(struct radeon_device *rdev)
3088 {
3089 	u32 gb_addr_config = 0;
3090 	u32 mc_shared_chmap, mc_arb_ramcfg;
3091 	u32 sx_debug_1;
3092 	u32 hdp_host_path_cntl;
3093 	u32 tmp;
3094 	int i, j;
3095 
3096 	switch (rdev->family) {
3097 	case CHIP_TAHITI:
3098 		rdev->config.si.max_shader_engines = 2;
3099 		rdev->config.si.max_tile_pipes = 12;
3100 		rdev->config.si.max_cu_per_sh = 8;
3101 		rdev->config.si.max_sh_per_se = 2;
3102 		rdev->config.si.max_backends_per_se = 4;
3103 		rdev->config.si.max_texture_channel_caches = 12;
3104 		rdev->config.si.max_gprs = 256;
3105 		rdev->config.si.max_gs_threads = 32;
3106 		rdev->config.si.max_hw_contexts = 8;
3107 
3108 		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3109 		rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3110 		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3111 		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3112 		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3113 		break;
3114 	case CHIP_PITCAIRN:
3115 		rdev->config.si.max_shader_engines = 2;
3116 		rdev->config.si.max_tile_pipes = 8;
3117 		rdev->config.si.max_cu_per_sh = 5;
3118 		rdev->config.si.max_sh_per_se = 2;
3119 		rdev->config.si.max_backends_per_se = 4;
3120 		rdev->config.si.max_texture_channel_caches = 8;
3121 		rdev->config.si.max_gprs = 256;
3122 		rdev->config.si.max_gs_threads = 32;
3123 		rdev->config.si.max_hw_contexts = 8;
3124 
3125 		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3126 		rdev->config.si.sc_prim_fifo_size_backend = 0x100;
3127 		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3128 		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3129 		gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
3130 		break;
3131 	case CHIP_VERDE:
3132 	default:
3133 		rdev->config.si.max_shader_engines = 1;
3134 		rdev->config.si.max_tile_pipes = 4;
3135 		rdev->config.si.max_cu_per_sh = 5;
3136 		rdev->config.si.max_sh_per_se = 2;
3137 		rdev->config.si.max_backends_per_se = 4;
3138 		rdev->config.si.max_texture_channel_caches = 4;
3139 		rdev->config.si.max_gprs = 256;
3140 		rdev->config.si.max_gs_threads = 32;
3141 		rdev->config.si.max_hw_contexts = 8;
3142 
3143 		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3144 		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3145 		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3146 		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3147 		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3148 		break;
3149 	case CHIP_OLAND:
3150 		rdev->config.si.max_shader_engines = 1;
3151 		rdev->config.si.max_tile_pipes = 4;
3152 		rdev->config.si.max_cu_per_sh = 6;
3153 		rdev->config.si.max_sh_per_se = 1;
3154 		rdev->config.si.max_backends_per_se = 2;
3155 		rdev->config.si.max_texture_channel_caches = 4;
3156 		rdev->config.si.max_gprs = 256;
3157 		rdev->config.si.max_gs_threads = 16;
3158 		rdev->config.si.max_hw_contexts = 8;
3159 
3160 		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3161 		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3162 		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3163 		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3164 		gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
3165 		break;
3166 	case CHIP_HAINAN:
3167 		rdev->config.si.max_shader_engines = 1;
3168 		rdev->config.si.max_tile_pipes = 4;
3169 		rdev->config.si.max_cu_per_sh = 5;
3170 		rdev->config.si.max_sh_per_se = 1;
3171 		rdev->config.si.max_backends_per_se = 1;
3172 		rdev->config.si.max_texture_channel_caches = 2;
3173 		rdev->config.si.max_gprs = 256;
3174 		rdev->config.si.max_gs_threads = 16;
3175 		rdev->config.si.max_hw_contexts = 8;
3176 
3177 		rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
3178 		rdev->config.si.sc_prim_fifo_size_backend = 0x40;
3179 		rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
3180 		rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
3181 		gb_addr_config = HAINAN_GB_ADDR_CONFIG_GOLDEN;
3182 		break;
3183 	}
3184 
3185 	/* Initialize HDP */
3186 	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3187 		WREG32((0x2c14 + j), 0x00000000);
3188 		WREG32((0x2c18 + j), 0x00000000);
3189 		WREG32((0x2c1c + j), 0x00000000);
3190 		WREG32((0x2c20 + j), 0x00000000);
3191 		WREG32((0x2c24 + j), 0x00000000);
3192 	}
3193 
3194 	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3195 	WREG32(SRBM_INT_CNTL, 1);
3196 	WREG32(SRBM_INT_ACK, 1);
3197 
3198 	evergreen_fix_pci_max_read_req_size(rdev);
3199 
3200 	WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
3201 
3202 	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
3203 	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3204 
3205 	rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
3206 	rdev->config.si.mem_max_burst_length_bytes = 256;
3207 	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
3208 	rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
3209 	if (rdev->config.si.mem_row_size_in_kb > 4)
3210 		rdev->config.si.mem_row_size_in_kb = 4;
3211 	/* XXX use MC settings? */
3212 	rdev->config.si.shader_engine_tile_size = 32;
3213 	rdev->config.si.num_gpus = 1;
3214 	rdev->config.si.multi_gpu_tile_size = 64;
3215 
3216 	/* fix up row size */
3217 	gb_addr_config &= ~ROW_SIZE_MASK;
3218 	switch (rdev->config.si.mem_row_size_in_kb) {
3219 	case 1:
3220 	default:
3221 		gb_addr_config |= ROW_SIZE(0);
3222 		break;
3223 	case 2:
3224 		gb_addr_config |= ROW_SIZE(1);
3225 		break;
3226 	case 4:
3227 		gb_addr_config |= ROW_SIZE(2);
3228 		break;
3229 	}
3230 
3231 	/* setup tiling info dword.  gb_addr_config is not adequate since it does
3232 	 * not have bank info, so create a custom tiling dword.
3233 	 * bits 3:0   num_pipes
3234 	 * bits 7:4   num_banks
3235 	 * bits 11:8  group_size
3236 	 * bits 15:12 row_size
3237 	 */
3238 	rdev->config.si.tile_config = 0;
3239 	switch (rdev->config.si.num_tile_pipes) {
3240 	case 1:
3241 		rdev->config.si.tile_config |= (0 << 0);
3242 		break;
3243 	case 2:
3244 		rdev->config.si.tile_config |= (1 << 0);
3245 		break;
3246 	case 4:
3247 		rdev->config.si.tile_config |= (2 << 0);
3248 		break;
3249 	case 8:
3250 	default:
3251 		/* XXX what about 12? */
3252 		rdev->config.si.tile_config |= (3 << 0);
3253 		break;
3254 	}
3255 	switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3256 	case 0: /* four banks */
3257 		rdev->config.si.tile_config |= 0 << 4;
3258 		break;
3259 	case 1: /* eight banks */
3260 		rdev->config.si.tile_config |= 1 << 4;
3261 		break;
3262 	case 2: /* sixteen banks */
3263 	default:
3264 		rdev->config.si.tile_config |= 2 << 4;
3265 		break;
3266 	}
3267 	rdev->config.si.tile_config |=
3268 		((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
3269 	rdev->config.si.tile_config |=
3270 		((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
3271 
3272 	WREG32(GB_ADDR_CONFIG, gb_addr_config);
3273 	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3274 	WREG32(DMIF_ADDR_CALC, gb_addr_config);
3275 	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3276 	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
3277 	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
3278 	if (rdev->has_uvd) {
3279 		WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3280 		WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3281 		WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3282 	}
3283 
3284 	si_tiling_mode_table_init(rdev);
3285 
3286 	si_setup_rb(rdev, rdev->config.si.max_shader_engines,
3287 		    rdev->config.si.max_sh_per_se,
3288 		    rdev->config.si.max_backends_per_se);
3289 
3290 	si_setup_spi(rdev, rdev->config.si.max_shader_engines,
3291 		     rdev->config.si.max_sh_per_se,
3292 		     rdev->config.si.max_cu_per_sh);
3293 
3294 	rdev->config.si.active_cus = 0;
3295 	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
3296 		for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
3297 			rdev->config.si.active_cus +=
3298 				hweight32(si_get_cu_active_bitmap(rdev, i, j));
3299 		}
3300 	}
3301 
3302 	/* set HW defaults for 3D engine */
3303 	WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3304 				     ROQ_IB2_START(0x2b)));
3305 	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
3306 
3307 	sx_debug_1 = RREG32(SX_DEBUG_1);
3308 	WREG32(SX_DEBUG_1, sx_debug_1);
3309 
3310 	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3311 
3312 	WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
3313 				 SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
3314 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
3315 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
3316 
3317 	WREG32(VGT_NUM_INSTANCES, 1);
3318 
3319 	WREG32(CP_PERFMON_CNTL, 0);
3320 
3321 	WREG32(SQ_CONFIG, 0);
3322 
3323 	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3324 					  FORCE_EOV_MAX_REZ_CNT(255)));
3325 
3326 	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
3327 	       AUTO_INVLD_EN(ES_AND_GS_AUTO));
3328 
3329 	WREG32(VGT_GS_VERTEX_REUSE, 16);
3330 	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3331 
3332 	WREG32(CB_PERFCOUNTER0_SELECT0, 0);
3333 	WREG32(CB_PERFCOUNTER0_SELECT1, 0);
3334 	WREG32(CB_PERFCOUNTER1_SELECT0, 0);
3335 	WREG32(CB_PERFCOUNTER1_SELECT1, 0);
3336 	WREG32(CB_PERFCOUNTER2_SELECT0, 0);
3337 	WREG32(CB_PERFCOUNTER2_SELECT1, 0);
3338 	WREG32(CB_PERFCOUNTER3_SELECT0, 0);
3339 	WREG32(CB_PERFCOUNTER3_SELECT1, 0);
3340 
3341 	tmp = RREG32(HDP_MISC_CNTL);
3342 	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3343 	WREG32(HDP_MISC_CNTL, tmp);
3344 
3345 	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3346 	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3347 
3348 	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3349 
3350 	udelay(50);
3351 }
3352 
3353 /*
3354  * GPU scratch registers helpers function.
3355  */
3356 static void si_scratch_init(struct radeon_device *rdev)
3357 {
3358 	int i;
3359 
3360 	rdev->scratch.num_reg = 7;
3361 	rdev->scratch.reg_base = SCRATCH_REG0;
3362 	for (i = 0; i < rdev->scratch.num_reg; i++) {
3363 		rdev->scratch.free[i] = true;
3364 		rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
3365 	}
3366 }
3367 
3368 void si_fence_ring_emit(struct radeon_device *rdev,
3369 			struct radeon_fence *fence)
3370 {
3371 	struct radeon_ring *ring = &rdev->ring[fence->ring];
3372 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
3373 
3374 	/* flush read cache over gart */
3375 	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3376 	radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3377 	radeon_ring_write(ring, 0);
3378 	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3379 	radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3380 			  PACKET3_TC_ACTION_ENA |
3381 			  PACKET3_SH_KCACHE_ACTION_ENA |
3382 			  PACKET3_SH_ICACHE_ACTION_ENA);
3383 	radeon_ring_write(ring, 0xFFFFFFFF);
3384 	radeon_ring_write(ring, 0);
3385 	radeon_ring_write(ring, 10); /* poll interval */
3386 	/* EVENT_WRITE_EOP - flush caches, send int */
3387 	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
3388 	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
3389 	radeon_ring_write(ring, lower_32_bits(addr));
3390 	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
3391 	radeon_ring_write(ring, fence->seq);
3392 	radeon_ring_write(ring, 0);
3393 }
3394 
3395 /*
3396  * IB stuff
3397  */
3398 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
3399 {
3400 	struct radeon_ring *ring = &rdev->ring[ib->ring];
3401 	unsigned vm_id = ib->vm ? ib->vm->ids[ib->ring].id : 0;
3402 	u32 header;
3403 
3404 	if (ib->is_const_ib) {
3405 		/* set switch buffer packet before const IB */
3406 		radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
3407 		radeon_ring_write(ring, 0);
3408 
3409 		header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
3410 	} else {
3411 		u32 next_rptr;
3412 		if (ring->rptr_save_reg) {
3413 			next_rptr = ring->wptr + 3 + 4 + 8;
3414 			radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3415 			radeon_ring_write(ring, ((ring->rptr_save_reg -
3416 						  PACKET3_SET_CONFIG_REG_START) >> 2));
3417 			radeon_ring_write(ring, next_rptr);
3418 		} else if (rdev->wb.enabled) {
3419 			next_rptr = ring->wptr + 5 + 4 + 8;
3420 			radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
3421 			radeon_ring_write(ring, (1 << 8));
3422 			radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
3423 			radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
3424 			radeon_ring_write(ring, next_rptr);
3425 		}
3426 
3427 		header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
3428 	}
3429 
3430 	radeon_ring_write(ring, header);
3431 	radeon_ring_write(ring,
3432 #ifdef __BIG_ENDIAN
3433 			  (2 << 0) |
3434 #endif
3435 			  (ib->gpu_addr & 0xFFFFFFFC));
3436 	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
3437 	radeon_ring_write(ring, ib->length_dw | (vm_id << 24));
3438 
3439 	if (!ib->is_const_ib) {
3440 		/* flush read cache over gart for this vmid */
3441 		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
3442 		radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
3443 		radeon_ring_write(ring, vm_id);
3444 		radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
3445 		radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
3446 				  PACKET3_TC_ACTION_ENA |
3447 				  PACKET3_SH_KCACHE_ACTION_ENA |
3448 				  PACKET3_SH_ICACHE_ACTION_ENA);
3449 		radeon_ring_write(ring, 0xFFFFFFFF);
3450 		radeon_ring_write(ring, 0);
3451 		radeon_ring_write(ring, 10); /* poll interval */
3452 	}
3453 }
3454 
3455 /*
3456  * CP.
3457  */
3458 static void si_cp_enable(struct radeon_device *rdev, bool enable)
3459 {
3460 	if (enable)
3461 		WREG32(CP_ME_CNTL, 0);
3462 	else {
3463 		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3464 			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
3465 		WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
3466 		WREG32(SCRATCH_UMSK, 0);
3467 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3468 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3469 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3470 	}
3471 	udelay(50);
3472 }
3473 
3474 static int si_cp_load_microcode(struct radeon_device *rdev)
3475 {
3476 	int i;
3477 
3478 	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw)
3479 		return -EINVAL;
3480 
3481 	si_cp_enable(rdev, false);
3482 
3483 	if (rdev->new_fw) {
3484 		const struct gfx_firmware_header_v1_0 *pfp_hdr =
3485 			(const struct gfx_firmware_header_v1_0 *)rdev->pfp_fw->data;
3486 		const struct gfx_firmware_header_v1_0 *ce_hdr =
3487 			(const struct gfx_firmware_header_v1_0 *)rdev->ce_fw->data;
3488 		const struct gfx_firmware_header_v1_0 *me_hdr =
3489 			(const struct gfx_firmware_header_v1_0 *)rdev->me_fw->data;
3490 		const __le32 *fw_data;
3491 		u32 fw_size;
3492 
3493 		radeon_ucode_print_gfx_hdr(&pfp_hdr->header);
3494 		radeon_ucode_print_gfx_hdr(&ce_hdr->header);
3495 		radeon_ucode_print_gfx_hdr(&me_hdr->header);
3496 
3497 		/* PFP */
3498 		fw_data = (const __le32 *)
3499 			(rdev->pfp_fw->data + le32_to_cpu(pfp_hdr->header.ucode_array_offset_bytes));
3500 		fw_size = le32_to_cpu(pfp_hdr->header.ucode_size_bytes) / 4;
3501 		WREG32(CP_PFP_UCODE_ADDR, 0);
3502 		for (i = 0; i < fw_size; i++)
3503 			WREG32(CP_PFP_UCODE_DATA, le32_to_cpup(fw_data++));
3504 		WREG32(CP_PFP_UCODE_ADDR, 0);
3505 
3506 		/* CE */
3507 		fw_data = (const __le32 *)
3508 			(rdev->ce_fw->data + le32_to_cpu(ce_hdr->header.ucode_array_offset_bytes));
3509 		fw_size = le32_to_cpu(ce_hdr->header.ucode_size_bytes) / 4;
3510 		WREG32(CP_CE_UCODE_ADDR, 0);
3511 		for (i = 0; i < fw_size; i++)
3512 			WREG32(CP_CE_UCODE_DATA, le32_to_cpup(fw_data++));
3513 		WREG32(CP_CE_UCODE_ADDR, 0);
3514 
3515 		/* ME */
3516 		fw_data = (const __be32 *)
3517 			(rdev->me_fw->data + le32_to_cpu(me_hdr->header.ucode_array_offset_bytes));
3518 		fw_size = le32_to_cpu(me_hdr->header.ucode_size_bytes) / 4;
3519 		WREG32(CP_ME_RAM_WADDR, 0);
3520 		for (i = 0; i < fw_size; i++)
3521 			WREG32(CP_ME_RAM_DATA, le32_to_cpup(fw_data++));
3522 		WREG32(CP_ME_RAM_WADDR, 0);
3523 	} else {
3524 		const __be32 *fw_data;
3525 
3526 		/* PFP */
3527 		fw_data = (const __be32 *)rdev->pfp_fw->data;
3528 		WREG32(CP_PFP_UCODE_ADDR, 0);
3529 		for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
3530 			WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
3531 		WREG32(CP_PFP_UCODE_ADDR, 0);
3532 
3533 		/* CE */
3534 		fw_data = (const __be32 *)rdev->ce_fw->data;
3535 		WREG32(CP_CE_UCODE_ADDR, 0);
3536 		for (i = 0; i < SI_CE_UCODE_SIZE; i++)
3537 			WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
3538 		WREG32(CP_CE_UCODE_ADDR, 0);
3539 
3540 		/* ME */
3541 		fw_data = (const __be32 *)rdev->me_fw->data;
3542 		WREG32(CP_ME_RAM_WADDR, 0);
3543 		for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
3544 			WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
3545 		WREG32(CP_ME_RAM_WADDR, 0);
3546 	}
3547 
3548 	WREG32(CP_PFP_UCODE_ADDR, 0);
3549 	WREG32(CP_CE_UCODE_ADDR, 0);
3550 	WREG32(CP_ME_RAM_WADDR, 0);
3551 	WREG32(CP_ME_RAM_RADDR, 0);
3552 	return 0;
3553 }
3554 
3555 static int si_cp_start(struct radeon_device *rdev)
3556 {
3557 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3558 	int r, i;
3559 
3560 	r = radeon_ring_lock(rdev, ring, 7 + 4);
3561 	if (r) {
3562 		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3563 		return r;
3564 	}
3565 	/* init the CP */
3566 	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3567 	radeon_ring_write(ring, 0x1);
3568 	radeon_ring_write(ring, 0x0);
3569 	radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
3570 	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3571 	radeon_ring_write(ring, 0);
3572 	radeon_ring_write(ring, 0);
3573 
3574 	/* init the CE partitions */
3575 	radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
3576 	radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
3577 	radeon_ring_write(ring, 0xc000);
3578 	radeon_ring_write(ring, 0xe000);
3579 	radeon_ring_unlock_commit(rdev, ring, false);
3580 
3581 	si_cp_enable(rdev, true);
3582 
3583 	r = radeon_ring_lock(rdev, ring, si_default_size + 10);
3584 	if (r) {
3585 		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3586 		return r;
3587 	}
3588 
3589 	/* setup clear context state */
3590 	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3591 	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3592 
3593 	for (i = 0; i < si_default_size; i++)
3594 		radeon_ring_write(ring, si_default_state[i]);
3595 
3596 	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3597 	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3598 
3599 	/* set clear context state */
3600 	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3601 	radeon_ring_write(ring, 0);
3602 
3603 	radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
3604 	radeon_ring_write(ring, 0x00000316);
3605 	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3606 	radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
3607 
3608 	radeon_ring_unlock_commit(rdev, ring, false);
3609 
3610 	for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
3611 		ring = &rdev->ring[i];
3612 		r = radeon_ring_lock(rdev, ring, 2);
3613 
3614 		/* clear the compute context state */
3615 		radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
3616 		radeon_ring_write(ring, 0);
3617 
3618 		radeon_ring_unlock_commit(rdev, ring, false);
3619 	}
3620 
3621 	return 0;
3622 }
3623 
3624 static void si_cp_fini(struct radeon_device *rdev)
3625 {
3626 	struct radeon_ring *ring;
3627 	si_cp_enable(rdev, false);
3628 
3629 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3630 	radeon_ring_fini(rdev, ring);
3631 	radeon_scratch_free(rdev, ring->rptr_save_reg);
3632 
3633 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3634 	radeon_ring_fini(rdev, ring);
3635 	radeon_scratch_free(rdev, ring->rptr_save_reg);
3636 
3637 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3638 	radeon_ring_fini(rdev, ring);
3639 	radeon_scratch_free(rdev, ring->rptr_save_reg);
3640 }
3641 
3642 static int si_cp_resume(struct radeon_device *rdev)
3643 {
3644 	struct radeon_ring *ring;
3645 	u32 tmp;
3646 	u32 rb_bufsz;
3647 	int r;
3648 
3649 	si_enable_gui_idle_interrupt(rdev, false);
3650 
3651 	WREG32(CP_SEM_WAIT_TIMER, 0x0);
3652 	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3653 
3654 	/* Set the write pointer delay */
3655 	WREG32(CP_RB_WPTR_DELAY, 0);
3656 
3657 	WREG32(CP_DEBUG, 0);
3658 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3659 
3660 	/* ring 0 - compute and gfx */
3661 	/* Set ring buffer size */
3662 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3663 	rb_bufsz = order_base_2(ring->ring_size / 8);
3664 	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3665 #ifdef __BIG_ENDIAN
3666 	tmp |= BUF_SWAP_32BIT;
3667 #endif
3668 	WREG32(CP_RB0_CNTL, tmp);
3669 
3670 	/* Initialize the ring buffer's read and write pointers */
3671 	WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
3672 	ring->wptr = 0;
3673 	WREG32(CP_RB0_WPTR, ring->wptr);
3674 
3675 	/* set the wb address whether it's enabled or not */
3676 	WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
3677 	WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3678 
3679 	if (rdev->wb.enabled)
3680 		WREG32(SCRATCH_UMSK, 0xff);
3681 	else {
3682 		tmp |= RB_NO_UPDATE;
3683 		WREG32(SCRATCH_UMSK, 0);
3684 	}
3685 
3686 	mdelay(1);
3687 	WREG32(CP_RB0_CNTL, tmp);
3688 
3689 	WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
3690 
3691 	/* ring1  - compute only */
3692 	/* Set ring buffer size */
3693 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
3694 	rb_bufsz = order_base_2(ring->ring_size / 8);
3695 	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3696 #ifdef __BIG_ENDIAN
3697 	tmp |= BUF_SWAP_32BIT;
3698 #endif
3699 	WREG32(CP_RB1_CNTL, tmp);
3700 
3701 	/* Initialize the ring buffer's read and write pointers */
3702 	WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
3703 	ring->wptr = 0;
3704 	WREG32(CP_RB1_WPTR, ring->wptr);
3705 
3706 	/* set the wb address whether it's enabled or not */
3707 	WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
3708 	WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
3709 
3710 	mdelay(1);
3711 	WREG32(CP_RB1_CNTL, tmp);
3712 
3713 	WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
3714 
3715 	/* ring2 - compute only */
3716 	/* Set ring buffer size */
3717 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
3718 	rb_bufsz = order_base_2(ring->ring_size / 8);
3719 	tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3720 #ifdef __BIG_ENDIAN
3721 	tmp |= BUF_SWAP_32BIT;
3722 #endif
3723 	WREG32(CP_RB2_CNTL, tmp);
3724 
3725 	/* Initialize the ring buffer's read and write pointers */
3726 	WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
3727 	ring->wptr = 0;
3728 	WREG32(CP_RB2_WPTR, ring->wptr);
3729 
3730 	/* set the wb address whether it's enabled or not */
3731 	WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
3732 	WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
3733 
3734 	mdelay(1);
3735 	WREG32(CP_RB2_CNTL, tmp);
3736 
3737 	WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
3738 
3739 	/* start the rings */
3740 	si_cp_start(rdev);
3741 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
3742 	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
3743 	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
3744 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
3745 	if (r) {
3746 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
3747 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3748 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3749 		return r;
3750 	}
3751 	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
3752 	if (r) {
3753 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3754 	}
3755 	r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3756 	if (r) {
3757 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3758 	}
3759 
3760 	si_enable_gui_idle_interrupt(rdev, true);
3761 
3762 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
3763 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
3764 
3765 	return 0;
3766 }
3767 
3768 u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3769 {
3770 	u32 reset_mask = 0;
3771 	u32 tmp;
3772 
3773 	/* GRBM_STATUS */
3774 	tmp = RREG32(GRBM_STATUS);
3775 	if (tmp & (PA_BUSY | SC_BUSY |
3776 		   BCI_BUSY | SX_BUSY |
3777 		   TA_BUSY | VGT_BUSY |
3778 		   DB_BUSY | CB_BUSY |
3779 		   GDS_BUSY | SPI_BUSY |
3780 		   IA_BUSY | IA_BUSY_NO_DMA))
3781 		reset_mask |= RADEON_RESET_GFX;
3782 
3783 	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3784 		   CP_BUSY | CP_COHERENCY_BUSY))
3785 		reset_mask |= RADEON_RESET_CP;
3786 
3787 	if (tmp & GRBM_EE_BUSY)
3788 		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3789 
3790 	/* GRBM_STATUS2 */
3791 	tmp = RREG32(GRBM_STATUS2);
3792 	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3793 		reset_mask |= RADEON_RESET_RLC;
3794 
3795 	/* DMA_STATUS_REG 0 */
3796 	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3797 	if (!(tmp & DMA_IDLE))
3798 		reset_mask |= RADEON_RESET_DMA;
3799 
3800 	/* DMA_STATUS_REG 1 */
3801 	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3802 	if (!(tmp & DMA_IDLE))
3803 		reset_mask |= RADEON_RESET_DMA1;
3804 
3805 	/* SRBM_STATUS2 */
3806 	tmp = RREG32(SRBM_STATUS2);
3807 	if (tmp & DMA_BUSY)
3808 		reset_mask |= RADEON_RESET_DMA;
3809 
3810 	if (tmp & DMA1_BUSY)
3811 		reset_mask |= RADEON_RESET_DMA1;
3812 
3813 	/* SRBM_STATUS */
3814 	tmp = RREG32(SRBM_STATUS);
3815 
3816 	if (tmp & IH_BUSY)
3817 		reset_mask |= RADEON_RESET_IH;
3818 
3819 	if (tmp & SEM_BUSY)
3820 		reset_mask |= RADEON_RESET_SEM;
3821 
3822 	if (tmp & GRBM_RQ_PENDING)
3823 		reset_mask |= RADEON_RESET_GRBM;
3824 
3825 	if (tmp & VMC_BUSY)
3826 		reset_mask |= RADEON_RESET_VMC;
3827 
3828 	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3829 		   MCC_BUSY | MCD_BUSY))
3830 		reset_mask |= RADEON_RESET_MC;
3831 
3832 	if (evergreen_is_display_hung(rdev))
3833 		reset_mask |= RADEON_RESET_DISPLAY;
3834 
3835 	/* VM_L2_STATUS */
3836 	tmp = RREG32(VM_L2_STATUS);
3837 	if (tmp & L2_BUSY)
3838 		reset_mask |= RADEON_RESET_VMC;
3839 
3840 	/* Skip MC reset as it's mostly likely not hung, just busy */
3841 	if (reset_mask & RADEON_RESET_MC) {
3842 		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3843 		reset_mask &= ~RADEON_RESET_MC;
3844 	}
3845 
3846 	return reset_mask;
3847 }
3848 
3849 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3850 {
3851 	struct evergreen_mc_save save;
3852 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3853 	u32 tmp;
3854 
3855 	if (reset_mask == 0)
3856 		return;
3857 
3858 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3859 
3860 	evergreen_print_gpu_status_regs(rdev);
3861 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3862 		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3863 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3864 		 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3865 
3866 	/* disable PG/CG */
3867 	si_fini_pg(rdev);
3868 	si_fini_cg(rdev);
3869 
3870 	/* stop the rlc */
3871 	si_rlc_stop(rdev);
3872 
3873 	/* Disable CP parsing/prefetching */
3874 	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3875 
3876 	if (reset_mask & RADEON_RESET_DMA) {
3877 		/* dma0 */
3878 		tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3879 		tmp &= ~DMA_RB_ENABLE;
3880 		WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3881 	}
3882 	if (reset_mask & RADEON_RESET_DMA1) {
3883 		/* dma1 */
3884 		tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3885 		tmp &= ~DMA_RB_ENABLE;
3886 		WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3887 	}
3888 
3889 	udelay(50);
3890 
3891 	evergreen_mc_stop(rdev, &save);
3892 	if (evergreen_mc_wait_for_idle(rdev)) {
3893 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3894 	}
3895 
3896 	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3897 		grbm_soft_reset = SOFT_RESET_CB |
3898 			SOFT_RESET_DB |
3899 			SOFT_RESET_GDS |
3900 			SOFT_RESET_PA |
3901 			SOFT_RESET_SC |
3902 			SOFT_RESET_BCI |
3903 			SOFT_RESET_SPI |
3904 			SOFT_RESET_SX |
3905 			SOFT_RESET_TC |
3906 			SOFT_RESET_TA |
3907 			SOFT_RESET_VGT |
3908 			SOFT_RESET_IA;
3909 	}
3910 
3911 	if (reset_mask & RADEON_RESET_CP) {
3912 		grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3913 
3914 		srbm_soft_reset |= SOFT_RESET_GRBM;
3915 	}
3916 
3917 	if (reset_mask & RADEON_RESET_DMA)
3918 		srbm_soft_reset |= SOFT_RESET_DMA;
3919 
3920 	if (reset_mask & RADEON_RESET_DMA1)
3921 		srbm_soft_reset |= SOFT_RESET_DMA1;
3922 
3923 	if (reset_mask & RADEON_RESET_DISPLAY)
3924 		srbm_soft_reset |= SOFT_RESET_DC;
3925 
3926 	if (reset_mask & RADEON_RESET_RLC)
3927 		grbm_soft_reset |= SOFT_RESET_RLC;
3928 
3929 	if (reset_mask & RADEON_RESET_SEM)
3930 		srbm_soft_reset |= SOFT_RESET_SEM;
3931 
3932 	if (reset_mask & RADEON_RESET_IH)
3933 		srbm_soft_reset |= SOFT_RESET_IH;
3934 
3935 	if (reset_mask & RADEON_RESET_GRBM)
3936 		srbm_soft_reset |= SOFT_RESET_GRBM;
3937 
3938 	if (reset_mask & RADEON_RESET_VMC)
3939 		srbm_soft_reset |= SOFT_RESET_VMC;
3940 
3941 	if (reset_mask & RADEON_RESET_MC)
3942 		srbm_soft_reset |= SOFT_RESET_MC;
3943 
3944 	if (grbm_soft_reset) {
3945 		tmp = RREG32(GRBM_SOFT_RESET);
3946 		tmp |= grbm_soft_reset;
3947 		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3948 		WREG32(GRBM_SOFT_RESET, tmp);
3949 		tmp = RREG32(GRBM_SOFT_RESET);
3950 
3951 		udelay(50);
3952 
3953 		tmp &= ~grbm_soft_reset;
3954 		WREG32(GRBM_SOFT_RESET, tmp);
3955 		tmp = RREG32(GRBM_SOFT_RESET);
3956 	}
3957 
3958 	if (srbm_soft_reset) {
3959 		tmp = RREG32(SRBM_SOFT_RESET);
3960 		tmp |= srbm_soft_reset;
3961 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3962 		WREG32(SRBM_SOFT_RESET, tmp);
3963 		tmp = RREG32(SRBM_SOFT_RESET);
3964 
3965 		udelay(50);
3966 
3967 		tmp &= ~srbm_soft_reset;
3968 		WREG32(SRBM_SOFT_RESET, tmp);
3969 		tmp = RREG32(SRBM_SOFT_RESET);
3970 	}
3971 
3972 	/* Wait a little for things to settle down */
3973 	udelay(50);
3974 
3975 	evergreen_mc_resume(rdev, &save);
3976 	udelay(50);
3977 
3978 	evergreen_print_gpu_status_regs(rdev);
3979 }
3980 
3981 static void si_set_clk_bypass_mode(struct radeon_device *rdev)
3982 {
3983 	u32 tmp, i;
3984 
3985 	tmp = RREG32(CG_SPLL_FUNC_CNTL);
3986 	tmp |= SPLL_BYPASS_EN;
3987 	WREG32(CG_SPLL_FUNC_CNTL, tmp);
3988 
3989 	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
3990 	tmp |= SPLL_CTLREQ_CHG;
3991 	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
3992 
3993 	for (i = 0; i < rdev->usec_timeout; i++) {
3994 		if (RREG32(SPLL_STATUS) & SPLL_CHG_STATUS)
3995 			break;
3996 		udelay(1);
3997 	}
3998 
3999 	tmp = RREG32(CG_SPLL_FUNC_CNTL_2);
4000 	tmp &= ~(SPLL_CTLREQ_CHG | SCLK_MUX_UPDATE);
4001 	WREG32(CG_SPLL_FUNC_CNTL_2, tmp);
4002 
4003 	tmp = RREG32(MPLL_CNTL_MODE);
4004 	tmp &= ~MPLL_MCLK_SEL;
4005 	WREG32(MPLL_CNTL_MODE, tmp);
4006 }
4007 
4008 static void si_spll_powerdown(struct radeon_device *rdev)
4009 {
4010 	u32 tmp;
4011 
4012 	tmp = RREG32(SPLL_CNTL_MODE);
4013 	tmp |= SPLL_SW_DIR_CONTROL;
4014 	WREG32(SPLL_CNTL_MODE, tmp);
4015 
4016 	tmp = RREG32(CG_SPLL_FUNC_CNTL);
4017 	tmp |= SPLL_RESET;
4018 	WREG32(CG_SPLL_FUNC_CNTL, tmp);
4019 
4020 	tmp = RREG32(CG_SPLL_FUNC_CNTL);
4021 	tmp |= SPLL_SLEEP;
4022 	WREG32(CG_SPLL_FUNC_CNTL, tmp);
4023 
4024 	tmp = RREG32(SPLL_CNTL_MODE);
4025 	tmp &= ~SPLL_SW_DIR_CONTROL;
4026 	WREG32(SPLL_CNTL_MODE, tmp);
4027 }
4028 
4029 static void si_gpu_pci_config_reset(struct radeon_device *rdev)
4030 {
4031 	struct evergreen_mc_save save;
4032 	u32 tmp, i;
4033 
4034 	dev_info(rdev->dev, "GPU pci config reset\n");
4035 
4036 	/* disable dpm? */
4037 
4038 	/* disable cg/pg */
4039 	si_fini_pg(rdev);
4040 	si_fini_cg(rdev);
4041 
4042 	/* Disable CP parsing/prefetching */
4043 	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
4044 	/* dma0 */
4045 	tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
4046 	tmp &= ~DMA_RB_ENABLE;
4047 	WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
4048 	/* dma1 */
4049 	tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
4050 	tmp &= ~DMA_RB_ENABLE;
4051 	WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
4052 	/* XXX other engines? */
4053 
4054 	/* halt the rlc, disable cp internal ints */
4055 	si_rlc_stop(rdev);
4056 
4057 	udelay(50);
4058 
4059 	/* disable mem access */
4060 	evergreen_mc_stop(rdev, &save);
4061 	if (evergreen_mc_wait_for_idle(rdev)) {
4062 		dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4063 	}
4064 
4065 	/* set mclk/sclk to bypass */
4066 	si_set_clk_bypass_mode(rdev);
4067 	/* powerdown spll */
4068 	si_spll_powerdown(rdev);
4069 	/* disable BM */
4070 	pci_clear_master(rdev->pdev);
4071 	/* reset */
4072 	radeon_pci_config_reset(rdev);
4073 	/* wait for asic to come out of reset */
4074 	for (i = 0; i < rdev->usec_timeout; i++) {
4075 		if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4076 			break;
4077 		udelay(1);
4078 	}
4079 }
4080 
4081 int si_asic_reset(struct radeon_device *rdev)
4082 {
4083 	u32 reset_mask;
4084 
4085 	reset_mask = si_gpu_check_soft_reset(rdev);
4086 
4087 	if (reset_mask)
4088 		r600_set_bios_scratch_engine_hung(rdev, true);
4089 
4090 	/* try soft reset */
4091 	si_gpu_soft_reset(rdev, reset_mask);
4092 
4093 	reset_mask = si_gpu_check_soft_reset(rdev);
4094 
4095 	/* try pci config reset */
4096 	if (reset_mask && radeon_hard_reset)
4097 		si_gpu_pci_config_reset(rdev);
4098 
4099 	reset_mask = si_gpu_check_soft_reset(rdev);
4100 
4101 	if (!reset_mask)
4102 		r600_set_bios_scratch_engine_hung(rdev, false);
4103 
4104 	return 0;
4105 }
4106 
4107 /**
4108  * si_gfx_is_lockup - Check if the GFX engine is locked up
4109  *
4110  * @rdev: radeon_device pointer
4111  * @ring: radeon_ring structure holding ring information
4112  *
4113  * Check if the GFX engine is locked up.
4114  * Returns true if the engine appears to be locked up, false if not.
4115  */
4116 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4117 {
4118 	u32 reset_mask = si_gpu_check_soft_reset(rdev);
4119 
4120 	if (!(reset_mask & (RADEON_RESET_GFX |
4121 			    RADEON_RESET_COMPUTE |
4122 			    RADEON_RESET_CP))) {
4123 		radeon_ring_lockup_update(rdev, ring);
4124 		return false;
4125 	}
4126 	return radeon_ring_test_lockup(rdev, ring);
4127 }
4128 
4129 /* MC */
4130 static void si_mc_program(struct radeon_device *rdev)
4131 {
4132 	struct evergreen_mc_save save;
4133 	u32 tmp;
4134 	int i, j;
4135 
4136 	/* Initialize HDP */
4137 	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
4138 		WREG32((0x2c14 + j), 0x00000000);
4139 		WREG32((0x2c18 + j), 0x00000000);
4140 		WREG32((0x2c1c + j), 0x00000000);
4141 		WREG32((0x2c20 + j), 0x00000000);
4142 		WREG32((0x2c24 + j), 0x00000000);
4143 	}
4144 	WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
4145 
4146 	evergreen_mc_stop(rdev, &save);
4147 	if (radeon_mc_wait_for_idle(rdev)) {
4148 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4149 	}
4150 	if (!ASIC_IS_NODCE(rdev))
4151 		/* Lockout access through VGA aperture*/
4152 		WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
4153 	/* Update configuration */
4154 	WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
4155 	       rdev->mc.vram_start >> 12);
4156 	WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
4157 	       rdev->mc.vram_end >> 12);
4158 	WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
4159 	       rdev->vram_scratch.gpu_addr >> 12);
4160 	tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
4161 	tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
4162 	WREG32(MC_VM_FB_LOCATION, tmp);
4163 	/* XXX double check these! */
4164 	WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
4165 	WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
4166 	WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
4167 	WREG32(MC_VM_AGP_BASE, 0);
4168 	WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
4169 	WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
4170 	if (radeon_mc_wait_for_idle(rdev)) {
4171 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
4172 	}
4173 	evergreen_mc_resume(rdev, &save);
4174 	if (!ASIC_IS_NODCE(rdev)) {
4175 		/* we need to own VRAM, so turn off the VGA renderer here
4176 		 * to stop it overwriting our objects */
4177 		rv515_vga_render_disable(rdev);
4178 	}
4179 }
4180 
4181 void si_vram_gtt_location(struct radeon_device *rdev,
4182 			  struct radeon_mc *mc)
4183 {
4184 	if (mc->mc_vram_size > 0xFFC0000000ULL) {
4185 		/* leave room for at least 1024M GTT */
4186 		dev_warn(rdev->dev, "limiting VRAM\n");
4187 		mc->real_vram_size = 0xFFC0000000ULL;
4188 		mc->mc_vram_size = 0xFFC0000000ULL;
4189 	}
4190 	radeon_vram_location(rdev, &rdev->mc, 0);
4191 	rdev->mc.gtt_base_align = 0;
4192 	radeon_gtt_location(rdev, mc);
4193 }
4194 
4195 static int si_mc_init(struct radeon_device *rdev)
4196 {
4197 	u32 tmp;
4198 	int chansize, numchan;
4199 
4200 	/* Get VRAM informations */
4201 	rdev->mc.vram_is_ddr = true;
4202 	tmp = RREG32(MC_ARB_RAMCFG);
4203 	if (tmp & CHANSIZE_OVERRIDE) {
4204 		chansize = 16;
4205 	} else if (tmp & CHANSIZE_MASK) {
4206 		chansize = 64;
4207 	} else {
4208 		chansize = 32;
4209 	}
4210 	tmp = RREG32(MC_SHARED_CHMAP);
4211 	switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
4212 	case 0:
4213 	default:
4214 		numchan = 1;
4215 		break;
4216 	case 1:
4217 		numchan = 2;
4218 		break;
4219 	case 2:
4220 		numchan = 4;
4221 		break;
4222 	case 3:
4223 		numchan = 8;
4224 		break;
4225 	case 4:
4226 		numchan = 3;
4227 		break;
4228 	case 5:
4229 		numchan = 6;
4230 		break;
4231 	case 6:
4232 		numchan = 10;
4233 		break;
4234 	case 7:
4235 		numchan = 12;
4236 		break;
4237 	case 8:
4238 		numchan = 16;
4239 		break;
4240 	}
4241 	rdev->mc.vram_width = numchan * chansize;
4242 	/* Could aper size report 0 ? */
4243 	rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
4244 	rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
4245 	/* size in MB on si */
4246 	tmp = RREG32(CONFIG_MEMSIZE);
4247 	/* some boards may have garbage in the upper 16 bits */
4248 	if (tmp & 0xffff0000) {
4249 		DRM_INFO("Probable bad vram size: 0x%08x\n", tmp);
4250 		if (tmp & 0xffff)
4251 			tmp &= 0xffff;
4252 	}
4253 	rdev->mc.mc_vram_size = tmp * 1024ULL * 1024ULL;
4254 	rdev->mc.real_vram_size = rdev->mc.mc_vram_size;
4255 	rdev->mc.visible_vram_size = rdev->mc.aper_size;
4256 	si_vram_gtt_location(rdev, &rdev->mc);
4257 	radeon_update_bandwidth_info(rdev);
4258 
4259 	return 0;
4260 }
4261 
4262 /*
4263  * GART
4264  */
4265 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
4266 {
4267 	/* flush hdp cache */
4268 	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
4269 
4270 	/* bits 0-15 are the VM contexts0-15 */
4271 	WREG32(VM_INVALIDATE_REQUEST, 1);
4272 }
4273 
4274 static int si_pcie_gart_enable(struct radeon_device *rdev)
4275 {
4276 	int r, i;
4277 
4278 	if (rdev->gart.robj == NULL) {
4279 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
4280 		return -EINVAL;
4281 	}
4282 	r = radeon_gart_table_vram_pin(rdev);
4283 	if (r)
4284 		return r;
4285 	/* Setup TLB control */
4286 	WREG32(MC_VM_MX_L1_TLB_CNTL,
4287 	       (0xA << 7) |
4288 	       ENABLE_L1_TLB |
4289 	       ENABLE_L1_FRAGMENT_PROCESSING |
4290 	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4291 	       ENABLE_ADVANCED_DRIVER_MODEL |
4292 	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4293 	/* Setup L2 cache */
4294 	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
4295 	       ENABLE_L2_FRAGMENT_PROCESSING |
4296 	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4297 	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4298 	       EFFECTIVE_L2_QUEUE_SIZE(7) |
4299 	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
4300 	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
4301 	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4302 	       BANK_SELECT(4) |
4303 	       L2_CACHE_BIGK_FRAGMENT_SIZE(4));
4304 	/* setup context0 */
4305 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
4306 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
4307 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
4308 	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
4309 			(u32)(rdev->dummy_page.addr >> 12));
4310 	WREG32(VM_CONTEXT0_CNTL2, 0);
4311 	WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
4312 				  RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
4313 
4314 	WREG32(0x15D4, 0);
4315 	WREG32(0x15D8, 0);
4316 	WREG32(0x15DC, 0);
4317 
4318 	/* empty context1-15 */
4319 	/* set vm size, must be a multiple of 4 */
4320 	WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
4321 	WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn - 1);
4322 	/* Assign the pt base to something valid for now; the pts used for
4323 	 * the VMs are determined by the application and setup and assigned
4324 	 * on the fly in the vm part of radeon_gart.c
4325 	 */
4326 	for (i = 1; i < 16; i++) {
4327 		if (i < 8)
4328 			WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
4329 			       rdev->vm_manager.saved_table_addr[i]);
4330 		else
4331 			WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
4332 			       rdev->vm_manager.saved_table_addr[i]);
4333 	}
4334 
4335 	/* enable context1-15 */
4336 	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
4337 	       (u32)(rdev->dummy_page.addr >> 12));
4338 	WREG32(VM_CONTEXT1_CNTL2, 4);
4339 	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
4340 				PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
4341 				RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4342 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4343 				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4344 				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
4345 				PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
4346 				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
4347 				VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
4348 				VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
4349 				READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
4350 				READ_PROTECTION_FAULT_ENABLE_DEFAULT |
4351 				WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
4352 				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
4353 
4354 	si_pcie_gart_tlb_flush(rdev);
4355 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
4356 		 (unsigned)(rdev->mc.gtt_size >> 20),
4357 		 (unsigned long long)rdev->gart.table_addr);
4358 	rdev->gart.ready = true;
4359 	return 0;
4360 }
4361 
4362 static void si_pcie_gart_disable(struct radeon_device *rdev)
4363 {
4364 	unsigned i;
4365 
4366 	for (i = 1; i < 16; ++i) {
4367 		uint32_t reg;
4368 		if (i < 8)
4369 			reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2);
4370 		else
4371 			reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2);
4372 		rdev->vm_manager.saved_table_addr[i] = RREG32(reg);
4373 	}
4374 
4375 	/* Disable all tables */
4376 	WREG32(VM_CONTEXT0_CNTL, 0);
4377 	WREG32(VM_CONTEXT1_CNTL, 0);
4378 	/* Setup TLB control */
4379 	WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
4380 	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
4381 	/* Setup L2 cache */
4382 	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
4383 	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
4384 	       EFFECTIVE_L2_QUEUE_SIZE(7) |
4385 	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
4386 	WREG32(VM_L2_CNTL2, 0);
4387 	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
4388 	       L2_CACHE_BIGK_FRAGMENT_SIZE(0));
4389 	radeon_gart_table_vram_unpin(rdev);
4390 }
4391 
4392 static void si_pcie_gart_fini(struct radeon_device *rdev)
4393 {
4394 	si_pcie_gart_disable(rdev);
4395 	radeon_gart_table_vram_free(rdev);
4396 	radeon_gart_fini(rdev);
4397 }
4398 
4399 /* vm parser */
4400 static bool si_vm_reg_valid(u32 reg)
4401 {
4402 	/* context regs are fine */
4403 	if (reg >= 0x28000)
4404 		return true;
4405 
4406 	/* check config regs */
4407 	switch (reg) {
4408 	case GRBM_GFX_INDEX:
4409 	case CP_STRMOUT_CNTL:
4410 	case VGT_VTX_VECT_EJECT_REG:
4411 	case VGT_CACHE_INVALIDATION:
4412 	case VGT_ESGS_RING_SIZE:
4413 	case VGT_GSVS_RING_SIZE:
4414 	case VGT_GS_VERTEX_REUSE:
4415 	case VGT_PRIMITIVE_TYPE:
4416 	case VGT_INDEX_TYPE:
4417 	case VGT_NUM_INDICES:
4418 	case VGT_NUM_INSTANCES:
4419 	case VGT_TF_RING_SIZE:
4420 	case VGT_HS_OFFCHIP_PARAM:
4421 	case VGT_TF_MEMORY_BASE:
4422 	case PA_CL_ENHANCE:
4423 	case PA_SU_LINE_STIPPLE_VALUE:
4424 	case PA_SC_LINE_STIPPLE_STATE:
4425 	case PA_SC_ENHANCE:
4426 	case SQC_CACHES:
4427 	case SPI_STATIC_THREAD_MGMT_1:
4428 	case SPI_STATIC_THREAD_MGMT_2:
4429 	case SPI_STATIC_THREAD_MGMT_3:
4430 	case SPI_PS_MAX_WAVE_ID:
4431 	case SPI_CONFIG_CNTL:
4432 	case SPI_CONFIG_CNTL_1:
4433 	case TA_CNTL_AUX:
4434 		return true;
4435 	default:
4436 		DRM_ERROR("Invalid register 0x%x in CS\n", reg);
4437 		return false;
4438 	}
4439 }
4440 
4441 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
4442 				  u32 *ib, struct radeon_cs_packet *pkt)
4443 {
4444 	switch (pkt->opcode) {
4445 	case PACKET3_NOP:
4446 	case PACKET3_SET_BASE:
4447 	case PACKET3_SET_CE_DE_COUNTERS:
4448 	case PACKET3_LOAD_CONST_RAM:
4449 	case PACKET3_WRITE_CONST_RAM:
4450 	case PACKET3_WRITE_CONST_RAM_OFFSET:
4451 	case PACKET3_DUMP_CONST_RAM:
4452 	case PACKET3_INCREMENT_CE_COUNTER:
4453 	case PACKET3_WAIT_ON_DE_COUNTER:
4454 	case PACKET3_CE_WRITE:
4455 		break;
4456 	default:
4457 		DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
4458 		return -EINVAL;
4459 	}
4460 	return 0;
4461 }
4462 
4463 static int si_vm_packet3_cp_dma_check(u32 *ib, u32 idx)
4464 {
4465 	u32 start_reg, reg, i;
4466 	u32 command = ib[idx + 4];
4467 	u32 info = ib[idx + 1];
4468 	u32 idx_value = ib[idx];
4469 	if (command & PACKET3_CP_DMA_CMD_SAS) {
4470 		/* src address space is register */
4471 		if (((info & 0x60000000) >> 29) == 0) {
4472 			start_reg = idx_value << 2;
4473 			if (command & PACKET3_CP_DMA_CMD_SAIC) {
4474 				reg = start_reg;
4475 				if (!si_vm_reg_valid(reg)) {
4476 					DRM_ERROR("CP DMA Bad SRC register\n");
4477 					return -EINVAL;
4478 				}
4479 			} else {
4480 				for (i = 0; i < (command & 0x1fffff); i++) {
4481 					reg = start_reg + (4 * i);
4482 					if (!si_vm_reg_valid(reg)) {
4483 						DRM_ERROR("CP DMA Bad SRC register\n");
4484 						return -EINVAL;
4485 					}
4486 				}
4487 			}
4488 		}
4489 	}
4490 	if (command & PACKET3_CP_DMA_CMD_DAS) {
4491 		/* dst address space is register */
4492 		if (((info & 0x00300000) >> 20) == 0) {
4493 			start_reg = ib[idx + 2];
4494 			if (command & PACKET3_CP_DMA_CMD_DAIC) {
4495 				reg = start_reg;
4496 				if (!si_vm_reg_valid(reg)) {
4497 					DRM_ERROR("CP DMA Bad DST register\n");
4498 					return -EINVAL;
4499 				}
4500 			} else {
4501 				for (i = 0; i < (command & 0x1fffff); i++) {
4502 					reg = start_reg + (4 * i);
4503 				if (!si_vm_reg_valid(reg)) {
4504 						DRM_ERROR("CP DMA Bad DST register\n");
4505 						return -EINVAL;
4506 					}
4507 				}
4508 			}
4509 		}
4510 	}
4511 	return 0;
4512 }
4513 
4514 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
4515 				   u32 *ib, struct radeon_cs_packet *pkt)
4516 {
4517 	int r;
4518 	u32 idx = pkt->idx + 1;
4519 	u32 idx_value = ib[idx];
4520 	u32 start_reg, end_reg, reg, i;
4521 
4522 	switch (pkt->opcode) {
4523 	case PACKET3_NOP:
4524 	case PACKET3_SET_BASE:
4525 	case PACKET3_CLEAR_STATE:
4526 	case PACKET3_INDEX_BUFFER_SIZE:
4527 	case PACKET3_DISPATCH_DIRECT:
4528 	case PACKET3_DISPATCH_INDIRECT:
4529 	case PACKET3_ALLOC_GDS:
4530 	case PACKET3_WRITE_GDS_RAM:
4531 	case PACKET3_ATOMIC_GDS:
4532 	case PACKET3_ATOMIC:
4533 	case PACKET3_OCCLUSION_QUERY:
4534 	case PACKET3_SET_PREDICATION:
4535 	case PACKET3_COND_EXEC:
4536 	case PACKET3_PRED_EXEC:
4537 	case PACKET3_DRAW_INDIRECT:
4538 	case PACKET3_DRAW_INDEX_INDIRECT:
4539 	case PACKET3_INDEX_BASE:
4540 	case PACKET3_DRAW_INDEX_2:
4541 	case PACKET3_CONTEXT_CONTROL:
4542 	case PACKET3_INDEX_TYPE:
4543 	case PACKET3_DRAW_INDIRECT_MULTI:
4544 	case PACKET3_DRAW_INDEX_AUTO:
4545 	case PACKET3_DRAW_INDEX_IMMD:
4546 	case PACKET3_NUM_INSTANCES:
4547 	case PACKET3_DRAW_INDEX_MULTI_AUTO:
4548 	case PACKET3_STRMOUT_BUFFER_UPDATE:
4549 	case PACKET3_DRAW_INDEX_OFFSET_2:
4550 	case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
4551 	case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
4552 	case PACKET3_MPEG_INDEX:
4553 	case PACKET3_WAIT_REG_MEM:
4554 	case PACKET3_MEM_WRITE:
4555 	case PACKET3_PFP_SYNC_ME:
4556 	case PACKET3_SURFACE_SYNC:
4557 	case PACKET3_EVENT_WRITE:
4558 	case PACKET3_EVENT_WRITE_EOP:
4559 	case PACKET3_EVENT_WRITE_EOS:
4560 	case PACKET3_SET_CONTEXT_REG:
4561 	case PACKET3_SET_CONTEXT_REG_INDIRECT:
4562 	case PACKET3_SET_SH_REG:
4563 	case PACKET3_SET_SH_REG_OFFSET:
4564 	case PACKET3_INCREMENT_DE_COUNTER:
4565 	case PACKET3_WAIT_ON_CE_COUNTER:
4566 	case PACKET3_WAIT_ON_AVAIL_BUFFER:
4567 	case PACKET3_ME_WRITE:
4568 		break;
4569 	case PACKET3_COPY_DATA:
4570 		if ((idx_value & 0xf00) == 0) {
4571 			reg = ib[idx + 3] * 4;
4572 			if (!si_vm_reg_valid(reg))
4573 				return -EINVAL;
4574 		}
4575 		break;
4576 	case PACKET3_WRITE_DATA:
4577 		if ((idx_value & 0xf00) == 0) {
4578 			start_reg = ib[idx + 1] * 4;
4579 			if (idx_value & 0x10000) {
4580 				if (!si_vm_reg_valid(start_reg))
4581 					return -EINVAL;
4582 			} else {
4583 				for (i = 0; i < (pkt->count - 2); i++) {
4584 					reg = start_reg + (4 * i);
4585 					if (!si_vm_reg_valid(reg))
4586 						return -EINVAL;
4587 				}
4588 			}
4589 		}
4590 		break;
4591 	case PACKET3_COND_WRITE:
4592 		if (idx_value & 0x100) {
4593 			reg = ib[idx + 5] * 4;
4594 			if (!si_vm_reg_valid(reg))
4595 				return -EINVAL;
4596 		}
4597 		break;
4598 	case PACKET3_COPY_DW:
4599 		if (idx_value & 0x2) {
4600 			reg = ib[idx + 3] * 4;
4601 			if (!si_vm_reg_valid(reg))
4602 				return -EINVAL;
4603 		}
4604 		break;
4605 	case PACKET3_SET_CONFIG_REG:
4606 		start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
4607 		end_reg = 4 * pkt->count + start_reg - 4;
4608 		if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
4609 		    (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
4610 		    (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
4611 			DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
4612 			return -EINVAL;
4613 		}
4614 		for (i = 0; i < pkt->count; i++) {
4615 			reg = start_reg + (4 * i);
4616 			if (!si_vm_reg_valid(reg))
4617 				return -EINVAL;
4618 		}
4619 		break;
4620 	case PACKET3_CP_DMA:
4621 		r = si_vm_packet3_cp_dma_check(ib, idx);
4622 		if (r)
4623 			return r;
4624 		break;
4625 	default:
4626 		DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
4627 		return -EINVAL;
4628 	}
4629 	return 0;
4630 }
4631 
4632 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
4633 				       u32 *ib, struct radeon_cs_packet *pkt)
4634 {
4635 	int r;
4636 	u32 idx = pkt->idx + 1;
4637 	u32 idx_value = ib[idx];
4638 	u32 start_reg, reg, i;
4639 
4640 	switch (pkt->opcode) {
4641 	case PACKET3_NOP:
4642 	case PACKET3_SET_BASE:
4643 	case PACKET3_CLEAR_STATE:
4644 	case PACKET3_DISPATCH_DIRECT:
4645 	case PACKET3_DISPATCH_INDIRECT:
4646 	case PACKET3_ALLOC_GDS:
4647 	case PACKET3_WRITE_GDS_RAM:
4648 	case PACKET3_ATOMIC_GDS:
4649 	case PACKET3_ATOMIC:
4650 	case PACKET3_OCCLUSION_QUERY:
4651 	case PACKET3_SET_PREDICATION:
4652 	case PACKET3_COND_EXEC:
4653 	case PACKET3_PRED_EXEC:
4654 	case PACKET3_CONTEXT_CONTROL:
4655 	case PACKET3_STRMOUT_BUFFER_UPDATE:
4656 	case PACKET3_WAIT_REG_MEM:
4657 	case PACKET3_MEM_WRITE:
4658 	case PACKET3_PFP_SYNC_ME:
4659 	case PACKET3_SURFACE_SYNC:
4660 	case PACKET3_EVENT_WRITE:
4661 	case PACKET3_EVENT_WRITE_EOP:
4662 	case PACKET3_EVENT_WRITE_EOS:
4663 	case PACKET3_SET_CONTEXT_REG:
4664 	case PACKET3_SET_CONTEXT_REG_INDIRECT:
4665 	case PACKET3_SET_SH_REG:
4666 	case PACKET3_SET_SH_REG_OFFSET:
4667 	case PACKET3_INCREMENT_DE_COUNTER:
4668 	case PACKET3_WAIT_ON_CE_COUNTER:
4669 	case PACKET3_WAIT_ON_AVAIL_BUFFER:
4670 	case PACKET3_ME_WRITE:
4671 		break;
4672 	case PACKET3_COPY_DATA:
4673 		if ((idx_value & 0xf00) == 0) {
4674 			reg = ib[idx + 3] * 4;
4675 			if (!si_vm_reg_valid(reg))
4676 				return -EINVAL;
4677 		}
4678 		break;
4679 	case PACKET3_WRITE_DATA:
4680 		if ((idx_value & 0xf00) == 0) {
4681 			start_reg = ib[idx + 1] * 4;
4682 			if (idx_value & 0x10000) {
4683 				if (!si_vm_reg_valid(start_reg))
4684 					return -EINVAL;
4685 			} else {
4686 				for (i = 0; i < (pkt->count - 2); i++) {
4687 					reg = start_reg + (4 * i);
4688 					if (!si_vm_reg_valid(reg))
4689 						return -EINVAL;
4690 				}
4691 			}
4692 		}
4693 		break;
4694 	case PACKET3_COND_WRITE:
4695 		if (idx_value & 0x100) {
4696 			reg = ib[idx + 5] * 4;
4697 			if (!si_vm_reg_valid(reg))
4698 				return -EINVAL;
4699 		}
4700 		break;
4701 	case PACKET3_COPY_DW:
4702 		if (idx_value & 0x2) {
4703 			reg = ib[idx + 3] * 4;
4704 			if (!si_vm_reg_valid(reg))
4705 				return -EINVAL;
4706 		}
4707 		break;
4708 	case PACKET3_CP_DMA:
4709 		r = si_vm_packet3_cp_dma_check(ib, idx);
4710 		if (r)
4711 			return r;
4712 		break;
4713 	default:
4714 		DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
4715 		return -EINVAL;
4716 	}
4717 	return 0;
4718 }
4719 
4720 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
4721 {
4722 	int ret = 0;
4723 	u32 idx = 0, i;
4724 	struct radeon_cs_packet pkt;
4725 
4726 	do {
4727 		pkt.idx = idx;
4728 		pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
4729 		pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
4730 		pkt.one_reg_wr = 0;
4731 		switch (pkt.type) {
4732 		case RADEON_PACKET_TYPE0:
4733 			dev_err(rdev->dev, "Packet0 not allowed!\n");
4734 			ret = -EINVAL;
4735 			break;
4736 		case RADEON_PACKET_TYPE2:
4737 			idx += 1;
4738 			break;
4739 		case RADEON_PACKET_TYPE3:
4740 			pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
4741 			if (ib->is_const_ib)
4742 				ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
4743 			else {
4744 				switch (ib->ring) {
4745 				case RADEON_RING_TYPE_GFX_INDEX:
4746 					ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
4747 					break;
4748 				case CAYMAN_RING_TYPE_CP1_INDEX:
4749 				case CAYMAN_RING_TYPE_CP2_INDEX:
4750 					ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
4751 					break;
4752 				default:
4753 					dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
4754 					ret = -EINVAL;
4755 					break;
4756 				}
4757 			}
4758 			idx += pkt.count + 2;
4759 			break;
4760 		default:
4761 			dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
4762 			ret = -EINVAL;
4763 			break;
4764 		}
4765 		if (ret) {
4766 			for (i = 0; i < ib->length_dw; i++) {
4767 				if (i == idx)
4768 					printk("\t0x%08x <---\n", ib->ptr[i]);
4769 				else
4770 					printk("\t0x%08x\n", ib->ptr[i]);
4771 			}
4772 			break;
4773 		}
4774 	} while (idx < ib->length_dw);
4775 
4776 	return ret;
4777 }
4778 
4779 /*
4780  * vm
4781  */
4782 int si_vm_init(struct radeon_device *rdev)
4783 {
4784 	/* number of VMs */
4785 	rdev->vm_manager.nvm = 16;
4786 	/* base offset of vram pages */
4787 	rdev->vm_manager.vram_base_offset = 0;
4788 
4789 	return 0;
4790 }
4791 
4792 void si_vm_fini(struct radeon_device *rdev)
4793 {
4794 }
4795 
4796 /**
4797  * si_vm_decode_fault - print human readable fault info
4798  *
4799  * @rdev: radeon_device pointer
4800  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
4801  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
4802  *
4803  * Print human readable fault information (SI).
4804  */
4805 static void si_vm_decode_fault(struct radeon_device *rdev,
4806 			       u32 status, u32 addr)
4807 {
4808 	u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
4809 	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
4810 	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
4811 	char *block;
4812 
4813 	if (rdev->family == CHIP_TAHITI) {
4814 		switch (mc_id) {
4815 		case 160:
4816 		case 144:
4817 		case 96:
4818 		case 80:
4819 		case 224:
4820 		case 208:
4821 		case 32:
4822 		case 16:
4823 			block = "CB";
4824 			break;
4825 		case 161:
4826 		case 145:
4827 		case 97:
4828 		case 81:
4829 		case 225:
4830 		case 209:
4831 		case 33:
4832 		case 17:
4833 			block = "CB_FMASK";
4834 			break;
4835 		case 162:
4836 		case 146:
4837 		case 98:
4838 		case 82:
4839 		case 226:
4840 		case 210:
4841 		case 34:
4842 		case 18:
4843 			block = "CB_CMASK";
4844 			break;
4845 		case 163:
4846 		case 147:
4847 		case 99:
4848 		case 83:
4849 		case 227:
4850 		case 211:
4851 		case 35:
4852 		case 19:
4853 			block = "CB_IMMED";
4854 			break;
4855 		case 164:
4856 		case 148:
4857 		case 100:
4858 		case 84:
4859 		case 228:
4860 		case 212:
4861 		case 36:
4862 		case 20:
4863 			block = "DB";
4864 			break;
4865 		case 165:
4866 		case 149:
4867 		case 101:
4868 		case 85:
4869 		case 229:
4870 		case 213:
4871 		case 37:
4872 		case 21:
4873 			block = "DB_HTILE";
4874 			break;
4875 		case 167:
4876 		case 151:
4877 		case 103:
4878 		case 87:
4879 		case 231:
4880 		case 215:
4881 		case 39:
4882 		case 23:
4883 			block = "DB_STEN";
4884 			break;
4885 		case 72:
4886 		case 68:
4887 		case 64:
4888 		case 8:
4889 		case 4:
4890 		case 0:
4891 		case 136:
4892 		case 132:
4893 		case 128:
4894 		case 200:
4895 		case 196:
4896 		case 192:
4897 			block = "TC";
4898 			break;
4899 		case 112:
4900 		case 48:
4901 			block = "CP";
4902 			break;
4903 		case 49:
4904 		case 177:
4905 		case 50:
4906 		case 178:
4907 			block = "SH";
4908 			break;
4909 		case 53:
4910 		case 190:
4911 			block = "VGT";
4912 			break;
4913 		case 117:
4914 			block = "IH";
4915 			break;
4916 		case 51:
4917 		case 115:
4918 			block = "RLC";
4919 			break;
4920 		case 119:
4921 		case 183:
4922 			block = "DMA0";
4923 			break;
4924 		case 61:
4925 			block = "DMA1";
4926 			break;
4927 		case 248:
4928 		case 120:
4929 			block = "HDP";
4930 			break;
4931 		default:
4932 			block = "unknown";
4933 			break;
4934 		}
4935 	} else {
4936 		switch (mc_id) {
4937 		case 32:
4938 		case 16:
4939 		case 96:
4940 		case 80:
4941 		case 160:
4942 		case 144:
4943 		case 224:
4944 		case 208:
4945 			block = "CB";
4946 			break;
4947 		case 33:
4948 		case 17:
4949 		case 97:
4950 		case 81:
4951 		case 161:
4952 		case 145:
4953 		case 225:
4954 		case 209:
4955 			block = "CB_FMASK";
4956 			break;
4957 		case 34:
4958 		case 18:
4959 		case 98:
4960 		case 82:
4961 		case 162:
4962 		case 146:
4963 		case 226:
4964 		case 210:
4965 			block = "CB_CMASK";
4966 			break;
4967 		case 35:
4968 		case 19:
4969 		case 99:
4970 		case 83:
4971 		case 163:
4972 		case 147:
4973 		case 227:
4974 		case 211:
4975 			block = "CB_IMMED";
4976 			break;
4977 		case 36:
4978 		case 20:
4979 		case 100:
4980 		case 84:
4981 		case 164:
4982 		case 148:
4983 		case 228:
4984 		case 212:
4985 			block = "DB";
4986 			break;
4987 		case 37:
4988 		case 21:
4989 		case 101:
4990 		case 85:
4991 		case 165:
4992 		case 149:
4993 		case 229:
4994 		case 213:
4995 			block = "DB_HTILE";
4996 			break;
4997 		case 39:
4998 		case 23:
4999 		case 103:
5000 		case 87:
5001 		case 167:
5002 		case 151:
5003 		case 231:
5004 		case 215:
5005 			block = "DB_STEN";
5006 			break;
5007 		case 72:
5008 		case 68:
5009 		case 8:
5010 		case 4:
5011 		case 136:
5012 		case 132:
5013 		case 200:
5014 		case 196:
5015 			block = "TC";
5016 			break;
5017 		case 112:
5018 		case 48:
5019 			block = "CP";
5020 			break;
5021 		case 49:
5022 		case 177:
5023 		case 50:
5024 		case 178:
5025 			block = "SH";
5026 			break;
5027 		case 53:
5028 			block = "VGT";
5029 			break;
5030 		case 117:
5031 			block = "IH";
5032 			break;
5033 		case 51:
5034 		case 115:
5035 			block = "RLC";
5036 			break;
5037 		case 119:
5038 		case 183:
5039 			block = "DMA0";
5040 			break;
5041 		case 61:
5042 			block = "DMA1";
5043 			break;
5044 		case 248:
5045 		case 120:
5046 			block = "HDP";
5047 			break;
5048 		default:
5049 			block = "unknown";
5050 			break;
5051 		}
5052 	}
5053 
5054 	printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
5055 	       protections, vmid, addr,
5056 	       (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
5057 	       block, mc_id);
5058 }
5059 
5060 void si_vm_flush(struct radeon_device *rdev, struct radeon_ring *ring,
5061 		 unsigned vm_id, uint64_t pd_addr)
5062 {
5063 	/* write new base address */
5064 	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5065 	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5066 				 WRITE_DATA_DST_SEL(0)));
5067 
5068 	if (vm_id < 8) {
5069 		radeon_ring_write(ring,
5070 				  (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm_id << 2)) >> 2);
5071 	} else {
5072 		radeon_ring_write(ring,
5073 				  (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm_id - 8) << 2)) >> 2);
5074 	}
5075 	radeon_ring_write(ring, 0);
5076 	radeon_ring_write(ring, pd_addr >> 12);
5077 
5078 	/* flush hdp cache */
5079 	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5080 	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5081 				 WRITE_DATA_DST_SEL(0)));
5082 	radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
5083 	radeon_ring_write(ring, 0);
5084 	radeon_ring_write(ring, 0x1);
5085 
5086 	/* bits 0-15 are the VM contexts0-15 */
5087 	radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
5088 	radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(1) |
5089 				 WRITE_DATA_DST_SEL(0)));
5090 	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5091 	radeon_ring_write(ring, 0);
5092 	radeon_ring_write(ring, 1 << vm_id);
5093 
5094 	/* wait for the invalidate to complete */
5095 	radeon_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
5096 	radeon_ring_write(ring, (WAIT_REG_MEM_FUNCTION(0) |  /* always */
5097 				 WAIT_REG_MEM_ENGINE(0))); /* me */
5098 	radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
5099 	radeon_ring_write(ring, 0);
5100 	radeon_ring_write(ring, 0); /* ref */
5101 	radeon_ring_write(ring, 0); /* mask */
5102 	radeon_ring_write(ring, 0x20); /* poll interval */
5103 
5104 	/* sync PFP to ME, otherwise we might get invalid PFP reads */
5105 	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
5106 	radeon_ring_write(ring, 0x0);
5107 }
5108 
5109 /*
5110  *  Power and clock gating
5111  */
5112 static void si_wait_for_rlc_serdes(struct radeon_device *rdev)
5113 {
5114 	int i;
5115 
5116 	for (i = 0; i < rdev->usec_timeout; i++) {
5117 		if (RREG32(RLC_SERDES_MASTER_BUSY_0) == 0)
5118 			break;
5119 		udelay(1);
5120 	}
5121 
5122 	for (i = 0; i < rdev->usec_timeout; i++) {
5123 		if (RREG32(RLC_SERDES_MASTER_BUSY_1) == 0)
5124 			break;
5125 		udelay(1);
5126 	}
5127 }
5128 
5129 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
5130 					 bool enable)
5131 {
5132 	u32 tmp = RREG32(CP_INT_CNTL_RING0);
5133 	u32 mask;
5134 	int i;
5135 
5136 	if (enable)
5137 		tmp |= (CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5138 	else
5139 		tmp &= ~(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5140 	WREG32(CP_INT_CNTL_RING0, tmp);
5141 
5142 	if (!enable) {
5143 		/* read a gfx register */
5144 		tmp = RREG32(DB_DEPTH_INFO);
5145 
5146 		mask = RLC_BUSY_STATUS | GFX_POWER_STATUS | GFX_CLOCK_STATUS | GFX_LS_STATUS;
5147 		for (i = 0; i < rdev->usec_timeout; i++) {
5148 			if ((RREG32(RLC_STAT) & mask) == (GFX_CLOCK_STATUS | GFX_POWER_STATUS))
5149 				break;
5150 			udelay(1);
5151 		}
5152 	}
5153 }
5154 
5155 static void si_set_uvd_dcm(struct radeon_device *rdev,
5156 			   bool sw_mode)
5157 {
5158 	u32 tmp, tmp2;
5159 
5160 	tmp = RREG32(UVD_CGC_CTRL);
5161 	tmp &= ~(CLK_OD_MASK | CG_DT_MASK);
5162 	tmp |= DCM | CG_DT(1) | CLK_OD(4);
5163 
5164 	if (sw_mode) {
5165 		tmp &= ~0x7ffff800;
5166 		tmp2 = DYN_OR_EN | DYN_RR_EN | G_DIV_ID(7);
5167 	} else {
5168 		tmp |= 0x7ffff800;
5169 		tmp2 = 0;
5170 	}
5171 
5172 	WREG32(UVD_CGC_CTRL, tmp);
5173 	WREG32_UVD_CTX(UVD_CGC_CTRL2, tmp2);
5174 }
5175 
5176 void si_init_uvd_internal_cg(struct radeon_device *rdev)
5177 {
5178 	bool hw_mode = true;
5179 
5180 	if (hw_mode) {
5181 		si_set_uvd_dcm(rdev, false);
5182 	} else {
5183 		u32 tmp = RREG32(UVD_CGC_CTRL);
5184 		tmp &= ~DCM;
5185 		WREG32(UVD_CGC_CTRL, tmp);
5186 	}
5187 }
5188 
5189 static u32 si_halt_rlc(struct radeon_device *rdev)
5190 {
5191 	u32 data, orig;
5192 
5193 	orig = data = RREG32(RLC_CNTL);
5194 
5195 	if (data & RLC_ENABLE) {
5196 		data &= ~RLC_ENABLE;
5197 		WREG32(RLC_CNTL, data);
5198 
5199 		si_wait_for_rlc_serdes(rdev);
5200 	}
5201 
5202 	return orig;
5203 }
5204 
5205 static void si_update_rlc(struct radeon_device *rdev, u32 rlc)
5206 {
5207 	u32 tmp;
5208 
5209 	tmp = RREG32(RLC_CNTL);
5210 	if (tmp != rlc)
5211 		WREG32(RLC_CNTL, rlc);
5212 }
5213 
5214 static void si_enable_dma_pg(struct radeon_device *rdev, bool enable)
5215 {
5216 	u32 data, orig;
5217 
5218 	orig = data = RREG32(DMA_PG);
5219 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA))
5220 		data |= PG_CNTL_ENABLE;
5221 	else
5222 		data &= ~PG_CNTL_ENABLE;
5223 	if (orig != data)
5224 		WREG32(DMA_PG, data);
5225 }
5226 
5227 static void si_init_dma_pg(struct radeon_device *rdev)
5228 {
5229 	u32 tmp;
5230 
5231 	WREG32(DMA_PGFSM_WRITE,  0x00002000);
5232 	WREG32(DMA_PGFSM_CONFIG, 0x100010ff);
5233 
5234 	for (tmp = 0; tmp < 5; tmp++)
5235 		WREG32(DMA_PGFSM_WRITE, 0);
5236 }
5237 
5238 static void si_enable_gfx_cgpg(struct radeon_device *rdev,
5239 			       bool enable)
5240 {
5241 	u32 tmp;
5242 
5243 	if (enable && (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG)) {
5244 		tmp = RLC_PUD(0x10) | RLC_PDD(0x10) | RLC_TTPD(0x10) | RLC_MSD(0x10);
5245 		WREG32(RLC_TTOP_D, tmp);
5246 
5247 		tmp = RREG32(RLC_PG_CNTL);
5248 		tmp |= GFX_PG_ENABLE;
5249 		WREG32(RLC_PG_CNTL, tmp);
5250 
5251 		tmp = RREG32(RLC_AUTO_PG_CTRL);
5252 		tmp |= AUTO_PG_EN;
5253 		WREG32(RLC_AUTO_PG_CTRL, tmp);
5254 	} else {
5255 		tmp = RREG32(RLC_AUTO_PG_CTRL);
5256 		tmp &= ~AUTO_PG_EN;
5257 		WREG32(RLC_AUTO_PG_CTRL, tmp);
5258 
5259 		tmp = RREG32(DB_RENDER_CONTROL);
5260 	}
5261 }
5262 
5263 static void si_init_gfx_cgpg(struct radeon_device *rdev)
5264 {
5265 	u32 tmp;
5266 
5267 	WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5268 
5269 	tmp = RREG32(RLC_PG_CNTL);
5270 	tmp |= GFX_PG_SRC;
5271 	WREG32(RLC_PG_CNTL, tmp);
5272 
5273 	WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5274 
5275 	tmp = RREG32(RLC_AUTO_PG_CTRL);
5276 
5277 	tmp &= ~GRBM_REG_SGIT_MASK;
5278 	tmp |= GRBM_REG_SGIT(0x700);
5279 	tmp &= ~PG_AFTER_GRBM_REG_ST_MASK;
5280 	WREG32(RLC_AUTO_PG_CTRL, tmp);
5281 }
5282 
5283 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh)
5284 {
5285 	u32 mask = 0, tmp, tmp1;
5286 	int i;
5287 
5288 	si_select_se_sh(rdev, se, sh);
5289 	tmp = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
5290 	tmp1 = RREG32(GC_USER_SHADER_ARRAY_CONFIG);
5291 	si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5292 
5293 	tmp &= 0xffff0000;
5294 
5295 	tmp |= tmp1;
5296 	tmp >>= 16;
5297 
5298 	for (i = 0; i < rdev->config.si.max_cu_per_sh; i ++) {
5299 		mask <<= 1;
5300 		mask |= 1;
5301 	}
5302 
5303 	return (~tmp) & mask;
5304 }
5305 
5306 static void si_init_ao_cu_mask(struct radeon_device *rdev)
5307 {
5308 	u32 i, j, k, active_cu_number = 0;
5309 	u32 mask, counter, cu_bitmap;
5310 	u32 tmp = 0;
5311 
5312 	for (i = 0; i < rdev->config.si.max_shader_engines; i++) {
5313 		for (j = 0; j < rdev->config.si.max_sh_per_se; j++) {
5314 			mask = 1;
5315 			cu_bitmap = 0;
5316 			counter  = 0;
5317 			for (k = 0; k < rdev->config.si.max_cu_per_sh; k++) {
5318 				if (si_get_cu_active_bitmap(rdev, i, j) & mask) {
5319 					if (counter < 2)
5320 						cu_bitmap |= mask;
5321 					counter++;
5322 				}
5323 				mask <<= 1;
5324 			}
5325 
5326 			active_cu_number += counter;
5327 			tmp |= (cu_bitmap << (i * 16 + j * 8));
5328 		}
5329 	}
5330 
5331 	WREG32(RLC_PG_AO_CU_MASK, tmp);
5332 
5333 	tmp = RREG32(RLC_MAX_PG_CU);
5334 	tmp &= ~MAX_PU_CU_MASK;
5335 	tmp |= MAX_PU_CU(active_cu_number);
5336 	WREG32(RLC_MAX_PG_CU, tmp);
5337 }
5338 
5339 static void si_enable_cgcg(struct radeon_device *rdev,
5340 			   bool enable)
5341 {
5342 	u32 data, orig, tmp;
5343 
5344 	orig = data = RREG32(RLC_CGCG_CGLS_CTRL);
5345 
5346 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CGCG)) {
5347 		si_enable_gui_idle_interrupt(rdev, true);
5348 
5349 		WREG32(RLC_GCPM_GENERAL_3, 0x00000080);
5350 
5351 		tmp = si_halt_rlc(rdev);
5352 
5353 		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5354 		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5355 		WREG32(RLC_SERDES_WR_CTRL, 0x00b000ff);
5356 
5357 		si_wait_for_rlc_serdes(rdev);
5358 
5359 		si_update_rlc(rdev, tmp);
5360 
5361 		WREG32(RLC_SERDES_WR_CTRL, 0x007000ff);
5362 
5363 		data |= CGCG_EN | CGLS_EN;
5364 	} else {
5365 		si_enable_gui_idle_interrupt(rdev, false);
5366 
5367 		RREG32(CB_CGTT_SCLK_CTRL);
5368 		RREG32(CB_CGTT_SCLK_CTRL);
5369 		RREG32(CB_CGTT_SCLK_CTRL);
5370 		RREG32(CB_CGTT_SCLK_CTRL);
5371 
5372 		data &= ~(CGCG_EN | CGLS_EN);
5373 	}
5374 
5375 	if (orig != data)
5376 		WREG32(RLC_CGCG_CGLS_CTRL, data);
5377 }
5378 
5379 static void si_enable_mgcg(struct radeon_device *rdev,
5380 			   bool enable)
5381 {
5382 	u32 data, orig, tmp = 0;
5383 
5384 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_MGCG)) {
5385 		orig = data = RREG32(CGTS_SM_CTRL_REG);
5386 		data = 0x96940200;
5387 		if (orig != data)
5388 			WREG32(CGTS_SM_CTRL_REG, data);
5389 
5390 		if (rdev->cg_flags & RADEON_CG_SUPPORT_GFX_CP_LS) {
5391 			orig = data = RREG32(CP_MEM_SLP_CNTL);
5392 			data |= CP_MEM_LS_EN;
5393 			if (orig != data)
5394 				WREG32(CP_MEM_SLP_CNTL, data);
5395 		}
5396 
5397 		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5398 		data &= 0xffffffc0;
5399 		if (orig != data)
5400 			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5401 
5402 		tmp = si_halt_rlc(rdev);
5403 
5404 		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5405 		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5406 		WREG32(RLC_SERDES_WR_CTRL, 0x00d000ff);
5407 
5408 		si_update_rlc(rdev, tmp);
5409 	} else {
5410 		orig = data = RREG32(RLC_CGTT_MGCG_OVERRIDE);
5411 		data |= 0x00000003;
5412 		if (orig != data)
5413 			WREG32(RLC_CGTT_MGCG_OVERRIDE, data);
5414 
5415 		data = RREG32(CP_MEM_SLP_CNTL);
5416 		if (data & CP_MEM_LS_EN) {
5417 			data &= ~CP_MEM_LS_EN;
5418 			WREG32(CP_MEM_SLP_CNTL, data);
5419 		}
5420 		orig = data = RREG32(CGTS_SM_CTRL_REG);
5421 		data |= LS_OVERRIDE | OVERRIDE;
5422 		if (orig != data)
5423 			WREG32(CGTS_SM_CTRL_REG, data);
5424 
5425 		tmp = si_halt_rlc(rdev);
5426 
5427 		WREG32(RLC_SERDES_WR_MASTER_MASK_0, 0xffffffff);
5428 		WREG32(RLC_SERDES_WR_MASTER_MASK_1, 0xffffffff);
5429 		WREG32(RLC_SERDES_WR_CTRL, 0x00e000ff);
5430 
5431 		si_update_rlc(rdev, tmp);
5432 	}
5433 }
5434 
5435 static void si_enable_uvd_mgcg(struct radeon_device *rdev,
5436 			       bool enable)
5437 {
5438 	u32 orig, data, tmp;
5439 
5440 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_UVD_MGCG)) {
5441 		tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5442 		tmp |= 0x3fff;
5443 		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5444 
5445 		orig = data = RREG32(UVD_CGC_CTRL);
5446 		data |= DCM;
5447 		if (orig != data)
5448 			WREG32(UVD_CGC_CTRL, data);
5449 
5450 		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0);
5451 		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0);
5452 	} else {
5453 		tmp = RREG32_UVD_CTX(UVD_CGC_MEM_CTRL);
5454 		tmp &= ~0x3fff;
5455 		WREG32_UVD_CTX(UVD_CGC_MEM_CTRL, tmp);
5456 
5457 		orig = data = RREG32(UVD_CGC_CTRL);
5458 		data &= ~DCM;
5459 		if (orig != data)
5460 			WREG32(UVD_CGC_CTRL, data);
5461 
5462 		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_0, 0xffffffff);
5463 		WREG32_SMC(SMC_CG_IND_START + CG_CGTT_LOCAL_1, 0xffffffff);
5464 	}
5465 }
5466 
5467 static const u32 mc_cg_registers[] =
5468 {
5469 	MC_HUB_MISC_HUB_CG,
5470 	MC_HUB_MISC_SIP_CG,
5471 	MC_HUB_MISC_VM_CG,
5472 	MC_XPB_CLK_GAT,
5473 	ATC_MISC_CG,
5474 	MC_CITF_MISC_WR_CG,
5475 	MC_CITF_MISC_RD_CG,
5476 	MC_CITF_MISC_VM_CG,
5477 	VM_L2_CG,
5478 };
5479 
5480 static void si_enable_mc_ls(struct radeon_device *rdev,
5481 			    bool enable)
5482 {
5483 	int i;
5484 	u32 orig, data;
5485 
5486 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5487 		orig = data = RREG32(mc_cg_registers[i]);
5488 		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_LS))
5489 			data |= MC_LS_ENABLE;
5490 		else
5491 			data &= ~MC_LS_ENABLE;
5492 		if (data != orig)
5493 			WREG32(mc_cg_registers[i], data);
5494 	}
5495 }
5496 
5497 static void si_enable_mc_mgcg(struct radeon_device *rdev,
5498 			       bool enable)
5499 {
5500 	int i;
5501 	u32 orig, data;
5502 
5503 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
5504 		orig = data = RREG32(mc_cg_registers[i]);
5505 		if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_MC_MGCG))
5506 			data |= MC_CG_ENABLE;
5507 		else
5508 			data &= ~MC_CG_ENABLE;
5509 		if (data != orig)
5510 			WREG32(mc_cg_registers[i], data);
5511 	}
5512 }
5513 
5514 static void si_enable_dma_mgcg(struct radeon_device *rdev,
5515 			       bool enable)
5516 {
5517 	u32 orig, data, offset;
5518 	int i;
5519 
5520 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_SDMA_MGCG)) {
5521 		for (i = 0; i < 2; i++) {
5522 			if (i == 0)
5523 				offset = DMA0_REGISTER_OFFSET;
5524 			else
5525 				offset = DMA1_REGISTER_OFFSET;
5526 			orig = data = RREG32(DMA_POWER_CNTL + offset);
5527 			data &= ~MEM_POWER_OVERRIDE;
5528 			if (data != orig)
5529 				WREG32(DMA_POWER_CNTL + offset, data);
5530 			WREG32(DMA_CLK_CTRL + offset, 0x00000100);
5531 		}
5532 	} else {
5533 		for (i = 0; i < 2; i++) {
5534 			if (i == 0)
5535 				offset = DMA0_REGISTER_OFFSET;
5536 			else
5537 				offset = DMA1_REGISTER_OFFSET;
5538 			orig = data = RREG32(DMA_POWER_CNTL + offset);
5539 			data |= MEM_POWER_OVERRIDE;
5540 			if (data != orig)
5541 				WREG32(DMA_POWER_CNTL + offset, data);
5542 
5543 			orig = data = RREG32(DMA_CLK_CTRL + offset);
5544 			data = 0xff000000;
5545 			if (data != orig)
5546 				WREG32(DMA_CLK_CTRL + offset, data);
5547 		}
5548 	}
5549 }
5550 
5551 static void si_enable_bif_mgls(struct radeon_device *rdev,
5552 			       bool enable)
5553 {
5554 	u32 orig, data;
5555 
5556 	orig = data = RREG32_PCIE(PCIE_CNTL2);
5557 
5558 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_BIF_LS))
5559 		data |= SLV_MEM_LS_EN | MST_MEM_LS_EN |
5560 			REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN;
5561 	else
5562 		data &= ~(SLV_MEM_LS_EN | MST_MEM_LS_EN |
5563 			  REPLAY_MEM_LS_EN | SLV_MEM_AGGRESSIVE_LS_EN);
5564 
5565 	if (orig != data)
5566 		WREG32_PCIE(PCIE_CNTL2, data);
5567 }
5568 
5569 static void si_enable_hdp_mgcg(struct radeon_device *rdev,
5570 			       bool enable)
5571 {
5572 	u32 orig, data;
5573 
5574 	orig = data = RREG32(HDP_HOST_PATH_CNTL);
5575 
5576 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_MGCG))
5577 		data &= ~CLOCK_GATING_DIS;
5578 	else
5579 		data |= CLOCK_GATING_DIS;
5580 
5581 	if (orig != data)
5582 		WREG32(HDP_HOST_PATH_CNTL, data);
5583 }
5584 
5585 static void si_enable_hdp_ls(struct radeon_device *rdev,
5586 			     bool enable)
5587 {
5588 	u32 orig, data;
5589 
5590 	orig = data = RREG32(HDP_MEM_POWER_LS);
5591 
5592 	if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_HDP_LS))
5593 		data |= HDP_LS_ENABLE;
5594 	else
5595 		data &= ~HDP_LS_ENABLE;
5596 
5597 	if (orig != data)
5598 		WREG32(HDP_MEM_POWER_LS, data);
5599 }
5600 
5601 static void si_update_cg(struct radeon_device *rdev,
5602 			 u32 block, bool enable)
5603 {
5604 	if (block & RADEON_CG_BLOCK_GFX) {
5605 		si_enable_gui_idle_interrupt(rdev, false);
5606 		/* order matters! */
5607 		if (enable) {
5608 			si_enable_mgcg(rdev, true);
5609 			si_enable_cgcg(rdev, true);
5610 		} else {
5611 			si_enable_cgcg(rdev, false);
5612 			si_enable_mgcg(rdev, false);
5613 		}
5614 		si_enable_gui_idle_interrupt(rdev, true);
5615 	}
5616 
5617 	if (block & RADEON_CG_BLOCK_MC) {
5618 		si_enable_mc_mgcg(rdev, enable);
5619 		si_enable_mc_ls(rdev, enable);
5620 	}
5621 
5622 	if (block & RADEON_CG_BLOCK_SDMA) {
5623 		si_enable_dma_mgcg(rdev, enable);
5624 	}
5625 
5626 	if (block & RADEON_CG_BLOCK_BIF) {
5627 		si_enable_bif_mgls(rdev, enable);
5628 	}
5629 
5630 	if (block & RADEON_CG_BLOCK_UVD) {
5631 		if (rdev->has_uvd) {
5632 			si_enable_uvd_mgcg(rdev, enable);
5633 		}
5634 	}
5635 
5636 	if (block & RADEON_CG_BLOCK_HDP) {
5637 		si_enable_hdp_mgcg(rdev, enable);
5638 		si_enable_hdp_ls(rdev, enable);
5639 	}
5640 }
5641 
5642 static void si_init_cg(struct radeon_device *rdev)
5643 {
5644 	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5645 			    RADEON_CG_BLOCK_MC |
5646 			    RADEON_CG_BLOCK_SDMA |
5647 			    RADEON_CG_BLOCK_BIF |
5648 			    RADEON_CG_BLOCK_HDP), true);
5649 	if (rdev->has_uvd) {
5650 		si_update_cg(rdev, RADEON_CG_BLOCK_UVD, true);
5651 		si_init_uvd_internal_cg(rdev);
5652 	}
5653 }
5654 
5655 static void si_fini_cg(struct radeon_device *rdev)
5656 {
5657 	if (rdev->has_uvd) {
5658 		si_update_cg(rdev, RADEON_CG_BLOCK_UVD, false);
5659 	}
5660 	si_update_cg(rdev, (RADEON_CG_BLOCK_GFX |
5661 			    RADEON_CG_BLOCK_MC |
5662 			    RADEON_CG_BLOCK_SDMA |
5663 			    RADEON_CG_BLOCK_BIF |
5664 			    RADEON_CG_BLOCK_HDP), false);
5665 }
5666 
5667 u32 si_get_csb_size(struct radeon_device *rdev)
5668 {
5669 	u32 count = 0;
5670 	const struct cs_section_def *sect = NULL;
5671 	const struct cs_extent_def *ext = NULL;
5672 
5673 	if (rdev->rlc.cs_data == NULL)
5674 		return 0;
5675 
5676 	/* begin clear state */
5677 	count += 2;
5678 	/* context control state */
5679 	count += 3;
5680 
5681 	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5682 		for (ext = sect->section; ext->extent != NULL; ++ext) {
5683 			if (sect->id == SECT_CONTEXT)
5684 				count += 2 + ext->reg_count;
5685 			else
5686 				return 0;
5687 		}
5688 	}
5689 	/* pa_sc_raster_config */
5690 	count += 3;
5691 	/* end clear state */
5692 	count += 2;
5693 	/* clear state */
5694 	count += 2;
5695 
5696 	return count;
5697 }
5698 
5699 void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
5700 {
5701 	u32 count = 0, i;
5702 	const struct cs_section_def *sect = NULL;
5703 	const struct cs_extent_def *ext = NULL;
5704 
5705 	if (rdev->rlc.cs_data == NULL)
5706 		return;
5707 	if (buffer == NULL)
5708 		return;
5709 
5710 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5711 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
5712 
5713 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
5714 	buffer[count++] = cpu_to_le32(0x80000000);
5715 	buffer[count++] = cpu_to_le32(0x80000000);
5716 
5717 	for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
5718 		for (ext = sect->section; ext->extent != NULL; ++ext) {
5719 			if (sect->id == SECT_CONTEXT) {
5720 				buffer[count++] =
5721 					cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
5722 				buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
5723 				for (i = 0; i < ext->reg_count; i++)
5724 					buffer[count++] = cpu_to_le32(ext->extent[i]);
5725 			} else {
5726 				return;
5727 			}
5728 		}
5729 	}
5730 
5731 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
5732 	buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
5733 	switch (rdev->family) {
5734 	case CHIP_TAHITI:
5735 	case CHIP_PITCAIRN:
5736 		buffer[count++] = cpu_to_le32(0x2a00126a);
5737 		break;
5738 	case CHIP_VERDE:
5739 		buffer[count++] = cpu_to_le32(0x0000124a);
5740 		break;
5741 	case CHIP_OLAND:
5742 		buffer[count++] = cpu_to_le32(0x00000082);
5743 		break;
5744 	case CHIP_HAINAN:
5745 		buffer[count++] = cpu_to_le32(0x00000000);
5746 		break;
5747 	default:
5748 		buffer[count++] = cpu_to_le32(0x00000000);
5749 		break;
5750 	}
5751 
5752 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
5753 	buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
5754 
5755 	buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
5756 	buffer[count++] = cpu_to_le32(0);
5757 }
5758 
5759 static void si_init_pg(struct radeon_device *rdev)
5760 {
5761 	if (rdev->pg_flags) {
5762 		if (rdev->pg_flags & RADEON_PG_SUPPORT_SDMA) {
5763 			si_init_dma_pg(rdev);
5764 		}
5765 		si_init_ao_cu_mask(rdev);
5766 		if (rdev->pg_flags & RADEON_PG_SUPPORT_GFX_PG) {
5767 			si_init_gfx_cgpg(rdev);
5768 		} else {
5769 			WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5770 			WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5771 		}
5772 		si_enable_dma_pg(rdev, true);
5773 		si_enable_gfx_cgpg(rdev, true);
5774 	} else {
5775 		WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
5776 		WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
5777 	}
5778 }
5779 
5780 static void si_fini_pg(struct radeon_device *rdev)
5781 {
5782 	if (rdev->pg_flags) {
5783 		si_enable_dma_pg(rdev, false);
5784 		si_enable_gfx_cgpg(rdev, false);
5785 	}
5786 }
5787 
5788 /*
5789  * RLC
5790  */
5791 void si_rlc_reset(struct radeon_device *rdev)
5792 {
5793 	u32 tmp = RREG32(GRBM_SOFT_RESET);
5794 
5795 	tmp |= SOFT_RESET_RLC;
5796 	WREG32(GRBM_SOFT_RESET, tmp);
5797 	udelay(50);
5798 	tmp &= ~SOFT_RESET_RLC;
5799 	WREG32(GRBM_SOFT_RESET, tmp);
5800 	udelay(50);
5801 }
5802 
5803 static void si_rlc_stop(struct radeon_device *rdev)
5804 {
5805 	WREG32(RLC_CNTL, 0);
5806 
5807 	si_enable_gui_idle_interrupt(rdev, false);
5808 
5809 	si_wait_for_rlc_serdes(rdev);
5810 }
5811 
5812 static void si_rlc_start(struct radeon_device *rdev)
5813 {
5814 	WREG32(RLC_CNTL, RLC_ENABLE);
5815 
5816 	si_enable_gui_idle_interrupt(rdev, true);
5817 
5818 	udelay(50);
5819 }
5820 
5821 static bool si_lbpw_supported(struct radeon_device *rdev)
5822 {
5823 	u32 tmp;
5824 
5825 	/* Enable LBPW only for DDR3 */
5826 	tmp = RREG32(MC_SEQ_MISC0);
5827 	if ((tmp & 0xF0000000) == 0xB0000000)
5828 		return true;
5829 	return false;
5830 }
5831 
5832 static void si_enable_lbpw(struct radeon_device *rdev, bool enable)
5833 {
5834 	u32 tmp;
5835 
5836 	tmp = RREG32(RLC_LB_CNTL);
5837 	if (enable)
5838 		tmp |= LOAD_BALANCE_ENABLE;
5839 	else
5840 		tmp &= ~LOAD_BALANCE_ENABLE;
5841 	WREG32(RLC_LB_CNTL, tmp);
5842 
5843 	if (!enable) {
5844 		si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
5845 		WREG32(SPI_LB_CU_MASK, 0x00ff);
5846 	}
5847 }
5848 
5849 static int si_rlc_resume(struct radeon_device *rdev)
5850 {
5851 	u32 i;
5852 
5853 	if (!rdev->rlc_fw)
5854 		return -EINVAL;
5855 
5856 	si_rlc_stop(rdev);
5857 
5858 	si_rlc_reset(rdev);
5859 
5860 	si_init_pg(rdev);
5861 
5862 	si_init_cg(rdev);
5863 
5864 	WREG32(RLC_RL_BASE, 0);
5865 	WREG32(RLC_RL_SIZE, 0);
5866 	WREG32(RLC_LB_CNTL, 0);
5867 	WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
5868 	WREG32(RLC_LB_CNTR_INIT, 0);
5869 	WREG32(RLC_LB_INIT_CU_MASK, 0xffffffff);
5870 
5871 	WREG32(RLC_MC_CNTL, 0);
5872 	WREG32(RLC_UCODE_CNTL, 0);
5873 
5874 	if (rdev->new_fw) {
5875 		const struct rlc_firmware_header_v1_0 *hdr =
5876 			(const struct rlc_firmware_header_v1_0 *)rdev->rlc_fw->data;
5877 		u32 fw_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
5878 		const __le32 *fw_data = (const __le32 *)
5879 			(rdev->rlc_fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
5880 
5881 		radeon_ucode_print_rlc_hdr(&hdr->header);
5882 
5883 		for (i = 0; i < fw_size; i++) {
5884 			WREG32(RLC_UCODE_ADDR, i);
5885 			WREG32(RLC_UCODE_DATA, le32_to_cpup(fw_data++));
5886 		}
5887 	} else {
5888 		const __be32 *fw_data =
5889 			(const __be32 *)rdev->rlc_fw->data;
5890 		for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
5891 			WREG32(RLC_UCODE_ADDR, i);
5892 			WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
5893 		}
5894 	}
5895 	WREG32(RLC_UCODE_ADDR, 0);
5896 
5897 	si_enable_lbpw(rdev, si_lbpw_supported(rdev));
5898 
5899 	si_rlc_start(rdev);
5900 
5901 	return 0;
5902 }
5903 
5904 static void si_enable_interrupts(struct radeon_device *rdev)
5905 {
5906 	u32 ih_cntl = RREG32(IH_CNTL);
5907 	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5908 
5909 	ih_cntl |= ENABLE_INTR;
5910 	ih_rb_cntl |= IH_RB_ENABLE;
5911 	WREG32(IH_CNTL, ih_cntl);
5912 	WREG32(IH_RB_CNTL, ih_rb_cntl);
5913 	rdev->ih.enabled = true;
5914 }
5915 
5916 static void si_disable_interrupts(struct radeon_device *rdev)
5917 {
5918 	u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
5919 	u32 ih_cntl = RREG32(IH_CNTL);
5920 
5921 	ih_rb_cntl &= ~IH_RB_ENABLE;
5922 	ih_cntl &= ~ENABLE_INTR;
5923 	WREG32(IH_RB_CNTL, ih_rb_cntl);
5924 	WREG32(IH_CNTL, ih_cntl);
5925 	/* set rptr, wptr to 0 */
5926 	WREG32(IH_RB_RPTR, 0);
5927 	WREG32(IH_RB_WPTR, 0);
5928 	rdev->ih.enabled = false;
5929 	rdev->ih.rptr = 0;
5930 }
5931 
5932 static void si_disable_interrupt_state(struct radeon_device *rdev)
5933 {
5934 	u32 tmp;
5935 
5936 	tmp = RREG32(CP_INT_CNTL_RING0) &
5937 		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
5938 	WREG32(CP_INT_CNTL_RING0, tmp);
5939 	WREG32(CP_INT_CNTL_RING1, 0);
5940 	WREG32(CP_INT_CNTL_RING2, 0);
5941 	tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
5942 	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
5943 	tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
5944 	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
5945 	WREG32(GRBM_INT_CNTL, 0);
5946 	WREG32(SRBM_INT_CNTL, 0);
5947 	if (rdev->num_crtc >= 2) {
5948 		WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5949 		WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5950 	}
5951 	if (rdev->num_crtc >= 4) {
5952 		WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5953 		WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5954 	}
5955 	if (rdev->num_crtc >= 6) {
5956 		WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5957 		WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5958 	}
5959 
5960 	if (rdev->num_crtc >= 2) {
5961 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
5962 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
5963 	}
5964 	if (rdev->num_crtc >= 4) {
5965 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
5966 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
5967 	}
5968 	if (rdev->num_crtc >= 6) {
5969 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
5970 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
5971 	}
5972 
5973 	if (!ASIC_IS_NODCE(rdev)) {
5974 		WREG32(DAC_AUTODETECT_INT_CONTROL, 0);
5975 
5976 		tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5977 		WREG32(DC_HPD1_INT_CONTROL, tmp);
5978 		tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5979 		WREG32(DC_HPD2_INT_CONTROL, tmp);
5980 		tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5981 		WREG32(DC_HPD3_INT_CONTROL, tmp);
5982 		tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5983 		WREG32(DC_HPD4_INT_CONTROL, tmp);
5984 		tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5985 		WREG32(DC_HPD5_INT_CONTROL, tmp);
5986 		tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
5987 		WREG32(DC_HPD6_INT_CONTROL, tmp);
5988 	}
5989 }
5990 
5991 static int si_irq_init(struct radeon_device *rdev)
5992 {
5993 	int ret = 0;
5994 	int rb_bufsz;
5995 	u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
5996 
5997 	/* allocate ring */
5998 	ret = r600_ih_ring_alloc(rdev);
5999 	if (ret)
6000 		return ret;
6001 
6002 	/* disable irqs */
6003 	si_disable_interrupts(rdev);
6004 
6005 	/* init rlc */
6006 	ret = si_rlc_resume(rdev);
6007 	if (ret) {
6008 		r600_ih_ring_fini(rdev);
6009 		return ret;
6010 	}
6011 
6012 	/* setup interrupt control */
6013 	/* set dummy read address to ring address */
6014 	WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
6015 	interrupt_cntl = RREG32(INTERRUPT_CNTL);
6016 	/* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
6017 	 * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
6018 	 */
6019 	interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
6020 	/* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
6021 	interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
6022 	WREG32(INTERRUPT_CNTL, interrupt_cntl);
6023 
6024 	WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
6025 	rb_bufsz = order_base_2(rdev->ih.ring_size / 4);
6026 
6027 	ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
6028 		      IH_WPTR_OVERFLOW_CLEAR |
6029 		      (rb_bufsz << 1));
6030 
6031 	if (rdev->wb.enabled)
6032 		ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
6033 
6034 	/* set the writeback address whether it's enabled or not */
6035 	WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
6036 	WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
6037 
6038 	WREG32(IH_RB_CNTL, ih_rb_cntl);
6039 
6040 	/* set rptr, wptr to 0 */
6041 	WREG32(IH_RB_RPTR, 0);
6042 	WREG32(IH_RB_WPTR, 0);
6043 
6044 	/* Default settings for IH_CNTL (disabled at first) */
6045 	ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
6046 	/* RPTR_REARM only works if msi's are enabled */
6047 	if (rdev->msi_enabled)
6048 		ih_cntl |= RPTR_REARM;
6049 	WREG32(IH_CNTL, ih_cntl);
6050 
6051 	/* force the active interrupt state to all disabled */
6052 	si_disable_interrupt_state(rdev);
6053 
6054 	pci_set_master(rdev->pdev);
6055 
6056 	/* enable irqs */
6057 	si_enable_interrupts(rdev);
6058 
6059 	return ret;
6060 }
6061 
6062 int si_irq_set(struct radeon_device *rdev)
6063 {
6064 	u32 cp_int_cntl;
6065 	u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
6066 	u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
6067 	u32 hpd1 = 0, hpd2 = 0, hpd3 = 0, hpd4 = 0, hpd5 = 0, hpd6 = 0;
6068 	u32 grbm_int_cntl = 0;
6069 	u32 dma_cntl, dma_cntl1;
6070 	u32 thermal_int = 0;
6071 
6072 	if (!rdev->irq.installed) {
6073 		WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
6074 		return -EINVAL;
6075 	}
6076 	/* don't enable anything if the ih is disabled */
6077 	if (!rdev->ih.enabled) {
6078 		si_disable_interrupts(rdev);
6079 		/* force the active interrupt state to all disabled */
6080 		si_disable_interrupt_state(rdev);
6081 		return 0;
6082 	}
6083 
6084 	cp_int_cntl = RREG32(CP_INT_CNTL_RING0) &
6085 		(CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
6086 
6087 	if (!ASIC_IS_NODCE(rdev)) {
6088 		hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6089 		hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6090 		hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6091 		hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6092 		hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6093 		hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~(DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN);
6094 	}
6095 
6096 	dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
6097 	dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
6098 
6099 	thermal_int = RREG32(CG_THERMAL_INT) &
6100 		~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
6101 
6102 	/* enable CP interrupts on all rings */
6103 	if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
6104 		DRM_DEBUG("si_irq_set: sw int gfx\n");
6105 		cp_int_cntl |= TIME_STAMP_INT_ENABLE;
6106 	}
6107 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
6108 		DRM_DEBUG("si_irq_set: sw int cp1\n");
6109 		cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
6110 	}
6111 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
6112 		DRM_DEBUG("si_irq_set: sw int cp2\n");
6113 		cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
6114 	}
6115 	if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
6116 		DRM_DEBUG("si_irq_set: sw int dma\n");
6117 		dma_cntl |= TRAP_ENABLE;
6118 	}
6119 
6120 	if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
6121 		DRM_DEBUG("si_irq_set: sw int dma1\n");
6122 		dma_cntl1 |= TRAP_ENABLE;
6123 	}
6124 	if (rdev->irq.crtc_vblank_int[0] ||
6125 	    atomic_read(&rdev->irq.pflip[0])) {
6126 		DRM_DEBUG("si_irq_set: vblank 0\n");
6127 		crtc1 |= VBLANK_INT_MASK;
6128 	}
6129 	if (rdev->irq.crtc_vblank_int[1] ||
6130 	    atomic_read(&rdev->irq.pflip[1])) {
6131 		DRM_DEBUG("si_irq_set: vblank 1\n");
6132 		crtc2 |= VBLANK_INT_MASK;
6133 	}
6134 	if (rdev->irq.crtc_vblank_int[2] ||
6135 	    atomic_read(&rdev->irq.pflip[2])) {
6136 		DRM_DEBUG("si_irq_set: vblank 2\n");
6137 		crtc3 |= VBLANK_INT_MASK;
6138 	}
6139 	if (rdev->irq.crtc_vblank_int[3] ||
6140 	    atomic_read(&rdev->irq.pflip[3])) {
6141 		DRM_DEBUG("si_irq_set: vblank 3\n");
6142 		crtc4 |= VBLANK_INT_MASK;
6143 	}
6144 	if (rdev->irq.crtc_vblank_int[4] ||
6145 	    atomic_read(&rdev->irq.pflip[4])) {
6146 		DRM_DEBUG("si_irq_set: vblank 4\n");
6147 		crtc5 |= VBLANK_INT_MASK;
6148 	}
6149 	if (rdev->irq.crtc_vblank_int[5] ||
6150 	    atomic_read(&rdev->irq.pflip[5])) {
6151 		DRM_DEBUG("si_irq_set: vblank 5\n");
6152 		crtc6 |= VBLANK_INT_MASK;
6153 	}
6154 	if (rdev->irq.hpd[0]) {
6155 		DRM_DEBUG("si_irq_set: hpd 1\n");
6156 		hpd1 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6157 	}
6158 	if (rdev->irq.hpd[1]) {
6159 		DRM_DEBUG("si_irq_set: hpd 2\n");
6160 		hpd2 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6161 	}
6162 	if (rdev->irq.hpd[2]) {
6163 		DRM_DEBUG("si_irq_set: hpd 3\n");
6164 		hpd3 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6165 	}
6166 	if (rdev->irq.hpd[3]) {
6167 		DRM_DEBUG("si_irq_set: hpd 4\n");
6168 		hpd4 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6169 	}
6170 	if (rdev->irq.hpd[4]) {
6171 		DRM_DEBUG("si_irq_set: hpd 5\n");
6172 		hpd5 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6173 	}
6174 	if (rdev->irq.hpd[5]) {
6175 		DRM_DEBUG("si_irq_set: hpd 6\n");
6176 		hpd6 |= DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN;
6177 	}
6178 
6179 	WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
6180 	WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
6181 	WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
6182 
6183 	WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
6184 	WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
6185 
6186 	WREG32(GRBM_INT_CNTL, grbm_int_cntl);
6187 
6188 	if (rdev->irq.dpm_thermal) {
6189 		DRM_DEBUG("dpm thermal\n");
6190 		thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
6191 	}
6192 
6193 	if (rdev->num_crtc >= 2) {
6194 		WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
6195 		WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
6196 	}
6197 	if (rdev->num_crtc >= 4) {
6198 		WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
6199 		WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
6200 	}
6201 	if (rdev->num_crtc >= 6) {
6202 		WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
6203 		WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
6204 	}
6205 
6206 	if (rdev->num_crtc >= 2) {
6207 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET,
6208 		       GRPH_PFLIP_INT_MASK);
6209 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET,
6210 		       GRPH_PFLIP_INT_MASK);
6211 	}
6212 	if (rdev->num_crtc >= 4) {
6213 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET,
6214 		       GRPH_PFLIP_INT_MASK);
6215 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET,
6216 		       GRPH_PFLIP_INT_MASK);
6217 	}
6218 	if (rdev->num_crtc >= 6) {
6219 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET,
6220 		       GRPH_PFLIP_INT_MASK);
6221 		WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET,
6222 		       GRPH_PFLIP_INT_MASK);
6223 	}
6224 
6225 	if (!ASIC_IS_NODCE(rdev)) {
6226 		WREG32(DC_HPD1_INT_CONTROL, hpd1);
6227 		WREG32(DC_HPD2_INT_CONTROL, hpd2);
6228 		WREG32(DC_HPD3_INT_CONTROL, hpd3);
6229 		WREG32(DC_HPD4_INT_CONTROL, hpd4);
6230 		WREG32(DC_HPD5_INT_CONTROL, hpd5);
6231 		WREG32(DC_HPD6_INT_CONTROL, hpd6);
6232 	}
6233 
6234 	WREG32(CG_THERMAL_INT, thermal_int);
6235 
6236 	/* posting read */
6237 	RREG32(SRBM_STATUS);
6238 
6239 	return 0;
6240 }
6241 
6242 static inline void si_irq_ack(struct radeon_device *rdev)
6243 {
6244 	u32 tmp;
6245 
6246 	if (ASIC_IS_NODCE(rdev))
6247 		return;
6248 
6249 	rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
6250 	rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
6251 	rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
6252 	rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
6253 	rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
6254 	rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
6255 	rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
6256 	rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
6257 	if (rdev->num_crtc >= 4) {
6258 		rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
6259 		rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
6260 	}
6261 	if (rdev->num_crtc >= 6) {
6262 		rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
6263 		rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
6264 	}
6265 
6266 	if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
6267 		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6268 	if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
6269 		WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6270 	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
6271 		WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
6272 	if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
6273 		WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
6274 	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
6275 		WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
6276 	if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
6277 		WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
6278 
6279 	if (rdev->num_crtc >= 4) {
6280 		if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
6281 			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6282 		if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
6283 			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6284 		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
6285 			WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
6286 		if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
6287 			WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
6288 		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
6289 			WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
6290 		if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
6291 			WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
6292 	}
6293 
6294 	if (rdev->num_crtc >= 6) {
6295 		if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
6296 			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6297 		if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
6298 			WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
6299 		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
6300 			WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
6301 		if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
6302 			WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
6303 		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
6304 			WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
6305 		if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
6306 			WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
6307 	}
6308 
6309 	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
6310 		tmp = RREG32(DC_HPD1_INT_CONTROL);
6311 		tmp |= DC_HPDx_INT_ACK;
6312 		WREG32(DC_HPD1_INT_CONTROL, tmp);
6313 	}
6314 	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
6315 		tmp = RREG32(DC_HPD2_INT_CONTROL);
6316 		tmp |= DC_HPDx_INT_ACK;
6317 		WREG32(DC_HPD2_INT_CONTROL, tmp);
6318 	}
6319 	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
6320 		tmp = RREG32(DC_HPD3_INT_CONTROL);
6321 		tmp |= DC_HPDx_INT_ACK;
6322 		WREG32(DC_HPD3_INT_CONTROL, tmp);
6323 	}
6324 	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
6325 		tmp = RREG32(DC_HPD4_INT_CONTROL);
6326 		tmp |= DC_HPDx_INT_ACK;
6327 		WREG32(DC_HPD4_INT_CONTROL, tmp);
6328 	}
6329 	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
6330 		tmp = RREG32(DC_HPD5_INT_CONTROL);
6331 		tmp |= DC_HPDx_INT_ACK;
6332 		WREG32(DC_HPD5_INT_CONTROL, tmp);
6333 	}
6334 	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
6335 		tmp = RREG32(DC_HPD5_INT_CONTROL);
6336 		tmp |= DC_HPDx_INT_ACK;
6337 		WREG32(DC_HPD6_INT_CONTROL, tmp);
6338 	}
6339 
6340 	if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT) {
6341 		tmp = RREG32(DC_HPD1_INT_CONTROL);
6342 		tmp |= DC_HPDx_RX_INT_ACK;
6343 		WREG32(DC_HPD1_INT_CONTROL, tmp);
6344 	}
6345 	if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT) {
6346 		tmp = RREG32(DC_HPD2_INT_CONTROL);
6347 		tmp |= DC_HPDx_RX_INT_ACK;
6348 		WREG32(DC_HPD2_INT_CONTROL, tmp);
6349 	}
6350 	if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT) {
6351 		tmp = RREG32(DC_HPD3_INT_CONTROL);
6352 		tmp |= DC_HPDx_RX_INT_ACK;
6353 		WREG32(DC_HPD3_INT_CONTROL, tmp);
6354 	}
6355 	if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT) {
6356 		tmp = RREG32(DC_HPD4_INT_CONTROL);
6357 		tmp |= DC_HPDx_RX_INT_ACK;
6358 		WREG32(DC_HPD4_INT_CONTROL, tmp);
6359 	}
6360 	if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT) {
6361 		tmp = RREG32(DC_HPD5_INT_CONTROL);
6362 		tmp |= DC_HPDx_RX_INT_ACK;
6363 		WREG32(DC_HPD5_INT_CONTROL, tmp);
6364 	}
6365 	if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT) {
6366 		tmp = RREG32(DC_HPD5_INT_CONTROL);
6367 		tmp |= DC_HPDx_RX_INT_ACK;
6368 		WREG32(DC_HPD6_INT_CONTROL, tmp);
6369 	}
6370 }
6371 
6372 static void si_irq_disable(struct radeon_device *rdev)
6373 {
6374 	si_disable_interrupts(rdev);
6375 	/* Wait and acknowledge irq */
6376 	mdelay(1);
6377 	si_irq_ack(rdev);
6378 	si_disable_interrupt_state(rdev);
6379 }
6380 
6381 static void si_irq_suspend(struct radeon_device *rdev)
6382 {
6383 	si_irq_disable(rdev);
6384 	si_rlc_stop(rdev);
6385 }
6386 
6387 static void si_irq_fini(struct radeon_device *rdev)
6388 {
6389 	si_irq_suspend(rdev);
6390 	r600_ih_ring_fini(rdev);
6391 }
6392 
6393 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
6394 {
6395 	u32 wptr, tmp;
6396 
6397 	if (rdev->wb.enabled)
6398 		wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
6399 	else
6400 		wptr = RREG32(IH_RB_WPTR);
6401 
6402 	if (wptr & RB_OVERFLOW) {
6403 		wptr &= ~RB_OVERFLOW;
6404 		/* When a ring buffer overflow happen start parsing interrupt
6405 		 * from the last not overwritten vector (wptr + 16). Hopefully
6406 		 * this should allow us to catchup.
6407 		 */
6408 		dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
6409 			 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
6410 		rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
6411 		tmp = RREG32(IH_RB_CNTL);
6412 		tmp |= IH_WPTR_OVERFLOW_CLEAR;
6413 		WREG32(IH_RB_CNTL, tmp);
6414 	}
6415 	return (wptr & rdev->ih.ptr_mask);
6416 }
6417 
6418 /*        SI IV Ring
6419  * Each IV ring entry is 128 bits:
6420  * [7:0]    - interrupt source id
6421  * [31:8]   - reserved
6422  * [59:32]  - interrupt source data
6423  * [63:60]  - reserved
6424  * [71:64]  - RINGID
6425  * [79:72]  - VMID
6426  * [127:80] - reserved
6427  */
6428 int si_irq_process(struct radeon_device *rdev)
6429 {
6430 	u32 wptr;
6431 	u32 rptr;
6432 	u32 src_id, src_data, ring_id;
6433 	u32 ring_index;
6434 	bool queue_hotplug = false;
6435 	bool queue_dp = false;
6436 	bool queue_thermal = false;
6437 	u32 status, addr;
6438 
6439 	if (!rdev->ih.enabled || rdev->shutdown)
6440 		return IRQ_NONE;
6441 
6442 	wptr = si_get_ih_wptr(rdev);
6443 
6444 restart_ih:
6445 	/* is somebody else already processing irqs? */
6446 	if (atomic_xchg(&rdev->ih.lock, 1))
6447 		return IRQ_NONE;
6448 
6449 	rptr = rdev->ih.rptr;
6450 	DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
6451 
6452 	/* Order reading of wptr vs. reading of IH ring data */
6453 	rmb();
6454 
6455 	/* display interrupts */
6456 	si_irq_ack(rdev);
6457 
6458 	while (rptr != wptr) {
6459 		/* wptr/rptr are in bytes! */
6460 		ring_index = rptr / 4;
6461 		src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
6462 		src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
6463 		ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
6464 
6465 		switch (src_id) {
6466 		case 1: /* D1 vblank/vline */
6467 			switch (src_data) {
6468 			case 0: /* D1 vblank */
6469 				if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT))
6470 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6471 
6472 				if (rdev->irq.crtc_vblank_int[0]) {
6473 					drm_handle_vblank(rdev->ddev, 0);
6474 					rdev->pm.vblank_sync = true;
6475 					wake_up(&rdev->irq.vblank_queue);
6476 				}
6477 				if (atomic_read(&rdev->irq.pflip[0]))
6478 					radeon_crtc_handle_vblank(rdev, 0);
6479 				rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
6480 				DRM_DEBUG("IH: D1 vblank\n");
6481 
6482 				break;
6483 			case 1: /* D1 vline */
6484 				if (!(rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT))
6485 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6486 
6487 				rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
6488 				DRM_DEBUG("IH: D1 vline\n");
6489 
6490 				break;
6491 			default:
6492 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6493 				break;
6494 			}
6495 			break;
6496 		case 2: /* D2 vblank/vline */
6497 			switch (src_data) {
6498 			case 0: /* D2 vblank */
6499 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT))
6500 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6501 
6502 				if (rdev->irq.crtc_vblank_int[1]) {
6503 					drm_handle_vblank(rdev->ddev, 1);
6504 					rdev->pm.vblank_sync = true;
6505 					wake_up(&rdev->irq.vblank_queue);
6506 				}
6507 				if (atomic_read(&rdev->irq.pflip[1]))
6508 					radeon_crtc_handle_vblank(rdev, 1);
6509 				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
6510 				DRM_DEBUG("IH: D2 vblank\n");
6511 
6512 				break;
6513 			case 1: /* D2 vline */
6514 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT))
6515 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6516 
6517 				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
6518 				DRM_DEBUG("IH: D2 vline\n");
6519 
6520 				break;
6521 			default:
6522 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6523 				break;
6524 			}
6525 			break;
6526 		case 3: /* D3 vblank/vline */
6527 			switch (src_data) {
6528 			case 0: /* D3 vblank */
6529 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT))
6530 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6531 
6532 				if (rdev->irq.crtc_vblank_int[2]) {
6533 					drm_handle_vblank(rdev->ddev, 2);
6534 					rdev->pm.vblank_sync = true;
6535 					wake_up(&rdev->irq.vblank_queue);
6536 				}
6537 				if (atomic_read(&rdev->irq.pflip[2]))
6538 					radeon_crtc_handle_vblank(rdev, 2);
6539 				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
6540 				DRM_DEBUG("IH: D3 vblank\n");
6541 
6542 				break;
6543 			case 1: /* D3 vline */
6544 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT))
6545 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6546 
6547 				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
6548 				DRM_DEBUG("IH: D3 vline\n");
6549 
6550 				break;
6551 			default:
6552 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6553 				break;
6554 			}
6555 			break;
6556 		case 4: /* D4 vblank/vline */
6557 			switch (src_data) {
6558 			case 0: /* D4 vblank */
6559 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT))
6560 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6561 
6562 				if (rdev->irq.crtc_vblank_int[3]) {
6563 					drm_handle_vblank(rdev->ddev, 3);
6564 					rdev->pm.vblank_sync = true;
6565 					wake_up(&rdev->irq.vblank_queue);
6566 				}
6567 				if (atomic_read(&rdev->irq.pflip[3]))
6568 					radeon_crtc_handle_vblank(rdev, 3);
6569 				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
6570 				DRM_DEBUG("IH: D4 vblank\n");
6571 
6572 				break;
6573 			case 1: /* D4 vline */
6574 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT))
6575 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6576 
6577 				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
6578 				DRM_DEBUG("IH: D4 vline\n");
6579 
6580 				break;
6581 			default:
6582 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6583 				break;
6584 			}
6585 			break;
6586 		case 5: /* D5 vblank/vline */
6587 			switch (src_data) {
6588 			case 0: /* D5 vblank */
6589 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT))
6590 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6591 
6592 				if (rdev->irq.crtc_vblank_int[4]) {
6593 					drm_handle_vblank(rdev->ddev, 4);
6594 					rdev->pm.vblank_sync = true;
6595 					wake_up(&rdev->irq.vblank_queue);
6596 				}
6597 				if (atomic_read(&rdev->irq.pflip[4]))
6598 					radeon_crtc_handle_vblank(rdev, 4);
6599 				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
6600 				DRM_DEBUG("IH: D5 vblank\n");
6601 
6602 				break;
6603 			case 1: /* D5 vline */
6604 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT))
6605 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6606 
6607 				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
6608 				DRM_DEBUG("IH: D5 vline\n");
6609 
6610 				break;
6611 			default:
6612 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6613 				break;
6614 			}
6615 			break;
6616 		case 6: /* D6 vblank/vline */
6617 			switch (src_data) {
6618 			case 0: /* D6 vblank */
6619 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT))
6620 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6621 
6622 				if (rdev->irq.crtc_vblank_int[5]) {
6623 					drm_handle_vblank(rdev->ddev, 5);
6624 					rdev->pm.vblank_sync = true;
6625 					wake_up(&rdev->irq.vblank_queue);
6626 				}
6627 				if (atomic_read(&rdev->irq.pflip[5]))
6628 					radeon_crtc_handle_vblank(rdev, 5);
6629 				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
6630 				DRM_DEBUG("IH: D6 vblank\n");
6631 
6632 				break;
6633 			case 1: /* D6 vline */
6634 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT))
6635 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6636 
6637 				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
6638 				DRM_DEBUG("IH: D6 vline\n");
6639 
6640 				break;
6641 			default:
6642 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6643 				break;
6644 			}
6645 			break;
6646 		case 8: /* D1 page flip */
6647 		case 10: /* D2 page flip */
6648 		case 12: /* D3 page flip */
6649 		case 14: /* D4 page flip */
6650 		case 16: /* D5 page flip */
6651 		case 18: /* D6 page flip */
6652 			DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
6653 			if (radeon_use_pflipirq > 0)
6654 				radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
6655 			break;
6656 		case 42: /* HPD hotplug */
6657 			switch (src_data) {
6658 			case 0:
6659 				if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT))
6660 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6661 
6662 				rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
6663 				queue_hotplug = true;
6664 				DRM_DEBUG("IH: HPD1\n");
6665 
6666 				break;
6667 			case 1:
6668 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT))
6669 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6670 
6671 				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
6672 				queue_hotplug = true;
6673 				DRM_DEBUG("IH: HPD2\n");
6674 
6675 				break;
6676 			case 2:
6677 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT))
6678 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6679 
6680 				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
6681 				queue_hotplug = true;
6682 				DRM_DEBUG("IH: HPD3\n");
6683 
6684 				break;
6685 			case 3:
6686 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT))
6687 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6688 
6689 				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
6690 				queue_hotplug = true;
6691 				DRM_DEBUG("IH: HPD4\n");
6692 
6693 				break;
6694 			case 4:
6695 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT))
6696 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6697 
6698 				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
6699 				queue_hotplug = true;
6700 				DRM_DEBUG("IH: HPD5\n");
6701 
6702 				break;
6703 			case 5:
6704 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT))
6705 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6706 
6707 				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
6708 				queue_hotplug = true;
6709 				DRM_DEBUG("IH: HPD6\n");
6710 
6711 				break;
6712 			case 6:
6713 				if (!(rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_RX_INTERRUPT))
6714 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6715 
6716 				rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_RX_INTERRUPT;
6717 				queue_dp = true;
6718 				DRM_DEBUG("IH: HPD_RX 1\n");
6719 
6720 				break;
6721 			case 7:
6722 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_RX_INTERRUPT))
6723 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6724 
6725 				rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_RX_INTERRUPT;
6726 				queue_dp = true;
6727 				DRM_DEBUG("IH: HPD_RX 2\n");
6728 
6729 				break;
6730 			case 8:
6731 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_RX_INTERRUPT))
6732 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6733 
6734 				rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_RX_INTERRUPT;
6735 				queue_dp = true;
6736 				DRM_DEBUG("IH: HPD_RX 3\n");
6737 
6738 				break;
6739 			case 9:
6740 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_RX_INTERRUPT))
6741 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6742 
6743 				rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_RX_INTERRUPT;
6744 				queue_dp = true;
6745 				DRM_DEBUG("IH: HPD_RX 4\n");
6746 
6747 				break;
6748 			case 10:
6749 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_RX_INTERRUPT))
6750 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6751 
6752 				rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_RX_INTERRUPT;
6753 				queue_dp = true;
6754 				DRM_DEBUG("IH: HPD_RX 5\n");
6755 
6756 				break;
6757 			case 11:
6758 				if (!(rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_RX_INTERRUPT))
6759 					DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
6760 
6761 				rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_RX_INTERRUPT;
6762 				queue_dp = true;
6763 				DRM_DEBUG("IH: HPD_RX 6\n");
6764 
6765 				break;
6766 			default:
6767 				DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6768 				break;
6769 			}
6770 			break;
6771 		case 96:
6772 			DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
6773 			WREG32(SRBM_INT_ACK, 0x1);
6774 			break;
6775 		case 124: /* UVD */
6776 			DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
6777 			radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
6778 			break;
6779 		case 146:
6780 		case 147:
6781 			addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
6782 			status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
6783 			/* reset addr and status */
6784 			WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
6785 			if (addr == 0x0 && status == 0x0)
6786 				break;
6787 			dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
6788 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
6789 				addr);
6790 			dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
6791 				status);
6792 			si_vm_decode_fault(rdev, status, addr);
6793 			break;
6794 		case 176: /* RINGID0 CP_INT */
6795 			radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6796 			break;
6797 		case 177: /* RINGID1 CP_INT */
6798 			radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6799 			break;
6800 		case 178: /* RINGID2 CP_INT */
6801 			radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6802 			break;
6803 		case 181: /* CP EOP event */
6804 			DRM_DEBUG("IH: CP EOP\n");
6805 			switch (ring_id) {
6806 			case 0:
6807 				radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
6808 				break;
6809 			case 1:
6810 				radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6811 				break;
6812 			case 2:
6813 				radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6814 				break;
6815 			}
6816 			break;
6817 		case 224: /* DMA trap event */
6818 			DRM_DEBUG("IH: DMA trap\n");
6819 			radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
6820 			break;
6821 		case 230: /* thermal low to high */
6822 			DRM_DEBUG("IH: thermal low to high\n");
6823 			rdev->pm.dpm.thermal.high_to_low = false;
6824 			queue_thermal = true;
6825 			break;
6826 		case 231: /* thermal high to low */
6827 			DRM_DEBUG("IH: thermal high to low\n");
6828 			rdev->pm.dpm.thermal.high_to_low = true;
6829 			queue_thermal = true;
6830 			break;
6831 		case 233: /* GUI IDLE */
6832 			DRM_DEBUG("IH: GUI idle\n");
6833 			break;
6834 		case 244: /* DMA trap event */
6835 			DRM_DEBUG("IH: DMA1 trap\n");
6836 			radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6837 			break;
6838 		default:
6839 			DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
6840 			break;
6841 		}
6842 
6843 		/* wptr/rptr are in bytes! */
6844 		rptr += 16;
6845 		rptr &= rdev->ih.ptr_mask;
6846 		WREG32(IH_RB_RPTR, rptr);
6847 	}
6848 	if (queue_dp)
6849 		schedule_work(&rdev->dp_work);
6850 	if (queue_hotplug)
6851 		schedule_work(&rdev->hotplug_work);
6852 	if (queue_thermal && rdev->pm.dpm_enabled)
6853 		schedule_work(&rdev->pm.dpm.thermal.work);
6854 	rdev->ih.rptr = rptr;
6855 	atomic_set(&rdev->ih.lock, 0);
6856 
6857 	/* make sure wptr hasn't changed while processing */
6858 	wptr = si_get_ih_wptr(rdev);
6859 	if (wptr != rptr)
6860 		goto restart_ih;
6861 
6862 	return IRQ_HANDLED;
6863 }
6864 
6865 /*
6866  * startup/shutdown callbacks
6867  */
6868 static int si_startup(struct radeon_device *rdev)
6869 {
6870 	struct radeon_ring *ring;
6871 	int r;
6872 
6873 	/* enable pcie gen2/3 link */
6874 	si_pcie_gen3_enable(rdev);
6875 	/* enable aspm */
6876 	si_program_aspm(rdev);
6877 
6878 	/* scratch needs to be initialized before MC */
6879 	r = r600_vram_scratch_init(rdev);
6880 	if (r)
6881 		return r;
6882 
6883 	si_mc_program(rdev);
6884 
6885 	if (!rdev->pm.dpm_enabled) {
6886 		r = si_mc_load_microcode(rdev);
6887 		if (r) {
6888 			DRM_ERROR("Failed to load MC firmware!\n");
6889 			return r;
6890 		}
6891 	}
6892 
6893 	r = si_pcie_gart_enable(rdev);
6894 	if (r)
6895 		return r;
6896 	si_gpu_init(rdev);
6897 
6898 	/* allocate rlc buffers */
6899 	if (rdev->family == CHIP_VERDE) {
6900 		rdev->rlc.reg_list = verde_rlc_save_restore_register_list;
6901 		rdev->rlc.reg_list_size =
6902 			(u32)ARRAY_SIZE(verde_rlc_save_restore_register_list);
6903 	}
6904 	rdev->rlc.cs_data = si_cs_data;
6905 	r = sumo_rlc_init(rdev);
6906 	if (r) {
6907 		DRM_ERROR("Failed to init rlc BOs!\n");
6908 		return r;
6909 	}
6910 
6911 	/* allocate wb buffer */
6912 	r = radeon_wb_init(rdev);
6913 	if (r)
6914 		return r;
6915 
6916 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
6917 	if (r) {
6918 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6919 		return r;
6920 	}
6921 
6922 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
6923 	if (r) {
6924 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6925 		return r;
6926 	}
6927 
6928 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
6929 	if (r) {
6930 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
6931 		return r;
6932 	}
6933 
6934 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
6935 	if (r) {
6936 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6937 		return r;
6938 	}
6939 
6940 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
6941 	if (r) {
6942 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
6943 		return r;
6944 	}
6945 
6946 	if (rdev->has_uvd) {
6947 		r = uvd_v2_2_resume(rdev);
6948 		if (!r) {
6949 			r = radeon_fence_driver_start_ring(rdev,
6950 							   R600_RING_TYPE_UVD_INDEX);
6951 			if (r)
6952 				dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
6953 		}
6954 		if (r)
6955 			rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
6956 	}
6957 
6958 	r = radeon_vce_resume(rdev);
6959 	if (!r) {
6960 		r = vce_v1_0_resume(rdev);
6961 		if (!r)
6962 			r = radeon_fence_driver_start_ring(rdev,
6963 							   TN_RING_TYPE_VCE1_INDEX);
6964 		if (!r)
6965 			r = radeon_fence_driver_start_ring(rdev,
6966 							   TN_RING_TYPE_VCE2_INDEX);
6967 	}
6968 	if (r) {
6969 		dev_err(rdev->dev, "VCE init error (%d).\n", r);
6970 		rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
6971 		rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
6972 	}
6973 
6974 	/* Enable IRQ */
6975 	if (!rdev->irq.installed) {
6976 		r = radeon_irq_kms_init(rdev);
6977 		if (r)
6978 			return r;
6979 	}
6980 
6981 	r = si_irq_init(rdev);
6982 	if (r) {
6983 		DRM_ERROR("radeon: IH init failed (%d).\n", r);
6984 		radeon_irq_kms_fini(rdev);
6985 		return r;
6986 	}
6987 	si_irq_set(rdev);
6988 
6989 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
6990 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
6991 			     RADEON_CP_PACKET2);
6992 	if (r)
6993 		return r;
6994 
6995 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
6996 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
6997 			     RADEON_CP_PACKET2);
6998 	if (r)
6999 		return r;
7000 
7001 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7002 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
7003 			     RADEON_CP_PACKET2);
7004 	if (r)
7005 		return r;
7006 
7007 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7008 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
7009 			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7010 	if (r)
7011 		return r;
7012 
7013 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7014 	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
7015 			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
7016 	if (r)
7017 		return r;
7018 
7019 	r = si_cp_load_microcode(rdev);
7020 	if (r)
7021 		return r;
7022 	r = si_cp_resume(rdev);
7023 	if (r)
7024 		return r;
7025 
7026 	r = cayman_dma_resume(rdev);
7027 	if (r)
7028 		return r;
7029 
7030 	if (rdev->has_uvd) {
7031 		ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7032 		if (ring->ring_size) {
7033 			r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7034 					     RADEON_CP_PACKET2);
7035 			if (!r)
7036 				r = uvd_v1_0_init(rdev);
7037 			if (r)
7038 				DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
7039 		}
7040 	}
7041 
7042 	r = -ENOENT;
7043 
7044 	ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7045 	if (ring->ring_size)
7046 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7047 				     VCE_CMD_NO_OP);
7048 
7049 	ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7050 	if (ring->ring_size)
7051 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
7052 				     VCE_CMD_NO_OP);
7053 
7054 	if (!r)
7055 		r = vce_v1_0_init(rdev);
7056 	else if (r != -ENOENT)
7057 		DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
7058 
7059 	r = radeon_ib_pool_init(rdev);
7060 	if (r) {
7061 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
7062 		return r;
7063 	}
7064 
7065 	r = radeon_vm_manager_init(rdev);
7066 	if (r) {
7067 		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
7068 		return r;
7069 	}
7070 
7071 	r = radeon_audio_init(rdev);
7072 	if (r)
7073 		return r;
7074 
7075 	return 0;
7076 }
7077 
7078 int si_resume(struct radeon_device *rdev)
7079 {
7080 	int r;
7081 
7082 	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
7083 	 * posting will perform necessary task to bring back GPU into good
7084 	 * shape.
7085 	 */
7086 	/* post card */
7087 	atom_asic_init(rdev->mode_info.atom_context);
7088 
7089 	/* init golden registers */
7090 	si_init_golden_registers(rdev);
7091 
7092 	if (rdev->pm.pm_method == PM_METHOD_DPM)
7093 		radeon_pm_resume(rdev);
7094 
7095 	rdev->accel_working = true;
7096 	r = si_startup(rdev);
7097 	if (r) {
7098 		DRM_ERROR("si startup failed on resume\n");
7099 		rdev->accel_working = false;
7100 		return r;
7101 	}
7102 
7103 	return r;
7104 
7105 }
7106 
7107 int si_suspend(struct radeon_device *rdev)
7108 {
7109 	radeon_pm_suspend(rdev);
7110 	radeon_audio_fini(rdev);
7111 	radeon_vm_manager_fini(rdev);
7112 	si_cp_enable(rdev, false);
7113 	cayman_dma_stop(rdev);
7114 	if (rdev->has_uvd) {
7115 		uvd_v1_0_fini(rdev);
7116 		radeon_uvd_suspend(rdev);
7117 		radeon_vce_suspend(rdev);
7118 	}
7119 	si_fini_pg(rdev);
7120 	si_fini_cg(rdev);
7121 	si_irq_suspend(rdev);
7122 	radeon_wb_disable(rdev);
7123 	si_pcie_gart_disable(rdev);
7124 	return 0;
7125 }
7126 
7127 /* Plan is to move initialization in that function and use
7128  * helper function so that radeon_device_init pretty much
7129  * do nothing more than calling asic specific function. This
7130  * should also allow to remove a bunch of callback function
7131  * like vram_info.
7132  */
7133 int si_init(struct radeon_device *rdev)
7134 {
7135 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7136 	int r;
7137 
7138 	/* Read BIOS */
7139 	if (!radeon_get_bios(rdev)) {
7140 		if (ASIC_IS_AVIVO(rdev))
7141 			return -EINVAL;
7142 	}
7143 	/* Must be an ATOMBIOS */
7144 	if (!rdev->is_atom_bios) {
7145 		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
7146 		return -EINVAL;
7147 	}
7148 	r = radeon_atombios_init(rdev);
7149 	if (r)
7150 		return r;
7151 
7152 	/* Post card if necessary */
7153 	if (!radeon_card_posted(rdev)) {
7154 		if (!rdev->bios) {
7155 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
7156 			return -EINVAL;
7157 		}
7158 		DRM_INFO("GPU not posted. posting now...\n");
7159 		atom_asic_init(rdev->mode_info.atom_context);
7160 	}
7161 	/* init golden registers */
7162 	si_init_golden_registers(rdev);
7163 	/* Initialize scratch registers */
7164 	si_scratch_init(rdev);
7165 	/* Initialize surface registers */
7166 	radeon_surface_init(rdev);
7167 	/* Initialize clocks */
7168 	radeon_get_clock_info(rdev->ddev);
7169 
7170 	/* Fence driver */
7171 	r = radeon_fence_driver_init(rdev);
7172 	if (r)
7173 		return r;
7174 
7175 	/* initialize memory controller */
7176 	r = si_mc_init(rdev);
7177 	if (r)
7178 		return r;
7179 	/* Memory manager */
7180 	r = radeon_bo_init(rdev);
7181 	if (r)
7182 		return r;
7183 
7184 	if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
7185 	    !rdev->rlc_fw || !rdev->mc_fw) {
7186 		r = si_init_microcode(rdev);
7187 		if (r) {
7188 			DRM_ERROR("Failed to load firmware!\n");
7189 			return r;
7190 		}
7191 	}
7192 
7193 	/* Initialize power management */
7194 	radeon_pm_init(rdev);
7195 
7196 	ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
7197 	ring->ring_obj = NULL;
7198 	r600_ring_init(rdev, ring, 1024 * 1024);
7199 
7200 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
7201 	ring->ring_obj = NULL;
7202 	r600_ring_init(rdev, ring, 1024 * 1024);
7203 
7204 	ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
7205 	ring->ring_obj = NULL;
7206 	r600_ring_init(rdev, ring, 1024 * 1024);
7207 
7208 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
7209 	ring->ring_obj = NULL;
7210 	r600_ring_init(rdev, ring, 64 * 1024);
7211 
7212 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
7213 	ring->ring_obj = NULL;
7214 	r600_ring_init(rdev, ring, 64 * 1024);
7215 
7216 	if (rdev->has_uvd) {
7217 		r = radeon_uvd_init(rdev);
7218 		if (!r) {
7219 			ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
7220 			ring->ring_obj = NULL;
7221 			r600_ring_init(rdev, ring, 4096);
7222 		}
7223 	}
7224 
7225 	r = radeon_vce_init(rdev);
7226 	if (!r) {
7227 		ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
7228 		ring->ring_obj = NULL;
7229 		r600_ring_init(rdev, ring, 4096);
7230 
7231 		ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
7232 		ring->ring_obj = NULL;
7233 		r600_ring_init(rdev, ring, 4096);
7234 	}
7235 
7236 	rdev->ih.ring_obj = NULL;
7237 	r600_ih_ring_init(rdev, 64 * 1024);
7238 
7239 	r = r600_pcie_gart_init(rdev);
7240 	if (r)
7241 		return r;
7242 
7243 	rdev->accel_working = true;
7244 	r = si_startup(rdev);
7245 	if (r) {
7246 		dev_err(rdev->dev, "disabling GPU acceleration\n");
7247 		si_cp_fini(rdev);
7248 		cayman_dma_fini(rdev);
7249 		si_irq_fini(rdev);
7250 		sumo_rlc_fini(rdev);
7251 		radeon_wb_fini(rdev);
7252 		radeon_ib_pool_fini(rdev);
7253 		radeon_vm_manager_fini(rdev);
7254 		radeon_irq_kms_fini(rdev);
7255 		si_pcie_gart_fini(rdev);
7256 		rdev->accel_working = false;
7257 	}
7258 
7259 	/* Don't start up if the MC ucode is missing.
7260 	 * The default clocks and voltages before the MC ucode
7261 	 * is loaded are not suffient for advanced operations.
7262 	 */
7263 	if (!rdev->mc_fw) {
7264 		DRM_ERROR("radeon: MC ucode required for NI+.\n");
7265 		return -EINVAL;
7266 	}
7267 
7268 	return 0;
7269 }
7270 
7271 void si_fini(struct radeon_device *rdev)
7272 {
7273 	radeon_pm_fini(rdev);
7274 	si_cp_fini(rdev);
7275 	cayman_dma_fini(rdev);
7276 	si_fini_pg(rdev);
7277 	si_fini_cg(rdev);
7278 	si_irq_fini(rdev);
7279 	sumo_rlc_fini(rdev);
7280 	radeon_wb_fini(rdev);
7281 	radeon_vm_manager_fini(rdev);
7282 	radeon_ib_pool_fini(rdev);
7283 	radeon_irq_kms_fini(rdev);
7284 	if (rdev->has_uvd) {
7285 		uvd_v1_0_fini(rdev);
7286 		radeon_uvd_fini(rdev);
7287 		radeon_vce_fini(rdev);
7288 	}
7289 	si_pcie_gart_fini(rdev);
7290 	r600_vram_scratch_fini(rdev);
7291 	radeon_gem_fini(rdev);
7292 	radeon_fence_driver_fini(rdev);
7293 	radeon_bo_fini(rdev);
7294 	radeon_atombios_fini(rdev);
7295 	kfree(rdev->bios);
7296 	rdev->bios = NULL;
7297 }
7298 
7299 /**
7300  * si_get_gpu_clock_counter - return GPU clock counter snapshot
7301  *
7302  * @rdev: radeon_device pointer
7303  *
7304  * Fetches a GPU clock counter snapshot (SI).
7305  * Returns the 64 bit clock counter snapshot.
7306  */
7307 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
7308 {
7309 	uint64_t clock;
7310 
7311 	mutex_lock(&rdev->gpu_clock_mutex);
7312 	WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
7313 	clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
7314 	        ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
7315 	mutex_unlock(&rdev->gpu_clock_mutex);
7316 	return clock;
7317 }
7318 
7319 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
7320 {
7321 	unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
7322 	int r;
7323 
7324 	/* bypass vclk and dclk with bclk */
7325 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7326 		VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
7327 		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7328 
7329 	/* put PLL in bypass mode */
7330 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
7331 
7332 	if (!vclk || !dclk) {
7333 		/* keep the Bypass mode */
7334 		return 0;
7335 	}
7336 
7337 	r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
7338 					  16384, 0x03FFFFFF, 0, 128, 5,
7339 					  &fb_div, &vclk_div, &dclk_div);
7340 	if (r)
7341 		return r;
7342 
7343 	/* set RESET_ANTI_MUX to 0 */
7344 	WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7345 
7346 	/* set VCO_MODE to 1 */
7347 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
7348 
7349 	/* disable sleep mode */
7350 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
7351 
7352 	/* deassert UPLL_RESET */
7353 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7354 
7355 	mdelay(1);
7356 
7357 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7358 	if (r)
7359 		return r;
7360 
7361 	/* assert UPLL_RESET again */
7362 	WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
7363 
7364 	/* disable spread spectrum. */
7365 	WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7366 
7367 	/* set feedback divider */
7368 	WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
7369 
7370 	/* set ref divider to 0 */
7371 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
7372 
7373 	if (fb_div < 307200)
7374 		WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
7375 	else
7376 		WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
7377 
7378 	/* set PDIV_A and PDIV_B */
7379 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7380 		UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
7381 		~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
7382 
7383 	/* give the PLL some time to settle */
7384 	mdelay(15);
7385 
7386 	/* deassert PLL_RESET */
7387 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
7388 
7389 	mdelay(15);
7390 
7391 	/* switch from bypass mode to normal mode */
7392 	WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
7393 
7394 	r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
7395 	if (r)
7396 		return r;
7397 
7398 	/* switch VCLK and DCLK selection */
7399 	WREG32_P(CG_UPLL_FUNC_CNTL_2,
7400 		VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
7401 		~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
7402 
7403 	mdelay(100);
7404 
7405 	return 0;
7406 }
7407 
7408 static void si_pcie_gen3_enable(struct radeon_device *rdev)
7409 {
7410 	struct pci_dev *root = rdev->pdev->bus->self;
7411 	int bridge_pos, gpu_pos;
7412 	u32 speed_cntl, mask, current_data_rate;
7413 	int ret, i;
7414 	u16 tmp16;
7415 
7416 	if (pci_is_root_bus(rdev->pdev->bus))
7417 		return;
7418 
7419 	if (radeon_pcie_gen2 == 0)
7420 		return;
7421 
7422 	if (rdev->flags & RADEON_IS_IGP)
7423 		return;
7424 
7425 	if (!(rdev->flags & RADEON_IS_PCIE))
7426 		return;
7427 
7428 	ret = drm_pcie_get_speed_cap_mask(rdev->ddev, &mask);
7429 	if (ret != 0)
7430 		return;
7431 
7432 	if (!(mask & (DRM_PCIE_SPEED_50 | DRM_PCIE_SPEED_80)))
7433 		return;
7434 
7435 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7436 	current_data_rate = (speed_cntl & LC_CURRENT_DATA_RATE_MASK) >>
7437 		LC_CURRENT_DATA_RATE_SHIFT;
7438 	if (mask & DRM_PCIE_SPEED_80) {
7439 		if (current_data_rate == 2) {
7440 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
7441 			return;
7442 		}
7443 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with radeon.pcie_gen2=0\n");
7444 	} else if (mask & DRM_PCIE_SPEED_50) {
7445 		if (current_data_rate == 1) {
7446 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
7447 			return;
7448 		}
7449 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
7450 	}
7451 
7452 	bridge_pos = pci_pcie_cap(root);
7453 	if (!bridge_pos)
7454 		return;
7455 
7456 	gpu_pos = pci_pcie_cap(rdev->pdev);
7457 	if (!gpu_pos)
7458 		return;
7459 
7460 	if (mask & DRM_PCIE_SPEED_80) {
7461 		/* re-try equalization if gen3 is not already enabled */
7462 		if (current_data_rate != 2) {
7463 			u16 bridge_cfg, gpu_cfg;
7464 			u16 bridge_cfg2, gpu_cfg2;
7465 			u32 max_lw, current_lw, tmp;
7466 
7467 			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7468 			pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7469 
7470 			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
7471 			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7472 
7473 			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
7474 			pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7475 
7476 			tmp = RREG32_PCIE(PCIE_LC_STATUS1);
7477 			max_lw = (tmp & LC_DETECTED_LINK_WIDTH_MASK) >> LC_DETECTED_LINK_WIDTH_SHIFT;
7478 			current_lw = (tmp & LC_OPERATING_LINK_WIDTH_MASK) >> LC_OPERATING_LINK_WIDTH_SHIFT;
7479 
7480 			if (current_lw < max_lw) {
7481 				tmp = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7482 				if (tmp & LC_RENEGOTIATION_SUPPORT) {
7483 					tmp &= ~(LC_LINK_WIDTH_MASK | LC_UPCONFIGURE_DIS);
7484 					tmp |= (max_lw << LC_LINK_WIDTH_SHIFT);
7485 					tmp |= LC_UPCONFIGURE_SUPPORT | LC_RENEGOTIATE_EN | LC_RECONFIG_NOW;
7486 					WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, tmp);
7487 				}
7488 			}
7489 
7490 			for (i = 0; i < 10; i++) {
7491 				/* check status */
7492 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
7493 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
7494 					break;
7495 
7496 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
7497 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
7498 
7499 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
7500 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
7501 
7502 				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7503 				tmp |= LC_SET_QUIESCE;
7504 				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7505 
7506 				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7507 				tmp |= LC_REDO_EQ;
7508 				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7509 
7510 				mdelay(100);
7511 
7512 				/* linkctl */
7513 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
7514 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7515 				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
7516 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
7517 
7518 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
7519 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
7520 				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
7521 				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
7522 
7523 				/* linkctl2 */
7524 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
7525 				tmp16 &= ~((1 << 4) | (7 << 9));
7526 				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
7527 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
7528 
7529 				pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7530 				tmp16 &= ~((1 << 4) | (7 << 9));
7531 				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
7532 				pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7533 
7534 				tmp = RREG32_PCIE_PORT(PCIE_LC_CNTL4);
7535 				tmp &= ~LC_SET_QUIESCE;
7536 				WREG32_PCIE_PORT(PCIE_LC_CNTL4, tmp);
7537 			}
7538 		}
7539 	}
7540 
7541 	/* set the link speed */
7542 	speed_cntl |= LC_FORCE_EN_SW_SPEED_CHANGE | LC_FORCE_DIS_HW_SPEED_CHANGE;
7543 	speed_cntl &= ~LC_FORCE_DIS_SW_SPEED_CHANGE;
7544 	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7545 
7546 	pci_read_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
7547 	tmp16 &= ~0xf;
7548 	if (mask & DRM_PCIE_SPEED_80)
7549 		tmp16 |= 3; /* gen3 */
7550 	else if (mask & DRM_PCIE_SPEED_50)
7551 		tmp16 |= 2; /* gen2 */
7552 	else
7553 		tmp16 |= 1; /* gen1 */
7554 	pci_write_config_word(rdev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
7555 
7556 	speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7557 	speed_cntl |= LC_INITIATE_LINK_SPEED_CHANGE;
7558 	WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
7559 
7560 	for (i = 0; i < rdev->usec_timeout; i++) {
7561 		speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
7562 		if ((speed_cntl & LC_INITIATE_LINK_SPEED_CHANGE) == 0)
7563 			break;
7564 		udelay(1);
7565 	}
7566 }
7567 
7568 static void si_program_aspm(struct radeon_device *rdev)
7569 {
7570 	u32 data, orig;
7571 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
7572 	bool disable_clkreq = false;
7573 
7574 	if (radeon_aspm == 0)
7575 		return;
7576 
7577 	if (!(rdev->flags & RADEON_IS_PCIE))
7578 		return;
7579 
7580 	orig = data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7581 	data &= ~LC_XMIT_N_FTS_MASK;
7582 	data |= LC_XMIT_N_FTS(0x24) | LC_XMIT_N_FTS_OVERRIDE_EN;
7583 	if (orig != data)
7584 		WREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL, data);
7585 
7586 	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL3);
7587 	data |= LC_GO_TO_RECOVERY;
7588 	if (orig != data)
7589 		WREG32_PCIE_PORT(PCIE_LC_CNTL3, data);
7590 
7591 	orig = data = RREG32_PCIE(PCIE_P_CNTL);
7592 	data |= P_IGNORE_EDB_ERR;
7593 	if (orig != data)
7594 		WREG32_PCIE(PCIE_P_CNTL, data);
7595 
7596 	orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7597 	data &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
7598 	data |= LC_PMI_TO_L1_DIS;
7599 	if (!disable_l0s)
7600 		data |= LC_L0S_INACTIVITY(7);
7601 
7602 	if (!disable_l1) {
7603 		data |= LC_L1_INACTIVITY(7);
7604 		data &= ~LC_PMI_TO_L1_DIS;
7605 		if (orig != data)
7606 			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7607 
7608 		if (!disable_plloff_in_l1) {
7609 			bool clk_req_support;
7610 
7611 			orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7612 			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7613 			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7614 			if (orig != data)
7615 				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7616 
7617 			orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7618 			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7619 			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7620 			if (orig != data)
7621 				WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7622 
7623 			orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7624 			data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
7625 			data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
7626 			if (orig != data)
7627 				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7628 
7629 			orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7630 			data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
7631 			data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
7632 			if (orig != data)
7633 				WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7634 
7635 			if ((rdev->family != CHIP_OLAND) && (rdev->family != CHIP_HAINAN)) {
7636 				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
7637 				data &= ~PLL_RAMP_UP_TIME_0_MASK;
7638 				if (orig != data)
7639 					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
7640 
7641 				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
7642 				data &= ~PLL_RAMP_UP_TIME_1_MASK;
7643 				if (orig != data)
7644 					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
7645 
7646 				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2);
7647 				data &= ~PLL_RAMP_UP_TIME_2_MASK;
7648 				if (orig != data)
7649 					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_2, data);
7650 
7651 				orig = data = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3);
7652 				data &= ~PLL_RAMP_UP_TIME_3_MASK;
7653 				if (orig != data)
7654 					WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_3, data);
7655 
7656 				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
7657 				data &= ~PLL_RAMP_UP_TIME_0_MASK;
7658 				if (orig != data)
7659 					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
7660 
7661 				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
7662 				data &= ~PLL_RAMP_UP_TIME_1_MASK;
7663 				if (orig != data)
7664 					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
7665 
7666 				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2);
7667 				data &= ~PLL_RAMP_UP_TIME_2_MASK;
7668 				if (orig != data)
7669 					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_2, data);
7670 
7671 				orig = data = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3);
7672 				data &= ~PLL_RAMP_UP_TIME_3_MASK;
7673 				if (orig != data)
7674 					WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_3, data);
7675 			}
7676 			orig = data = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
7677 			data &= ~LC_DYN_LANES_PWR_STATE_MASK;
7678 			data |= LC_DYN_LANES_PWR_STATE(3);
7679 			if (orig != data)
7680 				WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
7681 
7682 			orig = data = RREG32_PIF_PHY0(PB0_PIF_CNTL);
7683 			data &= ~LS2_EXIT_TIME_MASK;
7684 			if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7685 				data |= LS2_EXIT_TIME(5);
7686 			if (orig != data)
7687 				WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
7688 
7689 			orig = data = RREG32_PIF_PHY1(PB1_PIF_CNTL);
7690 			data &= ~LS2_EXIT_TIME_MASK;
7691 			if ((rdev->family == CHIP_OLAND) || (rdev->family == CHIP_HAINAN))
7692 				data |= LS2_EXIT_TIME(5);
7693 			if (orig != data)
7694 				WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
7695 
7696 			if (!disable_clkreq &&
7697 			    !pci_is_root_bus(rdev->pdev->bus)) {
7698 				struct pci_dev *root = rdev->pdev->bus->self;
7699 				u32 lnkcap;
7700 
7701 				clk_req_support = false;
7702 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
7703 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
7704 					clk_req_support = true;
7705 			} else {
7706 				clk_req_support = false;
7707 			}
7708 
7709 			if (clk_req_support) {
7710 				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL2);
7711 				data |= LC_ALLOW_PDWN_IN_L1 | LC_ALLOW_PDWN_IN_L23;
7712 				if (orig != data)
7713 					WREG32_PCIE_PORT(PCIE_LC_CNTL2, data);
7714 
7715 				orig = data = RREG32(THM_CLK_CNTL);
7716 				data &= ~(CMON_CLK_SEL_MASK | TMON_CLK_SEL_MASK);
7717 				data |= CMON_CLK_SEL(1) | TMON_CLK_SEL(1);
7718 				if (orig != data)
7719 					WREG32(THM_CLK_CNTL, data);
7720 
7721 				orig = data = RREG32(MISC_CLK_CNTL);
7722 				data &= ~(DEEP_SLEEP_CLK_SEL_MASK | ZCLK_SEL_MASK);
7723 				data |= DEEP_SLEEP_CLK_SEL(1) | ZCLK_SEL(1);
7724 				if (orig != data)
7725 					WREG32(MISC_CLK_CNTL, data);
7726 
7727 				orig = data = RREG32(CG_CLKPIN_CNTL);
7728 				data &= ~BCLK_AS_XCLK;
7729 				if (orig != data)
7730 					WREG32(CG_CLKPIN_CNTL, data);
7731 
7732 				orig = data = RREG32(CG_CLKPIN_CNTL_2);
7733 				data &= ~FORCE_BIF_REFCLK_EN;
7734 				if (orig != data)
7735 					WREG32(CG_CLKPIN_CNTL_2, data);
7736 
7737 				orig = data = RREG32(MPLL_BYPASSCLK_SEL);
7738 				data &= ~MPLL_CLKOUT_SEL_MASK;
7739 				data |= MPLL_CLKOUT_SEL(4);
7740 				if (orig != data)
7741 					WREG32(MPLL_BYPASSCLK_SEL, data);
7742 
7743 				orig = data = RREG32(SPLL_CNTL_MODE);
7744 				data &= ~SPLL_REFCLK_SEL_MASK;
7745 				if (orig != data)
7746 					WREG32(SPLL_CNTL_MODE, data);
7747 			}
7748 		}
7749 	} else {
7750 		if (orig != data)
7751 			WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7752 	}
7753 
7754 	orig = data = RREG32_PCIE(PCIE_CNTL2);
7755 	data |= SLV_MEM_LS_EN | MST_MEM_LS_EN | REPLAY_MEM_LS_EN;
7756 	if (orig != data)
7757 		WREG32_PCIE(PCIE_CNTL2, data);
7758 
7759 	if (!disable_l0s) {
7760 		data = RREG32_PCIE_PORT(PCIE_LC_N_FTS_CNTL);
7761 		if((data & LC_N_FTS_MASK) == LC_N_FTS_MASK) {
7762 			data = RREG32_PCIE(PCIE_LC_STATUS1);
7763 			if ((data & LC_REVERSE_XMIT) && (data & LC_REVERSE_RCVR)) {
7764 				orig = data = RREG32_PCIE_PORT(PCIE_LC_CNTL);
7765 				data &= ~LC_L0S_INACTIVITY_MASK;
7766 				if (orig != data)
7767 					WREG32_PCIE_PORT(PCIE_LC_CNTL, data);
7768 			}
7769 		}
7770 	}
7771 }
7772 
7773 int si_vce_send_vcepll_ctlreq(struct radeon_device *rdev)
7774 {
7775         unsigned i;
7776 
7777         /* make sure VCEPLL_CTLREQ is deasserted */
7778         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7779 
7780         mdelay(10);
7781 
7782         /* assert UPLL_CTLREQ */
7783         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, UPLL_CTLREQ_MASK, ~UPLL_CTLREQ_MASK);
7784 
7785         /* wait for CTLACK and CTLACK2 to get asserted */
7786         for (i = 0; i < 100; ++i) {
7787                 uint32_t mask = UPLL_CTLACK_MASK | UPLL_CTLACK2_MASK;
7788                 if ((RREG32_SMC(CG_VCEPLL_FUNC_CNTL) & mask) == mask)
7789                         break;
7790                 mdelay(10);
7791         }
7792 
7793         /* deassert UPLL_CTLREQ */
7794         WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~UPLL_CTLREQ_MASK);
7795 
7796         if (i == 100) {
7797                 DRM_ERROR("Timeout setting UVD clocks!\n");
7798                 return -ETIMEDOUT;
7799         }
7800 
7801         return 0;
7802 }
7803 
7804 int si_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
7805 {
7806 	unsigned fb_div = 0, evclk_div = 0, ecclk_div = 0;
7807 	int r;
7808 
7809 	/* bypass evclk and ecclk with bclk */
7810 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7811 		     EVCLK_SRC_SEL(1) | ECCLK_SRC_SEL(1),
7812 		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7813 
7814 	/* put PLL in bypass mode */
7815 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_BYPASS_EN_MASK,
7816 		     ~VCEPLL_BYPASS_EN_MASK);
7817 
7818 	if (!evclk || !ecclk) {
7819 		/* keep the Bypass mode, put PLL to sleep */
7820 		WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7821 			     ~VCEPLL_SLEEP_MASK);
7822 		return 0;
7823 	}
7824 
7825 	r = radeon_uvd_calc_upll_dividers(rdev, evclk, ecclk, 125000, 250000,
7826 					  16384, 0x03FFFFFF, 0, 128, 5,
7827 					  &fb_div, &evclk_div, &ecclk_div);
7828 	if (r)
7829 		return r;
7830 
7831 	/* set RESET_ANTI_MUX to 0 */
7832 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
7833 
7834 	/* set VCO_MODE to 1 */
7835 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_VCO_MODE_MASK,
7836 		     ~VCEPLL_VCO_MODE_MASK);
7837 
7838 	/* toggle VCEPLL_SLEEP to 1 then back to 0 */
7839 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_SLEEP_MASK,
7840 		     ~VCEPLL_SLEEP_MASK);
7841 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_SLEEP_MASK);
7842 
7843 	/* deassert VCEPLL_RESET */
7844 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7845 
7846 	mdelay(1);
7847 
7848 	r = si_vce_send_vcepll_ctlreq(rdev);
7849 	if (r)
7850 		return r;
7851 
7852 	/* assert VCEPLL_RESET again */
7853 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, VCEPLL_RESET_MASK, ~VCEPLL_RESET_MASK);
7854 
7855 	/* disable spread spectrum. */
7856 	WREG32_SMC_P(CG_VCEPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
7857 
7858 	/* set feedback divider */
7859 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_3, VCEPLL_FB_DIV(fb_div), ~VCEPLL_FB_DIV_MASK);
7860 
7861 	/* set ref divider to 0 */
7862 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_REF_DIV_MASK);
7863 
7864 	/* set PDIV_A and PDIV_B */
7865 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7866 		     VCEPLL_PDIV_A(evclk_div) | VCEPLL_PDIV_B(ecclk_div),
7867 		     ~(VCEPLL_PDIV_A_MASK | VCEPLL_PDIV_B_MASK));
7868 
7869 	/* give the PLL some time to settle */
7870 	mdelay(15);
7871 
7872 	/* deassert PLL_RESET */
7873 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_RESET_MASK);
7874 
7875 	mdelay(15);
7876 
7877 	/* switch from bypass mode to normal mode */
7878 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL, 0, ~VCEPLL_BYPASS_EN_MASK);
7879 
7880 	r = si_vce_send_vcepll_ctlreq(rdev);
7881 	if (r)
7882 		return r;
7883 
7884 	/* switch VCLK and DCLK selection */
7885 	WREG32_SMC_P(CG_VCEPLL_FUNC_CNTL_2,
7886 		     EVCLK_SRC_SEL(16) | ECCLK_SRC_SEL(16),
7887 		     ~(EVCLK_SRC_SEL_MASK | ECCLK_SRC_SEL_MASK));
7888 
7889 	mdelay(100);
7890 
7891 	return 0;
7892 }
7893