xref: /openbmc/linux/drivers/gpu/drm/radeon/ni.c (revision 6774def6)
1 /*
2  * Copyright 2010 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <drm/drmP.h>
28 #include "radeon.h"
29 #include "radeon_asic.h"
30 #include <drm/radeon_drm.h>
31 #include "nid.h"
32 #include "atom.h"
33 #include "ni_reg.h"
34 #include "cayman_blit_shaders.h"
35 #include "radeon_ucode.h"
36 #include "clearstate_cayman.h"
37 
38 static const u32 tn_rlc_save_restore_register_list[] =
39 {
40 	0x98fc,
41 	0x98f0,
42 	0x9834,
43 	0x9838,
44 	0x9870,
45 	0x9874,
46 	0x8a14,
47 	0x8b24,
48 	0x8bcc,
49 	0x8b10,
50 	0x8c30,
51 	0x8d00,
52 	0x8d04,
53 	0x8c00,
54 	0x8c04,
55 	0x8c10,
56 	0x8c14,
57 	0x8d8c,
58 	0x8cf0,
59 	0x8e38,
60 	0x9508,
61 	0x9688,
62 	0x9608,
63 	0x960c,
64 	0x9610,
65 	0x9614,
66 	0x88c4,
67 	0x8978,
68 	0x88d4,
69 	0x900c,
70 	0x9100,
71 	0x913c,
72 	0x90e8,
73 	0x9354,
74 	0xa008,
75 	0x98f8,
76 	0x9148,
77 	0x914c,
78 	0x3f94,
79 	0x98f4,
80 	0x9b7c,
81 	0x3f8c,
82 	0x8950,
83 	0x8954,
84 	0x8a18,
85 	0x8b28,
86 	0x9144,
87 	0x3f90,
88 	0x915c,
89 	0x9160,
90 	0x9178,
91 	0x917c,
92 	0x9180,
93 	0x918c,
94 	0x9190,
95 	0x9194,
96 	0x9198,
97 	0x919c,
98 	0x91a8,
99 	0x91ac,
100 	0x91b0,
101 	0x91b4,
102 	0x91b8,
103 	0x91c4,
104 	0x91c8,
105 	0x91cc,
106 	0x91d0,
107 	0x91d4,
108 	0x91e0,
109 	0x91e4,
110 	0x91ec,
111 	0x91f0,
112 	0x91f4,
113 	0x9200,
114 	0x9204,
115 	0x929c,
116 	0x8030,
117 	0x9150,
118 	0x9a60,
119 	0x920c,
120 	0x9210,
121 	0x9228,
122 	0x922c,
123 	0x9244,
124 	0x9248,
125 	0x91e8,
126 	0x9294,
127 	0x9208,
128 	0x9224,
129 	0x9240,
130 	0x9220,
131 	0x923c,
132 	0x9258,
133 	0x9744,
134 	0xa200,
135 	0xa204,
136 	0xa208,
137 	0xa20c,
138 	0x8d58,
139 	0x9030,
140 	0x9034,
141 	0x9038,
142 	0x903c,
143 	0x9040,
144 	0x9654,
145 	0x897c,
146 	0xa210,
147 	0xa214,
148 	0x9868,
149 	0xa02c,
150 	0x9664,
151 	0x9698,
152 	0x949c,
153 	0x8e10,
154 	0x8e18,
155 	0x8c50,
156 	0x8c58,
157 	0x8c60,
158 	0x8c68,
159 	0x89b4,
160 	0x9830,
161 	0x802c,
162 };
163 
164 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
165 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
166 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
167 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
168 extern int evergreen_mc_wait_for_idle(struct radeon_device *rdev);
169 extern void evergreen_mc_program(struct radeon_device *rdev);
170 extern void evergreen_irq_suspend(struct radeon_device *rdev);
171 extern int evergreen_mc_init(struct radeon_device *rdev);
172 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
173 extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
174 extern void evergreen_program_aspm(struct radeon_device *rdev);
175 extern void sumo_rlc_fini(struct radeon_device *rdev);
176 extern int sumo_rlc_init(struct radeon_device *rdev);
177 extern void evergreen_gpu_pci_config_reset(struct radeon_device *rdev);
178 
179 /* Firmware Names */
180 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
181 MODULE_FIRMWARE("radeon/BARTS_me.bin");
182 MODULE_FIRMWARE("radeon/BARTS_mc.bin");
183 MODULE_FIRMWARE("radeon/BARTS_smc.bin");
184 MODULE_FIRMWARE("radeon/BTC_rlc.bin");
185 MODULE_FIRMWARE("radeon/TURKS_pfp.bin");
186 MODULE_FIRMWARE("radeon/TURKS_me.bin");
187 MODULE_FIRMWARE("radeon/TURKS_mc.bin");
188 MODULE_FIRMWARE("radeon/TURKS_smc.bin");
189 MODULE_FIRMWARE("radeon/CAICOS_pfp.bin");
190 MODULE_FIRMWARE("radeon/CAICOS_me.bin");
191 MODULE_FIRMWARE("radeon/CAICOS_mc.bin");
192 MODULE_FIRMWARE("radeon/CAICOS_smc.bin");
193 MODULE_FIRMWARE("radeon/CAYMAN_pfp.bin");
194 MODULE_FIRMWARE("radeon/CAYMAN_me.bin");
195 MODULE_FIRMWARE("radeon/CAYMAN_mc.bin");
196 MODULE_FIRMWARE("radeon/CAYMAN_rlc.bin");
197 MODULE_FIRMWARE("radeon/CAYMAN_smc.bin");
198 MODULE_FIRMWARE("radeon/ARUBA_pfp.bin");
199 MODULE_FIRMWARE("radeon/ARUBA_me.bin");
200 MODULE_FIRMWARE("radeon/ARUBA_rlc.bin");
201 
202 
203 static const u32 cayman_golden_registers2[] =
204 {
205 	0x3e5c, 0xffffffff, 0x00000000,
206 	0x3e48, 0xffffffff, 0x00000000,
207 	0x3e4c, 0xffffffff, 0x00000000,
208 	0x3e64, 0xffffffff, 0x00000000,
209 	0x3e50, 0xffffffff, 0x00000000,
210 	0x3e60, 0xffffffff, 0x00000000
211 };
212 
213 static const u32 cayman_golden_registers[] =
214 {
215 	0x5eb4, 0xffffffff, 0x00000002,
216 	0x5e78, 0x8f311ff1, 0x001000f0,
217 	0x3f90, 0xffff0000, 0xff000000,
218 	0x9148, 0xffff0000, 0xff000000,
219 	0x3f94, 0xffff0000, 0xff000000,
220 	0x914c, 0xffff0000, 0xff000000,
221 	0xc78, 0x00000080, 0x00000080,
222 	0xbd4, 0x70073777, 0x00011003,
223 	0xd02c, 0xbfffff1f, 0x08421000,
224 	0xd0b8, 0x73773777, 0x02011003,
225 	0x5bc0, 0x00200000, 0x50100000,
226 	0x98f8, 0x33773777, 0x02011003,
227 	0x98fc, 0xffffffff, 0x76541032,
228 	0x7030, 0x31000311, 0x00000011,
229 	0x2f48, 0x33773777, 0x42010001,
230 	0x6b28, 0x00000010, 0x00000012,
231 	0x7728, 0x00000010, 0x00000012,
232 	0x10328, 0x00000010, 0x00000012,
233 	0x10f28, 0x00000010, 0x00000012,
234 	0x11b28, 0x00000010, 0x00000012,
235 	0x12728, 0x00000010, 0x00000012,
236 	0x240c, 0x000007ff, 0x00000000,
237 	0x8a14, 0xf000001f, 0x00000007,
238 	0x8b24, 0x3fff3fff, 0x00ff0fff,
239 	0x8b10, 0x0000ff0f, 0x00000000,
240 	0x28a4c, 0x07ffffff, 0x06000000,
241 	0x10c, 0x00000001, 0x00010003,
242 	0xa02c, 0xffffffff, 0x0000009b,
243 	0x913c, 0x0000010f, 0x01000100,
244 	0x8c04, 0xf8ff00ff, 0x40600060,
245 	0x28350, 0x00000f01, 0x00000000,
246 	0x9508, 0x3700001f, 0x00000002,
247 	0x960c, 0xffffffff, 0x54763210,
248 	0x88c4, 0x001f3ae3, 0x00000082,
249 	0x88d0, 0xffffffff, 0x0f40df40,
250 	0x88d4, 0x0000001f, 0x00000010,
251 	0x8974, 0xffffffff, 0x00000000
252 };
253 
254 static const u32 dvst_golden_registers2[] =
255 {
256 	0x8f8, 0xffffffff, 0,
257 	0x8fc, 0x00380000, 0,
258 	0x8f8, 0xffffffff, 1,
259 	0x8fc, 0x0e000000, 0
260 };
261 
262 static const u32 dvst_golden_registers[] =
263 {
264 	0x690, 0x3fff3fff, 0x20c00033,
265 	0x918c, 0x0fff0fff, 0x00010006,
266 	0x91a8, 0x0fff0fff, 0x00010006,
267 	0x9150, 0xffffdfff, 0x6e944040,
268 	0x917c, 0x0fff0fff, 0x00030002,
269 	0x9198, 0x0fff0fff, 0x00030002,
270 	0x915c, 0x0fff0fff, 0x00010000,
271 	0x3f90, 0xffff0001, 0xff000000,
272 	0x9178, 0x0fff0fff, 0x00070000,
273 	0x9194, 0x0fff0fff, 0x00070000,
274 	0x9148, 0xffff0001, 0xff000000,
275 	0x9190, 0x0fff0fff, 0x00090008,
276 	0x91ac, 0x0fff0fff, 0x00090008,
277 	0x3f94, 0xffff0000, 0xff000000,
278 	0x914c, 0xffff0000, 0xff000000,
279 	0x929c, 0x00000fff, 0x00000001,
280 	0x55e4, 0xff607fff, 0xfc000100,
281 	0x8a18, 0xff000fff, 0x00000100,
282 	0x8b28, 0xff000fff, 0x00000100,
283 	0x9144, 0xfffc0fff, 0x00000100,
284 	0x6ed8, 0x00010101, 0x00010000,
285 	0x9830, 0xffffffff, 0x00000000,
286 	0x9834, 0xf00fffff, 0x00000400,
287 	0x9838, 0xfffffffe, 0x00000000,
288 	0xd0c0, 0xff000fff, 0x00000100,
289 	0xd02c, 0xbfffff1f, 0x08421000,
290 	0xd0b8, 0x73773777, 0x12010001,
291 	0x5bb0, 0x000000f0, 0x00000070,
292 	0x98f8, 0x73773777, 0x12010001,
293 	0x98fc, 0xffffffff, 0x00000010,
294 	0x9b7c, 0x00ff0000, 0x00fc0000,
295 	0x8030, 0x00001f0f, 0x0000100a,
296 	0x2f48, 0x73773777, 0x12010001,
297 	0x2408, 0x00030000, 0x000c007f,
298 	0x8a14, 0xf000003f, 0x00000007,
299 	0x8b24, 0x3fff3fff, 0x00ff0fff,
300 	0x8b10, 0x0000ff0f, 0x00000000,
301 	0x28a4c, 0x07ffffff, 0x06000000,
302 	0x4d8, 0x00000fff, 0x00000100,
303 	0xa008, 0xffffffff, 0x00010000,
304 	0x913c, 0xffff03ff, 0x01000100,
305 	0x8c00, 0x000000ff, 0x00000003,
306 	0x8c04, 0xf8ff00ff, 0x40600060,
307 	0x8cf0, 0x1fff1fff, 0x08e00410,
308 	0x28350, 0x00000f01, 0x00000000,
309 	0x9508, 0xf700071f, 0x00000002,
310 	0x960c, 0xffffffff, 0x54763210,
311 	0x20ef8, 0x01ff01ff, 0x00000002,
312 	0x20e98, 0xfffffbff, 0x00200000,
313 	0x2015c, 0xffffffff, 0x00000f40,
314 	0x88c4, 0x001f3ae3, 0x00000082,
315 	0x8978, 0x3fffffff, 0x04050140,
316 	0x88d4, 0x0000001f, 0x00000010,
317 	0x8974, 0xffffffff, 0x00000000
318 };
319 
320 static const u32 scrapper_golden_registers[] =
321 {
322 	0x690, 0x3fff3fff, 0x20c00033,
323 	0x918c, 0x0fff0fff, 0x00010006,
324 	0x918c, 0x0fff0fff, 0x00010006,
325 	0x91a8, 0x0fff0fff, 0x00010006,
326 	0x91a8, 0x0fff0fff, 0x00010006,
327 	0x9150, 0xffffdfff, 0x6e944040,
328 	0x9150, 0xffffdfff, 0x6e944040,
329 	0x917c, 0x0fff0fff, 0x00030002,
330 	0x917c, 0x0fff0fff, 0x00030002,
331 	0x9198, 0x0fff0fff, 0x00030002,
332 	0x9198, 0x0fff0fff, 0x00030002,
333 	0x915c, 0x0fff0fff, 0x00010000,
334 	0x915c, 0x0fff0fff, 0x00010000,
335 	0x3f90, 0xffff0001, 0xff000000,
336 	0x3f90, 0xffff0001, 0xff000000,
337 	0x9178, 0x0fff0fff, 0x00070000,
338 	0x9178, 0x0fff0fff, 0x00070000,
339 	0x9194, 0x0fff0fff, 0x00070000,
340 	0x9194, 0x0fff0fff, 0x00070000,
341 	0x9148, 0xffff0001, 0xff000000,
342 	0x9148, 0xffff0001, 0xff000000,
343 	0x9190, 0x0fff0fff, 0x00090008,
344 	0x9190, 0x0fff0fff, 0x00090008,
345 	0x91ac, 0x0fff0fff, 0x00090008,
346 	0x91ac, 0x0fff0fff, 0x00090008,
347 	0x3f94, 0xffff0000, 0xff000000,
348 	0x3f94, 0xffff0000, 0xff000000,
349 	0x914c, 0xffff0000, 0xff000000,
350 	0x914c, 0xffff0000, 0xff000000,
351 	0x929c, 0x00000fff, 0x00000001,
352 	0x929c, 0x00000fff, 0x00000001,
353 	0x55e4, 0xff607fff, 0xfc000100,
354 	0x8a18, 0xff000fff, 0x00000100,
355 	0x8a18, 0xff000fff, 0x00000100,
356 	0x8b28, 0xff000fff, 0x00000100,
357 	0x8b28, 0xff000fff, 0x00000100,
358 	0x9144, 0xfffc0fff, 0x00000100,
359 	0x9144, 0xfffc0fff, 0x00000100,
360 	0x6ed8, 0x00010101, 0x00010000,
361 	0x9830, 0xffffffff, 0x00000000,
362 	0x9830, 0xffffffff, 0x00000000,
363 	0x9834, 0xf00fffff, 0x00000400,
364 	0x9834, 0xf00fffff, 0x00000400,
365 	0x9838, 0xfffffffe, 0x00000000,
366 	0x9838, 0xfffffffe, 0x00000000,
367 	0xd0c0, 0xff000fff, 0x00000100,
368 	0xd02c, 0xbfffff1f, 0x08421000,
369 	0xd02c, 0xbfffff1f, 0x08421000,
370 	0xd0b8, 0x73773777, 0x12010001,
371 	0xd0b8, 0x73773777, 0x12010001,
372 	0x5bb0, 0x000000f0, 0x00000070,
373 	0x98f8, 0x73773777, 0x12010001,
374 	0x98f8, 0x73773777, 0x12010001,
375 	0x98fc, 0xffffffff, 0x00000010,
376 	0x98fc, 0xffffffff, 0x00000010,
377 	0x9b7c, 0x00ff0000, 0x00fc0000,
378 	0x9b7c, 0x00ff0000, 0x00fc0000,
379 	0x8030, 0x00001f0f, 0x0000100a,
380 	0x8030, 0x00001f0f, 0x0000100a,
381 	0x2f48, 0x73773777, 0x12010001,
382 	0x2f48, 0x73773777, 0x12010001,
383 	0x2408, 0x00030000, 0x000c007f,
384 	0x8a14, 0xf000003f, 0x00000007,
385 	0x8a14, 0xf000003f, 0x00000007,
386 	0x8b24, 0x3fff3fff, 0x00ff0fff,
387 	0x8b24, 0x3fff3fff, 0x00ff0fff,
388 	0x8b10, 0x0000ff0f, 0x00000000,
389 	0x8b10, 0x0000ff0f, 0x00000000,
390 	0x28a4c, 0x07ffffff, 0x06000000,
391 	0x28a4c, 0x07ffffff, 0x06000000,
392 	0x4d8, 0x00000fff, 0x00000100,
393 	0x4d8, 0x00000fff, 0x00000100,
394 	0xa008, 0xffffffff, 0x00010000,
395 	0xa008, 0xffffffff, 0x00010000,
396 	0x913c, 0xffff03ff, 0x01000100,
397 	0x913c, 0xffff03ff, 0x01000100,
398 	0x90e8, 0x001fffff, 0x010400c0,
399 	0x8c00, 0x000000ff, 0x00000003,
400 	0x8c00, 0x000000ff, 0x00000003,
401 	0x8c04, 0xf8ff00ff, 0x40600060,
402 	0x8c04, 0xf8ff00ff, 0x40600060,
403 	0x8c30, 0x0000000f, 0x00040005,
404 	0x8cf0, 0x1fff1fff, 0x08e00410,
405 	0x8cf0, 0x1fff1fff, 0x08e00410,
406 	0x900c, 0x00ffffff, 0x0017071f,
407 	0x28350, 0x00000f01, 0x00000000,
408 	0x28350, 0x00000f01, 0x00000000,
409 	0x9508, 0xf700071f, 0x00000002,
410 	0x9508, 0xf700071f, 0x00000002,
411 	0x9688, 0x00300000, 0x0017000f,
412 	0x960c, 0xffffffff, 0x54763210,
413 	0x960c, 0xffffffff, 0x54763210,
414 	0x20ef8, 0x01ff01ff, 0x00000002,
415 	0x20e98, 0xfffffbff, 0x00200000,
416 	0x2015c, 0xffffffff, 0x00000f40,
417 	0x88c4, 0x001f3ae3, 0x00000082,
418 	0x88c4, 0x001f3ae3, 0x00000082,
419 	0x8978, 0x3fffffff, 0x04050140,
420 	0x8978, 0x3fffffff, 0x04050140,
421 	0x88d4, 0x0000001f, 0x00000010,
422 	0x88d4, 0x0000001f, 0x00000010,
423 	0x8974, 0xffffffff, 0x00000000,
424 	0x8974, 0xffffffff, 0x00000000
425 };
426 
427 static void ni_init_golden_registers(struct radeon_device *rdev)
428 {
429 	switch (rdev->family) {
430 	case CHIP_CAYMAN:
431 		radeon_program_register_sequence(rdev,
432 						 cayman_golden_registers,
433 						 (const u32)ARRAY_SIZE(cayman_golden_registers));
434 		radeon_program_register_sequence(rdev,
435 						 cayman_golden_registers2,
436 						 (const u32)ARRAY_SIZE(cayman_golden_registers2));
437 		break;
438 	case CHIP_ARUBA:
439 		if ((rdev->pdev->device == 0x9900) ||
440 		    (rdev->pdev->device == 0x9901) ||
441 		    (rdev->pdev->device == 0x9903) ||
442 		    (rdev->pdev->device == 0x9904) ||
443 		    (rdev->pdev->device == 0x9905) ||
444 		    (rdev->pdev->device == 0x9906) ||
445 		    (rdev->pdev->device == 0x9907) ||
446 		    (rdev->pdev->device == 0x9908) ||
447 		    (rdev->pdev->device == 0x9909) ||
448 		    (rdev->pdev->device == 0x990A) ||
449 		    (rdev->pdev->device == 0x990B) ||
450 		    (rdev->pdev->device == 0x990C) ||
451 		    (rdev->pdev->device == 0x990D) ||
452 		    (rdev->pdev->device == 0x990E) ||
453 		    (rdev->pdev->device == 0x990F) ||
454 		    (rdev->pdev->device == 0x9910) ||
455 		    (rdev->pdev->device == 0x9913) ||
456 		    (rdev->pdev->device == 0x9917) ||
457 		    (rdev->pdev->device == 0x9918)) {
458 			radeon_program_register_sequence(rdev,
459 							 dvst_golden_registers,
460 							 (const u32)ARRAY_SIZE(dvst_golden_registers));
461 			radeon_program_register_sequence(rdev,
462 							 dvst_golden_registers2,
463 							 (const u32)ARRAY_SIZE(dvst_golden_registers2));
464 		} else {
465 			radeon_program_register_sequence(rdev,
466 							 scrapper_golden_registers,
467 							 (const u32)ARRAY_SIZE(scrapper_golden_registers));
468 			radeon_program_register_sequence(rdev,
469 							 dvst_golden_registers2,
470 							 (const u32)ARRAY_SIZE(dvst_golden_registers2));
471 		}
472 		break;
473 	default:
474 		break;
475 	}
476 }
477 
478 #define BTC_IO_MC_REGS_SIZE 29
479 
480 static const u32 barts_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
481 	{0x00000077, 0xff010100},
482 	{0x00000078, 0x00000000},
483 	{0x00000079, 0x00001434},
484 	{0x0000007a, 0xcc08ec08},
485 	{0x0000007b, 0x00040000},
486 	{0x0000007c, 0x000080c0},
487 	{0x0000007d, 0x09000000},
488 	{0x0000007e, 0x00210404},
489 	{0x00000081, 0x08a8e800},
490 	{0x00000082, 0x00030444},
491 	{0x00000083, 0x00000000},
492 	{0x00000085, 0x00000001},
493 	{0x00000086, 0x00000002},
494 	{0x00000087, 0x48490000},
495 	{0x00000088, 0x20244647},
496 	{0x00000089, 0x00000005},
497 	{0x0000008b, 0x66030000},
498 	{0x0000008c, 0x00006603},
499 	{0x0000008d, 0x00000100},
500 	{0x0000008f, 0x00001c0a},
501 	{0x00000090, 0xff000001},
502 	{0x00000094, 0x00101101},
503 	{0x00000095, 0x00000fff},
504 	{0x00000096, 0x00116fff},
505 	{0x00000097, 0x60010000},
506 	{0x00000098, 0x10010000},
507 	{0x00000099, 0x00006000},
508 	{0x0000009a, 0x00001000},
509 	{0x0000009f, 0x00946a00}
510 };
511 
512 static const u32 turks_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
513 	{0x00000077, 0xff010100},
514 	{0x00000078, 0x00000000},
515 	{0x00000079, 0x00001434},
516 	{0x0000007a, 0xcc08ec08},
517 	{0x0000007b, 0x00040000},
518 	{0x0000007c, 0x000080c0},
519 	{0x0000007d, 0x09000000},
520 	{0x0000007e, 0x00210404},
521 	{0x00000081, 0x08a8e800},
522 	{0x00000082, 0x00030444},
523 	{0x00000083, 0x00000000},
524 	{0x00000085, 0x00000001},
525 	{0x00000086, 0x00000002},
526 	{0x00000087, 0x48490000},
527 	{0x00000088, 0x20244647},
528 	{0x00000089, 0x00000005},
529 	{0x0000008b, 0x66030000},
530 	{0x0000008c, 0x00006603},
531 	{0x0000008d, 0x00000100},
532 	{0x0000008f, 0x00001c0a},
533 	{0x00000090, 0xff000001},
534 	{0x00000094, 0x00101101},
535 	{0x00000095, 0x00000fff},
536 	{0x00000096, 0x00116fff},
537 	{0x00000097, 0x60010000},
538 	{0x00000098, 0x10010000},
539 	{0x00000099, 0x00006000},
540 	{0x0000009a, 0x00001000},
541 	{0x0000009f, 0x00936a00}
542 };
543 
544 static const u32 caicos_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
545 	{0x00000077, 0xff010100},
546 	{0x00000078, 0x00000000},
547 	{0x00000079, 0x00001434},
548 	{0x0000007a, 0xcc08ec08},
549 	{0x0000007b, 0x00040000},
550 	{0x0000007c, 0x000080c0},
551 	{0x0000007d, 0x09000000},
552 	{0x0000007e, 0x00210404},
553 	{0x00000081, 0x08a8e800},
554 	{0x00000082, 0x00030444},
555 	{0x00000083, 0x00000000},
556 	{0x00000085, 0x00000001},
557 	{0x00000086, 0x00000002},
558 	{0x00000087, 0x48490000},
559 	{0x00000088, 0x20244647},
560 	{0x00000089, 0x00000005},
561 	{0x0000008b, 0x66030000},
562 	{0x0000008c, 0x00006603},
563 	{0x0000008d, 0x00000100},
564 	{0x0000008f, 0x00001c0a},
565 	{0x00000090, 0xff000001},
566 	{0x00000094, 0x00101101},
567 	{0x00000095, 0x00000fff},
568 	{0x00000096, 0x00116fff},
569 	{0x00000097, 0x60010000},
570 	{0x00000098, 0x10010000},
571 	{0x00000099, 0x00006000},
572 	{0x0000009a, 0x00001000},
573 	{0x0000009f, 0x00916a00}
574 };
575 
576 static const u32 cayman_io_mc_regs[BTC_IO_MC_REGS_SIZE][2] = {
577 	{0x00000077, 0xff010100},
578 	{0x00000078, 0x00000000},
579 	{0x00000079, 0x00001434},
580 	{0x0000007a, 0xcc08ec08},
581 	{0x0000007b, 0x00040000},
582 	{0x0000007c, 0x000080c0},
583 	{0x0000007d, 0x09000000},
584 	{0x0000007e, 0x00210404},
585 	{0x00000081, 0x08a8e800},
586 	{0x00000082, 0x00030444},
587 	{0x00000083, 0x00000000},
588 	{0x00000085, 0x00000001},
589 	{0x00000086, 0x00000002},
590 	{0x00000087, 0x48490000},
591 	{0x00000088, 0x20244647},
592 	{0x00000089, 0x00000005},
593 	{0x0000008b, 0x66030000},
594 	{0x0000008c, 0x00006603},
595 	{0x0000008d, 0x00000100},
596 	{0x0000008f, 0x00001c0a},
597 	{0x00000090, 0xff000001},
598 	{0x00000094, 0x00101101},
599 	{0x00000095, 0x00000fff},
600 	{0x00000096, 0x00116fff},
601 	{0x00000097, 0x60010000},
602 	{0x00000098, 0x10010000},
603 	{0x00000099, 0x00006000},
604 	{0x0000009a, 0x00001000},
605 	{0x0000009f, 0x00976b00}
606 };
607 
608 int ni_mc_load_microcode(struct radeon_device *rdev)
609 {
610 	const __be32 *fw_data;
611 	u32 mem_type, running, blackout = 0;
612 	u32 *io_mc_regs;
613 	int i, ucode_size, regs_size;
614 
615 	if (!rdev->mc_fw)
616 		return -EINVAL;
617 
618 	switch (rdev->family) {
619 	case CHIP_BARTS:
620 		io_mc_regs = (u32 *)&barts_io_mc_regs;
621 		ucode_size = BTC_MC_UCODE_SIZE;
622 		regs_size = BTC_IO_MC_REGS_SIZE;
623 		break;
624 	case CHIP_TURKS:
625 		io_mc_regs = (u32 *)&turks_io_mc_regs;
626 		ucode_size = BTC_MC_UCODE_SIZE;
627 		regs_size = BTC_IO_MC_REGS_SIZE;
628 		break;
629 	case CHIP_CAICOS:
630 	default:
631 		io_mc_regs = (u32 *)&caicos_io_mc_regs;
632 		ucode_size = BTC_MC_UCODE_SIZE;
633 		regs_size = BTC_IO_MC_REGS_SIZE;
634 		break;
635 	case CHIP_CAYMAN:
636 		io_mc_regs = (u32 *)&cayman_io_mc_regs;
637 		ucode_size = CAYMAN_MC_UCODE_SIZE;
638 		regs_size = BTC_IO_MC_REGS_SIZE;
639 		break;
640 	}
641 
642 	mem_type = (RREG32(MC_SEQ_MISC0) & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT;
643 	running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
644 
645 	if ((mem_type == MC_SEQ_MISC0_GDDR5_VALUE) && (running == 0)) {
646 		if (running) {
647 			blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
648 			WREG32(MC_SHARED_BLACKOUT_CNTL, 1);
649 		}
650 
651 		/* reset the engine and set to writable */
652 		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
653 		WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
654 
655 		/* load mc io regs */
656 		for (i = 0; i < regs_size; i++) {
657 			WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
658 			WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
659 		}
660 		/* load the MC ucode */
661 		fw_data = (const __be32 *)rdev->mc_fw->data;
662 		for (i = 0; i < ucode_size; i++)
663 			WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
664 
665 		/* put the engine back into the active state */
666 		WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
667 		WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
668 		WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
669 
670 		/* wait for training to complete */
671 		for (i = 0; i < rdev->usec_timeout; i++) {
672 			if (RREG32(MC_IO_PAD_CNTL_D0) & MEM_FALL_OUT_CMD)
673 				break;
674 			udelay(1);
675 		}
676 
677 		if (running)
678 			WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
679 	}
680 
681 	return 0;
682 }
683 
684 int ni_init_microcode(struct radeon_device *rdev)
685 {
686 	const char *chip_name;
687 	const char *rlc_chip_name;
688 	size_t pfp_req_size, me_req_size, rlc_req_size, mc_req_size;
689 	size_t smc_req_size = 0;
690 	char fw_name[30];
691 	int err;
692 
693 	DRM_DEBUG("\n");
694 
695 	switch (rdev->family) {
696 	case CHIP_BARTS:
697 		chip_name = "BARTS";
698 		rlc_chip_name = "BTC";
699 		pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
700 		me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
701 		rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
702 		mc_req_size = BTC_MC_UCODE_SIZE * 4;
703 		smc_req_size = ALIGN(BARTS_SMC_UCODE_SIZE, 4);
704 		break;
705 	case CHIP_TURKS:
706 		chip_name = "TURKS";
707 		rlc_chip_name = "BTC";
708 		pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
709 		me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
710 		rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
711 		mc_req_size = BTC_MC_UCODE_SIZE * 4;
712 		smc_req_size = ALIGN(TURKS_SMC_UCODE_SIZE, 4);
713 		break;
714 	case CHIP_CAICOS:
715 		chip_name = "CAICOS";
716 		rlc_chip_name = "BTC";
717 		pfp_req_size = EVERGREEN_PFP_UCODE_SIZE * 4;
718 		me_req_size = EVERGREEN_PM4_UCODE_SIZE * 4;
719 		rlc_req_size = EVERGREEN_RLC_UCODE_SIZE * 4;
720 		mc_req_size = BTC_MC_UCODE_SIZE * 4;
721 		smc_req_size = ALIGN(CAICOS_SMC_UCODE_SIZE, 4);
722 		break;
723 	case CHIP_CAYMAN:
724 		chip_name = "CAYMAN";
725 		rlc_chip_name = "CAYMAN";
726 		pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
727 		me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
728 		rlc_req_size = CAYMAN_RLC_UCODE_SIZE * 4;
729 		mc_req_size = CAYMAN_MC_UCODE_SIZE * 4;
730 		smc_req_size = ALIGN(CAYMAN_SMC_UCODE_SIZE, 4);
731 		break;
732 	case CHIP_ARUBA:
733 		chip_name = "ARUBA";
734 		rlc_chip_name = "ARUBA";
735 		/* pfp/me same size as CAYMAN */
736 		pfp_req_size = CAYMAN_PFP_UCODE_SIZE * 4;
737 		me_req_size = CAYMAN_PM4_UCODE_SIZE * 4;
738 		rlc_req_size = ARUBA_RLC_UCODE_SIZE * 4;
739 		mc_req_size = 0;
740 		break;
741 	default: BUG();
742 	}
743 
744 	DRM_INFO("Loading %s Microcode\n", chip_name);
745 
746 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
747 	err = request_firmware(&rdev->pfp_fw, fw_name, rdev->dev);
748 	if (err)
749 		goto out;
750 	if (rdev->pfp_fw->size != pfp_req_size) {
751 		printk(KERN_ERR
752 		       "ni_cp: Bogus length %zu in firmware \"%s\"\n",
753 		       rdev->pfp_fw->size, fw_name);
754 		err = -EINVAL;
755 		goto out;
756 	}
757 
758 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
759 	err = request_firmware(&rdev->me_fw, fw_name, rdev->dev);
760 	if (err)
761 		goto out;
762 	if (rdev->me_fw->size != me_req_size) {
763 		printk(KERN_ERR
764 		       "ni_cp: Bogus length %zu in firmware \"%s\"\n",
765 		       rdev->me_fw->size, fw_name);
766 		err = -EINVAL;
767 	}
768 
769 	snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
770 	err = request_firmware(&rdev->rlc_fw, fw_name, rdev->dev);
771 	if (err)
772 		goto out;
773 	if (rdev->rlc_fw->size != rlc_req_size) {
774 		printk(KERN_ERR
775 		       "ni_rlc: Bogus length %zu in firmware \"%s\"\n",
776 		       rdev->rlc_fw->size, fw_name);
777 		err = -EINVAL;
778 	}
779 
780 	/* no MC ucode on TN */
781 	if (!(rdev->flags & RADEON_IS_IGP)) {
782 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
783 		err = request_firmware(&rdev->mc_fw, fw_name, rdev->dev);
784 		if (err)
785 			goto out;
786 		if (rdev->mc_fw->size != mc_req_size) {
787 			printk(KERN_ERR
788 			       "ni_mc: Bogus length %zu in firmware \"%s\"\n",
789 			       rdev->mc_fw->size, fw_name);
790 			err = -EINVAL;
791 		}
792 	}
793 
794 	if ((rdev->family >= CHIP_BARTS) && (rdev->family <= CHIP_CAYMAN)) {
795 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
796 		err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
797 		if (err) {
798 			printk(KERN_ERR
799 			       "smc: error loading firmware \"%s\"\n",
800 			       fw_name);
801 			release_firmware(rdev->smc_fw);
802 			rdev->smc_fw = NULL;
803 			err = 0;
804 		} else if (rdev->smc_fw->size != smc_req_size) {
805 			printk(KERN_ERR
806 			       "ni_mc: Bogus length %zu in firmware \"%s\"\n",
807 			       rdev->mc_fw->size, fw_name);
808 			err = -EINVAL;
809 		}
810 	}
811 
812 out:
813 	if (err) {
814 		if (err != -EINVAL)
815 			printk(KERN_ERR
816 			       "ni_cp: Failed to load firmware \"%s\"\n",
817 			       fw_name);
818 		release_firmware(rdev->pfp_fw);
819 		rdev->pfp_fw = NULL;
820 		release_firmware(rdev->me_fw);
821 		rdev->me_fw = NULL;
822 		release_firmware(rdev->rlc_fw);
823 		rdev->rlc_fw = NULL;
824 		release_firmware(rdev->mc_fw);
825 		rdev->mc_fw = NULL;
826 	}
827 	return err;
828 }
829 
830 int tn_get_temp(struct radeon_device *rdev)
831 {
832 	u32 temp = RREG32_SMC(TN_CURRENT_GNB_TEMP) & 0x7ff;
833 	int actual_temp = (temp / 8) - 49;
834 
835 	return actual_temp * 1000;
836 }
837 
838 /*
839  * Core functions
840  */
841 static void cayman_gpu_init(struct radeon_device *rdev)
842 {
843 	u32 gb_addr_config = 0;
844 	u32 mc_shared_chmap, mc_arb_ramcfg;
845 	u32 cgts_tcc_disable;
846 	u32 sx_debug_1;
847 	u32 smx_dc_ctl0;
848 	u32 cgts_sm_ctrl_reg;
849 	u32 hdp_host_path_cntl;
850 	u32 tmp;
851 	u32 disabled_rb_mask;
852 	int i, j;
853 
854 	switch (rdev->family) {
855 	case CHIP_CAYMAN:
856 		rdev->config.cayman.max_shader_engines = 2;
857 		rdev->config.cayman.max_pipes_per_simd = 4;
858 		rdev->config.cayman.max_tile_pipes = 8;
859 		rdev->config.cayman.max_simds_per_se = 12;
860 		rdev->config.cayman.max_backends_per_se = 4;
861 		rdev->config.cayman.max_texture_channel_caches = 8;
862 		rdev->config.cayman.max_gprs = 256;
863 		rdev->config.cayman.max_threads = 256;
864 		rdev->config.cayman.max_gs_threads = 32;
865 		rdev->config.cayman.max_stack_entries = 512;
866 		rdev->config.cayman.sx_num_of_sets = 8;
867 		rdev->config.cayman.sx_max_export_size = 256;
868 		rdev->config.cayman.sx_max_export_pos_size = 64;
869 		rdev->config.cayman.sx_max_export_smx_size = 192;
870 		rdev->config.cayman.max_hw_contexts = 8;
871 		rdev->config.cayman.sq_num_cf_insts = 2;
872 
873 		rdev->config.cayman.sc_prim_fifo_size = 0x100;
874 		rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
875 		rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
876 		gb_addr_config = CAYMAN_GB_ADDR_CONFIG_GOLDEN;
877 		break;
878 	case CHIP_ARUBA:
879 	default:
880 		rdev->config.cayman.max_shader_engines = 1;
881 		rdev->config.cayman.max_pipes_per_simd = 4;
882 		rdev->config.cayman.max_tile_pipes = 2;
883 		if ((rdev->pdev->device == 0x9900) ||
884 		    (rdev->pdev->device == 0x9901) ||
885 		    (rdev->pdev->device == 0x9905) ||
886 		    (rdev->pdev->device == 0x9906) ||
887 		    (rdev->pdev->device == 0x9907) ||
888 		    (rdev->pdev->device == 0x9908) ||
889 		    (rdev->pdev->device == 0x9909) ||
890 		    (rdev->pdev->device == 0x990B) ||
891 		    (rdev->pdev->device == 0x990C) ||
892 		    (rdev->pdev->device == 0x990F) ||
893 		    (rdev->pdev->device == 0x9910) ||
894 		    (rdev->pdev->device == 0x9917) ||
895 		    (rdev->pdev->device == 0x9999) ||
896 		    (rdev->pdev->device == 0x999C)) {
897 			rdev->config.cayman.max_simds_per_se = 6;
898 			rdev->config.cayman.max_backends_per_se = 2;
899 			rdev->config.cayman.max_hw_contexts = 8;
900 			rdev->config.cayman.sx_max_export_size = 256;
901 			rdev->config.cayman.sx_max_export_pos_size = 64;
902 			rdev->config.cayman.sx_max_export_smx_size = 192;
903 		} else if ((rdev->pdev->device == 0x9903) ||
904 			   (rdev->pdev->device == 0x9904) ||
905 			   (rdev->pdev->device == 0x990A) ||
906 			   (rdev->pdev->device == 0x990D) ||
907 			   (rdev->pdev->device == 0x990E) ||
908 			   (rdev->pdev->device == 0x9913) ||
909 			   (rdev->pdev->device == 0x9918) ||
910 			   (rdev->pdev->device == 0x999D)) {
911 			rdev->config.cayman.max_simds_per_se = 4;
912 			rdev->config.cayman.max_backends_per_se = 2;
913 			rdev->config.cayman.max_hw_contexts = 8;
914 			rdev->config.cayman.sx_max_export_size = 256;
915 			rdev->config.cayman.sx_max_export_pos_size = 64;
916 			rdev->config.cayman.sx_max_export_smx_size = 192;
917 		} else if ((rdev->pdev->device == 0x9919) ||
918 			   (rdev->pdev->device == 0x9990) ||
919 			   (rdev->pdev->device == 0x9991) ||
920 			   (rdev->pdev->device == 0x9994) ||
921 			   (rdev->pdev->device == 0x9995) ||
922 			   (rdev->pdev->device == 0x9996) ||
923 			   (rdev->pdev->device == 0x999A) ||
924 			   (rdev->pdev->device == 0x99A0)) {
925 			rdev->config.cayman.max_simds_per_se = 3;
926 			rdev->config.cayman.max_backends_per_se = 1;
927 			rdev->config.cayman.max_hw_contexts = 4;
928 			rdev->config.cayman.sx_max_export_size = 128;
929 			rdev->config.cayman.sx_max_export_pos_size = 32;
930 			rdev->config.cayman.sx_max_export_smx_size = 96;
931 		} else {
932 			rdev->config.cayman.max_simds_per_se = 2;
933 			rdev->config.cayman.max_backends_per_se = 1;
934 			rdev->config.cayman.max_hw_contexts = 4;
935 			rdev->config.cayman.sx_max_export_size = 128;
936 			rdev->config.cayman.sx_max_export_pos_size = 32;
937 			rdev->config.cayman.sx_max_export_smx_size = 96;
938 		}
939 		rdev->config.cayman.max_texture_channel_caches = 2;
940 		rdev->config.cayman.max_gprs = 256;
941 		rdev->config.cayman.max_threads = 256;
942 		rdev->config.cayman.max_gs_threads = 32;
943 		rdev->config.cayman.max_stack_entries = 512;
944 		rdev->config.cayman.sx_num_of_sets = 8;
945 		rdev->config.cayman.sq_num_cf_insts = 2;
946 
947 		rdev->config.cayman.sc_prim_fifo_size = 0x40;
948 		rdev->config.cayman.sc_hiz_tile_fifo_size = 0x30;
949 		rdev->config.cayman.sc_earlyz_tile_fifo_size = 0x130;
950 		gb_addr_config = ARUBA_GB_ADDR_CONFIG_GOLDEN;
951 		break;
952 	}
953 
954 	/* Initialize HDP */
955 	for (i = 0, j = 0; i < 32; i++, j += 0x18) {
956 		WREG32((0x2c14 + j), 0x00000000);
957 		WREG32((0x2c18 + j), 0x00000000);
958 		WREG32((0x2c1c + j), 0x00000000);
959 		WREG32((0x2c20 + j), 0x00000000);
960 		WREG32((0x2c24 + j), 0x00000000);
961 	}
962 
963 	WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
964 
965 	evergreen_fix_pci_max_read_req_size(rdev);
966 
967 	mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
968 	mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
969 
970 	tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
971 	rdev->config.cayman.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
972 	if (rdev->config.cayman.mem_row_size_in_kb > 4)
973 		rdev->config.cayman.mem_row_size_in_kb = 4;
974 	/* XXX use MC settings? */
975 	rdev->config.cayman.shader_engine_tile_size = 32;
976 	rdev->config.cayman.num_gpus = 1;
977 	rdev->config.cayman.multi_gpu_tile_size = 64;
978 
979 	tmp = (gb_addr_config & NUM_PIPES_MASK) >> NUM_PIPES_SHIFT;
980 	rdev->config.cayman.num_tile_pipes = (1 << tmp);
981 	tmp = (gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT;
982 	rdev->config.cayman.mem_max_burst_length_bytes = (tmp + 1) * 256;
983 	tmp = (gb_addr_config & NUM_SHADER_ENGINES_MASK) >> NUM_SHADER_ENGINES_SHIFT;
984 	rdev->config.cayman.num_shader_engines = tmp + 1;
985 	tmp = (gb_addr_config & NUM_GPUS_MASK) >> NUM_GPUS_SHIFT;
986 	rdev->config.cayman.num_gpus = tmp + 1;
987 	tmp = (gb_addr_config & MULTI_GPU_TILE_SIZE_MASK) >> MULTI_GPU_TILE_SIZE_SHIFT;
988 	rdev->config.cayman.multi_gpu_tile_size = 1 << tmp;
989 	tmp = (gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT;
990 	rdev->config.cayman.mem_row_size_in_kb = 1 << tmp;
991 
992 
993 	/* setup tiling info dword.  gb_addr_config is not adequate since it does
994 	 * not have bank info, so create a custom tiling dword.
995 	 * bits 3:0   num_pipes
996 	 * bits 7:4   num_banks
997 	 * bits 11:8  group_size
998 	 * bits 15:12 row_size
999 	 */
1000 	rdev->config.cayman.tile_config = 0;
1001 	switch (rdev->config.cayman.num_tile_pipes) {
1002 	case 1:
1003 	default:
1004 		rdev->config.cayman.tile_config |= (0 << 0);
1005 		break;
1006 	case 2:
1007 		rdev->config.cayman.tile_config |= (1 << 0);
1008 		break;
1009 	case 4:
1010 		rdev->config.cayman.tile_config |= (2 << 0);
1011 		break;
1012 	case 8:
1013 		rdev->config.cayman.tile_config |= (3 << 0);
1014 		break;
1015 	}
1016 
1017 	/* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
1018 	if (rdev->flags & RADEON_IS_IGP)
1019 		rdev->config.cayman.tile_config |= 1 << 4;
1020 	else {
1021 		switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
1022 		case 0: /* four banks */
1023 			rdev->config.cayman.tile_config |= 0 << 4;
1024 			break;
1025 		case 1: /* eight banks */
1026 			rdev->config.cayman.tile_config |= 1 << 4;
1027 			break;
1028 		case 2: /* sixteen banks */
1029 		default:
1030 			rdev->config.cayman.tile_config |= 2 << 4;
1031 			break;
1032 		}
1033 	}
1034 	rdev->config.cayman.tile_config |=
1035 		((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
1036 	rdev->config.cayman.tile_config |=
1037 		((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
1038 
1039 	tmp = 0;
1040 	for (i = (rdev->config.cayman.max_shader_engines - 1); i >= 0; i--) {
1041 		u32 rb_disable_bitmap;
1042 
1043 		WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1044 		WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1045 		rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
1046 		tmp <<= 4;
1047 		tmp |= rb_disable_bitmap;
1048 	}
1049 	/* enabled rb are just the one not disabled :) */
1050 	disabled_rb_mask = tmp;
1051 	tmp = 0;
1052 	for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1053 		tmp |= (1 << i);
1054 	/* if all the backends are disabled, fix it up here */
1055 	if ((disabled_rb_mask & tmp) == tmp) {
1056 		for (i = 0; i < (rdev->config.cayman.max_backends_per_se * rdev->config.cayman.max_shader_engines); i++)
1057 			disabled_rb_mask &= ~(1 << i);
1058 	}
1059 
1060 	for (i = 0; i < rdev->config.cayman.max_shader_engines; i++) {
1061 		u32 simd_disable_bitmap;
1062 
1063 		WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1064 		WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
1065 		simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
1066 		simd_disable_bitmap |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
1067 		tmp <<= 16;
1068 		tmp |= simd_disable_bitmap;
1069 	}
1070 	rdev->config.cayman.active_simds = hweight32(~tmp);
1071 
1072 	WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1073 	WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
1074 
1075 	WREG32(GB_ADDR_CONFIG, gb_addr_config);
1076 	WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
1077 	if (ASIC_IS_DCE6(rdev))
1078 		WREG32(DMIF_ADDR_CALC, gb_addr_config);
1079 	WREG32(HDP_ADDR_CONFIG, gb_addr_config);
1080 	WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
1081 	WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
1082 	WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
1083 	WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
1084 	WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
1085 
1086 	if ((rdev->config.cayman.max_backends_per_se == 1) &&
1087 	    (rdev->flags & RADEON_IS_IGP)) {
1088 		if ((disabled_rb_mask & 3) == 1) {
1089 			/* RB0 disabled, RB1 enabled */
1090 			tmp = 0x11111111;
1091 		} else {
1092 			/* RB1 disabled, RB0 enabled */
1093 			tmp = 0x00000000;
1094 		}
1095 	} else {
1096 		tmp = gb_addr_config & NUM_PIPES_MASK;
1097 		tmp = r6xx_remap_render_backend(rdev, tmp,
1098 						rdev->config.cayman.max_backends_per_se *
1099 						rdev->config.cayman.max_shader_engines,
1100 						CAYMAN_MAX_BACKENDS, disabled_rb_mask);
1101 	}
1102 	WREG32(GB_BACKEND_MAP, tmp);
1103 
1104 	cgts_tcc_disable = 0xffff0000;
1105 	for (i = 0; i < rdev->config.cayman.max_texture_channel_caches; i++)
1106 		cgts_tcc_disable &= ~(1 << (16 + i));
1107 	WREG32(CGTS_TCC_DISABLE, cgts_tcc_disable);
1108 	WREG32(CGTS_SYS_TCC_DISABLE, cgts_tcc_disable);
1109 	WREG32(CGTS_USER_SYS_TCC_DISABLE, cgts_tcc_disable);
1110 	WREG32(CGTS_USER_TCC_DISABLE, cgts_tcc_disable);
1111 
1112 	/* reprogram the shader complex */
1113 	cgts_sm_ctrl_reg = RREG32(CGTS_SM_CTRL_REG);
1114 	for (i = 0; i < 16; i++)
1115 		WREG32(CGTS_SM_CTRL_REG, OVERRIDE);
1116 	WREG32(CGTS_SM_CTRL_REG, cgts_sm_ctrl_reg);
1117 
1118 	/* set HW defaults for 3D engine */
1119 	WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
1120 
1121 	sx_debug_1 = RREG32(SX_DEBUG_1);
1122 	sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
1123 	WREG32(SX_DEBUG_1, sx_debug_1);
1124 
1125 	smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
1126 	smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
1127 	smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.cayman.sx_num_of_sets);
1128 	WREG32(SMX_DC_CTL0, smx_dc_ctl0);
1129 
1130 	WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4) | CRC_SIMD_ID_WADDR_DISABLE);
1131 
1132 	/* need to be explicitly zero-ed */
1133 	WREG32(VGT_OFFCHIP_LDS_BASE, 0);
1134 	WREG32(SQ_LSTMP_RING_BASE, 0);
1135 	WREG32(SQ_HSTMP_RING_BASE, 0);
1136 	WREG32(SQ_ESTMP_RING_BASE, 0);
1137 	WREG32(SQ_GSTMP_RING_BASE, 0);
1138 	WREG32(SQ_VSTMP_RING_BASE, 0);
1139 	WREG32(SQ_PSTMP_RING_BASE, 0);
1140 
1141 	WREG32(TA_CNTL_AUX, DISABLE_CUBE_ANISO);
1142 
1143 	WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.cayman.sx_max_export_size / 4) - 1) |
1144 					POSITION_BUFFER_SIZE((rdev->config.cayman.sx_max_export_pos_size / 4) - 1) |
1145 					SMX_BUFFER_SIZE((rdev->config.cayman.sx_max_export_smx_size / 4) - 1)));
1146 
1147 	WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.cayman.sc_prim_fifo_size) |
1148 				 SC_HIZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_hiz_tile_fifo_size) |
1149 				 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.cayman.sc_earlyz_tile_fifo_size)));
1150 
1151 
1152 	WREG32(VGT_NUM_INSTANCES, 1);
1153 
1154 	WREG32(CP_PERFMON_CNTL, 0);
1155 
1156 	WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.cayman.sq_num_cf_insts) |
1157 				  FETCH_FIFO_HIWATER(0x4) |
1158 				  DONE_FIFO_HIWATER(0xe0) |
1159 				  ALU_UPDATE_FIFO_HIWATER(0x8)));
1160 
1161 	WREG32(SQ_GPR_RESOURCE_MGMT_1, NUM_CLAUSE_TEMP_GPRS(4));
1162 	WREG32(SQ_CONFIG, (VC_ENABLE |
1163 			   EXPORT_SRC_C |
1164 			   GFX_PRIO(0) |
1165 			   CS1_PRIO(0) |
1166 			   CS2_PRIO(1)));
1167 	WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, DYN_GPR_ENABLE);
1168 
1169 	WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
1170 					  FORCE_EOV_MAX_REZ_CNT(255)));
1171 
1172 	WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
1173 	       AUTO_INVLD_EN(ES_AND_GS_AUTO));
1174 
1175 	WREG32(VGT_GS_VERTEX_REUSE, 16);
1176 	WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
1177 
1178 	WREG32(CB_PERF_CTR0_SEL_0, 0);
1179 	WREG32(CB_PERF_CTR0_SEL_1, 0);
1180 	WREG32(CB_PERF_CTR1_SEL_0, 0);
1181 	WREG32(CB_PERF_CTR1_SEL_1, 0);
1182 	WREG32(CB_PERF_CTR2_SEL_0, 0);
1183 	WREG32(CB_PERF_CTR2_SEL_1, 0);
1184 	WREG32(CB_PERF_CTR3_SEL_0, 0);
1185 	WREG32(CB_PERF_CTR3_SEL_1, 0);
1186 
1187 	tmp = RREG32(HDP_MISC_CNTL);
1188 	tmp |= HDP_FLUSH_INVALIDATE_CACHE;
1189 	WREG32(HDP_MISC_CNTL, tmp);
1190 
1191 	hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
1192 	WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
1193 
1194 	WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
1195 
1196 	udelay(50);
1197 
1198 	/* set clockgating golden values on TN */
1199 	if (rdev->family == CHIP_ARUBA) {
1200 		tmp = RREG32_CG(CG_CGTT_LOCAL_0);
1201 		tmp &= ~0x00380000;
1202 		WREG32_CG(CG_CGTT_LOCAL_0, tmp);
1203                 tmp = RREG32_CG(CG_CGTT_LOCAL_1);
1204 		tmp &= ~0x0e000000;
1205 		WREG32_CG(CG_CGTT_LOCAL_1, tmp);
1206 	}
1207 }
1208 
1209 /*
1210  * GART
1211  */
1212 void cayman_pcie_gart_tlb_flush(struct radeon_device *rdev)
1213 {
1214 	/* flush hdp cache */
1215 	WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
1216 
1217 	/* bits 0-7 are the VM contexts0-7 */
1218 	WREG32(VM_INVALIDATE_REQUEST, 1);
1219 }
1220 
1221 static int cayman_pcie_gart_enable(struct radeon_device *rdev)
1222 {
1223 	int i, r;
1224 
1225 	if (rdev->gart.robj == NULL) {
1226 		dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
1227 		return -EINVAL;
1228 	}
1229 	r = radeon_gart_table_vram_pin(rdev);
1230 	if (r)
1231 		return r;
1232 	/* Setup TLB control */
1233 	WREG32(MC_VM_MX_L1_TLB_CNTL,
1234 	       (0xA << 7) |
1235 	       ENABLE_L1_TLB |
1236 	       ENABLE_L1_FRAGMENT_PROCESSING |
1237 	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1238 	       ENABLE_ADVANCED_DRIVER_MODEL |
1239 	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1240 	/* Setup L2 cache */
1241 	WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
1242 	       ENABLE_L2_FRAGMENT_PROCESSING |
1243 	       ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1244 	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1245 	       EFFECTIVE_L2_QUEUE_SIZE(7) |
1246 	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
1247 	WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
1248 	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1249 	       BANK_SELECT(6) |
1250 	       L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1251 	/* setup context0 */
1252 	WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
1253 	WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
1254 	WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
1255 	WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
1256 			(u32)(rdev->dummy_page.addr >> 12));
1257 	WREG32(VM_CONTEXT0_CNTL2, 0);
1258 	WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
1259 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
1260 
1261 	WREG32(0x15D4, 0);
1262 	WREG32(0x15D8, 0);
1263 	WREG32(0x15DC, 0);
1264 
1265 	/* empty context1-7 */
1266 	/* Assign the pt base to something valid for now; the pts used for
1267 	 * the VMs are determined by the application and setup and assigned
1268 	 * on the fly in the vm part of radeon_gart.c
1269 	 */
1270 	for (i = 1; i < 8; i++) {
1271 		WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR + (i << 2), 0);
1272 		WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR + (i << 2), rdev->vm_manager.max_pfn);
1273 		WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
1274 		       rdev->vm_manager.saved_table_addr[i]);
1275 	}
1276 
1277 	/* enable context1-7 */
1278 	WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
1279 	       (u32)(rdev->dummy_page.addr >> 12));
1280 	WREG32(VM_CONTEXT1_CNTL2, 4);
1281 	WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
1282 				PAGE_TABLE_BLOCK_SIZE(radeon_vm_block_size - 9) |
1283 				RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1284 				RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1285 				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1286 				DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
1287 				PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
1288 				PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
1289 				VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
1290 				VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
1291 				READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
1292 				READ_PROTECTION_FAULT_ENABLE_DEFAULT |
1293 				WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
1294 				WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
1295 
1296 	cayman_pcie_gart_tlb_flush(rdev);
1297 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1298 		 (unsigned)(rdev->mc.gtt_size >> 20),
1299 		 (unsigned long long)rdev->gart.table_addr);
1300 	rdev->gart.ready = true;
1301 	return 0;
1302 }
1303 
1304 static void cayman_pcie_gart_disable(struct radeon_device *rdev)
1305 {
1306 	unsigned i;
1307 
1308 	for (i = 1; i < 8; ++i) {
1309 		rdev->vm_manager.saved_table_addr[i] = RREG32(
1310 			VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2));
1311 	}
1312 
1313 	/* Disable all tables */
1314 	WREG32(VM_CONTEXT0_CNTL, 0);
1315 	WREG32(VM_CONTEXT1_CNTL, 0);
1316 	/* Setup TLB control */
1317 	WREG32(MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING |
1318 	       SYSTEM_ACCESS_MODE_NOT_IN_SYS |
1319 	       SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
1320 	/* Setup L2 cache */
1321 	WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
1322 	       ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
1323 	       EFFECTIVE_L2_QUEUE_SIZE(7) |
1324 	       CONTEXT1_IDENTITY_ACCESS_MODE(1));
1325 	WREG32(VM_L2_CNTL2, 0);
1326 	WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
1327 	       L2_CACHE_BIGK_FRAGMENT_SIZE(6));
1328 	radeon_gart_table_vram_unpin(rdev);
1329 }
1330 
1331 static void cayman_pcie_gart_fini(struct radeon_device *rdev)
1332 {
1333 	cayman_pcie_gart_disable(rdev);
1334 	radeon_gart_table_vram_free(rdev);
1335 	radeon_gart_fini(rdev);
1336 }
1337 
1338 void cayman_cp_int_cntl_setup(struct radeon_device *rdev,
1339 			      int ring, u32 cp_int_cntl)
1340 {
1341 	u32 srbm_gfx_cntl = RREG32(SRBM_GFX_CNTL) & ~3;
1342 
1343 	WREG32(SRBM_GFX_CNTL, srbm_gfx_cntl | (ring & 3));
1344 	WREG32(CP_INT_CNTL, cp_int_cntl);
1345 }
1346 
1347 /*
1348  * CP.
1349  */
1350 void cayman_fence_ring_emit(struct radeon_device *rdev,
1351 			    struct radeon_fence *fence)
1352 {
1353 	struct radeon_ring *ring = &rdev->ring[fence->ring];
1354 	u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
1355 	u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1356 		PACKET3_SH_ACTION_ENA;
1357 
1358 	/* flush read cache over gart for this vmid */
1359 	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1360 	radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1361 	radeon_ring_write(ring, 0xFFFFFFFF);
1362 	radeon_ring_write(ring, 0);
1363 	radeon_ring_write(ring, 10); /* poll interval */
1364 	/* EVENT_WRITE_EOP - flush caches, send int */
1365 	radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
1366 	radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_EVENT_TS) | EVENT_INDEX(5));
1367 	radeon_ring_write(ring, lower_32_bits(addr));
1368 	radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
1369 	radeon_ring_write(ring, fence->seq);
1370 	radeon_ring_write(ring, 0);
1371 }
1372 
1373 void cayman_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
1374 {
1375 	struct radeon_ring *ring = &rdev->ring[ib->ring];
1376 	u32 cp_coher_cntl = PACKET3_FULL_CACHE_ENA | PACKET3_TC_ACTION_ENA |
1377 		PACKET3_SH_ACTION_ENA;
1378 
1379 	/* set to DX10/11 mode */
1380 	radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
1381 	radeon_ring_write(ring, 1);
1382 
1383 	if (ring->rptr_save_reg) {
1384 		uint32_t next_rptr = ring->wptr + 3 + 4 + 8;
1385 		radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
1386 		radeon_ring_write(ring, ((ring->rptr_save_reg -
1387 					  PACKET3_SET_CONFIG_REG_START) >> 2));
1388 		radeon_ring_write(ring, next_rptr);
1389 	}
1390 
1391 	radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
1392 	radeon_ring_write(ring,
1393 #ifdef __BIG_ENDIAN
1394 			  (2 << 0) |
1395 #endif
1396 			  (ib->gpu_addr & 0xFFFFFFFC));
1397 	radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
1398 	radeon_ring_write(ring, ib->length_dw |
1399 			  (ib->vm ? (ib->vm->id << 24) : 0));
1400 
1401 	/* flush read cache over gart for this vmid */
1402 	radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
1403 	radeon_ring_write(ring, PACKET3_ENGINE_ME | cp_coher_cntl);
1404 	radeon_ring_write(ring, 0xFFFFFFFF);
1405 	radeon_ring_write(ring, 0);
1406 	radeon_ring_write(ring, ((ib->vm ? ib->vm->id : 0) << 24) | 10); /* poll interval */
1407 }
1408 
1409 static void cayman_cp_enable(struct radeon_device *rdev, bool enable)
1410 {
1411 	if (enable)
1412 		WREG32(CP_ME_CNTL, 0);
1413 	else {
1414 		if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1415 			radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
1416 		WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT));
1417 		WREG32(SCRATCH_UMSK, 0);
1418 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1419 	}
1420 }
1421 
1422 u32 cayman_gfx_get_rptr(struct radeon_device *rdev,
1423 			struct radeon_ring *ring)
1424 {
1425 	u32 rptr;
1426 
1427 	if (rdev->wb.enabled)
1428 		rptr = rdev->wb.wb[ring->rptr_offs/4];
1429 	else {
1430 		if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1431 			rptr = RREG32(CP_RB0_RPTR);
1432 		else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1433 			rptr = RREG32(CP_RB1_RPTR);
1434 		else
1435 			rptr = RREG32(CP_RB2_RPTR);
1436 	}
1437 
1438 	return rptr;
1439 }
1440 
1441 u32 cayman_gfx_get_wptr(struct radeon_device *rdev,
1442 			struct radeon_ring *ring)
1443 {
1444 	u32 wptr;
1445 
1446 	if (ring->idx == RADEON_RING_TYPE_GFX_INDEX)
1447 		wptr = RREG32(CP_RB0_WPTR);
1448 	else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX)
1449 		wptr = RREG32(CP_RB1_WPTR);
1450 	else
1451 		wptr = RREG32(CP_RB2_WPTR);
1452 
1453 	return wptr;
1454 }
1455 
1456 void cayman_gfx_set_wptr(struct radeon_device *rdev,
1457 			 struct radeon_ring *ring)
1458 {
1459 	if (ring->idx == RADEON_RING_TYPE_GFX_INDEX) {
1460 		WREG32(CP_RB0_WPTR, ring->wptr);
1461 		(void)RREG32(CP_RB0_WPTR);
1462 	} else if (ring->idx == CAYMAN_RING_TYPE_CP1_INDEX) {
1463 		WREG32(CP_RB1_WPTR, ring->wptr);
1464 		(void)RREG32(CP_RB1_WPTR);
1465 	} else {
1466 		WREG32(CP_RB2_WPTR, ring->wptr);
1467 		(void)RREG32(CP_RB2_WPTR);
1468 	}
1469 }
1470 
1471 static int cayman_cp_load_microcode(struct radeon_device *rdev)
1472 {
1473 	const __be32 *fw_data;
1474 	int i;
1475 
1476 	if (!rdev->me_fw || !rdev->pfp_fw)
1477 		return -EINVAL;
1478 
1479 	cayman_cp_enable(rdev, false);
1480 
1481 	fw_data = (const __be32 *)rdev->pfp_fw->data;
1482 	WREG32(CP_PFP_UCODE_ADDR, 0);
1483 	for (i = 0; i < CAYMAN_PFP_UCODE_SIZE; i++)
1484 		WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
1485 	WREG32(CP_PFP_UCODE_ADDR, 0);
1486 
1487 	fw_data = (const __be32 *)rdev->me_fw->data;
1488 	WREG32(CP_ME_RAM_WADDR, 0);
1489 	for (i = 0; i < CAYMAN_PM4_UCODE_SIZE; i++)
1490 		WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
1491 
1492 	WREG32(CP_PFP_UCODE_ADDR, 0);
1493 	WREG32(CP_ME_RAM_WADDR, 0);
1494 	WREG32(CP_ME_RAM_RADDR, 0);
1495 	return 0;
1496 }
1497 
1498 static int cayman_cp_start(struct radeon_device *rdev)
1499 {
1500 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1501 	int r, i;
1502 
1503 	r = radeon_ring_lock(rdev, ring, 7);
1504 	if (r) {
1505 		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1506 		return r;
1507 	}
1508 	radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
1509 	radeon_ring_write(ring, 0x1);
1510 	radeon_ring_write(ring, 0x0);
1511 	radeon_ring_write(ring, rdev->config.cayman.max_hw_contexts - 1);
1512 	radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
1513 	radeon_ring_write(ring, 0);
1514 	radeon_ring_write(ring, 0);
1515 	radeon_ring_unlock_commit(rdev, ring, false);
1516 
1517 	cayman_cp_enable(rdev, true);
1518 
1519 	r = radeon_ring_lock(rdev, ring, cayman_default_size + 19);
1520 	if (r) {
1521 		DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
1522 		return r;
1523 	}
1524 
1525 	/* setup clear context state */
1526 	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1527 	radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
1528 
1529 	for (i = 0; i < cayman_default_size; i++)
1530 		radeon_ring_write(ring, cayman_default_state[i]);
1531 
1532 	radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
1533 	radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
1534 
1535 	/* set clear context state */
1536 	radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
1537 	radeon_ring_write(ring, 0);
1538 
1539 	/* SQ_VTX_BASE_VTX_LOC */
1540 	radeon_ring_write(ring, 0xc0026f00);
1541 	radeon_ring_write(ring, 0x00000000);
1542 	radeon_ring_write(ring, 0x00000000);
1543 	radeon_ring_write(ring, 0x00000000);
1544 
1545 	/* Clear consts */
1546 	radeon_ring_write(ring, 0xc0036f00);
1547 	radeon_ring_write(ring, 0x00000bc4);
1548 	radeon_ring_write(ring, 0xffffffff);
1549 	radeon_ring_write(ring, 0xffffffff);
1550 	radeon_ring_write(ring, 0xffffffff);
1551 
1552 	radeon_ring_write(ring, 0xc0026900);
1553 	radeon_ring_write(ring, 0x00000316);
1554 	radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
1555 	radeon_ring_write(ring, 0x00000010); /*  */
1556 
1557 	radeon_ring_unlock_commit(rdev, ring, false);
1558 
1559 	/* XXX init other rings */
1560 
1561 	return 0;
1562 }
1563 
1564 static void cayman_cp_fini(struct radeon_device *rdev)
1565 {
1566 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1567 	cayman_cp_enable(rdev, false);
1568 	radeon_ring_fini(rdev, ring);
1569 	radeon_scratch_free(rdev, ring->rptr_save_reg);
1570 }
1571 
1572 static int cayman_cp_resume(struct radeon_device *rdev)
1573 {
1574 	static const int ridx[] = {
1575 		RADEON_RING_TYPE_GFX_INDEX,
1576 		CAYMAN_RING_TYPE_CP1_INDEX,
1577 		CAYMAN_RING_TYPE_CP2_INDEX
1578 	};
1579 	static const unsigned cp_rb_cntl[] = {
1580 		CP_RB0_CNTL,
1581 		CP_RB1_CNTL,
1582 		CP_RB2_CNTL,
1583 	};
1584 	static const unsigned cp_rb_rptr_addr[] = {
1585 		CP_RB0_RPTR_ADDR,
1586 		CP_RB1_RPTR_ADDR,
1587 		CP_RB2_RPTR_ADDR
1588 	};
1589 	static const unsigned cp_rb_rptr_addr_hi[] = {
1590 		CP_RB0_RPTR_ADDR_HI,
1591 		CP_RB1_RPTR_ADDR_HI,
1592 		CP_RB2_RPTR_ADDR_HI
1593 	};
1594 	static const unsigned cp_rb_base[] = {
1595 		CP_RB0_BASE,
1596 		CP_RB1_BASE,
1597 		CP_RB2_BASE
1598 	};
1599 	static const unsigned cp_rb_rptr[] = {
1600 		CP_RB0_RPTR,
1601 		CP_RB1_RPTR,
1602 		CP_RB2_RPTR
1603 	};
1604 	static const unsigned cp_rb_wptr[] = {
1605 		CP_RB0_WPTR,
1606 		CP_RB1_WPTR,
1607 		CP_RB2_WPTR
1608 	};
1609 	struct radeon_ring *ring;
1610 	int i, r;
1611 
1612 	/* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
1613 	WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
1614 				 SOFT_RESET_PA |
1615 				 SOFT_RESET_SH |
1616 				 SOFT_RESET_VGT |
1617 				 SOFT_RESET_SPI |
1618 				 SOFT_RESET_SX));
1619 	RREG32(GRBM_SOFT_RESET);
1620 	mdelay(15);
1621 	WREG32(GRBM_SOFT_RESET, 0);
1622 	RREG32(GRBM_SOFT_RESET);
1623 
1624 	WREG32(CP_SEM_WAIT_TIMER, 0x0);
1625 	WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
1626 
1627 	/* Set the write pointer delay */
1628 	WREG32(CP_RB_WPTR_DELAY, 0);
1629 
1630 	WREG32(CP_DEBUG, (1 << 27));
1631 
1632 	/* set the wb address whether it's enabled or not */
1633 	WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
1634 	WREG32(SCRATCH_UMSK, 0xff);
1635 
1636 	for (i = 0; i < 3; ++i) {
1637 		uint32_t rb_cntl;
1638 		uint64_t addr;
1639 
1640 		/* Set ring buffer size */
1641 		ring = &rdev->ring[ridx[i]];
1642 		rb_cntl = order_base_2(ring->ring_size / 8);
1643 		rb_cntl |= order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8;
1644 #ifdef __BIG_ENDIAN
1645 		rb_cntl |= BUF_SWAP_32BIT;
1646 #endif
1647 		WREG32(cp_rb_cntl[i], rb_cntl);
1648 
1649 		/* set the wb address whether it's enabled or not */
1650 		addr = rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET;
1651 		WREG32(cp_rb_rptr_addr[i], addr & 0xFFFFFFFC);
1652 		WREG32(cp_rb_rptr_addr_hi[i], upper_32_bits(addr) & 0xFF);
1653 	}
1654 
1655 	/* set the rb base addr, this causes an internal reset of ALL rings */
1656 	for (i = 0; i < 3; ++i) {
1657 		ring = &rdev->ring[ridx[i]];
1658 		WREG32(cp_rb_base[i], ring->gpu_addr >> 8);
1659 	}
1660 
1661 	for (i = 0; i < 3; ++i) {
1662 		/* Initialize the ring buffer's read and write pointers */
1663 		ring = &rdev->ring[ridx[i]];
1664 		WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
1665 
1666 		ring->wptr = 0;
1667 		WREG32(cp_rb_rptr[i], 0);
1668 		WREG32(cp_rb_wptr[i], ring->wptr);
1669 
1670 		mdelay(1);
1671 		WREG32_P(cp_rb_cntl[i], 0, ~RB_RPTR_WR_ENA);
1672 	}
1673 
1674 	/* start the rings */
1675 	cayman_cp_start(rdev);
1676 	rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
1677 	rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1678 	rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1679 	/* this only test cp0 */
1680 	r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
1681 	if (r) {
1682 		rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
1683 		rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
1684 		rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
1685 		return r;
1686 	}
1687 
1688 	if (rdev->asic->copy.copy_ring_index == RADEON_RING_TYPE_GFX_INDEX)
1689 		radeon_ttm_set_active_vram_size(rdev, rdev->mc.real_vram_size);
1690 
1691 	return 0;
1692 }
1693 
1694 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev)
1695 {
1696 	u32 reset_mask = 0;
1697 	u32 tmp;
1698 
1699 	/* GRBM_STATUS */
1700 	tmp = RREG32(GRBM_STATUS);
1701 	if (tmp & (PA_BUSY | SC_BUSY |
1702 		   SH_BUSY | SX_BUSY |
1703 		   TA_BUSY | VGT_BUSY |
1704 		   DB_BUSY | CB_BUSY |
1705 		   GDS_BUSY | SPI_BUSY |
1706 		   IA_BUSY | IA_BUSY_NO_DMA))
1707 		reset_mask |= RADEON_RESET_GFX;
1708 
1709 	if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
1710 		   CP_BUSY | CP_COHERENCY_BUSY))
1711 		reset_mask |= RADEON_RESET_CP;
1712 
1713 	if (tmp & GRBM_EE_BUSY)
1714 		reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
1715 
1716 	/* DMA_STATUS_REG 0 */
1717 	tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
1718 	if (!(tmp & DMA_IDLE))
1719 		reset_mask |= RADEON_RESET_DMA;
1720 
1721 	/* DMA_STATUS_REG 1 */
1722 	tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
1723 	if (!(tmp & DMA_IDLE))
1724 		reset_mask |= RADEON_RESET_DMA1;
1725 
1726 	/* SRBM_STATUS2 */
1727 	tmp = RREG32(SRBM_STATUS2);
1728 	if (tmp & DMA_BUSY)
1729 		reset_mask |= RADEON_RESET_DMA;
1730 
1731 	if (tmp & DMA1_BUSY)
1732 		reset_mask |= RADEON_RESET_DMA1;
1733 
1734 	/* SRBM_STATUS */
1735 	tmp = RREG32(SRBM_STATUS);
1736 	if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
1737 		reset_mask |= RADEON_RESET_RLC;
1738 
1739 	if (tmp & IH_BUSY)
1740 		reset_mask |= RADEON_RESET_IH;
1741 
1742 	if (tmp & SEM_BUSY)
1743 		reset_mask |= RADEON_RESET_SEM;
1744 
1745 	if (tmp & GRBM_RQ_PENDING)
1746 		reset_mask |= RADEON_RESET_GRBM;
1747 
1748 	if (tmp & VMC_BUSY)
1749 		reset_mask |= RADEON_RESET_VMC;
1750 
1751 	if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
1752 		   MCC_BUSY | MCD_BUSY))
1753 		reset_mask |= RADEON_RESET_MC;
1754 
1755 	if (evergreen_is_display_hung(rdev))
1756 		reset_mask |= RADEON_RESET_DISPLAY;
1757 
1758 	/* VM_L2_STATUS */
1759 	tmp = RREG32(VM_L2_STATUS);
1760 	if (tmp & L2_BUSY)
1761 		reset_mask |= RADEON_RESET_VMC;
1762 
1763 	/* Skip MC reset as it's mostly likely not hung, just busy */
1764 	if (reset_mask & RADEON_RESET_MC) {
1765 		DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
1766 		reset_mask &= ~RADEON_RESET_MC;
1767 	}
1768 
1769 	return reset_mask;
1770 }
1771 
1772 static void cayman_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
1773 {
1774 	struct evergreen_mc_save save;
1775 	u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
1776 	u32 tmp;
1777 
1778 	if (reset_mask == 0)
1779 		return;
1780 
1781 	dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
1782 
1783 	evergreen_print_gpu_status_regs(rdev);
1784 	dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_ADDR   0x%08X\n",
1785 		 RREG32(0x14F8));
1786 	dev_info(rdev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_STATUS 0x%08X\n",
1787 		 RREG32(0x14D8));
1788 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1789 		 RREG32(0x14FC));
1790 	dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1791 		 RREG32(0x14DC));
1792 
1793 	/* Disable CP parsing/prefetching */
1794 	WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
1795 
1796 	if (reset_mask & RADEON_RESET_DMA) {
1797 		/* dma0 */
1798 		tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
1799 		tmp &= ~DMA_RB_ENABLE;
1800 		WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
1801 	}
1802 
1803 	if (reset_mask & RADEON_RESET_DMA1) {
1804 		/* dma1 */
1805 		tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
1806 		tmp &= ~DMA_RB_ENABLE;
1807 		WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
1808 	}
1809 
1810 	udelay(50);
1811 
1812 	evergreen_mc_stop(rdev, &save);
1813 	if (evergreen_mc_wait_for_idle(rdev)) {
1814 		dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
1815 	}
1816 
1817 	if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
1818 		grbm_soft_reset = SOFT_RESET_CB |
1819 			SOFT_RESET_DB |
1820 			SOFT_RESET_GDS |
1821 			SOFT_RESET_PA |
1822 			SOFT_RESET_SC |
1823 			SOFT_RESET_SPI |
1824 			SOFT_RESET_SH |
1825 			SOFT_RESET_SX |
1826 			SOFT_RESET_TC |
1827 			SOFT_RESET_TA |
1828 			SOFT_RESET_VGT |
1829 			SOFT_RESET_IA;
1830 	}
1831 
1832 	if (reset_mask & RADEON_RESET_CP) {
1833 		grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
1834 
1835 		srbm_soft_reset |= SOFT_RESET_GRBM;
1836 	}
1837 
1838 	if (reset_mask & RADEON_RESET_DMA)
1839 		srbm_soft_reset |= SOFT_RESET_DMA;
1840 
1841 	if (reset_mask & RADEON_RESET_DMA1)
1842 		srbm_soft_reset |= SOFT_RESET_DMA1;
1843 
1844 	if (reset_mask & RADEON_RESET_DISPLAY)
1845 		srbm_soft_reset |= SOFT_RESET_DC;
1846 
1847 	if (reset_mask & RADEON_RESET_RLC)
1848 		srbm_soft_reset |= SOFT_RESET_RLC;
1849 
1850 	if (reset_mask & RADEON_RESET_SEM)
1851 		srbm_soft_reset |= SOFT_RESET_SEM;
1852 
1853 	if (reset_mask & RADEON_RESET_IH)
1854 		srbm_soft_reset |= SOFT_RESET_IH;
1855 
1856 	if (reset_mask & RADEON_RESET_GRBM)
1857 		srbm_soft_reset |= SOFT_RESET_GRBM;
1858 
1859 	if (reset_mask & RADEON_RESET_VMC)
1860 		srbm_soft_reset |= SOFT_RESET_VMC;
1861 
1862 	if (!(rdev->flags & RADEON_IS_IGP)) {
1863 		if (reset_mask & RADEON_RESET_MC)
1864 			srbm_soft_reset |= SOFT_RESET_MC;
1865 	}
1866 
1867 	if (grbm_soft_reset) {
1868 		tmp = RREG32(GRBM_SOFT_RESET);
1869 		tmp |= grbm_soft_reset;
1870 		dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
1871 		WREG32(GRBM_SOFT_RESET, tmp);
1872 		tmp = RREG32(GRBM_SOFT_RESET);
1873 
1874 		udelay(50);
1875 
1876 		tmp &= ~grbm_soft_reset;
1877 		WREG32(GRBM_SOFT_RESET, tmp);
1878 		tmp = RREG32(GRBM_SOFT_RESET);
1879 	}
1880 
1881 	if (srbm_soft_reset) {
1882 		tmp = RREG32(SRBM_SOFT_RESET);
1883 		tmp |= srbm_soft_reset;
1884 		dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1885 		WREG32(SRBM_SOFT_RESET, tmp);
1886 		tmp = RREG32(SRBM_SOFT_RESET);
1887 
1888 		udelay(50);
1889 
1890 		tmp &= ~srbm_soft_reset;
1891 		WREG32(SRBM_SOFT_RESET, tmp);
1892 		tmp = RREG32(SRBM_SOFT_RESET);
1893 	}
1894 
1895 	/* Wait a little for things to settle down */
1896 	udelay(50);
1897 
1898 	evergreen_mc_resume(rdev, &save);
1899 	udelay(50);
1900 
1901 	evergreen_print_gpu_status_regs(rdev);
1902 }
1903 
1904 int cayman_asic_reset(struct radeon_device *rdev)
1905 {
1906 	u32 reset_mask;
1907 
1908 	reset_mask = cayman_gpu_check_soft_reset(rdev);
1909 
1910 	if (reset_mask)
1911 		r600_set_bios_scratch_engine_hung(rdev, true);
1912 
1913 	cayman_gpu_soft_reset(rdev, reset_mask);
1914 
1915 	reset_mask = cayman_gpu_check_soft_reset(rdev);
1916 
1917 	if (reset_mask)
1918 		evergreen_gpu_pci_config_reset(rdev);
1919 
1920 	r600_set_bios_scratch_engine_hung(rdev, false);
1921 
1922 	return 0;
1923 }
1924 
1925 /**
1926  * cayman_gfx_is_lockup - Check if the GFX engine is locked up
1927  *
1928  * @rdev: radeon_device pointer
1929  * @ring: radeon_ring structure holding ring information
1930  *
1931  * Check if the GFX engine is locked up.
1932  * Returns true if the engine appears to be locked up, false if not.
1933  */
1934 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
1935 {
1936 	u32 reset_mask = cayman_gpu_check_soft_reset(rdev);
1937 
1938 	if (!(reset_mask & (RADEON_RESET_GFX |
1939 			    RADEON_RESET_COMPUTE |
1940 			    RADEON_RESET_CP))) {
1941 		radeon_ring_lockup_update(rdev, ring);
1942 		return false;
1943 	}
1944 	return radeon_ring_test_lockup(rdev, ring);
1945 }
1946 
1947 static int cayman_startup(struct radeon_device *rdev)
1948 {
1949 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
1950 	int r;
1951 
1952 	/* enable pcie gen2 link */
1953 	evergreen_pcie_gen2_enable(rdev);
1954 	/* enable aspm */
1955 	evergreen_program_aspm(rdev);
1956 
1957 	/* scratch needs to be initialized before MC */
1958 	r = r600_vram_scratch_init(rdev);
1959 	if (r)
1960 		return r;
1961 
1962 	evergreen_mc_program(rdev);
1963 
1964 	if (!(rdev->flags & RADEON_IS_IGP) && !rdev->pm.dpm_enabled) {
1965 		r = ni_mc_load_microcode(rdev);
1966 		if (r) {
1967 			DRM_ERROR("Failed to load MC firmware!\n");
1968 			return r;
1969 		}
1970 	}
1971 
1972 	r = cayman_pcie_gart_enable(rdev);
1973 	if (r)
1974 		return r;
1975 	cayman_gpu_init(rdev);
1976 
1977 	/* allocate rlc buffers */
1978 	if (rdev->flags & RADEON_IS_IGP) {
1979 		rdev->rlc.reg_list = tn_rlc_save_restore_register_list;
1980 		rdev->rlc.reg_list_size =
1981 			(u32)ARRAY_SIZE(tn_rlc_save_restore_register_list);
1982 		rdev->rlc.cs_data = cayman_cs_data;
1983 		r = sumo_rlc_init(rdev);
1984 		if (r) {
1985 			DRM_ERROR("Failed to init rlc BOs!\n");
1986 			return r;
1987 		}
1988 	}
1989 
1990 	/* allocate wb buffer */
1991 	r = radeon_wb_init(rdev);
1992 	if (r)
1993 		return r;
1994 
1995 	r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
1996 	if (r) {
1997 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
1998 		return r;
1999 	}
2000 
2001 	r = uvd_v2_2_resume(rdev);
2002 	if (!r) {
2003 		r = radeon_fence_driver_start_ring(rdev,
2004 						   R600_RING_TYPE_UVD_INDEX);
2005 		if (r)
2006 			dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
2007 	}
2008 	if (r)
2009 		rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
2010 
2011 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
2012 	if (r) {
2013 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2014 		return r;
2015 	}
2016 
2017 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
2018 	if (r) {
2019 		dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
2020 		return r;
2021 	}
2022 
2023 	r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
2024 	if (r) {
2025 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2026 		return r;
2027 	}
2028 
2029 	r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
2030 	if (r) {
2031 		dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
2032 		return r;
2033 	}
2034 
2035 	/* Enable IRQ */
2036 	if (!rdev->irq.installed) {
2037 		r = radeon_irq_kms_init(rdev);
2038 		if (r)
2039 			return r;
2040 	}
2041 
2042 	r = r600_irq_init(rdev);
2043 	if (r) {
2044 		DRM_ERROR("radeon: IH init failed (%d).\n", r);
2045 		radeon_irq_kms_fini(rdev);
2046 		return r;
2047 	}
2048 	evergreen_irq_set(rdev);
2049 
2050 	r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
2051 			     RADEON_CP_PACKET2);
2052 	if (r)
2053 		return r;
2054 
2055 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2056 	r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
2057 			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2058 	if (r)
2059 		return r;
2060 
2061 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2062 	r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
2063 			     DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0));
2064 	if (r)
2065 		return r;
2066 
2067 	r = cayman_cp_load_microcode(rdev);
2068 	if (r)
2069 		return r;
2070 	r = cayman_cp_resume(rdev);
2071 	if (r)
2072 		return r;
2073 
2074 	r = cayman_dma_resume(rdev);
2075 	if (r)
2076 		return r;
2077 
2078 	ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2079 	if (ring->ring_size) {
2080 		r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
2081 				     RADEON_CP_PACKET2);
2082 		if (!r)
2083 			r = uvd_v1_0_init(rdev);
2084 		if (r)
2085 			DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
2086 	}
2087 
2088 	r = radeon_ib_pool_init(rdev);
2089 	if (r) {
2090 		dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
2091 		return r;
2092 	}
2093 
2094 	r = radeon_vm_manager_init(rdev);
2095 	if (r) {
2096 		dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
2097 		return r;
2098 	}
2099 
2100 	if (ASIC_IS_DCE6(rdev)) {
2101 		r = dce6_audio_init(rdev);
2102 		if (r)
2103 			return r;
2104 	} else {
2105 		r = r600_audio_init(rdev);
2106 		if (r)
2107 			return r;
2108 	}
2109 
2110 	return 0;
2111 }
2112 
2113 int cayman_resume(struct radeon_device *rdev)
2114 {
2115 	int r;
2116 
2117 	/* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
2118 	 * posting will perform necessary task to bring back GPU into good
2119 	 * shape.
2120 	 */
2121 	/* post card */
2122 	atom_asic_init(rdev->mode_info.atom_context);
2123 
2124 	/* init golden registers */
2125 	ni_init_golden_registers(rdev);
2126 
2127 	if (rdev->pm.pm_method == PM_METHOD_DPM)
2128 		radeon_pm_resume(rdev);
2129 
2130 	rdev->accel_working = true;
2131 	r = cayman_startup(rdev);
2132 	if (r) {
2133 		DRM_ERROR("cayman startup failed on resume\n");
2134 		rdev->accel_working = false;
2135 		return r;
2136 	}
2137 	return r;
2138 }
2139 
2140 int cayman_suspend(struct radeon_device *rdev)
2141 {
2142 	radeon_pm_suspend(rdev);
2143 	if (ASIC_IS_DCE6(rdev))
2144 		dce6_audio_fini(rdev);
2145 	else
2146 		r600_audio_fini(rdev);
2147 	radeon_vm_manager_fini(rdev);
2148 	cayman_cp_enable(rdev, false);
2149 	cayman_dma_stop(rdev);
2150 	uvd_v1_0_fini(rdev);
2151 	radeon_uvd_suspend(rdev);
2152 	evergreen_irq_suspend(rdev);
2153 	radeon_wb_disable(rdev);
2154 	cayman_pcie_gart_disable(rdev);
2155 	return 0;
2156 }
2157 
2158 /* Plan is to move initialization in that function and use
2159  * helper function so that radeon_device_init pretty much
2160  * do nothing more than calling asic specific function. This
2161  * should also allow to remove a bunch of callback function
2162  * like vram_info.
2163  */
2164 int cayman_init(struct radeon_device *rdev)
2165 {
2166 	struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2167 	int r;
2168 
2169 	/* Read BIOS */
2170 	if (!radeon_get_bios(rdev)) {
2171 		if (ASIC_IS_AVIVO(rdev))
2172 			return -EINVAL;
2173 	}
2174 	/* Must be an ATOMBIOS */
2175 	if (!rdev->is_atom_bios) {
2176 		dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
2177 		return -EINVAL;
2178 	}
2179 	r = radeon_atombios_init(rdev);
2180 	if (r)
2181 		return r;
2182 
2183 	/* Post card if necessary */
2184 	if (!radeon_card_posted(rdev)) {
2185 		if (!rdev->bios) {
2186 			dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
2187 			return -EINVAL;
2188 		}
2189 		DRM_INFO("GPU not posted. posting now...\n");
2190 		atom_asic_init(rdev->mode_info.atom_context);
2191 	}
2192 	/* init golden registers */
2193 	ni_init_golden_registers(rdev);
2194 	/* Initialize scratch registers */
2195 	r600_scratch_init(rdev);
2196 	/* Initialize surface registers */
2197 	radeon_surface_init(rdev);
2198 	/* Initialize clocks */
2199 	radeon_get_clock_info(rdev->ddev);
2200 	/* Fence driver */
2201 	r = radeon_fence_driver_init(rdev);
2202 	if (r)
2203 		return r;
2204 	/* initialize memory controller */
2205 	r = evergreen_mc_init(rdev);
2206 	if (r)
2207 		return r;
2208 	/* Memory manager */
2209 	r = radeon_bo_init(rdev);
2210 	if (r)
2211 		return r;
2212 
2213 	if (rdev->flags & RADEON_IS_IGP) {
2214 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
2215 			r = ni_init_microcode(rdev);
2216 			if (r) {
2217 				DRM_ERROR("Failed to load firmware!\n");
2218 				return r;
2219 			}
2220 		}
2221 	} else {
2222 		if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
2223 			r = ni_init_microcode(rdev);
2224 			if (r) {
2225 				DRM_ERROR("Failed to load firmware!\n");
2226 				return r;
2227 			}
2228 		}
2229 	}
2230 
2231 	/* Initialize power management */
2232 	radeon_pm_init(rdev);
2233 
2234 	ring->ring_obj = NULL;
2235 	r600_ring_init(rdev, ring, 1024 * 1024);
2236 
2237 	ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
2238 	ring->ring_obj = NULL;
2239 	r600_ring_init(rdev, ring, 64 * 1024);
2240 
2241 	ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
2242 	ring->ring_obj = NULL;
2243 	r600_ring_init(rdev, ring, 64 * 1024);
2244 
2245 	r = radeon_uvd_init(rdev);
2246 	if (!r) {
2247 		ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
2248 		ring->ring_obj = NULL;
2249 		r600_ring_init(rdev, ring, 4096);
2250 	}
2251 
2252 	rdev->ih.ring_obj = NULL;
2253 	r600_ih_ring_init(rdev, 64 * 1024);
2254 
2255 	r = r600_pcie_gart_init(rdev);
2256 	if (r)
2257 		return r;
2258 
2259 	rdev->accel_working = true;
2260 	r = cayman_startup(rdev);
2261 	if (r) {
2262 		dev_err(rdev->dev, "disabling GPU acceleration\n");
2263 		cayman_cp_fini(rdev);
2264 		cayman_dma_fini(rdev);
2265 		r600_irq_fini(rdev);
2266 		if (rdev->flags & RADEON_IS_IGP)
2267 			sumo_rlc_fini(rdev);
2268 		radeon_wb_fini(rdev);
2269 		radeon_ib_pool_fini(rdev);
2270 		radeon_vm_manager_fini(rdev);
2271 		radeon_irq_kms_fini(rdev);
2272 		cayman_pcie_gart_fini(rdev);
2273 		rdev->accel_working = false;
2274 	}
2275 
2276 	/* Don't start up if the MC ucode is missing.
2277 	 * The default clocks and voltages before the MC ucode
2278 	 * is loaded are not suffient for advanced operations.
2279 	 *
2280 	 * We can skip this check for TN, because there is no MC
2281 	 * ucode.
2282 	 */
2283 	if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
2284 		DRM_ERROR("radeon: MC ucode required for NI+.\n");
2285 		return -EINVAL;
2286 	}
2287 
2288 	return 0;
2289 }
2290 
2291 void cayman_fini(struct radeon_device *rdev)
2292 {
2293 	radeon_pm_fini(rdev);
2294 	cayman_cp_fini(rdev);
2295 	cayman_dma_fini(rdev);
2296 	r600_irq_fini(rdev);
2297 	if (rdev->flags & RADEON_IS_IGP)
2298 		sumo_rlc_fini(rdev);
2299 	radeon_wb_fini(rdev);
2300 	radeon_vm_manager_fini(rdev);
2301 	radeon_ib_pool_fini(rdev);
2302 	radeon_irq_kms_fini(rdev);
2303 	uvd_v1_0_fini(rdev);
2304 	radeon_uvd_fini(rdev);
2305 	cayman_pcie_gart_fini(rdev);
2306 	r600_vram_scratch_fini(rdev);
2307 	radeon_gem_fini(rdev);
2308 	radeon_fence_driver_fini(rdev);
2309 	radeon_bo_fini(rdev);
2310 	radeon_atombios_fini(rdev);
2311 	kfree(rdev->bios);
2312 	rdev->bios = NULL;
2313 }
2314 
2315 /*
2316  * vm
2317  */
2318 int cayman_vm_init(struct radeon_device *rdev)
2319 {
2320 	/* number of VMs */
2321 	rdev->vm_manager.nvm = 8;
2322 	/* base offset of vram pages */
2323 	if (rdev->flags & RADEON_IS_IGP) {
2324 		u64 tmp = RREG32(FUS_MC_VM_FB_OFFSET);
2325 		tmp <<= 22;
2326 		rdev->vm_manager.vram_base_offset = tmp;
2327 	} else
2328 		rdev->vm_manager.vram_base_offset = 0;
2329 	return 0;
2330 }
2331 
2332 void cayman_vm_fini(struct radeon_device *rdev)
2333 {
2334 }
2335 
2336 /**
2337  * cayman_vm_decode_fault - print human readable fault info
2338  *
2339  * @rdev: radeon_device pointer
2340  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
2341  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
2342  *
2343  * Print human readable fault information (cayman/TN).
2344  */
2345 void cayman_vm_decode_fault(struct radeon_device *rdev,
2346 			    u32 status, u32 addr)
2347 {
2348 	u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
2349 	u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
2350 	u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
2351 	char *block;
2352 
2353 	switch (mc_id) {
2354 	case 32:
2355 	case 16:
2356 	case 96:
2357 	case 80:
2358 	case 160:
2359 	case 144:
2360 	case 224:
2361 	case 208:
2362 		block = "CB";
2363 		break;
2364 	case 33:
2365 	case 17:
2366 	case 97:
2367 	case 81:
2368 	case 161:
2369 	case 145:
2370 	case 225:
2371 	case 209:
2372 		block = "CB_FMASK";
2373 		break;
2374 	case 34:
2375 	case 18:
2376 	case 98:
2377 	case 82:
2378 	case 162:
2379 	case 146:
2380 	case 226:
2381 	case 210:
2382 		block = "CB_CMASK";
2383 		break;
2384 	case 35:
2385 	case 19:
2386 	case 99:
2387 	case 83:
2388 	case 163:
2389 	case 147:
2390 	case 227:
2391 	case 211:
2392 		block = "CB_IMMED";
2393 		break;
2394 	case 36:
2395 	case 20:
2396 	case 100:
2397 	case 84:
2398 	case 164:
2399 	case 148:
2400 	case 228:
2401 	case 212:
2402 		block = "DB";
2403 		break;
2404 	case 37:
2405 	case 21:
2406 	case 101:
2407 	case 85:
2408 	case 165:
2409 	case 149:
2410 	case 229:
2411 	case 213:
2412 		block = "DB_HTILE";
2413 		break;
2414 	case 38:
2415 	case 22:
2416 	case 102:
2417 	case 86:
2418 	case 166:
2419 	case 150:
2420 	case 230:
2421 	case 214:
2422 		block = "SX";
2423 		break;
2424 	case 39:
2425 	case 23:
2426 	case 103:
2427 	case 87:
2428 	case 167:
2429 	case 151:
2430 	case 231:
2431 	case 215:
2432 		block = "DB_STEN";
2433 		break;
2434 	case 40:
2435 	case 24:
2436 	case 104:
2437 	case 88:
2438 	case 232:
2439 	case 216:
2440 	case 168:
2441 	case 152:
2442 		block = "TC_TFETCH";
2443 		break;
2444 	case 41:
2445 	case 25:
2446 	case 105:
2447 	case 89:
2448 	case 233:
2449 	case 217:
2450 	case 169:
2451 	case 153:
2452 		block = "TC_VFETCH";
2453 		break;
2454 	case 42:
2455 	case 26:
2456 	case 106:
2457 	case 90:
2458 	case 234:
2459 	case 218:
2460 	case 170:
2461 	case 154:
2462 		block = "VC";
2463 		break;
2464 	case 112:
2465 		block = "CP";
2466 		break;
2467 	case 113:
2468 	case 114:
2469 		block = "SH";
2470 		break;
2471 	case 115:
2472 		block = "VGT";
2473 		break;
2474 	case 178:
2475 		block = "IH";
2476 		break;
2477 	case 51:
2478 		block = "RLC";
2479 		break;
2480 	case 55:
2481 		block = "DMA";
2482 		break;
2483 	case 56:
2484 		block = "HDP";
2485 		break;
2486 	default:
2487 		block = "unknown";
2488 		break;
2489 	}
2490 
2491 	printk("VM fault (0x%02x, vmid %d) at page %u, %s from %s (%d)\n",
2492 	       protections, vmid, addr,
2493 	       (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
2494 	       block, mc_id);
2495 }
2496 
2497 /**
2498  * cayman_vm_flush - vm flush using the CP
2499  *
2500  * @rdev: radeon_device pointer
2501  *
2502  * Update the page table base and flush the VM TLB
2503  * using the CP (cayman-si).
2504  */
2505 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
2506 {
2507 	struct radeon_ring *ring = &rdev->ring[ridx];
2508 
2509 	if (vm == NULL)
2510 		return;
2511 
2512 	radeon_ring_write(ring, PACKET0(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2), 0));
2513 	radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
2514 
2515 	/* flush hdp cache */
2516 	radeon_ring_write(ring, PACKET0(HDP_MEM_COHERENCY_FLUSH_CNTL, 0));
2517 	radeon_ring_write(ring, 0x1);
2518 
2519 	/* bits 0-7 are the VM contexts0-7 */
2520 	radeon_ring_write(ring, PACKET0(VM_INVALIDATE_REQUEST, 0));
2521 	radeon_ring_write(ring, 1 << vm->id);
2522 
2523 	/* sync PFP to ME, otherwise we might get invalid PFP reads */
2524 	radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
2525 	radeon_ring_write(ring, 0x0);
2526 }
2527