xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision 8d81cd1a) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include <drm/amdgpu_drm.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_atombios.h"
33 #include "amdgpu_ih.h"
34 #include "amdgpu_uvd.h"
35 #include "amdgpu_vce.h"
36 #include "cikd.h"
37 #include "atom.h"
38 #include "amd_pcie.h"
39 
40 #include "cik.h"
41 #include "gmc_v7_0.h"
42 #include "cik_ih.h"
43 #include "dce_v8_0.h"
44 #include "gfx_v7_0.h"
45 #include "cik_sdma.h"
46 #include "uvd_v4_2.h"
47 #include "vce_v2_0.h"
48 #include "cik_dpm.h"
49 
50 #include "uvd/uvd_4_2_d.h"
51 
52 #include "smu/smu_7_0_1_d.h"
53 #include "smu/smu_7_0_1_sh_mask.h"
54 
55 #include "dce/dce_8_0_d.h"
56 #include "dce/dce_8_0_sh_mask.h"
57 
58 #include "bif/bif_4_1_d.h"
59 #include "bif/bif_4_1_sh_mask.h"
60 
61 #include "gca/gfx_7_2_d.h"
62 #include "gca/gfx_7_2_enum.h"
63 #include "gca/gfx_7_2_sh_mask.h"
64 
65 #include "gmc/gmc_7_1_d.h"
66 #include "gmc/gmc_7_1_sh_mask.h"
67 
68 #include "oss/oss_2_0_d.h"
69 #include "oss/oss_2_0_sh_mask.h"
70 
71 #include "amdgpu_dm.h"
72 #include "amdgpu_amdkfd.h"
73 #include "amdgpu_vkms.h"
74 
75 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
76 {
77 	{
78 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
79 		.max_width = 2048,
80 		.max_height = 1152,
81 		.max_pixels_per_frame = 2048 * 1152,
82 		.max_level = 0,
83 	},
84 };
85 
86 static const struct amdgpu_video_codecs cik_video_codecs_encode =
87 {
88 	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
89 	.codec_array = cik_video_codecs_encode_array,
90 };
91 
92 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
93 {
94 	{
95 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
96 		.max_width = 2048,
97 		.max_height = 1152,
98 		.max_pixels_per_frame = 2048 * 1152,
99 		.max_level = 3,
100 	},
101 	{
102 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
103 		.max_width = 2048,
104 		.max_height = 1152,
105 		.max_pixels_per_frame = 2048 * 1152,
106 		.max_level = 5,
107 	},
108 	{
109 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
110 		.max_width = 2048,
111 		.max_height = 1152,
112 		.max_pixels_per_frame = 2048 * 1152,
113 		.max_level = 41,
114 	},
115 	{
116 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
117 		.max_width = 2048,
118 		.max_height = 1152,
119 		.max_pixels_per_frame = 2048 * 1152,
120 		.max_level = 4,
121 	},
122 };
123 
124 static const struct amdgpu_video_codecs cik_video_codecs_decode =
125 {
126 	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
127 	.codec_array = cik_video_codecs_decode_array,
128 };
129 
130 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
131 				  const struct amdgpu_video_codecs **codecs)
132 {
133 	switch (adev->asic_type) {
134 	case CHIP_BONAIRE:
135 	case CHIP_HAWAII:
136 	case CHIP_KAVERI:
137 	case CHIP_KABINI:
138 	case CHIP_MULLINS:
139 		if (encode)
140 			*codecs = &cik_video_codecs_encode;
141 		else
142 			*codecs = &cik_video_codecs_decode;
143 		return 0;
144 	default:
145 		return -EINVAL;
146 	}
147 }
148 
149 /*
150  * Indirect registers accessor
151  */
152 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
153 {
154 	unsigned long flags;
155 	u32 r;
156 
157 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
158 	WREG32(mmPCIE_INDEX, reg);
159 	(void)RREG32(mmPCIE_INDEX);
160 	r = RREG32(mmPCIE_DATA);
161 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
162 	return r;
163 }
164 
165 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
166 {
167 	unsigned long flags;
168 
169 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
170 	WREG32(mmPCIE_INDEX, reg);
171 	(void)RREG32(mmPCIE_INDEX);
172 	WREG32(mmPCIE_DATA, v);
173 	(void)RREG32(mmPCIE_DATA);
174 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
175 }
176 
177 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
178 {
179 	unsigned long flags;
180 	u32 r;
181 
182 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
183 	WREG32(mmSMC_IND_INDEX_0, (reg));
184 	r = RREG32(mmSMC_IND_DATA_0);
185 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
186 	return r;
187 }
188 
189 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
190 {
191 	unsigned long flags;
192 
193 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
194 	WREG32(mmSMC_IND_INDEX_0, (reg));
195 	WREG32(mmSMC_IND_DATA_0, (v));
196 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
197 }
198 
199 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
200 {
201 	unsigned long flags;
202 	u32 r;
203 
204 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
205 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
206 	r = RREG32(mmUVD_CTX_DATA);
207 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
208 	return r;
209 }
210 
211 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
212 {
213 	unsigned long flags;
214 
215 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
216 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
217 	WREG32(mmUVD_CTX_DATA, (v));
218 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
219 }
220 
221 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
222 {
223 	unsigned long flags;
224 	u32 r;
225 
226 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
227 	WREG32(mmDIDT_IND_INDEX, (reg));
228 	r = RREG32(mmDIDT_IND_DATA);
229 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
230 	return r;
231 }
232 
233 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
234 {
235 	unsigned long flags;
236 
237 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
238 	WREG32(mmDIDT_IND_INDEX, (reg));
239 	WREG32(mmDIDT_IND_DATA, (v));
240 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
241 }
242 
243 static const u32 bonaire_golden_spm_registers[] =
244 {
245 	0xc200, 0xe0ffffff, 0xe0000000
246 };
247 
248 static const u32 bonaire_golden_common_registers[] =
249 {
250 	0x31dc, 0xffffffff, 0x00000800,
251 	0x31dd, 0xffffffff, 0x00000800,
252 	0x31e6, 0xffffffff, 0x00007fbf,
253 	0x31e7, 0xffffffff, 0x00007faf
254 };
255 
256 static const u32 bonaire_golden_registers[] =
257 {
258 	0xcd5, 0x00000333, 0x00000333,
259 	0xcd4, 0x000c0fc0, 0x00040200,
260 	0x2684, 0x00010000, 0x00058208,
261 	0xf000, 0xffff1fff, 0x00140000,
262 	0xf080, 0xfdfc0fff, 0x00000100,
263 	0xf08d, 0x40000000, 0x40000200,
264 	0x260c, 0xffffffff, 0x00000000,
265 	0x260d, 0xf00fffff, 0x00000400,
266 	0x260e, 0x0002021c, 0x00020200,
267 	0x31e, 0x00000080, 0x00000000,
268 	0x16ec, 0x000000f0, 0x00000070,
269 	0x16f0, 0xf0311fff, 0x80300000,
270 	0x263e, 0x73773777, 0x12010001,
271 	0xd43, 0x00810000, 0x408af000,
272 	0x1c0c, 0x31000111, 0x00000011,
273 	0xbd2, 0x73773777, 0x12010001,
274 	0x883, 0x00007fb6, 0x0021a1b1,
275 	0x884, 0x00007fb6, 0x002021b1,
276 	0x860, 0x00007fb6, 0x00002191,
277 	0x886, 0x00007fb6, 0x002121b1,
278 	0x887, 0x00007fb6, 0x002021b1,
279 	0x877, 0x00007fb6, 0x00002191,
280 	0x878, 0x00007fb6, 0x00002191,
281 	0xd8a, 0x0000003f, 0x0000000a,
282 	0xd8b, 0x0000003f, 0x0000000a,
283 	0xab9, 0x00073ffe, 0x000022a2,
284 	0x903, 0x000007ff, 0x00000000,
285 	0x2285, 0xf000003f, 0x00000007,
286 	0x22fc, 0x00002001, 0x00000001,
287 	0x22c9, 0xffffffff, 0x00ffffff,
288 	0xc281, 0x0000ff0f, 0x00000000,
289 	0xa293, 0x07ffffff, 0x06000000,
290 	0x136, 0x00000fff, 0x00000100,
291 	0xf9e, 0x00000001, 0x00000002,
292 	0x2440, 0x03000000, 0x0362c688,
293 	0x2300, 0x000000ff, 0x00000001,
294 	0x390, 0x00001fff, 0x00001fff,
295 	0x2418, 0x0000007f, 0x00000020,
296 	0x2542, 0x00010000, 0x00010000,
297 	0x2b05, 0x000003ff, 0x000000f3,
298 	0x2b03, 0xffffffff, 0x00001032
299 };
300 
301 static const u32 bonaire_mgcg_cgcg_init[] =
302 {
303 	0x3108, 0xffffffff, 0xfffffffc,
304 	0xc200, 0xffffffff, 0xe0000000,
305 	0xf0a8, 0xffffffff, 0x00000100,
306 	0xf082, 0xffffffff, 0x00000100,
307 	0xf0b0, 0xffffffff, 0xc0000100,
308 	0xf0b2, 0xffffffff, 0xc0000100,
309 	0xf0b1, 0xffffffff, 0xc0000100,
310 	0x1579, 0xffffffff, 0x00600100,
311 	0xf0a0, 0xffffffff, 0x00000100,
312 	0xf085, 0xffffffff, 0x06000100,
313 	0xf088, 0xffffffff, 0x00000100,
314 	0xf086, 0xffffffff, 0x06000100,
315 	0xf081, 0xffffffff, 0x00000100,
316 	0xf0b8, 0xffffffff, 0x00000100,
317 	0xf089, 0xffffffff, 0x00000100,
318 	0xf080, 0xffffffff, 0x00000100,
319 	0xf08c, 0xffffffff, 0x00000100,
320 	0xf08d, 0xffffffff, 0x00000100,
321 	0xf094, 0xffffffff, 0x00000100,
322 	0xf095, 0xffffffff, 0x00000100,
323 	0xf096, 0xffffffff, 0x00000100,
324 	0xf097, 0xffffffff, 0x00000100,
325 	0xf098, 0xffffffff, 0x00000100,
326 	0xf09f, 0xffffffff, 0x00000100,
327 	0xf09e, 0xffffffff, 0x00000100,
328 	0xf084, 0xffffffff, 0x06000100,
329 	0xf0a4, 0xffffffff, 0x00000100,
330 	0xf09d, 0xffffffff, 0x00000100,
331 	0xf0ad, 0xffffffff, 0x00000100,
332 	0xf0ac, 0xffffffff, 0x00000100,
333 	0xf09c, 0xffffffff, 0x00000100,
334 	0xc200, 0xffffffff, 0xe0000000,
335 	0xf008, 0xffffffff, 0x00010000,
336 	0xf009, 0xffffffff, 0x00030002,
337 	0xf00a, 0xffffffff, 0x00040007,
338 	0xf00b, 0xffffffff, 0x00060005,
339 	0xf00c, 0xffffffff, 0x00090008,
340 	0xf00d, 0xffffffff, 0x00010000,
341 	0xf00e, 0xffffffff, 0x00030002,
342 	0xf00f, 0xffffffff, 0x00040007,
343 	0xf010, 0xffffffff, 0x00060005,
344 	0xf011, 0xffffffff, 0x00090008,
345 	0xf012, 0xffffffff, 0x00010000,
346 	0xf013, 0xffffffff, 0x00030002,
347 	0xf014, 0xffffffff, 0x00040007,
348 	0xf015, 0xffffffff, 0x00060005,
349 	0xf016, 0xffffffff, 0x00090008,
350 	0xf017, 0xffffffff, 0x00010000,
351 	0xf018, 0xffffffff, 0x00030002,
352 	0xf019, 0xffffffff, 0x00040007,
353 	0xf01a, 0xffffffff, 0x00060005,
354 	0xf01b, 0xffffffff, 0x00090008,
355 	0xf01c, 0xffffffff, 0x00010000,
356 	0xf01d, 0xffffffff, 0x00030002,
357 	0xf01e, 0xffffffff, 0x00040007,
358 	0xf01f, 0xffffffff, 0x00060005,
359 	0xf020, 0xffffffff, 0x00090008,
360 	0xf021, 0xffffffff, 0x00010000,
361 	0xf022, 0xffffffff, 0x00030002,
362 	0xf023, 0xffffffff, 0x00040007,
363 	0xf024, 0xffffffff, 0x00060005,
364 	0xf025, 0xffffffff, 0x00090008,
365 	0xf026, 0xffffffff, 0x00010000,
366 	0xf027, 0xffffffff, 0x00030002,
367 	0xf028, 0xffffffff, 0x00040007,
368 	0xf029, 0xffffffff, 0x00060005,
369 	0xf02a, 0xffffffff, 0x00090008,
370 	0xf000, 0xffffffff, 0x96e00200,
371 	0x21c2, 0xffffffff, 0x00900100,
372 	0x3109, 0xffffffff, 0x0020003f,
373 	0xe, 0xffffffff, 0x0140001c,
374 	0xf, 0x000f0000, 0x000f0000,
375 	0x88, 0xffffffff, 0xc060000c,
376 	0x89, 0xc0000fff, 0x00000100,
377 	0x3e4, 0xffffffff, 0x00000100,
378 	0x3e6, 0x00000101, 0x00000000,
379 	0x82a, 0xffffffff, 0x00000104,
380 	0x1579, 0xff000fff, 0x00000100,
381 	0xc33, 0xc0000fff, 0x00000104,
382 	0x3079, 0x00000001, 0x00000001,
383 	0x3403, 0xff000ff0, 0x00000100,
384 	0x3603, 0xff000ff0, 0x00000100
385 };
386 
387 static const u32 spectre_golden_spm_registers[] =
388 {
389 	0xc200, 0xe0ffffff, 0xe0000000
390 };
391 
392 static const u32 spectre_golden_common_registers[] =
393 {
394 	0x31dc, 0xffffffff, 0x00000800,
395 	0x31dd, 0xffffffff, 0x00000800,
396 	0x31e6, 0xffffffff, 0x00007fbf,
397 	0x31e7, 0xffffffff, 0x00007faf
398 };
399 
400 static const u32 spectre_golden_registers[] =
401 {
402 	0xf000, 0xffff1fff, 0x96940200,
403 	0xf003, 0xffff0001, 0xff000000,
404 	0xf080, 0xfffc0fff, 0x00000100,
405 	0x1bb6, 0x00010101, 0x00010000,
406 	0x260d, 0xf00fffff, 0x00000400,
407 	0x260e, 0xfffffffc, 0x00020200,
408 	0x16ec, 0x000000f0, 0x00000070,
409 	0x16f0, 0xf0311fff, 0x80300000,
410 	0x263e, 0x73773777, 0x12010001,
411 	0x26df, 0x00ff0000, 0x00fc0000,
412 	0xbd2, 0x73773777, 0x12010001,
413 	0x2285, 0xf000003f, 0x00000007,
414 	0x22c9, 0xffffffff, 0x00ffffff,
415 	0xa0d4, 0x3f3f3fff, 0x00000082,
416 	0xa0d5, 0x0000003f, 0x00000000,
417 	0xf9e, 0x00000001, 0x00000002,
418 	0x244f, 0xffff03df, 0x00000004,
419 	0x31da, 0x00000008, 0x00000008,
420 	0x2300, 0x000008ff, 0x00000800,
421 	0x2542, 0x00010000, 0x00010000,
422 	0x2b03, 0xffffffff, 0x54763210,
423 	0x853e, 0x01ff01ff, 0x00000002,
424 	0x8526, 0x007ff800, 0x00200000,
425 	0x8057, 0xffffffff, 0x00000f40,
426 	0xc24d, 0xffffffff, 0x00000001
427 };
428 
429 static const u32 spectre_mgcg_cgcg_init[] =
430 {
431 	0x3108, 0xffffffff, 0xfffffffc,
432 	0xc200, 0xffffffff, 0xe0000000,
433 	0xf0a8, 0xffffffff, 0x00000100,
434 	0xf082, 0xffffffff, 0x00000100,
435 	0xf0b0, 0xffffffff, 0x00000100,
436 	0xf0b2, 0xffffffff, 0x00000100,
437 	0xf0b1, 0xffffffff, 0x00000100,
438 	0x1579, 0xffffffff, 0x00600100,
439 	0xf0a0, 0xffffffff, 0x00000100,
440 	0xf085, 0xffffffff, 0x06000100,
441 	0xf088, 0xffffffff, 0x00000100,
442 	0xf086, 0xffffffff, 0x06000100,
443 	0xf081, 0xffffffff, 0x00000100,
444 	0xf0b8, 0xffffffff, 0x00000100,
445 	0xf089, 0xffffffff, 0x00000100,
446 	0xf080, 0xffffffff, 0x00000100,
447 	0xf08c, 0xffffffff, 0x00000100,
448 	0xf08d, 0xffffffff, 0x00000100,
449 	0xf094, 0xffffffff, 0x00000100,
450 	0xf095, 0xffffffff, 0x00000100,
451 	0xf096, 0xffffffff, 0x00000100,
452 	0xf097, 0xffffffff, 0x00000100,
453 	0xf098, 0xffffffff, 0x00000100,
454 	0xf09f, 0xffffffff, 0x00000100,
455 	0xf09e, 0xffffffff, 0x00000100,
456 	0xf084, 0xffffffff, 0x06000100,
457 	0xf0a4, 0xffffffff, 0x00000100,
458 	0xf09d, 0xffffffff, 0x00000100,
459 	0xf0ad, 0xffffffff, 0x00000100,
460 	0xf0ac, 0xffffffff, 0x00000100,
461 	0xf09c, 0xffffffff, 0x00000100,
462 	0xc200, 0xffffffff, 0xe0000000,
463 	0xf008, 0xffffffff, 0x00010000,
464 	0xf009, 0xffffffff, 0x00030002,
465 	0xf00a, 0xffffffff, 0x00040007,
466 	0xf00b, 0xffffffff, 0x00060005,
467 	0xf00c, 0xffffffff, 0x00090008,
468 	0xf00d, 0xffffffff, 0x00010000,
469 	0xf00e, 0xffffffff, 0x00030002,
470 	0xf00f, 0xffffffff, 0x00040007,
471 	0xf010, 0xffffffff, 0x00060005,
472 	0xf011, 0xffffffff, 0x00090008,
473 	0xf012, 0xffffffff, 0x00010000,
474 	0xf013, 0xffffffff, 0x00030002,
475 	0xf014, 0xffffffff, 0x00040007,
476 	0xf015, 0xffffffff, 0x00060005,
477 	0xf016, 0xffffffff, 0x00090008,
478 	0xf017, 0xffffffff, 0x00010000,
479 	0xf018, 0xffffffff, 0x00030002,
480 	0xf019, 0xffffffff, 0x00040007,
481 	0xf01a, 0xffffffff, 0x00060005,
482 	0xf01b, 0xffffffff, 0x00090008,
483 	0xf01c, 0xffffffff, 0x00010000,
484 	0xf01d, 0xffffffff, 0x00030002,
485 	0xf01e, 0xffffffff, 0x00040007,
486 	0xf01f, 0xffffffff, 0x00060005,
487 	0xf020, 0xffffffff, 0x00090008,
488 	0xf021, 0xffffffff, 0x00010000,
489 	0xf022, 0xffffffff, 0x00030002,
490 	0xf023, 0xffffffff, 0x00040007,
491 	0xf024, 0xffffffff, 0x00060005,
492 	0xf025, 0xffffffff, 0x00090008,
493 	0xf026, 0xffffffff, 0x00010000,
494 	0xf027, 0xffffffff, 0x00030002,
495 	0xf028, 0xffffffff, 0x00040007,
496 	0xf029, 0xffffffff, 0x00060005,
497 	0xf02a, 0xffffffff, 0x00090008,
498 	0xf02b, 0xffffffff, 0x00010000,
499 	0xf02c, 0xffffffff, 0x00030002,
500 	0xf02d, 0xffffffff, 0x00040007,
501 	0xf02e, 0xffffffff, 0x00060005,
502 	0xf02f, 0xffffffff, 0x00090008,
503 	0xf000, 0xffffffff, 0x96e00200,
504 	0x21c2, 0xffffffff, 0x00900100,
505 	0x3109, 0xffffffff, 0x0020003f,
506 	0xe, 0xffffffff, 0x0140001c,
507 	0xf, 0x000f0000, 0x000f0000,
508 	0x88, 0xffffffff, 0xc060000c,
509 	0x89, 0xc0000fff, 0x00000100,
510 	0x3e4, 0xffffffff, 0x00000100,
511 	0x3e6, 0x00000101, 0x00000000,
512 	0x82a, 0xffffffff, 0x00000104,
513 	0x1579, 0xff000fff, 0x00000100,
514 	0xc33, 0xc0000fff, 0x00000104,
515 	0x3079, 0x00000001, 0x00000001,
516 	0x3403, 0xff000ff0, 0x00000100,
517 	0x3603, 0xff000ff0, 0x00000100
518 };
519 
520 static const u32 kalindi_golden_spm_registers[] =
521 {
522 	0xc200, 0xe0ffffff, 0xe0000000
523 };
524 
525 static const u32 kalindi_golden_common_registers[] =
526 {
527 	0x31dc, 0xffffffff, 0x00000800,
528 	0x31dd, 0xffffffff, 0x00000800,
529 	0x31e6, 0xffffffff, 0x00007fbf,
530 	0x31e7, 0xffffffff, 0x00007faf
531 };
532 
533 static const u32 kalindi_golden_registers[] =
534 {
535 	0xf000, 0xffffdfff, 0x6e944040,
536 	0x1579, 0xff607fff, 0xfc000100,
537 	0xf088, 0xff000fff, 0x00000100,
538 	0xf089, 0xff000fff, 0x00000100,
539 	0xf080, 0xfffc0fff, 0x00000100,
540 	0x1bb6, 0x00010101, 0x00010000,
541 	0x260c, 0xffffffff, 0x00000000,
542 	0x260d, 0xf00fffff, 0x00000400,
543 	0x16ec, 0x000000f0, 0x00000070,
544 	0x16f0, 0xf0311fff, 0x80300000,
545 	0x263e, 0x73773777, 0x12010001,
546 	0x263f, 0xffffffff, 0x00000010,
547 	0x26df, 0x00ff0000, 0x00fc0000,
548 	0x200c, 0x00001f0f, 0x0000100a,
549 	0xbd2, 0x73773777, 0x12010001,
550 	0x902, 0x000fffff, 0x000c007f,
551 	0x2285, 0xf000003f, 0x00000007,
552 	0x22c9, 0x3fff3fff, 0x00ffcfff,
553 	0xc281, 0x0000ff0f, 0x00000000,
554 	0xa293, 0x07ffffff, 0x06000000,
555 	0x136, 0x00000fff, 0x00000100,
556 	0xf9e, 0x00000001, 0x00000002,
557 	0x31da, 0x00000008, 0x00000008,
558 	0x2300, 0x000000ff, 0x00000003,
559 	0x853e, 0x01ff01ff, 0x00000002,
560 	0x8526, 0x007ff800, 0x00200000,
561 	0x8057, 0xffffffff, 0x00000f40,
562 	0x2231, 0x001f3ae3, 0x00000082,
563 	0x2235, 0x0000001f, 0x00000010,
564 	0xc24d, 0xffffffff, 0x00000000
565 };
566 
567 static const u32 kalindi_mgcg_cgcg_init[] =
568 {
569 	0x3108, 0xffffffff, 0xfffffffc,
570 	0xc200, 0xffffffff, 0xe0000000,
571 	0xf0a8, 0xffffffff, 0x00000100,
572 	0xf082, 0xffffffff, 0x00000100,
573 	0xf0b0, 0xffffffff, 0x00000100,
574 	0xf0b2, 0xffffffff, 0x00000100,
575 	0xf0b1, 0xffffffff, 0x00000100,
576 	0x1579, 0xffffffff, 0x00600100,
577 	0xf0a0, 0xffffffff, 0x00000100,
578 	0xf085, 0xffffffff, 0x06000100,
579 	0xf088, 0xffffffff, 0x00000100,
580 	0xf086, 0xffffffff, 0x06000100,
581 	0xf081, 0xffffffff, 0x00000100,
582 	0xf0b8, 0xffffffff, 0x00000100,
583 	0xf089, 0xffffffff, 0x00000100,
584 	0xf080, 0xffffffff, 0x00000100,
585 	0xf08c, 0xffffffff, 0x00000100,
586 	0xf08d, 0xffffffff, 0x00000100,
587 	0xf094, 0xffffffff, 0x00000100,
588 	0xf095, 0xffffffff, 0x00000100,
589 	0xf096, 0xffffffff, 0x00000100,
590 	0xf097, 0xffffffff, 0x00000100,
591 	0xf098, 0xffffffff, 0x00000100,
592 	0xf09f, 0xffffffff, 0x00000100,
593 	0xf09e, 0xffffffff, 0x00000100,
594 	0xf084, 0xffffffff, 0x06000100,
595 	0xf0a4, 0xffffffff, 0x00000100,
596 	0xf09d, 0xffffffff, 0x00000100,
597 	0xf0ad, 0xffffffff, 0x00000100,
598 	0xf0ac, 0xffffffff, 0x00000100,
599 	0xf09c, 0xffffffff, 0x00000100,
600 	0xc200, 0xffffffff, 0xe0000000,
601 	0xf008, 0xffffffff, 0x00010000,
602 	0xf009, 0xffffffff, 0x00030002,
603 	0xf00a, 0xffffffff, 0x00040007,
604 	0xf00b, 0xffffffff, 0x00060005,
605 	0xf00c, 0xffffffff, 0x00090008,
606 	0xf00d, 0xffffffff, 0x00010000,
607 	0xf00e, 0xffffffff, 0x00030002,
608 	0xf00f, 0xffffffff, 0x00040007,
609 	0xf010, 0xffffffff, 0x00060005,
610 	0xf011, 0xffffffff, 0x00090008,
611 	0xf000, 0xffffffff, 0x96e00200,
612 	0x21c2, 0xffffffff, 0x00900100,
613 	0x3109, 0xffffffff, 0x0020003f,
614 	0xe, 0xffffffff, 0x0140001c,
615 	0xf, 0x000f0000, 0x000f0000,
616 	0x88, 0xffffffff, 0xc060000c,
617 	0x89, 0xc0000fff, 0x00000100,
618 	0x82a, 0xffffffff, 0x00000104,
619 	0x1579, 0xff000fff, 0x00000100,
620 	0xc33, 0xc0000fff, 0x00000104,
621 	0x3079, 0x00000001, 0x00000001,
622 	0x3403, 0xff000ff0, 0x00000100,
623 	0x3603, 0xff000ff0, 0x00000100
624 };
625 
626 static const u32 hawaii_golden_spm_registers[] =
627 {
628 	0xc200, 0xe0ffffff, 0xe0000000
629 };
630 
631 static const u32 hawaii_golden_common_registers[] =
632 {
633 	0xc200, 0xffffffff, 0xe0000000,
634 	0xa0d4, 0xffffffff, 0x3a00161a,
635 	0xa0d5, 0xffffffff, 0x0000002e,
636 	0x2684, 0xffffffff, 0x00018208,
637 	0x263e, 0xffffffff, 0x12011003
638 };
639 
640 static const u32 hawaii_golden_registers[] =
641 {
642 	0xcd5, 0x00000333, 0x00000333,
643 	0x2684, 0x00010000, 0x00058208,
644 	0x260c, 0xffffffff, 0x00000000,
645 	0x260d, 0xf00fffff, 0x00000400,
646 	0x260e, 0x0002021c, 0x00020200,
647 	0x31e, 0x00000080, 0x00000000,
648 	0x16ec, 0x000000f0, 0x00000070,
649 	0x16f0, 0xf0311fff, 0x80300000,
650 	0xd43, 0x00810000, 0x408af000,
651 	0x1c0c, 0x31000111, 0x00000011,
652 	0xbd2, 0x73773777, 0x12010001,
653 	0x848, 0x0000007f, 0x0000001b,
654 	0x877, 0x00007fb6, 0x00002191,
655 	0xd8a, 0x0000003f, 0x0000000a,
656 	0xd8b, 0x0000003f, 0x0000000a,
657 	0xab9, 0x00073ffe, 0x000022a2,
658 	0x903, 0x000007ff, 0x00000000,
659 	0x22fc, 0x00002001, 0x00000001,
660 	0x22c9, 0xffffffff, 0x00ffffff,
661 	0xc281, 0x0000ff0f, 0x00000000,
662 	0xa293, 0x07ffffff, 0x06000000,
663 	0xf9e, 0x00000001, 0x00000002,
664 	0x31da, 0x00000008, 0x00000008,
665 	0x31dc, 0x00000f00, 0x00000800,
666 	0x31dd, 0x00000f00, 0x00000800,
667 	0x31e6, 0x00ffffff, 0x00ff7fbf,
668 	0x31e7, 0x00ffffff, 0x00ff7faf,
669 	0x2300, 0x000000ff, 0x00000800,
670 	0x390, 0x00001fff, 0x00001fff,
671 	0x2418, 0x0000007f, 0x00000020,
672 	0x2542, 0x00010000, 0x00010000,
673 	0x2b80, 0x00100000, 0x000ff07c,
674 	0x2b05, 0x000003ff, 0x0000000f,
675 	0x2b04, 0xffffffff, 0x7564fdec,
676 	0x2b03, 0xffffffff, 0x3120b9a8,
677 	0x2b02, 0x20000000, 0x0f9c0000
678 };
679 
680 static const u32 hawaii_mgcg_cgcg_init[] =
681 {
682 	0x3108, 0xffffffff, 0xfffffffd,
683 	0xc200, 0xffffffff, 0xe0000000,
684 	0xf0a8, 0xffffffff, 0x00000100,
685 	0xf082, 0xffffffff, 0x00000100,
686 	0xf0b0, 0xffffffff, 0x00000100,
687 	0xf0b2, 0xffffffff, 0x00000100,
688 	0xf0b1, 0xffffffff, 0x00000100,
689 	0x1579, 0xffffffff, 0x00200100,
690 	0xf0a0, 0xffffffff, 0x00000100,
691 	0xf085, 0xffffffff, 0x06000100,
692 	0xf088, 0xffffffff, 0x00000100,
693 	0xf086, 0xffffffff, 0x06000100,
694 	0xf081, 0xffffffff, 0x00000100,
695 	0xf0b8, 0xffffffff, 0x00000100,
696 	0xf089, 0xffffffff, 0x00000100,
697 	0xf080, 0xffffffff, 0x00000100,
698 	0xf08c, 0xffffffff, 0x00000100,
699 	0xf08d, 0xffffffff, 0x00000100,
700 	0xf094, 0xffffffff, 0x00000100,
701 	0xf095, 0xffffffff, 0x00000100,
702 	0xf096, 0xffffffff, 0x00000100,
703 	0xf097, 0xffffffff, 0x00000100,
704 	0xf098, 0xffffffff, 0x00000100,
705 	0xf09f, 0xffffffff, 0x00000100,
706 	0xf09e, 0xffffffff, 0x00000100,
707 	0xf084, 0xffffffff, 0x06000100,
708 	0xf0a4, 0xffffffff, 0x00000100,
709 	0xf09d, 0xffffffff, 0x00000100,
710 	0xf0ad, 0xffffffff, 0x00000100,
711 	0xf0ac, 0xffffffff, 0x00000100,
712 	0xf09c, 0xffffffff, 0x00000100,
713 	0xc200, 0xffffffff, 0xe0000000,
714 	0xf008, 0xffffffff, 0x00010000,
715 	0xf009, 0xffffffff, 0x00030002,
716 	0xf00a, 0xffffffff, 0x00040007,
717 	0xf00b, 0xffffffff, 0x00060005,
718 	0xf00c, 0xffffffff, 0x00090008,
719 	0xf00d, 0xffffffff, 0x00010000,
720 	0xf00e, 0xffffffff, 0x00030002,
721 	0xf00f, 0xffffffff, 0x00040007,
722 	0xf010, 0xffffffff, 0x00060005,
723 	0xf011, 0xffffffff, 0x00090008,
724 	0xf012, 0xffffffff, 0x00010000,
725 	0xf013, 0xffffffff, 0x00030002,
726 	0xf014, 0xffffffff, 0x00040007,
727 	0xf015, 0xffffffff, 0x00060005,
728 	0xf016, 0xffffffff, 0x00090008,
729 	0xf017, 0xffffffff, 0x00010000,
730 	0xf018, 0xffffffff, 0x00030002,
731 	0xf019, 0xffffffff, 0x00040007,
732 	0xf01a, 0xffffffff, 0x00060005,
733 	0xf01b, 0xffffffff, 0x00090008,
734 	0xf01c, 0xffffffff, 0x00010000,
735 	0xf01d, 0xffffffff, 0x00030002,
736 	0xf01e, 0xffffffff, 0x00040007,
737 	0xf01f, 0xffffffff, 0x00060005,
738 	0xf020, 0xffffffff, 0x00090008,
739 	0xf021, 0xffffffff, 0x00010000,
740 	0xf022, 0xffffffff, 0x00030002,
741 	0xf023, 0xffffffff, 0x00040007,
742 	0xf024, 0xffffffff, 0x00060005,
743 	0xf025, 0xffffffff, 0x00090008,
744 	0xf026, 0xffffffff, 0x00010000,
745 	0xf027, 0xffffffff, 0x00030002,
746 	0xf028, 0xffffffff, 0x00040007,
747 	0xf029, 0xffffffff, 0x00060005,
748 	0xf02a, 0xffffffff, 0x00090008,
749 	0xf02b, 0xffffffff, 0x00010000,
750 	0xf02c, 0xffffffff, 0x00030002,
751 	0xf02d, 0xffffffff, 0x00040007,
752 	0xf02e, 0xffffffff, 0x00060005,
753 	0xf02f, 0xffffffff, 0x00090008,
754 	0xf030, 0xffffffff, 0x00010000,
755 	0xf031, 0xffffffff, 0x00030002,
756 	0xf032, 0xffffffff, 0x00040007,
757 	0xf033, 0xffffffff, 0x00060005,
758 	0xf034, 0xffffffff, 0x00090008,
759 	0xf035, 0xffffffff, 0x00010000,
760 	0xf036, 0xffffffff, 0x00030002,
761 	0xf037, 0xffffffff, 0x00040007,
762 	0xf038, 0xffffffff, 0x00060005,
763 	0xf039, 0xffffffff, 0x00090008,
764 	0xf03a, 0xffffffff, 0x00010000,
765 	0xf03b, 0xffffffff, 0x00030002,
766 	0xf03c, 0xffffffff, 0x00040007,
767 	0xf03d, 0xffffffff, 0x00060005,
768 	0xf03e, 0xffffffff, 0x00090008,
769 	0x30c6, 0xffffffff, 0x00020200,
770 	0xcd4, 0xffffffff, 0x00000200,
771 	0x570, 0xffffffff, 0x00000400,
772 	0x157a, 0xffffffff, 0x00000000,
773 	0xbd4, 0xffffffff, 0x00000902,
774 	0xf000, 0xffffffff, 0x96940200,
775 	0x21c2, 0xffffffff, 0x00900100,
776 	0x3109, 0xffffffff, 0x0020003f,
777 	0xe, 0xffffffff, 0x0140001c,
778 	0xf, 0x000f0000, 0x000f0000,
779 	0x88, 0xffffffff, 0xc060000c,
780 	0x89, 0xc0000fff, 0x00000100,
781 	0x3e4, 0xffffffff, 0x00000100,
782 	0x3e6, 0x00000101, 0x00000000,
783 	0x82a, 0xffffffff, 0x00000104,
784 	0x1579, 0xff000fff, 0x00000100,
785 	0xc33, 0xc0000fff, 0x00000104,
786 	0x3079, 0x00000001, 0x00000001,
787 	0x3403, 0xff000ff0, 0x00000100,
788 	0x3603, 0xff000ff0, 0x00000100
789 };
790 
791 static const u32 godavari_golden_registers[] =
792 {
793 	0x1579, 0xff607fff, 0xfc000100,
794 	0x1bb6, 0x00010101, 0x00010000,
795 	0x260c, 0xffffffff, 0x00000000,
796 	0x260c0, 0xf00fffff, 0x00000400,
797 	0x184c, 0xffffffff, 0x00010000,
798 	0x16ec, 0x000000f0, 0x00000070,
799 	0x16f0, 0xf0311fff, 0x80300000,
800 	0x263e, 0x73773777, 0x12010001,
801 	0x263f, 0xffffffff, 0x00000010,
802 	0x200c, 0x00001f0f, 0x0000100a,
803 	0xbd2, 0x73773777, 0x12010001,
804 	0x902, 0x000fffff, 0x000c007f,
805 	0x2285, 0xf000003f, 0x00000007,
806 	0x22c9, 0xffffffff, 0x00ff0fff,
807 	0xc281, 0x0000ff0f, 0x00000000,
808 	0xa293, 0x07ffffff, 0x06000000,
809 	0x136, 0x00000fff, 0x00000100,
810 	0x3405, 0x00010000, 0x00810001,
811 	0x3605, 0x00010000, 0x00810001,
812 	0xf9e, 0x00000001, 0x00000002,
813 	0x31da, 0x00000008, 0x00000008,
814 	0x31dc, 0x00000f00, 0x00000800,
815 	0x31dd, 0x00000f00, 0x00000800,
816 	0x31e6, 0x00ffffff, 0x00ff7fbf,
817 	0x31e7, 0x00ffffff, 0x00ff7faf,
818 	0x2300, 0x000000ff, 0x00000001,
819 	0x853e, 0x01ff01ff, 0x00000002,
820 	0x8526, 0x007ff800, 0x00200000,
821 	0x8057, 0xffffffff, 0x00000f40,
822 	0x2231, 0x001f3ae3, 0x00000082,
823 	0x2235, 0x0000001f, 0x00000010,
824 	0xc24d, 0xffffffff, 0x00000000
825 };
826 
827 static void cik_init_golden_registers(struct amdgpu_device *adev)
828 {
829 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
830 	mutex_lock(&adev->grbm_idx_mutex);
831 
832 	switch (adev->asic_type) {
833 	case CHIP_BONAIRE:
834 		amdgpu_device_program_register_sequence(adev,
835 							bonaire_mgcg_cgcg_init,
836 							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
837 		amdgpu_device_program_register_sequence(adev,
838 							bonaire_golden_registers,
839 							ARRAY_SIZE(bonaire_golden_registers));
840 		amdgpu_device_program_register_sequence(adev,
841 							bonaire_golden_common_registers,
842 							ARRAY_SIZE(bonaire_golden_common_registers));
843 		amdgpu_device_program_register_sequence(adev,
844 							bonaire_golden_spm_registers,
845 							ARRAY_SIZE(bonaire_golden_spm_registers));
846 		break;
847 	case CHIP_KABINI:
848 		amdgpu_device_program_register_sequence(adev,
849 							kalindi_mgcg_cgcg_init,
850 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
851 		amdgpu_device_program_register_sequence(adev,
852 							kalindi_golden_registers,
853 							ARRAY_SIZE(kalindi_golden_registers));
854 		amdgpu_device_program_register_sequence(adev,
855 							kalindi_golden_common_registers,
856 							ARRAY_SIZE(kalindi_golden_common_registers));
857 		amdgpu_device_program_register_sequence(adev,
858 							kalindi_golden_spm_registers,
859 							ARRAY_SIZE(kalindi_golden_spm_registers));
860 		break;
861 	case CHIP_MULLINS:
862 		amdgpu_device_program_register_sequence(adev,
863 							kalindi_mgcg_cgcg_init,
864 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
865 		amdgpu_device_program_register_sequence(adev,
866 							godavari_golden_registers,
867 							ARRAY_SIZE(godavari_golden_registers));
868 		amdgpu_device_program_register_sequence(adev,
869 							kalindi_golden_common_registers,
870 							ARRAY_SIZE(kalindi_golden_common_registers));
871 		amdgpu_device_program_register_sequence(adev,
872 							kalindi_golden_spm_registers,
873 							ARRAY_SIZE(kalindi_golden_spm_registers));
874 		break;
875 	case CHIP_KAVERI:
876 		amdgpu_device_program_register_sequence(adev,
877 							spectre_mgcg_cgcg_init,
878 							ARRAY_SIZE(spectre_mgcg_cgcg_init));
879 		amdgpu_device_program_register_sequence(adev,
880 							spectre_golden_registers,
881 							ARRAY_SIZE(spectre_golden_registers));
882 		amdgpu_device_program_register_sequence(adev,
883 							spectre_golden_common_registers,
884 							ARRAY_SIZE(spectre_golden_common_registers));
885 		amdgpu_device_program_register_sequence(adev,
886 							spectre_golden_spm_registers,
887 							ARRAY_SIZE(spectre_golden_spm_registers));
888 		break;
889 	case CHIP_HAWAII:
890 		amdgpu_device_program_register_sequence(adev,
891 							hawaii_mgcg_cgcg_init,
892 							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
893 		amdgpu_device_program_register_sequence(adev,
894 							hawaii_golden_registers,
895 							ARRAY_SIZE(hawaii_golden_registers));
896 		amdgpu_device_program_register_sequence(adev,
897 							hawaii_golden_common_registers,
898 							ARRAY_SIZE(hawaii_golden_common_registers));
899 		amdgpu_device_program_register_sequence(adev,
900 							hawaii_golden_spm_registers,
901 							ARRAY_SIZE(hawaii_golden_spm_registers));
902 		break;
903 	default:
904 		break;
905 	}
906 	mutex_unlock(&adev->grbm_idx_mutex);
907 }
908 
909 /**
910  * cik_get_xclk - get the xclk
911  *
912  * @adev: amdgpu_device pointer
913  *
914  * Returns the reference clock used by the gfx engine
915  * (CIK).
916  */
917 static u32 cik_get_xclk(struct amdgpu_device *adev)
918 {
919 	u32 reference_clock = adev->clock.spll.reference_freq;
920 
921 	if (adev->flags & AMD_IS_APU) {
922 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
923 			return reference_clock / 2;
924 	} else {
925 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
926 			return reference_clock / 4;
927 	}
928 	return reference_clock;
929 }
930 
931 /**
932  * cik_srbm_select - select specific register instances
933  *
934  * @adev: amdgpu_device pointer
935  * @me: selected ME (micro engine)
936  * @pipe: pipe
937  * @queue: queue
938  * @vmid: VMID
939  *
940  * Switches the currently active registers instances.  Some
941  * registers are instanced per VMID, others are instanced per
942  * me/pipe/queue combination.
943  */
944 void cik_srbm_select(struct amdgpu_device *adev,
945 		     u32 me, u32 pipe, u32 queue, u32 vmid)
946 {
947 	u32 srbm_gfx_cntl =
948 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
949 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
950 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
951 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
952 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
953 }
954 
955 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
956 {
957 	uint32_t tmp;
958 
959 	tmp = RREG32(mmCONFIG_CNTL);
960 	if (!state)
961 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
962 	else
963 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
964 	WREG32(mmCONFIG_CNTL, tmp);
965 }
966 
967 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
968 {
969 	u32 bus_cntl;
970 	u32 d1vga_control = 0;
971 	u32 d2vga_control = 0;
972 	u32 vga_render_control = 0;
973 	u32 rom_cntl;
974 	bool r;
975 
976 	bus_cntl = RREG32(mmBUS_CNTL);
977 	if (adev->mode_info.num_crtc) {
978 		d1vga_control = RREG32(mmD1VGA_CONTROL);
979 		d2vga_control = RREG32(mmD2VGA_CONTROL);
980 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
981 	}
982 	rom_cntl = RREG32_SMC(ixROM_CNTL);
983 
984 	/* enable the rom */
985 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
986 	if (adev->mode_info.num_crtc) {
987 		/* Disable VGA mode */
988 		WREG32(mmD1VGA_CONTROL,
989 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
990 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
991 		WREG32(mmD2VGA_CONTROL,
992 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
993 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
994 		WREG32(mmVGA_RENDER_CONTROL,
995 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
996 	}
997 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
998 
999 	r = amdgpu_read_bios(adev);
1000 
1001 	/* restore regs */
1002 	WREG32(mmBUS_CNTL, bus_cntl);
1003 	if (adev->mode_info.num_crtc) {
1004 		WREG32(mmD1VGA_CONTROL, d1vga_control);
1005 		WREG32(mmD2VGA_CONTROL, d2vga_control);
1006 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007 	}
1008 	WREG32_SMC(ixROM_CNTL, rom_cntl);
1009 	return r;
1010 }
1011 
1012 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013 				   u8 *bios, u32 length_bytes)
1014 {
1015 	u32 *dw_ptr;
1016 	unsigned long flags;
1017 	u32 i, length_dw;
1018 
1019 	if (bios == NULL)
1020 		return false;
1021 	if (length_bytes == 0)
1022 		return false;
1023 	/* APU vbios image is part of sbios image */
1024 	if (adev->flags & AMD_IS_APU)
1025 		return false;
1026 
1027 	dw_ptr = (u32 *)bios;
1028 	length_dw = ALIGN(length_bytes, 4) / 4;
1029 	/* take the smc lock since we are using the smc index */
1030 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031 	/* set rom index to 0 */
1032 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033 	WREG32(mmSMC_IND_DATA_0, 0);
1034 	/* set index to data for continous read */
1035 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036 	for (i = 0; i < length_dw; i++)
1037 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039 
1040 	return true;
1041 }
1042 
1043 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044 	{mmGRBM_STATUS},
1045 	{mmGRBM_STATUS2},
1046 	{mmGRBM_STATUS_SE0},
1047 	{mmGRBM_STATUS_SE1},
1048 	{mmGRBM_STATUS_SE2},
1049 	{mmGRBM_STATUS_SE3},
1050 	{mmSRBM_STATUS},
1051 	{mmSRBM_STATUS2},
1052 	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053 	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054 	{mmCP_STAT},
1055 	{mmCP_STALLED_STAT1},
1056 	{mmCP_STALLED_STAT2},
1057 	{mmCP_STALLED_STAT3},
1058 	{mmCP_CPF_BUSY_STAT},
1059 	{mmCP_CPF_STALLED_STAT1},
1060 	{mmCP_CPF_STATUS},
1061 	{mmCP_CPC_BUSY_STAT},
1062 	{mmCP_CPC_STALLED_STAT1},
1063 	{mmCP_CPC_STATUS},
1064 	{mmGB_ADDR_CONFIG},
1065 	{mmMC_ARB_RAMCFG},
1066 	{mmGB_TILE_MODE0},
1067 	{mmGB_TILE_MODE1},
1068 	{mmGB_TILE_MODE2},
1069 	{mmGB_TILE_MODE3},
1070 	{mmGB_TILE_MODE4},
1071 	{mmGB_TILE_MODE5},
1072 	{mmGB_TILE_MODE6},
1073 	{mmGB_TILE_MODE7},
1074 	{mmGB_TILE_MODE8},
1075 	{mmGB_TILE_MODE9},
1076 	{mmGB_TILE_MODE10},
1077 	{mmGB_TILE_MODE11},
1078 	{mmGB_TILE_MODE12},
1079 	{mmGB_TILE_MODE13},
1080 	{mmGB_TILE_MODE14},
1081 	{mmGB_TILE_MODE15},
1082 	{mmGB_TILE_MODE16},
1083 	{mmGB_TILE_MODE17},
1084 	{mmGB_TILE_MODE18},
1085 	{mmGB_TILE_MODE19},
1086 	{mmGB_TILE_MODE20},
1087 	{mmGB_TILE_MODE21},
1088 	{mmGB_TILE_MODE22},
1089 	{mmGB_TILE_MODE23},
1090 	{mmGB_TILE_MODE24},
1091 	{mmGB_TILE_MODE25},
1092 	{mmGB_TILE_MODE26},
1093 	{mmGB_TILE_MODE27},
1094 	{mmGB_TILE_MODE28},
1095 	{mmGB_TILE_MODE29},
1096 	{mmGB_TILE_MODE30},
1097 	{mmGB_TILE_MODE31},
1098 	{mmGB_MACROTILE_MODE0},
1099 	{mmGB_MACROTILE_MODE1},
1100 	{mmGB_MACROTILE_MODE2},
1101 	{mmGB_MACROTILE_MODE3},
1102 	{mmGB_MACROTILE_MODE4},
1103 	{mmGB_MACROTILE_MODE5},
1104 	{mmGB_MACROTILE_MODE6},
1105 	{mmGB_MACROTILE_MODE7},
1106 	{mmGB_MACROTILE_MODE8},
1107 	{mmGB_MACROTILE_MODE9},
1108 	{mmGB_MACROTILE_MODE10},
1109 	{mmGB_MACROTILE_MODE11},
1110 	{mmGB_MACROTILE_MODE12},
1111 	{mmGB_MACROTILE_MODE13},
1112 	{mmGB_MACROTILE_MODE14},
1113 	{mmGB_MACROTILE_MODE15},
1114 	{mmCC_RB_BACKEND_DISABLE, true},
1115 	{mmGC_USER_RB_BACKEND_DISABLE, true},
1116 	{mmGB_BACKEND_MAP, false},
1117 	{mmPA_SC_RASTER_CONFIG, true},
1118 	{mmPA_SC_RASTER_CONFIG_1, true},
1119 };
1120 
1121 
1122 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123 				       bool indexed, u32 se_num,
1124 				       u32 sh_num, u32 reg_offset)
1125 {
1126 	if (indexed) {
1127 		uint32_t val;
1128 		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129 		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130 
1131 		switch (reg_offset) {
1132 		case mmCC_RB_BACKEND_DISABLE:
1133 			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134 		case mmGC_USER_RB_BACKEND_DISABLE:
1135 			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136 		case mmPA_SC_RASTER_CONFIG:
1137 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138 		case mmPA_SC_RASTER_CONFIG_1:
1139 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140 		}
1141 
1142 		mutex_lock(&adev->grbm_idx_mutex);
1143 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144 			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1145 
1146 		val = RREG32(reg_offset);
1147 
1148 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149 			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1150 		mutex_unlock(&adev->grbm_idx_mutex);
1151 		return val;
1152 	} else {
1153 		unsigned idx;
1154 
1155 		switch (reg_offset) {
1156 		case mmGB_ADDR_CONFIG:
1157 			return adev->gfx.config.gb_addr_config;
1158 		case mmMC_ARB_RAMCFG:
1159 			return adev->gfx.config.mc_arb_ramcfg;
1160 		case mmGB_TILE_MODE0:
1161 		case mmGB_TILE_MODE1:
1162 		case mmGB_TILE_MODE2:
1163 		case mmGB_TILE_MODE3:
1164 		case mmGB_TILE_MODE4:
1165 		case mmGB_TILE_MODE5:
1166 		case mmGB_TILE_MODE6:
1167 		case mmGB_TILE_MODE7:
1168 		case mmGB_TILE_MODE8:
1169 		case mmGB_TILE_MODE9:
1170 		case mmGB_TILE_MODE10:
1171 		case mmGB_TILE_MODE11:
1172 		case mmGB_TILE_MODE12:
1173 		case mmGB_TILE_MODE13:
1174 		case mmGB_TILE_MODE14:
1175 		case mmGB_TILE_MODE15:
1176 		case mmGB_TILE_MODE16:
1177 		case mmGB_TILE_MODE17:
1178 		case mmGB_TILE_MODE18:
1179 		case mmGB_TILE_MODE19:
1180 		case mmGB_TILE_MODE20:
1181 		case mmGB_TILE_MODE21:
1182 		case mmGB_TILE_MODE22:
1183 		case mmGB_TILE_MODE23:
1184 		case mmGB_TILE_MODE24:
1185 		case mmGB_TILE_MODE25:
1186 		case mmGB_TILE_MODE26:
1187 		case mmGB_TILE_MODE27:
1188 		case mmGB_TILE_MODE28:
1189 		case mmGB_TILE_MODE29:
1190 		case mmGB_TILE_MODE30:
1191 		case mmGB_TILE_MODE31:
1192 			idx = (reg_offset - mmGB_TILE_MODE0);
1193 			return adev->gfx.config.tile_mode_array[idx];
1194 		case mmGB_MACROTILE_MODE0:
1195 		case mmGB_MACROTILE_MODE1:
1196 		case mmGB_MACROTILE_MODE2:
1197 		case mmGB_MACROTILE_MODE3:
1198 		case mmGB_MACROTILE_MODE4:
1199 		case mmGB_MACROTILE_MODE5:
1200 		case mmGB_MACROTILE_MODE6:
1201 		case mmGB_MACROTILE_MODE7:
1202 		case mmGB_MACROTILE_MODE8:
1203 		case mmGB_MACROTILE_MODE9:
1204 		case mmGB_MACROTILE_MODE10:
1205 		case mmGB_MACROTILE_MODE11:
1206 		case mmGB_MACROTILE_MODE12:
1207 		case mmGB_MACROTILE_MODE13:
1208 		case mmGB_MACROTILE_MODE14:
1209 		case mmGB_MACROTILE_MODE15:
1210 			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211 			return adev->gfx.config.macrotile_mode_array[idx];
1212 		default:
1213 			return RREG32(reg_offset);
1214 		}
1215 	}
1216 }
1217 
1218 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219 			     u32 sh_num, u32 reg_offset, u32 *value)
1220 {
1221 	uint32_t i;
1222 
1223 	*value = 0;
1224 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225 		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226 
1227 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228 			continue;
1229 
1230 		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231 						reg_offset);
1232 		return 0;
1233 	}
1234 	return -EINVAL;
1235 }
1236 
1237 struct kv_reset_save_regs {
1238 	u32 gmcon_reng_execute;
1239 	u32 gmcon_misc;
1240 	u32 gmcon_misc3;
1241 };
1242 
1243 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244 				   struct kv_reset_save_regs *save)
1245 {
1246 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1248 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249 
1250 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1253 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255 }
1256 
1257 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258 				      struct kv_reset_save_regs *save)
1259 {
1260 	int i;
1261 
1262 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1263 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264 
1265 	for (i = 0; i < 5; i++)
1266 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1267 
1268 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1269 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270 
1271 	for (i = 0; i < 5; i++)
1272 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1273 
1274 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276 
1277 	for (i = 0; i < 5; i++)
1278 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1279 
1280 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282 
1283 	for (i = 0; i < 5; i++)
1284 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1285 
1286 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288 
1289 	for (i = 0; i < 5; i++)
1290 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1291 
1292 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1293 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294 
1295 	for (i = 0; i < 5; i++)
1296 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1297 
1298 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300 
1301 	for (i = 0; i < 5; i++)
1302 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1303 
1304 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306 
1307 	for (i = 0; i < 5; i++)
1308 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1309 
1310 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312 
1313 	for (i = 0; i < 5; i++)
1314 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1315 
1316 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318 
1319 	for (i = 0; i < 5; i++)
1320 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1321 
1322 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324 
1325 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1327 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328 }
1329 
1330 /**
1331  * cik_asic_pci_config_reset - soft reset GPU
1332  *
1333  * @adev: amdgpu_device pointer
1334  *
1335  * Use PCI Config method to reset the GPU.
1336  *
1337  * Returns 0 for success.
1338  */
1339 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340 {
1341 	struct kv_reset_save_regs kv_save = { 0 };
1342 	u32 i;
1343 	int r = -EINVAL;
1344 
1345 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346 
1347 	if (adev->flags & AMD_IS_APU)
1348 		kv_save_regs_for_reset(adev, &kv_save);
1349 
1350 	/* disable BM */
1351 	pci_clear_master(adev->pdev);
1352 	/* reset */
1353 	amdgpu_device_pci_config_reset(adev);
1354 
1355 	udelay(100);
1356 
1357 	/* wait for asic to come out of reset */
1358 	for (i = 0; i < adev->usec_timeout; i++) {
1359 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360 			/* enable BM */
1361 			pci_set_master(adev->pdev);
1362 			adev->has_hw_reset = true;
1363 			r = 0;
1364 			break;
1365 		}
1366 		udelay(1);
1367 	}
1368 
1369 	/* does asic init need to be run first??? */
1370 	if (adev->flags & AMD_IS_APU)
1371 		kv_restore_regs_for_reset(adev, &kv_save);
1372 
1373 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374 
1375 	return r;
1376 }
1377 
1378 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379 {
1380 	switch (adev->asic_type) {
1381 	case CHIP_BONAIRE:
1382 	case CHIP_HAWAII:
1383 		return amdgpu_dpm_is_baco_supported(adev);
1384 	default:
1385 		return false;
1386 	}
1387 }
1388 
1389 static enum amd_reset_method
1390 cik_asic_reset_method(struct amdgpu_device *adev)
1391 {
1392 	bool baco_reset;
1393 
1394 	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395 	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396 		return amdgpu_reset_method;
1397 
1398 	if (amdgpu_reset_method != -1)
1399 		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400 				  amdgpu_reset_method);
1401 
1402 	switch (adev->asic_type) {
1403 	case CHIP_BONAIRE:
1404 	case CHIP_HAWAII:
1405 		baco_reset = cik_asic_supports_baco(adev);
1406 		break;
1407 	default:
1408 		baco_reset = false;
1409 		break;
1410 	}
1411 
1412 	if (baco_reset)
1413 		return AMD_RESET_METHOD_BACO;
1414 	else
1415 		return AMD_RESET_METHOD_LEGACY;
1416 }
1417 
1418 /**
1419  * cik_asic_reset - soft reset GPU
1420  *
1421  * @adev: amdgpu_device pointer
1422  *
1423  * Look up which blocks are hung and attempt
1424  * to reset them.
1425  * Returns 0 for success.
1426  */
1427 static int cik_asic_reset(struct amdgpu_device *adev)
1428 {
1429 	int r;
1430 
1431 	/* APUs don't have full asic reset */
1432 	if (adev->flags & AMD_IS_APU)
1433 		return 0;
1434 
1435 	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436 		dev_info(adev->dev, "BACO reset\n");
1437 		r = amdgpu_dpm_baco_reset(adev);
1438 	} else {
1439 		dev_info(adev->dev, "PCI CONFIG reset\n");
1440 		r = cik_asic_pci_config_reset(adev);
1441 	}
1442 
1443 	return r;
1444 }
1445 
1446 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447 {
1448 	return RREG32(mmCONFIG_MEMSIZE);
1449 }
1450 
1451 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452 			      u32 cntl_reg, u32 status_reg)
1453 {
1454 	int r, i;
1455 	struct atom_clock_dividers dividers;
1456 	uint32_t tmp;
1457 
1458 	r = amdgpu_atombios_get_clock_dividers(adev,
1459 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460 					       clock, false, &dividers);
1461 	if (r)
1462 		return r;
1463 
1464 	tmp = RREG32_SMC(cntl_reg);
1465 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467 	tmp |= dividers.post_divider;
1468 	WREG32_SMC(cntl_reg, tmp);
1469 
1470 	for (i = 0; i < 100; i++) {
1471 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472 			break;
1473 		mdelay(10);
1474 	}
1475 	if (i == 100)
1476 		return -ETIMEDOUT;
1477 
1478 	return 0;
1479 }
1480 
1481 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482 {
1483 	int r = 0;
1484 
1485 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486 	if (r)
1487 		return r;
1488 
1489 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490 	return r;
1491 }
1492 
1493 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494 {
1495 	int r, i;
1496 	struct atom_clock_dividers dividers;
1497 	u32 tmp;
1498 
1499 	r = amdgpu_atombios_get_clock_dividers(adev,
1500 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501 					       ecclk, false, &dividers);
1502 	if (r)
1503 		return r;
1504 
1505 	for (i = 0; i < 100; i++) {
1506 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507 			break;
1508 		mdelay(10);
1509 	}
1510 	if (i == 100)
1511 		return -ETIMEDOUT;
1512 
1513 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516 	tmp |= dividers.post_divider;
1517 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518 
1519 	for (i = 0; i < 100; i++) {
1520 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521 			break;
1522 		mdelay(10);
1523 	}
1524 	if (i == 100)
1525 		return -ETIMEDOUT;
1526 
1527 	return 0;
1528 }
1529 
1530 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531 {
1532 	struct pci_dev *root = adev->pdev->bus->self;
1533 	u32 speed_cntl, current_data_rate;
1534 	int i;
1535 	u16 tmp16;
1536 
1537 	if (pci_is_root_bus(adev->pdev->bus))
1538 		return;
1539 
1540 	if (amdgpu_pcie_gen2 == 0)
1541 		return;
1542 
1543 	if (adev->flags & AMD_IS_APU)
1544 		return;
1545 
1546 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548 		return;
1549 
1550 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554 		if (current_data_rate == 2) {
1555 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556 			return;
1557 		}
1558 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560 		if (current_data_rate == 1) {
1561 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562 			return;
1563 		}
1564 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565 	}
1566 
1567 	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1568 		return;
1569 
1570 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571 		/* re-try equalization if gen3 is not already enabled */
1572 		if (current_data_rate != 2) {
1573 			u16 bridge_cfg, gpu_cfg;
1574 			u16 bridge_cfg2, gpu_cfg2;
1575 			u32 max_lw, current_lw, tmp;
1576 
1577 			pcie_capability_set_word(root, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1578 			pcie_capability_set_word(adev->pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_HAWD);
1579 
1580 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1581 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1582 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1583 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1584 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1585 
1586 			if (current_lw < max_lw) {
1587 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1588 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1589 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1590 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1591 					tmp |= (max_lw <<
1592 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1593 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1594 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1595 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1596 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1597 				}
1598 			}
1599 
1600 			for (i = 0; i < 10; i++) {
1601 				/* check status */
1602 				pcie_capability_read_word(adev->pdev,
1603 							  PCI_EXP_DEVSTA,
1604 							  &tmp16);
1605 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1606 					break;
1607 
1608 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1609 							  &bridge_cfg);
1610 				pcie_capability_read_word(adev->pdev,
1611 							  PCI_EXP_LNKCTL,
1612 							  &gpu_cfg);
1613 
1614 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1615 							  &bridge_cfg2);
1616 				pcie_capability_read_word(adev->pdev,
1617 							  PCI_EXP_LNKCTL2,
1618 							  &gpu_cfg2);
1619 
1620 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1621 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1622 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1623 
1624 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1625 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1626 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1627 
1628 				msleep(100);
1629 
1630 				/* linkctl */
1631 				pcie_capability_clear_and_set_word(root, PCI_EXP_LNKCTL,
1632 								   PCI_EXP_LNKCTL_HAWD,
1633 								   bridge_cfg &
1634 								   PCI_EXP_LNKCTL_HAWD);
1635 				pcie_capability_clear_and_set_word(adev->pdev, PCI_EXP_LNKCTL,
1636 								   PCI_EXP_LNKCTL_HAWD,
1637 								   gpu_cfg &
1638 								   PCI_EXP_LNKCTL_HAWD);
1639 
1640 				/* linkctl2 */
1641 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1642 							  &tmp16);
1643 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1644 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1645 				tmp16 |= (bridge_cfg2 &
1646 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1647 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1648 				pcie_capability_write_word(root,
1649 							   PCI_EXP_LNKCTL2,
1650 							   tmp16);
1651 
1652 				pcie_capability_read_word(adev->pdev,
1653 							  PCI_EXP_LNKCTL2,
1654 							  &tmp16);
1655 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1656 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1657 				tmp16 |= (gpu_cfg2 &
1658 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1659 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1660 				pcie_capability_write_word(adev->pdev,
1661 							   PCI_EXP_LNKCTL2,
1662 							   tmp16);
1663 
1664 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1665 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1666 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1667 			}
1668 		}
1669 	}
1670 
1671 	/* set the link speed */
1672 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1673 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1674 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1675 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1676 
1677 	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1678 	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1679 
1680 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1681 		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1682 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1683 		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1684 	else
1685 		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1686 	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1687 
1688 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1689 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1690 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1691 
1692 	for (i = 0; i < adev->usec_timeout; i++) {
1693 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1694 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1695 			break;
1696 		udelay(1);
1697 	}
1698 }
1699 
1700 static void cik_program_aspm(struct amdgpu_device *adev)
1701 {
1702 	u32 data, orig;
1703 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1704 	bool disable_clkreq = false;
1705 
1706 	if (!amdgpu_device_should_use_aspm(adev))
1707 		return;
1708 
1709 	if (pci_is_root_bus(adev->pdev->bus))
1710 		return;
1711 
1712 	/* XXX double check APUs */
1713 	if (adev->flags & AMD_IS_APU)
1714 		return;
1715 
1716 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1717 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1718 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1719 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1720 	if (orig != data)
1721 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1722 
1723 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1724 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1725 	if (orig != data)
1726 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1727 
1728 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1729 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1730 	if (orig != data)
1731 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1732 
1733 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1734 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1735 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1736 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1737 	if (!disable_l0s)
1738 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1739 
1740 	if (!disable_l1) {
1741 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1742 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1743 		if (orig != data)
1744 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1745 
1746 		if (!disable_plloff_in_l1) {
1747 			bool clk_req_support;
1748 
1749 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1750 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1751 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1752 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1753 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1754 			if (orig != data)
1755 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1756 
1757 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1758 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1759 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1760 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1761 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1762 			if (orig != data)
1763 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1764 
1765 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1766 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1767 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1768 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1769 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1770 			if (orig != data)
1771 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1772 
1773 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1774 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1775 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1776 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1777 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1778 			if (orig != data)
1779 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1780 
1781 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1782 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1783 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1784 			if (orig != data)
1785 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1786 
1787 			if (!disable_clkreq) {
1788 				struct pci_dev *root = adev->pdev->bus->self;
1789 				u32 lnkcap;
1790 
1791 				clk_req_support = false;
1792 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1793 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1794 					clk_req_support = true;
1795 			} else {
1796 				clk_req_support = false;
1797 			}
1798 
1799 			if (clk_req_support) {
1800 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1801 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1802 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1803 				if (orig != data)
1804 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1805 
1806 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1807 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1808 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1809 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1810 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1811 				if (orig != data)
1812 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1813 
1814 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1815 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1816 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1817 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1818 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1819 				if (orig != data)
1820 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1821 
1822 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1823 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1824 				if (orig != data)
1825 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1826 
1827 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1828 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1829 				if (orig != data)
1830 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1831 
1832 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1833 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1834 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1835 				if (orig != data)
1836 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1837 			}
1838 		}
1839 	} else {
1840 		if (orig != data)
1841 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1842 	}
1843 
1844 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1845 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1846 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1847 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1848 	if (orig != data)
1849 		WREG32_PCIE(ixPCIE_CNTL2, data);
1850 
1851 	if (!disable_l0s) {
1852 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1853 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1854 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1855 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1856 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1857 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1858 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1859 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1860 				if (orig != data)
1861 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1862 			}
1863 		}
1864 	}
1865 }
1866 
1867 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1868 {
1869 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1870 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1871 }
1872 
1873 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1874 {
1875 	if (!ring || !ring->funcs->emit_wreg) {
1876 		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1877 		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1878 	} else {
1879 		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1880 	}
1881 }
1882 
1883 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1884 			       struct amdgpu_ring *ring)
1885 {
1886 	if (!ring || !ring->funcs->emit_wreg) {
1887 		WREG32(mmHDP_DEBUG0, 1);
1888 		RREG32(mmHDP_DEBUG0);
1889 	} else {
1890 		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1891 	}
1892 }
1893 
1894 static bool cik_need_full_reset(struct amdgpu_device *adev)
1895 {
1896 	/* change this when we support soft reset */
1897 	return true;
1898 }
1899 
1900 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1901 			       uint64_t *count1)
1902 {
1903 	uint32_t perfctr = 0;
1904 	uint64_t cnt0_of, cnt1_of;
1905 	int tmp;
1906 
1907 	/* This reports 0 on APUs, so return to avoid writing/reading registers
1908 	 * that may or may not be different from their GPU counterparts
1909 	 */
1910 	if (adev->flags & AMD_IS_APU)
1911 		return;
1912 
1913 	/* Set the 2 events that we wish to watch, defined above */
1914 	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1915 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1916 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1917 
1918 	/* Write to enable desired perf counters */
1919 	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1920 	/* Zero out and enable the perf counters
1921 	 * Write 0x5:
1922 	 * Bit 0 = Start all counters(1)
1923 	 * Bit 2 = Global counter reset enable(1)
1924 	 */
1925 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1926 
1927 	msleep(1000);
1928 
1929 	/* Load the shadow and disable the perf counters
1930 	 * Write 0x2:
1931 	 * Bit 0 = Stop counters(0)
1932 	 * Bit 1 = Load the shadow counters(1)
1933 	 */
1934 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1935 
1936 	/* Read register values to get any >32bit overflow */
1937 	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1938 	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1939 	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1940 
1941 	/* Get the values and add the overflow */
1942 	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1943 	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1944 }
1945 
1946 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1947 {
1948 	u32 clock_cntl, pc;
1949 
1950 	if (adev->flags & AMD_IS_APU)
1951 		return false;
1952 
1953 	/* check if the SMC is already running */
1954 	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1955 	pc = RREG32_SMC(ixSMC_PC_C);
1956 	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1957 	    (0x20100 <= pc))
1958 		return true;
1959 
1960 	return false;
1961 }
1962 
1963 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1964 {
1965 	uint64_t nak_r, nak_g;
1966 
1967 	/* Get the number of NAKs received and generated */
1968 	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1969 	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1970 
1971 	/* Add the total number of NAKs, i.e the number of replays */
1972 	return (nak_r + nak_g);
1973 }
1974 
1975 static void cik_pre_asic_init(struct amdgpu_device *adev)
1976 {
1977 }
1978 
1979 static const struct amdgpu_asic_funcs cik_asic_funcs =
1980 {
1981 	.read_disabled_bios = &cik_read_disabled_bios,
1982 	.read_bios_from_rom = &cik_read_bios_from_rom,
1983 	.read_register = &cik_read_register,
1984 	.reset = &cik_asic_reset,
1985 	.reset_method = &cik_asic_reset_method,
1986 	.set_vga_state = &cik_vga_set_state,
1987 	.get_xclk = &cik_get_xclk,
1988 	.set_uvd_clocks = &cik_set_uvd_clocks,
1989 	.set_vce_clocks = &cik_set_vce_clocks,
1990 	.get_config_memsize = &cik_get_config_memsize,
1991 	.flush_hdp = &cik_flush_hdp,
1992 	.invalidate_hdp = &cik_invalidate_hdp,
1993 	.need_full_reset = &cik_need_full_reset,
1994 	.init_doorbell_index = &legacy_doorbell_index_init,
1995 	.get_pcie_usage = &cik_get_pcie_usage,
1996 	.need_reset_on_init = &cik_need_reset_on_init,
1997 	.get_pcie_replay_count = &cik_get_pcie_replay_count,
1998 	.supports_baco = &cik_asic_supports_baco,
1999 	.pre_asic_init = &cik_pre_asic_init,
2000 	.query_video_codecs = &cik_query_video_codecs,
2001 };
2002 
2003 static int cik_common_early_init(void *handle)
2004 {
2005 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2006 
2007 	adev->smc_rreg = &cik_smc_rreg;
2008 	adev->smc_wreg = &cik_smc_wreg;
2009 	adev->pcie_rreg = &cik_pcie_rreg;
2010 	adev->pcie_wreg = &cik_pcie_wreg;
2011 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2012 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2013 	adev->didt_rreg = &cik_didt_rreg;
2014 	adev->didt_wreg = &cik_didt_wreg;
2015 
2016 	adev->asic_funcs = &cik_asic_funcs;
2017 
2018 	adev->rev_id = cik_get_rev_id(adev);
2019 	adev->external_rev_id = 0xFF;
2020 	switch (adev->asic_type) {
2021 	case CHIP_BONAIRE:
2022 		adev->cg_flags =
2023 			AMD_CG_SUPPORT_GFX_MGCG |
2024 			AMD_CG_SUPPORT_GFX_MGLS |
2025 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2026 			AMD_CG_SUPPORT_GFX_CGLS |
2027 			AMD_CG_SUPPORT_GFX_CGTS |
2028 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2029 			AMD_CG_SUPPORT_GFX_CP_LS |
2030 			AMD_CG_SUPPORT_MC_LS |
2031 			AMD_CG_SUPPORT_MC_MGCG |
2032 			AMD_CG_SUPPORT_SDMA_MGCG |
2033 			AMD_CG_SUPPORT_SDMA_LS |
2034 			AMD_CG_SUPPORT_BIF_LS |
2035 			AMD_CG_SUPPORT_VCE_MGCG |
2036 			AMD_CG_SUPPORT_UVD_MGCG |
2037 			AMD_CG_SUPPORT_HDP_LS |
2038 			AMD_CG_SUPPORT_HDP_MGCG;
2039 		adev->pg_flags = 0;
2040 		adev->external_rev_id = adev->rev_id + 0x14;
2041 		break;
2042 	case CHIP_HAWAII:
2043 		adev->cg_flags =
2044 			AMD_CG_SUPPORT_GFX_MGCG |
2045 			AMD_CG_SUPPORT_GFX_MGLS |
2046 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2047 			AMD_CG_SUPPORT_GFX_CGLS |
2048 			AMD_CG_SUPPORT_GFX_CGTS |
2049 			AMD_CG_SUPPORT_GFX_CP_LS |
2050 			AMD_CG_SUPPORT_MC_LS |
2051 			AMD_CG_SUPPORT_MC_MGCG |
2052 			AMD_CG_SUPPORT_SDMA_MGCG |
2053 			AMD_CG_SUPPORT_SDMA_LS |
2054 			AMD_CG_SUPPORT_BIF_LS |
2055 			AMD_CG_SUPPORT_VCE_MGCG |
2056 			AMD_CG_SUPPORT_UVD_MGCG |
2057 			AMD_CG_SUPPORT_HDP_LS |
2058 			AMD_CG_SUPPORT_HDP_MGCG;
2059 		adev->pg_flags = 0;
2060 		adev->external_rev_id = 0x28;
2061 		break;
2062 	case CHIP_KAVERI:
2063 		adev->cg_flags =
2064 			AMD_CG_SUPPORT_GFX_MGCG |
2065 			AMD_CG_SUPPORT_GFX_MGLS |
2066 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2067 			AMD_CG_SUPPORT_GFX_CGLS |
2068 			AMD_CG_SUPPORT_GFX_CGTS |
2069 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2070 			AMD_CG_SUPPORT_GFX_CP_LS |
2071 			AMD_CG_SUPPORT_SDMA_MGCG |
2072 			AMD_CG_SUPPORT_SDMA_LS |
2073 			AMD_CG_SUPPORT_BIF_LS |
2074 			AMD_CG_SUPPORT_VCE_MGCG |
2075 			AMD_CG_SUPPORT_UVD_MGCG |
2076 			AMD_CG_SUPPORT_HDP_LS |
2077 			AMD_CG_SUPPORT_HDP_MGCG;
2078 		adev->pg_flags =
2079 			/*AMD_PG_SUPPORT_GFX_PG |
2080 			  AMD_PG_SUPPORT_GFX_SMG |
2081 			  AMD_PG_SUPPORT_GFX_DMG |*/
2082 			AMD_PG_SUPPORT_UVD |
2083 			AMD_PG_SUPPORT_VCE |
2084 			/*  AMD_PG_SUPPORT_CP |
2085 			  AMD_PG_SUPPORT_GDS |
2086 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2087 			  AMD_PG_SUPPORT_ACP |
2088 			  AMD_PG_SUPPORT_SAMU |*/
2089 			0;
2090 		if (adev->pdev->device == 0x1312 ||
2091 			adev->pdev->device == 0x1316 ||
2092 			adev->pdev->device == 0x1317)
2093 			adev->external_rev_id = 0x41;
2094 		else
2095 			adev->external_rev_id = 0x1;
2096 		break;
2097 	case CHIP_KABINI:
2098 	case CHIP_MULLINS:
2099 		adev->cg_flags =
2100 			AMD_CG_SUPPORT_GFX_MGCG |
2101 			AMD_CG_SUPPORT_GFX_MGLS |
2102 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2103 			AMD_CG_SUPPORT_GFX_CGLS |
2104 			AMD_CG_SUPPORT_GFX_CGTS |
2105 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2106 			AMD_CG_SUPPORT_GFX_CP_LS |
2107 			AMD_CG_SUPPORT_SDMA_MGCG |
2108 			AMD_CG_SUPPORT_SDMA_LS |
2109 			AMD_CG_SUPPORT_BIF_LS |
2110 			AMD_CG_SUPPORT_VCE_MGCG |
2111 			AMD_CG_SUPPORT_UVD_MGCG |
2112 			AMD_CG_SUPPORT_HDP_LS |
2113 			AMD_CG_SUPPORT_HDP_MGCG;
2114 		adev->pg_flags =
2115 			/*AMD_PG_SUPPORT_GFX_PG |
2116 			  AMD_PG_SUPPORT_GFX_SMG | */
2117 			AMD_PG_SUPPORT_UVD |
2118 			/*AMD_PG_SUPPORT_VCE |
2119 			  AMD_PG_SUPPORT_CP |
2120 			  AMD_PG_SUPPORT_GDS |
2121 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2122 			  AMD_PG_SUPPORT_SAMU |*/
2123 			0;
2124 		if (adev->asic_type == CHIP_KABINI) {
2125 			if (adev->rev_id == 0)
2126 				adev->external_rev_id = 0x81;
2127 			else if (adev->rev_id == 1)
2128 				adev->external_rev_id = 0x82;
2129 			else if (adev->rev_id == 2)
2130 				adev->external_rev_id = 0x85;
2131 		} else
2132 			adev->external_rev_id = adev->rev_id + 0xa1;
2133 		break;
2134 	default:
2135 		/* FIXME: not supported yet */
2136 		return -EINVAL;
2137 	}
2138 
2139 	return 0;
2140 }
2141 
2142 static int cik_common_sw_init(void *handle)
2143 {
2144 	return 0;
2145 }
2146 
2147 static int cik_common_sw_fini(void *handle)
2148 {
2149 	return 0;
2150 }
2151 
2152 static int cik_common_hw_init(void *handle)
2153 {
2154 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2155 
2156 	/* move the golden regs per IP block */
2157 	cik_init_golden_registers(adev);
2158 	/* enable pcie gen2/3 link */
2159 	cik_pcie_gen3_enable(adev);
2160 	/* enable aspm */
2161 	cik_program_aspm(adev);
2162 
2163 	return 0;
2164 }
2165 
2166 static int cik_common_hw_fini(void *handle)
2167 {
2168 	return 0;
2169 }
2170 
2171 static int cik_common_suspend(void *handle)
2172 {
2173 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2174 
2175 	return cik_common_hw_fini(adev);
2176 }
2177 
2178 static int cik_common_resume(void *handle)
2179 {
2180 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2181 
2182 	return cik_common_hw_init(adev);
2183 }
2184 
2185 static bool cik_common_is_idle(void *handle)
2186 {
2187 	return true;
2188 }
2189 
2190 static int cik_common_wait_for_idle(void *handle)
2191 {
2192 	return 0;
2193 }
2194 
2195 static int cik_common_soft_reset(void *handle)
2196 {
2197 	/* XXX hard reset?? */
2198 	return 0;
2199 }
2200 
2201 static int cik_common_set_clockgating_state(void *handle,
2202 					    enum amd_clockgating_state state)
2203 {
2204 	return 0;
2205 }
2206 
2207 static int cik_common_set_powergating_state(void *handle,
2208 					    enum amd_powergating_state state)
2209 {
2210 	return 0;
2211 }
2212 
2213 static const struct amd_ip_funcs cik_common_ip_funcs = {
2214 	.name = "cik_common",
2215 	.early_init = cik_common_early_init,
2216 	.late_init = NULL,
2217 	.sw_init = cik_common_sw_init,
2218 	.sw_fini = cik_common_sw_fini,
2219 	.hw_init = cik_common_hw_init,
2220 	.hw_fini = cik_common_hw_fini,
2221 	.suspend = cik_common_suspend,
2222 	.resume = cik_common_resume,
2223 	.is_idle = cik_common_is_idle,
2224 	.wait_for_idle = cik_common_wait_for_idle,
2225 	.soft_reset = cik_common_soft_reset,
2226 	.set_clockgating_state = cik_common_set_clockgating_state,
2227 	.set_powergating_state = cik_common_set_powergating_state,
2228 };
2229 
2230 static const struct amdgpu_ip_block_version cik_common_ip_block =
2231 {
2232 	.type = AMD_IP_BLOCK_TYPE_COMMON,
2233 	.major = 1,
2234 	.minor = 0,
2235 	.rev = 0,
2236 	.funcs = &cik_common_ip_funcs,
2237 };
2238 
2239 int cik_set_ip_blocks(struct amdgpu_device *adev)
2240 {
2241 	switch (adev->asic_type) {
2242 	case CHIP_BONAIRE:
2243 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2244 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2245 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2246 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2247 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2248 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2249 		if (adev->enable_virtual_display)
2250 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2251 #if defined(CONFIG_DRM_AMD_DC)
2252 		else if (amdgpu_device_has_dc_support(adev))
2253 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2254 #endif
2255 		else
2256 			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2257 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2258 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2259 		break;
2260 	case CHIP_HAWAII:
2261 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2262 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2263 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2264 		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2265 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2266 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2267 		if (adev->enable_virtual_display)
2268 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2269 #if defined(CONFIG_DRM_AMD_DC)
2270 		else if (amdgpu_device_has_dc_support(adev))
2271 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2272 #endif
2273 		else
2274 			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2275 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2276 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2277 		break;
2278 	case CHIP_KAVERI:
2279 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2280 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2281 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2282 		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2283 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2284 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2285 		if (adev->enable_virtual_display)
2286 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2287 #if defined(CONFIG_DRM_AMD_DC)
2288 		else if (amdgpu_device_has_dc_support(adev))
2289 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2290 #endif
2291 		else
2292 			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2293 
2294 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2295 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2296 		break;
2297 	case CHIP_KABINI:
2298 	case CHIP_MULLINS:
2299 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2300 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2301 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2302 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2303 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2304 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2305 		if (adev->enable_virtual_display)
2306 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2307 #if defined(CONFIG_DRM_AMD_DC)
2308 		else if (amdgpu_device_has_dc_support(adev))
2309 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2310 #endif
2311 		else
2312 			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2313 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2314 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2315 		break;
2316 	default:
2317 		/* FIXME: not supported yet */
2318 		return -EINVAL;
2319 	}
2320 	return 0;
2321 }
2322