xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision d7955ce4) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include <drm/amdgpu_drm.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_atombios.h"
33 #include "amdgpu_ih.h"
34 #include "amdgpu_uvd.h"
35 #include "amdgpu_vce.h"
36 #include "cikd.h"
37 #include "atom.h"
38 #include "amd_pcie.h"
39 
40 #include "cik.h"
41 #include "gmc_v7_0.h"
42 #include "cik_ih.h"
43 #include "dce_v8_0.h"
44 #include "gfx_v7_0.h"
45 #include "cik_sdma.h"
46 #include "uvd_v4_2.h"
47 #include "vce_v2_0.h"
48 #include "cik_dpm.h"
49 
50 #include "uvd/uvd_4_2_d.h"
51 
52 #include "smu/smu_7_0_1_d.h"
53 #include "smu/smu_7_0_1_sh_mask.h"
54 
55 #include "dce/dce_8_0_d.h"
56 #include "dce/dce_8_0_sh_mask.h"
57 
58 #include "bif/bif_4_1_d.h"
59 #include "bif/bif_4_1_sh_mask.h"
60 
61 #include "gca/gfx_7_2_d.h"
62 #include "gca/gfx_7_2_enum.h"
63 #include "gca/gfx_7_2_sh_mask.h"
64 
65 #include "gmc/gmc_7_1_d.h"
66 #include "gmc/gmc_7_1_sh_mask.h"
67 
68 #include "oss/oss_2_0_d.h"
69 #include "oss/oss_2_0_sh_mask.h"
70 
71 #include "amdgpu_dm.h"
72 #include "amdgpu_amdkfd.h"
73 #include "amdgpu_vkms.h"
74 
75 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
76 {
77 	{
78 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
79 		.max_width = 2048,
80 		.max_height = 1152,
81 		.max_pixels_per_frame = 2048 * 1152,
82 		.max_level = 0,
83 	},
84 };
85 
86 static const struct amdgpu_video_codecs cik_video_codecs_encode =
87 {
88 	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
89 	.codec_array = cik_video_codecs_encode_array,
90 };
91 
92 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
93 {
94 	{
95 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
96 		.max_width = 2048,
97 		.max_height = 1152,
98 		.max_pixels_per_frame = 2048 * 1152,
99 		.max_level = 3,
100 	},
101 	{
102 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
103 		.max_width = 2048,
104 		.max_height = 1152,
105 		.max_pixels_per_frame = 2048 * 1152,
106 		.max_level = 5,
107 	},
108 	{
109 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
110 		.max_width = 2048,
111 		.max_height = 1152,
112 		.max_pixels_per_frame = 2048 * 1152,
113 		.max_level = 41,
114 	},
115 	{
116 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
117 		.max_width = 2048,
118 		.max_height = 1152,
119 		.max_pixels_per_frame = 2048 * 1152,
120 		.max_level = 4,
121 	},
122 };
123 
124 static const struct amdgpu_video_codecs cik_video_codecs_decode =
125 {
126 	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
127 	.codec_array = cik_video_codecs_decode_array,
128 };
129 
130 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
131 				  const struct amdgpu_video_codecs **codecs)
132 {
133 	switch (adev->asic_type) {
134 	case CHIP_BONAIRE:
135 	case CHIP_HAWAII:
136 	case CHIP_KAVERI:
137 	case CHIP_KABINI:
138 	case CHIP_MULLINS:
139 		if (encode)
140 			*codecs = &cik_video_codecs_encode;
141 		else
142 			*codecs = &cik_video_codecs_decode;
143 		return 0;
144 	default:
145 		return -EINVAL;
146 	}
147 }
148 
149 /*
150  * Indirect registers accessor
151  */
152 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
153 {
154 	unsigned long flags;
155 	u32 r;
156 
157 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
158 	WREG32(mmPCIE_INDEX, reg);
159 	(void)RREG32(mmPCIE_INDEX);
160 	r = RREG32(mmPCIE_DATA);
161 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
162 	return r;
163 }
164 
165 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
166 {
167 	unsigned long flags;
168 
169 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
170 	WREG32(mmPCIE_INDEX, reg);
171 	(void)RREG32(mmPCIE_INDEX);
172 	WREG32(mmPCIE_DATA, v);
173 	(void)RREG32(mmPCIE_DATA);
174 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
175 }
176 
177 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
178 {
179 	unsigned long flags;
180 	u32 r;
181 
182 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
183 	WREG32(mmSMC_IND_INDEX_0, (reg));
184 	r = RREG32(mmSMC_IND_DATA_0);
185 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
186 	return r;
187 }
188 
189 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
190 {
191 	unsigned long flags;
192 
193 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
194 	WREG32(mmSMC_IND_INDEX_0, (reg));
195 	WREG32(mmSMC_IND_DATA_0, (v));
196 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
197 }
198 
199 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
200 {
201 	unsigned long flags;
202 	u32 r;
203 
204 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
205 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
206 	r = RREG32(mmUVD_CTX_DATA);
207 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
208 	return r;
209 }
210 
211 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
212 {
213 	unsigned long flags;
214 
215 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
216 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
217 	WREG32(mmUVD_CTX_DATA, (v));
218 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
219 }
220 
221 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
222 {
223 	unsigned long flags;
224 	u32 r;
225 
226 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
227 	WREG32(mmDIDT_IND_INDEX, (reg));
228 	r = RREG32(mmDIDT_IND_DATA);
229 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
230 	return r;
231 }
232 
233 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
234 {
235 	unsigned long flags;
236 
237 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
238 	WREG32(mmDIDT_IND_INDEX, (reg));
239 	WREG32(mmDIDT_IND_DATA, (v));
240 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
241 }
242 
243 static const u32 bonaire_golden_spm_registers[] =
244 {
245 	0xc200, 0xe0ffffff, 0xe0000000
246 };
247 
248 static const u32 bonaire_golden_common_registers[] =
249 {
250 	0x31dc, 0xffffffff, 0x00000800,
251 	0x31dd, 0xffffffff, 0x00000800,
252 	0x31e6, 0xffffffff, 0x00007fbf,
253 	0x31e7, 0xffffffff, 0x00007faf
254 };
255 
256 static const u32 bonaire_golden_registers[] =
257 {
258 	0xcd5, 0x00000333, 0x00000333,
259 	0xcd4, 0x000c0fc0, 0x00040200,
260 	0x2684, 0x00010000, 0x00058208,
261 	0xf000, 0xffff1fff, 0x00140000,
262 	0xf080, 0xfdfc0fff, 0x00000100,
263 	0xf08d, 0x40000000, 0x40000200,
264 	0x260c, 0xffffffff, 0x00000000,
265 	0x260d, 0xf00fffff, 0x00000400,
266 	0x260e, 0x0002021c, 0x00020200,
267 	0x31e, 0x00000080, 0x00000000,
268 	0x16ec, 0x000000f0, 0x00000070,
269 	0x16f0, 0xf0311fff, 0x80300000,
270 	0x263e, 0x73773777, 0x12010001,
271 	0xd43, 0x00810000, 0x408af000,
272 	0x1c0c, 0x31000111, 0x00000011,
273 	0xbd2, 0x73773777, 0x12010001,
274 	0x883, 0x00007fb6, 0x0021a1b1,
275 	0x884, 0x00007fb6, 0x002021b1,
276 	0x860, 0x00007fb6, 0x00002191,
277 	0x886, 0x00007fb6, 0x002121b1,
278 	0x887, 0x00007fb6, 0x002021b1,
279 	0x877, 0x00007fb6, 0x00002191,
280 	0x878, 0x00007fb6, 0x00002191,
281 	0xd8a, 0x0000003f, 0x0000000a,
282 	0xd8b, 0x0000003f, 0x0000000a,
283 	0xab9, 0x00073ffe, 0x000022a2,
284 	0x903, 0x000007ff, 0x00000000,
285 	0x2285, 0xf000003f, 0x00000007,
286 	0x22fc, 0x00002001, 0x00000001,
287 	0x22c9, 0xffffffff, 0x00ffffff,
288 	0xc281, 0x0000ff0f, 0x00000000,
289 	0xa293, 0x07ffffff, 0x06000000,
290 	0x136, 0x00000fff, 0x00000100,
291 	0xf9e, 0x00000001, 0x00000002,
292 	0x2440, 0x03000000, 0x0362c688,
293 	0x2300, 0x000000ff, 0x00000001,
294 	0x390, 0x00001fff, 0x00001fff,
295 	0x2418, 0x0000007f, 0x00000020,
296 	0x2542, 0x00010000, 0x00010000,
297 	0x2b05, 0x000003ff, 0x000000f3,
298 	0x2b03, 0xffffffff, 0x00001032
299 };
300 
301 static const u32 bonaire_mgcg_cgcg_init[] =
302 {
303 	0x3108, 0xffffffff, 0xfffffffc,
304 	0xc200, 0xffffffff, 0xe0000000,
305 	0xf0a8, 0xffffffff, 0x00000100,
306 	0xf082, 0xffffffff, 0x00000100,
307 	0xf0b0, 0xffffffff, 0xc0000100,
308 	0xf0b2, 0xffffffff, 0xc0000100,
309 	0xf0b1, 0xffffffff, 0xc0000100,
310 	0x1579, 0xffffffff, 0x00600100,
311 	0xf0a0, 0xffffffff, 0x00000100,
312 	0xf085, 0xffffffff, 0x06000100,
313 	0xf088, 0xffffffff, 0x00000100,
314 	0xf086, 0xffffffff, 0x06000100,
315 	0xf081, 0xffffffff, 0x00000100,
316 	0xf0b8, 0xffffffff, 0x00000100,
317 	0xf089, 0xffffffff, 0x00000100,
318 	0xf080, 0xffffffff, 0x00000100,
319 	0xf08c, 0xffffffff, 0x00000100,
320 	0xf08d, 0xffffffff, 0x00000100,
321 	0xf094, 0xffffffff, 0x00000100,
322 	0xf095, 0xffffffff, 0x00000100,
323 	0xf096, 0xffffffff, 0x00000100,
324 	0xf097, 0xffffffff, 0x00000100,
325 	0xf098, 0xffffffff, 0x00000100,
326 	0xf09f, 0xffffffff, 0x00000100,
327 	0xf09e, 0xffffffff, 0x00000100,
328 	0xf084, 0xffffffff, 0x06000100,
329 	0xf0a4, 0xffffffff, 0x00000100,
330 	0xf09d, 0xffffffff, 0x00000100,
331 	0xf0ad, 0xffffffff, 0x00000100,
332 	0xf0ac, 0xffffffff, 0x00000100,
333 	0xf09c, 0xffffffff, 0x00000100,
334 	0xc200, 0xffffffff, 0xe0000000,
335 	0xf008, 0xffffffff, 0x00010000,
336 	0xf009, 0xffffffff, 0x00030002,
337 	0xf00a, 0xffffffff, 0x00040007,
338 	0xf00b, 0xffffffff, 0x00060005,
339 	0xf00c, 0xffffffff, 0x00090008,
340 	0xf00d, 0xffffffff, 0x00010000,
341 	0xf00e, 0xffffffff, 0x00030002,
342 	0xf00f, 0xffffffff, 0x00040007,
343 	0xf010, 0xffffffff, 0x00060005,
344 	0xf011, 0xffffffff, 0x00090008,
345 	0xf012, 0xffffffff, 0x00010000,
346 	0xf013, 0xffffffff, 0x00030002,
347 	0xf014, 0xffffffff, 0x00040007,
348 	0xf015, 0xffffffff, 0x00060005,
349 	0xf016, 0xffffffff, 0x00090008,
350 	0xf017, 0xffffffff, 0x00010000,
351 	0xf018, 0xffffffff, 0x00030002,
352 	0xf019, 0xffffffff, 0x00040007,
353 	0xf01a, 0xffffffff, 0x00060005,
354 	0xf01b, 0xffffffff, 0x00090008,
355 	0xf01c, 0xffffffff, 0x00010000,
356 	0xf01d, 0xffffffff, 0x00030002,
357 	0xf01e, 0xffffffff, 0x00040007,
358 	0xf01f, 0xffffffff, 0x00060005,
359 	0xf020, 0xffffffff, 0x00090008,
360 	0xf021, 0xffffffff, 0x00010000,
361 	0xf022, 0xffffffff, 0x00030002,
362 	0xf023, 0xffffffff, 0x00040007,
363 	0xf024, 0xffffffff, 0x00060005,
364 	0xf025, 0xffffffff, 0x00090008,
365 	0xf026, 0xffffffff, 0x00010000,
366 	0xf027, 0xffffffff, 0x00030002,
367 	0xf028, 0xffffffff, 0x00040007,
368 	0xf029, 0xffffffff, 0x00060005,
369 	0xf02a, 0xffffffff, 0x00090008,
370 	0xf000, 0xffffffff, 0x96e00200,
371 	0x21c2, 0xffffffff, 0x00900100,
372 	0x3109, 0xffffffff, 0x0020003f,
373 	0xe, 0xffffffff, 0x0140001c,
374 	0xf, 0x000f0000, 0x000f0000,
375 	0x88, 0xffffffff, 0xc060000c,
376 	0x89, 0xc0000fff, 0x00000100,
377 	0x3e4, 0xffffffff, 0x00000100,
378 	0x3e6, 0x00000101, 0x00000000,
379 	0x82a, 0xffffffff, 0x00000104,
380 	0x1579, 0xff000fff, 0x00000100,
381 	0xc33, 0xc0000fff, 0x00000104,
382 	0x3079, 0x00000001, 0x00000001,
383 	0x3403, 0xff000ff0, 0x00000100,
384 	0x3603, 0xff000ff0, 0x00000100
385 };
386 
387 static const u32 spectre_golden_spm_registers[] =
388 {
389 	0xc200, 0xe0ffffff, 0xe0000000
390 };
391 
392 static const u32 spectre_golden_common_registers[] =
393 {
394 	0x31dc, 0xffffffff, 0x00000800,
395 	0x31dd, 0xffffffff, 0x00000800,
396 	0x31e6, 0xffffffff, 0x00007fbf,
397 	0x31e7, 0xffffffff, 0x00007faf
398 };
399 
400 static const u32 spectre_golden_registers[] =
401 {
402 	0xf000, 0xffff1fff, 0x96940200,
403 	0xf003, 0xffff0001, 0xff000000,
404 	0xf080, 0xfffc0fff, 0x00000100,
405 	0x1bb6, 0x00010101, 0x00010000,
406 	0x260d, 0xf00fffff, 0x00000400,
407 	0x260e, 0xfffffffc, 0x00020200,
408 	0x16ec, 0x000000f0, 0x00000070,
409 	0x16f0, 0xf0311fff, 0x80300000,
410 	0x263e, 0x73773777, 0x12010001,
411 	0x26df, 0x00ff0000, 0x00fc0000,
412 	0xbd2, 0x73773777, 0x12010001,
413 	0x2285, 0xf000003f, 0x00000007,
414 	0x22c9, 0xffffffff, 0x00ffffff,
415 	0xa0d4, 0x3f3f3fff, 0x00000082,
416 	0xa0d5, 0x0000003f, 0x00000000,
417 	0xf9e, 0x00000001, 0x00000002,
418 	0x244f, 0xffff03df, 0x00000004,
419 	0x31da, 0x00000008, 0x00000008,
420 	0x2300, 0x000008ff, 0x00000800,
421 	0x2542, 0x00010000, 0x00010000,
422 	0x2b03, 0xffffffff, 0x54763210,
423 	0x853e, 0x01ff01ff, 0x00000002,
424 	0x8526, 0x007ff800, 0x00200000,
425 	0x8057, 0xffffffff, 0x00000f40,
426 	0xc24d, 0xffffffff, 0x00000001
427 };
428 
429 static const u32 spectre_mgcg_cgcg_init[] =
430 {
431 	0x3108, 0xffffffff, 0xfffffffc,
432 	0xc200, 0xffffffff, 0xe0000000,
433 	0xf0a8, 0xffffffff, 0x00000100,
434 	0xf082, 0xffffffff, 0x00000100,
435 	0xf0b0, 0xffffffff, 0x00000100,
436 	0xf0b2, 0xffffffff, 0x00000100,
437 	0xf0b1, 0xffffffff, 0x00000100,
438 	0x1579, 0xffffffff, 0x00600100,
439 	0xf0a0, 0xffffffff, 0x00000100,
440 	0xf085, 0xffffffff, 0x06000100,
441 	0xf088, 0xffffffff, 0x00000100,
442 	0xf086, 0xffffffff, 0x06000100,
443 	0xf081, 0xffffffff, 0x00000100,
444 	0xf0b8, 0xffffffff, 0x00000100,
445 	0xf089, 0xffffffff, 0x00000100,
446 	0xf080, 0xffffffff, 0x00000100,
447 	0xf08c, 0xffffffff, 0x00000100,
448 	0xf08d, 0xffffffff, 0x00000100,
449 	0xf094, 0xffffffff, 0x00000100,
450 	0xf095, 0xffffffff, 0x00000100,
451 	0xf096, 0xffffffff, 0x00000100,
452 	0xf097, 0xffffffff, 0x00000100,
453 	0xf098, 0xffffffff, 0x00000100,
454 	0xf09f, 0xffffffff, 0x00000100,
455 	0xf09e, 0xffffffff, 0x00000100,
456 	0xf084, 0xffffffff, 0x06000100,
457 	0xf0a4, 0xffffffff, 0x00000100,
458 	0xf09d, 0xffffffff, 0x00000100,
459 	0xf0ad, 0xffffffff, 0x00000100,
460 	0xf0ac, 0xffffffff, 0x00000100,
461 	0xf09c, 0xffffffff, 0x00000100,
462 	0xc200, 0xffffffff, 0xe0000000,
463 	0xf008, 0xffffffff, 0x00010000,
464 	0xf009, 0xffffffff, 0x00030002,
465 	0xf00a, 0xffffffff, 0x00040007,
466 	0xf00b, 0xffffffff, 0x00060005,
467 	0xf00c, 0xffffffff, 0x00090008,
468 	0xf00d, 0xffffffff, 0x00010000,
469 	0xf00e, 0xffffffff, 0x00030002,
470 	0xf00f, 0xffffffff, 0x00040007,
471 	0xf010, 0xffffffff, 0x00060005,
472 	0xf011, 0xffffffff, 0x00090008,
473 	0xf012, 0xffffffff, 0x00010000,
474 	0xf013, 0xffffffff, 0x00030002,
475 	0xf014, 0xffffffff, 0x00040007,
476 	0xf015, 0xffffffff, 0x00060005,
477 	0xf016, 0xffffffff, 0x00090008,
478 	0xf017, 0xffffffff, 0x00010000,
479 	0xf018, 0xffffffff, 0x00030002,
480 	0xf019, 0xffffffff, 0x00040007,
481 	0xf01a, 0xffffffff, 0x00060005,
482 	0xf01b, 0xffffffff, 0x00090008,
483 	0xf01c, 0xffffffff, 0x00010000,
484 	0xf01d, 0xffffffff, 0x00030002,
485 	0xf01e, 0xffffffff, 0x00040007,
486 	0xf01f, 0xffffffff, 0x00060005,
487 	0xf020, 0xffffffff, 0x00090008,
488 	0xf021, 0xffffffff, 0x00010000,
489 	0xf022, 0xffffffff, 0x00030002,
490 	0xf023, 0xffffffff, 0x00040007,
491 	0xf024, 0xffffffff, 0x00060005,
492 	0xf025, 0xffffffff, 0x00090008,
493 	0xf026, 0xffffffff, 0x00010000,
494 	0xf027, 0xffffffff, 0x00030002,
495 	0xf028, 0xffffffff, 0x00040007,
496 	0xf029, 0xffffffff, 0x00060005,
497 	0xf02a, 0xffffffff, 0x00090008,
498 	0xf02b, 0xffffffff, 0x00010000,
499 	0xf02c, 0xffffffff, 0x00030002,
500 	0xf02d, 0xffffffff, 0x00040007,
501 	0xf02e, 0xffffffff, 0x00060005,
502 	0xf02f, 0xffffffff, 0x00090008,
503 	0xf000, 0xffffffff, 0x96e00200,
504 	0x21c2, 0xffffffff, 0x00900100,
505 	0x3109, 0xffffffff, 0x0020003f,
506 	0xe, 0xffffffff, 0x0140001c,
507 	0xf, 0x000f0000, 0x000f0000,
508 	0x88, 0xffffffff, 0xc060000c,
509 	0x89, 0xc0000fff, 0x00000100,
510 	0x3e4, 0xffffffff, 0x00000100,
511 	0x3e6, 0x00000101, 0x00000000,
512 	0x82a, 0xffffffff, 0x00000104,
513 	0x1579, 0xff000fff, 0x00000100,
514 	0xc33, 0xc0000fff, 0x00000104,
515 	0x3079, 0x00000001, 0x00000001,
516 	0x3403, 0xff000ff0, 0x00000100,
517 	0x3603, 0xff000ff0, 0x00000100
518 };
519 
520 static const u32 kalindi_golden_spm_registers[] =
521 {
522 	0xc200, 0xe0ffffff, 0xe0000000
523 };
524 
525 static const u32 kalindi_golden_common_registers[] =
526 {
527 	0x31dc, 0xffffffff, 0x00000800,
528 	0x31dd, 0xffffffff, 0x00000800,
529 	0x31e6, 0xffffffff, 0x00007fbf,
530 	0x31e7, 0xffffffff, 0x00007faf
531 };
532 
533 static const u32 kalindi_golden_registers[] =
534 {
535 	0xf000, 0xffffdfff, 0x6e944040,
536 	0x1579, 0xff607fff, 0xfc000100,
537 	0xf088, 0xff000fff, 0x00000100,
538 	0xf089, 0xff000fff, 0x00000100,
539 	0xf080, 0xfffc0fff, 0x00000100,
540 	0x1bb6, 0x00010101, 0x00010000,
541 	0x260c, 0xffffffff, 0x00000000,
542 	0x260d, 0xf00fffff, 0x00000400,
543 	0x16ec, 0x000000f0, 0x00000070,
544 	0x16f0, 0xf0311fff, 0x80300000,
545 	0x263e, 0x73773777, 0x12010001,
546 	0x263f, 0xffffffff, 0x00000010,
547 	0x26df, 0x00ff0000, 0x00fc0000,
548 	0x200c, 0x00001f0f, 0x0000100a,
549 	0xbd2, 0x73773777, 0x12010001,
550 	0x902, 0x000fffff, 0x000c007f,
551 	0x2285, 0xf000003f, 0x00000007,
552 	0x22c9, 0x3fff3fff, 0x00ffcfff,
553 	0xc281, 0x0000ff0f, 0x00000000,
554 	0xa293, 0x07ffffff, 0x06000000,
555 	0x136, 0x00000fff, 0x00000100,
556 	0xf9e, 0x00000001, 0x00000002,
557 	0x31da, 0x00000008, 0x00000008,
558 	0x2300, 0x000000ff, 0x00000003,
559 	0x853e, 0x01ff01ff, 0x00000002,
560 	0x8526, 0x007ff800, 0x00200000,
561 	0x8057, 0xffffffff, 0x00000f40,
562 	0x2231, 0x001f3ae3, 0x00000082,
563 	0x2235, 0x0000001f, 0x00000010,
564 	0xc24d, 0xffffffff, 0x00000000
565 };
566 
567 static const u32 kalindi_mgcg_cgcg_init[] =
568 {
569 	0x3108, 0xffffffff, 0xfffffffc,
570 	0xc200, 0xffffffff, 0xe0000000,
571 	0xf0a8, 0xffffffff, 0x00000100,
572 	0xf082, 0xffffffff, 0x00000100,
573 	0xf0b0, 0xffffffff, 0x00000100,
574 	0xf0b2, 0xffffffff, 0x00000100,
575 	0xf0b1, 0xffffffff, 0x00000100,
576 	0x1579, 0xffffffff, 0x00600100,
577 	0xf0a0, 0xffffffff, 0x00000100,
578 	0xf085, 0xffffffff, 0x06000100,
579 	0xf088, 0xffffffff, 0x00000100,
580 	0xf086, 0xffffffff, 0x06000100,
581 	0xf081, 0xffffffff, 0x00000100,
582 	0xf0b8, 0xffffffff, 0x00000100,
583 	0xf089, 0xffffffff, 0x00000100,
584 	0xf080, 0xffffffff, 0x00000100,
585 	0xf08c, 0xffffffff, 0x00000100,
586 	0xf08d, 0xffffffff, 0x00000100,
587 	0xf094, 0xffffffff, 0x00000100,
588 	0xf095, 0xffffffff, 0x00000100,
589 	0xf096, 0xffffffff, 0x00000100,
590 	0xf097, 0xffffffff, 0x00000100,
591 	0xf098, 0xffffffff, 0x00000100,
592 	0xf09f, 0xffffffff, 0x00000100,
593 	0xf09e, 0xffffffff, 0x00000100,
594 	0xf084, 0xffffffff, 0x06000100,
595 	0xf0a4, 0xffffffff, 0x00000100,
596 	0xf09d, 0xffffffff, 0x00000100,
597 	0xf0ad, 0xffffffff, 0x00000100,
598 	0xf0ac, 0xffffffff, 0x00000100,
599 	0xf09c, 0xffffffff, 0x00000100,
600 	0xc200, 0xffffffff, 0xe0000000,
601 	0xf008, 0xffffffff, 0x00010000,
602 	0xf009, 0xffffffff, 0x00030002,
603 	0xf00a, 0xffffffff, 0x00040007,
604 	0xf00b, 0xffffffff, 0x00060005,
605 	0xf00c, 0xffffffff, 0x00090008,
606 	0xf00d, 0xffffffff, 0x00010000,
607 	0xf00e, 0xffffffff, 0x00030002,
608 	0xf00f, 0xffffffff, 0x00040007,
609 	0xf010, 0xffffffff, 0x00060005,
610 	0xf011, 0xffffffff, 0x00090008,
611 	0xf000, 0xffffffff, 0x96e00200,
612 	0x21c2, 0xffffffff, 0x00900100,
613 	0x3109, 0xffffffff, 0x0020003f,
614 	0xe, 0xffffffff, 0x0140001c,
615 	0xf, 0x000f0000, 0x000f0000,
616 	0x88, 0xffffffff, 0xc060000c,
617 	0x89, 0xc0000fff, 0x00000100,
618 	0x82a, 0xffffffff, 0x00000104,
619 	0x1579, 0xff000fff, 0x00000100,
620 	0xc33, 0xc0000fff, 0x00000104,
621 	0x3079, 0x00000001, 0x00000001,
622 	0x3403, 0xff000ff0, 0x00000100,
623 	0x3603, 0xff000ff0, 0x00000100
624 };
625 
626 static const u32 hawaii_golden_spm_registers[] =
627 {
628 	0xc200, 0xe0ffffff, 0xe0000000
629 };
630 
631 static const u32 hawaii_golden_common_registers[] =
632 {
633 	0xc200, 0xffffffff, 0xe0000000,
634 	0xa0d4, 0xffffffff, 0x3a00161a,
635 	0xa0d5, 0xffffffff, 0x0000002e,
636 	0x2684, 0xffffffff, 0x00018208,
637 	0x263e, 0xffffffff, 0x12011003
638 };
639 
640 static const u32 hawaii_golden_registers[] =
641 {
642 	0xcd5, 0x00000333, 0x00000333,
643 	0x2684, 0x00010000, 0x00058208,
644 	0x260c, 0xffffffff, 0x00000000,
645 	0x260d, 0xf00fffff, 0x00000400,
646 	0x260e, 0x0002021c, 0x00020200,
647 	0x31e, 0x00000080, 0x00000000,
648 	0x16ec, 0x000000f0, 0x00000070,
649 	0x16f0, 0xf0311fff, 0x80300000,
650 	0xd43, 0x00810000, 0x408af000,
651 	0x1c0c, 0x31000111, 0x00000011,
652 	0xbd2, 0x73773777, 0x12010001,
653 	0x848, 0x0000007f, 0x0000001b,
654 	0x877, 0x00007fb6, 0x00002191,
655 	0xd8a, 0x0000003f, 0x0000000a,
656 	0xd8b, 0x0000003f, 0x0000000a,
657 	0xab9, 0x00073ffe, 0x000022a2,
658 	0x903, 0x000007ff, 0x00000000,
659 	0x22fc, 0x00002001, 0x00000001,
660 	0x22c9, 0xffffffff, 0x00ffffff,
661 	0xc281, 0x0000ff0f, 0x00000000,
662 	0xa293, 0x07ffffff, 0x06000000,
663 	0xf9e, 0x00000001, 0x00000002,
664 	0x31da, 0x00000008, 0x00000008,
665 	0x31dc, 0x00000f00, 0x00000800,
666 	0x31dd, 0x00000f00, 0x00000800,
667 	0x31e6, 0x00ffffff, 0x00ff7fbf,
668 	0x31e7, 0x00ffffff, 0x00ff7faf,
669 	0x2300, 0x000000ff, 0x00000800,
670 	0x390, 0x00001fff, 0x00001fff,
671 	0x2418, 0x0000007f, 0x00000020,
672 	0x2542, 0x00010000, 0x00010000,
673 	0x2b80, 0x00100000, 0x000ff07c,
674 	0x2b05, 0x000003ff, 0x0000000f,
675 	0x2b04, 0xffffffff, 0x7564fdec,
676 	0x2b03, 0xffffffff, 0x3120b9a8,
677 	0x2b02, 0x20000000, 0x0f9c0000
678 };
679 
680 static const u32 hawaii_mgcg_cgcg_init[] =
681 {
682 	0x3108, 0xffffffff, 0xfffffffd,
683 	0xc200, 0xffffffff, 0xe0000000,
684 	0xf0a8, 0xffffffff, 0x00000100,
685 	0xf082, 0xffffffff, 0x00000100,
686 	0xf0b0, 0xffffffff, 0x00000100,
687 	0xf0b2, 0xffffffff, 0x00000100,
688 	0xf0b1, 0xffffffff, 0x00000100,
689 	0x1579, 0xffffffff, 0x00200100,
690 	0xf0a0, 0xffffffff, 0x00000100,
691 	0xf085, 0xffffffff, 0x06000100,
692 	0xf088, 0xffffffff, 0x00000100,
693 	0xf086, 0xffffffff, 0x06000100,
694 	0xf081, 0xffffffff, 0x00000100,
695 	0xf0b8, 0xffffffff, 0x00000100,
696 	0xf089, 0xffffffff, 0x00000100,
697 	0xf080, 0xffffffff, 0x00000100,
698 	0xf08c, 0xffffffff, 0x00000100,
699 	0xf08d, 0xffffffff, 0x00000100,
700 	0xf094, 0xffffffff, 0x00000100,
701 	0xf095, 0xffffffff, 0x00000100,
702 	0xf096, 0xffffffff, 0x00000100,
703 	0xf097, 0xffffffff, 0x00000100,
704 	0xf098, 0xffffffff, 0x00000100,
705 	0xf09f, 0xffffffff, 0x00000100,
706 	0xf09e, 0xffffffff, 0x00000100,
707 	0xf084, 0xffffffff, 0x06000100,
708 	0xf0a4, 0xffffffff, 0x00000100,
709 	0xf09d, 0xffffffff, 0x00000100,
710 	0xf0ad, 0xffffffff, 0x00000100,
711 	0xf0ac, 0xffffffff, 0x00000100,
712 	0xf09c, 0xffffffff, 0x00000100,
713 	0xc200, 0xffffffff, 0xe0000000,
714 	0xf008, 0xffffffff, 0x00010000,
715 	0xf009, 0xffffffff, 0x00030002,
716 	0xf00a, 0xffffffff, 0x00040007,
717 	0xf00b, 0xffffffff, 0x00060005,
718 	0xf00c, 0xffffffff, 0x00090008,
719 	0xf00d, 0xffffffff, 0x00010000,
720 	0xf00e, 0xffffffff, 0x00030002,
721 	0xf00f, 0xffffffff, 0x00040007,
722 	0xf010, 0xffffffff, 0x00060005,
723 	0xf011, 0xffffffff, 0x00090008,
724 	0xf012, 0xffffffff, 0x00010000,
725 	0xf013, 0xffffffff, 0x00030002,
726 	0xf014, 0xffffffff, 0x00040007,
727 	0xf015, 0xffffffff, 0x00060005,
728 	0xf016, 0xffffffff, 0x00090008,
729 	0xf017, 0xffffffff, 0x00010000,
730 	0xf018, 0xffffffff, 0x00030002,
731 	0xf019, 0xffffffff, 0x00040007,
732 	0xf01a, 0xffffffff, 0x00060005,
733 	0xf01b, 0xffffffff, 0x00090008,
734 	0xf01c, 0xffffffff, 0x00010000,
735 	0xf01d, 0xffffffff, 0x00030002,
736 	0xf01e, 0xffffffff, 0x00040007,
737 	0xf01f, 0xffffffff, 0x00060005,
738 	0xf020, 0xffffffff, 0x00090008,
739 	0xf021, 0xffffffff, 0x00010000,
740 	0xf022, 0xffffffff, 0x00030002,
741 	0xf023, 0xffffffff, 0x00040007,
742 	0xf024, 0xffffffff, 0x00060005,
743 	0xf025, 0xffffffff, 0x00090008,
744 	0xf026, 0xffffffff, 0x00010000,
745 	0xf027, 0xffffffff, 0x00030002,
746 	0xf028, 0xffffffff, 0x00040007,
747 	0xf029, 0xffffffff, 0x00060005,
748 	0xf02a, 0xffffffff, 0x00090008,
749 	0xf02b, 0xffffffff, 0x00010000,
750 	0xf02c, 0xffffffff, 0x00030002,
751 	0xf02d, 0xffffffff, 0x00040007,
752 	0xf02e, 0xffffffff, 0x00060005,
753 	0xf02f, 0xffffffff, 0x00090008,
754 	0xf030, 0xffffffff, 0x00010000,
755 	0xf031, 0xffffffff, 0x00030002,
756 	0xf032, 0xffffffff, 0x00040007,
757 	0xf033, 0xffffffff, 0x00060005,
758 	0xf034, 0xffffffff, 0x00090008,
759 	0xf035, 0xffffffff, 0x00010000,
760 	0xf036, 0xffffffff, 0x00030002,
761 	0xf037, 0xffffffff, 0x00040007,
762 	0xf038, 0xffffffff, 0x00060005,
763 	0xf039, 0xffffffff, 0x00090008,
764 	0xf03a, 0xffffffff, 0x00010000,
765 	0xf03b, 0xffffffff, 0x00030002,
766 	0xf03c, 0xffffffff, 0x00040007,
767 	0xf03d, 0xffffffff, 0x00060005,
768 	0xf03e, 0xffffffff, 0x00090008,
769 	0x30c6, 0xffffffff, 0x00020200,
770 	0xcd4, 0xffffffff, 0x00000200,
771 	0x570, 0xffffffff, 0x00000400,
772 	0x157a, 0xffffffff, 0x00000000,
773 	0xbd4, 0xffffffff, 0x00000902,
774 	0xf000, 0xffffffff, 0x96940200,
775 	0x21c2, 0xffffffff, 0x00900100,
776 	0x3109, 0xffffffff, 0x0020003f,
777 	0xe, 0xffffffff, 0x0140001c,
778 	0xf, 0x000f0000, 0x000f0000,
779 	0x88, 0xffffffff, 0xc060000c,
780 	0x89, 0xc0000fff, 0x00000100,
781 	0x3e4, 0xffffffff, 0x00000100,
782 	0x3e6, 0x00000101, 0x00000000,
783 	0x82a, 0xffffffff, 0x00000104,
784 	0x1579, 0xff000fff, 0x00000100,
785 	0xc33, 0xc0000fff, 0x00000104,
786 	0x3079, 0x00000001, 0x00000001,
787 	0x3403, 0xff000ff0, 0x00000100,
788 	0x3603, 0xff000ff0, 0x00000100
789 };
790 
791 static const u32 godavari_golden_registers[] =
792 {
793 	0x1579, 0xff607fff, 0xfc000100,
794 	0x1bb6, 0x00010101, 0x00010000,
795 	0x260c, 0xffffffff, 0x00000000,
796 	0x260c0, 0xf00fffff, 0x00000400,
797 	0x184c, 0xffffffff, 0x00010000,
798 	0x16ec, 0x000000f0, 0x00000070,
799 	0x16f0, 0xf0311fff, 0x80300000,
800 	0x263e, 0x73773777, 0x12010001,
801 	0x263f, 0xffffffff, 0x00000010,
802 	0x200c, 0x00001f0f, 0x0000100a,
803 	0xbd2, 0x73773777, 0x12010001,
804 	0x902, 0x000fffff, 0x000c007f,
805 	0x2285, 0xf000003f, 0x00000007,
806 	0x22c9, 0xffffffff, 0x00ff0fff,
807 	0xc281, 0x0000ff0f, 0x00000000,
808 	0xa293, 0x07ffffff, 0x06000000,
809 	0x136, 0x00000fff, 0x00000100,
810 	0x3405, 0x00010000, 0x00810001,
811 	0x3605, 0x00010000, 0x00810001,
812 	0xf9e, 0x00000001, 0x00000002,
813 	0x31da, 0x00000008, 0x00000008,
814 	0x31dc, 0x00000f00, 0x00000800,
815 	0x31dd, 0x00000f00, 0x00000800,
816 	0x31e6, 0x00ffffff, 0x00ff7fbf,
817 	0x31e7, 0x00ffffff, 0x00ff7faf,
818 	0x2300, 0x000000ff, 0x00000001,
819 	0x853e, 0x01ff01ff, 0x00000002,
820 	0x8526, 0x007ff800, 0x00200000,
821 	0x8057, 0xffffffff, 0x00000f40,
822 	0x2231, 0x001f3ae3, 0x00000082,
823 	0x2235, 0x0000001f, 0x00000010,
824 	0xc24d, 0xffffffff, 0x00000000
825 };
826 
827 static void cik_init_golden_registers(struct amdgpu_device *adev)
828 {
829 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
830 	mutex_lock(&adev->grbm_idx_mutex);
831 
832 	switch (adev->asic_type) {
833 	case CHIP_BONAIRE:
834 		amdgpu_device_program_register_sequence(adev,
835 							bonaire_mgcg_cgcg_init,
836 							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
837 		amdgpu_device_program_register_sequence(adev,
838 							bonaire_golden_registers,
839 							ARRAY_SIZE(bonaire_golden_registers));
840 		amdgpu_device_program_register_sequence(adev,
841 							bonaire_golden_common_registers,
842 							ARRAY_SIZE(bonaire_golden_common_registers));
843 		amdgpu_device_program_register_sequence(adev,
844 							bonaire_golden_spm_registers,
845 							ARRAY_SIZE(bonaire_golden_spm_registers));
846 		break;
847 	case CHIP_KABINI:
848 		amdgpu_device_program_register_sequence(adev,
849 							kalindi_mgcg_cgcg_init,
850 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
851 		amdgpu_device_program_register_sequence(adev,
852 							kalindi_golden_registers,
853 							ARRAY_SIZE(kalindi_golden_registers));
854 		amdgpu_device_program_register_sequence(adev,
855 							kalindi_golden_common_registers,
856 							ARRAY_SIZE(kalindi_golden_common_registers));
857 		amdgpu_device_program_register_sequence(adev,
858 							kalindi_golden_spm_registers,
859 							ARRAY_SIZE(kalindi_golden_spm_registers));
860 		break;
861 	case CHIP_MULLINS:
862 		amdgpu_device_program_register_sequence(adev,
863 							kalindi_mgcg_cgcg_init,
864 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
865 		amdgpu_device_program_register_sequence(adev,
866 							godavari_golden_registers,
867 							ARRAY_SIZE(godavari_golden_registers));
868 		amdgpu_device_program_register_sequence(adev,
869 							kalindi_golden_common_registers,
870 							ARRAY_SIZE(kalindi_golden_common_registers));
871 		amdgpu_device_program_register_sequence(adev,
872 							kalindi_golden_spm_registers,
873 							ARRAY_SIZE(kalindi_golden_spm_registers));
874 		break;
875 	case CHIP_KAVERI:
876 		amdgpu_device_program_register_sequence(adev,
877 							spectre_mgcg_cgcg_init,
878 							ARRAY_SIZE(spectre_mgcg_cgcg_init));
879 		amdgpu_device_program_register_sequence(adev,
880 							spectre_golden_registers,
881 							ARRAY_SIZE(spectre_golden_registers));
882 		amdgpu_device_program_register_sequence(adev,
883 							spectre_golden_common_registers,
884 							ARRAY_SIZE(spectre_golden_common_registers));
885 		amdgpu_device_program_register_sequence(adev,
886 							spectre_golden_spm_registers,
887 							ARRAY_SIZE(spectre_golden_spm_registers));
888 		break;
889 	case CHIP_HAWAII:
890 		amdgpu_device_program_register_sequence(adev,
891 							hawaii_mgcg_cgcg_init,
892 							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
893 		amdgpu_device_program_register_sequence(adev,
894 							hawaii_golden_registers,
895 							ARRAY_SIZE(hawaii_golden_registers));
896 		amdgpu_device_program_register_sequence(adev,
897 							hawaii_golden_common_registers,
898 							ARRAY_SIZE(hawaii_golden_common_registers));
899 		amdgpu_device_program_register_sequence(adev,
900 							hawaii_golden_spm_registers,
901 							ARRAY_SIZE(hawaii_golden_spm_registers));
902 		break;
903 	default:
904 		break;
905 	}
906 	mutex_unlock(&adev->grbm_idx_mutex);
907 }
908 
909 /**
910  * cik_get_xclk - get the xclk
911  *
912  * @adev: amdgpu_device pointer
913  *
914  * Returns the reference clock used by the gfx engine
915  * (CIK).
916  */
917 static u32 cik_get_xclk(struct amdgpu_device *adev)
918 {
919 	u32 reference_clock = adev->clock.spll.reference_freq;
920 
921 	if (adev->flags & AMD_IS_APU) {
922 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
923 			return reference_clock / 2;
924 	} else {
925 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
926 			return reference_clock / 4;
927 	}
928 	return reference_clock;
929 }
930 
931 /**
932  * cik_srbm_select - select specific register instances
933  *
934  * @adev: amdgpu_device pointer
935  * @me: selected ME (micro engine)
936  * @pipe: pipe
937  * @queue: queue
938  * @vmid: VMID
939  *
940  * Switches the currently active registers instances.  Some
941  * registers are instanced per VMID, others are instanced per
942  * me/pipe/queue combination.
943  */
944 void cik_srbm_select(struct amdgpu_device *adev,
945 		     u32 me, u32 pipe, u32 queue, u32 vmid)
946 {
947 	u32 srbm_gfx_cntl =
948 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
949 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
950 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
951 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
952 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
953 }
954 
955 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
956 {
957 	uint32_t tmp;
958 
959 	tmp = RREG32(mmCONFIG_CNTL);
960 	if (!state)
961 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
962 	else
963 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
964 	WREG32(mmCONFIG_CNTL, tmp);
965 }
966 
967 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
968 {
969 	u32 bus_cntl;
970 	u32 d1vga_control = 0;
971 	u32 d2vga_control = 0;
972 	u32 vga_render_control = 0;
973 	u32 rom_cntl;
974 	bool r;
975 
976 	bus_cntl = RREG32(mmBUS_CNTL);
977 	if (adev->mode_info.num_crtc) {
978 		d1vga_control = RREG32(mmD1VGA_CONTROL);
979 		d2vga_control = RREG32(mmD2VGA_CONTROL);
980 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
981 	}
982 	rom_cntl = RREG32_SMC(ixROM_CNTL);
983 
984 	/* enable the rom */
985 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
986 	if (adev->mode_info.num_crtc) {
987 		/* Disable VGA mode */
988 		WREG32(mmD1VGA_CONTROL,
989 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
990 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
991 		WREG32(mmD2VGA_CONTROL,
992 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
993 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
994 		WREG32(mmVGA_RENDER_CONTROL,
995 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
996 	}
997 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
998 
999 	r = amdgpu_read_bios(adev);
1000 
1001 	/* restore regs */
1002 	WREG32(mmBUS_CNTL, bus_cntl);
1003 	if (adev->mode_info.num_crtc) {
1004 		WREG32(mmD1VGA_CONTROL, d1vga_control);
1005 		WREG32(mmD2VGA_CONTROL, d2vga_control);
1006 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007 	}
1008 	WREG32_SMC(ixROM_CNTL, rom_cntl);
1009 	return r;
1010 }
1011 
1012 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013 				   u8 *bios, u32 length_bytes)
1014 {
1015 	u32 *dw_ptr;
1016 	unsigned long flags;
1017 	u32 i, length_dw;
1018 
1019 	if (bios == NULL)
1020 		return false;
1021 	if (length_bytes == 0)
1022 		return false;
1023 	/* APU vbios image is part of sbios image */
1024 	if (adev->flags & AMD_IS_APU)
1025 		return false;
1026 
1027 	dw_ptr = (u32 *)bios;
1028 	length_dw = ALIGN(length_bytes, 4) / 4;
1029 	/* take the smc lock since we are using the smc index */
1030 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031 	/* set rom index to 0 */
1032 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033 	WREG32(mmSMC_IND_DATA_0, 0);
1034 	/* set index to data for continous read */
1035 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036 	for (i = 0; i < length_dw; i++)
1037 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039 
1040 	return true;
1041 }
1042 
1043 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044 	{mmGRBM_STATUS},
1045 	{mmGRBM_STATUS2},
1046 	{mmGRBM_STATUS_SE0},
1047 	{mmGRBM_STATUS_SE1},
1048 	{mmGRBM_STATUS_SE2},
1049 	{mmGRBM_STATUS_SE3},
1050 	{mmSRBM_STATUS},
1051 	{mmSRBM_STATUS2},
1052 	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053 	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054 	{mmCP_STAT},
1055 	{mmCP_STALLED_STAT1},
1056 	{mmCP_STALLED_STAT2},
1057 	{mmCP_STALLED_STAT3},
1058 	{mmCP_CPF_BUSY_STAT},
1059 	{mmCP_CPF_STALLED_STAT1},
1060 	{mmCP_CPF_STATUS},
1061 	{mmCP_CPC_BUSY_STAT},
1062 	{mmCP_CPC_STALLED_STAT1},
1063 	{mmCP_CPC_STATUS},
1064 	{mmGB_ADDR_CONFIG},
1065 	{mmMC_ARB_RAMCFG},
1066 	{mmGB_TILE_MODE0},
1067 	{mmGB_TILE_MODE1},
1068 	{mmGB_TILE_MODE2},
1069 	{mmGB_TILE_MODE3},
1070 	{mmGB_TILE_MODE4},
1071 	{mmGB_TILE_MODE5},
1072 	{mmGB_TILE_MODE6},
1073 	{mmGB_TILE_MODE7},
1074 	{mmGB_TILE_MODE8},
1075 	{mmGB_TILE_MODE9},
1076 	{mmGB_TILE_MODE10},
1077 	{mmGB_TILE_MODE11},
1078 	{mmGB_TILE_MODE12},
1079 	{mmGB_TILE_MODE13},
1080 	{mmGB_TILE_MODE14},
1081 	{mmGB_TILE_MODE15},
1082 	{mmGB_TILE_MODE16},
1083 	{mmGB_TILE_MODE17},
1084 	{mmGB_TILE_MODE18},
1085 	{mmGB_TILE_MODE19},
1086 	{mmGB_TILE_MODE20},
1087 	{mmGB_TILE_MODE21},
1088 	{mmGB_TILE_MODE22},
1089 	{mmGB_TILE_MODE23},
1090 	{mmGB_TILE_MODE24},
1091 	{mmGB_TILE_MODE25},
1092 	{mmGB_TILE_MODE26},
1093 	{mmGB_TILE_MODE27},
1094 	{mmGB_TILE_MODE28},
1095 	{mmGB_TILE_MODE29},
1096 	{mmGB_TILE_MODE30},
1097 	{mmGB_TILE_MODE31},
1098 	{mmGB_MACROTILE_MODE0},
1099 	{mmGB_MACROTILE_MODE1},
1100 	{mmGB_MACROTILE_MODE2},
1101 	{mmGB_MACROTILE_MODE3},
1102 	{mmGB_MACROTILE_MODE4},
1103 	{mmGB_MACROTILE_MODE5},
1104 	{mmGB_MACROTILE_MODE6},
1105 	{mmGB_MACROTILE_MODE7},
1106 	{mmGB_MACROTILE_MODE8},
1107 	{mmGB_MACROTILE_MODE9},
1108 	{mmGB_MACROTILE_MODE10},
1109 	{mmGB_MACROTILE_MODE11},
1110 	{mmGB_MACROTILE_MODE12},
1111 	{mmGB_MACROTILE_MODE13},
1112 	{mmGB_MACROTILE_MODE14},
1113 	{mmGB_MACROTILE_MODE15},
1114 	{mmCC_RB_BACKEND_DISABLE, true},
1115 	{mmGC_USER_RB_BACKEND_DISABLE, true},
1116 	{mmGB_BACKEND_MAP, false},
1117 	{mmPA_SC_RASTER_CONFIG, true},
1118 	{mmPA_SC_RASTER_CONFIG_1, true},
1119 };
1120 
1121 
1122 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123 				       bool indexed, u32 se_num,
1124 				       u32 sh_num, u32 reg_offset)
1125 {
1126 	if (indexed) {
1127 		uint32_t val;
1128 		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129 		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130 
1131 		switch (reg_offset) {
1132 		case mmCC_RB_BACKEND_DISABLE:
1133 			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134 		case mmGC_USER_RB_BACKEND_DISABLE:
1135 			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136 		case mmPA_SC_RASTER_CONFIG:
1137 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138 		case mmPA_SC_RASTER_CONFIG_1:
1139 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140 		}
1141 
1142 		mutex_lock(&adev->grbm_idx_mutex);
1143 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144 			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
1145 
1146 		val = RREG32(reg_offset);
1147 
1148 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149 			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
1150 		mutex_unlock(&adev->grbm_idx_mutex);
1151 		return val;
1152 	} else {
1153 		unsigned idx;
1154 
1155 		switch (reg_offset) {
1156 		case mmGB_ADDR_CONFIG:
1157 			return adev->gfx.config.gb_addr_config;
1158 		case mmMC_ARB_RAMCFG:
1159 			return adev->gfx.config.mc_arb_ramcfg;
1160 		case mmGB_TILE_MODE0:
1161 		case mmGB_TILE_MODE1:
1162 		case mmGB_TILE_MODE2:
1163 		case mmGB_TILE_MODE3:
1164 		case mmGB_TILE_MODE4:
1165 		case mmGB_TILE_MODE5:
1166 		case mmGB_TILE_MODE6:
1167 		case mmGB_TILE_MODE7:
1168 		case mmGB_TILE_MODE8:
1169 		case mmGB_TILE_MODE9:
1170 		case mmGB_TILE_MODE10:
1171 		case mmGB_TILE_MODE11:
1172 		case mmGB_TILE_MODE12:
1173 		case mmGB_TILE_MODE13:
1174 		case mmGB_TILE_MODE14:
1175 		case mmGB_TILE_MODE15:
1176 		case mmGB_TILE_MODE16:
1177 		case mmGB_TILE_MODE17:
1178 		case mmGB_TILE_MODE18:
1179 		case mmGB_TILE_MODE19:
1180 		case mmGB_TILE_MODE20:
1181 		case mmGB_TILE_MODE21:
1182 		case mmGB_TILE_MODE22:
1183 		case mmGB_TILE_MODE23:
1184 		case mmGB_TILE_MODE24:
1185 		case mmGB_TILE_MODE25:
1186 		case mmGB_TILE_MODE26:
1187 		case mmGB_TILE_MODE27:
1188 		case mmGB_TILE_MODE28:
1189 		case mmGB_TILE_MODE29:
1190 		case mmGB_TILE_MODE30:
1191 		case mmGB_TILE_MODE31:
1192 			idx = (reg_offset - mmGB_TILE_MODE0);
1193 			return adev->gfx.config.tile_mode_array[idx];
1194 		case mmGB_MACROTILE_MODE0:
1195 		case mmGB_MACROTILE_MODE1:
1196 		case mmGB_MACROTILE_MODE2:
1197 		case mmGB_MACROTILE_MODE3:
1198 		case mmGB_MACROTILE_MODE4:
1199 		case mmGB_MACROTILE_MODE5:
1200 		case mmGB_MACROTILE_MODE6:
1201 		case mmGB_MACROTILE_MODE7:
1202 		case mmGB_MACROTILE_MODE8:
1203 		case mmGB_MACROTILE_MODE9:
1204 		case mmGB_MACROTILE_MODE10:
1205 		case mmGB_MACROTILE_MODE11:
1206 		case mmGB_MACROTILE_MODE12:
1207 		case mmGB_MACROTILE_MODE13:
1208 		case mmGB_MACROTILE_MODE14:
1209 		case mmGB_MACROTILE_MODE15:
1210 			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211 			return adev->gfx.config.macrotile_mode_array[idx];
1212 		default:
1213 			return RREG32(reg_offset);
1214 		}
1215 	}
1216 }
1217 
1218 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219 			     u32 sh_num, u32 reg_offset, u32 *value)
1220 {
1221 	uint32_t i;
1222 
1223 	*value = 0;
1224 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225 		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226 
1227 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228 			continue;
1229 
1230 		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231 						reg_offset);
1232 		return 0;
1233 	}
1234 	return -EINVAL;
1235 }
1236 
1237 struct kv_reset_save_regs {
1238 	u32 gmcon_reng_execute;
1239 	u32 gmcon_misc;
1240 	u32 gmcon_misc3;
1241 };
1242 
1243 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244 				   struct kv_reset_save_regs *save)
1245 {
1246 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1248 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249 
1250 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1253 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255 }
1256 
1257 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258 				      struct kv_reset_save_regs *save)
1259 {
1260 	int i;
1261 
1262 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1263 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264 
1265 	for (i = 0; i < 5; i++)
1266 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1267 
1268 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1269 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270 
1271 	for (i = 0; i < 5; i++)
1272 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1273 
1274 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276 
1277 	for (i = 0; i < 5; i++)
1278 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1279 
1280 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282 
1283 	for (i = 0; i < 5; i++)
1284 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1285 
1286 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288 
1289 	for (i = 0; i < 5; i++)
1290 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1291 
1292 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1293 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294 
1295 	for (i = 0; i < 5; i++)
1296 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1297 
1298 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300 
1301 	for (i = 0; i < 5; i++)
1302 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1303 
1304 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306 
1307 	for (i = 0; i < 5; i++)
1308 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1309 
1310 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312 
1313 	for (i = 0; i < 5; i++)
1314 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1315 
1316 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318 
1319 	for (i = 0; i < 5; i++)
1320 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1321 
1322 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324 
1325 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1327 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328 }
1329 
1330 /**
1331  * cik_asic_pci_config_reset - soft reset GPU
1332  *
1333  * @adev: amdgpu_device pointer
1334  *
1335  * Use PCI Config method to reset the GPU.
1336  *
1337  * Returns 0 for success.
1338  */
1339 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340 {
1341 	struct kv_reset_save_regs kv_save = { 0 };
1342 	u32 i;
1343 	int r = -EINVAL;
1344 
1345 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346 
1347 	if (adev->flags & AMD_IS_APU)
1348 		kv_save_regs_for_reset(adev, &kv_save);
1349 
1350 	/* disable BM */
1351 	pci_clear_master(adev->pdev);
1352 	/* reset */
1353 	amdgpu_device_pci_config_reset(adev);
1354 
1355 	udelay(100);
1356 
1357 	/* wait for asic to come out of reset */
1358 	for (i = 0; i < adev->usec_timeout; i++) {
1359 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360 			/* enable BM */
1361 			pci_set_master(adev->pdev);
1362 			adev->has_hw_reset = true;
1363 			r = 0;
1364 			break;
1365 		}
1366 		udelay(1);
1367 	}
1368 
1369 	/* does asic init need to be run first??? */
1370 	if (adev->flags & AMD_IS_APU)
1371 		kv_restore_regs_for_reset(adev, &kv_save);
1372 
1373 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374 
1375 	return r;
1376 }
1377 
1378 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379 {
1380 	switch (adev->asic_type) {
1381 	case CHIP_BONAIRE:
1382 	case CHIP_HAWAII:
1383 		return amdgpu_dpm_is_baco_supported(adev);
1384 	default:
1385 		return false;
1386 	}
1387 }
1388 
1389 static enum amd_reset_method
1390 cik_asic_reset_method(struct amdgpu_device *adev)
1391 {
1392 	bool baco_reset;
1393 
1394 	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395 	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396 		return amdgpu_reset_method;
1397 
1398 	if (amdgpu_reset_method != -1)
1399 		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400 				  amdgpu_reset_method);
1401 
1402 	switch (adev->asic_type) {
1403 	case CHIP_BONAIRE:
1404 	case CHIP_HAWAII:
1405 		baco_reset = cik_asic_supports_baco(adev);
1406 		break;
1407 	default:
1408 		baco_reset = false;
1409 		break;
1410 	}
1411 
1412 	if (baco_reset)
1413 		return AMD_RESET_METHOD_BACO;
1414 	else
1415 		return AMD_RESET_METHOD_LEGACY;
1416 }
1417 
1418 /**
1419  * cik_asic_reset - soft reset GPU
1420  *
1421  * @adev: amdgpu_device pointer
1422  *
1423  * Look up which blocks are hung and attempt
1424  * to reset them.
1425  * Returns 0 for success.
1426  */
1427 static int cik_asic_reset(struct amdgpu_device *adev)
1428 {
1429 	int r;
1430 
1431 	/* APUs don't have full asic reset */
1432 	if (adev->flags & AMD_IS_APU)
1433 		return 0;
1434 
1435 	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1436 		dev_info(adev->dev, "BACO reset\n");
1437 		r = amdgpu_dpm_baco_reset(adev);
1438 	} else {
1439 		dev_info(adev->dev, "PCI CONFIG reset\n");
1440 		r = cik_asic_pci_config_reset(adev);
1441 	}
1442 
1443 	return r;
1444 }
1445 
1446 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1447 {
1448 	return RREG32(mmCONFIG_MEMSIZE);
1449 }
1450 
1451 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1452 			      u32 cntl_reg, u32 status_reg)
1453 {
1454 	int r, i;
1455 	struct atom_clock_dividers dividers;
1456 	uint32_t tmp;
1457 
1458 	r = amdgpu_atombios_get_clock_dividers(adev,
1459 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1460 					       clock, false, &dividers);
1461 	if (r)
1462 		return r;
1463 
1464 	tmp = RREG32_SMC(cntl_reg);
1465 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1466 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1467 	tmp |= dividers.post_divider;
1468 	WREG32_SMC(cntl_reg, tmp);
1469 
1470 	for (i = 0; i < 100; i++) {
1471 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1472 			break;
1473 		mdelay(10);
1474 	}
1475 	if (i == 100)
1476 		return -ETIMEDOUT;
1477 
1478 	return 0;
1479 }
1480 
1481 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1482 {
1483 	int r = 0;
1484 
1485 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1486 	if (r)
1487 		return r;
1488 
1489 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1490 	return r;
1491 }
1492 
1493 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1494 {
1495 	int r, i;
1496 	struct atom_clock_dividers dividers;
1497 	u32 tmp;
1498 
1499 	r = amdgpu_atombios_get_clock_dividers(adev,
1500 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1501 					       ecclk, false, &dividers);
1502 	if (r)
1503 		return r;
1504 
1505 	for (i = 0; i < 100; i++) {
1506 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1507 			break;
1508 		mdelay(10);
1509 	}
1510 	if (i == 100)
1511 		return -ETIMEDOUT;
1512 
1513 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1514 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1515 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1516 	tmp |= dividers.post_divider;
1517 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1518 
1519 	for (i = 0; i < 100; i++) {
1520 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1521 			break;
1522 		mdelay(10);
1523 	}
1524 	if (i == 100)
1525 		return -ETIMEDOUT;
1526 
1527 	return 0;
1528 }
1529 
1530 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1531 {
1532 	struct pci_dev *root = adev->pdev->bus->self;
1533 	u32 speed_cntl, current_data_rate;
1534 	int i;
1535 	u16 tmp16;
1536 
1537 	if (pci_is_root_bus(adev->pdev->bus))
1538 		return;
1539 
1540 	if (amdgpu_pcie_gen2 == 0)
1541 		return;
1542 
1543 	if (adev->flags & AMD_IS_APU)
1544 		return;
1545 
1546 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1547 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1548 		return;
1549 
1550 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1551 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1552 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1553 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1554 		if (current_data_rate == 2) {
1555 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1556 			return;
1557 		}
1558 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1559 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1560 		if (current_data_rate == 1) {
1561 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1562 			return;
1563 		}
1564 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1565 	}
1566 
1567 	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1568 		return;
1569 
1570 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1571 		/* re-try equalization if gen3 is not already enabled */
1572 		if (current_data_rate != 2) {
1573 			u16 bridge_cfg, gpu_cfg;
1574 			u16 bridge_cfg2, gpu_cfg2;
1575 			u32 max_lw, current_lw, tmp;
1576 
1577 			pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1578 						  &bridge_cfg);
1579 			pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1580 						  &gpu_cfg);
1581 
1582 			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1583 			pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1584 
1585 			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1586 			pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1587 						   tmp16);
1588 
1589 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1590 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1591 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1592 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1593 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1594 
1595 			if (current_lw < max_lw) {
1596 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1597 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1598 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1599 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1600 					tmp |= (max_lw <<
1601 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1602 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1603 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1604 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1605 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1606 				}
1607 			}
1608 
1609 			for (i = 0; i < 10; i++) {
1610 				/* check status */
1611 				pcie_capability_read_word(adev->pdev,
1612 							  PCI_EXP_DEVSTA,
1613 							  &tmp16);
1614 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1615 					break;
1616 
1617 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1618 							  &bridge_cfg);
1619 				pcie_capability_read_word(adev->pdev,
1620 							  PCI_EXP_LNKCTL,
1621 							  &gpu_cfg);
1622 
1623 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1624 							  &bridge_cfg2);
1625 				pcie_capability_read_word(adev->pdev,
1626 							  PCI_EXP_LNKCTL2,
1627 							  &gpu_cfg2);
1628 
1629 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1630 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1631 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1632 
1633 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1634 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1635 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1636 
1637 				msleep(100);
1638 
1639 				/* linkctl */
1640 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1641 							  &tmp16);
1642 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1643 				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1644 				pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1645 							   tmp16);
1646 
1647 				pcie_capability_read_word(adev->pdev,
1648 							  PCI_EXP_LNKCTL,
1649 							  &tmp16);
1650 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1651 				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1652 				pcie_capability_write_word(adev->pdev,
1653 							   PCI_EXP_LNKCTL,
1654 							   tmp16);
1655 
1656 				/* linkctl2 */
1657 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1658 							  &tmp16);
1659 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1660 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1661 				tmp16 |= (bridge_cfg2 &
1662 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1663 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1664 				pcie_capability_write_word(root,
1665 							   PCI_EXP_LNKCTL2,
1666 							   tmp16);
1667 
1668 				pcie_capability_read_word(adev->pdev,
1669 							  PCI_EXP_LNKCTL2,
1670 							  &tmp16);
1671 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1672 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1673 				tmp16 |= (gpu_cfg2 &
1674 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1675 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1676 				pcie_capability_write_word(adev->pdev,
1677 							   PCI_EXP_LNKCTL2,
1678 							   tmp16);
1679 
1680 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1681 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1682 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1683 			}
1684 		}
1685 	}
1686 
1687 	/* set the link speed */
1688 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1689 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1690 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1691 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1692 
1693 	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1694 	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1695 
1696 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1697 		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1698 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1699 		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1700 	else
1701 		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1702 	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1703 
1704 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1705 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1706 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1707 
1708 	for (i = 0; i < adev->usec_timeout; i++) {
1709 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1710 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1711 			break;
1712 		udelay(1);
1713 	}
1714 }
1715 
1716 static void cik_program_aspm(struct amdgpu_device *adev)
1717 {
1718 	u32 data, orig;
1719 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1720 	bool disable_clkreq = false;
1721 
1722 	if (!amdgpu_device_should_use_aspm(adev))
1723 		return;
1724 
1725 	if (pci_is_root_bus(adev->pdev->bus))
1726 		return;
1727 
1728 	/* XXX double check APUs */
1729 	if (adev->flags & AMD_IS_APU)
1730 		return;
1731 
1732 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1733 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1734 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1735 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1736 	if (orig != data)
1737 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1738 
1739 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1740 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1741 	if (orig != data)
1742 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1743 
1744 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1745 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1746 	if (orig != data)
1747 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1748 
1749 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1750 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1751 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1752 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1753 	if (!disable_l0s)
1754 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1755 
1756 	if (!disable_l1) {
1757 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1758 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1759 		if (orig != data)
1760 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1761 
1762 		if (!disable_plloff_in_l1) {
1763 			bool clk_req_support;
1764 
1765 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1766 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1767 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1768 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1769 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1770 			if (orig != data)
1771 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1772 
1773 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1774 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1775 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1776 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1777 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1778 			if (orig != data)
1779 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1780 
1781 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1782 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1783 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1784 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1785 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1786 			if (orig != data)
1787 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1788 
1789 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1790 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1791 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1792 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1793 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1794 			if (orig != data)
1795 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1796 
1797 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1798 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1799 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1800 			if (orig != data)
1801 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1802 
1803 			if (!disable_clkreq) {
1804 				struct pci_dev *root = adev->pdev->bus->self;
1805 				u32 lnkcap;
1806 
1807 				clk_req_support = false;
1808 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1809 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1810 					clk_req_support = true;
1811 			} else {
1812 				clk_req_support = false;
1813 			}
1814 
1815 			if (clk_req_support) {
1816 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1817 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1818 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1819 				if (orig != data)
1820 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1821 
1822 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1823 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1824 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1825 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1826 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1827 				if (orig != data)
1828 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1829 
1830 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1831 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1832 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1833 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1834 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1835 				if (orig != data)
1836 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1837 
1838 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1839 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1840 				if (orig != data)
1841 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1842 
1843 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1844 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1845 				if (orig != data)
1846 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1847 
1848 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1849 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1850 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1851 				if (orig != data)
1852 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1853 			}
1854 		}
1855 	} else {
1856 		if (orig != data)
1857 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1858 	}
1859 
1860 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1861 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1862 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1863 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1864 	if (orig != data)
1865 		WREG32_PCIE(ixPCIE_CNTL2, data);
1866 
1867 	if (!disable_l0s) {
1868 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1869 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1870 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1871 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1872 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1873 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1874 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1875 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1876 				if (orig != data)
1877 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1878 			}
1879 		}
1880 	}
1881 }
1882 
1883 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1884 {
1885 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1886 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1887 }
1888 
1889 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1890 {
1891 	if (!ring || !ring->funcs->emit_wreg) {
1892 		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1893 		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1894 	} else {
1895 		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1896 	}
1897 }
1898 
1899 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1900 			       struct amdgpu_ring *ring)
1901 {
1902 	if (!ring || !ring->funcs->emit_wreg) {
1903 		WREG32(mmHDP_DEBUG0, 1);
1904 		RREG32(mmHDP_DEBUG0);
1905 	} else {
1906 		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1907 	}
1908 }
1909 
1910 static bool cik_need_full_reset(struct amdgpu_device *adev)
1911 {
1912 	/* change this when we support soft reset */
1913 	return true;
1914 }
1915 
1916 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1917 			       uint64_t *count1)
1918 {
1919 	uint32_t perfctr = 0;
1920 	uint64_t cnt0_of, cnt1_of;
1921 	int tmp;
1922 
1923 	/* This reports 0 on APUs, so return to avoid writing/reading registers
1924 	 * that may or may not be different from their GPU counterparts
1925 	 */
1926 	if (adev->flags & AMD_IS_APU)
1927 		return;
1928 
1929 	/* Set the 2 events that we wish to watch, defined above */
1930 	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1931 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1932 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1933 
1934 	/* Write to enable desired perf counters */
1935 	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1936 	/* Zero out and enable the perf counters
1937 	 * Write 0x5:
1938 	 * Bit 0 = Start all counters(1)
1939 	 * Bit 2 = Global counter reset enable(1)
1940 	 */
1941 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1942 
1943 	msleep(1000);
1944 
1945 	/* Load the shadow and disable the perf counters
1946 	 * Write 0x2:
1947 	 * Bit 0 = Stop counters(0)
1948 	 * Bit 1 = Load the shadow counters(1)
1949 	 */
1950 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1951 
1952 	/* Read register values to get any >32bit overflow */
1953 	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1954 	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1955 	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1956 
1957 	/* Get the values and add the overflow */
1958 	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1959 	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1960 }
1961 
1962 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1963 {
1964 	u32 clock_cntl, pc;
1965 
1966 	if (adev->flags & AMD_IS_APU)
1967 		return false;
1968 
1969 	/* check if the SMC is already running */
1970 	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1971 	pc = RREG32_SMC(ixSMC_PC_C);
1972 	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1973 	    (0x20100 <= pc))
1974 		return true;
1975 
1976 	return false;
1977 }
1978 
1979 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1980 {
1981 	uint64_t nak_r, nak_g;
1982 
1983 	/* Get the number of NAKs received and generated */
1984 	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1985 	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1986 
1987 	/* Add the total number of NAKs, i.e the number of replays */
1988 	return (nak_r + nak_g);
1989 }
1990 
1991 static void cik_pre_asic_init(struct amdgpu_device *adev)
1992 {
1993 }
1994 
1995 static const struct amdgpu_asic_funcs cik_asic_funcs =
1996 {
1997 	.read_disabled_bios = &cik_read_disabled_bios,
1998 	.read_bios_from_rom = &cik_read_bios_from_rom,
1999 	.read_register = &cik_read_register,
2000 	.reset = &cik_asic_reset,
2001 	.reset_method = &cik_asic_reset_method,
2002 	.set_vga_state = &cik_vga_set_state,
2003 	.get_xclk = &cik_get_xclk,
2004 	.set_uvd_clocks = &cik_set_uvd_clocks,
2005 	.set_vce_clocks = &cik_set_vce_clocks,
2006 	.get_config_memsize = &cik_get_config_memsize,
2007 	.flush_hdp = &cik_flush_hdp,
2008 	.invalidate_hdp = &cik_invalidate_hdp,
2009 	.need_full_reset = &cik_need_full_reset,
2010 	.init_doorbell_index = &legacy_doorbell_index_init,
2011 	.get_pcie_usage = &cik_get_pcie_usage,
2012 	.need_reset_on_init = &cik_need_reset_on_init,
2013 	.get_pcie_replay_count = &cik_get_pcie_replay_count,
2014 	.supports_baco = &cik_asic_supports_baco,
2015 	.pre_asic_init = &cik_pre_asic_init,
2016 	.query_video_codecs = &cik_query_video_codecs,
2017 };
2018 
2019 static int cik_common_early_init(void *handle)
2020 {
2021 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2022 
2023 	adev->smc_rreg = &cik_smc_rreg;
2024 	adev->smc_wreg = &cik_smc_wreg;
2025 	adev->pcie_rreg = &cik_pcie_rreg;
2026 	adev->pcie_wreg = &cik_pcie_wreg;
2027 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2028 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2029 	adev->didt_rreg = &cik_didt_rreg;
2030 	adev->didt_wreg = &cik_didt_wreg;
2031 
2032 	adev->asic_funcs = &cik_asic_funcs;
2033 
2034 	adev->rev_id = cik_get_rev_id(adev);
2035 	adev->external_rev_id = 0xFF;
2036 	switch (adev->asic_type) {
2037 	case CHIP_BONAIRE:
2038 		adev->cg_flags =
2039 			AMD_CG_SUPPORT_GFX_MGCG |
2040 			AMD_CG_SUPPORT_GFX_MGLS |
2041 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2042 			AMD_CG_SUPPORT_GFX_CGLS |
2043 			AMD_CG_SUPPORT_GFX_CGTS |
2044 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2045 			AMD_CG_SUPPORT_GFX_CP_LS |
2046 			AMD_CG_SUPPORT_MC_LS |
2047 			AMD_CG_SUPPORT_MC_MGCG |
2048 			AMD_CG_SUPPORT_SDMA_MGCG |
2049 			AMD_CG_SUPPORT_SDMA_LS |
2050 			AMD_CG_SUPPORT_BIF_LS |
2051 			AMD_CG_SUPPORT_VCE_MGCG |
2052 			AMD_CG_SUPPORT_UVD_MGCG |
2053 			AMD_CG_SUPPORT_HDP_LS |
2054 			AMD_CG_SUPPORT_HDP_MGCG;
2055 		adev->pg_flags = 0;
2056 		adev->external_rev_id = adev->rev_id + 0x14;
2057 		break;
2058 	case CHIP_HAWAII:
2059 		adev->cg_flags =
2060 			AMD_CG_SUPPORT_GFX_MGCG |
2061 			AMD_CG_SUPPORT_GFX_MGLS |
2062 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2063 			AMD_CG_SUPPORT_GFX_CGLS |
2064 			AMD_CG_SUPPORT_GFX_CGTS |
2065 			AMD_CG_SUPPORT_GFX_CP_LS |
2066 			AMD_CG_SUPPORT_MC_LS |
2067 			AMD_CG_SUPPORT_MC_MGCG |
2068 			AMD_CG_SUPPORT_SDMA_MGCG |
2069 			AMD_CG_SUPPORT_SDMA_LS |
2070 			AMD_CG_SUPPORT_BIF_LS |
2071 			AMD_CG_SUPPORT_VCE_MGCG |
2072 			AMD_CG_SUPPORT_UVD_MGCG |
2073 			AMD_CG_SUPPORT_HDP_LS |
2074 			AMD_CG_SUPPORT_HDP_MGCG;
2075 		adev->pg_flags = 0;
2076 		adev->external_rev_id = 0x28;
2077 		break;
2078 	case CHIP_KAVERI:
2079 		adev->cg_flags =
2080 			AMD_CG_SUPPORT_GFX_MGCG |
2081 			AMD_CG_SUPPORT_GFX_MGLS |
2082 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2083 			AMD_CG_SUPPORT_GFX_CGLS |
2084 			AMD_CG_SUPPORT_GFX_CGTS |
2085 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2086 			AMD_CG_SUPPORT_GFX_CP_LS |
2087 			AMD_CG_SUPPORT_SDMA_MGCG |
2088 			AMD_CG_SUPPORT_SDMA_LS |
2089 			AMD_CG_SUPPORT_BIF_LS |
2090 			AMD_CG_SUPPORT_VCE_MGCG |
2091 			AMD_CG_SUPPORT_UVD_MGCG |
2092 			AMD_CG_SUPPORT_HDP_LS |
2093 			AMD_CG_SUPPORT_HDP_MGCG;
2094 		adev->pg_flags =
2095 			/*AMD_PG_SUPPORT_GFX_PG |
2096 			  AMD_PG_SUPPORT_GFX_SMG |
2097 			  AMD_PG_SUPPORT_GFX_DMG |*/
2098 			AMD_PG_SUPPORT_UVD |
2099 			AMD_PG_SUPPORT_VCE |
2100 			/*  AMD_PG_SUPPORT_CP |
2101 			  AMD_PG_SUPPORT_GDS |
2102 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2103 			  AMD_PG_SUPPORT_ACP |
2104 			  AMD_PG_SUPPORT_SAMU |*/
2105 			0;
2106 		if (adev->pdev->device == 0x1312 ||
2107 			adev->pdev->device == 0x1316 ||
2108 			adev->pdev->device == 0x1317)
2109 			adev->external_rev_id = 0x41;
2110 		else
2111 			adev->external_rev_id = 0x1;
2112 		break;
2113 	case CHIP_KABINI:
2114 	case CHIP_MULLINS:
2115 		adev->cg_flags =
2116 			AMD_CG_SUPPORT_GFX_MGCG |
2117 			AMD_CG_SUPPORT_GFX_MGLS |
2118 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2119 			AMD_CG_SUPPORT_GFX_CGLS |
2120 			AMD_CG_SUPPORT_GFX_CGTS |
2121 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2122 			AMD_CG_SUPPORT_GFX_CP_LS |
2123 			AMD_CG_SUPPORT_SDMA_MGCG |
2124 			AMD_CG_SUPPORT_SDMA_LS |
2125 			AMD_CG_SUPPORT_BIF_LS |
2126 			AMD_CG_SUPPORT_VCE_MGCG |
2127 			AMD_CG_SUPPORT_UVD_MGCG |
2128 			AMD_CG_SUPPORT_HDP_LS |
2129 			AMD_CG_SUPPORT_HDP_MGCG;
2130 		adev->pg_flags =
2131 			/*AMD_PG_SUPPORT_GFX_PG |
2132 			  AMD_PG_SUPPORT_GFX_SMG | */
2133 			AMD_PG_SUPPORT_UVD |
2134 			/*AMD_PG_SUPPORT_VCE |
2135 			  AMD_PG_SUPPORT_CP |
2136 			  AMD_PG_SUPPORT_GDS |
2137 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2138 			  AMD_PG_SUPPORT_SAMU |*/
2139 			0;
2140 		if (adev->asic_type == CHIP_KABINI) {
2141 			if (adev->rev_id == 0)
2142 				adev->external_rev_id = 0x81;
2143 			else if (adev->rev_id == 1)
2144 				adev->external_rev_id = 0x82;
2145 			else if (adev->rev_id == 2)
2146 				adev->external_rev_id = 0x85;
2147 		} else
2148 			adev->external_rev_id = adev->rev_id + 0xa1;
2149 		break;
2150 	default:
2151 		/* FIXME: not supported yet */
2152 		return -EINVAL;
2153 	}
2154 
2155 	return 0;
2156 }
2157 
2158 static int cik_common_sw_init(void *handle)
2159 {
2160 	return 0;
2161 }
2162 
2163 static int cik_common_sw_fini(void *handle)
2164 {
2165 	return 0;
2166 }
2167 
2168 static int cik_common_hw_init(void *handle)
2169 {
2170 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2171 
2172 	/* move the golden regs per IP block */
2173 	cik_init_golden_registers(adev);
2174 	/* enable pcie gen2/3 link */
2175 	cik_pcie_gen3_enable(adev);
2176 	/* enable aspm */
2177 	cik_program_aspm(adev);
2178 
2179 	return 0;
2180 }
2181 
2182 static int cik_common_hw_fini(void *handle)
2183 {
2184 	return 0;
2185 }
2186 
2187 static int cik_common_suspend(void *handle)
2188 {
2189 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2190 
2191 	return cik_common_hw_fini(adev);
2192 }
2193 
2194 static int cik_common_resume(void *handle)
2195 {
2196 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2197 
2198 	return cik_common_hw_init(adev);
2199 }
2200 
2201 static bool cik_common_is_idle(void *handle)
2202 {
2203 	return true;
2204 }
2205 
2206 static int cik_common_wait_for_idle(void *handle)
2207 {
2208 	return 0;
2209 }
2210 
2211 static int cik_common_soft_reset(void *handle)
2212 {
2213 	/* XXX hard reset?? */
2214 	return 0;
2215 }
2216 
2217 static int cik_common_set_clockgating_state(void *handle,
2218 					    enum amd_clockgating_state state)
2219 {
2220 	return 0;
2221 }
2222 
2223 static int cik_common_set_powergating_state(void *handle,
2224 					    enum amd_powergating_state state)
2225 {
2226 	return 0;
2227 }
2228 
2229 static const struct amd_ip_funcs cik_common_ip_funcs = {
2230 	.name = "cik_common",
2231 	.early_init = cik_common_early_init,
2232 	.late_init = NULL,
2233 	.sw_init = cik_common_sw_init,
2234 	.sw_fini = cik_common_sw_fini,
2235 	.hw_init = cik_common_hw_init,
2236 	.hw_fini = cik_common_hw_fini,
2237 	.suspend = cik_common_suspend,
2238 	.resume = cik_common_resume,
2239 	.is_idle = cik_common_is_idle,
2240 	.wait_for_idle = cik_common_wait_for_idle,
2241 	.soft_reset = cik_common_soft_reset,
2242 	.set_clockgating_state = cik_common_set_clockgating_state,
2243 	.set_powergating_state = cik_common_set_powergating_state,
2244 };
2245 
2246 static const struct amdgpu_ip_block_version cik_common_ip_block =
2247 {
2248 	.type = AMD_IP_BLOCK_TYPE_COMMON,
2249 	.major = 1,
2250 	.minor = 0,
2251 	.rev = 0,
2252 	.funcs = &cik_common_ip_funcs,
2253 };
2254 
2255 int cik_set_ip_blocks(struct amdgpu_device *adev)
2256 {
2257 	switch (adev->asic_type) {
2258 	case CHIP_BONAIRE:
2259 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2260 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2261 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2262 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2263 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2264 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2265 		if (adev->enable_virtual_display)
2266 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2267 #if defined(CONFIG_DRM_AMD_DC)
2268 		else if (amdgpu_device_has_dc_support(adev))
2269 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2270 #endif
2271 		else
2272 			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2273 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2274 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2275 		break;
2276 	case CHIP_HAWAII:
2277 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2278 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2279 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2280 		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2281 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2282 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2283 		if (adev->enable_virtual_display)
2284 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2285 #if defined(CONFIG_DRM_AMD_DC)
2286 		else if (amdgpu_device_has_dc_support(adev))
2287 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2288 #endif
2289 		else
2290 			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2291 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2292 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2293 		break;
2294 	case CHIP_KAVERI:
2295 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2296 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2297 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2298 		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2299 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2300 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2301 		if (adev->enable_virtual_display)
2302 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2303 #if defined(CONFIG_DRM_AMD_DC)
2304 		else if (amdgpu_device_has_dc_support(adev))
2305 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2306 #endif
2307 		else
2308 			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2309 
2310 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2311 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2312 		break;
2313 	case CHIP_KABINI:
2314 	case CHIP_MULLINS:
2315 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2316 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2317 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2318 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2319 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2320 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2321 		if (adev->enable_virtual_display)
2322 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2323 #if defined(CONFIG_DRM_AMD_DC)
2324 		else if (amdgpu_device_has_dc_support(adev))
2325 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2326 #endif
2327 		else
2328 			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2329 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2330 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2331 		break;
2332 	default:
2333 		/* FIXME: not supported yet */
2334 		return -EINVAL;
2335 	}
2336 	return 0;
2337 }
2338