xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision 8dce88fe) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include <linux/pci.h>
28 
29 #include <drm/amdgpu_drm.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_atombios.h"
33 #include "amdgpu_ih.h"
34 #include "amdgpu_uvd.h"
35 #include "amdgpu_vce.h"
36 #include "cikd.h"
37 #include "atom.h"
38 #include "amd_pcie.h"
39 
40 #include "cik.h"
41 #include "gmc_v7_0.h"
42 #include "cik_ih.h"
43 #include "dce_v8_0.h"
44 #include "gfx_v7_0.h"
45 #include "cik_sdma.h"
46 #include "uvd_v4_2.h"
47 #include "vce_v2_0.h"
48 #include "cik_dpm.h"
49 
50 #include "uvd/uvd_4_2_d.h"
51 
52 #include "smu/smu_7_0_1_d.h"
53 #include "smu/smu_7_0_1_sh_mask.h"
54 
55 #include "dce/dce_8_0_d.h"
56 #include "dce/dce_8_0_sh_mask.h"
57 
58 #include "bif/bif_4_1_d.h"
59 #include "bif/bif_4_1_sh_mask.h"
60 
61 #include "gca/gfx_7_2_d.h"
62 #include "gca/gfx_7_2_enum.h"
63 #include "gca/gfx_7_2_sh_mask.h"
64 
65 #include "gmc/gmc_7_1_d.h"
66 #include "gmc/gmc_7_1_sh_mask.h"
67 
68 #include "oss/oss_2_0_d.h"
69 #include "oss/oss_2_0_sh_mask.h"
70 
71 #include "amdgpu_dm.h"
72 #include "amdgpu_amdkfd.h"
73 #include "amdgpu_vkms.h"
74 
75 static const struct amdgpu_video_codec_info cik_video_codecs_encode_array[] =
76 {
77 	{
78 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
79 		.max_width = 2048,
80 		.max_height = 1152,
81 		.max_pixels_per_frame = 2048 * 1152,
82 		.max_level = 0,
83 	},
84 };
85 
86 static const struct amdgpu_video_codecs cik_video_codecs_encode =
87 {
88 	.codec_count = ARRAY_SIZE(cik_video_codecs_encode_array),
89 	.codec_array = cik_video_codecs_encode_array,
90 };
91 
92 static const struct amdgpu_video_codec_info cik_video_codecs_decode_array[] =
93 {
94 	{
95 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG2,
96 		.max_width = 2048,
97 		.max_height = 1152,
98 		.max_pixels_per_frame = 2048 * 1152,
99 		.max_level = 3,
100 	},
101 	{
102 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4,
103 		.max_width = 2048,
104 		.max_height = 1152,
105 		.max_pixels_per_frame = 2048 * 1152,
106 		.max_level = 5,
107 	},
108 	{
109 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_MPEG4_AVC,
110 		.max_width = 2048,
111 		.max_height = 1152,
112 		.max_pixels_per_frame = 2048 * 1152,
113 		.max_level = 41,
114 	},
115 	{
116 		.codec_type = AMDGPU_INFO_VIDEO_CAPS_CODEC_IDX_VC1,
117 		.max_width = 2048,
118 		.max_height = 1152,
119 		.max_pixels_per_frame = 2048 * 1152,
120 		.max_level = 4,
121 	},
122 };
123 
124 static const struct amdgpu_video_codecs cik_video_codecs_decode =
125 {
126 	.codec_count = ARRAY_SIZE(cik_video_codecs_decode_array),
127 	.codec_array = cik_video_codecs_decode_array,
128 };
129 
130 static int cik_query_video_codecs(struct amdgpu_device *adev, bool encode,
131 				  const struct amdgpu_video_codecs **codecs)
132 {
133 	switch (adev->asic_type) {
134 	case CHIP_BONAIRE:
135 	case CHIP_HAWAII:
136 	case CHIP_KAVERI:
137 	case CHIP_KABINI:
138 	case CHIP_MULLINS:
139 		if (encode)
140 			*codecs = &cik_video_codecs_encode;
141 		else
142 			*codecs = &cik_video_codecs_decode;
143 		return 0;
144 	default:
145 		return -EINVAL;
146 	}
147 }
148 
149 /*
150  * Indirect registers accessor
151  */
152 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
153 {
154 	unsigned long flags;
155 	u32 r;
156 
157 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
158 	WREG32(mmPCIE_INDEX, reg);
159 	(void)RREG32(mmPCIE_INDEX);
160 	r = RREG32(mmPCIE_DATA);
161 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
162 	return r;
163 }
164 
165 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
166 {
167 	unsigned long flags;
168 
169 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
170 	WREG32(mmPCIE_INDEX, reg);
171 	(void)RREG32(mmPCIE_INDEX);
172 	WREG32(mmPCIE_DATA, v);
173 	(void)RREG32(mmPCIE_DATA);
174 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
175 }
176 
177 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
178 {
179 	unsigned long flags;
180 	u32 r;
181 
182 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
183 	WREG32(mmSMC_IND_INDEX_0, (reg));
184 	r = RREG32(mmSMC_IND_DATA_0);
185 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
186 	return r;
187 }
188 
189 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
190 {
191 	unsigned long flags;
192 
193 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
194 	WREG32(mmSMC_IND_INDEX_0, (reg));
195 	WREG32(mmSMC_IND_DATA_0, (v));
196 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
197 }
198 
199 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
200 {
201 	unsigned long flags;
202 	u32 r;
203 
204 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
205 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
206 	r = RREG32(mmUVD_CTX_DATA);
207 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
208 	return r;
209 }
210 
211 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
212 {
213 	unsigned long flags;
214 
215 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
216 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
217 	WREG32(mmUVD_CTX_DATA, (v));
218 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
219 }
220 
221 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
222 {
223 	unsigned long flags;
224 	u32 r;
225 
226 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
227 	WREG32(mmDIDT_IND_INDEX, (reg));
228 	r = RREG32(mmDIDT_IND_DATA);
229 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
230 	return r;
231 }
232 
233 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
234 {
235 	unsigned long flags;
236 
237 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
238 	WREG32(mmDIDT_IND_INDEX, (reg));
239 	WREG32(mmDIDT_IND_DATA, (v));
240 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
241 }
242 
243 static const u32 bonaire_golden_spm_registers[] =
244 {
245 	0xc200, 0xe0ffffff, 0xe0000000
246 };
247 
248 static const u32 bonaire_golden_common_registers[] =
249 {
250 	0x31dc, 0xffffffff, 0x00000800,
251 	0x31dd, 0xffffffff, 0x00000800,
252 	0x31e6, 0xffffffff, 0x00007fbf,
253 	0x31e7, 0xffffffff, 0x00007faf
254 };
255 
256 static const u32 bonaire_golden_registers[] =
257 {
258 	0xcd5, 0x00000333, 0x00000333,
259 	0xcd4, 0x000c0fc0, 0x00040200,
260 	0x2684, 0x00010000, 0x00058208,
261 	0xf000, 0xffff1fff, 0x00140000,
262 	0xf080, 0xfdfc0fff, 0x00000100,
263 	0xf08d, 0x40000000, 0x40000200,
264 	0x260c, 0xffffffff, 0x00000000,
265 	0x260d, 0xf00fffff, 0x00000400,
266 	0x260e, 0x0002021c, 0x00020200,
267 	0x31e, 0x00000080, 0x00000000,
268 	0x16ec, 0x000000f0, 0x00000070,
269 	0x16f0, 0xf0311fff, 0x80300000,
270 	0x263e, 0x73773777, 0x12010001,
271 	0xd43, 0x00810000, 0x408af000,
272 	0x1c0c, 0x31000111, 0x00000011,
273 	0xbd2, 0x73773777, 0x12010001,
274 	0x883, 0x00007fb6, 0x0021a1b1,
275 	0x884, 0x00007fb6, 0x002021b1,
276 	0x860, 0x00007fb6, 0x00002191,
277 	0x886, 0x00007fb6, 0x002121b1,
278 	0x887, 0x00007fb6, 0x002021b1,
279 	0x877, 0x00007fb6, 0x00002191,
280 	0x878, 0x00007fb6, 0x00002191,
281 	0xd8a, 0x0000003f, 0x0000000a,
282 	0xd8b, 0x0000003f, 0x0000000a,
283 	0xab9, 0x00073ffe, 0x000022a2,
284 	0x903, 0x000007ff, 0x00000000,
285 	0x2285, 0xf000003f, 0x00000007,
286 	0x22fc, 0x00002001, 0x00000001,
287 	0x22c9, 0xffffffff, 0x00ffffff,
288 	0xc281, 0x0000ff0f, 0x00000000,
289 	0xa293, 0x07ffffff, 0x06000000,
290 	0x136, 0x00000fff, 0x00000100,
291 	0xf9e, 0x00000001, 0x00000002,
292 	0x2440, 0x03000000, 0x0362c688,
293 	0x2300, 0x000000ff, 0x00000001,
294 	0x390, 0x00001fff, 0x00001fff,
295 	0x2418, 0x0000007f, 0x00000020,
296 	0x2542, 0x00010000, 0x00010000,
297 	0x2b05, 0x000003ff, 0x000000f3,
298 	0x2b03, 0xffffffff, 0x00001032
299 };
300 
301 static const u32 bonaire_mgcg_cgcg_init[] =
302 {
303 	0x3108, 0xffffffff, 0xfffffffc,
304 	0xc200, 0xffffffff, 0xe0000000,
305 	0xf0a8, 0xffffffff, 0x00000100,
306 	0xf082, 0xffffffff, 0x00000100,
307 	0xf0b0, 0xffffffff, 0xc0000100,
308 	0xf0b2, 0xffffffff, 0xc0000100,
309 	0xf0b1, 0xffffffff, 0xc0000100,
310 	0x1579, 0xffffffff, 0x00600100,
311 	0xf0a0, 0xffffffff, 0x00000100,
312 	0xf085, 0xffffffff, 0x06000100,
313 	0xf088, 0xffffffff, 0x00000100,
314 	0xf086, 0xffffffff, 0x06000100,
315 	0xf081, 0xffffffff, 0x00000100,
316 	0xf0b8, 0xffffffff, 0x00000100,
317 	0xf089, 0xffffffff, 0x00000100,
318 	0xf080, 0xffffffff, 0x00000100,
319 	0xf08c, 0xffffffff, 0x00000100,
320 	0xf08d, 0xffffffff, 0x00000100,
321 	0xf094, 0xffffffff, 0x00000100,
322 	0xf095, 0xffffffff, 0x00000100,
323 	0xf096, 0xffffffff, 0x00000100,
324 	0xf097, 0xffffffff, 0x00000100,
325 	0xf098, 0xffffffff, 0x00000100,
326 	0xf09f, 0xffffffff, 0x00000100,
327 	0xf09e, 0xffffffff, 0x00000100,
328 	0xf084, 0xffffffff, 0x06000100,
329 	0xf0a4, 0xffffffff, 0x00000100,
330 	0xf09d, 0xffffffff, 0x00000100,
331 	0xf0ad, 0xffffffff, 0x00000100,
332 	0xf0ac, 0xffffffff, 0x00000100,
333 	0xf09c, 0xffffffff, 0x00000100,
334 	0xc200, 0xffffffff, 0xe0000000,
335 	0xf008, 0xffffffff, 0x00010000,
336 	0xf009, 0xffffffff, 0x00030002,
337 	0xf00a, 0xffffffff, 0x00040007,
338 	0xf00b, 0xffffffff, 0x00060005,
339 	0xf00c, 0xffffffff, 0x00090008,
340 	0xf00d, 0xffffffff, 0x00010000,
341 	0xf00e, 0xffffffff, 0x00030002,
342 	0xf00f, 0xffffffff, 0x00040007,
343 	0xf010, 0xffffffff, 0x00060005,
344 	0xf011, 0xffffffff, 0x00090008,
345 	0xf012, 0xffffffff, 0x00010000,
346 	0xf013, 0xffffffff, 0x00030002,
347 	0xf014, 0xffffffff, 0x00040007,
348 	0xf015, 0xffffffff, 0x00060005,
349 	0xf016, 0xffffffff, 0x00090008,
350 	0xf017, 0xffffffff, 0x00010000,
351 	0xf018, 0xffffffff, 0x00030002,
352 	0xf019, 0xffffffff, 0x00040007,
353 	0xf01a, 0xffffffff, 0x00060005,
354 	0xf01b, 0xffffffff, 0x00090008,
355 	0xf01c, 0xffffffff, 0x00010000,
356 	0xf01d, 0xffffffff, 0x00030002,
357 	0xf01e, 0xffffffff, 0x00040007,
358 	0xf01f, 0xffffffff, 0x00060005,
359 	0xf020, 0xffffffff, 0x00090008,
360 	0xf021, 0xffffffff, 0x00010000,
361 	0xf022, 0xffffffff, 0x00030002,
362 	0xf023, 0xffffffff, 0x00040007,
363 	0xf024, 0xffffffff, 0x00060005,
364 	0xf025, 0xffffffff, 0x00090008,
365 	0xf026, 0xffffffff, 0x00010000,
366 	0xf027, 0xffffffff, 0x00030002,
367 	0xf028, 0xffffffff, 0x00040007,
368 	0xf029, 0xffffffff, 0x00060005,
369 	0xf02a, 0xffffffff, 0x00090008,
370 	0xf000, 0xffffffff, 0x96e00200,
371 	0x21c2, 0xffffffff, 0x00900100,
372 	0x3109, 0xffffffff, 0x0020003f,
373 	0xe, 0xffffffff, 0x0140001c,
374 	0xf, 0x000f0000, 0x000f0000,
375 	0x88, 0xffffffff, 0xc060000c,
376 	0x89, 0xc0000fff, 0x00000100,
377 	0x3e4, 0xffffffff, 0x00000100,
378 	0x3e6, 0x00000101, 0x00000000,
379 	0x82a, 0xffffffff, 0x00000104,
380 	0x1579, 0xff000fff, 0x00000100,
381 	0xc33, 0xc0000fff, 0x00000104,
382 	0x3079, 0x00000001, 0x00000001,
383 	0x3403, 0xff000ff0, 0x00000100,
384 	0x3603, 0xff000ff0, 0x00000100
385 };
386 
387 static const u32 spectre_golden_spm_registers[] =
388 {
389 	0xc200, 0xe0ffffff, 0xe0000000
390 };
391 
392 static const u32 spectre_golden_common_registers[] =
393 {
394 	0x31dc, 0xffffffff, 0x00000800,
395 	0x31dd, 0xffffffff, 0x00000800,
396 	0x31e6, 0xffffffff, 0x00007fbf,
397 	0x31e7, 0xffffffff, 0x00007faf
398 };
399 
400 static const u32 spectre_golden_registers[] =
401 {
402 	0xf000, 0xffff1fff, 0x96940200,
403 	0xf003, 0xffff0001, 0xff000000,
404 	0xf080, 0xfffc0fff, 0x00000100,
405 	0x1bb6, 0x00010101, 0x00010000,
406 	0x260d, 0xf00fffff, 0x00000400,
407 	0x260e, 0xfffffffc, 0x00020200,
408 	0x16ec, 0x000000f0, 0x00000070,
409 	0x16f0, 0xf0311fff, 0x80300000,
410 	0x263e, 0x73773777, 0x12010001,
411 	0x26df, 0x00ff0000, 0x00fc0000,
412 	0xbd2, 0x73773777, 0x12010001,
413 	0x2285, 0xf000003f, 0x00000007,
414 	0x22c9, 0xffffffff, 0x00ffffff,
415 	0xa0d4, 0x3f3f3fff, 0x00000082,
416 	0xa0d5, 0x0000003f, 0x00000000,
417 	0xf9e, 0x00000001, 0x00000002,
418 	0x244f, 0xffff03df, 0x00000004,
419 	0x31da, 0x00000008, 0x00000008,
420 	0x2300, 0x000008ff, 0x00000800,
421 	0x2542, 0x00010000, 0x00010000,
422 	0x2b03, 0xffffffff, 0x54763210,
423 	0x853e, 0x01ff01ff, 0x00000002,
424 	0x8526, 0x007ff800, 0x00200000,
425 	0x8057, 0xffffffff, 0x00000f40,
426 	0xc24d, 0xffffffff, 0x00000001
427 };
428 
429 static const u32 spectre_mgcg_cgcg_init[] =
430 {
431 	0x3108, 0xffffffff, 0xfffffffc,
432 	0xc200, 0xffffffff, 0xe0000000,
433 	0xf0a8, 0xffffffff, 0x00000100,
434 	0xf082, 0xffffffff, 0x00000100,
435 	0xf0b0, 0xffffffff, 0x00000100,
436 	0xf0b2, 0xffffffff, 0x00000100,
437 	0xf0b1, 0xffffffff, 0x00000100,
438 	0x1579, 0xffffffff, 0x00600100,
439 	0xf0a0, 0xffffffff, 0x00000100,
440 	0xf085, 0xffffffff, 0x06000100,
441 	0xf088, 0xffffffff, 0x00000100,
442 	0xf086, 0xffffffff, 0x06000100,
443 	0xf081, 0xffffffff, 0x00000100,
444 	0xf0b8, 0xffffffff, 0x00000100,
445 	0xf089, 0xffffffff, 0x00000100,
446 	0xf080, 0xffffffff, 0x00000100,
447 	0xf08c, 0xffffffff, 0x00000100,
448 	0xf08d, 0xffffffff, 0x00000100,
449 	0xf094, 0xffffffff, 0x00000100,
450 	0xf095, 0xffffffff, 0x00000100,
451 	0xf096, 0xffffffff, 0x00000100,
452 	0xf097, 0xffffffff, 0x00000100,
453 	0xf098, 0xffffffff, 0x00000100,
454 	0xf09f, 0xffffffff, 0x00000100,
455 	0xf09e, 0xffffffff, 0x00000100,
456 	0xf084, 0xffffffff, 0x06000100,
457 	0xf0a4, 0xffffffff, 0x00000100,
458 	0xf09d, 0xffffffff, 0x00000100,
459 	0xf0ad, 0xffffffff, 0x00000100,
460 	0xf0ac, 0xffffffff, 0x00000100,
461 	0xf09c, 0xffffffff, 0x00000100,
462 	0xc200, 0xffffffff, 0xe0000000,
463 	0xf008, 0xffffffff, 0x00010000,
464 	0xf009, 0xffffffff, 0x00030002,
465 	0xf00a, 0xffffffff, 0x00040007,
466 	0xf00b, 0xffffffff, 0x00060005,
467 	0xf00c, 0xffffffff, 0x00090008,
468 	0xf00d, 0xffffffff, 0x00010000,
469 	0xf00e, 0xffffffff, 0x00030002,
470 	0xf00f, 0xffffffff, 0x00040007,
471 	0xf010, 0xffffffff, 0x00060005,
472 	0xf011, 0xffffffff, 0x00090008,
473 	0xf012, 0xffffffff, 0x00010000,
474 	0xf013, 0xffffffff, 0x00030002,
475 	0xf014, 0xffffffff, 0x00040007,
476 	0xf015, 0xffffffff, 0x00060005,
477 	0xf016, 0xffffffff, 0x00090008,
478 	0xf017, 0xffffffff, 0x00010000,
479 	0xf018, 0xffffffff, 0x00030002,
480 	0xf019, 0xffffffff, 0x00040007,
481 	0xf01a, 0xffffffff, 0x00060005,
482 	0xf01b, 0xffffffff, 0x00090008,
483 	0xf01c, 0xffffffff, 0x00010000,
484 	0xf01d, 0xffffffff, 0x00030002,
485 	0xf01e, 0xffffffff, 0x00040007,
486 	0xf01f, 0xffffffff, 0x00060005,
487 	0xf020, 0xffffffff, 0x00090008,
488 	0xf021, 0xffffffff, 0x00010000,
489 	0xf022, 0xffffffff, 0x00030002,
490 	0xf023, 0xffffffff, 0x00040007,
491 	0xf024, 0xffffffff, 0x00060005,
492 	0xf025, 0xffffffff, 0x00090008,
493 	0xf026, 0xffffffff, 0x00010000,
494 	0xf027, 0xffffffff, 0x00030002,
495 	0xf028, 0xffffffff, 0x00040007,
496 	0xf029, 0xffffffff, 0x00060005,
497 	0xf02a, 0xffffffff, 0x00090008,
498 	0xf02b, 0xffffffff, 0x00010000,
499 	0xf02c, 0xffffffff, 0x00030002,
500 	0xf02d, 0xffffffff, 0x00040007,
501 	0xf02e, 0xffffffff, 0x00060005,
502 	0xf02f, 0xffffffff, 0x00090008,
503 	0xf000, 0xffffffff, 0x96e00200,
504 	0x21c2, 0xffffffff, 0x00900100,
505 	0x3109, 0xffffffff, 0x0020003f,
506 	0xe, 0xffffffff, 0x0140001c,
507 	0xf, 0x000f0000, 0x000f0000,
508 	0x88, 0xffffffff, 0xc060000c,
509 	0x89, 0xc0000fff, 0x00000100,
510 	0x3e4, 0xffffffff, 0x00000100,
511 	0x3e6, 0x00000101, 0x00000000,
512 	0x82a, 0xffffffff, 0x00000104,
513 	0x1579, 0xff000fff, 0x00000100,
514 	0xc33, 0xc0000fff, 0x00000104,
515 	0x3079, 0x00000001, 0x00000001,
516 	0x3403, 0xff000ff0, 0x00000100,
517 	0x3603, 0xff000ff0, 0x00000100
518 };
519 
520 static const u32 kalindi_golden_spm_registers[] =
521 {
522 	0xc200, 0xe0ffffff, 0xe0000000
523 };
524 
525 static const u32 kalindi_golden_common_registers[] =
526 {
527 	0x31dc, 0xffffffff, 0x00000800,
528 	0x31dd, 0xffffffff, 0x00000800,
529 	0x31e6, 0xffffffff, 0x00007fbf,
530 	0x31e7, 0xffffffff, 0x00007faf
531 };
532 
533 static const u32 kalindi_golden_registers[] =
534 {
535 	0xf000, 0xffffdfff, 0x6e944040,
536 	0x1579, 0xff607fff, 0xfc000100,
537 	0xf088, 0xff000fff, 0x00000100,
538 	0xf089, 0xff000fff, 0x00000100,
539 	0xf080, 0xfffc0fff, 0x00000100,
540 	0x1bb6, 0x00010101, 0x00010000,
541 	0x260c, 0xffffffff, 0x00000000,
542 	0x260d, 0xf00fffff, 0x00000400,
543 	0x16ec, 0x000000f0, 0x00000070,
544 	0x16f0, 0xf0311fff, 0x80300000,
545 	0x263e, 0x73773777, 0x12010001,
546 	0x263f, 0xffffffff, 0x00000010,
547 	0x26df, 0x00ff0000, 0x00fc0000,
548 	0x200c, 0x00001f0f, 0x0000100a,
549 	0xbd2, 0x73773777, 0x12010001,
550 	0x902, 0x000fffff, 0x000c007f,
551 	0x2285, 0xf000003f, 0x00000007,
552 	0x22c9, 0x3fff3fff, 0x00ffcfff,
553 	0xc281, 0x0000ff0f, 0x00000000,
554 	0xa293, 0x07ffffff, 0x06000000,
555 	0x136, 0x00000fff, 0x00000100,
556 	0xf9e, 0x00000001, 0x00000002,
557 	0x31da, 0x00000008, 0x00000008,
558 	0x2300, 0x000000ff, 0x00000003,
559 	0x853e, 0x01ff01ff, 0x00000002,
560 	0x8526, 0x007ff800, 0x00200000,
561 	0x8057, 0xffffffff, 0x00000f40,
562 	0x2231, 0x001f3ae3, 0x00000082,
563 	0x2235, 0x0000001f, 0x00000010,
564 	0xc24d, 0xffffffff, 0x00000000
565 };
566 
567 static const u32 kalindi_mgcg_cgcg_init[] =
568 {
569 	0x3108, 0xffffffff, 0xfffffffc,
570 	0xc200, 0xffffffff, 0xe0000000,
571 	0xf0a8, 0xffffffff, 0x00000100,
572 	0xf082, 0xffffffff, 0x00000100,
573 	0xf0b0, 0xffffffff, 0x00000100,
574 	0xf0b2, 0xffffffff, 0x00000100,
575 	0xf0b1, 0xffffffff, 0x00000100,
576 	0x1579, 0xffffffff, 0x00600100,
577 	0xf0a0, 0xffffffff, 0x00000100,
578 	0xf085, 0xffffffff, 0x06000100,
579 	0xf088, 0xffffffff, 0x00000100,
580 	0xf086, 0xffffffff, 0x06000100,
581 	0xf081, 0xffffffff, 0x00000100,
582 	0xf0b8, 0xffffffff, 0x00000100,
583 	0xf089, 0xffffffff, 0x00000100,
584 	0xf080, 0xffffffff, 0x00000100,
585 	0xf08c, 0xffffffff, 0x00000100,
586 	0xf08d, 0xffffffff, 0x00000100,
587 	0xf094, 0xffffffff, 0x00000100,
588 	0xf095, 0xffffffff, 0x00000100,
589 	0xf096, 0xffffffff, 0x00000100,
590 	0xf097, 0xffffffff, 0x00000100,
591 	0xf098, 0xffffffff, 0x00000100,
592 	0xf09f, 0xffffffff, 0x00000100,
593 	0xf09e, 0xffffffff, 0x00000100,
594 	0xf084, 0xffffffff, 0x06000100,
595 	0xf0a4, 0xffffffff, 0x00000100,
596 	0xf09d, 0xffffffff, 0x00000100,
597 	0xf0ad, 0xffffffff, 0x00000100,
598 	0xf0ac, 0xffffffff, 0x00000100,
599 	0xf09c, 0xffffffff, 0x00000100,
600 	0xc200, 0xffffffff, 0xe0000000,
601 	0xf008, 0xffffffff, 0x00010000,
602 	0xf009, 0xffffffff, 0x00030002,
603 	0xf00a, 0xffffffff, 0x00040007,
604 	0xf00b, 0xffffffff, 0x00060005,
605 	0xf00c, 0xffffffff, 0x00090008,
606 	0xf00d, 0xffffffff, 0x00010000,
607 	0xf00e, 0xffffffff, 0x00030002,
608 	0xf00f, 0xffffffff, 0x00040007,
609 	0xf010, 0xffffffff, 0x00060005,
610 	0xf011, 0xffffffff, 0x00090008,
611 	0xf000, 0xffffffff, 0x96e00200,
612 	0x21c2, 0xffffffff, 0x00900100,
613 	0x3109, 0xffffffff, 0x0020003f,
614 	0xe, 0xffffffff, 0x0140001c,
615 	0xf, 0x000f0000, 0x000f0000,
616 	0x88, 0xffffffff, 0xc060000c,
617 	0x89, 0xc0000fff, 0x00000100,
618 	0x82a, 0xffffffff, 0x00000104,
619 	0x1579, 0xff000fff, 0x00000100,
620 	0xc33, 0xc0000fff, 0x00000104,
621 	0x3079, 0x00000001, 0x00000001,
622 	0x3403, 0xff000ff0, 0x00000100,
623 	0x3603, 0xff000ff0, 0x00000100
624 };
625 
626 static const u32 hawaii_golden_spm_registers[] =
627 {
628 	0xc200, 0xe0ffffff, 0xe0000000
629 };
630 
631 static const u32 hawaii_golden_common_registers[] =
632 {
633 	0xc200, 0xffffffff, 0xe0000000,
634 	0xa0d4, 0xffffffff, 0x3a00161a,
635 	0xa0d5, 0xffffffff, 0x0000002e,
636 	0x2684, 0xffffffff, 0x00018208,
637 	0x263e, 0xffffffff, 0x12011003
638 };
639 
640 static const u32 hawaii_golden_registers[] =
641 {
642 	0xcd5, 0x00000333, 0x00000333,
643 	0x2684, 0x00010000, 0x00058208,
644 	0x260c, 0xffffffff, 0x00000000,
645 	0x260d, 0xf00fffff, 0x00000400,
646 	0x260e, 0x0002021c, 0x00020200,
647 	0x31e, 0x00000080, 0x00000000,
648 	0x16ec, 0x000000f0, 0x00000070,
649 	0x16f0, 0xf0311fff, 0x80300000,
650 	0xd43, 0x00810000, 0x408af000,
651 	0x1c0c, 0x31000111, 0x00000011,
652 	0xbd2, 0x73773777, 0x12010001,
653 	0x848, 0x0000007f, 0x0000001b,
654 	0x877, 0x00007fb6, 0x00002191,
655 	0xd8a, 0x0000003f, 0x0000000a,
656 	0xd8b, 0x0000003f, 0x0000000a,
657 	0xab9, 0x00073ffe, 0x000022a2,
658 	0x903, 0x000007ff, 0x00000000,
659 	0x22fc, 0x00002001, 0x00000001,
660 	0x22c9, 0xffffffff, 0x00ffffff,
661 	0xc281, 0x0000ff0f, 0x00000000,
662 	0xa293, 0x07ffffff, 0x06000000,
663 	0xf9e, 0x00000001, 0x00000002,
664 	0x31da, 0x00000008, 0x00000008,
665 	0x31dc, 0x00000f00, 0x00000800,
666 	0x31dd, 0x00000f00, 0x00000800,
667 	0x31e6, 0x00ffffff, 0x00ff7fbf,
668 	0x31e7, 0x00ffffff, 0x00ff7faf,
669 	0x2300, 0x000000ff, 0x00000800,
670 	0x390, 0x00001fff, 0x00001fff,
671 	0x2418, 0x0000007f, 0x00000020,
672 	0x2542, 0x00010000, 0x00010000,
673 	0x2b80, 0x00100000, 0x000ff07c,
674 	0x2b05, 0x000003ff, 0x0000000f,
675 	0x2b04, 0xffffffff, 0x7564fdec,
676 	0x2b03, 0xffffffff, 0x3120b9a8,
677 	0x2b02, 0x20000000, 0x0f9c0000
678 };
679 
680 static const u32 hawaii_mgcg_cgcg_init[] =
681 {
682 	0x3108, 0xffffffff, 0xfffffffd,
683 	0xc200, 0xffffffff, 0xe0000000,
684 	0xf0a8, 0xffffffff, 0x00000100,
685 	0xf082, 0xffffffff, 0x00000100,
686 	0xf0b0, 0xffffffff, 0x00000100,
687 	0xf0b2, 0xffffffff, 0x00000100,
688 	0xf0b1, 0xffffffff, 0x00000100,
689 	0x1579, 0xffffffff, 0x00200100,
690 	0xf0a0, 0xffffffff, 0x00000100,
691 	0xf085, 0xffffffff, 0x06000100,
692 	0xf088, 0xffffffff, 0x00000100,
693 	0xf086, 0xffffffff, 0x06000100,
694 	0xf081, 0xffffffff, 0x00000100,
695 	0xf0b8, 0xffffffff, 0x00000100,
696 	0xf089, 0xffffffff, 0x00000100,
697 	0xf080, 0xffffffff, 0x00000100,
698 	0xf08c, 0xffffffff, 0x00000100,
699 	0xf08d, 0xffffffff, 0x00000100,
700 	0xf094, 0xffffffff, 0x00000100,
701 	0xf095, 0xffffffff, 0x00000100,
702 	0xf096, 0xffffffff, 0x00000100,
703 	0xf097, 0xffffffff, 0x00000100,
704 	0xf098, 0xffffffff, 0x00000100,
705 	0xf09f, 0xffffffff, 0x00000100,
706 	0xf09e, 0xffffffff, 0x00000100,
707 	0xf084, 0xffffffff, 0x06000100,
708 	0xf0a4, 0xffffffff, 0x00000100,
709 	0xf09d, 0xffffffff, 0x00000100,
710 	0xf0ad, 0xffffffff, 0x00000100,
711 	0xf0ac, 0xffffffff, 0x00000100,
712 	0xf09c, 0xffffffff, 0x00000100,
713 	0xc200, 0xffffffff, 0xe0000000,
714 	0xf008, 0xffffffff, 0x00010000,
715 	0xf009, 0xffffffff, 0x00030002,
716 	0xf00a, 0xffffffff, 0x00040007,
717 	0xf00b, 0xffffffff, 0x00060005,
718 	0xf00c, 0xffffffff, 0x00090008,
719 	0xf00d, 0xffffffff, 0x00010000,
720 	0xf00e, 0xffffffff, 0x00030002,
721 	0xf00f, 0xffffffff, 0x00040007,
722 	0xf010, 0xffffffff, 0x00060005,
723 	0xf011, 0xffffffff, 0x00090008,
724 	0xf012, 0xffffffff, 0x00010000,
725 	0xf013, 0xffffffff, 0x00030002,
726 	0xf014, 0xffffffff, 0x00040007,
727 	0xf015, 0xffffffff, 0x00060005,
728 	0xf016, 0xffffffff, 0x00090008,
729 	0xf017, 0xffffffff, 0x00010000,
730 	0xf018, 0xffffffff, 0x00030002,
731 	0xf019, 0xffffffff, 0x00040007,
732 	0xf01a, 0xffffffff, 0x00060005,
733 	0xf01b, 0xffffffff, 0x00090008,
734 	0xf01c, 0xffffffff, 0x00010000,
735 	0xf01d, 0xffffffff, 0x00030002,
736 	0xf01e, 0xffffffff, 0x00040007,
737 	0xf01f, 0xffffffff, 0x00060005,
738 	0xf020, 0xffffffff, 0x00090008,
739 	0xf021, 0xffffffff, 0x00010000,
740 	0xf022, 0xffffffff, 0x00030002,
741 	0xf023, 0xffffffff, 0x00040007,
742 	0xf024, 0xffffffff, 0x00060005,
743 	0xf025, 0xffffffff, 0x00090008,
744 	0xf026, 0xffffffff, 0x00010000,
745 	0xf027, 0xffffffff, 0x00030002,
746 	0xf028, 0xffffffff, 0x00040007,
747 	0xf029, 0xffffffff, 0x00060005,
748 	0xf02a, 0xffffffff, 0x00090008,
749 	0xf02b, 0xffffffff, 0x00010000,
750 	0xf02c, 0xffffffff, 0x00030002,
751 	0xf02d, 0xffffffff, 0x00040007,
752 	0xf02e, 0xffffffff, 0x00060005,
753 	0xf02f, 0xffffffff, 0x00090008,
754 	0xf030, 0xffffffff, 0x00010000,
755 	0xf031, 0xffffffff, 0x00030002,
756 	0xf032, 0xffffffff, 0x00040007,
757 	0xf033, 0xffffffff, 0x00060005,
758 	0xf034, 0xffffffff, 0x00090008,
759 	0xf035, 0xffffffff, 0x00010000,
760 	0xf036, 0xffffffff, 0x00030002,
761 	0xf037, 0xffffffff, 0x00040007,
762 	0xf038, 0xffffffff, 0x00060005,
763 	0xf039, 0xffffffff, 0x00090008,
764 	0xf03a, 0xffffffff, 0x00010000,
765 	0xf03b, 0xffffffff, 0x00030002,
766 	0xf03c, 0xffffffff, 0x00040007,
767 	0xf03d, 0xffffffff, 0x00060005,
768 	0xf03e, 0xffffffff, 0x00090008,
769 	0x30c6, 0xffffffff, 0x00020200,
770 	0xcd4, 0xffffffff, 0x00000200,
771 	0x570, 0xffffffff, 0x00000400,
772 	0x157a, 0xffffffff, 0x00000000,
773 	0xbd4, 0xffffffff, 0x00000902,
774 	0xf000, 0xffffffff, 0x96940200,
775 	0x21c2, 0xffffffff, 0x00900100,
776 	0x3109, 0xffffffff, 0x0020003f,
777 	0xe, 0xffffffff, 0x0140001c,
778 	0xf, 0x000f0000, 0x000f0000,
779 	0x88, 0xffffffff, 0xc060000c,
780 	0x89, 0xc0000fff, 0x00000100,
781 	0x3e4, 0xffffffff, 0x00000100,
782 	0x3e6, 0x00000101, 0x00000000,
783 	0x82a, 0xffffffff, 0x00000104,
784 	0x1579, 0xff000fff, 0x00000100,
785 	0xc33, 0xc0000fff, 0x00000104,
786 	0x3079, 0x00000001, 0x00000001,
787 	0x3403, 0xff000ff0, 0x00000100,
788 	0x3603, 0xff000ff0, 0x00000100
789 };
790 
791 static const u32 godavari_golden_registers[] =
792 {
793 	0x1579, 0xff607fff, 0xfc000100,
794 	0x1bb6, 0x00010101, 0x00010000,
795 	0x260c, 0xffffffff, 0x00000000,
796 	0x260c0, 0xf00fffff, 0x00000400,
797 	0x184c, 0xffffffff, 0x00010000,
798 	0x16ec, 0x000000f0, 0x00000070,
799 	0x16f0, 0xf0311fff, 0x80300000,
800 	0x263e, 0x73773777, 0x12010001,
801 	0x263f, 0xffffffff, 0x00000010,
802 	0x200c, 0x00001f0f, 0x0000100a,
803 	0xbd2, 0x73773777, 0x12010001,
804 	0x902, 0x000fffff, 0x000c007f,
805 	0x2285, 0xf000003f, 0x00000007,
806 	0x22c9, 0xffffffff, 0x00ff0fff,
807 	0xc281, 0x0000ff0f, 0x00000000,
808 	0xa293, 0x07ffffff, 0x06000000,
809 	0x136, 0x00000fff, 0x00000100,
810 	0x3405, 0x00010000, 0x00810001,
811 	0x3605, 0x00010000, 0x00810001,
812 	0xf9e, 0x00000001, 0x00000002,
813 	0x31da, 0x00000008, 0x00000008,
814 	0x31dc, 0x00000f00, 0x00000800,
815 	0x31dd, 0x00000f00, 0x00000800,
816 	0x31e6, 0x00ffffff, 0x00ff7fbf,
817 	0x31e7, 0x00ffffff, 0x00ff7faf,
818 	0x2300, 0x000000ff, 0x00000001,
819 	0x853e, 0x01ff01ff, 0x00000002,
820 	0x8526, 0x007ff800, 0x00200000,
821 	0x8057, 0xffffffff, 0x00000f40,
822 	0x2231, 0x001f3ae3, 0x00000082,
823 	0x2235, 0x0000001f, 0x00000010,
824 	0xc24d, 0xffffffff, 0x00000000
825 };
826 
827 static void cik_init_golden_registers(struct amdgpu_device *adev)
828 {
829 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
830 	mutex_lock(&adev->grbm_idx_mutex);
831 
832 	switch (adev->asic_type) {
833 	case CHIP_BONAIRE:
834 		amdgpu_device_program_register_sequence(adev,
835 							bonaire_mgcg_cgcg_init,
836 							ARRAY_SIZE(bonaire_mgcg_cgcg_init));
837 		amdgpu_device_program_register_sequence(adev,
838 							bonaire_golden_registers,
839 							ARRAY_SIZE(bonaire_golden_registers));
840 		amdgpu_device_program_register_sequence(adev,
841 							bonaire_golden_common_registers,
842 							ARRAY_SIZE(bonaire_golden_common_registers));
843 		amdgpu_device_program_register_sequence(adev,
844 							bonaire_golden_spm_registers,
845 							ARRAY_SIZE(bonaire_golden_spm_registers));
846 		break;
847 	case CHIP_KABINI:
848 		amdgpu_device_program_register_sequence(adev,
849 							kalindi_mgcg_cgcg_init,
850 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
851 		amdgpu_device_program_register_sequence(adev,
852 							kalindi_golden_registers,
853 							ARRAY_SIZE(kalindi_golden_registers));
854 		amdgpu_device_program_register_sequence(adev,
855 							kalindi_golden_common_registers,
856 							ARRAY_SIZE(kalindi_golden_common_registers));
857 		amdgpu_device_program_register_sequence(adev,
858 							kalindi_golden_spm_registers,
859 							ARRAY_SIZE(kalindi_golden_spm_registers));
860 		break;
861 	case CHIP_MULLINS:
862 		amdgpu_device_program_register_sequence(adev,
863 							kalindi_mgcg_cgcg_init,
864 							ARRAY_SIZE(kalindi_mgcg_cgcg_init));
865 		amdgpu_device_program_register_sequence(adev,
866 							godavari_golden_registers,
867 							ARRAY_SIZE(godavari_golden_registers));
868 		amdgpu_device_program_register_sequence(adev,
869 							kalindi_golden_common_registers,
870 							ARRAY_SIZE(kalindi_golden_common_registers));
871 		amdgpu_device_program_register_sequence(adev,
872 							kalindi_golden_spm_registers,
873 							ARRAY_SIZE(kalindi_golden_spm_registers));
874 		break;
875 	case CHIP_KAVERI:
876 		amdgpu_device_program_register_sequence(adev,
877 							spectre_mgcg_cgcg_init,
878 							ARRAY_SIZE(spectre_mgcg_cgcg_init));
879 		amdgpu_device_program_register_sequence(adev,
880 							spectre_golden_registers,
881 							ARRAY_SIZE(spectre_golden_registers));
882 		amdgpu_device_program_register_sequence(adev,
883 							spectre_golden_common_registers,
884 							ARRAY_SIZE(spectre_golden_common_registers));
885 		amdgpu_device_program_register_sequence(adev,
886 							spectre_golden_spm_registers,
887 							ARRAY_SIZE(spectre_golden_spm_registers));
888 		break;
889 	case CHIP_HAWAII:
890 		amdgpu_device_program_register_sequence(adev,
891 							hawaii_mgcg_cgcg_init,
892 							ARRAY_SIZE(hawaii_mgcg_cgcg_init));
893 		amdgpu_device_program_register_sequence(adev,
894 							hawaii_golden_registers,
895 							ARRAY_SIZE(hawaii_golden_registers));
896 		amdgpu_device_program_register_sequence(adev,
897 							hawaii_golden_common_registers,
898 							ARRAY_SIZE(hawaii_golden_common_registers));
899 		amdgpu_device_program_register_sequence(adev,
900 							hawaii_golden_spm_registers,
901 							ARRAY_SIZE(hawaii_golden_spm_registers));
902 		break;
903 	default:
904 		break;
905 	}
906 	mutex_unlock(&adev->grbm_idx_mutex);
907 }
908 
909 /**
910  * cik_get_xclk - get the xclk
911  *
912  * @adev: amdgpu_device pointer
913  *
914  * Returns the reference clock used by the gfx engine
915  * (CIK).
916  */
917 static u32 cik_get_xclk(struct amdgpu_device *adev)
918 {
919 	u32 reference_clock = adev->clock.spll.reference_freq;
920 
921 	if (adev->flags & AMD_IS_APU) {
922 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
923 			return reference_clock / 2;
924 	} else {
925 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
926 			return reference_clock / 4;
927 	}
928 	return reference_clock;
929 }
930 
931 /**
932  * cik_srbm_select - select specific register instances
933  *
934  * @adev: amdgpu_device pointer
935  * @me: selected ME (micro engine)
936  * @pipe: pipe
937  * @queue: queue
938  * @vmid: VMID
939  *
940  * Switches the currently active registers instances.  Some
941  * registers are instanced per VMID, others are instanced per
942  * me/pipe/queue combination.
943  */
944 void cik_srbm_select(struct amdgpu_device *adev,
945 		     u32 me, u32 pipe, u32 queue, u32 vmid)
946 {
947 	u32 srbm_gfx_cntl =
948 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
949 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
950 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
951 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
952 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
953 }
954 
955 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
956 {
957 	uint32_t tmp;
958 
959 	tmp = RREG32(mmCONFIG_CNTL);
960 	if (!state)
961 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
962 	else
963 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
964 	WREG32(mmCONFIG_CNTL, tmp);
965 }
966 
967 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
968 {
969 	u32 bus_cntl;
970 	u32 d1vga_control = 0;
971 	u32 d2vga_control = 0;
972 	u32 vga_render_control = 0;
973 	u32 rom_cntl;
974 	bool r;
975 
976 	bus_cntl = RREG32(mmBUS_CNTL);
977 	if (adev->mode_info.num_crtc) {
978 		d1vga_control = RREG32(mmD1VGA_CONTROL);
979 		d2vga_control = RREG32(mmD2VGA_CONTROL);
980 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
981 	}
982 	rom_cntl = RREG32_SMC(ixROM_CNTL);
983 
984 	/* enable the rom */
985 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
986 	if (adev->mode_info.num_crtc) {
987 		/* Disable VGA mode */
988 		WREG32(mmD1VGA_CONTROL,
989 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
990 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
991 		WREG32(mmD2VGA_CONTROL,
992 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
993 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
994 		WREG32(mmVGA_RENDER_CONTROL,
995 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
996 	}
997 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
998 
999 	r = amdgpu_read_bios(adev);
1000 
1001 	/* restore regs */
1002 	WREG32(mmBUS_CNTL, bus_cntl);
1003 	if (adev->mode_info.num_crtc) {
1004 		WREG32(mmD1VGA_CONTROL, d1vga_control);
1005 		WREG32(mmD2VGA_CONTROL, d2vga_control);
1006 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
1007 	}
1008 	WREG32_SMC(ixROM_CNTL, rom_cntl);
1009 	return r;
1010 }
1011 
1012 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
1013 				   u8 *bios, u32 length_bytes)
1014 {
1015 	u32 *dw_ptr;
1016 	unsigned long flags;
1017 	u32 i, length_dw;
1018 
1019 	if (bios == NULL)
1020 		return false;
1021 	if (length_bytes == 0)
1022 		return false;
1023 	/* APU vbios image is part of sbios image */
1024 	if (adev->flags & AMD_IS_APU)
1025 		return false;
1026 
1027 	dw_ptr = (u32 *)bios;
1028 	length_dw = ALIGN(length_bytes, 4) / 4;
1029 	/* take the smc lock since we are using the smc index */
1030 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
1031 	/* set rom index to 0 */
1032 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
1033 	WREG32(mmSMC_IND_DATA_0, 0);
1034 	/* set index to data for continous read */
1035 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
1036 	for (i = 0; i < length_dw; i++)
1037 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
1038 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
1039 
1040 	return true;
1041 }
1042 
1043 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
1044 	{mmGRBM_STATUS},
1045 	{mmGRBM_STATUS2},
1046 	{mmGRBM_STATUS_SE0},
1047 	{mmGRBM_STATUS_SE1},
1048 	{mmGRBM_STATUS_SE2},
1049 	{mmGRBM_STATUS_SE3},
1050 	{mmSRBM_STATUS},
1051 	{mmSRBM_STATUS2},
1052 	{mmSDMA0_STATUS_REG + SDMA0_REGISTER_OFFSET},
1053 	{mmSDMA0_STATUS_REG + SDMA1_REGISTER_OFFSET},
1054 	{mmCP_STAT},
1055 	{mmCP_STALLED_STAT1},
1056 	{mmCP_STALLED_STAT2},
1057 	{mmCP_STALLED_STAT3},
1058 	{mmCP_CPF_BUSY_STAT},
1059 	{mmCP_CPF_STALLED_STAT1},
1060 	{mmCP_CPF_STATUS},
1061 	{mmCP_CPC_BUSY_STAT},
1062 	{mmCP_CPC_STALLED_STAT1},
1063 	{mmCP_CPC_STATUS},
1064 	{mmGB_ADDR_CONFIG},
1065 	{mmMC_ARB_RAMCFG},
1066 	{mmGB_TILE_MODE0},
1067 	{mmGB_TILE_MODE1},
1068 	{mmGB_TILE_MODE2},
1069 	{mmGB_TILE_MODE3},
1070 	{mmGB_TILE_MODE4},
1071 	{mmGB_TILE_MODE5},
1072 	{mmGB_TILE_MODE6},
1073 	{mmGB_TILE_MODE7},
1074 	{mmGB_TILE_MODE8},
1075 	{mmGB_TILE_MODE9},
1076 	{mmGB_TILE_MODE10},
1077 	{mmGB_TILE_MODE11},
1078 	{mmGB_TILE_MODE12},
1079 	{mmGB_TILE_MODE13},
1080 	{mmGB_TILE_MODE14},
1081 	{mmGB_TILE_MODE15},
1082 	{mmGB_TILE_MODE16},
1083 	{mmGB_TILE_MODE17},
1084 	{mmGB_TILE_MODE18},
1085 	{mmGB_TILE_MODE19},
1086 	{mmGB_TILE_MODE20},
1087 	{mmGB_TILE_MODE21},
1088 	{mmGB_TILE_MODE22},
1089 	{mmGB_TILE_MODE23},
1090 	{mmGB_TILE_MODE24},
1091 	{mmGB_TILE_MODE25},
1092 	{mmGB_TILE_MODE26},
1093 	{mmGB_TILE_MODE27},
1094 	{mmGB_TILE_MODE28},
1095 	{mmGB_TILE_MODE29},
1096 	{mmGB_TILE_MODE30},
1097 	{mmGB_TILE_MODE31},
1098 	{mmGB_MACROTILE_MODE0},
1099 	{mmGB_MACROTILE_MODE1},
1100 	{mmGB_MACROTILE_MODE2},
1101 	{mmGB_MACROTILE_MODE3},
1102 	{mmGB_MACROTILE_MODE4},
1103 	{mmGB_MACROTILE_MODE5},
1104 	{mmGB_MACROTILE_MODE6},
1105 	{mmGB_MACROTILE_MODE7},
1106 	{mmGB_MACROTILE_MODE8},
1107 	{mmGB_MACROTILE_MODE9},
1108 	{mmGB_MACROTILE_MODE10},
1109 	{mmGB_MACROTILE_MODE11},
1110 	{mmGB_MACROTILE_MODE12},
1111 	{mmGB_MACROTILE_MODE13},
1112 	{mmGB_MACROTILE_MODE14},
1113 	{mmGB_MACROTILE_MODE15},
1114 	{mmCC_RB_BACKEND_DISABLE, true},
1115 	{mmGC_USER_RB_BACKEND_DISABLE, true},
1116 	{mmGB_BACKEND_MAP, false},
1117 	{mmPA_SC_RASTER_CONFIG, true},
1118 	{mmPA_SC_RASTER_CONFIG_1, true},
1119 };
1120 
1121 
1122 static uint32_t cik_get_register_value(struct amdgpu_device *adev,
1123 				       bool indexed, u32 se_num,
1124 				       u32 sh_num, u32 reg_offset)
1125 {
1126 	if (indexed) {
1127 		uint32_t val;
1128 		unsigned se_idx = (se_num == 0xffffffff) ? 0 : se_num;
1129 		unsigned sh_idx = (sh_num == 0xffffffff) ? 0 : sh_num;
1130 
1131 		switch (reg_offset) {
1132 		case mmCC_RB_BACKEND_DISABLE:
1133 			return adev->gfx.config.rb_config[se_idx][sh_idx].rb_backend_disable;
1134 		case mmGC_USER_RB_BACKEND_DISABLE:
1135 			return adev->gfx.config.rb_config[se_idx][sh_idx].user_rb_backend_disable;
1136 		case mmPA_SC_RASTER_CONFIG:
1137 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config;
1138 		case mmPA_SC_RASTER_CONFIG_1:
1139 			return adev->gfx.config.rb_config[se_idx][sh_idx].raster_config_1;
1140 		}
1141 
1142 		mutex_lock(&adev->grbm_idx_mutex);
1143 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1144 			amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1145 
1146 		val = RREG32(reg_offset);
1147 
1148 		if (se_num != 0xffffffff || sh_num != 0xffffffff)
1149 			amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1150 		mutex_unlock(&adev->grbm_idx_mutex);
1151 		return val;
1152 	} else {
1153 		unsigned idx;
1154 
1155 		switch (reg_offset) {
1156 		case mmGB_ADDR_CONFIG:
1157 			return adev->gfx.config.gb_addr_config;
1158 		case mmMC_ARB_RAMCFG:
1159 			return adev->gfx.config.mc_arb_ramcfg;
1160 		case mmGB_TILE_MODE0:
1161 		case mmGB_TILE_MODE1:
1162 		case mmGB_TILE_MODE2:
1163 		case mmGB_TILE_MODE3:
1164 		case mmGB_TILE_MODE4:
1165 		case mmGB_TILE_MODE5:
1166 		case mmGB_TILE_MODE6:
1167 		case mmGB_TILE_MODE7:
1168 		case mmGB_TILE_MODE8:
1169 		case mmGB_TILE_MODE9:
1170 		case mmGB_TILE_MODE10:
1171 		case mmGB_TILE_MODE11:
1172 		case mmGB_TILE_MODE12:
1173 		case mmGB_TILE_MODE13:
1174 		case mmGB_TILE_MODE14:
1175 		case mmGB_TILE_MODE15:
1176 		case mmGB_TILE_MODE16:
1177 		case mmGB_TILE_MODE17:
1178 		case mmGB_TILE_MODE18:
1179 		case mmGB_TILE_MODE19:
1180 		case mmGB_TILE_MODE20:
1181 		case mmGB_TILE_MODE21:
1182 		case mmGB_TILE_MODE22:
1183 		case mmGB_TILE_MODE23:
1184 		case mmGB_TILE_MODE24:
1185 		case mmGB_TILE_MODE25:
1186 		case mmGB_TILE_MODE26:
1187 		case mmGB_TILE_MODE27:
1188 		case mmGB_TILE_MODE28:
1189 		case mmGB_TILE_MODE29:
1190 		case mmGB_TILE_MODE30:
1191 		case mmGB_TILE_MODE31:
1192 			idx = (reg_offset - mmGB_TILE_MODE0);
1193 			return adev->gfx.config.tile_mode_array[idx];
1194 		case mmGB_MACROTILE_MODE0:
1195 		case mmGB_MACROTILE_MODE1:
1196 		case mmGB_MACROTILE_MODE2:
1197 		case mmGB_MACROTILE_MODE3:
1198 		case mmGB_MACROTILE_MODE4:
1199 		case mmGB_MACROTILE_MODE5:
1200 		case mmGB_MACROTILE_MODE6:
1201 		case mmGB_MACROTILE_MODE7:
1202 		case mmGB_MACROTILE_MODE8:
1203 		case mmGB_MACROTILE_MODE9:
1204 		case mmGB_MACROTILE_MODE10:
1205 		case mmGB_MACROTILE_MODE11:
1206 		case mmGB_MACROTILE_MODE12:
1207 		case mmGB_MACROTILE_MODE13:
1208 		case mmGB_MACROTILE_MODE14:
1209 		case mmGB_MACROTILE_MODE15:
1210 			idx = (reg_offset - mmGB_MACROTILE_MODE0);
1211 			return adev->gfx.config.macrotile_mode_array[idx];
1212 		default:
1213 			return RREG32(reg_offset);
1214 		}
1215 	}
1216 }
1217 
1218 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1219 			     u32 sh_num, u32 reg_offset, u32 *value)
1220 {
1221 	uint32_t i;
1222 
1223 	*value = 0;
1224 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1225 		bool indexed = cik_allowed_read_registers[i].grbm_indexed;
1226 
1227 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1228 			continue;
1229 
1230 		*value = cik_get_register_value(adev, indexed, se_num, sh_num,
1231 						reg_offset);
1232 		return 0;
1233 	}
1234 	return -EINVAL;
1235 }
1236 
1237 struct kv_reset_save_regs {
1238 	u32 gmcon_reng_execute;
1239 	u32 gmcon_misc;
1240 	u32 gmcon_misc3;
1241 };
1242 
1243 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1244 				   struct kv_reset_save_regs *save)
1245 {
1246 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1247 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1248 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1249 
1250 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1251 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1252 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1253 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1254 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1255 }
1256 
1257 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1258 				      struct kv_reset_save_regs *save)
1259 {
1260 	int i;
1261 
1262 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1263 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1264 
1265 	for (i = 0; i < 5; i++)
1266 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1267 
1268 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1269 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1270 
1271 	for (i = 0; i < 5; i++)
1272 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1273 
1274 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1275 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1276 
1277 	for (i = 0; i < 5; i++)
1278 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1279 
1280 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1281 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1282 
1283 	for (i = 0; i < 5; i++)
1284 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1285 
1286 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1287 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1288 
1289 	for (i = 0; i < 5; i++)
1290 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1291 
1292 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1293 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1294 
1295 	for (i = 0; i < 5; i++)
1296 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1297 
1298 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1299 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1300 
1301 	for (i = 0; i < 5; i++)
1302 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1303 
1304 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1305 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1306 
1307 	for (i = 0; i < 5; i++)
1308 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1309 
1310 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1311 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1312 
1313 	for (i = 0; i < 5; i++)
1314 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1315 
1316 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1317 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1318 
1319 	for (i = 0; i < 5; i++)
1320 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1321 
1322 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1323 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1324 
1325 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1326 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1327 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1328 }
1329 
1330 /**
1331  * cik_asic_pci_config_reset - soft reset GPU
1332  *
1333  * @adev: amdgpu_device pointer
1334  *
1335  * Use PCI Config method to reset the GPU.
1336  *
1337  * Returns 0 for success.
1338  */
1339 static int cik_asic_pci_config_reset(struct amdgpu_device *adev)
1340 {
1341 	struct kv_reset_save_regs kv_save = { 0 };
1342 	u32 i;
1343 	int r = -EINVAL;
1344 
1345 	amdgpu_atombios_scratch_regs_engine_hung(adev, true);
1346 
1347 	if (adev->flags & AMD_IS_APU)
1348 		kv_save_regs_for_reset(adev, &kv_save);
1349 
1350 	/* disable BM */
1351 	pci_clear_master(adev->pdev);
1352 	/* reset */
1353 	amdgpu_device_pci_config_reset(adev);
1354 
1355 	udelay(100);
1356 
1357 	/* wait for asic to come out of reset */
1358 	for (i = 0; i < adev->usec_timeout; i++) {
1359 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1360 			/* enable BM */
1361 			pci_set_master(adev->pdev);
1362 			adev->has_hw_reset = true;
1363 			r = 0;
1364 			break;
1365 		}
1366 		udelay(1);
1367 	}
1368 
1369 	/* does asic init need to be run first??? */
1370 	if (adev->flags & AMD_IS_APU)
1371 		kv_restore_regs_for_reset(adev, &kv_save);
1372 
1373 	amdgpu_atombios_scratch_regs_engine_hung(adev, false);
1374 
1375 	return r;
1376 }
1377 
1378 static bool cik_asic_supports_baco(struct amdgpu_device *adev)
1379 {
1380 	switch (adev->asic_type) {
1381 	case CHIP_BONAIRE:
1382 	case CHIP_HAWAII:
1383 		return amdgpu_dpm_is_baco_supported(adev);
1384 	default:
1385 		return false;
1386 	}
1387 }
1388 
1389 static enum amd_reset_method
1390 cik_asic_reset_method(struct amdgpu_device *adev)
1391 {
1392 	bool baco_reset;
1393 
1394 	if (amdgpu_reset_method == AMD_RESET_METHOD_LEGACY ||
1395 	    amdgpu_reset_method == AMD_RESET_METHOD_BACO)
1396 		return amdgpu_reset_method;
1397 
1398 	if (amdgpu_reset_method != -1)
1399 		dev_warn(adev->dev, "Specified reset:%d isn't supported, using AUTO instead.\n",
1400 				  amdgpu_reset_method);
1401 
1402 	switch (adev->asic_type) {
1403 	case CHIP_BONAIRE:
1404 	case CHIP_HAWAII:
1405 		baco_reset = cik_asic_supports_baco(adev);
1406 		break;
1407 	default:
1408 		baco_reset = false;
1409 		break;
1410 	}
1411 
1412 	if (baco_reset)
1413 		return AMD_RESET_METHOD_BACO;
1414 	else
1415 		return AMD_RESET_METHOD_LEGACY;
1416 }
1417 
1418 /**
1419  * cik_asic_reset - soft reset GPU
1420  *
1421  * @adev: amdgpu_device pointer
1422  *
1423  * Look up which blocks are hung and attempt
1424  * to reset them.
1425  * Returns 0 for success.
1426  */
1427 static int cik_asic_reset(struct amdgpu_device *adev)
1428 {
1429 	int r;
1430 
1431 	if (cik_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
1432 		dev_info(adev->dev, "BACO reset\n");
1433 		r = amdgpu_dpm_baco_reset(adev);
1434 	} else {
1435 		dev_info(adev->dev, "PCI CONFIG reset\n");
1436 		r = cik_asic_pci_config_reset(adev);
1437 	}
1438 
1439 	return r;
1440 }
1441 
1442 static u32 cik_get_config_memsize(struct amdgpu_device *adev)
1443 {
1444 	return RREG32(mmCONFIG_MEMSIZE);
1445 }
1446 
1447 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1448 			      u32 cntl_reg, u32 status_reg)
1449 {
1450 	int r, i;
1451 	struct atom_clock_dividers dividers;
1452 	uint32_t tmp;
1453 
1454 	r = amdgpu_atombios_get_clock_dividers(adev,
1455 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1456 					       clock, false, &dividers);
1457 	if (r)
1458 		return r;
1459 
1460 	tmp = RREG32_SMC(cntl_reg);
1461 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1462 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1463 	tmp |= dividers.post_divider;
1464 	WREG32_SMC(cntl_reg, tmp);
1465 
1466 	for (i = 0; i < 100; i++) {
1467 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1468 			break;
1469 		mdelay(10);
1470 	}
1471 	if (i == 100)
1472 		return -ETIMEDOUT;
1473 
1474 	return 0;
1475 }
1476 
1477 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1478 {
1479 	int r = 0;
1480 
1481 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1482 	if (r)
1483 		return r;
1484 
1485 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1486 	return r;
1487 }
1488 
1489 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1490 {
1491 	int r, i;
1492 	struct atom_clock_dividers dividers;
1493 	u32 tmp;
1494 
1495 	r = amdgpu_atombios_get_clock_dividers(adev,
1496 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1497 					       ecclk, false, &dividers);
1498 	if (r)
1499 		return r;
1500 
1501 	for (i = 0; i < 100; i++) {
1502 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1503 			break;
1504 		mdelay(10);
1505 	}
1506 	if (i == 100)
1507 		return -ETIMEDOUT;
1508 
1509 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1510 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1511 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1512 	tmp |= dividers.post_divider;
1513 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1514 
1515 	for (i = 0; i < 100; i++) {
1516 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1517 			break;
1518 		mdelay(10);
1519 	}
1520 	if (i == 100)
1521 		return -ETIMEDOUT;
1522 
1523 	return 0;
1524 }
1525 
1526 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1527 {
1528 	struct pci_dev *root = adev->pdev->bus->self;
1529 	u32 speed_cntl, current_data_rate;
1530 	int i;
1531 	u16 tmp16;
1532 
1533 	if (pci_is_root_bus(adev->pdev->bus))
1534 		return;
1535 
1536 	if (amdgpu_pcie_gen2 == 0)
1537 		return;
1538 
1539 	if (adev->flags & AMD_IS_APU)
1540 		return;
1541 
1542 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1543 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1544 		return;
1545 
1546 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1547 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1548 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1549 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1550 		if (current_data_rate == 2) {
1551 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1552 			return;
1553 		}
1554 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1555 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1556 		if (current_data_rate == 1) {
1557 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1558 			return;
1559 		}
1560 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1561 	}
1562 
1563 	if (!pci_is_pcie(root) || !pci_is_pcie(adev->pdev))
1564 		return;
1565 
1566 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1567 		/* re-try equalization if gen3 is not already enabled */
1568 		if (current_data_rate != 2) {
1569 			u16 bridge_cfg, gpu_cfg;
1570 			u16 bridge_cfg2, gpu_cfg2;
1571 			u32 max_lw, current_lw, tmp;
1572 
1573 			pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1574 						  &bridge_cfg);
1575 			pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL,
1576 						  &gpu_cfg);
1577 
1578 			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1579 			pcie_capability_write_word(root, PCI_EXP_LNKCTL, tmp16);
1580 
1581 			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1582 			pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL,
1583 						   tmp16);
1584 
1585 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1586 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1587 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1588 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1589 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1590 
1591 			if (current_lw < max_lw) {
1592 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1593 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1594 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1595 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1596 					tmp |= (max_lw <<
1597 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1598 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1599 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1600 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1601 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1602 				}
1603 			}
1604 
1605 			for (i = 0; i < 10; i++) {
1606 				/* check status */
1607 				pcie_capability_read_word(adev->pdev,
1608 							  PCI_EXP_DEVSTA,
1609 							  &tmp16);
1610 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1611 					break;
1612 
1613 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1614 							  &bridge_cfg);
1615 				pcie_capability_read_word(adev->pdev,
1616 							  PCI_EXP_LNKCTL,
1617 							  &gpu_cfg);
1618 
1619 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1620 							  &bridge_cfg2);
1621 				pcie_capability_read_word(adev->pdev,
1622 							  PCI_EXP_LNKCTL2,
1623 							  &gpu_cfg2);
1624 
1625 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1626 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1627 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1628 
1629 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1630 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1631 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1632 
1633 				msleep(100);
1634 
1635 				/* linkctl */
1636 				pcie_capability_read_word(root, PCI_EXP_LNKCTL,
1637 							  &tmp16);
1638 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1639 				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1640 				pcie_capability_write_word(root, PCI_EXP_LNKCTL,
1641 							   tmp16);
1642 
1643 				pcie_capability_read_word(adev->pdev,
1644 							  PCI_EXP_LNKCTL,
1645 							  &tmp16);
1646 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1647 				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1648 				pcie_capability_write_word(adev->pdev,
1649 							   PCI_EXP_LNKCTL,
1650 							   tmp16);
1651 
1652 				/* linkctl2 */
1653 				pcie_capability_read_word(root, PCI_EXP_LNKCTL2,
1654 							  &tmp16);
1655 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1656 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1657 				tmp16 |= (bridge_cfg2 &
1658 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1659 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1660 				pcie_capability_write_word(root,
1661 							   PCI_EXP_LNKCTL2,
1662 							   tmp16);
1663 
1664 				pcie_capability_read_word(adev->pdev,
1665 							  PCI_EXP_LNKCTL2,
1666 							  &tmp16);
1667 				tmp16 &= ~(PCI_EXP_LNKCTL2_ENTER_COMP |
1668 					   PCI_EXP_LNKCTL2_TX_MARGIN);
1669 				tmp16 |= (gpu_cfg2 &
1670 					  (PCI_EXP_LNKCTL2_ENTER_COMP |
1671 					   PCI_EXP_LNKCTL2_TX_MARGIN));
1672 				pcie_capability_write_word(adev->pdev,
1673 							   PCI_EXP_LNKCTL2,
1674 							   tmp16);
1675 
1676 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1677 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1678 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1679 			}
1680 		}
1681 	}
1682 
1683 	/* set the link speed */
1684 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1685 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1686 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1687 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1688 
1689 	pcie_capability_read_word(adev->pdev, PCI_EXP_LNKCTL2, &tmp16);
1690 	tmp16 &= ~PCI_EXP_LNKCTL2_TLS;
1691 
1692 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1693 		tmp16 |= PCI_EXP_LNKCTL2_TLS_8_0GT; /* gen3 */
1694 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1695 		tmp16 |= PCI_EXP_LNKCTL2_TLS_5_0GT; /* gen2 */
1696 	else
1697 		tmp16 |= PCI_EXP_LNKCTL2_TLS_2_5GT; /* gen1 */
1698 	pcie_capability_write_word(adev->pdev, PCI_EXP_LNKCTL2, tmp16);
1699 
1700 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1701 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1702 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1703 
1704 	for (i = 0; i < adev->usec_timeout; i++) {
1705 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1706 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1707 			break;
1708 		udelay(1);
1709 	}
1710 }
1711 
1712 static void cik_program_aspm(struct amdgpu_device *adev)
1713 {
1714 	u32 data, orig;
1715 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1716 	bool disable_clkreq = false;
1717 
1718 	if (amdgpu_aspm == 0)
1719 		return;
1720 
1721 	if (pci_is_root_bus(adev->pdev->bus))
1722 		return;
1723 
1724 	/* XXX double check APUs */
1725 	if (adev->flags & AMD_IS_APU)
1726 		return;
1727 
1728 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1729 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1730 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1731 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1732 	if (orig != data)
1733 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1734 
1735 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1736 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1737 	if (orig != data)
1738 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1739 
1740 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1741 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1742 	if (orig != data)
1743 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1744 
1745 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1746 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1747 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1748 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1749 	if (!disable_l0s)
1750 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1751 
1752 	if (!disable_l1) {
1753 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1754 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1755 		if (orig != data)
1756 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1757 
1758 		if (!disable_plloff_in_l1) {
1759 			bool clk_req_support;
1760 
1761 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1762 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1763 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1764 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1765 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1766 			if (orig != data)
1767 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1768 
1769 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1770 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1771 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1772 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1773 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1774 			if (orig != data)
1775 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1776 
1777 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1778 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1779 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1780 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1781 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1782 			if (orig != data)
1783 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1784 
1785 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1786 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1787 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1788 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1789 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1790 			if (orig != data)
1791 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1792 
1793 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1794 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1795 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1796 			if (orig != data)
1797 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1798 
1799 			if (!disable_clkreq) {
1800 				struct pci_dev *root = adev->pdev->bus->self;
1801 				u32 lnkcap;
1802 
1803 				clk_req_support = false;
1804 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1805 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1806 					clk_req_support = true;
1807 			} else {
1808 				clk_req_support = false;
1809 			}
1810 
1811 			if (clk_req_support) {
1812 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1813 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1814 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1815 				if (orig != data)
1816 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1817 
1818 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1819 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1820 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1821 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1822 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1823 				if (orig != data)
1824 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1825 
1826 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1827 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1828 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1829 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1830 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1831 				if (orig != data)
1832 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1833 
1834 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1835 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1836 				if (orig != data)
1837 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1838 
1839 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1840 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1841 				if (orig != data)
1842 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1843 
1844 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1845 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1846 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1847 				if (orig != data)
1848 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1849 			}
1850 		}
1851 	} else {
1852 		if (orig != data)
1853 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1854 	}
1855 
1856 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1857 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1858 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1859 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1860 	if (orig != data)
1861 		WREG32_PCIE(ixPCIE_CNTL2, data);
1862 
1863 	if (!disable_l0s) {
1864 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1865 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1866 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1867 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1868 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1869 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1870 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1871 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1872 				if (orig != data)
1873 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1874 			}
1875 		}
1876 	}
1877 }
1878 
1879 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1880 {
1881 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1882 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1883 }
1884 
1885 static void cik_flush_hdp(struct amdgpu_device *adev, struct amdgpu_ring *ring)
1886 {
1887 	if (!ring || !ring->funcs->emit_wreg) {
1888 		WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1889 		RREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL);
1890 	} else {
1891 		amdgpu_ring_emit_wreg(ring, mmHDP_MEM_COHERENCY_FLUSH_CNTL, 1);
1892 	}
1893 }
1894 
1895 static void cik_invalidate_hdp(struct amdgpu_device *adev,
1896 			       struct amdgpu_ring *ring)
1897 {
1898 	if (!ring || !ring->funcs->emit_wreg) {
1899 		WREG32(mmHDP_DEBUG0, 1);
1900 		RREG32(mmHDP_DEBUG0);
1901 	} else {
1902 		amdgpu_ring_emit_wreg(ring, mmHDP_DEBUG0, 1);
1903 	}
1904 }
1905 
1906 static bool cik_need_full_reset(struct amdgpu_device *adev)
1907 {
1908 	/* change this when we support soft reset */
1909 	return true;
1910 }
1911 
1912 static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0,
1913 			       uint64_t *count1)
1914 {
1915 	uint32_t perfctr = 0;
1916 	uint64_t cnt0_of, cnt1_of;
1917 	int tmp;
1918 
1919 	/* This reports 0 on APUs, so return to avoid writing/reading registers
1920 	 * that may or may not be different from their GPU counterparts
1921 	 */
1922 	if (adev->flags & AMD_IS_APU)
1923 		return;
1924 
1925 	/* Set the 2 events that we wish to watch, defined above */
1926 	/* Reg 40 is # received msgs, Reg 104 is # of posted requests sent */
1927 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT0_SEL, 40);
1928 	perfctr = REG_SET_FIELD(perfctr, PCIE_PERF_CNTL_TXCLK, EVENT1_SEL, 104);
1929 
1930 	/* Write to enable desired perf counters */
1931 	WREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK, perfctr);
1932 	/* Zero out and enable the perf counters
1933 	 * Write 0x5:
1934 	 * Bit 0 = Start all counters(1)
1935 	 * Bit 2 = Global counter reset enable(1)
1936 	 */
1937 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000005);
1938 
1939 	msleep(1000);
1940 
1941 	/* Load the shadow and disable the perf counters
1942 	 * Write 0x2:
1943 	 * Bit 0 = Stop counters(0)
1944 	 * Bit 1 = Load the shadow counters(1)
1945 	 */
1946 	WREG32_PCIE(ixPCIE_PERF_COUNT_CNTL, 0x00000002);
1947 
1948 	/* Read register values to get any >32bit overflow */
1949 	tmp = RREG32_PCIE(ixPCIE_PERF_CNTL_TXCLK);
1950 	cnt0_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER0_UPPER);
1951 	cnt1_of = REG_GET_FIELD(tmp, PCIE_PERF_CNTL_TXCLK, COUNTER1_UPPER);
1952 
1953 	/* Get the values and add the overflow */
1954 	*count0 = RREG32_PCIE(ixPCIE_PERF_COUNT0_TXCLK) | (cnt0_of << 32);
1955 	*count1 = RREG32_PCIE(ixPCIE_PERF_COUNT1_TXCLK) | (cnt1_of << 32);
1956 }
1957 
1958 static bool cik_need_reset_on_init(struct amdgpu_device *adev)
1959 {
1960 	u32 clock_cntl, pc;
1961 
1962 	if (adev->flags & AMD_IS_APU)
1963 		return false;
1964 
1965 	/* check if the SMC is already running */
1966 	clock_cntl = RREG32_SMC(ixSMC_SYSCON_CLOCK_CNTL_0);
1967 	pc = RREG32_SMC(ixSMC_PC_C);
1968 	if ((0 == REG_GET_FIELD(clock_cntl, SMC_SYSCON_CLOCK_CNTL_0, ck_disable)) &&
1969 	    (0x20100 <= pc))
1970 		return true;
1971 
1972 	return false;
1973 }
1974 
1975 static uint64_t cik_get_pcie_replay_count(struct amdgpu_device *adev)
1976 {
1977 	uint64_t nak_r, nak_g;
1978 
1979 	/* Get the number of NAKs received and generated */
1980 	nak_r = RREG32_PCIE(ixPCIE_RX_NUM_NAK);
1981 	nak_g = RREG32_PCIE(ixPCIE_RX_NUM_NAK_GENERATED);
1982 
1983 	/* Add the total number of NAKs, i.e the number of replays */
1984 	return (nak_r + nak_g);
1985 }
1986 
1987 static void cik_pre_asic_init(struct amdgpu_device *adev)
1988 {
1989 }
1990 
1991 static const struct amdgpu_asic_funcs cik_asic_funcs =
1992 {
1993 	.read_disabled_bios = &cik_read_disabled_bios,
1994 	.read_bios_from_rom = &cik_read_bios_from_rom,
1995 	.read_register = &cik_read_register,
1996 	.reset = &cik_asic_reset,
1997 	.reset_method = &cik_asic_reset_method,
1998 	.set_vga_state = &cik_vga_set_state,
1999 	.get_xclk = &cik_get_xclk,
2000 	.set_uvd_clocks = &cik_set_uvd_clocks,
2001 	.set_vce_clocks = &cik_set_vce_clocks,
2002 	.get_config_memsize = &cik_get_config_memsize,
2003 	.flush_hdp = &cik_flush_hdp,
2004 	.invalidate_hdp = &cik_invalidate_hdp,
2005 	.need_full_reset = &cik_need_full_reset,
2006 	.init_doorbell_index = &legacy_doorbell_index_init,
2007 	.get_pcie_usage = &cik_get_pcie_usage,
2008 	.need_reset_on_init = &cik_need_reset_on_init,
2009 	.get_pcie_replay_count = &cik_get_pcie_replay_count,
2010 	.supports_baco = &cik_asic_supports_baco,
2011 	.pre_asic_init = &cik_pre_asic_init,
2012 	.query_video_codecs = &cik_query_video_codecs,
2013 };
2014 
2015 static int cik_common_early_init(void *handle)
2016 {
2017 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2018 
2019 	adev->smc_rreg = &cik_smc_rreg;
2020 	adev->smc_wreg = &cik_smc_wreg;
2021 	adev->pcie_rreg = &cik_pcie_rreg;
2022 	adev->pcie_wreg = &cik_pcie_wreg;
2023 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2024 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2025 	adev->didt_rreg = &cik_didt_rreg;
2026 	adev->didt_wreg = &cik_didt_wreg;
2027 
2028 	adev->asic_funcs = &cik_asic_funcs;
2029 
2030 	adev->rev_id = cik_get_rev_id(adev);
2031 	adev->external_rev_id = 0xFF;
2032 	switch (adev->asic_type) {
2033 	case CHIP_BONAIRE:
2034 		adev->cg_flags =
2035 			AMD_CG_SUPPORT_GFX_MGCG |
2036 			AMD_CG_SUPPORT_GFX_MGLS |
2037 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2038 			AMD_CG_SUPPORT_GFX_CGLS |
2039 			AMD_CG_SUPPORT_GFX_CGTS |
2040 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2041 			AMD_CG_SUPPORT_GFX_CP_LS |
2042 			AMD_CG_SUPPORT_MC_LS |
2043 			AMD_CG_SUPPORT_MC_MGCG |
2044 			AMD_CG_SUPPORT_SDMA_MGCG |
2045 			AMD_CG_SUPPORT_SDMA_LS |
2046 			AMD_CG_SUPPORT_BIF_LS |
2047 			AMD_CG_SUPPORT_VCE_MGCG |
2048 			AMD_CG_SUPPORT_UVD_MGCG |
2049 			AMD_CG_SUPPORT_HDP_LS |
2050 			AMD_CG_SUPPORT_HDP_MGCG;
2051 		adev->pg_flags = 0;
2052 		adev->external_rev_id = adev->rev_id + 0x14;
2053 		break;
2054 	case CHIP_HAWAII:
2055 		adev->cg_flags =
2056 			AMD_CG_SUPPORT_GFX_MGCG |
2057 			AMD_CG_SUPPORT_GFX_MGLS |
2058 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2059 			AMD_CG_SUPPORT_GFX_CGLS |
2060 			AMD_CG_SUPPORT_GFX_CGTS |
2061 			AMD_CG_SUPPORT_GFX_CP_LS |
2062 			AMD_CG_SUPPORT_MC_LS |
2063 			AMD_CG_SUPPORT_MC_MGCG |
2064 			AMD_CG_SUPPORT_SDMA_MGCG |
2065 			AMD_CG_SUPPORT_SDMA_LS |
2066 			AMD_CG_SUPPORT_BIF_LS |
2067 			AMD_CG_SUPPORT_VCE_MGCG |
2068 			AMD_CG_SUPPORT_UVD_MGCG |
2069 			AMD_CG_SUPPORT_HDP_LS |
2070 			AMD_CG_SUPPORT_HDP_MGCG;
2071 		adev->pg_flags = 0;
2072 		adev->external_rev_id = 0x28;
2073 		break;
2074 	case CHIP_KAVERI:
2075 		adev->cg_flags =
2076 			AMD_CG_SUPPORT_GFX_MGCG |
2077 			AMD_CG_SUPPORT_GFX_MGLS |
2078 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2079 			AMD_CG_SUPPORT_GFX_CGLS |
2080 			AMD_CG_SUPPORT_GFX_CGTS |
2081 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2082 			AMD_CG_SUPPORT_GFX_CP_LS |
2083 			AMD_CG_SUPPORT_SDMA_MGCG |
2084 			AMD_CG_SUPPORT_SDMA_LS |
2085 			AMD_CG_SUPPORT_BIF_LS |
2086 			AMD_CG_SUPPORT_VCE_MGCG |
2087 			AMD_CG_SUPPORT_UVD_MGCG |
2088 			AMD_CG_SUPPORT_HDP_LS |
2089 			AMD_CG_SUPPORT_HDP_MGCG;
2090 		adev->pg_flags =
2091 			/*AMD_PG_SUPPORT_GFX_PG |
2092 			  AMD_PG_SUPPORT_GFX_SMG |
2093 			  AMD_PG_SUPPORT_GFX_DMG |*/
2094 			AMD_PG_SUPPORT_UVD |
2095 			AMD_PG_SUPPORT_VCE |
2096 			/*  AMD_PG_SUPPORT_CP |
2097 			  AMD_PG_SUPPORT_GDS |
2098 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2099 			  AMD_PG_SUPPORT_ACP |
2100 			  AMD_PG_SUPPORT_SAMU |*/
2101 			0;
2102 		if (adev->pdev->device == 0x1312 ||
2103 			adev->pdev->device == 0x1316 ||
2104 			adev->pdev->device == 0x1317)
2105 			adev->external_rev_id = 0x41;
2106 		else
2107 			adev->external_rev_id = 0x1;
2108 		break;
2109 	case CHIP_KABINI:
2110 	case CHIP_MULLINS:
2111 		adev->cg_flags =
2112 			AMD_CG_SUPPORT_GFX_MGCG |
2113 			AMD_CG_SUPPORT_GFX_MGLS |
2114 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2115 			AMD_CG_SUPPORT_GFX_CGLS |
2116 			AMD_CG_SUPPORT_GFX_CGTS |
2117 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2118 			AMD_CG_SUPPORT_GFX_CP_LS |
2119 			AMD_CG_SUPPORT_SDMA_MGCG |
2120 			AMD_CG_SUPPORT_SDMA_LS |
2121 			AMD_CG_SUPPORT_BIF_LS |
2122 			AMD_CG_SUPPORT_VCE_MGCG |
2123 			AMD_CG_SUPPORT_UVD_MGCG |
2124 			AMD_CG_SUPPORT_HDP_LS |
2125 			AMD_CG_SUPPORT_HDP_MGCG;
2126 		adev->pg_flags =
2127 			/*AMD_PG_SUPPORT_GFX_PG |
2128 			  AMD_PG_SUPPORT_GFX_SMG | */
2129 			AMD_PG_SUPPORT_UVD |
2130 			/*AMD_PG_SUPPORT_VCE |
2131 			  AMD_PG_SUPPORT_CP |
2132 			  AMD_PG_SUPPORT_GDS |
2133 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2134 			  AMD_PG_SUPPORT_SAMU |*/
2135 			0;
2136 		if (adev->asic_type == CHIP_KABINI) {
2137 			if (adev->rev_id == 0)
2138 				adev->external_rev_id = 0x81;
2139 			else if (adev->rev_id == 1)
2140 				adev->external_rev_id = 0x82;
2141 			else if (adev->rev_id == 2)
2142 				adev->external_rev_id = 0x85;
2143 		} else
2144 			adev->external_rev_id = adev->rev_id + 0xa1;
2145 		break;
2146 	default:
2147 		/* FIXME: not supported yet */
2148 		return -EINVAL;
2149 	}
2150 
2151 	return 0;
2152 }
2153 
2154 static int cik_common_sw_init(void *handle)
2155 {
2156 	return 0;
2157 }
2158 
2159 static int cik_common_sw_fini(void *handle)
2160 {
2161 	return 0;
2162 }
2163 
2164 static int cik_common_hw_init(void *handle)
2165 {
2166 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2167 
2168 	/* move the golden regs per IP block */
2169 	cik_init_golden_registers(adev);
2170 	/* enable pcie gen2/3 link */
2171 	cik_pcie_gen3_enable(adev);
2172 	/* enable aspm */
2173 	cik_program_aspm(adev);
2174 
2175 	return 0;
2176 }
2177 
2178 static int cik_common_hw_fini(void *handle)
2179 {
2180 	return 0;
2181 }
2182 
2183 static int cik_common_suspend(void *handle)
2184 {
2185 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2186 
2187 	return cik_common_hw_fini(adev);
2188 }
2189 
2190 static int cik_common_resume(void *handle)
2191 {
2192 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2193 
2194 	return cik_common_hw_init(adev);
2195 }
2196 
2197 static bool cik_common_is_idle(void *handle)
2198 {
2199 	return true;
2200 }
2201 
2202 static int cik_common_wait_for_idle(void *handle)
2203 {
2204 	return 0;
2205 }
2206 
2207 static int cik_common_soft_reset(void *handle)
2208 {
2209 	/* XXX hard reset?? */
2210 	return 0;
2211 }
2212 
2213 static int cik_common_set_clockgating_state(void *handle,
2214 					    enum amd_clockgating_state state)
2215 {
2216 	return 0;
2217 }
2218 
2219 static int cik_common_set_powergating_state(void *handle,
2220 					    enum amd_powergating_state state)
2221 {
2222 	return 0;
2223 }
2224 
2225 static const struct amd_ip_funcs cik_common_ip_funcs = {
2226 	.name = "cik_common",
2227 	.early_init = cik_common_early_init,
2228 	.late_init = NULL,
2229 	.sw_init = cik_common_sw_init,
2230 	.sw_fini = cik_common_sw_fini,
2231 	.hw_init = cik_common_hw_init,
2232 	.hw_fini = cik_common_hw_fini,
2233 	.suspend = cik_common_suspend,
2234 	.resume = cik_common_resume,
2235 	.is_idle = cik_common_is_idle,
2236 	.wait_for_idle = cik_common_wait_for_idle,
2237 	.soft_reset = cik_common_soft_reset,
2238 	.set_clockgating_state = cik_common_set_clockgating_state,
2239 	.set_powergating_state = cik_common_set_powergating_state,
2240 };
2241 
2242 static const struct amdgpu_ip_block_version cik_common_ip_block =
2243 {
2244 	.type = AMD_IP_BLOCK_TYPE_COMMON,
2245 	.major = 1,
2246 	.minor = 0,
2247 	.rev = 0,
2248 	.funcs = &cik_common_ip_funcs,
2249 };
2250 
2251 int cik_set_ip_blocks(struct amdgpu_device *adev)
2252 {
2253 	switch (adev->asic_type) {
2254 	case CHIP_BONAIRE:
2255 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2256 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2257 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2258 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2259 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2260 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2261 		if (adev->enable_virtual_display)
2262 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2263 #if defined(CONFIG_DRM_AMD_DC)
2264 		else if (amdgpu_device_has_dc_support(adev))
2265 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2266 #endif
2267 		else
2268 			amdgpu_device_ip_block_add(adev, &dce_v8_2_ip_block);
2269 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2270 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2271 		break;
2272 	case CHIP_HAWAII:
2273 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2274 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2275 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2276 		amdgpu_device_ip_block_add(adev, &gfx_v7_3_ip_block);
2277 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2278 		amdgpu_device_ip_block_add(adev, &pp_smu_ip_block);
2279 		if (adev->enable_virtual_display)
2280 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2281 #if defined(CONFIG_DRM_AMD_DC)
2282 		else if (amdgpu_device_has_dc_support(adev))
2283 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2284 #endif
2285 		else
2286 			amdgpu_device_ip_block_add(adev, &dce_v8_5_ip_block);
2287 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2288 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2289 		break;
2290 	case CHIP_KAVERI:
2291 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2292 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2293 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2294 		amdgpu_device_ip_block_add(adev, &gfx_v7_1_ip_block);
2295 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2296 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2297 		if (adev->enable_virtual_display)
2298 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2299 #if defined(CONFIG_DRM_AMD_DC)
2300 		else if (amdgpu_device_has_dc_support(adev))
2301 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2302 #endif
2303 		else
2304 			amdgpu_device_ip_block_add(adev, &dce_v8_1_ip_block);
2305 
2306 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2307 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2308 		break;
2309 	case CHIP_KABINI:
2310 	case CHIP_MULLINS:
2311 		amdgpu_device_ip_block_add(adev, &cik_common_ip_block);
2312 		amdgpu_device_ip_block_add(adev, &gmc_v7_0_ip_block);
2313 		amdgpu_device_ip_block_add(adev, &cik_ih_ip_block);
2314 		amdgpu_device_ip_block_add(adev, &gfx_v7_2_ip_block);
2315 		amdgpu_device_ip_block_add(adev, &cik_sdma_ip_block);
2316 		amdgpu_device_ip_block_add(adev, &kv_smu_ip_block);
2317 		if (adev->enable_virtual_display)
2318 			amdgpu_device_ip_block_add(adev, &amdgpu_vkms_ip_block);
2319 #if defined(CONFIG_DRM_AMD_DC)
2320 		else if (amdgpu_device_has_dc_support(adev))
2321 			amdgpu_device_ip_block_add(adev, &dm_ip_block);
2322 #endif
2323 		else
2324 			amdgpu_device_ip_block_add(adev, &dce_v8_3_ip_block);
2325 		amdgpu_device_ip_block_add(adev, &uvd_v4_2_ip_block);
2326 		amdgpu_device_ip_block_add(adev, &vce_v2_0_ip_block);
2327 		break;
2328 	default:
2329 		/* FIXME: not supported yet */
2330 		return -EINVAL;
2331 	}
2332 	return 0;
2333 }
2334