xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision 23c2b932) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "amdgpu.h"
29 #include "amdgpu_atombios.h"
30 #include "amdgpu_ih.h"
31 #include "amdgpu_uvd.h"
32 #include "amdgpu_vce.h"
33 #include "cikd.h"
34 #include "atom.h"
35 #include "amd_pcie.h"
36 
37 #include "cik.h"
38 #include "gmc_v7_0.h"
39 #include "cik_ih.h"
40 #include "dce_v8_0.h"
41 #include "gfx_v7_0.h"
42 #include "cik_sdma.h"
43 #include "uvd_v4_2.h"
44 #include "vce_v2_0.h"
45 #include "cik_dpm.h"
46 
47 #include "uvd/uvd_4_2_d.h"
48 
49 #include "smu/smu_7_0_1_d.h"
50 #include "smu/smu_7_0_1_sh_mask.h"
51 
52 #include "dce/dce_8_0_d.h"
53 #include "dce/dce_8_0_sh_mask.h"
54 
55 #include "bif/bif_4_1_d.h"
56 #include "bif/bif_4_1_sh_mask.h"
57 
58 #include "gca/gfx_7_2_d.h"
59 #include "gca/gfx_7_2_enum.h"
60 #include "gca/gfx_7_2_sh_mask.h"
61 
62 #include "gmc/gmc_7_1_d.h"
63 #include "gmc/gmc_7_1_sh_mask.h"
64 
65 #include "oss/oss_2_0_d.h"
66 #include "oss/oss_2_0_sh_mask.h"
67 
68 #include "amdgpu_amdkfd.h"
69 #include "amdgpu_powerplay.h"
70 
71 /*
72  * Indirect registers accessor
73  */
74 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
75 {
76 	unsigned long flags;
77 	u32 r;
78 
79 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
80 	WREG32(mmPCIE_INDEX, reg);
81 	(void)RREG32(mmPCIE_INDEX);
82 	r = RREG32(mmPCIE_DATA);
83 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
84 	return r;
85 }
86 
87 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
88 {
89 	unsigned long flags;
90 
91 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
92 	WREG32(mmPCIE_INDEX, reg);
93 	(void)RREG32(mmPCIE_INDEX);
94 	WREG32(mmPCIE_DATA, v);
95 	(void)RREG32(mmPCIE_DATA);
96 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
97 }
98 
99 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
100 {
101 	unsigned long flags;
102 	u32 r;
103 
104 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
105 	WREG32(mmSMC_IND_INDEX_0, (reg));
106 	r = RREG32(mmSMC_IND_DATA_0);
107 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
108 	return r;
109 }
110 
111 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
112 {
113 	unsigned long flags;
114 
115 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
116 	WREG32(mmSMC_IND_INDEX_0, (reg));
117 	WREG32(mmSMC_IND_DATA_0, (v));
118 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
119 }
120 
121 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
122 {
123 	unsigned long flags;
124 	u32 r;
125 
126 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
127 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
128 	r = RREG32(mmUVD_CTX_DATA);
129 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
130 	return r;
131 }
132 
133 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
134 {
135 	unsigned long flags;
136 
137 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
138 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
139 	WREG32(mmUVD_CTX_DATA, (v));
140 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
141 }
142 
143 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
144 {
145 	unsigned long flags;
146 	u32 r;
147 
148 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
149 	WREG32(mmDIDT_IND_INDEX, (reg));
150 	r = RREG32(mmDIDT_IND_DATA);
151 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
152 	return r;
153 }
154 
155 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
156 {
157 	unsigned long flags;
158 
159 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
160 	WREG32(mmDIDT_IND_INDEX, (reg));
161 	WREG32(mmDIDT_IND_DATA, (v));
162 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
163 }
164 
165 static const u32 bonaire_golden_spm_registers[] =
166 {
167 	0xc200, 0xe0ffffff, 0xe0000000
168 };
169 
170 static const u32 bonaire_golden_common_registers[] =
171 {
172 	0x31dc, 0xffffffff, 0x00000800,
173 	0x31dd, 0xffffffff, 0x00000800,
174 	0x31e6, 0xffffffff, 0x00007fbf,
175 	0x31e7, 0xffffffff, 0x00007faf
176 };
177 
178 static const u32 bonaire_golden_registers[] =
179 {
180 	0xcd5, 0x00000333, 0x00000333,
181 	0xcd4, 0x000c0fc0, 0x00040200,
182 	0x2684, 0x00010000, 0x00058208,
183 	0xf000, 0xffff1fff, 0x00140000,
184 	0xf080, 0xfdfc0fff, 0x00000100,
185 	0xf08d, 0x40000000, 0x40000200,
186 	0x260c, 0xffffffff, 0x00000000,
187 	0x260d, 0xf00fffff, 0x00000400,
188 	0x260e, 0x0002021c, 0x00020200,
189 	0x31e, 0x00000080, 0x00000000,
190 	0x16ec, 0x000000f0, 0x00000070,
191 	0x16f0, 0xf0311fff, 0x80300000,
192 	0x263e, 0x73773777, 0x12010001,
193 	0xd43, 0x00810000, 0x408af000,
194 	0x1c0c, 0x31000111, 0x00000011,
195 	0xbd2, 0x73773777, 0x12010001,
196 	0x883, 0x00007fb6, 0x0021a1b1,
197 	0x884, 0x00007fb6, 0x002021b1,
198 	0x860, 0x00007fb6, 0x00002191,
199 	0x886, 0x00007fb6, 0x002121b1,
200 	0x887, 0x00007fb6, 0x002021b1,
201 	0x877, 0x00007fb6, 0x00002191,
202 	0x878, 0x00007fb6, 0x00002191,
203 	0xd8a, 0x0000003f, 0x0000000a,
204 	0xd8b, 0x0000003f, 0x0000000a,
205 	0xab9, 0x00073ffe, 0x000022a2,
206 	0x903, 0x000007ff, 0x00000000,
207 	0x2285, 0xf000003f, 0x00000007,
208 	0x22fc, 0x00002001, 0x00000001,
209 	0x22c9, 0xffffffff, 0x00ffffff,
210 	0xc281, 0x0000ff0f, 0x00000000,
211 	0xa293, 0x07ffffff, 0x06000000,
212 	0x136, 0x00000fff, 0x00000100,
213 	0xf9e, 0x00000001, 0x00000002,
214 	0x2440, 0x03000000, 0x0362c688,
215 	0x2300, 0x000000ff, 0x00000001,
216 	0x390, 0x00001fff, 0x00001fff,
217 	0x2418, 0x0000007f, 0x00000020,
218 	0x2542, 0x00010000, 0x00010000,
219 	0x2b05, 0x000003ff, 0x000000f3,
220 	0x2b03, 0xffffffff, 0x00001032
221 };
222 
223 static const u32 bonaire_mgcg_cgcg_init[] =
224 {
225 	0x3108, 0xffffffff, 0xfffffffc,
226 	0xc200, 0xffffffff, 0xe0000000,
227 	0xf0a8, 0xffffffff, 0x00000100,
228 	0xf082, 0xffffffff, 0x00000100,
229 	0xf0b0, 0xffffffff, 0xc0000100,
230 	0xf0b2, 0xffffffff, 0xc0000100,
231 	0xf0b1, 0xffffffff, 0xc0000100,
232 	0x1579, 0xffffffff, 0x00600100,
233 	0xf0a0, 0xffffffff, 0x00000100,
234 	0xf085, 0xffffffff, 0x06000100,
235 	0xf088, 0xffffffff, 0x00000100,
236 	0xf086, 0xffffffff, 0x06000100,
237 	0xf081, 0xffffffff, 0x00000100,
238 	0xf0b8, 0xffffffff, 0x00000100,
239 	0xf089, 0xffffffff, 0x00000100,
240 	0xf080, 0xffffffff, 0x00000100,
241 	0xf08c, 0xffffffff, 0x00000100,
242 	0xf08d, 0xffffffff, 0x00000100,
243 	0xf094, 0xffffffff, 0x00000100,
244 	0xf095, 0xffffffff, 0x00000100,
245 	0xf096, 0xffffffff, 0x00000100,
246 	0xf097, 0xffffffff, 0x00000100,
247 	0xf098, 0xffffffff, 0x00000100,
248 	0xf09f, 0xffffffff, 0x00000100,
249 	0xf09e, 0xffffffff, 0x00000100,
250 	0xf084, 0xffffffff, 0x06000100,
251 	0xf0a4, 0xffffffff, 0x00000100,
252 	0xf09d, 0xffffffff, 0x00000100,
253 	0xf0ad, 0xffffffff, 0x00000100,
254 	0xf0ac, 0xffffffff, 0x00000100,
255 	0xf09c, 0xffffffff, 0x00000100,
256 	0xc200, 0xffffffff, 0xe0000000,
257 	0xf008, 0xffffffff, 0x00010000,
258 	0xf009, 0xffffffff, 0x00030002,
259 	0xf00a, 0xffffffff, 0x00040007,
260 	0xf00b, 0xffffffff, 0x00060005,
261 	0xf00c, 0xffffffff, 0x00090008,
262 	0xf00d, 0xffffffff, 0x00010000,
263 	0xf00e, 0xffffffff, 0x00030002,
264 	0xf00f, 0xffffffff, 0x00040007,
265 	0xf010, 0xffffffff, 0x00060005,
266 	0xf011, 0xffffffff, 0x00090008,
267 	0xf012, 0xffffffff, 0x00010000,
268 	0xf013, 0xffffffff, 0x00030002,
269 	0xf014, 0xffffffff, 0x00040007,
270 	0xf015, 0xffffffff, 0x00060005,
271 	0xf016, 0xffffffff, 0x00090008,
272 	0xf017, 0xffffffff, 0x00010000,
273 	0xf018, 0xffffffff, 0x00030002,
274 	0xf019, 0xffffffff, 0x00040007,
275 	0xf01a, 0xffffffff, 0x00060005,
276 	0xf01b, 0xffffffff, 0x00090008,
277 	0xf01c, 0xffffffff, 0x00010000,
278 	0xf01d, 0xffffffff, 0x00030002,
279 	0xf01e, 0xffffffff, 0x00040007,
280 	0xf01f, 0xffffffff, 0x00060005,
281 	0xf020, 0xffffffff, 0x00090008,
282 	0xf021, 0xffffffff, 0x00010000,
283 	0xf022, 0xffffffff, 0x00030002,
284 	0xf023, 0xffffffff, 0x00040007,
285 	0xf024, 0xffffffff, 0x00060005,
286 	0xf025, 0xffffffff, 0x00090008,
287 	0xf026, 0xffffffff, 0x00010000,
288 	0xf027, 0xffffffff, 0x00030002,
289 	0xf028, 0xffffffff, 0x00040007,
290 	0xf029, 0xffffffff, 0x00060005,
291 	0xf02a, 0xffffffff, 0x00090008,
292 	0xf000, 0xffffffff, 0x96e00200,
293 	0x21c2, 0xffffffff, 0x00900100,
294 	0x3109, 0xffffffff, 0x0020003f,
295 	0xe, 0xffffffff, 0x0140001c,
296 	0xf, 0x000f0000, 0x000f0000,
297 	0x88, 0xffffffff, 0xc060000c,
298 	0x89, 0xc0000fff, 0x00000100,
299 	0x3e4, 0xffffffff, 0x00000100,
300 	0x3e6, 0x00000101, 0x00000000,
301 	0x82a, 0xffffffff, 0x00000104,
302 	0x1579, 0xff000fff, 0x00000100,
303 	0xc33, 0xc0000fff, 0x00000104,
304 	0x3079, 0x00000001, 0x00000001,
305 	0x3403, 0xff000ff0, 0x00000100,
306 	0x3603, 0xff000ff0, 0x00000100
307 };
308 
309 static const u32 spectre_golden_spm_registers[] =
310 {
311 	0xc200, 0xe0ffffff, 0xe0000000
312 };
313 
314 static const u32 spectre_golden_common_registers[] =
315 {
316 	0x31dc, 0xffffffff, 0x00000800,
317 	0x31dd, 0xffffffff, 0x00000800,
318 	0x31e6, 0xffffffff, 0x00007fbf,
319 	0x31e7, 0xffffffff, 0x00007faf
320 };
321 
322 static const u32 spectre_golden_registers[] =
323 {
324 	0xf000, 0xffff1fff, 0x96940200,
325 	0xf003, 0xffff0001, 0xff000000,
326 	0xf080, 0xfffc0fff, 0x00000100,
327 	0x1bb6, 0x00010101, 0x00010000,
328 	0x260d, 0xf00fffff, 0x00000400,
329 	0x260e, 0xfffffffc, 0x00020200,
330 	0x16ec, 0x000000f0, 0x00000070,
331 	0x16f0, 0xf0311fff, 0x80300000,
332 	0x263e, 0x73773777, 0x12010001,
333 	0x26df, 0x00ff0000, 0x00fc0000,
334 	0xbd2, 0x73773777, 0x12010001,
335 	0x2285, 0xf000003f, 0x00000007,
336 	0x22c9, 0xffffffff, 0x00ffffff,
337 	0xa0d4, 0x3f3f3fff, 0x00000082,
338 	0xa0d5, 0x0000003f, 0x00000000,
339 	0xf9e, 0x00000001, 0x00000002,
340 	0x244f, 0xffff03df, 0x00000004,
341 	0x31da, 0x00000008, 0x00000008,
342 	0x2300, 0x000008ff, 0x00000800,
343 	0x2542, 0x00010000, 0x00010000,
344 	0x2b03, 0xffffffff, 0x54763210,
345 	0x853e, 0x01ff01ff, 0x00000002,
346 	0x8526, 0x007ff800, 0x00200000,
347 	0x8057, 0xffffffff, 0x00000f40,
348 	0xc24d, 0xffffffff, 0x00000001
349 };
350 
351 static const u32 spectre_mgcg_cgcg_init[] =
352 {
353 	0x3108, 0xffffffff, 0xfffffffc,
354 	0xc200, 0xffffffff, 0xe0000000,
355 	0xf0a8, 0xffffffff, 0x00000100,
356 	0xf082, 0xffffffff, 0x00000100,
357 	0xf0b0, 0xffffffff, 0x00000100,
358 	0xf0b2, 0xffffffff, 0x00000100,
359 	0xf0b1, 0xffffffff, 0x00000100,
360 	0x1579, 0xffffffff, 0x00600100,
361 	0xf0a0, 0xffffffff, 0x00000100,
362 	0xf085, 0xffffffff, 0x06000100,
363 	0xf088, 0xffffffff, 0x00000100,
364 	0xf086, 0xffffffff, 0x06000100,
365 	0xf081, 0xffffffff, 0x00000100,
366 	0xf0b8, 0xffffffff, 0x00000100,
367 	0xf089, 0xffffffff, 0x00000100,
368 	0xf080, 0xffffffff, 0x00000100,
369 	0xf08c, 0xffffffff, 0x00000100,
370 	0xf08d, 0xffffffff, 0x00000100,
371 	0xf094, 0xffffffff, 0x00000100,
372 	0xf095, 0xffffffff, 0x00000100,
373 	0xf096, 0xffffffff, 0x00000100,
374 	0xf097, 0xffffffff, 0x00000100,
375 	0xf098, 0xffffffff, 0x00000100,
376 	0xf09f, 0xffffffff, 0x00000100,
377 	0xf09e, 0xffffffff, 0x00000100,
378 	0xf084, 0xffffffff, 0x06000100,
379 	0xf0a4, 0xffffffff, 0x00000100,
380 	0xf09d, 0xffffffff, 0x00000100,
381 	0xf0ad, 0xffffffff, 0x00000100,
382 	0xf0ac, 0xffffffff, 0x00000100,
383 	0xf09c, 0xffffffff, 0x00000100,
384 	0xc200, 0xffffffff, 0xe0000000,
385 	0xf008, 0xffffffff, 0x00010000,
386 	0xf009, 0xffffffff, 0x00030002,
387 	0xf00a, 0xffffffff, 0x00040007,
388 	0xf00b, 0xffffffff, 0x00060005,
389 	0xf00c, 0xffffffff, 0x00090008,
390 	0xf00d, 0xffffffff, 0x00010000,
391 	0xf00e, 0xffffffff, 0x00030002,
392 	0xf00f, 0xffffffff, 0x00040007,
393 	0xf010, 0xffffffff, 0x00060005,
394 	0xf011, 0xffffffff, 0x00090008,
395 	0xf012, 0xffffffff, 0x00010000,
396 	0xf013, 0xffffffff, 0x00030002,
397 	0xf014, 0xffffffff, 0x00040007,
398 	0xf015, 0xffffffff, 0x00060005,
399 	0xf016, 0xffffffff, 0x00090008,
400 	0xf017, 0xffffffff, 0x00010000,
401 	0xf018, 0xffffffff, 0x00030002,
402 	0xf019, 0xffffffff, 0x00040007,
403 	0xf01a, 0xffffffff, 0x00060005,
404 	0xf01b, 0xffffffff, 0x00090008,
405 	0xf01c, 0xffffffff, 0x00010000,
406 	0xf01d, 0xffffffff, 0x00030002,
407 	0xf01e, 0xffffffff, 0x00040007,
408 	0xf01f, 0xffffffff, 0x00060005,
409 	0xf020, 0xffffffff, 0x00090008,
410 	0xf021, 0xffffffff, 0x00010000,
411 	0xf022, 0xffffffff, 0x00030002,
412 	0xf023, 0xffffffff, 0x00040007,
413 	0xf024, 0xffffffff, 0x00060005,
414 	0xf025, 0xffffffff, 0x00090008,
415 	0xf026, 0xffffffff, 0x00010000,
416 	0xf027, 0xffffffff, 0x00030002,
417 	0xf028, 0xffffffff, 0x00040007,
418 	0xf029, 0xffffffff, 0x00060005,
419 	0xf02a, 0xffffffff, 0x00090008,
420 	0xf02b, 0xffffffff, 0x00010000,
421 	0xf02c, 0xffffffff, 0x00030002,
422 	0xf02d, 0xffffffff, 0x00040007,
423 	0xf02e, 0xffffffff, 0x00060005,
424 	0xf02f, 0xffffffff, 0x00090008,
425 	0xf000, 0xffffffff, 0x96e00200,
426 	0x21c2, 0xffffffff, 0x00900100,
427 	0x3109, 0xffffffff, 0x0020003f,
428 	0xe, 0xffffffff, 0x0140001c,
429 	0xf, 0x000f0000, 0x000f0000,
430 	0x88, 0xffffffff, 0xc060000c,
431 	0x89, 0xc0000fff, 0x00000100,
432 	0x3e4, 0xffffffff, 0x00000100,
433 	0x3e6, 0x00000101, 0x00000000,
434 	0x82a, 0xffffffff, 0x00000104,
435 	0x1579, 0xff000fff, 0x00000100,
436 	0xc33, 0xc0000fff, 0x00000104,
437 	0x3079, 0x00000001, 0x00000001,
438 	0x3403, 0xff000ff0, 0x00000100,
439 	0x3603, 0xff000ff0, 0x00000100
440 };
441 
442 static const u32 kalindi_golden_spm_registers[] =
443 {
444 	0xc200, 0xe0ffffff, 0xe0000000
445 };
446 
447 static const u32 kalindi_golden_common_registers[] =
448 {
449 	0x31dc, 0xffffffff, 0x00000800,
450 	0x31dd, 0xffffffff, 0x00000800,
451 	0x31e6, 0xffffffff, 0x00007fbf,
452 	0x31e7, 0xffffffff, 0x00007faf
453 };
454 
455 static const u32 kalindi_golden_registers[] =
456 {
457 	0xf000, 0xffffdfff, 0x6e944040,
458 	0x1579, 0xff607fff, 0xfc000100,
459 	0xf088, 0xff000fff, 0x00000100,
460 	0xf089, 0xff000fff, 0x00000100,
461 	0xf080, 0xfffc0fff, 0x00000100,
462 	0x1bb6, 0x00010101, 0x00010000,
463 	0x260c, 0xffffffff, 0x00000000,
464 	0x260d, 0xf00fffff, 0x00000400,
465 	0x16ec, 0x000000f0, 0x00000070,
466 	0x16f0, 0xf0311fff, 0x80300000,
467 	0x263e, 0x73773777, 0x12010001,
468 	0x263f, 0xffffffff, 0x00000010,
469 	0x26df, 0x00ff0000, 0x00fc0000,
470 	0x200c, 0x00001f0f, 0x0000100a,
471 	0xbd2, 0x73773777, 0x12010001,
472 	0x902, 0x000fffff, 0x000c007f,
473 	0x2285, 0xf000003f, 0x00000007,
474 	0x22c9, 0x3fff3fff, 0x00ffcfff,
475 	0xc281, 0x0000ff0f, 0x00000000,
476 	0xa293, 0x07ffffff, 0x06000000,
477 	0x136, 0x00000fff, 0x00000100,
478 	0xf9e, 0x00000001, 0x00000002,
479 	0x31da, 0x00000008, 0x00000008,
480 	0x2300, 0x000000ff, 0x00000003,
481 	0x853e, 0x01ff01ff, 0x00000002,
482 	0x8526, 0x007ff800, 0x00200000,
483 	0x8057, 0xffffffff, 0x00000f40,
484 	0x2231, 0x001f3ae3, 0x00000082,
485 	0x2235, 0x0000001f, 0x00000010,
486 	0xc24d, 0xffffffff, 0x00000000
487 };
488 
489 static const u32 kalindi_mgcg_cgcg_init[] =
490 {
491 	0x3108, 0xffffffff, 0xfffffffc,
492 	0xc200, 0xffffffff, 0xe0000000,
493 	0xf0a8, 0xffffffff, 0x00000100,
494 	0xf082, 0xffffffff, 0x00000100,
495 	0xf0b0, 0xffffffff, 0x00000100,
496 	0xf0b2, 0xffffffff, 0x00000100,
497 	0xf0b1, 0xffffffff, 0x00000100,
498 	0x1579, 0xffffffff, 0x00600100,
499 	0xf0a0, 0xffffffff, 0x00000100,
500 	0xf085, 0xffffffff, 0x06000100,
501 	0xf088, 0xffffffff, 0x00000100,
502 	0xf086, 0xffffffff, 0x06000100,
503 	0xf081, 0xffffffff, 0x00000100,
504 	0xf0b8, 0xffffffff, 0x00000100,
505 	0xf089, 0xffffffff, 0x00000100,
506 	0xf080, 0xffffffff, 0x00000100,
507 	0xf08c, 0xffffffff, 0x00000100,
508 	0xf08d, 0xffffffff, 0x00000100,
509 	0xf094, 0xffffffff, 0x00000100,
510 	0xf095, 0xffffffff, 0x00000100,
511 	0xf096, 0xffffffff, 0x00000100,
512 	0xf097, 0xffffffff, 0x00000100,
513 	0xf098, 0xffffffff, 0x00000100,
514 	0xf09f, 0xffffffff, 0x00000100,
515 	0xf09e, 0xffffffff, 0x00000100,
516 	0xf084, 0xffffffff, 0x06000100,
517 	0xf0a4, 0xffffffff, 0x00000100,
518 	0xf09d, 0xffffffff, 0x00000100,
519 	0xf0ad, 0xffffffff, 0x00000100,
520 	0xf0ac, 0xffffffff, 0x00000100,
521 	0xf09c, 0xffffffff, 0x00000100,
522 	0xc200, 0xffffffff, 0xe0000000,
523 	0xf008, 0xffffffff, 0x00010000,
524 	0xf009, 0xffffffff, 0x00030002,
525 	0xf00a, 0xffffffff, 0x00040007,
526 	0xf00b, 0xffffffff, 0x00060005,
527 	0xf00c, 0xffffffff, 0x00090008,
528 	0xf00d, 0xffffffff, 0x00010000,
529 	0xf00e, 0xffffffff, 0x00030002,
530 	0xf00f, 0xffffffff, 0x00040007,
531 	0xf010, 0xffffffff, 0x00060005,
532 	0xf011, 0xffffffff, 0x00090008,
533 	0xf000, 0xffffffff, 0x96e00200,
534 	0x21c2, 0xffffffff, 0x00900100,
535 	0x3109, 0xffffffff, 0x0020003f,
536 	0xe, 0xffffffff, 0x0140001c,
537 	0xf, 0x000f0000, 0x000f0000,
538 	0x88, 0xffffffff, 0xc060000c,
539 	0x89, 0xc0000fff, 0x00000100,
540 	0x82a, 0xffffffff, 0x00000104,
541 	0x1579, 0xff000fff, 0x00000100,
542 	0xc33, 0xc0000fff, 0x00000104,
543 	0x3079, 0x00000001, 0x00000001,
544 	0x3403, 0xff000ff0, 0x00000100,
545 	0x3603, 0xff000ff0, 0x00000100
546 };
547 
548 static const u32 hawaii_golden_spm_registers[] =
549 {
550 	0xc200, 0xe0ffffff, 0xe0000000
551 };
552 
553 static const u32 hawaii_golden_common_registers[] =
554 {
555 	0xc200, 0xffffffff, 0xe0000000,
556 	0xa0d4, 0xffffffff, 0x3a00161a,
557 	0xa0d5, 0xffffffff, 0x0000002e,
558 	0x2684, 0xffffffff, 0x00018208,
559 	0x263e, 0xffffffff, 0x12011003
560 };
561 
562 static const u32 hawaii_golden_registers[] =
563 {
564 	0xcd5, 0x00000333, 0x00000333,
565 	0x2684, 0x00010000, 0x00058208,
566 	0x260c, 0xffffffff, 0x00000000,
567 	0x260d, 0xf00fffff, 0x00000400,
568 	0x260e, 0x0002021c, 0x00020200,
569 	0x31e, 0x00000080, 0x00000000,
570 	0x16ec, 0x000000f0, 0x00000070,
571 	0x16f0, 0xf0311fff, 0x80300000,
572 	0xd43, 0x00810000, 0x408af000,
573 	0x1c0c, 0x31000111, 0x00000011,
574 	0xbd2, 0x73773777, 0x12010001,
575 	0x848, 0x0000007f, 0x0000001b,
576 	0x877, 0x00007fb6, 0x00002191,
577 	0xd8a, 0x0000003f, 0x0000000a,
578 	0xd8b, 0x0000003f, 0x0000000a,
579 	0xab9, 0x00073ffe, 0x000022a2,
580 	0x903, 0x000007ff, 0x00000000,
581 	0x22fc, 0x00002001, 0x00000001,
582 	0x22c9, 0xffffffff, 0x00ffffff,
583 	0xc281, 0x0000ff0f, 0x00000000,
584 	0xa293, 0x07ffffff, 0x06000000,
585 	0xf9e, 0x00000001, 0x00000002,
586 	0x31da, 0x00000008, 0x00000008,
587 	0x31dc, 0x00000f00, 0x00000800,
588 	0x31dd, 0x00000f00, 0x00000800,
589 	0x31e6, 0x00ffffff, 0x00ff7fbf,
590 	0x31e7, 0x00ffffff, 0x00ff7faf,
591 	0x2300, 0x000000ff, 0x00000800,
592 	0x390, 0x00001fff, 0x00001fff,
593 	0x2418, 0x0000007f, 0x00000020,
594 	0x2542, 0x00010000, 0x00010000,
595 	0x2b80, 0x00100000, 0x000ff07c,
596 	0x2b05, 0x000003ff, 0x0000000f,
597 	0x2b04, 0xffffffff, 0x7564fdec,
598 	0x2b03, 0xffffffff, 0x3120b9a8,
599 	0x2b02, 0x20000000, 0x0f9c0000
600 };
601 
602 static const u32 hawaii_mgcg_cgcg_init[] =
603 {
604 	0x3108, 0xffffffff, 0xfffffffd,
605 	0xc200, 0xffffffff, 0xe0000000,
606 	0xf0a8, 0xffffffff, 0x00000100,
607 	0xf082, 0xffffffff, 0x00000100,
608 	0xf0b0, 0xffffffff, 0x00000100,
609 	0xf0b2, 0xffffffff, 0x00000100,
610 	0xf0b1, 0xffffffff, 0x00000100,
611 	0x1579, 0xffffffff, 0x00200100,
612 	0xf0a0, 0xffffffff, 0x00000100,
613 	0xf085, 0xffffffff, 0x06000100,
614 	0xf088, 0xffffffff, 0x00000100,
615 	0xf086, 0xffffffff, 0x06000100,
616 	0xf081, 0xffffffff, 0x00000100,
617 	0xf0b8, 0xffffffff, 0x00000100,
618 	0xf089, 0xffffffff, 0x00000100,
619 	0xf080, 0xffffffff, 0x00000100,
620 	0xf08c, 0xffffffff, 0x00000100,
621 	0xf08d, 0xffffffff, 0x00000100,
622 	0xf094, 0xffffffff, 0x00000100,
623 	0xf095, 0xffffffff, 0x00000100,
624 	0xf096, 0xffffffff, 0x00000100,
625 	0xf097, 0xffffffff, 0x00000100,
626 	0xf098, 0xffffffff, 0x00000100,
627 	0xf09f, 0xffffffff, 0x00000100,
628 	0xf09e, 0xffffffff, 0x00000100,
629 	0xf084, 0xffffffff, 0x06000100,
630 	0xf0a4, 0xffffffff, 0x00000100,
631 	0xf09d, 0xffffffff, 0x00000100,
632 	0xf0ad, 0xffffffff, 0x00000100,
633 	0xf0ac, 0xffffffff, 0x00000100,
634 	0xf09c, 0xffffffff, 0x00000100,
635 	0xc200, 0xffffffff, 0xe0000000,
636 	0xf008, 0xffffffff, 0x00010000,
637 	0xf009, 0xffffffff, 0x00030002,
638 	0xf00a, 0xffffffff, 0x00040007,
639 	0xf00b, 0xffffffff, 0x00060005,
640 	0xf00c, 0xffffffff, 0x00090008,
641 	0xf00d, 0xffffffff, 0x00010000,
642 	0xf00e, 0xffffffff, 0x00030002,
643 	0xf00f, 0xffffffff, 0x00040007,
644 	0xf010, 0xffffffff, 0x00060005,
645 	0xf011, 0xffffffff, 0x00090008,
646 	0xf012, 0xffffffff, 0x00010000,
647 	0xf013, 0xffffffff, 0x00030002,
648 	0xf014, 0xffffffff, 0x00040007,
649 	0xf015, 0xffffffff, 0x00060005,
650 	0xf016, 0xffffffff, 0x00090008,
651 	0xf017, 0xffffffff, 0x00010000,
652 	0xf018, 0xffffffff, 0x00030002,
653 	0xf019, 0xffffffff, 0x00040007,
654 	0xf01a, 0xffffffff, 0x00060005,
655 	0xf01b, 0xffffffff, 0x00090008,
656 	0xf01c, 0xffffffff, 0x00010000,
657 	0xf01d, 0xffffffff, 0x00030002,
658 	0xf01e, 0xffffffff, 0x00040007,
659 	0xf01f, 0xffffffff, 0x00060005,
660 	0xf020, 0xffffffff, 0x00090008,
661 	0xf021, 0xffffffff, 0x00010000,
662 	0xf022, 0xffffffff, 0x00030002,
663 	0xf023, 0xffffffff, 0x00040007,
664 	0xf024, 0xffffffff, 0x00060005,
665 	0xf025, 0xffffffff, 0x00090008,
666 	0xf026, 0xffffffff, 0x00010000,
667 	0xf027, 0xffffffff, 0x00030002,
668 	0xf028, 0xffffffff, 0x00040007,
669 	0xf029, 0xffffffff, 0x00060005,
670 	0xf02a, 0xffffffff, 0x00090008,
671 	0xf02b, 0xffffffff, 0x00010000,
672 	0xf02c, 0xffffffff, 0x00030002,
673 	0xf02d, 0xffffffff, 0x00040007,
674 	0xf02e, 0xffffffff, 0x00060005,
675 	0xf02f, 0xffffffff, 0x00090008,
676 	0xf030, 0xffffffff, 0x00010000,
677 	0xf031, 0xffffffff, 0x00030002,
678 	0xf032, 0xffffffff, 0x00040007,
679 	0xf033, 0xffffffff, 0x00060005,
680 	0xf034, 0xffffffff, 0x00090008,
681 	0xf035, 0xffffffff, 0x00010000,
682 	0xf036, 0xffffffff, 0x00030002,
683 	0xf037, 0xffffffff, 0x00040007,
684 	0xf038, 0xffffffff, 0x00060005,
685 	0xf039, 0xffffffff, 0x00090008,
686 	0xf03a, 0xffffffff, 0x00010000,
687 	0xf03b, 0xffffffff, 0x00030002,
688 	0xf03c, 0xffffffff, 0x00040007,
689 	0xf03d, 0xffffffff, 0x00060005,
690 	0xf03e, 0xffffffff, 0x00090008,
691 	0x30c6, 0xffffffff, 0x00020200,
692 	0xcd4, 0xffffffff, 0x00000200,
693 	0x570, 0xffffffff, 0x00000400,
694 	0x157a, 0xffffffff, 0x00000000,
695 	0xbd4, 0xffffffff, 0x00000902,
696 	0xf000, 0xffffffff, 0x96940200,
697 	0x21c2, 0xffffffff, 0x00900100,
698 	0x3109, 0xffffffff, 0x0020003f,
699 	0xe, 0xffffffff, 0x0140001c,
700 	0xf, 0x000f0000, 0x000f0000,
701 	0x88, 0xffffffff, 0xc060000c,
702 	0x89, 0xc0000fff, 0x00000100,
703 	0x3e4, 0xffffffff, 0x00000100,
704 	0x3e6, 0x00000101, 0x00000000,
705 	0x82a, 0xffffffff, 0x00000104,
706 	0x1579, 0xff000fff, 0x00000100,
707 	0xc33, 0xc0000fff, 0x00000104,
708 	0x3079, 0x00000001, 0x00000001,
709 	0x3403, 0xff000ff0, 0x00000100,
710 	0x3603, 0xff000ff0, 0x00000100
711 };
712 
713 static const u32 godavari_golden_registers[] =
714 {
715 	0x1579, 0xff607fff, 0xfc000100,
716 	0x1bb6, 0x00010101, 0x00010000,
717 	0x260c, 0xffffffff, 0x00000000,
718 	0x260c0, 0xf00fffff, 0x00000400,
719 	0x184c, 0xffffffff, 0x00010000,
720 	0x16ec, 0x000000f0, 0x00000070,
721 	0x16f0, 0xf0311fff, 0x80300000,
722 	0x263e, 0x73773777, 0x12010001,
723 	0x263f, 0xffffffff, 0x00000010,
724 	0x200c, 0x00001f0f, 0x0000100a,
725 	0xbd2, 0x73773777, 0x12010001,
726 	0x902, 0x000fffff, 0x000c007f,
727 	0x2285, 0xf000003f, 0x00000007,
728 	0x22c9, 0xffffffff, 0x00ff0fff,
729 	0xc281, 0x0000ff0f, 0x00000000,
730 	0xa293, 0x07ffffff, 0x06000000,
731 	0x136, 0x00000fff, 0x00000100,
732 	0x3405, 0x00010000, 0x00810001,
733 	0x3605, 0x00010000, 0x00810001,
734 	0xf9e, 0x00000001, 0x00000002,
735 	0x31da, 0x00000008, 0x00000008,
736 	0x31dc, 0x00000f00, 0x00000800,
737 	0x31dd, 0x00000f00, 0x00000800,
738 	0x31e6, 0x00ffffff, 0x00ff7fbf,
739 	0x31e7, 0x00ffffff, 0x00ff7faf,
740 	0x2300, 0x000000ff, 0x00000001,
741 	0x853e, 0x01ff01ff, 0x00000002,
742 	0x8526, 0x007ff800, 0x00200000,
743 	0x8057, 0xffffffff, 0x00000f40,
744 	0x2231, 0x001f3ae3, 0x00000082,
745 	0x2235, 0x0000001f, 0x00000010,
746 	0xc24d, 0xffffffff, 0x00000000
747 };
748 
749 static void cik_init_golden_registers(struct amdgpu_device *adev)
750 {
751 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
752 	mutex_lock(&adev->grbm_idx_mutex);
753 
754 	switch (adev->asic_type) {
755 	case CHIP_BONAIRE:
756 		amdgpu_program_register_sequence(adev,
757 						 bonaire_mgcg_cgcg_init,
758 						 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
759 		amdgpu_program_register_sequence(adev,
760 						 bonaire_golden_registers,
761 						 (const u32)ARRAY_SIZE(bonaire_golden_registers));
762 		amdgpu_program_register_sequence(adev,
763 						 bonaire_golden_common_registers,
764 						 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
765 		amdgpu_program_register_sequence(adev,
766 						 bonaire_golden_spm_registers,
767 						 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
768 		break;
769 	case CHIP_KABINI:
770 		amdgpu_program_register_sequence(adev,
771 						 kalindi_mgcg_cgcg_init,
772 						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
773 		amdgpu_program_register_sequence(adev,
774 						 kalindi_golden_registers,
775 						 (const u32)ARRAY_SIZE(kalindi_golden_registers));
776 		amdgpu_program_register_sequence(adev,
777 						 kalindi_golden_common_registers,
778 						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
779 		amdgpu_program_register_sequence(adev,
780 						 kalindi_golden_spm_registers,
781 						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
782 		break;
783 	case CHIP_MULLINS:
784 		amdgpu_program_register_sequence(adev,
785 						 kalindi_mgcg_cgcg_init,
786 						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
787 		amdgpu_program_register_sequence(adev,
788 						 godavari_golden_registers,
789 						 (const u32)ARRAY_SIZE(godavari_golden_registers));
790 		amdgpu_program_register_sequence(adev,
791 						 kalindi_golden_common_registers,
792 						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
793 		amdgpu_program_register_sequence(adev,
794 						 kalindi_golden_spm_registers,
795 						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
796 		break;
797 	case CHIP_KAVERI:
798 		amdgpu_program_register_sequence(adev,
799 						 spectre_mgcg_cgcg_init,
800 						 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
801 		amdgpu_program_register_sequence(adev,
802 						 spectre_golden_registers,
803 						 (const u32)ARRAY_SIZE(spectre_golden_registers));
804 		amdgpu_program_register_sequence(adev,
805 						 spectre_golden_common_registers,
806 						 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
807 		amdgpu_program_register_sequence(adev,
808 						 spectre_golden_spm_registers,
809 						 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
810 		break;
811 	case CHIP_HAWAII:
812 		amdgpu_program_register_sequence(adev,
813 						 hawaii_mgcg_cgcg_init,
814 						 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
815 		amdgpu_program_register_sequence(adev,
816 						 hawaii_golden_registers,
817 						 (const u32)ARRAY_SIZE(hawaii_golden_registers));
818 		amdgpu_program_register_sequence(adev,
819 						 hawaii_golden_common_registers,
820 						 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
821 		amdgpu_program_register_sequence(adev,
822 						 hawaii_golden_spm_registers,
823 						 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
824 		break;
825 	default:
826 		break;
827 	}
828 	mutex_unlock(&adev->grbm_idx_mutex);
829 }
830 
831 /**
832  * cik_get_xclk - get the xclk
833  *
834  * @adev: amdgpu_device pointer
835  *
836  * Returns the reference clock used by the gfx engine
837  * (CIK).
838  */
839 static u32 cik_get_xclk(struct amdgpu_device *adev)
840 {
841 	u32 reference_clock = adev->clock.spll.reference_freq;
842 
843 	if (adev->flags & AMD_IS_APU) {
844 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
845 			return reference_clock / 2;
846 	} else {
847 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
848 			return reference_clock / 4;
849 	}
850 	return reference_clock;
851 }
852 
853 /**
854  * cik_srbm_select - select specific register instances
855  *
856  * @adev: amdgpu_device pointer
857  * @me: selected ME (micro engine)
858  * @pipe: pipe
859  * @queue: queue
860  * @vmid: VMID
861  *
862  * Switches the currently active registers instances.  Some
863  * registers are instanced per VMID, others are instanced per
864  * me/pipe/queue combination.
865  */
866 void cik_srbm_select(struct amdgpu_device *adev,
867 		     u32 me, u32 pipe, u32 queue, u32 vmid)
868 {
869 	u32 srbm_gfx_cntl =
870 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
871 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
872 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
873 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
874 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
875 }
876 
877 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
878 {
879 	uint32_t tmp;
880 
881 	tmp = RREG32(mmCONFIG_CNTL);
882 	if (state == false)
883 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
884 	else
885 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
886 	WREG32(mmCONFIG_CNTL, tmp);
887 }
888 
889 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
890 {
891 	u32 bus_cntl;
892 	u32 d1vga_control = 0;
893 	u32 d2vga_control = 0;
894 	u32 vga_render_control = 0;
895 	u32 rom_cntl;
896 	bool r;
897 
898 	bus_cntl = RREG32(mmBUS_CNTL);
899 	if (adev->mode_info.num_crtc) {
900 		d1vga_control = RREG32(mmD1VGA_CONTROL);
901 		d2vga_control = RREG32(mmD2VGA_CONTROL);
902 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
903 	}
904 	rom_cntl = RREG32_SMC(ixROM_CNTL);
905 
906 	/* enable the rom */
907 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
908 	if (adev->mode_info.num_crtc) {
909 		/* Disable VGA mode */
910 		WREG32(mmD1VGA_CONTROL,
911 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
912 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
913 		WREG32(mmD2VGA_CONTROL,
914 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
915 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
916 		WREG32(mmVGA_RENDER_CONTROL,
917 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
918 	}
919 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
920 
921 	r = amdgpu_read_bios(adev);
922 
923 	/* restore regs */
924 	WREG32(mmBUS_CNTL, bus_cntl);
925 	if (adev->mode_info.num_crtc) {
926 		WREG32(mmD1VGA_CONTROL, d1vga_control);
927 		WREG32(mmD2VGA_CONTROL, d2vga_control);
928 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
929 	}
930 	WREG32_SMC(ixROM_CNTL, rom_cntl);
931 	return r;
932 }
933 
934 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
935 				   u8 *bios, u32 length_bytes)
936 {
937 	u32 *dw_ptr;
938 	unsigned long flags;
939 	u32 i, length_dw;
940 
941 	if (bios == NULL)
942 		return false;
943 	if (length_bytes == 0)
944 		return false;
945 	/* APU vbios image is part of sbios image */
946 	if (adev->flags & AMD_IS_APU)
947 		return false;
948 
949 	dw_ptr = (u32 *)bios;
950 	length_dw = ALIGN(length_bytes, 4) / 4;
951 	/* take the smc lock since we are using the smc index */
952 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
953 	/* set rom index to 0 */
954 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
955 	WREG32(mmSMC_IND_DATA_0, 0);
956 	/* set index to data for continous read */
957 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
958 	for (i = 0; i < length_dw; i++)
959 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
960 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
961 
962 	return true;
963 }
964 
965 static u32 cik_get_virtual_caps(struct amdgpu_device *adev)
966 {
967 	/* CIK does not support SR-IOV */
968 	return 0;
969 }
970 
971 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
972 	{mmGRBM_STATUS, false},
973 	{mmGB_ADDR_CONFIG, false},
974 	{mmMC_ARB_RAMCFG, false},
975 	{mmGB_TILE_MODE0, false},
976 	{mmGB_TILE_MODE1, false},
977 	{mmGB_TILE_MODE2, false},
978 	{mmGB_TILE_MODE3, false},
979 	{mmGB_TILE_MODE4, false},
980 	{mmGB_TILE_MODE5, false},
981 	{mmGB_TILE_MODE6, false},
982 	{mmGB_TILE_MODE7, false},
983 	{mmGB_TILE_MODE8, false},
984 	{mmGB_TILE_MODE9, false},
985 	{mmGB_TILE_MODE10, false},
986 	{mmGB_TILE_MODE11, false},
987 	{mmGB_TILE_MODE12, false},
988 	{mmGB_TILE_MODE13, false},
989 	{mmGB_TILE_MODE14, false},
990 	{mmGB_TILE_MODE15, false},
991 	{mmGB_TILE_MODE16, false},
992 	{mmGB_TILE_MODE17, false},
993 	{mmGB_TILE_MODE18, false},
994 	{mmGB_TILE_MODE19, false},
995 	{mmGB_TILE_MODE20, false},
996 	{mmGB_TILE_MODE21, false},
997 	{mmGB_TILE_MODE22, false},
998 	{mmGB_TILE_MODE23, false},
999 	{mmGB_TILE_MODE24, false},
1000 	{mmGB_TILE_MODE25, false},
1001 	{mmGB_TILE_MODE26, false},
1002 	{mmGB_TILE_MODE27, false},
1003 	{mmGB_TILE_MODE28, false},
1004 	{mmGB_TILE_MODE29, false},
1005 	{mmGB_TILE_MODE30, false},
1006 	{mmGB_TILE_MODE31, false},
1007 	{mmGB_MACROTILE_MODE0, false},
1008 	{mmGB_MACROTILE_MODE1, false},
1009 	{mmGB_MACROTILE_MODE2, false},
1010 	{mmGB_MACROTILE_MODE3, false},
1011 	{mmGB_MACROTILE_MODE4, false},
1012 	{mmGB_MACROTILE_MODE5, false},
1013 	{mmGB_MACROTILE_MODE6, false},
1014 	{mmGB_MACROTILE_MODE7, false},
1015 	{mmGB_MACROTILE_MODE8, false},
1016 	{mmGB_MACROTILE_MODE9, false},
1017 	{mmGB_MACROTILE_MODE10, false},
1018 	{mmGB_MACROTILE_MODE11, false},
1019 	{mmGB_MACROTILE_MODE12, false},
1020 	{mmGB_MACROTILE_MODE13, false},
1021 	{mmGB_MACROTILE_MODE14, false},
1022 	{mmGB_MACROTILE_MODE15, false},
1023 	{mmCC_RB_BACKEND_DISABLE, false, true},
1024 	{mmGC_USER_RB_BACKEND_DISABLE, false, true},
1025 	{mmGB_BACKEND_MAP, false, false},
1026 	{mmPA_SC_RASTER_CONFIG, false, true},
1027 	{mmPA_SC_RASTER_CONFIG_1, false, true},
1028 };
1029 
1030 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1031 					  u32 se_num, u32 sh_num,
1032 					  u32 reg_offset)
1033 {
1034 	uint32_t val;
1035 
1036 	mutex_lock(&adev->grbm_idx_mutex);
1037 	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1038 		gfx_v7_0_select_se_sh(adev, se_num, sh_num);
1039 
1040 	val = RREG32(reg_offset);
1041 
1042 	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1043 		gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
1044 	mutex_unlock(&adev->grbm_idx_mutex);
1045 	return val;
1046 }
1047 
1048 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1049 			     u32 sh_num, u32 reg_offset, u32 *value)
1050 {
1051 	uint32_t i;
1052 
1053 	*value = 0;
1054 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1055 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1056 			continue;
1057 
1058 		if (!cik_allowed_read_registers[i].untouched)
1059 			*value = cik_allowed_read_registers[i].grbm_indexed ?
1060 				 cik_read_indexed_register(adev, se_num,
1061 							   sh_num, reg_offset) :
1062 				 RREG32(reg_offset);
1063 		return 0;
1064 	}
1065 	return -EINVAL;
1066 }
1067 
1068 struct kv_reset_save_regs {
1069 	u32 gmcon_reng_execute;
1070 	u32 gmcon_misc;
1071 	u32 gmcon_misc3;
1072 };
1073 
1074 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1075 				   struct kv_reset_save_regs *save)
1076 {
1077 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1078 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1079 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1080 
1081 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1082 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1083 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1084 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1085 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1086 }
1087 
1088 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1089 				      struct kv_reset_save_regs *save)
1090 {
1091 	int i;
1092 
1093 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1094 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1095 
1096 	for (i = 0; i < 5; i++)
1097 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1098 
1099 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1100 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1101 
1102 	for (i = 0; i < 5; i++)
1103 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1104 
1105 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1106 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1107 
1108 	for (i = 0; i < 5; i++)
1109 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1110 
1111 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1112 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1113 
1114 	for (i = 0; i < 5; i++)
1115 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1116 
1117 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1118 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1119 
1120 	for (i = 0; i < 5; i++)
1121 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1122 
1123 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1124 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1125 
1126 	for (i = 0; i < 5; i++)
1127 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1128 
1129 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1130 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1131 
1132 	for (i = 0; i < 5; i++)
1133 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1134 
1135 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1136 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1137 
1138 	for (i = 0; i < 5; i++)
1139 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1140 
1141 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1142 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1143 
1144 	for (i = 0; i < 5; i++)
1145 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1146 
1147 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1148 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1149 
1150 	for (i = 0; i < 5; i++)
1151 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1152 
1153 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1154 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1155 
1156 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1157 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1158 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1159 }
1160 
1161 static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1162 {
1163 	struct kv_reset_save_regs kv_save = { 0 };
1164 	u32 i;
1165 
1166 	dev_info(adev->dev, "GPU pci config reset\n");
1167 
1168 	if (adev->flags & AMD_IS_APU)
1169 		kv_save_regs_for_reset(adev, &kv_save);
1170 
1171 	/* disable BM */
1172 	pci_clear_master(adev->pdev);
1173 	/* reset */
1174 	amdgpu_pci_config_reset(adev);
1175 
1176 	udelay(100);
1177 
1178 	/* wait for asic to come out of reset */
1179 	for (i = 0; i < adev->usec_timeout; i++) {
1180 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
1181 			break;
1182 		udelay(1);
1183 	}
1184 
1185 	/* does asic init need to be run first??? */
1186 	if (adev->flags & AMD_IS_APU)
1187 		kv_restore_regs_for_reset(adev, &kv_save);
1188 }
1189 
1190 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1191 {
1192 	u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1193 
1194 	if (hung)
1195 		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1196 	else
1197 		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1198 
1199 	WREG32(mmBIOS_SCRATCH_3, tmp);
1200 }
1201 
1202 /**
1203  * cik_asic_reset - soft reset GPU
1204  *
1205  * @adev: amdgpu_device pointer
1206  *
1207  * Look up which blocks are hung and attempt
1208  * to reset them.
1209  * Returns 0 for success.
1210  */
1211 static int cik_asic_reset(struct amdgpu_device *adev)
1212 {
1213 	cik_set_bios_scratch_engine_hung(adev, true);
1214 
1215 	cik_gpu_pci_config_reset(adev);
1216 
1217 	cik_set_bios_scratch_engine_hung(adev, false);
1218 
1219 	return 0;
1220 }
1221 
1222 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1223 			      u32 cntl_reg, u32 status_reg)
1224 {
1225 	int r, i;
1226 	struct atom_clock_dividers dividers;
1227 	uint32_t tmp;
1228 
1229 	r = amdgpu_atombios_get_clock_dividers(adev,
1230 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1231 					       clock, false, &dividers);
1232 	if (r)
1233 		return r;
1234 
1235 	tmp = RREG32_SMC(cntl_reg);
1236 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1237 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1238 	tmp |= dividers.post_divider;
1239 	WREG32_SMC(cntl_reg, tmp);
1240 
1241 	for (i = 0; i < 100; i++) {
1242 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1243 			break;
1244 		mdelay(10);
1245 	}
1246 	if (i == 100)
1247 		return -ETIMEDOUT;
1248 
1249 	return 0;
1250 }
1251 
1252 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1253 {
1254 	int r = 0;
1255 
1256 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1257 	if (r)
1258 		return r;
1259 
1260 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1261 	return r;
1262 }
1263 
1264 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1265 {
1266 	int r, i;
1267 	struct atom_clock_dividers dividers;
1268 	u32 tmp;
1269 
1270 	r = amdgpu_atombios_get_clock_dividers(adev,
1271 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1272 					       ecclk, false, &dividers);
1273 	if (r)
1274 		return r;
1275 
1276 	for (i = 0; i < 100; i++) {
1277 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1278 			break;
1279 		mdelay(10);
1280 	}
1281 	if (i == 100)
1282 		return -ETIMEDOUT;
1283 
1284 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1285 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1286 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1287 	tmp |= dividers.post_divider;
1288 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1289 
1290 	for (i = 0; i < 100; i++) {
1291 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1292 			break;
1293 		mdelay(10);
1294 	}
1295 	if (i == 100)
1296 		return -ETIMEDOUT;
1297 
1298 	return 0;
1299 }
1300 
1301 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1302 {
1303 	struct pci_dev *root = adev->pdev->bus->self;
1304 	int bridge_pos, gpu_pos;
1305 	u32 speed_cntl, current_data_rate;
1306 	int i;
1307 	u16 tmp16;
1308 
1309 	if (pci_is_root_bus(adev->pdev->bus))
1310 		return;
1311 
1312 	if (amdgpu_pcie_gen2 == 0)
1313 		return;
1314 
1315 	if (adev->flags & AMD_IS_APU)
1316 		return;
1317 
1318 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1319 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1320 		return;
1321 
1322 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1323 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1324 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1325 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1326 		if (current_data_rate == 2) {
1327 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1328 			return;
1329 		}
1330 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1331 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1332 		if (current_data_rate == 1) {
1333 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1334 			return;
1335 		}
1336 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1337 	}
1338 
1339 	bridge_pos = pci_pcie_cap(root);
1340 	if (!bridge_pos)
1341 		return;
1342 
1343 	gpu_pos = pci_pcie_cap(adev->pdev);
1344 	if (!gpu_pos)
1345 		return;
1346 
1347 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1348 		/* re-try equalization if gen3 is not already enabled */
1349 		if (current_data_rate != 2) {
1350 			u16 bridge_cfg, gpu_cfg;
1351 			u16 bridge_cfg2, gpu_cfg2;
1352 			u32 max_lw, current_lw, tmp;
1353 
1354 			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1355 			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1356 
1357 			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1358 			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1359 
1360 			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1361 			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1362 
1363 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1364 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1365 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1366 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1367 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1368 
1369 			if (current_lw < max_lw) {
1370 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1371 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1372 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1373 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1374 					tmp |= (max_lw <<
1375 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1376 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1377 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1378 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1379 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1380 				}
1381 			}
1382 
1383 			for (i = 0; i < 10; i++) {
1384 				/* check status */
1385 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1386 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1387 					break;
1388 
1389 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1390 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1391 
1392 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1393 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1394 
1395 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1396 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1397 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1398 
1399 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1400 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1401 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1402 
1403 				mdelay(100);
1404 
1405 				/* linkctl */
1406 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1407 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1408 				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1409 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1410 
1411 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1412 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1413 				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1414 				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1415 
1416 				/* linkctl2 */
1417 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1418 				tmp16 &= ~((1 << 4) | (7 << 9));
1419 				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1420 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1421 
1422 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1423 				tmp16 &= ~((1 << 4) | (7 << 9));
1424 				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1425 				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1426 
1427 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1428 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1429 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1430 			}
1431 		}
1432 	}
1433 
1434 	/* set the link speed */
1435 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1436 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1437 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1438 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1439 
1440 	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1441 	tmp16 &= ~0xf;
1442 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1443 		tmp16 |= 3; /* gen3 */
1444 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1445 		tmp16 |= 2; /* gen2 */
1446 	else
1447 		tmp16 |= 1; /* gen1 */
1448 	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1449 
1450 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1451 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1452 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1453 
1454 	for (i = 0; i < adev->usec_timeout; i++) {
1455 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1456 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1457 			break;
1458 		udelay(1);
1459 	}
1460 }
1461 
1462 static void cik_program_aspm(struct amdgpu_device *adev)
1463 {
1464 	u32 data, orig;
1465 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1466 	bool disable_clkreq = false;
1467 
1468 	if (amdgpu_aspm == 0)
1469 		return;
1470 
1471 	if (pci_is_root_bus(adev->pdev->bus))
1472 		return;
1473 
1474 	/* XXX double check APUs */
1475 	if (adev->flags & AMD_IS_APU)
1476 		return;
1477 
1478 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1479 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1480 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1481 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1482 	if (orig != data)
1483 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1484 
1485 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1486 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1487 	if (orig != data)
1488 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1489 
1490 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1491 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1492 	if (orig != data)
1493 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1494 
1495 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1496 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1497 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1498 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1499 	if (!disable_l0s)
1500 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1501 
1502 	if (!disable_l1) {
1503 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1504 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1505 		if (orig != data)
1506 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1507 
1508 		if (!disable_plloff_in_l1) {
1509 			bool clk_req_support;
1510 
1511 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1512 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1513 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1514 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1515 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1516 			if (orig != data)
1517 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1518 
1519 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1520 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1521 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1522 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1523 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1524 			if (orig != data)
1525 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1526 
1527 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1528 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1529 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1530 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1531 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1532 			if (orig != data)
1533 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1534 
1535 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1536 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1537 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1538 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1539 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1540 			if (orig != data)
1541 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1542 
1543 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1544 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1545 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1546 			if (orig != data)
1547 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1548 
1549 			if (!disable_clkreq) {
1550 				struct pci_dev *root = adev->pdev->bus->self;
1551 				u32 lnkcap;
1552 
1553 				clk_req_support = false;
1554 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1555 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1556 					clk_req_support = true;
1557 			} else {
1558 				clk_req_support = false;
1559 			}
1560 
1561 			if (clk_req_support) {
1562 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1563 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1564 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1565 				if (orig != data)
1566 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1567 
1568 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1569 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1570 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1571 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1572 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1573 				if (orig != data)
1574 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1575 
1576 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1577 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1578 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1579 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1580 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1581 				if (orig != data)
1582 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1583 
1584 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1585 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1586 				if (orig != data)
1587 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1588 
1589 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1590 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1591 				if (orig != data)
1592 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1593 
1594 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1595 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1596 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1597 				if (orig != data)
1598 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1599 			}
1600 		}
1601 	} else {
1602 		if (orig != data)
1603 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1604 	}
1605 
1606 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1607 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1608 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1609 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1610 	if (orig != data)
1611 		WREG32_PCIE(ixPCIE_CNTL2, data);
1612 
1613 	if (!disable_l0s) {
1614 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1615 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1616 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1617 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1618 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1619 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1620 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1621 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1622 				if (orig != data)
1623 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1624 			}
1625 		}
1626 	}
1627 }
1628 
1629 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1630 {
1631 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1632 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1633 }
1634 
1635 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1636 {
1637 	/* ORDER MATTERS! */
1638 	{
1639 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1640 		.major = 1,
1641 		.minor = 0,
1642 		.rev = 0,
1643 		.funcs = &cik_common_ip_funcs,
1644 	},
1645 	{
1646 		.type = AMD_IP_BLOCK_TYPE_GMC,
1647 		.major = 7,
1648 		.minor = 0,
1649 		.rev = 0,
1650 		.funcs = &gmc_v7_0_ip_funcs,
1651 	},
1652 	{
1653 		.type = AMD_IP_BLOCK_TYPE_IH,
1654 		.major = 2,
1655 		.minor = 0,
1656 		.rev = 0,
1657 		.funcs = &cik_ih_ip_funcs,
1658 	},
1659 	{
1660 		.type = AMD_IP_BLOCK_TYPE_SMC,
1661 		.major = 7,
1662 		.minor = 0,
1663 		.rev = 0,
1664 		.funcs = &amdgpu_pp_ip_funcs,
1665 	},
1666 	{
1667 		.type = AMD_IP_BLOCK_TYPE_DCE,
1668 		.major = 8,
1669 		.minor = 2,
1670 		.rev = 0,
1671 		.funcs = &dce_v8_0_ip_funcs,
1672 	},
1673 	{
1674 		.type = AMD_IP_BLOCK_TYPE_GFX,
1675 		.major = 7,
1676 		.minor = 2,
1677 		.rev = 0,
1678 		.funcs = &gfx_v7_0_ip_funcs,
1679 	},
1680 	{
1681 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1682 		.major = 2,
1683 		.minor = 0,
1684 		.rev = 0,
1685 		.funcs = &cik_sdma_ip_funcs,
1686 	},
1687 	{
1688 		.type = AMD_IP_BLOCK_TYPE_UVD,
1689 		.major = 4,
1690 		.minor = 2,
1691 		.rev = 0,
1692 		.funcs = &uvd_v4_2_ip_funcs,
1693 	},
1694 	{
1695 		.type = AMD_IP_BLOCK_TYPE_VCE,
1696 		.major = 2,
1697 		.minor = 0,
1698 		.rev = 0,
1699 		.funcs = &vce_v2_0_ip_funcs,
1700 	},
1701 };
1702 
1703 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1704 {
1705 	/* ORDER MATTERS! */
1706 	{
1707 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1708 		.major = 1,
1709 		.minor = 0,
1710 		.rev = 0,
1711 		.funcs = &cik_common_ip_funcs,
1712 	},
1713 	{
1714 		.type = AMD_IP_BLOCK_TYPE_GMC,
1715 		.major = 7,
1716 		.minor = 0,
1717 		.rev = 0,
1718 		.funcs = &gmc_v7_0_ip_funcs,
1719 	},
1720 	{
1721 		.type = AMD_IP_BLOCK_TYPE_IH,
1722 		.major = 2,
1723 		.minor = 0,
1724 		.rev = 0,
1725 		.funcs = &cik_ih_ip_funcs,
1726 	},
1727 	{
1728 		.type = AMD_IP_BLOCK_TYPE_SMC,
1729 		.major = 7,
1730 		.minor = 0,
1731 		.rev = 0,
1732 		.funcs = &amdgpu_pp_ip_funcs,
1733 	},
1734 	{
1735 		.type = AMD_IP_BLOCK_TYPE_DCE,
1736 		.major = 8,
1737 		.minor = 5,
1738 		.rev = 0,
1739 		.funcs = &dce_v8_0_ip_funcs,
1740 	},
1741 	{
1742 		.type = AMD_IP_BLOCK_TYPE_GFX,
1743 		.major = 7,
1744 		.minor = 3,
1745 		.rev = 0,
1746 		.funcs = &gfx_v7_0_ip_funcs,
1747 	},
1748 	{
1749 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1750 		.major = 2,
1751 		.minor = 0,
1752 		.rev = 0,
1753 		.funcs = &cik_sdma_ip_funcs,
1754 	},
1755 	{
1756 		.type = AMD_IP_BLOCK_TYPE_UVD,
1757 		.major = 4,
1758 		.minor = 2,
1759 		.rev = 0,
1760 		.funcs = &uvd_v4_2_ip_funcs,
1761 	},
1762 	{
1763 		.type = AMD_IP_BLOCK_TYPE_VCE,
1764 		.major = 2,
1765 		.minor = 0,
1766 		.rev = 0,
1767 		.funcs = &vce_v2_0_ip_funcs,
1768 	},
1769 };
1770 
1771 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
1772 {
1773 	/* ORDER MATTERS! */
1774 	{
1775 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1776 		.major = 1,
1777 		.minor = 0,
1778 		.rev = 0,
1779 		.funcs = &cik_common_ip_funcs,
1780 	},
1781 	{
1782 		.type = AMD_IP_BLOCK_TYPE_GMC,
1783 		.major = 7,
1784 		.minor = 0,
1785 		.rev = 0,
1786 		.funcs = &gmc_v7_0_ip_funcs,
1787 	},
1788 	{
1789 		.type = AMD_IP_BLOCK_TYPE_IH,
1790 		.major = 2,
1791 		.minor = 0,
1792 		.rev = 0,
1793 		.funcs = &cik_ih_ip_funcs,
1794 	},
1795 	{
1796 		.type = AMD_IP_BLOCK_TYPE_SMC,
1797 		.major = 7,
1798 		.minor = 0,
1799 		.rev = 0,
1800 		.funcs = &amdgpu_pp_ip_funcs,
1801 	},
1802 	{
1803 		.type = AMD_IP_BLOCK_TYPE_DCE,
1804 		.major = 8,
1805 		.minor = 3,
1806 		.rev = 0,
1807 		.funcs = &dce_v8_0_ip_funcs,
1808 	},
1809 	{
1810 		.type = AMD_IP_BLOCK_TYPE_GFX,
1811 		.major = 7,
1812 		.minor = 2,
1813 		.rev = 0,
1814 		.funcs = &gfx_v7_0_ip_funcs,
1815 	},
1816 	{
1817 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1818 		.major = 2,
1819 		.minor = 0,
1820 		.rev = 0,
1821 		.funcs = &cik_sdma_ip_funcs,
1822 	},
1823 	{
1824 		.type = AMD_IP_BLOCK_TYPE_UVD,
1825 		.major = 4,
1826 		.minor = 2,
1827 		.rev = 0,
1828 		.funcs = &uvd_v4_2_ip_funcs,
1829 	},
1830 	{
1831 		.type = AMD_IP_BLOCK_TYPE_VCE,
1832 		.major = 2,
1833 		.minor = 0,
1834 		.rev = 0,
1835 		.funcs = &vce_v2_0_ip_funcs,
1836 	},
1837 };
1838 
1839 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
1840 {
1841 	/* ORDER MATTERS! */
1842 	{
1843 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1844 		.major = 1,
1845 		.minor = 0,
1846 		.rev = 0,
1847 		.funcs = &cik_common_ip_funcs,
1848 	},
1849 	{
1850 		.type = AMD_IP_BLOCK_TYPE_GMC,
1851 		.major = 7,
1852 		.minor = 0,
1853 		.rev = 0,
1854 		.funcs = &gmc_v7_0_ip_funcs,
1855 	},
1856 	{
1857 		.type = AMD_IP_BLOCK_TYPE_IH,
1858 		.major = 2,
1859 		.minor = 0,
1860 		.rev = 0,
1861 		.funcs = &cik_ih_ip_funcs,
1862 	},
1863 	{
1864 		.type = AMD_IP_BLOCK_TYPE_SMC,
1865 		.major = 7,
1866 		.minor = 0,
1867 		.rev = 0,
1868 		.funcs = &amdgpu_pp_ip_funcs,
1869 	},
1870 	{
1871 		.type = AMD_IP_BLOCK_TYPE_DCE,
1872 		.major = 8,
1873 		.minor = 3,
1874 		.rev = 0,
1875 		.funcs = &dce_v8_0_ip_funcs,
1876 	},
1877 	{
1878 		.type = AMD_IP_BLOCK_TYPE_GFX,
1879 		.major = 7,
1880 		.minor = 2,
1881 		.rev = 0,
1882 		.funcs = &gfx_v7_0_ip_funcs,
1883 	},
1884 	{
1885 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1886 		.major = 2,
1887 		.minor = 0,
1888 		.rev = 0,
1889 		.funcs = &cik_sdma_ip_funcs,
1890 	},
1891 	{
1892 		.type = AMD_IP_BLOCK_TYPE_UVD,
1893 		.major = 4,
1894 		.minor = 2,
1895 		.rev = 0,
1896 		.funcs = &uvd_v4_2_ip_funcs,
1897 	},
1898 	{
1899 		.type = AMD_IP_BLOCK_TYPE_VCE,
1900 		.major = 2,
1901 		.minor = 0,
1902 		.rev = 0,
1903 		.funcs = &vce_v2_0_ip_funcs,
1904 	},
1905 };
1906 
1907 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
1908 {
1909 	/* ORDER MATTERS! */
1910 	{
1911 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1912 		.major = 1,
1913 		.minor = 0,
1914 		.rev = 0,
1915 		.funcs = &cik_common_ip_funcs,
1916 	},
1917 	{
1918 		.type = AMD_IP_BLOCK_TYPE_GMC,
1919 		.major = 7,
1920 		.minor = 0,
1921 		.rev = 0,
1922 		.funcs = &gmc_v7_0_ip_funcs,
1923 	},
1924 	{
1925 		.type = AMD_IP_BLOCK_TYPE_IH,
1926 		.major = 2,
1927 		.minor = 0,
1928 		.rev = 0,
1929 		.funcs = &cik_ih_ip_funcs,
1930 	},
1931 	{
1932 		.type = AMD_IP_BLOCK_TYPE_SMC,
1933 		.major = 7,
1934 		.minor = 0,
1935 		.rev = 0,
1936 		.funcs = &amdgpu_pp_ip_funcs,
1937 	},
1938 	{
1939 		.type = AMD_IP_BLOCK_TYPE_DCE,
1940 		.major = 8,
1941 		.minor = 1,
1942 		.rev = 0,
1943 		.funcs = &dce_v8_0_ip_funcs,
1944 	},
1945 	{
1946 		.type = AMD_IP_BLOCK_TYPE_GFX,
1947 		.major = 7,
1948 		.minor = 1,
1949 		.rev = 0,
1950 		.funcs = &gfx_v7_0_ip_funcs,
1951 	},
1952 	{
1953 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1954 		.major = 2,
1955 		.minor = 0,
1956 		.rev = 0,
1957 		.funcs = &cik_sdma_ip_funcs,
1958 	},
1959 	{
1960 		.type = AMD_IP_BLOCK_TYPE_UVD,
1961 		.major = 4,
1962 		.minor = 2,
1963 		.rev = 0,
1964 		.funcs = &uvd_v4_2_ip_funcs,
1965 	},
1966 	{
1967 		.type = AMD_IP_BLOCK_TYPE_VCE,
1968 		.major = 2,
1969 		.minor = 0,
1970 		.rev = 0,
1971 		.funcs = &vce_v2_0_ip_funcs,
1972 	},
1973 };
1974 
1975 int cik_set_ip_blocks(struct amdgpu_device *adev)
1976 {
1977 	switch (adev->asic_type) {
1978 	case CHIP_BONAIRE:
1979 		adev->ip_blocks = bonaire_ip_blocks;
1980 		adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
1981 		break;
1982 	case CHIP_HAWAII:
1983 		adev->ip_blocks = hawaii_ip_blocks;
1984 		adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
1985 		break;
1986 	case CHIP_KAVERI:
1987 		adev->ip_blocks = kaveri_ip_blocks;
1988 		adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
1989 		break;
1990 	case CHIP_KABINI:
1991 		adev->ip_blocks = kabini_ip_blocks;
1992 		adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
1993 		break;
1994 	case CHIP_MULLINS:
1995 		adev->ip_blocks = mullins_ip_blocks;
1996 		adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
1997 		break;
1998 	default:
1999 		/* FIXME: not supported yet */
2000 		return -EINVAL;
2001 	}
2002 
2003 	return 0;
2004 }
2005 
2006 static const struct amdgpu_asic_funcs cik_asic_funcs =
2007 {
2008 	.read_disabled_bios = &cik_read_disabled_bios,
2009 	.read_bios_from_rom = &cik_read_bios_from_rom,
2010 	.read_register = &cik_read_register,
2011 	.reset = &cik_asic_reset,
2012 	.set_vga_state = &cik_vga_set_state,
2013 	.get_xclk = &cik_get_xclk,
2014 	.set_uvd_clocks = &cik_set_uvd_clocks,
2015 	.set_vce_clocks = &cik_set_vce_clocks,
2016 	.get_virtual_caps = &cik_get_virtual_caps,
2017 	/* these should be moved to their own ip modules */
2018 	.get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
2019 	.wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
2020 };
2021 
2022 static int cik_common_early_init(void *handle)
2023 {
2024 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2025 
2026 	adev->smc_rreg = &cik_smc_rreg;
2027 	adev->smc_wreg = &cik_smc_wreg;
2028 	adev->pcie_rreg = &cik_pcie_rreg;
2029 	adev->pcie_wreg = &cik_pcie_wreg;
2030 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2031 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2032 	adev->didt_rreg = &cik_didt_rreg;
2033 	adev->didt_wreg = &cik_didt_wreg;
2034 
2035 	adev->asic_funcs = &cik_asic_funcs;
2036 
2037 	adev->rev_id = cik_get_rev_id(adev);
2038 	adev->external_rev_id = 0xFF;
2039 	switch (adev->asic_type) {
2040 	case CHIP_BONAIRE:
2041 		adev->cg_flags =
2042 			AMD_CG_SUPPORT_GFX_MGCG |
2043 			AMD_CG_SUPPORT_GFX_MGLS |
2044 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2045 			AMD_CG_SUPPORT_GFX_CGLS |
2046 			AMD_CG_SUPPORT_GFX_CGTS |
2047 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2048 			AMD_CG_SUPPORT_GFX_CP_LS |
2049 			AMD_CG_SUPPORT_MC_LS |
2050 			AMD_CG_SUPPORT_MC_MGCG |
2051 			AMD_CG_SUPPORT_SDMA_MGCG |
2052 			AMD_CG_SUPPORT_SDMA_LS |
2053 			AMD_CG_SUPPORT_BIF_LS |
2054 			AMD_CG_SUPPORT_VCE_MGCG |
2055 			AMD_CG_SUPPORT_UVD_MGCG |
2056 			AMD_CG_SUPPORT_HDP_LS |
2057 			AMD_CG_SUPPORT_HDP_MGCG;
2058 		adev->pg_flags = 0;
2059 		adev->external_rev_id = adev->rev_id + 0x14;
2060 		break;
2061 	case CHIP_HAWAII:
2062 		adev->cg_flags =
2063 			AMD_CG_SUPPORT_GFX_MGCG |
2064 			AMD_CG_SUPPORT_GFX_MGLS |
2065 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2066 			AMD_CG_SUPPORT_GFX_CGLS |
2067 			AMD_CG_SUPPORT_GFX_CGTS |
2068 			AMD_CG_SUPPORT_GFX_CP_LS |
2069 			AMD_CG_SUPPORT_MC_LS |
2070 			AMD_CG_SUPPORT_MC_MGCG |
2071 			AMD_CG_SUPPORT_SDMA_MGCG |
2072 			AMD_CG_SUPPORT_SDMA_LS |
2073 			AMD_CG_SUPPORT_BIF_LS |
2074 			AMD_CG_SUPPORT_VCE_MGCG |
2075 			AMD_CG_SUPPORT_UVD_MGCG |
2076 			AMD_CG_SUPPORT_HDP_LS |
2077 			AMD_CG_SUPPORT_HDP_MGCG;
2078 		adev->pg_flags = 0;
2079 		adev->external_rev_id = 0x28;
2080 		break;
2081 	case CHIP_KAVERI:
2082 		adev->cg_flags =
2083 			AMD_CG_SUPPORT_GFX_MGCG |
2084 			AMD_CG_SUPPORT_GFX_MGLS |
2085 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2086 			AMD_CG_SUPPORT_GFX_CGLS |
2087 			AMD_CG_SUPPORT_GFX_CGTS |
2088 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2089 			AMD_CG_SUPPORT_GFX_CP_LS |
2090 			AMD_CG_SUPPORT_SDMA_MGCG |
2091 			AMD_CG_SUPPORT_SDMA_LS |
2092 			AMD_CG_SUPPORT_BIF_LS |
2093 			AMD_CG_SUPPORT_VCE_MGCG |
2094 			AMD_CG_SUPPORT_UVD_MGCG |
2095 			AMD_CG_SUPPORT_HDP_LS |
2096 			AMD_CG_SUPPORT_HDP_MGCG;
2097 		adev->pg_flags =
2098 			/*AMD_PG_SUPPORT_GFX_PG |
2099 			  AMD_PG_SUPPORT_GFX_SMG |
2100 			  AMD_PG_SUPPORT_GFX_DMG |*/
2101 			AMD_PG_SUPPORT_UVD |
2102 			/*AMD_PG_SUPPORT_VCE |
2103 			  AMD_PG_SUPPORT_CP |
2104 			  AMD_PG_SUPPORT_GDS |
2105 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2106 			  AMD_PG_SUPPORT_ACP |
2107 			  AMD_PG_SUPPORT_SAMU |*/
2108 			0;
2109 		if (adev->pdev->device == 0x1312 ||
2110 			adev->pdev->device == 0x1316 ||
2111 			adev->pdev->device == 0x1317)
2112 			adev->external_rev_id = 0x41;
2113 		else
2114 			adev->external_rev_id = 0x1;
2115 		break;
2116 	case CHIP_KABINI:
2117 	case CHIP_MULLINS:
2118 		adev->cg_flags =
2119 			AMD_CG_SUPPORT_GFX_MGCG |
2120 			AMD_CG_SUPPORT_GFX_MGLS |
2121 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2122 			AMD_CG_SUPPORT_GFX_CGLS |
2123 			AMD_CG_SUPPORT_GFX_CGTS |
2124 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2125 			AMD_CG_SUPPORT_GFX_CP_LS |
2126 			AMD_CG_SUPPORT_SDMA_MGCG |
2127 			AMD_CG_SUPPORT_SDMA_LS |
2128 			AMD_CG_SUPPORT_BIF_LS |
2129 			AMD_CG_SUPPORT_VCE_MGCG |
2130 			AMD_CG_SUPPORT_UVD_MGCG |
2131 			AMD_CG_SUPPORT_HDP_LS |
2132 			AMD_CG_SUPPORT_HDP_MGCG;
2133 		adev->pg_flags =
2134 			/*AMD_PG_SUPPORT_GFX_PG |
2135 			  AMD_PG_SUPPORT_GFX_SMG | */
2136 			AMD_PG_SUPPORT_UVD |
2137 			/*AMD_PG_SUPPORT_VCE |
2138 			  AMD_PG_SUPPORT_CP |
2139 			  AMD_PG_SUPPORT_GDS |
2140 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2141 			  AMD_PG_SUPPORT_SAMU |*/
2142 			0;
2143 		if (adev->asic_type == CHIP_KABINI) {
2144 			if (adev->rev_id == 0)
2145 				adev->external_rev_id = 0x81;
2146 			else if (adev->rev_id == 1)
2147 				adev->external_rev_id = 0x82;
2148 			else if (adev->rev_id == 2)
2149 				adev->external_rev_id = 0x85;
2150 		} else
2151 			adev->external_rev_id = adev->rev_id + 0xa1;
2152 		break;
2153 	default:
2154 		/* FIXME: not supported yet */
2155 		return -EINVAL;
2156 	}
2157 
2158 	amdgpu_get_pcie_info(adev);
2159 
2160 	return 0;
2161 }
2162 
2163 static int cik_common_sw_init(void *handle)
2164 {
2165 	return 0;
2166 }
2167 
2168 static int cik_common_sw_fini(void *handle)
2169 {
2170 	return 0;
2171 }
2172 
2173 static int cik_common_hw_init(void *handle)
2174 {
2175 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2176 
2177 	/* move the golden regs per IP block */
2178 	cik_init_golden_registers(adev);
2179 	/* enable pcie gen2/3 link */
2180 	cik_pcie_gen3_enable(adev);
2181 	/* enable aspm */
2182 	cik_program_aspm(adev);
2183 
2184 	return 0;
2185 }
2186 
2187 static int cik_common_hw_fini(void *handle)
2188 {
2189 	return 0;
2190 }
2191 
2192 static int cik_common_suspend(void *handle)
2193 {
2194 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2195 
2196 	amdgpu_amdkfd_suspend(adev);
2197 
2198 	return cik_common_hw_fini(adev);
2199 }
2200 
2201 static int cik_common_resume(void *handle)
2202 {
2203 	int r;
2204 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2205 
2206 	r = cik_common_hw_init(adev);
2207 	if (r)
2208 		return r;
2209 
2210 	return amdgpu_amdkfd_resume(adev);
2211 }
2212 
2213 static bool cik_common_is_idle(void *handle)
2214 {
2215 	return true;
2216 }
2217 
2218 static int cik_common_wait_for_idle(void *handle)
2219 {
2220 	return 0;
2221 }
2222 
2223 static int cik_common_soft_reset(void *handle)
2224 {
2225 	/* XXX hard reset?? */
2226 	return 0;
2227 }
2228 
2229 static int cik_common_set_clockgating_state(void *handle,
2230 					    enum amd_clockgating_state state)
2231 {
2232 	return 0;
2233 }
2234 
2235 static int cik_common_set_powergating_state(void *handle,
2236 					    enum amd_powergating_state state)
2237 {
2238 	return 0;
2239 }
2240 
2241 const struct amd_ip_funcs cik_common_ip_funcs = {
2242 	.name = "cik_common",
2243 	.early_init = cik_common_early_init,
2244 	.late_init = NULL,
2245 	.sw_init = cik_common_sw_init,
2246 	.sw_fini = cik_common_sw_fini,
2247 	.hw_init = cik_common_hw_init,
2248 	.hw_fini = cik_common_hw_fini,
2249 	.suspend = cik_common_suspend,
2250 	.resume = cik_common_resume,
2251 	.is_idle = cik_common_is_idle,
2252 	.wait_for_idle = cik_common_wait_for_idle,
2253 	.soft_reset = cik_common_soft_reset,
2254 	.set_clockgating_state = cik_common_set_clockgating_state,
2255 	.set_powergating_state = cik_common_set_powergating_state,
2256 };
2257