xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/cik.c (revision bc5aa3a0) 
1 /*
2  * Copyright 2012 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Alex Deucher
23  */
24 #include <linux/firmware.h>
25 #include <linux/slab.h>
26 #include <linux/module.h>
27 #include "drmP.h"
28 #include "amdgpu.h"
29 #include "amdgpu_atombios.h"
30 #include "amdgpu_ih.h"
31 #include "amdgpu_uvd.h"
32 #include "amdgpu_vce.h"
33 #include "cikd.h"
34 #include "atom.h"
35 #include "amd_pcie.h"
36 
37 #include "cik.h"
38 #include "gmc_v7_0.h"
39 #include "cik_ih.h"
40 #include "dce_v8_0.h"
41 #include "gfx_v7_0.h"
42 #include "cik_sdma.h"
43 #include "uvd_v4_2.h"
44 #include "vce_v2_0.h"
45 #include "cik_dpm.h"
46 
47 #include "uvd/uvd_4_2_d.h"
48 
49 #include "smu/smu_7_0_1_d.h"
50 #include "smu/smu_7_0_1_sh_mask.h"
51 
52 #include "dce/dce_8_0_d.h"
53 #include "dce/dce_8_0_sh_mask.h"
54 
55 #include "bif/bif_4_1_d.h"
56 #include "bif/bif_4_1_sh_mask.h"
57 
58 #include "gca/gfx_7_2_d.h"
59 #include "gca/gfx_7_2_enum.h"
60 #include "gca/gfx_7_2_sh_mask.h"
61 
62 #include "gmc/gmc_7_1_d.h"
63 #include "gmc/gmc_7_1_sh_mask.h"
64 
65 #include "oss/oss_2_0_d.h"
66 #include "oss/oss_2_0_sh_mask.h"
67 
68 #include "amdgpu_amdkfd.h"
69 #include "amdgpu_powerplay.h"
70 
71 /*
72  * Indirect registers accessor
73  */
74 static u32 cik_pcie_rreg(struct amdgpu_device *adev, u32 reg)
75 {
76 	unsigned long flags;
77 	u32 r;
78 
79 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
80 	WREG32(mmPCIE_INDEX, reg);
81 	(void)RREG32(mmPCIE_INDEX);
82 	r = RREG32(mmPCIE_DATA);
83 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
84 	return r;
85 }
86 
87 static void cik_pcie_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
88 {
89 	unsigned long flags;
90 
91 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
92 	WREG32(mmPCIE_INDEX, reg);
93 	(void)RREG32(mmPCIE_INDEX);
94 	WREG32(mmPCIE_DATA, v);
95 	(void)RREG32(mmPCIE_DATA);
96 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
97 }
98 
99 static u32 cik_smc_rreg(struct amdgpu_device *adev, u32 reg)
100 {
101 	unsigned long flags;
102 	u32 r;
103 
104 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
105 	WREG32(mmSMC_IND_INDEX_0, (reg));
106 	r = RREG32(mmSMC_IND_DATA_0);
107 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
108 	return r;
109 }
110 
111 static void cik_smc_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
112 {
113 	unsigned long flags;
114 
115 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
116 	WREG32(mmSMC_IND_INDEX_0, (reg));
117 	WREG32(mmSMC_IND_DATA_0, (v));
118 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
119 }
120 
121 static u32 cik_uvd_ctx_rreg(struct amdgpu_device *adev, u32 reg)
122 {
123 	unsigned long flags;
124 	u32 r;
125 
126 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
127 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
128 	r = RREG32(mmUVD_CTX_DATA);
129 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
130 	return r;
131 }
132 
133 static void cik_uvd_ctx_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
134 {
135 	unsigned long flags;
136 
137 	spin_lock_irqsave(&adev->uvd_ctx_idx_lock, flags);
138 	WREG32(mmUVD_CTX_INDEX, ((reg) & 0x1ff));
139 	WREG32(mmUVD_CTX_DATA, (v));
140 	spin_unlock_irqrestore(&adev->uvd_ctx_idx_lock, flags);
141 }
142 
143 static u32 cik_didt_rreg(struct amdgpu_device *adev, u32 reg)
144 {
145 	unsigned long flags;
146 	u32 r;
147 
148 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
149 	WREG32(mmDIDT_IND_INDEX, (reg));
150 	r = RREG32(mmDIDT_IND_DATA);
151 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
152 	return r;
153 }
154 
155 static void cik_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
156 {
157 	unsigned long flags;
158 
159 	spin_lock_irqsave(&adev->didt_idx_lock, flags);
160 	WREG32(mmDIDT_IND_INDEX, (reg));
161 	WREG32(mmDIDT_IND_DATA, (v));
162 	spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
163 }
164 
165 static const u32 bonaire_golden_spm_registers[] =
166 {
167 	0xc200, 0xe0ffffff, 0xe0000000
168 };
169 
170 static const u32 bonaire_golden_common_registers[] =
171 {
172 	0x31dc, 0xffffffff, 0x00000800,
173 	0x31dd, 0xffffffff, 0x00000800,
174 	0x31e6, 0xffffffff, 0x00007fbf,
175 	0x31e7, 0xffffffff, 0x00007faf
176 };
177 
178 static const u32 bonaire_golden_registers[] =
179 {
180 	0xcd5, 0x00000333, 0x00000333,
181 	0xcd4, 0x000c0fc0, 0x00040200,
182 	0x2684, 0x00010000, 0x00058208,
183 	0xf000, 0xffff1fff, 0x00140000,
184 	0xf080, 0xfdfc0fff, 0x00000100,
185 	0xf08d, 0x40000000, 0x40000200,
186 	0x260c, 0xffffffff, 0x00000000,
187 	0x260d, 0xf00fffff, 0x00000400,
188 	0x260e, 0x0002021c, 0x00020200,
189 	0x31e, 0x00000080, 0x00000000,
190 	0x16ec, 0x000000f0, 0x00000070,
191 	0x16f0, 0xf0311fff, 0x80300000,
192 	0x263e, 0x73773777, 0x12010001,
193 	0xd43, 0x00810000, 0x408af000,
194 	0x1c0c, 0x31000111, 0x00000011,
195 	0xbd2, 0x73773777, 0x12010001,
196 	0x883, 0x00007fb6, 0x0021a1b1,
197 	0x884, 0x00007fb6, 0x002021b1,
198 	0x860, 0x00007fb6, 0x00002191,
199 	0x886, 0x00007fb6, 0x002121b1,
200 	0x887, 0x00007fb6, 0x002021b1,
201 	0x877, 0x00007fb6, 0x00002191,
202 	0x878, 0x00007fb6, 0x00002191,
203 	0xd8a, 0x0000003f, 0x0000000a,
204 	0xd8b, 0x0000003f, 0x0000000a,
205 	0xab9, 0x00073ffe, 0x000022a2,
206 	0x903, 0x000007ff, 0x00000000,
207 	0x2285, 0xf000003f, 0x00000007,
208 	0x22fc, 0x00002001, 0x00000001,
209 	0x22c9, 0xffffffff, 0x00ffffff,
210 	0xc281, 0x0000ff0f, 0x00000000,
211 	0xa293, 0x07ffffff, 0x06000000,
212 	0x136, 0x00000fff, 0x00000100,
213 	0xf9e, 0x00000001, 0x00000002,
214 	0x2440, 0x03000000, 0x0362c688,
215 	0x2300, 0x000000ff, 0x00000001,
216 	0x390, 0x00001fff, 0x00001fff,
217 	0x2418, 0x0000007f, 0x00000020,
218 	0x2542, 0x00010000, 0x00010000,
219 	0x2b05, 0x000003ff, 0x000000f3,
220 	0x2b03, 0xffffffff, 0x00001032
221 };
222 
223 static const u32 bonaire_mgcg_cgcg_init[] =
224 {
225 	0x3108, 0xffffffff, 0xfffffffc,
226 	0xc200, 0xffffffff, 0xe0000000,
227 	0xf0a8, 0xffffffff, 0x00000100,
228 	0xf082, 0xffffffff, 0x00000100,
229 	0xf0b0, 0xffffffff, 0xc0000100,
230 	0xf0b2, 0xffffffff, 0xc0000100,
231 	0xf0b1, 0xffffffff, 0xc0000100,
232 	0x1579, 0xffffffff, 0x00600100,
233 	0xf0a0, 0xffffffff, 0x00000100,
234 	0xf085, 0xffffffff, 0x06000100,
235 	0xf088, 0xffffffff, 0x00000100,
236 	0xf086, 0xffffffff, 0x06000100,
237 	0xf081, 0xffffffff, 0x00000100,
238 	0xf0b8, 0xffffffff, 0x00000100,
239 	0xf089, 0xffffffff, 0x00000100,
240 	0xf080, 0xffffffff, 0x00000100,
241 	0xf08c, 0xffffffff, 0x00000100,
242 	0xf08d, 0xffffffff, 0x00000100,
243 	0xf094, 0xffffffff, 0x00000100,
244 	0xf095, 0xffffffff, 0x00000100,
245 	0xf096, 0xffffffff, 0x00000100,
246 	0xf097, 0xffffffff, 0x00000100,
247 	0xf098, 0xffffffff, 0x00000100,
248 	0xf09f, 0xffffffff, 0x00000100,
249 	0xf09e, 0xffffffff, 0x00000100,
250 	0xf084, 0xffffffff, 0x06000100,
251 	0xf0a4, 0xffffffff, 0x00000100,
252 	0xf09d, 0xffffffff, 0x00000100,
253 	0xf0ad, 0xffffffff, 0x00000100,
254 	0xf0ac, 0xffffffff, 0x00000100,
255 	0xf09c, 0xffffffff, 0x00000100,
256 	0xc200, 0xffffffff, 0xe0000000,
257 	0xf008, 0xffffffff, 0x00010000,
258 	0xf009, 0xffffffff, 0x00030002,
259 	0xf00a, 0xffffffff, 0x00040007,
260 	0xf00b, 0xffffffff, 0x00060005,
261 	0xf00c, 0xffffffff, 0x00090008,
262 	0xf00d, 0xffffffff, 0x00010000,
263 	0xf00e, 0xffffffff, 0x00030002,
264 	0xf00f, 0xffffffff, 0x00040007,
265 	0xf010, 0xffffffff, 0x00060005,
266 	0xf011, 0xffffffff, 0x00090008,
267 	0xf012, 0xffffffff, 0x00010000,
268 	0xf013, 0xffffffff, 0x00030002,
269 	0xf014, 0xffffffff, 0x00040007,
270 	0xf015, 0xffffffff, 0x00060005,
271 	0xf016, 0xffffffff, 0x00090008,
272 	0xf017, 0xffffffff, 0x00010000,
273 	0xf018, 0xffffffff, 0x00030002,
274 	0xf019, 0xffffffff, 0x00040007,
275 	0xf01a, 0xffffffff, 0x00060005,
276 	0xf01b, 0xffffffff, 0x00090008,
277 	0xf01c, 0xffffffff, 0x00010000,
278 	0xf01d, 0xffffffff, 0x00030002,
279 	0xf01e, 0xffffffff, 0x00040007,
280 	0xf01f, 0xffffffff, 0x00060005,
281 	0xf020, 0xffffffff, 0x00090008,
282 	0xf021, 0xffffffff, 0x00010000,
283 	0xf022, 0xffffffff, 0x00030002,
284 	0xf023, 0xffffffff, 0x00040007,
285 	0xf024, 0xffffffff, 0x00060005,
286 	0xf025, 0xffffffff, 0x00090008,
287 	0xf026, 0xffffffff, 0x00010000,
288 	0xf027, 0xffffffff, 0x00030002,
289 	0xf028, 0xffffffff, 0x00040007,
290 	0xf029, 0xffffffff, 0x00060005,
291 	0xf02a, 0xffffffff, 0x00090008,
292 	0xf000, 0xffffffff, 0x96e00200,
293 	0x21c2, 0xffffffff, 0x00900100,
294 	0x3109, 0xffffffff, 0x0020003f,
295 	0xe, 0xffffffff, 0x0140001c,
296 	0xf, 0x000f0000, 0x000f0000,
297 	0x88, 0xffffffff, 0xc060000c,
298 	0x89, 0xc0000fff, 0x00000100,
299 	0x3e4, 0xffffffff, 0x00000100,
300 	0x3e6, 0x00000101, 0x00000000,
301 	0x82a, 0xffffffff, 0x00000104,
302 	0x1579, 0xff000fff, 0x00000100,
303 	0xc33, 0xc0000fff, 0x00000104,
304 	0x3079, 0x00000001, 0x00000001,
305 	0x3403, 0xff000ff0, 0x00000100,
306 	0x3603, 0xff000ff0, 0x00000100
307 };
308 
309 static const u32 spectre_golden_spm_registers[] =
310 {
311 	0xc200, 0xe0ffffff, 0xe0000000
312 };
313 
314 static const u32 spectre_golden_common_registers[] =
315 {
316 	0x31dc, 0xffffffff, 0x00000800,
317 	0x31dd, 0xffffffff, 0x00000800,
318 	0x31e6, 0xffffffff, 0x00007fbf,
319 	0x31e7, 0xffffffff, 0x00007faf
320 };
321 
322 static const u32 spectre_golden_registers[] =
323 {
324 	0xf000, 0xffff1fff, 0x96940200,
325 	0xf003, 0xffff0001, 0xff000000,
326 	0xf080, 0xfffc0fff, 0x00000100,
327 	0x1bb6, 0x00010101, 0x00010000,
328 	0x260d, 0xf00fffff, 0x00000400,
329 	0x260e, 0xfffffffc, 0x00020200,
330 	0x16ec, 0x000000f0, 0x00000070,
331 	0x16f0, 0xf0311fff, 0x80300000,
332 	0x263e, 0x73773777, 0x12010001,
333 	0x26df, 0x00ff0000, 0x00fc0000,
334 	0xbd2, 0x73773777, 0x12010001,
335 	0x2285, 0xf000003f, 0x00000007,
336 	0x22c9, 0xffffffff, 0x00ffffff,
337 	0xa0d4, 0x3f3f3fff, 0x00000082,
338 	0xa0d5, 0x0000003f, 0x00000000,
339 	0xf9e, 0x00000001, 0x00000002,
340 	0x244f, 0xffff03df, 0x00000004,
341 	0x31da, 0x00000008, 0x00000008,
342 	0x2300, 0x000008ff, 0x00000800,
343 	0x2542, 0x00010000, 0x00010000,
344 	0x2b03, 0xffffffff, 0x54763210,
345 	0x853e, 0x01ff01ff, 0x00000002,
346 	0x8526, 0x007ff800, 0x00200000,
347 	0x8057, 0xffffffff, 0x00000f40,
348 	0xc24d, 0xffffffff, 0x00000001
349 };
350 
351 static const u32 spectre_mgcg_cgcg_init[] =
352 {
353 	0x3108, 0xffffffff, 0xfffffffc,
354 	0xc200, 0xffffffff, 0xe0000000,
355 	0xf0a8, 0xffffffff, 0x00000100,
356 	0xf082, 0xffffffff, 0x00000100,
357 	0xf0b0, 0xffffffff, 0x00000100,
358 	0xf0b2, 0xffffffff, 0x00000100,
359 	0xf0b1, 0xffffffff, 0x00000100,
360 	0x1579, 0xffffffff, 0x00600100,
361 	0xf0a0, 0xffffffff, 0x00000100,
362 	0xf085, 0xffffffff, 0x06000100,
363 	0xf088, 0xffffffff, 0x00000100,
364 	0xf086, 0xffffffff, 0x06000100,
365 	0xf081, 0xffffffff, 0x00000100,
366 	0xf0b8, 0xffffffff, 0x00000100,
367 	0xf089, 0xffffffff, 0x00000100,
368 	0xf080, 0xffffffff, 0x00000100,
369 	0xf08c, 0xffffffff, 0x00000100,
370 	0xf08d, 0xffffffff, 0x00000100,
371 	0xf094, 0xffffffff, 0x00000100,
372 	0xf095, 0xffffffff, 0x00000100,
373 	0xf096, 0xffffffff, 0x00000100,
374 	0xf097, 0xffffffff, 0x00000100,
375 	0xf098, 0xffffffff, 0x00000100,
376 	0xf09f, 0xffffffff, 0x00000100,
377 	0xf09e, 0xffffffff, 0x00000100,
378 	0xf084, 0xffffffff, 0x06000100,
379 	0xf0a4, 0xffffffff, 0x00000100,
380 	0xf09d, 0xffffffff, 0x00000100,
381 	0xf0ad, 0xffffffff, 0x00000100,
382 	0xf0ac, 0xffffffff, 0x00000100,
383 	0xf09c, 0xffffffff, 0x00000100,
384 	0xc200, 0xffffffff, 0xe0000000,
385 	0xf008, 0xffffffff, 0x00010000,
386 	0xf009, 0xffffffff, 0x00030002,
387 	0xf00a, 0xffffffff, 0x00040007,
388 	0xf00b, 0xffffffff, 0x00060005,
389 	0xf00c, 0xffffffff, 0x00090008,
390 	0xf00d, 0xffffffff, 0x00010000,
391 	0xf00e, 0xffffffff, 0x00030002,
392 	0xf00f, 0xffffffff, 0x00040007,
393 	0xf010, 0xffffffff, 0x00060005,
394 	0xf011, 0xffffffff, 0x00090008,
395 	0xf012, 0xffffffff, 0x00010000,
396 	0xf013, 0xffffffff, 0x00030002,
397 	0xf014, 0xffffffff, 0x00040007,
398 	0xf015, 0xffffffff, 0x00060005,
399 	0xf016, 0xffffffff, 0x00090008,
400 	0xf017, 0xffffffff, 0x00010000,
401 	0xf018, 0xffffffff, 0x00030002,
402 	0xf019, 0xffffffff, 0x00040007,
403 	0xf01a, 0xffffffff, 0x00060005,
404 	0xf01b, 0xffffffff, 0x00090008,
405 	0xf01c, 0xffffffff, 0x00010000,
406 	0xf01d, 0xffffffff, 0x00030002,
407 	0xf01e, 0xffffffff, 0x00040007,
408 	0xf01f, 0xffffffff, 0x00060005,
409 	0xf020, 0xffffffff, 0x00090008,
410 	0xf021, 0xffffffff, 0x00010000,
411 	0xf022, 0xffffffff, 0x00030002,
412 	0xf023, 0xffffffff, 0x00040007,
413 	0xf024, 0xffffffff, 0x00060005,
414 	0xf025, 0xffffffff, 0x00090008,
415 	0xf026, 0xffffffff, 0x00010000,
416 	0xf027, 0xffffffff, 0x00030002,
417 	0xf028, 0xffffffff, 0x00040007,
418 	0xf029, 0xffffffff, 0x00060005,
419 	0xf02a, 0xffffffff, 0x00090008,
420 	0xf02b, 0xffffffff, 0x00010000,
421 	0xf02c, 0xffffffff, 0x00030002,
422 	0xf02d, 0xffffffff, 0x00040007,
423 	0xf02e, 0xffffffff, 0x00060005,
424 	0xf02f, 0xffffffff, 0x00090008,
425 	0xf000, 0xffffffff, 0x96e00200,
426 	0x21c2, 0xffffffff, 0x00900100,
427 	0x3109, 0xffffffff, 0x0020003f,
428 	0xe, 0xffffffff, 0x0140001c,
429 	0xf, 0x000f0000, 0x000f0000,
430 	0x88, 0xffffffff, 0xc060000c,
431 	0x89, 0xc0000fff, 0x00000100,
432 	0x3e4, 0xffffffff, 0x00000100,
433 	0x3e6, 0x00000101, 0x00000000,
434 	0x82a, 0xffffffff, 0x00000104,
435 	0x1579, 0xff000fff, 0x00000100,
436 	0xc33, 0xc0000fff, 0x00000104,
437 	0x3079, 0x00000001, 0x00000001,
438 	0x3403, 0xff000ff0, 0x00000100,
439 	0x3603, 0xff000ff0, 0x00000100
440 };
441 
442 static const u32 kalindi_golden_spm_registers[] =
443 {
444 	0xc200, 0xe0ffffff, 0xe0000000
445 };
446 
447 static const u32 kalindi_golden_common_registers[] =
448 {
449 	0x31dc, 0xffffffff, 0x00000800,
450 	0x31dd, 0xffffffff, 0x00000800,
451 	0x31e6, 0xffffffff, 0x00007fbf,
452 	0x31e7, 0xffffffff, 0x00007faf
453 };
454 
455 static const u32 kalindi_golden_registers[] =
456 {
457 	0xf000, 0xffffdfff, 0x6e944040,
458 	0x1579, 0xff607fff, 0xfc000100,
459 	0xf088, 0xff000fff, 0x00000100,
460 	0xf089, 0xff000fff, 0x00000100,
461 	0xf080, 0xfffc0fff, 0x00000100,
462 	0x1bb6, 0x00010101, 0x00010000,
463 	0x260c, 0xffffffff, 0x00000000,
464 	0x260d, 0xf00fffff, 0x00000400,
465 	0x16ec, 0x000000f0, 0x00000070,
466 	0x16f0, 0xf0311fff, 0x80300000,
467 	0x263e, 0x73773777, 0x12010001,
468 	0x263f, 0xffffffff, 0x00000010,
469 	0x26df, 0x00ff0000, 0x00fc0000,
470 	0x200c, 0x00001f0f, 0x0000100a,
471 	0xbd2, 0x73773777, 0x12010001,
472 	0x902, 0x000fffff, 0x000c007f,
473 	0x2285, 0xf000003f, 0x00000007,
474 	0x22c9, 0x3fff3fff, 0x00ffcfff,
475 	0xc281, 0x0000ff0f, 0x00000000,
476 	0xa293, 0x07ffffff, 0x06000000,
477 	0x136, 0x00000fff, 0x00000100,
478 	0xf9e, 0x00000001, 0x00000002,
479 	0x31da, 0x00000008, 0x00000008,
480 	0x2300, 0x000000ff, 0x00000003,
481 	0x853e, 0x01ff01ff, 0x00000002,
482 	0x8526, 0x007ff800, 0x00200000,
483 	0x8057, 0xffffffff, 0x00000f40,
484 	0x2231, 0x001f3ae3, 0x00000082,
485 	0x2235, 0x0000001f, 0x00000010,
486 	0xc24d, 0xffffffff, 0x00000000
487 };
488 
489 static const u32 kalindi_mgcg_cgcg_init[] =
490 {
491 	0x3108, 0xffffffff, 0xfffffffc,
492 	0xc200, 0xffffffff, 0xe0000000,
493 	0xf0a8, 0xffffffff, 0x00000100,
494 	0xf082, 0xffffffff, 0x00000100,
495 	0xf0b0, 0xffffffff, 0x00000100,
496 	0xf0b2, 0xffffffff, 0x00000100,
497 	0xf0b1, 0xffffffff, 0x00000100,
498 	0x1579, 0xffffffff, 0x00600100,
499 	0xf0a0, 0xffffffff, 0x00000100,
500 	0xf085, 0xffffffff, 0x06000100,
501 	0xf088, 0xffffffff, 0x00000100,
502 	0xf086, 0xffffffff, 0x06000100,
503 	0xf081, 0xffffffff, 0x00000100,
504 	0xf0b8, 0xffffffff, 0x00000100,
505 	0xf089, 0xffffffff, 0x00000100,
506 	0xf080, 0xffffffff, 0x00000100,
507 	0xf08c, 0xffffffff, 0x00000100,
508 	0xf08d, 0xffffffff, 0x00000100,
509 	0xf094, 0xffffffff, 0x00000100,
510 	0xf095, 0xffffffff, 0x00000100,
511 	0xf096, 0xffffffff, 0x00000100,
512 	0xf097, 0xffffffff, 0x00000100,
513 	0xf098, 0xffffffff, 0x00000100,
514 	0xf09f, 0xffffffff, 0x00000100,
515 	0xf09e, 0xffffffff, 0x00000100,
516 	0xf084, 0xffffffff, 0x06000100,
517 	0xf0a4, 0xffffffff, 0x00000100,
518 	0xf09d, 0xffffffff, 0x00000100,
519 	0xf0ad, 0xffffffff, 0x00000100,
520 	0xf0ac, 0xffffffff, 0x00000100,
521 	0xf09c, 0xffffffff, 0x00000100,
522 	0xc200, 0xffffffff, 0xe0000000,
523 	0xf008, 0xffffffff, 0x00010000,
524 	0xf009, 0xffffffff, 0x00030002,
525 	0xf00a, 0xffffffff, 0x00040007,
526 	0xf00b, 0xffffffff, 0x00060005,
527 	0xf00c, 0xffffffff, 0x00090008,
528 	0xf00d, 0xffffffff, 0x00010000,
529 	0xf00e, 0xffffffff, 0x00030002,
530 	0xf00f, 0xffffffff, 0x00040007,
531 	0xf010, 0xffffffff, 0x00060005,
532 	0xf011, 0xffffffff, 0x00090008,
533 	0xf000, 0xffffffff, 0x96e00200,
534 	0x21c2, 0xffffffff, 0x00900100,
535 	0x3109, 0xffffffff, 0x0020003f,
536 	0xe, 0xffffffff, 0x0140001c,
537 	0xf, 0x000f0000, 0x000f0000,
538 	0x88, 0xffffffff, 0xc060000c,
539 	0x89, 0xc0000fff, 0x00000100,
540 	0x82a, 0xffffffff, 0x00000104,
541 	0x1579, 0xff000fff, 0x00000100,
542 	0xc33, 0xc0000fff, 0x00000104,
543 	0x3079, 0x00000001, 0x00000001,
544 	0x3403, 0xff000ff0, 0x00000100,
545 	0x3603, 0xff000ff0, 0x00000100
546 };
547 
548 static const u32 hawaii_golden_spm_registers[] =
549 {
550 	0xc200, 0xe0ffffff, 0xe0000000
551 };
552 
553 static const u32 hawaii_golden_common_registers[] =
554 {
555 	0xc200, 0xffffffff, 0xe0000000,
556 	0xa0d4, 0xffffffff, 0x3a00161a,
557 	0xa0d5, 0xffffffff, 0x0000002e,
558 	0x2684, 0xffffffff, 0x00018208,
559 	0x263e, 0xffffffff, 0x12011003
560 };
561 
562 static const u32 hawaii_golden_registers[] =
563 {
564 	0xcd5, 0x00000333, 0x00000333,
565 	0x2684, 0x00010000, 0x00058208,
566 	0x260c, 0xffffffff, 0x00000000,
567 	0x260d, 0xf00fffff, 0x00000400,
568 	0x260e, 0x0002021c, 0x00020200,
569 	0x31e, 0x00000080, 0x00000000,
570 	0x16ec, 0x000000f0, 0x00000070,
571 	0x16f0, 0xf0311fff, 0x80300000,
572 	0xd43, 0x00810000, 0x408af000,
573 	0x1c0c, 0x31000111, 0x00000011,
574 	0xbd2, 0x73773777, 0x12010001,
575 	0x848, 0x0000007f, 0x0000001b,
576 	0x877, 0x00007fb6, 0x00002191,
577 	0xd8a, 0x0000003f, 0x0000000a,
578 	0xd8b, 0x0000003f, 0x0000000a,
579 	0xab9, 0x00073ffe, 0x000022a2,
580 	0x903, 0x000007ff, 0x00000000,
581 	0x22fc, 0x00002001, 0x00000001,
582 	0x22c9, 0xffffffff, 0x00ffffff,
583 	0xc281, 0x0000ff0f, 0x00000000,
584 	0xa293, 0x07ffffff, 0x06000000,
585 	0xf9e, 0x00000001, 0x00000002,
586 	0x31da, 0x00000008, 0x00000008,
587 	0x31dc, 0x00000f00, 0x00000800,
588 	0x31dd, 0x00000f00, 0x00000800,
589 	0x31e6, 0x00ffffff, 0x00ff7fbf,
590 	0x31e7, 0x00ffffff, 0x00ff7faf,
591 	0x2300, 0x000000ff, 0x00000800,
592 	0x390, 0x00001fff, 0x00001fff,
593 	0x2418, 0x0000007f, 0x00000020,
594 	0x2542, 0x00010000, 0x00010000,
595 	0x2b80, 0x00100000, 0x000ff07c,
596 	0x2b05, 0x000003ff, 0x0000000f,
597 	0x2b04, 0xffffffff, 0x7564fdec,
598 	0x2b03, 0xffffffff, 0x3120b9a8,
599 	0x2b02, 0x20000000, 0x0f9c0000
600 };
601 
602 static const u32 hawaii_mgcg_cgcg_init[] =
603 {
604 	0x3108, 0xffffffff, 0xfffffffd,
605 	0xc200, 0xffffffff, 0xe0000000,
606 	0xf0a8, 0xffffffff, 0x00000100,
607 	0xf082, 0xffffffff, 0x00000100,
608 	0xf0b0, 0xffffffff, 0x00000100,
609 	0xf0b2, 0xffffffff, 0x00000100,
610 	0xf0b1, 0xffffffff, 0x00000100,
611 	0x1579, 0xffffffff, 0x00200100,
612 	0xf0a0, 0xffffffff, 0x00000100,
613 	0xf085, 0xffffffff, 0x06000100,
614 	0xf088, 0xffffffff, 0x00000100,
615 	0xf086, 0xffffffff, 0x06000100,
616 	0xf081, 0xffffffff, 0x00000100,
617 	0xf0b8, 0xffffffff, 0x00000100,
618 	0xf089, 0xffffffff, 0x00000100,
619 	0xf080, 0xffffffff, 0x00000100,
620 	0xf08c, 0xffffffff, 0x00000100,
621 	0xf08d, 0xffffffff, 0x00000100,
622 	0xf094, 0xffffffff, 0x00000100,
623 	0xf095, 0xffffffff, 0x00000100,
624 	0xf096, 0xffffffff, 0x00000100,
625 	0xf097, 0xffffffff, 0x00000100,
626 	0xf098, 0xffffffff, 0x00000100,
627 	0xf09f, 0xffffffff, 0x00000100,
628 	0xf09e, 0xffffffff, 0x00000100,
629 	0xf084, 0xffffffff, 0x06000100,
630 	0xf0a4, 0xffffffff, 0x00000100,
631 	0xf09d, 0xffffffff, 0x00000100,
632 	0xf0ad, 0xffffffff, 0x00000100,
633 	0xf0ac, 0xffffffff, 0x00000100,
634 	0xf09c, 0xffffffff, 0x00000100,
635 	0xc200, 0xffffffff, 0xe0000000,
636 	0xf008, 0xffffffff, 0x00010000,
637 	0xf009, 0xffffffff, 0x00030002,
638 	0xf00a, 0xffffffff, 0x00040007,
639 	0xf00b, 0xffffffff, 0x00060005,
640 	0xf00c, 0xffffffff, 0x00090008,
641 	0xf00d, 0xffffffff, 0x00010000,
642 	0xf00e, 0xffffffff, 0x00030002,
643 	0xf00f, 0xffffffff, 0x00040007,
644 	0xf010, 0xffffffff, 0x00060005,
645 	0xf011, 0xffffffff, 0x00090008,
646 	0xf012, 0xffffffff, 0x00010000,
647 	0xf013, 0xffffffff, 0x00030002,
648 	0xf014, 0xffffffff, 0x00040007,
649 	0xf015, 0xffffffff, 0x00060005,
650 	0xf016, 0xffffffff, 0x00090008,
651 	0xf017, 0xffffffff, 0x00010000,
652 	0xf018, 0xffffffff, 0x00030002,
653 	0xf019, 0xffffffff, 0x00040007,
654 	0xf01a, 0xffffffff, 0x00060005,
655 	0xf01b, 0xffffffff, 0x00090008,
656 	0xf01c, 0xffffffff, 0x00010000,
657 	0xf01d, 0xffffffff, 0x00030002,
658 	0xf01e, 0xffffffff, 0x00040007,
659 	0xf01f, 0xffffffff, 0x00060005,
660 	0xf020, 0xffffffff, 0x00090008,
661 	0xf021, 0xffffffff, 0x00010000,
662 	0xf022, 0xffffffff, 0x00030002,
663 	0xf023, 0xffffffff, 0x00040007,
664 	0xf024, 0xffffffff, 0x00060005,
665 	0xf025, 0xffffffff, 0x00090008,
666 	0xf026, 0xffffffff, 0x00010000,
667 	0xf027, 0xffffffff, 0x00030002,
668 	0xf028, 0xffffffff, 0x00040007,
669 	0xf029, 0xffffffff, 0x00060005,
670 	0xf02a, 0xffffffff, 0x00090008,
671 	0xf02b, 0xffffffff, 0x00010000,
672 	0xf02c, 0xffffffff, 0x00030002,
673 	0xf02d, 0xffffffff, 0x00040007,
674 	0xf02e, 0xffffffff, 0x00060005,
675 	0xf02f, 0xffffffff, 0x00090008,
676 	0xf030, 0xffffffff, 0x00010000,
677 	0xf031, 0xffffffff, 0x00030002,
678 	0xf032, 0xffffffff, 0x00040007,
679 	0xf033, 0xffffffff, 0x00060005,
680 	0xf034, 0xffffffff, 0x00090008,
681 	0xf035, 0xffffffff, 0x00010000,
682 	0xf036, 0xffffffff, 0x00030002,
683 	0xf037, 0xffffffff, 0x00040007,
684 	0xf038, 0xffffffff, 0x00060005,
685 	0xf039, 0xffffffff, 0x00090008,
686 	0xf03a, 0xffffffff, 0x00010000,
687 	0xf03b, 0xffffffff, 0x00030002,
688 	0xf03c, 0xffffffff, 0x00040007,
689 	0xf03d, 0xffffffff, 0x00060005,
690 	0xf03e, 0xffffffff, 0x00090008,
691 	0x30c6, 0xffffffff, 0x00020200,
692 	0xcd4, 0xffffffff, 0x00000200,
693 	0x570, 0xffffffff, 0x00000400,
694 	0x157a, 0xffffffff, 0x00000000,
695 	0xbd4, 0xffffffff, 0x00000902,
696 	0xf000, 0xffffffff, 0x96940200,
697 	0x21c2, 0xffffffff, 0x00900100,
698 	0x3109, 0xffffffff, 0x0020003f,
699 	0xe, 0xffffffff, 0x0140001c,
700 	0xf, 0x000f0000, 0x000f0000,
701 	0x88, 0xffffffff, 0xc060000c,
702 	0x89, 0xc0000fff, 0x00000100,
703 	0x3e4, 0xffffffff, 0x00000100,
704 	0x3e6, 0x00000101, 0x00000000,
705 	0x82a, 0xffffffff, 0x00000104,
706 	0x1579, 0xff000fff, 0x00000100,
707 	0xc33, 0xc0000fff, 0x00000104,
708 	0x3079, 0x00000001, 0x00000001,
709 	0x3403, 0xff000ff0, 0x00000100,
710 	0x3603, 0xff000ff0, 0x00000100
711 };
712 
713 static const u32 godavari_golden_registers[] =
714 {
715 	0x1579, 0xff607fff, 0xfc000100,
716 	0x1bb6, 0x00010101, 0x00010000,
717 	0x260c, 0xffffffff, 0x00000000,
718 	0x260c0, 0xf00fffff, 0x00000400,
719 	0x184c, 0xffffffff, 0x00010000,
720 	0x16ec, 0x000000f0, 0x00000070,
721 	0x16f0, 0xf0311fff, 0x80300000,
722 	0x263e, 0x73773777, 0x12010001,
723 	0x263f, 0xffffffff, 0x00000010,
724 	0x200c, 0x00001f0f, 0x0000100a,
725 	0xbd2, 0x73773777, 0x12010001,
726 	0x902, 0x000fffff, 0x000c007f,
727 	0x2285, 0xf000003f, 0x00000007,
728 	0x22c9, 0xffffffff, 0x00ff0fff,
729 	0xc281, 0x0000ff0f, 0x00000000,
730 	0xa293, 0x07ffffff, 0x06000000,
731 	0x136, 0x00000fff, 0x00000100,
732 	0x3405, 0x00010000, 0x00810001,
733 	0x3605, 0x00010000, 0x00810001,
734 	0xf9e, 0x00000001, 0x00000002,
735 	0x31da, 0x00000008, 0x00000008,
736 	0x31dc, 0x00000f00, 0x00000800,
737 	0x31dd, 0x00000f00, 0x00000800,
738 	0x31e6, 0x00ffffff, 0x00ff7fbf,
739 	0x31e7, 0x00ffffff, 0x00ff7faf,
740 	0x2300, 0x000000ff, 0x00000001,
741 	0x853e, 0x01ff01ff, 0x00000002,
742 	0x8526, 0x007ff800, 0x00200000,
743 	0x8057, 0xffffffff, 0x00000f40,
744 	0x2231, 0x001f3ae3, 0x00000082,
745 	0x2235, 0x0000001f, 0x00000010,
746 	0xc24d, 0xffffffff, 0x00000000
747 };
748 
749 static void cik_init_golden_registers(struct amdgpu_device *adev)
750 {
751 	/* Some of the registers might be dependent on GRBM_GFX_INDEX */
752 	mutex_lock(&adev->grbm_idx_mutex);
753 
754 	switch (adev->asic_type) {
755 	case CHIP_BONAIRE:
756 		amdgpu_program_register_sequence(adev,
757 						 bonaire_mgcg_cgcg_init,
758 						 (const u32)ARRAY_SIZE(bonaire_mgcg_cgcg_init));
759 		amdgpu_program_register_sequence(adev,
760 						 bonaire_golden_registers,
761 						 (const u32)ARRAY_SIZE(bonaire_golden_registers));
762 		amdgpu_program_register_sequence(adev,
763 						 bonaire_golden_common_registers,
764 						 (const u32)ARRAY_SIZE(bonaire_golden_common_registers));
765 		amdgpu_program_register_sequence(adev,
766 						 bonaire_golden_spm_registers,
767 						 (const u32)ARRAY_SIZE(bonaire_golden_spm_registers));
768 		break;
769 	case CHIP_KABINI:
770 		amdgpu_program_register_sequence(adev,
771 						 kalindi_mgcg_cgcg_init,
772 						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
773 		amdgpu_program_register_sequence(adev,
774 						 kalindi_golden_registers,
775 						 (const u32)ARRAY_SIZE(kalindi_golden_registers));
776 		amdgpu_program_register_sequence(adev,
777 						 kalindi_golden_common_registers,
778 						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
779 		amdgpu_program_register_sequence(adev,
780 						 kalindi_golden_spm_registers,
781 						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
782 		break;
783 	case CHIP_MULLINS:
784 		amdgpu_program_register_sequence(adev,
785 						 kalindi_mgcg_cgcg_init,
786 						 (const u32)ARRAY_SIZE(kalindi_mgcg_cgcg_init));
787 		amdgpu_program_register_sequence(adev,
788 						 godavari_golden_registers,
789 						 (const u32)ARRAY_SIZE(godavari_golden_registers));
790 		amdgpu_program_register_sequence(adev,
791 						 kalindi_golden_common_registers,
792 						 (const u32)ARRAY_SIZE(kalindi_golden_common_registers));
793 		amdgpu_program_register_sequence(adev,
794 						 kalindi_golden_spm_registers,
795 						 (const u32)ARRAY_SIZE(kalindi_golden_spm_registers));
796 		break;
797 	case CHIP_KAVERI:
798 		amdgpu_program_register_sequence(adev,
799 						 spectre_mgcg_cgcg_init,
800 						 (const u32)ARRAY_SIZE(spectre_mgcg_cgcg_init));
801 		amdgpu_program_register_sequence(adev,
802 						 spectre_golden_registers,
803 						 (const u32)ARRAY_SIZE(spectre_golden_registers));
804 		amdgpu_program_register_sequence(adev,
805 						 spectre_golden_common_registers,
806 						 (const u32)ARRAY_SIZE(spectre_golden_common_registers));
807 		amdgpu_program_register_sequence(adev,
808 						 spectre_golden_spm_registers,
809 						 (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
810 		break;
811 	case CHIP_HAWAII:
812 		amdgpu_program_register_sequence(adev,
813 						 hawaii_mgcg_cgcg_init,
814 						 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
815 		amdgpu_program_register_sequence(adev,
816 						 hawaii_golden_registers,
817 						 (const u32)ARRAY_SIZE(hawaii_golden_registers));
818 		amdgpu_program_register_sequence(adev,
819 						 hawaii_golden_common_registers,
820 						 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
821 		amdgpu_program_register_sequence(adev,
822 						 hawaii_golden_spm_registers,
823 						 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
824 		break;
825 	default:
826 		break;
827 	}
828 	mutex_unlock(&adev->grbm_idx_mutex);
829 }
830 
831 /**
832  * cik_get_xclk - get the xclk
833  *
834  * @adev: amdgpu_device pointer
835  *
836  * Returns the reference clock used by the gfx engine
837  * (CIK).
838  */
839 static u32 cik_get_xclk(struct amdgpu_device *adev)
840 {
841 	u32 reference_clock = adev->clock.spll.reference_freq;
842 
843 	if (adev->flags & AMD_IS_APU) {
844 		if (RREG32_SMC(ixGENERAL_PWRMGT) & GENERAL_PWRMGT__GPU_COUNTER_CLK_MASK)
845 			return reference_clock / 2;
846 	} else {
847 		if (RREG32_SMC(ixCG_CLKPIN_CNTL) & CG_CLKPIN_CNTL__XTALIN_DIVIDE_MASK)
848 			return reference_clock / 4;
849 	}
850 	return reference_clock;
851 }
852 
853 /**
854  * cik_srbm_select - select specific register instances
855  *
856  * @adev: amdgpu_device pointer
857  * @me: selected ME (micro engine)
858  * @pipe: pipe
859  * @queue: queue
860  * @vmid: VMID
861  *
862  * Switches the currently active registers instances.  Some
863  * registers are instanced per VMID, others are instanced per
864  * me/pipe/queue combination.
865  */
866 void cik_srbm_select(struct amdgpu_device *adev,
867 		     u32 me, u32 pipe, u32 queue, u32 vmid)
868 {
869 	u32 srbm_gfx_cntl =
870 		(((pipe << SRBM_GFX_CNTL__PIPEID__SHIFT) & SRBM_GFX_CNTL__PIPEID_MASK)|
871 		((me << SRBM_GFX_CNTL__MEID__SHIFT) & SRBM_GFX_CNTL__MEID_MASK)|
872 		((vmid << SRBM_GFX_CNTL__VMID__SHIFT) & SRBM_GFX_CNTL__VMID_MASK)|
873 		((queue << SRBM_GFX_CNTL__QUEUEID__SHIFT) & SRBM_GFX_CNTL__QUEUEID_MASK));
874 	WREG32(mmSRBM_GFX_CNTL, srbm_gfx_cntl);
875 }
876 
877 static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
878 {
879 	uint32_t tmp;
880 
881 	tmp = RREG32(mmCONFIG_CNTL);
882 	if (!state)
883 		tmp |= CONFIG_CNTL__VGA_DIS_MASK;
884 	else
885 		tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
886 	WREG32(mmCONFIG_CNTL, tmp);
887 }
888 
889 static bool cik_read_disabled_bios(struct amdgpu_device *adev)
890 {
891 	u32 bus_cntl;
892 	u32 d1vga_control = 0;
893 	u32 d2vga_control = 0;
894 	u32 vga_render_control = 0;
895 	u32 rom_cntl;
896 	bool r;
897 
898 	bus_cntl = RREG32(mmBUS_CNTL);
899 	if (adev->mode_info.num_crtc) {
900 		d1vga_control = RREG32(mmD1VGA_CONTROL);
901 		d2vga_control = RREG32(mmD2VGA_CONTROL);
902 		vga_render_control = RREG32(mmVGA_RENDER_CONTROL);
903 	}
904 	rom_cntl = RREG32_SMC(ixROM_CNTL);
905 
906 	/* enable the rom */
907 	WREG32(mmBUS_CNTL, (bus_cntl & ~BUS_CNTL__BIOS_ROM_DIS_MASK));
908 	if (adev->mode_info.num_crtc) {
909 		/* Disable VGA mode */
910 		WREG32(mmD1VGA_CONTROL,
911 		       (d1vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
912 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
913 		WREG32(mmD2VGA_CONTROL,
914 		       (d2vga_control & ~(D1VGA_CONTROL__D1VGA_MODE_ENABLE_MASK |
915 					  D1VGA_CONTROL__D1VGA_TIMING_SELECT_MASK)));
916 		WREG32(mmVGA_RENDER_CONTROL,
917 		       (vga_render_control & ~VGA_RENDER_CONTROL__VGA_VSTATUS_CNTL_MASK));
918 	}
919 	WREG32_SMC(ixROM_CNTL, rom_cntl | ROM_CNTL__SCK_OVERWRITE_MASK);
920 
921 	r = amdgpu_read_bios(adev);
922 
923 	/* restore regs */
924 	WREG32(mmBUS_CNTL, bus_cntl);
925 	if (adev->mode_info.num_crtc) {
926 		WREG32(mmD1VGA_CONTROL, d1vga_control);
927 		WREG32(mmD2VGA_CONTROL, d2vga_control);
928 		WREG32(mmVGA_RENDER_CONTROL, vga_render_control);
929 	}
930 	WREG32_SMC(ixROM_CNTL, rom_cntl);
931 	return r;
932 }
933 
934 static bool cik_read_bios_from_rom(struct amdgpu_device *adev,
935 				   u8 *bios, u32 length_bytes)
936 {
937 	u32 *dw_ptr;
938 	unsigned long flags;
939 	u32 i, length_dw;
940 
941 	if (bios == NULL)
942 		return false;
943 	if (length_bytes == 0)
944 		return false;
945 	/* APU vbios image is part of sbios image */
946 	if (adev->flags & AMD_IS_APU)
947 		return false;
948 
949 	dw_ptr = (u32 *)bios;
950 	length_dw = ALIGN(length_bytes, 4) / 4;
951 	/* take the smc lock since we are using the smc index */
952 	spin_lock_irqsave(&adev->smc_idx_lock, flags);
953 	/* set rom index to 0 */
954 	WREG32(mmSMC_IND_INDEX_0, ixROM_INDEX);
955 	WREG32(mmSMC_IND_DATA_0, 0);
956 	/* set index to data for continous read */
957 	WREG32(mmSMC_IND_INDEX_0, ixROM_DATA);
958 	for (i = 0; i < length_dw; i++)
959 		dw_ptr[i] = RREG32(mmSMC_IND_DATA_0);
960 	spin_unlock_irqrestore(&adev->smc_idx_lock, flags);
961 
962 	return true;
963 }
964 
965 static u32 cik_get_virtual_caps(struct amdgpu_device *adev)
966 {
967 	/* CIK does not support SR-IOV */
968 	return 0;
969 }
970 
971 static const struct amdgpu_allowed_register_entry cik_allowed_read_registers[] = {
972 	{mmGRBM_STATUS, false},
973 	{mmGB_ADDR_CONFIG, false},
974 	{mmMC_ARB_RAMCFG, false},
975 	{mmGB_TILE_MODE0, false},
976 	{mmGB_TILE_MODE1, false},
977 	{mmGB_TILE_MODE2, false},
978 	{mmGB_TILE_MODE3, false},
979 	{mmGB_TILE_MODE4, false},
980 	{mmGB_TILE_MODE5, false},
981 	{mmGB_TILE_MODE6, false},
982 	{mmGB_TILE_MODE7, false},
983 	{mmGB_TILE_MODE8, false},
984 	{mmGB_TILE_MODE9, false},
985 	{mmGB_TILE_MODE10, false},
986 	{mmGB_TILE_MODE11, false},
987 	{mmGB_TILE_MODE12, false},
988 	{mmGB_TILE_MODE13, false},
989 	{mmGB_TILE_MODE14, false},
990 	{mmGB_TILE_MODE15, false},
991 	{mmGB_TILE_MODE16, false},
992 	{mmGB_TILE_MODE17, false},
993 	{mmGB_TILE_MODE18, false},
994 	{mmGB_TILE_MODE19, false},
995 	{mmGB_TILE_MODE20, false},
996 	{mmGB_TILE_MODE21, false},
997 	{mmGB_TILE_MODE22, false},
998 	{mmGB_TILE_MODE23, false},
999 	{mmGB_TILE_MODE24, false},
1000 	{mmGB_TILE_MODE25, false},
1001 	{mmGB_TILE_MODE26, false},
1002 	{mmGB_TILE_MODE27, false},
1003 	{mmGB_TILE_MODE28, false},
1004 	{mmGB_TILE_MODE29, false},
1005 	{mmGB_TILE_MODE30, false},
1006 	{mmGB_TILE_MODE31, false},
1007 	{mmGB_MACROTILE_MODE0, false},
1008 	{mmGB_MACROTILE_MODE1, false},
1009 	{mmGB_MACROTILE_MODE2, false},
1010 	{mmGB_MACROTILE_MODE3, false},
1011 	{mmGB_MACROTILE_MODE4, false},
1012 	{mmGB_MACROTILE_MODE5, false},
1013 	{mmGB_MACROTILE_MODE6, false},
1014 	{mmGB_MACROTILE_MODE7, false},
1015 	{mmGB_MACROTILE_MODE8, false},
1016 	{mmGB_MACROTILE_MODE9, false},
1017 	{mmGB_MACROTILE_MODE10, false},
1018 	{mmGB_MACROTILE_MODE11, false},
1019 	{mmGB_MACROTILE_MODE12, false},
1020 	{mmGB_MACROTILE_MODE13, false},
1021 	{mmGB_MACROTILE_MODE14, false},
1022 	{mmGB_MACROTILE_MODE15, false},
1023 	{mmCC_RB_BACKEND_DISABLE, false, true},
1024 	{mmGC_USER_RB_BACKEND_DISABLE, false, true},
1025 	{mmGB_BACKEND_MAP, false, false},
1026 	{mmPA_SC_RASTER_CONFIG, false, true},
1027 	{mmPA_SC_RASTER_CONFIG_1, false, true},
1028 };
1029 
1030 static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
1031 					  u32 se_num, u32 sh_num,
1032 					  u32 reg_offset)
1033 {
1034 	uint32_t val;
1035 
1036 	mutex_lock(&adev->grbm_idx_mutex);
1037 	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1038 		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
1039 
1040 	val = RREG32(reg_offset);
1041 
1042 	if (se_num != 0xffffffff || sh_num != 0xffffffff)
1043 		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
1044 	mutex_unlock(&adev->grbm_idx_mutex);
1045 	return val;
1046 }
1047 
1048 static int cik_read_register(struct amdgpu_device *adev, u32 se_num,
1049 			     u32 sh_num, u32 reg_offset, u32 *value)
1050 {
1051 	uint32_t i;
1052 
1053 	*value = 0;
1054 	for (i = 0; i < ARRAY_SIZE(cik_allowed_read_registers); i++) {
1055 		if (reg_offset != cik_allowed_read_registers[i].reg_offset)
1056 			continue;
1057 
1058 		if (!cik_allowed_read_registers[i].untouched)
1059 			*value = cik_allowed_read_registers[i].grbm_indexed ?
1060 				 cik_read_indexed_register(adev, se_num,
1061 							   sh_num, reg_offset) :
1062 				 RREG32(reg_offset);
1063 		return 0;
1064 	}
1065 	return -EINVAL;
1066 }
1067 
1068 struct kv_reset_save_regs {
1069 	u32 gmcon_reng_execute;
1070 	u32 gmcon_misc;
1071 	u32 gmcon_misc3;
1072 };
1073 
1074 static void kv_save_regs_for_reset(struct amdgpu_device *adev,
1075 				   struct kv_reset_save_regs *save)
1076 {
1077 	save->gmcon_reng_execute = RREG32(mmGMCON_RENG_EXECUTE);
1078 	save->gmcon_misc = RREG32(mmGMCON_MISC);
1079 	save->gmcon_misc3 = RREG32(mmGMCON_MISC3);
1080 
1081 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute &
1082 		~GMCON_RENG_EXECUTE__RENG_EXECUTE_ON_PWR_UP_MASK);
1083 	WREG32(mmGMCON_MISC, save->gmcon_misc &
1084 		~(GMCON_MISC__RENG_EXECUTE_ON_REG_UPDATE_MASK |
1085 			GMCON_MISC__STCTRL_STUTTER_EN_MASK));
1086 }
1087 
1088 static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
1089 				      struct kv_reset_save_regs *save)
1090 {
1091 	int i;
1092 
1093 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1094 	WREG32(mmGMCON_PGFSM_CONFIG, 0x200010ff);
1095 
1096 	for (i = 0; i < 5; i++)
1097 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1098 
1099 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1100 	WREG32(mmGMCON_PGFSM_CONFIG, 0x300010ff);
1101 
1102 	for (i = 0; i < 5; i++)
1103 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1104 
1105 	WREG32(mmGMCON_PGFSM_WRITE, 0x210000);
1106 	WREG32(mmGMCON_PGFSM_CONFIG, 0xa00010ff);
1107 
1108 	for (i = 0; i < 5; i++)
1109 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1110 
1111 	WREG32(mmGMCON_PGFSM_WRITE, 0x21003);
1112 	WREG32(mmGMCON_PGFSM_CONFIG, 0xb00010ff);
1113 
1114 	for (i = 0; i < 5; i++)
1115 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1116 
1117 	WREG32(mmGMCON_PGFSM_WRITE, 0x2b00);
1118 	WREG32(mmGMCON_PGFSM_CONFIG, 0xc00010ff);
1119 
1120 	for (i = 0; i < 5; i++)
1121 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1122 
1123 	WREG32(mmGMCON_PGFSM_WRITE, 0);
1124 	WREG32(mmGMCON_PGFSM_CONFIG, 0xd00010ff);
1125 
1126 	for (i = 0; i < 5; i++)
1127 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1128 
1129 	WREG32(mmGMCON_PGFSM_WRITE, 0x420000);
1130 	WREG32(mmGMCON_PGFSM_CONFIG, 0x100010ff);
1131 
1132 	for (i = 0; i < 5; i++)
1133 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1134 
1135 	WREG32(mmGMCON_PGFSM_WRITE, 0x120202);
1136 	WREG32(mmGMCON_PGFSM_CONFIG, 0x500010ff);
1137 
1138 	for (i = 0; i < 5; i++)
1139 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1140 
1141 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e3e36);
1142 	WREG32(mmGMCON_PGFSM_CONFIG, 0x600010ff);
1143 
1144 	for (i = 0; i < 5; i++)
1145 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1146 
1147 	WREG32(mmGMCON_PGFSM_WRITE, 0x373f3e);
1148 	WREG32(mmGMCON_PGFSM_CONFIG, 0x700010ff);
1149 
1150 	for (i = 0; i < 5; i++)
1151 		WREG32(mmGMCON_PGFSM_WRITE, 0);
1152 
1153 	WREG32(mmGMCON_PGFSM_WRITE, 0x3e1332);
1154 	WREG32(mmGMCON_PGFSM_CONFIG, 0xe00010ff);
1155 
1156 	WREG32(mmGMCON_MISC3, save->gmcon_misc3);
1157 	WREG32(mmGMCON_MISC, save->gmcon_misc);
1158 	WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
1159 }
1160 
1161 static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
1162 {
1163 	struct kv_reset_save_regs kv_save = { 0 };
1164 	u32 i;
1165 	int r = -EINVAL;
1166 
1167 	dev_info(adev->dev, "GPU pci config reset\n");
1168 
1169 	if (adev->flags & AMD_IS_APU)
1170 		kv_save_regs_for_reset(adev, &kv_save);
1171 
1172 	/* disable BM */
1173 	pci_clear_master(adev->pdev);
1174 	/* reset */
1175 	amdgpu_pci_config_reset(adev);
1176 
1177 	udelay(100);
1178 
1179 	/* wait for asic to come out of reset */
1180 	for (i = 0; i < adev->usec_timeout; i++) {
1181 		if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
1182 			/* enable BM */
1183 			pci_set_master(adev->pdev);
1184 			r = 0;
1185 			break;
1186 		}
1187 		udelay(1);
1188 	}
1189 
1190 	/* does asic init need to be run first??? */
1191 	if (adev->flags & AMD_IS_APU)
1192 		kv_restore_regs_for_reset(adev, &kv_save);
1193 
1194 	return r;
1195 }
1196 
1197 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
1198 {
1199 	u32 tmp = RREG32(mmBIOS_SCRATCH_3);
1200 
1201 	if (hung)
1202 		tmp |= ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1203 	else
1204 		tmp &= ~ATOM_S3_ASIC_GUI_ENGINE_HUNG;
1205 
1206 	WREG32(mmBIOS_SCRATCH_3, tmp);
1207 }
1208 
1209 /**
1210  * cik_asic_reset - soft reset GPU
1211  *
1212  * @adev: amdgpu_device pointer
1213  *
1214  * Look up which blocks are hung and attempt
1215  * to reset them.
1216  * Returns 0 for success.
1217  */
1218 static int cik_asic_reset(struct amdgpu_device *adev)
1219 {
1220 	int r;
1221 	cik_set_bios_scratch_engine_hung(adev, true);
1222 
1223 	r = cik_gpu_pci_config_reset(adev);
1224 
1225 	cik_set_bios_scratch_engine_hung(adev, false);
1226 
1227 	return r;
1228 }
1229 
1230 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
1231 			      u32 cntl_reg, u32 status_reg)
1232 {
1233 	int r, i;
1234 	struct atom_clock_dividers dividers;
1235 	uint32_t tmp;
1236 
1237 	r = amdgpu_atombios_get_clock_dividers(adev,
1238 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1239 					       clock, false, &dividers);
1240 	if (r)
1241 		return r;
1242 
1243 	tmp = RREG32_SMC(cntl_reg);
1244 	tmp &= ~(CG_DCLK_CNTL__DCLK_DIR_CNTL_EN_MASK |
1245 		CG_DCLK_CNTL__DCLK_DIVIDER_MASK);
1246 	tmp |= dividers.post_divider;
1247 	WREG32_SMC(cntl_reg, tmp);
1248 
1249 	for (i = 0; i < 100; i++) {
1250 		if (RREG32_SMC(status_reg) & CG_DCLK_STATUS__DCLK_STATUS_MASK)
1251 			break;
1252 		mdelay(10);
1253 	}
1254 	if (i == 100)
1255 		return -ETIMEDOUT;
1256 
1257 	return 0;
1258 }
1259 
1260 static int cik_set_uvd_clocks(struct amdgpu_device *adev, u32 vclk, u32 dclk)
1261 {
1262 	int r = 0;
1263 
1264 	r = cik_set_uvd_clock(adev, vclk, ixCG_VCLK_CNTL, ixCG_VCLK_STATUS);
1265 	if (r)
1266 		return r;
1267 
1268 	r = cik_set_uvd_clock(adev, dclk, ixCG_DCLK_CNTL, ixCG_DCLK_STATUS);
1269 	return r;
1270 }
1271 
1272 static int cik_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
1273 {
1274 	int r, i;
1275 	struct atom_clock_dividers dividers;
1276 	u32 tmp;
1277 
1278 	r = amdgpu_atombios_get_clock_dividers(adev,
1279 					       COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
1280 					       ecclk, false, &dividers);
1281 	if (r)
1282 		return r;
1283 
1284 	for (i = 0; i < 100; i++) {
1285 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1286 			break;
1287 		mdelay(10);
1288 	}
1289 	if (i == 100)
1290 		return -ETIMEDOUT;
1291 
1292 	tmp = RREG32_SMC(ixCG_ECLK_CNTL);
1293 	tmp &= ~(CG_ECLK_CNTL__ECLK_DIR_CNTL_EN_MASK |
1294 		CG_ECLK_CNTL__ECLK_DIVIDER_MASK);
1295 	tmp |= dividers.post_divider;
1296 	WREG32_SMC(ixCG_ECLK_CNTL, tmp);
1297 
1298 	for (i = 0; i < 100; i++) {
1299 		if (RREG32_SMC(ixCG_ECLK_STATUS) & CG_ECLK_STATUS__ECLK_STATUS_MASK)
1300 			break;
1301 		mdelay(10);
1302 	}
1303 	if (i == 100)
1304 		return -ETIMEDOUT;
1305 
1306 	return 0;
1307 }
1308 
1309 static void cik_pcie_gen3_enable(struct amdgpu_device *adev)
1310 {
1311 	struct pci_dev *root = adev->pdev->bus->self;
1312 	int bridge_pos, gpu_pos;
1313 	u32 speed_cntl, current_data_rate;
1314 	int i;
1315 	u16 tmp16;
1316 
1317 	if (pci_is_root_bus(adev->pdev->bus))
1318 		return;
1319 
1320 	if (amdgpu_pcie_gen2 == 0)
1321 		return;
1322 
1323 	if (adev->flags & AMD_IS_APU)
1324 		return;
1325 
1326 	if (!(adev->pm.pcie_gen_mask & (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
1327 					CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)))
1328 		return;
1329 
1330 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1331 	current_data_rate = (speed_cntl & PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE_MASK) >>
1332 		PCIE_LC_SPEED_CNTL__LC_CURRENT_DATA_RATE__SHIFT;
1333 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1334 		if (current_data_rate == 2) {
1335 			DRM_INFO("PCIE gen 3 link speeds already enabled\n");
1336 			return;
1337 		}
1338 		DRM_INFO("enabling PCIE gen 3 link speeds, disable with amdgpu.pcie_gen2=0\n");
1339 	} else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2) {
1340 		if (current_data_rate == 1) {
1341 			DRM_INFO("PCIE gen 2 link speeds already enabled\n");
1342 			return;
1343 		}
1344 		DRM_INFO("enabling PCIE gen 2 link speeds, disable with amdgpu.pcie_gen2=0\n");
1345 	}
1346 
1347 	bridge_pos = pci_pcie_cap(root);
1348 	if (!bridge_pos)
1349 		return;
1350 
1351 	gpu_pos = pci_pcie_cap(adev->pdev);
1352 	if (!gpu_pos)
1353 		return;
1354 
1355 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3) {
1356 		/* re-try equalization if gen3 is not already enabled */
1357 		if (current_data_rate != 2) {
1358 			u16 bridge_cfg, gpu_cfg;
1359 			u16 bridge_cfg2, gpu_cfg2;
1360 			u32 max_lw, current_lw, tmp;
1361 
1362 			pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1363 			pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1364 
1365 			tmp16 = bridge_cfg | PCI_EXP_LNKCTL_HAWD;
1366 			pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1367 
1368 			tmp16 = gpu_cfg | PCI_EXP_LNKCTL_HAWD;
1369 			pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1370 
1371 			tmp = RREG32_PCIE(ixPCIE_LC_STATUS1);
1372 			max_lw = (tmp & PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH_MASK) >>
1373 				PCIE_LC_STATUS1__LC_DETECTED_LINK_WIDTH__SHIFT;
1374 			current_lw = (tmp & PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH_MASK)
1375 				>> PCIE_LC_STATUS1__LC_OPERATING_LINK_WIDTH__SHIFT;
1376 
1377 			if (current_lw < max_lw) {
1378 				tmp = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1379 				if (tmp & PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATION_SUPPORT_MASK) {
1380 					tmp &= ~(PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH_MASK |
1381 						PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_DIS_MASK);
1382 					tmp |= (max_lw <<
1383 						PCIE_LC_LINK_WIDTH_CNTL__LC_LINK_WIDTH__SHIFT);
1384 					tmp |= PCIE_LC_LINK_WIDTH_CNTL__LC_UPCONFIGURE_SUPPORT_MASK |
1385 					PCIE_LC_LINK_WIDTH_CNTL__LC_RENEGOTIATE_EN_MASK |
1386 					PCIE_LC_LINK_WIDTH_CNTL__LC_RECONFIG_NOW_MASK;
1387 					WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, tmp);
1388 				}
1389 			}
1390 
1391 			for (i = 0; i < 10; i++) {
1392 				/* check status */
1393 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_DEVSTA, &tmp16);
1394 				if (tmp16 & PCI_EXP_DEVSTA_TRPND)
1395 					break;
1396 
1397 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &bridge_cfg);
1398 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &gpu_cfg);
1399 
1400 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &bridge_cfg2);
1401 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &gpu_cfg2);
1402 
1403 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1404 				tmp |= PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1405 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1406 
1407 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1408 				tmp |= PCIE_LC_CNTL4__LC_REDO_EQ_MASK;
1409 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1410 
1411 				mdelay(100);
1412 
1413 				/* linkctl */
1414 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL, &tmp16);
1415 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1416 				tmp16 |= (bridge_cfg & PCI_EXP_LNKCTL_HAWD);
1417 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL, tmp16);
1418 
1419 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, &tmp16);
1420 				tmp16 &= ~PCI_EXP_LNKCTL_HAWD;
1421 				tmp16 |= (gpu_cfg & PCI_EXP_LNKCTL_HAWD);
1422 				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL, tmp16);
1423 
1424 				/* linkctl2 */
1425 				pci_read_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, &tmp16);
1426 				tmp16 &= ~((1 << 4) | (7 << 9));
1427 				tmp16 |= (bridge_cfg2 & ((1 << 4) | (7 << 9)));
1428 				pci_write_config_word(root, bridge_pos + PCI_EXP_LNKCTL2, tmp16);
1429 
1430 				pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1431 				tmp16 &= ~((1 << 4) | (7 << 9));
1432 				tmp16 |= (gpu_cfg2 & ((1 << 4) | (7 << 9)));
1433 				pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1434 
1435 				tmp = RREG32_PCIE(ixPCIE_LC_CNTL4);
1436 				tmp &= ~PCIE_LC_CNTL4__LC_SET_QUIESCE_MASK;
1437 				WREG32_PCIE(ixPCIE_LC_CNTL4, tmp);
1438 			}
1439 		}
1440 	}
1441 
1442 	/* set the link speed */
1443 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_FORCE_EN_SW_SPEED_CHANGE_MASK |
1444 		PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_HW_SPEED_CHANGE_MASK;
1445 	speed_cntl &= ~PCIE_LC_SPEED_CNTL__LC_FORCE_DIS_SW_SPEED_CHANGE_MASK;
1446 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1447 
1448 	pci_read_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, &tmp16);
1449 	tmp16 &= ~0xf;
1450 	if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
1451 		tmp16 |= 3; /* gen3 */
1452 	else if (adev->pm.pcie_gen_mask & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2)
1453 		tmp16 |= 2; /* gen2 */
1454 	else
1455 		tmp16 |= 1; /* gen1 */
1456 	pci_write_config_word(adev->pdev, gpu_pos + PCI_EXP_LNKCTL2, tmp16);
1457 
1458 	speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1459 	speed_cntl |= PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK;
1460 	WREG32_PCIE(ixPCIE_LC_SPEED_CNTL, speed_cntl);
1461 
1462 	for (i = 0; i < adev->usec_timeout; i++) {
1463 		speed_cntl = RREG32_PCIE(ixPCIE_LC_SPEED_CNTL);
1464 		if ((speed_cntl & PCIE_LC_SPEED_CNTL__LC_INITIATE_LINK_SPEED_CHANGE_MASK) == 0)
1465 			break;
1466 		udelay(1);
1467 	}
1468 }
1469 
1470 static void cik_program_aspm(struct amdgpu_device *adev)
1471 {
1472 	u32 data, orig;
1473 	bool disable_l0s = false, disable_l1 = false, disable_plloff_in_l1 = false;
1474 	bool disable_clkreq = false;
1475 
1476 	if (amdgpu_aspm == 0)
1477 		return;
1478 
1479 	if (pci_is_root_bus(adev->pdev->bus))
1480 		return;
1481 
1482 	/* XXX double check APUs */
1483 	if (adev->flags & AMD_IS_APU)
1484 		return;
1485 
1486 	orig = data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1487 	data &= ~PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_MASK;
1488 	data |= (0x24 << PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS__SHIFT) |
1489 		PCIE_LC_N_FTS_CNTL__LC_XMIT_N_FTS_OVERRIDE_EN_MASK;
1490 	if (orig != data)
1491 		WREG32_PCIE(ixPCIE_LC_N_FTS_CNTL, data);
1492 
1493 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL3);
1494 	data |= PCIE_LC_CNTL3__LC_GO_TO_RECOVERY_MASK;
1495 	if (orig != data)
1496 		WREG32_PCIE(ixPCIE_LC_CNTL3, data);
1497 
1498 	orig = data = RREG32_PCIE(ixPCIE_P_CNTL);
1499 	data |= PCIE_P_CNTL__P_IGNORE_EDB_ERR_MASK;
1500 	if (orig != data)
1501 		WREG32_PCIE(ixPCIE_P_CNTL, data);
1502 
1503 	orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1504 	data &= ~(PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK |
1505 		PCIE_LC_CNTL__LC_L1_INACTIVITY_MASK);
1506 	data |= PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1507 	if (!disable_l0s)
1508 		data |= (7 << PCIE_LC_CNTL__LC_L0S_INACTIVITY__SHIFT);
1509 
1510 	if (!disable_l1) {
1511 		data |= (7 << PCIE_LC_CNTL__LC_L1_INACTIVITY__SHIFT);
1512 		data &= ~PCIE_LC_CNTL__LC_PMI_TO_L1_DIS_MASK;
1513 		if (orig != data)
1514 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1515 
1516 		if (!disable_plloff_in_l1) {
1517 			bool clk_req_support;
1518 
1519 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_0);
1520 			data &= ~(PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1521 				PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1522 			data |= (7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1523 				(7 << PB0_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1524 			if (orig != data)
1525 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_0, data);
1526 
1527 			orig = data = RREG32_PCIE(ixPB0_PIF_PWRDOWN_1);
1528 			data &= ~(PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1529 				PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1530 			data |= (7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1531 				(7 << PB0_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1532 			if (orig != data)
1533 				WREG32_PCIE(ixPB0_PIF_PWRDOWN_1, data);
1534 
1535 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_0);
1536 			data &= ~(PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0_MASK |
1537 				PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0_MASK);
1538 			data |= (7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_OFF_0__SHIFT) |
1539 				(7 << PB1_PIF_PWRDOWN_0__PLL_POWER_STATE_IN_TXS2_0__SHIFT);
1540 			if (orig != data)
1541 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_0, data);
1542 
1543 			orig = data = RREG32_PCIE(ixPB1_PIF_PWRDOWN_1);
1544 			data &= ~(PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1_MASK |
1545 				PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1_MASK);
1546 			data |= (7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_OFF_1__SHIFT) |
1547 				(7 << PB1_PIF_PWRDOWN_1__PLL_POWER_STATE_IN_TXS2_1__SHIFT);
1548 			if (orig != data)
1549 				WREG32_PCIE(ixPB1_PIF_PWRDOWN_1, data);
1550 
1551 			orig = data = RREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL);
1552 			data &= ~PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE_MASK;
1553 			data |= ~(3 << PCIE_LC_LINK_WIDTH_CNTL__LC_DYN_LANES_PWR_STATE__SHIFT);
1554 			if (orig != data)
1555 				WREG32_PCIE(ixPCIE_LC_LINK_WIDTH_CNTL, data);
1556 
1557 			if (!disable_clkreq) {
1558 				struct pci_dev *root = adev->pdev->bus->self;
1559 				u32 lnkcap;
1560 
1561 				clk_req_support = false;
1562 				pcie_capability_read_dword(root, PCI_EXP_LNKCAP, &lnkcap);
1563 				if (lnkcap & PCI_EXP_LNKCAP_CLKPM)
1564 					clk_req_support = true;
1565 			} else {
1566 				clk_req_support = false;
1567 			}
1568 
1569 			if (clk_req_support) {
1570 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL2);
1571 				data |= PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L1_MASK |
1572 					PCIE_LC_CNTL2__LC_ALLOW_PDWN_IN_L23_MASK;
1573 				if (orig != data)
1574 					WREG32_PCIE(ixPCIE_LC_CNTL2, data);
1575 
1576 				orig = data = RREG32_SMC(ixTHM_CLK_CNTL);
1577 				data &= ~(THM_CLK_CNTL__CMON_CLK_SEL_MASK |
1578 					THM_CLK_CNTL__TMON_CLK_SEL_MASK);
1579 				data |= (1 << THM_CLK_CNTL__CMON_CLK_SEL__SHIFT) |
1580 					(1 << THM_CLK_CNTL__TMON_CLK_SEL__SHIFT);
1581 				if (orig != data)
1582 					WREG32_SMC(ixTHM_CLK_CNTL, data);
1583 
1584 				orig = data = RREG32_SMC(ixMISC_CLK_CTRL);
1585 				data &= ~(MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL_MASK |
1586 					MISC_CLK_CTRL__ZCLK_SEL_MASK);
1587 				data |= (1 << MISC_CLK_CTRL__DEEP_SLEEP_CLK_SEL__SHIFT) |
1588 					(1 << MISC_CLK_CTRL__ZCLK_SEL__SHIFT);
1589 				if (orig != data)
1590 					WREG32_SMC(ixMISC_CLK_CTRL, data);
1591 
1592 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL);
1593 				data &= ~CG_CLKPIN_CNTL__BCLK_AS_XCLK_MASK;
1594 				if (orig != data)
1595 					WREG32_SMC(ixCG_CLKPIN_CNTL, data);
1596 
1597 				orig = data = RREG32_SMC(ixCG_CLKPIN_CNTL_2);
1598 				data &= ~CG_CLKPIN_CNTL_2__FORCE_BIF_REFCLK_EN_MASK;
1599 				if (orig != data)
1600 					WREG32_SMC(ixCG_CLKPIN_CNTL_2, data);
1601 
1602 				orig = data = RREG32_SMC(ixMPLL_BYPASSCLK_SEL);
1603 				data &= ~MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL_MASK;
1604 				data |= (4 << MPLL_BYPASSCLK_SEL__MPLL_CLKOUT_SEL__SHIFT);
1605 				if (orig != data)
1606 					WREG32_SMC(ixMPLL_BYPASSCLK_SEL, data);
1607 			}
1608 		}
1609 	} else {
1610 		if (orig != data)
1611 			WREG32_PCIE(ixPCIE_LC_CNTL, data);
1612 	}
1613 
1614 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
1615 	data |= PCIE_CNTL2__SLV_MEM_LS_EN_MASK |
1616 		PCIE_CNTL2__MST_MEM_LS_EN_MASK |
1617 		PCIE_CNTL2__REPLAY_MEM_LS_EN_MASK;
1618 	if (orig != data)
1619 		WREG32_PCIE(ixPCIE_CNTL2, data);
1620 
1621 	if (!disable_l0s) {
1622 		data = RREG32_PCIE(ixPCIE_LC_N_FTS_CNTL);
1623 		if ((data & PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) ==
1624 				PCIE_LC_N_FTS_CNTL__LC_N_FTS_MASK) {
1625 			data = RREG32_PCIE(ixPCIE_LC_STATUS1);
1626 			if ((data & PCIE_LC_STATUS1__LC_REVERSE_XMIT_MASK) &&
1627 			(data & PCIE_LC_STATUS1__LC_REVERSE_RCVR_MASK)) {
1628 				orig = data = RREG32_PCIE(ixPCIE_LC_CNTL);
1629 				data &= ~PCIE_LC_CNTL__LC_L0S_INACTIVITY_MASK;
1630 				if (orig != data)
1631 					WREG32_PCIE(ixPCIE_LC_CNTL, data);
1632 			}
1633 		}
1634 	}
1635 }
1636 
1637 static uint32_t cik_get_rev_id(struct amdgpu_device *adev)
1638 {
1639 	return (RREG32(mmCC_DRM_ID_STRAPS) & CC_DRM_ID_STRAPS__ATI_REV_ID_MASK)
1640 		>> CC_DRM_ID_STRAPS__ATI_REV_ID__SHIFT;
1641 }
1642 
1643 static const struct amdgpu_ip_block_version bonaire_ip_blocks[] =
1644 {
1645 	/* ORDER MATTERS! */
1646 	{
1647 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1648 		.major = 1,
1649 		.minor = 0,
1650 		.rev = 0,
1651 		.funcs = &cik_common_ip_funcs,
1652 	},
1653 	{
1654 		.type = AMD_IP_BLOCK_TYPE_GMC,
1655 		.major = 7,
1656 		.minor = 0,
1657 		.rev = 0,
1658 		.funcs = &gmc_v7_0_ip_funcs,
1659 	},
1660 	{
1661 		.type = AMD_IP_BLOCK_TYPE_IH,
1662 		.major = 2,
1663 		.minor = 0,
1664 		.rev = 0,
1665 		.funcs = &cik_ih_ip_funcs,
1666 	},
1667 	{
1668 		.type = AMD_IP_BLOCK_TYPE_SMC,
1669 		.major = 7,
1670 		.minor = 0,
1671 		.rev = 0,
1672 		.funcs = &amdgpu_pp_ip_funcs,
1673 	},
1674 	{
1675 		.type = AMD_IP_BLOCK_TYPE_DCE,
1676 		.major = 8,
1677 		.minor = 2,
1678 		.rev = 0,
1679 		.funcs = &dce_v8_0_ip_funcs,
1680 	},
1681 	{
1682 		.type = AMD_IP_BLOCK_TYPE_GFX,
1683 		.major = 7,
1684 		.minor = 2,
1685 		.rev = 0,
1686 		.funcs = &gfx_v7_0_ip_funcs,
1687 	},
1688 	{
1689 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1690 		.major = 2,
1691 		.minor = 0,
1692 		.rev = 0,
1693 		.funcs = &cik_sdma_ip_funcs,
1694 	},
1695 	{
1696 		.type = AMD_IP_BLOCK_TYPE_UVD,
1697 		.major = 4,
1698 		.minor = 2,
1699 		.rev = 0,
1700 		.funcs = &uvd_v4_2_ip_funcs,
1701 	},
1702 	{
1703 		.type = AMD_IP_BLOCK_TYPE_VCE,
1704 		.major = 2,
1705 		.minor = 0,
1706 		.rev = 0,
1707 		.funcs = &vce_v2_0_ip_funcs,
1708 	},
1709 };
1710 
1711 static const struct amdgpu_ip_block_version hawaii_ip_blocks[] =
1712 {
1713 	/* ORDER MATTERS! */
1714 	{
1715 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1716 		.major = 1,
1717 		.minor = 0,
1718 		.rev = 0,
1719 		.funcs = &cik_common_ip_funcs,
1720 	},
1721 	{
1722 		.type = AMD_IP_BLOCK_TYPE_GMC,
1723 		.major = 7,
1724 		.minor = 0,
1725 		.rev = 0,
1726 		.funcs = &gmc_v7_0_ip_funcs,
1727 	},
1728 	{
1729 		.type = AMD_IP_BLOCK_TYPE_IH,
1730 		.major = 2,
1731 		.minor = 0,
1732 		.rev = 0,
1733 		.funcs = &cik_ih_ip_funcs,
1734 	},
1735 	{
1736 		.type = AMD_IP_BLOCK_TYPE_SMC,
1737 		.major = 7,
1738 		.minor = 0,
1739 		.rev = 0,
1740 		.funcs = &amdgpu_pp_ip_funcs,
1741 	},
1742 	{
1743 		.type = AMD_IP_BLOCK_TYPE_DCE,
1744 		.major = 8,
1745 		.minor = 5,
1746 		.rev = 0,
1747 		.funcs = &dce_v8_0_ip_funcs,
1748 	},
1749 	{
1750 		.type = AMD_IP_BLOCK_TYPE_GFX,
1751 		.major = 7,
1752 		.minor = 3,
1753 		.rev = 0,
1754 		.funcs = &gfx_v7_0_ip_funcs,
1755 	},
1756 	{
1757 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1758 		.major = 2,
1759 		.minor = 0,
1760 		.rev = 0,
1761 		.funcs = &cik_sdma_ip_funcs,
1762 	},
1763 	{
1764 		.type = AMD_IP_BLOCK_TYPE_UVD,
1765 		.major = 4,
1766 		.minor = 2,
1767 		.rev = 0,
1768 		.funcs = &uvd_v4_2_ip_funcs,
1769 	},
1770 	{
1771 		.type = AMD_IP_BLOCK_TYPE_VCE,
1772 		.major = 2,
1773 		.minor = 0,
1774 		.rev = 0,
1775 		.funcs = &vce_v2_0_ip_funcs,
1776 	},
1777 };
1778 
1779 static const struct amdgpu_ip_block_version kabini_ip_blocks[] =
1780 {
1781 	/* ORDER MATTERS! */
1782 	{
1783 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1784 		.major = 1,
1785 		.minor = 0,
1786 		.rev = 0,
1787 		.funcs = &cik_common_ip_funcs,
1788 	},
1789 	{
1790 		.type = AMD_IP_BLOCK_TYPE_GMC,
1791 		.major = 7,
1792 		.minor = 0,
1793 		.rev = 0,
1794 		.funcs = &gmc_v7_0_ip_funcs,
1795 	},
1796 	{
1797 		.type = AMD_IP_BLOCK_TYPE_IH,
1798 		.major = 2,
1799 		.minor = 0,
1800 		.rev = 0,
1801 		.funcs = &cik_ih_ip_funcs,
1802 	},
1803 	{
1804 		.type = AMD_IP_BLOCK_TYPE_SMC,
1805 		.major = 7,
1806 		.minor = 0,
1807 		.rev = 0,
1808 		.funcs = &amdgpu_pp_ip_funcs,
1809 	},
1810 	{
1811 		.type = AMD_IP_BLOCK_TYPE_DCE,
1812 		.major = 8,
1813 		.minor = 3,
1814 		.rev = 0,
1815 		.funcs = &dce_v8_0_ip_funcs,
1816 	},
1817 	{
1818 		.type = AMD_IP_BLOCK_TYPE_GFX,
1819 		.major = 7,
1820 		.minor = 2,
1821 		.rev = 0,
1822 		.funcs = &gfx_v7_0_ip_funcs,
1823 	},
1824 	{
1825 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1826 		.major = 2,
1827 		.minor = 0,
1828 		.rev = 0,
1829 		.funcs = &cik_sdma_ip_funcs,
1830 	},
1831 	{
1832 		.type = AMD_IP_BLOCK_TYPE_UVD,
1833 		.major = 4,
1834 		.minor = 2,
1835 		.rev = 0,
1836 		.funcs = &uvd_v4_2_ip_funcs,
1837 	},
1838 	{
1839 		.type = AMD_IP_BLOCK_TYPE_VCE,
1840 		.major = 2,
1841 		.minor = 0,
1842 		.rev = 0,
1843 		.funcs = &vce_v2_0_ip_funcs,
1844 	},
1845 };
1846 
1847 static const struct amdgpu_ip_block_version mullins_ip_blocks[] =
1848 {
1849 	/* ORDER MATTERS! */
1850 	{
1851 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1852 		.major = 1,
1853 		.minor = 0,
1854 		.rev = 0,
1855 		.funcs = &cik_common_ip_funcs,
1856 	},
1857 	{
1858 		.type = AMD_IP_BLOCK_TYPE_GMC,
1859 		.major = 7,
1860 		.minor = 0,
1861 		.rev = 0,
1862 		.funcs = &gmc_v7_0_ip_funcs,
1863 	},
1864 	{
1865 		.type = AMD_IP_BLOCK_TYPE_IH,
1866 		.major = 2,
1867 		.minor = 0,
1868 		.rev = 0,
1869 		.funcs = &cik_ih_ip_funcs,
1870 	},
1871 	{
1872 		.type = AMD_IP_BLOCK_TYPE_SMC,
1873 		.major = 7,
1874 		.minor = 0,
1875 		.rev = 0,
1876 		.funcs = &amdgpu_pp_ip_funcs,
1877 	},
1878 	{
1879 		.type = AMD_IP_BLOCK_TYPE_DCE,
1880 		.major = 8,
1881 		.minor = 3,
1882 		.rev = 0,
1883 		.funcs = &dce_v8_0_ip_funcs,
1884 	},
1885 	{
1886 		.type = AMD_IP_BLOCK_TYPE_GFX,
1887 		.major = 7,
1888 		.minor = 2,
1889 		.rev = 0,
1890 		.funcs = &gfx_v7_0_ip_funcs,
1891 	},
1892 	{
1893 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1894 		.major = 2,
1895 		.minor = 0,
1896 		.rev = 0,
1897 		.funcs = &cik_sdma_ip_funcs,
1898 	},
1899 	{
1900 		.type = AMD_IP_BLOCK_TYPE_UVD,
1901 		.major = 4,
1902 		.minor = 2,
1903 		.rev = 0,
1904 		.funcs = &uvd_v4_2_ip_funcs,
1905 	},
1906 	{
1907 		.type = AMD_IP_BLOCK_TYPE_VCE,
1908 		.major = 2,
1909 		.minor = 0,
1910 		.rev = 0,
1911 		.funcs = &vce_v2_0_ip_funcs,
1912 	},
1913 };
1914 
1915 static const struct amdgpu_ip_block_version kaveri_ip_blocks[] =
1916 {
1917 	/* ORDER MATTERS! */
1918 	{
1919 		.type = AMD_IP_BLOCK_TYPE_COMMON,
1920 		.major = 1,
1921 		.minor = 0,
1922 		.rev = 0,
1923 		.funcs = &cik_common_ip_funcs,
1924 	},
1925 	{
1926 		.type = AMD_IP_BLOCK_TYPE_GMC,
1927 		.major = 7,
1928 		.minor = 0,
1929 		.rev = 0,
1930 		.funcs = &gmc_v7_0_ip_funcs,
1931 	},
1932 	{
1933 		.type = AMD_IP_BLOCK_TYPE_IH,
1934 		.major = 2,
1935 		.minor = 0,
1936 		.rev = 0,
1937 		.funcs = &cik_ih_ip_funcs,
1938 	},
1939 	{
1940 		.type = AMD_IP_BLOCK_TYPE_SMC,
1941 		.major = 7,
1942 		.minor = 0,
1943 		.rev = 0,
1944 		.funcs = &amdgpu_pp_ip_funcs,
1945 	},
1946 	{
1947 		.type = AMD_IP_BLOCK_TYPE_DCE,
1948 		.major = 8,
1949 		.minor = 1,
1950 		.rev = 0,
1951 		.funcs = &dce_v8_0_ip_funcs,
1952 	},
1953 	{
1954 		.type = AMD_IP_BLOCK_TYPE_GFX,
1955 		.major = 7,
1956 		.minor = 1,
1957 		.rev = 0,
1958 		.funcs = &gfx_v7_0_ip_funcs,
1959 	},
1960 	{
1961 		.type = AMD_IP_BLOCK_TYPE_SDMA,
1962 		.major = 2,
1963 		.minor = 0,
1964 		.rev = 0,
1965 		.funcs = &cik_sdma_ip_funcs,
1966 	},
1967 	{
1968 		.type = AMD_IP_BLOCK_TYPE_UVD,
1969 		.major = 4,
1970 		.minor = 2,
1971 		.rev = 0,
1972 		.funcs = &uvd_v4_2_ip_funcs,
1973 	},
1974 	{
1975 		.type = AMD_IP_BLOCK_TYPE_VCE,
1976 		.major = 2,
1977 		.minor = 0,
1978 		.rev = 0,
1979 		.funcs = &vce_v2_0_ip_funcs,
1980 	},
1981 };
1982 
1983 int cik_set_ip_blocks(struct amdgpu_device *adev)
1984 {
1985 	switch (adev->asic_type) {
1986 	case CHIP_BONAIRE:
1987 		adev->ip_blocks = bonaire_ip_blocks;
1988 		adev->num_ip_blocks = ARRAY_SIZE(bonaire_ip_blocks);
1989 		break;
1990 	case CHIP_HAWAII:
1991 		adev->ip_blocks = hawaii_ip_blocks;
1992 		adev->num_ip_blocks = ARRAY_SIZE(hawaii_ip_blocks);
1993 		break;
1994 	case CHIP_KAVERI:
1995 		adev->ip_blocks = kaveri_ip_blocks;
1996 		adev->num_ip_blocks = ARRAY_SIZE(kaveri_ip_blocks);
1997 		break;
1998 	case CHIP_KABINI:
1999 		adev->ip_blocks = kabini_ip_blocks;
2000 		adev->num_ip_blocks = ARRAY_SIZE(kabini_ip_blocks);
2001 		break;
2002 	case CHIP_MULLINS:
2003 		adev->ip_blocks = mullins_ip_blocks;
2004 		adev->num_ip_blocks = ARRAY_SIZE(mullins_ip_blocks);
2005 		break;
2006 	default:
2007 		/* FIXME: not supported yet */
2008 		return -EINVAL;
2009 	}
2010 
2011 	return 0;
2012 }
2013 
2014 static const struct amdgpu_asic_funcs cik_asic_funcs =
2015 {
2016 	.read_disabled_bios = &cik_read_disabled_bios,
2017 	.read_bios_from_rom = &cik_read_bios_from_rom,
2018 	.read_register = &cik_read_register,
2019 	.reset = &cik_asic_reset,
2020 	.set_vga_state = &cik_vga_set_state,
2021 	.get_xclk = &cik_get_xclk,
2022 	.set_uvd_clocks = &cik_set_uvd_clocks,
2023 	.set_vce_clocks = &cik_set_vce_clocks,
2024 	.get_virtual_caps = &cik_get_virtual_caps,
2025 };
2026 
2027 static int cik_common_early_init(void *handle)
2028 {
2029 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2030 
2031 	adev->smc_rreg = &cik_smc_rreg;
2032 	adev->smc_wreg = &cik_smc_wreg;
2033 	adev->pcie_rreg = &cik_pcie_rreg;
2034 	adev->pcie_wreg = &cik_pcie_wreg;
2035 	adev->uvd_ctx_rreg = &cik_uvd_ctx_rreg;
2036 	adev->uvd_ctx_wreg = &cik_uvd_ctx_wreg;
2037 	adev->didt_rreg = &cik_didt_rreg;
2038 	adev->didt_wreg = &cik_didt_wreg;
2039 
2040 	adev->asic_funcs = &cik_asic_funcs;
2041 
2042 	adev->rev_id = cik_get_rev_id(adev);
2043 	adev->external_rev_id = 0xFF;
2044 	switch (adev->asic_type) {
2045 	case CHIP_BONAIRE:
2046 		adev->cg_flags =
2047 			AMD_CG_SUPPORT_GFX_MGCG |
2048 			AMD_CG_SUPPORT_GFX_MGLS |
2049 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2050 			AMD_CG_SUPPORT_GFX_CGLS |
2051 			AMD_CG_SUPPORT_GFX_CGTS |
2052 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2053 			AMD_CG_SUPPORT_GFX_CP_LS |
2054 			AMD_CG_SUPPORT_MC_LS |
2055 			AMD_CG_SUPPORT_MC_MGCG |
2056 			AMD_CG_SUPPORT_SDMA_MGCG |
2057 			AMD_CG_SUPPORT_SDMA_LS |
2058 			AMD_CG_SUPPORT_BIF_LS |
2059 			AMD_CG_SUPPORT_VCE_MGCG |
2060 			AMD_CG_SUPPORT_UVD_MGCG |
2061 			AMD_CG_SUPPORT_HDP_LS |
2062 			AMD_CG_SUPPORT_HDP_MGCG;
2063 		adev->pg_flags = 0;
2064 		adev->external_rev_id = adev->rev_id + 0x14;
2065 		break;
2066 	case CHIP_HAWAII:
2067 		adev->cg_flags =
2068 			AMD_CG_SUPPORT_GFX_MGCG |
2069 			AMD_CG_SUPPORT_GFX_MGLS |
2070 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2071 			AMD_CG_SUPPORT_GFX_CGLS |
2072 			AMD_CG_SUPPORT_GFX_CGTS |
2073 			AMD_CG_SUPPORT_GFX_CP_LS |
2074 			AMD_CG_SUPPORT_MC_LS |
2075 			AMD_CG_SUPPORT_MC_MGCG |
2076 			AMD_CG_SUPPORT_SDMA_MGCG |
2077 			AMD_CG_SUPPORT_SDMA_LS |
2078 			AMD_CG_SUPPORT_BIF_LS |
2079 			AMD_CG_SUPPORT_VCE_MGCG |
2080 			AMD_CG_SUPPORT_UVD_MGCG |
2081 			AMD_CG_SUPPORT_HDP_LS |
2082 			AMD_CG_SUPPORT_HDP_MGCG;
2083 		adev->pg_flags = 0;
2084 		adev->external_rev_id = 0x28;
2085 		break;
2086 	case CHIP_KAVERI:
2087 		adev->cg_flags =
2088 			AMD_CG_SUPPORT_GFX_MGCG |
2089 			AMD_CG_SUPPORT_GFX_MGLS |
2090 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2091 			AMD_CG_SUPPORT_GFX_CGLS |
2092 			AMD_CG_SUPPORT_GFX_CGTS |
2093 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2094 			AMD_CG_SUPPORT_GFX_CP_LS |
2095 			AMD_CG_SUPPORT_SDMA_MGCG |
2096 			AMD_CG_SUPPORT_SDMA_LS |
2097 			AMD_CG_SUPPORT_BIF_LS |
2098 			AMD_CG_SUPPORT_VCE_MGCG |
2099 			AMD_CG_SUPPORT_UVD_MGCG |
2100 			AMD_CG_SUPPORT_HDP_LS |
2101 			AMD_CG_SUPPORT_HDP_MGCG;
2102 		adev->pg_flags =
2103 			/*AMD_PG_SUPPORT_GFX_PG |
2104 			  AMD_PG_SUPPORT_GFX_SMG |
2105 			  AMD_PG_SUPPORT_GFX_DMG |*/
2106 			AMD_PG_SUPPORT_UVD |
2107 			/*AMD_PG_SUPPORT_VCE |
2108 			  AMD_PG_SUPPORT_CP |
2109 			  AMD_PG_SUPPORT_GDS |
2110 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2111 			  AMD_PG_SUPPORT_ACP |
2112 			  AMD_PG_SUPPORT_SAMU |*/
2113 			0;
2114 		if (adev->pdev->device == 0x1312 ||
2115 			adev->pdev->device == 0x1316 ||
2116 			adev->pdev->device == 0x1317)
2117 			adev->external_rev_id = 0x41;
2118 		else
2119 			adev->external_rev_id = 0x1;
2120 		break;
2121 	case CHIP_KABINI:
2122 	case CHIP_MULLINS:
2123 		adev->cg_flags =
2124 			AMD_CG_SUPPORT_GFX_MGCG |
2125 			AMD_CG_SUPPORT_GFX_MGLS |
2126 			/*AMD_CG_SUPPORT_GFX_CGCG |*/
2127 			AMD_CG_SUPPORT_GFX_CGLS |
2128 			AMD_CG_SUPPORT_GFX_CGTS |
2129 			AMD_CG_SUPPORT_GFX_CGTS_LS |
2130 			AMD_CG_SUPPORT_GFX_CP_LS |
2131 			AMD_CG_SUPPORT_SDMA_MGCG |
2132 			AMD_CG_SUPPORT_SDMA_LS |
2133 			AMD_CG_SUPPORT_BIF_LS |
2134 			AMD_CG_SUPPORT_VCE_MGCG |
2135 			AMD_CG_SUPPORT_UVD_MGCG |
2136 			AMD_CG_SUPPORT_HDP_LS |
2137 			AMD_CG_SUPPORT_HDP_MGCG;
2138 		adev->pg_flags =
2139 			/*AMD_PG_SUPPORT_GFX_PG |
2140 			  AMD_PG_SUPPORT_GFX_SMG | */
2141 			AMD_PG_SUPPORT_UVD |
2142 			/*AMD_PG_SUPPORT_VCE |
2143 			  AMD_PG_SUPPORT_CP |
2144 			  AMD_PG_SUPPORT_GDS |
2145 			  AMD_PG_SUPPORT_RLC_SMU_HS |
2146 			  AMD_PG_SUPPORT_SAMU |*/
2147 			0;
2148 		if (adev->asic_type == CHIP_KABINI) {
2149 			if (adev->rev_id == 0)
2150 				adev->external_rev_id = 0x81;
2151 			else if (adev->rev_id == 1)
2152 				adev->external_rev_id = 0x82;
2153 			else if (adev->rev_id == 2)
2154 				adev->external_rev_id = 0x85;
2155 		} else
2156 			adev->external_rev_id = adev->rev_id + 0xa1;
2157 		break;
2158 	default:
2159 		/* FIXME: not supported yet */
2160 		return -EINVAL;
2161 	}
2162 
2163 	amdgpu_get_pcie_info(adev);
2164 
2165 	return 0;
2166 }
2167 
2168 static int cik_common_sw_init(void *handle)
2169 {
2170 	return 0;
2171 }
2172 
2173 static int cik_common_sw_fini(void *handle)
2174 {
2175 	return 0;
2176 }
2177 
2178 static int cik_common_hw_init(void *handle)
2179 {
2180 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2181 
2182 	/* move the golden regs per IP block */
2183 	cik_init_golden_registers(adev);
2184 	/* enable pcie gen2/3 link */
2185 	cik_pcie_gen3_enable(adev);
2186 	/* enable aspm */
2187 	cik_program_aspm(adev);
2188 
2189 	return 0;
2190 }
2191 
2192 static int cik_common_hw_fini(void *handle)
2193 {
2194 	return 0;
2195 }
2196 
2197 static int cik_common_suspend(void *handle)
2198 {
2199 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2200 
2201 	amdgpu_amdkfd_suspend(adev);
2202 
2203 	return cik_common_hw_fini(adev);
2204 }
2205 
2206 static int cik_common_resume(void *handle)
2207 {
2208 	int r;
2209 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2210 
2211 	r = cik_common_hw_init(adev);
2212 	if (r)
2213 		return r;
2214 
2215 	return amdgpu_amdkfd_resume(adev);
2216 }
2217 
2218 static bool cik_common_is_idle(void *handle)
2219 {
2220 	return true;
2221 }
2222 
2223 static int cik_common_wait_for_idle(void *handle)
2224 {
2225 	return 0;
2226 }
2227 
2228 static int cik_common_soft_reset(void *handle)
2229 {
2230 	/* XXX hard reset?? */
2231 	return 0;
2232 }
2233 
2234 static int cik_common_set_clockgating_state(void *handle,
2235 					    enum amd_clockgating_state state)
2236 {
2237 	return 0;
2238 }
2239 
2240 static int cik_common_set_powergating_state(void *handle,
2241 					    enum amd_powergating_state state)
2242 {
2243 	return 0;
2244 }
2245 
2246 const struct amd_ip_funcs cik_common_ip_funcs = {
2247 	.name = "cik_common",
2248 	.early_init = cik_common_early_init,
2249 	.late_init = NULL,
2250 	.sw_init = cik_common_sw_init,
2251 	.sw_fini = cik_common_sw_fini,
2252 	.hw_init = cik_common_hw_init,
2253 	.hw_fini = cik_common_hw_fini,
2254 	.suspend = cik_common_suspend,
2255 	.resume = cik_common_resume,
2256 	.is_idle = cik_common_is_idle,
2257 	.wait_for_idle = cik_common_wait_for_idle,
2258 	.soft_reset = cik_common_soft_reset,
2259 	.set_clockgating_state = cik_common_set_clockgating_state,
2260 	.set_powergating_state = cik_common_set_powergating_state,
2261 };
2262