xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c (revision 272d7089)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/pci.h>
26 
27 #include <drm/drm_cache.h>
28 
29 #include "amdgpu.h"
30 #include "gmc_v9_0.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "amdgpu_gem.h"
33 
34 #include "hdp/hdp_4_0_offset.h"
35 #include "hdp/hdp_4_0_sh_mask.h"
36 #include "gc/gc_9_0_sh_mask.h"
37 #include "dce/dce_12_0_offset.h"
38 #include "dce/dce_12_0_sh_mask.h"
39 #include "vega10_enum.h"
40 #include "mmhub/mmhub_1_0_offset.h"
41 #include "athub/athub_1_0_sh_mask.h"
42 #include "athub/athub_1_0_offset.h"
43 #include "oss/osssys_4_0_offset.h"
44 
45 #include "soc15.h"
46 #include "soc15d.h"
47 #include "soc15_common.h"
48 #include "umc/umc_6_0_sh_mask.h"
49 
50 #include "gfxhub_v1_0.h"
51 #include "mmhub_v1_0.h"
52 #include "athub_v1_0.h"
53 #include "gfxhub_v1_1.h"
54 #include "mmhub_v9_4.h"
55 #include "umc_v6_1.h"
56 #include "umc_v6_0.h"
57 
58 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
59 
60 #include "amdgpu_ras.h"
61 #include "amdgpu_xgmi.h"
62 
63 /* add these here since we already include dce12 headers and these are for DCN */
64 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
65 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
66 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
67 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
68 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
69 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
70 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0                                                                  0x049d
71 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX                                                         2
72 
73 
74 static const char *gfxhub_client_ids[] = {
75 	"CB",
76 	"DB",
77 	"IA",
78 	"WD",
79 	"CPF",
80 	"CPC",
81 	"CPG",
82 	"RLC",
83 	"TCP",
84 	"SQC (inst)",
85 	"SQC (data)",
86 	"SQG",
87 	"PA",
88 };
89 
90 static const char *mmhub_client_ids_raven[][2] = {
91 	[0][0] = "MP1",
92 	[1][0] = "MP0",
93 	[2][0] = "VCN",
94 	[3][0] = "VCNU",
95 	[4][0] = "HDP",
96 	[5][0] = "DCE",
97 	[13][0] = "UTCL2",
98 	[19][0] = "TLS",
99 	[26][0] = "OSS",
100 	[27][0] = "SDMA0",
101 	[0][1] = "MP1",
102 	[1][1] = "MP0",
103 	[2][1] = "VCN",
104 	[3][1] = "VCNU",
105 	[4][1] = "HDP",
106 	[5][1] = "XDP",
107 	[6][1] = "DBGU0",
108 	[7][1] = "DCE",
109 	[8][1] = "DCEDWB0",
110 	[9][1] = "DCEDWB1",
111 	[26][1] = "OSS",
112 	[27][1] = "SDMA0",
113 };
114 
115 static const char *mmhub_client_ids_renoir[][2] = {
116 	[0][0] = "MP1",
117 	[1][0] = "MP0",
118 	[2][0] = "HDP",
119 	[4][0] = "DCEDMC",
120 	[5][0] = "DCEVGA",
121 	[13][0] = "UTCL2",
122 	[19][0] = "TLS",
123 	[26][0] = "OSS",
124 	[27][0] = "SDMA0",
125 	[28][0] = "VCN",
126 	[29][0] = "VCNU",
127 	[30][0] = "JPEG",
128 	[0][1] = "MP1",
129 	[1][1] = "MP0",
130 	[2][1] = "HDP",
131 	[3][1] = "XDP",
132 	[6][1] = "DBGU0",
133 	[7][1] = "DCEDMC",
134 	[8][1] = "DCEVGA",
135 	[9][1] = "DCEDWB",
136 	[26][1] = "OSS",
137 	[27][1] = "SDMA0",
138 	[28][1] = "VCN",
139 	[29][1] = "VCNU",
140 	[30][1] = "JPEG",
141 };
142 
143 static const char *mmhub_client_ids_vega10[][2] = {
144 	[0][0] = "MP0",
145 	[1][0] = "UVD",
146 	[2][0] = "UVDU",
147 	[3][0] = "HDP",
148 	[13][0] = "UTCL2",
149 	[14][0] = "OSS",
150 	[15][0] = "SDMA1",
151 	[32+0][0] = "VCE0",
152 	[32+1][0] = "VCE0U",
153 	[32+2][0] = "XDMA",
154 	[32+3][0] = "DCE",
155 	[32+4][0] = "MP1",
156 	[32+14][0] = "SDMA0",
157 	[0][1] = "MP0",
158 	[1][1] = "UVD",
159 	[2][1] = "UVDU",
160 	[3][1] = "DBGU0",
161 	[4][1] = "HDP",
162 	[5][1] = "XDP",
163 	[14][1] = "OSS",
164 	[15][1] = "SDMA0",
165 	[32+0][1] = "VCE0",
166 	[32+1][1] = "VCE0U",
167 	[32+2][1] = "XDMA",
168 	[32+3][1] = "DCE",
169 	[32+4][1] = "DCEDWB",
170 	[32+5][1] = "MP1",
171 	[32+6][1] = "DBGU1",
172 	[32+14][1] = "SDMA1",
173 };
174 
175 static const char *mmhub_client_ids_vega12[][2] = {
176 	[0][0] = "MP0",
177 	[1][0] = "VCE0",
178 	[2][0] = "VCE0U",
179 	[3][0] = "HDP",
180 	[13][0] = "UTCL2",
181 	[14][0] = "OSS",
182 	[15][0] = "SDMA1",
183 	[32+0][0] = "DCE",
184 	[32+1][0] = "XDMA",
185 	[32+2][0] = "UVD",
186 	[32+3][0] = "UVDU",
187 	[32+4][0] = "MP1",
188 	[32+15][0] = "SDMA0",
189 	[0][1] = "MP0",
190 	[1][1] = "VCE0",
191 	[2][1] = "VCE0U",
192 	[3][1] = "DBGU0",
193 	[4][1] = "HDP",
194 	[5][1] = "XDP",
195 	[14][1] = "OSS",
196 	[15][1] = "SDMA0",
197 	[32+0][1] = "DCE",
198 	[32+1][1] = "DCEDWB",
199 	[32+2][1] = "XDMA",
200 	[32+3][1] = "UVD",
201 	[32+4][1] = "UVDU",
202 	[32+5][1] = "MP1",
203 	[32+6][1] = "DBGU1",
204 	[32+15][1] = "SDMA1",
205 };
206 
207 static const char *mmhub_client_ids_vega20[][2] = {
208 	[0][0] = "XDMA",
209 	[1][0] = "DCE",
210 	[2][0] = "VCE0",
211 	[3][0] = "VCE0U",
212 	[4][0] = "UVD",
213 	[5][0] = "UVD1U",
214 	[13][0] = "OSS",
215 	[14][0] = "HDP",
216 	[15][0] = "SDMA0",
217 	[32+0][0] = "UVD",
218 	[32+1][0] = "UVDU",
219 	[32+2][0] = "MP1",
220 	[32+3][0] = "MP0",
221 	[32+12][0] = "UTCL2",
222 	[32+14][0] = "SDMA1",
223 	[0][1] = "XDMA",
224 	[1][1] = "DCE",
225 	[2][1] = "DCEDWB",
226 	[3][1] = "VCE0",
227 	[4][1] = "VCE0U",
228 	[5][1] = "UVD1",
229 	[6][1] = "UVD1U",
230 	[7][1] = "DBGU0",
231 	[8][1] = "XDP",
232 	[13][1] = "OSS",
233 	[14][1] = "HDP",
234 	[15][1] = "SDMA0",
235 	[32+0][1] = "UVD",
236 	[32+1][1] = "UVDU",
237 	[32+2][1] = "DBGU1",
238 	[32+3][1] = "MP1",
239 	[32+4][1] = "MP0",
240 	[32+14][1] = "SDMA1",
241 };
242 
243 static const char *mmhub_client_ids_arcturus[][2] = {
244 	[2][0] = "MP1",
245 	[3][0] = "MP0",
246 	[10][0] = "UTCL2",
247 	[13][0] = "OSS",
248 	[14][0] = "HDP",
249 	[15][0] = "SDMA0",
250 	[32+15][0] = "SDMA1",
251 	[64+15][0] = "SDMA2",
252 	[96+15][0] = "SDMA3",
253 	[128+15][0] = "SDMA4",
254 	[160+11][0] = "JPEG",
255 	[160+12][0] = "VCN",
256 	[160+13][0] = "VCNU",
257 	[160+15][0] = "SDMA5",
258 	[192+10][0] = "UTCL2",
259 	[192+11][0] = "JPEG1",
260 	[192+12][0] = "VCN1",
261 	[192+13][0] = "VCN1U",
262 	[192+15][0] = "SDMA6",
263 	[224+15][0] = "SDMA7",
264 	[0][1] = "DBGU1",
265 	[1][1] = "XDP",
266 	[2][1] = "MP1",
267 	[3][1] = "MP0",
268 	[13][1] = "OSS",
269 	[14][1] = "HDP",
270 	[15][1] = "SDMA0",
271 	[32+15][1] = "SDMA1",
272 	[32+15][1] = "SDMA1",
273 	[64+15][1] = "SDMA2",
274 	[96+15][1] = "SDMA3",
275 	[128+15][1] = "SDMA4",
276 	[160+11][1] = "JPEG",
277 	[160+12][1] = "VCN",
278 	[160+13][1] = "VCNU",
279 	[160+15][1] = "SDMA5",
280 	[192+11][1] = "JPEG1",
281 	[192+12][1] = "VCN1",
282 	[192+13][1] = "VCN1U",
283 	[192+15][1] = "SDMA6",
284 	[224+15][1] = "SDMA7",
285 };
286 
287 static const u32 golden_settings_vega10_hdp[] =
288 {
289 	0xf64, 0x0fffffff, 0x00000000,
290 	0xf65, 0x0fffffff, 0x00000000,
291 	0xf66, 0x0fffffff, 0x00000000,
292 	0xf67, 0x0fffffff, 0x00000000,
293 	0xf68, 0x0fffffff, 0x00000000,
294 	0xf6a, 0x0fffffff, 0x00000000,
295 	0xf6b, 0x0fffffff, 0x00000000,
296 	0xf6c, 0x0fffffff, 0x00000000,
297 	0xf6d, 0x0fffffff, 0x00000000,
298 	0xf6e, 0x0fffffff, 0x00000000,
299 };
300 
301 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
302 {
303 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
304 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
305 };
306 
307 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
308 {
309 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
310 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
311 };
312 
313 static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
314 	(0x000143c0 + 0x00000000),
315 	(0x000143c0 + 0x00000800),
316 	(0x000143c0 + 0x00001000),
317 	(0x000143c0 + 0x00001800),
318 	(0x000543c0 + 0x00000000),
319 	(0x000543c0 + 0x00000800),
320 	(0x000543c0 + 0x00001000),
321 	(0x000543c0 + 0x00001800),
322 	(0x000943c0 + 0x00000000),
323 	(0x000943c0 + 0x00000800),
324 	(0x000943c0 + 0x00001000),
325 	(0x000943c0 + 0x00001800),
326 	(0x000d43c0 + 0x00000000),
327 	(0x000d43c0 + 0x00000800),
328 	(0x000d43c0 + 0x00001000),
329 	(0x000d43c0 + 0x00001800),
330 	(0x001143c0 + 0x00000000),
331 	(0x001143c0 + 0x00000800),
332 	(0x001143c0 + 0x00001000),
333 	(0x001143c0 + 0x00001800),
334 	(0x001543c0 + 0x00000000),
335 	(0x001543c0 + 0x00000800),
336 	(0x001543c0 + 0x00001000),
337 	(0x001543c0 + 0x00001800),
338 	(0x001943c0 + 0x00000000),
339 	(0x001943c0 + 0x00000800),
340 	(0x001943c0 + 0x00001000),
341 	(0x001943c0 + 0x00001800),
342 	(0x001d43c0 + 0x00000000),
343 	(0x001d43c0 + 0x00000800),
344 	(0x001d43c0 + 0x00001000),
345 	(0x001d43c0 + 0x00001800),
346 };
347 
348 static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
349 	(0x000143e0 + 0x00000000),
350 	(0x000143e0 + 0x00000800),
351 	(0x000143e0 + 0x00001000),
352 	(0x000143e0 + 0x00001800),
353 	(0x000543e0 + 0x00000000),
354 	(0x000543e0 + 0x00000800),
355 	(0x000543e0 + 0x00001000),
356 	(0x000543e0 + 0x00001800),
357 	(0x000943e0 + 0x00000000),
358 	(0x000943e0 + 0x00000800),
359 	(0x000943e0 + 0x00001000),
360 	(0x000943e0 + 0x00001800),
361 	(0x000d43e0 + 0x00000000),
362 	(0x000d43e0 + 0x00000800),
363 	(0x000d43e0 + 0x00001000),
364 	(0x000d43e0 + 0x00001800),
365 	(0x001143e0 + 0x00000000),
366 	(0x001143e0 + 0x00000800),
367 	(0x001143e0 + 0x00001000),
368 	(0x001143e0 + 0x00001800),
369 	(0x001543e0 + 0x00000000),
370 	(0x001543e0 + 0x00000800),
371 	(0x001543e0 + 0x00001000),
372 	(0x001543e0 + 0x00001800),
373 	(0x001943e0 + 0x00000000),
374 	(0x001943e0 + 0x00000800),
375 	(0x001943e0 + 0x00001000),
376 	(0x001943e0 + 0x00001800),
377 	(0x001d43e0 + 0x00000000),
378 	(0x001d43e0 + 0x00000800),
379 	(0x001d43e0 + 0x00001000),
380 	(0x001d43e0 + 0x00001800),
381 };
382 
383 static const uint32_t ecc_umc_mcumc_status_addrs[] = {
384 	(0x000143c2 + 0x00000000),
385 	(0x000143c2 + 0x00000800),
386 	(0x000143c2 + 0x00001000),
387 	(0x000143c2 + 0x00001800),
388 	(0x000543c2 + 0x00000000),
389 	(0x000543c2 + 0x00000800),
390 	(0x000543c2 + 0x00001000),
391 	(0x000543c2 + 0x00001800),
392 	(0x000943c2 + 0x00000000),
393 	(0x000943c2 + 0x00000800),
394 	(0x000943c2 + 0x00001000),
395 	(0x000943c2 + 0x00001800),
396 	(0x000d43c2 + 0x00000000),
397 	(0x000d43c2 + 0x00000800),
398 	(0x000d43c2 + 0x00001000),
399 	(0x000d43c2 + 0x00001800),
400 	(0x001143c2 + 0x00000000),
401 	(0x001143c2 + 0x00000800),
402 	(0x001143c2 + 0x00001000),
403 	(0x001143c2 + 0x00001800),
404 	(0x001543c2 + 0x00000000),
405 	(0x001543c2 + 0x00000800),
406 	(0x001543c2 + 0x00001000),
407 	(0x001543c2 + 0x00001800),
408 	(0x001943c2 + 0x00000000),
409 	(0x001943c2 + 0x00000800),
410 	(0x001943c2 + 0x00001000),
411 	(0x001943c2 + 0x00001800),
412 	(0x001d43c2 + 0x00000000),
413 	(0x001d43c2 + 0x00000800),
414 	(0x001d43c2 + 0x00001000),
415 	(0x001d43c2 + 0x00001800),
416 };
417 
418 static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
419 		struct amdgpu_irq_src *src,
420 		unsigned type,
421 		enum amdgpu_interrupt_state state)
422 {
423 	u32 bits, i, tmp, reg;
424 
425 	/* Devices newer then VEGA10/12 shall have these programming
426 	     sequences performed by PSP BL */
427 	if (adev->asic_type >= CHIP_VEGA20)
428 		return 0;
429 
430 	bits = 0x7f;
431 
432 	switch (state) {
433 	case AMDGPU_IRQ_STATE_DISABLE:
434 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
435 			reg = ecc_umc_mcumc_ctrl_addrs[i];
436 			tmp = RREG32(reg);
437 			tmp &= ~bits;
438 			WREG32(reg, tmp);
439 		}
440 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
441 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
442 			tmp = RREG32(reg);
443 			tmp &= ~bits;
444 			WREG32(reg, tmp);
445 		}
446 		break;
447 	case AMDGPU_IRQ_STATE_ENABLE:
448 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
449 			reg = ecc_umc_mcumc_ctrl_addrs[i];
450 			tmp = RREG32(reg);
451 			tmp |= bits;
452 			WREG32(reg, tmp);
453 		}
454 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
455 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
456 			tmp = RREG32(reg);
457 			tmp |= bits;
458 			WREG32(reg, tmp);
459 		}
460 		break;
461 	default:
462 		break;
463 	}
464 
465 	return 0;
466 }
467 
468 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
469 					struct amdgpu_irq_src *src,
470 					unsigned type,
471 					enum amdgpu_interrupt_state state)
472 {
473 	struct amdgpu_vmhub *hub;
474 	u32 tmp, reg, bits, i, j;
475 
476 	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477 		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
478 		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
479 		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
480 		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
481 		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
482 		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
483 
484 	switch (state) {
485 	case AMDGPU_IRQ_STATE_DISABLE:
486 		for (j = 0; j < adev->num_vmhubs; j++) {
487 			hub = &adev->vmhub[j];
488 			for (i = 0; i < 16; i++) {
489 				reg = hub->vm_context0_cntl + i;
490 				tmp = RREG32(reg);
491 				tmp &= ~bits;
492 				WREG32(reg, tmp);
493 			}
494 		}
495 		break;
496 	case AMDGPU_IRQ_STATE_ENABLE:
497 		for (j = 0; j < adev->num_vmhubs; j++) {
498 			hub = &adev->vmhub[j];
499 			for (i = 0; i < 16; i++) {
500 				reg = hub->vm_context0_cntl + i;
501 				tmp = RREG32(reg);
502 				tmp |= bits;
503 				WREG32(reg, tmp);
504 			}
505 		}
506 	default:
507 		break;
508 	}
509 
510 	return 0;
511 }
512 
513 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
514 				struct amdgpu_irq_src *source,
515 				struct amdgpu_iv_entry *entry)
516 {
517 	struct amdgpu_vmhub *hub;
518 	bool retry_fault = !!(entry->src_data[1] & 0x80);
519 	uint32_t status = 0, cid = 0, rw = 0;
520 	u64 addr;
521 	char hub_name[10];
522 	const char *mmhub_cid;
523 
524 	addr = (u64)entry->src_data[0] << 12;
525 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
526 
527 	if (retry_fault && amdgpu_gmc_filter_faults(adev, addr, entry->pasid,
528 						    entry->timestamp))
529 		return 1; /* This also prevents sending it to KFD */
530 
531 	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
532 		snprintf(hub_name, sizeof(hub_name), "mmhub0");
533 		hub = &adev->vmhub[AMDGPU_MMHUB_0];
534 	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
535 		snprintf(hub_name, sizeof(hub_name), "mmhub1");
536 		hub = &adev->vmhub[AMDGPU_MMHUB_1];
537 	} else {
538 		snprintf(hub_name, sizeof(hub_name), "gfxhub0");
539 		hub = &adev->vmhub[AMDGPU_GFXHUB_0];
540 	}
541 
542 	/* If it's the first fault for this address, process it normally */
543 	if (retry_fault && !in_interrupt() &&
544 	    amdgpu_vm_handle_fault(adev, entry->pasid, addr))
545 		return 1; /* This also prevents sending it to KFD */
546 
547 	if (!amdgpu_sriov_vf(adev)) {
548 		/*
549 		 * Issue a dummy read to wait for the status register to
550 		 * be updated to avoid reading an incorrect value due to
551 		 * the new fast GRBM interface.
552 		 */
553 		if (entry->vmid_src == AMDGPU_GFXHUB_0)
554 			RREG32(hub->vm_l2_pro_fault_status);
555 
556 		status = RREG32(hub->vm_l2_pro_fault_status);
557 		cid = REG_GET_FIELD(status,
558 				    VM_L2_PROTECTION_FAULT_STATUS, CID);
559 		rw = REG_GET_FIELD(status,
560 				   VM_L2_PROTECTION_FAULT_STATUS, RW);
561 		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
562 	}
563 
564 	if (printk_ratelimit()) {
565 		struct amdgpu_task_info task_info;
566 
567 		memset(&task_info, 0, sizeof(struct amdgpu_task_info));
568 		amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
569 
570 		dev_err(adev->dev,
571 			"[%s] %s page fault (src_id:%u ring:%u vmid:%u "
572 			"pasid:%u, for process %s pid %d thread %s pid %d)\n",
573 			hub_name, retry_fault ? "retry" : "no-retry",
574 			entry->src_id, entry->ring_id, entry->vmid,
575 			entry->pasid, task_info.process_name, task_info.tgid,
576 			task_info.task_name, task_info.pid);
577 		dev_err(adev->dev, "  in page starting at address 0x%016llx from client %d\n",
578 			addr, entry->client_id);
579 		if (!amdgpu_sriov_vf(adev)) {
580 			dev_err(adev->dev,
581 				"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
582 				status);
583 			if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) {
584 				dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
585 					cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid],
586 					cid);
587 			} else {
588 				switch (adev->asic_type) {
589 				case CHIP_VEGA10:
590 					mmhub_cid = mmhub_client_ids_vega10[cid][rw];
591 					break;
592 				case CHIP_VEGA12:
593 					mmhub_cid = mmhub_client_ids_vega12[cid][rw];
594 					break;
595 				case CHIP_VEGA20:
596 					mmhub_cid = mmhub_client_ids_vega20[cid][rw];
597 					break;
598 				case CHIP_ARCTURUS:
599 					mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
600 					break;
601 				case CHIP_RAVEN:
602 					mmhub_cid = mmhub_client_ids_raven[cid][rw];
603 					break;
604 				case CHIP_RENOIR:
605 					mmhub_cid = mmhub_client_ids_renoir[cid][rw];
606 					break;
607 				default:
608 					mmhub_cid = NULL;
609 					break;
610 				}
611 				dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
612 					mmhub_cid ? mmhub_cid : "unknown", cid);
613 			}
614 			dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
615 				REG_GET_FIELD(status,
616 				VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
617 			dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
618 				REG_GET_FIELD(status,
619 				VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
620 			dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
621 				REG_GET_FIELD(status,
622 				VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
623 			dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
624 				REG_GET_FIELD(status,
625 				VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
626 			dev_err(adev->dev, "\t RW: 0x%x\n", rw);
627 		}
628 	}
629 
630 	return 0;
631 }
632 
633 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
634 	.set = gmc_v9_0_vm_fault_interrupt_state,
635 	.process = gmc_v9_0_process_interrupt,
636 };
637 
638 
639 static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
640 	.set = gmc_v9_0_ecc_interrupt_state,
641 	.process = amdgpu_umc_process_ecc_irq,
642 };
643 
644 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
645 {
646 	adev->gmc.vm_fault.num_types = 1;
647 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
648 
649 	if (!amdgpu_sriov_vf(adev)) {
650 		adev->gmc.ecc_irq.num_types = 1;
651 		adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
652 	}
653 }
654 
655 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
656 					uint32_t flush_type)
657 {
658 	u32 req = 0;
659 
660 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
661 			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
662 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
663 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
664 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
665 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
666 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
667 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
668 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
669 			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
670 
671 	return req;
672 }
673 
674 /**
675  * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
676  *
677  * @adev: amdgpu_device pointer
678  * @vmhub: vmhub type
679  *
680  */
681 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
682 				       uint32_t vmhub)
683 {
684 	return ((vmhub == AMDGPU_MMHUB_0 ||
685 		 vmhub == AMDGPU_MMHUB_1) &&
686 		(!amdgpu_sriov_vf(adev)) &&
687 		(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
688 		   (adev->apu_flags & AMD_APU_IS_PICASSO))));
689 }
690 
691 static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
692 					uint8_t vmid, uint16_t *p_pasid)
693 {
694 	uint32_t value;
695 
696 	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
697 		     + vmid);
698 	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
699 
700 	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
701 }
702 
703 /*
704  * GART
705  * VMID 0 is the physical GPU addresses as used by the kernel.
706  * VMIDs 1-15 are used for userspace clients and are handled
707  * by the amdgpu vm/hsa code.
708  */
709 
710 /**
711  * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
712  *
713  * @adev: amdgpu_device pointer
714  * @vmid: vm instance to flush
715  * @flush_type: the flush type
716  *
717  * Flush the TLB for the requested page table using certain type.
718  */
719 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
720 					uint32_t vmhub, uint32_t flush_type)
721 {
722 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
723 	const unsigned eng = 17;
724 	u32 j, inv_req, inv_req2, tmp;
725 	struct amdgpu_vmhub *hub;
726 
727 	BUG_ON(vmhub >= adev->num_vmhubs);
728 
729 	hub = &adev->vmhub[vmhub];
730 	if (adev->gmc.xgmi.num_physical_nodes &&
731 	    adev->asic_type == CHIP_VEGA20) {
732 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
733 		 * heavy-weight TLB flush (type 2), which flushes
734 		 * both. Due to a race condition with concurrent
735 		 * memory accesses using the same TLB cache line, we
736 		 * still need a second TLB flush after this.
737 		 */
738 		inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
739 		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
740 	} else {
741 		inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
742 		inv_req2 = 0;
743 	}
744 
745 	/* This is necessary for a HW workaround under SRIOV as well
746 	 * as GFXOFF under bare metal
747 	 */
748 	if (adev->gfx.kiq.ring.sched.ready &&
749 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
750 	    down_read_trylock(&adev->reset_sem)) {
751 		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
752 		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
753 
754 		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
755 						   1 << vmid);
756 		up_read(&adev->reset_sem);
757 		return;
758 	}
759 
760 	spin_lock(&adev->gmc.invalidate_lock);
761 
762 	/*
763 	 * It may lose gpuvm invalidate acknowldege state across power-gating
764 	 * off cycle, add semaphore acquire before invalidation and semaphore
765 	 * release after invalidation to avoid entering power gated state
766 	 * to WA the Issue
767 	 */
768 
769 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
770 	if (use_semaphore) {
771 		for (j = 0; j < adev->usec_timeout; j++) {
772 			/* a read return value of 1 means semaphore acuqire */
773 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem +
774 					    hub->eng_distance * eng);
775 			if (tmp & 0x1)
776 				break;
777 			udelay(1);
778 		}
779 
780 		if (j >= adev->usec_timeout)
781 			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
782 	}
783 
784 	do {
785 		WREG32_NO_KIQ(hub->vm_inv_eng0_req +
786 			      hub->eng_distance * eng, inv_req);
787 
788 		/*
789 		 * Issue a dummy read to wait for the ACK register to
790 		 * be cleared to avoid a false ACK due to the new fast
791 		 * GRBM interface.
792 		 */
793 		if (vmhub == AMDGPU_GFXHUB_0)
794 			RREG32_NO_KIQ(hub->vm_inv_eng0_req +
795 				      hub->eng_distance * eng);
796 
797 		for (j = 0; j < adev->usec_timeout; j++) {
798 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack +
799 					    hub->eng_distance * eng);
800 			if (tmp & (1 << vmid))
801 				break;
802 			udelay(1);
803 		}
804 
805 		inv_req = inv_req2;
806 		inv_req2 = 0;
807 	} while (inv_req);
808 
809 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
810 	if (use_semaphore)
811 		/*
812 		 * add semaphore release after invalidation,
813 		 * write with 0 means semaphore release
814 		 */
815 		WREG32_NO_KIQ(hub->vm_inv_eng0_sem +
816 			      hub->eng_distance * eng, 0);
817 
818 	spin_unlock(&adev->gmc.invalidate_lock);
819 
820 	if (j < adev->usec_timeout)
821 		return;
822 
823 	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
824 }
825 
826 /**
827  * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
828  *
829  * @adev: amdgpu_device pointer
830  * @pasid: pasid to be flush
831  *
832  * Flush the TLB for the requested pasid.
833  */
834 static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
835 					uint16_t pasid, uint32_t flush_type,
836 					bool all_hub)
837 {
838 	int vmid, i;
839 	signed long r;
840 	uint32_t seq;
841 	uint16_t queried_pasid;
842 	bool ret;
843 	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
844 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
845 
846 	if (amdgpu_in_reset(adev))
847 		return -EIO;
848 
849 	if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) {
850 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
851 		 * heavy-weight TLB flush (type 2), which flushes
852 		 * both. Due to a race condition with concurrent
853 		 * memory accesses using the same TLB cache line, we
854 		 * still need a second TLB flush after this.
855 		 */
856 		bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
857 				       adev->asic_type == CHIP_VEGA20);
858 		/* 2 dwords flush + 8 dwords fence */
859 		unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
860 
861 		if (vega20_xgmi_wa)
862 			ndw += kiq->pmf->invalidate_tlbs_size;
863 
864 		spin_lock(&adev->gfx.kiq.ring_lock);
865 		/* 2 dwords flush + 8 dwords fence */
866 		amdgpu_ring_alloc(ring, ndw);
867 		if (vega20_xgmi_wa)
868 			kiq->pmf->kiq_invalidate_tlbs(ring,
869 						      pasid, 2, all_hub);
870 		kiq->pmf->kiq_invalidate_tlbs(ring,
871 					pasid, flush_type, all_hub);
872 		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
873 		if (r) {
874 			amdgpu_ring_undo(ring);
875 			spin_unlock(&adev->gfx.kiq.ring_lock);
876 			up_read(&adev->reset_sem);
877 			return -ETIME;
878 		}
879 
880 		amdgpu_ring_commit(ring);
881 		spin_unlock(&adev->gfx.kiq.ring_lock);
882 		r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
883 		if (r < 1) {
884 			dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
885 			up_read(&adev->reset_sem);
886 			return -ETIME;
887 		}
888 		up_read(&adev->reset_sem);
889 		return 0;
890 	}
891 
892 	for (vmid = 1; vmid < 16; vmid++) {
893 
894 		ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
895 				&queried_pasid);
896 		if (ret && queried_pasid == pasid) {
897 			if (all_hub) {
898 				for (i = 0; i < adev->num_vmhubs; i++)
899 					gmc_v9_0_flush_gpu_tlb(adev, vmid,
900 							i, flush_type);
901 			} else {
902 				gmc_v9_0_flush_gpu_tlb(adev, vmid,
903 						AMDGPU_GFXHUB_0, flush_type);
904 			}
905 			break;
906 		}
907 	}
908 
909 	return 0;
910 
911 }
912 
913 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
914 					    unsigned vmid, uint64_t pd_addr)
915 {
916 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
917 	struct amdgpu_device *adev = ring->adev;
918 	struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
919 	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
920 	unsigned eng = ring->vm_inv_eng;
921 
922 	/*
923 	 * It may lose gpuvm invalidate acknowldege state across power-gating
924 	 * off cycle, add semaphore acquire before invalidation and semaphore
925 	 * release after invalidation to avoid entering power gated state
926 	 * to WA the Issue
927 	 */
928 
929 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
930 	if (use_semaphore)
931 		/* a read return value of 1 means semaphore acuqire */
932 		amdgpu_ring_emit_reg_wait(ring,
933 					  hub->vm_inv_eng0_sem +
934 					  hub->eng_distance * eng, 0x1, 0x1);
935 
936 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
937 			      (hub->ctx_addr_distance * vmid),
938 			      lower_32_bits(pd_addr));
939 
940 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
941 			      (hub->ctx_addr_distance * vmid),
942 			      upper_32_bits(pd_addr));
943 
944 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
945 					    hub->eng_distance * eng,
946 					    hub->vm_inv_eng0_ack +
947 					    hub->eng_distance * eng,
948 					    req, 1 << vmid);
949 
950 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
951 	if (use_semaphore)
952 		/*
953 		 * add semaphore release after invalidation,
954 		 * write with 0 means semaphore release
955 		 */
956 		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
957 				      hub->eng_distance * eng, 0);
958 
959 	return pd_addr;
960 }
961 
962 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
963 					unsigned pasid)
964 {
965 	struct amdgpu_device *adev = ring->adev;
966 	uint32_t reg;
967 
968 	/* Do nothing because there's no lut register for mmhub1. */
969 	if (ring->funcs->vmhub == AMDGPU_MMHUB_1)
970 		return;
971 
972 	if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
973 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
974 	else
975 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
976 
977 	amdgpu_ring_emit_wreg(ring, reg, pasid);
978 }
979 
980 /*
981  * PTE format on VEGA 10:
982  * 63:59 reserved
983  * 58:57 mtype
984  * 56 F
985  * 55 L
986  * 54 P
987  * 53 SW
988  * 52 T
989  * 50:48 reserved
990  * 47:12 4k physical page base address
991  * 11:7 fragment
992  * 6 write
993  * 5 read
994  * 4 exe
995  * 3 Z
996  * 2 snooped
997  * 1 system
998  * 0 valid
999  *
1000  * PDE format on VEGA 10:
1001  * 63:59 block fragment size
1002  * 58:55 reserved
1003  * 54 P
1004  * 53:48 reserved
1005  * 47:6 physical base address of PD or PTE
1006  * 5:3 reserved
1007  * 2 C
1008  * 1 system
1009  * 0 valid
1010  */
1011 
1012 static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1013 
1014 {
1015 	switch (flags) {
1016 	case AMDGPU_VM_MTYPE_DEFAULT:
1017 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1018 	case AMDGPU_VM_MTYPE_NC:
1019 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1020 	case AMDGPU_VM_MTYPE_WC:
1021 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1022 	case AMDGPU_VM_MTYPE_RW:
1023 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1024 	case AMDGPU_VM_MTYPE_CC:
1025 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1026 	case AMDGPU_VM_MTYPE_UC:
1027 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1028 	default:
1029 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1030 	}
1031 }
1032 
1033 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1034 				uint64_t *addr, uint64_t *flags)
1035 {
1036 	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1037 		*addr = adev->vm_manager.vram_base_offset + *addr -
1038 			adev->gmc.vram_start;
1039 	BUG_ON(*addr & 0xFFFF00000000003FULL);
1040 
1041 	if (!adev->gmc.translate_further)
1042 		return;
1043 
1044 	if (level == AMDGPU_VM_PDB1) {
1045 		/* Set the block fragment size */
1046 		if (!(*flags & AMDGPU_PDE_PTE))
1047 			*flags |= AMDGPU_PDE_BFS(0x9);
1048 
1049 	} else if (level == AMDGPU_VM_PDB0) {
1050 		if (*flags & AMDGPU_PDE_PTE)
1051 			*flags &= ~AMDGPU_PDE_PTE;
1052 		else
1053 			*flags |= AMDGPU_PTE_TF;
1054 	}
1055 }
1056 
1057 static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1058 				struct amdgpu_bo_va_mapping *mapping,
1059 				uint64_t *flags)
1060 {
1061 	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1062 	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1063 
1064 	*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1065 	*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1066 
1067 	if (mapping->flags & AMDGPU_PTE_PRT) {
1068 		*flags |= AMDGPU_PTE_PRT;
1069 		*flags &= ~AMDGPU_PTE_VALID;
1070 	}
1071 
1072 	if (adev->asic_type == CHIP_ARCTURUS &&
1073 	    !(*flags & AMDGPU_PTE_SYSTEM) &&
1074 	    mapping->bo_va->is_xgmi)
1075 		*flags |= AMDGPU_PTE_SNOOPED;
1076 }
1077 
1078 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1079 {
1080 	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1081 	unsigned size;
1082 
1083 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1084 		size = AMDGPU_VBIOS_VGA_ALLOCATION;
1085 	} else {
1086 		u32 viewport;
1087 
1088 		switch (adev->asic_type) {
1089 		case CHIP_RAVEN:
1090 		case CHIP_RENOIR:
1091 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1092 			size = (REG_GET_FIELD(viewport,
1093 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1094 				REG_GET_FIELD(viewport,
1095 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1096 				4);
1097 			break;
1098 		case CHIP_VEGA10:
1099 		case CHIP_VEGA12:
1100 		case CHIP_VEGA20:
1101 		default:
1102 			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1103 			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1104 				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1105 				4);
1106 			break;
1107 		}
1108 	}
1109 
1110 	return size;
1111 }
1112 
1113 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1114 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1115 	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1116 	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1117 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1118 	.map_mtype = gmc_v9_0_map_mtype,
1119 	.get_vm_pde = gmc_v9_0_get_vm_pde,
1120 	.get_vm_pte = gmc_v9_0_get_vm_pte,
1121 	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1122 };
1123 
1124 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1125 {
1126 	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1127 }
1128 
1129 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1130 {
1131 	switch (adev->asic_type) {
1132 	case CHIP_VEGA10:
1133 		adev->umc.funcs = &umc_v6_0_funcs;
1134 		break;
1135 	case CHIP_VEGA20:
1136 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1137 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1138 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1139 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1140 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1141 		adev->umc.funcs = &umc_v6_1_funcs;
1142 		break;
1143 	case CHIP_ARCTURUS:
1144 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1145 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1146 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1147 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1148 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1149 		adev->umc.funcs = &umc_v6_1_funcs;
1150 		break;
1151 	default:
1152 		break;
1153 	}
1154 }
1155 
1156 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1157 {
1158 	switch (adev->asic_type) {
1159 	case CHIP_ARCTURUS:
1160 		adev->mmhub.funcs = &mmhub_v9_4_funcs;
1161 		break;
1162 	default:
1163 		adev->mmhub.funcs = &mmhub_v1_0_funcs;
1164 		break;
1165 	}
1166 }
1167 
1168 static int gmc_v9_0_early_init(void *handle)
1169 {
1170 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1171 
1172 	gmc_v9_0_set_gmc_funcs(adev);
1173 	gmc_v9_0_set_irq_funcs(adev);
1174 	gmc_v9_0_set_umc_funcs(adev);
1175 	gmc_v9_0_set_mmhub_funcs(adev);
1176 
1177 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1178 	adev->gmc.shared_aperture_end =
1179 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1180 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1181 	adev->gmc.private_aperture_end =
1182 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1183 
1184 	return 0;
1185 }
1186 
1187 static int gmc_v9_0_late_init(void *handle)
1188 {
1189 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1190 	int r;
1191 
1192 	amdgpu_bo_late_init(adev);
1193 
1194 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1195 	if (r)
1196 		return r;
1197 	/* Check if ecc is available */
1198 	if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) {
1199 		r = amdgpu_atomfirmware_mem_ecc_supported(adev);
1200 		if (!r) {
1201 			DRM_INFO("ECC is not present.\n");
1202 			if (adev->df.funcs->enable_ecc_force_par_wr_rmw)
1203 				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1204 		} else
1205 			DRM_INFO("ECC is active.\n");
1206 
1207 		r = amdgpu_atomfirmware_sram_ecc_supported(adev);
1208 		if (!r)
1209 			DRM_INFO("SRAM ECC is not present.\n");
1210 		else
1211 			DRM_INFO("SRAM ECC is active.\n");
1212 	}
1213 
1214 	if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)
1215 		adev->mmhub.funcs->reset_ras_error_count(adev);
1216 
1217 	r = amdgpu_gmc_ras_late_init(adev);
1218 	if (r)
1219 		return r;
1220 
1221 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1222 }
1223 
1224 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1225 					struct amdgpu_gmc *mc)
1226 {
1227 	u64 base = 0;
1228 
1229 	if (!amdgpu_sriov_vf(adev))
1230 		base = adev->mmhub.funcs->get_fb_location(adev);
1231 
1232 	/* add the xgmi offset of the physical node */
1233 	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1234 	amdgpu_gmc_vram_location(adev, mc, base);
1235 	amdgpu_gmc_gart_location(adev, mc);
1236 	amdgpu_gmc_agp_location(adev, mc);
1237 	/* base offset of vram pages */
1238 	adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
1239 
1240 	/* XXX: add the xgmi offset of the physical node? */
1241 	adev->vm_manager.vram_base_offset +=
1242 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1243 }
1244 
1245 /**
1246  * gmc_v9_0_mc_init - initialize the memory controller driver params
1247  *
1248  * @adev: amdgpu_device pointer
1249  *
1250  * Look up the amount of vram, vram width, and decide how to place
1251  * vram and gart within the GPU's physical address space.
1252  * Returns 0 for success.
1253  */
1254 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1255 {
1256 	int r;
1257 
1258 	/* size in MB on si */
1259 	adev->gmc.mc_vram_size =
1260 		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1261 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1262 
1263 	if (!(adev->flags & AMD_IS_APU)) {
1264 		r = amdgpu_device_resize_fb_bar(adev);
1265 		if (r)
1266 			return r;
1267 	}
1268 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1269 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1270 
1271 #ifdef CONFIG_X86_64
1272 	if (adev->flags & AMD_IS_APU) {
1273 		adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev);
1274 		adev->gmc.aper_size = adev->gmc.real_vram_size;
1275 	}
1276 #endif
1277 	/* In case the PCI BAR is larger than the actual amount of vram */
1278 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1279 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
1280 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
1281 
1282 	/* set the gart size */
1283 	if (amdgpu_gart_size == -1) {
1284 		switch (adev->asic_type) {
1285 		case CHIP_VEGA10:  /* all engines support GPUVM */
1286 		case CHIP_VEGA12:  /* all engines support GPUVM */
1287 		case CHIP_VEGA20:
1288 		case CHIP_ARCTURUS:
1289 		default:
1290 			adev->gmc.gart_size = 512ULL << 20;
1291 			break;
1292 		case CHIP_RAVEN:   /* DCE SG support */
1293 		case CHIP_RENOIR:
1294 			adev->gmc.gart_size = 1024ULL << 20;
1295 			break;
1296 		}
1297 	} else {
1298 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1299 	}
1300 
1301 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1302 
1303 	return 0;
1304 }
1305 
1306 static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1307 {
1308 	int r;
1309 
1310 	if (adev->gart.bo) {
1311 		WARN(1, "VEGA10 PCIE GART already initialized\n");
1312 		return 0;
1313 	}
1314 	/* Initialize common gart structure */
1315 	r = amdgpu_gart_init(adev);
1316 	if (r)
1317 		return r;
1318 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1319 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1320 				 AMDGPU_PTE_EXECUTABLE;
1321 	return amdgpu_gart_table_vram_alloc(adev);
1322 }
1323 
1324 /**
1325  * gmc_v9_0_save_registers - saves regs
1326  *
1327  * @adev: amdgpu_device pointer
1328  *
1329  * This saves potential register values that should be
1330  * restored upon resume
1331  */
1332 static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1333 {
1334 	if (adev->asic_type == CHIP_RAVEN)
1335 		adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1336 }
1337 
1338 static int gmc_v9_0_sw_init(void *handle)
1339 {
1340 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
1341 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1342 
1343 	gfxhub_v1_0_init(adev);
1344 
1345 	adev->mmhub.funcs->init(adev);
1346 
1347 	spin_lock_init(&adev->gmc.invalidate_lock);
1348 
1349 	r = amdgpu_atomfirmware_get_vram_info(adev,
1350 		&vram_width, &vram_type, &vram_vendor);
1351 	if (amdgpu_sriov_vf(adev))
1352 		/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1353 		 * and DF related registers is not readable, seems hardcord is the
1354 		 * only way to set the correct vram_width
1355 		 */
1356 		adev->gmc.vram_width = 2048;
1357 	else if (amdgpu_emu_mode != 1)
1358 		adev->gmc.vram_width = vram_width;
1359 
1360 	if (!adev->gmc.vram_width) {
1361 		int chansize, numchan;
1362 
1363 		/* hbm memory channel size */
1364 		if (adev->flags & AMD_IS_APU)
1365 			chansize = 64;
1366 		else
1367 			chansize = 128;
1368 
1369 		numchan = adev->df.funcs->get_hbm_channel_number(adev);
1370 		adev->gmc.vram_width = numchan * chansize;
1371 	}
1372 
1373 	adev->gmc.vram_type = vram_type;
1374 	adev->gmc.vram_vendor = vram_vendor;
1375 	switch (adev->asic_type) {
1376 	case CHIP_RAVEN:
1377 		adev->num_vmhubs = 2;
1378 
1379 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
1380 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1381 		} else {
1382 			/* vm_size is 128TB + 512GB for legacy 3-level page support */
1383 			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
1384 			adev->gmc.translate_further =
1385 				adev->vm_manager.num_level > 1;
1386 		}
1387 		break;
1388 	case CHIP_VEGA10:
1389 	case CHIP_VEGA12:
1390 	case CHIP_VEGA20:
1391 	case CHIP_RENOIR:
1392 		adev->num_vmhubs = 2;
1393 
1394 
1395 		/*
1396 		 * To fulfill 4-level page support,
1397 		 * vm size is 256TB (48bit), maximum size of Vega10,
1398 		 * block size 512 (9bit)
1399 		 */
1400 		/* sriov restrict max_pfn below AMDGPU_GMC_HOLE */
1401 		if (amdgpu_sriov_vf(adev))
1402 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
1403 		else
1404 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1405 		break;
1406 	case CHIP_ARCTURUS:
1407 		adev->num_vmhubs = 3;
1408 
1409 		/* Keep the vm size same with Vega20 */
1410 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1411 		break;
1412 	default:
1413 		break;
1414 	}
1415 
1416 	/* This interrupt is VMC page fault.*/
1417 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
1418 				&adev->gmc.vm_fault);
1419 	if (r)
1420 		return r;
1421 
1422 	if (adev->asic_type == CHIP_ARCTURUS) {
1423 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1424 					&adev->gmc.vm_fault);
1425 		if (r)
1426 			return r;
1427 	}
1428 
1429 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
1430 				&adev->gmc.vm_fault);
1431 
1432 	if (r)
1433 		return r;
1434 
1435 	if (!amdgpu_sriov_vf(adev)) {
1436 		/* interrupt sent to DF. */
1437 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
1438 				      &adev->gmc.ecc_irq);
1439 		if (r)
1440 			return r;
1441 	}
1442 
1443 	/* Set the internal MC address mask
1444 	 * This is the max address of the GPU's
1445 	 * internal address space.
1446 	 */
1447 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
1448 
1449 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
1450 	if (r) {
1451 		printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
1452 		return r;
1453 	}
1454 	adev->need_swiotlb = drm_need_swiotlb(44);
1455 
1456 	if (adev->gmc.xgmi.supported) {
1457 		r = gfxhub_v1_1_get_xgmi_info(adev);
1458 		if (r)
1459 			return r;
1460 	}
1461 
1462 	r = gmc_v9_0_mc_init(adev);
1463 	if (r)
1464 		return r;
1465 
1466 	amdgpu_gmc_get_vbios_allocations(adev);
1467 
1468 	/* Memory manager */
1469 	r = amdgpu_bo_init(adev);
1470 	if (r)
1471 		return r;
1472 
1473 	r = gmc_v9_0_gart_init(adev);
1474 	if (r)
1475 		return r;
1476 
1477 	/*
1478 	 * number of VMs
1479 	 * VMID 0 is reserved for System
1480 	 * amdgpu graphics/compute will use VMIDs 1..n-1
1481 	 * amdkfd will use VMIDs n..15
1482 	 *
1483 	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
1484 	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
1485 	 * for video processing.
1486 	 */
1487 	adev->vm_manager.first_kfd_vmid =
1488 		adev->asic_type == CHIP_ARCTURUS ? 3 : 8;
1489 
1490 	amdgpu_vm_manager_init(adev);
1491 
1492 	gmc_v9_0_save_registers(adev);
1493 
1494 	return 0;
1495 }
1496 
1497 static int gmc_v9_0_sw_fini(void *handle)
1498 {
1499 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1500 
1501 	amdgpu_gmc_ras_fini(adev);
1502 	amdgpu_gem_force_release(adev);
1503 	amdgpu_vm_manager_fini(adev);
1504 	amdgpu_gart_table_vram_free(adev);
1505 	amdgpu_bo_fini(adev);
1506 	amdgpu_gart_fini(adev);
1507 
1508 	return 0;
1509 }
1510 
1511 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
1512 {
1513 
1514 	switch (adev->asic_type) {
1515 	case CHIP_VEGA10:
1516 		if (amdgpu_sriov_vf(adev))
1517 			break;
1518 		fallthrough;
1519 	case CHIP_VEGA20:
1520 		soc15_program_register_sequence(adev,
1521 						golden_settings_mmhub_1_0_0,
1522 						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
1523 		soc15_program_register_sequence(adev,
1524 						golden_settings_athub_1_0_0,
1525 						ARRAY_SIZE(golden_settings_athub_1_0_0));
1526 		break;
1527 	case CHIP_VEGA12:
1528 		break;
1529 	case CHIP_RAVEN:
1530 		/* TODO for renoir */
1531 		soc15_program_register_sequence(adev,
1532 						golden_settings_athub_1_0_0,
1533 						ARRAY_SIZE(golden_settings_athub_1_0_0));
1534 		break;
1535 	default:
1536 		break;
1537 	}
1538 }
1539 
1540 /**
1541  * gmc_v9_0_restore_registers - restores regs
1542  *
1543  * @adev: amdgpu_device pointer
1544  *
1545  * This restores register values, saved at suspend.
1546  */
1547 void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
1548 {
1549 	if (adev->asic_type == CHIP_RAVEN)
1550 		WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
1551 }
1552 
1553 /**
1554  * gmc_v9_0_gart_enable - gart enable
1555  *
1556  * @adev: amdgpu_device pointer
1557  */
1558 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1559 {
1560 	int r;
1561 
1562 	if (adev->gart.bo == NULL) {
1563 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1564 		return -EINVAL;
1565 	}
1566 	r = amdgpu_gart_table_vram_pin(adev);
1567 	if (r)
1568 		return r;
1569 
1570 	r = gfxhub_v1_0_gart_enable(adev);
1571 	if (r)
1572 		return r;
1573 
1574 	r = adev->mmhub.funcs->gart_enable(adev);
1575 	if (r)
1576 		return r;
1577 
1578 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1579 		 (unsigned)(adev->gmc.gart_size >> 20),
1580 		 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
1581 	adev->gart.ready = true;
1582 	return 0;
1583 }
1584 
1585 static int gmc_v9_0_hw_init(void *handle)
1586 {
1587 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1588 	bool value;
1589 	int r, i;
1590 	u32 tmp;
1591 
1592 	/* The sequence of these two function calls matters.*/
1593 	gmc_v9_0_init_golden_registers(adev);
1594 
1595 	if (adev->mode_info.num_crtc) {
1596 		if (adev->asic_type != CHIP_ARCTURUS) {
1597 			/* Lockout access through VGA aperture*/
1598 			WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1599 
1600 			/* disable VGA render */
1601 			WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
1602 		}
1603 	}
1604 
1605 	amdgpu_device_program_register_sequence(adev,
1606 						golden_settings_vega10_hdp,
1607 						ARRAY_SIZE(golden_settings_vega10_hdp));
1608 
1609 	if (adev->mmhub.funcs->update_power_gating)
1610 		adev->mmhub.funcs->update_power_gating(adev, true);
1611 
1612 	switch (adev->asic_type) {
1613 	case CHIP_ARCTURUS:
1614 		WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1);
1615 		break;
1616 	default:
1617 		break;
1618 	}
1619 
1620 	WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
1621 
1622 	tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
1623 	WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
1624 
1625 	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8));
1626 	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));
1627 
1628 	/* After HDP is initialized, flush HDP.*/
1629 	adev->nbio.funcs->hdp_flush(adev, NULL);
1630 
1631 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1632 		value = false;
1633 	else
1634 		value = true;
1635 
1636 	if (!amdgpu_sriov_vf(adev)) {
1637 		gfxhub_v1_0_set_fault_enable_default(adev, value);
1638 		adev->mmhub.funcs->set_fault_enable_default(adev, value);
1639 	}
1640 	for (i = 0; i < adev->num_vmhubs; ++i)
1641 		gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
1642 
1643 	if (adev->umc.funcs && adev->umc.funcs->init_registers)
1644 		adev->umc.funcs->init_registers(adev);
1645 
1646 	r = gmc_v9_0_gart_enable(adev);
1647 
1648 	return r;
1649 }
1650 
1651 /**
1652  * gmc_v9_0_gart_disable - gart disable
1653  *
1654  * @adev: amdgpu_device pointer
1655  *
1656  * This disables all VM page table.
1657  */
1658 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1659 {
1660 	gfxhub_v1_0_gart_disable(adev);
1661 	adev->mmhub.funcs->gart_disable(adev);
1662 	amdgpu_gart_table_vram_unpin(adev);
1663 }
1664 
1665 static int gmc_v9_0_hw_fini(void *handle)
1666 {
1667 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1668 
1669 	if (amdgpu_sriov_vf(adev)) {
1670 		/* full access mode, so don't touch any GMC register */
1671 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1672 		return 0;
1673 	}
1674 
1675 	amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
1676 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1677 	gmc_v9_0_gart_disable(adev);
1678 
1679 	return 0;
1680 }
1681 
1682 static int gmc_v9_0_suspend(void *handle)
1683 {
1684 	int r;
1685 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1686 
1687 	r = gmc_v9_0_hw_fini(adev);
1688 	if (r)
1689 		return r;
1690 
1691 	return 0;
1692 }
1693 
1694 static int gmc_v9_0_resume(void *handle)
1695 {
1696 	int r;
1697 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1698 
1699 	r = gmc_v9_0_hw_init(adev);
1700 	if (r)
1701 		return r;
1702 
1703 	amdgpu_vmid_reset_all(adev);
1704 
1705 	return 0;
1706 }
1707 
1708 static bool gmc_v9_0_is_idle(void *handle)
1709 {
1710 	/* MC is always ready in GMC v9.*/
1711 	return true;
1712 }
1713 
1714 static int gmc_v9_0_wait_for_idle(void *handle)
1715 {
1716 	/* There is no need to wait for MC idle in GMC v9.*/
1717 	return 0;
1718 }
1719 
1720 static int gmc_v9_0_soft_reset(void *handle)
1721 {
1722 	/* XXX for emulation.*/
1723 	return 0;
1724 }
1725 
1726 static int gmc_v9_0_set_clockgating_state(void *handle,
1727 					enum amd_clockgating_state state)
1728 {
1729 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1730 
1731 	adev->mmhub.funcs->set_clockgating(adev, state);
1732 
1733 	athub_v1_0_set_clockgating(adev, state);
1734 
1735 	return 0;
1736 }
1737 
1738 static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1739 {
1740 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1741 
1742 	adev->mmhub.funcs->get_clockgating(adev, flags);
1743 
1744 	athub_v1_0_get_clockgating(adev, flags);
1745 }
1746 
1747 static int gmc_v9_0_set_powergating_state(void *handle,
1748 					enum amd_powergating_state state)
1749 {
1750 	return 0;
1751 }
1752 
1753 const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1754 	.name = "gmc_v9_0",
1755 	.early_init = gmc_v9_0_early_init,
1756 	.late_init = gmc_v9_0_late_init,
1757 	.sw_init = gmc_v9_0_sw_init,
1758 	.sw_fini = gmc_v9_0_sw_fini,
1759 	.hw_init = gmc_v9_0_hw_init,
1760 	.hw_fini = gmc_v9_0_hw_fini,
1761 	.suspend = gmc_v9_0_suspend,
1762 	.resume = gmc_v9_0_resume,
1763 	.is_idle = gmc_v9_0_is_idle,
1764 	.wait_for_idle = gmc_v9_0_wait_for_idle,
1765 	.soft_reset = gmc_v9_0_soft_reset,
1766 	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
1767 	.set_powergating_state = gmc_v9_0_set_powergating_state,
1768 	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
1769 };
1770 
1771 const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1772 {
1773 	.type = AMD_IP_BLOCK_TYPE_GMC,
1774 	.major = 9,
1775 	.minor = 0,
1776 	.rev = 0,
1777 	.funcs = &gmc_v9_0_ip_funcs,
1778 };
1779