xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c (revision a2475e62)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/pci.h>
26 
27 #include <drm/drm_cache.h>
28 
29 #include "amdgpu.h"
30 #include "gmc_v9_0.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "amdgpu_gem.h"
33 
34 #include "hdp/hdp_4_0_offset.h"
35 #include "hdp/hdp_4_0_sh_mask.h"
36 #include "gc/gc_9_0_sh_mask.h"
37 #include "dce/dce_12_0_offset.h"
38 #include "dce/dce_12_0_sh_mask.h"
39 #include "vega10_enum.h"
40 #include "mmhub/mmhub_1_0_offset.h"
41 #include "athub/athub_1_0_sh_mask.h"
42 #include "athub/athub_1_0_offset.h"
43 #include "oss/osssys_4_0_offset.h"
44 
45 #include "soc15.h"
46 #include "soc15d.h"
47 #include "soc15_common.h"
48 #include "umc/umc_6_0_sh_mask.h"
49 
50 #include "gfxhub_v1_0.h"
51 #include "mmhub_v1_0.h"
52 #include "athub_v1_0.h"
53 #include "gfxhub_v1_1.h"
54 #include "mmhub_v9_4.h"
55 #include "umc_v6_1.h"
56 #include "umc_v6_0.h"
57 
58 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
59 
60 #include "amdgpu_ras.h"
61 #include "amdgpu_xgmi.h"
62 
63 /* add these here since we already include dce12 headers and these are for DCN */
64 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
65 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
66 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
67 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
68 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
69 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
70 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0                                                                  0x049d
71 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX                                                         2
72 
73 
74 static const char *gfxhub_client_ids[] = {
75 	"CB",
76 	"DB",
77 	"IA",
78 	"WD",
79 	"CPF",
80 	"CPC",
81 	"CPG",
82 	"RLC",
83 	"TCP",
84 	"SQC (inst)",
85 	"SQC (data)",
86 	"SQG",
87 	"PA",
88 };
89 
90 static const char *mmhub_client_ids_raven[][2] = {
91 	[0][0] = "MP1",
92 	[1][0] = "MP0",
93 	[2][0] = "VCN",
94 	[3][0] = "VCNU",
95 	[4][0] = "HDP",
96 	[5][0] = "DCE",
97 	[13][0] = "UTCL2",
98 	[19][0] = "TLS",
99 	[26][0] = "OSS",
100 	[27][0] = "SDMA0",
101 	[0][1] = "MP1",
102 	[1][1] = "MP0",
103 	[2][1] = "VCN",
104 	[3][1] = "VCNU",
105 	[4][1] = "HDP",
106 	[5][1] = "XDP",
107 	[6][1] = "DBGU0",
108 	[7][1] = "DCE",
109 	[8][1] = "DCEDWB0",
110 	[9][1] = "DCEDWB1",
111 	[26][1] = "OSS",
112 	[27][1] = "SDMA0",
113 };
114 
115 static const char *mmhub_client_ids_renoir[][2] = {
116 	[0][0] = "MP1",
117 	[1][0] = "MP0",
118 	[2][0] = "HDP",
119 	[4][0] = "DCEDMC",
120 	[5][0] = "DCEVGA",
121 	[13][0] = "UTCL2",
122 	[19][0] = "TLS",
123 	[26][0] = "OSS",
124 	[27][0] = "SDMA0",
125 	[28][0] = "VCN",
126 	[29][0] = "VCNU",
127 	[30][0] = "JPEG",
128 	[0][1] = "MP1",
129 	[1][1] = "MP0",
130 	[2][1] = "HDP",
131 	[3][1] = "XDP",
132 	[6][1] = "DBGU0",
133 	[7][1] = "DCEDMC",
134 	[8][1] = "DCEVGA",
135 	[9][1] = "DCEDWB",
136 	[26][1] = "OSS",
137 	[27][1] = "SDMA0",
138 	[28][1] = "VCN",
139 	[29][1] = "VCNU",
140 	[30][1] = "JPEG",
141 };
142 
143 static const char *mmhub_client_ids_vega10[][2] = {
144 	[0][0] = "MP0",
145 	[1][0] = "UVD",
146 	[2][0] = "UVDU",
147 	[3][0] = "HDP",
148 	[13][0] = "UTCL2",
149 	[14][0] = "OSS",
150 	[15][0] = "SDMA1",
151 	[32+0][0] = "VCE0",
152 	[32+1][0] = "VCE0U",
153 	[32+2][0] = "XDMA",
154 	[32+3][0] = "DCE",
155 	[32+4][0] = "MP1",
156 	[32+14][0] = "SDMA0",
157 	[0][1] = "MP0",
158 	[1][1] = "UVD",
159 	[2][1] = "UVDU",
160 	[3][1] = "DBGU0",
161 	[4][1] = "HDP",
162 	[5][1] = "XDP",
163 	[14][1] = "OSS",
164 	[15][1] = "SDMA0",
165 	[32+0][1] = "VCE0",
166 	[32+1][1] = "VCE0U",
167 	[32+2][1] = "XDMA",
168 	[32+3][1] = "DCE",
169 	[32+4][1] = "DCEDWB",
170 	[32+5][1] = "MP1",
171 	[32+6][1] = "DBGU1",
172 	[32+14][1] = "SDMA1",
173 };
174 
175 static const char *mmhub_client_ids_vega12[][2] = {
176 	[0][0] = "MP0",
177 	[1][0] = "VCE0",
178 	[2][0] = "VCE0U",
179 	[3][0] = "HDP",
180 	[13][0] = "UTCL2",
181 	[14][0] = "OSS",
182 	[15][0] = "SDMA1",
183 	[32+0][0] = "DCE",
184 	[32+1][0] = "XDMA",
185 	[32+2][0] = "UVD",
186 	[32+3][0] = "UVDU",
187 	[32+4][0] = "MP1",
188 	[32+15][0] = "SDMA0",
189 	[0][1] = "MP0",
190 	[1][1] = "VCE0",
191 	[2][1] = "VCE0U",
192 	[3][1] = "DBGU0",
193 	[4][1] = "HDP",
194 	[5][1] = "XDP",
195 	[14][1] = "OSS",
196 	[15][1] = "SDMA0",
197 	[32+0][1] = "DCE",
198 	[32+1][1] = "DCEDWB",
199 	[32+2][1] = "XDMA",
200 	[32+3][1] = "UVD",
201 	[32+4][1] = "UVDU",
202 	[32+5][1] = "MP1",
203 	[32+6][1] = "DBGU1",
204 	[32+15][1] = "SDMA1",
205 };
206 
207 static const char *mmhub_client_ids_vega20[][2] = {
208 	[0][0] = "XDMA",
209 	[1][0] = "DCE",
210 	[2][0] = "VCE0",
211 	[3][0] = "VCE0U",
212 	[4][0] = "UVD",
213 	[5][0] = "UVD1U",
214 	[13][0] = "OSS",
215 	[14][0] = "HDP",
216 	[15][0] = "SDMA0",
217 	[32+0][0] = "UVD",
218 	[32+1][0] = "UVDU",
219 	[32+2][0] = "MP1",
220 	[32+3][0] = "MP0",
221 	[32+12][0] = "UTCL2",
222 	[32+14][0] = "SDMA1",
223 	[0][1] = "XDMA",
224 	[1][1] = "DCE",
225 	[2][1] = "DCEDWB",
226 	[3][1] = "VCE0",
227 	[4][1] = "VCE0U",
228 	[5][1] = "UVD1",
229 	[6][1] = "UVD1U",
230 	[7][1] = "DBGU0",
231 	[8][1] = "XDP",
232 	[13][1] = "OSS",
233 	[14][1] = "HDP",
234 	[15][1] = "SDMA0",
235 	[32+0][1] = "UVD",
236 	[32+1][1] = "UVDU",
237 	[32+2][1] = "DBGU1",
238 	[32+3][1] = "MP1",
239 	[32+4][1] = "MP0",
240 	[32+14][1] = "SDMA1",
241 };
242 
243 static const char *mmhub_client_ids_arcturus[][2] = {
244 	[2][0] = "MP1",
245 	[3][0] = "MP0",
246 	[10][0] = "UTCL2",
247 	[13][0] = "OSS",
248 	[14][0] = "HDP",
249 	[15][0] = "SDMA0",
250 	[32+15][0] = "SDMA1",
251 	[64+15][0] = "SDMA2",
252 	[96+15][0] = "SDMA3",
253 	[128+15][0] = "SDMA4",
254 	[160+11][0] = "JPEG",
255 	[160+12][0] = "VCN",
256 	[160+13][0] = "VCNU",
257 	[160+15][0] = "SDMA5",
258 	[192+10][0] = "UTCL2",
259 	[192+11][0] = "JPEG1",
260 	[192+12][0] = "VCN1",
261 	[192+13][0] = "VCN1U",
262 	[192+15][0] = "SDMA6",
263 	[224+15][0] = "SDMA7",
264 	[0][1] = "DBGU1",
265 	[1][1] = "XDP",
266 	[2][1] = "MP1",
267 	[3][1] = "MP0",
268 	[13][1] = "OSS",
269 	[14][1] = "HDP",
270 	[15][1] = "SDMA0",
271 	[32+15][1] = "SDMA1",
272 	[64+15][1] = "SDMA2",
273 	[96+15][1] = "SDMA3",
274 	[128+15][1] = "SDMA4",
275 	[160+11][1] = "JPEG",
276 	[160+12][1] = "VCN",
277 	[160+13][1] = "VCNU",
278 	[160+15][1] = "SDMA5",
279 	[192+11][1] = "JPEG1",
280 	[192+12][1] = "VCN1",
281 	[192+13][1] = "VCN1U",
282 	[192+15][1] = "SDMA6",
283 	[224+15][1] = "SDMA7",
284 };
285 
286 static const u32 golden_settings_vega10_hdp[] =
287 {
288 	0xf64, 0x0fffffff, 0x00000000,
289 	0xf65, 0x0fffffff, 0x00000000,
290 	0xf66, 0x0fffffff, 0x00000000,
291 	0xf67, 0x0fffffff, 0x00000000,
292 	0xf68, 0x0fffffff, 0x00000000,
293 	0xf6a, 0x0fffffff, 0x00000000,
294 	0xf6b, 0x0fffffff, 0x00000000,
295 	0xf6c, 0x0fffffff, 0x00000000,
296 	0xf6d, 0x0fffffff, 0x00000000,
297 	0xf6e, 0x0fffffff, 0x00000000,
298 };
299 
300 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
301 {
302 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
303 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
304 };
305 
306 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
307 {
308 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
309 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
310 };
311 
312 static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
313 	(0x000143c0 + 0x00000000),
314 	(0x000143c0 + 0x00000800),
315 	(0x000143c0 + 0x00001000),
316 	(0x000143c0 + 0x00001800),
317 	(0x000543c0 + 0x00000000),
318 	(0x000543c0 + 0x00000800),
319 	(0x000543c0 + 0x00001000),
320 	(0x000543c0 + 0x00001800),
321 	(0x000943c0 + 0x00000000),
322 	(0x000943c0 + 0x00000800),
323 	(0x000943c0 + 0x00001000),
324 	(0x000943c0 + 0x00001800),
325 	(0x000d43c0 + 0x00000000),
326 	(0x000d43c0 + 0x00000800),
327 	(0x000d43c0 + 0x00001000),
328 	(0x000d43c0 + 0x00001800),
329 	(0x001143c0 + 0x00000000),
330 	(0x001143c0 + 0x00000800),
331 	(0x001143c0 + 0x00001000),
332 	(0x001143c0 + 0x00001800),
333 	(0x001543c0 + 0x00000000),
334 	(0x001543c0 + 0x00000800),
335 	(0x001543c0 + 0x00001000),
336 	(0x001543c0 + 0x00001800),
337 	(0x001943c0 + 0x00000000),
338 	(0x001943c0 + 0x00000800),
339 	(0x001943c0 + 0x00001000),
340 	(0x001943c0 + 0x00001800),
341 	(0x001d43c0 + 0x00000000),
342 	(0x001d43c0 + 0x00000800),
343 	(0x001d43c0 + 0x00001000),
344 	(0x001d43c0 + 0x00001800),
345 };
346 
347 static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
348 	(0x000143e0 + 0x00000000),
349 	(0x000143e0 + 0x00000800),
350 	(0x000143e0 + 0x00001000),
351 	(0x000143e0 + 0x00001800),
352 	(0x000543e0 + 0x00000000),
353 	(0x000543e0 + 0x00000800),
354 	(0x000543e0 + 0x00001000),
355 	(0x000543e0 + 0x00001800),
356 	(0x000943e0 + 0x00000000),
357 	(0x000943e0 + 0x00000800),
358 	(0x000943e0 + 0x00001000),
359 	(0x000943e0 + 0x00001800),
360 	(0x000d43e0 + 0x00000000),
361 	(0x000d43e0 + 0x00000800),
362 	(0x000d43e0 + 0x00001000),
363 	(0x000d43e0 + 0x00001800),
364 	(0x001143e0 + 0x00000000),
365 	(0x001143e0 + 0x00000800),
366 	(0x001143e0 + 0x00001000),
367 	(0x001143e0 + 0x00001800),
368 	(0x001543e0 + 0x00000000),
369 	(0x001543e0 + 0x00000800),
370 	(0x001543e0 + 0x00001000),
371 	(0x001543e0 + 0x00001800),
372 	(0x001943e0 + 0x00000000),
373 	(0x001943e0 + 0x00000800),
374 	(0x001943e0 + 0x00001000),
375 	(0x001943e0 + 0x00001800),
376 	(0x001d43e0 + 0x00000000),
377 	(0x001d43e0 + 0x00000800),
378 	(0x001d43e0 + 0x00001000),
379 	(0x001d43e0 + 0x00001800),
380 };
381 
382 static const uint32_t ecc_umc_mcumc_status_addrs[] = {
383 	(0x000143c2 + 0x00000000),
384 	(0x000143c2 + 0x00000800),
385 	(0x000143c2 + 0x00001000),
386 	(0x000143c2 + 0x00001800),
387 	(0x000543c2 + 0x00000000),
388 	(0x000543c2 + 0x00000800),
389 	(0x000543c2 + 0x00001000),
390 	(0x000543c2 + 0x00001800),
391 	(0x000943c2 + 0x00000000),
392 	(0x000943c2 + 0x00000800),
393 	(0x000943c2 + 0x00001000),
394 	(0x000943c2 + 0x00001800),
395 	(0x000d43c2 + 0x00000000),
396 	(0x000d43c2 + 0x00000800),
397 	(0x000d43c2 + 0x00001000),
398 	(0x000d43c2 + 0x00001800),
399 	(0x001143c2 + 0x00000000),
400 	(0x001143c2 + 0x00000800),
401 	(0x001143c2 + 0x00001000),
402 	(0x001143c2 + 0x00001800),
403 	(0x001543c2 + 0x00000000),
404 	(0x001543c2 + 0x00000800),
405 	(0x001543c2 + 0x00001000),
406 	(0x001543c2 + 0x00001800),
407 	(0x001943c2 + 0x00000000),
408 	(0x001943c2 + 0x00000800),
409 	(0x001943c2 + 0x00001000),
410 	(0x001943c2 + 0x00001800),
411 	(0x001d43c2 + 0x00000000),
412 	(0x001d43c2 + 0x00000800),
413 	(0x001d43c2 + 0x00001000),
414 	(0x001d43c2 + 0x00001800),
415 };
416 
417 static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
418 		struct amdgpu_irq_src *src,
419 		unsigned type,
420 		enum amdgpu_interrupt_state state)
421 {
422 	u32 bits, i, tmp, reg;
423 
424 	/* Devices newer then VEGA10/12 shall have these programming
425 	     sequences performed by PSP BL */
426 	if (adev->asic_type >= CHIP_VEGA20)
427 		return 0;
428 
429 	bits = 0x7f;
430 
431 	switch (state) {
432 	case AMDGPU_IRQ_STATE_DISABLE:
433 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
434 			reg = ecc_umc_mcumc_ctrl_addrs[i];
435 			tmp = RREG32(reg);
436 			tmp &= ~bits;
437 			WREG32(reg, tmp);
438 		}
439 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
440 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
441 			tmp = RREG32(reg);
442 			tmp &= ~bits;
443 			WREG32(reg, tmp);
444 		}
445 		break;
446 	case AMDGPU_IRQ_STATE_ENABLE:
447 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
448 			reg = ecc_umc_mcumc_ctrl_addrs[i];
449 			tmp = RREG32(reg);
450 			tmp |= bits;
451 			WREG32(reg, tmp);
452 		}
453 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
454 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
455 			tmp = RREG32(reg);
456 			tmp |= bits;
457 			WREG32(reg, tmp);
458 		}
459 		break;
460 	default:
461 		break;
462 	}
463 
464 	return 0;
465 }
466 
467 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
468 					struct amdgpu_irq_src *src,
469 					unsigned type,
470 					enum amdgpu_interrupt_state state)
471 {
472 	struct amdgpu_vmhub *hub;
473 	u32 tmp, reg, bits, i, j;
474 
475 	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
476 		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477 		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
478 		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
479 		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
480 		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
481 		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
482 
483 	switch (state) {
484 	case AMDGPU_IRQ_STATE_DISABLE:
485 		for (j = 0; j < adev->num_vmhubs; j++) {
486 			hub = &adev->vmhub[j];
487 			for (i = 0; i < 16; i++) {
488 				reg = hub->vm_context0_cntl + i;
489 				tmp = RREG32(reg);
490 				tmp &= ~bits;
491 				WREG32(reg, tmp);
492 			}
493 		}
494 		break;
495 	case AMDGPU_IRQ_STATE_ENABLE:
496 		for (j = 0; j < adev->num_vmhubs; j++) {
497 			hub = &adev->vmhub[j];
498 			for (i = 0; i < 16; i++) {
499 				reg = hub->vm_context0_cntl + i;
500 				tmp = RREG32(reg);
501 				tmp |= bits;
502 				WREG32(reg, tmp);
503 			}
504 		}
505 	default:
506 		break;
507 	}
508 
509 	return 0;
510 }
511 
512 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
513 				struct amdgpu_irq_src *source,
514 				struct amdgpu_iv_entry *entry)
515 {
516 	struct amdgpu_vmhub *hub;
517 	bool retry_fault = !!(entry->src_data[1] & 0x80);
518 	uint32_t status = 0, cid = 0, rw = 0;
519 	u64 addr;
520 	char hub_name[10];
521 	const char *mmhub_cid;
522 
523 	addr = (u64)entry->src_data[0] << 12;
524 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
525 
526 	if (retry_fault && amdgpu_gmc_filter_faults(adev, addr, entry->pasid,
527 						    entry->timestamp))
528 		return 1; /* This also prevents sending it to KFD */
529 
530 	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
531 		snprintf(hub_name, sizeof(hub_name), "mmhub0");
532 		hub = &adev->vmhub[AMDGPU_MMHUB_0];
533 	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
534 		snprintf(hub_name, sizeof(hub_name), "mmhub1");
535 		hub = &adev->vmhub[AMDGPU_MMHUB_1];
536 	} else {
537 		snprintf(hub_name, sizeof(hub_name), "gfxhub0");
538 		hub = &adev->vmhub[AMDGPU_GFXHUB_0];
539 	}
540 
541 	/* If it's the first fault for this address, process it normally */
542 	if (retry_fault && !in_interrupt() &&
543 	    amdgpu_vm_handle_fault(adev, entry->pasid, addr))
544 		return 1; /* This also prevents sending it to KFD */
545 
546 	if (!amdgpu_sriov_vf(adev)) {
547 		/*
548 		 * Issue a dummy read to wait for the status register to
549 		 * be updated to avoid reading an incorrect value due to
550 		 * the new fast GRBM interface.
551 		 */
552 		if (entry->vmid_src == AMDGPU_GFXHUB_0)
553 			RREG32(hub->vm_l2_pro_fault_status);
554 
555 		status = RREG32(hub->vm_l2_pro_fault_status);
556 		cid = REG_GET_FIELD(status,
557 				    VM_L2_PROTECTION_FAULT_STATUS, CID);
558 		rw = REG_GET_FIELD(status,
559 				   VM_L2_PROTECTION_FAULT_STATUS, RW);
560 		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
561 	}
562 
563 	if (printk_ratelimit()) {
564 		struct amdgpu_task_info task_info;
565 
566 		memset(&task_info, 0, sizeof(struct amdgpu_task_info));
567 		amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
568 
569 		dev_err(adev->dev,
570 			"[%s] %s page fault (src_id:%u ring:%u vmid:%u "
571 			"pasid:%u, for process %s pid %d thread %s pid %d)\n",
572 			hub_name, retry_fault ? "retry" : "no-retry",
573 			entry->src_id, entry->ring_id, entry->vmid,
574 			entry->pasid, task_info.process_name, task_info.tgid,
575 			task_info.task_name, task_info.pid);
576 		dev_err(adev->dev, "  in page starting at address 0x%016llx from client %d\n",
577 			addr, entry->client_id);
578 		if (!amdgpu_sriov_vf(adev)) {
579 			dev_err(adev->dev,
580 				"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
581 				status);
582 			if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) {
583 				dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
584 					cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid],
585 					cid);
586 			} else {
587 				switch (adev->asic_type) {
588 				case CHIP_VEGA10:
589 					mmhub_cid = mmhub_client_ids_vega10[cid][rw];
590 					break;
591 				case CHIP_VEGA12:
592 					mmhub_cid = mmhub_client_ids_vega12[cid][rw];
593 					break;
594 				case CHIP_VEGA20:
595 					mmhub_cid = mmhub_client_ids_vega20[cid][rw];
596 					break;
597 				case CHIP_ARCTURUS:
598 					mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
599 					break;
600 				case CHIP_RAVEN:
601 					mmhub_cid = mmhub_client_ids_raven[cid][rw];
602 					break;
603 				case CHIP_RENOIR:
604 					mmhub_cid = mmhub_client_ids_renoir[cid][rw];
605 					break;
606 				default:
607 					mmhub_cid = NULL;
608 					break;
609 				}
610 				dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
611 					mmhub_cid ? mmhub_cid : "unknown", cid);
612 			}
613 			dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
614 				REG_GET_FIELD(status,
615 				VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
616 			dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
617 				REG_GET_FIELD(status,
618 				VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
619 			dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
620 				REG_GET_FIELD(status,
621 				VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
622 			dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
623 				REG_GET_FIELD(status,
624 				VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
625 			dev_err(adev->dev, "\t RW: 0x%x\n", rw);
626 		}
627 	}
628 
629 	return 0;
630 }
631 
632 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
633 	.set = gmc_v9_0_vm_fault_interrupt_state,
634 	.process = gmc_v9_0_process_interrupt,
635 };
636 
637 
638 static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
639 	.set = gmc_v9_0_ecc_interrupt_state,
640 	.process = amdgpu_umc_process_ecc_irq,
641 };
642 
643 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
644 {
645 	adev->gmc.vm_fault.num_types = 1;
646 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
647 
648 	if (!amdgpu_sriov_vf(adev)) {
649 		adev->gmc.ecc_irq.num_types = 1;
650 		adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
651 	}
652 }
653 
654 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
655 					uint32_t flush_type)
656 {
657 	u32 req = 0;
658 
659 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
660 			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
661 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
662 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
663 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
664 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
665 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
666 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
667 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
668 			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
669 
670 	return req;
671 }
672 
673 /**
674  * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
675  *
676  * @adev: amdgpu_device pointer
677  * @vmhub: vmhub type
678  *
679  */
680 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
681 				       uint32_t vmhub)
682 {
683 	return ((vmhub == AMDGPU_MMHUB_0 ||
684 		 vmhub == AMDGPU_MMHUB_1) &&
685 		(!amdgpu_sriov_vf(adev)) &&
686 		(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
687 		   (adev->apu_flags & AMD_APU_IS_PICASSO))));
688 }
689 
690 static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
691 					uint8_t vmid, uint16_t *p_pasid)
692 {
693 	uint32_t value;
694 
695 	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
696 		     + vmid);
697 	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
698 
699 	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
700 }
701 
702 /*
703  * GART
704  * VMID 0 is the physical GPU addresses as used by the kernel.
705  * VMIDs 1-15 are used for userspace clients and are handled
706  * by the amdgpu vm/hsa code.
707  */
708 
709 /**
710  * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
711  *
712  * @adev: amdgpu_device pointer
713  * @vmid: vm instance to flush
714  * @flush_type: the flush type
715  *
716  * Flush the TLB for the requested page table using certain type.
717  */
718 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
719 					uint32_t vmhub, uint32_t flush_type)
720 {
721 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
722 	const unsigned eng = 17;
723 	u32 j, inv_req, inv_req2, tmp;
724 	struct amdgpu_vmhub *hub;
725 
726 	BUG_ON(vmhub >= adev->num_vmhubs);
727 
728 	hub = &adev->vmhub[vmhub];
729 	if (adev->gmc.xgmi.num_physical_nodes &&
730 	    adev->asic_type == CHIP_VEGA20) {
731 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
732 		 * heavy-weight TLB flush (type 2), which flushes
733 		 * both. Due to a race condition with concurrent
734 		 * memory accesses using the same TLB cache line, we
735 		 * still need a second TLB flush after this.
736 		 */
737 		inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
738 		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
739 	} else {
740 		inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
741 		inv_req2 = 0;
742 	}
743 
744 	/* This is necessary for a HW workaround under SRIOV as well
745 	 * as GFXOFF under bare metal
746 	 */
747 	if (adev->gfx.kiq.ring.sched.ready &&
748 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
749 	    down_read_trylock(&adev->reset_sem)) {
750 		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
751 		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
752 
753 		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
754 						   1 << vmid);
755 		up_read(&adev->reset_sem);
756 		return;
757 	}
758 
759 	spin_lock(&adev->gmc.invalidate_lock);
760 
761 	/*
762 	 * It may lose gpuvm invalidate acknowldege state across power-gating
763 	 * off cycle, add semaphore acquire before invalidation and semaphore
764 	 * release after invalidation to avoid entering power gated state
765 	 * to WA the Issue
766 	 */
767 
768 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
769 	if (use_semaphore) {
770 		for (j = 0; j < adev->usec_timeout; j++) {
771 			/* a read return value of 1 means semaphore acuqire */
772 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem +
773 					    hub->eng_distance * eng);
774 			if (tmp & 0x1)
775 				break;
776 			udelay(1);
777 		}
778 
779 		if (j >= adev->usec_timeout)
780 			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
781 	}
782 
783 	do {
784 		WREG32_NO_KIQ(hub->vm_inv_eng0_req +
785 			      hub->eng_distance * eng, inv_req);
786 
787 		/*
788 		 * Issue a dummy read to wait for the ACK register to
789 		 * be cleared to avoid a false ACK due to the new fast
790 		 * GRBM interface.
791 		 */
792 		if (vmhub == AMDGPU_GFXHUB_0)
793 			RREG32_NO_KIQ(hub->vm_inv_eng0_req +
794 				      hub->eng_distance * eng);
795 
796 		for (j = 0; j < adev->usec_timeout; j++) {
797 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack +
798 					    hub->eng_distance * eng);
799 			if (tmp & (1 << vmid))
800 				break;
801 			udelay(1);
802 		}
803 
804 		inv_req = inv_req2;
805 		inv_req2 = 0;
806 	} while (inv_req);
807 
808 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
809 	if (use_semaphore)
810 		/*
811 		 * add semaphore release after invalidation,
812 		 * write with 0 means semaphore release
813 		 */
814 		WREG32_NO_KIQ(hub->vm_inv_eng0_sem +
815 			      hub->eng_distance * eng, 0);
816 
817 	spin_unlock(&adev->gmc.invalidate_lock);
818 
819 	if (j < adev->usec_timeout)
820 		return;
821 
822 	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
823 }
824 
825 /**
826  * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
827  *
828  * @adev: amdgpu_device pointer
829  * @pasid: pasid to be flush
830  *
831  * Flush the TLB for the requested pasid.
832  */
833 static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
834 					uint16_t pasid, uint32_t flush_type,
835 					bool all_hub)
836 {
837 	int vmid, i;
838 	signed long r;
839 	uint32_t seq;
840 	uint16_t queried_pasid;
841 	bool ret;
842 	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
843 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
844 
845 	if (amdgpu_in_reset(adev))
846 		return -EIO;
847 
848 	if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) {
849 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
850 		 * heavy-weight TLB flush (type 2), which flushes
851 		 * both. Due to a race condition with concurrent
852 		 * memory accesses using the same TLB cache line, we
853 		 * still need a second TLB flush after this.
854 		 */
855 		bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
856 				       adev->asic_type == CHIP_VEGA20);
857 		/* 2 dwords flush + 8 dwords fence */
858 		unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
859 
860 		if (vega20_xgmi_wa)
861 			ndw += kiq->pmf->invalidate_tlbs_size;
862 
863 		spin_lock(&adev->gfx.kiq.ring_lock);
864 		/* 2 dwords flush + 8 dwords fence */
865 		amdgpu_ring_alloc(ring, ndw);
866 		if (vega20_xgmi_wa)
867 			kiq->pmf->kiq_invalidate_tlbs(ring,
868 						      pasid, 2, all_hub);
869 		kiq->pmf->kiq_invalidate_tlbs(ring,
870 					pasid, flush_type, all_hub);
871 		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
872 		if (r) {
873 			amdgpu_ring_undo(ring);
874 			spin_unlock(&adev->gfx.kiq.ring_lock);
875 			up_read(&adev->reset_sem);
876 			return -ETIME;
877 		}
878 
879 		amdgpu_ring_commit(ring);
880 		spin_unlock(&adev->gfx.kiq.ring_lock);
881 		r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
882 		if (r < 1) {
883 			dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
884 			up_read(&adev->reset_sem);
885 			return -ETIME;
886 		}
887 		up_read(&adev->reset_sem);
888 		return 0;
889 	}
890 
891 	for (vmid = 1; vmid < 16; vmid++) {
892 
893 		ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
894 				&queried_pasid);
895 		if (ret && queried_pasid == pasid) {
896 			if (all_hub) {
897 				for (i = 0; i < adev->num_vmhubs; i++)
898 					gmc_v9_0_flush_gpu_tlb(adev, vmid,
899 							i, flush_type);
900 			} else {
901 				gmc_v9_0_flush_gpu_tlb(adev, vmid,
902 						AMDGPU_GFXHUB_0, flush_type);
903 			}
904 			break;
905 		}
906 	}
907 
908 	return 0;
909 
910 }
911 
912 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
913 					    unsigned vmid, uint64_t pd_addr)
914 {
915 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
916 	struct amdgpu_device *adev = ring->adev;
917 	struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
918 	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
919 	unsigned eng = ring->vm_inv_eng;
920 
921 	/*
922 	 * It may lose gpuvm invalidate acknowldege state across power-gating
923 	 * off cycle, add semaphore acquire before invalidation and semaphore
924 	 * release after invalidation to avoid entering power gated state
925 	 * to WA the Issue
926 	 */
927 
928 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
929 	if (use_semaphore)
930 		/* a read return value of 1 means semaphore acuqire */
931 		amdgpu_ring_emit_reg_wait(ring,
932 					  hub->vm_inv_eng0_sem +
933 					  hub->eng_distance * eng, 0x1, 0x1);
934 
935 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
936 			      (hub->ctx_addr_distance * vmid),
937 			      lower_32_bits(pd_addr));
938 
939 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
940 			      (hub->ctx_addr_distance * vmid),
941 			      upper_32_bits(pd_addr));
942 
943 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
944 					    hub->eng_distance * eng,
945 					    hub->vm_inv_eng0_ack +
946 					    hub->eng_distance * eng,
947 					    req, 1 << vmid);
948 
949 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
950 	if (use_semaphore)
951 		/*
952 		 * add semaphore release after invalidation,
953 		 * write with 0 means semaphore release
954 		 */
955 		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
956 				      hub->eng_distance * eng, 0);
957 
958 	return pd_addr;
959 }
960 
961 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
962 					unsigned pasid)
963 {
964 	struct amdgpu_device *adev = ring->adev;
965 	uint32_t reg;
966 
967 	/* Do nothing because there's no lut register for mmhub1. */
968 	if (ring->funcs->vmhub == AMDGPU_MMHUB_1)
969 		return;
970 
971 	if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
972 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
973 	else
974 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
975 
976 	amdgpu_ring_emit_wreg(ring, reg, pasid);
977 }
978 
979 /*
980  * PTE format on VEGA 10:
981  * 63:59 reserved
982  * 58:57 mtype
983  * 56 F
984  * 55 L
985  * 54 P
986  * 53 SW
987  * 52 T
988  * 50:48 reserved
989  * 47:12 4k physical page base address
990  * 11:7 fragment
991  * 6 write
992  * 5 read
993  * 4 exe
994  * 3 Z
995  * 2 snooped
996  * 1 system
997  * 0 valid
998  *
999  * PDE format on VEGA 10:
1000  * 63:59 block fragment size
1001  * 58:55 reserved
1002  * 54 P
1003  * 53:48 reserved
1004  * 47:6 physical base address of PD or PTE
1005  * 5:3 reserved
1006  * 2 C
1007  * 1 system
1008  * 0 valid
1009  */
1010 
1011 static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1012 
1013 {
1014 	switch (flags) {
1015 	case AMDGPU_VM_MTYPE_DEFAULT:
1016 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1017 	case AMDGPU_VM_MTYPE_NC:
1018 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1019 	case AMDGPU_VM_MTYPE_WC:
1020 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1021 	case AMDGPU_VM_MTYPE_RW:
1022 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1023 	case AMDGPU_VM_MTYPE_CC:
1024 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1025 	case AMDGPU_VM_MTYPE_UC:
1026 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1027 	default:
1028 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1029 	}
1030 }
1031 
1032 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1033 				uint64_t *addr, uint64_t *flags)
1034 {
1035 	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1036 		*addr = adev->vm_manager.vram_base_offset + *addr -
1037 			adev->gmc.vram_start;
1038 	BUG_ON(*addr & 0xFFFF00000000003FULL);
1039 
1040 	if (!adev->gmc.translate_further)
1041 		return;
1042 
1043 	if (level == AMDGPU_VM_PDB1) {
1044 		/* Set the block fragment size */
1045 		if (!(*flags & AMDGPU_PDE_PTE))
1046 			*flags |= AMDGPU_PDE_BFS(0x9);
1047 
1048 	} else if (level == AMDGPU_VM_PDB0) {
1049 		if (*flags & AMDGPU_PDE_PTE)
1050 			*flags &= ~AMDGPU_PDE_PTE;
1051 		else
1052 			*flags |= AMDGPU_PTE_TF;
1053 	}
1054 }
1055 
1056 static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1057 				struct amdgpu_bo_va_mapping *mapping,
1058 				uint64_t *flags)
1059 {
1060 	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1061 	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1062 
1063 	*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1064 	*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1065 
1066 	if (mapping->flags & AMDGPU_PTE_PRT) {
1067 		*flags |= AMDGPU_PTE_PRT;
1068 		*flags &= ~AMDGPU_PTE_VALID;
1069 	}
1070 
1071 	if (adev->asic_type == CHIP_ARCTURUS &&
1072 	    !(*flags & AMDGPU_PTE_SYSTEM) &&
1073 	    mapping->bo_va->is_xgmi)
1074 		*flags |= AMDGPU_PTE_SNOOPED;
1075 }
1076 
1077 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1078 {
1079 	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1080 	unsigned size;
1081 
1082 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1083 		size = AMDGPU_VBIOS_VGA_ALLOCATION;
1084 	} else {
1085 		u32 viewport;
1086 
1087 		switch (adev->asic_type) {
1088 		case CHIP_RAVEN:
1089 		case CHIP_RENOIR:
1090 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1091 			size = (REG_GET_FIELD(viewport,
1092 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1093 				REG_GET_FIELD(viewport,
1094 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1095 				4);
1096 			break;
1097 		case CHIP_VEGA10:
1098 		case CHIP_VEGA12:
1099 		case CHIP_VEGA20:
1100 		default:
1101 			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1102 			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1103 				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1104 				4);
1105 			break;
1106 		}
1107 	}
1108 
1109 	return size;
1110 }
1111 
1112 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1113 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1114 	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1115 	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1116 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1117 	.map_mtype = gmc_v9_0_map_mtype,
1118 	.get_vm_pde = gmc_v9_0_get_vm_pde,
1119 	.get_vm_pte = gmc_v9_0_get_vm_pte,
1120 	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1121 };
1122 
1123 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1124 {
1125 	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1126 }
1127 
1128 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1129 {
1130 	switch (adev->asic_type) {
1131 	case CHIP_VEGA10:
1132 		adev->umc.funcs = &umc_v6_0_funcs;
1133 		break;
1134 	case CHIP_VEGA20:
1135 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1136 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1137 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1138 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1139 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1140 		adev->umc.funcs = &umc_v6_1_funcs;
1141 		break;
1142 	case CHIP_ARCTURUS:
1143 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1144 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1145 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1146 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1147 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1148 		adev->umc.funcs = &umc_v6_1_funcs;
1149 		break;
1150 	default:
1151 		break;
1152 	}
1153 }
1154 
1155 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1156 {
1157 	switch (adev->asic_type) {
1158 	case CHIP_ARCTURUS:
1159 		adev->mmhub.funcs = &mmhub_v9_4_funcs;
1160 		break;
1161 	default:
1162 		adev->mmhub.funcs = &mmhub_v1_0_funcs;
1163 		break;
1164 	}
1165 }
1166 
1167 static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1168 {
1169 	adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1170 }
1171 
1172 static int gmc_v9_0_early_init(void *handle)
1173 {
1174 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1175 
1176 	gmc_v9_0_set_gmc_funcs(adev);
1177 	gmc_v9_0_set_irq_funcs(adev);
1178 	gmc_v9_0_set_umc_funcs(adev);
1179 	gmc_v9_0_set_mmhub_funcs(adev);
1180 	gmc_v9_0_set_gfxhub_funcs(adev);
1181 
1182 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1183 	adev->gmc.shared_aperture_end =
1184 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1185 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1186 	adev->gmc.private_aperture_end =
1187 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1188 
1189 	return 0;
1190 }
1191 
1192 static int gmc_v9_0_late_init(void *handle)
1193 {
1194 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1195 	int r;
1196 
1197 	amdgpu_bo_late_init(adev);
1198 
1199 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1200 	if (r)
1201 		return r;
1202 
1203 	/*
1204 	 * Workaround performance drop issue with VBIOS enables partial
1205 	 * writes, while disables HBM ECC for vega10.
1206 	 */
1207 	if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) {
1208 		if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1209 			if (adev->df.funcs->enable_ecc_force_par_wr_rmw)
1210 				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1211 		}
1212 	}
1213 
1214 	if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)
1215 		adev->mmhub.funcs->reset_ras_error_count(adev);
1216 
1217 	r = amdgpu_gmc_ras_late_init(adev);
1218 	if (r)
1219 		return r;
1220 
1221 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1222 }
1223 
1224 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1225 					struct amdgpu_gmc *mc)
1226 {
1227 	u64 base = 0;
1228 
1229 	if (!amdgpu_sriov_vf(adev))
1230 		base = adev->mmhub.funcs->get_fb_location(adev);
1231 
1232 	/* add the xgmi offset of the physical node */
1233 	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1234 	amdgpu_gmc_vram_location(adev, mc, base);
1235 	amdgpu_gmc_gart_location(adev, mc);
1236 	amdgpu_gmc_agp_location(adev, mc);
1237 	/* base offset of vram pages */
1238 	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1239 
1240 	/* XXX: add the xgmi offset of the physical node? */
1241 	adev->vm_manager.vram_base_offset +=
1242 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1243 }
1244 
1245 /**
1246  * gmc_v9_0_mc_init - initialize the memory controller driver params
1247  *
1248  * @adev: amdgpu_device pointer
1249  *
1250  * Look up the amount of vram, vram width, and decide how to place
1251  * vram and gart within the GPU's physical address space.
1252  * Returns 0 for success.
1253  */
1254 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1255 {
1256 	int r;
1257 
1258 	/* size in MB on si */
1259 	adev->gmc.mc_vram_size =
1260 		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1261 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1262 
1263 	if (!(adev->flags & AMD_IS_APU)) {
1264 		r = amdgpu_device_resize_fb_bar(adev);
1265 		if (r)
1266 			return r;
1267 	}
1268 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1269 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1270 
1271 #ifdef CONFIG_X86_64
1272 	if (adev->flags & AMD_IS_APU) {
1273 		adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1274 		adev->gmc.aper_size = adev->gmc.real_vram_size;
1275 	}
1276 #endif
1277 	/* In case the PCI BAR is larger than the actual amount of vram */
1278 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1279 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
1280 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
1281 
1282 	/* set the gart size */
1283 	if (amdgpu_gart_size == -1) {
1284 		switch (adev->asic_type) {
1285 		case CHIP_VEGA10:  /* all engines support GPUVM */
1286 		case CHIP_VEGA12:  /* all engines support GPUVM */
1287 		case CHIP_VEGA20:
1288 		case CHIP_ARCTURUS:
1289 		default:
1290 			adev->gmc.gart_size = 512ULL << 20;
1291 			break;
1292 		case CHIP_RAVEN:   /* DCE SG support */
1293 		case CHIP_RENOIR:
1294 			adev->gmc.gart_size = 1024ULL << 20;
1295 			break;
1296 		}
1297 	} else {
1298 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1299 	}
1300 
1301 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1302 
1303 	return 0;
1304 }
1305 
1306 static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1307 {
1308 	int r;
1309 
1310 	if (adev->gart.bo) {
1311 		WARN(1, "VEGA10 PCIE GART already initialized\n");
1312 		return 0;
1313 	}
1314 	/* Initialize common gart structure */
1315 	r = amdgpu_gart_init(adev);
1316 	if (r)
1317 		return r;
1318 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1319 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1320 				 AMDGPU_PTE_EXECUTABLE;
1321 	return amdgpu_gart_table_vram_alloc(adev);
1322 }
1323 
1324 /**
1325  * gmc_v9_0_save_registers - saves regs
1326  *
1327  * @adev: amdgpu_device pointer
1328  *
1329  * This saves potential register values that should be
1330  * restored upon resume
1331  */
1332 static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1333 {
1334 	if (adev->asic_type == CHIP_RAVEN)
1335 		adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1336 }
1337 
1338 static int gmc_v9_0_sw_init(void *handle)
1339 {
1340 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
1341 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1342 
1343 	adev->gfxhub.funcs->init(adev);
1344 
1345 	adev->mmhub.funcs->init(adev);
1346 
1347 	spin_lock_init(&adev->gmc.invalidate_lock);
1348 
1349 	r = amdgpu_atomfirmware_get_vram_info(adev,
1350 		&vram_width, &vram_type, &vram_vendor);
1351 	if (amdgpu_sriov_vf(adev))
1352 		/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1353 		 * and DF related registers is not readable, seems hardcord is the
1354 		 * only way to set the correct vram_width
1355 		 */
1356 		adev->gmc.vram_width = 2048;
1357 	else if (amdgpu_emu_mode != 1)
1358 		adev->gmc.vram_width = vram_width;
1359 
1360 	if (!adev->gmc.vram_width) {
1361 		int chansize, numchan;
1362 
1363 		/* hbm memory channel size */
1364 		if (adev->flags & AMD_IS_APU)
1365 			chansize = 64;
1366 		else
1367 			chansize = 128;
1368 
1369 		numchan = adev->df.funcs->get_hbm_channel_number(adev);
1370 		adev->gmc.vram_width = numchan * chansize;
1371 	}
1372 
1373 	adev->gmc.vram_type = vram_type;
1374 	adev->gmc.vram_vendor = vram_vendor;
1375 	switch (adev->asic_type) {
1376 	case CHIP_RAVEN:
1377 		adev->num_vmhubs = 2;
1378 
1379 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
1380 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1381 		} else {
1382 			/* vm_size is 128TB + 512GB for legacy 3-level page support */
1383 			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
1384 			adev->gmc.translate_further =
1385 				adev->vm_manager.num_level > 1;
1386 		}
1387 		break;
1388 	case CHIP_VEGA10:
1389 	case CHIP_VEGA12:
1390 	case CHIP_VEGA20:
1391 	case CHIP_RENOIR:
1392 		adev->num_vmhubs = 2;
1393 
1394 
1395 		/*
1396 		 * To fulfill 4-level page support,
1397 		 * vm size is 256TB (48bit), maximum size of Vega10,
1398 		 * block size 512 (9bit)
1399 		 */
1400 		/* sriov restrict max_pfn below AMDGPU_GMC_HOLE */
1401 		if (amdgpu_sriov_vf(adev))
1402 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
1403 		else
1404 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1405 		break;
1406 	case CHIP_ARCTURUS:
1407 		adev->num_vmhubs = 3;
1408 
1409 		/* Keep the vm size same with Vega20 */
1410 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1411 		break;
1412 	default:
1413 		break;
1414 	}
1415 
1416 	/* This interrupt is VMC page fault.*/
1417 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
1418 				&adev->gmc.vm_fault);
1419 	if (r)
1420 		return r;
1421 
1422 	if (adev->asic_type == CHIP_ARCTURUS) {
1423 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1424 					&adev->gmc.vm_fault);
1425 		if (r)
1426 			return r;
1427 	}
1428 
1429 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
1430 				&adev->gmc.vm_fault);
1431 
1432 	if (r)
1433 		return r;
1434 
1435 	if (!amdgpu_sriov_vf(adev)) {
1436 		/* interrupt sent to DF. */
1437 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
1438 				      &adev->gmc.ecc_irq);
1439 		if (r)
1440 			return r;
1441 	}
1442 
1443 	/* Set the internal MC address mask
1444 	 * This is the max address of the GPU's
1445 	 * internal address space.
1446 	 */
1447 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
1448 
1449 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
1450 	if (r) {
1451 		printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
1452 		return r;
1453 	}
1454 	adev->need_swiotlb = drm_need_swiotlb(44);
1455 
1456 	if (adev->gmc.xgmi.supported) {
1457 		r = adev->gfxhub.funcs->get_xgmi_info(adev);
1458 		if (r)
1459 			return r;
1460 	}
1461 
1462 	r = gmc_v9_0_mc_init(adev);
1463 	if (r)
1464 		return r;
1465 
1466 	amdgpu_gmc_get_vbios_allocations(adev);
1467 
1468 	/* Memory manager */
1469 	r = amdgpu_bo_init(adev);
1470 	if (r)
1471 		return r;
1472 
1473 	r = gmc_v9_0_gart_init(adev);
1474 	if (r)
1475 		return r;
1476 
1477 	/*
1478 	 * number of VMs
1479 	 * VMID 0 is reserved for System
1480 	 * amdgpu graphics/compute will use VMIDs 1..n-1
1481 	 * amdkfd will use VMIDs n..15
1482 	 *
1483 	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
1484 	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
1485 	 * for video processing.
1486 	 */
1487 	adev->vm_manager.first_kfd_vmid =
1488 		adev->asic_type == CHIP_ARCTURUS ? 3 : 8;
1489 
1490 	amdgpu_vm_manager_init(adev);
1491 
1492 	gmc_v9_0_save_registers(adev);
1493 
1494 	return 0;
1495 }
1496 
1497 static int gmc_v9_0_sw_fini(void *handle)
1498 {
1499 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1500 
1501 	amdgpu_gmc_ras_fini(adev);
1502 	amdgpu_gem_force_release(adev);
1503 	amdgpu_vm_manager_fini(adev);
1504 	amdgpu_gart_table_vram_free(adev);
1505 	amdgpu_bo_fini(adev);
1506 	amdgpu_gart_fini(adev);
1507 
1508 	return 0;
1509 }
1510 
1511 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
1512 {
1513 
1514 	switch (adev->asic_type) {
1515 	case CHIP_VEGA10:
1516 		if (amdgpu_sriov_vf(adev))
1517 			break;
1518 		fallthrough;
1519 	case CHIP_VEGA20:
1520 		soc15_program_register_sequence(adev,
1521 						golden_settings_mmhub_1_0_0,
1522 						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
1523 		soc15_program_register_sequence(adev,
1524 						golden_settings_athub_1_0_0,
1525 						ARRAY_SIZE(golden_settings_athub_1_0_0));
1526 		break;
1527 	case CHIP_VEGA12:
1528 		break;
1529 	case CHIP_RAVEN:
1530 		/* TODO for renoir */
1531 		soc15_program_register_sequence(adev,
1532 						golden_settings_athub_1_0_0,
1533 						ARRAY_SIZE(golden_settings_athub_1_0_0));
1534 		break;
1535 	default:
1536 		break;
1537 	}
1538 }
1539 
1540 /**
1541  * gmc_v9_0_restore_registers - restores regs
1542  *
1543  * @adev: amdgpu_device pointer
1544  *
1545  * This restores register values, saved at suspend.
1546  */
1547 void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
1548 {
1549 	if (adev->asic_type == CHIP_RAVEN) {
1550 		WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
1551 		WARN_ON(adev->gmc.sdpif_register !=
1552 			RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
1553 	}
1554 }
1555 
1556 /**
1557  * gmc_v9_0_gart_enable - gart enable
1558  *
1559  * @adev: amdgpu_device pointer
1560  */
1561 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1562 {
1563 	int r;
1564 
1565 	if (adev->gart.bo == NULL) {
1566 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1567 		return -EINVAL;
1568 	}
1569 	r = amdgpu_gart_table_vram_pin(adev);
1570 	if (r)
1571 		return r;
1572 
1573 	r = adev->gfxhub.funcs->gart_enable(adev);
1574 	if (r)
1575 		return r;
1576 
1577 	r = adev->mmhub.funcs->gart_enable(adev);
1578 	if (r)
1579 		return r;
1580 
1581 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1582 		 (unsigned)(adev->gmc.gart_size >> 20),
1583 		 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
1584 	adev->gart.ready = true;
1585 	return 0;
1586 }
1587 
1588 static int gmc_v9_0_hw_init(void *handle)
1589 {
1590 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1591 	bool value;
1592 	int r, i;
1593 	u32 tmp;
1594 
1595 	/* The sequence of these two function calls matters.*/
1596 	gmc_v9_0_init_golden_registers(adev);
1597 
1598 	if (adev->mode_info.num_crtc) {
1599 		if (adev->asic_type != CHIP_ARCTURUS) {
1600 			/* Lockout access through VGA aperture*/
1601 			WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1602 
1603 			/* disable VGA render */
1604 			WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
1605 		}
1606 	}
1607 
1608 	amdgpu_device_program_register_sequence(adev,
1609 						golden_settings_vega10_hdp,
1610 						ARRAY_SIZE(golden_settings_vega10_hdp));
1611 
1612 	if (adev->mmhub.funcs->update_power_gating)
1613 		adev->mmhub.funcs->update_power_gating(adev, true);
1614 
1615 	switch (adev->asic_type) {
1616 	case CHIP_ARCTURUS:
1617 		WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1);
1618 		break;
1619 	default:
1620 		break;
1621 	}
1622 
1623 	WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
1624 
1625 	tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
1626 	WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
1627 
1628 	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8));
1629 	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));
1630 
1631 	/* After HDP is initialized, flush HDP.*/
1632 	adev->nbio.funcs->hdp_flush(adev, NULL);
1633 
1634 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1635 		value = false;
1636 	else
1637 		value = true;
1638 
1639 	if (!amdgpu_sriov_vf(adev)) {
1640 		adev->gfxhub.funcs->set_fault_enable_default(adev, value);
1641 		adev->mmhub.funcs->set_fault_enable_default(adev, value);
1642 	}
1643 	for (i = 0; i < adev->num_vmhubs; ++i)
1644 		gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
1645 
1646 	if (adev->umc.funcs && adev->umc.funcs->init_registers)
1647 		adev->umc.funcs->init_registers(adev);
1648 
1649 	r = gmc_v9_0_gart_enable(adev);
1650 
1651 	return r;
1652 }
1653 
1654 /**
1655  * gmc_v9_0_gart_disable - gart disable
1656  *
1657  * @adev: amdgpu_device pointer
1658  *
1659  * This disables all VM page table.
1660  */
1661 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1662 {
1663 	adev->gfxhub.funcs->gart_disable(adev);
1664 	adev->mmhub.funcs->gart_disable(adev);
1665 	amdgpu_gart_table_vram_unpin(adev);
1666 }
1667 
1668 static int gmc_v9_0_hw_fini(void *handle)
1669 {
1670 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1671 
1672 	if (amdgpu_sriov_vf(adev)) {
1673 		/* full access mode, so don't touch any GMC register */
1674 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1675 		return 0;
1676 	}
1677 
1678 	amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
1679 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1680 	gmc_v9_0_gart_disable(adev);
1681 
1682 	return 0;
1683 }
1684 
1685 static int gmc_v9_0_suspend(void *handle)
1686 {
1687 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1688 
1689 	return gmc_v9_0_hw_fini(adev);
1690 }
1691 
1692 static int gmc_v9_0_resume(void *handle)
1693 {
1694 	int r;
1695 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1696 
1697 	r = gmc_v9_0_hw_init(adev);
1698 	if (r)
1699 		return r;
1700 
1701 	amdgpu_vmid_reset_all(adev);
1702 
1703 	return 0;
1704 }
1705 
1706 static bool gmc_v9_0_is_idle(void *handle)
1707 {
1708 	/* MC is always ready in GMC v9.*/
1709 	return true;
1710 }
1711 
1712 static int gmc_v9_0_wait_for_idle(void *handle)
1713 {
1714 	/* There is no need to wait for MC idle in GMC v9.*/
1715 	return 0;
1716 }
1717 
1718 static int gmc_v9_0_soft_reset(void *handle)
1719 {
1720 	/* XXX for emulation.*/
1721 	return 0;
1722 }
1723 
1724 static int gmc_v9_0_set_clockgating_state(void *handle,
1725 					enum amd_clockgating_state state)
1726 {
1727 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1728 
1729 	adev->mmhub.funcs->set_clockgating(adev, state);
1730 
1731 	athub_v1_0_set_clockgating(adev, state);
1732 
1733 	return 0;
1734 }
1735 
1736 static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1737 {
1738 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1739 
1740 	adev->mmhub.funcs->get_clockgating(adev, flags);
1741 
1742 	athub_v1_0_get_clockgating(adev, flags);
1743 }
1744 
1745 static int gmc_v9_0_set_powergating_state(void *handle,
1746 					enum amd_powergating_state state)
1747 {
1748 	return 0;
1749 }
1750 
1751 const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1752 	.name = "gmc_v9_0",
1753 	.early_init = gmc_v9_0_early_init,
1754 	.late_init = gmc_v9_0_late_init,
1755 	.sw_init = gmc_v9_0_sw_init,
1756 	.sw_fini = gmc_v9_0_sw_fini,
1757 	.hw_init = gmc_v9_0_hw_init,
1758 	.hw_fini = gmc_v9_0_hw_fini,
1759 	.suspend = gmc_v9_0_suspend,
1760 	.resume = gmc_v9_0_resume,
1761 	.is_idle = gmc_v9_0_is_idle,
1762 	.wait_for_idle = gmc_v9_0_wait_for_idle,
1763 	.soft_reset = gmc_v9_0_soft_reset,
1764 	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
1765 	.set_powergating_state = gmc_v9_0_set_powergating_state,
1766 	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
1767 };
1768 
1769 const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1770 {
1771 	.type = AMD_IP_BLOCK_TYPE_GMC,
1772 	.major = 9,
1773 	.minor = 0,
1774 	.rev = 0,
1775 	.funcs = &gmc_v9_0_ip_funcs,
1776 };
1777