xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c (revision 2208f39c)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/pci.h>
26 
27 #include <drm/drm_cache.h>
28 
29 #include "amdgpu.h"
30 #include "gmc_v9_0.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "amdgpu_gem.h"
33 
34 #include "hdp/hdp_4_0_offset.h"
35 #include "hdp/hdp_4_0_sh_mask.h"
36 #include "gc/gc_9_0_sh_mask.h"
37 #include "dce/dce_12_0_offset.h"
38 #include "dce/dce_12_0_sh_mask.h"
39 #include "vega10_enum.h"
40 #include "mmhub/mmhub_1_0_offset.h"
41 #include "athub/athub_1_0_sh_mask.h"
42 #include "athub/athub_1_0_offset.h"
43 #include "oss/osssys_4_0_offset.h"
44 
45 #include "soc15.h"
46 #include "soc15d.h"
47 #include "soc15_common.h"
48 #include "umc/umc_6_0_sh_mask.h"
49 
50 #include "gfxhub_v1_0.h"
51 #include "mmhub_v1_0.h"
52 #include "athub_v1_0.h"
53 #include "gfxhub_v1_1.h"
54 #include "mmhub_v9_4.h"
55 #include "umc_v6_1.h"
56 #include "umc_v6_0.h"
57 
58 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
59 
60 #include "amdgpu_ras.h"
61 #include "amdgpu_xgmi.h"
62 
63 /* add these here since we already include dce12 headers and these are for DCN */
64 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
65 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
66 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
67 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
68 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
69 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
70 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0                                                                  0x049d
71 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX                                                         2
72 
73 
74 static const char *gfxhub_client_ids[] = {
75 	"CB",
76 	"DB",
77 	"IA",
78 	"WD",
79 	"CPF",
80 	"CPC",
81 	"CPG",
82 	"RLC",
83 	"TCP",
84 	"SQC (inst)",
85 	"SQC (data)",
86 	"SQG",
87 	"PA",
88 };
89 
90 static const char *mmhub_client_ids_raven[][2] = {
91 	[0][0] = "MP1",
92 	[1][0] = "MP0",
93 	[2][0] = "VCN",
94 	[3][0] = "VCNU",
95 	[4][0] = "HDP",
96 	[5][0] = "DCE",
97 	[13][0] = "UTCL2",
98 	[19][0] = "TLS",
99 	[26][0] = "OSS",
100 	[27][0] = "SDMA0",
101 	[0][1] = "MP1",
102 	[1][1] = "MP0",
103 	[2][1] = "VCN",
104 	[3][1] = "VCNU",
105 	[4][1] = "HDP",
106 	[5][1] = "XDP",
107 	[6][1] = "DBGU0",
108 	[7][1] = "DCE",
109 	[8][1] = "DCEDWB0",
110 	[9][1] = "DCEDWB1",
111 	[26][1] = "OSS",
112 	[27][1] = "SDMA0",
113 };
114 
115 static const char *mmhub_client_ids_renoir[][2] = {
116 	[0][0] = "MP1",
117 	[1][0] = "MP0",
118 	[2][0] = "HDP",
119 	[4][0] = "DCEDMC",
120 	[5][0] = "DCEVGA",
121 	[13][0] = "UTCL2",
122 	[19][0] = "TLS",
123 	[26][0] = "OSS",
124 	[27][0] = "SDMA0",
125 	[28][0] = "VCN",
126 	[29][0] = "VCNU",
127 	[30][0] = "JPEG",
128 	[0][1] = "MP1",
129 	[1][1] = "MP0",
130 	[2][1] = "HDP",
131 	[3][1] = "XDP",
132 	[6][1] = "DBGU0",
133 	[7][1] = "DCEDMC",
134 	[8][1] = "DCEVGA",
135 	[9][1] = "DCEDWB",
136 	[26][1] = "OSS",
137 	[27][1] = "SDMA0",
138 	[28][1] = "VCN",
139 	[29][1] = "VCNU",
140 	[30][1] = "JPEG",
141 };
142 
143 static const char *mmhub_client_ids_vega10[][2] = {
144 	[0][0] = "MP0",
145 	[1][0] = "UVD",
146 	[2][0] = "UVDU",
147 	[3][0] = "HDP",
148 	[13][0] = "UTCL2",
149 	[14][0] = "OSS",
150 	[15][0] = "SDMA1",
151 	[32+0][0] = "VCE0",
152 	[32+1][0] = "VCE0U",
153 	[32+2][0] = "XDMA",
154 	[32+3][0] = "DCE",
155 	[32+4][0] = "MP1",
156 	[32+14][0] = "SDMA0",
157 	[0][1] = "MP0",
158 	[1][1] = "UVD",
159 	[2][1] = "UVDU",
160 	[3][1] = "DBGU0",
161 	[4][1] = "HDP",
162 	[5][1] = "XDP",
163 	[14][1] = "OSS",
164 	[15][1] = "SDMA0",
165 	[32+0][1] = "VCE0",
166 	[32+1][1] = "VCE0U",
167 	[32+2][1] = "XDMA",
168 	[32+3][1] = "DCE",
169 	[32+4][1] = "DCEDWB",
170 	[32+5][1] = "MP1",
171 	[32+6][1] = "DBGU1",
172 	[32+14][1] = "SDMA1",
173 };
174 
175 static const char *mmhub_client_ids_vega12[][2] = {
176 	[0][0] = "MP0",
177 	[1][0] = "VCE0",
178 	[2][0] = "VCE0U",
179 	[3][0] = "HDP",
180 	[13][0] = "UTCL2",
181 	[14][0] = "OSS",
182 	[15][0] = "SDMA1",
183 	[32+0][0] = "DCE",
184 	[32+1][0] = "XDMA",
185 	[32+2][0] = "UVD",
186 	[32+3][0] = "UVDU",
187 	[32+4][0] = "MP1",
188 	[32+15][0] = "SDMA0",
189 	[0][1] = "MP0",
190 	[1][1] = "VCE0",
191 	[2][1] = "VCE0U",
192 	[3][1] = "DBGU0",
193 	[4][1] = "HDP",
194 	[5][1] = "XDP",
195 	[14][1] = "OSS",
196 	[15][1] = "SDMA0",
197 	[32+0][1] = "DCE",
198 	[32+1][1] = "DCEDWB",
199 	[32+2][1] = "XDMA",
200 	[32+3][1] = "UVD",
201 	[32+4][1] = "UVDU",
202 	[32+5][1] = "MP1",
203 	[32+6][1] = "DBGU1",
204 	[32+15][1] = "SDMA1",
205 };
206 
207 static const char *mmhub_client_ids_vega20[][2] = {
208 	[0][0] = "XDMA",
209 	[1][0] = "DCE",
210 	[2][0] = "VCE0",
211 	[3][0] = "VCE0U",
212 	[4][0] = "UVD",
213 	[5][0] = "UVD1U",
214 	[13][0] = "OSS",
215 	[14][0] = "HDP",
216 	[15][0] = "SDMA0",
217 	[32+0][0] = "UVD",
218 	[32+1][0] = "UVDU",
219 	[32+2][0] = "MP1",
220 	[32+3][0] = "MP0",
221 	[32+12][0] = "UTCL2",
222 	[32+14][0] = "SDMA1",
223 	[0][1] = "XDMA",
224 	[1][1] = "DCE",
225 	[2][1] = "DCEDWB",
226 	[3][1] = "VCE0",
227 	[4][1] = "VCE0U",
228 	[5][1] = "UVD1",
229 	[6][1] = "UVD1U",
230 	[7][1] = "DBGU0",
231 	[8][1] = "XDP",
232 	[13][1] = "OSS",
233 	[14][1] = "HDP",
234 	[15][1] = "SDMA0",
235 	[32+0][1] = "UVD",
236 	[32+1][1] = "UVDU",
237 	[32+2][1] = "DBGU1",
238 	[32+3][1] = "MP1",
239 	[32+4][1] = "MP0",
240 	[32+14][1] = "SDMA1",
241 };
242 
243 static const char *mmhub_client_ids_arcturus[][2] = {
244 	[2][0] = "MP1",
245 	[3][0] = "MP0",
246 	[10][0] = "UTCL2",
247 	[13][0] = "OSS",
248 	[14][0] = "HDP",
249 	[15][0] = "SDMA0",
250 	[32+15][0] = "SDMA1",
251 	[64+15][0] = "SDMA2",
252 	[96+15][0] = "SDMA3",
253 	[128+15][0] = "SDMA4",
254 	[160+11][0] = "JPEG",
255 	[160+12][0] = "VCN",
256 	[160+13][0] = "VCNU",
257 	[160+15][0] = "SDMA5",
258 	[192+10][0] = "UTCL2",
259 	[192+11][0] = "JPEG1",
260 	[192+12][0] = "VCN1",
261 	[192+13][0] = "VCN1U",
262 	[192+15][0] = "SDMA6",
263 	[224+15][0] = "SDMA7",
264 	[0][1] = "DBGU1",
265 	[1][1] = "XDP",
266 	[2][1] = "MP1",
267 	[3][1] = "MP0",
268 	[13][1] = "OSS",
269 	[14][1] = "HDP",
270 	[15][1] = "SDMA0",
271 	[32+15][1] = "SDMA1",
272 	[64+15][1] = "SDMA2",
273 	[96+15][1] = "SDMA3",
274 	[128+15][1] = "SDMA4",
275 	[160+11][1] = "JPEG",
276 	[160+12][1] = "VCN",
277 	[160+13][1] = "VCNU",
278 	[160+15][1] = "SDMA5",
279 	[192+11][1] = "JPEG1",
280 	[192+12][1] = "VCN1",
281 	[192+13][1] = "VCN1U",
282 	[192+15][1] = "SDMA6",
283 	[224+15][1] = "SDMA7",
284 };
285 
286 static const u32 golden_settings_vega10_hdp[] =
287 {
288 	0xf64, 0x0fffffff, 0x00000000,
289 	0xf65, 0x0fffffff, 0x00000000,
290 	0xf66, 0x0fffffff, 0x00000000,
291 	0xf67, 0x0fffffff, 0x00000000,
292 	0xf68, 0x0fffffff, 0x00000000,
293 	0xf6a, 0x0fffffff, 0x00000000,
294 	0xf6b, 0x0fffffff, 0x00000000,
295 	0xf6c, 0x0fffffff, 0x00000000,
296 	0xf6d, 0x0fffffff, 0x00000000,
297 	0xf6e, 0x0fffffff, 0x00000000,
298 };
299 
300 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
301 {
302 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
303 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
304 };
305 
306 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
307 {
308 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
309 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
310 };
311 
312 static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
313 	(0x000143c0 + 0x00000000),
314 	(0x000143c0 + 0x00000800),
315 	(0x000143c0 + 0x00001000),
316 	(0x000143c0 + 0x00001800),
317 	(0x000543c0 + 0x00000000),
318 	(0x000543c0 + 0x00000800),
319 	(0x000543c0 + 0x00001000),
320 	(0x000543c0 + 0x00001800),
321 	(0x000943c0 + 0x00000000),
322 	(0x000943c0 + 0x00000800),
323 	(0x000943c0 + 0x00001000),
324 	(0x000943c0 + 0x00001800),
325 	(0x000d43c0 + 0x00000000),
326 	(0x000d43c0 + 0x00000800),
327 	(0x000d43c0 + 0x00001000),
328 	(0x000d43c0 + 0x00001800),
329 	(0x001143c0 + 0x00000000),
330 	(0x001143c0 + 0x00000800),
331 	(0x001143c0 + 0x00001000),
332 	(0x001143c0 + 0x00001800),
333 	(0x001543c0 + 0x00000000),
334 	(0x001543c0 + 0x00000800),
335 	(0x001543c0 + 0x00001000),
336 	(0x001543c0 + 0x00001800),
337 	(0x001943c0 + 0x00000000),
338 	(0x001943c0 + 0x00000800),
339 	(0x001943c0 + 0x00001000),
340 	(0x001943c0 + 0x00001800),
341 	(0x001d43c0 + 0x00000000),
342 	(0x001d43c0 + 0x00000800),
343 	(0x001d43c0 + 0x00001000),
344 	(0x001d43c0 + 0x00001800),
345 };
346 
347 static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
348 	(0x000143e0 + 0x00000000),
349 	(0x000143e0 + 0x00000800),
350 	(0x000143e0 + 0x00001000),
351 	(0x000143e0 + 0x00001800),
352 	(0x000543e0 + 0x00000000),
353 	(0x000543e0 + 0x00000800),
354 	(0x000543e0 + 0x00001000),
355 	(0x000543e0 + 0x00001800),
356 	(0x000943e0 + 0x00000000),
357 	(0x000943e0 + 0x00000800),
358 	(0x000943e0 + 0x00001000),
359 	(0x000943e0 + 0x00001800),
360 	(0x000d43e0 + 0x00000000),
361 	(0x000d43e0 + 0x00000800),
362 	(0x000d43e0 + 0x00001000),
363 	(0x000d43e0 + 0x00001800),
364 	(0x001143e0 + 0x00000000),
365 	(0x001143e0 + 0x00000800),
366 	(0x001143e0 + 0x00001000),
367 	(0x001143e0 + 0x00001800),
368 	(0x001543e0 + 0x00000000),
369 	(0x001543e0 + 0x00000800),
370 	(0x001543e0 + 0x00001000),
371 	(0x001543e0 + 0x00001800),
372 	(0x001943e0 + 0x00000000),
373 	(0x001943e0 + 0x00000800),
374 	(0x001943e0 + 0x00001000),
375 	(0x001943e0 + 0x00001800),
376 	(0x001d43e0 + 0x00000000),
377 	(0x001d43e0 + 0x00000800),
378 	(0x001d43e0 + 0x00001000),
379 	(0x001d43e0 + 0x00001800),
380 };
381 
382 static const uint32_t ecc_umc_mcumc_status_addrs[] = {
383 	(0x000143c2 + 0x00000000),
384 	(0x000143c2 + 0x00000800),
385 	(0x000143c2 + 0x00001000),
386 	(0x000143c2 + 0x00001800),
387 	(0x000543c2 + 0x00000000),
388 	(0x000543c2 + 0x00000800),
389 	(0x000543c2 + 0x00001000),
390 	(0x000543c2 + 0x00001800),
391 	(0x000943c2 + 0x00000000),
392 	(0x000943c2 + 0x00000800),
393 	(0x000943c2 + 0x00001000),
394 	(0x000943c2 + 0x00001800),
395 	(0x000d43c2 + 0x00000000),
396 	(0x000d43c2 + 0x00000800),
397 	(0x000d43c2 + 0x00001000),
398 	(0x000d43c2 + 0x00001800),
399 	(0x001143c2 + 0x00000000),
400 	(0x001143c2 + 0x00000800),
401 	(0x001143c2 + 0x00001000),
402 	(0x001143c2 + 0x00001800),
403 	(0x001543c2 + 0x00000000),
404 	(0x001543c2 + 0x00000800),
405 	(0x001543c2 + 0x00001000),
406 	(0x001543c2 + 0x00001800),
407 	(0x001943c2 + 0x00000000),
408 	(0x001943c2 + 0x00000800),
409 	(0x001943c2 + 0x00001000),
410 	(0x001943c2 + 0x00001800),
411 	(0x001d43c2 + 0x00000000),
412 	(0x001d43c2 + 0x00000800),
413 	(0x001d43c2 + 0x00001000),
414 	(0x001d43c2 + 0x00001800),
415 };
416 
417 static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
418 		struct amdgpu_irq_src *src,
419 		unsigned type,
420 		enum amdgpu_interrupt_state state)
421 {
422 	u32 bits, i, tmp, reg;
423 
424 	/* Devices newer then VEGA10/12 shall have these programming
425 	     sequences performed by PSP BL */
426 	if (adev->asic_type >= CHIP_VEGA20)
427 		return 0;
428 
429 	bits = 0x7f;
430 
431 	switch (state) {
432 	case AMDGPU_IRQ_STATE_DISABLE:
433 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
434 			reg = ecc_umc_mcumc_ctrl_addrs[i];
435 			tmp = RREG32(reg);
436 			tmp &= ~bits;
437 			WREG32(reg, tmp);
438 		}
439 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
440 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
441 			tmp = RREG32(reg);
442 			tmp &= ~bits;
443 			WREG32(reg, tmp);
444 		}
445 		break;
446 	case AMDGPU_IRQ_STATE_ENABLE:
447 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
448 			reg = ecc_umc_mcumc_ctrl_addrs[i];
449 			tmp = RREG32(reg);
450 			tmp |= bits;
451 			WREG32(reg, tmp);
452 		}
453 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
454 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
455 			tmp = RREG32(reg);
456 			tmp |= bits;
457 			WREG32(reg, tmp);
458 		}
459 		break;
460 	default:
461 		break;
462 	}
463 
464 	return 0;
465 }
466 
467 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
468 					struct amdgpu_irq_src *src,
469 					unsigned type,
470 					enum amdgpu_interrupt_state state)
471 {
472 	struct amdgpu_vmhub *hub;
473 	u32 tmp, reg, bits, i, j;
474 
475 	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
476 		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477 		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
478 		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
479 		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
480 		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
481 		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
482 
483 	switch (state) {
484 	case AMDGPU_IRQ_STATE_DISABLE:
485 		for (j = 0; j < adev->num_vmhubs; j++) {
486 			hub = &adev->vmhub[j];
487 			for (i = 0; i < 16; i++) {
488 				reg = hub->vm_context0_cntl + i;
489 				tmp = RREG32(reg);
490 				tmp &= ~bits;
491 				WREG32(reg, tmp);
492 			}
493 		}
494 		break;
495 	case AMDGPU_IRQ_STATE_ENABLE:
496 		for (j = 0; j < adev->num_vmhubs; j++) {
497 			hub = &adev->vmhub[j];
498 			for (i = 0; i < 16; i++) {
499 				reg = hub->vm_context0_cntl + i;
500 				tmp = RREG32(reg);
501 				tmp |= bits;
502 				WREG32(reg, tmp);
503 			}
504 		}
505 		break;
506 	default:
507 		break;
508 	}
509 
510 	return 0;
511 }
512 
513 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
514 				      struct amdgpu_irq_src *source,
515 				      struct amdgpu_iv_entry *entry)
516 {
517 	bool retry_fault = !!(entry->src_data[1] & 0x80);
518 	uint32_t status = 0, cid = 0, rw = 0;
519 	struct amdgpu_task_info task_info;
520 	struct amdgpu_vmhub *hub;
521 	const char *mmhub_cid;
522 	const char *hub_name;
523 	u64 addr;
524 
525 	addr = (u64)entry->src_data[0] << 12;
526 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
527 
528 	if (retry_fault) {
529 		/* Returning 1 here also prevents sending the IV to the KFD */
530 
531 		/* Process it onyl if it's the first fault for this address */
532 		if (entry->ih != &adev->irq.ih_soft &&
533 		    amdgpu_gmc_filter_faults(adev, addr, entry->pasid,
534 					     entry->timestamp))
535 			return 1;
536 
537 		/* Delegate it to a different ring if the hardware hasn't
538 		 * already done it.
539 		 */
540 		if (in_interrupt()) {
541 			amdgpu_irq_delegate(adev, entry, 8);
542 			return 1;
543 		}
544 
545 		/* Try to handle the recoverable page faults by filling page
546 		 * tables
547 		 */
548 		if (amdgpu_vm_handle_fault(adev, entry->pasid, addr))
549 			return 1;
550 	}
551 
552 	if (!printk_ratelimit())
553 		return 0;
554 
555 	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
556 		hub_name = "mmhub0";
557 		hub = &adev->vmhub[AMDGPU_MMHUB_0];
558 	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
559 		hub_name = "mmhub1";
560 		hub = &adev->vmhub[AMDGPU_MMHUB_1];
561 	} else {
562 		hub_name = "gfxhub0";
563 		hub = &adev->vmhub[AMDGPU_GFXHUB_0];
564 	}
565 
566 	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
567 	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
568 
569 	dev_err(adev->dev,
570 		"[%s] %s page fault (src_id:%u ring:%u vmid:%u "
571 		"pasid:%u, for process %s pid %d thread %s pid %d)\n",
572 		hub_name, retry_fault ? "retry" : "no-retry",
573 		entry->src_id, entry->ring_id, entry->vmid,
574 		entry->pasid, task_info.process_name, task_info.tgid,
575 		task_info.task_name, task_info.pid);
576 	dev_err(adev->dev, "  in page starting at address 0x%012llx from client %d\n",
577 		addr, entry->client_id);
578 
579 	if (amdgpu_sriov_vf(adev))
580 		return 0;
581 
582 	/*
583 	 * Issue a dummy read to wait for the status register to
584 	 * be updated to avoid reading an incorrect value due to
585 	 * the new fast GRBM interface.
586 	 */
587 	if (entry->vmid_src == AMDGPU_GFXHUB_0)
588 		RREG32(hub->vm_l2_pro_fault_status);
589 
590 	status = RREG32(hub->vm_l2_pro_fault_status);
591 	cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
592 	rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
593 	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
594 
595 
596 	dev_err(adev->dev,
597 		"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
598 		status);
599 	if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) {
600 		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
601 			cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
602 			gfxhub_client_ids[cid],
603 			cid);
604 	} else {
605 		switch (adev->asic_type) {
606 		case CHIP_VEGA10:
607 			mmhub_cid = mmhub_client_ids_vega10[cid][rw];
608 			break;
609 		case CHIP_VEGA12:
610 			mmhub_cid = mmhub_client_ids_vega12[cid][rw];
611 			break;
612 		case CHIP_VEGA20:
613 			mmhub_cid = mmhub_client_ids_vega20[cid][rw];
614 			break;
615 		case CHIP_ARCTURUS:
616 			mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
617 			break;
618 		case CHIP_RAVEN:
619 			mmhub_cid = mmhub_client_ids_raven[cid][rw];
620 			break;
621 		case CHIP_RENOIR:
622 			mmhub_cid = mmhub_client_ids_renoir[cid][rw];
623 			break;
624 		default:
625 			mmhub_cid = NULL;
626 			break;
627 		}
628 		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
629 			mmhub_cid ? mmhub_cid : "unknown", cid);
630 	}
631 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
632 		REG_GET_FIELD(status,
633 		VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
634 	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
635 		REG_GET_FIELD(status,
636 		VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
637 	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
638 		REG_GET_FIELD(status,
639 		VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
640 	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
641 		REG_GET_FIELD(status,
642 		VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
643 	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
644 	return 0;
645 }
646 
647 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
648 	.set = gmc_v9_0_vm_fault_interrupt_state,
649 	.process = gmc_v9_0_process_interrupt,
650 };
651 
652 
653 static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
654 	.set = gmc_v9_0_ecc_interrupt_state,
655 	.process = amdgpu_umc_process_ecc_irq,
656 };
657 
658 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
659 {
660 	adev->gmc.vm_fault.num_types = 1;
661 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
662 
663 	if (!amdgpu_sriov_vf(adev)) {
664 		adev->gmc.ecc_irq.num_types = 1;
665 		adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
666 	}
667 }
668 
669 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
670 					uint32_t flush_type)
671 {
672 	u32 req = 0;
673 
674 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
675 			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
676 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
677 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
678 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
679 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
680 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
681 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
682 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
683 			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
684 
685 	return req;
686 }
687 
688 /**
689  * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
690  *
691  * @adev: amdgpu_device pointer
692  * @vmhub: vmhub type
693  *
694  */
695 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
696 				       uint32_t vmhub)
697 {
698 	return ((vmhub == AMDGPU_MMHUB_0 ||
699 		 vmhub == AMDGPU_MMHUB_1) &&
700 		(!amdgpu_sriov_vf(adev)) &&
701 		(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
702 		   (adev->apu_flags & AMD_APU_IS_PICASSO))));
703 }
704 
705 static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
706 					uint8_t vmid, uint16_t *p_pasid)
707 {
708 	uint32_t value;
709 
710 	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
711 		     + vmid);
712 	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
713 
714 	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
715 }
716 
717 /*
718  * GART
719  * VMID 0 is the physical GPU addresses as used by the kernel.
720  * VMIDs 1-15 are used for userspace clients and are handled
721  * by the amdgpu vm/hsa code.
722  */
723 
724 /**
725  * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
726  *
727  * @adev: amdgpu_device pointer
728  * @vmid: vm instance to flush
729  * @flush_type: the flush type
730  *
731  * Flush the TLB for the requested page table using certain type.
732  */
733 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
734 					uint32_t vmhub, uint32_t flush_type)
735 {
736 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
737 	const unsigned eng = 17;
738 	u32 j, inv_req, inv_req2, tmp;
739 	struct amdgpu_vmhub *hub;
740 
741 	BUG_ON(vmhub >= adev->num_vmhubs);
742 
743 	hub = &adev->vmhub[vmhub];
744 	if (adev->gmc.xgmi.num_physical_nodes &&
745 	    adev->asic_type == CHIP_VEGA20) {
746 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
747 		 * heavy-weight TLB flush (type 2), which flushes
748 		 * both. Due to a race condition with concurrent
749 		 * memory accesses using the same TLB cache line, we
750 		 * still need a second TLB flush after this.
751 		 */
752 		inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
753 		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
754 	} else {
755 		inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
756 		inv_req2 = 0;
757 	}
758 
759 	/* This is necessary for a HW workaround under SRIOV as well
760 	 * as GFXOFF under bare metal
761 	 */
762 	if (adev->gfx.kiq.ring.sched.ready &&
763 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
764 	    down_read_trylock(&adev->reset_sem)) {
765 		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
766 		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
767 
768 		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
769 						   1 << vmid);
770 		up_read(&adev->reset_sem);
771 		return;
772 	}
773 
774 	spin_lock(&adev->gmc.invalidate_lock);
775 
776 	/*
777 	 * It may lose gpuvm invalidate acknowldege state across power-gating
778 	 * off cycle, add semaphore acquire before invalidation and semaphore
779 	 * release after invalidation to avoid entering power gated state
780 	 * to WA the Issue
781 	 */
782 
783 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
784 	if (use_semaphore) {
785 		for (j = 0; j < adev->usec_timeout; j++) {
786 			/* a read return value of 1 means semaphore acuqire */
787 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem +
788 					    hub->eng_distance * eng);
789 			if (tmp & 0x1)
790 				break;
791 			udelay(1);
792 		}
793 
794 		if (j >= adev->usec_timeout)
795 			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
796 	}
797 
798 	do {
799 		WREG32_NO_KIQ(hub->vm_inv_eng0_req +
800 			      hub->eng_distance * eng, inv_req);
801 
802 		/*
803 		 * Issue a dummy read to wait for the ACK register to
804 		 * be cleared to avoid a false ACK due to the new fast
805 		 * GRBM interface.
806 		 */
807 		if (vmhub == AMDGPU_GFXHUB_0)
808 			RREG32_NO_KIQ(hub->vm_inv_eng0_req +
809 				      hub->eng_distance * eng);
810 
811 		for (j = 0; j < adev->usec_timeout; j++) {
812 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack +
813 					    hub->eng_distance * eng);
814 			if (tmp & (1 << vmid))
815 				break;
816 			udelay(1);
817 		}
818 
819 		inv_req = inv_req2;
820 		inv_req2 = 0;
821 	} while (inv_req);
822 
823 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
824 	if (use_semaphore)
825 		/*
826 		 * add semaphore release after invalidation,
827 		 * write with 0 means semaphore release
828 		 */
829 		WREG32_NO_KIQ(hub->vm_inv_eng0_sem +
830 			      hub->eng_distance * eng, 0);
831 
832 	spin_unlock(&adev->gmc.invalidate_lock);
833 
834 	if (j < adev->usec_timeout)
835 		return;
836 
837 	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
838 }
839 
840 /**
841  * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
842  *
843  * @adev: amdgpu_device pointer
844  * @pasid: pasid to be flush
845  *
846  * Flush the TLB for the requested pasid.
847  */
848 static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
849 					uint16_t pasid, uint32_t flush_type,
850 					bool all_hub)
851 {
852 	int vmid, i;
853 	signed long r;
854 	uint32_t seq;
855 	uint16_t queried_pasid;
856 	bool ret;
857 	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
858 	struct amdgpu_kiq *kiq = &adev->gfx.kiq;
859 
860 	if (amdgpu_in_reset(adev))
861 		return -EIO;
862 
863 	if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) {
864 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
865 		 * heavy-weight TLB flush (type 2), which flushes
866 		 * both. Due to a race condition with concurrent
867 		 * memory accesses using the same TLB cache line, we
868 		 * still need a second TLB flush after this.
869 		 */
870 		bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
871 				       adev->asic_type == CHIP_VEGA20);
872 		/* 2 dwords flush + 8 dwords fence */
873 		unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
874 
875 		if (vega20_xgmi_wa)
876 			ndw += kiq->pmf->invalidate_tlbs_size;
877 
878 		spin_lock(&adev->gfx.kiq.ring_lock);
879 		/* 2 dwords flush + 8 dwords fence */
880 		amdgpu_ring_alloc(ring, ndw);
881 		if (vega20_xgmi_wa)
882 			kiq->pmf->kiq_invalidate_tlbs(ring,
883 						      pasid, 2, all_hub);
884 		kiq->pmf->kiq_invalidate_tlbs(ring,
885 					pasid, flush_type, all_hub);
886 		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
887 		if (r) {
888 			amdgpu_ring_undo(ring);
889 			spin_unlock(&adev->gfx.kiq.ring_lock);
890 			up_read(&adev->reset_sem);
891 			return -ETIME;
892 		}
893 
894 		amdgpu_ring_commit(ring);
895 		spin_unlock(&adev->gfx.kiq.ring_lock);
896 		r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout);
897 		if (r < 1) {
898 			dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
899 			up_read(&adev->reset_sem);
900 			return -ETIME;
901 		}
902 		up_read(&adev->reset_sem);
903 		return 0;
904 	}
905 
906 	for (vmid = 1; vmid < 16; vmid++) {
907 
908 		ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
909 				&queried_pasid);
910 		if (ret && queried_pasid == pasid) {
911 			if (all_hub) {
912 				for (i = 0; i < adev->num_vmhubs; i++)
913 					gmc_v9_0_flush_gpu_tlb(adev, vmid,
914 							i, flush_type);
915 			} else {
916 				gmc_v9_0_flush_gpu_tlb(adev, vmid,
917 						AMDGPU_GFXHUB_0, flush_type);
918 			}
919 			break;
920 		}
921 	}
922 
923 	return 0;
924 
925 }
926 
927 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
928 					    unsigned vmid, uint64_t pd_addr)
929 {
930 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub);
931 	struct amdgpu_device *adev = ring->adev;
932 	struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
933 	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
934 	unsigned eng = ring->vm_inv_eng;
935 
936 	/*
937 	 * It may lose gpuvm invalidate acknowldege state across power-gating
938 	 * off cycle, add semaphore acquire before invalidation and semaphore
939 	 * release after invalidation to avoid entering power gated state
940 	 * to WA the Issue
941 	 */
942 
943 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
944 	if (use_semaphore)
945 		/* a read return value of 1 means semaphore acuqire */
946 		amdgpu_ring_emit_reg_wait(ring,
947 					  hub->vm_inv_eng0_sem +
948 					  hub->eng_distance * eng, 0x1, 0x1);
949 
950 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
951 			      (hub->ctx_addr_distance * vmid),
952 			      lower_32_bits(pd_addr));
953 
954 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
955 			      (hub->ctx_addr_distance * vmid),
956 			      upper_32_bits(pd_addr));
957 
958 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
959 					    hub->eng_distance * eng,
960 					    hub->vm_inv_eng0_ack +
961 					    hub->eng_distance * eng,
962 					    req, 1 << vmid);
963 
964 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
965 	if (use_semaphore)
966 		/*
967 		 * add semaphore release after invalidation,
968 		 * write with 0 means semaphore release
969 		 */
970 		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
971 				      hub->eng_distance * eng, 0);
972 
973 	return pd_addr;
974 }
975 
976 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
977 					unsigned pasid)
978 {
979 	struct amdgpu_device *adev = ring->adev;
980 	uint32_t reg;
981 
982 	/* Do nothing because there's no lut register for mmhub1. */
983 	if (ring->funcs->vmhub == AMDGPU_MMHUB_1)
984 		return;
985 
986 	if (ring->funcs->vmhub == AMDGPU_GFXHUB_0)
987 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
988 	else
989 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
990 
991 	amdgpu_ring_emit_wreg(ring, reg, pasid);
992 }
993 
994 /*
995  * PTE format on VEGA 10:
996  * 63:59 reserved
997  * 58:57 mtype
998  * 56 F
999  * 55 L
1000  * 54 P
1001  * 53 SW
1002  * 52 T
1003  * 50:48 reserved
1004  * 47:12 4k physical page base address
1005  * 11:7 fragment
1006  * 6 write
1007  * 5 read
1008  * 4 exe
1009  * 3 Z
1010  * 2 snooped
1011  * 1 system
1012  * 0 valid
1013  *
1014  * PDE format on VEGA 10:
1015  * 63:59 block fragment size
1016  * 58:55 reserved
1017  * 54 P
1018  * 53:48 reserved
1019  * 47:6 physical base address of PD or PTE
1020  * 5:3 reserved
1021  * 2 C
1022  * 1 system
1023  * 0 valid
1024  */
1025 
1026 static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1027 
1028 {
1029 	switch (flags) {
1030 	case AMDGPU_VM_MTYPE_DEFAULT:
1031 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1032 	case AMDGPU_VM_MTYPE_NC:
1033 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1034 	case AMDGPU_VM_MTYPE_WC:
1035 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1036 	case AMDGPU_VM_MTYPE_RW:
1037 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1038 	case AMDGPU_VM_MTYPE_CC:
1039 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1040 	case AMDGPU_VM_MTYPE_UC:
1041 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1042 	default:
1043 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1044 	}
1045 }
1046 
1047 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1048 				uint64_t *addr, uint64_t *flags)
1049 {
1050 	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1051 		*addr = adev->vm_manager.vram_base_offset + *addr -
1052 			adev->gmc.vram_start;
1053 	BUG_ON(*addr & 0xFFFF00000000003FULL);
1054 
1055 	if (!adev->gmc.translate_further)
1056 		return;
1057 
1058 	if (level == AMDGPU_VM_PDB1) {
1059 		/* Set the block fragment size */
1060 		if (!(*flags & AMDGPU_PDE_PTE))
1061 			*flags |= AMDGPU_PDE_BFS(0x9);
1062 
1063 	} else if (level == AMDGPU_VM_PDB0) {
1064 		if (*flags & AMDGPU_PDE_PTE)
1065 			*flags &= ~AMDGPU_PDE_PTE;
1066 		else
1067 			*flags |= AMDGPU_PTE_TF;
1068 	}
1069 }
1070 
1071 static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1072 				struct amdgpu_bo_va_mapping *mapping,
1073 				uint64_t *flags)
1074 {
1075 	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1076 	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1077 
1078 	*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1079 	*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1080 
1081 	if (mapping->flags & AMDGPU_PTE_PRT) {
1082 		*flags |= AMDGPU_PTE_PRT;
1083 		*flags &= ~AMDGPU_PTE_VALID;
1084 	}
1085 
1086 	if (adev->asic_type == CHIP_ARCTURUS &&
1087 	    !(*flags & AMDGPU_PTE_SYSTEM) &&
1088 	    mapping->bo_va->is_xgmi)
1089 		*flags |= AMDGPU_PTE_SNOOPED;
1090 }
1091 
1092 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1093 {
1094 	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1095 	unsigned size;
1096 
1097 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1098 		size = AMDGPU_VBIOS_VGA_ALLOCATION;
1099 	} else {
1100 		u32 viewport;
1101 
1102 		switch (adev->asic_type) {
1103 		case CHIP_RAVEN:
1104 		case CHIP_RENOIR:
1105 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1106 			size = (REG_GET_FIELD(viewport,
1107 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1108 				REG_GET_FIELD(viewport,
1109 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1110 				4);
1111 			break;
1112 		case CHIP_VEGA10:
1113 		case CHIP_VEGA12:
1114 		case CHIP_VEGA20:
1115 		default:
1116 			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1117 			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1118 				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1119 				4);
1120 			break;
1121 		}
1122 	}
1123 
1124 	return size;
1125 }
1126 
1127 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1128 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1129 	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1130 	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1131 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1132 	.map_mtype = gmc_v9_0_map_mtype,
1133 	.get_vm_pde = gmc_v9_0_get_vm_pde,
1134 	.get_vm_pte = gmc_v9_0_get_vm_pte,
1135 	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1136 };
1137 
1138 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1139 {
1140 	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1141 }
1142 
1143 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1144 {
1145 	switch (adev->asic_type) {
1146 	case CHIP_VEGA10:
1147 		adev->umc.funcs = &umc_v6_0_funcs;
1148 		break;
1149 	case CHIP_VEGA20:
1150 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1151 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1152 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1153 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1154 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1155 		adev->umc.funcs = &umc_v6_1_funcs;
1156 		break;
1157 	case CHIP_ARCTURUS:
1158 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1159 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1160 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1161 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1162 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1163 		adev->umc.funcs = &umc_v6_1_funcs;
1164 		break;
1165 	default:
1166 		break;
1167 	}
1168 }
1169 
1170 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1171 {
1172 	switch (adev->asic_type) {
1173 	case CHIP_ARCTURUS:
1174 		adev->mmhub.funcs = &mmhub_v9_4_funcs;
1175 		break;
1176 	default:
1177 		adev->mmhub.funcs = &mmhub_v1_0_funcs;
1178 		break;
1179 	}
1180 }
1181 
1182 static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1183 {
1184 	adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1185 }
1186 
1187 static int gmc_v9_0_early_init(void *handle)
1188 {
1189 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1190 
1191 	gmc_v9_0_set_gmc_funcs(adev);
1192 	gmc_v9_0_set_irq_funcs(adev);
1193 	gmc_v9_0_set_umc_funcs(adev);
1194 	gmc_v9_0_set_mmhub_funcs(adev);
1195 	gmc_v9_0_set_gfxhub_funcs(adev);
1196 
1197 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1198 	adev->gmc.shared_aperture_end =
1199 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1200 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1201 	adev->gmc.private_aperture_end =
1202 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1203 
1204 	return 0;
1205 }
1206 
1207 static int gmc_v9_0_late_init(void *handle)
1208 {
1209 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1210 	int r;
1211 
1212 	amdgpu_bo_late_init(adev);
1213 
1214 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1215 	if (r)
1216 		return r;
1217 
1218 	/*
1219 	 * Workaround performance drop issue with VBIOS enables partial
1220 	 * writes, while disables HBM ECC for vega10.
1221 	 */
1222 	if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) {
1223 		if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1224 			if (adev->df.funcs->enable_ecc_force_par_wr_rmw)
1225 				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1226 		}
1227 	}
1228 
1229 	if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count)
1230 		adev->mmhub.funcs->reset_ras_error_count(adev);
1231 
1232 	r = amdgpu_gmc_ras_late_init(adev);
1233 	if (r)
1234 		return r;
1235 
1236 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1237 }
1238 
1239 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1240 					struct amdgpu_gmc *mc)
1241 {
1242 	u64 base = 0;
1243 
1244 	if (!amdgpu_sriov_vf(adev))
1245 		base = adev->mmhub.funcs->get_fb_location(adev);
1246 
1247 	/* add the xgmi offset of the physical node */
1248 	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1249 	amdgpu_gmc_vram_location(adev, mc, base);
1250 	amdgpu_gmc_gart_location(adev, mc);
1251 	amdgpu_gmc_agp_location(adev, mc);
1252 	/* base offset of vram pages */
1253 	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1254 
1255 	/* XXX: add the xgmi offset of the physical node? */
1256 	adev->vm_manager.vram_base_offset +=
1257 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1258 }
1259 
1260 /**
1261  * gmc_v9_0_mc_init - initialize the memory controller driver params
1262  *
1263  * @adev: amdgpu_device pointer
1264  *
1265  * Look up the amount of vram, vram width, and decide how to place
1266  * vram and gart within the GPU's physical address space.
1267  * Returns 0 for success.
1268  */
1269 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1270 {
1271 	int r;
1272 
1273 	/* size in MB on si */
1274 	adev->gmc.mc_vram_size =
1275 		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1276 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1277 
1278 	if (!(adev->flags & AMD_IS_APU)) {
1279 		r = amdgpu_device_resize_fb_bar(adev);
1280 		if (r)
1281 			return r;
1282 	}
1283 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1284 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1285 
1286 #ifdef CONFIG_X86_64
1287 	if (adev->flags & AMD_IS_APU) {
1288 		adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1289 		adev->gmc.aper_size = adev->gmc.real_vram_size;
1290 	}
1291 #endif
1292 	/* In case the PCI BAR is larger than the actual amount of vram */
1293 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1294 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
1295 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
1296 
1297 	/* set the gart size */
1298 	if (amdgpu_gart_size == -1) {
1299 		switch (adev->asic_type) {
1300 		case CHIP_VEGA10:  /* all engines support GPUVM */
1301 		case CHIP_VEGA12:  /* all engines support GPUVM */
1302 		case CHIP_VEGA20:
1303 		case CHIP_ARCTURUS:
1304 		default:
1305 			adev->gmc.gart_size = 512ULL << 20;
1306 			break;
1307 		case CHIP_RAVEN:   /* DCE SG support */
1308 		case CHIP_RENOIR:
1309 			adev->gmc.gart_size = 1024ULL << 20;
1310 			break;
1311 		}
1312 	} else {
1313 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1314 	}
1315 
1316 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1317 
1318 	return 0;
1319 }
1320 
1321 static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1322 {
1323 	int r;
1324 
1325 	if (adev->gart.bo) {
1326 		WARN(1, "VEGA10 PCIE GART already initialized\n");
1327 		return 0;
1328 	}
1329 	/* Initialize common gart structure */
1330 	r = amdgpu_gart_init(adev);
1331 	if (r)
1332 		return r;
1333 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1334 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1335 				 AMDGPU_PTE_EXECUTABLE;
1336 	return amdgpu_gart_table_vram_alloc(adev);
1337 }
1338 
1339 /**
1340  * gmc_v9_0_save_registers - saves regs
1341  *
1342  * @adev: amdgpu_device pointer
1343  *
1344  * This saves potential register values that should be
1345  * restored upon resume
1346  */
1347 static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1348 {
1349 	if (adev->asic_type == CHIP_RAVEN)
1350 		adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1351 }
1352 
1353 static int gmc_v9_0_sw_init(void *handle)
1354 {
1355 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
1356 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1357 
1358 	adev->gfxhub.funcs->init(adev);
1359 
1360 	adev->mmhub.funcs->init(adev);
1361 
1362 	spin_lock_init(&adev->gmc.invalidate_lock);
1363 
1364 	r = amdgpu_atomfirmware_get_vram_info(adev,
1365 		&vram_width, &vram_type, &vram_vendor);
1366 	if (amdgpu_sriov_vf(adev))
1367 		/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
1368 		 * and DF related registers is not readable, seems hardcord is the
1369 		 * only way to set the correct vram_width
1370 		 */
1371 		adev->gmc.vram_width = 2048;
1372 	else if (amdgpu_emu_mode != 1)
1373 		adev->gmc.vram_width = vram_width;
1374 
1375 	if (!adev->gmc.vram_width) {
1376 		int chansize, numchan;
1377 
1378 		/* hbm memory channel size */
1379 		if (adev->flags & AMD_IS_APU)
1380 			chansize = 64;
1381 		else
1382 			chansize = 128;
1383 
1384 		numchan = adev->df.funcs->get_hbm_channel_number(adev);
1385 		adev->gmc.vram_width = numchan * chansize;
1386 	}
1387 
1388 	adev->gmc.vram_type = vram_type;
1389 	adev->gmc.vram_vendor = vram_vendor;
1390 	switch (adev->asic_type) {
1391 	case CHIP_RAVEN:
1392 		adev->num_vmhubs = 2;
1393 
1394 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
1395 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1396 		} else {
1397 			/* vm_size is 128TB + 512GB for legacy 3-level page support */
1398 			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
1399 			adev->gmc.translate_further =
1400 				adev->vm_manager.num_level > 1;
1401 		}
1402 		break;
1403 	case CHIP_VEGA10:
1404 	case CHIP_VEGA12:
1405 	case CHIP_VEGA20:
1406 	case CHIP_RENOIR:
1407 		adev->num_vmhubs = 2;
1408 
1409 
1410 		/*
1411 		 * To fulfill 4-level page support,
1412 		 * vm size is 256TB (48bit), maximum size of Vega10,
1413 		 * block size 512 (9bit)
1414 		 */
1415 		/* sriov restrict max_pfn below AMDGPU_GMC_HOLE */
1416 		if (amdgpu_sriov_vf(adev))
1417 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
1418 		else
1419 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1420 		break;
1421 	case CHIP_ARCTURUS:
1422 		adev->num_vmhubs = 3;
1423 
1424 		/* Keep the vm size same with Vega20 */
1425 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
1426 		break;
1427 	default:
1428 		break;
1429 	}
1430 
1431 	/* This interrupt is VMC page fault.*/
1432 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
1433 				&adev->gmc.vm_fault);
1434 	if (r)
1435 		return r;
1436 
1437 	if (adev->asic_type == CHIP_ARCTURUS) {
1438 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
1439 					&adev->gmc.vm_fault);
1440 		if (r)
1441 			return r;
1442 	}
1443 
1444 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
1445 				&adev->gmc.vm_fault);
1446 
1447 	if (r)
1448 		return r;
1449 
1450 	if (!amdgpu_sriov_vf(adev)) {
1451 		/* interrupt sent to DF. */
1452 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
1453 				      &adev->gmc.ecc_irq);
1454 		if (r)
1455 			return r;
1456 	}
1457 
1458 	/* Set the internal MC address mask
1459 	 * This is the max address of the GPU's
1460 	 * internal address space.
1461 	 */
1462 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
1463 
1464 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
1465 	if (r) {
1466 		printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
1467 		return r;
1468 	}
1469 	adev->need_swiotlb = drm_need_swiotlb(44);
1470 
1471 	if (adev->gmc.xgmi.supported) {
1472 		r = adev->gfxhub.funcs->get_xgmi_info(adev);
1473 		if (r)
1474 			return r;
1475 	}
1476 
1477 	r = gmc_v9_0_mc_init(adev);
1478 	if (r)
1479 		return r;
1480 
1481 	amdgpu_gmc_get_vbios_allocations(adev);
1482 
1483 	/* Memory manager */
1484 	r = amdgpu_bo_init(adev);
1485 	if (r)
1486 		return r;
1487 
1488 	r = gmc_v9_0_gart_init(adev);
1489 	if (r)
1490 		return r;
1491 
1492 	/*
1493 	 * number of VMs
1494 	 * VMID 0 is reserved for System
1495 	 * amdgpu graphics/compute will use VMIDs 1..n-1
1496 	 * amdkfd will use VMIDs n..15
1497 	 *
1498 	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
1499 	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
1500 	 * for video processing.
1501 	 */
1502 	adev->vm_manager.first_kfd_vmid =
1503 		adev->asic_type == CHIP_ARCTURUS ? 3 : 8;
1504 
1505 	amdgpu_vm_manager_init(adev);
1506 
1507 	gmc_v9_0_save_registers(adev);
1508 
1509 	return 0;
1510 }
1511 
1512 static int gmc_v9_0_sw_fini(void *handle)
1513 {
1514 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1515 
1516 	amdgpu_gmc_ras_fini(adev);
1517 	amdgpu_gem_force_release(adev);
1518 	amdgpu_vm_manager_fini(adev);
1519 	amdgpu_gart_table_vram_free(adev);
1520 	amdgpu_bo_fini(adev);
1521 	amdgpu_gart_fini(adev);
1522 
1523 	return 0;
1524 }
1525 
1526 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
1527 {
1528 
1529 	switch (adev->asic_type) {
1530 	case CHIP_VEGA10:
1531 		if (amdgpu_sriov_vf(adev))
1532 			break;
1533 		fallthrough;
1534 	case CHIP_VEGA20:
1535 		soc15_program_register_sequence(adev,
1536 						golden_settings_mmhub_1_0_0,
1537 						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
1538 		soc15_program_register_sequence(adev,
1539 						golden_settings_athub_1_0_0,
1540 						ARRAY_SIZE(golden_settings_athub_1_0_0));
1541 		break;
1542 	case CHIP_VEGA12:
1543 		break;
1544 	case CHIP_RAVEN:
1545 		/* TODO for renoir */
1546 		soc15_program_register_sequence(adev,
1547 						golden_settings_athub_1_0_0,
1548 						ARRAY_SIZE(golden_settings_athub_1_0_0));
1549 		break;
1550 	default:
1551 		break;
1552 	}
1553 }
1554 
1555 /**
1556  * gmc_v9_0_restore_registers - restores regs
1557  *
1558  * @adev: amdgpu_device pointer
1559  *
1560  * This restores register values, saved at suspend.
1561  */
1562 void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
1563 {
1564 	if (adev->asic_type == CHIP_RAVEN) {
1565 		WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
1566 		WARN_ON(adev->gmc.sdpif_register !=
1567 			RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
1568 	}
1569 }
1570 
1571 /**
1572  * gmc_v9_0_gart_enable - gart enable
1573  *
1574  * @adev: amdgpu_device pointer
1575  */
1576 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1577 {
1578 	int r;
1579 
1580 	if (adev->gart.bo == NULL) {
1581 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1582 		return -EINVAL;
1583 	}
1584 	r = amdgpu_gart_table_vram_pin(adev);
1585 	if (r)
1586 		return r;
1587 
1588 	r = adev->gfxhub.funcs->gart_enable(adev);
1589 	if (r)
1590 		return r;
1591 
1592 	r = adev->mmhub.funcs->gart_enable(adev);
1593 	if (r)
1594 		return r;
1595 
1596 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1597 		 (unsigned)(adev->gmc.gart_size >> 20),
1598 		 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
1599 	adev->gart.ready = true;
1600 	return 0;
1601 }
1602 
1603 static int gmc_v9_0_hw_init(void *handle)
1604 {
1605 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1606 	bool value;
1607 	int r, i;
1608 	u32 tmp;
1609 
1610 	/* The sequence of these two function calls matters.*/
1611 	gmc_v9_0_init_golden_registers(adev);
1612 
1613 	if (adev->mode_info.num_crtc) {
1614 		if (adev->asic_type != CHIP_ARCTURUS) {
1615 			/* Lockout access through VGA aperture*/
1616 			WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1617 
1618 			/* disable VGA render */
1619 			WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
1620 		}
1621 	}
1622 
1623 	amdgpu_device_program_register_sequence(adev,
1624 						golden_settings_vega10_hdp,
1625 						ARRAY_SIZE(golden_settings_vega10_hdp));
1626 
1627 	if (adev->mmhub.funcs->update_power_gating)
1628 		adev->mmhub.funcs->update_power_gating(adev, true);
1629 
1630 	switch (adev->asic_type) {
1631 	case CHIP_ARCTURUS:
1632 		WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1);
1633 		break;
1634 	default:
1635 		break;
1636 	}
1637 
1638 	WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
1639 
1640 	tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
1641 	WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
1642 
1643 	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8));
1644 	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));
1645 
1646 	/* After HDP is initialized, flush HDP.*/
1647 	adev->nbio.funcs->hdp_flush(adev, NULL);
1648 
1649 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1650 		value = false;
1651 	else
1652 		value = true;
1653 
1654 	if (!amdgpu_sriov_vf(adev)) {
1655 		adev->gfxhub.funcs->set_fault_enable_default(adev, value);
1656 		adev->mmhub.funcs->set_fault_enable_default(adev, value);
1657 	}
1658 	for (i = 0; i < adev->num_vmhubs; ++i)
1659 		gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
1660 
1661 	if (adev->umc.funcs && adev->umc.funcs->init_registers)
1662 		adev->umc.funcs->init_registers(adev);
1663 
1664 	r = gmc_v9_0_gart_enable(adev);
1665 
1666 	return r;
1667 }
1668 
1669 /**
1670  * gmc_v9_0_gart_disable - gart disable
1671  *
1672  * @adev: amdgpu_device pointer
1673  *
1674  * This disables all VM page table.
1675  */
1676 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1677 {
1678 	adev->gfxhub.funcs->gart_disable(adev);
1679 	adev->mmhub.funcs->gart_disable(adev);
1680 	amdgpu_gart_table_vram_unpin(adev);
1681 }
1682 
1683 static int gmc_v9_0_hw_fini(void *handle)
1684 {
1685 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1686 
1687 	if (amdgpu_sriov_vf(adev)) {
1688 		/* full access mode, so don't touch any GMC register */
1689 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1690 		return 0;
1691 	}
1692 
1693 	amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
1694 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1695 	gmc_v9_0_gart_disable(adev);
1696 
1697 	return 0;
1698 }
1699 
1700 static int gmc_v9_0_suspend(void *handle)
1701 {
1702 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1703 
1704 	return gmc_v9_0_hw_fini(adev);
1705 }
1706 
1707 static int gmc_v9_0_resume(void *handle)
1708 {
1709 	int r;
1710 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1711 
1712 	r = gmc_v9_0_hw_init(adev);
1713 	if (r)
1714 		return r;
1715 
1716 	amdgpu_vmid_reset_all(adev);
1717 
1718 	return 0;
1719 }
1720 
1721 static bool gmc_v9_0_is_idle(void *handle)
1722 {
1723 	/* MC is always ready in GMC v9.*/
1724 	return true;
1725 }
1726 
1727 static int gmc_v9_0_wait_for_idle(void *handle)
1728 {
1729 	/* There is no need to wait for MC idle in GMC v9.*/
1730 	return 0;
1731 }
1732 
1733 static int gmc_v9_0_soft_reset(void *handle)
1734 {
1735 	/* XXX for emulation.*/
1736 	return 0;
1737 }
1738 
1739 static int gmc_v9_0_set_clockgating_state(void *handle,
1740 					enum amd_clockgating_state state)
1741 {
1742 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1743 
1744 	adev->mmhub.funcs->set_clockgating(adev, state);
1745 
1746 	athub_v1_0_set_clockgating(adev, state);
1747 
1748 	return 0;
1749 }
1750 
1751 static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1752 {
1753 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1754 
1755 	adev->mmhub.funcs->get_clockgating(adev, flags);
1756 
1757 	athub_v1_0_get_clockgating(adev, flags);
1758 }
1759 
1760 static int gmc_v9_0_set_powergating_state(void *handle,
1761 					enum amd_powergating_state state)
1762 {
1763 	return 0;
1764 }
1765 
1766 const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1767 	.name = "gmc_v9_0",
1768 	.early_init = gmc_v9_0_early_init,
1769 	.late_init = gmc_v9_0_late_init,
1770 	.sw_init = gmc_v9_0_sw_init,
1771 	.sw_fini = gmc_v9_0_sw_fini,
1772 	.hw_init = gmc_v9_0_hw_init,
1773 	.hw_fini = gmc_v9_0_hw_fini,
1774 	.suspend = gmc_v9_0_suspend,
1775 	.resume = gmc_v9_0_resume,
1776 	.is_idle = gmc_v9_0_is_idle,
1777 	.wait_for_idle = gmc_v9_0_wait_for_idle,
1778 	.soft_reset = gmc_v9_0_soft_reset,
1779 	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
1780 	.set_powergating_state = gmc_v9_0_set_powergating_state,
1781 	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
1782 };
1783 
1784 const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1785 {
1786 	.type = AMD_IP_BLOCK_TYPE_GMC,
1787 	.major = 9,
1788 	.minor = 0,
1789 	.rev = 0,
1790 	.funcs = &gmc_v9_0_ip_funcs,
1791 };
1792