xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c (revision 993cace4)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 
24 #include <linux/firmware.h>
25 #include <linux/pci.h>
26 
27 #include <drm/drm_cache.h>
28 
29 #include "amdgpu.h"
30 #include "gmc_v9_0.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "amdgpu_gem.h"
33 
34 #include "gc/gc_9_0_sh_mask.h"
35 #include "dce/dce_12_0_offset.h"
36 #include "dce/dce_12_0_sh_mask.h"
37 #include "vega10_enum.h"
38 #include "mmhub/mmhub_1_0_offset.h"
39 #include "athub/athub_1_0_sh_mask.h"
40 #include "athub/athub_1_0_offset.h"
41 #include "oss/osssys_4_0_offset.h"
42 
43 #include "soc15.h"
44 #include "soc15d.h"
45 #include "soc15_common.h"
46 #include "umc/umc_6_0_sh_mask.h"
47 
48 #include "gfxhub_v1_0.h"
49 #include "mmhub_v1_0.h"
50 #include "athub_v1_0.h"
51 #include "gfxhub_v1_1.h"
52 #include "gfxhub_v1_2.h"
53 #include "mmhub_v9_4.h"
54 #include "mmhub_v1_7.h"
55 #include "mmhub_v1_8.h"
56 #include "umc_v6_1.h"
57 #include "umc_v6_0.h"
58 #include "umc_v6_7.h"
59 #include "hdp_v4_0.h"
60 #include "mca_v3_0.h"
61 
62 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
63 
64 #include "amdgpu_ras.h"
65 #include "amdgpu_xgmi.h"
66 
67 #include "amdgpu_reset.h"
68 
69 /* add these here since we already include dce12 headers and these are for DCN */
70 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
71 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
72 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
73 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
74 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
75 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
76 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0                                                                  0x049d
77 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX                                                         2
78 
79 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2                                                          0x05ea
80 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX                                                 2
81 
82 #define MAX_MEM_RANGES 8
83 
84 static const char * const gfxhub_client_ids[] = {
85 	"CB",
86 	"DB",
87 	"IA",
88 	"WD",
89 	"CPF",
90 	"CPC",
91 	"CPG",
92 	"RLC",
93 	"TCP",
94 	"SQC (inst)",
95 	"SQC (data)",
96 	"SQG",
97 	"PA",
98 };
99 
100 static const char *mmhub_client_ids_raven[][2] = {
101 	[0][0] = "MP1",
102 	[1][0] = "MP0",
103 	[2][0] = "VCN",
104 	[3][0] = "VCNU",
105 	[4][0] = "HDP",
106 	[5][0] = "DCE",
107 	[13][0] = "UTCL2",
108 	[19][0] = "TLS",
109 	[26][0] = "OSS",
110 	[27][0] = "SDMA0",
111 	[0][1] = "MP1",
112 	[1][1] = "MP0",
113 	[2][1] = "VCN",
114 	[3][1] = "VCNU",
115 	[4][1] = "HDP",
116 	[5][1] = "XDP",
117 	[6][1] = "DBGU0",
118 	[7][1] = "DCE",
119 	[8][1] = "DCEDWB0",
120 	[9][1] = "DCEDWB1",
121 	[26][1] = "OSS",
122 	[27][1] = "SDMA0",
123 };
124 
125 static const char *mmhub_client_ids_renoir[][2] = {
126 	[0][0] = "MP1",
127 	[1][0] = "MP0",
128 	[2][0] = "HDP",
129 	[4][0] = "DCEDMC",
130 	[5][0] = "DCEVGA",
131 	[13][0] = "UTCL2",
132 	[19][0] = "TLS",
133 	[26][0] = "OSS",
134 	[27][0] = "SDMA0",
135 	[28][0] = "VCN",
136 	[29][0] = "VCNU",
137 	[30][0] = "JPEG",
138 	[0][1] = "MP1",
139 	[1][1] = "MP0",
140 	[2][1] = "HDP",
141 	[3][1] = "XDP",
142 	[6][1] = "DBGU0",
143 	[7][1] = "DCEDMC",
144 	[8][1] = "DCEVGA",
145 	[9][1] = "DCEDWB",
146 	[26][1] = "OSS",
147 	[27][1] = "SDMA0",
148 	[28][1] = "VCN",
149 	[29][1] = "VCNU",
150 	[30][1] = "JPEG",
151 };
152 
153 static const char *mmhub_client_ids_vega10[][2] = {
154 	[0][0] = "MP0",
155 	[1][0] = "UVD",
156 	[2][0] = "UVDU",
157 	[3][0] = "HDP",
158 	[13][0] = "UTCL2",
159 	[14][0] = "OSS",
160 	[15][0] = "SDMA1",
161 	[32+0][0] = "VCE0",
162 	[32+1][0] = "VCE0U",
163 	[32+2][0] = "XDMA",
164 	[32+3][0] = "DCE",
165 	[32+4][0] = "MP1",
166 	[32+14][0] = "SDMA0",
167 	[0][1] = "MP0",
168 	[1][1] = "UVD",
169 	[2][1] = "UVDU",
170 	[3][1] = "DBGU0",
171 	[4][1] = "HDP",
172 	[5][1] = "XDP",
173 	[14][1] = "OSS",
174 	[15][1] = "SDMA0",
175 	[32+0][1] = "VCE0",
176 	[32+1][1] = "VCE0U",
177 	[32+2][1] = "XDMA",
178 	[32+3][1] = "DCE",
179 	[32+4][1] = "DCEDWB",
180 	[32+5][1] = "MP1",
181 	[32+6][1] = "DBGU1",
182 	[32+14][1] = "SDMA1",
183 };
184 
185 static const char *mmhub_client_ids_vega12[][2] = {
186 	[0][0] = "MP0",
187 	[1][0] = "VCE0",
188 	[2][0] = "VCE0U",
189 	[3][0] = "HDP",
190 	[13][0] = "UTCL2",
191 	[14][0] = "OSS",
192 	[15][0] = "SDMA1",
193 	[32+0][0] = "DCE",
194 	[32+1][0] = "XDMA",
195 	[32+2][0] = "UVD",
196 	[32+3][0] = "UVDU",
197 	[32+4][0] = "MP1",
198 	[32+15][0] = "SDMA0",
199 	[0][1] = "MP0",
200 	[1][1] = "VCE0",
201 	[2][1] = "VCE0U",
202 	[3][1] = "DBGU0",
203 	[4][1] = "HDP",
204 	[5][1] = "XDP",
205 	[14][1] = "OSS",
206 	[15][1] = "SDMA0",
207 	[32+0][1] = "DCE",
208 	[32+1][1] = "DCEDWB",
209 	[32+2][1] = "XDMA",
210 	[32+3][1] = "UVD",
211 	[32+4][1] = "UVDU",
212 	[32+5][1] = "MP1",
213 	[32+6][1] = "DBGU1",
214 	[32+15][1] = "SDMA1",
215 };
216 
217 static const char *mmhub_client_ids_vega20[][2] = {
218 	[0][0] = "XDMA",
219 	[1][0] = "DCE",
220 	[2][0] = "VCE0",
221 	[3][0] = "VCE0U",
222 	[4][0] = "UVD",
223 	[5][0] = "UVD1U",
224 	[13][0] = "OSS",
225 	[14][0] = "HDP",
226 	[15][0] = "SDMA0",
227 	[32+0][0] = "UVD",
228 	[32+1][0] = "UVDU",
229 	[32+2][0] = "MP1",
230 	[32+3][0] = "MP0",
231 	[32+12][0] = "UTCL2",
232 	[32+14][0] = "SDMA1",
233 	[0][1] = "XDMA",
234 	[1][1] = "DCE",
235 	[2][1] = "DCEDWB",
236 	[3][1] = "VCE0",
237 	[4][1] = "VCE0U",
238 	[5][1] = "UVD1",
239 	[6][1] = "UVD1U",
240 	[7][1] = "DBGU0",
241 	[8][1] = "XDP",
242 	[13][1] = "OSS",
243 	[14][1] = "HDP",
244 	[15][1] = "SDMA0",
245 	[32+0][1] = "UVD",
246 	[32+1][1] = "UVDU",
247 	[32+2][1] = "DBGU1",
248 	[32+3][1] = "MP1",
249 	[32+4][1] = "MP0",
250 	[32+14][1] = "SDMA1",
251 };
252 
253 static const char *mmhub_client_ids_arcturus[][2] = {
254 	[0][0] = "DBGU1",
255 	[1][0] = "XDP",
256 	[2][0] = "MP1",
257 	[14][0] = "HDP",
258 	[171][0] = "JPEG",
259 	[172][0] = "VCN",
260 	[173][0] = "VCNU",
261 	[203][0] = "JPEG1",
262 	[204][0] = "VCN1",
263 	[205][0] = "VCN1U",
264 	[256][0] = "SDMA0",
265 	[257][0] = "SDMA1",
266 	[258][0] = "SDMA2",
267 	[259][0] = "SDMA3",
268 	[260][0] = "SDMA4",
269 	[261][0] = "SDMA5",
270 	[262][0] = "SDMA6",
271 	[263][0] = "SDMA7",
272 	[384][0] = "OSS",
273 	[0][1] = "DBGU1",
274 	[1][1] = "XDP",
275 	[2][1] = "MP1",
276 	[14][1] = "HDP",
277 	[171][1] = "JPEG",
278 	[172][1] = "VCN",
279 	[173][1] = "VCNU",
280 	[203][1] = "JPEG1",
281 	[204][1] = "VCN1",
282 	[205][1] = "VCN1U",
283 	[256][1] = "SDMA0",
284 	[257][1] = "SDMA1",
285 	[258][1] = "SDMA2",
286 	[259][1] = "SDMA3",
287 	[260][1] = "SDMA4",
288 	[261][1] = "SDMA5",
289 	[262][1] = "SDMA6",
290 	[263][1] = "SDMA7",
291 	[384][1] = "OSS",
292 };
293 
294 static const char *mmhub_client_ids_aldebaran[][2] = {
295 	[2][0] = "MP1",
296 	[3][0] = "MP0",
297 	[32+1][0] = "DBGU_IO0",
298 	[32+2][0] = "DBGU_IO2",
299 	[32+4][0] = "MPIO",
300 	[96+11][0] = "JPEG0",
301 	[96+12][0] = "VCN0",
302 	[96+13][0] = "VCNU0",
303 	[128+11][0] = "JPEG1",
304 	[128+12][0] = "VCN1",
305 	[128+13][0] = "VCNU1",
306 	[160+1][0] = "XDP",
307 	[160+14][0] = "HDP",
308 	[256+0][0] = "SDMA0",
309 	[256+1][0] = "SDMA1",
310 	[256+2][0] = "SDMA2",
311 	[256+3][0] = "SDMA3",
312 	[256+4][0] = "SDMA4",
313 	[384+0][0] = "OSS",
314 	[2][1] = "MP1",
315 	[3][1] = "MP0",
316 	[32+1][1] = "DBGU_IO0",
317 	[32+2][1] = "DBGU_IO2",
318 	[32+4][1] = "MPIO",
319 	[96+11][1] = "JPEG0",
320 	[96+12][1] = "VCN0",
321 	[96+13][1] = "VCNU0",
322 	[128+11][1] = "JPEG1",
323 	[128+12][1] = "VCN1",
324 	[128+13][1] = "VCNU1",
325 	[160+1][1] = "XDP",
326 	[160+14][1] = "HDP",
327 	[256+0][1] = "SDMA0",
328 	[256+1][1] = "SDMA1",
329 	[256+2][1] = "SDMA2",
330 	[256+3][1] = "SDMA3",
331 	[256+4][1] = "SDMA4",
332 	[384+0][1] = "OSS",
333 };
334 
335 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = {
336 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
337 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
338 };
339 
340 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = {
341 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
342 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
343 };
344 
345 static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = {
346 	(0x000143c0 + 0x00000000),
347 	(0x000143c0 + 0x00000800),
348 	(0x000143c0 + 0x00001000),
349 	(0x000143c0 + 0x00001800),
350 	(0x000543c0 + 0x00000000),
351 	(0x000543c0 + 0x00000800),
352 	(0x000543c0 + 0x00001000),
353 	(0x000543c0 + 0x00001800),
354 	(0x000943c0 + 0x00000000),
355 	(0x000943c0 + 0x00000800),
356 	(0x000943c0 + 0x00001000),
357 	(0x000943c0 + 0x00001800),
358 	(0x000d43c0 + 0x00000000),
359 	(0x000d43c0 + 0x00000800),
360 	(0x000d43c0 + 0x00001000),
361 	(0x000d43c0 + 0x00001800),
362 	(0x001143c0 + 0x00000000),
363 	(0x001143c0 + 0x00000800),
364 	(0x001143c0 + 0x00001000),
365 	(0x001143c0 + 0x00001800),
366 	(0x001543c0 + 0x00000000),
367 	(0x001543c0 + 0x00000800),
368 	(0x001543c0 + 0x00001000),
369 	(0x001543c0 + 0x00001800),
370 	(0x001943c0 + 0x00000000),
371 	(0x001943c0 + 0x00000800),
372 	(0x001943c0 + 0x00001000),
373 	(0x001943c0 + 0x00001800),
374 	(0x001d43c0 + 0x00000000),
375 	(0x001d43c0 + 0x00000800),
376 	(0x001d43c0 + 0x00001000),
377 	(0x001d43c0 + 0x00001800),
378 };
379 
380 static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = {
381 	(0x000143e0 + 0x00000000),
382 	(0x000143e0 + 0x00000800),
383 	(0x000143e0 + 0x00001000),
384 	(0x000143e0 + 0x00001800),
385 	(0x000543e0 + 0x00000000),
386 	(0x000543e0 + 0x00000800),
387 	(0x000543e0 + 0x00001000),
388 	(0x000543e0 + 0x00001800),
389 	(0x000943e0 + 0x00000000),
390 	(0x000943e0 + 0x00000800),
391 	(0x000943e0 + 0x00001000),
392 	(0x000943e0 + 0x00001800),
393 	(0x000d43e0 + 0x00000000),
394 	(0x000d43e0 + 0x00000800),
395 	(0x000d43e0 + 0x00001000),
396 	(0x000d43e0 + 0x00001800),
397 	(0x001143e0 + 0x00000000),
398 	(0x001143e0 + 0x00000800),
399 	(0x001143e0 + 0x00001000),
400 	(0x001143e0 + 0x00001800),
401 	(0x001543e0 + 0x00000000),
402 	(0x001543e0 + 0x00000800),
403 	(0x001543e0 + 0x00001000),
404 	(0x001543e0 + 0x00001800),
405 	(0x001943e0 + 0x00000000),
406 	(0x001943e0 + 0x00000800),
407 	(0x001943e0 + 0x00001000),
408 	(0x001943e0 + 0x00001800),
409 	(0x001d43e0 + 0x00000000),
410 	(0x001d43e0 + 0x00000800),
411 	(0x001d43e0 + 0x00001000),
412 	(0x001d43e0 + 0x00001800),
413 };
414 
415 static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev,
416 		struct amdgpu_irq_src *src,
417 		unsigned int type,
418 		enum amdgpu_interrupt_state state)
419 {
420 	u32 bits, i, tmp, reg;
421 
422 	/* Devices newer then VEGA10/12 shall have these programming
423 	 * sequences performed by PSP BL
424 	 */
425 	if (adev->asic_type >= CHIP_VEGA20)
426 		return 0;
427 
428 	bits = 0x7f;
429 
430 	switch (state) {
431 	case AMDGPU_IRQ_STATE_DISABLE:
432 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
433 			reg = ecc_umc_mcumc_ctrl_addrs[i];
434 			tmp = RREG32(reg);
435 			tmp &= ~bits;
436 			WREG32(reg, tmp);
437 		}
438 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
439 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
440 			tmp = RREG32(reg);
441 			tmp &= ~bits;
442 			WREG32(reg, tmp);
443 		}
444 		break;
445 	case AMDGPU_IRQ_STATE_ENABLE:
446 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) {
447 			reg = ecc_umc_mcumc_ctrl_addrs[i];
448 			tmp = RREG32(reg);
449 			tmp |= bits;
450 			WREG32(reg, tmp);
451 		}
452 		for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) {
453 			reg = ecc_umc_mcumc_ctrl_mask_addrs[i];
454 			tmp = RREG32(reg);
455 			tmp |= bits;
456 			WREG32(reg, tmp);
457 		}
458 		break;
459 	default:
460 		break;
461 	}
462 
463 	return 0;
464 }
465 
466 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
467 					struct amdgpu_irq_src *src,
468 					unsigned int type,
469 					enum amdgpu_interrupt_state state)
470 {
471 	struct amdgpu_vmhub *hub;
472 	u32 tmp, reg, bits, i, j;
473 
474 	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
475 		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
476 		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
477 		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
478 		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
479 		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
480 		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
481 
482 	switch (state) {
483 	case AMDGPU_IRQ_STATE_DISABLE:
484 		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
485 			hub = &adev->vmhub[j];
486 			for (i = 0; i < 16; i++) {
487 				reg = hub->vm_context0_cntl + i;
488 
489 				/* This works because this interrupt is only
490 				 * enabled at init/resume and disabled in
491 				 * fini/suspend, so the overall state doesn't
492 				 * change over the course of suspend/resume.
493 				 */
494 				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
495 					continue;
496 
497 				if (j >= AMDGPU_MMHUB0(0))
498 					tmp = RREG32_SOC15_IP(MMHUB, reg);
499 				else
500 					tmp = RREG32_SOC15_IP(GC, reg);
501 
502 				tmp &= ~bits;
503 
504 				if (j >= AMDGPU_MMHUB0(0))
505 					WREG32_SOC15_IP(MMHUB, reg, tmp);
506 				else
507 					WREG32_SOC15_IP(GC, reg, tmp);
508 			}
509 		}
510 		break;
511 	case AMDGPU_IRQ_STATE_ENABLE:
512 		for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
513 			hub = &adev->vmhub[j];
514 			for (i = 0; i < 16; i++) {
515 				reg = hub->vm_context0_cntl + i;
516 
517 				/* This works because this interrupt is only
518 				 * enabled at init/resume and disabled in
519 				 * fini/suspend, so the overall state doesn't
520 				 * change over the course of suspend/resume.
521 				 */
522 				if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0)))
523 					continue;
524 
525 				if (j >= AMDGPU_MMHUB0(0))
526 					tmp = RREG32_SOC15_IP(MMHUB, reg);
527 				else
528 					tmp = RREG32_SOC15_IP(GC, reg);
529 
530 				tmp |= bits;
531 
532 				if (j >= AMDGPU_MMHUB0(0))
533 					WREG32_SOC15_IP(MMHUB, reg, tmp);
534 				else
535 					WREG32_SOC15_IP(GC, reg, tmp);
536 			}
537 		}
538 		break;
539 	default:
540 		break;
541 	}
542 
543 	return 0;
544 }
545 
546 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
547 				      struct amdgpu_irq_src *source,
548 				      struct amdgpu_iv_entry *entry)
549 {
550 	bool retry_fault = !!(entry->src_data[1] & 0x80);
551 	bool write_fault = !!(entry->src_data[1] & 0x20);
552 	uint32_t status = 0, cid = 0, rw = 0;
553 	struct amdgpu_task_info task_info;
554 	struct amdgpu_vmhub *hub;
555 	const char *mmhub_cid;
556 	const char *hub_name;
557 	u64 addr;
558 	uint32_t cam_index = 0;
559 	int ret, xcc_id = 0;
560 	uint32_t node_id;
561 
562 	node_id = entry->node_id;
563 
564 	addr = (u64)entry->src_data[0] << 12;
565 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
566 
567 	if (entry->client_id == SOC15_IH_CLIENTID_VMC) {
568 		hub_name = "mmhub0";
569 		hub = &adev->vmhub[AMDGPU_MMHUB0(node_id / 4)];
570 	} else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) {
571 		hub_name = "mmhub1";
572 		hub = &adev->vmhub[AMDGPU_MMHUB1(0)];
573 	} else {
574 		hub_name = "gfxhub0";
575 		if (adev->gfx.funcs->ih_node_to_logical_xcc) {
576 			xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev,
577 				node_id);
578 			if (xcc_id < 0)
579 				xcc_id = 0;
580 		}
581 		hub = &adev->vmhub[xcc_id];
582 	}
583 
584 	if (retry_fault) {
585 		if (adev->irq.retry_cam_enabled) {
586 			/* Delegate it to a different ring if the hardware hasn't
587 			 * already done it.
588 			 */
589 			if (entry->ih == &adev->irq.ih) {
590 				amdgpu_irq_delegate(adev, entry, 8);
591 				return 1;
592 			}
593 
594 			cam_index = entry->src_data[2] & 0x3ff;
595 
596 			ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
597 						     addr, write_fault);
598 			WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index);
599 			if (ret)
600 				return 1;
601 		} else {
602 			/* Process it onyl if it's the first fault for this address */
603 			if (entry->ih != &adev->irq.ih_soft &&
604 			    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
605 					     entry->timestamp))
606 				return 1;
607 
608 			/* Delegate it to a different ring if the hardware hasn't
609 			 * already done it.
610 			 */
611 			if (entry->ih == &adev->irq.ih) {
612 				amdgpu_irq_delegate(adev, entry, 8);
613 				return 1;
614 			}
615 
616 			/* Try to handle the recoverable page faults by filling page
617 			 * tables
618 			 */
619 			if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id,
620 						   addr, write_fault))
621 				return 1;
622 		}
623 	}
624 
625 	if (!printk_ratelimit())
626 		return 0;
627 
628 
629 	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
630 	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
631 
632 	dev_err(adev->dev,
633 		"[%s] %s page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n",
634 		hub_name, retry_fault ? "retry" : "no-retry",
635 		entry->src_id, entry->ring_id, entry->vmid,
636 		entry->pasid, task_info.process_name, task_info.tgid,
637 		task_info.task_name, task_info.pid);
638 	dev_err(adev->dev, "  in page starting at address 0x%016llx from IH client 0x%x (%s)\n",
639 		addr, entry->client_id,
640 		soc15_ih_clientid_name[entry->client_id]);
641 
642 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
643 		dev_err(adev->dev, "  cookie node_id %d fault from die %s%d%s\n",
644 			node_id, node_id % 4 == 3 ? "RSV" : "AID", node_id / 4,
645 			node_id % 4 == 1 ? ".XCD0" : node_id % 4 == 2 ? ".XCD1" : "");
646 
647 	if (amdgpu_sriov_vf(adev))
648 		return 0;
649 
650 	/*
651 	 * Issue a dummy read to wait for the status register to
652 	 * be updated to avoid reading an incorrect value due to
653 	 * the new fast GRBM interface.
654 	 */
655 	if ((entry->vmid_src == AMDGPU_GFXHUB(0)) &&
656 	    (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2)))
657 		RREG32(hub->vm_l2_pro_fault_status);
658 
659 	status = RREG32(hub->vm_l2_pro_fault_status);
660 	cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID);
661 	rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW);
662 	WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
663 
664 	dev_err(adev->dev,
665 		"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
666 		status);
667 	if (entry->vmid_src == AMDGPU_GFXHUB(0)) {
668 		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
669 			cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" :
670 			gfxhub_client_ids[cid],
671 			cid);
672 	} else {
673 		switch (adev->ip_versions[MMHUB_HWIP][0]) {
674 		case IP_VERSION(9, 0, 0):
675 			mmhub_cid = mmhub_client_ids_vega10[cid][rw];
676 			break;
677 		case IP_VERSION(9, 3, 0):
678 			mmhub_cid = mmhub_client_ids_vega12[cid][rw];
679 			break;
680 		case IP_VERSION(9, 4, 0):
681 			mmhub_cid = mmhub_client_ids_vega20[cid][rw];
682 			break;
683 		case IP_VERSION(9, 4, 1):
684 			mmhub_cid = mmhub_client_ids_arcturus[cid][rw];
685 			break;
686 		case IP_VERSION(9, 1, 0):
687 		case IP_VERSION(9, 2, 0):
688 			mmhub_cid = mmhub_client_ids_raven[cid][rw];
689 			break;
690 		case IP_VERSION(1, 5, 0):
691 		case IP_VERSION(2, 4, 0):
692 			mmhub_cid = mmhub_client_ids_renoir[cid][rw];
693 			break;
694 		case IP_VERSION(1, 8, 0):
695 		case IP_VERSION(9, 4, 2):
696 			mmhub_cid = mmhub_client_ids_aldebaran[cid][rw];
697 			break;
698 		default:
699 			mmhub_cid = NULL;
700 			break;
701 		}
702 		dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
703 			mmhub_cid ? mmhub_cid : "unknown", cid);
704 	}
705 	dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
706 		REG_GET_FIELD(status,
707 		VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
708 	dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
709 		REG_GET_FIELD(status,
710 		VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
711 	dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
712 		REG_GET_FIELD(status,
713 		VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
714 	dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
715 		REG_GET_FIELD(status,
716 		VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
717 	dev_err(adev->dev, "\t RW: 0x%x\n", rw);
718 	return 0;
719 }
720 
721 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
722 	.set = gmc_v9_0_vm_fault_interrupt_state,
723 	.process = gmc_v9_0_process_interrupt,
724 };
725 
726 
727 static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = {
728 	.set = gmc_v9_0_ecc_interrupt_state,
729 	.process = amdgpu_umc_process_ecc_irq,
730 };
731 
732 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
733 {
734 	adev->gmc.vm_fault.num_types = 1;
735 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
736 
737 	if (!amdgpu_sriov_vf(adev) &&
738 	    !adev->gmc.xgmi.connected_to_cpu) {
739 		adev->gmc.ecc_irq.num_types = 1;
740 		adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs;
741 	}
742 }
743 
744 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid,
745 					uint32_t flush_type)
746 {
747 	u32 req = 0;
748 
749 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
750 			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
751 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
752 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
753 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
754 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
755 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
756 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
757 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
758 			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
759 
760 	return req;
761 }
762 
763 /**
764  * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore
765  *
766  * @adev: amdgpu_device pointer
767  * @vmhub: vmhub type
768  *
769  */
770 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev,
771 				       uint32_t vmhub)
772 {
773 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
774 	    adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
775 		return false;
776 
777 	return ((vmhub == AMDGPU_MMHUB0(0) ||
778 		 vmhub == AMDGPU_MMHUB1(0)) &&
779 		(!amdgpu_sriov_vf(adev)) &&
780 		(!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) &&
781 		   (adev->apu_flags & AMD_APU_IS_PICASSO))));
782 }
783 
784 static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev,
785 					uint8_t vmid, uint16_t *p_pasid)
786 {
787 	uint32_t value;
788 
789 	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
790 		     + vmid);
791 	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
792 
793 	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
794 }
795 
796 /*
797  * GART
798  * VMID 0 is the physical GPU addresses as used by the kernel.
799  * VMIDs 1-15 are used for userspace clients and are handled
800  * by the amdgpu vm/hsa code.
801  */
802 
803 /**
804  * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type
805  *
806  * @adev: amdgpu_device pointer
807  * @vmid: vm instance to flush
808  * @vmhub: which hub to flush
809  * @flush_type: the flush type
810  *
811  * Flush the TLB for the requested page table using certain type.
812  */
813 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
814 					uint32_t vmhub, uint32_t flush_type)
815 {
816 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub);
817 	const unsigned int eng = 17;
818 	u32 j, inv_req, inv_req2, tmp;
819 	struct amdgpu_vmhub *hub;
820 
821 	BUG_ON(vmhub >= AMDGPU_MAX_VMHUBS);
822 
823 	hub = &adev->vmhub[vmhub];
824 	if (adev->gmc.xgmi.num_physical_nodes &&
825 	    adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0)) {
826 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
827 		 * heavy-weight TLB flush (type 2), which flushes
828 		 * both. Due to a race condition with concurrent
829 		 * memory accesses using the same TLB cache line, we
830 		 * still need a second TLB flush after this.
831 		 */
832 		inv_req = gmc_v9_0_get_invalidate_req(vmid, 2);
833 		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
834 	} else if (flush_type == 2 &&
835 		   adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3) &&
836 		   adev->rev_id == 0) {
837 		inv_req = gmc_v9_0_get_invalidate_req(vmid, 0);
838 		inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type);
839 	} else {
840 		inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type);
841 		inv_req2 = 0;
842 	}
843 
844 	/* This is necessary for a HW workaround under SRIOV as well
845 	 * as GFXOFF under bare metal
846 	 */
847 	if (adev->gfx.kiq[0].ring.sched.ready &&
848 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
849 	    down_read_trylock(&adev->reset_domain->sem)) {
850 		uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
851 		uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
852 
853 		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
854 						   1 << vmid);
855 		up_read(&adev->reset_domain->sem);
856 		return;
857 	}
858 
859 	spin_lock(&adev->gmc.invalidate_lock);
860 
861 	/*
862 	 * It may lose gpuvm invalidate acknowldege state across power-gating
863 	 * off cycle, add semaphore acquire before invalidation and semaphore
864 	 * release after invalidation to avoid entering power gated state
865 	 * to WA the Issue
866 	 */
867 
868 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
869 	if (use_semaphore) {
870 		for (j = 0; j < adev->usec_timeout; j++) {
871 			/* a read return value of 1 means semaphore acquire */
872 			if (vmhub >= AMDGPU_MMHUB0(0))
873 				tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng);
874 			else
875 				tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng);
876 			if (tmp & 0x1)
877 				break;
878 			udelay(1);
879 		}
880 
881 		if (j >= adev->usec_timeout)
882 			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
883 	}
884 
885 	do {
886 		if (vmhub >= AMDGPU_MMHUB0(0))
887 			WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req);
888 		else
889 			WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req);
890 
891 		/*
892 		 * Issue a dummy read to wait for the ACK register to
893 		 * be cleared to avoid a false ACK due to the new fast
894 		 * GRBM interface.
895 		 */
896 		if ((vmhub == AMDGPU_GFXHUB(0)) &&
897 		    (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2)))
898 			RREG32_NO_KIQ(hub->vm_inv_eng0_req +
899 				      hub->eng_distance * eng);
900 
901 		for (j = 0; j < adev->usec_timeout; j++) {
902 			if (vmhub >= AMDGPU_MMHUB0(0))
903 				tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_ack + hub->eng_distance * eng);
904 			else
905 				tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_ack + hub->eng_distance * eng);
906 			if (tmp & (1 << vmid))
907 				break;
908 			udelay(1);
909 		}
910 
911 		inv_req = inv_req2;
912 		inv_req2 = 0;
913 	} while (inv_req);
914 
915 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
916 	if (use_semaphore) {
917 		/*
918 		 * add semaphore release after invalidation,
919 		 * write with 0 means semaphore release
920 		 */
921 		if (vmhub >= AMDGPU_MMHUB0(0))
922 			WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0);
923 		else
924 			WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0);
925 	}
926 
927 	spin_unlock(&adev->gmc.invalidate_lock);
928 
929 	if (j < adev->usec_timeout)
930 		return;
931 
932 	DRM_ERROR("Timeout waiting for VM flush ACK!\n");
933 }
934 
935 /**
936  * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid
937  *
938  * @adev: amdgpu_device pointer
939  * @pasid: pasid to be flush
940  * @flush_type: the flush type
941  * @all_hub: flush all hubs
942  * @inst: is used to select which instance of KIQ to use for the invalidation
943  *
944  * Flush the TLB for the requested pasid.
945  */
946 static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
947 					uint16_t pasid, uint32_t flush_type,
948 					bool all_hub, uint32_t inst)
949 {
950 	int vmid, i;
951 	signed long r;
952 	uint32_t seq;
953 	uint16_t queried_pasid;
954 	bool ret;
955 	u32 usec_timeout = amdgpu_sriov_vf(adev) ? SRIOV_USEC_TIMEOUT : adev->usec_timeout;
956 	struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring;
957 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst];
958 
959 	if (amdgpu_in_reset(adev))
960 		return -EIO;
961 
962 	if (ring->sched.ready && down_read_trylock(&adev->reset_domain->sem)) {
963 		/* Vega20+XGMI caches PTEs in TC and TLB. Add a
964 		 * heavy-weight TLB flush (type 2), which flushes
965 		 * both. Due to a race condition with concurrent
966 		 * memory accesses using the same TLB cache line, we
967 		 * still need a second TLB flush after this.
968 		 */
969 		bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes &&
970 				       adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0));
971 		/* 2 dwords flush + 8 dwords fence */
972 		unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8;
973 
974 		if (vega20_xgmi_wa)
975 			ndw += kiq->pmf->invalidate_tlbs_size;
976 
977 		spin_lock(&adev->gfx.kiq[inst].ring_lock);
978 		/* 2 dwords flush + 8 dwords fence */
979 		amdgpu_ring_alloc(ring, ndw);
980 		if (vega20_xgmi_wa)
981 			kiq->pmf->kiq_invalidate_tlbs(ring,
982 						      pasid, 2, all_hub);
983 
984 		if (flush_type == 2 &&
985 		    adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3) &&
986 		    adev->rev_id == 0)
987 			kiq->pmf->kiq_invalidate_tlbs(ring,
988 						pasid, 0, all_hub);
989 
990 		kiq->pmf->kiq_invalidate_tlbs(ring,
991 					pasid, flush_type, all_hub);
992 		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
993 		if (r) {
994 			amdgpu_ring_undo(ring);
995 			spin_unlock(&adev->gfx.kiq[inst].ring_lock);
996 			up_read(&adev->reset_domain->sem);
997 			return -ETIME;
998 		}
999 
1000 		amdgpu_ring_commit(ring);
1001 		spin_unlock(&adev->gfx.kiq[inst].ring_lock);
1002 		r = amdgpu_fence_wait_polling(ring, seq, usec_timeout);
1003 		if (r < 1) {
1004 			dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
1005 			up_read(&adev->reset_domain->sem);
1006 			return -ETIME;
1007 		}
1008 		up_read(&adev->reset_domain->sem);
1009 		return 0;
1010 	}
1011 
1012 	for (vmid = 1; vmid < 16; vmid++) {
1013 
1014 		ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
1015 				&queried_pasid);
1016 		if (ret && queried_pasid == pasid) {
1017 			if (all_hub) {
1018 				for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
1019 					gmc_v9_0_flush_gpu_tlb(adev, vmid,
1020 							i, flush_type);
1021 			} else {
1022 				gmc_v9_0_flush_gpu_tlb(adev, vmid,
1023 						AMDGPU_GFXHUB(0), flush_type);
1024 			}
1025 			break;
1026 		}
1027 	}
1028 
1029 	return 0;
1030 
1031 }
1032 
1033 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
1034 					    unsigned int vmid, uint64_t pd_addr)
1035 {
1036 	bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
1037 	struct amdgpu_device *adev = ring->adev;
1038 	struct amdgpu_vmhub *hub = &adev->vmhub[ring->vm_hub];
1039 	uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0);
1040 	unsigned int eng = ring->vm_inv_eng;
1041 
1042 	/*
1043 	 * It may lose gpuvm invalidate acknowldege state across power-gating
1044 	 * off cycle, add semaphore acquire before invalidation and semaphore
1045 	 * release after invalidation to avoid entering power gated state
1046 	 * to WA the Issue
1047 	 */
1048 
1049 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
1050 	if (use_semaphore)
1051 		/* a read return value of 1 means semaphore acuqire */
1052 		amdgpu_ring_emit_reg_wait(ring,
1053 					  hub->vm_inv_eng0_sem +
1054 					  hub->eng_distance * eng, 0x1, 0x1);
1055 
1056 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
1057 			      (hub->ctx_addr_distance * vmid),
1058 			      lower_32_bits(pd_addr));
1059 
1060 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
1061 			      (hub->ctx_addr_distance * vmid),
1062 			      upper_32_bits(pd_addr));
1063 
1064 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
1065 					    hub->eng_distance * eng,
1066 					    hub->vm_inv_eng0_ack +
1067 					    hub->eng_distance * eng,
1068 					    req, 1 << vmid);
1069 
1070 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
1071 	if (use_semaphore)
1072 		/*
1073 		 * add semaphore release after invalidation,
1074 		 * write with 0 means semaphore release
1075 		 */
1076 		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
1077 				      hub->eng_distance * eng, 0);
1078 
1079 	return pd_addr;
1080 }
1081 
1082 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
1083 					unsigned int pasid)
1084 {
1085 	struct amdgpu_device *adev = ring->adev;
1086 	uint32_t reg;
1087 
1088 	/* Do nothing because there's no lut register for mmhub1. */
1089 	if (ring->vm_hub == AMDGPU_MMHUB1(0))
1090 		return;
1091 
1092 	if (ring->vm_hub == AMDGPU_GFXHUB(0))
1093 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
1094 	else
1095 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
1096 
1097 	amdgpu_ring_emit_wreg(ring, reg, pasid);
1098 }
1099 
1100 /*
1101  * PTE format on VEGA 10:
1102  * 63:59 reserved
1103  * 58:57 mtype
1104  * 56 F
1105  * 55 L
1106  * 54 P
1107  * 53 SW
1108  * 52 T
1109  * 50:48 reserved
1110  * 47:12 4k physical page base address
1111  * 11:7 fragment
1112  * 6 write
1113  * 5 read
1114  * 4 exe
1115  * 3 Z
1116  * 2 snooped
1117  * 1 system
1118  * 0 valid
1119  *
1120  * PDE format on VEGA 10:
1121  * 63:59 block fragment size
1122  * 58:55 reserved
1123  * 54 P
1124  * 53:48 reserved
1125  * 47:6 physical base address of PD or PTE
1126  * 5:3 reserved
1127  * 2 C
1128  * 1 system
1129  * 0 valid
1130  */
1131 
1132 static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
1133 
1134 {
1135 	switch (flags) {
1136 	case AMDGPU_VM_MTYPE_DEFAULT:
1137 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1138 	case AMDGPU_VM_MTYPE_NC:
1139 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1140 	case AMDGPU_VM_MTYPE_WC:
1141 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC);
1142 	case AMDGPU_VM_MTYPE_RW:
1143 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW);
1144 	case AMDGPU_VM_MTYPE_CC:
1145 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC);
1146 	case AMDGPU_VM_MTYPE_UC:
1147 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC);
1148 	default:
1149 		return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC);
1150 	}
1151 }
1152 
1153 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
1154 				uint64_t *addr, uint64_t *flags)
1155 {
1156 	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
1157 		*addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
1158 	BUG_ON(*addr & 0xFFFF00000000003FULL);
1159 
1160 	if (!adev->gmc.translate_further)
1161 		return;
1162 
1163 	if (level == AMDGPU_VM_PDB1) {
1164 		/* Set the block fragment size */
1165 		if (!(*flags & AMDGPU_PDE_PTE))
1166 			*flags |= AMDGPU_PDE_BFS(0x9);
1167 
1168 	} else if (level == AMDGPU_VM_PDB0) {
1169 		if (*flags & AMDGPU_PDE_PTE) {
1170 			*flags &= ~AMDGPU_PDE_PTE;
1171 			if (!(*flags & AMDGPU_PTE_VALID))
1172 				*addr |= 1 << PAGE_SHIFT;
1173 		} else {
1174 			*flags |= AMDGPU_PTE_TF;
1175 		}
1176 	}
1177 }
1178 
1179 static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev,
1180 					 struct amdgpu_bo *bo,
1181 					 struct amdgpu_bo_va_mapping *mapping,
1182 					 uint64_t *flags)
1183 {
1184 	struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev);
1185 	bool is_vram = bo->tbo.resource->mem_type == TTM_PL_VRAM;
1186 	bool coherent = bo->flags & AMDGPU_GEM_CREATE_COHERENT;
1187 	bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED;
1188 	struct amdgpu_vm *vm = mapping->bo_va->base.vm;
1189 	unsigned int mtype_local, mtype;
1190 	bool snoop = false;
1191 	bool is_local;
1192 
1193 	switch (adev->ip_versions[GC_HWIP][0]) {
1194 	case IP_VERSION(9, 4, 1):
1195 	case IP_VERSION(9, 4, 2):
1196 		if (is_vram) {
1197 			if (bo_adev == adev) {
1198 				if (uncached)
1199 					mtype = MTYPE_UC;
1200 				else if (coherent)
1201 					mtype = MTYPE_CC;
1202 				else
1203 					mtype = MTYPE_RW;
1204 				/* FIXME: is this still needed? Or does
1205 				 * amdgpu_ttm_tt_pde_flags already handle this?
1206 				 */
1207 				if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
1208 				     adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) &&
1209 				    adev->gmc.xgmi.connected_to_cpu)
1210 					snoop = true;
1211 			} else {
1212 				if (uncached || coherent)
1213 					mtype = MTYPE_UC;
1214 				else
1215 					mtype = MTYPE_NC;
1216 				if (mapping->bo_va->is_xgmi)
1217 					snoop = true;
1218 			}
1219 		} else {
1220 			if (uncached || coherent)
1221 				mtype = MTYPE_UC;
1222 			else
1223 				mtype = MTYPE_NC;
1224 			/* FIXME: is this still needed? Or does
1225 			 * amdgpu_ttm_tt_pde_flags already handle this?
1226 			 */
1227 			snoop = true;
1228 		}
1229 		break;
1230 	case IP_VERSION(9, 4, 3):
1231 		/* Only local VRAM BOs or system memory on non-NUMA APUs
1232 		 * can be assumed to be local in their entirety. Choose
1233 		 * MTYPE_NC as safe fallback for all system memory BOs on
1234 		 * NUMA systems. Their MTYPE can be overridden per-page in
1235 		 * gmc_v9_0_override_vm_pte_flags.
1236 		 */
1237 		mtype_local = MTYPE_RW;
1238 		if (amdgpu_mtype_local == 1) {
1239 			DRM_INFO_ONCE("Using MTYPE_NC for local memory\n");
1240 			mtype_local = MTYPE_NC;
1241 		} else if (amdgpu_mtype_local == 2) {
1242 			DRM_INFO_ONCE("Using MTYPE_CC for local memory\n");
1243 			mtype_local = MTYPE_CC;
1244 		} else {
1245 			DRM_INFO_ONCE("Using MTYPE_RW for local memory\n");
1246 		}
1247 		is_local = (!is_vram && (adev->flags & AMD_IS_APU) &&
1248 			    num_possible_nodes() <= 1) ||
1249 			   (is_vram && adev == bo_adev &&
1250 			    KFD_XCP_MEM_ID(adev, bo->xcp_id) == vm->mem_id);
1251 		snoop = true;
1252 		if (uncached) {
1253 			mtype = MTYPE_UC;
1254 		} else if (adev->flags & AMD_IS_APU) {
1255 			mtype = is_local ? mtype_local : MTYPE_NC;
1256 		} else {
1257 			/* dGPU */
1258 			if (is_local)
1259 				mtype = mtype_local;
1260 			else if (is_vram)
1261 				mtype = MTYPE_NC;
1262 			else
1263 				mtype = MTYPE_UC;
1264 		}
1265 
1266 		break;
1267 	default:
1268 		if (uncached || coherent)
1269 			mtype = MTYPE_UC;
1270 		else
1271 			mtype = MTYPE_NC;
1272 
1273 		/* FIXME: is this still needed? Or does
1274 		 * amdgpu_ttm_tt_pde_flags already handle this?
1275 		 */
1276 		if (!is_vram)
1277 			snoop = true;
1278 	}
1279 
1280 	if (mtype != MTYPE_NC)
1281 		*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1282 			 AMDGPU_PTE_MTYPE_VG10(mtype);
1283 	*flags |= snoop ? AMDGPU_PTE_SNOOPED : 0;
1284 }
1285 
1286 static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev,
1287 				struct amdgpu_bo_va_mapping *mapping,
1288 				uint64_t *flags)
1289 {
1290 	struct amdgpu_bo *bo = mapping->bo_va->base.bo;
1291 
1292 	*flags &= ~AMDGPU_PTE_EXECUTABLE;
1293 	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
1294 
1295 	*flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK;
1296 	*flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK;
1297 
1298 	if (mapping->flags & AMDGPU_PTE_PRT) {
1299 		*flags |= AMDGPU_PTE_PRT;
1300 		*flags &= ~AMDGPU_PTE_VALID;
1301 	}
1302 
1303 	if (bo && bo->tbo.resource)
1304 		gmc_v9_0_get_coherence_flags(adev, mapping->bo_va->base.bo,
1305 					     mapping, flags);
1306 }
1307 
1308 static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
1309 					   struct amdgpu_vm *vm,
1310 					   uint64_t addr, uint64_t *flags)
1311 {
1312 	int local_node, nid;
1313 
1314 	/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system
1315 	 * memory can use more efficient MTYPEs.
1316 	 */
1317 	if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 3))
1318 		return;
1319 
1320 	/* Only direct-mapped memory allows us to determine the NUMA node from
1321 	 * the DMA address.
1322 	 */
1323 	if (!adev->ram_is_direct_mapped) {
1324 		dev_dbg(adev->dev, "RAM is not direct mapped\n");
1325 		return;
1326 	}
1327 
1328 	/* Only override mappings with MTYPE_NC, which is the safe default for
1329 	 * cacheable memory.
1330 	 */
1331 	if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) !=
1332 	    AMDGPU_PTE_MTYPE_VG10(MTYPE_NC)) {
1333 		dev_dbg(adev->dev, "MTYPE is not NC\n");
1334 		return;
1335 	}
1336 
1337 	/* FIXME: Only supported on native mode for now. For carve-out, the
1338 	 * NUMA affinity of the GPU/VM needs to come from the PCI info because
1339 	 * memory partitions are not associated with different NUMA nodes.
1340 	 */
1341 	if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
1342 		local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
1343 	} else {
1344 		dev_dbg(adev->dev, "Only native mode APU is supported.\n");
1345 		return;
1346 	}
1347 
1348 	/* Only handle real RAM. Mappings of PCIe resources don't have struct
1349 	 * page or NUMA nodes.
1350 	 */
1351 	if (!page_is_ram(addr >> PAGE_SHIFT)) {
1352 		dev_dbg(adev->dev, "Page is not RAM.\n");
1353 		return;
1354 	}
1355 	nid = pfn_to_nid(addr >> PAGE_SHIFT);
1356 	dev_dbg(adev->dev, "vm->mem_id=%d, local_node=%d, nid=%d\n",
1357 		vm->mem_id, local_node, nid);
1358 	if (nid == local_node) {
1359 		uint64_t old_flags = *flags;
1360 		unsigned int mtype_local = MTYPE_RW;
1361 
1362 		if (amdgpu_mtype_local == 1)
1363 			mtype_local = MTYPE_NC;
1364 		else if (amdgpu_mtype_local == 2)
1365 			mtype_local = MTYPE_CC;
1366 
1367 		*flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) |
1368 			 AMDGPU_PTE_MTYPE_VG10(mtype_local);
1369 		dev_dbg(adev->dev, "flags updated from %llx to %llx\n",
1370 			old_flags, *flags);
1371 	}
1372 }
1373 
1374 static unsigned int gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
1375 {
1376 	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
1377 	unsigned int size;
1378 
1379 	/* TODO move to DC so GMC doesn't need to hard-code DCN registers */
1380 
1381 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
1382 		size = AMDGPU_VBIOS_VGA_ALLOCATION;
1383 	} else {
1384 		u32 viewport;
1385 
1386 		switch (adev->ip_versions[DCE_HWIP][0]) {
1387 		case IP_VERSION(1, 0, 0):
1388 		case IP_VERSION(1, 0, 1):
1389 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
1390 			size = (REG_GET_FIELD(viewport,
1391 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1392 				REG_GET_FIELD(viewport,
1393 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1394 				4);
1395 			break;
1396 		case IP_VERSION(2, 1, 0):
1397 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2);
1398 			size = (REG_GET_FIELD(viewport,
1399 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
1400 				REG_GET_FIELD(viewport,
1401 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
1402 				4);
1403 			break;
1404 		default:
1405 			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
1406 			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
1407 				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
1408 				4);
1409 			break;
1410 		}
1411 	}
1412 
1413 	return size;
1414 }
1415 
1416 static enum amdgpu_memory_partition
1417 gmc_v9_0_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes)
1418 {
1419 	enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE;
1420 
1421 	if (adev->nbio.funcs->get_memory_partition_mode)
1422 		mode = adev->nbio.funcs->get_memory_partition_mode(adev,
1423 								   supp_modes);
1424 
1425 	return mode;
1426 }
1427 
1428 static enum amdgpu_memory_partition
1429 gmc_v9_0_query_memory_partition(struct amdgpu_device *adev)
1430 {
1431 	if (amdgpu_sriov_vf(adev))
1432 		return AMDGPU_NPS1_PARTITION_MODE;
1433 
1434 	return gmc_v9_0_get_memory_partition(adev, NULL);
1435 }
1436 
1437 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
1438 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
1439 	.flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid,
1440 	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
1441 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
1442 	.map_mtype = gmc_v9_0_map_mtype,
1443 	.get_vm_pde = gmc_v9_0_get_vm_pde,
1444 	.get_vm_pte = gmc_v9_0_get_vm_pte,
1445 	.override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags,
1446 	.get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size,
1447 	.query_mem_partition_mode = &gmc_v9_0_query_memory_partition,
1448 };
1449 
1450 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
1451 {
1452 	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
1453 }
1454 
1455 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
1456 {
1457 	switch (adev->ip_versions[UMC_HWIP][0]) {
1458 	case IP_VERSION(6, 0, 0):
1459 		adev->umc.funcs = &umc_v6_0_funcs;
1460 		break;
1461 	case IP_VERSION(6, 1, 1):
1462 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1463 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1464 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1465 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20;
1466 		adev->umc.retire_unit = 1;
1467 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1468 		adev->umc.ras = &umc_v6_1_ras;
1469 		break;
1470 	case IP_VERSION(6, 1, 2):
1471 		adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM;
1472 		adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM;
1473 		adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM;
1474 		adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT;
1475 		adev->umc.retire_unit = 1;
1476 		adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0];
1477 		adev->umc.ras = &umc_v6_1_ras;
1478 		break;
1479 	case IP_VERSION(6, 7, 0):
1480 		adev->umc.max_ras_err_cnt_per_query =
1481 			UMC_V6_7_TOTAL_CHANNEL_NUM * UMC_V6_7_BAD_PAGE_NUM_PER_CHANNEL;
1482 		adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM;
1483 		adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM;
1484 		adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET;
1485 		adev->umc.retire_unit = (UMC_V6_7_NA_MAP_PA_NUM * 2);
1486 		if (!adev->gmc.xgmi.connected_to_cpu)
1487 			adev->umc.ras = &umc_v6_7_ras;
1488 		if (1 & adev->smuio.funcs->get_die_id(adev))
1489 			adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0];
1490 		else
1491 			adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0];
1492 		break;
1493 	default:
1494 		break;
1495 	}
1496 }
1497 
1498 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev)
1499 {
1500 	switch (adev->ip_versions[MMHUB_HWIP][0]) {
1501 	case IP_VERSION(9, 4, 1):
1502 		adev->mmhub.funcs = &mmhub_v9_4_funcs;
1503 		break;
1504 	case IP_VERSION(9, 4, 2):
1505 		adev->mmhub.funcs = &mmhub_v1_7_funcs;
1506 		break;
1507 	case IP_VERSION(1, 8, 0):
1508 		adev->mmhub.funcs = &mmhub_v1_8_funcs;
1509 		break;
1510 	default:
1511 		adev->mmhub.funcs = &mmhub_v1_0_funcs;
1512 		break;
1513 	}
1514 }
1515 
1516 static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev)
1517 {
1518 	switch (adev->ip_versions[MMHUB_HWIP][0]) {
1519 	case IP_VERSION(9, 4, 0):
1520 		adev->mmhub.ras = &mmhub_v1_0_ras;
1521 		break;
1522 	case IP_VERSION(9, 4, 1):
1523 		adev->mmhub.ras = &mmhub_v9_4_ras;
1524 		break;
1525 	case IP_VERSION(9, 4, 2):
1526 		adev->mmhub.ras = &mmhub_v1_7_ras;
1527 		break;
1528 	case IP_VERSION(1, 8, 0):
1529 		adev->mmhub.ras = &mmhub_v1_8_ras;
1530 		break;
1531 	default:
1532 		/* mmhub ras is not available */
1533 		break;
1534 	}
1535 }
1536 
1537 static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev)
1538 {
1539 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
1540 		adev->gfxhub.funcs = &gfxhub_v1_2_funcs;
1541 	else
1542 		adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1543 }
1544 
1545 static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
1546 {
1547 	adev->hdp.ras = &hdp_v4_0_ras;
1548 }
1549 
1550 static void gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device *adev)
1551 {
1552 	struct amdgpu_mca *mca = &adev->mca;
1553 
1554 	/* is UMC the right IP to check for MCA?  Maybe DF? */
1555 	switch (adev->ip_versions[UMC_HWIP][0]) {
1556 	case IP_VERSION(6, 7, 0):
1557 		if (!adev->gmc.xgmi.connected_to_cpu) {
1558 			mca->mp0.ras = &mca_v3_0_mp0_ras;
1559 			mca->mp1.ras = &mca_v3_0_mp1_ras;
1560 			mca->mpio.ras = &mca_v3_0_mpio_ras;
1561 		}
1562 		break;
1563 	default:
1564 		break;
1565 	}
1566 }
1567 
1568 static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev)
1569 {
1570 	if (!adev->gmc.xgmi.connected_to_cpu)
1571 		adev->gmc.xgmi.ras = &xgmi_ras;
1572 }
1573 
1574 static int gmc_v9_0_early_init(void *handle)
1575 {
1576 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1577 
1578 	/*
1579 	 * 9.4.0, 9.4.1 and 9.4.3 don't have XGMI defined
1580 	 * in their IP discovery tables
1581 	 */
1582 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0) ||
1583 	    adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
1584 	    adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
1585 		adev->gmc.xgmi.supported = true;
1586 
1587 	if (adev->ip_versions[XGMI_HWIP][0] == IP_VERSION(6, 1, 0)) {
1588 		adev->gmc.xgmi.supported = true;
1589 		adev->gmc.xgmi.connected_to_cpu =
1590 			adev->smuio.funcs->is_host_gpu_xgmi_supported(adev);
1591 	}
1592 
1593 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
1594 		enum amdgpu_pkg_type pkg_type =
1595 			adev->smuio.funcs->get_pkg_type(adev);
1596 		/* On GFXIP 9.4.3. APU, there is no physical VRAM domain present
1597 		 * and the APU, can be in used two possible modes:
1598 		 *  - carveout mode
1599 		 *  - native APU mode
1600 		 * "is_app_apu" can be used to identify the APU in the native
1601 		 * mode.
1602 		 */
1603 		adev->gmc.is_app_apu = (pkg_type == AMDGPU_PKG_TYPE_APU &&
1604 					!pci_resource_len(adev->pdev, 0));
1605 	}
1606 
1607 	gmc_v9_0_set_gmc_funcs(adev);
1608 	gmc_v9_0_set_irq_funcs(adev);
1609 	gmc_v9_0_set_umc_funcs(adev);
1610 	gmc_v9_0_set_mmhub_funcs(adev);
1611 	gmc_v9_0_set_mmhub_ras_funcs(adev);
1612 	gmc_v9_0_set_gfxhub_funcs(adev);
1613 	gmc_v9_0_set_hdp_ras_funcs(adev);
1614 	gmc_v9_0_set_mca_ras_funcs(adev);
1615 	gmc_v9_0_set_xgmi_ras_funcs(adev);
1616 
1617 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1618 	adev->gmc.shared_aperture_end =
1619 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
1620 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
1621 	adev->gmc.private_aperture_end =
1622 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
1623 	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
1624 
1625 	return 0;
1626 }
1627 
1628 static int gmc_v9_0_late_init(void *handle)
1629 {
1630 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1631 	int r;
1632 
1633 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
1634 	if (r)
1635 		return r;
1636 
1637 	/*
1638 	 * Workaround performance drop issue with VBIOS enables partial
1639 	 * writes, while disables HBM ECC for vega10.
1640 	 */
1641 	if (!amdgpu_sriov_vf(adev) &&
1642 	    (adev->ip_versions[UMC_HWIP][0] == IP_VERSION(6, 0, 0))) {
1643 		if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1644 			if (adev->df.funcs &&
1645 			    adev->df.funcs->enable_ecc_force_par_wr_rmw)
1646 				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1647 		}
1648 	}
1649 
1650 	if (!amdgpu_persistent_edc_harvesting_supported(adev)) {
1651 		if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
1652 		    adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
1653 			adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
1654 
1655 		if (adev->hdp.ras && adev->hdp.ras->ras_block.hw_ops &&
1656 		    adev->hdp.ras->ras_block.hw_ops->reset_ras_error_count)
1657 			adev->hdp.ras->ras_block.hw_ops->reset_ras_error_count(adev);
1658 	}
1659 
1660 	r = amdgpu_gmc_ras_late_init(adev);
1661 	if (r)
1662 		return r;
1663 
1664 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
1665 }
1666 
1667 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
1668 					struct amdgpu_gmc *mc)
1669 {
1670 	u64 base = adev->mmhub.funcs->get_fb_location(adev);
1671 
1672 	/* add the xgmi offset of the physical node */
1673 	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1674 	if (adev->gmc.xgmi.connected_to_cpu) {
1675 		amdgpu_gmc_sysvm_location(adev, mc);
1676 	} else {
1677 		amdgpu_gmc_vram_location(adev, mc, base);
1678 		amdgpu_gmc_gart_location(adev, mc);
1679 		amdgpu_gmc_agp_location(adev, mc);
1680 	}
1681 	/* base offset of vram pages */
1682 	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
1683 
1684 	/* XXX: add the xgmi offset of the physical node? */
1685 	adev->vm_manager.vram_base_offset +=
1686 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
1687 }
1688 
1689 /**
1690  * gmc_v9_0_mc_init - initialize the memory controller driver params
1691  *
1692  * @adev: amdgpu_device pointer
1693  *
1694  * Look up the amount of vram, vram width, and decide how to place
1695  * vram and gart within the GPU's physical address space.
1696  * Returns 0 for success.
1697  */
1698 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
1699 {
1700 	int r;
1701 
1702 	/* size in MB on si */
1703 	if (!adev->gmc.is_app_apu) {
1704 		adev->gmc.mc_vram_size =
1705 			adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
1706 	} else {
1707 		DRM_DEBUG("Set mc_vram_size = 0 for APP APU\n");
1708 		adev->gmc.mc_vram_size = 0;
1709 	}
1710 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
1711 
1712 	if (!(adev->flags & AMD_IS_APU) &&
1713 	    !adev->gmc.xgmi.connected_to_cpu) {
1714 		r = amdgpu_device_resize_fb_bar(adev);
1715 		if (r)
1716 			return r;
1717 	}
1718 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
1719 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
1720 
1721 #ifdef CONFIG_X86_64
1722 	/*
1723 	 * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi
1724 	 * interface can use VRAM through here as it appears system reserved
1725 	 * memory in host address space.
1726 	 *
1727 	 * For APUs, VRAM is just the stolen system memory and can be accessed
1728 	 * directly.
1729 	 *
1730 	 * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR.
1731 	 */
1732 
1733 	/* check whether both host-gpu and gpu-gpu xgmi links exist */
1734 	if ((!amdgpu_sriov_vf(adev) &&
1735 		(adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) ||
1736 	    (adev->gmc.xgmi.supported &&
1737 	     adev->gmc.xgmi.connected_to_cpu)) {
1738 		adev->gmc.aper_base =
1739 			adev->gfxhub.funcs->get_mc_fb_offset(adev) +
1740 			adev->gmc.xgmi.physical_node_id *
1741 			adev->gmc.xgmi.node_segment_size;
1742 		adev->gmc.aper_size = adev->gmc.real_vram_size;
1743 	}
1744 
1745 #endif
1746 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
1747 
1748 	/* set the gart size */
1749 	if (amdgpu_gart_size == -1) {
1750 		switch (adev->ip_versions[GC_HWIP][0]) {
1751 		case IP_VERSION(9, 0, 1):  /* all engines support GPUVM */
1752 		case IP_VERSION(9, 2, 1):  /* all engines support GPUVM */
1753 		case IP_VERSION(9, 4, 0):
1754 		case IP_VERSION(9, 4, 1):
1755 		case IP_VERSION(9, 4, 2):
1756 		case IP_VERSION(9, 4, 3):
1757 		default:
1758 			adev->gmc.gart_size = 512ULL << 20;
1759 			break;
1760 		case IP_VERSION(9, 1, 0):   /* DCE SG support */
1761 		case IP_VERSION(9, 2, 2):   /* DCE SG support */
1762 		case IP_VERSION(9, 3, 0):
1763 			adev->gmc.gart_size = 1024ULL << 20;
1764 			break;
1765 		}
1766 	} else {
1767 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
1768 	}
1769 
1770 	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
1771 
1772 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
1773 
1774 	return 0;
1775 }
1776 
1777 static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
1778 {
1779 	int r;
1780 
1781 	if (adev->gart.bo) {
1782 		WARN(1, "VEGA10 PCIE GART already initialized\n");
1783 		return 0;
1784 	}
1785 
1786 	if (adev->gmc.xgmi.connected_to_cpu) {
1787 		adev->gmc.vmid0_page_table_depth = 1;
1788 		adev->gmc.vmid0_page_table_block_size = 12;
1789 	} else {
1790 		adev->gmc.vmid0_page_table_depth = 0;
1791 		adev->gmc.vmid0_page_table_block_size = 0;
1792 	}
1793 
1794 	/* Initialize common gart structure */
1795 	r = amdgpu_gart_init(adev);
1796 	if (r)
1797 		return r;
1798 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
1799 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) |
1800 				 AMDGPU_PTE_EXECUTABLE;
1801 
1802 	if (!adev->gmc.real_vram_size) {
1803 		dev_info(adev->dev, "Put GART in system memory for APU\n");
1804 		r = amdgpu_gart_table_ram_alloc(adev);
1805 		if (r)
1806 			dev_err(adev->dev, "Failed to allocate GART in system memory\n");
1807 	} else {
1808 		r = amdgpu_gart_table_vram_alloc(adev);
1809 		if (r)
1810 			return r;
1811 
1812 		if (adev->gmc.xgmi.connected_to_cpu)
1813 			r = amdgpu_gmc_pdb0_alloc(adev);
1814 	}
1815 
1816 	return r;
1817 }
1818 
1819 /**
1820  * gmc_v9_0_save_registers - saves regs
1821  *
1822  * @adev: amdgpu_device pointer
1823  *
1824  * This saves potential register values that should be
1825  * restored upon resume
1826  */
1827 static void gmc_v9_0_save_registers(struct amdgpu_device *adev)
1828 {
1829 	if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) ||
1830 	    (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1)))
1831 		adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0);
1832 }
1833 
1834 static bool gmc_v9_0_validate_partition_info(struct amdgpu_device *adev)
1835 {
1836 	enum amdgpu_memory_partition mode;
1837 	u32 supp_modes;
1838 	bool valid;
1839 
1840 	mode = gmc_v9_0_get_memory_partition(adev, &supp_modes);
1841 
1842 	/* Mode detected by hardware not present in supported modes */
1843 	if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) &&
1844 	    !(BIT(mode - 1) & supp_modes))
1845 		return false;
1846 
1847 	switch (mode) {
1848 	case UNKNOWN_MEMORY_PARTITION_MODE:
1849 	case AMDGPU_NPS1_PARTITION_MODE:
1850 		valid = (adev->gmc.num_mem_partitions == 1);
1851 		break;
1852 	case AMDGPU_NPS2_PARTITION_MODE:
1853 		valid = (adev->gmc.num_mem_partitions == 2);
1854 		break;
1855 	case AMDGPU_NPS4_PARTITION_MODE:
1856 		valid = (adev->gmc.num_mem_partitions == 3 ||
1857 			 adev->gmc.num_mem_partitions == 4);
1858 		break;
1859 	default:
1860 		valid = false;
1861 	}
1862 
1863 	return valid;
1864 }
1865 
1866 static bool gmc_v9_0_is_node_present(int *node_ids, int num_ids, int nid)
1867 {
1868 	int i;
1869 
1870 	/* Check if node with id 'nid' is present in 'node_ids' array */
1871 	for (i = 0; i < num_ids; ++i)
1872 		if (node_ids[i] == nid)
1873 			return true;
1874 
1875 	return false;
1876 }
1877 
1878 static void
1879 gmc_v9_0_init_acpi_mem_ranges(struct amdgpu_device *adev,
1880 			      struct amdgpu_mem_partition_info *mem_ranges)
1881 {
1882 	int num_ranges = 0, ret, mem_groups;
1883 	struct amdgpu_numa_info numa_info;
1884 	int node_ids[MAX_MEM_RANGES];
1885 	int num_xcc, xcc_id;
1886 	uint32_t xcc_mask;
1887 
1888 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
1889 	xcc_mask = (1U << num_xcc) - 1;
1890 	mem_groups = hweight32(adev->aid_mask);
1891 
1892 	for_each_inst(xcc_id, xcc_mask)	{
1893 		ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info);
1894 		if (ret)
1895 			continue;
1896 
1897 		if (numa_info.nid == NUMA_NO_NODE) {
1898 			mem_ranges[0].size = numa_info.size;
1899 			mem_ranges[0].numa.node = numa_info.nid;
1900 			num_ranges = 1;
1901 			break;
1902 		}
1903 
1904 		if (gmc_v9_0_is_node_present(node_ids, num_ranges,
1905 					     numa_info.nid))
1906 			continue;
1907 
1908 		node_ids[num_ranges] = numa_info.nid;
1909 		mem_ranges[num_ranges].numa.node = numa_info.nid;
1910 		mem_ranges[num_ranges].size = numa_info.size;
1911 		++num_ranges;
1912 	}
1913 
1914 	adev->gmc.num_mem_partitions = num_ranges;
1915 
1916 	/* If there is only partition, don't use entire size */
1917 	if (adev->gmc.num_mem_partitions == 1) {
1918 		mem_ranges[0].size = mem_ranges[0].size * (mem_groups - 1);
1919 		do_div(mem_ranges[0].size, mem_groups);
1920 	}
1921 }
1922 
1923 static void
1924 gmc_v9_0_init_sw_mem_ranges(struct amdgpu_device *adev,
1925 			    struct amdgpu_mem_partition_info *mem_ranges)
1926 {
1927 	enum amdgpu_memory_partition mode;
1928 	u32 start_addr = 0, size;
1929 	int i;
1930 
1931 	mode = gmc_v9_0_query_memory_partition(adev);
1932 
1933 	switch (mode) {
1934 	case UNKNOWN_MEMORY_PARTITION_MODE:
1935 	case AMDGPU_NPS1_PARTITION_MODE:
1936 		adev->gmc.num_mem_partitions = 1;
1937 		break;
1938 	case AMDGPU_NPS2_PARTITION_MODE:
1939 		adev->gmc.num_mem_partitions = 2;
1940 		break;
1941 	case AMDGPU_NPS4_PARTITION_MODE:
1942 		if (adev->flags & AMD_IS_APU)
1943 			adev->gmc.num_mem_partitions = 3;
1944 		else
1945 			adev->gmc.num_mem_partitions = 4;
1946 		break;
1947 	default:
1948 		adev->gmc.num_mem_partitions = 1;
1949 		break;
1950 	}
1951 
1952 	size = (adev->gmc.real_vram_size + SZ_16M) >> AMDGPU_GPU_PAGE_SHIFT;
1953 	size /= adev->gmc.num_mem_partitions;
1954 
1955 	for (i = 0; i < adev->gmc.num_mem_partitions; ++i) {
1956 		mem_ranges[i].range.fpfn = start_addr;
1957 		mem_ranges[i].size = ((u64)size << AMDGPU_GPU_PAGE_SHIFT);
1958 		mem_ranges[i].range.lpfn = start_addr + size - 1;
1959 		start_addr += size;
1960 	}
1961 
1962 	/* Adjust the last one */
1963 	mem_ranges[adev->gmc.num_mem_partitions - 1].range.lpfn =
1964 		(adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1;
1965 	mem_ranges[adev->gmc.num_mem_partitions - 1].size =
1966 		adev->gmc.real_vram_size -
1967 		((u64)mem_ranges[adev->gmc.num_mem_partitions - 1].range.fpfn
1968 		 << AMDGPU_GPU_PAGE_SHIFT);
1969 }
1970 
1971 static int gmc_v9_0_init_mem_ranges(struct amdgpu_device *adev)
1972 {
1973 	bool valid;
1974 
1975 	adev->gmc.mem_partitions = kzalloc(
1976 		MAX_MEM_RANGES * sizeof(struct amdgpu_mem_partition_info),
1977 		GFP_KERNEL);
1978 
1979 	if (!adev->gmc.mem_partitions)
1980 		return -ENOMEM;
1981 
1982 	/* TODO : Get the range from PSP/Discovery for dGPU */
1983 	if (adev->gmc.is_app_apu)
1984 		gmc_v9_0_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions);
1985 	else
1986 		gmc_v9_0_init_sw_mem_ranges(adev, adev->gmc.mem_partitions);
1987 
1988 	if (amdgpu_sriov_vf(adev))
1989 		valid = true;
1990 	else
1991 		valid = gmc_v9_0_validate_partition_info(adev);
1992 	if (!valid) {
1993 		/* TODO: handle invalid case */
1994 		dev_WARN(adev->dev,
1995 			 "Mem ranges not matching with hardware config");
1996 	}
1997 
1998 	return 0;
1999 }
2000 
2001 static void gmc_v9_4_3_init_vram_info(struct amdgpu_device *adev)
2002 {
2003 	static const u32 regBIF_BIOS_SCRATCH_4 = 0x50;
2004 	u32 vram_info;
2005 
2006 	if (!amdgpu_sriov_vf(adev)) {
2007 		vram_info = RREG32(regBIF_BIOS_SCRATCH_4);
2008 		adev->gmc.vram_vendor = vram_info & 0xF;
2009 	}
2010 	adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
2011 	adev->gmc.vram_width = 128 * 64;
2012 }
2013 
2014 static int gmc_v9_0_sw_init(void *handle)
2015 {
2016 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits;
2017 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2018 	unsigned long inst_mask = adev->aid_mask;
2019 
2020 	adev->gfxhub.funcs->init(adev);
2021 
2022 	adev->mmhub.funcs->init(adev);
2023 
2024 	spin_lock_init(&adev->gmc.invalidate_lock);
2025 
2026 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
2027 		gmc_v9_4_3_init_vram_info(adev);
2028 	} else if (!adev->bios) {
2029 		if (adev->flags & AMD_IS_APU) {
2030 			adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4;
2031 			adev->gmc.vram_width = 64 * 64;
2032 		} else {
2033 			adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM;
2034 			adev->gmc.vram_width = 128 * 64;
2035 		}
2036 	} else {
2037 		r = amdgpu_atomfirmware_get_vram_info(adev,
2038 			&vram_width, &vram_type, &vram_vendor);
2039 		if (amdgpu_sriov_vf(adev))
2040 			/* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN,
2041 			 * and DF related registers is not readable, seems hardcord is the
2042 			 * only way to set the correct vram_width
2043 			 */
2044 			adev->gmc.vram_width = 2048;
2045 		else if (amdgpu_emu_mode != 1)
2046 			adev->gmc.vram_width = vram_width;
2047 
2048 		if (!adev->gmc.vram_width) {
2049 			int chansize, numchan;
2050 
2051 			/* hbm memory channel size */
2052 			if (adev->flags & AMD_IS_APU)
2053 				chansize = 64;
2054 			else
2055 				chansize = 128;
2056 			if (adev->df.funcs &&
2057 			    adev->df.funcs->get_hbm_channel_number) {
2058 				numchan = adev->df.funcs->get_hbm_channel_number(adev);
2059 				adev->gmc.vram_width = numchan * chansize;
2060 			}
2061 		}
2062 
2063 		adev->gmc.vram_type = vram_type;
2064 		adev->gmc.vram_vendor = vram_vendor;
2065 	}
2066 	switch (adev->ip_versions[GC_HWIP][0]) {
2067 	case IP_VERSION(9, 1, 0):
2068 	case IP_VERSION(9, 2, 2):
2069 		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2070 		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
2071 
2072 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
2073 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2074 		} else {
2075 			/* vm_size is 128TB + 512GB for legacy 3-level page support */
2076 			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
2077 			adev->gmc.translate_further =
2078 				adev->vm_manager.num_level > 1;
2079 		}
2080 		break;
2081 	case IP_VERSION(9, 0, 1):
2082 	case IP_VERSION(9, 2, 1):
2083 	case IP_VERSION(9, 4, 0):
2084 	case IP_VERSION(9, 3, 0):
2085 	case IP_VERSION(9, 4, 2):
2086 		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2087 		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
2088 
2089 		/*
2090 		 * To fulfill 4-level page support,
2091 		 * vm size is 256TB (48bit), maximum size of Vega10,
2092 		 * block size 512 (9bit)
2093 		 */
2094 		/* sriov restrict max_pfn below AMDGPU_GMC_HOLE */
2095 		if (amdgpu_sriov_vf(adev))
2096 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47);
2097 		else
2098 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2099 		if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
2100 			adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2101 		break;
2102 	case IP_VERSION(9, 4, 1):
2103 		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
2104 		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
2105 		set_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask);
2106 
2107 		/* Keep the vm size same with Vega20 */
2108 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2109 		adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2110 		break;
2111 	case IP_VERSION(9, 4, 3):
2112 		bitmap_set(adev->vmhubs_mask, AMDGPU_GFXHUB(0),
2113 				  NUM_XCC(adev->gfx.xcc_mask));
2114 
2115 		inst_mask <<= AMDGPU_MMHUB0(0);
2116 		bitmap_or(adev->vmhubs_mask, adev->vmhubs_mask, &inst_mask, 32);
2117 
2118 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
2119 		adev->gmc.translate_further = adev->vm_manager.num_level > 1;
2120 		break;
2121 	default:
2122 		break;
2123 	}
2124 
2125 	/* This interrupt is VMC page fault.*/
2126 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT,
2127 				&adev->gmc.vm_fault);
2128 	if (r)
2129 		return r;
2130 
2131 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) {
2132 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT,
2133 					&adev->gmc.vm_fault);
2134 		if (r)
2135 			return r;
2136 	}
2137 
2138 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT,
2139 				&adev->gmc.vm_fault);
2140 
2141 	if (r)
2142 		return r;
2143 
2144 	if (!amdgpu_sriov_vf(adev) &&
2145 	    !adev->gmc.xgmi.connected_to_cpu) {
2146 		/* interrupt sent to DF. */
2147 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
2148 				      &adev->gmc.ecc_irq);
2149 		if (r)
2150 			return r;
2151 	}
2152 
2153 	/* Set the internal MC address mask
2154 	 * This is the max address of the GPU's
2155 	 * internal address space.
2156 	 */
2157 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
2158 
2159 	dma_addr_bits = adev->ip_versions[GC_HWIP][0] >= IP_VERSION(9, 4, 2) ? 48:44;
2160 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(dma_addr_bits));
2161 	if (r) {
2162 		dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
2163 		return r;
2164 	}
2165 	adev->need_swiotlb = drm_need_swiotlb(dma_addr_bits);
2166 
2167 	r = gmc_v9_0_mc_init(adev);
2168 	if (r)
2169 		return r;
2170 
2171 	amdgpu_gmc_get_vbios_allocations(adev);
2172 
2173 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) {
2174 		r = gmc_v9_0_init_mem_ranges(adev);
2175 		if (r)
2176 			return r;
2177 	}
2178 
2179 	/* Memory manager */
2180 	r = amdgpu_bo_init(adev);
2181 	if (r)
2182 		return r;
2183 
2184 	r = gmc_v9_0_gart_init(adev);
2185 	if (r)
2186 		return r;
2187 
2188 	/*
2189 	 * number of VMs
2190 	 * VMID 0 is reserved for System
2191 	 * amdgpu graphics/compute will use VMIDs 1..n-1
2192 	 * amdkfd will use VMIDs n..15
2193 	 *
2194 	 * The first KFD VMID is 8 for GPUs with graphics, 3 for
2195 	 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs
2196 	 * for video processing.
2197 	 */
2198 	adev->vm_manager.first_kfd_vmid =
2199 		(adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) ||
2200 		 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
2201 		 adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) ? 3 : 8;
2202 
2203 	amdgpu_vm_manager_init(adev);
2204 
2205 	gmc_v9_0_save_registers(adev);
2206 
2207 	r = amdgpu_gmc_ras_sw_init(adev);
2208 	if (r)
2209 		return r;
2210 
2211 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
2212 		amdgpu_gmc_sysfs_init(adev);
2213 
2214 	return 0;
2215 }
2216 
2217 static int gmc_v9_0_sw_fini(void *handle)
2218 {
2219 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2220 
2221 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3))
2222 		amdgpu_gmc_sysfs_fini(adev);
2223 
2224 	amdgpu_gmc_ras_fini(adev);
2225 	amdgpu_gem_force_release(adev);
2226 	amdgpu_vm_manager_fini(adev);
2227 	if (!adev->gmc.real_vram_size) {
2228 		dev_info(adev->dev, "Put GART in system memory for APU free\n");
2229 		amdgpu_gart_table_ram_free(adev);
2230 	} else {
2231 		amdgpu_gart_table_vram_free(adev);
2232 	}
2233 	amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0);
2234 	amdgpu_bo_fini(adev);
2235 
2236 	adev->gmc.num_mem_partitions = 0;
2237 	kfree(adev->gmc.mem_partitions);
2238 
2239 	return 0;
2240 }
2241 
2242 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
2243 {
2244 
2245 	switch (adev->ip_versions[MMHUB_HWIP][0]) {
2246 	case IP_VERSION(9, 0, 0):
2247 		if (amdgpu_sriov_vf(adev))
2248 			break;
2249 		fallthrough;
2250 	case IP_VERSION(9, 4, 0):
2251 		soc15_program_register_sequence(adev,
2252 						golden_settings_mmhub_1_0_0,
2253 						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
2254 		soc15_program_register_sequence(adev,
2255 						golden_settings_athub_1_0_0,
2256 						ARRAY_SIZE(golden_settings_athub_1_0_0));
2257 		break;
2258 	case IP_VERSION(9, 1, 0):
2259 	case IP_VERSION(9, 2, 0):
2260 		/* TODO for renoir */
2261 		soc15_program_register_sequence(adev,
2262 						golden_settings_athub_1_0_0,
2263 						ARRAY_SIZE(golden_settings_athub_1_0_0));
2264 		break;
2265 	default:
2266 		break;
2267 	}
2268 }
2269 
2270 /**
2271  * gmc_v9_0_restore_registers - restores regs
2272  *
2273  * @adev: amdgpu_device pointer
2274  *
2275  * This restores register values, saved at suspend.
2276  */
2277 void gmc_v9_0_restore_registers(struct amdgpu_device *adev)
2278 {
2279 	if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) ||
2280 	    (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1))) {
2281 		WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register);
2282 		WARN_ON(adev->gmc.sdpif_register !=
2283 			RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0));
2284 	}
2285 }
2286 
2287 /**
2288  * gmc_v9_0_gart_enable - gart enable
2289  *
2290  * @adev: amdgpu_device pointer
2291  */
2292 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
2293 {
2294 	int r;
2295 
2296 	if (adev->gmc.xgmi.connected_to_cpu)
2297 		amdgpu_gmc_init_pdb0(adev);
2298 
2299 	if (adev->gart.bo == NULL) {
2300 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
2301 		return -EINVAL;
2302 	}
2303 
2304 	amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
2305 
2306 	if (!adev->in_s0ix) {
2307 		r = adev->gfxhub.funcs->gart_enable(adev);
2308 		if (r)
2309 			return r;
2310 	}
2311 
2312 	r = adev->mmhub.funcs->gart_enable(adev);
2313 	if (r)
2314 		return r;
2315 
2316 	DRM_INFO("PCIE GART of %uM enabled.\n",
2317 		 (unsigned int)(adev->gmc.gart_size >> 20));
2318 	if (adev->gmc.pdb0_bo)
2319 		DRM_INFO("PDB0 located at 0x%016llX\n",
2320 				(unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo));
2321 	DRM_INFO("PTB located at 0x%016llX\n",
2322 			(unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
2323 
2324 	return 0;
2325 }
2326 
2327 static int gmc_v9_0_hw_init(void *handle)
2328 {
2329 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2330 	bool value;
2331 	int i, r;
2332 
2333 	/* The sequence of these two function calls matters.*/
2334 	gmc_v9_0_init_golden_registers(adev);
2335 
2336 	if (adev->mode_info.num_crtc) {
2337 		/* Lockout access through VGA aperture*/
2338 		WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
2339 		/* disable VGA render */
2340 		WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
2341 	}
2342 
2343 	if (adev->mmhub.funcs->update_power_gating)
2344 		adev->mmhub.funcs->update_power_gating(adev, true);
2345 
2346 	adev->hdp.funcs->init_registers(adev);
2347 
2348 	/* After HDP is initialized, flush HDP.*/
2349 	adev->hdp.funcs->flush_hdp(adev, NULL);
2350 
2351 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
2352 		value = false;
2353 	else
2354 		value = true;
2355 
2356 	if (!amdgpu_sriov_vf(adev)) {
2357 		if (!adev->in_s0ix)
2358 			adev->gfxhub.funcs->set_fault_enable_default(adev, value);
2359 		adev->mmhub.funcs->set_fault_enable_default(adev, value);
2360 	}
2361 	for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) {
2362 		if (adev->in_s0ix && (i == AMDGPU_GFXHUB(0)))
2363 			continue;
2364 		gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0);
2365 	}
2366 
2367 	if (adev->umc.funcs && adev->umc.funcs->init_registers)
2368 		adev->umc.funcs->init_registers(adev);
2369 
2370 	r = gmc_v9_0_gart_enable(adev);
2371 	if (r)
2372 		return r;
2373 
2374 	if (amdgpu_emu_mode == 1)
2375 		return amdgpu_gmc_vram_checking(adev);
2376 
2377 	return 0;
2378 }
2379 
2380 /**
2381  * gmc_v9_0_gart_disable - gart disable
2382  *
2383  * @adev: amdgpu_device pointer
2384  *
2385  * This disables all VM page table.
2386  */
2387 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
2388 {
2389 	if (!adev->in_s0ix)
2390 		adev->gfxhub.funcs->gart_disable(adev);
2391 	adev->mmhub.funcs->gart_disable(adev);
2392 }
2393 
2394 static int gmc_v9_0_hw_fini(void *handle)
2395 {
2396 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2397 
2398 	gmc_v9_0_gart_disable(adev);
2399 
2400 	if (amdgpu_sriov_vf(adev)) {
2401 		/* full access mode, so don't touch any GMC register */
2402 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
2403 		return 0;
2404 	}
2405 
2406 	/*
2407 	 * Pair the operations did in gmc_v9_0_hw_init and thus maintain
2408 	 * a correct cached state for GMC. Otherwise, the "gate" again
2409 	 * operation on S3 resuming will fail due to wrong cached state.
2410 	 */
2411 	if (adev->mmhub.funcs->update_power_gating)
2412 		adev->mmhub.funcs->update_power_gating(adev, false);
2413 
2414 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
2415 
2416 	if (adev->gmc.ecc_irq.funcs &&
2417 		amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC))
2418 		amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0);
2419 
2420 	return 0;
2421 }
2422 
2423 static int gmc_v9_0_suspend(void *handle)
2424 {
2425 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2426 
2427 	return gmc_v9_0_hw_fini(adev);
2428 }
2429 
2430 static int gmc_v9_0_resume(void *handle)
2431 {
2432 	int r;
2433 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2434 
2435 	r = gmc_v9_0_hw_init(adev);
2436 	if (r)
2437 		return r;
2438 
2439 	amdgpu_vmid_reset_all(adev);
2440 
2441 	return 0;
2442 }
2443 
2444 static bool gmc_v9_0_is_idle(void *handle)
2445 {
2446 	/* MC is always ready in GMC v9.*/
2447 	return true;
2448 }
2449 
2450 static int gmc_v9_0_wait_for_idle(void *handle)
2451 {
2452 	/* There is no need to wait for MC idle in GMC v9.*/
2453 	return 0;
2454 }
2455 
2456 static int gmc_v9_0_soft_reset(void *handle)
2457 {
2458 	/* XXX for emulation.*/
2459 	return 0;
2460 }
2461 
2462 static int gmc_v9_0_set_clockgating_state(void *handle,
2463 					enum amd_clockgating_state state)
2464 {
2465 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2466 
2467 	adev->mmhub.funcs->set_clockgating(adev, state);
2468 
2469 	athub_v1_0_set_clockgating(adev, state);
2470 
2471 	return 0;
2472 }
2473 
2474 static void gmc_v9_0_get_clockgating_state(void *handle, u64 *flags)
2475 {
2476 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2477 
2478 	adev->mmhub.funcs->get_clockgating(adev, flags);
2479 
2480 	athub_v1_0_get_clockgating(adev, flags);
2481 }
2482 
2483 static int gmc_v9_0_set_powergating_state(void *handle,
2484 					enum amd_powergating_state state)
2485 {
2486 	return 0;
2487 }
2488 
2489 const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
2490 	.name = "gmc_v9_0",
2491 	.early_init = gmc_v9_0_early_init,
2492 	.late_init = gmc_v9_0_late_init,
2493 	.sw_init = gmc_v9_0_sw_init,
2494 	.sw_fini = gmc_v9_0_sw_fini,
2495 	.hw_init = gmc_v9_0_hw_init,
2496 	.hw_fini = gmc_v9_0_hw_fini,
2497 	.suspend = gmc_v9_0_suspend,
2498 	.resume = gmc_v9_0_resume,
2499 	.is_idle = gmc_v9_0_is_idle,
2500 	.wait_for_idle = gmc_v9_0_wait_for_idle,
2501 	.soft_reset = gmc_v9_0_soft_reset,
2502 	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
2503 	.set_powergating_state = gmc_v9_0_set_powergating_state,
2504 	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
2505 };
2506 
2507 const struct amdgpu_ip_block_version gmc_v9_0_ip_block = {
2508 	.type = AMD_IP_BLOCK_TYPE_GMC,
2509 	.major = 9,
2510 	.minor = 0,
2511 	.rev = 0,
2512 	.funcs = &gmc_v9_0_ip_funcs,
2513 };
2514