xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c (revision 98ddec80)
1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include <drm/drm_cache.h>
25 #include "amdgpu.h"
26 #include "gmc_v9_0.h"
27 #include "amdgpu_atomfirmware.h"
28 
29 #include "hdp/hdp_4_0_offset.h"
30 #include "hdp/hdp_4_0_sh_mask.h"
31 #include "gc/gc_9_0_sh_mask.h"
32 #include "dce/dce_12_0_offset.h"
33 #include "dce/dce_12_0_sh_mask.h"
34 #include "vega10_enum.h"
35 #include "mmhub/mmhub_1_0_offset.h"
36 #include "athub/athub_1_0_offset.h"
37 #include "oss/osssys_4_0_offset.h"
38 
39 #include "soc15.h"
40 #include "soc15_common.h"
41 #include "umc/umc_6_0_sh_mask.h"
42 
43 #include "gfxhub_v1_0.h"
44 #include "mmhub_v1_0.h"
45 
46 /* add these here since we already include dce12 headers and these are for DCN */
47 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION                                                          0x055d
48 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX                                                 2
49 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT                                        0x0
50 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT                                       0x10
51 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK                                          0x00003FFFL
52 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK                                         0x3FFF0000L
53 
54 /* XXX Move this macro to VEGA10 header file, which is like vid.h for VI.*/
55 #define AMDGPU_NUM_OF_VMIDS			8
56 
57 static const u32 golden_settings_vega10_hdp[] =
58 {
59 	0xf64, 0x0fffffff, 0x00000000,
60 	0xf65, 0x0fffffff, 0x00000000,
61 	0xf66, 0x0fffffff, 0x00000000,
62 	0xf67, 0x0fffffff, 0x00000000,
63 	0xf68, 0x0fffffff, 0x00000000,
64 	0xf6a, 0x0fffffff, 0x00000000,
65 	0xf6b, 0x0fffffff, 0x00000000,
66 	0xf6c, 0x0fffffff, 0x00000000,
67 	0xf6d, 0x0fffffff, 0x00000000,
68 	0xf6e, 0x0fffffff, 0x00000000,
69 };
70 
71 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] =
72 {
73 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa),
74 	SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565)
75 };
76 
77 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] =
78 {
79 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800),
80 	SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008)
81 };
82 
83 /* Ecc related register addresses, (BASE + reg offset) */
84 /* Universal Memory Controller caps (may be fused). */
85 /* UMCCH:UmcLocalCap */
86 #define UMCLOCALCAPS_ADDR0	(0x00014306 + 0x00000000)
87 #define UMCLOCALCAPS_ADDR1	(0x00014306 + 0x00000800)
88 #define UMCLOCALCAPS_ADDR2	(0x00014306 + 0x00001000)
89 #define UMCLOCALCAPS_ADDR3	(0x00014306 + 0x00001800)
90 #define UMCLOCALCAPS_ADDR4	(0x00054306 + 0x00000000)
91 #define UMCLOCALCAPS_ADDR5	(0x00054306 + 0x00000800)
92 #define UMCLOCALCAPS_ADDR6	(0x00054306 + 0x00001000)
93 #define UMCLOCALCAPS_ADDR7	(0x00054306 + 0x00001800)
94 #define UMCLOCALCAPS_ADDR8	(0x00094306 + 0x00000000)
95 #define UMCLOCALCAPS_ADDR9	(0x00094306 + 0x00000800)
96 #define UMCLOCALCAPS_ADDR10	(0x00094306 + 0x00001000)
97 #define UMCLOCALCAPS_ADDR11	(0x00094306 + 0x00001800)
98 #define UMCLOCALCAPS_ADDR12	(0x000d4306 + 0x00000000)
99 #define UMCLOCALCAPS_ADDR13	(0x000d4306 + 0x00000800)
100 #define UMCLOCALCAPS_ADDR14	(0x000d4306 + 0x00001000)
101 #define UMCLOCALCAPS_ADDR15	(0x000d4306 + 0x00001800)
102 
103 /* Universal Memory Controller Channel config. */
104 /* UMCCH:UMC_CONFIG */
105 #define UMCCH_UMC_CONFIG_ADDR0	(0x00014040 + 0x00000000)
106 #define UMCCH_UMC_CONFIG_ADDR1	(0x00014040 + 0x00000800)
107 #define UMCCH_UMC_CONFIG_ADDR2	(0x00014040 + 0x00001000)
108 #define UMCCH_UMC_CONFIG_ADDR3	(0x00014040 + 0x00001800)
109 #define UMCCH_UMC_CONFIG_ADDR4	(0x00054040 + 0x00000000)
110 #define UMCCH_UMC_CONFIG_ADDR5	(0x00054040 + 0x00000800)
111 #define UMCCH_UMC_CONFIG_ADDR6	(0x00054040 + 0x00001000)
112 #define UMCCH_UMC_CONFIG_ADDR7	(0x00054040 + 0x00001800)
113 #define UMCCH_UMC_CONFIG_ADDR8	(0x00094040 + 0x00000000)
114 #define UMCCH_UMC_CONFIG_ADDR9	(0x00094040 + 0x00000800)
115 #define UMCCH_UMC_CONFIG_ADDR10	(0x00094040 + 0x00001000)
116 #define UMCCH_UMC_CONFIG_ADDR11	(0x00094040 + 0x00001800)
117 #define UMCCH_UMC_CONFIG_ADDR12	(0x000d4040 + 0x00000000)
118 #define UMCCH_UMC_CONFIG_ADDR13	(0x000d4040 + 0x00000800)
119 #define UMCCH_UMC_CONFIG_ADDR14	(0x000d4040 + 0x00001000)
120 #define UMCCH_UMC_CONFIG_ADDR15	(0x000d4040 + 0x00001800)
121 
122 /* Universal Memory Controller Channel Ecc config. */
123 /* UMCCH:EccCtrl */
124 #define UMCCH_ECCCTRL_ADDR0	(0x00014053 + 0x00000000)
125 #define UMCCH_ECCCTRL_ADDR1	(0x00014053 + 0x00000800)
126 #define UMCCH_ECCCTRL_ADDR2	(0x00014053 + 0x00001000)
127 #define UMCCH_ECCCTRL_ADDR3	(0x00014053 + 0x00001800)
128 #define UMCCH_ECCCTRL_ADDR4	(0x00054053 + 0x00000000)
129 #define UMCCH_ECCCTRL_ADDR5	(0x00054053 + 0x00000800)
130 #define UMCCH_ECCCTRL_ADDR6	(0x00054053 + 0x00001000)
131 #define UMCCH_ECCCTRL_ADDR7	(0x00054053 + 0x00001800)
132 #define UMCCH_ECCCTRL_ADDR8	(0x00094053 + 0x00000000)
133 #define UMCCH_ECCCTRL_ADDR9	(0x00094053 + 0x00000800)
134 #define UMCCH_ECCCTRL_ADDR10	(0x00094053 + 0x00001000)
135 #define UMCCH_ECCCTRL_ADDR11	(0x00094053 + 0x00001800)
136 #define UMCCH_ECCCTRL_ADDR12	(0x000d4053 + 0x00000000)
137 #define UMCCH_ECCCTRL_ADDR13	(0x000d4053 + 0x00000800)
138 #define UMCCH_ECCCTRL_ADDR14	(0x000d4053 + 0x00001000)
139 #define UMCCH_ECCCTRL_ADDR15	(0x000d4053 + 0x00001800)
140 
141 static const uint32_t ecc_umclocalcap_addrs[] = {
142 	UMCLOCALCAPS_ADDR0,
143 	UMCLOCALCAPS_ADDR1,
144 	UMCLOCALCAPS_ADDR2,
145 	UMCLOCALCAPS_ADDR3,
146 	UMCLOCALCAPS_ADDR4,
147 	UMCLOCALCAPS_ADDR5,
148 	UMCLOCALCAPS_ADDR6,
149 	UMCLOCALCAPS_ADDR7,
150 	UMCLOCALCAPS_ADDR8,
151 	UMCLOCALCAPS_ADDR9,
152 	UMCLOCALCAPS_ADDR10,
153 	UMCLOCALCAPS_ADDR11,
154 	UMCLOCALCAPS_ADDR12,
155 	UMCLOCALCAPS_ADDR13,
156 	UMCLOCALCAPS_ADDR14,
157 	UMCLOCALCAPS_ADDR15,
158 };
159 
160 static const uint32_t ecc_umcch_umc_config_addrs[] = {
161 	UMCCH_UMC_CONFIG_ADDR0,
162 	UMCCH_UMC_CONFIG_ADDR1,
163 	UMCCH_UMC_CONFIG_ADDR2,
164 	UMCCH_UMC_CONFIG_ADDR3,
165 	UMCCH_UMC_CONFIG_ADDR4,
166 	UMCCH_UMC_CONFIG_ADDR5,
167 	UMCCH_UMC_CONFIG_ADDR6,
168 	UMCCH_UMC_CONFIG_ADDR7,
169 	UMCCH_UMC_CONFIG_ADDR8,
170 	UMCCH_UMC_CONFIG_ADDR9,
171 	UMCCH_UMC_CONFIG_ADDR10,
172 	UMCCH_UMC_CONFIG_ADDR11,
173 	UMCCH_UMC_CONFIG_ADDR12,
174 	UMCCH_UMC_CONFIG_ADDR13,
175 	UMCCH_UMC_CONFIG_ADDR14,
176 	UMCCH_UMC_CONFIG_ADDR15,
177 };
178 
179 static const uint32_t ecc_umcch_eccctrl_addrs[] = {
180 	UMCCH_ECCCTRL_ADDR0,
181 	UMCCH_ECCCTRL_ADDR1,
182 	UMCCH_ECCCTRL_ADDR2,
183 	UMCCH_ECCCTRL_ADDR3,
184 	UMCCH_ECCCTRL_ADDR4,
185 	UMCCH_ECCCTRL_ADDR5,
186 	UMCCH_ECCCTRL_ADDR6,
187 	UMCCH_ECCCTRL_ADDR7,
188 	UMCCH_ECCCTRL_ADDR8,
189 	UMCCH_ECCCTRL_ADDR9,
190 	UMCCH_ECCCTRL_ADDR10,
191 	UMCCH_ECCCTRL_ADDR11,
192 	UMCCH_ECCCTRL_ADDR12,
193 	UMCCH_ECCCTRL_ADDR13,
194 	UMCCH_ECCCTRL_ADDR14,
195 	UMCCH_ECCCTRL_ADDR15,
196 };
197 
198 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
199 					struct amdgpu_irq_src *src,
200 					unsigned type,
201 					enum amdgpu_interrupt_state state)
202 {
203 	struct amdgpu_vmhub *hub;
204 	u32 tmp, reg, bits, i, j;
205 
206 	bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
207 		VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
208 		VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
209 		VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
210 		VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
211 		VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
212 		VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
213 
214 	switch (state) {
215 	case AMDGPU_IRQ_STATE_DISABLE:
216 		for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
217 			hub = &adev->vmhub[j];
218 			for (i = 0; i < 16; i++) {
219 				reg = hub->vm_context0_cntl + i;
220 				tmp = RREG32(reg);
221 				tmp &= ~bits;
222 				WREG32(reg, tmp);
223 			}
224 		}
225 		break;
226 	case AMDGPU_IRQ_STATE_ENABLE:
227 		for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) {
228 			hub = &adev->vmhub[j];
229 			for (i = 0; i < 16; i++) {
230 				reg = hub->vm_context0_cntl + i;
231 				tmp = RREG32(reg);
232 				tmp |= bits;
233 				WREG32(reg, tmp);
234 			}
235 		}
236 	default:
237 		break;
238 	}
239 
240 	return 0;
241 }
242 
243 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev,
244 				struct amdgpu_irq_src *source,
245 				struct amdgpu_iv_entry *entry)
246 {
247 	struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
248 	uint32_t status = 0;
249 	u64 addr;
250 
251 	addr = (u64)entry->src_data[0] << 12;
252 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
253 
254 	if (!amdgpu_sriov_vf(adev)) {
255 		status = RREG32(hub->vm_l2_pro_fault_status);
256 		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
257 	}
258 
259 	if (printk_ratelimit()) {
260 		dev_err(adev->dev,
261 			"[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u)\n",
262 			entry->vmid_src ? "mmhub" : "gfxhub",
263 			entry->src_id, entry->ring_id, entry->vmid,
264 			entry->pasid);
265 		dev_err(adev->dev, "  at page 0x%016llx from %d\n",
266 			addr, entry->client_id);
267 		if (!amdgpu_sriov_vf(adev))
268 			dev_err(adev->dev,
269 				"VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
270 				status);
271 	}
272 
273 	return 0;
274 }
275 
276 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = {
277 	.set = gmc_v9_0_vm_fault_interrupt_state,
278 	.process = gmc_v9_0_process_interrupt,
279 };
280 
281 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev)
282 {
283 	adev->gmc.vm_fault.num_types = 1;
284 	adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs;
285 }
286 
287 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid)
288 {
289 	u32 req = 0;
290 
291 	/* invalidate using legacy mode on vmid*/
292 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
293 			    PER_VMID_INVALIDATE_REQ, 1 << vmid);
294 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0);
295 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
296 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
297 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
298 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
299 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
300 	req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ,
301 			    CLEAR_PROTECTION_FAULT_STATUS_ADDR,	0);
302 
303 	return req;
304 }
305 
306 /*
307  * GART
308  * VMID 0 is the physical GPU addresses as used by the kernel.
309  * VMIDs 1-15 are used for userspace clients and are handled
310  * by the amdgpu vm/hsa code.
311  */
312 
313 /**
314  * gmc_v9_0_flush_gpu_tlb - gart tlb flush callback
315  *
316  * @adev: amdgpu_device pointer
317  * @vmid: vm instance to flush
318  *
319  * Flush the TLB for the requested page table.
320  */
321 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev,
322 					uint32_t vmid)
323 {
324 	/* Use register 17 for GART */
325 	const unsigned eng = 17;
326 	unsigned i, j;
327 
328 	spin_lock(&adev->gmc.invalidate_lock);
329 
330 	for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
331 		struct amdgpu_vmhub *hub = &adev->vmhub[i];
332 		u32 tmp = gmc_v9_0_get_invalidate_req(vmid);
333 
334 		WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp);
335 
336 		/* Busy wait for ACK.*/
337 		for (j = 0; j < 100; j++) {
338 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng);
339 			tmp &= 1 << vmid;
340 			if (tmp)
341 				break;
342 			cpu_relax();
343 		}
344 		if (j < 100)
345 			continue;
346 
347 		/* Wait for ACK with a delay.*/
348 		for (j = 0; j < adev->usec_timeout; j++) {
349 			tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng);
350 			tmp &= 1 << vmid;
351 			if (tmp)
352 				break;
353 			udelay(1);
354 		}
355 		if (j < adev->usec_timeout)
356 			continue;
357 
358 		DRM_ERROR("Timeout waiting for VM flush ACK!\n");
359 	}
360 
361 	spin_unlock(&adev->gmc.invalidate_lock);
362 }
363 
364 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
365 					    unsigned vmid, uint64_t pd_addr)
366 {
367 	struct amdgpu_device *adev = ring->adev;
368 	struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub];
369 	uint32_t req = gmc_v9_0_get_invalidate_req(vmid);
370 	uint64_t flags = AMDGPU_PTE_VALID;
371 	unsigned eng = ring->vm_inv_eng;
372 
373 	amdgpu_gmc_get_vm_pde(adev, -1, &pd_addr, &flags);
374 	pd_addr |= flags;
375 
376 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid),
377 			      lower_32_bits(pd_addr));
378 
379 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid),
380 			      upper_32_bits(pd_addr));
381 
382 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng,
383 					    hub->vm_inv_eng0_ack + eng,
384 					    req, 1 << vmid);
385 
386 	return pd_addr;
387 }
388 
389 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
390 					unsigned pasid)
391 {
392 	struct amdgpu_device *adev = ring->adev;
393 	uint32_t reg;
394 
395 	if (ring->funcs->vmhub == AMDGPU_GFXHUB)
396 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
397 	else
398 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
399 
400 	amdgpu_ring_emit_wreg(ring, reg, pasid);
401 }
402 
403 /**
404  * gmc_v9_0_set_pte_pde - update the page tables using MMIO
405  *
406  * @adev: amdgpu_device pointer
407  * @cpu_pt_addr: cpu address of the page table
408  * @gpu_page_idx: entry in the page table to update
409  * @addr: dst addr to write into pte/pde
410  * @flags: access flags
411  *
412  * Update the page tables using the CPU.
413  */
414 static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
415 				uint32_t gpu_page_idx, uint64_t addr,
416 				uint64_t flags)
417 {
418 	void __iomem *ptr = (void *)cpu_pt_addr;
419 	uint64_t value;
420 
421 	/*
422 	 * PTE format on VEGA 10:
423 	 * 63:59 reserved
424 	 * 58:57 mtype
425 	 * 56 F
426 	 * 55 L
427 	 * 54 P
428 	 * 53 SW
429 	 * 52 T
430 	 * 50:48 reserved
431 	 * 47:12 4k physical page base address
432 	 * 11:7 fragment
433 	 * 6 write
434 	 * 5 read
435 	 * 4 exe
436 	 * 3 Z
437 	 * 2 snooped
438 	 * 1 system
439 	 * 0 valid
440 	 *
441 	 * PDE format on VEGA 10:
442 	 * 63:59 block fragment size
443 	 * 58:55 reserved
444 	 * 54 P
445 	 * 53:48 reserved
446 	 * 47:6 physical base address of PD or PTE
447 	 * 5:3 reserved
448 	 * 2 C
449 	 * 1 system
450 	 * 0 valid
451 	 */
452 
453 	/*
454 	 * The following is for PTE only. GART does not have PDEs.
455 	*/
456 	value = addr & 0x0000FFFFFFFFF000ULL;
457 	value |= flags;
458 	writeq(value, ptr + (gpu_page_idx * 8));
459 	return 0;
460 }
461 
462 static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev,
463 						uint32_t flags)
464 
465 {
466 	uint64_t pte_flag = 0;
467 
468 	if (flags & AMDGPU_VM_PAGE_EXECUTABLE)
469 		pte_flag |= AMDGPU_PTE_EXECUTABLE;
470 	if (flags & AMDGPU_VM_PAGE_READABLE)
471 		pte_flag |= AMDGPU_PTE_READABLE;
472 	if (flags & AMDGPU_VM_PAGE_WRITEABLE)
473 		pte_flag |= AMDGPU_PTE_WRITEABLE;
474 
475 	switch (flags & AMDGPU_VM_MTYPE_MASK) {
476 	case AMDGPU_VM_MTYPE_DEFAULT:
477 		pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
478 		break;
479 	case AMDGPU_VM_MTYPE_NC:
480 		pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
481 		break;
482 	case AMDGPU_VM_MTYPE_WC:
483 		pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC);
484 		break;
485 	case AMDGPU_VM_MTYPE_CC:
486 		pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC);
487 		break;
488 	case AMDGPU_VM_MTYPE_UC:
489 		pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC);
490 		break;
491 	default:
492 		pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC);
493 		break;
494 	}
495 
496 	if (flags & AMDGPU_VM_PAGE_PRT)
497 		pte_flag |= AMDGPU_PTE_PRT;
498 
499 	return pte_flag;
500 }
501 
502 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level,
503 				uint64_t *addr, uint64_t *flags)
504 {
505 	if (!(*flags & AMDGPU_PDE_PTE))
506 		*addr = adev->vm_manager.vram_base_offset + *addr -
507 			adev->gmc.vram_start;
508 	BUG_ON(*addr & 0xFFFF00000000003FULL);
509 
510 	if (!adev->gmc.translate_further)
511 		return;
512 
513 	if (level == AMDGPU_VM_PDB1) {
514 		/* Set the block fragment size */
515 		if (!(*flags & AMDGPU_PDE_PTE))
516 			*flags |= AMDGPU_PDE_BFS(0x9);
517 
518 	} else if (level == AMDGPU_VM_PDB0) {
519 		if (*flags & AMDGPU_PDE_PTE)
520 			*flags &= ~AMDGPU_PDE_PTE;
521 		else
522 			*flags |= AMDGPU_PTE_TF;
523 	}
524 }
525 
526 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
527 	.flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb,
528 	.emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb,
529 	.emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping,
530 	.set_pte_pde = gmc_v9_0_set_pte_pde,
531 	.get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags,
532 	.get_vm_pde = gmc_v9_0_get_vm_pde
533 };
534 
535 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
536 {
537 	if (adev->gmc.gmc_funcs == NULL)
538 		adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
539 }
540 
541 static int gmc_v9_0_early_init(void *handle)
542 {
543 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
544 
545 	gmc_v9_0_set_gmc_funcs(adev);
546 	gmc_v9_0_set_irq_funcs(adev);
547 
548 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
549 	adev->gmc.shared_aperture_end =
550 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
551 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
552 	adev->gmc.private_aperture_end =
553 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
554 
555 	return 0;
556 }
557 
558 static int gmc_v9_0_ecc_available(struct amdgpu_device *adev)
559 {
560 	uint32_t reg_val;
561 	uint32_t reg_addr;
562 	uint32_t field_val;
563 	size_t i;
564 	uint32_t fv2;
565 	size_t lost_sheep;
566 
567 	DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n");
568 
569 	lost_sheep = 0;
570 	for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) {
571 		reg_addr = ecc_umclocalcap_addrs[i];
572 		DRM_DEBUG("ecc: "
573 			  "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n",
574 			  i, reg_addr);
575 		reg_val = RREG32(reg_addr);
576 		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap,
577 					  EccDis);
578 		DRM_DEBUG("ecc: "
579 			  "reg_val: 0x%08x, "
580 			  "EccDis: 0x%08x, ",
581 			  reg_val, field_val);
582 		if (field_val) {
583 			DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n");
584 			++lost_sheep;
585 		}
586 	}
587 
588 	for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) {
589 		reg_addr = ecc_umcch_umc_config_addrs[i];
590 		DRM_DEBUG("ecc: "
591 			  "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x",
592 			  i, reg_addr);
593 		reg_val = RREG32(reg_addr);
594 		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG,
595 					  DramReady);
596 		DRM_DEBUG("ecc: "
597 			  "reg_val: 0x%08x, "
598 			  "DramReady: 0x%08x\n",
599 			  reg_val, field_val);
600 
601 		if (!field_val) {
602 			DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n");
603 			++lost_sheep;
604 		}
605 	}
606 
607 	for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) {
608 		reg_addr = ecc_umcch_eccctrl_addrs[i];
609 		DRM_DEBUG("ecc: "
610 			  "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ",
611 			  i, reg_addr);
612 		reg_val = RREG32(reg_addr);
613 		field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
614 					  WrEccEn);
615 		fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl,
616 				    RdEccEn);
617 		DRM_DEBUG("ecc: "
618 			  "reg_val: 0x%08x, "
619 			  "WrEccEn: 0x%08x, "
620 			  "RdEccEn: 0x%08x\n",
621 			  reg_val, field_val, fv2);
622 
623 		if (!field_val) {
624 			DRM_DEBUG("ecc: WrEccEn is not set\n");
625 			++lost_sheep;
626 		}
627 		if (!fv2) {
628 			DRM_DEBUG("ecc: RdEccEn is not set\n");
629 			++lost_sheep;
630 		}
631 	}
632 
633 	DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep);
634 	return lost_sheep == 0;
635 }
636 
637 static int gmc_v9_0_late_init(void *handle)
638 {
639 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
640 	/*
641 	 * The latest engine allocation on gfx9 is:
642 	 * Engine 0, 1: idle
643 	 * Engine 2, 3: firmware
644 	 * Engine 4~13: amdgpu ring, subject to change when ring number changes
645 	 * Engine 14~15: idle
646 	 * Engine 16: kfd tlb invalidation
647 	 * Engine 17: Gart flushes
648 	 */
649 	unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 };
650 	unsigned i;
651 	int r;
652 
653 	/*
654 	 * TODO - Uncomment once GART corruption issue is fixed.
655 	 */
656 	/* amdgpu_bo_late_init(adev); */
657 
658 	for(i = 0; i < adev->num_rings; ++i) {
659 		struct amdgpu_ring *ring = adev->rings[i];
660 		unsigned vmhub = ring->funcs->vmhub;
661 
662 		ring->vm_inv_eng = vm_inv_eng[vmhub]++;
663 		dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n",
664 			 ring->idx, ring->name, ring->vm_inv_eng,
665 			 ring->funcs->vmhub);
666 	}
667 
668 	/* Engine 16 is used for KFD and 17 for GART flushes */
669 	for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i)
670 		BUG_ON(vm_inv_eng[i] > 16);
671 
672 	if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) {
673 		r = gmc_v9_0_ecc_available(adev);
674 		if (r == 1) {
675 			DRM_INFO("ECC is active.\n");
676 		} else if (r == 0) {
677 			DRM_INFO("ECC is not present.\n");
678 			adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false);
679 		} else {
680 			DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r);
681 			return r;
682 		}
683 	}
684 
685 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
686 }
687 
688 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev,
689 					struct amdgpu_gmc *mc)
690 {
691 	u64 base = 0;
692 	if (!amdgpu_sriov_vf(adev))
693 		base = mmhub_v1_0_get_fb_location(adev);
694 	amdgpu_device_vram_location(adev, &adev->gmc, base);
695 	amdgpu_device_gart_location(adev, mc);
696 	/* base offset of vram pages */
697 	adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev);
698 }
699 
700 /**
701  * gmc_v9_0_mc_init - initialize the memory controller driver params
702  *
703  * @adev: amdgpu_device pointer
704  *
705  * Look up the amount of vram, vram width, and decide how to place
706  * vram and gart within the GPU's physical address space.
707  * Returns 0 for success.
708  */
709 static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
710 {
711 	int chansize, numchan;
712 	int r;
713 
714 	if (amdgpu_emu_mode != 1)
715 		adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev);
716 	if (!adev->gmc.vram_width) {
717 		/* hbm memory channel size */
718 		if (adev->flags & AMD_IS_APU)
719 			chansize = 64;
720 		else
721 			chansize = 128;
722 
723 		numchan = adev->df_funcs->get_hbm_channel_number(adev);
724 		adev->gmc.vram_width = numchan * chansize;
725 	}
726 
727 	/* size in MB on si */
728 	adev->gmc.mc_vram_size =
729 		adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL;
730 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
731 
732 	if (!(adev->flags & AMD_IS_APU)) {
733 		r = amdgpu_device_resize_fb_bar(adev);
734 		if (r)
735 			return r;
736 	}
737 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
738 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
739 
740 #ifdef CONFIG_X86_64
741 	if (adev->flags & AMD_IS_APU) {
742 		adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev);
743 		adev->gmc.aper_size = adev->gmc.real_vram_size;
744 	}
745 #endif
746 	/* In case the PCI BAR is larger than the actual amount of vram */
747 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
748 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
749 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
750 
751 	/* set the gart size */
752 	if (amdgpu_gart_size == -1) {
753 		switch (adev->asic_type) {
754 		case CHIP_VEGA10:  /* all engines support GPUVM */
755 		case CHIP_VEGA12:  /* all engines support GPUVM */
756 		case CHIP_VEGA20:
757 		default:
758 			adev->gmc.gart_size = 512ULL << 20;
759 			break;
760 		case CHIP_RAVEN:   /* DCE SG support */
761 			adev->gmc.gart_size = 1024ULL << 20;
762 			break;
763 		}
764 	} else {
765 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
766 	}
767 
768 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
769 
770 	return 0;
771 }
772 
773 static int gmc_v9_0_gart_init(struct amdgpu_device *adev)
774 {
775 	int r;
776 
777 	if (adev->gart.robj) {
778 		WARN(1, "VEGA10 PCIE GART already initialized\n");
779 		return 0;
780 	}
781 	/* Initialize common gart structure */
782 	r = amdgpu_gart_init(adev);
783 	if (r)
784 		return r;
785 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
786 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) |
787 				 AMDGPU_PTE_EXECUTABLE;
788 	return amdgpu_gart_table_vram_alloc(adev);
789 }
790 
791 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev)
792 {
793 #if 0
794 	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
795 #endif
796 	unsigned size;
797 
798 	/*
799 	 * TODO Remove once GART corruption is resolved
800 	 * Check related code in gmc_v9_0_sw_fini
801 	 * */
802 	size = 9 * 1024 * 1024;
803 
804 #if 0
805 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
806 		size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
807 	} else {
808 		u32 viewport;
809 
810 		switch (adev->asic_type) {
811 		case CHIP_RAVEN:
812 			viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
813 			size = (REG_GET_FIELD(viewport,
814 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
815 				REG_GET_FIELD(viewport,
816 					      HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) *
817 				4);
818 			break;
819 		case CHIP_VEGA10:
820 		case CHIP_VEGA12:
821 		default:
822 			viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE);
823 			size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
824 				REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) *
825 				4);
826 			break;
827 		}
828 	}
829 	/* return 0 if the pre-OS buffer uses up most of vram */
830 	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
831 		return 0;
832 
833 #endif
834 	return size;
835 }
836 
837 static int gmc_v9_0_sw_init(void *handle)
838 {
839 	int r;
840 	int dma_bits;
841 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
842 
843 	gfxhub_v1_0_init(adev);
844 	mmhub_v1_0_init(adev);
845 
846 	spin_lock_init(&adev->gmc.invalidate_lock);
847 
848 	adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev);
849 	switch (adev->asic_type) {
850 	case CHIP_RAVEN:
851 		if (adev->rev_id == 0x0 || adev->rev_id == 0x1) {
852 			amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
853 		} else {
854 			/* vm_size is 128TB + 512GB for legacy 3-level page support */
855 			amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48);
856 			adev->gmc.translate_further =
857 				adev->vm_manager.num_level > 1;
858 		}
859 		break;
860 	case CHIP_VEGA10:
861 	case CHIP_VEGA12:
862 	case CHIP_VEGA20:
863 		/*
864 		 * To fulfill 4-level page support,
865 		 * vm size is 256TB (48bit), maximum size of Vega10,
866 		 * block size 512 (9bit)
867 		 */
868 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
869 		break;
870 	default:
871 		break;
872 	}
873 
874 	/* This interrupt is VMC page fault.*/
875 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, 0,
876 				&adev->gmc.vm_fault);
877 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, 0,
878 				&adev->gmc.vm_fault);
879 
880 	if (r)
881 		return r;
882 
883 	/* Set the internal MC address mask
884 	 * This is the max address of the GPU's
885 	 * internal address space.
886 	 */
887 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
888 
889 	/* set DMA mask + need_dma32 flags.
890 	 * PCIE - can handle 44-bits.
891 	 * IGP - can handle 44-bits
892 	 * PCI - dma32 for legacy pci gart, 44 bits on vega10
893 	 */
894 	adev->need_dma32 = false;
895 	dma_bits = adev->need_dma32 ? 32 : 44;
896 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
897 	if (r) {
898 		adev->need_dma32 = true;
899 		dma_bits = 32;
900 		printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
901 	}
902 	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
903 	if (r) {
904 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
905 		printk(KERN_WARNING "amdgpu: No coherent DMA available.\n");
906 	}
907 	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
908 
909 	r = gmc_v9_0_mc_init(adev);
910 	if (r)
911 		return r;
912 
913 	adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev);
914 
915 	/* Memory manager */
916 	r = amdgpu_bo_init(adev);
917 	if (r)
918 		return r;
919 
920 	r = gmc_v9_0_gart_init(adev);
921 	if (r)
922 		return r;
923 
924 	/*
925 	 * number of VMs
926 	 * VMID 0 is reserved for System
927 	 * amdgpu graphics/compute will use VMIDs 1-7
928 	 * amdkfd will use VMIDs 8-15
929 	 */
930 	adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
931 	adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS;
932 
933 	amdgpu_vm_manager_init(adev);
934 
935 	return 0;
936 }
937 
938 /**
939  * gmc_v9_0_gart_fini - vm fini callback
940  *
941  * @adev: amdgpu_device pointer
942  *
943  * Tears down the driver GART/VM setup (CIK).
944  */
945 static void gmc_v9_0_gart_fini(struct amdgpu_device *adev)
946 {
947 	amdgpu_gart_table_vram_free(adev);
948 	amdgpu_gart_fini(adev);
949 }
950 
951 static int gmc_v9_0_sw_fini(void *handle)
952 {
953 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
954 
955 	amdgpu_gem_force_release(adev);
956 	amdgpu_vm_manager_fini(adev);
957 	gmc_v9_0_gart_fini(adev);
958 
959 	/*
960 	* TODO:
961 	* Currently there is a bug where some memory client outside
962 	* of the driver writes to first 8M of VRAM on S3 resume,
963 	* this overrides GART which by default gets placed in first 8M and
964 	* causes VM_FAULTS once GTT is accessed.
965 	* Keep the stolen memory reservation until the while this is not solved.
966 	* Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init
967 	*/
968 	amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL);
969 
970 	amdgpu_bo_fini(adev);
971 
972 	return 0;
973 }
974 
975 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev)
976 {
977 
978 	switch (adev->asic_type) {
979 	case CHIP_VEGA10:
980 	case CHIP_VEGA20:
981 		soc15_program_register_sequence(adev,
982 						golden_settings_mmhub_1_0_0,
983 						ARRAY_SIZE(golden_settings_mmhub_1_0_0));
984 		soc15_program_register_sequence(adev,
985 						golden_settings_athub_1_0_0,
986 						ARRAY_SIZE(golden_settings_athub_1_0_0));
987 		break;
988 	case CHIP_VEGA12:
989 		break;
990 	case CHIP_RAVEN:
991 		soc15_program_register_sequence(adev,
992 						golden_settings_athub_1_0_0,
993 						ARRAY_SIZE(golden_settings_athub_1_0_0));
994 		break;
995 	default:
996 		break;
997 	}
998 }
999 
1000 /**
1001  * gmc_v9_0_gart_enable - gart enable
1002  *
1003  * @adev: amdgpu_device pointer
1004  */
1005 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev)
1006 {
1007 	int r;
1008 	bool value;
1009 	u32 tmp;
1010 
1011 	amdgpu_device_program_register_sequence(adev,
1012 						golden_settings_vega10_hdp,
1013 						ARRAY_SIZE(golden_settings_vega10_hdp));
1014 
1015 	if (adev->gart.robj == NULL) {
1016 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1017 		return -EINVAL;
1018 	}
1019 	r = amdgpu_gart_table_vram_pin(adev);
1020 	if (r)
1021 		return r;
1022 
1023 	switch (adev->asic_type) {
1024 	case CHIP_RAVEN:
1025 		mmhub_v1_0_initialize_power_gating(adev);
1026 		mmhub_v1_0_update_power_gating(adev, true);
1027 		break;
1028 	default:
1029 		break;
1030 	}
1031 
1032 	r = gfxhub_v1_0_gart_enable(adev);
1033 	if (r)
1034 		return r;
1035 
1036 	r = mmhub_v1_0_gart_enable(adev);
1037 	if (r)
1038 		return r;
1039 
1040 	WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
1041 
1042 	tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL);
1043 	WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp);
1044 
1045 	/* After HDP is initialized, flush HDP.*/
1046 	adev->nbio_funcs->hdp_flush(adev, NULL);
1047 
1048 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
1049 		value = false;
1050 	else
1051 		value = true;
1052 
1053 	gfxhub_v1_0_set_fault_enable_default(adev, value);
1054 	mmhub_v1_0_set_fault_enable_default(adev, value);
1055 	gmc_v9_0_flush_gpu_tlb(adev, 0);
1056 
1057 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1058 		 (unsigned)(adev->gmc.gart_size >> 20),
1059 		 (unsigned long long)adev->gart.table_addr);
1060 	adev->gart.ready = true;
1061 	return 0;
1062 }
1063 
1064 static int gmc_v9_0_hw_init(void *handle)
1065 {
1066 	int r;
1067 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1068 
1069 	/* The sequence of these two function calls matters.*/
1070 	gmc_v9_0_init_golden_registers(adev);
1071 
1072 	if (adev->mode_info.num_crtc) {
1073 		/* Lockout access through VGA aperture*/
1074 		WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
1075 
1076 		/* disable VGA render */
1077 		WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
1078 	}
1079 
1080 	r = gmc_v9_0_gart_enable(adev);
1081 
1082 	return r;
1083 }
1084 
1085 /**
1086  * gmc_v9_0_gart_disable - gart disable
1087  *
1088  * @adev: amdgpu_device pointer
1089  *
1090  * This disables all VM page table.
1091  */
1092 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev)
1093 {
1094 	gfxhub_v1_0_gart_disable(adev);
1095 	mmhub_v1_0_gart_disable(adev);
1096 	amdgpu_gart_table_vram_unpin(adev);
1097 }
1098 
1099 static int gmc_v9_0_hw_fini(void *handle)
1100 {
1101 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1102 
1103 	if (amdgpu_sriov_vf(adev)) {
1104 		/* full access mode, so don't touch any GMC register */
1105 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1106 		return 0;
1107 	}
1108 
1109 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1110 	gmc_v9_0_gart_disable(adev);
1111 
1112 	return 0;
1113 }
1114 
1115 static int gmc_v9_0_suspend(void *handle)
1116 {
1117 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1118 
1119 	return gmc_v9_0_hw_fini(adev);
1120 }
1121 
1122 static int gmc_v9_0_resume(void *handle)
1123 {
1124 	int r;
1125 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1126 
1127 	r = gmc_v9_0_hw_init(adev);
1128 	if (r)
1129 		return r;
1130 
1131 	amdgpu_vmid_reset_all(adev);
1132 
1133 	return 0;
1134 }
1135 
1136 static bool gmc_v9_0_is_idle(void *handle)
1137 {
1138 	/* MC is always ready in GMC v9.*/
1139 	return true;
1140 }
1141 
1142 static int gmc_v9_0_wait_for_idle(void *handle)
1143 {
1144 	/* There is no need to wait for MC idle in GMC v9.*/
1145 	return 0;
1146 }
1147 
1148 static int gmc_v9_0_soft_reset(void *handle)
1149 {
1150 	/* XXX for emulation.*/
1151 	return 0;
1152 }
1153 
1154 static int gmc_v9_0_set_clockgating_state(void *handle,
1155 					enum amd_clockgating_state state)
1156 {
1157 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1158 
1159 	return mmhub_v1_0_set_clockgating(adev, state);
1160 }
1161 
1162 static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags)
1163 {
1164 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1165 
1166 	mmhub_v1_0_get_clockgating(adev, flags);
1167 }
1168 
1169 static int gmc_v9_0_set_powergating_state(void *handle,
1170 					enum amd_powergating_state state)
1171 {
1172 	return 0;
1173 }
1174 
1175 const struct amd_ip_funcs gmc_v9_0_ip_funcs = {
1176 	.name = "gmc_v9_0",
1177 	.early_init = gmc_v9_0_early_init,
1178 	.late_init = gmc_v9_0_late_init,
1179 	.sw_init = gmc_v9_0_sw_init,
1180 	.sw_fini = gmc_v9_0_sw_fini,
1181 	.hw_init = gmc_v9_0_hw_init,
1182 	.hw_fini = gmc_v9_0_hw_fini,
1183 	.suspend = gmc_v9_0_suspend,
1184 	.resume = gmc_v9_0_resume,
1185 	.is_idle = gmc_v9_0_is_idle,
1186 	.wait_for_idle = gmc_v9_0_wait_for_idle,
1187 	.soft_reset = gmc_v9_0_soft_reset,
1188 	.set_clockgating_state = gmc_v9_0_set_clockgating_state,
1189 	.set_powergating_state = gmc_v9_0_set_powergating_state,
1190 	.get_clockgating_state = gmc_v9_0_get_clockgating_state,
1191 };
1192 
1193 const struct amdgpu_ip_block_version gmc_v9_0_ip_block =
1194 {
1195 	.type = AMD_IP_BLOCK_TYPE_GMC,
1196 	.major = 9,
1197 	.minor = 0,
1198 	.rev = 0,
1199 	.funcs = &gmc_v9_0_ip_funcs,
1200 };
1201