1 /*
2  * Copyright 2019 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include <linux/pci.h>
25 
26 #include <drm/drm_cache.h>
27 
28 #include "amdgpu.h"
29 #include "amdgpu_atomfirmware.h"
30 #include "gmc_v10_0.h"
31 #include "umc_v8_7.h"
32 
33 #include "athub/athub_2_0_0_sh_mask.h"
34 #include "athub/athub_2_0_0_offset.h"
35 #include "dcn/dcn_2_0_0_offset.h"
36 #include "dcn/dcn_2_0_0_sh_mask.h"
37 #include "oss/osssys_5_0_0_offset.h"
38 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
39 #include "navi10_enum.h"
40 
41 #include "soc15.h"
42 #include "soc15d.h"
43 #include "soc15_common.h"
44 
45 #include "nbio_v2_3.h"
46 
47 #include "gfxhub_v2_0.h"
48 #include "gfxhub_v2_1.h"
49 #include "mmhub_v2_0.h"
50 #include "mmhub_v2_3.h"
51 #include "athub_v2_0.h"
52 #include "athub_v2_1.h"
53 
54 #include "amdgpu_reset.h"
55 
56 static int gmc_v10_0_ecc_interrupt_state(struct amdgpu_device *adev,
57 					 struct amdgpu_irq_src *src,
58 					 unsigned int type,
59 					 enum amdgpu_interrupt_state state)
60 {
61 	return 0;
62 }
63 
64 static int
65 gmc_v10_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
66 				   struct amdgpu_irq_src *src, unsigned int type,
67 				   enum amdgpu_interrupt_state state)
68 {
69 	switch (state) {
70 	case AMDGPU_IRQ_STATE_DISABLE:
71 		/* MM HUB */
72 		amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), false);
73 		/* GFX HUB */
74 		/* This works because this interrupt is only
75 		 * enabled at init/resume and disabled in
76 		 * fini/suspend, so the overall state doesn't
77 		 * change over the course of suspend/resume.
78 		 */
79 		if (!adev->in_s0ix)
80 			amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), false);
81 		break;
82 	case AMDGPU_IRQ_STATE_ENABLE:
83 		/* MM HUB */
84 		amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_MMHUB0(0), true);
85 		/* GFX HUB */
86 		/* This works because this interrupt is only
87 		 * enabled at init/resume and disabled in
88 		 * fini/suspend, so the overall state doesn't
89 		 * change over the course of suspend/resume.
90 		 */
91 		if (!adev->in_s0ix)
92 			amdgpu_gmc_set_vm_fault_masks(adev, AMDGPU_GFXHUB(0), true);
93 		break;
94 	default:
95 		break;
96 	}
97 
98 	return 0;
99 }
100 
101 static int gmc_v10_0_process_interrupt(struct amdgpu_device *adev,
102 				       struct amdgpu_irq_src *source,
103 				       struct amdgpu_iv_entry *entry)
104 {
105 	bool retry_fault = !!(entry->src_data[1] & 0x80);
106 	bool write_fault = !!(entry->src_data[1] & 0x20);
107 	struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src];
108 	struct amdgpu_task_info task_info;
109 	uint32_t status = 0;
110 	u64 addr;
111 
112 	addr = (u64)entry->src_data[0] << 12;
113 	addr |= ((u64)entry->src_data[1] & 0xf) << 44;
114 
115 	if (retry_fault) {
116 		/* Returning 1 here also prevents sending the IV to the KFD */
117 
118 		/* Process it onyl if it's the first fault for this address */
119 		if (entry->ih != &adev->irq.ih_soft &&
120 		    amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid,
121 					     entry->timestamp))
122 			return 1;
123 
124 		/* Delegate it to a different ring if the hardware hasn't
125 		 * already done it.
126 		 */
127 		if (entry->ih == &adev->irq.ih) {
128 			amdgpu_irq_delegate(adev, entry, 8);
129 			return 1;
130 		}
131 
132 		/* Try to handle the recoverable page faults by filling page
133 		 * tables
134 		 */
135 		if (amdgpu_vm_handle_fault(adev, entry->pasid, 0, 0, addr, write_fault))
136 			return 1;
137 	}
138 
139 	if (!amdgpu_sriov_vf(adev)) {
140 		/*
141 		 * Issue a dummy read to wait for the status register to
142 		 * be updated to avoid reading an incorrect value due to
143 		 * the new fast GRBM interface.
144 		 */
145 		if ((entry->vmid_src == AMDGPU_GFXHUB(0)) &&
146 		    (adev->ip_versions[GC_HWIP][0] < IP_VERSION(10, 3, 0)))
147 			RREG32(hub->vm_l2_pro_fault_status);
148 
149 		status = RREG32(hub->vm_l2_pro_fault_status);
150 		WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1);
151 	}
152 
153 	if (!printk_ratelimit())
154 		return 0;
155 
156 	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
157 	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
158 
159 	dev_err(adev->dev,
160 		"[%s] page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n",
161 		entry->vmid_src ? "mmhub" : "gfxhub",
162 		entry->src_id, entry->ring_id, entry->vmid,
163 		entry->pasid, task_info.process_name, task_info.tgid,
164 		task_info.task_name, task_info.pid);
165 	dev_err(adev->dev, "  in page starting at address 0x%016llx from client 0x%x (%s)\n",
166 		addr, entry->client_id,
167 		soc15_ih_clientid_name[entry->client_id]);
168 
169 	if (!amdgpu_sriov_vf(adev))
170 		hub->vmhub_funcs->print_l2_protection_fault_status(adev,
171 								   status);
172 
173 	return 0;
174 }
175 
176 static const struct amdgpu_irq_src_funcs gmc_v10_0_irq_funcs = {
177 	.set = gmc_v10_0_vm_fault_interrupt_state,
178 	.process = gmc_v10_0_process_interrupt,
179 };
180 
181 static const struct amdgpu_irq_src_funcs gmc_v10_0_ecc_funcs = {
182 	.set = gmc_v10_0_ecc_interrupt_state,
183 	.process = amdgpu_umc_process_ecc_irq,
184 };
185 
186 static void gmc_v10_0_set_irq_funcs(struct amdgpu_device *adev)
187 {
188 	adev->gmc.vm_fault.num_types = 1;
189 	adev->gmc.vm_fault.funcs = &gmc_v10_0_irq_funcs;
190 
191 	if (!amdgpu_sriov_vf(adev)) {
192 		adev->gmc.ecc_irq.num_types = 1;
193 		adev->gmc.ecc_irq.funcs = &gmc_v10_0_ecc_funcs;
194 	}
195 }
196 
197 /**
198  * gmc_v10_0_use_invalidate_semaphore - judge whether to use semaphore
199  *
200  * @adev: amdgpu_device pointer
201  * @vmhub: vmhub type
202  *
203  */
204 static bool gmc_v10_0_use_invalidate_semaphore(struct amdgpu_device *adev,
205 				       uint32_t vmhub)
206 {
207 	return ((vmhub == AMDGPU_MMHUB0(0)) &&
208 		(!amdgpu_sriov_vf(adev)));
209 }
210 
211 static bool gmc_v10_0_get_atc_vmid_pasid_mapping_info(
212 					struct amdgpu_device *adev,
213 					uint8_t vmid, uint16_t *p_pasid)
214 {
215 	uint32_t value;
216 
217 	value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING)
218 		     + vmid);
219 	*p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK;
220 
221 	return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK);
222 }
223 
224 /*
225  * GART
226  * VMID 0 is the physical GPU addresses as used by the kernel.
227  * VMIDs 1-15 are used for userspace clients and are handled
228  * by the amdgpu vm/hsa code.
229  */
230 
231 static void gmc_v10_0_flush_vm_hub(struct amdgpu_device *adev, uint32_t vmid,
232 				   unsigned int vmhub, uint32_t flush_type)
233 {
234 	bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(adev, vmhub);
235 	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
236 	u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
237 	u32 tmp;
238 	/* Use register 17 for GART */
239 	const unsigned int eng = 17;
240 	unsigned int i;
241 	unsigned char hub_ip = 0;
242 
243 	hub_ip = (vmhub == AMDGPU_GFXHUB(0)) ?
244 		   GC_HWIP : MMHUB_HWIP;
245 
246 	spin_lock(&adev->gmc.invalidate_lock);
247 	/*
248 	 * It may lose gpuvm invalidate acknowldege state across power-gating
249 	 * off cycle, add semaphore acquire before invalidation and semaphore
250 	 * release after invalidation to avoid entering power gated state
251 	 * to WA the Issue
252 	 */
253 
254 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
255 	if (use_semaphore) {
256 		for (i = 0; i < adev->usec_timeout; i++) {
257 			/* a read return value of 1 means semaphore acuqire */
258 			tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +
259 					 hub->eng_distance * eng, hub_ip);
260 
261 			if (tmp & 0x1)
262 				break;
263 			udelay(1);
264 		}
265 
266 		if (i >= adev->usec_timeout)
267 			DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n");
268 	}
269 
270 	WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req +
271 			  hub->eng_distance * eng,
272 			  inv_req, hub_ip);
273 
274 	/*
275 	 * Issue a dummy read to wait for the ACK register to be cleared
276 	 * to avoid a false ACK due to the new fast GRBM interface.
277 	 */
278 	if ((vmhub == AMDGPU_GFXHUB(0)) &&
279 	    (adev->ip_versions[GC_HWIP][0] < IP_VERSION(10, 3, 0)))
280 		RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_req +
281 				  hub->eng_distance * eng, hub_ip);
282 
283 	/* Wait for ACK with a delay.*/
284 	for (i = 0; i < adev->usec_timeout; i++) {
285 		tmp = RREG32_RLC_NO_KIQ(hub->vm_inv_eng0_ack +
286 				  hub->eng_distance * eng, hub_ip);
287 
288 		tmp &= 1 << vmid;
289 		if (tmp)
290 			break;
291 
292 		udelay(1);
293 	}
294 
295 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
296 	if (use_semaphore)
297 		/*
298 		 * add semaphore release after invalidation,
299 		 * write with 0 means semaphore release
300 		 */
301 		WREG32_RLC_NO_KIQ(hub->vm_inv_eng0_sem +
302 				  hub->eng_distance * eng, 0, hub_ip);
303 
304 	spin_unlock(&adev->gmc.invalidate_lock);
305 
306 	if (i < adev->usec_timeout)
307 		return;
308 
309 	DRM_ERROR("Timeout waiting for VM flush hub: %d!\n", vmhub);
310 }
311 
312 /**
313  * gmc_v10_0_flush_gpu_tlb - gart tlb flush callback
314  *
315  * @adev: amdgpu_device pointer
316  * @vmid: vm instance to flush
317  * @vmhub: vmhub type
318  * @flush_type: the flush type
319  *
320  * Flush the TLB for the requested page table.
321  */
322 static void gmc_v10_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
323 					uint32_t vmhub, uint32_t flush_type)
324 {
325 	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
326 	struct dma_fence *fence;
327 	struct amdgpu_job *job;
328 
329 	int r;
330 
331 	/* flush hdp cache */
332 	adev->hdp.funcs->flush_hdp(adev, NULL);
333 
334 	/* For SRIOV run time, driver shouldn't access the register through MMIO
335 	 * Directly use kiq to do the vm invalidation instead
336 	 */
337 	if (adev->gfx.kiq[0].ring.sched.ready && !adev->enable_mes &&
338 	    (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) &&
339 	    down_read_trylock(&adev->reset_domain->sem)) {
340 		struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
341 		const unsigned int eng = 17;
342 		u32 inv_req = hub->vmhub_funcs->get_invalidate_req(vmid, flush_type);
343 		u32 req = hub->vm_inv_eng0_req + hub->eng_distance * eng;
344 		u32 ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng;
345 
346 		amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req,
347 				1 << vmid);
348 
349 		up_read(&adev->reset_domain->sem);
350 		return;
351 	}
352 
353 	mutex_lock(&adev->mman.gtt_window_lock);
354 
355 	if (vmhub == AMDGPU_MMHUB0(0)) {
356 		gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_MMHUB0(0), 0);
357 		mutex_unlock(&adev->mman.gtt_window_lock);
358 		return;
359 	}
360 
361 	BUG_ON(vmhub != AMDGPU_GFXHUB(0));
362 
363 	if (!adev->mman.buffer_funcs_enabled ||
364 	    !adev->ib_pool_ready ||
365 	    amdgpu_in_reset(adev) ||
366 	    ring->sched.ready == false) {
367 		gmc_v10_0_flush_vm_hub(adev, vmid, AMDGPU_GFXHUB(0), 0);
368 		mutex_unlock(&adev->mman.gtt_window_lock);
369 		return;
370 	}
371 
372 	/* The SDMA on Navi has a bug which can theoretically result in memory
373 	 * corruption if an invalidation happens at the same time as an VA
374 	 * translation. Avoid this by doing the invalidation from the SDMA
375 	 * itself.
376 	 */
377 	r = amdgpu_job_alloc_with_ib(ring->adev, &adev->mman.high_pr,
378 				     AMDGPU_FENCE_OWNER_UNDEFINED,
379 				     16 * 4, AMDGPU_IB_POOL_IMMEDIATE,
380 				     &job);
381 	if (r)
382 		goto error_alloc;
383 
384 	job->vm_pd_addr = amdgpu_gmc_pd_addr(adev->gart.bo);
385 	job->vm_needs_flush = true;
386 	job->ibs->ptr[job->ibs->length_dw++] = ring->funcs->nop;
387 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
388 	fence = amdgpu_job_submit(job);
389 
390 	mutex_unlock(&adev->mman.gtt_window_lock);
391 
392 	dma_fence_wait(fence, false);
393 	dma_fence_put(fence);
394 
395 	return;
396 
397 error_alloc:
398 	mutex_unlock(&adev->mman.gtt_window_lock);
399 	DRM_ERROR("Error flushing GPU TLB using the SDMA (%d)!\n", r);
400 }
401 
402 /**
403  * gmc_v10_0_flush_gpu_tlb_pasid - tlb flush via pasid
404  *
405  * @adev: amdgpu_device pointer
406  * @pasid: pasid to be flush
407  * @flush_type: the flush type
408  * @all_hub: Used with PACKET3_INVALIDATE_TLBS_ALL_HUB()
409  * @inst: is used to select which instance of KIQ to use for the invalidation
410  *
411  * Flush the TLB for the requested pasid.
412  */
413 static int gmc_v10_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev,
414 					uint16_t pasid, uint32_t flush_type,
415 					bool all_hub, uint32_t inst)
416 {
417 	int vmid, i;
418 	signed long r;
419 	uint32_t seq;
420 	uint16_t queried_pasid;
421 	bool ret;
422 	u32 usec_timeout = amdgpu_sriov_vf(adev) ? SRIOV_USEC_TIMEOUT : adev->usec_timeout;
423 	struct amdgpu_ring *ring = &adev->gfx.kiq[0].ring;
424 	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
425 
426 	if (amdgpu_emu_mode == 0 && ring->sched.ready) {
427 		spin_lock(&adev->gfx.kiq[0].ring_lock);
428 		/* 2 dwords flush + 8 dwords fence */
429 		amdgpu_ring_alloc(ring, kiq->pmf->invalidate_tlbs_size + 8);
430 		kiq->pmf->kiq_invalidate_tlbs(ring,
431 					pasid, flush_type, all_hub);
432 		r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT);
433 		if (r) {
434 			amdgpu_ring_undo(ring);
435 			spin_unlock(&adev->gfx.kiq[0].ring_lock);
436 			return -ETIME;
437 		}
438 
439 		amdgpu_ring_commit(ring);
440 		spin_unlock(&adev->gfx.kiq[0].ring_lock);
441 		r = amdgpu_fence_wait_polling(ring, seq, usec_timeout);
442 		if (r < 1) {
443 			dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r);
444 			return -ETIME;
445 		}
446 
447 		return 0;
448 	}
449 
450 	for (vmid = 1; vmid < AMDGPU_NUM_VMID; vmid++) {
451 
452 		ret = gmc_v10_0_get_atc_vmid_pasid_mapping_info(adev, vmid,
453 				&queried_pasid);
454 		if (ret	&& queried_pasid == pasid) {
455 			if (all_hub) {
456 				for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS)
457 					gmc_v10_0_flush_gpu_tlb(adev, vmid,
458 							i, flush_type);
459 			} else {
460 				gmc_v10_0_flush_gpu_tlb(adev, vmid,
461 						AMDGPU_GFXHUB(0), flush_type);
462 			}
463 			if (!adev->enable_mes)
464 				break;
465 		}
466 	}
467 
468 	return 0;
469 }
470 
471 static uint64_t gmc_v10_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
472 					     unsigned int vmid, uint64_t pd_addr)
473 {
474 	bool use_semaphore = gmc_v10_0_use_invalidate_semaphore(ring->adev, ring->vm_hub);
475 	struct amdgpu_vmhub *hub = &ring->adev->vmhub[ring->vm_hub];
476 	uint32_t req = hub->vmhub_funcs->get_invalidate_req(vmid, 0);
477 	unsigned int eng = ring->vm_inv_eng;
478 
479 	/*
480 	 * It may lose gpuvm invalidate acknowldege state across power-gating
481 	 * off cycle, add semaphore acquire before invalidation and semaphore
482 	 * release after invalidation to avoid entering power gated state
483 	 * to WA the Issue
484 	 */
485 
486 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
487 	if (use_semaphore)
488 		/* a read return value of 1 means semaphore acuqire */
489 		amdgpu_ring_emit_reg_wait(ring,
490 					  hub->vm_inv_eng0_sem +
491 					  hub->eng_distance * eng, 0x1, 0x1);
492 
493 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 +
494 			      (hub->ctx_addr_distance * vmid),
495 			      lower_32_bits(pd_addr));
496 
497 	amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 +
498 			      (hub->ctx_addr_distance * vmid),
499 			      upper_32_bits(pd_addr));
500 
501 	amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req +
502 					    hub->eng_distance * eng,
503 					    hub->vm_inv_eng0_ack +
504 					    hub->eng_distance * eng,
505 					    req, 1 << vmid);
506 
507 	/* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */
508 	if (use_semaphore)
509 		/*
510 		 * add semaphore release after invalidation,
511 		 * write with 0 means semaphore release
512 		 */
513 		amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem +
514 				      hub->eng_distance * eng, 0);
515 
516 	return pd_addr;
517 }
518 
519 static void gmc_v10_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned int vmid,
520 					 unsigned int pasid)
521 {
522 	struct amdgpu_device *adev = ring->adev;
523 	uint32_t reg;
524 
525 	/* MES fw manages IH_VMID_x_LUT updating */
526 	if (ring->is_mes_queue)
527 		return;
528 
529 	if (ring->vm_hub == AMDGPU_GFXHUB(0))
530 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid;
531 	else
532 		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid;
533 
534 	amdgpu_ring_emit_wreg(ring, reg, pasid);
535 }
536 
537 /*
538  * PTE format on NAVI 10:
539  * 63:59 reserved
540  * 58 reserved and for sienna_cichlid is used for MALL noalloc
541  * 57 reserved
542  * 56 F
543  * 55 L
544  * 54 reserved
545  * 53:52 SW
546  * 51 T
547  * 50:48 mtype
548  * 47:12 4k physical page base address
549  * 11:7 fragment
550  * 6 write
551  * 5 read
552  * 4 exe
553  * 3 Z
554  * 2 snooped
555  * 1 system
556  * 0 valid
557  *
558  * PDE format on NAVI 10:
559  * 63:59 block fragment size
560  * 58:55 reserved
561  * 54 P
562  * 53:48 reserved
563  * 47:6 physical base address of PD or PTE
564  * 5:3 reserved
565  * 2 C
566  * 1 system
567  * 0 valid
568  */
569 
570 static uint64_t gmc_v10_0_map_mtype(struct amdgpu_device *adev, uint32_t flags)
571 {
572 	switch (flags) {
573 	case AMDGPU_VM_MTYPE_DEFAULT:
574 		return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
575 	case AMDGPU_VM_MTYPE_NC:
576 		return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
577 	case AMDGPU_VM_MTYPE_WC:
578 		return AMDGPU_PTE_MTYPE_NV10(MTYPE_WC);
579 	case AMDGPU_VM_MTYPE_CC:
580 		return AMDGPU_PTE_MTYPE_NV10(MTYPE_CC);
581 	case AMDGPU_VM_MTYPE_UC:
582 		return AMDGPU_PTE_MTYPE_NV10(MTYPE_UC);
583 	default:
584 		return AMDGPU_PTE_MTYPE_NV10(MTYPE_NC);
585 	}
586 }
587 
588 static void gmc_v10_0_get_vm_pde(struct amdgpu_device *adev, int level,
589 				 uint64_t *addr, uint64_t *flags)
590 {
591 	if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM))
592 		*addr = amdgpu_gmc_vram_mc2pa(adev, *addr);
593 	BUG_ON(*addr & 0xFFFF00000000003FULL);
594 
595 	if (!adev->gmc.translate_further)
596 		return;
597 
598 	if (level == AMDGPU_VM_PDB1) {
599 		/* Set the block fragment size */
600 		if (!(*flags & AMDGPU_PDE_PTE))
601 			*flags |= AMDGPU_PDE_BFS(0x9);
602 
603 	} else if (level == AMDGPU_VM_PDB0) {
604 		if (*flags & AMDGPU_PDE_PTE)
605 			*flags &= ~AMDGPU_PDE_PTE;
606 		else
607 			*flags |= AMDGPU_PTE_TF;
608 	}
609 }
610 
611 static void gmc_v10_0_get_vm_pte(struct amdgpu_device *adev,
612 				 struct amdgpu_bo_va_mapping *mapping,
613 				 uint64_t *flags)
614 {
615 	struct amdgpu_bo *bo = mapping->bo_va->base.bo;
616 
617 	*flags &= ~AMDGPU_PTE_EXECUTABLE;
618 	*flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE;
619 
620 	*flags &= ~AMDGPU_PTE_MTYPE_NV10_MASK;
621 	*flags |= (mapping->flags & AMDGPU_PTE_MTYPE_NV10_MASK);
622 
623 	*flags &= ~AMDGPU_PTE_NOALLOC;
624 	*flags |= (mapping->flags & AMDGPU_PTE_NOALLOC);
625 
626 	if (mapping->flags & AMDGPU_PTE_PRT) {
627 		*flags |= AMDGPU_PTE_PRT;
628 		*flags |= AMDGPU_PTE_SNOOPED;
629 		*flags |= AMDGPU_PTE_LOG;
630 		*flags |= AMDGPU_PTE_SYSTEM;
631 		*flags &= ~AMDGPU_PTE_VALID;
632 	}
633 
634 	if (bo && bo->flags & (AMDGPU_GEM_CREATE_COHERENT |
635 			       AMDGPU_GEM_CREATE_UNCACHED))
636 		*flags = (*flags & ~AMDGPU_PTE_MTYPE_NV10_MASK) |
637 			 AMDGPU_PTE_MTYPE_NV10(MTYPE_UC);
638 }
639 
640 static unsigned int gmc_v10_0_get_vbios_fb_size(struct amdgpu_device *adev)
641 {
642 	u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL);
643 	unsigned int size;
644 
645 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
646 		size = AMDGPU_VBIOS_VGA_ALLOCATION;
647 	} else {
648 		u32 viewport;
649 		u32 pitch;
650 
651 		viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION);
652 		pitch = RREG32_SOC15(DCE, 0, mmHUBPREQ0_DCSURF_SURFACE_PITCH);
653 		size = (REG_GET_FIELD(viewport,
654 					HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) *
655 				REG_GET_FIELD(pitch, HUBPREQ0_DCSURF_SURFACE_PITCH, PITCH) *
656 				4);
657 	}
658 
659 	return size;
660 }
661 
662 static const struct amdgpu_gmc_funcs gmc_v10_0_gmc_funcs = {
663 	.flush_gpu_tlb = gmc_v10_0_flush_gpu_tlb,
664 	.flush_gpu_tlb_pasid = gmc_v10_0_flush_gpu_tlb_pasid,
665 	.emit_flush_gpu_tlb = gmc_v10_0_emit_flush_gpu_tlb,
666 	.emit_pasid_mapping = gmc_v10_0_emit_pasid_mapping,
667 	.map_mtype = gmc_v10_0_map_mtype,
668 	.get_vm_pde = gmc_v10_0_get_vm_pde,
669 	.get_vm_pte = gmc_v10_0_get_vm_pte,
670 	.get_vbios_fb_size = gmc_v10_0_get_vbios_fb_size,
671 };
672 
673 static void gmc_v10_0_set_gmc_funcs(struct amdgpu_device *adev)
674 {
675 	if (adev->gmc.gmc_funcs == NULL)
676 		adev->gmc.gmc_funcs = &gmc_v10_0_gmc_funcs;
677 }
678 
679 static void gmc_v10_0_set_umc_funcs(struct amdgpu_device *adev)
680 {
681 	switch (adev->ip_versions[UMC_HWIP][0]) {
682 	case IP_VERSION(8, 7, 0):
683 		adev->umc.max_ras_err_cnt_per_query = UMC_V8_7_TOTAL_CHANNEL_NUM;
684 		adev->umc.channel_inst_num = UMC_V8_7_CHANNEL_INSTANCE_NUM;
685 		adev->umc.umc_inst_num = UMC_V8_7_UMC_INSTANCE_NUM;
686 		adev->umc.channel_offs = UMC_V8_7_PER_CHANNEL_OFFSET_SIENNA;
687 		adev->umc.retire_unit = 1;
688 		adev->umc.channel_idx_tbl = &umc_v8_7_channel_idx_tbl[0][0];
689 		adev->umc.ras = &umc_v8_7_ras;
690 		break;
691 	default:
692 		break;
693 	}
694 }
695 
696 static void gmc_v10_0_set_mmhub_funcs(struct amdgpu_device *adev)
697 {
698 	switch (adev->ip_versions[MMHUB_HWIP][0]) {
699 	case IP_VERSION(2, 3, 0):
700 	case IP_VERSION(2, 4, 0):
701 	case IP_VERSION(2, 4, 1):
702 		adev->mmhub.funcs = &mmhub_v2_3_funcs;
703 		break;
704 	default:
705 		adev->mmhub.funcs = &mmhub_v2_0_funcs;
706 		break;
707 	}
708 }
709 
710 static void gmc_v10_0_set_gfxhub_funcs(struct amdgpu_device *adev)
711 {
712 	switch (adev->ip_versions[GC_HWIP][0]) {
713 	case IP_VERSION(10, 3, 0):
714 	case IP_VERSION(10, 3, 2):
715 	case IP_VERSION(10, 3, 1):
716 	case IP_VERSION(10, 3, 4):
717 	case IP_VERSION(10, 3, 5):
718 	case IP_VERSION(10, 3, 6):
719 	case IP_VERSION(10, 3, 3):
720 	case IP_VERSION(10, 3, 7):
721 		adev->gfxhub.funcs = &gfxhub_v2_1_funcs;
722 		break;
723 	default:
724 		adev->gfxhub.funcs = &gfxhub_v2_0_funcs;
725 		break;
726 	}
727 }
728 
729 
730 static int gmc_v10_0_early_init(void *handle)
731 {
732 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
733 
734 	gmc_v10_0_set_mmhub_funcs(adev);
735 	gmc_v10_0_set_gfxhub_funcs(adev);
736 	gmc_v10_0_set_gmc_funcs(adev);
737 	gmc_v10_0_set_irq_funcs(adev);
738 	gmc_v10_0_set_umc_funcs(adev);
739 
740 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
741 	adev->gmc.shared_aperture_end =
742 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
743 	adev->gmc.private_aperture_start = 0x1000000000000000ULL;
744 	adev->gmc.private_aperture_end =
745 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
746 	adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF;
747 
748 	return 0;
749 }
750 
751 static int gmc_v10_0_late_init(void *handle)
752 {
753 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
754 	int r;
755 
756 	r = amdgpu_gmc_allocate_vm_inv_eng(adev);
757 	if (r)
758 		return r;
759 
760 	r = amdgpu_gmc_ras_late_init(adev);
761 	if (r)
762 		return r;
763 
764 	return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
765 }
766 
767 static void gmc_v10_0_vram_gtt_location(struct amdgpu_device *adev,
768 					struct amdgpu_gmc *mc)
769 {
770 	u64 base = 0;
771 
772 	base = adev->gfxhub.funcs->get_fb_location(adev);
773 
774 	/* add the xgmi offset of the physical node */
775 	base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
776 
777 	amdgpu_gmc_vram_location(adev, &adev->gmc, base);
778 	amdgpu_gmc_gart_location(adev, mc);
779 	amdgpu_gmc_agp_location(adev, mc);
780 
781 	/* base offset of vram pages */
782 	adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev);
783 
784 	/* add the xgmi offset of the physical node */
785 	adev->vm_manager.vram_base_offset +=
786 		adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size;
787 }
788 
789 /**
790  * gmc_v10_0_mc_init - initialize the memory controller driver params
791  *
792  * @adev: amdgpu_device pointer
793  *
794  * Look up the amount of vram, vram width, and decide how to place
795  * vram and gart within the GPU's physical address space.
796  * Returns 0 for success.
797  */
798 static int gmc_v10_0_mc_init(struct amdgpu_device *adev)
799 {
800 	int r;
801 
802 	/* size in MB on si */
803 	adev->gmc.mc_vram_size =
804 		adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
805 	adev->gmc.real_vram_size = adev->gmc.mc_vram_size;
806 
807 	if (!(adev->flags & AMD_IS_APU)) {
808 		r = amdgpu_device_resize_fb_bar(adev);
809 		if (r)
810 			return r;
811 	}
812 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
813 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
814 
815 #ifdef CONFIG_X86_64
816 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) {
817 		adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev);
818 		adev->gmc.aper_size = adev->gmc.real_vram_size;
819 	}
820 #endif
821 
822 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
823 
824 	/* set the gart size */
825 	if (amdgpu_gart_size == -1) {
826 		switch (adev->ip_versions[GC_HWIP][0]) {
827 		default:
828 			adev->gmc.gart_size = 512ULL << 20;
829 			break;
830 		case IP_VERSION(10, 3, 1):   /* DCE SG support */
831 		case IP_VERSION(10, 3, 3):   /* DCE SG support */
832 		case IP_VERSION(10, 3, 6):   /* DCE SG support */
833 		case IP_VERSION(10, 3, 7):   /* DCE SG support */
834 			adev->gmc.gart_size = 1024ULL << 20;
835 			break;
836 		}
837 	} else {
838 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
839 	}
840 
841 	gmc_v10_0_vram_gtt_location(adev, &adev->gmc);
842 
843 	return 0;
844 }
845 
846 static int gmc_v10_0_gart_init(struct amdgpu_device *adev)
847 {
848 	int r;
849 
850 	if (adev->gart.bo) {
851 		WARN(1, "NAVI10 PCIE GART already initialized\n");
852 		return 0;
853 	}
854 
855 	/* Initialize common gart structure */
856 	r = amdgpu_gart_init(adev);
857 	if (r)
858 		return r;
859 
860 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
861 	adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_NV10(MTYPE_UC) |
862 				 AMDGPU_PTE_EXECUTABLE;
863 
864 	return amdgpu_gart_table_vram_alloc(adev);
865 }
866 
867 static int gmc_v10_0_sw_init(void *handle)
868 {
869 	int r, vram_width = 0, vram_type = 0, vram_vendor = 0;
870 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
871 
872 	adev->gfxhub.funcs->init(adev);
873 
874 	adev->mmhub.funcs->init(adev);
875 
876 	spin_lock_init(&adev->gmc.invalidate_lock);
877 
878 	if ((adev->flags & AMD_IS_APU) && amdgpu_emu_mode == 1) {
879 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4;
880 		adev->gmc.vram_width = 64;
881 	} else if (amdgpu_emu_mode == 1) {
882 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_GDDR6;
883 		adev->gmc.vram_width = 1 * 128; /* numchan * chansize */
884 	} else {
885 		r = amdgpu_atomfirmware_get_vram_info(adev,
886 				&vram_width, &vram_type, &vram_vendor);
887 		adev->gmc.vram_width = vram_width;
888 
889 		adev->gmc.vram_type = vram_type;
890 		adev->gmc.vram_vendor = vram_vendor;
891 	}
892 
893 	switch (adev->ip_versions[GC_HWIP][0]) {
894 	case IP_VERSION(10, 3, 0):
895 		adev->gmc.mall_size = 128 * 1024 * 1024;
896 		break;
897 	case IP_VERSION(10, 3, 2):
898 		adev->gmc.mall_size = 96 * 1024 * 1024;
899 		break;
900 	case IP_VERSION(10, 3, 4):
901 		adev->gmc.mall_size = 32 * 1024 * 1024;
902 		break;
903 	case IP_VERSION(10, 3, 5):
904 		adev->gmc.mall_size = 16 * 1024 * 1024;
905 		break;
906 	default:
907 		adev->gmc.mall_size = 0;
908 		break;
909 	}
910 
911 	switch (adev->ip_versions[GC_HWIP][0]) {
912 	case IP_VERSION(10, 1, 10):
913 	case IP_VERSION(10, 1, 1):
914 	case IP_VERSION(10, 1, 2):
915 	case IP_VERSION(10, 1, 3):
916 	case IP_VERSION(10, 1, 4):
917 	case IP_VERSION(10, 3, 0):
918 	case IP_VERSION(10, 3, 2):
919 	case IP_VERSION(10, 3, 1):
920 	case IP_VERSION(10, 3, 4):
921 	case IP_VERSION(10, 3, 5):
922 	case IP_VERSION(10, 3, 6):
923 	case IP_VERSION(10, 3, 3):
924 	case IP_VERSION(10, 3, 7):
925 		set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
926 		set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask);
927 		/*
928 		 * To fulfill 4-level page support,
929 		 * vm size is 256TB (48bit), maximum size of Navi10/Navi14/Navi12,
930 		 * block size 512 (9bit)
931 		 */
932 		amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48);
933 		break;
934 	default:
935 		break;
936 	}
937 
938 	/* This interrupt is VMC page fault.*/
939 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC,
940 			      VMC_1_0__SRCID__VM_FAULT,
941 			      &adev->gmc.vm_fault);
942 
943 	if (r)
944 		return r;
945 
946 	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2,
947 			      UTCL2_1_0__SRCID__FAULT,
948 			      &adev->gmc.vm_fault);
949 	if (r)
950 		return r;
951 
952 	if (!amdgpu_sriov_vf(adev)) {
953 		/* interrupt sent to DF. */
954 		r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0,
955 				      &adev->gmc.ecc_irq);
956 		if (r)
957 			return r;
958 	}
959 
960 	/*
961 	 * Set the internal MC address mask This is the max address of the GPU's
962 	 * internal address space.
963 	 */
964 	adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */
965 
966 	r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44));
967 	if (r) {
968 		dev_warn(adev->dev, "amdgpu: No suitable DMA available.\n");
969 		return r;
970 	}
971 
972 	adev->need_swiotlb = drm_need_swiotlb(44);
973 
974 	r = gmc_v10_0_mc_init(adev);
975 	if (r)
976 		return r;
977 
978 	amdgpu_gmc_get_vbios_allocations(adev);
979 
980 	/* Memory manager */
981 	r = amdgpu_bo_init(adev);
982 	if (r)
983 		return r;
984 
985 	r = gmc_v10_0_gart_init(adev);
986 	if (r)
987 		return r;
988 
989 	/*
990 	 * number of VMs
991 	 * VMID 0 is reserved for System
992 	 * amdgpu graphics/compute will use VMIDs 1-7
993 	 * amdkfd will use VMIDs 8-15
994 	 */
995 	adev->vm_manager.first_kfd_vmid = 8;
996 
997 	amdgpu_vm_manager_init(adev);
998 
999 	r = amdgpu_gmc_ras_sw_init(adev);
1000 	if (r)
1001 		return r;
1002 
1003 	return 0;
1004 }
1005 
1006 /**
1007  * gmc_v10_0_gart_fini - vm fini callback
1008  *
1009  * @adev: amdgpu_device pointer
1010  *
1011  * Tears down the driver GART/VM setup (CIK).
1012  */
1013 static void gmc_v10_0_gart_fini(struct amdgpu_device *adev)
1014 {
1015 	amdgpu_gart_table_vram_free(adev);
1016 }
1017 
1018 static int gmc_v10_0_sw_fini(void *handle)
1019 {
1020 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1021 
1022 	amdgpu_vm_manager_fini(adev);
1023 	gmc_v10_0_gart_fini(adev);
1024 	amdgpu_gem_force_release(adev);
1025 	amdgpu_bo_fini(adev);
1026 
1027 	return 0;
1028 }
1029 
1030 static void gmc_v10_0_init_golden_registers(struct amdgpu_device *adev)
1031 {
1032 }
1033 
1034 /**
1035  * gmc_v10_0_gart_enable - gart enable
1036  *
1037  * @adev: amdgpu_device pointer
1038  */
1039 static int gmc_v10_0_gart_enable(struct amdgpu_device *adev)
1040 {
1041 	int r;
1042 	bool value;
1043 
1044 	if (adev->gart.bo == NULL) {
1045 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
1046 		return -EINVAL;
1047 	}
1048 
1049 	amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
1050 
1051 	if (!adev->in_s0ix) {
1052 		r = adev->gfxhub.funcs->gart_enable(adev);
1053 		if (r)
1054 			return r;
1055 	}
1056 
1057 	r = adev->mmhub.funcs->gart_enable(adev);
1058 	if (r)
1059 		return r;
1060 
1061 	adev->hdp.funcs->init_registers(adev);
1062 
1063 	/* Flush HDP after it is initialized */
1064 	adev->hdp.funcs->flush_hdp(adev, NULL);
1065 
1066 	value = (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) ?
1067 		false : true;
1068 
1069 	if (!adev->in_s0ix)
1070 		adev->gfxhub.funcs->set_fault_enable_default(adev, value);
1071 	adev->mmhub.funcs->set_fault_enable_default(adev, value);
1072 	gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_MMHUB0(0), 0);
1073 	if (!adev->in_s0ix)
1074 		gmc_v10_0_flush_gpu_tlb(adev, 0, AMDGPU_GFXHUB(0), 0);
1075 
1076 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
1077 		 (unsigned int)(adev->gmc.gart_size >> 20),
1078 		 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo));
1079 
1080 	return 0;
1081 }
1082 
1083 static int gmc_v10_0_hw_init(void *handle)
1084 {
1085 	int r;
1086 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1087 
1088 	/* The sequence of these two function calls matters.*/
1089 	gmc_v10_0_init_golden_registers(adev);
1090 
1091 	/*
1092 	 * harvestable groups in gc_utcl2 need to be programmed before any GFX block
1093 	 * register setup within GMC, or else system hang when harvesting SA.
1094 	 */
1095 	if (!adev->in_s0ix && adev->gfxhub.funcs && adev->gfxhub.funcs->utcl2_harvest)
1096 		adev->gfxhub.funcs->utcl2_harvest(adev);
1097 
1098 	r = gmc_v10_0_gart_enable(adev);
1099 	if (r)
1100 		return r;
1101 
1102 	if (amdgpu_emu_mode == 1) {
1103 		r = amdgpu_gmc_vram_checking(adev);
1104 		if (r)
1105 			return r;
1106 	}
1107 
1108 	if (adev->umc.funcs && adev->umc.funcs->init_registers)
1109 		adev->umc.funcs->init_registers(adev);
1110 
1111 	return 0;
1112 }
1113 
1114 /**
1115  * gmc_v10_0_gart_disable - gart disable
1116  *
1117  * @adev: amdgpu_device pointer
1118  *
1119  * This disables all VM page table.
1120  */
1121 static void gmc_v10_0_gart_disable(struct amdgpu_device *adev)
1122 {
1123 	if (!adev->in_s0ix)
1124 		adev->gfxhub.funcs->gart_disable(adev);
1125 	adev->mmhub.funcs->gart_disable(adev);
1126 }
1127 
1128 static int gmc_v10_0_hw_fini(void *handle)
1129 {
1130 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1131 
1132 	gmc_v10_0_gart_disable(adev);
1133 
1134 	if (amdgpu_sriov_vf(adev)) {
1135 		/* full access mode, so don't touch any GMC register */
1136 		DRM_DEBUG("For SRIOV client, shouldn't do anything.\n");
1137 		return 0;
1138 	}
1139 
1140 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1141 
1142 	return 0;
1143 }
1144 
1145 static int gmc_v10_0_suspend(void *handle)
1146 {
1147 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1148 
1149 	gmc_v10_0_hw_fini(adev);
1150 
1151 	return 0;
1152 }
1153 
1154 static int gmc_v10_0_resume(void *handle)
1155 {
1156 	int r;
1157 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1158 
1159 	r = gmc_v10_0_hw_init(adev);
1160 	if (r)
1161 		return r;
1162 
1163 	amdgpu_vmid_reset_all(adev);
1164 
1165 	return 0;
1166 }
1167 
1168 static bool gmc_v10_0_is_idle(void *handle)
1169 {
1170 	/* MC is always ready in GMC v10.*/
1171 	return true;
1172 }
1173 
1174 static int gmc_v10_0_wait_for_idle(void *handle)
1175 {
1176 	/* There is no need to wait for MC idle in GMC v10.*/
1177 	return 0;
1178 }
1179 
1180 static int gmc_v10_0_soft_reset(void *handle)
1181 {
1182 	return 0;
1183 }
1184 
1185 static int gmc_v10_0_set_clockgating_state(void *handle,
1186 					   enum amd_clockgating_state state)
1187 {
1188 	int r;
1189 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1190 
1191 	/*
1192 	 * The issue mmhub can't disconnect from DF with MMHUB clock gating being disabled
1193 	 * is a new problem observed at DF 3.0.3, however with the same suspend sequence not
1194 	 * seen any issue on the DF 3.0.2 series platform.
1195 	 */
1196 	if (adev->in_s0ix && adev->ip_versions[DF_HWIP][0] > IP_VERSION(3, 0, 2)) {
1197 		dev_dbg(adev->dev, "keep mmhub clock gating being enabled for s0ix\n");
1198 		return 0;
1199 	}
1200 
1201 	r = adev->mmhub.funcs->set_clockgating(adev, state);
1202 	if (r)
1203 		return r;
1204 
1205 	if (adev->ip_versions[ATHUB_HWIP][0] >= IP_VERSION(2, 1, 0))
1206 		return athub_v2_1_set_clockgating(adev, state);
1207 	else
1208 		return athub_v2_0_set_clockgating(adev, state);
1209 }
1210 
1211 static void gmc_v10_0_get_clockgating_state(void *handle, u64 *flags)
1212 {
1213 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1214 
1215 	if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(10, 1, 3) ||
1216 	    adev->ip_versions[GC_HWIP][0] == IP_VERSION(10, 1, 4))
1217 		return;
1218 
1219 	adev->mmhub.funcs->get_clockgating(adev, flags);
1220 
1221 	if (adev->ip_versions[ATHUB_HWIP][0] >= IP_VERSION(2, 1, 0))
1222 		athub_v2_1_get_clockgating(adev, flags);
1223 	else
1224 		athub_v2_0_get_clockgating(adev, flags);
1225 }
1226 
1227 static int gmc_v10_0_set_powergating_state(void *handle,
1228 					   enum amd_powergating_state state)
1229 {
1230 	return 0;
1231 }
1232 
1233 const struct amd_ip_funcs gmc_v10_0_ip_funcs = {
1234 	.name = "gmc_v10_0",
1235 	.early_init = gmc_v10_0_early_init,
1236 	.late_init = gmc_v10_0_late_init,
1237 	.sw_init = gmc_v10_0_sw_init,
1238 	.sw_fini = gmc_v10_0_sw_fini,
1239 	.hw_init = gmc_v10_0_hw_init,
1240 	.hw_fini = gmc_v10_0_hw_fini,
1241 	.suspend = gmc_v10_0_suspend,
1242 	.resume = gmc_v10_0_resume,
1243 	.is_idle = gmc_v10_0_is_idle,
1244 	.wait_for_idle = gmc_v10_0_wait_for_idle,
1245 	.soft_reset = gmc_v10_0_soft_reset,
1246 	.set_clockgating_state = gmc_v10_0_set_clockgating_state,
1247 	.set_powergating_state = gmc_v10_0_set_powergating_state,
1248 	.get_clockgating_state = gmc_v10_0_get_clockgating_state,
1249 };
1250 
1251 const struct amdgpu_ip_block_version gmc_v10_0_ip_block = {
1252 	.type = AMD_IP_BLOCK_TYPE_GMC,
1253 	.major = 10,
1254 	.minor = 0,
1255 	.rev = 0,
1256 	.funcs = &gmc_v10_0_ip_funcs,
1257 };
1258