xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c (revision 82003e04)
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include "drmP.h"
25 #include "amdgpu.h"
26 #include "cikd.h"
27 #include "cik.h"
28 #include "gmc_v7_0.h"
29 #include "amdgpu_ucode.h"
30 
31 #include "bif/bif_4_1_d.h"
32 #include "bif/bif_4_1_sh_mask.h"
33 
34 #include "gmc/gmc_7_1_d.h"
35 #include "gmc/gmc_7_1_sh_mask.h"
36 
37 #include "oss/oss_2_0_d.h"
38 #include "oss/oss_2_0_sh_mask.h"
39 
40 static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev);
41 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
42 static int gmc_v7_0_wait_for_idle(void *handle);
43 
44 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
45 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
46 MODULE_FIRMWARE("amdgpu/topaz_mc.bin");
47 
48 static const u32 golden_settings_iceland_a11[] =
49 {
50 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
51 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
52 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
53 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
54 };
55 
56 static const u32 iceland_mgcg_cgcg_init[] =
57 {
58 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
59 };
60 
61 static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
62 {
63 	switch (adev->asic_type) {
64 	case CHIP_TOPAZ:
65 		amdgpu_program_register_sequence(adev,
66 						 iceland_mgcg_cgcg_init,
67 						 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
68 		amdgpu_program_register_sequence(adev,
69 						 golden_settings_iceland_a11,
70 						 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
71 		break;
72 	default:
73 		break;
74 	}
75 }
76 
77 static void gmc_v7_0_mc_stop(struct amdgpu_device *adev,
78 			     struct amdgpu_mode_mc_save *save)
79 {
80 	u32 blackout;
81 
82 	if (adev->mode_info.num_crtc)
83 		amdgpu_display_stop_mc_access(adev, save);
84 
85 	gmc_v7_0_wait_for_idle((void *)adev);
86 
87 	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
88 	if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
89 		/* Block CPU access */
90 		WREG32(mmBIF_FB_EN, 0);
91 		/* blackout the MC */
92 		blackout = REG_SET_FIELD(blackout,
93 					 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
94 		WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
95 	}
96 	/* wait for the MC to settle */
97 	udelay(100);
98 }
99 
100 static void gmc_v7_0_mc_resume(struct amdgpu_device *adev,
101 			       struct amdgpu_mode_mc_save *save)
102 {
103 	u32 tmp;
104 
105 	/* unblackout the MC */
106 	tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
107 	tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
108 	WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
109 	/* allow CPU access */
110 	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
111 	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
112 	WREG32(mmBIF_FB_EN, tmp);
113 
114 	if (adev->mode_info.num_crtc)
115 		amdgpu_display_resume_mc_access(adev, save);
116 }
117 
118 /**
119  * gmc_v7_0_init_microcode - load ucode images from disk
120  *
121  * @adev: amdgpu_device pointer
122  *
123  * Use the firmware interface to load the ucode images into
124  * the driver (not loaded into hw).
125  * Returns 0 on success, error on failure.
126  */
127 static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)
128 {
129 	const char *chip_name;
130 	char fw_name[30];
131 	int err;
132 
133 	DRM_DEBUG("\n");
134 
135 	switch (adev->asic_type) {
136 	case CHIP_BONAIRE:
137 		chip_name = "bonaire";
138 		break;
139 	case CHIP_HAWAII:
140 		chip_name = "hawaii";
141 		break;
142 	case CHIP_TOPAZ:
143 		chip_name = "topaz";
144 		break;
145 	case CHIP_KAVERI:
146 	case CHIP_KABINI:
147 	case CHIP_MULLINS:
148 		return 0;
149 	default: BUG();
150 	}
151 
152 	if (adev->asic_type == CHIP_TOPAZ)
153 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
154 	else
155 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
156 
157 	err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
158 	if (err)
159 		goto out;
160 	err = amdgpu_ucode_validate(adev->mc.fw);
161 
162 out:
163 	if (err) {
164 		printk(KERN_ERR
165 		       "cik_mc: Failed to load firmware \"%s\"\n",
166 		       fw_name);
167 		release_firmware(adev->mc.fw);
168 		adev->mc.fw = NULL;
169 	}
170 	return err;
171 }
172 
173 /**
174  * gmc_v7_0_mc_load_microcode - load MC ucode into the hw
175  *
176  * @adev: amdgpu_device pointer
177  *
178  * Load the GDDR MC ucode into the hw (CIK).
179  * Returns 0 on success, error on failure.
180  */
181 static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
182 {
183 	const struct mc_firmware_header_v1_0 *hdr;
184 	const __le32 *fw_data = NULL;
185 	const __le32 *io_mc_regs = NULL;
186 	u32 running;
187 	int i, ucode_size, regs_size;
188 
189 	if (!adev->mc.fw)
190 		return -EINVAL;
191 
192 	hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
193 	amdgpu_ucode_print_mc_hdr(&hdr->header);
194 
195 	adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
196 	regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
197 	io_mc_regs = (const __le32 *)
198 		(adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
199 	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
200 	fw_data = (const __le32 *)
201 		(adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
202 
203 	running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
204 
205 	if (running == 0) {
206 		/* reset the engine and set to writable */
207 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
208 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
209 
210 		/* load mc io regs */
211 		for (i = 0; i < regs_size; i++) {
212 			WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
213 			WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
214 		}
215 		/* load the MC ucode */
216 		for (i = 0; i < ucode_size; i++)
217 			WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
218 
219 		/* put the engine back into the active state */
220 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
221 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
222 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
223 
224 		/* wait for training to complete */
225 		for (i = 0; i < adev->usec_timeout; i++) {
226 			if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
227 					  MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
228 				break;
229 			udelay(1);
230 		}
231 		for (i = 0; i < adev->usec_timeout; i++) {
232 			if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
233 					  MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
234 				break;
235 			udelay(1);
236 		}
237 	}
238 
239 	return 0;
240 }
241 
242 static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
243 				       struct amdgpu_mc *mc)
244 {
245 	if (mc->mc_vram_size > 0xFFC0000000ULL) {
246 		/* leave room for at least 1024M GTT */
247 		dev_warn(adev->dev, "limiting VRAM\n");
248 		mc->real_vram_size = 0xFFC0000000ULL;
249 		mc->mc_vram_size = 0xFFC0000000ULL;
250 	}
251 	amdgpu_vram_location(adev, &adev->mc, 0);
252 	adev->mc.gtt_base_align = 0;
253 	amdgpu_gtt_location(adev, mc);
254 }
255 
256 /**
257  * gmc_v7_0_mc_program - program the GPU memory controller
258  *
259  * @adev: amdgpu_device pointer
260  *
261  * Set the location of vram, gart, and AGP in the GPU's
262  * physical address space (CIK).
263  */
264 static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
265 {
266 	struct amdgpu_mode_mc_save save;
267 	u32 tmp;
268 	int i, j;
269 
270 	/* Initialize HDP */
271 	for (i = 0, j = 0; i < 32; i++, j += 0x6) {
272 		WREG32((0xb05 + j), 0x00000000);
273 		WREG32((0xb06 + j), 0x00000000);
274 		WREG32((0xb07 + j), 0x00000000);
275 		WREG32((0xb08 + j), 0x00000000);
276 		WREG32((0xb09 + j), 0x00000000);
277 	}
278 	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
279 
280 	if (adev->mode_info.num_crtc)
281 		amdgpu_display_set_vga_render_state(adev, false);
282 
283 	gmc_v7_0_mc_stop(adev, &save);
284 	if (gmc_v7_0_wait_for_idle((void *)adev)) {
285 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
286 	}
287 	/* Update configuration */
288 	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
289 	       adev->mc.vram_start >> 12);
290 	WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
291 	       adev->mc.vram_end >> 12);
292 	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
293 	       adev->vram_scratch.gpu_addr >> 12);
294 	tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
295 	tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
296 	WREG32(mmMC_VM_FB_LOCATION, tmp);
297 	/* XXX double check these! */
298 	WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
299 	WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
300 	WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
301 	WREG32(mmMC_VM_AGP_BASE, 0);
302 	WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
303 	WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
304 	if (gmc_v7_0_wait_for_idle((void *)adev)) {
305 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
306 	}
307 	gmc_v7_0_mc_resume(adev, &save);
308 
309 	WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
310 
311 	tmp = RREG32(mmHDP_MISC_CNTL);
312 	tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
313 	WREG32(mmHDP_MISC_CNTL, tmp);
314 
315 	tmp = RREG32(mmHDP_HOST_PATH_CNTL);
316 	WREG32(mmHDP_HOST_PATH_CNTL, tmp);
317 }
318 
319 /**
320  * gmc_v7_0_mc_init - initialize the memory controller driver params
321  *
322  * @adev: amdgpu_device pointer
323  *
324  * Look up the amount of vram, vram width, and decide how to place
325  * vram and gart within the GPU's physical address space (CIK).
326  * Returns 0 for success.
327  */
328 static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
329 {
330 	u32 tmp;
331 	int chansize, numchan;
332 
333 	/* Get VRAM informations */
334 	tmp = RREG32(mmMC_ARB_RAMCFG);
335 	if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
336 		chansize = 64;
337 	} else {
338 		chansize = 32;
339 	}
340 	tmp = RREG32(mmMC_SHARED_CHMAP);
341 	switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
342 	case 0:
343 	default:
344 		numchan = 1;
345 		break;
346 	case 1:
347 		numchan = 2;
348 		break;
349 	case 2:
350 		numchan = 4;
351 		break;
352 	case 3:
353 		numchan = 8;
354 		break;
355 	case 4:
356 		numchan = 3;
357 		break;
358 	case 5:
359 		numchan = 6;
360 		break;
361 	case 6:
362 		numchan = 10;
363 		break;
364 	case 7:
365 		numchan = 12;
366 		break;
367 	case 8:
368 		numchan = 16;
369 		break;
370 	}
371 	adev->mc.vram_width = numchan * chansize;
372 	/* Could aper size report 0 ? */
373 	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
374 	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
375 	/* size in MB on si */
376 	adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
377 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
378 	adev->mc.visible_vram_size = adev->mc.aper_size;
379 
380 	/* In case the PCI BAR is larger than the actual amount of vram */
381 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
382 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
383 
384 	/* unless the user had overridden it, set the gart
385 	 * size equal to the 1024 or vram, whichever is larger.
386 	 */
387 	if (amdgpu_gart_size == -1)
388 		adev->mc.gtt_size = amdgpu_ttm_get_gtt_mem_size(adev);
389 	else
390 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
391 
392 	gmc_v7_0_vram_gtt_location(adev, &adev->mc);
393 
394 	return 0;
395 }
396 
397 /*
398  * GART
399  * VMID 0 is the physical GPU addresses as used by the kernel.
400  * VMIDs 1-15 are used for userspace clients and are handled
401  * by the amdgpu vm/hsa code.
402  */
403 
404 /**
405  * gmc_v7_0_gart_flush_gpu_tlb - gart tlb flush callback
406  *
407  * @adev: amdgpu_device pointer
408  * @vmid: vm instance to flush
409  *
410  * Flush the TLB for the requested page table (CIK).
411  */
412 static void gmc_v7_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
413 					uint32_t vmid)
414 {
415 	/* flush hdp cache */
416 	WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
417 
418 	/* bits 0-15 are the VM contexts0-15 */
419 	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
420 }
421 
422 /**
423  * gmc_v7_0_gart_set_pte_pde - update the page tables using MMIO
424  *
425  * @adev: amdgpu_device pointer
426  * @cpu_pt_addr: cpu address of the page table
427  * @gpu_page_idx: entry in the page table to update
428  * @addr: dst addr to write into pte/pde
429  * @flags: access flags
430  *
431  * Update the page tables using the CPU.
432  */
433 static int gmc_v7_0_gart_set_pte_pde(struct amdgpu_device *adev,
434 				     void *cpu_pt_addr,
435 				     uint32_t gpu_page_idx,
436 				     uint64_t addr,
437 				     uint32_t flags)
438 {
439 	void __iomem *ptr = (void *)cpu_pt_addr;
440 	uint64_t value;
441 
442 	value = addr & 0xFFFFFFFFFFFFF000ULL;
443 	value |= flags;
444 	writeq(value, ptr + (gpu_page_idx * 8));
445 
446 	return 0;
447 }
448 
449 /**
450  * gmc_v8_0_set_fault_enable_default - update VM fault handling
451  *
452  * @adev: amdgpu_device pointer
453  * @value: true redirects VM faults to the default page
454  */
455 static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,
456 					      bool value)
457 {
458 	u32 tmp;
459 
460 	tmp = RREG32(mmVM_CONTEXT1_CNTL);
461 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
462 			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
463 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
464 			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
465 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
466 			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
467 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
468 			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
469 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
470 			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
471 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
472 			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
473 	WREG32(mmVM_CONTEXT1_CNTL, tmp);
474 }
475 
476 /**
477  * gmc_v7_0_gart_enable - gart enable
478  *
479  * @adev: amdgpu_device pointer
480  *
481  * This sets up the TLBs, programs the page tables for VMID0,
482  * sets up the hw for VMIDs 1-15 which are allocated on
483  * demand, and sets up the global locations for the LDS, GDS,
484  * and GPUVM for FSA64 clients (CIK).
485  * Returns 0 for success, errors for failure.
486  */
487 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
488 {
489 	int r, i;
490 	u32 tmp;
491 
492 	if (adev->gart.robj == NULL) {
493 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
494 		return -EINVAL;
495 	}
496 	r = amdgpu_gart_table_vram_pin(adev);
497 	if (r)
498 		return r;
499 	/* Setup TLB control */
500 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
501 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
502 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
503 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
504 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
505 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
506 	WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
507 	/* Setup L2 cache */
508 	tmp = RREG32(mmVM_L2_CNTL);
509 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
510 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
511 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
512 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
513 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
514 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
515 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
516 	WREG32(mmVM_L2_CNTL, tmp);
517 	tmp = REG_SET_FIELD(0, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
518 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
519 	WREG32(mmVM_L2_CNTL2, tmp);
520 	tmp = RREG32(mmVM_L2_CNTL3);
521 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
522 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
523 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
524 	WREG32(mmVM_L2_CNTL3, tmp);
525 	/* setup context0 */
526 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
527 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gtt_end >> 12);
528 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
529 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
530 			(u32)(adev->dummy_page.addr >> 12));
531 	WREG32(mmVM_CONTEXT0_CNTL2, 0);
532 	tmp = RREG32(mmVM_CONTEXT0_CNTL);
533 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
534 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
535 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
536 	WREG32(mmVM_CONTEXT0_CNTL, tmp);
537 
538 	WREG32(0x575, 0);
539 	WREG32(0x576, 0);
540 	WREG32(0x577, 0);
541 
542 	/* empty context1-15 */
543 	/* FIXME start with 4G, once using 2 level pt switch to full
544 	 * vm size space
545 	 */
546 	/* set vm size, must be a multiple of 4 */
547 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
548 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
549 	for (i = 1; i < 16; i++) {
550 		if (i < 8)
551 			WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
552 			       adev->gart.table_addr >> 12);
553 		else
554 			WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
555 			       adev->gart.table_addr >> 12);
556 	}
557 
558 	/* enable context1-15 */
559 	WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
560 	       (u32)(adev->dummy_page.addr >> 12));
561 	WREG32(mmVM_CONTEXT1_CNTL2, 4);
562 	tmp = RREG32(mmVM_CONTEXT1_CNTL);
563 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
564 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
565 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
566 			    amdgpu_vm_block_size - 9);
567 	WREG32(mmVM_CONTEXT1_CNTL, tmp);
568 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
569 		gmc_v7_0_set_fault_enable_default(adev, false);
570 	else
571 		gmc_v7_0_set_fault_enable_default(adev, true);
572 
573 	if (adev->asic_type == CHIP_KAVERI) {
574 		tmp = RREG32(mmCHUB_CONTROL);
575 		tmp &= ~BYPASS_VM;
576 		WREG32(mmCHUB_CONTROL, tmp);
577 	}
578 
579 	gmc_v7_0_gart_flush_gpu_tlb(adev, 0);
580 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
581 		 (unsigned)(adev->mc.gtt_size >> 20),
582 		 (unsigned long long)adev->gart.table_addr);
583 	adev->gart.ready = true;
584 	return 0;
585 }
586 
587 static int gmc_v7_0_gart_init(struct amdgpu_device *adev)
588 {
589 	int r;
590 
591 	if (adev->gart.robj) {
592 		WARN(1, "R600 PCIE GART already initialized\n");
593 		return 0;
594 	}
595 	/* Initialize common gart structure */
596 	r = amdgpu_gart_init(adev);
597 	if (r)
598 		return r;
599 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
600 	return amdgpu_gart_table_vram_alloc(adev);
601 }
602 
603 /**
604  * gmc_v7_0_gart_disable - gart disable
605  *
606  * @adev: amdgpu_device pointer
607  *
608  * This disables all VM page table (CIK).
609  */
610 static void gmc_v7_0_gart_disable(struct amdgpu_device *adev)
611 {
612 	u32 tmp;
613 
614 	/* Disable all tables */
615 	WREG32(mmVM_CONTEXT0_CNTL, 0);
616 	WREG32(mmVM_CONTEXT1_CNTL, 0);
617 	/* Setup TLB control */
618 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
619 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
620 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
621 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
622 	WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
623 	/* Setup L2 cache */
624 	tmp = RREG32(mmVM_L2_CNTL);
625 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
626 	WREG32(mmVM_L2_CNTL, tmp);
627 	WREG32(mmVM_L2_CNTL2, 0);
628 	amdgpu_gart_table_vram_unpin(adev);
629 }
630 
631 /**
632  * gmc_v7_0_gart_fini - vm fini callback
633  *
634  * @adev: amdgpu_device pointer
635  *
636  * Tears down the driver GART/VM setup (CIK).
637  */
638 static void gmc_v7_0_gart_fini(struct amdgpu_device *adev)
639 {
640 	amdgpu_gart_table_vram_free(adev);
641 	amdgpu_gart_fini(adev);
642 }
643 
644 /*
645  * vm
646  * VMID 0 is the physical GPU addresses as used by the kernel.
647  * VMIDs 1-15 are used for userspace clients and are handled
648  * by the amdgpu vm/hsa code.
649  */
650 /**
651  * gmc_v7_0_vm_init - cik vm init callback
652  *
653  * @adev: amdgpu_device pointer
654  *
655  * Inits cik specific vm parameters (number of VMs, base of vram for
656  * VMIDs 1-15) (CIK).
657  * Returns 0 for success.
658  */
659 static int gmc_v7_0_vm_init(struct amdgpu_device *adev)
660 {
661 	/*
662 	 * number of VMs
663 	 * VMID 0 is reserved for System
664 	 * amdgpu graphics/compute will use VMIDs 1-7
665 	 * amdkfd will use VMIDs 8-15
666 	 */
667 	adev->vm_manager.num_ids = AMDGPU_NUM_OF_VMIDS;
668 	amdgpu_vm_manager_init(adev);
669 
670 	/* base offset of vram pages */
671 	if (adev->flags & AMD_IS_APU) {
672 		u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
673 		tmp <<= 22;
674 		adev->vm_manager.vram_base_offset = tmp;
675 	} else
676 		adev->vm_manager.vram_base_offset = 0;
677 
678 	return 0;
679 }
680 
681 /**
682  * gmc_v7_0_vm_fini - cik vm fini callback
683  *
684  * @adev: amdgpu_device pointer
685  *
686  * Tear down any asic specific VM setup (CIK).
687  */
688 static void gmc_v7_0_vm_fini(struct amdgpu_device *adev)
689 {
690 }
691 
692 /**
693  * gmc_v7_0_vm_decode_fault - print human readable fault info
694  *
695  * @adev: amdgpu_device pointer
696  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
697  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
698  *
699  * Print human readable fault information (CIK).
700  */
701 static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev,
702 				     u32 status, u32 addr, u32 mc_client)
703 {
704 	u32 mc_id;
705 	u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
706 	u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
707 					PROTECTIONS);
708 	char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
709 		(mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
710 
711 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
712 			      MEMORY_CLIENT_ID);
713 
714 	printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
715 	       protections, vmid, addr,
716 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
717 			     MEMORY_CLIENT_RW) ?
718 	       "write" : "read", block, mc_client, mc_id);
719 }
720 
721 
722 static const u32 mc_cg_registers[] = {
723 	mmMC_HUB_MISC_HUB_CG,
724 	mmMC_HUB_MISC_SIP_CG,
725 	mmMC_HUB_MISC_VM_CG,
726 	mmMC_XPB_CLK_GAT,
727 	mmATC_MISC_CG,
728 	mmMC_CITF_MISC_WR_CG,
729 	mmMC_CITF_MISC_RD_CG,
730 	mmMC_CITF_MISC_VM_CG,
731 	mmVM_L2_CG,
732 };
733 
734 static const u32 mc_cg_ls_en[] = {
735 	MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
736 	MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
737 	MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
738 	MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
739 	ATC_MISC_CG__MEM_LS_ENABLE_MASK,
740 	MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
741 	MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
742 	MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
743 	VM_L2_CG__MEM_LS_ENABLE_MASK,
744 };
745 
746 static const u32 mc_cg_en[] = {
747 	MC_HUB_MISC_HUB_CG__ENABLE_MASK,
748 	MC_HUB_MISC_SIP_CG__ENABLE_MASK,
749 	MC_HUB_MISC_VM_CG__ENABLE_MASK,
750 	MC_XPB_CLK_GAT__ENABLE_MASK,
751 	ATC_MISC_CG__ENABLE_MASK,
752 	MC_CITF_MISC_WR_CG__ENABLE_MASK,
753 	MC_CITF_MISC_RD_CG__ENABLE_MASK,
754 	MC_CITF_MISC_VM_CG__ENABLE_MASK,
755 	VM_L2_CG__ENABLE_MASK,
756 };
757 
758 static void gmc_v7_0_enable_mc_ls(struct amdgpu_device *adev,
759 				  bool enable)
760 {
761 	int i;
762 	u32 orig, data;
763 
764 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
765 		orig = data = RREG32(mc_cg_registers[i]);
766 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
767 			data |= mc_cg_ls_en[i];
768 		else
769 			data &= ~mc_cg_ls_en[i];
770 		if (data != orig)
771 			WREG32(mc_cg_registers[i], data);
772 	}
773 }
774 
775 static void gmc_v7_0_enable_mc_mgcg(struct amdgpu_device *adev,
776 				    bool enable)
777 {
778 	int i;
779 	u32 orig, data;
780 
781 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
782 		orig = data = RREG32(mc_cg_registers[i]);
783 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
784 			data |= mc_cg_en[i];
785 		else
786 			data &= ~mc_cg_en[i];
787 		if (data != orig)
788 			WREG32(mc_cg_registers[i], data);
789 	}
790 }
791 
792 static void gmc_v7_0_enable_bif_mgls(struct amdgpu_device *adev,
793 				     bool enable)
794 {
795 	u32 orig, data;
796 
797 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
798 
799 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
800 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
801 		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
802 		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
803 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
804 	} else {
805 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
806 		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
807 		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
808 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
809 	}
810 
811 	if (orig != data)
812 		WREG32_PCIE(ixPCIE_CNTL2, data);
813 }
814 
815 static void gmc_v7_0_enable_hdp_mgcg(struct amdgpu_device *adev,
816 				     bool enable)
817 {
818 	u32 orig, data;
819 
820 	orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
821 
822 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
823 		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
824 	else
825 		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
826 
827 	if (orig != data)
828 		WREG32(mmHDP_HOST_PATH_CNTL, data);
829 }
830 
831 static void gmc_v7_0_enable_hdp_ls(struct amdgpu_device *adev,
832 				   bool enable)
833 {
834 	u32 orig, data;
835 
836 	orig = data = RREG32(mmHDP_MEM_POWER_LS);
837 
838 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
839 		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
840 	else
841 		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
842 
843 	if (orig != data)
844 		WREG32(mmHDP_MEM_POWER_LS, data);
845 }
846 
847 static int gmc_v7_0_convert_vram_type(int mc_seq_vram_type)
848 {
849 	switch (mc_seq_vram_type) {
850 	case MC_SEQ_MISC0__MT__GDDR1:
851 		return AMDGPU_VRAM_TYPE_GDDR1;
852 	case MC_SEQ_MISC0__MT__DDR2:
853 		return AMDGPU_VRAM_TYPE_DDR2;
854 	case MC_SEQ_MISC0__MT__GDDR3:
855 		return AMDGPU_VRAM_TYPE_GDDR3;
856 	case MC_SEQ_MISC0__MT__GDDR4:
857 		return AMDGPU_VRAM_TYPE_GDDR4;
858 	case MC_SEQ_MISC0__MT__GDDR5:
859 		return AMDGPU_VRAM_TYPE_GDDR5;
860 	case MC_SEQ_MISC0__MT__HBM:
861 		return AMDGPU_VRAM_TYPE_HBM;
862 	case MC_SEQ_MISC0__MT__DDR3:
863 		return AMDGPU_VRAM_TYPE_DDR3;
864 	default:
865 		return AMDGPU_VRAM_TYPE_UNKNOWN;
866 	}
867 }
868 
869 static int gmc_v7_0_early_init(void *handle)
870 {
871 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
872 
873 	gmc_v7_0_set_gart_funcs(adev);
874 	gmc_v7_0_set_irq_funcs(adev);
875 
876 	return 0;
877 }
878 
879 static int gmc_v7_0_late_init(void *handle)
880 {
881 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
882 
883 	if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
884 		return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
885 	else
886 		return 0;
887 }
888 
889 static int gmc_v7_0_sw_init(void *handle)
890 {
891 	int r;
892 	int dma_bits;
893 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
894 
895 	if (adev->flags & AMD_IS_APU) {
896 		adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
897 	} else {
898 		u32 tmp = RREG32(mmMC_SEQ_MISC0);
899 		tmp &= MC_SEQ_MISC0__MT__MASK;
900 		adev->mc.vram_type = gmc_v7_0_convert_vram_type(tmp);
901 	}
902 
903 	r = amdgpu_irq_add_id(adev, 146, &adev->mc.vm_fault);
904 	if (r)
905 		return r;
906 
907 	r = amdgpu_irq_add_id(adev, 147, &adev->mc.vm_fault);
908 	if (r)
909 		return r;
910 
911 	/* Adjust VM size here.
912 	 * Currently set to 4GB ((1 << 20) 4k pages).
913 	 * Max GPUVM size for cayman and SI is 40 bits.
914 	 */
915 	adev->vm_manager.max_pfn = amdgpu_vm_size << 18;
916 
917 	/* Set the internal MC address mask
918 	 * This is the max address of the GPU's
919 	 * internal address space.
920 	 */
921 	adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
922 
923 	/* set DMA mask + need_dma32 flags.
924 	 * PCIE - can handle 40-bits.
925 	 * IGP - can handle 40-bits
926 	 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
927 	 */
928 	adev->need_dma32 = false;
929 	dma_bits = adev->need_dma32 ? 32 : 40;
930 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
931 	if (r) {
932 		adev->need_dma32 = true;
933 		dma_bits = 32;
934 		printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
935 	}
936 	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
937 	if (r) {
938 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
939 		printk(KERN_WARNING "amdgpu: No coherent DMA available.\n");
940 	}
941 
942 	r = gmc_v7_0_init_microcode(adev);
943 	if (r) {
944 		DRM_ERROR("Failed to load mc firmware!\n");
945 		return r;
946 	}
947 
948 	r = amdgpu_ttm_global_init(adev);
949 	if (r) {
950 		return r;
951 	}
952 
953 	r = gmc_v7_0_mc_init(adev);
954 	if (r)
955 		return r;
956 
957 	/* Memory manager */
958 	r = amdgpu_bo_init(adev);
959 	if (r)
960 		return r;
961 
962 	r = gmc_v7_0_gart_init(adev);
963 	if (r)
964 		return r;
965 
966 	if (!adev->vm_manager.enabled) {
967 		r = gmc_v7_0_vm_init(adev);
968 		if (r) {
969 			dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
970 			return r;
971 		}
972 		adev->vm_manager.enabled = true;
973 	}
974 
975 	return r;
976 }
977 
978 static int gmc_v7_0_sw_fini(void *handle)
979 {
980 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
981 
982 	if (adev->vm_manager.enabled) {
983 		amdgpu_vm_manager_fini(adev);
984 		gmc_v7_0_vm_fini(adev);
985 		adev->vm_manager.enabled = false;
986 	}
987 	gmc_v7_0_gart_fini(adev);
988 	amdgpu_gem_force_release(adev);
989 	amdgpu_bo_fini(adev);
990 
991 	return 0;
992 }
993 
994 static int gmc_v7_0_hw_init(void *handle)
995 {
996 	int r;
997 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
998 
999 	gmc_v7_0_init_golden_registers(adev);
1000 
1001 	gmc_v7_0_mc_program(adev);
1002 
1003 	if (!(adev->flags & AMD_IS_APU)) {
1004 		r = gmc_v7_0_mc_load_microcode(adev);
1005 		if (r) {
1006 			DRM_ERROR("Failed to load MC firmware!\n");
1007 			return r;
1008 		}
1009 	}
1010 
1011 	r = gmc_v7_0_gart_enable(adev);
1012 	if (r)
1013 		return r;
1014 
1015 	return r;
1016 }
1017 
1018 static int gmc_v7_0_hw_fini(void *handle)
1019 {
1020 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1021 
1022 	amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
1023 	gmc_v7_0_gart_disable(adev);
1024 
1025 	return 0;
1026 }
1027 
1028 static int gmc_v7_0_suspend(void *handle)
1029 {
1030 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1031 
1032 	if (adev->vm_manager.enabled) {
1033 		gmc_v7_0_vm_fini(adev);
1034 		adev->vm_manager.enabled = false;
1035 	}
1036 	gmc_v7_0_hw_fini(adev);
1037 
1038 	return 0;
1039 }
1040 
1041 static int gmc_v7_0_resume(void *handle)
1042 {
1043 	int r;
1044 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1045 
1046 	r = gmc_v7_0_hw_init(adev);
1047 	if (r)
1048 		return r;
1049 
1050 	if (!adev->vm_manager.enabled) {
1051 		r = gmc_v7_0_vm_init(adev);
1052 		if (r) {
1053 			dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
1054 			return r;
1055 		}
1056 		adev->vm_manager.enabled = true;
1057 	}
1058 
1059 	return r;
1060 }
1061 
1062 static bool gmc_v7_0_is_idle(void *handle)
1063 {
1064 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1065 	u32 tmp = RREG32(mmSRBM_STATUS);
1066 
1067 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1068 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
1069 		return false;
1070 
1071 	return true;
1072 }
1073 
1074 static int gmc_v7_0_wait_for_idle(void *handle)
1075 {
1076 	unsigned i;
1077 	u32 tmp;
1078 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1079 
1080 	for (i = 0; i < adev->usec_timeout; i++) {
1081 		/* read MC_STATUS */
1082 		tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1083 					       SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1084 					       SRBM_STATUS__MCC_BUSY_MASK |
1085 					       SRBM_STATUS__MCD_BUSY_MASK |
1086 					       SRBM_STATUS__VMC_BUSY_MASK);
1087 		if (!tmp)
1088 			return 0;
1089 		udelay(1);
1090 	}
1091 	return -ETIMEDOUT;
1092 
1093 }
1094 
1095 static int gmc_v7_0_soft_reset(void *handle)
1096 {
1097 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1098 	struct amdgpu_mode_mc_save save;
1099 	u32 srbm_soft_reset = 0;
1100 	u32 tmp = RREG32(mmSRBM_STATUS);
1101 
1102 	if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1103 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1104 						SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1105 
1106 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1107 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
1108 		if (!(adev->flags & AMD_IS_APU))
1109 			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1110 							SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1111 	}
1112 
1113 	if (srbm_soft_reset) {
1114 		gmc_v7_0_mc_stop(adev, &save);
1115 		if (gmc_v7_0_wait_for_idle((void *)adev)) {
1116 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1117 		}
1118 
1119 
1120 		tmp = RREG32(mmSRBM_SOFT_RESET);
1121 		tmp |= srbm_soft_reset;
1122 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1123 		WREG32(mmSRBM_SOFT_RESET, tmp);
1124 		tmp = RREG32(mmSRBM_SOFT_RESET);
1125 
1126 		udelay(50);
1127 
1128 		tmp &= ~srbm_soft_reset;
1129 		WREG32(mmSRBM_SOFT_RESET, tmp);
1130 		tmp = RREG32(mmSRBM_SOFT_RESET);
1131 
1132 		/* Wait a little for things to settle down */
1133 		udelay(50);
1134 
1135 		gmc_v7_0_mc_resume(adev, &save);
1136 		udelay(50);
1137 	}
1138 
1139 	return 0;
1140 }
1141 
1142 static int gmc_v7_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1143 					     struct amdgpu_irq_src *src,
1144 					     unsigned type,
1145 					     enum amdgpu_interrupt_state state)
1146 {
1147 	u32 tmp;
1148 	u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1149 		    VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1150 		    VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1151 		    VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1152 		    VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1153 		    VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1154 
1155 	switch (state) {
1156 	case AMDGPU_IRQ_STATE_DISABLE:
1157 		/* system context */
1158 		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1159 		tmp &= ~bits;
1160 		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1161 		/* VMs */
1162 		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1163 		tmp &= ~bits;
1164 		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1165 		break;
1166 	case AMDGPU_IRQ_STATE_ENABLE:
1167 		/* system context */
1168 		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1169 		tmp |= bits;
1170 		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1171 		/* VMs */
1172 		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1173 		tmp |= bits;
1174 		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1175 		break;
1176 	default:
1177 		break;
1178 	}
1179 
1180 	return 0;
1181 }
1182 
1183 static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
1184 				      struct amdgpu_irq_src *source,
1185 				      struct amdgpu_iv_entry *entry)
1186 {
1187 	u32 addr, status, mc_client;
1188 
1189 	addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1190 	status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1191 	mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
1192 	/* reset addr and status */
1193 	WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1194 
1195 	if (!addr && !status)
1196 		return 0;
1197 
1198 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1199 		gmc_v7_0_set_fault_enable_default(adev, false);
1200 
1201 	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1202 		entry->src_id, entry->src_data);
1203 	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1204 		addr);
1205 	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1206 		status);
1207 	gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client);
1208 
1209 	return 0;
1210 }
1211 
1212 static int gmc_v7_0_set_clockgating_state(void *handle,
1213 					  enum amd_clockgating_state state)
1214 {
1215 	bool gate = false;
1216 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1217 
1218 	if (state == AMD_CG_STATE_GATE)
1219 		gate = true;
1220 
1221 	if (!(adev->flags & AMD_IS_APU)) {
1222 		gmc_v7_0_enable_mc_mgcg(adev, gate);
1223 		gmc_v7_0_enable_mc_ls(adev, gate);
1224 	}
1225 	gmc_v7_0_enable_bif_mgls(adev, gate);
1226 	gmc_v7_0_enable_hdp_mgcg(adev, gate);
1227 	gmc_v7_0_enable_hdp_ls(adev, gate);
1228 
1229 	return 0;
1230 }
1231 
1232 static int gmc_v7_0_set_powergating_state(void *handle,
1233 					  enum amd_powergating_state state)
1234 {
1235 	return 0;
1236 }
1237 
1238 const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
1239 	.name = "gmc_v7_0",
1240 	.early_init = gmc_v7_0_early_init,
1241 	.late_init = gmc_v7_0_late_init,
1242 	.sw_init = gmc_v7_0_sw_init,
1243 	.sw_fini = gmc_v7_0_sw_fini,
1244 	.hw_init = gmc_v7_0_hw_init,
1245 	.hw_fini = gmc_v7_0_hw_fini,
1246 	.suspend = gmc_v7_0_suspend,
1247 	.resume = gmc_v7_0_resume,
1248 	.is_idle = gmc_v7_0_is_idle,
1249 	.wait_for_idle = gmc_v7_0_wait_for_idle,
1250 	.soft_reset = gmc_v7_0_soft_reset,
1251 	.set_clockgating_state = gmc_v7_0_set_clockgating_state,
1252 	.set_powergating_state = gmc_v7_0_set_powergating_state,
1253 };
1254 
1255 static const struct amdgpu_gart_funcs gmc_v7_0_gart_funcs = {
1256 	.flush_gpu_tlb = gmc_v7_0_gart_flush_gpu_tlb,
1257 	.set_pte_pde = gmc_v7_0_gart_set_pte_pde,
1258 };
1259 
1260 static const struct amdgpu_irq_src_funcs gmc_v7_0_irq_funcs = {
1261 	.set = gmc_v7_0_vm_fault_interrupt_state,
1262 	.process = gmc_v7_0_process_interrupt,
1263 };
1264 
1265 static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev)
1266 {
1267 	if (adev->gart.gart_funcs == NULL)
1268 		adev->gart.gart_funcs = &gmc_v7_0_gart_funcs;
1269 }
1270 
1271 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)
1272 {
1273 	adev->mc.vm_fault.num_types = 1;
1274 	adev->mc.vm_fault.funcs = &gmc_v7_0_irq_funcs;
1275 }
1276