xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c (revision e2f1cf25)
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include "drmP.h"
25 #include "amdgpu.h"
26 #include "gmc_v8_0.h"
27 #include "amdgpu_ucode.h"
28 
29 #include "gmc/gmc_8_1_d.h"
30 #include "gmc/gmc_8_1_sh_mask.h"
31 
32 #include "bif/bif_5_0_d.h"
33 #include "bif/bif_5_0_sh_mask.h"
34 
35 #include "oss/oss_3_0_d.h"
36 #include "oss/oss_3_0_sh_mask.h"
37 
38 #include "vid.h"
39 #include "vi.h"
40 
41 
42 static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev);
43 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);
44 
45 MODULE_FIRMWARE("amdgpu/topaz_mc.bin");
46 MODULE_FIRMWARE("amdgpu/tonga_mc.bin");
47 MODULE_FIRMWARE("amdgpu/fiji_mc.bin");
48 
49 static const u32 golden_settings_tonga_a11[] =
50 {
51 	mmMC_ARB_WTM_GRPWT_RD, 0x00000003, 0x00000000,
52 	mmMC_HUB_RDREQ_DMIF_LIMIT, 0x0000007f, 0x00000028,
53 	mmMC_HUB_WDP_UMC, 0x00007fb6, 0x00000991,
54 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
55 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
56 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
57 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
58 };
59 
60 static const u32 tonga_mgcg_cgcg_init[] =
61 {
62 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
63 };
64 
65 static const u32 golden_settings_fiji_a10[] =
66 {
67 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
68 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
69 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
70 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff,
71 };
72 
73 static const u32 fiji_mgcg_cgcg_init[] =
74 {
75 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
76 };
77 
78 static const u32 golden_settings_iceland_a11[] =
79 {
80 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
81 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
82 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
83 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
84 };
85 
86 static const u32 iceland_mgcg_cgcg_init[] =
87 {
88 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
89 };
90 
91 static const u32 cz_mgcg_cgcg_init[] =
92 {
93 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
94 };
95 
96 static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
97 {
98 	switch (adev->asic_type) {
99 	case CHIP_TOPAZ:
100 		amdgpu_program_register_sequence(adev,
101 						 iceland_mgcg_cgcg_init,
102 						 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
103 		amdgpu_program_register_sequence(adev,
104 						 golden_settings_iceland_a11,
105 						 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
106 		break;
107 	case CHIP_FIJI:
108 		amdgpu_program_register_sequence(adev,
109 						 fiji_mgcg_cgcg_init,
110 						 (const u32)ARRAY_SIZE(fiji_mgcg_cgcg_init));
111 		amdgpu_program_register_sequence(adev,
112 						 golden_settings_fiji_a10,
113 						 (const u32)ARRAY_SIZE(golden_settings_fiji_a10));
114 		break;
115 	case CHIP_TONGA:
116 		amdgpu_program_register_sequence(adev,
117 						 tonga_mgcg_cgcg_init,
118 						 (const u32)ARRAY_SIZE(tonga_mgcg_cgcg_init));
119 		amdgpu_program_register_sequence(adev,
120 						 golden_settings_tonga_a11,
121 						 (const u32)ARRAY_SIZE(golden_settings_tonga_a11));
122 		break;
123 	case CHIP_CARRIZO:
124 		amdgpu_program_register_sequence(adev,
125 						 cz_mgcg_cgcg_init,
126 						 (const u32)ARRAY_SIZE(cz_mgcg_cgcg_init));
127 		break;
128 	default:
129 		break;
130 	}
131 }
132 
133 /**
134  * gmc8_mc_wait_for_idle - wait for MC idle callback.
135  *
136  * @adev: amdgpu_device pointer
137  *
138  * Wait for the MC (memory controller) to be idle.
139  * (evergreen+).
140  * Returns 0 if the MC is idle, -1 if not.
141  */
142 int gmc_v8_0_mc_wait_for_idle(struct amdgpu_device *adev)
143 {
144 	unsigned i;
145 	u32 tmp;
146 
147 	for (i = 0; i < adev->usec_timeout; i++) {
148 		/* read MC_STATUS */
149 		tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__VMC_BUSY_MASK |
150 					       SRBM_STATUS__MCB_BUSY_MASK |
151 					       SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
152 					       SRBM_STATUS__MCC_BUSY_MASK |
153 					       SRBM_STATUS__MCD_BUSY_MASK |
154 					       SRBM_STATUS__VMC1_BUSY_MASK);
155 		if (!tmp)
156 			return 0;
157 		udelay(1);
158 	}
159 	return -1;
160 }
161 
162 void gmc_v8_0_mc_stop(struct amdgpu_device *adev,
163 		      struct amdgpu_mode_mc_save *save)
164 {
165 	u32 blackout;
166 
167 	if (adev->mode_info.num_crtc)
168 		amdgpu_display_stop_mc_access(adev, save);
169 
170 	amdgpu_asic_wait_for_mc_idle(adev);
171 
172 	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
173 	if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
174 		/* Block CPU access */
175 		WREG32(mmBIF_FB_EN, 0);
176 		/* blackout the MC */
177 		blackout = REG_SET_FIELD(blackout,
178 					 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1);
179 		WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
180 	}
181 	/* wait for the MC to settle */
182 	udelay(100);
183 }
184 
185 void gmc_v8_0_mc_resume(struct amdgpu_device *adev,
186 			struct amdgpu_mode_mc_save *save)
187 {
188 	u32 tmp;
189 
190 	/* unblackout the MC */
191 	tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
192 	tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
193 	WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
194 	/* allow CPU access */
195 	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
196 	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
197 	WREG32(mmBIF_FB_EN, tmp);
198 
199 	if (adev->mode_info.num_crtc)
200 		amdgpu_display_resume_mc_access(adev, save);
201 }
202 
203 /**
204  * gmc_v8_0_init_microcode - load ucode images from disk
205  *
206  * @adev: amdgpu_device pointer
207  *
208  * Use the firmware interface to load the ucode images into
209  * the driver (not loaded into hw).
210  * Returns 0 on success, error on failure.
211  */
212 static int gmc_v8_0_init_microcode(struct amdgpu_device *adev)
213 {
214 	const char *chip_name;
215 	char fw_name[30];
216 	int err;
217 
218 	DRM_DEBUG("\n");
219 
220 	switch (adev->asic_type) {
221 	case CHIP_TOPAZ:
222 		chip_name = "topaz";
223 		break;
224 	case CHIP_TONGA:
225 		chip_name = "tonga";
226 		break;
227 	case CHIP_FIJI:
228 		chip_name = "fiji";
229 		break;
230 	case CHIP_CARRIZO:
231 		return 0;
232 	default: BUG();
233 	}
234 
235 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
236 	err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
237 	if (err)
238 		goto out;
239 	err = amdgpu_ucode_validate(adev->mc.fw);
240 
241 out:
242 	if (err) {
243 		printk(KERN_ERR
244 		       "mc: Failed to load firmware \"%s\"\n",
245 		       fw_name);
246 		release_firmware(adev->mc.fw);
247 		adev->mc.fw = NULL;
248 	}
249 	return err;
250 }
251 
252 /**
253  * gmc_v8_0_mc_load_microcode - load MC ucode into the hw
254  *
255  * @adev: amdgpu_device pointer
256  *
257  * Load the GDDR MC ucode into the hw (CIK).
258  * Returns 0 on success, error on failure.
259  */
260 static int gmc_v8_0_mc_load_microcode(struct amdgpu_device *adev)
261 {
262 	const struct mc_firmware_header_v1_0 *hdr;
263 	const __le32 *fw_data = NULL;
264 	const __le32 *io_mc_regs = NULL;
265 	u32 running, blackout = 0;
266 	int i, ucode_size, regs_size;
267 
268 	if (!adev->mc.fw)
269 		return -EINVAL;
270 
271 	hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
272 	amdgpu_ucode_print_mc_hdr(&hdr->header);
273 
274 	adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
275 	regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
276 	io_mc_regs = (const __le32 *)
277 		(adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
278 	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
279 	fw_data = (const __le32 *)
280 		(adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
281 
282 	running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
283 
284 	if (running == 0) {
285 		if (running) {
286 			blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
287 			WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
288 		}
289 
290 		/* reset the engine and set to writable */
291 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
292 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
293 
294 		/* load mc io regs */
295 		for (i = 0; i < regs_size; i++) {
296 			WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
297 			WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
298 		}
299 		/* load the MC ucode */
300 		for (i = 0; i < ucode_size; i++)
301 			WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
302 
303 		/* put the engine back into the active state */
304 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
305 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
306 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
307 
308 		/* wait for training to complete */
309 		for (i = 0; i < adev->usec_timeout; i++) {
310 			if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
311 					  MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
312 				break;
313 			udelay(1);
314 		}
315 		for (i = 0; i < adev->usec_timeout; i++) {
316 			if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
317 					  MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
318 				break;
319 			udelay(1);
320 		}
321 
322 		if (running)
323 			WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout);
324 	}
325 
326 	return 0;
327 }
328 
329 static void gmc_v8_0_vram_gtt_location(struct amdgpu_device *adev,
330 				       struct amdgpu_mc *mc)
331 {
332 	if (mc->mc_vram_size > 0xFFC0000000ULL) {
333 		/* leave room for at least 1024M GTT */
334 		dev_warn(adev->dev, "limiting VRAM\n");
335 		mc->real_vram_size = 0xFFC0000000ULL;
336 		mc->mc_vram_size = 0xFFC0000000ULL;
337 	}
338 	amdgpu_vram_location(adev, &adev->mc, 0);
339 	adev->mc.gtt_base_align = 0;
340 	amdgpu_gtt_location(adev, mc);
341 }
342 
343 /**
344  * gmc_v8_0_mc_program - program the GPU memory controller
345  *
346  * @adev: amdgpu_device pointer
347  *
348  * Set the location of vram, gart, and AGP in the GPU's
349  * physical address space (CIK).
350  */
351 static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
352 {
353 	struct amdgpu_mode_mc_save save;
354 	u32 tmp;
355 	int i, j;
356 
357 	/* Initialize HDP */
358 	for (i = 0, j = 0; i < 32; i++, j += 0x6) {
359 		WREG32((0xb05 + j), 0x00000000);
360 		WREG32((0xb06 + j), 0x00000000);
361 		WREG32((0xb07 + j), 0x00000000);
362 		WREG32((0xb08 + j), 0x00000000);
363 		WREG32((0xb09 + j), 0x00000000);
364 	}
365 	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
366 
367 	if (adev->mode_info.num_crtc)
368 		amdgpu_display_set_vga_render_state(adev, false);
369 
370 	gmc_v8_0_mc_stop(adev, &save);
371 	if (amdgpu_asic_wait_for_mc_idle(adev)) {
372 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
373 	}
374 	/* Update configuration */
375 	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
376 	       adev->mc.vram_start >> 12);
377 	WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
378 	       adev->mc.vram_end >> 12);
379 	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
380 	       adev->vram_scratch.gpu_addr >> 12);
381 	tmp = ((adev->mc.vram_end >> 24) & 0xFFFF) << 16;
382 	tmp |= ((adev->mc.vram_start >> 24) & 0xFFFF);
383 	WREG32(mmMC_VM_FB_LOCATION, tmp);
384 	/* XXX double check these! */
385 	WREG32(mmHDP_NONSURFACE_BASE, (adev->mc.vram_start >> 8));
386 	WREG32(mmHDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
387 	WREG32(mmHDP_NONSURFACE_SIZE, 0x3FFFFFFF);
388 	WREG32(mmMC_VM_AGP_BASE, 0);
389 	WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
390 	WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
391 	if (amdgpu_asic_wait_for_mc_idle(adev)) {
392 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
393 	}
394 	gmc_v8_0_mc_resume(adev, &save);
395 
396 	WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
397 
398 	tmp = RREG32(mmHDP_MISC_CNTL);
399 	tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1);
400 	WREG32(mmHDP_MISC_CNTL, tmp);
401 
402 	tmp = RREG32(mmHDP_HOST_PATH_CNTL);
403 	WREG32(mmHDP_HOST_PATH_CNTL, tmp);
404 }
405 
406 /**
407  * gmc_v8_0_mc_init - initialize the memory controller driver params
408  *
409  * @adev: amdgpu_device pointer
410  *
411  * Look up the amount of vram, vram width, and decide how to place
412  * vram and gart within the GPU's physical address space (CIK).
413  * Returns 0 for success.
414  */
415 static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
416 {
417 	u32 tmp;
418 	int chansize, numchan;
419 
420 	/* Get VRAM informations */
421 	tmp = RREG32(mmMC_ARB_RAMCFG);
422 	if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
423 		chansize = 64;
424 	} else {
425 		chansize = 32;
426 	}
427 	tmp = RREG32(mmMC_SHARED_CHMAP);
428 	switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
429 	case 0:
430 	default:
431 		numchan = 1;
432 		break;
433 	case 1:
434 		numchan = 2;
435 		break;
436 	case 2:
437 		numchan = 4;
438 		break;
439 	case 3:
440 		numchan = 8;
441 		break;
442 	case 4:
443 		numchan = 3;
444 		break;
445 	case 5:
446 		numchan = 6;
447 		break;
448 	case 6:
449 		numchan = 10;
450 		break;
451 	case 7:
452 		numchan = 12;
453 		break;
454 	case 8:
455 		numchan = 16;
456 		break;
457 	}
458 	adev->mc.vram_width = numchan * chansize;
459 	/* Could aper size report 0 ? */
460 	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
461 	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
462 	/* size in MB on si */
463 	adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
464 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
465 	adev->mc.visible_vram_size = adev->mc.aper_size;
466 
467 	/* unless the user had overridden it, set the gart
468 	 * size equal to the 1024 or vram, whichever is larger.
469 	 */
470 	if (amdgpu_gart_size == -1)
471 		adev->mc.gtt_size = max((1024ULL << 20), adev->mc.mc_vram_size);
472 	else
473 		adev->mc.gtt_size = (uint64_t)amdgpu_gart_size << 20;
474 
475 	gmc_v8_0_vram_gtt_location(adev, &adev->mc);
476 
477 	return 0;
478 }
479 
480 /*
481  * GART
482  * VMID 0 is the physical GPU addresses as used by the kernel.
483  * VMIDs 1-15 are used for userspace clients and are handled
484  * by the amdgpu vm/hsa code.
485  */
486 
487 /**
488  * gmc_v8_0_gart_flush_gpu_tlb - gart tlb flush callback
489  *
490  * @adev: amdgpu_device pointer
491  * @vmid: vm instance to flush
492  *
493  * Flush the TLB for the requested page table (CIK).
494  */
495 static void gmc_v8_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
496 					uint32_t vmid)
497 {
498 	/* flush hdp cache */
499 	WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
500 
501 	/* bits 0-15 are the VM contexts0-15 */
502 	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
503 }
504 
505 /**
506  * gmc_v8_0_gart_set_pte_pde - update the page tables using MMIO
507  *
508  * @adev: amdgpu_device pointer
509  * @cpu_pt_addr: cpu address of the page table
510  * @gpu_page_idx: entry in the page table to update
511  * @addr: dst addr to write into pte/pde
512  * @flags: access flags
513  *
514  * Update the page tables using the CPU.
515  */
516 static int gmc_v8_0_gart_set_pte_pde(struct amdgpu_device *adev,
517 				     void *cpu_pt_addr,
518 				     uint32_t gpu_page_idx,
519 				     uint64_t addr,
520 				     uint32_t flags)
521 {
522 	void __iomem *ptr = (void *)cpu_pt_addr;
523 	uint64_t value;
524 
525 	/*
526 	 * PTE format on VI:
527 	 * 63:40 reserved
528 	 * 39:12 4k physical page base address
529 	 * 11:7 fragment
530 	 * 6 write
531 	 * 5 read
532 	 * 4 exe
533 	 * 3 reserved
534 	 * 2 snooped
535 	 * 1 system
536 	 * 0 valid
537 	 *
538 	 * PDE format on VI:
539 	 * 63:59 block fragment size
540 	 * 58:40 reserved
541 	 * 39:1 physical base address of PTE
542 	 * bits 5:1 must be 0.
543 	 * 0 valid
544 	 */
545 	value = addr & 0x000000FFFFFFF000ULL;
546 	value |= flags;
547 	writeq(value, ptr + (gpu_page_idx * 8));
548 
549 	return 0;
550 }
551 
552 /**
553  * gmc_v8_0_gart_enable - gart enable
554  *
555  * @adev: amdgpu_device pointer
556  *
557  * This sets up the TLBs, programs the page tables for VMID0,
558  * sets up the hw for VMIDs 1-15 which are allocated on
559  * demand, and sets up the global locations for the LDS, GDS,
560  * and GPUVM for FSA64 clients (CIK).
561  * Returns 0 for success, errors for failure.
562  */
563 static int gmc_v8_0_gart_enable(struct amdgpu_device *adev)
564 {
565 	int r, i;
566 	u32 tmp;
567 
568 	if (adev->gart.robj == NULL) {
569 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
570 		return -EINVAL;
571 	}
572 	r = amdgpu_gart_table_vram_pin(adev);
573 	if (r)
574 		return r;
575 	/* Setup TLB control */
576 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
577 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
578 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
579 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
580 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
581 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
582 	WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
583 	/* Setup L2 cache */
584 	tmp = RREG32(mmVM_L2_CNTL);
585 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
586 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
587 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
588 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
589 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
590 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
591 	WREG32(mmVM_L2_CNTL, tmp);
592 	tmp = RREG32(mmVM_L2_CNTL2);
593 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
594 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
595 	WREG32(mmVM_L2_CNTL2, tmp);
596 	tmp = RREG32(mmVM_L2_CNTL3);
597 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
598 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 4);
599 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, 4);
600 	WREG32(mmVM_L2_CNTL3, tmp);
601 	/* XXX: set to enable PTE/PDE in system memory */
602 	tmp = RREG32(mmVM_L2_CNTL4);
603 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_PHYSICAL, 0);
604 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SHARED, 0);
605 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PDE_REQUEST_SNOOP, 0);
606 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_PHYSICAL, 0);
607 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SHARED, 0);
608 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT0_PTE_REQUEST_SNOOP, 0);
609 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_PHYSICAL, 0);
610 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SHARED, 0);
611 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PDE_REQUEST_SNOOP, 0);
612 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_PHYSICAL, 0);
613 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SHARED, 0);
614 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_CONTEXT1_PTE_REQUEST_SNOOP, 0);
615 	WREG32(mmVM_L2_CNTL4, tmp);
616 	/* setup context0 */
617 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gtt_start >> 12);
618 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, (adev->mc.gtt_end >> 12) - 1);
619 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
620 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
621 			(u32)(adev->dummy_page.addr >> 12));
622 	WREG32(mmVM_CONTEXT0_CNTL2, 0);
623 	tmp = RREG32(mmVM_CONTEXT0_CNTL);
624 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
625 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
626 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
627 	WREG32(mmVM_CONTEXT0_CNTL, tmp);
628 
629 	WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR, 0);
630 	WREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR, 0);
631 	WREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET, 0);
632 
633 	/* empty context1-15 */
634 	/* FIXME start with 4G, once using 2 level pt switch to full
635 	 * vm size space
636 	 */
637 	/* set vm size, must be a multiple of 4 */
638 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
639 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
640 	for (i = 1; i < 16; i++) {
641 		if (i < 8)
642 			WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
643 			       adev->gart.table_addr >> 12);
644 		else
645 			WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
646 			       adev->gart.table_addr >> 12);
647 	}
648 
649 	/* enable context1-15 */
650 	WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
651 	       (u32)(adev->dummy_page.addr >> 12));
652 	WREG32(mmVM_CONTEXT1_CNTL2, 4);
653 	tmp = RREG32(mmVM_CONTEXT1_CNTL);
654 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
655 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
656 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
657 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
658 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
659 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
660 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
661 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
662 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
663 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
664 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
665 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
666 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
667 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
668 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT, 1);
669 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
670 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
671 			    amdgpu_vm_block_size - 9);
672 	WREG32(mmVM_CONTEXT1_CNTL, tmp);
673 
674 	gmc_v8_0_gart_flush_gpu_tlb(adev, 0);
675 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
676 		 (unsigned)(adev->mc.gtt_size >> 20),
677 		 (unsigned long long)adev->gart.table_addr);
678 	adev->gart.ready = true;
679 	return 0;
680 }
681 
682 static int gmc_v8_0_gart_init(struct amdgpu_device *adev)
683 {
684 	int r;
685 
686 	if (adev->gart.robj) {
687 		WARN(1, "R600 PCIE GART already initialized\n");
688 		return 0;
689 	}
690 	/* Initialize common gart structure */
691 	r = amdgpu_gart_init(adev);
692 	if (r)
693 		return r;
694 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
695 	return amdgpu_gart_table_vram_alloc(adev);
696 }
697 
698 /**
699  * gmc_v8_0_gart_disable - gart disable
700  *
701  * @adev: amdgpu_device pointer
702  *
703  * This disables all VM page table (CIK).
704  */
705 static void gmc_v8_0_gart_disable(struct amdgpu_device *adev)
706 {
707 	u32 tmp;
708 
709 	/* Disable all tables */
710 	WREG32(mmVM_CONTEXT0_CNTL, 0);
711 	WREG32(mmVM_CONTEXT1_CNTL, 0);
712 	/* Setup TLB control */
713 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
714 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
715 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
716 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
717 	WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
718 	/* Setup L2 cache */
719 	tmp = RREG32(mmVM_L2_CNTL);
720 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
721 	WREG32(mmVM_L2_CNTL, tmp);
722 	WREG32(mmVM_L2_CNTL2, 0);
723 	amdgpu_gart_table_vram_unpin(adev);
724 }
725 
726 /**
727  * gmc_v8_0_gart_fini - vm fini callback
728  *
729  * @adev: amdgpu_device pointer
730  *
731  * Tears down the driver GART/VM setup (CIK).
732  */
733 static void gmc_v8_0_gart_fini(struct amdgpu_device *adev)
734 {
735 	amdgpu_gart_table_vram_free(adev);
736 	amdgpu_gart_fini(adev);
737 }
738 
739 /*
740  * vm
741  * VMID 0 is the physical GPU addresses as used by the kernel.
742  * VMIDs 1-15 are used for userspace clients and are handled
743  * by the amdgpu vm/hsa code.
744  */
745 /**
746  * gmc_v8_0_vm_init - cik vm init callback
747  *
748  * @adev: amdgpu_device pointer
749  *
750  * Inits cik specific vm parameters (number of VMs, base of vram for
751  * VMIDs 1-15) (CIK).
752  * Returns 0 for success.
753  */
754 static int gmc_v8_0_vm_init(struct amdgpu_device *adev)
755 {
756 	/*
757 	 * number of VMs
758 	 * VMID 0 is reserved for System
759 	 * amdgpu graphics/compute will use VMIDs 1-7
760 	 * amdkfd will use VMIDs 8-15
761 	 */
762 	adev->vm_manager.nvm = AMDGPU_NUM_OF_VMIDS;
763 
764 	/* base offset of vram pages */
765 	if (adev->flags & AMD_IS_APU) {
766 		u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
767 		tmp <<= 22;
768 		adev->vm_manager.vram_base_offset = tmp;
769 	} else
770 		adev->vm_manager.vram_base_offset = 0;
771 
772 	return 0;
773 }
774 
775 /**
776  * gmc_v8_0_vm_fini - cik vm fini callback
777  *
778  * @adev: amdgpu_device pointer
779  *
780  * Tear down any asic specific VM setup (CIK).
781  */
782 static void gmc_v8_0_vm_fini(struct amdgpu_device *adev)
783 {
784 }
785 
786 /**
787  * gmc_v8_0_vm_decode_fault - print human readable fault info
788  *
789  * @adev: amdgpu_device pointer
790  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
791  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
792  *
793  * Print human readable fault information (CIK).
794  */
795 static void gmc_v8_0_vm_decode_fault(struct amdgpu_device *adev,
796 				     u32 status, u32 addr, u32 mc_client)
797 {
798 	u32 mc_id;
799 	u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
800 	u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
801 					PROTECTIONS);
802 	char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
803 		(mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
804 
805 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
806 			      MEMORY_CLIENT_ID);
807 
808 	printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
809 	       protections, vmid, addr,
810 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
811 			     MEMORY_CLIENT_RW) ?
812 	       "write" : "read", block, mc_client, mc_id);
813 }
814 
815 static int gmc_v8_0_convert_vram_type(int mc_seq_vram_type)
816 {
817 	switch (mc_seq_vram_type) {
818 	case MC_SEQ_MISC0__MT__GDDR1:
819 		return AMDGPU_VRAM_TYPE_GDDR1;
820 	case MC_SEQ_MISC0__MT__DDR2:
821 		return AMDGPU_VRAM_TYPE_DDR2;
822 	case MC_SEQ_MISC0__MT__GDDR3:
823 		return AMDGPU_VRAM_TYPE_GDDR3;
824 	case MC_SEQ_MISC0__MT__GDDR4:
825 		return AMDGPU_VRAM_TYPE_GDDR4;
826 	case MC_SEQ_MISC0__MT__GDDR5:
827 		return AMDGPU_VRAM_TYPE_GDDR5;
828 	case MC_SEQ_MISC0__MT__HBM:
829 		return AMDGPU_VRAM_TYPE_HBM;
830 	case MC_SEQ_MISC0__MT__DDR3:
831 		return AMDGPU_VRAM_TYPE_DDR3;
832 	default:
833 		return AMDGPU_VRAM_TYPE_UNKNOWN;
834 	}
835 }
836 
837 static int gmc_v8_0_early_init(void *handle)
838 {
839 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
840 
841 	gmc_v8_0_set_gart_funcs(adev);
842 	gmc_v8_0_set_irq_funcs(adev);
843 
844 	if (adev->flags & AMD_IS_APU) {
845 		adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
846 	} else {
847 		u32 tmp = RREG32(mmMC_SEQ_MISC0);
848 		tmp &= MC_SEQ_MISC0__MT__MASK;
849 		adev->mc.vram_type = gmc_v8_0_convert_vram_type(tmp);
850 	}
851 
852 	return 0;
853 }
854 
855 static int gmc_v8_0_sw_init(void *handle)
856 {
857 	int r;
858 	int dma_bits;
859 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
860 
861 	r = amdgpu_gem_init(adev);
862 	if (r)
863 		return r;
864 
865 	r = amdgpu_irq_add_id(adev, 146, &adev->mc.vm_fault);
866 	if (r)
867 		return r;
868 
869 	r = amdgpu_irq_add_id(adev, 147, &adev->mc.vm_fault);
870 	if (r)
871 		return r;
872 
873 	/* Adjust VM size here.
874 	 * Currently set to 4GB ((1 << 20) 4k pages).
875 	 * Max GPUVM size for cayman and SI is 40 bits.
876 	 */
877 	adev->vm_manager.max_pfn = amdgpu_vm_size << 18;
878 
879 	/* Set the internal MC address mask
880 	 * This is the max address of the GPU's
881 	 * internal address space.
882 	 */
883 	adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
884 
885 	/* set DMA mask + need_dma32 flags.
886 	 * PCIE - can handle 40-bits.
887 	 * IGP - can handle 40-bits
888 	 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
889 	 */
890 	adev->need_dma32 = false;
891 	dma_bits = adev->need_dma32 ? 32 : 40;
892 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
893 	if (r) {
894 		adev->need_dma32 = true;
895 		dma_bits = 32;
896 		printk(KERN_WARNING "amdgpu: No suitable DMA available.\n");
897 	}
898 	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
899 	if (r) {
900 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
901 		printk(KERN_WARNING "amdgpu: No coherent DMA available.\n");
902 	}
903 
904 	r = gmc_v8_0_init_microcode(adev);
905 	if (r) {
906 		DRM_ERROR("Failed to load mc firmware!\n");
907 		return r;
908 	}
909 
910 	r = gmc_v8_0_mc_init(adev);
911 	if (r)
912 		return r;
913 
914 	/* Memory manager */
915 	r = amdgpu_bo_init(adev);
916 	if (r)
917 		return r;
918 
919 	r = gmc_v8_0_gart_init(adev);
920 	if (r)
921 		return r;
922 
923 	if (!adev->vm_manager.enabled) {
924 		r = gmc_v8_0_vm_init(adev);
925 		if (r) {
926 			dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
927 			return r;
928 		}
929 		adev->vm_manager.enabled = true;
930 	}
931 
932 	return r;
933 }
934 
935 static int gmc_v8_0_sw_fini(void *handle)
936 {
937 	int i;
938 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
939 
940 	if (adev->vm_manager.enabled) {
941 		for (i = 0; i < AMDGPU_NUM_VM; ++i)
942 			amdgpu_fence_unref(&adev->vm_manager.active[i]);
943 		gmc_v8_0_vm_fini(adev);
944 		adev->vm_manager.enabled = false;
945 	}
946 	gmc_v8_0_gart_fini(adev);
947 	amdgpu_gem_fini(adev);
948 	amdgpu_bo_fini(adev);
949 
950 	return 0;
951 }
952 
953 static int gmc_v8_0_hw_init(void *handle)
954 {
955 	int r;
956 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
957 
958 	gmc_v8_0_init_golden_registers(adev);
959 
960 	gmc_v8_0_mc_program(adev);
961 
962 	if (!(adev->flags & AMD_IS_APU)) {
963 		r = gmc_v8_0_mc_load_microcode(adev);
964 		if (r) {
965 			DRM_ERROR("Failed to load MC firmware!\n");
966 			return r;
967 		}
968 	}
969 
970 	r = gmc_v8_0_gart_enable(adev);
971 	if (r)
972 		return r;
973 
974 	return r;
975 }
976 
977 static int gmc_v8_0_hw_fini(void *handle)
978 {
979 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
980 
981 	gmc_v8_0_gart_disable(adev);
982 
983 	return 0;
984 }
985 
986 static int gmc_v8_0_suspend(void *handle)
987 {
988 	int i;
989 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
990 
991 	if (adev->vm_manager.enabled) {
992 		for (i = 0; i < AMDGPU_NUM_VM; ++i)
993 			amdgpu_fence_unref(&adev->vm_manager.active[i]);
994 		gmc_v8_0_vm_fini(adev);
995 		adev->vm_manager.enabled = false;
996 	}
997 	gmc_v8_0_hw_fini(adev);
998 
999 	return 0;
1000 }
1001 
1002 static int gmc_v8_0_resume(void *handle)
1003 {
1004 	int r;
1005 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1006 
1007 	r = gmc_v8_0_hw_init(adev);
1008 	if (r)
1009 		return r;
1010 
1011 	if (!adev->vm_manager.enabled) {
1012 		r = gmc_v8_0_vm_init(adev);
1013 		if (r) {
1014 			dev_err(adev->dev, "vm manager initialization failed (%d).\n", r);
1015 			return r;
1016 		}
1017 		adev->vm_manager.enabled = true;
1018 	}
1019 
1020 	return r;
1021 }
1022 
1023 static bool gmc_v8_0_is_idle(void *handle)
1024 {
1025 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1026 	u32 tmp = RREG32(mmSRBM_STATUS);
1027 
1028 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1029 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
1030 		return false;
1031 
1032 	return true;
1033 }
1034 
1035 static int gmc_v8_0_wait_for_idle(void *handle)
1036 {
1037 	unsigned i;
1038 	u32 tmp;
1039 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1040 
1041 	for (i = 0; i < adev->usec_timeout; i++) {
1042 		/* read MC_STATUS */
1043 		tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1044 					       SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1045 					       SRBM_STATUS__MCC_BUSY_MASK |
1046 					       SRBM_STATUS__MCD_BUSY_MASK |
1047 					       SRBM_STATUS__VMC_BUSY_MASK |
1048 					       SRBM_STATUS__VMC1_BUSY_MASK);
1049 		if (!tmp)
1050 			return 0;
1051 		udelay(1);
1052 	}
1053 	return -ETIMEDOUT;
1054 
1055 }
1056 
1057 static void gmc_v8_0_print_status(void *handle)
1058 {
1059 	int i, j;
1060 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1061 
1062 	dev_info(adev->dev, "GMC 8.x registers\n");
1063 	dev_info(adev->dev, "  SRBM_STATUS=0x%08X\n",
1064 		RREG32(mmSRBM_STATUS));
1065 	dev_info(adev->dev, "  SRBM_STATUS2=0x%08X\n",
1066 		RREG32(mmSRBM_STATUS2));
1067 
1068 	dev_info(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1069 		 RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR));
1070 	dev_info(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1071 		 RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS));
1072 	dev_info(adev->dev, "  MC_VM_MX_L1_TLB_CNTL=0x%08X\n",
1073 		 RREG32(mmMC_VM_MX_L1_TLB_CNTL));
1074 	dev_info(adev->dev, "  VM_L2_CNTL=0x%08X\n",
1075 		 RREG32(mmVM_L2_CNTL));
1076 	dev_info(adev->dev, "  VM_L2_CNTL2=0x%08X\n",
1077 		 RREG32(mmVM_L2_CNTL2));
1078 	dev_info(adev->dev, "  VM_L2_CNTL3=0x%08X\n",
1079 		 RREG32(mmVM_L2_CNTL3));
1080 	dev_info(adev->dev, "  VM_L2_CNTL4=0x%08X\n",
1081 		 RREG32(mmVM_L2_CNTL4));
1082 	dev_info(adev->dev, "  VM_CONTEXT0_PAGE_TABLE_START_ADDR=0x%08X\n",
1083 		 RREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR));
1084 	dev_info(adev->dev, "  VM_CONTEXT0_PAGE_TABLE_END_ADDR=0x%08X\n",
1085 		 RREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR));
1086 	dev_info(adev->dev, "  VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR=0x%08X\n",
1087 		 RREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR));
1088 	dev_info(adev->dev, "  VM_CONTEXT0_CNTL2=0x%08X\n",
1089 		 RREG32(mmVM_CONTEXT0_CNTL2));
1090 	dev_info(adev->dev, "  VM_CONTEXT0_CNTL=0x%08X\n",
1091 		 RREG32(mmVM_CONTEXT0_CNTL));
1092 	dev_info(adev->dev, "  VM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR=0x%08X\n",
1093 		 RREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR));
1094 	dev_info(adev->dev, "  VM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR=0x%08X\n",
1095 		 RREG32(mmVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR));
1096 	dev_info(adev->dev, "  mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET=0x%08X\n",
1097 		 RREG32(mmVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET));
1098 	dev_info(adev->dev, "  VM_CONTEXT1_PAGE_TABLE_START_ADDR=0x%08X\n",
1099 		 RREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR));
1100 	dev_info(adev->dev, "  VM_CONTEXT1_PAGE_TABLE_END_ADDR=0x%08X\n",
1101 		 RREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR));
1102 	dev_info(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR=0x%08X\n",
1103 		 RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR));
1104 	dev_info(adev->dev, "  VM_CONTEXT1_CNTL2=0x%08X\n",
1105 		 RREG32(mmVM_CONTEXT1_CNTL2));
1106 	dev_info(adev->dev, "  VM_CONTEXT1_CNTL=0x%08X\n",
1107 		 RREG32(mmVM_CONTEXT1_CNTL));
1108 	for (i = 0; i < 16; i++) {
1109 		if (i < 8)
1110 			dev_info(adev->dev, "  VM_CONTEXT%d_PAGE_TABLE_BASE_ADDR=0x%08X\n",
1111 				 i, RREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i));
1112 		else
1113 			dev_info(adev->dev, "  VM_CONTEXT%d_PAGE_TABLE_BASE_ADDR=0x%08X\n",
1114 				 i, RREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8));
1115 	}
1116 	dev_info(adev->dev, "  MC_VM_SYSTEM_APERTURE_LOW_ADDR=0x%08X\n",
1117 		 RREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR));
1118 	dev_info(adev->dev, "  MC_VM_SYSTEM_APERTURE_HIGH_ADDR=0x%08X\n",
1119 		 RREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR));
1120 	dev_info(adev->dev, "  MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR=0x%08X\n",
1121 		 RREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR));
1122 	dev_info(adev->dev, "  MC_VM_FB_LOCATION=0x%08X\n",
1123 		 RREG32(mmMC_VM_FB_LOCATION));
1124 	dev_info(adev->dev, "  MC_VM_AGP_BASE=0x%08X\n",
1125 		 RREG32(mmMC_VM_AGP_BASE));
1126 	dev_info(adev->dev, "  MC_VM_AGP_TOP=0x%08X\n",
1127 		 RREG32(mmMC_VM_AGP_TOP));
1128 	dev_info(adev->dev, "  MC_VM_AGP_BOT=0x%08X\n",
1129 		 RREG32(mmMC_VM_AGP_BOT));
1130 
1131 	dev_info(adev->dev, "  HDP_REG_COHERENCY_FLUSH_CNTL=0x%08X\n",
1132 		 RREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL));
1133 	dev_info(adev->dev, "  HDP_NONSURFACE_BASE=0x%08X\n",
1134 		 RREG32(mmHDP_NONSURFACE_BASE));
1135 	dev_info(adev->dev, "  HDP_NONSURFACE_INFO=0x%08X\n",
1136 		 RREG32(mmHDP_NONSURFACE_INFO));
1137 	dev_info(adev->dev, "  HDP_NONSURFACE_SIZE=0x%08X\n",
1138 		 RREG32(mmHDP_NONSURFACE_SIZE));
1139 	dev_info(adev->dev, "  HDP_MISC_CNTL=0x%08X\n",
1140 		 RREG32(mmHDP_MISC_CNTL));
1141 	dev_info(adev->dev, "  HDP_HOST_PATH_CNTL=0x%08X\n",
1142 		 RREG32(mmHDP_HOST_PATH_CNTL));
1143 
1144 	for (i = 0, j = 0; i < 32; i++, j += 0x6) {
1145 		dev_info(adev->dev, "  %d:\n", i);
1146 		dev_info(adev->dev, "  0x%04X=0x%08X\n",
1147 			 0xb05 + j, RREG32(0xb05 + j));
1148 		dev_info(adev->dev, "  0x%04X=0x%08X\n",
1149 			 0xb06 + j, RREG32(0xb06 + j));
1150 		dev_info(adev->dev, "  0x%04X=0x%08X\n",
1151 			 0xb07 + j, RREG32(0xb07 + j));
1152 		dev_info(adev->dev, "  0x%04X=0x%08X\n",
1153 			 0xb08 + j, RREG32(0xb08 + j));
1154 		dev_info(adev->dev, "  0x%04X=0x%08X\n",
1155 			 0xb09 + j, RREG32(0xb09 + j));
1156 	}
1157 
1158 	dev_info(adev->dev, "  BIF_FB_EN=0x%08X\n",
1159 		 RREG32(mmBIF_FB_EN));
1160 }
1161 
1162 static int gmc_v8_0_soft_reset(void *handle)
1163 {
1164 	struct amdgpu_mode_mc_save save;
1165 	u32 srbm_soft_reset = 0;
1166 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1167 	u32 tmp = RREG32(mmSRBM_STATUS);
1168 
1169 	if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1170 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1171 						SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1172 
1173 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1174 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
1175 		if (!(adev->flags & AMD_IS_APU))
1176 			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1177 							SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1178 	}
1179 
1180 	if (srbm_soft_reset) {
1181 		gmc_v8_0_print_status((void *)adev);
1182 
1183 		gmc_v8_0_mc_stop(adev, &save);
1184 		if (gmc_v8_0_wait_for_idle(adev)) {
1185 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1186 		}
1187 
1188 
1189 		tmp = RREG32(mmSRBM_SOFT_RESET);
1190 		tmp |= srbm_soft_reset;
1191 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1192 		WREG32(mmSRBM_SOFT_RESET, tmp);
1193 		tmp = RREG32(mmSRBM_SOFT_RESET);
1194 
1195 		udelay(50);
1196 
1197 		tmp &= ~srbm_soft_reset;
1198 		WREG32(mmSRBM_SOFT_RESET, tmp);
1199 		tmp = RREG32(mmSRBM_SOFT_RESET);
1200 
1201 		/* Wait a little for things to settle down */
1202 		udelay(50);
1203 
1204 		gmc_v8_0_mc_resume(adev, &save);
1205 		udelay(50);
1206 
1207 		gmc_v8_0_print_status((void *)adev);
1208 	}
1209 
1210 	return 0;
1211 }
1212 
1213 static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1214 					     struct amdgpu_irq_src *src,
1215 					     unsigned type,
1216 					     enum amdgpu_interrupt_state state)
1217 {
1218 	u32 tmp;
1219 	u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1220 		    VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1221 		    VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1222 		    VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1223 		    VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1224 		    VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1225 		    VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1226 
1227 	switch (state) {
1228 	case AMDGPU_IRQ_STATE_DISABLE:
1229 		/* system context */
1230 		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1231 		tmp &= ~bits;
1232 		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1233 		/* VMs */
1234 		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1235 		tmp &= ~bits;
1236 		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1237 		break;
1238 	case AMDGPU_IRQ_STATE_ENABLE:
1239 		/* system context */
1240 		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1241 		tmp |= bits;
1242 		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1243 		/* VMs */
1244 		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1245 		tmp |= bits;
1246 		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1247 		break;
1248 	default:
1249 		break;
1250 	}
1251 
1252 	return 0;
1253 }
1254 
1255 static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev,
1256 				      struct amdgpu_irq_src *source,
1257 				      struct amdgpu_iv_entry *entry)
1258 {
1259 	u32 addr, status, mc_client;
1260 
1261 	addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1262 	status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1263 	mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
1264 	dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1265 		entry->src_id, entry->src_data);
1266 	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1267 		addr);
1268 	dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1269 		status);
1270 	gmc_v8_0_vm_decode_fault(adev, status, addr, mc_client);
1271 	/* reset addr and status */
1272 	WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1273 
1274 	return 0;
1275 }
1276 
1277 static int gmc_v8_0_set_clockgating_state(void *handle,
1278 					  enum amd_clockgating_state state)
1279 {
1280 	return 0;
1281 }
1282 
1283 static int gmc_v8_0_set_powergating_state(void *handle,
1284 					  enum amd_powergating_state state)
1285 {
1286 	return 0;
1287 }
1288 
1289 const struct amd_ip_funcs gmc_v8_0_ip_funcs = {
1290 	.early_init = gmc_v8_0_early_init,
1291 	.late_init = NULL,
1292 	.sw_init = gmc_v8_0_sw_init,
1293 	.sw_fini = gmc_v8_0_sw_fini,
1294 	.hw_init = gmc_v8_0_hw_init,
1295 	.hw_fini = gmc_v8_0_hw_fini,
1296 	.suspend = gmc_v8_0_suspend,
1297 	.resume = gmc_v8_0_resume,
1298 	.is_idle = gmc_v8_0_is_idle,
1299 	.wait_for_idle = gmc_v8_0_wait_for_idle,
1300 	.soft_reset = gmc_v8_0_soft_reset,
1301 	.print_status = gmc_v8_0_print_status,
1302 	.set_clockgating_state = gmc_v8_0_set_clockgating_state,
1303 	.set_powergating_state = gmc_v8_0_set_powergating_state,
1304 };
1305 
1306 static const struct amdgpu_gart_funcs gmc_v8_0_gart_funcs = {
1307 	.flush_gpu_tlb = gmc_v8_0_gart_flush_gpu_tlb,
1308 	.set_pte_pde = gmc_v8_0_gart_set_pte_pde,
1309 };
1310 
1311 static const struct amdgpu_irq_src_funcs gmc_v8_0_irq_funcs = {
1312 	.set = gmc_v8_0_vm_fault_interrupt_state,
1313 	.process = gmc_v8_0_process_interrupt,
1314 };
1315 
1316 static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev)
1317 {
1318 	if (adev->gart.gart_funcs == NULL)
1319 		adev->gart.gart_funcs = &gmc_v8_0_gart_funcs;
1320 }
1321 
1322 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev)
1323 {
1324 	adev->mc.vm_fault.num_types = 1;
1325 	adev->mc.vm_fault.funcs = &gmc_v8_0_irq_funcs;
1326 }
1327