xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c (revision 93f5715e)
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include <drm/drmP.h>
25 #include <drm/drm_cache.h>
26 #include "amdgpu.h"
27 #include "cikd.h"
28 #include "cik.h"
29 #include "gmc_v7_0.h"
30 #include "amdgpu_ucode.h"
31 #include "amdgpu_amdkfd.h"
32 
33 #include "bif/bif_4_1_d.h"
34 #include "bif/bif_4_1_sh_mask.h"
35 
36 #include "gmc/gmc_7_1_d.h"
37 #include "gmc/gmc_7_1_sh_mask.h"
38 
39 #include "oss/oss_2_0_d.h"
40 #include "oss/oss_2_0_sh_mask.h"
41 
42 #include "dce/dce_8_0_d.h"
43 #include "dce/dce_8_0_sh_mask.h"
44 
45 #include "amdgpu_atombios.h"
46 
47 #include "ivsrcid/ivsrcid_vislands30.h"
48 
49 static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev);
50 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
51 static int gmc_v7_0_wait_for_idle(void *handle);
52 
53 MODULE_FIRMWARE("amdgpu/bonaire_mc.bin");
54 MODULE_FIRMWARE("amdgpu/hawaii_mc.bin");
55 MODULE_FIRMWARE("amdgpu/topaz_mc.bin");
56 
57 static const u32 golden_settings_iceland_a11[] =
58 {
59 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
60 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
61 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
62 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
63 };
64 
65 static const u32 iceland_mgcg_cgcg_init[] =
66 {
67 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
68 };
69 
70 static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
71 {
72 	switch (adev->asic_type) {
73 	case CHIP_TOPAZ:
74 		amdgpu_device_program_register_sequence(adev,
75 							iceland_mgcg_cgcg_init,
76 							ARRAY_SIZE(iceland_mgcg_cgcg_init));
77 		amdgpu_device_program_register_sequence(adev,
78 							golden_settings_iceland_a11,
79 							ARRAY_SIZE(golden_settings_iceland_a11));
80 		break;
81 	default:
82 		break;
83 	}
84 }
85 
86 static void gmc_v7_0_mc_stop(struct amdgpu_device *adev)
87 {
88 	u32 blackout;
89 
90 	gmc_v7_0_wait_for_idle((void *)adev);
91 
92 	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
93 	if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
94 		/* Block CPU access */
95 		WREG32(mmBIF_FB_EN, 0);
96 		/* blackout the MC */
97 		blackout = REG_SET_FIELD(blackout,
98 					 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
99 		WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
100 	}
101 	/* wait for the MC to settle */
102 	udelay(100);
103 }
104 
105 static void gmc_v7_0_mc_resume(struct amdgpu_device *adev)
106 {
107 	u32 tmp;
108 
109 	/* unblackout the MC */
110 	tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
111 	tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
112 	WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
113 	/* allow CPU access */
114 	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
115 	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
116 	WREG32(mmBIF_FB_EN, tmp);
117 }
118 
119 /**
120  * gmc_v7_0_init_microcode - load ucode images from disk
121  *
122  * @adev: amdgpu_device pointer
123  *
124  * Use the firmware interface to load the ucode images into
125  * the driver (not loaded into hw).
126  * Returns 0 on success, error on failure.
127  */
128 static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)
129 {
130 	const char *chip_name;
131 	char fw_name[30];
132 	int err;
133 
134 	DRM_DEBUG("\n");
135 
136 	switch (adev->asic_type) {
137 	case CHIP_BONAIRE:
138 		chip_name = "bonaire";
139 		break;
140 	case CHIP_HAWAII:
141 		chip_name = "hawaii";
142 		break;
143 	case CHIP_TOPAZ:
144 		chip_name = "topaz";
145 		break;
146 	case CHIP_KAVERI:
147 	case CHIP_KABINI:
148 	case CHIP_MULLINS:
149 		return 0;
150 	default: BUG();
151 	}
152 
153 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
154 
155 	err = request_firmware(&adev->gmc.fw, fw_name, adev->dev);
156 	if (err)
157 		goto out;
158 	err = amdgpu_ucode_validate(adev->gmc.fw);
159 
160 out:
161 	if (err) {
162 		pr_err("cik_mc: Failed to load firmware \"%s\"\n", fw_name);
163 		release_firmware(adev->gmc.fw);
164 		adev->gmc.fw = NULL;
165 	}
166 	return err;
167 }
168 
169 /**
170  * gmc_v7_0_mc_load_microcode - load MC ucode into the hw
171  *
172  * @adev: amdgpu_device pointer
173  *
174  * Load the GDDR MC ucode into the hw (CIK).
175  * Returns 0 on success, error on failure.
176  */
177 static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
178 {
179 	const struct mc_firmware_header_v1_0 *hdr;
180 	const __le32 *fw_data = NULL;
181 	const __le32 *io_mc_regs = NULL;
182 	u32 running;
183 	int i, ucode_size, regs_size;
184 
185 	if (!adev->gmc.fw)
186 		return -EINVAL;
187 
188 	hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;
189 	amdgpu_ucode_print_mc_hdr(&hdr->header);
190 
191 	adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);
192 	regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
193 	io_mc_regs = (const __le32 *)
194 		(adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
195 	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
196 	fw_data = (const __le32 *)
197 		(adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
198 
199 	running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
200 
201 	if (running == 0) {
202 		/* reset the engine and set to writable */
203 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
204 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
205 
206 		/* load mc io regs */
207 		for (i = 0; i < regs_size; i++) {
208 			WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
209 			WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
210 		}
211 		/* load the MC ucode */
212 		for (i = 0; i < ucode_size; i++)
213 			WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
214 
215 		/* put the engine back into the active state */
216 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
217 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
218 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
219 
220 		/* wait for training to complete */
221 		for (i = 0; i < adev->usec_timeout; i++) {
222 			if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
223 					  MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
224 				break;
225 			udelay(1);
226 		}
227 		for (i = 0; i < adev->usec_timeout; i++) {
228 			if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
229 					  MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
230 				break;
231 			udelay(1);
232 		}
233 	}
234 
235 	return 0;
236 }
237 
238 static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
239 				       struct amdgpu_gmc *mc)
240 {
241 	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
242 	base <<= 24;
243 
244 	amdgpu_device_vram_location(adev, &adev->gmc, base);
245 	amdgpu_device_gart_location(adev, mc);
246 }
247 
248 /**
249  * gmc_v7_0_mc_program - program the GPU memory controller
250  *
251  * @adev: amdgpu_device pointer
252  *
253  * Set the location of vram, gart, and AGP in the GPU's
254  * physical address space (CIK).
255  */
256 static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
257 {
258 	u32 tmp;
259 	int i, j;
260 
261 	/* Initialize HDP */
262 	for (i = 0, j = 0; i < 32; i++, j += 0x6) {
263 		WREG32((0xb05 + j), 0x00000000);
264 		WREG32((0xb06 + j), 0x00000000);
265 		WREG32((0xb07 + j), 0x00000000);
266 		WREG32((0xb08 + j), 0x00000000);
267 		WREG32((0xb09 + j), 0x00000000);
268 	}
269 	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
270 
271 	if (gmc_v7_0_wait_for_idle((void *)adev)) {
272 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
273 	}
274 	if (adev->mode_info.num_crtc) {
275 		/* Lockout access through VGA aperture*/
276 		tmp = RREG32(mmVGA_HDP_CONTROL);
277 		tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
278 		WREG32(mmVGA_HDP_CONTROL, tmp);
279 
280 		/* disable VGA render */
281 		tmp = RREG32(mmVGA_RENDER_CONTROL);
282 		tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
283 		WREG32(mmVGA_RENDER_CONTROL, tmp);
284 	}
285 	/* Update configuration */
286 	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
287 	       adev->gmc.vram_start >> 12);
288 	WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
289 	       adev->gmc.vram_end >> 12);
290 	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
291 	       adev->vram_scratch.gpu_addr >> 12);
292 	WREG32(mmMC_VM_AGP_BASE, 0);
293 	WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
294 	WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
295 	if (gmc_v7_0_wait_for_idle((void *)adev)) {
296 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
297 	}
298 
299 	WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
300 
301 	tmp = RREG32(mmHDP_MISC_CNTL);
302 	tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
303 	WREG32(mmHDP_MISC_CNTL, tmp);
304 
305 	tmp = RREG32(mmHDP_HOST_PATH_CNTL);
306 	WREG32(mmHDP_HOST_PATH_CNTL, tmp);
307 }
308 
309 /**
310  * gmc_v7_0_mc_init - initialize the memory controller driver params
311  *
312  * @adev: amdgpu_device pointer
313  *
314  * Look up the amount of vram, vram width, and decide how to place
315  * vram and gart within the GPU's physical address space (CIK).
316  * Returns 0 for success.
317  */
318 static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
319 {
320 	int r;
321 
322 	adev->gmc.vram_width = amdgpu_atombios_get_vram_width(adev);
323 	if (!adev->gmc.vram_width) {
324 		u32 tmp;
325 		int chansize, numchan;
326 
327 		/* Get VRAM informations */
328 		tmp = RREG32(mmMC_ARB_RAMCFG);
329 		if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
330 			chansize = 64;
331 		} else {
332 			chansize = 32;
333 		}
334 		tmp = RREG32(mmMC_SHARED_CHMAP);
335 		switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
336 		case 0:
337 		default:
338 			numchan = 1;
339 			break;
340 		case 1:
341 			numchan = 2;
342 			break;
343 		case 2:
344 			numchan = 4;
345 			break;
346 		case 3:
347 			numchan = 8;
348 			break;
349 		case 4:
350 			numchan = 3;
351 			break;
352 		case 5:
353 			numchan = 6;
354 			break;
355 		case 6:
356 			numchan = 10;
357 			break;
358 		case 7:
359 			numchan = 12;
360 			break;
361 		case 8:
362 			numchan = 16;
363 			break;
364 		}
365 		adev->gmc.vram_width = numchan * chansize;
366 	}
367 	/* size in MB on si */
368 	adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
369 	adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
370 
371 	if (!(adev->flags & AMD_IS_APU)) {
372 		r = amdgpu_device_resize_fb_bar(adev);
373 		if (r)
374 			return r;
375 	}
376 	adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
377 	adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
378 
379 #ifdef CONFIG_X86_64
380 	if (adev->flags & AMD_IS_APU) {
381 		adev->gmc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
382 		adev->gmc.aper_size = adev->gmc.real_vram_size;
383 	}
384 #endif
385 
386 	/* In case the PCI BAR is larger than the actual amount of vram */
387 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
388 	if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size)
389 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
390 
391 	/* set the gart size */
392 	if (amdgpu_gart_size == -1) {
393 		switch (adev->asic_type) {
394 		case CHIP_TOPAZ:     /* no MM engines */
395 		default:
396 			adev->gmc.gart_size = 256ULL << 20;
397 			break;
398 #ifdef CONFIG_DRM_AMDGPU_CIK
399 		case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
400 		case CHIP_HAWAII:  /* UVD, VCE do not support GPUVM */
401 		case CHIP_KAVERI:  /* UVD, VCE do not support GPUVM */
402 		case CHIP_KABINI:  /* UVD, VCE do not support GPUVM */
403 		case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
404 			adev->gmc.gart_size = 1024ULL << 20;
405 			break;
406 #endif
407 		}
408 	} else {
409 		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
410 	}
411 
412 	gmc_v7_0_vram_gtt_location(adev, &adev->gmc);
413 
414 	return 0;
415 }
416 
417 /*
418  * GART
419  * VMID 0 is the physical GPU addresses as used by the kernel.
420  * VMIDs 1-15 are used for userspace clients and are handled
421  * by the amdgpu vm/hsa code.
422  */
423 
424 /**
425  * gmc_v7_0_flush_gpu_tlb - gart tlb flush callback
426  *
427  * @adev: amdgpu_device pointer
428  * @vmid: vm instance to flush
429  *
430  * Flush the TLB for the requested page table (CIK).
431  */
432 static void gmc_v7_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid)
433 {
434 	/* bits 0-15 are the VM contexts0-15 */
435 	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
436 }
437 
438 static uint64_t gmc_v7_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
439 					    unsigned vmid, uint64_t pd_addr)
440 {
441 	uint32_t reg;
442 
443 	if (vmid < 8)
444 		reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid;
445 	else
446 		reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vmid - 8;
447 	amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12);
448 
449 	/* bits 0-15 are the VM contexts0-15 */
450 	amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid);
451 
452 	return pd_addr;
453 }
454 
455 static void gmc_v7_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid,
456 					unsigned pasid)
457 {
458 	amdgpu_ring_emit_wreg(ring, mmIH_VMID_0_LUT + vmid, pasid);
459 }
460 
461 /**
462  * gmc_v7_0_set_pte_pde - update the page tables using MMIO
463  *
464  * @adev: amdgpu_device pointer
465  * @cpu_pt_addr: cpu address of the page table
466  * @gpu_page_idx: entry in the page table to update
467  * @addr: dst addr to write into pte/pde
468  * @flags: access flags
469  *
470  * Update the page tables using the CPU.
471  */
472 static int gmc_v7_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr,
473 				 uint32_t gpu_page_idx, uint64_t addr,
474 				 uint64_t flags)
475 {
476 	void __iomem *ptr = (void *)cpu_pt_addr;
477 	uint64_t value;
478 
479 	value = addr & 0xFFFFFFFFFFFFF000ULL;
480 	value |= flags;
481 	writeq(value, ptr + (gpu_page_idx * 8));
482 
483 	return 0;
484 }
485 
486 static uint64_t gmc_v7_0_get_vm_pte_flags(struct amdgpu_device *adev,
487 					  uint32_t flags)
488 {
489 	uint64_t pte_flag = 0;
490 
491 	if (flags & AMDGPU_VM_PAGE_READABLE)
492 		pte_flag |= AMDGPU_PTE_READABLE;
493 	if (flags & AMDGPU_VM_PAGE_WRITEABLE)
494 		pte_flag |= AMDGPU_PTE_WRITEABLE;
495 	if (flags & AMDGPU_VM_PAGE_PRT)
496 		pte_flag |= AMDGPU_PTE_PRT;
497 
498 	return pte_flag;
499 }
500 
501 static void gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, int level,
502 				uint64_t *addr, uint64_t *flags)
503 {
504 	BUG_ON(*addr & 0xFFFFFF0000000FFFULL);
505 }
506 
507 /**
508  * gmc_v8_0_set_fault_enable_default - update VM fault handling
509  *
510  * @adev: amdgpu_device pointer
511  * @value: true redirects VM faults to the default page
512  */
513 static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,
514 					      bool value)
515 {
516 	u32 tmp;
517 
518 	tmp = RREG32(mmVM_CONTEXT1_CNTL);
519 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
520 			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
521 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
522 			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
523 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
524 			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
525 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
526 			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
527 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
528 			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
529 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
530 			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
531 	WREG32(mmVM_CONTEXT1_CNTL, tmp);
532 }
533 
534 /**
535  * gmc_v7_0_set_prt - set PRT VM fault
536  *
537  * @adev: amdgpu_device pointer
538  * @enable: enable/disable VM fault handling for PRT
539  */
540 static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)
541 {
542 	uint32_t tmp;
543 
544 	if (enable && !adev->gmc.prt_warning) {
545 		dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
546 		adev->gmc.prt_warning = true;
547 	}
548 
549 	tmp = RREG32(mmVM_PRT_CNTL);
550 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
551 			    CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
552 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
553 			    CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
554 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
555 			    TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
556 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
557 			    TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
558 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
559 			    L2_CACHE_STORE_INVALID_ENTRIES, enable);
560 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
561 			    L1_TLB_STORE_INVALID_ENTRIES, enable);
562 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
563 			    MASK_PDE0_FAULT, enable);
564 	WREG32(mmVM_PRT_CNTL, tmp);
565 
566 	if (enable) {
567 		uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
568 		uint32_t high = adev->vm_manager.max_pfn -
569 			(AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT);
570 
571 		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
572 		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
573 		WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
574 		WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
575 		WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
576 		WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
577 		WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
578 		WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
579 	} else {
580 		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
581 		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
582 		WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
583 		WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
584 		WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
585 		WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
586 		WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
587 		WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
588 	}
589 }
590 
591 /**
592  * gmc_v7_0_gart_enable - gart enable
593  *
594  * @adev: amdgpu_device pointer
595  *
596  * This sets up the TLBs, programs the page tables for VMID0,
597  * sets up the hw for VMIDs 1-15 which are allocated on
598  * demand, and sets up the global locations for the LDS, GDS,
599  * and GPUVM for FSA64 clients (CIK).
600  * Returns 0 for success, errors for failure.
601  */
602 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
603 {
604 	int r, i;
605 	u32 tmp, field;
606 
607 	if (adev->gart.robj == NULL) {
608 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
609 		return -EINVAL;
610 	}
611 	r = amdgpu_gart_table_vram_pin(adev);
612 	if (r)
613 		return r;
614 	/* Setup TLB control */
615 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
616 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
617 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
618 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
619 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
620 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
621 	WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
622 	/* Setup L2 cache */
623 	tmp = RREG32(mmVM_L2_CNTL);
624 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
625 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
626 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
627 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
628 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
629 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
630 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
631 	WREG32(mmVM_L2_CNTL, tmp);
632 	tmp = REG_SET_FIELD(0, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
633 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
634 	WREG32(mmVM_L2_CNTL2, tmp);
635 
636 	field = adev->vm_manager.fragment_size;
637 	tmp = RREG32(mmVM_L2_CNTL3);
638 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
639 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);
640 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);
641 	WREG32(mmVM_L2_CNTL3, tmp);
642 	/* setup context0 */
643 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);
644 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);
645 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
646 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
647 			(u32)(adev->dummy_page_addr >> 12));
648 	WREG32(mmVM_CONTEXT0_CNTL2, 0);
649 	tmp = RREG32(mmVM_CONTEXT0_CNTL);
650 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
651 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
652 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
653 	WREG32(mmVM_CONTEXT0_CNTL, tmp);
654 
655 	WREG32(0x575, 0);
656 	WREG32(0x576, 0);
657 	WREG32(0x577, 0);
658 
659 	/* empty context1-15 */
660 	/* FIXME start with 4G, once using 2 level pt switch to full
661 	 * vm size space
662 	 */
663 	/* set vm size, must be a multiple of 4 */
664 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
665 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
666 	for (i = 1; i < 16; i++) {
667 		if (i < 8)
668 			WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
669 			       adev->gart.table_addr >> 12);
670 		else
671 			WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
672 			       adev->gart.table_addr >> 12);
673 	}
674 
675 	/* enable context1-15 */
676 	WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
677 	       (u32)(adev->dummy_page_addr >> 12));
678 	WREG32(mmVM_CONTEXT1_CNTL2, 4);
679 	tmp = RREG32(mmVM_CONTEXT1_CNTL);
680 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
681 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
682 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
683 			    adev->vm_manager.block_size - 9);
684 	WREG32(mmVM_CONTEXT1_CNTL, tmp);
685 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
686 		gmc_v7_0_set_fault_enable_default(adev, false);
687 	else
688 		gmc_v7_0_set_fault_enable_default(adev, true);
689 
690 	if (adev->asic_type == CHIP_KAVERI) {
691 		tmp = RREG32(mmCHUB_CONTROL);
692 		tmp &= ~BYPASS_VM;
693 		WREG32(mmCHUB_CONTROL, tmp);
694 	}
695 
696 	gmc_v7_0_flush_gpu_tlb(adev, 0);
697 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
698 		 (unsigned)(adev->gmc.gart_size >> 20),
699 		 (unsigned long long)adev->gart.table_addr);
700 	adev->gart.ready = true;
701 	return 0;
702 }
703 
704 static int gmc_v7_0_gart_init(struct amdgpu_device *adev)
705 {
706 	int r;
707 
708 	if (adev->gart.robj) {
709 		WARN(1, "R600 PCIE GART already initialized\n");
710 		return 0;
711 	}
712 	/* Initialize common gart structure */
713 	r = amdgpu_gart_init(adev);
714 	if (r)
715 		return r;
716 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
717 	adev->gart.gart_pte_flags = 0;
718 	return amdgpu_gart_table_vram_alloc(adev);
719 }
720 
721 /**
722  * gmc_v7_0_gart_disable - gart disable
723  *
724  * @adev: amdgpu_device pointer
725  *
726  * This disables all VM page table (CIK).
727  */
728 static void gmc_v7_0_gart_disable(struct amdgpu_device *adev)
729 {
730 	u32 tmp;
731 
732 	/* Disable all tables */
733 	WREG32(mmVM_CONTEXT0_CNTL, 0);
734 	WREG32(mmVM_CONTEXT1_CNTL, 0);
735 	/* Setup TLB control */
736 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
737 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
738 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
739 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
740 	WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
741 	/* Setup L2 cache */
742 	tmp = RREG32(mmVM_L2_CNTL);
743 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
744 	WREG32(mmVM_L2_CNTL, tmp);
745 	WREG32(mmVM_L2_CNTL2, 0);
746 	amdgpu_gart_table_vram_unpin(adev);
747 }
748 
749 /**
750  * gmc_v7_0_gart_fini - vm fini callback
751  *
752  * @adev: amdgpu_device pointer
753  *
754  * Tears down the driver GART/VM setup (CIK).
755  */
756 static void gmc_v7_0_gart_fini(struct amdgpu_device *adev)
757 {
758 	amdgpu_gart_table_vram_free(adev);
759 	amdgpu_gart_fini(adev);
760 }
761 
762 /**
763  * gmc_v7_0_vm_decode_fault - print human readable fault info
764  *
765  * @adev: amdgpu_device pointer
766  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
767  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
768  *
769  * Print human readable fault information (CIK).
770  */
771 static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev, u32 status,
772 				     u32 addr, u32 mc_client, unsigned pasid)
773 {
774 	u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
775 	u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
776 					PROTECTIONS);
777 	char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
778 		(mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
779 	u32 mc_id;
780 
781 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
782 			      MEMORY_CLIENT_ID);
783 
784 	dev_err(adev->dev, "VM fault (0x%02x, vmid %d, pasid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
785 	       protections, vmid, pasid, addr,
786 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
787 			     MEMORY_CLIENT_RW) ?
788 	       "write" : "read", block, mc_client, mc_id);
789 }
790 
791 
792 static const u32 mc_cg_registers[] = {
793 	mmMC_HUB_MISC_HUB_CG,
794 	mmMC_HUB_MISC_SIP_CG,
795 	mmMC_HUB_MISC_VM_CG,
796 	mmMC_XPB_CLK_GAT,
797 	mmATC_MISC_CG,
798 	mmMC_CITF_MISC_WR_CG,
799 	mmMC_CITF_MISC_RD_CG,
800 	mmMC_CITF_MISC_VM_CG,
801 	mmVM_L2_CG,
802 };
803 
804 static const u32 mc_cg_ls_en[] = {
805 	MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
806 	MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
807 	MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
808 	MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
809 	ATC_MISC_CG__MEM_LS_ENABLE_MASK,
810 	MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
811 	MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
812 	MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
813 	VM_L2_CG__MEM_LS_ENABLE_MASK,
814 };
815 
816 static const u32 mc_cg_en[] = {
817 	MC_HUB_MISC_HUB_CG__ENABLE_MASK,
818 	MC_HUB_MISC_SIP_CG__ENABLE_MASK,
819 	MC_HUB_MISC_VM_CG__ENABLE_MASK,
820 	MC_XPB_CLK_GAT__ENABLE_MASK,
821 	ATC_MISC_CG__ENABLE_MASK,
822 	MC_CITF_MISC_WR_CG__ENABLE_MASK,
823 	MC_CITF_MISC_RD_CG__ENABLE_MASK,
824 	MC_CITF_MISC_VM_CG__ENABLE_MASK,
825 	VM_L2_CG__ENABLE_MASK,
826 };
827 
828 static void gmc_v7_0_enable_mc_ls(struct amdgpu_device *adev,
829 				  bool enable)
830 {
831 	int i;
832 	u32 orig, data;
833 
834 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
835 		orig = data = RREG32(mc_cg_registers[i]);
836 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
837 			data |= mc_cg_ls_en[i];
838 		else
839 			data &= ~mc_cg_ls_en[i];
840 		if (data != orig)
841 			WREG32(mc_cg_registers[i], data);
842 	}
843 }
844 
845 static void gmc_v7_0_enable_mc_mgcg(struct amdgpu_device *adev,
846 				    bool enable)
847 {
848 	int i;
849 	u32 orig, data;
850 
851 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
852 		orig = data = RREG32(mc_cg_registers[i]);
853 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
854 			data |= mc_cg_en[i];
855 		else
856 			data &= ~mc_cg_en[i];
857 		if (data != orig)
858 			WREG32(mc_cg_registers[i], data);
859 	}
860 }
861 
862 static void gmc_v7_0_enable_bif_mgls(struct amdgpu_device *adev,
863 				     bool enable)
864 {
865 	u32 orig, data;
866 
867 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
868 
869 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
870 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
871 		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
872 		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
873 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
874 	} else {
875 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
876 		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
877 		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
878 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
879 	}
880 
881 	if (orig != data)
882 		WREG32_PCIE(ixPCIE_CNTL2, data);
883 }
884 
885 static void gmc_v7_0_enable_hdp_mgcg(struct amdgpu_device *adev,
886 				     bool enable)
887 {
888 	u32 orig, data;
889 
890 	orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
891 
892 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
893 		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
894 	else
895 		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
896 
897 	if (orig != data)
898 		WREG32(mmHDP_HOST_PATH_CNTL, data);
899 }
900 
901 static void gmc_v7_0_enable_hdp_ls(struct amdgpu_device *adev,
902 				   bool enable)
903 {
904 	u32 orig, data;
905 
906 	orig = data = RREG32(mmHDP_MEM_POWER_LS);
907 
908 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
909 		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
910 	else
911 		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
912 
913 	if (orig != data)
914 		WREG32(mmHDP_MEM_POWER_LS, data);
915 }
916 
917 static int gmc_v7_0_convert_vram_type(int mc_seq_vram_type)
918 {
919 	switch (mc_seq_vram_type) {
920 	case MC_SEQ_MISC0__MT__GDDR1:
921 		return AMDGPU_VRAM_TYPE_GDDR1;
922 	case MC_SEQ_MISC0__MT__DDR2:
923 		return AMDGPU_VRAM_TYPE_DDR2;
924 	case MC_SEQ_MISC0__MT__GDDR3:
925 		return AMDGPU_VRAM_TYPE_GDDR3;
926 	case MC_SEQ_MISC0__MT__GDDR4:
927 		return AMDGPU_VRAM_TYPE_GDDR4;
928 	case MC_SEQ_MISC0__MT__GDDR5:
929 		return AMDGPU_VRAM_TYPE_GDDR5;
930 	case MC_SEQ_MISC0__MT__HBM:
931 		return AMDGPU_VRAM_TYPE_HBM;
932 	case MC_SEQ_MISC0__MT__DDR3:
933 		return AMDGPU_VRAM_TYPE_DDR3;
934 	default:
935 		return AMDGPU_VRAM_TYPE_UNKNOWN;
936 	}
937 }
938 
939 static int gmc_v7_0_early_init(void *handle)
940 {
941 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
942 
943 	gmc_v7_0_set_gmc_funcs(adev);
944 	gmc_v7_0_set_irq_funcs(adev);
945 
946 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
947 	adev->gmc.shared_aperture_end =
948 		adev->gmc.shared_aperture_start + (4ULL << 30) - 1;
949 	adev->gmc.private_aperture_start =
950 		adev->gmc.shared_aperture_end + 1;
951 	adev->gmc.private_aperture_end =
952 		adev->gmc.private_aperture_start + (4ULL << 30) - 1;
953 
954 	return 0;
955 }
956 
957 static int gmc_v7_0_late_init(void *handle)
958 {
959 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
960 
961 	amdgpu_bo_late_init(adev);
962 
963 	if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
964 		return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
965 	else
966 		return 0;
967 }
968 
969 static unsigned gmc_v7_0_get_vbios_fb_size(struct amdgpu_device *adev)
970 {
971 	u32 d1vga_control = RREG32(mmD1VGA_CONTROL);
972 	unsigned size;
973 
974 	if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
975 		size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */
976 	} else {
977 		u32 viewport = RREG32(mmVIEWPORT_SIZE);
978 		size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
979 			REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *
980 			4);
981 	}
982 	/* return 0 if the pre-OS buffer uses up most of vram */
983 	if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024))
984 		return 0;
985 	return size;
986 }
987 
988 static int gmc_v7_0_sw_init(void *handle)
989 {
990 	int r;
991 	int dma_bits;
992 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
993 
994 	if (adev->flags & AMD_IS_APU) {
995 		adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
996 	} else {
997 		u32 tmp = RREG32(mmMC_SEQ_MISC0);
998 		tmp &= MC_SEQ_MISC0__MT__MASK;
999 		adev->gmc.vram_type = gmc_v7_0_convert_vram_type(tmp);
1000 	}
1001 
1002 	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_PAGE_INV_FAULT, &adev->gmc.vm_fault);
1003 	if (r)
1004 		return r;
1005 
1006 	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, VISLANDS30_IV_SRCID_GFX_MEM_PROT_FAULT, &adev->gmc.vm_fault);
1007 	if (r)
1008 		return r;
1009 
1010 	/* Adjust VM size here.
1011 	 * Currently set to 4GB ((1 << 20) 4k pages).
1012 	 * Max GPUVM size for cayman and SI is 40 bits.
1013 	 */
1014 	amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
1015 
1016 	/* Set the internal MC address mask
1017 	 * This is the max address of the GPU's
1018 	 * internal address space.
1019 	 */
1020 	adev->gmc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
1021 
1022 	/* set DMA mask + need_dma32 flags.
1023 	 * PCIE - can handle 40-bits.
1024 	 * IGP - can handle 40-bits
1025 	 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
1026 	 */
1027 	adev->need_dma32 = false;
1028 	dma_bits = adev->need_dma32 ? 32 : 40;
1029 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1030 	if (r) {
1031 		adev->need_dma32 = true;
1032 		dma_bits = 32;
1033 		pr_warn("amdgpu: No suitable DMA available\n");
1034 	}
1035 	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
1036 	if (r) {
1037 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
1038 		pr_warn("amdgpu: No coherent DMA available\n");
1039 	}
1040 	adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits);
1041 
1042 	r = gmc_v7_0_init_microcode(adev);
1043 	if (r) {
1044 		DRM_ERROR("Failed to load mc firmware!\n");
1045 		return r;
1046 	}
1047 
1048 	r = gmc_v7_0_mc_init(adev);
1049 	if (r)
1050 		return r;
1051 
1052 	adev->gmc.stolen_size = gmc_v7_0_get_vbios_fb_size(adev);
1053 
1054 	/* Memory manager */
1055 	r = amdgpu_bo_init(adev);
1056 	if (r)
1057 		return r;
1058 
1059 	r = gmc_v7_0_gart_init(adev);
1060 	if (r)
1061 		return r;
1062 
1063 	/*
1064 	 * number of VMs
1065 	 * VMID 0 is reserved for System
1066 	 * amdgpu graphics/compute will use VMIDs 1-7
1067 	 * amdkfd will use VMIDs 8-15
1068 	 */
1069 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1070 	amdgpu_vm_manager_init(adev);
1071 
1072 	/* base offset of vram pages */
1073 	if (adev->flags & AMD_IS_APU) {
1074 		u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
1075 
1076 		tmp <<= 22;
1077 		adev->vm_manager.vram_base_offset = tmp;
1078 	} else {
1079 		adev->vm_manager.vram_base_offset = 0;
1080 	}
1081 
1082 	adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),
1083 					GFP_KERNEL);
1084 	if (!adev->gmc.vm_fault_info)
1085 		return -ENOMEM;
1086 	atomic_set(&adev->gmc.vm_fault_info_updated, 0);
1087 
1088 	return 0;
1089 }
1090 
1091 static int gmc_v7_0_sw_fini(void *handle)
1092 {
1093 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1094 
1095 	amdgpu_gem_force_release(adev);
1096 	amdgpu_vm_manager_fini(adev);
1097 	kfree(adev->gmc.vm_fault_info);
1098 	gmc_v7_0_gart_fini(adev);
1099 	amdgpu_bo_fini(adev);
1100 	release_firmware(adev->gmc.fw);
1101 	adev->gmc.fw = NULL;
1102 
1103 	return 0;
1104 }
1105 
1106 static int gmc_v7_0_hw_init(void *handle)
1107 {
1108 	int r;
1109 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1110 
1111 	gmc_v7_0_init_golden_registers(adev);
1112 
1113 	gmc_v7_0_mc_program(adev);
1114 
1115 	if (!(adev->flags & AMD_IS_APU)) {
1116 		r = gmc_v7_0_mc_load_microcode(adev);
1117 		if (r) {
1118 			DRM_ERROR("Failed to load MC firmware!\n");
1119 			return r;
1120 		}
1121 	}
1122 
1123 	r = gmc_v7_0_gart_enable(adev);
1124 	if (r)
1125 		return r;
1126 
1127 	return r;
1128 }
1129 
1130 static int gmc_v7_0_hw_fini(void *handle)
1131 {
1132 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1133 
1134 	amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
1135 	gmc_v7_0_gart_disable(adev);
1136 
1137 	return 0;
1138 }
1139 
1140 static int gmc_v7_0_suspend(void *handle)
1141 {
1142 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1143 
1144 	gmc_v7_0_hw_fini(adev);
1145 
1146 	return 0;
1147 }
1148 
1149 static int gmc_v7_0_resume(void *handle)
1150 {
1151 	int r;
1152 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1153 
1154 	r = gmc_v7_0_hw_init(adev);
1155 	if (r)
1156 		return r;
1157 
1158 	amdgpu_vmid_reset_all(adev);
1159 
1160 	return 0;
1161 }
1162 
1163 static bool gmc_v7_0_is_idle(void *handle)
1164 {
1165 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1166 	u32 tmp = RREG32(mmSRBM_STATUS);
1167 
1168 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1169 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
1170 		return false;
1171 
1172 	return true;
1173 }
1174 
1175 static int gmc_v7_0_wait_for_idle(void *handle)
1176 {
1177 	unsigned i;
1178 	u32 tmp;
1179 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1180 
1181 	for (i = 0; i < adev->usec_timeout; i++) {
1182 		/* read MC_STATUS */
1183 		tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1184 					       SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1185 					       SRBM_STATUS__MCC_BUSY_MASK |
1186 					       SRBM_STATUS__MCD_BUSY_MASK |
1187 					       SRBM_STATUS__VMC_BUSY_MASK);
1188 		if (!tmp)
1189 			return 0;
1190 		udelay(1);
1191 	}
1192 	return -ETIMEDOUT;
1193 
1194 }
1195 
1196 static int gmc_v7_0_soft_reset(void *handle)
1197 {
1198 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1199 	u32 srbm_soft_reset = 0;
1200 	u32 tmp = RREG32(mmSRBM_STATUS);
1201 
1202 	if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1203 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1204 						SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1205 
1206 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1207 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
1208 		if (!(adev->flags & AMD_IS_APU))
1209 			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1210 							SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1211 	}
1212 
1213 	if (srbm_soft_reset) {
1214 		gmc_v7_0_mc_stop(adev);
1215 		if (gmc_v7_0_wait_for_idle((void *)adev)) {
1216 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1217 		}
1218 
1219 
1220 		tmp = RREG32(mmSRBM_SOFT_RESET);
1221 		tmp |= srbm_soft_reset;
1222 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1223 		WREG32(mmSRBM_SOFT_RESET, tmp);
1224 		tmp = RREG32(mmSRBM_SOFT_RESET);
1225 
1226 		udelay(50);
1227 
1228 		tmp &= ~srbm_soft_reset;
1229 		WREG32(mmSRBM_SOFT_RESET, tmp);
1230 		tmp = RREG32(mmSRBM_SOFT_RESET);
1231 
1232 		/* Wait a little for things to settle down */
1233 		udelay(50);
1234 
1235 		gmc_v7_0_mc_resume(adev);
1236 		udelay(50);
1237 	}
1238 
1239 	return 0;
1240 }
1241 
1242 static int gmc_v7_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1243 					     struct amdgpu_irq_src *src,
1244 					     unsigned type,
1245 					     enum amdgpu_interrupt_state state)
1246 {
1247 	u32 tmp;
1248 	u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1249 		    VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1250 		    VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1251 		    VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1252 		    VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1253 		    VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1254 
1255 	switch (state) {
1256 	case AMDGPU_IRQ_STATE_DISABLE:
1257 		/* system context */
1258 		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1259 		tmp &= ~bits;
1260 		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1261 		/* VMs */
1262 		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1263 		tmp &= ~bits;
1264 		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1265 		break;
1266 	case AMDGPU_IRQ_STATE_ENABLE:
1267 		/* system context */
1268 		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1269 		tmp |= bits;
1270 		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1271 		/* VMs */
1272 		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1273 		tmp |= bits;
1274 		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1275 		break;
1276 	default:
1277 		break;
1278 	}
1279 
1280 	return 0;
1281 }
1282 
1283 static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
1284 				      struct amdgpu_irq_src *source,
1285 				      struct amdgpu_iv_entry *entry)
1286 {
1287 	u32 addr, status, mc_client, vmid;
1288 
1289 	addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1290 	status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1291 	mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
1292 	/* reset addr and status */
1293 	WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1294 
1295 	if (!addr && !status)
1296 		return 0;
1297 
1298 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1299 		gmc_v7_0_set_fault_enable_default(adev, false);
1300 
1301 	if (printk_ratelimit()) {
1302 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1303 			entry->src_id, entry->src_data[0]);
1304 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1305 			addr);
1306 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1307 			status);
1308 		gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client,
1309 					 entry->pasid);
1310 	}
1311 
1312 	vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1313 			     VMID);
1314 	if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)
1315 		&& !atomic_read(&adev->gmc.vm_fault_info_updated)) {
1316 		struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;
1317 		u32 protections = REG_GET_FIELD(status,
1318 					VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1319 					PROTECTIONS);
1320 
1321 		info->vmid = vmid;
1322 		info->mc_id = REG_GET_FIELD(status,
1323 					    VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1324 					    MEMORY_CLIENT_ID);
1325 		info->status = status;
1326 		info->page_addr = addr;
1327 		info->prot_valid = protections & 0x7 ? true : false;
1328 		info->prot_read = protections & 0x8 ? true : false;
1329 		info->prot_write = protections & 0x10 ? true : false;
1330 		info->prot_exec = protections & 0x20 ? true : false;
1331 		mb();
1332 		atomic_set(&adev->gmc.vm_fault_info_updated, 1);
1333 	}
1334 
1335 	return 0;
1336 }
1337 
1338 static int gmc_v7_0_set_clockgating_state(void *handle,
1339 					  enum amd_clockgating_state state)
1340 {
1341 	bool gate = false;
1342 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1343 
1344 	if (state == AMD_CG_STATE_GATE)
1345 		gate = true;
1346 
1347 	if (!(adev->flags & AMD_IS_APU)) {
1348 		gmc_v7_0_enable_mc_mgcg(adev, gate);
1349 		gmc_v7_0_enable_mc_ls(adev, gate);
1350 	}
1351 	gmc_v7_0_enable_bif_mgls(adev, gate);
1352 	gmc_v7_0_enable_hdp_mgcg(adev, gate);
1353 	gmc_v7_0_enable_hdp_ls(adev, gate);
1354 
1355 	return 0;
1356 }
1357 
1358 static int gmc_v7_0_set_powergating_state(void *handle,
1359 					  enum amd_powergating_state state)
1360 {
1361 	return 0;
1362 }
1363 
1364 static const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
1365 	.name = "gmc_v7_0",
1366 	.early_init = gmc_v7_0_early_init,
1367 	.late_init = gmc_v7_0_late_init,
1368 	.sw_init = gmc_v7_0_sw_init,
1369 	.sw_fini = gmc_v7_0_sw_fini,
1370 	.hw_init = gmc_v7_0_hw_init,
1371 	.hw_fini = gmc_v7_0_hw_fini,
1372 	.suspend = gmc_v7_0_suspend,
1373 	.resume = gmc_v7_0_resume,
1374 	.is_idle = gmc_v7_0_is_idle,
1375 	.wait_for_idle = gmc_v7_0_wait_for_idle,
1376 	.soft_reset = gmc_v7_0_soft_reset,
1377 	.set_clockgating_state = gmc_v7_0_set_clockgating_state,
1378 	.set_powergating_state = gmc_v7_0_set_powergating_state,
1379 };
1380 
1381 static const struct amdgpu_gmc_funcs gmc_v7_0_gmc_funcs = {
1382 	.flush_gpu_tlb = gmc_v7_0_flush_gpu_tlb,
1383 	.emit_flush_gpu_tlb = gmc_v7_0_emit_flush_gpu_tlb,
1384 	.emit_pasid_mapping = gmc_v7_0_emit_pasid_mapping,
1385 	.set_pte_pde = gmc_v7_0_set_pte_pde,
1386 	.set_prt = gmc_v7_0_set_prt,
1387 	.get_vm_pte_flags = gmc_v7_0_get_vm_pte_flags,
1388 	.get_vm_pde = gmc_v7_0_get_vm_pde
1389 };
1390 
1391 static const struct amdgpu_irq_src_funcs gmc_v7_0_irq_funcs = {
1392 	.set = gmc_v7_0_vm_fault_interrupt_state,
1393 	.process = gmc_v7_0_process_interrupt,
1394 };
1395 
1396 static void gmc_v7_0_set_gmc_funcs(struct amdgpu_device *adev)
1397 {
1398 	if (adev->gmc.gmc_funcs == NULL)
1399 		adev->gmc.gmc_funcs = &gmc_v7_0_gmc_funcs;
1400 }
1401 
1402 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)
1403 {
1404 	adev->gmc.vm_fault.num_types = 1;
1405 	adev->gmc.vm_fault.funcs = &gmc_v7_0_irq_funcs;
1406 }
1407 
1408 const struct amdgpu_ip_block_version gmc_v7_0_ip_block =
1409 {
1410 	.type = AMD_IP_BLOCK_TYPE_GMC,
1411 	.major = 7,
1412 	.minor = 0,
1413 	.rev = 0,
1414 	.funcs = &gmc_v7_0_ip_funcs,
1415 };
1416 
1417 const struct amdgpu_ip_block_version gmc_v7_4_ip_block =
1418 {
1419 	.type = AMD_IP_BLOCK_TYPE_GMC,
1420 	.major = 7,
1421 	.minor = 4,
1422 	.rev = 0,
1423 	.funcs = &gmc_v7_0_ip_funcs,
1424 };
1425