xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c (revision 6aa7de05)
1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  */
23 #include <linux/firmware.h>
24 #include <drm/drmP.h>
25 #include "amdgpu.h"
26 #include "cikd.h"
27 #include "cik.h"
28 #include "gmc_v7_0.h"
29 #include "amdgpu_ucode.h"
30 
31 #include "bif/bif_4_1_d.h"
32 #include "bif/bif_4_1_sh_mask.h"
33 
34 #include "gmc/gmc_7_1_d.h"
35 #include "gmc/gmc_7_1_sh_mask.h"
36 
37 #include "oss/oss_2_0_d.h"
38 #include "oss/oss_2_0_sh_mask.h"
39 
40 #include "dce/dce_8_0_d.h"
41 #include "dce/dce_8_0_sh_mask.h"
42 
43 #include "amdgpu_atombios.h"
44 
45 static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev);
46 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
47 static int gmc_v7_0_wait_for_idle(void *handle);
48 
49 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
50 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
51 MODULE_FIRMWARE("amdgpu/topaz_mc.bin");
52 
53 static const u32 golden_settings_iceland_a11[] =
54 {
55 	mmVM_PRT_APERTURE0_LOW_ADDR, 0x0fffffff, 0x0fffffff,
56 	mmVM_PRT_APERTURE1_LOW_ADDR, 0x0fffffff, 0x0fffffff,
57 	mmVM_PRT_APERTURE2_LOW_ADDR, 0x0fffffff, 0x0fffffff,
58 	mmVM_PRT_APERTURE3_LOW_ADDR, 0x0fffffff, 0x0fffffff
59 };
60 
61 static const u32 iceland_mgcg_cgcg_init[] =
62 {
63 	mmMC_MEM_POWER_LS, 0xffffffff, 0x00000104
64 };
65 
66 static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
67 {
68 	switch (adev->asic_type) {
69 	case CHIP_TOPAZ:
70 		amdgpu_program_register_sequence(adev,
71 						 iceland_mgcg_cgcg_init,
72 						 (const u32)ARRAY_SIZE(iceland_mgcg_cgcg_init));
73 		amdgpu_program_register_sequence(adev,
74 						 golden_settings_iceland_a11,
75 						 (const u32)ARRAY_SIZE(golden_settings_iceland_a11));
76 		break;
77 	default:
78 		break;
79 	}
80 }
81 
82 static void gmc_v7_0_mc_stop(struct amdgpu_device *adev)
83 {
84 	u32 blackout;
85 
86 	gmc_v7_0_wait_for_idle((void *)adev);
87 
88 	blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
89 	if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
90 		/* Block CPU access */
91 		WREG32(mmBIF_FB_EN, 0);
92 		/* blackout the MC */
93 		blackout = REG_SET_FIELD(blackout,
94 					 MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
95 		WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
96 	}
97 	/* wait for the MC to settle */
98 	udelay(100);
99 }
100 
101 static void gmc_v7_0_mc_resume(struct amdgpu_device *adev)
102 {
103 	u32 tmp;
104 
105 	/* unblackout the MC */
106 	tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
107 	tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
108 	WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
109 	/* allow CPU access */
110 	tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
111 	tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
112 	WREG32(mmBIF_FB_EN, tmp);
113 }
114 
115 /**
116  * gmc_v7_0_init_microcode - load ucode images from disk
117  *
118  * @adev: amdgpu_device pointer
119  *
120  * Use the firmware interface to load the ucode images into
121  * the driver (not loaded into hw).
122  * Returns 0 on success, error on failure.
123  */
124 static int gmc_v7_0_init_microcode(struct amdgpu_device *adev)
125 {
126 	const char *chip_name;
127 	char fw_name[30];
128 	int err;
129 
130 	DRM_DEBUG("\n");
131 
132 	switch (adev->asic_type) {
133 	case CHIP_BONAIRE:
134 		chip_name = "bonaire";
135 		break;
136 	case CHIP_HAWAII:
137 		chip_name = "hawaii";
138 		break;
139 	case CHIP_TOPAZ:
140 		chip_name = "topaz";
141 		break;
142 	case CHIP_KAVERI:
143 	case CHIP_KABINI:
144 	case CHIP_MULLINS:
145 		return 0;
146 	default: BUG();
147 	}
148 
149 	if (adev->asic_type == CHIP_TOPAZ)
150 		snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
151 	else
152 		snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
153 
154 	err = request_firmware(&adev->mc.fw, fw_name, adev->dev);
155 	if (err)
156 		goto out;
157 	err = amdgpu_ucode_validate(adev->mc.fw);
158 
159 out:
160 	if (err) {
161 		pr_err("cik_mc: Failed to load firmware \"%s\"\n", fw_name);
162 		release_firmware(adev->mc.fw);
163 		adev->mc.fw = NULL;
164 	}
165 	return err;
166 }
167 
168 /**
169  * gmc_v7_0_mc_load_microcode - load MC ucode into the hw
170  *
171  * @adev: amdgpu_device pointer
172  *
173  * Load the GDDR MC ucode into the hw (CIK).
174  * Returns 0 on success, error on failure.
175  */
176 static int gmc_v7_0_mc_load_microcode(struct amdgpu_device *adev)
177 {
178 	const struct mc_firmware_header_v1_0 *hdr;
179 	const __le32 *fw_data = NULL;
180 	const __le32 *io_mc_regs = NULL;
181 	u32 running;
182 	int i, ucode_size, regs_size;
183 
184 	if (!adev->mc.fw)
185 		return -EINVAL;
186 
187 	hdr = (const struct mc_firmware_header_v1_0 *)adev->mc.fw->data;
188 	amdgpu_ucode_print_mc_hdr(&hdr->header);
189 
190 	adev->mc.fw_version = le32_to_cpu(hdr->header.ucode_version);
191 	regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
192 	io_mc_regs = (const __le32 *)
193 		(adev->mc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
194 	ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
195 	fw_data = (const __le32 *)
196 		(adev->mc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
197 
198 	running = REG_GET_FIELD(RREG32(mmMC_SEQ_SUP_CNTL), MC_SEQ_SUP_CNTL, RUN);
199 
200 	if (running == 0) {
201 		/* reset the engine and set to writable */
202 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
203 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
204 
205 		/* load mc io regs */
206 		for (i = 0; i < regs_size; i++) {
207 			WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(io_mc_regs++));
208 			WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(io_mc_regs++));
209 		}
210 		/* load the MC ucode */
211 		for (i = 0; i < ucode_size; i++)
212 			WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(fw_data++));
213 
214 		/* put the engine back into the active state */
215 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
216 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
217 		WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
218 
219 		/* wait for training to complete */
220 		for (i = 0; i < adev->usec_timeout; i++) {
221 			if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
222 					  MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D0))
223 				break;
224 			udelay(1);
225 		}
226 		for (i = 0; i < adev->usec_timeout; i++) {
227 			if (REG_GET_FIELD(RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL),
228 					  MC_SEQ_TRAIN_WAKEUP_CNTL, TRAIN_DONE_D1))
229 				break;
230 			udelay(1);
231 		}
232 	}
233 
234 	return 0;
235 }
236 
237 static void gmc_v7_0_vram_gtt_location(struct amdgpu_device *adev,
238 				       struct amdgpu_mc *mc)
239 {
240 	u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
241 	base <<= 24;
242 
243 	if (mc->mc_vram_size > 0xFFC0000000ULL) {
244 		/* leave room for at least 1024M GTT */
245 		dev_warn(adev->dev, "limiting VRAM\n");
246 		mc->real_vram_size = 0xFFC0000000ULL;
247 		mc->mc_vram_size = 0xFFC0000000ULL;
248 	}
249 	amdgpu_vram_location(adev, &adev->mc, base);
250 	amdgpu_gart_location(adev, mc);
251 }
252 
253 /**
254  * gmc_v7_0_mc_program - program the GPU memory controller
255  *
256  * @adev: amdgpu_device pointer
257  *
258  * Set the location of vram, gart, and AGP in the GPU's
259  * physical address space (CIK).
260  */
261 static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
262 {
263 	u32 tmp;
264 	int i, j;
265 
266 	/* Initialize HDP */
267 	for (i = 0, j = 0; i < 32; i++, j += 0x6) {
268 		WREG32((0xb05 + j), 0x00000000);
269 		WREG32((0xb06 + j), 0x00000000);
270 		WREG32((0xb07 + j), 0x00000000);
271 		WREG32((0xb08 + j), 0x00000000);
272 		WREG32((0xb09 + j), 0x00000000);
273 	}
274 	WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
275 
276 	if (gmc_v7_0_wait_for_idle((void *)adev)) {
277 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
278 	}
279 	if (adev->mode_info.num_crtc) {
280 		/* Lockout access through VGA aperture*/
281 		tmp = RREG32(mmVGA_HDP_CONTROL);
282 		tmp = REG_SET_FIELD(tmp, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1);
283 		WREG32(mmVGA_HDP_CONTROL, tmp);
284 
285 		/* disable VGA render */
286 		tmp = RREG32(mmVGA_RENDER_CONTROL);
287 		tmp = REG_SET_FIELD(tmp, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0);
288 		WREG32(mmVGA_RENDER_CONTROL, tmp);
289 	}
290 	/* Update configuration */
291 	WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
292 	       adev->mc.vram_start >> 12);
293 	WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
294 	       adev->mc.vram_end >> 12);
295 	WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
296 	       adev->vram_scratch.gpu_addr >> 12);
297 	WREG32(mmMC_VM_AGP_BASE, 0);
298 	WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
299 	WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
300 	if (gmc_v7_0_wait_for_idle((void *)adev)) {
301 		dev_warn(adev->dev, "Wait for MC idle timedout !\n");
302 	}
303 
304 	WREG32(mmBIF_FB_EN, BIF_FB_EN__FB_READ_EN_MASK | BIF_FB_EN__FB_WRITE_EN_MASK);
305 
306 	tmp = RREG32(mmHDP_MISC_CNTL);
307 	tmp = REG_SET_FIELD(tmp, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 0);
308 	WREG32(mmHDP_MISC_CNTL, tmp);
309 
310 	tmp = RREG32(mmHDP_HOST_PATH_CNTL);
311 	WREG32(mmHDP_HOST_PATH_CNTL, tmp);
312 }
313 
314 /**
315  * gmc_v7_0_mc_init - initialize the memory controller driver params
316  *
317  * @adev: amdgpu_device pointer
318  *
319  * Look up the amount of vram, vram width, and decide how to place
320  * vram and gart within the GPU's physical address space (CIK).
321  * Returns 0 for success.
322  */
323 static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
324 {
325 	adev->mc.vram_width = amdgpu_atombios_get_vram_width(adev);
326 	if (!adev->mc.vram_width) {
327 		u32 tmp;
328 		int chansize, numchan;
329 
330 		/* Get VRAM informations */
331 		tmp = RREG32(mmMC_ARB_RAMCFG);
332 		if (REG_GET_FIELD(tmp, MC_ARB_RAMCFG, CHANSIZE)) {
333 			chansize = 64;
334 		} else {
335 			chansize = 32;
336 		}
337 		tmp = RREG32(mmMC_SHARED_CHMAP);
338 		switch (REG_GET_FIELD(tmp, MC_SHARED_CHMAP, NOOFCHAN)) {
339 		case 0:
340 		default:
341 			numchan = 1;
342 			break;
343 		case 1:
344 			numchan = 2;
345 			break;
346 		case 2:
347 			numchan = 4;
348 			break;
349 		case 3:
350 			numchan = 8;
351 			break;
352 		case 4:
353 			numchan = 3;
354 			break;
355 		case 5:
356 			numchan = 6;
357 			break;
358 		case 6:
359 			numchan = 10;
360 			break;
361 		case 7:
362 			numchan = 12;
363 			break;
364 		case 8:
365 			numchan = 16;
366 			break;
367 		}
368 		adev->mc.vram_width = numchan * chansize;
369 	}
370 	/* Could aper size report 0 ? */
371 	adev->mc.aper_base = pci_resource_start(adev->pdev, 0);
372 	adev->mc.aper_size = pci_resource_len(adev->pdev, 0);
373 	/* size in MB on si */
374 	adev->mc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
375 	adev->mc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
376 
377 #ifdef CONFIG_X86_64
378 	if (adev->flags & AMD_IS_APU) {
379 		adev->mc.aper_base = ((u64)RREG32(mmMC_VM_FB_OFFSET)) << 22;
380 		adev->mc.aper_size = adev->mc.real_vram_size;
381 	}
382 #endif
383 
384 	/* In case the PCI BAR is larger than the actual amount of vram */
385 	adev->mc.visible_vram_size = adev->mc.aper_size;
386 	if (adev->mc.visible_vram_size > adev->mc.real_vram_size)
387 		adev->mc.visible_vram_size = adev->mc.real_vram_size;
388 
389 	/* set the gart size */
390 	if (amdgpu_gart_size == -1) {
391 		switch (adev->asic_type) {
392 		case CHIP_TOPAZ:     /* no MM engines */
393 		default:
394 			adev->mc.gart_size = 256ULL << 20;
395 			break;
396 #ifdef CONFIG_DRM_AMDGPU_CIK
397 		case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
398 		case CHIP_HAWAII:  /* UVD, VCE do not support GPUVM */
399 		case CHIP_KAVERI:  /* UVD, VCE do not support GPUVM */
400 		case CHIP_KABINI:  /* UVD, VCE do not support GPUVM */
401 		case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
402 			adev->mc.gart_size = 1024ULL << 20;
403 			break;
404 #endif
405 		}
406 	} else {
407 		adev->mc.gart_size = (u64)amdgpu_gart_size << 20;
408 	}
409 
410 	gmc_v7_0_vram_gtt_location(adev, &adev->mc);
411 
412 	return 0;
413 }
414 
415 /*
416  * GART
417  * VMID 0 is the physical GPU addresses as used by the kernel.
418  * VMIDs 1-15 are used for userspace clients and are handled
419  * by the amdgpu vm/hsa code.
420  */
421 
422 /**
423  * gmc_v7_0_gart_flush_gpu_tlb - gart tlb flush callback
424  *
425  * @adev: amdgpu_device pointer
426  * @vmid: vm instance to flush
427  *
428  * Flush the TLB for the requested page table (CIK).
429  */
430 static void gmc_v7_0_gart_flush_gpu_tlb(struct amdgpu_device *adev,
431 					uint32_t vmid)
432 {
433 	/* flush hdp cache */
434 	WREG32(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0);
435 
436 	/* bits 0-15 are the VM contexts0-15 */
437 	WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
438 }
439 
440 /**
441  * gmc_v7_0_gart_set_pte_pde - update the page tables using MMIO
442  *
443  * @adev: amdgpu_device pointer
444  * @cpu_pt_addr: cpu address of the page table
445  * @gpu_page_idx: entry in the page table to update
446  * @addr: dst addr to write into pte/pde
447  * @flags: access flags
448  *
449  * Update the page tables using the CPU.
450  */
451 static int gmc_v7_0_gart_set_pte_pde(struct amdgpu_device *adev,
452 				     void *cpu_pt_addr,
453 				     uint32_t gpu_page_idx,
454 				     uint64_t addr,
455 				     uint64_t flags)
456 {
457 	void __iomem *ptr = (void *)cpu_pt_addr;
458 	uint64_t value;
459 
460 	value = addr & 0xFFFFFFFFFFFFF000ULL;
461 	value |= flags;
462 	writeq(value, ptr + (gpu_page_idx * 8));
463 
464 	return 0;
465 }
466 
467 static uint64_t gmc_v7_0_get_vm_pte_flags(struct amdgpu_device *adev,
468 					  uint32_t flags)
469 {
470 	uint64_t pte_flag = 0;
471 
472 	if (flags & AMDGPU_VM_PAGE_READABLE)
473 		pte_flag |= AMDGPU_PTE_READABLE;
474 	if (flags & AMDGPU_VM_PAGE_WRITEABLE)
475 		pte_flag |= AMDGPU_PTE_WRITEABLE;
476 	if (flags & AMDGPU_VM_PAGE_PRT)
477 		pte_flag |= AMDGPU_PTE_PRT;
478 
479 	return pte_flag;
480 }
481 
482 static uint64_t gmc_v7_0_get_vm_pde(struct amdgpu_device *adev, uint64_t addr)
483 {
484 	BUG_ON(addr & 0xFFFFFF0000000FFFULL);
485 	return addr;
486 }
487 
488 /**
489  * gmc_v8_0_set_fault_enable_default - update VM fault handling
490  *
491  * @adev: amdgpu_device pointer
492  * @value: true redirects VM faults to the default page
493  */
494 static void gmc_v7_0_set_fault_enable_default(struct amdgpu_device *adev,
495 					      bool value)
496 {
497 	u32 tmp;
498 
499 	tmp = RREG32(mmVM_CONTEXT1_CNTL);
500 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
501 			    RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
502 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
503 			    DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
504 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
505 			    PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
506 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
507 			    VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
508 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
509 			    READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
510 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
511 			    WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
512 	WREG32(mmVM_CONTEXT1_CNTL, tmp);
513 }
514 
515 /**
516  * gmc_v7_0_set_prt - set PRT VM fault
517  *
518  * @adev: amdgpu_device pointer
519  * @enable: enable/disable VM fault handling for PRT
520  */
521 static void gmc_v7_0_set_prt(struct amdgpu_device *adev, bool enable)
522 {
523 	uint32_t tmp;
524 
525 	if (enable && !adev->mc.prt_warning) {
526 		dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
527 		adev->mc.prt_warning = true;
528 	}
529 
530 	tmp = RREG32(mmVM_PRT_CNTL);
531 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
532 			    CB_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
533 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
534 			    CB_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
535 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
536 			    TC_DISABLE_READ_FAULT_ON_UNMAPPED_ACCESS, enable);
537 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
538 			    TC_DISABLE_WRITE_FAULT_ON_UNMAPPED_ACCESS, enable);
539 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
540 			    L2_CACHE_STORE_INVALID_ENTRIES, enable);
541 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
542 			    L1_TLB_STORE_INVALID_ENTRIES, enable);
543 	tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
544 			    MASK_PDE0_FAULT, enable);
545 	WREG32(mmVM_PRT_CNTL, tmp);
546 
547 	if (enable) {
548 		uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
549 		uint32_t high = adev->vm_manager.max_pfn;
550 
551 		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
552 		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
553 		WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
554 		WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
555 		WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
556 		WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
557 		WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
558 		WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
559 	} else {
560 		WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
561 		WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
562 		WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
563 		WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
564 		WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
565 		WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
566 		WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
567 		WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
568 	}
569 }
570 
571 /**
572  * gmc_v7_0_gart_enable - gart enable
573  *
574  * @adev: amdgpu_device pointer
575  *
576  * This sets up the TLBs, programs the page tables for VMID0,
577  * sets up the hw for VMIDs 1-15 which are allocated on
578  * demand, and sets up the global locations for the LDS, GDS,
579  * and GPUVM for FSA64 clients (CIK).
580  * Returns 0 for success, errors for failure.
581  */
582 static int gmc_v7_0_gart_enable(struct amdgpu_device *adev)
583 {
584 	int r, i;
585 	u32 tmp, field;
586 
587 	if (adev->gart.robj == NULL) {
588 		dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
589 		return -EINVAL;
590 	}
591 	r = amdgpu_gart_table_vram_pin(adev);
592 	if (r)
593 		return r;
594 	/* Setup TLB control */
595 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
596 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
597 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 1);
598 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
599 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 1);
600 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
601 	WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
602 	/* Setup L2 cache */
603 	tmp = RREG32(mmVM_L2_CNTL);
604 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
605 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
606 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE, 1);
607 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE, 1);
608 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, EFFECTIVE_L2_QUEUE_SIZE, 7);
609 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
610 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
611 	WREG32(mmVM_L2_CNTL, tmp);
612 	tmp = REG_SET_FIELD(0, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
613 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
614 	WREG32(mmVM_L2_CNTL2, tmp);
615 
616 	field = adev->vm_manager.fragment_size;
617 	tmp = RREG32(mmVM_L2_CNTL3);
618 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY, 1);
619 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, field);
620 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, L2_CACHE_BIGK_FRAGMENT_SIZE, field);
621 	WREG32(mmVM_L2_CNTL3, tmp);
622 	/* setup context0 */
623 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->mc.gart_start >> 12);
624 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->mc.gart_end >> 12);
625 	WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, adev->gart.table_addr >> 12);
626 	WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
627 			(u32)(adev->dummy_page.addr >> 12));
628 	WREG32(mmVM_CONTEXT0_CNTL2, 0);
629 	tmp = RREG32(mmVM_CONTEXT0_CNTL);
630 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
631 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
632 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
633 	WREG32(mmVM_CONTEXT0_CNTL, tmp);
634 
635 	WREG32(0x575, 0);
636 	WREG32(0x576, 0);
637 	WREG32(0x577, 0);
638 
639 	/* empty context1-15 */
640 	/* FIXME start with 4G, once using 2 level pt switch to full
641 	 * vm size space
642 	 */
643 	/* set vm size, must be a multiple of 4 */
644 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
645 	WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
646 	for (i = 1; i < 16; i++) {
647 		if (i < 8)
648 			WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
649 			       adev->gart.table_addr >> 12);
650 		else
651 			WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
652 			       adev->gart.table_addr >> 12);
653 	}
654 
655 	/* enable context1-15 */
656 	WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
657 	       (u32)(adev->dummy_page.addr >> 12));
658 	WREG32(mmVM_CONTEXT1_CNTL2, 4);
659 	tmp = RREG32(mmVM_CONTEXT1_CNTL);
660 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
661 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 1);
662 	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_BLOCK_SIZE,
663 			    adev->vm_manager.block_size - 9);
664 	WREG32(mmVM_CONTEXT1_CNTL, tmp);
665 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
666 		gmc_v7_0_set_fault_enable_default(adev, false);
667 	else
668 		gmc_v7_0_set_fault_enable_default(adev, true);
669 
670 	if (adev->asic_type == CHIP_KAVERI) {
671 		tmp = RREG32(mmCHUB_CONTROL);
672 		tmp &= ~BYPASS_VM;
673 		WREG32(mmCHUB_CONTROL, tmp);
674 	}
675 
676 	gmc_v7_0_gart_flush_gpu_tlb(adev, 0);
677 	DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
678 		 (unsigned)(adev->mc.gart_size >> 20),
679 		 (unsigned long long)adev->gart.table_addr);
680 	adev->gart.ready = true;
681 	return 0;
682 }
683 
684 static int gmc_v7_0_gart_init(struct amdgpu_device *adev)
685 {
686 	int r;
687 
688 	if (adev->gart.robj) {
689 		WARN(1, "R600 PCIE GART already initialized\n");
690 		return 0;
691 	}
692 	/* Initialize common gart structure */
693 	r = amdgpu_gart_init(adev);
694 	if (r)
695 		return r;
696 	adev->gart.table_size = adev->gart.num_gpu_pages * 8;
697 	adev->gart.gart_pte_flags = 0;
698 	return amdgpu_gart_table_vram_alloc(adev);
699 }
700 
701 /**
702  * gmc_v7_0_gart_disable - gart disable
703  *
704  * @adev: amdgpu_device pointer
705  *
706  * This disables all VM page table (CIK).
707  */
708 static void gmc_v7_0_gart_disable(struct amdgpu_device *adev)
709 {
710 	u32 tmp;
711 
712 	/* Disable all tables */
713 	WREG32(mmVM_CONTEXT0_CNTL, 0);
714 	WREG32(mmVM_CONTEXT1_CNTL, 0);
715 	/* Setup TLB control */
716 	tmp = RREG32(mmMC_VM_MX_L1_TLB_CNTL);
717 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
718 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_FRAGMENT_PROCESSING, 0);
719 	tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_ADVANCED_DRIVER_MODEL, 0);
720 	WREG32(mmMC_VM_MX_L1_TLB_CNTL, tmp);
721 	/* Setup L2 cache */
722 	tmp = RREG32(mmVM_L2_CNTL);
723 	tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
724 	WREG32(mmVM_L2_CNTL, tmp);
725 	WREG32(mmVM_L2_CNTL2, 0);
726 	amdgpu_gart_table_vram_unpin(adev);
727 }
728 
729 /**
730  * gmc_v7_0_gart_fini - vm fini callback
731  *
732  * @adev: amdgpu_device pointer
733  *
734  * Tears down the driver GART/VM setup (CIK).
735  */
736 static void gmc_v7_0_gart_fini(struct amdgpu_device *adev)
737 {
738 	amdgpu_gart_table_vram_free(adev);
739 	amdgpu_gart_fini(adev);
740 }
741 
742 /**
743  * gmc_v7_0_vm_decode_fault - print human readable fault info
744  *
745  * @adev: amdgpu_device pointer
746  * @status: VM_CONTEXT1_PROTECTION_FAULT_STATUS register value
747  * @addr: VM_CONTEXT1_PROTECTION_FAULT_ADDR register value
748  *
749  * Print human readable fault information (CIK).
750  */
751 static void gmc_v7_0_vm_decode_fault(struct amdgpu_device *adev,
752 				     u32 status, u32 addr, u32 mc_client)
753 {
754 	u32 mc_id;
755 	u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
756 	u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
757 					PROTECTIONS);
758 	char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
759 		(mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
760 
761 	mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
762 			      MEMORY_CLIENT_ID);
763 
764 	dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
765 	       protections, vmid, addr,
766 	       REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
767 			     MEMORY_CLIENT_RW) ?
768 	       "write" : "read", block, mc_client, mc_id);
769 }
770 
771 
772 static const u32 mc_cg_registers[] = {
773 	mmMC_HUB_MISC_HUB_CG,
774 	mmMC_HUB_MISC_SIP_CG,
775 	mmMC_HUB_MISC_VM_CG,
776 	mmMC_XPB_CLK_GAT,
777 	mmATC_MISC_CG,
778 	mmMC_CITF_MISC_WR_CG,
779 	mmMC_CITF_MISC_RD_CG,
780 	mmMC_CITF_MISC_VM_CG,
781 	mmVM_L2_CG,
782 };
783 
784 static const u32 mc_cg_ls_en[] = {
785 	MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
786 	MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
787 	MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
788 	MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
789 	ATC_MISC_CG__MEM_LS_ENABLE_MASK,
790 	MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
791 	MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
792 	MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
793 	VM_L2_CG__MEM_LS_ENABLE_MASK,
794 };
795 
796 static const u32 mc_cg_en[] = {
797 	MC_HUB_MISC_HUB_CG__ENABLE_MASK,
798 	MC_HUB_MISC_SIP_CG__ENABLE_MASK,
799 	MC_HUB_MISC_VM_CG__ENABLE_MASK,
800 	MC_XPB_CLK_GAT__ENABLE_MASK,
801 	ATC_MISC_CG__ENABLE_MASK,
802 	MC_CITF_MISC_WR_CG__ENABLE_MASK,
803 	MC_CITF_MISC_RD_CG__ENABLE_MASK,
804 	MC_CITF_MISC_VM_CG__ENABLE_MASK,
805 	VM_L2_CG__ENABLE_MASK,
806 };
807 
808 static void gmc_v7_0_enable_mc_ls(struct amdgpu_device *adev,
809 				  bool enable)
810 {
811 	int i;
812 	u32 orig, data;
813 
814 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
815 		orig = data = RREG32(mc_cg_registers[i]);
816 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS))
817 			data |= mc_cg_ls_en[i];
818 		else
819 			data &= ~mc_cg_ls_en[i];
820 		if (data != orig)
821 			WREG32(mc_cg_registers[i], data);
822 	}
823 }
824 
825 static void gmc_v7_0_enable_mc_mgcg(struct amdgpu_device *adev,
826 				    bool enable)
827 {
828 	int i;
829 	u32 orig, data;
830 
831 	for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
832 		orig = data = RREG32(mc_cg_registers[i]);
833 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG))
834 			data |= mc_cg_en[i];
835 		else
836 			data &= ~mc_cg_en[i];
837 		if (data != orig)
838 			WREG32(mc_cg_registers[i], data);
839 	}
840 }
841 
842 static void gmc_v7_0_enable_bif_mgls(struct amdgpu_device *adev,
843 				     bool enable)
844 {
845 	u32 orig, data;
846 
847 	orig = data = RREG32_PCIE(ixPCIE_CNTL2);
848 
849 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_BIF_LS)) {
850 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
851 		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
852 		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
853 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
854 	} else {
855 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
856 		data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
857 		data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
858 		data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
859 	}
860 
861 	if (orig != data)
862 		WREG32_PCIE(ixPCIE_CNTL2, data);
863 }
864 
865 static void gmc_v7_0_enable_hdp_mgcg(struct amdgpu_device *adev,
866 				     bool enable)
867 {
868 	u32 orig, data;
869 
870 	orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
871 
872 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_MGCG))
873 		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
874 	else
875 		data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
876 
877 	if (orig != data)
878 		WREG32(mmHDP_HOST_PATH_CNTL, data);
879 }
880 
881 static void gmc_v7_0_enable_hdp_ls(struct amdgpu_device *adev,
882 				   bool enable)
883 {
884 	u32 orig, data;
885 
886 	orig = data = RREG32(mmHDP_MEM_POWER_LS);
887 
888 	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_HDP_LS))
889 		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
890 	else
891 		data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
892 
893 	if (orig != data)
894 		WREG32(mmHDP_MEM_POWER_LS, data);
895 }
896 
897 static int gmc_v7_0_convert_vram_type(int mc_seq_vram_type)
898 {
899 	switch (mc_seq_vram_type) {
900 	case MC_SEQ_MISC0__MT__GDDR1:
901 		return AMDGPU_VRAM_TYPE_GDDR1;
902 	case MC_SEQ_MISC0__MT__DDR2:
903 		return AMDGPU_VRAM_TYPE_DDR2;
904 	case MC_SEQ_MISC0__MT__GDDR3:
905 		return AMDGPU_VRAM_TYPE_GDDR3;
906 	case MC_SEQ_MISC0__MT__GDDR4:
907 		return AMDGPU_VRAM_TYPE_GDDR4;
908 	case MC_SEQ_MISC0__MT__GDDR5:
909 		return AMDGPU_VRAM_TYPE_GDDR5;
910 	case MC_SEQ_MISC0__MT__HBM:
911 		return AMDGPU_VRAM_TYPE_HBM;
912 	case MC_SEQ_MISC0__MT__DDR3:
913 		return AMDGPU_VRAM_TYPE_DDR3;
914 	default:
915 		return AMDGPU_VRAM_TYPE_UNKNOWN;
916 	}
917 }
918 
919 static int gmc_v7_0_early_init(void *handle)
920 {
921 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
922 
923 	gmc_v7_0_set_gart_funcs(adev);
924 	gmc_v7_0_set_irq_funcs(adev);
925 
926 	adev->mc.shared_aperture_start = 0x2000000000000000ULL;
927 	adev->mc.shared_aperture_end =
928 		adev->mc.shared_aperture_start + (4ULL << 30) - 1;
929 	adev->mc.private_aperture_start =
930 		adev->mc.shared_aperture_end + 1;
931 	adev->mc.private_aperture_end =
932 		adev->mc.private_aperture_start + (4ULL << 30) - 1;
933 
934 	return 0;
935 }
936 
937 static int gmc_v7_0_late_init(void *handle)
938 {
939 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
940 
941 	if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
942 		return amdgpu_irq_get(adev, &adev->mc.vm_fault, 0);
943 	else
944 		return 0;
945 }
946 
947 static int gmc_v7_0_sw_init(void *handle)
948 {
949 	int r;
950 	int dma_bits;
951 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
952 
953 	if (adev->flags & AMD_IS_APU) {
954 		adev->mc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
955 	} else {
956 		u32 tmp = RREG32(mmMC_SEQ_MISC0);
957 		tmp &= MC_SEQ_MISC0__MT__MASK;
958 		adev->mc.vram_type = gmc_v7_0_convert_vram_type(tmp);
959 	}
960 
961 	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 146, &adev->mc.vm_fault);
962 	if (r)
963 		return r;
964 
965 	r = amdgpu_irq_add_id(adev, AMDGPU_IH_CLIENTID_LEGACY, 147, &adev->mc.vm_fault);
966 	if (r)
967 		return r;
968 
969 	/* Adjust VM size here.
970 	 * Currently set to 4GB ((1 << 20) 4k pages).
971 	 * Max GPUVM size for cayman and SI is 40 bits.
972 	 */
973 	amdgpu_vm_adjust_size(adev, 64, 4);
974 	adev->vm_manager.max_pfn = adev->vm_manager.vm_size << 18;
975 
976 	/* Set the internal MC address mask
977 	 * This is the max address of the GPU's
978 	 * internal address space.
979 	 */
980 	adev->mc.mc_mask = 0xffffffffffULL; /* 40 bit MC */
981 
982 	adev->mc.stolen_size = 256 * 1024;
983 
984 	/* set DMA mask + need_dma32 flags.
985 	 * PCIE - can handle 40-bits.
986 	 * IGP - can handle 40-bits
987 	 * PCI - dma32 for legacy pci gart, 40 bits on newer asics
988 	 */
989 	adev->need_dma32 = false;
990 	dma_bits = adev->need_dma32 ? 32 : 40;
991 	r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
992 	if (r) {
993 		adev->need_dma32 = true;
994 		dma_bits = 32;
995 		pr_warn("amdgpu: No suitable DMA available\n");
996 	}
997 	r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits));
998 	if (r) {
999 		pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32));
1000 		pr_warn("amdgpu: No coherent DMA available\n");
1001 	}
1002 
1003 	r = gmc_v7_0_init_microcode(adev);
1004 	if (r) {
1005 		DRM_ERROR("Failed to load mc firmware!\n");
1006 		return r;
1007 	}
1008 
1009 	r = gmc_v7_0_mc_init(adev);
1010 	if (r)
1011 		return r;
1012 
1013 	/* Memory manager */
1014 	r = amdgpu_bo_init(adev);
1015 	if (r)
1016 		return r;
1017 
1018 	r = gmc_v7_0_gart_init(adev);
1019 	if (r)
1020 		return r;
1021 
1022 	/*
1023 	 * number of VMs
1024 	 * VMID 0 is reserved for System
1025 	 * amdgpu graphics/compute will use VMIDs 1-7
1026 	 * amdkfd will use VMIDs 8-15
1027 	 */
1028 	adev->vm_manager.id_mgr[0].num_ids = AMDGPU_NUM_OF_VMIDS;
1029 	adev->vm_manager.num_level = 1;
1030 	amdgpu_vm_manager_init(adev);
1031 
1032 	/* base offset of vram pages */
1033 	if (adev->flags & AMD_IS_APU) {
1034 		u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
1035 
1036 		tmp <<= 22;
1037 		adev->vm_manager.vram_base_offset = tmp;
1038 	} else {
1039 		adev->vm_manager.vram_base_offset = 0;
1040 	}
1041 
1042 	return 0;
1043 }
1044 
1045 static int gmc_v7_0_sw_fini(void *handle)
1046 {
1047 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1048 
1049 	amdgpu_vm_manager_fini(adev);
1050 	gmc_v7_0_gart_fini(adev);
1051 	amdgpu_gem_force_release(adev);
1052 	amdgpu_bo_fini(adev);
1053 
1054 	return 0;
1055 }
1056 
1057 static int gmc_v7_0_hw_init(void *handle)
1058 {
1059 	int r;
1060 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1061 
1062 	gmc_v7_0_init_golden_registers(adev);
1063 
1064 	gmc_v7_0_mc_program(adev);
1065 
1066 	if (!(adev->flags & AMD_IS_APU)) {
1067 		r = gmc_v7_0_mc_load_microcode(adev);
1068 		if (r) {
1069 			DRM_ERROR("Failed to load MC firmware!\n");
1070 			return r;
1071 		}
1072 	}
1073 
1074 	r = gmc_v7_0_gart_enable(adev);
1075 	if (r)
1076 		return r;
1077 
1078 	return r;
1079 }
1080 
1081 static int gmc_v7_0_hw_fini(void *handle)
1082 {
1083 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1084 
1085 	amdgpu_irq_put(adev, &adev->mc.vm_fault, 0);
1086 	gmc_v7_0_gart_disable(adev);
1087 
1088 	return 0;
1089 }
1090 
1091 static int gmc_v7_0_suspend(void *handle)
1092 {
1093 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1094 
1095 	gmc_v7_0_hw_fini(adev);
1096 
1097 	return 0;
1098 }
1099 
1100 static int gmc_v7_0_resume(void *handle)
1101 {
1102 	int r;
1103 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1104 
1105 	r = gmc_v7_0_hw_init(adev);
1106 	if (r)
1107 		return r;
1108 
1109 	amdgpu_vm_reset_all_ids(adev);
1110 
1111 	return 0;
1112 }
1113 
1114 static bool gmc_v7_0_is_idle(void *handle)
1115 {
1116 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1117 	u32 tmp = RREG32(mmSRBM_STATUS);
1118 
1119 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1120 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
1121 		return false;
1122 
1123 	return true;
1124 }
1125 
1126 static int gmc_v7_0_wait_for_idle(void *handle)
1127 {
1128 	unsigned i;
1129 	u32 tmp;
1130 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1131 
1132 	for (i = 0; i < adev->usec_timeout; i++) {
1133 		/* read MC_STATUS */
1134 		tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__MCB_BUSY_MASK |
1135 					       SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1136 					       SRBM_STATUS__MCC_BUSY_MASK |
1137 					       SRBM_STATUS__MCD_BUSY_MASK |
1138 					       SRBM_STATUS__VMC_BUSY_MASK);
1139 		if (!tmp)
1140 			return 0;
1141 		udelay(1);
1142 	}
1143 	return -ETIMEDOUT;
1144 
1145 }
1146 
1147 static int gmc_v7_0_soft_reset(void *handle)
1148 {
1149 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1150 	u32 srbm_soft_reset = 0;
1151 	u32 tmp = RREG32(mmSRBM_STATUS);
1152 
1153 	if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
1154 		srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1155 						SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
1156 
1157 	if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
1158 		   SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
1159 		if (!(adev->flags & AMD_IS_APU))
1160 			srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
1161 							SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
1162 	}
1163 
1164 	if (srbm_soft_reset) {
1165 		gmc_v7_0_mc_stop(adev);
1166 		if (gmc_v7_0_wait_for_idle((void *)adev)) {
1167 			dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1168 		}
1169 
1170 
1171 		tmp = RREG32(mmSRBM_SOFT_RESET);
1172 		tmp |= srbm_soft_reset;
1173 		dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
1174 		WREG32(mmSRBM_SOFT_RESET, tmp);
1175 		tmp = RREG32(mmSRBM_SOFT_RESET);
1176 
1177 		udelay(50);
1178 
1179 		tmp &= ~srbm_soft_reset;
1180 		WREG32(mmSRBM_SOFT_RESET, tmp);
1181 		tmp = RREG32(mmSRBM_SOFT_RESET);
1182 
1183 		/* Wait a little for things to settle down */
1184 		udelay(50);
1185 
1186 		gmc_v7_0_mc_resume(adev);
1187 		udelay(50);
1188 	}
1189 
1190 	return 0;
1191 }
1192 
1193 static int gmc_v7_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1194 					     struct amdgpu_irq_src *src,
1195 					     unsigned type,
1196 					     enum amdgpu_interrupt_state state)
1197 {
1198 	u32 tmp;
1199 	u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1200 		    VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1201 		    VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1202 		    VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1203 		    VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1204 		    VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1205 
1206 	switch (state) {
1207 	case AMDGPU_IRQ_STATE_DISABLE:
1208 		/* system context */
1209 		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1210 		tmp &= ~bits;
1211 		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1212 		/* VMs */
1213 		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1214 		tmp &= ~bits;
1215 		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1216 		break;
1217 	case AMDGPU_IRQ_STATE_ENABLE:
1218 		/* system context */
1219 		tmp = RREG32(mmVM_CONTEXT0_CNTL);
1220 		tmp |= bits;
1221 		WREG32(mmVM_CONTEXT0_CNTL, tmp);
1222 		/* VMs */
1223 		tmp = RREG32(mmVM_CONTEXT1_CNTL);
1224 		tmp |= bits;
1225 		WREG32(mmVM_CONTEXT1_CNTL, tmp);
1226 		break;
1227 	default:
1228 		break;
1229 	}
1230 
1231 	return 0;
1232 }
1233 
1234 static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev,
1235 				      struct amdgpu_irq_src *source,
1236 				      struct amdgpu_iv_entry *entry)
1237 {
1238 	u32 addr, status, mc_client;
1239 
1240 	addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
1241 	status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
1242 	mc_client = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_MCCLIENT);
1243 	/* reset addr and status */
1244 	WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1245 
1246 	if (!addr && !status)
1247 		return 0;
1248 
1249 	if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1250 		gmc_v7_0_set_fault_enable_default(adev, false);
1251 
1252 	if (printk_ratelimit()) {
1253 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
1254 			entry->src_id, entry->src_data[0]);
1255 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
1256 			addr);
1257 		dev_err(adev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1258 			status);
1259 		gmc_v7_0_vm_decode_fault(adev, status, addr, mc_client);
1260 	}
1261 
1262 	return 0;
1263 }
1264 
1265 static int gmc_v7_0_set_clockgating_state(void *handle,
1266 					  enum amd_clockgating_state state)
1267 {
1268 	bool gate = false;
1269 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1270 
1271 	if (state == AMD_CG_STATE_GATE)
1272 		gate = true;
1273 
1274 	if (!(adev->flags & AMD_IS_APU)) {
1275 		gmc_v7_0_enable_mc_mgcg(adev, gate);
1276 		gmc_v7_0_enable_mc_ls(adev, gate);
1277 	}
1278 	gmc_v7_0_enable_bif_mgls(adev, gate);
1279 	gmc_v7_0_enable_hdp_mgcg(adev, gate);
1280 	gmc_v7_0_enable_hdp_ls(adev, gate);
1281 
1282 	return 0;
1283 }
1284 
1285 static int gmc_v7_0_set_powergating_state(void *handle,
1286 					  enum amd_powergating_state state)
1287 {
1288 	return 0;
1289 }
1290 
1291 static const struct amd_ip_funcs gmc_v7_0_ip_funcs = {
1292 	.name = "gmc_v7_0",
1293 	.early_init = gmc_v7_0_early_init,
1294 	.late_init = gmc_v7_0_late_init,
1295 	.sw_init = gmc_v7_0_sw_init,
1296 	.sw_fini = gmc_v7_0_sw_fini,
1297 	.hw_init = gmc_v7_0_hw_init,
1298 	.hw_fini = gmc_v7_0_hw_fini,
1299 	.suspend = gmc_v7_0_suspend,
1300 	.resume = gmc_v7_0_resume,
1301 	.is_idle = gmc_v7_0_is_idle,
1302 	.wait_for_idle = gmc_v7_0_wait_for_idle,
1303 	.soft_reset = gmc_v7_0_soft_reset,
1304 	.set_clockgating_state = gmc_v7_0_set_clockgating_state,
1305 	.set_powergating_state = gmc_v7_0_set_powergating_state,
1306 };
1307 
1308 static const struct amdgpu_gart_funcs gmc_v7_0_gart_funcs = {
1309 	.flush_gpu_tlb = gmc_v7_0_gart_flush_gpu_tlb,
1310 	.set_pte_pde = gmc_v7_0_gart_set_pte_pde,
1311 	.set_prt = gmc_v7_0_set_prt,
1312 	.get_vm_pte_flags = gmc_v7_0_get_vm_pte_flags,
1313 	.get_vm_pde = gmc_v7_0_get_vm_pde
1314 };
1315 
1316 static const struct amdgpu_irq_src_funcs gmc_v7_0_irq_funcs = {
1317 	.set = gmc_v7_0_vm_fault_interrupt_state,
1318 	.process = gmc_v7_0_process_interrupt,
1319 };
1320 
1321 static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev)
1322 {
1323 	if (adev->gart.gart_funcs == NULL)
1324 		adev->gart.gart_funcs = &gmc_v7_0_gart_funcs;
1325 }
1326 
1327 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev)
1328 {
1329 	adev->mc.vm_fault.num_types = 1;
1330 	adev->mc.vm_fault.funcs = &gmc_v7_0_irq_funcs;
1331 }
1332 
1333 const struct amdgpu_ip_block_version gmc_v7_0_ip_block =
1334 {
1335 	.type = AMD_IP_BLOCK_TYPE_GMC,
1336 	.major = 7,
1337 	.minor = 0,
1338 	.rev = 0,
1339 	.funcs = &gmc_v7_0_ip_funcs,
1340 };
1341 
1342 const struct amdgpu_ip_block_version gmc_v7_4_ip_block =
1343 {
1344 	.type = AMD_IP_BLOCK_TYPE_GMC,
1345 	.major = 7,
1346 	.minor = 4,
1347 	.rev = 0,
1348 	.funcs = &gmc_v7_0_ip_funcs,
1349 };
1350