1df70502eSKen Wang /*
2df70502eSKen Wang * Copyright 2014 Advanced Micro Devices, Inc.
3df70502eSKen Wang *
4df70502eSKen Wang * Permission is hereby granted, free of charge, to any person obtaining a
5df70502eSKen Wang * copy of this software and associated documentation files (the "Software"),
6df70502eSKen Wang * to deal in the Software without restriction, including without limitation
7df70502eSKen Wang * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8df70502eSKen Wang * and/or sell copies of the Software, and to permit persons to whom the
9df70502eSKen Wang * Software is furnished to do so, subject to the following conditions:
10df70502eSKen Wang *
11df70502eSKen Wang * The above copyright notice and this permission notice shall be included in
12df70502eSKen Wang * all copies or substantial portions of the Software.
13df70502eSKen Wang *
14df70502eSKen Wang * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15df70502eSKen Wang * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16df70502eSKen Wang * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17df70502eSKen Wang * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18df70502eSKen Wang * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19df70502eSKen Wang * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20df70502eSKen Wang * OTHER DEALINGS IN THE SOFTWARE.
21df70502eSKen Wang *
22df70502eSKen Wang */
2347b757fbSSam Ravnborg
24df70502eSKen Wang #include <linux/firmware.h>
2547b757fbSSam Ravnborg #include <linux/module.h>
2647b757fbSSam Ravnborg #include <linux/pci.h>
2747b757fbSSam Ravnborg
28fd5fd480SChunming Zhou #include <drm/drm_cache.h>
29df70502eSKen Wang #include "amdgpu.h"
30df70502eSKen Wang #include "gmc_v6_0.h"
31df70502eSKen Wang #include "amdgpu_ucode.h"
322cddc50eSHuang Rui #include "amdgpu_gem.h"
3372518269STom St Denis
3472518269STom St Denis #include "bif/bif_3_0_d.h"
3572518269STom St Denis #include "bif/bif_3_0_sh_mask.h"
3672518269STom St Denis #include "oss/oss_1_0_d.h"
3772518269STom St Denis #include "oss/oss_1_0_sh_mask.h"
3872518269STom St Denis #include "gmc/gmc_6_0_d.h"
3972518269STom St Denis #include "gmc/gmc_6_0_sh_mask.h"
4072518269STom St Denis #include "dce/dce_6_0_d.h"
4172518269STom St Denis #include "dce/dce_6_0_sh_mask.h"
4272518269STom St Denis #include "si_enums.h"
43df70502eSKen Wang
44132f34e4SChristian König static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev);
45df70502eSKen Wang static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev);
46df70502eSKen Wang static int gmc_v6_0_wait_for_idle(void *handle);
47df70502eSKen Wang
488eaf2b1fSAlex Deucher MODULE_FIRMWARE("amdgpu/tahiti_mc.bin");
498eaf2b1fSAlex Deucher MODULE_FIRMWARE("amdgpu/pitcairn_mc.bin");
508eaf2b1fSAlex Deucher MODULE_FIRMWARE("amdgpu/verde_mc.bin");
518eaf2b1fSAlex Deucher MODULE_FIRMWARE("amdgpu/oland_mc.bin");
528d4d7c58STakashi Iwai MODULE_FIRMWARE("amdgpu/hainan_mc.bin");
538eaf2b1fSAlex Deucher MODULE_FIRMWARE("amdgpu/si58_mc.bin");
54df70502eSKen Wang
5572518269STom St Denis #define MC_SEQ_MISC0__MT__MASK 0xf0000000
5672518269STom St Denis #define MC_SEQ_MISC0__MT__GDDR1 0x10000000
5772518269STom St Denis #define MC_SEQ_MISC0__MT__DDR2 0x20000000
5872518269STom St Denis #define MC_SEQ_MISC0__MT__GDDR3 0x30000000
5972518269STom St Denis #define MC_SEQ_MISC0__MT__GDDR4 0x40000000
6072518269STom St Denis #define MC_SEQ_MISC0__MT__GDDR5 0x50000000
6172518269STom St Denis #define MC_SEQ_MISC0__MT__HBM 0x60000000
6272518269STom St Denis #define MC_SEQ_MISC0__MT__DDR3 0xB0000000
6372518269STom St Denis
gmc_v6_0_mc_stop(struct amdgpu_device * adev)64e4f6b39eSAlex Deucher static void gmc_v6_0_mc_stop(struct amdgpu_device *adev)
65df70502eSKen Wang {
66df70502eSKen Wang u32 blackout;
67df70502eSKen Wang
68df70502eSKen Wang gmc_v6_0_wait_for_idle((void *)adev);
69df70502eSKen Wang
7072518269STom St Denis blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
7172518269STom St Denis if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
72df70502eSKen Wang /* Block CPU access */
7372518269STom St Denis WREG32(mmBIF_FB_EN, 0);
74df70502eSKen Wang /* blackout the MC */
75df70502eSKen Wang blackout = REG_SET_FIELD(blackout,
7672518269STom St Denis MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
7772518269STom St Denis WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout | 1);
78df70502eSKen Wang }
79df70502eSKen Wang /* wait for the MC to settle */
80df70502eSKen Wang udelay(100);
81df70502eSKen Wang
82df70502eSKen Wang }
83df70502eSKen Wang
gmc_v6_0_mc_resume(struct amdgpu_device * adev)84e4f6b39eSAlex Deucher static void gmc_v6_0_mc_resume(struct amdgpu_device *adev)
85df70502eSKen Wang {
86df70502eSKen Wang u32 tmp;
87df70502eSKen Wang
88df70502eSKen Wang /* unblackout the MC */
8972518269STom St Denis tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
9072518269STom St Denis tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0);
9172518269STom St Denis WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp);
92df70502eSKen Wang /* allow CPU access */
9372518269STom St Denis tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1);
9472518269STom St Denis tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1);
9572518269STom St Denis WREG32(mmBIF_FB_EN, tmp);
96df70502eSKen Wang }
97df70502eSKen Wang
gmc_v6_0_init_microcode(struct amdgpu_device * adev)98df70502eSKen Wang static int gmc_v6_0_init_microcode(struct amdgpu_device *adev)
99df70502eSKen Wang {
100df70502eSKen Wang const char *chip_name;
101df70502eSKen Wang char fw_name[30];
102df70502eSKen Wang int err;
103f1d877beSAlex Deucher bool is_58_fw = false;
104df70502eSKen Wang
105df70502eSKen Wang DRM_DEBUG("\n");
106df70502eSKen Wang
107df70502eSKen Wang switch (adev->asic_type) {
108df70502eSKen Wang case CHIP_TAHITI:
109df70502eSKen Wang chip_name = "tahiti";
110df70502eSKen Wang break;
111df70502eSKen Wang case CHIP_PITCAIRN:
112df70502eSKen Wang chip_name = "pitcairn";
113df70502eSKen Wang break;
114df70502eSKen Wang case CHIP_VERDE:
115df70502eSKen Wang chip_name = "verde";
116df70502eSKen Wang break;
117df70502eSKen Wang case CHIP_OLAND:
118df70502eSKen Wang chip_name = "oland";
119df70502eSKen Wang break;
120df70502eSKen Wang case CHIP_HAINAN:
121df70502eSKen Wang chip_name = "hainan";
122df70502eSKen Wang break;
1230cfc1d68SSrinivasan Shanmugam default:
1240cfc1d68SSrinivasan Shanmugam BUG();
125df70502eSKen Wang }
126df70502eSKen Wang
127f1d877beSAlex Deucher /* this memory configuration requires special firmware */
128f1d877beSAlex Deucher if (((RREG32(mmMC_SEQ_MISC0) & 0xff000000) >> 24) == 0x58)
129f1d877beSAlex Deucher is_58_fw = true;
130f1d877beSAlex Deucher
131f1d877beSAlex Deucher if (is_58_fw)
1328eaf2b1fSAlex Deucher snprintf(fw_name, sizeof(fw_name), "amdgpu/si58_mc.bin");
133f1d877beSAlex Deucher else
1348eaf2b1fSAlex Deucher snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_mc.bin", chip_name);
1352d70575bSMario Limonciello err = amdgpu_ucode_request(adev, &adev->gmc.fw, fw_name);
136df70502eSKen Wang if (err) {
137075719c3STom St Denis dev_err(adev->dev,
138df70502eSKen Wang "si_mc: Failed to load firmware \"%s\"\n",
139df70502eSKen Wang fw_name);
1402d70575bSMario Limonciello amdgpu_ucode_release(&adev->gmc.fw);
141df70502eSKen Wang }
142df70502eSKen Wang return err;
143df70502eSKen Wang }
144df70502eSKen Wang
gmc_v6_0_mc_load_microcode(struct amdgpu_device * adev)145df70502eSKen Wang static int gmc_v6_0_mc_load_microcode(struct amdgpu_device *adev)
146df70502eSKen Wang {
147df70502eSKen Wang const __le32 *new_fw_data = NULL;
148df70502eSKen Wang u32 running;
149df70502eSKen Wang const __le32 *new_io_mc_regs = NULL;
150df70502eSKen Wang int i, regs_size, ucode_size;
151df70502eSKen Wang const struct mc_firmware_header_v1_0 *hdr;
152df70502eSKen Wang
153770d13b1SChristian König if (!adev->gmc.fw)
154df70502eSKen Wang return -EINVAL;
155df70502eSKen Wang
156770d13b1SChristian König hdr = (const struct mc_firmware_header_v1_0 *)adev->gmc.fw->data;
157df70502eSKen Wang
158df70502eSKen Wang amdgpu_ucode_print_mc_hdr(&hdr->header);
159df70502eSKen Wang
160770d13b1SChristian König adev->gmc.fw_version = le32_to_cpu(hdr->header.ucode_version);
161df70502eSKen Wang regs_size = le32_to_cpu(hdr->io_debug_size_bytes) / (4 * 2);
162df70502eSKen Wang new_io_mc_regs = (const __le32 *)
163770d13b1SChristian König (adev->gmc.fw->data + le32_to_cpu(hdr->io_debug_array_offset_bytes));
164df70502eSKen Wang ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes) / 4;
165df70502eSKen Wang new_fw_data = (const __le32 *)
166770d13b1SChristian König (adev->gmc.fw->data + le32_to_cpu(hdr->header.ucode_array_offset_bytes));
167df70502eSKen Wang
16872518269STom St Denis running = RREG32(mmMC_SEQ_SUP_CNTL) & MC_SEQ_SUP_CNTL__RUN_MASK;
169df70502eSKen Wang
170df70502eSKen Wang if (running == 0) {
171df70502eSKen Wang
172df70502eSKen Wang /* reset the engine and set to writable */
17372518269STom St Denis WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
17472518269STom St Denis WREG32(mmMC_SEQ_SUP_CNTL, 0x00000010);
175df70502eSKen Wang
176df70502eSKen Wang /* load mc io regs */
177df70502eSKen Wang for (i = 0; i < regs_size; i++) {
17872518269STom St Denis WREG32(mmMC_SEQ_IO_DEBUG_INDEX, le32_to_cpup(new_io_mc_regs++));
17972518269STom St Denis WREG32(mmMC_SEQ_IO_DEBUG_DATA, le32_to_cpup(new_io_mc_regs++));
180df70502eSKen Wang }
181df70502eSKen Wang /* load the MC ucode */
1820cfc1d68SSrinivasan Shanmugam for (i = 0; i < ucode_size; i++)
18372518269STom St Denis WREG32(mmMC_SEQ_SUP_PGM, le32_to_cpup(new_fw_data++));
184df70502eSKen Wang
185df70502eSKen Wang /* put the engine back into the active state */
18672518269STom St Denis WREG32(mmMC_SEQ_SUP_CNTL, 0x00000008);
18772518269STom St Denis WREG32(mmMC_SEQ_SUP_CNTL, 0x00000004);
18872518269STom St Denis WREG32(mmMC_SEQ_SUP_CNTL, 0x00000001);
189df70502eSKen Wang
190df70502eSKen Wang /* wait for training to complete */
191df70502eSKen Wang for (i = 0; i < adev->usec_timeout; i++) {
19272518269STom St Denis if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D0_MASK)
193df70502eSKen Wang break;
194df70502eSKen Wang udelay(1);
195df70502eSKen Wang }
196df70502eSKen Wang for (i = 0; i < adev->usec_timeout; i++) {
19772518269STom St Denis if (RREG32(mmMC_SEQ_TRAIN_WAKEUP_CNTL) & MC_SEQ_TRAIN_WAKEUP_CNTL__TRAIN_DONE_D1_MASK)
198df70502eSKen Wang break;
199df70502eSKen Wang udelay(1);
200df70502eSKen Wang }
201df70502eSKen Wang
202df70502eSKen Wang }
203df70502eSKen Wang
204df70502eSKen Wang return 0;
205df70502eSKen Wang }
206df70502eSKen Wang
gmc_v6_0_vram_gtt_location(struct amdgpu_device * adev,struct amdgpu_gmc * mc)207df70502eSKen Wang static void gmc_v6_0_vram_gtt_location(struct amdgpu_device *adev,
208770d13b1SChristian König struct amdgpu_gmc *mc)
209df70502eSKen Wang {
210ba3a5b83SAlex Deucher u64 base = RREG32(mmMC_VM_FB_LOCATION) & 0xFFFF;
2110cfc1d68SSrinivasan Shanmugam
212ba3a5b83SAlex Deucher base <<= 24;
213ba3a5b83SAlex Deucher
21483afe835SOak Zeng amdgpu_gmc_vram_location(adev, mc, base);
215961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc);
216df70502eSKen Wang }
217df70502eSKen Wang
gmc_v6_0_mc_program(struct amdgpu_device * adev)218df70502eSKen Wang static void gmc_v6_0_mc_program(struct amdgpu_device *adev)
219df70502eSKen Wang {
220df70502eSKen Wang int i, j;
221df70502eSKen Wang
222df70502eSKen Wang /* Initialize HDP */
223df70502eSKen Wang for (i = 0, j = 0; i < 32; i++, j += 0x6) {
224df70502eSKen Wang WREG32((0xb05 + j), 0x00000000);
225df70502eSKen Wang WREG32((0xb06 + j), 0x00000000);
226df70502eSKen Wang WREG32((0xb07 + j), 0x00000000);
227df70502eSKen Wang WREG32((0xb08 + j), 0x00000000);
228df70502eSKen Wang WREG32((0xb09 + j), 0x00000000);
229df70502eSKen Wang }
23072518269STom St Denis WREG32(mmHDP_REG_COHERENCY_FLUSH_CNTL, 0);
231df70502eSKen Wang
2320cfc1d68SSrinivasan Shanmugam if (gmc_v6_0_wait_for_idle((void *)adev))
233df70502eSKen Wang dev_warn(adev->dev, "Wait for MC idle timedout !\n");
234df70502eSKen Wang
23503ba88cfSAlex Deucher if (adev->mode_info.num_crtc) {
23603ba88cfSAlex Deucher u32 tmp;
23703ba88cfSAlex Deucher
23803ba88cfSAlex Deucher /* Lockout access through VGA aperture*/
23903ba88cfSAlex Deucher tmp = RREG32(mmVGA_HDP_CONTROL);
24003ba88cfSAlex Deucher tmp |= VGA_HDP_CONTROL__VGA_MEMORY_DISABLE_MASK;
24103ba88cfSAlex Deucher WREG32(mmVGA_HDP_CONTROL, tmp);
24203ba88cfSAlex Deucher
24303ba88cfSAlex Deucher /* disable VGA render */
24403ba88cfSAlex Deucher tmp = RREG32(mmVGA_RENDER_CONTROL);
24503ba88cfSAlex Deucher tmp &= ~VGA_VSTATUS_CNTL;
24603ba88cfSAlex Deucher WREG32(mmVGA_RENDER_CONTROL, tmp);
24703ba88cfSAlex Deucher }
248df70502eSKen Wang /* Update configuration */
24972518269STom St Denis WREG32(mmMC_VM_SYSTEM_APERTURE_LOW_ADDR,
250770d13b1SChristian König adev->gmc.vram_start >> 12);
25172518269STom St Denis WREG32(mmMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
252770d13b1SChristian König adev->gmc.vram_end >> 12);
25372518269STom St Denis WREG32(mmMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
2547ccfd79fSChristian König adev->mem_scratch.gpu_addr >> 12);
25572518269STom St Denis WREG32(mmMC_VM_AGP_BASE, 0);
25672518269STom St Denis WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
25772518269STom St Denis WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
258df70502eSKen Wang
2590cfc1d68SSrinivasan Shanmugam if (gmc_v6_0_wait_for_idle((void *)adev))
260df70502eSKen Wang dev_warn(adev->dev, "Wait for MC idle timedout !\n");
261df70502eSKen Wang }
262df70502eSKen Wang
gmc_v6_0_mc_init(struct amdgpu_device * adev)263df70502eSKen Wang static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
264df70502eSKen Wang {
265df70502eSKen Wang
266df70502eSKen Wang u32 tmp;
267df70502eSKen Wang int chansize, numchan;
268d6895ad3SChristian König int r;
269df70502eSKen Wang
27072518269STom St Denis tmp = RREG32(mmMC_ARB_RAMCFG);
2710cfc1d68SSrinivasan Shanmugam if (tmp & (1 << 11))
272df70502eSKen Wang chansize = 16;
2730cfc1d68SSrinivasan Shanmugam else if (tmp & MC_ARB_RAMCFG__CHANSIZE_MASK)
274df70502eSKen Wang chansize = 64;
2750cfc1d68SSrinivasan Shanmugam else
276df70502eSKen Wang chansize = 32;
2770cfc1d68SSrinivasan Shanmugam
27872518269STom St Denis tmp = RREG32(mmMC_SHARED_CHMAP);
27972518269STom St Denis switch ((tmp & MC_SHARED_CHMAP__NOOFCHAN_MASK) >> MC_SHARED_CHMAP__NOOFCHAN__SHIFT) {
280df70502eSKen Wang case 0:
281df70502eSKen Wang default:
282df70502eSKen Wang numchan = 1;
283df70502eSKen Wang break;
284df70502eSKen Wang case 1:
285df70502eSKen Wang numchan = 2;
286df70502eSKen Wang break;
287df70502eSKen Wang case 2:
288df70502eSKen Wang numchan = 4;
289df70502eSKen Wang break;
290df70502eSKen Wang case 3:
291df70502eSKen Wang numchan = 8;
292df70502eSKen Wang break;
293df70502eSKen Wang case 4:
294df70502eSKen Wang numchan = 3;
295df70502eSKen Wang break;
296df70502eSKen Wang case 5:
297df70502eSKen Wang numchan = 6;
298df70502eSKen Wang break;
299df70502eSKen Wang case 6:
300df70502eSKen Wang numchan = 10;
301df70502eSKen Wang break;
302df70502eSKen Wang case 7:
303df70502eSKen Wang numchan = 12;
304df70502eSKen Wang break;
305df70502eSKen Wang case 8:
306df70502eSKen Wang numchan = 16;
307df70502eSKen Wang break;
308df70502eSKen Wang }
309770d13b1SChristian König adev->gmc.vram_width = numchan * chansize;
310df70502eSKen Wang /* size in MB on si */
311770d13b1SChristian König adev->gmc.mc_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
312770d13b1SChristian König adev->gmc.real_vram_size = RREG32(mmCONFIG_MEMSIZE) * 1024ULL * 1024ULL;
313d6895ad3SChristian König
314d6895ad3SChristian König if (!(adev->flags & AMD_IS_APU)) {
315d6895ad3SChristian König r = amdgpu_device_resize_fb_bar(adev);
316d6895ad3SChristian König if (r)
317d6895ad3SChristian König return r;
318d6895ad3SChristian König }
319770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0);
320770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0);
321770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size;
322df70502eSKen Wang
323c3db7b5aSAlex Deucher /* set the gart size */
324c3db7b5aSAlex Deucher if (amdgpu_gart_size == -1) {
325c3db7b5aSAlex Deucher switch (adev->asic_type) {
326c3db7b5aSAlex Deucher case CHIP_HAINAN: /* no MM engines */
327c3db7b5aSAlex Deucher default:
328770d13b1SChristian König adev->gmc.gart_size = 256ULL << 20;
329c3db7b5aSAlex Deucher break;
330c3db7b5aSAlex Deucher case CHIP_VERDE: /* UVD, VCE do not support GPUVM */
331c3db7b5aSAlex Deucher case CHIP_TAHITI: /* UVD, VCE do not support GPUVM */
332c3db7b5aSAlex Deucher case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */
333c3db7b5aSAlex Deucher case CHIP_OLAND: /* UVD, VCE do not support GPUVM */
334770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20;
335c3db7b5aSAlex Deucher break;
336c3db7b5aSAlex Deucher }
337c3db7b5aSAlex Deucher } else {
338770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
339c3db7b5aSAlex Deucher }
340c3db7b5aSAlex Deucher
341f1dc12caSOak Zeng adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
342770d13b1SChristian König gmc_v6_0_vram_gtt_location(adev, &adev->gmc);
343df70502eSKen Wang
344df70502eSKen Wang return 0;
345df70502eSKen Wang }
346df70502eSKen Wang
gmc_v6_0_flush_gpu_tlb(struct amdgpu_device * adev,uint32_t vmid,uint32_t vmhub,uint32_t flush_type)3473ff98548SOak Zeng static void gmc_v6_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
3483ff98548SOak Zeng uint32_t vmhub, uint32_t flush_type)
349df70502eSKen Wang {
35072518269STom St Denis WREG32(mmVM_INVALIDATE_REQUEST, 1 << vmid);
351df70502eSKen Wang }
352df70502eSKen Wang
gmc_v6_0_emit_flush_gpu_tlb(struct amdgpu_ring * ring,unsigned int vmid,uint64_t pd_addr)3534fef88bdSChristian König static uint64_t gmc_v6_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring,
3540cfc1d68SSrinivasan Shanmugam unsigned int vmid, uint64_t pd_addr)
3554fef88bdSChristian König {
3564fef88bdSChristian König uint32_t reg;
3574fef88bdSChristian König
3584fef88bdSChristian König /* write new base address */
3594fef88bdSChristian König if (vmid < 8)
3604fef88bdSChristian König reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vmid;
3614fef88bdSChristian König else
3624fef88bdSChristian König reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (vmid - 8);
3634fef88bdSChristian König amdgpu_ring_emit_wreg(ring, reg, pd_addr >> 12);
3644fef88bdSChristian König
3654fef88bdSChristian König /* bits 0-15 are the VM contexts0-15 */
3664fef88bdSChristian König amdgpu_ring_emit_wreg(ring, mmVM_INVALIDATE_REQUEST, 1 << vmid);
3674fef88bdSChristian König
3684fef88bdSChristian König return pd_addr;
3694fef88bdSChristian König }
3704fef88bdSChristian König
gmc_v6_0_get_vm_pde(struct amdgpu_device * adev,int level,uint64_t * addr,uint64_t * flags)3713de676d8SChristian König static void gmc_v6_0_get_vm_pde(struct amdgpu_device *adev, int level,
3723de676d8SChristian König uint64_t *addr, uint64_t *flags)
373b1166325SChristian König {
3743de676d8SChristian König BUG_ON(*addr & 0xFFFFFF0000000FFFULL);
375b1166325SChristian König }
376b1166325SChristian König
gmc_v6_0_get_vm_pte(struct amdgpu_device * adev,struct amdgpu_bo_va_mapping * mapping,uint64_t * flags)377cbfae36cSChristian König static void gmc_v6_0_get_vm_pte(struct amdgpu_device *adev,
378cbfae36cSChristian König struct amdgpu_bo_va_mapping *mapping,
379cbfae36cSChristian König uint64_t *flags)
380cbfae36cSChristian König {
381cbfae36cSChristian König *flags &= ~AMDGPU_PTE_EXECUTABLE;
382cbfae36cSChristian König *flags &= ~AMDGPU_PTE_PRT;
383cbfae36cSChristian König }
384cbfae36cSChristian König
gmc_v6_0_set_fault_enable_default(struct amdgpu_device * adev,bool value)385df70502eSKen Wang static void gmc_v6_0_set_fault_enable_default(struct amdgpu_device *adev,
386df70502eSKen Wang bool value)
387df70502eSKen Wang {
388df70502eSKen Wang u32 tmp;
389df70502eSKen Wang
39072518269STom St Denis tmp = RREG32(mmVM_CONTEXT1_CNTL);
39172518269STom St Denis tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
39272518269STom St Denis RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
39372518269STom St Denis tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
39472518269STom St Denis DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
39572518269STom St Denis tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
39672518269STom St Denis PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
39772518269STom St Denis tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
39872518269STom St Denis VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
39972518269STom St Denis tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
40072518269STom St Denis READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
40172518269STom St Denis tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
40272518269STom St Denis WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
40372518269STom St Denis WREG32(mmVM_CONTEXT1_CNTL, tmp);
404df70502eSKen Wang }
405df70502eSKen Wang
406f7c35abeSChristian König /**
4070cfc1d68SSrinivasan Shanmugam * gmc_v8_0_set_prt() - set PRT VM fault
4080cfc1d68SSrinivasan Shanmugam *
4090cfc1d68SSrinivasan Shanmugam * @adev: amdgpu_device pointer
4100cfc1d68SSrinivasan Shanmugam * @enable: enable/disable VM fault handling for PRT
4110cfc1d68SSrinivasan Shanmugam */
gmc_v6_0_set_prt(struct amdgpu_device * adev,bool enable)412f7c35abeSChristian König static void gmc_v6_0_set_prt(struct amdgpu_device *adev, bool enable)
413f7c35abeSChristian König {
414f7c35abeSChristian König u32 tmp;
415f7c35abeSChristian König
416770d13b1SChristian König if (enable && !adev->gmc.prt_warning) {
417f7c35abeSChristian König dev_warn(adev->dev, "Disabling VM faults because of PRT request!\n");
418770d13b1SChristian König adev->gmc.prt_warning = true;
419f7c35abeSChristian König }
420f7c35abeSChristian König
421f7c35abeSChristian König tmp = RREG32(mmVM_PRT_CNTL);
422f7c35abeSChristian König tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
423f7c35abeSChristian König CB_DISABLE_FAULT_ON_UNMAPPED_ACCESS,
424f7c35abeSChristian König enable);
425f7c35abeSChristian König tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
426f7c35abeSChristian König TC_DISABLE_FAULT_ON_UNMAPPED_ACCESS,
427f7c35abeSChristian König enable);
428f7c35abeSChristian König tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
429f7c35abeSChristian König L2_CACHE_STORE_INVALID_ENTRIES,
430f7c35abeSChristian König enable);
431f7c35abeSChristian König tmp = REG_SET_FIELD(tmp, VM_PRT_CNTL,
432f7c35abeSChristian König L1_TLB_STORE_INVALID_ENTRIES,
433f7c35abeSChristian König enable);
434f7c35abeSChristian König WREG32(mmVM_PRT_CNTL, tmp);
435f7c35abeSChristian König
436f7c35abeSChristian König if (enable) {
437f7c35abeSChristian König uint32_t low = AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT;
438a3e9a15aSChristian König uint32_t high = adev->vm_manager.max_pfn -
439a3e9a15aSChristian König (AMDGPU_VA_RESERVED_SIZE >> AMDGPU_GPU_PAGE_SHIFT);
440f7c35abeSChristian König
441f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, low);
442f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, low);
443f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, low);
444f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, low);
445f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, high);
446f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, high);
447f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, high);
448f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, high);
449f7c35abeSChristian König } else {
450f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE0_LOW_ADDR, 0xfffffff);
451f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE1_LOW_ADDR, 0xfffffff);
452f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE2_LOW_ADDR, 0xfffffff);
453f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE3_LOW_ADDR, 0xfffffff);
454f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE0_HIGH_ADDR, 0x0);
455f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE1_HIGH_ADDR, 0x0);
456f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE2_HIGH_ADDR, 0x0);
457f7c35abeSChristian König WREG32(mmVM_PRT_APERTURE3_HIGH_ADDR, 0x0);
458f7c35abeSChristian König }
459f7c35abeSChristian König }
460f7c35abeSChristian König
gmc_v6_0_gart_enable(struct amdgpu_device * adev)461df70502eSKen Wang static int gmc_v6_0_gart_enable(struct amdgpu_device *adev)
462df70502eSKen Wang {
463bdb1922aSMichel Dänzer uint64_t table_addr;
464e618d306SRoger He u32 field;
4651b08dfb8SChristian König int i;
466df70502eSKen Wang
4671123b989SChristian König if (adev->gart.bo == NULL) {
468df70502eSKen Wang dev_err(adev->dev, "No VRAM object for PCIE GART.\n");
469df70502eSKen Wang return -EINVAL;
470df70502eSKen Wang }
4711b08dfb8SChristian König amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr);
472bdb1922aSMichel Dänzer
473bdb1922aSMichel Dänzer table_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
474bdb1922aSMichel Dänzer
475df70502eSKen Wang /* Setup TLB control */
47672518269STom St Denis WREG32(mmMC_VM_MX_L1_TLB_CNTL,
477df70502eSKen Wang (0xA << 7) |
47872518269STom St Denis MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_TLB_MASK |
47972518269STom St Denis MC_VM_MX_L1_TLB_CNTL__ENABLE_L1_FRAGMENT_PROCESSING_MASK |
48072518269STom St Denis MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
48172518269STom St Denis MC_VM_MX_L1_TLB_CNTL__ENABLE_ADVANCED_DRIVER_MODEL_MASK |
48272518269STom St Denis (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
483df70502eSKen Wang /* Setup L2 cache */
48472518269STom St Denis WREG32(mmVM_L2_CNTL,
48572518269STom St Denis VM_L2_CNTL__ENABLE_L2_CACHE_MASK |
48672518269STom St Denis VM_L2_CNTL__ENABLE_L2_FRAGMENT_PROCESSING_MASK |
48772518269STom St Denis VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
48872518269STom St Denis VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
48972518269STom St Denis (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
49072518269STom St Denis (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
49172518269STom St Denis WREG32(mmVM_L2_CNTL2,
49272518269STom St Denis VM_L2_CNTL2__INVALIDATE_ALL_L1_TLBS_MASK |
49372518269STom St Denis VM_L2_CNTL2__INVALIDATE_L2_CACHE_MASK);
494e618d306SRoger He
495e618d306SRoger He field = adev->vm_manager.fragment_size;
49672518269STom St Denis WREG32(mmVM_L2_CNTL3,
49772518269STom St Denis VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
498e618d306SRoger He (field << VM_L2_CNTL3__BANK_SELECT__SHIFT) |
499e618d306SRoger He (field << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
500df70502eSKen Wang /* setup context0 */
501770d13b1SChristian König WREG32(mmVM_CONTEXT0_PAGE_TABLE_START_ADDR, adev->gmc.gart_start >> 12);
502770d13b1SChristian König WREG32(mmVM_CONTEXT0_PAGE_TABLE_END_ADDR, adev->gmc.gart_end >> 12);
5034e830fb1SChristian König WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR, table_addr >> 12);
50472518269STom St Denis WREG32(mmVM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
50592e71b06SChristian König (u32)(adev->dummy_page_addr >> 12));
50672518269STom St Denis WREG32(mmVM_CONTEXT0_CNTL2, 0);
50772518269STom St Denis WREG32(mmVM_CONTEXT0_CNTL,
50872518269STom St Denis VM_CONTEXT0_CNTL__ENABLE_CONTEXT_MASK |
50972518269STom St Denis (0UL << VM_CONTEXT0_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
51072518269STom St Denis VM_CONTEXT0_CNTL__RANGE_PROTECTION_FAULT_ENABLE_DEFAULT_MASK);
511df70502eSKen Wang
512df70502eSKen Wang WREG32(0x575, 0);
513df70502eSKen Wang WREG32(0x576, 0);
514df70502eSKen Wang WREG32(0x577, 0);
515df70502eSKen Wang
516df70502eSKen Wang /* empty context1-15 */
517df70502eSKen Wang /* set vm size, must be a multiple of 4 */
51872518269STom St Denis WREG32(mmVM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
51972518269STom St Denis WREG32(mmVM_CONTEXT1_PAGE_TABLE_END_ADDR, adev->vm_manager.max_pfn - 1);
520df70502eSKen Wang /* Assign the pt base to something valid for now; the pts used for
521df70502eSKen Wang * the VMs are determined by the application and setup and assigned
522df70502eSKen Wang * on the fly in the vm part of radeon_gart.c
523df70502eSKen Wang */
52468fce5f0SNirmoy Das for (i = 1; i < AMDGPU_NUM_VMID; i++) {
525df70502eSKen Wang if (i < 8)
52672518269STom St Denis WREG32(mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i,
5274e830fb1SChristian König table_addr >> 12);
528df70502eSKen Wang else
52972518269STom St Denis WREG32(mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + i - 8,
5304e830fb1SChristian König table_addr >> 12);
531df70502eSKen Wang }
532df70502eSKen Wang
533df70502eSKen Wang /* enable context1-15 */
53472518269STom St Denis WREG32(mmVM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
53592e71b06SChristian König (u32)(adev->dummy_page_addr >> 12));
53672518269STom St Denis WREG32(mmVM_CONTEXT1_CNTL2, 4);
53772518269STom St Denis WREG32(mmVM_CONTEXT1_CNTL,
53872518269STom St Denis VM_CONTEXT1_CNTL__ENABLE_CONTEXT_MASK |
53972518269STom St Denis (1UL << VM_CONTEXT1_CNTL__PAGE_TABLE_DEPTH__SHIFT) |
54036b32a68SZhang, Jerry ((adev->vm_manager.block_size - 9)
54136b32a68SZhang, Jerry << VM_CONTEXT1_CNTL__PAGE_TABLE_BLOCK_SIZE__SHIFT));
542a8447647SFlora Cui if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
543a8447647SFlora Cui gmc_v6_0_set_fault_enable_default(adev, false);
544a8447647SFlora Cui else
545a8447647SFlora Cui gmc_v6_0_set_fault_enable_default(adev, true);
546df70502eSKen Wang
5473ff98548SOak Zeng gmc_v6_0_flush_gpu_tlb(adev, 0, 0, 0);
548075719c3STom St Denis dev_info(adev->dev, "PCIE GART of %uM enabled (table at 0x%016llX).\n",
5490cfc1d68SSrinivasan Shanmugam (unsigned int)(adev->gmc.gart_size >> 20),
5504e830fb1SChristian König (unsigned long long)table_addr);
551df70502eSKen Wang return 0;
552df70502eSKen Wang }
553df70502eSKen Wang
gmc_v6_0_gart_init(struct amdgpu_device * adev)554df70502eSKen Wang static int gmc_v6_0_gart_init(struct amdgpu_device *adev)
555df70502eSKen Wang {
556df70502eSKen Wang int r;
557df70502eSKen Wang
5581123b989SChristian König if (adev->gart.bo) {
559075719c3STom St Denis dev_warn(adev->dev, "gmc_v6_0 PCIE GART already initialized\n");
560df70502eSKen Wang return 0;
561df70502eSKen Wang }
562df70502eSKen Wang r = amdgpu_gart_init(adev);
563df70502eSKen Wang if (r)
564df70502eSKen Wang return r;
565df70502eSKen Wang adev->gart.table_size = adev->gart.num_gpu_pages * 8;
5664b98e0c4SAlex Xie adev->gart.gart_pte_flags = 0;
567df70502eSKen Wang return amdgpu_gart_table_vram_alloc(adev);
568df70502eSKen Wang }
569df70502eSKen Wang
gmc_v6_0_gart_disable(struct amdgpu_device * adev)570df70502eSKen Wang static void gmc_v6_0_gart_disable(struct amdgpu_device *adev)
571df70502eSKen Wang {
572df70502eSKen Wang /*unsigned i;
573df70502eSKen Wang
574df70502eSKen Wang for (i = 1; i < 16; ++i) {
575df70502eSKen Wang uint32_t reg;
576df70502eSKen Wang if (i < 8)
577df70502eSKen Wang reg = VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + i ;
578df70502eSKen Wang else
579df70502eSKen Wang reg = VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + (i - 8);
580df70502eSKen Wang adev->vm_manager.saved_table_addr[i] = RREG32(reg);
581df70502eSKen Wang }*/
582df70502eSKen Wang
583df70502eSKen Wang /* Disable all tables */
58472518269STom St Denis WREG32(mmVM_CONTEXT0_CNTL, 0);
58572518269STom St Denis WREG32(mmVM_CONTEXT1_CNTL, 0);
586df70502eSKen Wang /* Setup TLB control */
58772518269STom St Denis WREG32(mmMC_VM_MX_L1_TLB_CNTL,
58872518269STom St Denis MC_VM_MX_L1_TLB_CNTL__SYSTEM_ACCESS_MODE_MASK |
58972518269STom St Denis (0UL << MC_VM_MX_L1_TLB_CNTL__SYSTEM_APERTURE_UNMAPPED_ACCESS__SHIFT));
590df70502eSKen Wang /* Setup L2 cache */
59172518269STom St Denis WREG32(mmVM_L2_CNTL,
59272518269STom St Denis VM_L2_CNTL__ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE_MASK |
59372518269STom St Denis VM_L2_CNTL__ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE_MASK |
59472518269STom St Denis (7UL << VM_L2_CNTL__EFFECTIVE_L2_QUEUE_SIZE__SHIFT) |
59572518269STom St Denis (1UL << VM_L2_CNTL__CONTEXT1_IDENTITY_ACCESS_MODE__SHIFT));
59672518269STom St Denis WREG32(mmVM_L2_CNTL2, 0);
59772518269STom St Denis WREG32(mmVM_L2_CNTL3,
59872518269STom St Denis VM_L2_CNTL3__L2_CACHE_BIGK_ASSOCIATIVITY_MASK |
59972518269STom St Denis (0UL << VM_L2_CNTL3__L2_CACHE_BIGK_FRAGMENT_SIZE__SHIFT));
600df70502eSKen Wang }
601df70502eSKen Wang
gmc_v6_0_vm_decode_fault(struct amdgpu_device * adev,u32 status,u32 addr,u32 mc_client)602df70502eSKen Wang static void gmc_v6_0_vm_decode_fault(struct amdgpu_device *adev,
603df70502eSKen Wang u32 status, u32 addr, u32 mc_client)
604df70502eSKen Wang {
605df70502eSKen Wang u32 mc_id;
60672518269STom St Denis u32 vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
60772518269STom St Denis u32 protections = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
60872518269STom St Denis PROTECTIONS);
609df70502eSKen Wang char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
610df70502eSKen Wang (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
611df70502eSKen Wang
61272518269STom St Denis mc_id = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
61372518269STom St Denis MEMORY_CLIENT_ID);
614df70502eSKen Wang
615075719c3STom St Denis dev_err(adev->dev, "VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
616df70502eSKen Wang protections, vmid, addr,
61772518269STom St Denis REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
61872518269STom St Denis MEMORY_CLIENT_RW) ?
619df70502eSKen Wang "write" : "read", block, mc_client, mc_id);
620df70502eSKen Wang }
621df70502eSKen Wang
622df70502eSKen Wang /*
623df70502eSKen Wang static const u32 mc_cg_registers[] = {
624df70502eSKen Wang MC_HUB_MISC_HUB_CG,
625df70502eSKen Wang MC_HUB_MISC_SIP_CG,
626df70502eSKen Wang MC_HUB_MISC_VM_CG,
627df70502eSKen Wang MC_XPB_CLK_GAT,
628df70502eSKen Wang ATC_MISC_CG,
629df70502eSKen Wang MC_CITF_MISC_WR_CG,
630df70502eSKen Wang MC_CITF_MISC_RD_CG,
631df70502eSKen Wang MC_CITF_MISC_VM_CG,
632df70502eSKen Wang VM_L2_CG,
633df70502eSKen Wang };
634df70502eSKen Wang
635df70502eSKen Wang static const u32 mc_cg_ls_en[] = {
636df70502eSKen Wang MC_HUB_MISC_HUB_CG__MEM_LS_ENABLE_MASK,
637df70502eSKen Wang MC_HUB_MISC_SIP_CG__MEM_LS_ENABLE_MASK,
638df70502eSKen Wang MC_HUB_MISC_VM_CG__MEM_LS_ENABLE_MASK,
639df70502eSKen Wang MC_XPB_CLK_GAT__MEM_LS_ENABLE_MASK,
640df70502eSKen Wang ATC_MISC_CG__MEM_LS_ENABLE_MASK,
641df70502eSKen Wang MC_CITF_MISC_WR_CG__MEM_LS_ENABLE_MASK,
642df70502eSKen Wang MC_CITF_MISC_RD_CG__MEM_LS_ENABLE_MASK,
643df70502eSKen Wang MC_CITF_MISC_VM_CG__MEM_LS_ENABLE_MASK,
644df70502eSKen Wang VM_L2_CG__MEM_LS_ENABLE_MASK,
645df70502eSKen Wang };
646df70502eSKen Wang
647df70502eSKen Wang static const u32 mc_cg_en[] = {
648df70502eSKen Wang MC_HUB_MISC_HUB_CG__ENABLE_MASK,
649df70502eSKen Wang MC_HUB_MISC_SIP_CG__ENABLE_MASK,
650df70502eSKen Wang MC_HUB_MISC_VM_CG__ENABLE_MASK,
651df70502eSKen Wang MC_XPB_CLK_GAT__ENABLE_MASK,
652df70502eSKen Wang ATC_MISC_CG__ENABLE_MASK,
653df70502eSKen Wang MC_CITF_MISC_WR_CG__ENABLE_MASK,
654df70502eSKen Wang MC_CITF_MISC_RD_CG__ENABLE_MASK,
655df70502eSKen Wang MC_CITF_MISC_VM_CG__ENABLE_MASK,
656df70502eSKen Wang VM_L2_CG__ENABLE_MASK,
657df70502eSKen Wang };
658df70502eSKen Wang
659df70502eSKen Wang static void gmc_v6_0_enable_mc_ls(struct amdgpu_device *adev,
660df70502eSKen Wang bool enable)
661df70502eSKen Wang {
662df70502eSKen Wang int i;
663df70502eSKen Wang u32 orig, data;
664df70502eSKen Wang
665df70502eSKen Wang for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
666df70502eSKen Wang orig = data = RREG32(mc_cg_registers[i]);
667df70502eSKen Wang if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_MC_LS))
668df70502eSKen Wang data |= mc_cg_ls_en[i];
669df70502eSKen Wang else
670df70502eSKen Wang data &= ~mc_cg_ls_en[i];
671df70502eSKen Wang if (data != orig)
672df70502eSKen Wang WREG32(mc_cg_registers[i], data);
673df70502eSKen Wang }
674df70502eSKen Wang }
675df70502eSKen Wang
676df70502eSKen Wang static void gmc_v6_0_enable_mc_mgcg(struct amdgpu_device *adev,
677df70502eSKen Wang bool enable)
678df70502eSKen Wang {
679df70502eSKen Wang int i;
680df70502eSKen Wang u32 orig, data;
681df70502eSKen Wang
682df70502eSKen Wang for (i = 0; i < ARRAY_SIZE(mc_cg_registers); i++) {
683df70502eSKen Wang orig = data = RREG32(mc_cg_registers[i]);
684df70502eSKen Wang if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_MC_MGCG))
685df70502eSKen Wang data |= mc_cg_en[i];
686df70502eSKen Wang else
687df70502eSKen Wang data &= ~mc_cg_en[i];
688df70502eSKen Wang if (data != orig)
689df70502eSKen Wang WREG32(mc_cg_registers[i], data);
690df70502eSKen Wang }
691df70502eSKen Wang }
692df70502eSKen Wang
693df70502eSKen Wang static void gmc_v6_0_enable_bif_mgls(struct amdgpu_device *adev,
694df70502eSKen Wang bool enable)
695df70502eSKen Wang {
696df70502eSKen Wang u32 orig, data;
697df70502eSKen Wang
698df70502eSKen Wang orig = data = RREG32_PCIE(ixPCIE_CNTL2);
699df70502eSKen Wang
700df70502eSKen Wang if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_BIF_LS)) {
701df70502eSKen Wang data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 1);
702df70502eSKen Wang data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 1);
703df70502eSKen Wang data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 1);
704df70502eSKen Wang data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 1);
705df70502eSKen Wang } else {
706df70502eSKen Wang data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_LS_EN, 0);
707df70502eSKen Wang data = REG_SET_FIELD(data, PCIE_CNTL2, MST_MEM_LS_EN, 0);
708df70502eSKen Wang data = REG_SET_FIELD(data, PCIE_CNTL2, REPLAY_MEM_LS_EN, 0);
709df70502eSKen Wang data = REG_SET_FIELD(data, PCIE_CNTL2, SLV_MEM_AGGRESSIVE_LS_EN, 0);
710df70502eSKen Wang }
711df70502eSKen Wang
712df70502eSKen Wang if (orig != data)
713df70502eSKen Wang WREG32_PCIE(ixPCIE_CNTL2, data);
714df70502eSKen Wang }
715df70502eSKen Wang
716df70502eSKen Wang static void gmc_v6_0_enable_hdp_mgcg(struct amdgpu_device *adev,
717df70502eSKen Wang bool enable)
718df70502eSKen Wang {
719df70502eSKen Wang u32 orig, data;
720df70502eSKen Wang
72172518269STom St Denis orig = data = RREG32(mmHDP_HOST_PATH_CNTL);
722df70502eSKen Wang
723df70502eSKen Wang if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_HDP_MGCG))
724df70502eSKen Wang data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 0);
725df70502eSKen Wang else
726df70502eSKen Wang data = REG_SET_FIELD(data, HDP_HOST_PATH_CNTL, CLOCK_GATING_DIS, 1);
727df70502eSKen Wang
728df70502eSKen Wang if (orig != data)
72972518269STom St Denis WREG32(mmHDP_HOST_PATH_CNTL, data);
730df70502eSKen Wang }
731df70502eSKen Wang
732df70502eSKen Wang static void gmc_v6_0_enable_hdp_ls(struct amdgpu_device *adev,
733df70502eSKen Wang bool enable)
734df70502eSKen Wang {
735df70502eSKen Wang u32 orig, data;
736df70502eSKen Wang
73772518269STom St Denis orig = data = RREG32(mmHDP_MEM_POWER_LS);
738df70502eSKen Wang
739df70502eSKen Wang if (enable && (adev->cg_flags & AMDGPU_CG_SUPPORT_HDP_LS))
740df70502eSKen Wang data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 1);
741df70502eSKen Wang else
742df70502eSKen Wang data = REG_SET_FIELD(data, HDP_MEM_POWER_LS, LS_ENABLE, 0);
743df70502eSKen Wang
744df70502eSKen Wang if (orig != data)
74572518269STom St Denis WREG32(mmHDP_MEM_POWER_LS, data);
746df70502eSKen Wang }
747df70502eSKen Wang */
748df70502eSKen Wang
gmc_v6_0_convert_vram_type(int mc_seq_vram_type)749df70502eSKen Wang static int gmc_v6_0_convert_vram_type(int mc_seq_vram_type)
750df70502eSKen Wang {
751df70502eSKen Wang switch (mc_seq_vram_type) {
752df70502eSKen Wang case MC_SEQ_MISC0__MT__GDDR1:
753df70502eSKen Wang return AMDGPU_VRAM_TYPE_GDDR1;
754df70502eSKen Wang case MC_SEQ_MISC0__MT__DDR2:
755df70502eSKen Wang return AMDGPU_VRAM_TYPE_DDR2;
756df70502eSKen Wang case MC_SEQ_MISC0__MT__GDDR3:
757df70502eSKen Wang return AMDGPU_VRAM_TYPE_GDDR3;
758df70502eSKen Wang case MC_SEQ_MISC0__MT__GDDR4:
759df70502eSKen Wang return AMDGPU_VRAM_TYPE_GDDR4;
760df70502eSKen Wang case MC_SEQ_MISC0__MT__GDDR5:
761df70502eSKen Wang return AMDGPU_VRAM_TYPE_GDDR5;
762df70502eSKen Wang case MC_SEQ_MISC0__MT__DDR3:
763df70502eSKen Wang return AMDGPU_VRAM_TYPE_DDR3;
764df70502eSKen Wang default:
765df70502eSKen Wang return AMDGPU_VRAM_TYPE_UNKNOWN;
766df70502eSKen Wang }
767df70502eSKen Wang }
768df70502eSKen Wang
gmc_v6_0_early_init(void * handle)769df70502eSKen Wang static int gmc_v6_0_early_init(void *handle)
770df70502eSKen Wang {
771df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
772df70502eSKen Wang
773132f34e4SChristian König gmc_v6_0_set_gmc_funcs(adev);
774df70502eSKen Wang gmc_v6_0_set_irq_funcs(adev);
775df70502eSKen Wang
776df70502eSKen Wang return 0;
777df70502eSKen Wang }
778df70502eSKen Wang
gmc_v6_0_late_init(void * handle)779df70502eSKen Wang static int gmc_v6_0_late_init(void *handle)
780df70502eSKen Wang {
781df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
782df70502eSKen Wang
783a8447647SFlora Cui if (amdgpu_vm_fault_stop != AMDGPU_VM_FAULT_STOP_ALWAYS)
784770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0);
785a8447647SFlora Cui else
786a8447647SFlora Cui return 0;
787df70502eSKen Wang }
788df70502eSKen Wang
gmc_v6_0_get_vbios_fb_size(struct amdgpu_device * adev)7890cfc1d68SSrinivasan Shanmugam static unsigned int gmc_v6_0_get_vbios_fb_size(struct amdgpu_device *adev)
790ebdef28eSAlex Deucher {
791ebdef28eSAlex Deucher u32 d1vga_control = RREG32(mmD1VGA_CONTROL);
7920cfc1d68SSrinivasan Shanmugam unsigned int size;
793ebdef28eSAlex Deucher
794ebdef28eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) {
79581b54fb7SAlex Deucher size = AMDGPU_VBIOS_VGA_ALLOCATION;
796ebdef28eSAlex Deucher } else {
797ebdef28eSAlex Deucher u32 viewport = RREG32(mmVIEWPORT_SIZE);
7980cfc1d68SSrinivasan Shanmugam
799ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_HEIGHT) *
800ebdef28eSAlex Deucher REG_GET_FIELD(viewport, VIEWPORT_SIZE, VIEWPORT_WIDTH) *
801ebdef28eSAlex Deucher 4);
802ebdef28eSAlex Deucher }
803ebdef28eSAlex Deucher return size;
804ebdef28eSAlex Deucher }
805ebdef28eSAlex Deucher
gmc_v6_0_sw_init(void * handle)806df70502eSKen Wang static int gmc_v6_0_sw_init(void *handle)
807df70502eSKen Wang {
808df70502eSKen Wang int r;
809df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
810df70502eSKen Wang
811d9426c3dSLe Ma set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask);
812bad4c3e6SAlex Deucher
813b8691c76SJim Qu if (adev->flags & AMD_IS_APU) {
814770d13b1SChristian König adev->gmc.vram_type = AMDGPU_VRAM_TYPE_UNKNOWN;
815b8691c76SJim Qu } else {
816b8691c76SJim Qu u32 tmp = RREG32(mmMC_SEQ_MISC0);
8170cfc1d68SSrinivasan Shanmugam
818b8691c76SJim Qu tmp &= MC_SEQ_MISC0__MT__MASK;
819770d13b1SChristian König adev->gmc.vram_type = gmc_v6_0_convert_vram_type(tmp);
820b8691c76SJim Qu }
821b8691c76SJim Qu
8221ffdeca6SChristian König r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 146, &adev->gmc.vm_fault);
823df70502eSKen Wang if (r)
824df70502eSKen Wang return r;
825df70502eSKen Wang
8261ffdeca6SChristian König r = amdgpu_irq_add_id(adev, AMDGPU_IRQ_CLIENTID_LEGACY, 147, &adev->gmc.vm_fault);
827df70502eSKen Wang if (r)
828df70502eSKen Wang return r;
829df70502eSKen Wang
830f3368128SChristian König amdgpu_vm_adjust_size(adev, 64, 9, 1, 40);
83136b32a68SZhang, Jerry
832770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffULL;
833df70502eSKen Wang
834403475beSAlex Deucher r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(40));
835df70502eSKen Wang if (r) {
836dd4fa6c1SAurabindo Pillai dev_warn(adev->dev, "No suitable DMA available.\n");
837244511f3SChristoph Hellwig return r;
838df70502eSKen Wang }
839403475beSAlex Deucher adev->need_swiotlb = drm_need_swiotlb(40);
840df70502eSKen Wang
841df70502eSKen Wang r = gmc_v6_0_init_microcode(adev);
842df70502eSKen Wang if (r) {
843075719c3STom St Denis dev_err(adev->dev, "Failed to load mc firmware!\n");
844df70502eSKen Wang return r;
845df70502eSKen Wang }
846df70502eSKen Wang
847df70502eSKen Wang r = gmc_v6_0_mc_init(adev);
848df70502eSKen Wang if (r)
849df70502eSKen Wang return r;
850df70502eSKen Wang
851422fe8d2SAlex Deucher amdgpu_gmc_get_vbios_allocations(adev);
852ebdef28eSAlex Deucher
853df70502eSKen Wang r = amdgpu_bo_init(adev);
854df70502eSKen Wang if (r)
855df70502eSKen Wang return r;
856df70502eSKen Wang
857df70502eSKen Wang r = gmc_v6_0_gart_init(adev);
858df70502eSKen Wang if (r)
859df70502eSKen Wang return r;
860df70502eSKen Wang
86105ec3edaSChristian König /*
86205ec3edaSChristian König * number of VMs
86305ec3edaSChristian König * VMID 0 is reserved for System
86405ec3edaSChristian König * amdgpu graphics/compute will use VMIDs 1-7
86505ec3edaSChristian König * amdkfd will use VMIDs 8-15
86605ec3edaSChristian König */
86740111ec2SFelix Kuehling adev->vm_manager.first_kfd_vmid = 8;
86805ec3edaSChristian König amdgpu_vm_manager_init(adev);
86905ec3edaSChristian König
87005ec3edaSChristian König /* base offset of vram pages */
87105ec3edaSChristian König if (adev->flags & AMD_IS_APU) {
87205ec3edaSChristian König u64 tmp = RREG32(mmMC_VM_FB_OFFSET);
87305ec3edaSChristian König
87405ec3edaSChristian König tmp <<= 22;
87505ec3edaSChristian König adev->vm_manager.vram_base_offset = tmp;
87605ec3edaSChristian König } else {
87705ec3edaSChristian König adev->vm_manager.vram_base_offset = 0;
878df70502eSKen Wang }
879df70502eSKen Wang
88005ec3edaSChristian König return 0;
881df70502eSKen Wang }
882df70502eSKen Wang
gmc_v6_0_sw_fini(void * handle)883df70502eSKen Wang static int gmc_v6_0_sw_fini(void *handle)
884df70502eSKen Wang {
885df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
886df70502eSKen Wang
887f59548c8SMonk Liu amdgpu_gem_force_release(adev);
88805ec3edaSChristian König amdgpu_vm_manager_fini(adev);
889a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev);
890df70502eSKen Wang amdgpu_bo_fini(adev);
8912d70575bSMario Limonciello amdgpu_ucode_release(&adev->gmc.fw);
892df70502eSKen Wang
893df70502eSKen Wang return 0;
894df70502eSKen Wang }
895df70502eSKen Wang
gmc_v6_0_hw_init(void * handle)896df70502eSKen Wang static int gmc_v6_0_hw_init(void *handle)
897df70502eSKen Wang {
898df70502eSKen Wang int r;
899df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
900df70502eSKen Wang
901df70502eSKen Wang gmc_v6_0_mc_program(adev);
902df70502eSKen Wang
903df70502eSKen Wang if (!(adev->flags & AMD_IS_APU)) {
904df70502eSKen Wang r = gmc_v6_0_mc_load_microcode(adev);
905df70502eSKen Wang if (r) {
906075719c3STom St Denis dev_err(adev->dev, "Failed to load MC firmware!\n");
907df70502eSKen Wang return r;
908df70502eSKen Wang }
909df70502eSKen Wang }
910df70502eSKen Wang
911df70502eSKen Wang r = gmc_v6_0_gart_enable(adev);
912df70502eSKen Wang if (r)
913df70502eSKen Wang return r;
914df70502eSKen Wang
915479e3b02SXiaojian Du if (amdgpu_emu_mode == 1)
916479e3b02SXiaojian Du return amdgpu_gmc_vram_checking(adev);
917*7110e988SSrinivasan Shanmugam
918*7110e988SSrinivasan Shanmugam return 0;
919df70502eSKen Wang }
920df70502eSKen Wang
gmc_v6_0_hw_fini(void * handle)921df70502eSKen Wang static int gmc_v6_0_hw_fini(void *handle)
922df70502eSKen Wang {
923df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
924df70502eSKen Wang
925770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0);
926df70502eSKen Wang gmc_v6_0_gart_disable(adev);
927df70502eSKen Wang
928df70502eSKen Wang return 0;
929df70502eSKen Wang }
930df70502eSKen Wang
gmc_v6_0_suspend(void * handle)931df70502eSKen Wang static int gmc_v6_0_suspend(void *handle)
932df70502eSKen Wang {
933df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
934df70502eSKen Wang
935df70502eSKen Wang gmc_v6_0_hw_fini(adev);
936df70502eSKen Wang
937df70502eSKen Wang return 0;
938df70502eSKen Wang }
939df70502eSKen Wang
gmc_v6_0_resume(void * handle)940df70502eSKen Wang static int gmc_v6_0_resume(void *handle)
941df70502eSKen Wang {
942df70502eSKen Wang int r;
943df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
944df70502eSKen Wang
945df70502eSKen Wang r = gmc_v6_0_hw_init(adev);
946df70502eSKen Wang if (r)
947df70502eSKen Wang return r;
948df70502eSKen Wang
949620f774fSChristian König amdgpu_vmid_reset_all(adev);
950df70502eSKen Wang
951b3c85a0fSChristian König return 0;
952df70502eSKen Wang }
953df70502eSKen Wang
gmc_v6_0_is_idle(void * handle)954df70502eSKen Wang static bool gmc_v6_0_is_idle(void *handle)
955df70502eSKen Wang {
956df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
95772518269STom St Denis u32 tmp = RREG32(mmSRBM_STATUS);
958df70502eSKen Wang
959df70502eSKen Wang if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
960df70502eSKen Wang SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK | SRBM_STATUS__VMC_BUSY_MASK))
961df70502eSKen Wang return false;
962df70502eSKen Wang
963df70502eSKen Wang return true;
964df70502eSKen Wang }
965df70502eSKen Wang
gmc_v6_0_wait_for_idle(void * handle)966df70502eSKen Wang static int gmc_v6_0_wait_for_idle(void *handle)
967df70502eSKen Wang {
9680cfc1d68SSrinivasan Shanmugam unsigned int i;
969df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
970df70502eSKen Wang
971df70502eSKen Wang for (i = 0; i < adev->usec_timeout; i++) {
9729c2e1ae3STom St Denis if (gmc_v6_0_is_idle(handle))
973df70502eSKen Wang return 0;
974df70502eSKen Wang udelay(1);
975df70502eSKen Wang }
976df70502eSKen Wang return -ETIMEDOUT;
977df70502eSKen Wang
978df70502eSKen Wang }
979df70502eSKen Wang
gmc_v6_0_soft_reset(void * handle)980df70502eSKen Wang static int gmc_v6_0_soft_reset(void *handle)
981df70502eSKen Wang {
982df70502eSKen Wang struct amdgpu_device *adev = (struct amdgpu_device *)handle;
983df70502eSKen Wang u32 srbm_soft_reset = 0;
98472518269STom St Denis u32 tmp = RREG32(mmSRBM_STATUS);
985df70502eSKen Wang
986df70502eSKen Wang if (tmp & SRBM_STATUS__VMC_BUSY_MASK)
987df70502eSKen Wang srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
98872518269STom St Denis SRBM_SOFT_RESET, SOFT_RESET_VMC, 1);
989df70502eSKen Wang
990df70502eSKen Wang if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
991df70502eSKen Wang SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) {
992df70502eSKen Wang if (!(adev->flags & AMD_IS_APU))
993df70502eSKen Wang srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset,
99472518269STom St Denis SRBM_SOFT_RESET, SOFT_RESET_MC, 1);
995df70502eSKen Wang }
996df70502eSKen Wang
997df70502eSKen Wang if (srbm_soft_reset) {
998e4f6b39eSAlex Deucher gmc_v6_0_mc_stop(adev);
9990cfc1d68SSrinivasan Shanmugam if (gmc_v6_0_wait_for_idle(adev))
1000df70502eSKen Wang dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
1001df70502eSKen Wang
100272518269STom St Denis tmp = RREG32(mmSRBM_SOFT_RESET);
1003df70502eSKen Wang tmp |= srbm_soft_reset;
1004df70502eSKen Wang dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
100572518269STom St Denis WREG32(mmSRBM_SOFT_RESET, tmp);
100672518269STom St Denis tmp = RREG32(mmSRBM_SOFT_RESET);
1007df70502eSKen Wang
1008df70502eSKen Wang udelay(50);
1009df70502eSKen Wang
1010df70502eSKen Wang tmp &= ~srbm_soft_reset;
101172518269STom St Denis WREG32(mmSRBM_SOFT_RESET, tmp);
101272518269STom St Denis tmp = RREG32(mmSRBM_SOFT_RESET);
1013df70502eSKen Wang
1014df70502eSKen Wang udelay(50);
1015df70502eSKen Wang
1016e4f6b39eSAlex Deucher gmc_v6_0_mc_resume(adev);
1017df70502eSKen Wang udelay(50);
1018df70502eSKen Wang }
1019df70502eSKen Wang
1020df70502eSKen Wang return 0;
1021df70502eSKen Wang }
1022df70502eSKen Wang
gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device * adev,struct amdgpu_irq_src * src,unsigned int type,enum amdgpu_interrupt_state state)1023df70502eSKen Wang static int gmc_v6_0_vm_fault_interrupt_state(struct amdgpu_device *adev,
1024df70502eSKen Wang struct amdgpu_irq_src *src,
10250cfc1d68SSrinivasan Shanmugam unsigned int type,
1026df70502eSKen Wang enum amdgpu_interrupt_state state)
1027df70502eSKen Wang {
1028df70502eSKen Wang u32 tmp;
1029df70502eSKen Wang u32 bits = (VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1030df70502eSKen Wang VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1031df70502eSKen Wang VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1032df70502eSKen Wang VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1033df70502eSKen Wang VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
1034df70502eSKen Wang VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK);
1035df70502eSKen Wang
1036df70502eSKen Wang switch (state) {
1037df70502eSKen Wang case AMDGPU_IRQ_STATE_DISABLE:
103872518269STom St Denis tmp = RREG32(mmVM_CONTEXT0_CNTL);
1039df70502eSKen Wang tmp &= ~bits;
104072518269STom St Denis WREG32(mmVM_CONTEXT0_CNTL, tmp);
104172518269STom St Denis tmp = RREG32(mmVM_CONTEXT1_CNTL);
1042df70502eSKen Wang tmp &= ~bits;
104372518269STom St Denis WREG32(mmVM_CONTEXT1_CNTL, tmp);
1044df70502eSKen Wang break;
1045df70502eSKen Wang case AMDGPU_IRQ_STATE_ENABLE:
104672518269STom St Denis tmp = RREG32(mmVM_CONTEXT0_CNTL);
1047df70502eSKen Wang tmp |= bits;
104872518269STom St Denis WREG32(mmVM_CONTEXT0_CNTL, tmp);
104972518269STom St Denis tmp = RREG32(mmVM_CONTEXT1_CNTL);
1050df70502eSKen Wang tmp |= bits;
105172518269STom St Denis WREG32(mmVM_CONTEXT1_CNTL, tmp);
1052df70502eSKen Wang break;
1053df70502eSKen Wang default:
1054df70502eSKen Wang break;
1055df70502eSKen Wang }
1056df70502eSKen Wang
1057df70502eSKen Wang return 0;
1058df70502eSKen Wang }
1059df70502eSKen Wang
gmc_v6_0_process_interrupt(struct amdgpu_device * adev,struct amdgpu_irq_src * source,struct amdgpu_iv_entry * entry)1060df70502eSKen Wang static int gmc_v6_0_process_interrupt(struct amdgpu_device *adev,
1061df70502eSKen Wang struct amdgpu_irq_src *source,
1062df70502eSKen Wang struct amdgpu_iv_entry *entry)
1063df70502eSKen Wang {
1064df70502eSKen Wang u32 addr, status;
1065df70502eSKen Wang
106672518269STom St Denis addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
106772518269STom St Denis status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
106872518269STom St Denis WREG32_P(mmVM_CONTEXT1_CNTL2, 1, ~1);
1069df70502eSKen Wang
1070df70502eSKen Wang if (!addr && !status)
1071df70502eSKen Wang return 0;
1072df70502eSKen Wang
1073df70502eSKen Wang if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_FIRST)
1074df70502eSKen Wang gmc_v6_0_set_fault_enable_default(adev, false);
1075df70502eSKen Wang
107601615881SEdward O'Callaghan if (printk_ratelimit()) {
1077df70502eSKen Wang dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
10787ccf5aa8SAlex Deucher entry->src_id, entry->src_data[0]);
1079df70502eSKen Wang dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
1080df70502eSKen Wang addr);
1081df70502eSKen Wang dev_err(adev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
1082df70502eSKen Wang status);
1083df70502eSKen Wang gmc_v6_0_vm_decode_fault(adev, status, addr, 0);
108401615881SEdward O'Callaghan }
1085df70502eSKen Wang
1086df70502eSKen Wang return 0;
1087df70502eSKen Wang }
1088df70502eSKen Wang
gmc_v6_0_set_clockgating_state(void * handle,enum amd_clockgating_state state)1089df70502eSKen Wang static int gmc_v6_0_set_clockgating_state(void *handle,
1090df70502eSKen Wang enum amd_clockgating_state state)
1091df70502eSKen Wang {
1092df70502eSKen Wang return 0;
1093df70502eSKen Wang }
1094df70502eSKen Wang
gmc_v6_0_set_powergating_state(void * handle,enum amd_powergating_state state)1095df70502eSKen Wang static int gmc_v6_0_set_powergating_state(void *handle,
1096df70502eSKen Wang enum amd_powergating_state state)
1097df70502eSKen Wang {
1098df70502eSKen Wang return 0;
1099df70502eSKen Wang }
1100df70502eSKen Wang
1101a1255107SAlex Deucher static const struct amd_ip_funcs gmc_v6_0_ip_funcs = {
1102df70502eSKen Wang .name = "gmc_v6_0",
1103df70502eSKen Wang .early_init = gmc_v6_0_early_init,
1104df70502eSKen Wang .late_init = gmc_v6_0_late_init,
1105df70502eSKen Wang .sw_init = gmc_v6_0_sw_init,
1106df70502eSKen Wang .sw_fini = gmc_v6_0_sw_fini,
1107df70502eSKen Wang .hw_init = gmc_v6_0_hw_init,
1108df70502eSKen Wang .hw_fini = gmc_v6_0_hw_fini,
1109df70502eSKen Wang .suspend = gmc_v6_0_suspend,
1110df70502eSKen Wang .resume = gmc_v6_0_resume,
1111df70502eSKen Wang .is_idle = gmc_v6_0_is_idle,
1112df70502eSKen Wang .wait_for_idle = gmc_v6_0_wait_for_idle,
1113df70502eSKen Wang .soft_reset = gmc_v6_0_soft_reset,
1114df70502eSKen Wang .set_clockgating_state = gmc_v6_0_set_clockgating_state,
1115df70502eSKen Wang .set_powergating_state = gmc_v6_0_set_powergating_state,
1116df70502eSKen Wang };
1117df70502eSKen Wang
1118132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v6_0_gmc_funcs = {
1119132f34e4SChristian König .flush_gpu_tlb = gmc_v6_0_flush_gpu_tlb,
11204fef88bdSChristian König .emit_flush_gpu_tlb = gmc_v6_0_emit_flush_gpu_tlb,
1121f7c35abeSChristian König .set_prt = gmc_v6_0_set_prt,
1122b1166325SChristian König .get_vm_pde = gmc_v6_0_get_vm_pde,
1123cbfae36cSChristian König .get_vm_pte = gmc_v6_0_get_vm_pte,
1124422fe8d2SAlex Deucher .get_vbios_fb_size = gmc_v6_0_get_vbios_fb_size,
1125df70502eSKen Wang };
1126df70502eSKen Wang
1127df70502eSKen Wang static const struct amdgpu_irq_src_funcs gmc_v6_0_irq_funcs = {
1128df70502eSKen Wang .set = gmc_v6_0_vm_fault_interrupt_state,
1129df70502eSKen Wang .process = gmc_v6_0_process_interrupt,
1130df70502eSKen Wang };
1131df70502eSKen Wang
gmc_v6_0_set_gmc_funcs(struct amdgpu_device * adev)1132132f34e4SChristian König static void gmc_v6_0_set_gmc_funcs(struct amdgpu_device *adev)
1133df70502eSKen Wang {
1134132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v6_0_gmc_funcs;
1135df70502eSKen Wang }
1136df70502eSKen Wang
gmc_v6_0_set_irq_funcs(struct amdgpu_device * adev)1137df70502eSKen Wang static void gmc_v6_0_set_irq_funcs(struct amdgpu_device *adev)
1138df70502eSKen Wang {
1139770d13b1SChristian König adev->gmc.vm_fault.num_types = 1;
1140770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v6_0_irq_funcs;
1141df70502eSKen Wang }
1142df70502eSKen Wang
11430cfc1d68SSrinivasan Shanmugam const struct amdgpu_ip_block_version gmc_v6_0_ip_block = {
1144a1255107SAlex Deucher .type = AMD_IP_BLOCK_TYPE_GMC,
1145a1255107SAlex Deucher .major = 6,
1146a1255107SAlex Deucher .minor = 0,
1147a1255107SAlex Deucher .rev = 0,
1148a1255107SAlex Deucher .funcs = &gmc_v6_0_ip_funcs,
1149a1255107SAlex Deucher };
1150