1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 #include <linux/firmware.h> 24 #include <drm/drm_cache.h> 25 #include "amdgpu.h" 26 #include "gmc_v9_0.h" 27 #include "amdgpu_atomfirmware.h" 28 29 #include "hdp/hdp_4_0_offset.h" 30 #include "hdp/hdp_4_0_sh_mask.h" 31 #include "gc/gc_9_0_sh_mask.h" 32 #include "dce/dce_12_0_offset.h" 33 #include "dce/dce_12_0_sh_mask.h" 34 #include "vega10_enum.h" 35 #include "mmhub/mmhub_1_0_offset.h" 36 #include "athub/athub_1_0_offset.h" 37 #include "oss/osssys_4_0_offset.h" 38 39 #include "soc15.h" 40 #include "soc15_common.h" 41 #include "umc/umc_6_0_sh_mask.h" 42 43 #include "gfxhub_v1_0.h" 44 #include "mmhub_v1_0.h" 45 46 /* add these here since we already include dce12 headers and these are for DCN */ 47 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 48 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 49 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 50 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 51 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 52 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 53 54 /* XXX Move this macro to VEGA10 header file, which is like vid.h for VI.*/ 55 #define AMDGPU_NUM_OF_VMIDS 8 56 57 static const u32 golden_settings_vega10_hdp[] = 58 { 59 0xf64, 0x0fffffff, 0x00000000, 60 0xf65, 0x0fffffff, 0x00000000, 61 0xf66, 0x0fffffff, 0x00000000, 62 0xf67, 0x0fffffff, 0x00000000, 63 0xf68, 0x0fffffff, 0x00000000, 64 0xf6a, 0x0fffffff, 0x00000000, 65 0xf6b, 0x0fffffff, 0x00000000, 66 0xf6c, 0x0fffffff, 0x00000000, 67 0xf6d, 0x0fffffff, 0x00000000, 68 0xf6e, 0x0fffffff, 0x00000000, 69 }; 70 71 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 72 { 73 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 74 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 75 }; 76 77 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 78 { 79 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 80 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 81 }; 82 83 /* Ecc related register addresses, (BASE + reg offset) */ 84 /* Universal Memory Controller caps (may be fused). */ 85 /* UMCCH:UmcLocalCap */ 86 #define UMCLOCALCAPS_ADDR0 (0x00014306 + 0x00000000) 87 #define UMCLOCALCAPS_ADDR1 (0x00014306 + 0x00000800) 88 #define UMCLOCALCAPS_ADDR2 (0x00014306 + 0x00001000) 89 #define UMCLOCALCAPS_ADDR3 (0x00014306 + 0x00001800) 90 #define UMCLOCALCAPS_ADDR4 (0x00054306 + 0x00000000) 91 #define UMCLOCALCAPS_ADDR5 (0x00054306 + 0x00000800) 92 #define UMCLOCALCAPS_ADDR6 (0x00054306 + 0x00001000) 93 #define UMCLOCALCAPS_ADDR7 (0x00054306 + 0x00001800) 94 #define UMCLOCALCAPS_ADDR8 (0x00094306 + 0x00000000) 95 #define UMCLOCALCAPS_ADDR9 (0x00094306 + 0x00000800) 96 #define UMCLOCALCAPS_ADDR10 (0x00094306 + 0x00001000) 97 #define UMCLOCALCAPS_ADDR11 (0x00094306 + 0x00001800) 98 #define UMCLOCALCAPS_ADDR12 (0x000d4306 + 0x00000000) 99 #define UMCLOCALCAPS_ADDR13 (0x000d4306 + 0x00000800) 100 #define UMCLOCALCAPS_ADDR14 (0x000d4306 + 0x00001000) 101 #define UMCLOCALCAPS_ADDR15 (0x000d4306 + 0x00001800) 102 103 /* Universal Memory Controller Channel config. */ 104 /* UMCCH:UMC_CONFIG */ 105 #define UMCCH_UMC_CONFIG_ADDR0 (0x00014040 + 0x00000000) 106 #define UMCCH_UMC_CONFIG_ADDR1 (0x00014040 + 0x00000800) 107 #define UMCCH_UMC_CONFIG_ADDR2 (0x00014040 + 0x00001000) 108 #define UMCCH_UMC_CONFIG_ADDR3 (0x00014040 + 0x00001800) 109 #define UMCCH_UMC_CONFIG_ADDR4 (0x00054040 + 0x00000000) 110 #define UMCCH_UMC_CONFIG_ADDR5 (0x00054040 + 0x00000800) 111 #define UMCCH_UMC_CONFIG_ADDR6 (0x00054040 + 0x00001000) 112 #define UMCCH_UMC_CONFIG_ADDR7 (0x00054040 + 0x00001800) 113 #define UMCCH_UMC_CONFIG_ADDR8 (0x00094040 + 0x00000000) 114 #define UMCCH_UMC_CONFIG_ADDR9 (0x00094040 + 0x00000800) 115 #define UMCCH_UMC_CONFIG_ADDR10 (0x00094040 + 0x00001000) 116 #define UMCCH_UMC_CONFIG_ADDR11 (0x00094040 + 0x00001800) 117 #define UMCCH_UMC_CONFIG_ADDR12 (0x000d4040 + 0x00000000) 118 #define UMCCH_UMC_CONFIG_ADDR13 (0x000d4040 + 0x00000800) 119 #define UMCCH_UMC_CONFIG_ADDR14 (0x000d4040 + 0x00001000) 120 #define UMCCH_UMC_CONFIG_ADDR15 (0x000d4040 + 0x00001800) 121 122 /* Universal Memory Controller Channel Ecc config. */ 123 /* UMCCH:EccCtrl */ 124 #define UMCCH_ECCCTRL_ADDR0 (0x00014053 + 0x00000000) 125 #define UMCCH_ECCCTRL_ADDR1 (0x00014053 + 0x00000800) 126 #define UMCCH_ECCCTRL_ADDR2 (0x00014053 + 0x00001000) 127 #define UMCCH_ECCCTRL_ADDR3 (0x00014053 + 0x00001800) 128 #define UMCCH_ECCCTRL_ADDR4 (0x00054053 + 0x00000000) 129 #define UMCCH_ECCCTRL_ADDR5 (0x00054053 + 0x00000800) 130 #define UMCCH_ECCCTRL_ADDR6 (0x00054053 + 0x00001000) 131 #define UMCCH_ECCCTRL_ADDR7 (0x00054053 + 0x00001800) 132 #define UMCCH_ECCCTRL_ADDR8 (0x00094053 + 0x00000000) 133 #define UMCCH_ECCCTRL_ADDR9 (0x00094053 + 0x00000800) 134 #define UMCCH_ECCCTRL_ADDR10 (0x00094053 + 0x00001000) 135 #define UMCCH_ECCCTRL_ADDR11 (0x00094053 + 0x00001800) 136 #define UMCCH_ECCCTRL_ADDR12 (0x000d4053 + 0x00000000) 137 #define UMCCH_ECCCTRL_ADDR13 (0x000d4053 + 0x00000800) 138 #define UMCCH_ECCCTRL_ADDR14 (0x000d4053 + 0x00001000) 139 #define UMCCH_ECCCTRL_ADDR15 (0x000d4053 + 0x00001800) 140 141 static const uint32_t ecc_umclocalcap_addrs[] = { 142 UMCLOCALCAPS_ADDR0, 143 UMCLOCALCAPS_ADDR1, 144 UMCLOCALCAPS_ADDR2, 145 UMCLOCALCAPS_ADDR3, 146 UMCLOCALCAPS_ADDR4, 147 UMCLOCALCAPS_ADDR5, 148 UMCLOCALCAPS_ADDR6, 149 UMCLOCALCAPS_ADDR7, 150 UMCLOCALCAPS_ADDR8, 151 UMCLOCALCAPS_ADDR9, 152 UMCLOCALCAPS_ADDR10, 153 UMCLOCALCAPS_ADDR11, 154 UMCLOCALCAPS_ADDR12, 155 UMCLOCALCAPS_ADDR13, 156 UMCLOCALCAPS_ADDR14, 157 UMCLOCALCAPS_ADDR15, 158 }; 159 160 static const uint32_t ecc_umcch_umc_config_addrs[] = { 161 UMCCH_UMC_CONFIG_ADDR0, 162 UMCCH_UMC_CONFIG_ADDR1, 163 UMCCH_UMC_CONFIG_ADDR2, 164 UMCCH_UMC_CONFIG_ADDR3, 165 UMCCH_UMC_CONFIG_ADDR4, 166 UMCCH_UMC_CONFIG_ADDR5, 167 UMCCH_UMC_CONFIG_ADDR6, 168 UMCCH_UMC_CONFIG_ADDR7, 169 UMCCH_UMC_CONFIG_ADDR8, 170 UMCCH_UMC_CONFIG_ADDR9, 171 UMCCH_UMC_CONFIG_ADDR10, 172 UMCCH_UMC_CONFIG_ADDR11, 173 UMCCH_UMC_CONFIG_ADDR12, 174 UMCCH_UMC_CONFIG_ADDR13, 175 UMCCH_UMC_CONFIG_ADDR14, 176 UMCCH_UMC_CONFIG_ADDR15, 177 }; 178 179 static const uint32_t ecc_umcch_eccctrl_addrs[] = { 180 UMCCH_ECCCTRL_ADDR0, 181 UMCCH_ECCCTRL_ADDR1, 182 UMCCH_ECCCTRL_ADDR2, 183 UMCCH_ECCCTRL_ADDR3, 184 UMCCH_ECCCTRL_ADDR4, 185 UMCCH_ECCCTRL_ADDR5, 186 UMCCH_ECCCTRL_ADDR6, 187 UMCCH_ECCCTRL_ADDR7, 188 UMCCH_ECCCTRL_ADDR8, 189 UMCCH_ECCCTRL_ADDR9, 190 UMCCH_ECCCTRL_ADDR10, 191 UMCCH_ECCCTRL_ADDR11, 192 UMCCH_ECCCTRL_ADDR12, 193 UMCCH_ECCCTRL_ADDR13, 194 UMCCH_ECCCTRL_ADDR14, 195 UMCCH_ECCCTRL_ADDR15, 196 }; 197 198 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 199 struct amdgpu_irq_src *src, 200 unsigned type, 201 enum amdgpu_interrupt_state state) 202 { 203 struct amdgpu_vmhub *hub; 204 u32 tmp, reg, bits, i, j; 205 206 bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 207 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 208 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 209 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 210 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 211 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 212 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 213 214 switch (state) { 215 case AMDGPU_IRQ_STATE_DISABLE: 216 for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 217 hub = &adev->vmhub[j]; 218 for (i = 0; i < 16; i++) { 219 reg = hub->vm_context0_cntl + i; 220 tmp = RREG32(reg); 221 tmp &= ~bits; 222 WREG32(reg, tmp); 223 } 224 } 225 break; 226 case AMDGPU_IRQ_STATE_ENABLE: 227 for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 228 hub = &adev->vmhub[j]; 229 for (i = 0; i < 16; i++) { 230 reg = hub->vm_context0_cntl + i; 231 tmp = RREG32(reg); 232 tmp |= bits; 233 WREG32(reg, tmp); 234 } 235 } 236 default: 237 break; 238 } 239 240 return 0; 241 } 242 243 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 244 struct amdgpu_irq_src *source, 245 struct amdgpu_iv_entry *entry) 246 { 247 struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src]; 248 uint32_t status = 0; 249 u64 addr; 250 251 addr = (u64)entry->src_data[0] << 12; 252 addr |= ((u64)entry->src_data[1] & 0xf) << 44; 253 254 if (!amdgpu_sriov_vf(adev)) { 255 status = RREG32(hub->vm_l2_pro_fault_status); 256 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 257 } 258 259 if (printk_ratelimit()) { 260 struct amdgpu_task_info task_info = { 0 }; 261 262 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 263 264 dev_err(adev->dev, 265 "[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d\n)\n", 266 entry->vmid_src ? "mmhub" : "gfxhub", 267 entry->src_id, entry->ring_id, entry->vmid, 268 entry->pasid, task_info.process_name, task_info.tgid, 269 task_info.task_name, task_info.pid); 270 dev_err(adev->dev, " at page 0x%016llx from %d\n", 271 addr, entry->client_id); 272 if (!amdgpu_sriov_vf(adev)) 273 dev_err(adev->dev, 274 "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 275 status); 276 } 277 278 return 0; 279 } 280 281 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 282 .set = gmc_v9_0_vm_fault_interrupt_state, 283 .process = gmc_v9_0_process_interrupt, 284 }; 285 286 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 287 { 288 adev->gmc.vm_fault.num_types = 1; 289 adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 290 } 291 292 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid) 293 { 294 u32 req = 0; 295 296 /* invalidate using legacy mode on vmid*/ 297 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 298 PER_VMID_INVALIDATE_REQ, 1 << vmid); 299 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0); 300 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 301 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 302 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 303 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 304 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 305 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 306 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 307 308 return req; 309 } 310 311 /* 312 * GART 313 * VMID 0 is the physical GPU addresses as used by the kernel. 314 * VMIDs 1-15 are used for userspace clients and are handled 315 * by the amdgpu vm/hsa code. 316 */ 317 318 /** 319 * gmc_v9_0_flush_gpu_tlb - gart tlb flush callback 320 * 321 * @adev: amdgpu_device pointer 322 * @vmid: vm instance to flush 323 * 324 * Flush the TLB for the requested page table. 325 */ 326 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, 327 uint32_t vmid) 328 { 329 /* Use register 17 for GART */ 330 const unsigned eng = 17; 331 unsigned i, j; 332 333 spin_lock(&adev->gmc.invalidate_lock); 334 335 for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 336 struct amdgpu_vmhub *hub = &adev->vmhub[i]; 337 u32 tmp = gmc_v9_0_get_invalidate_req(vmid); 338 339 WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp); 340 341 /* Busy wait for ACK.*/ 342 for (j = 0; j < 100; j++) { 343 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng); 344 tmp &= 1 << vmid; 345 if (tmp) 346 break; 347 cpu_relax(); 348 } 349 if (j < 100) 350 continue; 351 352 /* Wait for ACK with a delay.*/ 353 for (j = 0; j < adev->usec_timeout; j++) { 354 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng); 355 tmp &= 1 << vmid; 356 if (tmp) 357 break; 358 udelay(1); 359 } 360 if (j < adev->usec_timeout) 361 continue; 362 363 DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 364 } 365 366 spin_unlock(&adev->gmc.invalidate_lock); 367 } 368 369 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 370 unsigned vmid, uint64_t pd_addr) 371 { 372 struct amdgpu_device *adev = ring->adev; 373 struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 374 uint32_t req = gmc_v9_0_get_invalidate_req(vmid); 375 uint64_t flags = AMDGPU_PTE_VALID; 376 unsigned eng = ring->vm_inv_eng; 377 378 amdgpu_gmc_get_vm_pde(adev, -1, &pd_addr, &flags); 379 pd_addr |= flags; 380 381 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid), 382 lower_32_bits(pd_addr)); 383 384 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid), 385 upper_32_bits(pd_addr)); 386 387 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng, 388 hub->vm_inv_eng0_ack + eng, 389 req, 1 << vmid); 390 391 return pd_addr; 392 } 393 394 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 395 unsigned pasid) 396 { 397 struct amdgpu_device *adev = ring->adev; 398 uint32_t reg; 399 400 if (ring->funcs->vmhub == AMDGPU_GFXHUB) 401 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 402 else 403 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 404 405 amdgpu_ring_emit_wreg(ring, reg, pasid); 406 } 407 408 /** 409 * gmc_v9_0_set_pte_pde - update the page tables using MMIO 410 * 411 * @adev: amdgpu_device pointer 412 * @cpu_pt_addr: cpu address of the page table 413 * @gpu_page_idx: entry in the page table to update 414 * @addr: dst addr to write into pte/pde 415 * @flags: access flags 416 * 417 * Update the page tables using the CPU. 418 */ 419 static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, 420 uint32_t gpu_page_idx, uint64_t addr, 421 uint64_t flags) 422 { 423 void __iomem *ptr = (void *)cpu_pt_addr; 424 uint64_t value; 425 426 /* 427 * PTE format on VEGA 10: 428 * 63:59 reserved 429 * 58:57 mtype 430 * 56 F 431 * 55 L 432 * 54 P 433 * 53 SW 434 * 52 T 435 * 50:48 reserved 436 * 47:12 4k physical page base address 437 * 11:7 fragment 438 * 6 write 439 * 5 read 440 * 4 exe 441 * 3 Z 442 * 2 snooped 443 * 1 system 444 * 0 valid 445 * 446 * PDE format on VEGA 10: 447 * 63:59 block fragment size 448 * 58:55 reserved 449 * 54 P 450 * 53:48 reserved 451 * 47:6 physical base address of PD or PTE 452 * 5:3 reserved 453 * 2 C 454 * 1 system 455 * 0 valid 456 */ 457 458 /* 459 * The following is for PTE only. GART does not have PDEs. 460 */ 461 value = addr & 0x0000FFFFFFFFF000ULL; 462 value |= flags; 463 writeq(value, ptr + (gpu_page_idx * 8)); 464 return 0; 465 } 466 467 static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev, 468 uint32_t flags) 469 470 { 471 uint64_t pte_flag = 0; 472 473 if (flags & AMDGPU_VM_PAGE_EXECUTABLE) 474 pte_flag |= AMDGPU_PTE_EXECUTABLE; 475 if (flags & AMDGPU_VM_PAGE_READABLE) 476 pte_flag |= AMDGPU_PTE_READABLE; 477 if (flags & AMDGPU_VM_PAGE_WRITEABLE) 478 pte_flag |= AMDGPU_PTE_WRITEABLE; 479 480 switch (flags & AMDGPU_VM_MTYPE_MASK) { 481 case AMDGPU_VM_MTYPE_DEFAULT: 482 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 483 break; 484 case AMDGPU_VM_MTYPE_NC: 485 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 486 break; 487 case AMDGPU_VM_MTYPE_WC: 488 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC); 489 break; 490 case AMDGPU_VM_MTYPE_CC: 491 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC); 492 break; 493 case AMDGPU_VM_MTYPE_UC: 494 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC); 495 break; 496 default: 497 pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 498 break; 499 } 500 501 if (flags & AMDGPU_VM_PAGE_PRT) 502 pte_flag |= AMDGPU_PTE_PRT; 503 504 return pte_flag; 505 } 506 507 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 508 uint64_t *addr, uint64_t *flags) 509 { 510 if (!(*flags & AMDGPU_PDE_PTE)) 511 *addr = adev->vm_manager.vram_base_offset + *addr - 512 adev->gmc.vram_start; 513 BUG_ON(*addr & 0xFFFF00000000003FULL); 514 515 if (!adev->gmc.translate_further) 516 return; 517 518 if (level == AMDGPU_VM_PDB1) { 519 /* Set the block fragment size */ 520 if (!(*flags & AMDGPU_PDE_PTE)) 521 *flags |= AMDGPU_PDE_BFS(0x9); 522 523 } else if (level == AMDGPU_VM_PDB0) { 524 if (*flags & AMDGPU_PDE_PTE) 525 *flags &= ~AMDGPU_PDE_PTE; 526 else 527 *flags |= AMDGPU_PTE_TF; 528 } 529 } 530 531 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 532 .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 533 .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 534 .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 535 .set_pte_pde = gmc_v9_0_set_pte_pde, 536 .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags, 537 .get_vm_pde = gmc_v9_0_get_vm_pde 538 }; 539 540 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 541 { 542 if (adev->gmc.gmc_funcs == NULL) 543 adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 544 } 545 546 static int gmc_v9_0_early_init(void *handle) 547 { 548 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 549 550 gmc_v9_0_set_gmc_funcs(adev); 551 gmc_v9_0_set_irq_funcs(adev); 552 553 adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 554 adev->gmc.shared_aperture_end = 555 adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 556 adev->gmc.private_aperture_start = 0x1000000000000000ULL; 557 adev->gmc.private_aperture_end = 558 adev->gmc.private_aperture_start + (4ULL << 30) - 1; 559 560 return 0; 561 } 562 563 static int gmc_v9_0_ecc_available(struct amdgpu_device *adev) 564 { 565 uint32_t reg_val; 566 uint32_t reg_addr; 567 uint32_t field_val; 568 size_t i; 569 uint32_t fv2; 570 size_t lost_sheep; 571 572 DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n"); 573 574 lost_sheep = 0; 575 for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) { 576 reg_addr = ecc_umclocalcap_addrs[i]; 577 DRM_DEBUG("ecc: " 578 "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n", 579 i, reg_addr); 580 reg_val = RREG32(reg_addr); 581 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap, 582 EccDis); 583 DRM_DEBUG("ecc: " 584 "reg_val: 0x%08x, " 585 "EccDis: 0x%08x, ", 586 reg_val, field_val); 587 if (field_val) { 588 DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n"); 589 ++lost_sheep; 590 } 591 } 592 593 for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) { 594 reg_addr = ecc_umcch_umc_config_addrs[i]; 595 DRM_DEBUG("ecc: " 596 "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x", 597 i, reg_addr); 598 reg_val = RREG32(reg_addr); 599 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG, 600 DramReady); 601 DRM_DEBUG("ecc: " 602 "reg_val: 0x%08x, " 603 "DramReady: 0x%08x\n", 604 reg_val, field_val); 605 606 if (!field_val) { 607 DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n"); 608 ++lost_sheep; 609 } 610 } 611 612 for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) { 613 reg_addr = ecc_umcch_eccctrl_addrs[i]; 614 DRM_DEBUG("ecc: " 615 "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ", 616 i, reg_addr); 617 reg_val = RREG32(reg_addr); 618 field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl, 619 WrEccEn); 620 fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl, 621 RdEccEn); 622 DRM_DEBUG("ecc: " 623 "reg_val: 0x%08x, " 624 "WrEccEn: 0x%08x, " 625 "RdEccEn: 0x%08x\n", 626 reg_val, field_val, fv2); 627 628 if (!field_val) { 629 DRM_DEBUG("ecc: WrEccEn is not set\n"); 630 ++lost_sheep; 631 } 632 if (!fv2) { 633 DRM_DEBUG("ecc: RdEccEn is not set\n"); 634 ++lost_sheep; 635 } 636 } 637 638 DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep); 639 return lost_sheep == 0; 640 } 641 642 static int gmc_v9_0_late_init(void *handle) 643 { 644 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 645 /* 646 * The latest engine allocation on gfx9 is: 647 * Engine 0, 1: idle 648 * Engine 2, 3: firmware 649 * Engine 4~13: amdgpu ring, subject to change when ring number changes 650 * Engine 14~15: idle 651 * Engine 16: kfd tlb invalidation 652 * Engine 17: Gart flushes 653 */ 654 unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 }; 655 unsigned i; 656 int r; 657 658 /* 659 * TODO - Uncomment once GART corruption issue is fixed. 660 */ 661 /* amdgpu_bo_late_init(adev); */ 662 663 for(i = 0; i < adev->num_rings; ++i) { 664 struct amdgpu_ring *ring = adev->rings[i]; 665 unsigned vmhub = ring->funcs->vmhub; 666 667 ring->vm_inv_eng = vm_inv_eng[vmhub]++; 668 dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n", 669 ring->idx, ring->name, ring->vm_inv_eng, 670 ring->funcs->vmhub); 671 } 672 673 /* Engine 16 is used for KFD and 17 for GART flushes */ 674 for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i) 675 BUG_ON(vm_inv_eng[i] > 16); 676 677 if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) { 678 r = gmc_v9_0_ecc_available(adev); 679 if (r == 1) { 680 DRM_INFO("ECC is active.\n"); 681 } else if (r == 0) { 682 DRM_INFO("ECC is not present.\n"); 683 adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false); 684 } else { 685 DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r); 686 return r; 687 } 688 } 689 690 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 691 } 692 693 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 694 struct amdgpu_gmc *mc) 695 { 696 u64 base = 0; 697 if (!amdgpu_sriov_vf(adev)) 698 base = mmhub_v1_0_get_fb_location(adev); 699 amdgpu_device_vram_location(adev, &adev->gmc, base); 700 amdgpu_device_gart_location(adev, mc); 701 /* base offset of vram pages */ 702 adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev); 703 } 704 705 /** 706 * gmc_v9_0_mc_init - initialize the memory controller driver params 707 * 708 * @adev: amdgpu_device pointer 709 * 710 * Look up the amount of vram, vram width, and decide how to place 711 * vram and gart within the GPU's physical address space. 712 * Returns 0 for success. 713 */ 714 static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 715 { 716 int chansize, numchan; 717 int r; 718 719 if (amdgpu_emu_mode != 1) 720 adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev); 721 if (!adev->gmc.vram_width) { 722 /* hbm memory channel size */ 723 if (adev->flags & AMD_IS_APU) 724 chansize = 64; 725 else 726 chansize = 128; 727 728 numchan = adev->df_funcs->get_hbm_channel_number(adev); 729 adev->gmc.vram_width = numchan * chansize; 730 } 731 732 /* size in MB on si */ 733 adev->gmc.mc_vram_size = 734 adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL; 735 adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 736 737 if (!(adev->flags & AMD_IS_APU)) { 738 r = amdgpu_device_resize_fb_bar(adev); 739 if (r) 740 return r; 741 } 742 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 743 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 744 745 #ifdef CONFIG_X86_64 746 if (adev->flags & AMD_IS_APU) { 747 adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev); 748 adev->gmc.aper_size = adev->gmc.real_vram_size; 749 } 750 #endif 751 /* In case the PCI BAR is larger than the actual amount of vram */ 752 adev->gmc.visible_vram_size = adev->gmc.aper_size; 753 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 754 adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 755 756 /* set the gart size */ 757 if (amdgpu_gart_size == -1) { 758 switch (adev->asic_type) { 759 case CHIP_VEGA10: /* all engines support GPUVM */ 760 case CHIP_VEGA12: /* all engines support GPUVM */ 761 case CHIP_VEGA20: 762 default: 763 adev->gmc.gart_size = 512ULL << 20; 764 break; 765 case CHIP_RAVEN: /* DCE SG support */ 766 adev->gmc.gart_size = 1024ULL << 20; 767 break; 768 } 769 } else { 770 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 771 } 772 773 gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 774 775 return 0; 776 } 777 778 static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 779 { 780 int r; 781 782 if (adev->gart.robj) { 783 WARN(1, "VEGA10 PCIE GART already initialized\n"); 784 return 0; 785 } 786 /* Initialize common gart structure */ 787 r = amdgpu_gart_init(adev); 788 if (r) 789 return r; 790 adev->gart.table_size = adev->gart.num_gpu_pages * 8; 791 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) | 792 AMDGPU_PTE_EXECUTABLE; 793 return amdgpu_gart_table_vram_alloc(adev); 794 } 795 796 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 797 { 798 #if 0 799 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 800 #endif 801 unsigned size; 802 803 /* 804 * TODO Remove once GART corruption is resolved 805 * Check related code in gmc_v9_0_sw_fini 806 * */ 807 size = 9 * 1024 * 1024; 808 809 #if 0 810 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 811 size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ 812 } else { 813 u32 viewport; 814 815 switch (adev->asic_type) { 816 case CHIP_RAVEN: 817 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 818 size = (REG_GET_FIELD(viewport, 819 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 820 REG_GET_FIELD(viewport, 821 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 822 4); 823 break; 824 case CHIP_VEGA10: 825 case CHIP_VEGA12: 826 default: 827 viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 828 size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 829 REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 830 4); 831 break; 832 } 833 } 834 /* return 0 if the pre-OS buffer uses up most of vram */ 835 if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 836 return 0; 837 838 #endif 839 return size; 840 } 841 842 static int gmc_v9_0_sw_init(void *handle) 843 { 844 int r; 845 int dma_bits; 846 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 847 848 gfxhub_v1_0_init(adev); 849 mmhub_v1_0_init(adev); 850 851 spin_lock_init(&adev->gmc.invalidate_lock); 852 853 adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev); 854 switch (adev->asic_type) { 855 case CHIP_RAVEN: 856 if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 857 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 858 } else { 859 /* vm_size is 128TB + 512GB for legacy 3-level page support */ 860 amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 861 adev->gmc.translate_further = 862 adev->vm_manager.num_level > 1; 863 } 864 break; 865 case CHIP_VEGA10: 866 case CHIP_VEGA12: 867 case CHIP_VEGA20: 868 /* 869 * To fulfill 4-level page support, 870 * vm size is 256TB (48bit), maximum size of Vega10, 871 * block size 512 (9bit) 872 */ 873 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 874 break; 875 default: 876 break; 877 } 878 879 /* This interrupt is VMC page fault.*/ 880 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, 0, 881 &adev->gmc.vm_fault); 882 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, 0, 883 &adev->gmc.vm_fault); 884 885 if (r) 886 return r; 887 888 /* Set the internal MC address mask 889 * This is the max address of the GPU's 890 * internal address space. 891 */ 892 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 893 894 /* set DMA mask + need_dma32 flags. 895 * PCIE - can handle 44-bits. 896 * IGP - can handle 44-bits 897 * PCI - dma32 for legacy pci gart, 44 bits on vega10 898 */ 899 adev->need_dma32 = false; 900 dma_bits = adev->need_dma32 ? 32 : 44; 901 r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 902 if (r) { 903 adev->need_dma32 = true; 904 dma_bits = 32; 905 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 906 } 907 r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 908 if (r) { 909 pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32)); 910 printk(KERN_WARNING "amdgpu: No coherent DMA available.\n"); 911 } 912 adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits); 913 914 r = gmc_v9_0_mc_init(adev); 915 if (r) 916 return r; 917 918 adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev); 919 920 /* Memory manager */ 921 r = amdgpu_bo_init(adev); 922 if (r) 923 return r; 924 925 r = gmc_v9_0_gart_init(adev); 926 if (r) 927 return r; 928 929 /* 930 * number of VMs 931 * VMID 0 is reserved for System 932 * amdgpu graphics/compute will use VMIDs 1-7 933 * amdkfd will use VMIDs 8-15 934 */ 935 adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 936 adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 937 938 amdgpu_vm_manager_init(adev); 939 940 return 0; 941 } 942 943 /** 944 * gmc_v9_0_gart_fini - vm fini callback 945 * 946 * @adev: amdgpu_device pointer 947 * 948 * Tears down the driver GART/VM setup (CIK). 949 */ 950 static void gmc_v9_0_gart_fini(struct amdgpu_device *adev) 951 { 952 amdgpu_gart_table_vram_free(adev); 953 amdgpu_gart_fini(adev); 954 } 955 956 static int gmc_v9_0_sw_fini(void *handle) 957 { 958 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 959 960 amdgpu_gem_force_release(adev); 961 amdgpu_vm_manager_fini(adev); 962 gmc_v9_0_gart_fini(adev); 963 964 /* 965 * TODO: 966 * Currently there is a bug where some memory client outside 967 * of the driver writes to first 8M of VRAM on S3 resume, 968 * this overrides GART which by default gets placed in first 8M and 969 * causes VM_FAULTS once GTT is accessed. 970 * Keep the stolen memory reservation until the while this is not solved. 971 * Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init 972 */ 973 amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); 974 975 amdgpu_bo_fini(adev); 976 977 return 0; 978 } 979 980 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 981 { 982 983 switch (adev->asic_type) { 984 case CHIP_VEGA10: 985 case CHIP_VEGA20: 986 soc15_program_register_sequence(adev, 987 golden_settings_mmhub_1_0_0, 988 ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 989 soc15_program_register_sequence(adev, 990 golden_settings_athub_1_0_0, 991 ARRAY_SIZE(golden_settings_athub_1_0_0)); 992 break; 993 case CHIP_VEGA12: 994 break; 995 case CHIP_RAVEN: 996 soc15_program_register_sequence(adev, 997 golden_settings_athub_1_0_0, 998 ARRAY_SIZE(golden_settings_athub_1_0_0)); 999 break; 1000 default: 1001 break; 1002 } 1003 } 1004 1005 /** 1006 * gmc_v9_0_gart_enable - gart enable 1007 * 1008 * @adev: amdgpu_device pointer 1009 */ 1010 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1011 { 1012 int r; 1013 bool value; 1014 u32 tmp; 1015 1016 amdgpu_device_program_register_sequence(adev, 1017 golden_settings_vega10_hdp, 1018 ARRAY_SIZE(golden_settings_vega10_hdp)); 1019 1020 if (adev->gart.robj == NULL) { 1021 dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1022 return -EINVAL; 1023 } 1024 r = amdgpu_gart_table_vram_pin(adev); 1025 if (r) 1026 return r; 1027 1028 switch (adev->asic_type) { 1029 case CHIP_RAVEN: 1030 mmhub_v1_0_initialize_power_gating(adev); 1031 mmhub_v1_0_update_power_gating(adev, true); 1032 break; 1033 default: 1034 break; 1035 } 1036 1037 r = gfxhub_v1_0_gart_enable(adev); 1038 if (r) 1039 return r; 1040 1041 r = mmhub_v1_0_gart_enable(adev); 1042 if (r) 1043 return r; 1044 1045 WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1046 1047 tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1048 WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1049 1050 /* After HDP is initialized, flush HDP.*/ 1051 adev->nbio_funcs->hdp_flush(adev, NULL); 1052 1053 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1054 value = false; 1055 else 1056 value = true; 1057 1058 gfxhub_v1_0_set_fault_enable_default(adev, value); 1059 mmhub_v1_0_set_fault_enable_default(adev, value); 1060 gmc_v9_0_flush_gpu_tlb(adev, 0); 1061 1062 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1063 (unsigned)(adev->gmc.gart_size >> 20), 1064 (unsigned long long)adev->gart.table_addr); 1065 adev->gart.ready = true; 1066 return 0; 1067 } 1068 1069 static int gmc_v9_0_hw_init(void *handle) 1070 { 1071 int r; 1072 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1073 1074 /* The sequence of these two function calls matters.*/ 1075 gmc_v9_0_init_golden_registers(adev); 1076 1077 if (adev->mode_info.num_crtc) { 1078 /* Lockout access through VGA aperture*/ 1079 WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1080 1081 /* disable VGA render */ 1082 WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1083 } 1084 1085 r = gmc_v9_0_gart_enable(adev); 1086 1087 return r; 1088 } 1089 1090 /** 1091 * gmc_v9_0_gart_disable - gart disable 1092 * 1093 * @adev: amdgpu_device pointer 1094 * 1095 * This disables all VM page table. 1096 */ 1097 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1098 { 1099 gfxhub_v1_0_gart_disable(adev); 1100 mmhub_v1_0_gart_disable(adev); 1101 amdgpu_gart_table_vram_unpin(adev); 1102 } 1103 1104 static int gmc_v9_0_hw_fini(void *handle) 1105 { 1106 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1107 1108 if (amdgpu_sriov_vf(adev)) { 1109 /* full access mode, so don't touch any GMC register */ 1110 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 1111 return 0; 1112 } 1113 1114 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1115 gmc_v9_0_gart_disable(adev); 1116 1117 return 0; 1118 } 1119 1120 static int gmc_v9_0_suspend(void *handle) 1121 { 1122 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1123 1124 return gmc_v9_0_hw_fini(adev); 1125 } 1126 1127 static int gmc_v9_0_resume(void *handle) 1128 { 1129 int r; 1130 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1131 1132 r = gmc_v9_0_hw_init(adev); 1133 if (r) 1134 return r; 1135 1136 amdgpu_vmid_reset_all(adev); 1137 1138 return 0; 1139 } 1140 1141 static bool gmc_v9_0_is_idle(void *handle) 1142 { 1143 /* MC is always ready in GMC v9.*/ 1144 return true; 1145 } 1146 1147 static int gmc_v9_0_wait_for_idle(void *handle) 1148 { 1149 /* There is no need to wait for MC idle in GMC v9.*/ 1150 return 0; 1151 } 1152 1153 static int gmc_v9_0_soft_reset(void *handle) 1154 { 1155 /* XXX for emulation.*/ 1156 return 0; 1157 } 1158 1159 static int gmc_v9_0_set_clockgating_state(void *handle, 1160 enum amd_clockgating_state state) 1161 { 1162 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1163 1164 return mmhub_v1_0_set_clockgating(adev, state); 1165 } 1166 1167 static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 1168 { 1169 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1170 1171 mmhub_v1_0_get_clockgating(adev, flags); 1172 } 1173 1174 static int gmc_v9_0_set_powergating_state(void *handle, 1175 enum amd_powergating_state state) 1176 { 1177 return 0; 1178 } 1179 1180 const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1181 .name = "gmc_v9_0", 1182 .early_init = gmc_v9_0_early_init, 1183 .late_init = gmc_v9_0_late_init, 1184 .sw_init = gmc_v9_0_sw_init, 1185 .sw_fini = gmc_v9_0_sw_fini, 1186 .hw_init = gmc_v9_0_hw_init, 1187 .hw_fini = gmc_v9_0_hw_fini, 1188 .suspend = gmc_v9_0_suspend, 1189 .resume = gmc_v9_0_resume, 1190 .is_idle = gmc_v9_0_is_idle, 1191 .wait_for_idle = gmc_v9_0_wait_for_idle, 1192 .soft_reset = gmc_v9_0_soft_reset, 1193 .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1194 .set_powergating_state = gmc_v9_0_set_powergating_state, 1195 .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1196 }; 1197 1198 const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1199 { 1200 .type = AMD_IP_BLOCK_TYPE_GMC, 1201 .major = 9, 1202 .minor = 0, 1203 .rev = 0, 1204 .funcs = &gmc_v9_0_ip_funcs, 1205 }; 1206