12279b4e5STianci.Yin /* 22279b4e5STianci.Yin * Copyright 2021 Advanced Micro Devices, Inc. 32279b4e5STianci.Yin * 42279b4e5STianci.Yin * Permission is hereby granted, free of charge, to any person obtaining a 52279b4e5STianci.Yin * copy of this software and associated documentation files (the "Software"), 62279b4e5STianci.Yin * to deal in the Software without restriction, including without limitation 72279b4e5STianci.Yin * the rights to use, copy, modify, merge, publish, distribute, sublicense, 82279b4e5STianci.Yin * and/or sell copies of the Software, and to permit persons to whom the 92279b4e5STianci.Yin * Software is furnished to do so, subject to the following conditions: 102279b4e5STianci.Yin * 112279b4e5STianci.Yin * The above copyright notice and this permission notice shall be included in 122279b4e5STianci.Yin * all copies or substantial portions of the Software. 132279b4e5STianci.Yin * 142279b4e5STianci.Yin * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 152279b4e5STianci.Yin * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 162279b4e5STianci.Yin * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 172279b4e5STianci.Yin * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 182279b4e5STianci.Yin * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 192279b4e5STianci.Yin * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 202279b4e5STianci.Yin * OTHER DEALINGS IN THE SOFTWARE. 212279b4e5STianci.Yin * 222279b4e5STianci.Yin */ 232279b4e5STianci.Yin 242279b4e5STianci.Yin #include "amdgpu.h" 252279b4e5STianci.Yin #include "gfxhub_v3_0.h" 262279b4e5STianci.Yin 272279b4e5STianci.Yin #include "gc/gc_11_0_0_offset.h" 282279b4e5STianci.Yin #include "gc/gc_11_0_0_sh_mask.h" 297c8e4a25SChengming Gui #include "gc/gc_11_0_0_default.h" 302279b4e5STianci.Yin #include "navi10_enum.h" 312279b4e5STianci.Yin #include "soc15_common.h" 322279b4e5STianci.Yin 332279b4e5STianci.Yin static const char *gfxhub_client_ids[] = { 342279b4e5STianci.Yin "CB/DB", 352279b4e5STianci.Yin "Reserved", 362279b4e5STianci.Yin "GE1", 372279b4e5STianci.Yin "GE2", 382279b4e5STianci.Yin "CPF", 392279b4e5STianci.Yin "CPC", 402279b4e5STianci.Yin "CPG", 412279b4e5STianci.Yin "RLC", 422279b4e5STianci.Yin "TCP", 432279b4e5STianci.Yin "SQC (inst)", 442279b4e5STianci.Yin "SQC (data)", 452279b4e5STianci.Yin "SQG", 462279b4e5STianci.Yin "Reserved", 472279b4e5STianci.Yin "SDMA0", 482279b4e5STianci.Yin "SDMA1", 492279b4e5STianci.Yin "GCR", 502279b4e5STianci.Yin "SDMA2", 512279b4e5STianci.Yin "SDMA3", 522279b4e5STianci.Yin }; 532279b4e5STianci.Yin 542279b4e5STianci.Yin static uint32_t gfxhub_v3_0_get_invalidate_req(unsigned int vmid, 552279b4e5STianci.Yin uint32_t flush_type) 562279b4e5STianci.Yin { 572279b4e5STianci.Yin u32 req = 0; 582279b4e5STianci.Yin 592279b4e5STianci.Yin /* invalidate using legacy mode on vmid*/ 602279b4e5STianci.Yin req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 612279b4e5STianci.Yin PER_VMID_INVALIDATE_REQ, 1 << vmid); 622279b4e5STianci.Yin req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 632279b4e5STianci.Yin req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 642279b4e5STianci.Yin req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 652279b4e5STianci.Yin req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 662279b4e5STianci.Yin req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 672279b4e5STianci.Yin req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 682279b4e5STianci.Yin req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 692279b4e5STianci.Yin CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 702279b4e5STianci.Yin 712279b4e5STianci.Yin return req; 722279b4e5STianci.Yin } 732279b4e5STianci.Yin 742279b4e5STianci.Yin static void 752279b4e5STianci.Yin gfxhub_v3_0_print_l2_protection_fault_status(struct amdgpu_device *adev, 762279b4e5STianci.Yin uint32_t status) 772279b4e5STianci.Yin { 782279b4e5STianci.Yin u32 cid = REG_GET_FIELD(status, 792279b4e5STianci.Yin GCVM_L2_PROTECTION_FAULT_STATUS, CID); 802279b4e5STianci.Yin 812279b4e5STianci.Yin dev_err(adev->dev, 822279b4e5STianci.Yin "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 832279b4e5STianci.Yin status); 842279b4e5STianci.Yin dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 852279b4e5STianci.Yin cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid], 862279b4e5STianci.Yin cid); 872279b4e5STianci.Yin dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 882279b4e5STianci.Yin REG_GET_FIELD(status, 892279b4e5STianci.Yin GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 902279b4e5STianci.Yin dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 912279b4e5STianci.Yin REG_GET_FIELD(status, 922279b4e5STianci.Yin GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 932279b4e5STianci.Yin dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 942279b4e5STianci.Yin REG_GET_FIELD(status, 952279b4e5STianci.Yin GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 962279b4e5STianci.Yin dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 972279b4e5STianci.Yin REG_GET_FIELD(status, 982279b4e5STianci.Yin GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 992279b4e5STianci.Yin dev_err(adev->dev, "\t RW: 0x%lx\n", 1002279b4e5STianci.Yin REG_GET_FIELD(status, 1012279b4e5STianci.Yin GCVM_L2_PROTECTION_FAULT_STATUS, RW)); 1022279b4e5STianci.Yin } 1032279b4e5STianci.Yin 1042279b4e5STianci.Yin static u64 gfxhub_v3_0_get_fb_location(struct amdgpu_device *adev) 1052279b4e5STianci.Yin { 1062279b4e5STianci.Yin u64 base = RREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_BASE); 1072279b4e5STianci.Yin 1082279b4e5STianci.Yin base &= GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK; 1092279b4e5STianci.Yin base <<= 24; 1102279b4e5STianci.Yin 1112279b4e5STianci.Yin return base; 1122279b4e5STianci.Yin } 1132279b4e5STianci.Yin 1142279b4e5STianci.Yin static u64 gfxhub_v3_0_get_mc_fb_offset(struct amdgpu_device *adev) 1152279b4e5STianci.Yin { 1162279b4e5STianci.Yin return (u64)RREG32_SOC15(GC, 0, regGCMC_VM_FB_OFFSET) << 24; 1172279b4e5STianci.Yin } 1182279b4e5STianci.Yin 1192279b4e5STianci.Yin static void gfxhub_v3_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, 1202279b4e5STianci.Yin uint64_t page_table_base) 1212279b4e5STianci.Yin { 1222279b4e5STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 1232279b4e5STianci.Yin 1242279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 1252279b4e5STianci.Yin hub->ctx_addr_distance * vmid, 1262279b4e5STianci.Yin lower_32_bits(page_table_base)); 1272279b4e5STianci.Yin 1282279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 1292279b4e5STianci.Yin hub->ctx_addr_distance * vmid, 1302279b4e5STianci.Yin upper_32_bits(page_table_base)); 1312279b4e5STianci.Yin } 1322279b4e5STianci.Yin 1332279b4e5STianci.Yin static void gfxhub_v3_0_init_gart_aperture_regs(struct amdgpu_device *adev) 1342279b4e5STianci.Yin { 1352279b4e5STianci.Yin uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 1362279b4e5STianci.Yin 1372279b4e5STianci.Yin gfxhub_v3_0_setup_vm_pt_regs(adev, 0, pt_base); 1382279b4e5STianci.Yin 1392279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 1402279b4e5STianci.Yin (u32)(adev->gmc.gart_start >> 12)); 1412279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 1422279b4e5STianci.Yin (u32)(adev->gmc.gart_start >> 44)); 1432279b4e5STianci.Yin 1442279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 1452279b4e5STianci.Yin (u32)(adev->gmc.gart_end >> 12)); 1462279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 1472279b4e5STianci.Yin (u32)(adev->gmc.gart_end >> 44)); 1482279b4e5STianci.Yin } 1492279b4e5STianci.Yin 1502279b4e5STianci.Yin static void gfxhub_v3_0_init_system_aperture_regs(struct amdgpu_device *adev) 1512279b4e5STianci.Yin { 1522279b4e5STianci.Yin uint64_t value; 1532279b4e5STianci.Yin 154*c6eafee0SAlex Deucher /* Program the AGP BAR */ 1552279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BASE, 0); 156*c6eafee0SAlex Deucher WREG32_SOC15(GC, 0, regGCMC_VM_AGP_BOT, adev->gmc.agp_start >> 24); 157*c6eafee0SAlex Deucher WREG32_SOC15(GC, 0, regGCMC_VM_AGP_TOP, adev->gmc.agp_end >> 24); 158*c6eafee0SAlex Deucher 1592279b4e5STianci.Yin 1602279b4e5STianci.Yin /* Program the system aperture low logical page number. */ 1612279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_LOW_ADDR, 1622279b4e5STianci.Yin adev->gmc.vram_start >> 18); 1632279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 1642279b4e5STianci.Yin adev->gmc.vram_end >> 18); 1652279b4e5STianci.Yin 1662279b4e5STianci.Yin /* Set default page address. */ 1677ccfd79fSChristian König value = adev->mem_scratch.gpu_addr - adev->gmc.vram_start 1682279b4e5STianci.Yin + adev->vm_manager.vram_base_offset; 1692279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 1702279b4e5STianci.Yin (u32)(value >> 12)); 1712279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 1722279b4e5STianci.Yin (u32)(value >> 44)); 1732279b4e5STianci.Yin 1742279b4e5STianci.Yin /* Program "protection fault". */ 1752279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 1762279b4e5STianci.Yin (u32)(adev->dummy_page_addr >> 12)); 1772279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 1782279b4e5STianci.Yin (u32)((u64)adev->dummy_page_addr >> 44)); 1792279b4e5STianci.Yin 1802279b4e5STianci.Yin WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2, 1812279b4e5STianci.Yin ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 1822279b4e5STianci.Yin } 1832279b4e5STianci.Yin 1842279b4e5STianci.Yin 1852279b4e5STianci.Yin static void gfxhub_v3_0_init_tlb_regs(struct amdgpu_device *adev) 1862279b4e5STianci.Yin { 1872279b4e5STianci.Yin uint32_t tmp; 1882279b4e5STianci.Yin 1892279b4e5STianci.Yin /* Setup TLB control */ 1902279b4e5STianci.Yin tmp = RREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL); 1912279b4e5STianci.Yin 1922279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1); 1932279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3); 1942279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 1952279b4e5STianci.Yin ENABLE_ADVANCED_DRIVER_MODEL, 1); 1962279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 1972279b4e5STianci.Yin SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 1982279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0); 1992279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 2002279b4e5STianci.Yin MTYPE, MTYPE_UC); /* UC, uncached */ 2012279b4e5STianci.Yin 2022279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL, tmp); 2032279b4e5STianci.Yin } 2042279b4e5STianci.Yin 2052279b4e5STianci.Yin static void gfxhub_v3_0_init_cache_regs(struct amdgpu_device *adev) 2062279b4e5STianci.Yin { 2072279b4e5STianci.Yin uint32_t tmp; 2082279b4e5STianci.Yin 2092279b4e5STianci.Yin /* These registers are not accessible to VF-SRIOV. 2102279b4e5STianci.Yin * The PF will program them instead. 2112279b4e5STianci.Yin */ 2122279b4e5STianci.Yin if (amdgpu_sriov_vf(adev)) 2132279b4e5STianci.Yin return; 2142279b4e5STianci.Yin 2152279b4e5STianci.Yin /* Setup L2 cache */ 2162279b4e5STianci.Yin tmp = RREG32_SOC15(GC, 0, regGCVM_L2_CNTL); 2172279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1); 2182279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0); 2192279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 2202279b4e5STianci.Yin ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); 2212279b4e5STianci.Yin /* XXX for emulation, Refer to closed source code.*/ 2222279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 2232279b4e5STianci.Yin L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 2242279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0); 2252279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1); 2262279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0); 2272279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CNTL, tmp); 2282279b4e5STianci.Yin 2292279b4e5STianci.Yin tmp = RREG32_SOC15(GC, 0, regGCVM_L2_CNTL2); 2302279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1); 2312279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 2322279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CNTL2, tmp); 2332279b4e5STianci.Yin 2342279b4e5STianci.Yin tmp = regGCVM_L2_CNTL3_DEFAULT; 2352279b4e5STianci.Yin if (adev->gmc.translate_further) { 2362279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12); 2372279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 2382279b4e5STianci.Yin L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 2392279b4e5STianci.Yin } else { 2402279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9); 2412279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 2422279b4e5STianci.Yin L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 2432279b4e5STianci.Yin } 2442279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CNTL3, tmp); 2452279b4e5STianci.Yin 2462279b4e5STianci.Yin tmp = regGCVM_L2_CNTL4_DEFAULT; 2472279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 2482279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 2492279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CNTL4, tmp); 2502279b4e5STianci.Yin 2512279b4e5STianci.Yin tmp = regGCVM_L2_CNTL5_DEFAULT; 2522279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0); 2532279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CNTL5, tmp); 2542279b4e5STianci.Yin } 2552279b4e5STianci.Yin 2562279b4e5STianci.Yin static void gfxhub_v3_0_enable_system_domain(struct amdgpu_device *adev) 2572279b4e5STianci.Yin { 2582279b4e5STianci.Yin uint32_t tmp; 2592279b4e5STianci.Yin 2602279b4e5STianci.Yin tmp = RREG32_SOC15(GC, 0, regGCVM_CONTEXT0_CNTL); 2612279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 2622279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0); 2632279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 2642279b4e5STianci.Yin RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 2652279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_CONTEXT0_CNTL, tmp); 2662279b4e5STianci.Yin } 2672279b4e5STianci.Yin 2682279b4e5STianci.Yin static void gfxhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) 2692279b4e5STianci.Yin { 2702279b4e5STianci.Yin /* These registers are not accessible to VF-SRIOV. 2712279b4e5STianci.Yin * The PF will program them instead. 2722279b4e5STianci.Yin */ 2732279b4e5STianci.Yin if (amdgpu_sriov_vf(adev)) 2742279b4e5STianci.Yin return; 2752279b4e5STianci.Yin 2762279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 2772279b4e5STianci.Yin 0xFFFFFFFF); 2782279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 2792279b4e5STianci.Yin 0x0000000F); 2802279b4e5STianci.Yin 2812279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 2822279b4e5STianci.Yin 0); 2832279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 2842279b4e5STianci.Yin 0); 2852279b4e5STianci.Yin 2862279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0); 2872279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0); 2882279b4e5STianci.Yin 2892279b4e5STianci.Yin } 2902279b4e5STianci.Yin 2912279b4e5STianci.Yin static void gfxhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) 2922279b4e5STianci.Yin { 2932279b4e5STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 2942279b4e5STianci.Yin int i; 2952279b4e5STianci.Yin uint32_t tmp; 2962279b4e5STianci.Yin 2972279b4e5STianci.Yin for (i = 0; i <= 14; i++) { 2982279b4e5STianci.Yin tmp = RREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL, i); 2992279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); 3002279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 3012279b4e5STianci.Yin adev->vm_manager.num_level); 3022279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3032279b4e5STianci.Yin RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3042279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3052279b4e5STianci.Yin DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3062279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3072279b4e5STianci.Yin PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3082279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3092279b4e5STianci.Yin VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3102279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3112279b4e5STianci.Yin READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3122279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3132279b4e5STianci.Yin WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3142279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3152279b4e5STianci.Yin EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3162279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3172279b4e5STianci.Yin PAGE_TABLE_BLOCK_SIZE, 3182279b4e5STianci.Yin adev->vm_manager.block_size - 9); 3192279b4e5STianci.Yin /* Send no-retry XNACK on fault to suppress VM fault storm. */ 3202279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 3212279b4e5STianci.Yin RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 3222279b4e5STianci.Yin !amdgpu_noretry); 3232279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_CNTL, 3242279b4e5STianci.Yin i * hub->ctx_distance, tmp); 3252279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 3262279b4e5STianci.Yin i * hub->ctx_addr_distance, 0); 3272279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 3282279b4e5STianci.Yin i * hub->ctx_addr_distance, 0); 3292279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 3302279b4e5STianci.Yin i * hub->ctx_addr_distance, 3312279b4e5STianci.Yin lower_32_bits(adev->vm_manager.max_pfn - 1)); 3322279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 3332279b4e5STianci.Yin i * hub->ctx_addr_distance, 3342279b4e5STianci.Yin upper_32_bits(adev->vm_manager.max_pfn - 1)); 3352279b4e5STianci.Yin } 336d7dab4fcSJack Xiao 337d7dab4fcSJack Xiao hub->vm_cntx_cntl = tmp; 3382279b4e5STianci.Yin } 3392279b4e5STianci.Yin 3402279b4e5STianci.Yin static void gfxhub_v3_0_program_invalidation(struct amdgpu_device *adev) 3412279b4e5STianci.Yin { 3422279b4e5STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 3432279b4e5STianci.Yin unsigned i; 3442279b4e5STianci.Yin 3452279b4e5STianci.Yin for (i = 0 ; i < 18; ++i) { 3462279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 3472279b4e5STianci.Yin i * hub->eng_addr_distance, 0xffffffff); 3482279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 3492279b4e5STianci.Yin i * hub->eng_addr_distance, 0x1f); 3502279b4e5STianci.Yin } 3512279b4e5STianci.Yin } 3522279b4e5STianci.Yin 3532279b4e5STianci.Yin static int gfxhub_v3_0_gart_enable(struct amdgpu_device *adev) 3542279b4e5STianci.Yin { 3552279b4e5STianci.Yin if (amdgpu_sriov_vf(adev)) { 3562279b4e5STianci.Yin /* 3572279b4e5STianci.Yin * GCMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are 3582279b4e5STianci.Yin * VF copy registers so vbios post doesn't program them, for 3592279b4e5STianci.Yin * SRIOV driver need to program them 3602279b4e5STianci.Yin */ 3612279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_BASE, 3622279b4e5STianci.Yin adev->gmc.vram_start >> 24); 3632279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_FB_LOCATION_TOP, 3642279b4e5STianci.Yin adev->gmc.vram_end >> 24); 3652279b4e5STianci.Yin } 3662279b4e5STianci.Yin 3672279b4e5STianci.Yin /* GART Enable. */ 3682279b4e5STianci.Yin gfxhub_v3_0_init_gart_aperture_regs(adev); 3692279b4e5STianci.Yin gfxhub_v3_0_init_system_aperture_regs(adev); 3702279b4e5STianci.Yin gfxhub_v3_0_init_tlb_regs(adev); 3712279b4e5STianci.Yin gfxhub_v3_0_init_cache_regs(adev); 3722279b4e5STianci.Yin 3732279b4e5STianci.Yin gfxhub_v3_0_enable_system_domain(adev); 3742279b4e5STianci.Yin gfxhub_v3_0_disable_identity_aperture(adev); 3752279b4e5STianci.Yin gfxhub_v3_0_setup_vmid_config(adev); 3762279b4e5STianci.Yin gfxhub_v3_0_program_invalidation(adev); 3772279b4e5STianci.Yin 3782279b4e5STianci.Yin return 0; 3792279b4e5STianci.Yin } 3802279b4e5STianci.Yin 3812279b4e5STianci.Yin static void gfxhub_v3_0_gart_disable(struct amdgpu_device *adev) 3822279b4e5STianci.Yin { 3832279b4e5STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 3842279b4e5STianci.Yin u32 tmp; 3852279b4e5STianci.Yin u32 i; 3862279b4e5STianci.Yin 3872279b4e5STianci.Yin /* Disable all tables */ 3882279b4e5STianci.Yin for (i = 0; i < 16; i++) 3892279b4e5STianci.Yin WREG32_SOC15_OFFSET(GC, 0, regGCVM_CONTEXT0_CNTL, 3902279b4e5STianci.Yin i * hub->ctx_distance, 0); 3912279b4e5STianci.Yin 3922279b4e5STianci.Yin /* Setup TLB control */ 3932279b4e5STianci.Yin tmp = RREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL); 3942279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); 3952279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 3962279b4e5STianci.Yin ENABLE_ADVANCED_DRIVER_MODEL, 0); 3972279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCMC_VM_MX_L1_TLB_CNTL, tmp); 3982279b4e5STianci.Yin 3992279b4e5STianci.Yin /* Setup L2 cache */ 4002279b4e5STianci.Yin WREG32_FIELD15_PREREG(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0); 4012279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_CNTL3, 0); 4022279b4e5STianci.Yin } 4032279b4e5STianci.Yin 4042279b4e5STianci.Yin /** 4052279b4e5STianci.Yin * gfxhub_v3_0_set_fault_enable_default - update GART/VM fault handling 4062279b4e5STianci.Yin * 4072279b4e5STianci.Yin * @adev: amdgpu_device pointer 4082279b4e5STianci.Yin * @value: true redirects VM faults to the default page 4092279b4e5STianci.Yin */ 4102279b4e5STianci.Yin static void gfxhub_v3_0_set_fault_enable_default(struct amdgpu_device *adev, 4112279b4e5STianci.Yin bool value) 4122279b4e5STianci.Yin { 4132279b4e5STianci.Yin u32 tmp; 4142279b4e5STianci.Yin 4157c8e4a25SChengming Gui /* NO halt CP when page fault */ 4167c8e4a25SChengming Gui tmp = RREG32_SOC15(GC, 0, regCP_DEBUG); 4177c8e4a25SChengming Gui tmp = REG_SET_FIELD(tmp, CP_DEBUG, CPG_UTCL1_ERROR_HALT_DISABLE, 1); 4187c8e4a25SChengming Gui WREG32_SOC15(GC, 0, regCP_DEBUG, tmp); 4197c8e4a25SChengming Gui 4207c8e4a25SChengming Gui /** 4217c8e4a25SChengming Gui * Set GRBM_GFX_INDEX in broad cast mode 4227c8e4a25SChengming Gui * before programming GL1C_UTCL0_CNTL1 and SQG_CONFIG 4237c8e4a25SChengming Gui */ 4247c8e4a25SChengming Gui WREG32_SOC15(GC, 0, regGRBM_GFX_INDEX, regGRBM_GFX_INDEX_DEFAULT); 4257c8e4a25SChengming Gui 4267c8e4a25SChengming Gui /** 4277c8e4a25SChengming Gui * Retry respond mode: RETRY 4287c8e4a25SChengming Gui * Error (no retry) respond mode: SUCCESS 4297c8e4a25SChengming Gui */ 4307c8e4a25SChengming Gui tmp = RREG32_SOC15(GC, 0, regGL1C_UTCL0_CNTL1); 4317c8e4a25SChengming Gui tmp = REG_SET_FIELD(tmp, GL1C_UTCL0_CNTL1, RESP_MODE, 0); 4327c8e4a25SChengming Gui tmp = REG_SET_FIELD(tmp, GL1C_UTCL0_CNTL1, RESP_FAULT_MODE, 0x2); 4337c8e4a25SChengming Gui WREG32_SOC15(GC, 0, regGL1C_UTCL0_CNTL1, tmp); 4347c8e4a25SChengming Gui 4352279b4e5STianci.Yin /* These registers are not accessible to VF-SRIOV. 4362279b4e5STianci.Yin * The PF will program them instead. 4372279b4e5STianci.Yin */ 4382279b4e5STianci.Yin if (amdgpu_sriov_vf(adev)) 4392279b4e5STianci.Yin return; 4402279b4e5STianci.Yin 4417c8e4a25SChengming Gui /* Disable SQ XNACK interrupt for all VMIDs */ 4427c8e4a25SChengming Gui tmp = RREG32_SOC15(GC, 0, regSQG_CONFIG); 4437c8e4a25SChengming Gui tmp = REG_SET_FIELD(tmp, SQG_CONFIG, XNACK_INTR_MASK, 4447c8e4a25SChengming Gui SQG_CONFIG__XNACK_INTR_MASK_MASK >> 4457c8e4a25SChengming Gui SQG_CONFIG__XNACK_INTR_MASK__SHIFT); 4467c8e4a25SChengming Gui WREG32_SOC15(GC, 0, regSQG_CONFIG, tmp); 4477c8e4a25SChengming Gui 4482279b4e5STianci.Yin tmp = RREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL); 4492279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4502279b4e5STianci.Yin RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4512279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4522279b4e5STianci.Yin PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4532279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4542279b4e5STianci.Yin PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4552279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4562279b4e5STianci.Yin PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4572279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4582279b4e5STianci.Yin TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 4592279b4e5STianci.Yin value); 4602279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4612279b4e5STianci.Yin NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4622279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4632279b4e5STianci.Yin DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4642279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4652279b4e5STianci.Yin VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4662279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4672279b4e5STianci.Yin READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4682279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4692279b4e5STianci.Yin WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4702279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4712279b4e5STianci.Yin EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4722279b4e5STianci.Yin if (!value) { 4732279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4742279b4e5STianci.Yin CRASH_ON_NO_RETRY_FAULT, 1); 4752279b4e5STianci.Yin tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 4762279b4e5STianci.Yin CRASH_ON_RETRY_FAULT, 1); 4772279b4e5STianci.Yin } 4782279b4e5STianci.Yin WREG32_SOC15(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL, tmp); 4792279b4e5STianci.Yin } 4802279b4e5STianci.Yin 4812279b4e5STianci.Yin static const struct amdgpu_vmhub_funcs gfxhub_v3_0_vmhub_funcs = { 4822279b4e5STianci.Yin .print_l2_protection_fault_status = gfxhub_v3_0_print_l2_protection_fault_status, 4832279b4e5STianci.Yin .get_invalidate_req = gfxhub_v3_0_get_invalidate_req, 4842279b4e5STianci.Yin }; 4852279b4e5STianci.Yin 4862279b4e5STianci.Yin static void gfxhub_v3_0_init(struct amdgpu_device *adev) 4872279b4e5STianci.Yin { 4882279b4e5STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 4892279b4e5STianci.Yin 4902279b4e5STianci.Yin hub->ctx0_ptb_addr_lo32 = 4912279b4e5STianci.Yin SOC15_REG_OFFSET(GC, 0, 4922279b4e5STianci.Yin regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 4932279b4e5STianci.Yin hub->ctx0_ptb_addr_hi32 = 4942279b4e5STianci.Yin SOC15_REG_OFFSET(GC, 0, 4952279b4e5STianci.Yin regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 4962279b4e5STianci.Yin hub->vm_inv_eng0_sem = 4972279b4e5STianci.Yin SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_SEM); 4982279b4e5STianci.Yin hub->vm_inv_eng0_req = 4992279b4e5STianci.Yin SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_REQ); 5002279b4e5STianci.Yin hub->vm_inv_eng0_ack = 5012279b4e5STianci.Yin SOC15_REG_OFFSET(GC, 0, regGCVM_INVALIDATE_ENG0_ACK); 5022279b4e5STianci.Yin hub->vm_context0_cntl = 5032279b4e5STianci.Yin SOC15_REG_OFFSET(GC, 0, regGCVM_CONTEXT0_CNTL); 5042279b4e5STianci.Yin hub->vm_l2_pro_fault_status = 5052279b4e5STianci.Yin SOC15_REG_OFFSET(GC, 0, regGCVM_L2_PROTECTION_FAULT_STATUS); 5062279b4e5STianci.Yin hub->vm_l2_pro_fault_cntl = 5072279b4e5STianci.Yin SOC15_REG_OFFSET(GC, 0, regGCVM_L2_PROTECTION_FAULT_CNTL); 5082279b4e5STianci.Yin 5092279b4e5STianci.Yin hub->ctx_distance = regGCVM_CONTEXT1_CNTL - regGCVM_CONTEXT0_CNTL; 5102279b4e5STianci.Yin hub->ctx_addr_distance = regGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 5112279b4e5STianci.Yin regGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 5122279b4e5STianci.Yin hub->eng_distance = regGCVM_INVALIDATE_ENG1_REQ - 5132279b4e5STianci.Yin regGCVM_INVALIDATE_ENG0_REQ; 5142279b4e5STianci.Yin hub->eng_addr_distance = regGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 5152279b4e5STianci.Yin regGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 5162279b4e5STianci.Yin 5172279b4e5STianci.Yin hub->vm_cntx_cntl_vm_fault = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 5182279b4e5STianci.Yin GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 5192279b4e5STianci.Yin GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 5202279b4e5STianci.Yin GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 5212279b4e5STianci.Yin GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 5222279b4e5STianci.Yin GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 5232279b4e5STianci.Yin GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 5242279b4e5STianci.Yin 5252279b4e5STianci.Yin hub->vmhub_funcs = &gfxhub_v3_0_vmhub_funcs; 5262279b4e5STianci.Yin } 5272279b4e5STianci.Yin 5282279b4e5STianci.Yin const struct amdgpu_gfxhub_funcs gfxhub_v3_0_funcs = { 5292279b4e5STianci.Yin .get_fb_location = gfxhub_v3_0_get_fb_location, 5302279b4e5STianci.Yin .get_mc_fb_offset = gfxhub_v3_0_get_mc_fb_offset, 5312279b4e5STianci.Yin .setup_vm_pt_regs = gfxhub_v3_0_setup_vm_pt_regs, 5322279b4e5STianci.Yin .gart_enable = gfxhub_v3_0_gart_enable, 5332279b4e5STianci.Yin .gart_disable = gfxhub_v3_0_gart_disable, 5342279b4e5STianci.Yin .set_fault_enable_default = gfxhub_v3_0_set_fault_enable_default, 5352279b4e5STianci.Yin .init = gfxhub_v3_0_init, 5362279b4e5STianci.Yin }; 537