1af01d47dSLikun Gao /* 2af01d47dSLikun Gao * Copyright 2019 Advanced Micro Devices, Inc. 3af01d47dSLikun Gao * 4af01d47dSLikun Gao * Permission is hereby granted, free of charge, to any person obtaining a 5af01d47dSLikun Gao * copy of this software and associated documentation files (the "Software"), 6af01d47dSLikun Gao * to deal in the Software without restriction, including without limitation 7af01d47dSLikun Gao * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8af01d47dSLikun Gao * and/or sell copies of the Software, and to permit persons to whom the 9af01d47dSLikun Gao * Software is furnished to do so, subject to the following conditions: 10af01d47dSLikun Gao * 11af01d47dSLikun Gao * The above copyright notice and this permission notice shall be included in 12af01d47dSLikun Gao * all copies or substantial portions of the Software. 13af01d47dSLikun Gao * 14af01d47dSLikun Gao * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15af01d47dSLikun Gao * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16af01d47dSLikun Gao * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17af01d47dSLikun Gao * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18af01d47dSLikun Gao * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19af01d47dSLikun Gao * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20af01d47dSLikun Gao * OTHER DEALINGS IN THE SOFTWARE. 21af01d47dSLikun Gao * 22af01d47dSLikun Gao */ 23af01d47dSLikun Gao 24af01d47dSLikun Gao #include "amdgpu.h" 25af01d47dSLikun Gao #include "gfxhub_v2_1.h" 26af01d47dSLikun Gao 27af01d47dSLikun Gao #include "gc/gc_10_3_0_offset.h" 28af01d47dSLikun Gao #include "gc/gc_10_3_0_sh_mask.h" 29af01d47dSLikun Gao #include "gc/gc_10_3_0_default.h" 30af01d47dSLikun Gao #include "navi10_enum.h" 31af01d47dSLikun Gao 32af01d47dSLikun Gao #include "soc15_common.h" 33af01d47dSLikun Gao 3493fabd84SAlex Deucher static const char *gfxhub_client_ids[] = { 3593fabd84SAlex Deucher "CB/DB", 3693fabd84SAlex Deucher "Reserved", 3793fabd84SAlex Deucher "GE1", 3893fabd84SAlex Deucher "GE2", 3993fabd84SAlex Deucher "CPF", 4093fabd84SAlex Deucher "CPC", 4193fabd84SAlex Deucher "CPG", 4293fabd84SAlex Deucher "RLC", 4393fabd84SAlex Deucher "TCP", 4493fabd84SAlex Deucher "SQC (inst)", 4593fabd84SAlex Deucher "SQC (data)", 4693fabd84SAlex Deucher "SQG", 4793fabd84SAlex Deucher "Reserved", 4893fabd84SAlex Deucher "SDMA0", 4993fabd84SAlex Deucher "SDMA1", 5093fabd84SAlex Deucher "GCR", 5193fabd84SAlex Deucher "SDMA2", 5293fabd84SAlex Deucher "SDMA3", 5393fabd84SAlex Deucher }; 5493fabd84SAlex Deucher 55caa9f483SHuang Rui static uint32_t gfxhub_v2_1_get_invalidate_req(unsigned int vmid, 56caa9f483SHuang Rui uint32_t flush_type) 57caa9f483SHuang Rui { 58caa9f483SHuang Rui u32 req = 0; 59caa9f483SHuang Rui 60caa9f483SHuang Rui /* invalidate using legacy mode on vmid*/ 61caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 62caa9f483SHuang Rui PER_VMID_INVALIDATE_REQ, 1 << vmid); 63caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 64caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 65caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 66caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 67caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 68caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 69caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, 70caa9f483SHuang Rui CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 71caa9f483SHuang Rui 72caa9f483SHuang Rui return req; 73caa9f483SHuang Rui } 74caa9f483SHuang Rui 752577db91SHuang Rui static void 762577db91SHuang Rui gfxhub_v2_1_print_l2_protection_fault_status(struct amdgpu_device *adev, 772577db91SHuang Rui uint32_t status) 782577db91SHuang Rui { 7993fabd84SAlex Deucher u32 cid = REG_GET_FIELD(status, 8093fabd84SAlex Deucher GCVM_L2_PROTECTION_FAULT_STATUS, CID); 8193fabd84SAlex Deucher 822577db91SHuang Rui dev_err(adev->dev, 832577db91SHuang Rui "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 842577db91SHuang Rui status); 8593fabd84SAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 8693fabd84SAlex Deucher cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid], 8793fabd84SAlex Deucher cid); 882577db91SHuang Rui dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 892577db91SHuang Rui REG_GET_FIELD(status, 902577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 912577db91SHuang Rui dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 922577db91SHuang Rui REG_GET_FIELD(status, 932577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 942577db91SHuang Rui dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 952577db91SHuang Rui REG_GET_FIELD(status, 962577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 972577db91SHuang Rui dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 982577db91SHuang Rui REG_GET_FIELD(status, 992577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 1002577db91SHuang Rui dev_err(adev->dev, "\t RW: 0x%lx\n", 1012577db91SHuang Rui REG_GET_FIELD(status, 1022577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, RW)); 1032577db91SHuang Rui } 1042577db91SHuang Rui 1058ffff9b4SOak Zeng static u64 gfxhub_v2_1_get_fb_location(struct amdgpu_device *adev) 106af01d47dSLikun Gao { 107af01d47dSLikun Gao u64 base = RREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE); 108af01d47dSLikun Gao 109af01d47dSLikun Gao base &= GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK; 110af01d47dSLikun Gao base <<= 24; 111af01d47dSLikun Gao 112af01d47dSLikun Gao return base; 113af01d47dSLikun Gao } 114af01d47dSLikun Gao 1158ffff9b4SOak Zeng static u64 gfxhub_v2_1_get_mc_fb_offset(struct amdgpu_device *adev) 116af01d47dSLikun Gao { 117af01d47dSLikun Gao return (u64)RREG32_SOC15(GC, 0, mmGCMC_VM_FB_OFFSET) << 24; 118af01d47dSLikun Gao } 119af01d47dSLikun Gao 1208ffff9b4SOak Zeng static void gfxhub_v2_1_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, 121af01d47dSLikun Gao uint64_t page_table_base) 122af01d47dSLikun Gao { 12313ae12d9SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 124af01d47dSLikun Gao 125af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 12613ae12d9SHuang Rui hub->ctx_addr_distance * vmid, 12713ae12d9SHuang Rui lower_32_bits(page_table_base)); 128af01d47dSLikun Gao 129af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 13013ae12d9SHuang Rui hub->ctx_addr_distance * vmid, 13113ae12d9SHuang Rui upper_32_bits(page_table_base)); 132af01d47dSLikun Gao } 133af01d47dSLikun Gao 134af01d47dSLikun Gao static void gfxhub_v2_1_init_gart_aperture_regs(struct amdgpu_device *adev) 135af01d47dSLikun Gao { 136af01d47dSLikun Gao uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 137af01d47dSLikun Gao 138af01d47dSLikun Gao gfxhub_v2_1_setup_vm_pt_regs(adev, 0, pt_base); 139af01d47dSLikun Gao 140af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 141af01d47dSLikun Gao (u32)(adev->gmc.gart_start >> 12)); 142af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 143af01d47dSLikun Gao (u32)(adev->gmc.gart_start >> 44)); 144af01d47dSLikun Gao 145af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 146af01d47dSLikun Gao (u32)(adev->gmc.gart_end >> 12)); 147af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 148af01d47dSLikun Gao (u32)(adev->gmc.gart_end >> 44)); 149af01d47dSLikun Gao } 150af01d47dSLikun Gao 151af01d47dSLikun Gao static void gfxhub_v2_1_init_system_aperture_regs(struct amdgpu_device *adev) 152af01d47dSLikun Gao { 153af01d47dSLikun Gao uint64_t value; 154af01d47dSLikun Gao 155*99698b51SAlex Deucher /* Program the AGP BAR */ 156af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BASE, 0); 157*99698b51SAlex Deucher WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BOT, adev->gmc.agp_start >> 24); 158*99698b51SAlex Deucher WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_TOP, adev->gmc.agp_end >> 24); 159af01d47dSLikun Gao 160af01d47dSLikun Gao /* Program the system aperture low logical page number. */ 161af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_LOW_ADDR, 162*99698b51SAlex Deucher min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18); 163af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 164*99698b51SAlex Deucher max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18); 165af01d47dSLikun Gao 166af01d47dSLikun Gao /* Set default page address. */ 167af01d47dSLikun Gao value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start 168af01d47dSLikun Gao + adev->vm_manager.vram_base_offset; 169af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 170af01d47dSLikun Gao (u32)(value >> 12)); 171af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 172af01d47dSLikun Gao (u32)(value >> 44)); 173af01d47dSLikun Gao 174af01d47dSLikun Gao /* Program "protection fault". */ 175af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 176af01d47dSLikun Gao (u32)(adev->dummy_page_addr >> 12)); 177af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 178af01d47dSLikun Gao (u32)((u64)adev->dummy_page_addr >> 44)); 179af01d47dSLikun Gao 180af01d47dSLikun Gao WREG32_FIELD15(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2, 181af01d47dSLikun Gao ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 182af01d47dSLikun Gao } 183af01d47dSLikun Gao 184af01d47dSLikun Gao 185af01d47dSLikun Gao static void gfxhub_v2_1_init_tlb_regs(struct amdgpu_device *adev) 186af01d47dSLikun Gao { 187af01d47dSLikun Gao uint32_t tmp; 188af01d47dSLikun Gao 189af01d47dSLikun Gao /* Setup TLB control */ 190af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL); 191af01d47dSLikun Gao 192af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1); 193af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3); 194af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 195af01d47dSLikun Gao ENABLE_ADVANCED_DRIVER_MODEL, 1); 196af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 197af01d47dSLikun Gao SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 198af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0); 199af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 200af01d47dSLikun Gao MTYPE, MTYPE_UC); /* UC, uncached */ 201af01d47dSLikun Gao 202af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp); 203af01d47dSLikun Gao } 204af01d47dSLikun Gao 205af01d47dSLikun Gao static void gfxhub_v2_1_init_cache_regs(struct amdgpu_device *adev) 206af01d47dSLikun Gao { 207af01d47dSLikun Gao uint32_t tmp; 208af01d47dSLikun Gao 2091d447326SLiu ChengZhe /* These registers are not accessible to VF-SRIOV. 2101d447326SLiu ChengZhe * The PF will program them instead. 2111d447326SLiu ChengZhe */ 2121d447326SLiu ChengZhe if (amdgpu_sriov_vf(adev)) 2131d447326SLiu ChengZhe return; 2141d447326SLiu ChengZhe 215af01d47dSLikun Gao /* Setup L2 cache */ 216af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL); 217af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1); 218af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0); 219af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 220af01d47dSLikun Gao ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); 221af01d47dSLikun Gao /* XXX for emulation, Refer to closed source code.*/ 222af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 223af01d47dSLikun Gao L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 224af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0); 225af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1); 226af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0); 227af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL, tmp); 228af01d47dSLikun Gao 229af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2); 230af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1); 231af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 232af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2, tmp); 233af01d47dSLikun Gao 234af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL3_DEFAULT; 235af01d47dSLikun Gao if (adev->gmc.translate_further) { 236af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12); 237af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 238af01d47dSLikun Gao L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 239af01d47dSLikun Gao } else { 240af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9); 241af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 242af01d47dSLikun Gao L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 243af01d47dSLikun Gao } 244af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, tmp); 245af01d47dSLikun Gao 246af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL4_DEFAULT; 247af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 248af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 249af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL4, tmp); 250af01d47dSLikun Gao 251af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL5_DEFAULT; 252af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0); 253af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL5, tmp); 254af01d47dSLikun Gao } 255af01d47dSLikun Gao 256af01d47dSLikun Gao static void gfxhub_v2_1_enable_system_domain(struct amdgpu_device *adev) 257af01d47dSLikun Gao { 258af01d47dSLikun Gao uint32_t tmp; 259af01d47dSLikun Gao 260af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL); 261af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 262af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0); 263af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 264af01d47dSLikun Gao RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 265af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL, tmp); 266af01d47dSLikun Gao } 267af01d47dSLikun Gao 268af01d47dSLikun Gao static void gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device *adev) 269af01d47dSLikun Gao { 2701d447326SLiu ChengZhe /* These registers are not accessible to VF-SRIOV. 2711d447326SLiu ChengZhe * The PF will program them instead. 2721d447326SLiu ChengZhe */ 2731d447326SLiu ChengZhe if (amdgpu_sriov_vf(adev)) 2741d447326SLiu ChengZhe return; 2751d447326SLiu ChengZhe 276af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 277af01d47dSLikun Gao 0xFFFFFFFF); 278af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 279af01d47dSLikun Gao 0x0000000F); 280af01d47dSLikun Gao 281af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 282af01d47dSLikun Gao 0); 283af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 284af01d47dSLikun Gao 0); 285af01d47dSLikun Gao 286af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0); 287af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0); 288af01d47dSLikun Gao 289af01d47dSLikun Gao } 290af01d47dSLikun Gao 291af01d47dSLikun Gao static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev) 292af01d47dSLikun Gao { 29313ae12d9SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 294af01d47dSLikun Gao int i; 295af01d47dSLikun Gao uint32_t tmp; 296af01d47dSLikun Gao 297af01d47dSLikun Gao for (i = 0; i <= 14; i++) { 298af01d47dSLikun Gao tmp = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, i); 299af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); 300af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 301af01d47dSLikun Gao adev->vm_manager.num_level); 302af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 303af01d47dSLikun Gao RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 304af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 305af01d47dSLikun Gao DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 306af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 307af01d47dSLikun Gao PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 308af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 309af01d47dSLikun Gao VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 310af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 311af01d47dSLikun Gao READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 312af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 313af01d47dSLikun Gao WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 314af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 315af01d47dSLikun Gao EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 316af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 317af01d47dSLikun Gao PAGE_TABLE_BLOCK_SIZE, 318af01d47dSLikun Gao adev->vm_manager.block_size - 9); 319af01d47dSLikun Gao /* Send no-retry XNACK on fault to suppress VM fault storm. */ 320af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 321af01d47dSLikun Gao RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 3229b498efaSAlex Deucher !adev->gmc.noretry); 32313ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, 32413ae12d9SHuang Rui i * hub->ctx_distance, tmp); 32513ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 32613ae12d9SHuang Rui i * hub->ctx_addr_distance, 0); 32713ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 32813ae12d9SHuang Rui i * hub->ctx_addr_distance, 0); 32913ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 33013ae12d9SHuang Rui i * hub->ctx_addr_distance, 331af01d47dSLikun Gao lower_32_bits(adev->vm_manager.max_pfn - 1)); 33213ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 33313ae12d9SHuang Rui i * hub->ctx_addr_distance, 334af01d47dSLikun Gao upper_32_bits(adev->vm_manager.max_pfn - 1)); 335af01d47dSLikun Gao } 336af01d47dSLikun Gao } 337af01d47dSLikun Gao 338af01d47dSLikun Gao static void gfxhub_v2_1_program_invalidation(struct amdgpu_device *adev) 339af01d47dSLikun Gao { 34013ae12d9SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 341af01d47dSLikun Gao unsigned i; 342af01d47dSLikun Gao 343af01d47dSLikun Gao for (i = 0 ; i < 18; ++i) { 344af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 34513ae12d9SHuang Rui i * hub->eng_addr_distance, 0xffffffff); 346af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 34713ae12d9SHuang Rui i * hub->eng_addr_distance, 0x1f); 348af01d47dSLikun Gao } 349af01d47dSLikun Gao } 350af01d47dSLikun Gao 3518ffff9b4SOak Zeng static int gfxhub_v2_1_gart_enable(struct amdgpu_device *adev) 352af01d47dSLikun Gao { 353af01d47dSLikun Gao if (amdgpu_sriov_vf(adev)) { 354af01d47dSLikun Gao /* 355af01d47dSLikun Gao * GCMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are 356af01d47dSLikun Gao * VF copy registers so vbios post doesn't program them, for 357af01d47dSLikun Gao * SRIOV driver need to program them 358af01d47dSLikun Gao */ 359af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE, 360af01d47dSLikun Gao adev->gmc.vram_start >> 24); 361af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_TOP, 362af01d47dSLikun Gao adev->gmc.vram_end >> 24); 363af01d47dSLikun Gao } 364af01d47dSLikun Gao 365af01d47dSLikun Gao /* GART Enable. */ 366af01d47dSLikun Gao gfxhub_v2_1_init_gart_aperture_regs(adev); 367af01d47dSLikun Gao gfxhub_v2_1_init_system_aperture_regs(adev); 368af01d47dSLikun Gao gfxhub_v2_1_init_tlb_regs(adev); 369af01d47dSLikun Gao gfxhub_v2_1_init_cache_regs(adev); 370af01d47dSLikun Gao 371af01d47dSLikun Gao gfxhub_v2_1_enable_system_domain(adev); 372af01d47dSLikun Gao gfxhub_v2_1_disable_identity_aperture(adev); 373af01d47dSLikun Gao gfxhub_v2_1_setup_vmid_config(adev); 374af01d47dSLikun Gao gfxhub_v2_1_program_invalidation(adev); 375af01d47dSLikun Gao 376af01d47dSLikun Gao return 0; 377af01d47dSLikun Gao } 378af01d47dSLikun Gao 3798ffff9b4SOak Zeng static void gfxhub_v2_1_gart_disable(struct amdgpu_device *adev) 380af01d47dSLikun Gao { 38113ae12d9SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 382af01d47dSLikun Gao u32 tmp; 383af01d47dSLikun Gao u32 i; 384af01d47dSLikun Gao 385af01d47dSLikun Gao /* Disable all tables */ 386af01d47dSLikun Gao for (i = 0; i < 16; i++) 38713ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL, 38813ae12d9SHuang Rui i * hub->ctx_distance, 0); 389af01d47dSLikun Gao 390af01d47dSLikun Gao /* Setup TLB control */ 391af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL); 392af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); 393af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 394af01d47dSLikun Gao ENABLE_ADVANCED_DRIVER_MODEL, 0); 395af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp); 396af01d47dSLikun Gao 397af01d47dSLikun Gao /* Setup L2 cache */ 398af01d47dSLikun Gao WREG32_FIELD15(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0); 399af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, 0); 400af01d47dSLikun Gao } 401af01d47dSLikun Gao 402af01d47dSLikun Gao /** 403af01d47dSLikun Gao * gfxhub_v2_1_set_fault_enable_default - update GART/VM fault handling 404af01d47dSLikun Gao * 405af01d47dSLikun Gao * @adev: amdgpu_device pointer 406af01d47dSLikun Gao * @value: true redirects VM faults to the default page 407af01d47dSLikun Gao */ 4088ffff9b4SOak Zeng static void gfxhub_v2_1_set_fault_enable_default(struct amdgpu_device *adev, 409af01d47dSLikun Gao bool value) 410af01d47dSLikun Gao { 411af01d47dSLikun Gao u32 tmp; 4121d447326SLiu ChengZhe 4131d447326SLiu ChengZhe /* These registers are not accessible to VF-SRIOV. 4141d447326SLiu ChengZhe * The PF will program them instead. 4151d447326SLiu ChengZhe */ 4161d447326SLiu ChengZhe if (amdgpu_sriov_vf(adev)) 4171d447326SLiu ChengZhe return; 4181d447326SLiu ChengZhe 419af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL); 420af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 421af01d47dSLikun Gao RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 422af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 423af01d47dSLikun Gao PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 424af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 425af01d47dSLikun Gao PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 426af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 427af01d47dSLikun Gao PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 428af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 429af01d47dSLikun Gao TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 430af01d47dSLikun Gao value); 431af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 432af01d47dSLikun Gao NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 433af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 434af01d47dSLikun Gao DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 435af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 436af01d47dSLikun Gao VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 437af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 438af01d47dSLikun Gao READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 439af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 440af01d47dSLikun Gao WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 441af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 442af01d47dSLikun Gao EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 443af01d47dSLikun Gao if (!value) { 444af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 445af01d47dSLikun Gao CRASH_ON_NO_RETRY_FAULT, 1); 446af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 447af01d47dSLikun Gao CRASH_ON_RETRY_FAULT, 1); 448af01d47dSLikun Gao } 449af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, tmp); 450af01d47dSLikun Gao } 451af01d47dSLikun Gao 4522577db91SHuang Rui static const struct amdgpu_vmhub_funcs gfxhub_v2_1_vmhub_funcs = { 4532577db91SHuang Rui .print_l2_protection_fault_status = gfxhub_v2_1_print_l2_protection_fault_status, 454caa9f483SHuang Rui .get_invalidate_req = gfxhub_v2_1_get_invalidate_req, 4552577db91SHuang Rui }; 4562577db91SHuang Rui 4578ffff9b4SOak Zeng static void gfxhub_v2_1_init(struct amdgpu_device *adev) 458af01d47dSLikun Gao { 459af01d47dSLikun Gao struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 460af01d47dSLikun Gao 461af01d47dSLikun Gao hub->ctx0_ptb_addr_lo32 = 462af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, 463af01d47dSLikun Gao mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 464af01d47dSLikun Gao hub->ctx0_ptb_addr_hi32 = 465af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, 466af01d47dSLikun Gao mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 467af01d47dSLikun Gao hub->vm_inv_eng0_sem = 468af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_SEM); 469af01d47dSLikun Gao hub->vm_inv_eng0_req = 470af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_REQ); 471af01d47dSLikun Gao hub->vm_inv_eng0_ack = 472af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ACK); 473af01d47dSLikun Gao hub->vm_context0_cntl = 474af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL); 475af01d47dSLikun Gao hub->vm_l2_pro_fault_status = 476af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_STATUS); 477af01d47dSLikun Gao hub->vm_l2_pro_fault_cntl = 478af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL); 4791f9d56c3SHuang Rui 4801f9d56c3SHuang Rui hub->ctx_distance = mmGCVM_CONTEXT1_CNTL - mmGCVM_CONTEXT0_CNTL; 4811f9d56c3SHuang Rui hub->ctx_addr_distance = mmGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 4821f9d56c3SHuang Rui mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 4831f9d56c3SHuang Rui hub->eng_distance = mmGCVM_INVALIDATE_ENG1_REQ - 4841f9d56c3SHuang Rui mmGCVM_INVALIDATE_ENG0_REQ; 4851f9d56c3SHuang Rui hub->eng_addr_distance = mmGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 4861f9d56c3SHuang Rui mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 4875befb6fcSHuang Rui 4885befb6fcSHuang Rui hub->vm_cntx_cntl_vm_fault = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4895befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4905befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4915befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4925befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4935befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4945befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 4952577db91SHuang Rui 4962577db91SHuang Rui hub->vmhub_funcs = &gfxhub_v2_1_vmhub_funcs; 497af01d47dSLikun Gao } 498fdb8483bSJohn Clements 4998ffff9b4SOak Zeng static int gfxhub_v2_1_get_xgmi_info(struct amdgpu_device *adev) 500fdb8483bSJohn Clements { 501fdb8483bSJohn Clements u32 xgmi_lfb_cntl = RREG32_SOC15(GC, 0, mmGCMC_VM_XGMI_LFB_CNTL); 502fdb8483bSJohn Clements u32 max_region = 503fdb8483bSJohn Clements REG_GET_FIELD(xgmi_lfb_cntl, GCMC_VM_XGMI_LFB_CNTL, PF_MAX_REGION); 504fdb8483bSJohn Clements u32 max_num_physical_nodes = 0; 505fdb8483bSJohn Clements u32 max_physical_node_id = 0; 506fdb8483bSJohn Clements 507fdb8483bSJohn Clements switch (adev->asic_type) { 508fdb8483bSJohn Clements case CHIP_SIENNA_CICHLID: 509fdb8483bSJohn Clements max_num_physical_nodes = 4; 510fdb8483bSJohn Clements max_physical_node_id = 3; 511fdb8483bSJohn Clements break; 512fdb8483bSJohn Clements default: 513fdb8483bSJohn Clements return -EINVAL; 514fdb8483bSJohn Clements } 515fdb8483bSJohn Clements 516fdb8483bSJohn Clements /* PF_MAX_REGION=0 means xgmi is disabled */ 517fdb8483bSJohn Clements if (max_region) { 518fdb8483bSJohn Clements adev->gmc.xgmi.num_physical_nodes = max_region + 1; 519fdb8483bSJohn Clements if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes) 520fdb8483bSJohn Clements return -EINVAL; 521fdb8483bSJohn Clements 522fdb8483bSJohn Clements adev->gmc.xgmi.physical_node_id = 523fdb8483bSJohn Clements REG_GET_FIELD(xgmi_lfb_cntl, GCMC_VM_XGMI_LFB_CNTL, PF_LFB_REGION); 524fdb8483bSJohn Clements if (adev->gmc.xgmi.physical_node_id > max_physical_node_id) 525fdb8483bSJohn Clements return -EINVAL; 526fdb8483bSJohn Clements 527fdb8483bSJohn Clements adev->gmc.xgmi.node_segment_size = REG_GET_FIELD( 528fdb8483bSJohn Clements RREG32_SOC15(GC, 0, mmGCMC_VM_XGMI_LFB_SIZE), 529fdb8483bSJohn Clements GCMC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24; 530fdb8483bSJohn Clements } 531fdb8483bSJohn Clements 532fdb8483bSJohn Clements return 0; 533fdb8483bSJohn Clements } 5348ffff9b4SOak Zeng 5358ffff9b4SOak Zeng const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = { 5368ffff9b4SOak Zeng .get_fb_location = gfxhub_v2_1_get_fb_location, 5378ffff9b4SOak Zeng .get_mc_fb_offset = gfxhub_v2_1_get_mc_fb_offset, 5388ffff9b4SOak Zeng .setup_vm_pt_regs = gfxhub_v2_1_setup_vm_pt_regs, 5398ffff9b4SOak Zeng .gart_enable = gfxhub_v2_1_gart_enable, 5408ffff9b4SOak Zeng .gart_disable = gfxhub_v2_1_gart_disable, 5418ffff9b4SOak Zeng .set_fault_enable_default = gfxhub_v2_1_set_fault_enable_default, 5428ffff9b4SOak Zeng .init = gfxhub_v2_1_init, 5438ffff9b4SOak Zeng .get_xgmi_info = gfxhub_v2_1_get_xgmi_info, 5448ffff9b4SOak Zeng }; 545