1af01d47dSLikun Gao /* 2af01d47dSLikun Gao * Copyright 2019 Advanced Micro Devices, Inc. 3af01d47dSLikun Gao * 4af01d47dSLikun Gao * Permission is hereby granted, free of charge, to any person obtaining a 5af01d47dSLikun Gao * copy of this software and associated documentation files (the "Software"), 6af01d47dSLikun Gao * to deal in the Software without restriction, including without limitation 7af01d47dSLikun Gao * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8af01d47dSLikun Gao * and/or sell copies of the Software, and to permit persons to whom the 9af01d47dSLikun Gao * Software is furnished to do so, subject to the following conditions: 10af01d47dSLikun Gao * 11af01d47dSLikun Gao * The above copyright notice and this permission notice shall be included in 12af01d47dSLikun Gao * all copies or substantial portions of the Software. 13af01d47dSLikun Gao * 14af01d47dSLikun Gao * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15af01d47dSLikun Gao * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16af01d47dSLikun Gao * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17af01d47dSLikun Gao * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18af01d47dSLikun Gao * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19af01d47dSLikun Gao * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20af01d47dSLikun Gao * OTHER DEALINGS IN THE SOFTWARE. 21af01d47dSLikun Gao * 22af01d47dSLikun Gao */ 23af01d47dSLikun Gao 24af01d47dSLikun Gao #include "amdgpu.h" 25af01d47dSLikun Gao #include "gfxhub_v2_1.h" 26af01d47dSLikun Gao 27af01d47dSLikun Gao #include "gc/gc_10_3_0_offset.h" 28af01d47dSLikun Gao #include "gc/gc_10_3_0_sh_mask.h" 29af01d47dSLikun Gao #include "gc/gc_10_3_0_default.h" 30af01d47dSLikun Gao #include "navi10_enum.h" 31af01d47dSLikun Gao 32af01d47dSLikun Gao #include "soc15_common.h" 33af01d47dSLikun Gao 342577db91SHuang Rui static void 352577db91SHuang Rui gfxhub_v2_1_print_l2_protection_fault_status(struct amdgpu_device *adev, 362577db91SHuang Rui uint32_t status) 372577db91SHuang Rui { 382577db91SHuang Rui dev_err(adev->dev, 392577db91SHuang Rui "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 402577db91SHuang Rui status); 412577db91SHuang Rui dev_err(adev->dev, "\t Faulty UTCL2 client ID: 0x%lx\n", 422577db91SHuang Rui REG_GET_FIELD(status, 432577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, CID)); 442577db91SHuang Rui dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 452577db91SHuang Rui REG_GET_FIELD(status, 462577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 472577db91SHuang Rui dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 482577db91SHuang Rui REG_GET_FIELD(status, 492577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 502577db91SHuang Rui dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 512577db91SHuang Rui REG_GET_FIELD(status, 522577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 532577db91SHuang Rui dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 542577db91SHuang Rui REG_GET_FIELD(status, 552577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 562577db91SHuang Rui dev_err(adev->dev, "\t RW: 0x%lx\n", 572577db91SHuang Rui REG_GET_FIELD(status, 582577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, RW)); 592577db91SHuang Rui } 602577db91SHuang Rui 61af01d47dSLikun Gao u64 gfxhub_v2_1_get_fb_location(struct amdgpu_device *adev) 62af01d47dSLikun Gao { 63af01d47dSLikun Gao u64 base = RREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE); 64af01d47dSLikun Gao 65af01d47dSLikun Gao base &= GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK; 66af01d47dSLikun Gao base <<= 24; 67af01d47dSLikun Gao 68af01d47dSLikun Gao return base; 69af01d47dSLikun Gao } 70af01d47dSLikun Gao 71af01d47dSLikun Gao u64 gfxhub_v2_1_get_mc_fb_offset(struct amdgpu_device *adev) 72af01d47dSLikun Gao { 73af01d47dSLikun Gao return (u64)RREG32_SOC15(GC, 0, mmGCMC_VM_FB_OFFSET) << 24; 74af01d47dSLikun Gao } 75af01d47dSLikun Gao 76af01d47dSLikun Gao void gfxhub_v2_1_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, 77af01d47dSLikun Gao uint64_t page_table_base) 78af01d47dSLikun Gao { 7913ae12d9SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 80af01d47dSLikun Gao 81af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 8213ae12d9SHuang Rui hub->ctx_addr_distance * vmid, 8313ae12d9SHuang Rui lower_32_bits(page_table_base)); 84af01d47dSLikun Gao 85af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 8613ae12d9SHuang Rui hub->ctx_addr_distance * vmid, 8713ae12d9SHuang Rui upper_32_bits(page_table_base)); 88af01d47dSLikun Gao } 89af01d47dSLikun Gao 90af01d47dSLikun Gao static void gfxhub_v2_1_init_gart_aperture_regs(struct amdgpu_device *adev) 91af01d47dSLikun Gao { 92af01d47dSLikun Gao uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 93af01d47dSLikun Gao 94af01d47dSLikun Gao gfxhub_v2_1_setup_vm_pt_regs(adev, 0, pt_base); 95af01d47dSLikun Gao 96af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 97af01d47dSLikun Gao (u32)(adev->gmc.gart_start >> 12)); 98af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 99af01d47dSLikun Gao (u32)(adev->gmc.gart_start >> 44)); 100af01d47dSLikun Gao 101af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 102af01d47dSLikun Gao (u32)(adev->gmc.gart_end >> 12)); 103af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 104af01d47dSLikun Gao (u32)(adev->gmc.gart_end >> 44)); 105af01d47dSLikun Gao } 106af01d47dSLikun Gao 107af01d47dSLikun Gao static void gfxhub_v2_1_init_system_aperture_regs(struct amdgpu_device *adev) 108af01d47dSLikun Gao { 109af01d47dSLikun Gao uint64_t value; 110af01d47dSLikun Gao 111af01d47dSLikun Gao /* Disable AGP. */ 112af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BASE, 0); 113af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_TOP, 0); 114af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BOT, 0x00FFFFFF); 115af01d47dSLikun Gao 116af01d47dSLikun Gao /* Program the system aperture low logical page number. */ 117af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_LOW_ADDR, 118af01d47dSLikun Gao adev->gmc.vram_start >> 18); 119af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 120af01d47dSLikun Gao adev->gmc.vram_end >> 18); 121af01d47dSLikun Gao 122af01d47dSLikun Gao /* Set default page address. */ 123af01d47dSLikun Gao value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start 124af01d47dSLikun Gao + adev->vm_manager.vram_base_offset; 125af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 126af01d47dSLikun Gao (u32)(value >> 12)); 127af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 128af01d47dSLikun Gao (u32)(value >> 44)); 129af01d47dSLikun Gao 130af01d47dSLikun Gao /* Program "protection fault". */ 131af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 132af01d47dSLikun Gao (u32)(adev->dummy_page_addr >> 12)); 133af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 134af01d47dSLikun Gao (u32)((u64)adev->dummy_page_addr >> 44)); 135af01d47dSLikun Gao 136af01d47dSLikun Gao WREG32_FIELD15(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2, 137af01d47dSLikun Gao ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 138af01d47dSLikun Gao } 139af01d47dSLikun Gao 140af01d47dSLikun Gao 141af01d47dSLikun Gao static void gfxhub_v2_1_init_tlb_regs(struct amdgpu_device *adev) 142af01d47dSLikun Gao { 143af01d47dSLikun Gao uint32_t tmp; 144af01d47dSLikun Gao 145af01d47dSLikun Gao /* Setup TLB control */ 146af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL); 147af01d47dSLikun Gao 148af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1); 149af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3); 150af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 151af01d47dSLikun Gao ENABLE_ADVANCED_DRIVER_MODEL, 1); 152af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 153af01d47dSLikun Gao SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 154af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0); 155af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 156af01d47dSLikun Gao MTYPE, MTYPE_UC); /* UC, uncached */ 157af01d47dSLikun Gao 158af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp); 159af01d47dSLikun Gao } 160af01d47dSLikun Gao 161af01d47dSLikun Gao static void gfxhub_v2_1_init_cache_regs(struct amdgpu_device *adev) 162af01d47dSLikun Gao { 163af01d47dSLikun Gao uint32_t tmp; 164af01d47dSLikun Gao 165af01d47dSLikun Gao /* Setup L2 cache */ 166af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL); 167af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1); 168af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0); 169af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 170af01d47dSLikun Gao ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); 171af01d47dSLikun Gao /* XXX for emulation, Refer to closed source code.*/ 172af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, 173af01d47dSLikun Gao L2_PDE0_CACHE_TAG_GENERATION_MODE, 0); 174af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0); 175af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1); 176af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0); 177af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL, tmp); 178af01d47dSLikun Gao 179af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2); 180af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1); 181af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 182af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2, tmp); 183af01d47dSLikun Gao 184af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL3_DEFAULT; 185af01d47dSLikun Gao if (adev->gmc.translate_further) { 186af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12); 187af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 188af01d47dSLikun Gao L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 189af01d47dSLikun Gao } else { 190af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9); 191af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, 192af01d47dSLikun Gao L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 193af01d47dSLikun Gao } 194af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, tmp); 195af01d47dSLikun Gao 196af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL4_DEFAULT; 197af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 198af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 199af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL4, tmp); 200af01d47dSLikun Gao 201af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL5_DEFAULT; 202af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0); 203af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL5, tmp); 204af01d47dSLikun Gao } 205af01d47dSLikun Gao 206af01d47dSLikun Gao static void gfxhub_v2_1_enable_system_domain(struct amdgpu_device *adev) 207af01d47dSLikun Gao { 208af01d47dSLikun Gao uint32_t tmp; 209af01d47dSLikun Gao 210af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL); 211af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 212af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0); 213af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, 214af01d47dSLikun Gao RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 215af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL, tmp); 216af01d47dSLikun Gao } 217af01d47dSLikun Gao 218af01d47dSLikun Gao static void gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device *adev) 219af01d47dSLikun Gao { 220af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 221af01d47dSLikun Gao 0xFFFFFFFF); 222af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 223af01d47dSLikun Gao 0x0000000F); 224af01d47dSLikun Gao 225af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 226af01d47dSLikun Gao 0); 227af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 228af01d47dSLikun Gao 0); 229af01d47dSLikun Gao 230af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0); 231af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0); 232af01d47dSLikun Gao 233af01d47dSLikun Gao } 234af01d47dSLikun Gao 235af01d47dSLikun Gao static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev) 236af01d47dSLikun Gao { 23713ae12d9SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 238af01d47dSLikun Gao int i; 239af01d47dSLikun Gao uint32_t tmp; 240af01d47dSLikun Gao 241af01d47dSLikun Gao for (i = 0; i <= 14; i++) { 242af01d47dSLikun Gao tmp = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, i); 243af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); 244af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 245af01d47dSLikun Gao adev->vm_manager.num_level); 246af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 247af01d47dSLikun Gao RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 248af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 249af01d47dSLikun Gao DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 250af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 251af01d47dSLikun Gao PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 252af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 253af01d47dSLikun Gao VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 254af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 255af01d47dSLikun Gao READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 256af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 257af01d47dSLikun Gao WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 258af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 259af01d47dSLikun Gao EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 260af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 261af01d47dSLikun Gao PAGE_TABLE_BLOCK_SIZE, 262af01d47dSLikun Gao adev->vm_manager.block_size - 9); 263af01d47dSLikun Gao /* Send no-retry XNACK on fault to suppress VM fault storm. */ 264af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, 265af01d47dSLikun Gao RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 266af01d47dSLikun Gao !amdgpu_noretry); 26713ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, 26813ae12d9SHuang Rui i * hub->ctx_distance, tmp); 26913ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 27013ae12d9SHuang Rui i * hub->ctx_addr_distance, 0); 27113ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 27213ae12d9SHuang Rui i * hub->ctx_addr_distance, 0); 27313ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 27413ae12d9SHuang Rui i * hub->ctx_addr_distance, 275af01d47dSLikun Gao lower_32_bits(adev->vm_manager.max_pfn - 1)); 27613ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 27713ae12d9SHuang Rui i * hub->ctx_addr_distance, 278af01d47dSLikun Gao upper_32_bits(adev->vm_manager.max_pfn - 1)); 279af01d47dSLikun Gao } 280af01d47dSLikun Gao } 281af01d47dSLikun Gao 282af01d47dSLikun Gao static void gfxhub_v2_1_program_invalidation(struct amdgpu_device *adev) 283af01d47dSLikun Gao { 28413ae12d9SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 285af01d47dSLikun Gao unsigned i; 286af01d47dSLikun Gao 287af01d47dSLikun Gao for (i = 0 ; i < 18; ++i) { 288af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 28913ae12d9SHuang Rui i * hub->eng_addr_distance, 0xffffffff); 290af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 29113ae12d9SHuang Rui i * hub->eng_addr_distance, 0x1f); 292af01d47dSLikun Gao } 293af01d47dSLikun Gao } 294af01d47dSLikun Gao 295af01d47dSLikun Gao int gfxhub_v2_1_gart_enable(struct amdgpu_device *adev) 296af01d47dSLikun Gao { 297af01d47dSLikun Gao if (amdgpu_sriov_vf(adev)) { 298af01d47dSLikun Gao /* 299af01d47dSLikun Gao * GCMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are 300af01d47dSLikun Gao * VF copy registers so vbios post doesn't program them, for 301af01d47dSLikun Gao * SRIOV driver need to program them 302af01d47dSLikun Gao */ 303af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE, 304af01d47dSLikun Gao adev->gmc.vram_start >> 24); 305af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_TOP, 306af01d47dSLikun Gao adev->gmc.vram_end >> 24); 307af01d47dSLikun Gao } 308af01d47dSLikun Gao 309af01d47dSLikun Gao /* GART Enable. */ 310af01d47dSLikun Gao gfxhub_v2_1_init_gart_aperture_regs(adev); 311af01d47dSLikun Gao gfxhub_v2_1_init_system_aperture_regs(adev); 312af01d47dSLikun Gao gfxhub_v2_1_init_tlb_regs(adev); 313af01d47dSLikun Gao gfxhub_v2_1_init_cache_regs(adev); 314af01d47dSLikun Gao 315af01d47dSLikun Gao gfxhub_v2_1_enable_system_domain(adev); 316af01d47dSLikun Gao gfxhub_v2_1_disable_identity_aperture(adev); 317af01d47dSLikun Gao gfxhub_v2_1_setup_vmid_config(adev); 318af01d47dSLikun Gao gfxhub_v2_1_program_invalidation(adev); 319af01d47dSLikun Gao 320af01d47dSLikun Gao return 0; 321af01d47dSLikun Gao } 322af01d47dSLikun Gao 323af01d47dSLikun Gao void gfxhub_v2_1_gart_disable(struct amdgpu_device *adev) 324af01d47dSLikun Gao { 32513ae12d9SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 326af01d47dSLikun Gao u32 tmp; 327af01d47dSLikun Gao u32 i; 328af01d47dSLikun Gao 329af01d47dSLikun Gao /* Disable all tables */ 330af01d47dSLikun Gao for (i = 0; i < 16; i++) 33113ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL, 33213ae12d9SHuang Rui i * hub->ctx_distance, 0); 333af01d47dSLikun Gao 334af01d47dSLikun Gao /* Setup TLB control */ 335af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL); 336af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); 337af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, 338af01d47dSLikun Gao ENABLE_ADVANCED_DRIVER_MODEL, 0); 339af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp); 340af01d47dSLikun Gao 341af01d47dSLikun Gao /* Setup L2 cache */ 342af01d47dSLikun Gao WREG32_FIELD15(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0); 343af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, 0); 344af01d47dSLikun Gao } 345af01d47dSLikun Gao 346af01d47dSLikun Gao /** 347af01d47dSLikun Gao * gfxhub_v2_1_set_fault_enable_default - update GART/VM fault handling 348af01d47dSLikun Gao * 349af01d47dSLikun Gao * @adev: amdgpu_device pointer 350af01d47dSLikun Gao * @value: true redirects VM faults to the default page 351af01d47dSLikun Gao */ 352af01d47dSLikun Gao void gfxhub_v2_1_set_fault_enable_default(struct amdgpu_device *adev, 353af01d47dSLikun Gao bool value) 354af01d47dSLikun Gao { 355af01d47dSLikun Gao u32 tmp; 356af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL); 357af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 358af01d47dSLikun Gao RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 359af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 360af01d47dSLikun Gao PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 361af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 362af01d47dSLikun Gao PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 363af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 364af01d47dSLikun Gao PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 365af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 366af01d47dSLikun Gao TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 367af01d47dSLikun Gao value); 368af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 369af01d47dSLikun Gao NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 370af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 371af01d47dSLikun Gao DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 372af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 373af01d47dSLikun Gao VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 374af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 375af01d47dSLikun Gao READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 376af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 377af01d47dSLikun Gao WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 378af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 379af01d47dSLikun Gao EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 380af01d47dSLikun Gao if (!value) { 381af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 382af01d47dSLikun Gao CRASH_ON_NO_RETRY_FAULT, 1); 383af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL, 384af01d47dSLikun Gao CRASH_ON_RETRY_FAULT, 1); 385af01d47dSLikun Gao } 386af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, tmp); 387af01d47dSLikun Gao } 388af01d47dSLikun Gao 3892577db91SHuang Rui static const struct amdgpu_vmhub_funcs gfxhub_v2_1_vmhub_funcs = { 3902577db91SHuang Rui .print_l2_protection_fault_status = gfxhub_v2_1_print_l2_protection_fault_status, 3912577db91SHuang Rui }; 3922577db91SHuang Rui 393af01d47dSLikun Gao void gfxhub_v2_1_init(struct amdgpu_device *adev) 394af01d47dSLikun Gao { 395af01d47dSLikun Gao struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 396af01d47dSLikun Gao 397af01d47dSLikun Gao hub->ctx0_ptb_addr_lo32 = 398af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, 399af01d47dSLikun Gao mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 400af01d47dSLikun Gao hub->ctx0_ptb_addr_hi32 = 401af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, 402af01d47dSLikun Gao mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 403af01d47dSLikun Gao hub->vm_inv_eng0_sem = 404af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_SEM); 405af01d47dSLikun Gao hub->vm_inv_eng0_req = 406af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_REQ); 407af01d47dSLikun Gao hub->vm_inv_eng0_ack = 408af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ACK); 409af01d47dSLikun Gao hub->vm_context0_cntl = 410af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL); 411af01d47dSLikun Gao hub->vm_l2_pro_fault_status = 412af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_STATUS); 413af01d47dSLikun Gao hub->vm_l2_pro_fault_cntl = 414af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL); 4151f9d56c3SHuang Rui 4161f9d56c3SHuang Rui hub->ctx_distance = mmGCVM_CONTEXT1_CNTL - mmGCVM_CONTEXT0_CNTL; 4171f9d56c3SHuang Rui hub->ctx_addr_distance = mmGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 4181f9d56c3SHuang Rui mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 4191f9d56c3SHuang Rui hub->eng_distance = mmGCVM_INVALIDATE_ENG1_REQ - 4201f9d56c3SHuang Rui mmGCVM_INVALIDATE_ENG0_REQ; 4211f9d56c3SHuang Rui hub->eng_addr_distance = mmGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 4221f9d56c3SHuang Rui mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 4235befb6fcSHuang Rui 4245befb6fcSHuang Rui hub->vm_cntx_cntl_vm_fault = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4255befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4265befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4275befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4285befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4295befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4305befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 4312577db91SHuang Rui 4322577db91SHuang Rui hub->vmhub_funcs = &gfxhub_v2_1_vmhub_funcs; 433af01d47dSLikun Gao } 434fdb8483bSJohn Clements 435fdb8483bSJohn Clements int gfxhub_v2_1_get_xgmi_info(struct amdgpu_device *adev) 436fdb8483bSJohn Clements { 437fdb8483bSJohn Clements u32 xgmi_lfb_cntl = RREG32_SOC15(GC, 0, mmGCMC_VM_XGMI_LFB_CNTL); 438fdb8483bSJohn Clements u32 max_region = 439fdb8483bSJohn Clements REG_GET_FIELD(xgmi_lfb_cntl, GCMC_VM_XGMI_LFB_CNTL, PF_MAX_REGION); 440fdb8483bSJohn Clements u32 max_num_physical_nodes = 0; 441fdb8483bSJohn Clements u32 max_physical_node_id = 0; 442fdb8483bSJohn Clements 443fdb8483bSJohn Clements switch (adev->asic_type) { 444fdb8483bSJohn Clements case CHIP_SIENNA_CICHLID: 445fdb8483bSJohn Clements max_num_physical_nodes = 4; 446fdb8483bSJohn Clements max_physical_node_id = 3; 447fdb8483bSJohn Clements break; 448fdb8483bSJohn Clements default: 449fdb8483bSJohn Clements return -EINVAL; 450fdb8483bSJohn Clements } 451fdb8483bSJohn Clements 452fdb8483bSJohn Clements /* PF_MAX_REGION=0 means xgmi is disabled */ 453fdb8483bSJohn Clements if (max_region) { 454fdb8483bSJohn Clements adev->gmc.xgmi.num_physical_nodes = max_region + 1; 455fdb8483bSJohn Clements if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes) 456fdb8483bSJohn Clements return -EINVAL; 457fdb8483bSJohn Clements 458fdb8483bSJohn Clements adev->gmc.xgmi.physical_node_id = 459fdb8483bSJohn Clements REG_GET_FIELD(xgmi_lfb_cntl, GCMC_VM_XGMI_LFB_CNTL, PF_LFB_REGION); 460fdb8483bSJohn Clements if (adev->gmc.xgmi.physical_node_id > max_physical_node_id) 461fdb8483bSJohn Clements return -EINVAL; 462fdb8483bSJohn Clements 463fdb8483bSJohn Clements adev->gmc.xgmi.node_segment_size = REG_GET_FIELD( 464fdb8483bSJohn Clements RREG32_SOC15(GC, 0, mmGCMC_VM_XGMI_LFB_SIZE), 465fdb8483bSJohn Clements GCMC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24; 466fdb8483bSJohn Clements } 467fdb8483bSJohn Clements 468fdb8483bSJohn Clements return 0; 469fdb8483bSJohn Clements } 470