198a0f868STianci.Yin /* 298a0f868STianci.Yin * Copyright 2021 Advanced Micro Devices, Inc. 398a0f868STianci.Yin * 498a0f868STianci.Yin * Permission is hereby granted, free of charge, to any person obtaining a 598a0f868STianci.Yin * copy of this software and associated documentation files (the "Software"), 698a0f868STianci.Yin * to deal in the Software without restriction, including without limitation 798a0f868STianci.Yin * the rights to use, copy, modify, merge, publish, distribute, sublicense, 898a0f868STianci.Yin * and/or sell copies of the Software, and to permit persons to whom the 998a0f868STianci.Yin * Software is furnished to do so, subject to the following conditions: 1098a0f868STianci.Yin * 1198a0f868STianci.Yin * The above copyright notice and this permission notice shall be included in 1298a0f868STianci.Yin * all copies or substantial portions of the Software. 1398a0f868STianci.Yin * 1498a0f868STianci.Yin * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 1598a0f868STianci.Yin * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 1698a0f868STianci.Yin * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 1798a0f868STianci.Yin * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 1898a0f868STianci.Yin * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 1998a0f868STianci.Yin * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 2098a0f868STianci.Yin * OTHER DEALINGS IN THE SOFTWARE. 2198a0f868STianci.Yin * 2298a0f868STianci.Yin */ 2398a0f868STianci.Yin 2498a0f868STianci.Yin #include "amdgpu.h" 2598a0f868STianci.Yin #include "mmhub_v3_0.h" 2698a0f868STianci.Yin 2798a0f868STianci.Yin #include "mmhub/mmhub_3_0_0_offset.h" 2898a0f868STianci.Yin #include "mmhub/mmhub_3_0_0_sh_mask.h" 2998a0f868STianci.Yin #include "navi10_enum.h" 3098a0f868STianci.Yin 3198a0f868STianci.Yin #include "soc15_common.h" 3298a0f868STianci.Yin 3398a0f868STianci.Yin #define regMMVM_L2_CNTL3_DEFAULT 0x80100007 3498a0f868STianci.Yin #define regMMVM_L2_CNTL4_DEFAULT 0x000000c1 3598a0f868STianci.Yin #define regMMVM_L2_CNTL5_DEFAULT 0x00003fe0 3698a0f868STianci.Yin 3798a0f868STianci.Yin static const char *mmhub_client_ids_v3_0_0[][2] = { 3898a0f868STianci.Yin [0][0] = "VMC", 3998a0f868STianci.Yin [4][0] = "DCEDMC", 4098a0f868STianci.Yin [5][0] = "DCEVGA", 4198a0f868STianci.Yin [6][0] = "MP0", 4298a0f868STianci.Yin [7][0] = "MP1", 4398a0f868STianci.Yin [8][0] = "MPIO", 4498a0f868STianci.Yin [16][0] = "HDP", 4598a0f868STianci.Yin [17][0] = "LSDMA", 4698a0f868STianci.Yin [18][0] = "JPEG", 4798a0f868STianci.Yin [19][0] = "VCNU0", 4898a0f868STianci.Yin [21][0] = "VSCH", 4998a0f868STianci.Yin [22][0] = "VCNU1", 5098a0f868STianci.Yin [23][0] = "VCN1", 5198a0f868STianci.Yin [32+20][0] = "VCN0", 5298a0f868STianci.Yin [2][1] = "DBGUNBIO", 5398a0f868STianci.Yin [3][1] = "DCEDWB", 5498a0f868STianci.Yin [4][1] = "DCEDMC", 5598a0f868STianci.Yin [5][1] = "DCEVGA", 5698a0f868STianci.Yin [6][1] = "MP0", 5798a0f868STianci.Yin [7][1] = "MP1", 5898a0f868STianci.Yin [8][1] = "MPIO", 5998a0f868STianci.Yin [10][1] = "DBGU0", 6098a0f868STianci.Yin [11][1] = "DBGU1", 6198a0f868STianci.Yin [12][1] = "DBGU2", 6298a0f868STianci.Yin [13][1] = "DBGU3", 6398a0f868STianci.Yin [14][1] = "XDP", 6498a0f868STianci.Yin [15][1] = "OSSSYS", 6598a0f868STianci.Yin [16][1] = "HDP", 6698a0f868STianci.Yin [17][1] = "LSDMA", 6798a0f868STianci.Yin [18][1] = "JPEG", 6898a0f868STianci.Yin [19][1] = "VCNU0", 6998a0f868STianci.Yin [20][1] = "VCN0", 7098a0f868STianci.Yin [21][1] = "VSCH", 7198a0f868STianci.Yin [22][1] = "VCNU1", 7298a0f868STianci.Yin [23][1] = "VCN1", 7398a0f868STianci.Yin }; 7498a0f868STianci.Yin 7598a0f868STianci.Yin static uint32_t mmhub_v3_0_get_invalidate_req(unsigned int vmid, 7698a0f868STianci.Yin uint32_t flush_type) 7798a0f868STianci.Yin { 7898a0f868STianci.Yin u32 req = 0; 7998a0f868STianci.Yin 8098a0f868STianci.Yin /* invalidate using legacy mode on vmid*/ 8198a0f868STianci.Yin req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, 8298a0f868STianci.Yin PER_VMID_INVALIDATE_REQ, 1 << vmid); 8398a0f868STianci.Yin req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 8498a0f868STianci.Yin req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 8598a0f868STianci.Yin req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 8698a0f868STianci.Yin req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 8798a0f868STianci.Yin req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 8898a0f868STianci.Yin req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 8998a0f868STianci.Yin req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, 9098a0f868STianci.Yin CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 9198a0f868STianci.Yin 9298a0f868STianci.Yin return req; 9398a0f868STianci.Yin } 9498a0f868STianci.Yin 9598a0f868STianci.Yin static void 9698a0f868STianci.Yin mmhub_v3_0_print_l2_protection_fault_status(struct amdgpu_device *adev, 9798a0f868STianci.Yin uint32_t status) 9898a0f868STianci.Yin { 9998a0f868STianci.Yin uint32_t cid, rw; 10098a0f868STianci.Yin const char *mmhub_cid = NULL; 10198a0f868STianci.Yin 10298a0f868STianci.Yin cid = REG_GET_FIELD(status, 10398a0f868STianci.Yin MMVM_L2_PROTECTION_FAULT_STATUS, CID); 10498a0f868STianci.Yin rw = REG_GET_FIELD(status, 10598a0f868STianci.Yin MMVM_L2_PROTECTION_FAULT_STATUS, RW); 10698a0f868STianci.Yin 10798a0f868STianci.Yin dev_err(adev->dev, 10898a0f868STianci.Yin "MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 10998a0f868STianci.Yin status); 11098a0f868STianci.Yin switch (adev->ip_versions[MMHUB_HWIP][0]) { 11198a0f868STianci.Yin case IP_VERSION(3, 0, 0): 112*50e9f54dSHuang Rui case IP_VERSION(3, 0, 1): 11398a0f868STianci.Yin mmhub_cid = mmhub_client_ids_v3_0_0[cid][rw]; 11498a0f868STianci.Yin break; 11598a0f868STianci.Yin default: 11698a0f868STianci.Yin mmhub_cid = NULL; 11798a0f868STianci.Yin break; 11898a0f868STianci.Yin } 11998a0f868STianci.Yin dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 12098a0f868STianci.Yin mmhub_cid ? mmhub_cid : "unknown", cid); 12198a0f868STianci.Yin dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 12298a0f868STianci.Yin REG_GET_FIELD(status, 12398a0f868STianci.Yin MMVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 12498a0f868STianci.Yin dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 12598a0f868STianci.Yin REG_GET_FIELD(status, 12698a0f868STianci.Yin MMVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 12798a0f868STianci.Yin dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 12898a0f868STianci.Yin REG_GET_FIELD(status, 12998a0f868STianci.Yin MMVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 13098a0f868STianci.Yin dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 13198a0f868STianci.Yin REG_GET_FIELD(status, 13298a0f868STianci.Yin MMVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 13398a0f868STianci.Yin dev_err(adev->dev, "\t RW: 0x%x\n", rw); 13498a0f868STianci.Yin } 13598a0f868STianci.Yin 13698a0f868STianci.Yin static void mmhub_v3_0_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid, 13798a0f868STianci.Yin uint64_t page_table_base) 13898a0f868STianci.Yin { 13998a0f868STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 14098a0f868STianci.Yin 14198a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 14298a0f868STianci.Yin hub->ctx_addr_distance * vmid, 14398a0f868STianci.Yin lower_32_bits(page_table_base)); 14498a0f868STianci.Yin 14598a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 14698a0f868STianci.Yin hub->ctx_addr_distance * vmid, 14798a0f868STianci.Yin upper_32_bits(page_table_base)); 14898a0f868STianci.Yin } 14998a0f868STianci.Yin 15098a0f868STianci.Yin static void mmhub_v3_0_init_gart_aperture_regs(struct amdgpu_device *adev) 15198a0f868STianci.Yin { 15298a0f868STianci.Yin uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 15398a0f868STianci.Yin 15498a0f868STianci.Yin mmhub_v3_0_setup_vm_pt_regs(adev, 0, pt_base); 15598a0f868STianci.Yin 15698a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 15798a0f868STianci.Yin (u32)(adev->gmc.gart_start >> 12)); 15898a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 15998a0f868STianci.Yin (u32)(adev->gmc.gart_start >> 44)); 16098a0f868STianci.Yin 16198a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 16298a0f868STianci.Yin (u32)(adev->gmc.gart_end >> 12)); 16398a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 16498a0f868STianci.Yin (u32)(adev->gmc.gart_end >> 44)); 16598a0f868STianci.Yin } 16698a0f868STianci.Yin 16798a0f868STianci.Yin static void mmhub_v3_0_init_system_aperture_regs(struct amdgpu_device *adev) 16898a0f868STianci.Yin { 16998a0f868STianci.Yin uint64_t value; 17098a0f868STianci.Yin uint32_t tmp; 17198a0f868STianci.Yin 17298a0f868STianci.Yin /* Disable AGP. */ 17398a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BASE, 0); 17498a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_TOP, 0); 17598a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_AGP_BOT, 0x00FFFFFF); 17698a0f868STianci.Yin 17798a0f868STianci.Yin if (!amdgpu_sriov_vf(adev)) { 17898a0f868STianci.Yin /* 17998a0f868STianci.Yin * the new L1 policy will block SRIOV guest from writing 18098a0f868STianci.Yin * these regs, and they will be programed at host. 18198a0f868STianci.Yin * so skip programing these regs. 18298a0f868STianci.Yin */ 18398a0f868STianci.Yin /* Program the system aperture low logical page number. */ 18498a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_LOW_ADDR, 18598a0f868STianci.Yin adev->gmc.vram_start >> 18); 18698a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 18798a0f868STianci.Yin adev->gmc.vram_end >> 18); 18898a0f868STianci.Yin } 18998a0f868STianci.Yin 19098a0f868STianci.Yin /* Set default page address. */ 19198a0f868STianci.Yin value = adev->vram_scratch.gpu_addr - adev->gmc.vram_start + 19298a0f868STianci.Yin adev->vm_manager.vram_base_offset; 19398a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 19498a0f868STianci.Yin (u32)(value >> 12)); 19598a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 19698a0f868STianci.Yin (u32)(value >> 44)); 19798a0f868STianci.Yin 19898a0f868STianci.Yin /* Program "protection fault". */ 19998a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 20098a0f868STianci.Yin (u32)(adev->dummy_page_addr >> 12)); 20198a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 20298a0f868STianci.Yin (u32)((u64)adev->dummy_page_addr >> 44)); 20398a0f868STianci.Yin 20498a0f868STianci.Yin tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL2); 20598a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2, 20698a0f868STianci.Yin ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 20798a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL2, tmp); 20898a0f868STianci.Yin } 20998a0f868STianci.Yin 21098a0f868STianci.Yin static void mmhub_v3_0_init_tlb_regs(struct amdgpu_device *adev) 21198a0f868STianci.Yin { 21298a0f868STianci.Yin uint32_t tmp; 21398a0f868STianci.Yin 21498a0f868STianci.Yin /* Setup TLB control */ 21598a0f868STianci.Yin tmp = RREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL); 21698a0f868STianci.Yin 21798a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1); 21898a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3); 21998a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, 22098a0f868STianci.Yin ENABLE_ADVANCED_DRIVER_MODEL, 1); 22198a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, 22298a0f868STianci.Yin SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 22398a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ECO_BITS, 0); 22498a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, 22598a0f868STianci.Yin MTYPE, MTYPE_UC); /* UC, uncached */ 22698a0f868STianci.Yin 22798a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL, tmp); 22898a0f868STianci.Yin } 22998a0f868STianci.Yin 23098a0f868STianci.Yin static void mmhub_v3_0_init_cache_regs(struct amdgpu_device *adev) 23198a0f868STianci.Yin { 23298a0f868STianci.Yin uint32_t tmp; 23398a0f868STianci.Yin 23498a0f868STianci.Yin /* These registers are not accessible to VF-SRIOV. 23598a0f868STianci.Yin * The PF will program them instead. 23698a0f868STianci.Yin */ 23798a0f868STianci.Yin if (amdgpu_sriov_vf(adev)) 23898a0f868STianci.Yin return; 23998a0f868STianci.Yin 24098a0f868STianci.Yin /* Setup L2 cache */ 24198a0f868STianci.Yin tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL); 24298a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 1); 24398a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0); 24498a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, 24598a0f868STianci.Yin ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); 24698a0f868STianci.Yin /* XXX for emulation, Refer to closed source code.*/ 24798a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE, 24898a0f868STianci.Yin 0); 24998a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0); 25098a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1); 25198a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0); 25298a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL, tmp); 25398a0f868STianci.Yin 25498a0f868STianci.Yin tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL2); 25598a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1); 25698a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 25798a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL2, tmp); 25898a0f868STianci.Yin 25998a0f868STianci.Yin tmp = regMMVM_L2_CNTL3_DEFAULT; 26098a0f868STianci.Yin if (adev->gmc.translate_further) { 26198a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 12); 26298a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, 26398a0f868STianci.Yin L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 26498a0f868STianci.Yin } else { 26598a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 9); 26698a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, 26798a0f868STianci.Yin L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 26898a0f868STianci.Yin } 26998a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL3, tmp); 27098a0f868STianci.Yin 27198a0f868STianci.Yin tmp = regMMVM_L2_CNTL4_DEFAULT; 27298a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 27398a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 27498a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL4, tmp); 27598a0f868STianci.Yin 27698a0f868STianci.Yin tmp = regMMVM_L2_CNTL5_DEFAULT; 27798a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0); 27898a0f868STianci.Yin WREG32_SOC15(GC, 0, regMMVM_L2_CNTL5, tmp); 27998a0f868STianci.Yin } 28098a0f868STianci.Yin 28198a0f868STianci.Yin static void mmhub_v3_0_enable_system_domain(struct amdgpu_device *adev) 28298a0f868STianci.Yin { 28398a0f868STianci.Yin uint32_t tmp; 28498a0f868STianci.Yin 28598a0f868STianci.Yin tmp = RREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_CNTL); 28698a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 28798a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0); 28898a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, 28998a0f868STianci.Yin RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 29098a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_CONTEXT0_CNTL, tmp); 29198a0f868STianci.Yin } 29298a0f868STianci.Yin 29398a0f868STianci.Yin static void mmhub_v3_0_disable_identity_aperture(struct amdgpu_device *adev) 29498a0f868STianci.Yin { 29598a0f868STianci.Yin /* These registers are not accessible to VF-SRIOV. 29698a0f868STianci.Yin * The PF will program them instead. 29798a0f868STianci.Yin */ 29898a0f868STianci.Yin if (amdgpu_sriov_vf(adev)) 29998a0f868STianci.Yin return; 30098a0f868STianci.Yin 30198a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, 30298a0f868STianci.Yin regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 30398a0f868STianci.Yin 0xFFFFFFFF); 30498a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, 30598a0f868STianci.Yin regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 30698a0f868STianci.Yin 0x0000000F); 30798a0f868STianci.Yin 30898a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, 30998a0f868STianci.Yin regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0); 31098a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, 31198a0f868STianci.Yin regMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0); 31298a0f868STianci.Yin 31398a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 31498a0f868STianci.Yin 0); 31598a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 31698a0f868STianci.Yin 0); 31798a0f868STianci.Yin } 31898a0f868STianci.Yin 31998a0f868STianci.Yin static void mmhub_v3_0_setup_vmid_config(struct amdgpu_device *adev) 32098a0f868STianci.Yin { 32198a0f868STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 32298a0f868STianci.Yin int i; 32398a0f868STianci.Yin uint32_t tmp; 32498a0f868STianci.Yin 32598a0f868STianci.Yin for (i = 0; i <= 14; i++) { 32698a0f868STianci.Yin tmp = RREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, i); 32798a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); 32898a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 32998a0f868STianci.Yin adev->vm_manager.num_level); 33098a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 33198a0f868STianci.Yin RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 33298a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 33398a0f868STianci.Yin DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 33498a0f868STianci.Yin 1); 33598a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 33698a0f868STianci.Yin PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 33798a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 33898a0f868STianci.Yin VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 33998a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 34098a0f868STianci.Yin READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 34198a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 34298a0f868STianci.Yin WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 34398a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 34498a0f868STianci.Yin EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 34598a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 34698a0f868STianci.Yin PAGE_TABLE_BLOCK_SIZE, 34798a0f868STianci.Yin adev->vm_manager.block_size - 9); 34898a0f868STianci.Yin /* Send no-retry XNACK on fault to suppress VM fault storm. */ 34998a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 35098a0f868STianci.Yin RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 35198a0f868STianci.Yin !amdgpu_noretry); 35298a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_CNTL, 35398a0f868STianci.Yin i * hub->ctx_distance, tmp); 35498a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 35598a0f868STianci.Yin i * hub->ctx_addr_distance, 0); 35698a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 35798a0f868STianci.Yin i * hub->ctx_addr_distance, 0); 35898a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 35998a0f868STianci.Yin i * hub->ctx_addr_distance, 36098a0f868STianci.Yin lower_32_bits(adev->vm_manager.max_pfn - 1)); 36198a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 36298a0f868STianci.Yin i * hub->ctx_addr_distance, 36398a0f868STianci.Yin upper_32_bits(adev->vm_manager.max_pfn - 1)); 36498a0f868STianci.Yin } 365d7dab4fcSJack Xiao 366d7dab4fcSJack Xiao hub->vm_cntx_cntl = tmp; 36798a0f868STianci.Yin } 36898a0f868STianci.Yin 36998a0f868STianci.Yin static void mmhub_v3_0_program_invalidation(struct amdgpu_device *adev) 37098a0f868STianci.Yin { 37198a0f868STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 37298a0f868STianci.Yin unsigned i; 37398a0f868STianci.Yin 37498a0f868STianci.Yin for (i = 0; i < 18; ++i) { 37598a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 37698a0f868STianci.Yin i * hub->eng_addr_distance, 0xffffffff); 37798a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 37898a0f868STianci.Yin i * hub->eng_addr_distance, 0x1f); 37998a0f868STianci.Yin } 38098a0f868STianci.Yin } 38198a0f868STianci.Yin 38298a0f868STianci.Yin static int mmhub_v3_0_gart_enable(struct amdgpu_device *adev) 38398a0f868STianci.Yin { 38498a0f868STianci.Yin /* GART Enable. */ 38598a0f868STianci.Yin mmhub_v3_0_init_gart_aperture_regs(adev); 38698a0f868STianci.Yin mmhub_v3_0_init_system_aperture_regs(adev); 38798a0f868STianci.Yin mmhub_v3_0_init_tlb_regs(adev); 38898a0f868STianci.Yin mmhub_v3_0_init_cache_regs(adev); 38998a0f868STianci.Yin 39098a0f868STianci.Yin mmhub_v3_0_enable_system_domain(adev); 39198a0f868STianci.Yin mmhub_v3_0_disable_identity_aperture(adev); 39298a0f868STianci.Yin mmhub_v3_0_setup_vmid_config(adev); 39398a0f868STianci.Yin mmhub_v3_0_program_invalidation(adev); 39498a0f868STianci.Yin 39598a0f868STianci.Yin return 0; 39698a0f868STianci.Yin } 39798a0f868STianci.Yin 39898a0f868STianci.Yin static void mmhub_v3_0_gart_disable(struct amdgpu_device *adev) 39998a0f868STianci.Yin { 40098a0f868STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 40198a0f868STianci.Yin u32 tmp; 40298a0f868STianci.Yin u32 i; 40398a0f868STianci.Yin 40498a0f868STianci.Yin /* Disable all tables */ 40598a0f868STianci.Yin for (i = 0; i < 16; i++) 40698a0f868STianci.Yin WREG32_SOC15_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_CNTL, 40798a0f868STianci.Yin i * hub->ctx_distance, 0); 40898a0f868STianci.Yin 40998a0f868STianci.Yin /* Setup TLB control */ 41098a0f868STianci.Yin tmp = RREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL); 41198a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); 41298a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, 41398a0f868STianci.Yin ENABLE_ADVANCED_DRIVER_MODEL, 0); 41498a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMMC_VM_MX_L1_TLB_CNTL, tmp); 41598a0f868STianci.Yin 41698a0f868STianci.Yin /* Setup L2 cache */ 41798a0f868STianci.Yin tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL); 41898a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 0); 41998a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL, tmp); 42098a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_CNTL3, 0); 42198a0f868STianci.Yin } 42298a0f868STianci.Yin 42398a0f868STianci.Yin /** 42498a0f868STianci.Yin * mmhub_v3_0_set_fault_enable_default - update GART/VM fault handling 42598a0f868STianci.Yin * 42698a0f868STianci.Yin * @adev: amdgpu_device pointer 42798a0f868STianci.Yin * @value: true redirects VM faults to the default page 42898a0f868STianci.Yin */ 42998a0f868STianci.Yin static void mmhub_v3_0_set_fault_enable_default(struct amdgpu_device *adev, bool value) 43098a0f868STianci.Yin { 43198a0f868STianci.Yin u32 tmp; 43298a0f868STianci.Yin 43398a0f868STianci.Yin /* These registers are not accessible to VF-SRIOV. 43498a0f868STianci.Yin * The PF will program them instead. 43598a0f868STianci.Yin */ 43698a0f868STianci.Yin if (amdgpu_sriov_vf(adev)) 43798a0f868STianci.Yin return; 43898a0f868STianci.Yin 43998a0f868STianci.Yin tmp = RREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL); 44098a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 44198a0f868STianci.Yin RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 44298a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 44398a0f868STianci.Yin PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 44498a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 44598a0f868STianci.Yin PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 44698a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 44798a0f868STianci.Yin PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 44898a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 44998a0f868STianci.Yin TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 45098a0f868STianci.Yin value); 45198a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 45298a0f868STianci.Yin NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 45398a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 45498a0f868STianci.Yin DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 45598a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 45698a0f868STianci.Yin VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 45798a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 45898a0f868STianci.Yin READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 45998a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 46098a0f868STianci.Yin WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 46198a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 46298a0f868STianci.Yin EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 46398a0f868STianci.Yin if (!value) { 46498a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 46598a0f868STianci.Yin CRASH_ON_NO_RETRY_FAULT, 1); 46698a0f868STianci.Yin tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 46798a0f868STianci.Yin CRASH_ON_RETRY_FAULT, 1); 46898a0f868STianci.Yin } 46998a0f868STianci.Yin WREG32_SOC15(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL, tmp); 47098a0f868STianci.Yin } 47198a0f868STianci.Yin 47298a0f868STianci.Yin static const struct amdgpu_vmhub_funcs mmhub_v3_0_vmhub_funcs = { 47398a0f868STianci.Yin .print_l2_protection_fault_status = mmhub_v3_0_print_l2_protection_fault_status, 47498a0f868STianci.Yin .get_invalidate_req = mmhub_v3_0_get_invalidate_req, 47598a0f868STianci.Yin }; 47698a0f868STianci.Yin 47798a0f868STianci.Yin static void mmhub_v3_0_init(struct amdgpu_device *adev) 47898a0f868STianci.Yin { 47998a0f868STianci.Yin struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 48098a0f868STianci.Yin 48198a0f868STianci.Yin hub->ctx0_ptb_addr_lo32 = 48298a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, 48398a0f868STianci.Yin regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 48498a0f868STianci.Yin hub->ctx0_ptb_addr_hi32 = 48598a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, 48698a0f868STianci.Yin regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 48798a0f868STianci.Yin hub->vm_inv_eng0_sem = 48898a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_SEM); 48998a0f868STianci.Yin hub->vm_inv_eng0_req = 49098a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_REQ); 49198a0f868STianci.Yin hub->vm_inv_eng0_ack = 49298a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, regMMVM_INVALIDATE_ENG0_ACK); 49398a0f868STianci.Yin hub->vm_context0_cntl = 49498a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, regMMVM_CONTEXT0_CNTL); 49598a0f868STianci.Yin hub->vm_l2_pro_fault_status = 49698a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_STATUS); 49798a0f868STianci.Yin hub->vm_l2_pro_fault_cntl = 49898a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_PROTECTION_FAULT_CNTL); 49998a0f868STianci.Yin 50098a0f868STianci.Yin hub->ctx_distance = regMMVM_CONTEXT1_CNTL - regMMVM_CONTEXT0_CNTL; 50198a0f868STianci.Yin hub->ctx_addr_distance = regMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 50298a0f868STianci.Yin regMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 50398a0f868STianci.Yin hub->eng_distance = regMMVM_INVALIDATE_ENG1_REQ - 50498a0f868STianci.Yin regMMVM_INVALIDATE_ENG0_REQ; 50598a0f868STianci.Yin hub->eng_addr_distance = regMMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 50698a0f868STianci.Yin regMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 50798a0f868STianci.Yin 50898a0f868STianci.Yin hub->vm_cntx_cntl_vm_fault = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 50998a0f868STianci.Yin MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 51098a0f868STianci.Yin MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 51198a0f868STianci.Yin MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 51298a0f868STianci.Yin MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 51398a0f868STianci.Yin MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 51498a0f868STianci.Yin MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 51598a0f868STianci.Yin 51698a0f868STianci.Yin hub->vm_l2_bank_select_reserved_cid2 = 51798a0f868STianci.Yin SOC15_REG_OFFSET(MMHUB, 0, regMMVM_L2_BANK_SELECT_RESERVED_CID2); 51898a0f868STianci.Yin 51998a0f868STianci.Yin hub->vmhub_funcs = &mmhub_v3_0_vmhub_funcs; 52098a0f868STianci.Yin } 52198a0f868STianci.Yin 52298a0f868STianci.Yin static u64 mmhub_v3_0_get_fb_location(struct amdgpu_device *adev) 52398a0f868STianci.Yin { 52498a0f868STianci.Yin u64 base; 52598a0f868STianci.Yin 52698a0f868STianci.Yin base = RREG32_SOC15(MMHUB, 0, regMMMC_VM_FB_LOCATION_BASE); 52798a0f868STianci.Yin 52898a0f868STianci.Yin base &= MMMC_VM_FB_LOCATION_BASE__FB_BASE_MASK; 52998a0f868STianci.Yin base <<= 24; 53098a0f868STianci.Yin 53198a0f868STianci.Yin return base; 53298a0f868STianci.Yin } 53398a0f868STianci.Yin 53498a0f868STianci.Yin static u64 mmhub_v3_0_get_mc_fb_offset(struct amdgpu_device *adev) 53598a0f868STianci.Yin { 53698a0f868STianci.Yin return (u64)RREG32_SOC15(MMHUB, 0, regMMMC_VM_FB_OFFSET) << 24; 53798a0f868STianci.Yin } 53898a0f868STianci.Yin 53998a0f868STianci.Yin static void mmhub_v3_0_update_medium_grain_clock_gating(struct amdgpu_device *adev, 54098a0f868STianci.Yin bool enable) 54198a0f868STianci.Yin { 5427ccf6eb0SEvan Quan uint32_t def, data; 5437ccf6eb0SEvan Quan #if 0 5447ccf6eb0SEvan Quan uint32_t def1, data1, def2 = 0, data2 = 0; 5457ccf6eb0SEvan Quan #endif 5467ccf6eb0SEvan Quan 5477ccf6eb0SEvan Quan def = data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG); 5487ccf6eb0SEvan Quan #if 0 5497ccf6eb0SEvan Quan def1 = data1 = RREG32_SOC15(MMHUB, 0, regDAGB0_CNTL_MISC2); 5507ccf6eb0SEvan Quan def2 = data2 = RREG32_SOC15(MMHUB, 0, regDAGB1_CNTL_MISC2); 5517ccf6eb0SEvan Quan #endif 5527ccf6eb0SEvan Quan 5537ccf6eb0SEvan Quan if (enable) { 5547ccf6eb0SEvan Quan data |= MM_ATC_L2_MISC_CG__ENABLE_MASK; 5557ccf6eb0SEvan Quan #if 0 5567ccf6eb0SEvan Quan data1 &= ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | 5577ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK | 5587ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | 5597ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK | 5607ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | 5617ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); 5627ccf6eb0SEvan Quan 5637ccf6eb0SEvan Quan data2 &= ~(DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | 5647ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK | 5657ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | 5667ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK | 5677ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | 5687ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); 5697ccf6eb0SEvan Quan #endif 5707ccf6eb0SEvan Quan } else { 5717ccf6eb0SEvan Quan data &= ~MM_ATC_L2_MISC_CG__ENABLE_MASK; 5727ccf6eb0SEvan Quan #if 0 5737ccf6eb0SEvan Quan data1 |= (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | 5747ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK | 5757ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | 5767ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK | 5777ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | 5787ccf6eb0SEvan Quan DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); 5797ccf6eb0SEvan Quan 5807ccf6eb0SEvan Quan data2 |= (DAGB1_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | 5817ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_WRRET_CG_MASK | 5827ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | 5837ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_RDRET_CG_MASK | 5847ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | 5857ccf6eb0SEvan Quan DAGB1_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); 5867ccf6eb0SEvan Quan #endif 5877ccf6eb0SEvan Quan } 5887ccf6eb0SEvan Quan 5897ccf6eb0SEvan Quan if (def != data) 5907ccf6eb0SEvan Quan WREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG, data); 5917ccf6eb0SEvan Quan #if 0 5927ccf6eb0SEvan Quan if (def1 != data1) 5937ccf6eb0SEvan Quan WREG32_SOC15(MMHUB, 0, regDAGB0_CNTL_MISC2, data1); 5947ccf6eb0SEvan Quan 5957ccf6eb0SEvan Quan if (def2 != data2) 5967ccf6eb0SEvan Quan WREG32_SOC15(MMHUB, 0, regDAGB1_CNTL_MISC2, data2); 5977ccf6eb0SEvan Quan #endif 59898a0f868STianci.Yin } 59998a0f868STianci.Yin 60098a0f868STianci.Yin static void mmhub_v3_0_update_medium_grain_light_sleep(struct amdgpu_device *adev, 60198a0f868STianci.Yin bool enable) 60298a0f868STianci.Yin { 6037ccf6eb0SEvan Quan uint32_t def, data; 6047ccf6eb0SEvan Quan 6057ccf6eb0SEvan Quan def = data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG); 6067ccf6eb0SEvan Quan 6077ccf6eb0SEvan Quan if (enable) 6087ccf6eb0SEvan Quan data |= MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK; 6097ccf6eb0SEvan Quan else 6107ccf6eb0SEvan Quan data &= ~MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK; 6117ccf6eb0SEvan Quan 6127ccf6eb0SEvan Quan if (def != data) 6137ccf6eb0SEvan Quan WREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG, data); 61498a0f868STianci.Yin } 61598a0f868STianci.Yin 61698a0f868STianci.Yin static int mmhub_v3_0_set_clockgating(struct amdgpu_device *adev, 61798a0f868STianci.Yin enum amd_clockgating_state state) 61898a0f868STianci.Yin { 61998a0f868STianci.Yin if (amdgpu_sriov_vf(adev)) 62098a0f868STianci.Yin return 0; 62198a0f868STianci.Yin 6227ccf6eb0SEvan Quan if (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG) 62398a0f868STianci.Yin mmhub_v3_0_update_medium_grain_clock_gating(adev, 62498a0f868STianci.Yin state == AMD_CG_STATE_GATE); 6257ccf6eb0SEvan Quan 6267ccf6eb0SEvan Quan if (adev->cg_flags & AMD_CG_SUPPORT_MC_LS) 62798a0f868STianci.Yin mmhub_v3_0_update_medium_grain_light_sleep(adev, 62898a0f868STianci.Yin state == AMD_CG_STATE_GATE); 6297ccf6eb0SEvan Quan 63098a0f868STianci.Yin return 0; 63198a0f868STianci.Yin } 63298a0f868STianci.Yin 63398a0f868STianci.Yin static void mmhub_v3_0_get_clockgating(struct amdgpu_device *adev, u64 *flags) 63498a0f868STianci.Yin { 6357ccf6eb0SEvan Quan int data; 6367ccf6eb0SEvan Quan 6377ccf6eb0SEvan Quan if (amdgpu_sriov_vf(adev)) 6387ccf6eb0SEvan Quan *flags = 0; 6397ccf6eb0SEvan Quan 6407ccf6eb0SEvan Quan data = RREG32_SOC15(MMHUB, 0, regMM_ATC_L2_MISC_CG); 6417ccf6eb0SEvan Quan 6427ccf6eb0SEvan Quan /* AMD_CG_SUPPORT_MC_MGCG */ 6437ccf6eb0SEvan Quan if (data & MM_ATC_L2_MISC_CG__ENABLE_MASK) 6447ccf6eb0SEvan Quan *flags |= AMD_CG_SUPPORT_MC_MGCG; 6457ccf6eb0SEvan Quan 6467ccf6eb0SEvan Quan /* AMD_CG_SUPPORT_MC_LS */ 6477ccf6eb0SEvan Quan if (data & MM_ATC_L2_MISC_CG__MEM_LS_ENABLE_MASK) 6487ccf6eb0SEvan Quan *flags |= AMD_CG_SUPPORT_MC_LS; 64998a0f868STianci.Yin } 65098a0f868STianci.Yin 65198a0f868STianci.Yin const struct amdgpu_mmhub_funcs mmhub_v3_0_funcs = { 65298a0f868STianci.Yin .init = mmhub_v3_0_init, 65398a0f868STianci.Yin .get_fb_location = mmhub_v3_0_get_fb_location, 65498a0f868STianci.Yin .get_mc_fb_offset = mmhub_v3_0_get_mc_fb_offset, 65598a0f868STianci.Yin .gart_enable = mmhub_v3_0_gart_enable, 65698a0f868STianci.Yin .set_fault_enable_default = mmhub_v3_0_set_fault_enable_default, 65798a0f868STianci.Yin .gart_disable = mmhub_v3_0_gart_disable, 65898a0f868STianci.Yin .set_clockgating = mmhub_v3_0_set_clockgating, 65998a0f868STianci.Yin .get_clockgating = mmhub_v3_0_get_clockgating, 66098a0f868STianci.Yin .setup_vm_pt_regs = mmhub_v3_0_setup_vm_pt_regs, 66198a0f868STianci.Yin }; 662