14d8d75a4SHuang Rui /* 24d8d75a4SHuang Rui * Copyright 2019 Advanced Micro Devices, Inc. 34d8d75a4SHuang Rui * 44d8d75a4SHuang Rui * Permission is hereby granted, free of charge, to any person obtaining a 54d8d75a4SHuang Rui * copy of this software and associated documentation files (the "Software"), 64d8d75a4SHuang Rui * to deal in the Software without restriction, including without limitation 74d8d75a4SHuang Rui * the rights to use, copy, modify, merge, publish, distribute, sublicense, 84d8d75a4SHuang Rui * and/or sell copies of the Software, and to permit persons to whom the 94d8d75a4SHuang Rui * Software is furnished to do so, subject to the following conditions: 104d8d75a4SHuang Rui * 114d8d75a4SHuang Rui * The above copyright notice and this permission notice shall be included in 124d8d75a4SHuang Rui * all copies or substantial portions of the Software. 134d8d75a4SHuang Rui * 144d8d75a4SHuang Rui * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 154d8d75a4SHuang Rui * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 164d8d75a4SHuang Rui * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 174d8d75a4SHuang Rui * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 184d8d75a4SHuang Rui * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 194d8d75a4SHuang Rui * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 204d8d75a4SHuang Rui * OTHER DEALINGS IN THE SOFTWARE. 214d8d75a4SHuang Rui * 224d8d75a4SHuang Rui */ 234d8d75a4SHuang Rui 244d8d75a4SHuang Rui #include "amdgpu.h" 254d8d75a4SHuang Rui #include "mmhub_v2_3.h" 264d8d75a4SHuang Rui 274d8d75a4SHuang Rui #include "mmhub/mmhub_2_3_0_offset.h" 284d8d75a4SHuang Rui #include "mmhub/mmhub_2_3_0_sh_mask.h" 294d8d75a4SHuang Rui #include "mmhub/mmhub_2_3_0_default.h" 304d8d75a4SHuang Rui #include "navi10_enum.h" 314d8d75a4SHuang Rui 324d8d75a4SHuang Rui #include "soc15_common.h" 334d8d75a4SHuang Rui 34682b1f4cSAlex Deucher static const char *mmhub_client_ids_vangogh[][2] = { 35682b1f4cSAlex Deucher [0][0] = "MP0", 36682b1f4cSAlex Deucher [1][0] = "MP1", 37682b1f4cSAlex Deucher [2][0] = "DCEDMC", 38682b1f4cSAlex Deucher [3][0] = "DCEVGA", 39682b1f4cSAlex Deucher [13][0] = "UTCL2", 40682b1f4cSAlex Deucher [26][0] = "OSS", 41682b1f4cSAlex Deucher [27][0] = "HDP", 42682b1f4cSAlex Deucher [28][0] = "VCN", 43682b1f4cSAlex Deucher [29][0] = "VCNU", 44682b1f4cSAlex Deucher [30][0] = "JPEG", 45682b1f4cSAlex Deucher [0][1] = "MP0", 46682b1f4cSAlex Deucher [1][1] = "MP1", 47682b1f4cSAlex Deucher [2][1] = "DCEDMC", 48682b1f4cSAlex Deucher [3][1] = "DCEVGA", 49682b1f4cSAlex Deucher [4][1] = "DCEDWB", 50682b1f4cSAlex Deucher [5][1] = "XDP", 51682b1f4cSAlex Deucher [26][1] = "OSS", 52682b1f4cSAlex Deucher [27][1] = "HDP", 53682b1f4cSAlex Deucher [28][1] = "VCN", 54682b1f4cSAlex Deucher [29][1] = "VCNU", 55682b1f4cSAlex Deucher [30][1] = "JPEG", 56682b1f4cSAlex Deucher }; 57682b1f4cSAlex Deucher 584d8d75a4SHuang Rui static uint32_t mmhub_v2_3_get_invalidate_req(unsigned int vmid, 594d8d75a4SHuang Rui uint32_t flush_type) 604d8d75a4SHuang Rui { 614d8d75a4SHuang Rui u32 req = 0; 624d8d75a4SHuang Rui 634d8d75a4SHuang Rui /* invalidate using legacy mode on vmid*/ 644d8d75a4SHuang Rui req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, 654d8d75a4SHuang Rui PER_VMID_INVALIDATE_REQ, 1 << vmid); 664d8d75a4SHuang Rui req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 674d8d75a4SHuang Rui req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 684d8d75a4SHuang Rui req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 694d8d75a4SHuang Rui req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 704d8d75a4SHuang Rui req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 714d8d75a4SHuang Rui req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 724d8d75a4SHuang Rui req = REG_SET_FIELD(req, MMVM_INVALIDATE_ENG0_REQ, 734d8d75a4SHuang Rui CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 744d8d75a4SHuang Rui 754d8d75a4SHuang Rui return req; 764d8d75a4SHuang Rui } 774d8d75a4SHuang Rui 784d8d75a4SHuang Rui static void 794d8d75a4SHuang Rui mmhub_v2_3_print_l2_protection_fault_status(struct amdgpu_device *adev, 804d8d75a4SHuang Rui uint32_t status) 814d8d75a4SHuang Rui { 82682b1f4cSAlex Deucher uint32_t cid, rw; 83682b1f4cSAlex Deucher const char *mmhub_cid = NULL; 84682b1f4cSAlex Deucher 85682b1f4cSAlex Deucher cid = REG_GET_FIELD(status, 86682b1f4cSAlex Deucher MMVM_L2_PROTECTION_FAULT_STATUS, CID); 87682b1f4cSAlex Deucher rw = REG_GET_FIELD(status, 88682b1f4cSAlex Deucher MMVM_L2_PROTECTION_FAULT_STATUS, RW); 89682b1f4cSAlex Deucher 904d8d75a4SHuang Rui dev_err(adev->dev, 914d8d75a4SHuang Rui "MMVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 924d8d75a4SHuang Rui status); 931d789535SAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 94bc7c3d1dSAlex Deucher case IP_VERSION(2, 3, 0): 95bc7c3d1dSAlex Deucher case IP_VERSION(2, 4, 0): 96a142606dSYifan Zhang case IP_VERSION(2, 4, 1): 97682b1f4cSAlex Deucher mmhub_cid = mmhub_client_ids_vangogh[cid][rw]; 98682b1f4cSAlex Deucher break; 99682b1f4cSAlex Deucher default: 100682b1f4cSAlex Deucher mmhub_cid = NULL; 101682b1f4cSAlex Deucher break; 102682b1f4cSAlex Deucher } 103682b1f4cSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 104682b1f4cSAlex Deucher mmhub_cid ? mmhub_cid : "unknown", cid); 1054d8d75a4SHuang Rui dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 1064d8d75a4SHuang Rui REG_GET_FIELD(status, 1074d8d75a4SHuang Rui MMVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 1084d8d75a4SHuang Rui dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 1094d8d75a4SHuang Rui REG_GET_FIELD(status, 1104d8d75a4SHuang Rui MMVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 1114d8d75a4SHuang Rui dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 1124d8d75a4SHuang Rui REG_GET_FIELD(status, 1134d8d75a4SHuang Rui MMVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 1144d8d75a4SHuang Rui dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 1154d8d75a4SHuang Rui REG_GET_FIELD(status, 1164d8d75a4SHuang Rui MMVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 117682b1f4cSAlex Deucher dev_err(adev->dev, "\t RW: 0x%x\n", rw); 1184d8d75a4SHuang Rui } 1194d8d75a4SHuang Rui 1204d8d75a4SHuang Rui static void mmhub_v2_3_setup_vm_pt_regs(struct amdgpu_device *adev, 1214d8d75a4SHuang Rui uint32_t vmid, 1224d8d75a4SHuang Rui uint64_t page_table_base) 1234d8d75a4SHuang Rui { 1244d8d75a4SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 1254d8d75a4SHuang Rui 1264d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, 1274d8d75a4SHuang Rui hub->ctx_addr_distance * vmid, lower_32_bits(page_table_base)); 1284d8d75a4SHuang Rui 1294d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, 1304d8d75a4SHuang Rui hub->ctx_addr_distance * vmid, upper_32_bits(page_table_base)); 1314d8d75a4SHuang Rui } 1324d8d75a4SHuang Rui 1334d8d75a4SHuang Rui static void mmhub_v2_3_init_gart_aperture_regs(struct amdgpu_device *adev) 1344d8d75a4SHuang Rui { 1354d8d75a4SHuang Rui uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); 1364d8d75a4SHuang Rui 1374d8d75a4SHuang Rui mmhub_v2_3_setup_vm_pt_regs(adev, 0, pt_base); 1384d8d75a4SHuang Rui 1394d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, 1404d8d75a4SHuang Rui (u32)(adev->gmc.gart_start >> 12)); 1414d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, 1424d8d75a4SHuang Rui (u32)(adev->gmc.gart_start >> 44)); 1434d8d75a4SHuang Rui 1444d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, 1454d8d75a4SHuang Rui (u32)(adev->gmc.gart_end >> 12)); 1464d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, 1474d8d75a4SHuang Rui (u32)(adev->gmc.gart_end >> 44)); 1484d8d75a4SHuang Rui } 1494d8d75a4SHuang Rui 1504d8d75a4SHuang Rui static void mmhub_v2_3_init_system_aperture_regs(struct amdgpu_device *adev) 1514d8d75a4SHuang Rui { 1524d8d75a4SHuang Rui uint64_t value; 1534d8d75a4SHuang Rui uint32_t tmp; 1544d8d75a4SHuang Rui 1554d8d75a4SHuang Rui /* Disable AGP. */ 1564d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_AGP_BASE, 0); 15799698b51SAlex Deucher WREG32_SOC15(MMHUB, 0, mmMMMC_VM_AGP_BOT, adev->gmc.agp_start >> 24); 15899698b51SAlex Deucher WREG32_SOC15(MMHUB, 0, mmMMMC_VM_AGP_TOP, adev->gmc.agp_end >> 24); 1594d8d75a4SHuang Rui 1604d8d75a4SHuang Rui /* Program the system aperture low logical page number. */ 1614d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_SYSTEM_APERTURE_LOW_ADDR, 16299698b51SAlex Deucher min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18); 1634d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 16499698b51SAlex Deucher max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18); 1654d8d75a4SHuang Rui 1664d8d75a4SHuang Rui /* Set default page address. */ 1670ca565abSOak Zeng value = amdgpu_gmc_vram_mc2pa(adev, adev->vram_scratch.gpu_addr); 1684d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, 1694d8d75a4SHuang Rui (u32)(value >> 12)); 1704d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB, 1714d8d75a4SHuang Rui (u32)(value >> 44)); 1724d8d75a4SHuang Rui 1734d8d75a4SHuang Rui /* Program "protection fault". */ 1744d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32, 1754d8d75a4SHuang Rui (u32)(adev->dummy_page_addr >> 12)); 1764d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32, 1774d8d75a4SHuang Rui (u32)((u64)adev->dummy_page_addr >> 44)); 1784d8d75a4SHuang Rui 1794d8d75a4SHuang Rui tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL2); 1804d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL2, 1814d8d75a4SHuang Rui ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1); 1824d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL2, tmp); 1834d8d75a4SHuang Rui } 1844d8d75a4SHuang Rui 1854d8d75a4SHuang Rui static void mmhub_v2_3_init_tlb_regs(struct amdgpu_device *adev) 1864d8d75a4SHuang Rui { 1874d8d75a4SHuang Rui uint32_t tmp; 1884d8d75a4SHuang Rui 1894d8d75a4SHuang Rui /* Setup TLB control */ 1904d8d75a4SHuang Rui tmp = RREG32_SOC15(MMHUB, 0, mmMMMC_VM_MX_L1_TLB_CNTL); 1914d8d75a4SHuang Rui 1924d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1); 1934d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3); 1944d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, 1954d8d75a4SHuang Rui ENABLE_ADVANCED_DRIVER_MODEL, 1); 1964d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, 1974d8d75a4SHuang Rui SYSTEM_APERTURE_UNMAPPED_ACCESS, 0); 1984d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, 1994d8d75a4SHuang Rui MTYPE, MTYPE_UC); /* UC, uncached */ 2004d8d75a4SHuang Rui 2014d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_MX_L1_TLB_CNTL, tmp); 2024d8d75a4SHuang Rui } 2034d8d75a4SHuang Rui 2044d8d75a4SHuang Rui static void mmhub_v2_3_init_cache_regs(struct amdgpu_device *adev) 2054d8d75a4SHuang Rui { 2064d8d75a4SHuang Rui uint32_t tmp; 2074d8d75a4SHuang Rui 2084d8d75a4SHuang Rui /* Setup L2 cache */ 2094d8d75a4SHuang Rui tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL); 2104d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 1); 2114d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0); 2124d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, 2134d8d75a4SHuang Rui ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1); 2144d8d75a4SHuang Rui /* XXX for emulation, Refer to closed source code.*/ 2154d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE, 2164d8d75a4SHuang Rui 0); 2174d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0); 2184d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1); 2194d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0); 2204d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL, tmp); 2214d8d75a4SHuang Rui 2224d8d75a4SHuang Rui tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL2); 2234d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1); 2244d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1); 2254d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL2, tmp); 2264d8d75a4SHuang Rui 2274d8d75a4SHuang Rui tmp = mmMMVM_L2_CNTL3_DEFAULT; 2284d8d75a4SHuang Rui if (adev->gmc.translate_further) { 2294d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 12); 2304d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, 2314d8d75a4SHuang Rui L2_CACHE_BIGK_FRAGMENT_SIZE, 9); 2324d8d75a4SHuang Rui } else { 2334d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, BANK_SELECT, 9); 2344d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL3, 2354d8d75a4SHuang Rui L2_CACHE_BIGK_FRAGMENT_SIZE, 6); 2364d8d75a4SHuang Rui } 2374d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL3, tmp); 2384d8d75a4SHuang Rui 2394d8d75a4SHuang Rui tmp = mmMMVM_L2_CNTL4_DEFAULT; 2404d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0); 2414d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0); 2424d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL4, tmp); 2434d8d75a4SHuang Rui 2444d8d75a4SHuang Rui tmp = mmMMVM_L2_CNTL5_DEFAULT; 2454d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0); 2464d8d75a4SHuang Rui WREG32_SOC15(GC, 0, mmMMVM_L2_CNTL5, tmp); 2474d8d75a4SHuang Rui } 2484d8d75a4SHuang Rui 2494d8d75a4SHuang Rui static void mmhub_v2_3_enable_system_domain(struct amdgpu_device *adev) 2504d8d75a4SHuang Rui { 2514d8d75a4SHuang Rui uint32_t tmp; 2524d8d75a4SHuang Rui 2534d8d75a4SHuang Rui tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_CNTL); 2544d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1); 2554d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0); 2564d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT0_CNTL, 2574d8d75a4SHuang Rui RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0); 2584d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_CONTEXT0_CNTL, tmp); 2594d8d75a4SHuang Rui } 2604d8d75a4SHuang Rui 2614d8d75a4SHuang Rui static void mmhub_v2_3_disable_identity_aperture(struct amdgpu_device *adev) 2624d8d75a4SHuang Rui { 2634d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, 2644d8d75a4SHuang Rui mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32, 2654d8d75a4SHuang Rui 0xFFFFFFFF); 2664d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, 2674d8d75a4SHuang Rui mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32, 2684d8d75a4SHuang Rui 0x0000000F); 2694d8d75a4SHuang Rui 2704d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, 2714d8d75a4SHuang Rui mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32, 0); 2724d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, 2734d8d75a4SHuang Rui mmMMVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32, 0); 2744d8d75a4SHuang Rui 2754d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 2764d8d75a4SHuang Rui 0); 2774d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 2784d8d75a4SHuang Rui 0); 2794d8d75a4SHuang Rui } 2804d8d75a4SHuang Rui 2814d8d75a4SHuang Rui static void mmhub_v2_3_setup_vmid_config(struct amdgpu_device *adev) 2824d8d75a4SHuang Rui { 2834d8d75a4SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 2844d8d75a4SHuang Rui int i; 2854d8d75a4SHuang Rui uint32_t tmp; 2864d8d75a4SHuang Rui 2874d8d75a4SHuang Rui for (i = 0; i <= 14; i++) { 2884d8d75a4SHuang Rui tmp = RREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_CNTL, i); 2894d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1); 2904d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH, 2914d8d75a4SHuang Rui adev->vm_manager.num_level); 2924d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 2934d8d75a4SHuang Rui RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 2944d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 2954d8d75a4SHuang Rui DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 2964d8d75a4SHuang Rui 1); 2974d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 2984d8d75a4SHuang Rui PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 2994d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 3004d8d75a4SHuang Rui VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3014d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 3024d8d75a4SHuang Rui READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3034d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 3044d8d75a4SHuang Rui WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3054d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 3064d8d75a4SHuang Rui EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1); 3074d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 3084d8d75a4SHuang Rui PAGE_TABLE_BLOCK_SIZE, 3094d8d75a4SHuang Rui adev->vm_manager.block_size - 9); 3104d8d75a4SHuang Rui /* Send no-retry XNACK on fault to suppress VM fault storm. */ 3114d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_CONTEXT1_CNTL, 3124d8d75a4SHuang Rui RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 313fdcf0167SHawking Zhang !adev->gmc.noretry); 3144d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_CNTL, 3154d8d75a4SHuang Rui i * hub->ctx_distance, tmp); 3164d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32, 3174d8d75a4SHuang Rui i * hub->ctx_addr_distance, 0); 3184d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32, 3194d8d75a4SHuang Rui i * hub->ctx_addr_distance, 0); 3204d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32, 3214d8d75a4SHuang Rui i * hub->ctx_addr_distance, 3224d8d75a4SHuang Rui lower_32_bits(adev->vm_manager.max_pfn - 1)); 3234d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32, 3244d8d75a4SHuang Rui i * hub->ctx_addr_distance, 3254d8d75a4SHuang Rui upper_32_bits(adev->vm_manager.max_pfn - 1)); 3264d8d75a4SHuang Rui } 327*d7dab4fcSJack Xiao 328*d7dab4fcSJack Xiao hub->vm_cntx_cntl = tmp; 3294d8d75a4SHuang Rui } 3304d8d75a4SHuang Rui 3314d8d75a4SHuang Rui static void mmhub_v2_3_program_invalidation(struct amdgpu_device *adev) 3324d8d75a4SHuang Rui { 3334d8d75a4SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 3344d8d75a4SHuang Rui unsigned i; 3354d8d75a4SHuang Rui 3364d8d75a4SHuang Rui for (i = 0; i < 18; ++i) { 3374d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, 3384d8d75a4SHuang Rui mmMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32, 3394d8d75a4SHuang Rui i * hub->eng_addr_distance, 0xffffffff); 3404d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, 3414d8d75a4SHuang Rui mmMMVM_INVALIDATE_ENG0_ADDR_RANGE_HI32, 3424d8d75a4SHuang Rui i * hub->eng_addr_distance, 0x1f); 3434d8d75a4SHuang Rui } 3444d8d75a4SHuang Rui } 3454d8d75a4SHuang Rui 3464d8d75a4SHuang Rui static int mmhub_v2_3_gart_enable(struct amdgpu_device *adev) 3474d8d75a4SHuang Rui { 3484d8d75a4SHuang Rui if (amdgpu_sriov_vf(adev)) { 3494d8d75a4SHuang Rui /* 3504d8d75a4SHuang Rui * MMMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are 3514d8d75a4SHuang Rui * VF copy registers so vbios post doesn't program them, for 3524d8d75a4SHuang Rui * SRIOV driver need to program them 3534d8d75a4SHuang Rui */ 3544d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_FB_LOCATION_BASE, 3554d8d75a4SHuang Rui adev->gmc.vram_start >> 24); 3564d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_FB_LOCATION_TOP, 3574d8d75a4SHuang Rui adev->gmc.vram_end >> 24); 3584d8d75a4SHuang Rui } 3594d8d75a4SHuang Rui 3604d8d75a4SHuang Rui /* GART Enable. */ 3614d8d75a4SHuang Rui mmhub_v2_3_init_gart_aperture_regs(adev); 3624d8d75a4SHuang Rui mmhub_v2_3_init_system_aperture_regs(adev); 3634d8d75a4SHuang Rui mmhub_v2_3_init_tlb_regs(adev); 3644d8d75a4SHuang Rui mmhub_v2_3_init_cache_regs(adev); 3654d8d75a4SHuang Rui 3664d8d75a4SHuang Rui mmhub_v2_3_enable_system_domain(adev); 3674d8d75a4SHuang Rui mmhub_v2_3_disable_identity_aperture(adev); 3684d8d75a4SHuang Rui mmhub_v2_3_setup_vmid_config(adev); 3694d8d75a4SHuang Rui mmhub_v2_3_program_invalidation(adev); 3704d8d75a4SHuang Rui 3714d8d75a4SHuang Rui return 0; 3724d8d75a4SHuang Rui } 3734d8d75a4SHuang Rui 3744d8d75a4SHuang Rui static void mmhub_v2_3_gart_disable(struct amdgpu_device *adev) 3754d8d75a4SHuang Rui { 3764d8d75a4SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 3774d8d75a4SHuang Rui u32 tmp; 3784d8d75a4SHuang Rui u32 i; 3794d8d75a4SHuang Rui 3804d8d75a4SHuang Rui /* Disable all tables */ 38168fce5f0SNirmoy Das for (i = 0; i < AMDGPU_NUM_VMID; i++) 3824d8d75a4SHuang Rui WREG32_SOC15_OFFSET(MMHUB, 0, mmMMVM_CONTEXT0_CNTL, 3834d8d75a4SHuang Rui i * hub->ctx_distance, 0); 3844d8d75a4SHuang Rui 3854d8d75a4SHuang Rui /* Setup TLB control */ 3864d8d75a4SHuang Rui tmp = RREG32_SOC15(MMHUB, 0, mmMMMC_VM_MX_L1_TLB_CNTL); 3874d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0); 3884d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMMC_VM_MX_L1_TLB_CNTL, 3894d8d75a4SHuang Rui ENABLE_ADVANCED_DRIVER_MODEL, 0); 3904d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMMC_VM_MX_L1_TLB_CNTL, tmp); 3914d8d75a4SHuang Rui 3924d8d75a4SHuang Rui /* Setup L2 cache */ 3934d8d75a4SHuang Rui tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL); 3944d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_CNTL, ENABLE_L2_CACHE, 0); 3954d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL, tmp); 3964d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_CNTL3, 0); 3974d8d75a4SHuang Rui } 3984d8d75a4SHuang Rui 3994d8d75a4SHuang Rui /** 4004d8d75a4SHuang Rui * mmhub_v2_3_set_fault_enable_default - update GART/VM fault handling 4014d8d75a4SHuang Rui * 4024d8d75a4SHuang Rui * @adev: amdgpu_device pointer 4034d8d75a4SHuang Rui * @value: true redirects VM faults to the default page 4044d8d75a4SHuang Rui */ 4054d8d75a4SHuang Rui static void mmhub_v2_3_set_fault_enable_default(struct amdgpu_device *adev, 4064d8d75a4SHuang Rui bool value) 4074d8d75a4SHuang Rui { 4084d8d75a4SHuang Rui u32 tmp; 4094d8d75a4SHuang Rui tmp = RREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL); 4104d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4114d8d75a4SHuang Rui RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4124d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4134d8d75a4SHuang Rui PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4144d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4154d8d75a4SHuang Rui PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4164d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4174d8d75a4SHuang Rui PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4184d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4194d8d75a4SHuang Rui TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT, 4204d8d75a4SHuang Rui value); 4214d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4224d8d75a4SHuang Rui NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4234d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4244d8d75a4SHuang Rui DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4254d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4264d8d75a4SHuang Rui VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4274d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4284d8d75a4SHuang Rui READ_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4294d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4304d8d75a4SHuang Rui WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4314d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4324d8d75a4SHuang Rui EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value); 4334d8d75a4SHuang Rui if (!value) { 4344d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4354d8d75a4SHuang Rui CRASH_ON_NO_RETRY_FAULT, 1); 4364d8d75a4SHuang Rui tmp = REG_SET_FIELD(tmp, MMVM_L2_PROTECTION_FAULT_CNTL, 4374d8d75a4SHuang Rui CRASH_ON_RETRY_FAULT, 1); 4384d8d75a4SHuang Rui } 4394d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL, tmp); 4404d8d75a4SHuang Rui } 4414d8d75a4SHuang Rui 4424d8d75a4SHuang Rui static const struct amdgpu_vmhub_funcs mmhub_v2_3_vmhub_funcs = { 4434d8d75a4SHuang Rui .print_l2_protection_fault_status = mmhub_v2_3_print_l2_protection_fault_status, 4444d8d75a4SHuang Rui .get_invalidate_req = mmhub_v2_3_get_invalidate_req, 4454d8d75a4SHuang Rui }; 4464d8d75a4SHuang Rui 4474d8d75a4SHuang Rui static void mmhub_v2_3_init(struct amdgpu_device *adev) 4484d8d75a4SHuang Rui { 4494d8d75a4SHuang Rui struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_MMHUB_0]; 4504d8d75a4SHuang Rui 4514d8d75a4SHuang Rui hub->ctx0_ptb_addr_lo32 = 4524d8d75a4SHuang Rui SOC15_REG_OFFSET(MMHUB, 0, 4534d8d75a4SHuang Rui mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32); 4544d8d75a4SHuang Rui hub->ctx0_ptb_addr_hi32 = 4554d8d75a4SHuang Rui SOC15_REG_OFFSET(MMHUB, 0, 4564d8d75a4SHuang Rui mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32); 4574d8d75a4SHuang Rui hub->vm_inv_eng0_sem = 4584d8d75a4SHuang Rui SOC15_REG_OFFSET(MMHUB, 0, 4594d8d75a4SHuang Rui mmMMVM_INVALIDATE_ENG0_SEM); 4604d8d75a4SHuang Rui hub->vm_inv_eng0_req = 4614d8d75a4SHuang Rui SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_INVALIDATE_ENG0_REQ); 4624d8d75a4SHuang Rui hub->vm_inv_eng0_ack = 4634d8d75a4SHuang Rui SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_INVALIDATE_ENG0_ACK); 4644d8d75a4SHuang Rui hub->vm_context0_cntl = 4654d8d75a4SHuang Rui SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_CONTEXT0_CNTL); 4664d8d75a4SHuang Rui hub->vm_l2_pro_fault_status = 4674d8d75a4SHuang Rui SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_STATUS); 4684d8d75a4SHuang Rui hub->vm_l2_pro_fault_cntl = 4694d8d75a4SHuang Rui SOC15_REG_OFFSET(MMHUB, 0, mmMMVM_L2_PROTECTION_FAULT_CNTL); 4704d8d75a4SHuang Rui 4714d8d75a4SHuang Rui hub->ctx_distance = mmMMVM_CONTEXT1_CNTL - mmMMVM_CONTEXT0_CNTL; 4724d8d75a4SHuang Rui hub->ctx_addr_distance = mmMMVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 - 4734d8d75a4SHuang Rui mmMMVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32; 4744d8d75a4SHuang Rui hub->eng_distance = mmMMVM_INVALIDATE_ENG1_REQ - 4754d8d75a4SHuang Rui mmMMVM_INVALIDATE_ENG0_REQ; 4764d8d75a4SHuang Rui hub->eng_addr_distance = mmMMVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 - 4774d8d75a4SHuang Rui mmMMVM_INVALIDATE_ENG0_ADDR_RANGE_LO32; 4784d8d75a4SHuang Rui 4794d8d75a4SHuang Rui hub->vm_cntx_cntl_vm_fault = MMVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4804d8d75a4SHuang Rui MMVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4814d8d75a4SHuang Rui MMVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4824d8d75a4SHuang Rui MMVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4834d8d75a4SHuang Rui MMVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4844d8d75a4SHuang Rui MMVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 4854d8d75a4SHuang Rui MMVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 4864d8d75a4SHuang Rui 4874d8d75a4SHuang Rui hub->vmhub_funcs = &mmhub_v2_3_vmhub_funcs; 4884d8d75a4SHuang Rui } 4894d8d75a4SHuang Rui 4904d8d75a4SHuang Rui static void 4914d8d75a4SHuang Rui mmhub_v2_3_update_medium_grain_clock_gating(struct amdgpu_device *adev, 4924d8d75a4SHuang Rui bool enable) 4934d8d75a4SHuang Rui { 4944d8d75a4SHuang Rui uint32_t def, data, def1, data1; 4954d8d75a4SHuang Rui 4963c9a7b7dSAaron Liu def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_CGTT_CLK_CTRL); 4974d8d75a4SHuang Rui def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2); 4984d8d75a4SHuang Rui 4994d8d75a4SHuang Rui if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_MGCG)) { 5003c9a7b7dSAaron Liu data &= ~MM_ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK; 5014d8d75a4SHuang Rui data1 &= ~(DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | 5024d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK | 5034d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | 5044d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK | 5054d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | 5064d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); 5074d8d75a4SHuang Rui 5084d8d75a4SHuang Rui } else { 5093c9a7b7dSAaron Liu data |= MM_ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK; 5104d8d75a4SHuang Rui data1 |= (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | 5114d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK | 5124d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | 5134d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK | 5144d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | 5154d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK); 5164d8d75a4SHuang Rui } 5174d8d75a4SHuang Rui 5184d8d75a4SHuang Rui if (def != data) 5193c9a7b7dSAaron Liu WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_CGTT_CLK_CTRL, data); 5204d8d75a4SHuang Rui if (def1 != data1) 5214d8d75a4SHuang Rui WREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2, data1); 5224d8d75a4SHuang Rui } 5234d8d75a4SHuang Rui 5244d8d75a4SHuang Rui static void 5254d8d75a4SHuang Rui mmhub_v2_3_update_medium_grain_light_sleep(struct amdgpu_device *adev, 5264d8d75a4SHuang Rui bool enable) 5274d8d75a4SHuang Rui { 5283c9a7b7dSAaron Liu uint32_t def, data, def1, data1, def2, data2; 5294d8d75a4SHuang Rui 5303c9a7b7dSAaron Liu def = data = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_CGTT_CLK_CTRL); 5313c9a7b7dSAaron Liu def1 = data1 = RREG32_SOC15(MMHUB, 0, mmDAGB0_WR_CGTT_CLK_CTRL); 5323c9a7b7dSAaron Liu def2 = data2 = RREG32_SOC15(MMHUB, 0, mmDAGB0_RD_CGTT_CLK_CTRL); 5334d8d75a4SHuang Rui 5343c9a7b7dSAaron Liu if (enable && (adev->cg_flags & AMD_CG_SUPPORT_MC_LS)) { 5353c9a7b7dSAaron Liu data &= ~MM_ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK; 5365993e793SColin Ian King data1 &= ~(DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | 5373c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | 5383c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | 5393c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | 5403c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_REGISTER_MASK); 5415993e793SColin Ian King data2 &= ~(DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | 5423c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | 5433c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | 5443c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | 5453c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_REGISTER_MASK); 5463c9a7b7dSAaron Liu } else { 5473c9a7b7dSAaron Liu data |= MM_ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK; 5483c9a7b7dSAaron Liu data1 |= (DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | 5493c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | 5503c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | 5513c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | 5523c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_REGISTER_MASK); 5533c9a7b7dSAaron Liu data2 |= (DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | 5543c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | 5553c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | 5563c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | 5573c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_REGISTER_MASK); 5583c9a7b7dSAaron Liu } 5594d8d75a4SHuang Rui 5604d8d75a4SHuang Rui if (def != data) 5613c9a7b7dSAaron Liu WREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_CGTT_CLK_CTRL, data); 5623c9a7b7dSAaron Liu if (def1 != data1) 5633c9a7b7dSAaron Liu WREG32_SOC15(MMHUB, 0, mmDAGB0_WR_CGTT_CLK_CTRL, data1); 5643c9a7b7dSAaron Liu if (def2 != data2) 5653c9a7b7dSAaron Liu WREG32_SOC15(MMHUB, 0, mmDAGB0_RD_CGTT_CLK_CTRL, data2); 5664d8d75a4SHuang Rui } 5674d8d75a4SHuang Rui 5684d8d75a4SHuang Rui static int mmhub_v2_3_set_clockgating(struct amdgpu_device *adev, 5694d8d75a4SHuang Rui enum amd_clockgating_state state) 5704d8d75a4SHuang Rui { 5714d8d75a4SHuang Rui if (amdgpu_sriov_vf(adev)) 5724d8d75a4SHuang Rui return 0; 5734d8d75a4SHuang Rui 5744d8d75a4SHuang Rui mmhub_v2_3_update_medium_grain_clock_gating(adev, 5750bb6d3dbSZhen Lei state == AMD_CG_STATE_GATE); 5764d8d75a4SHuang Rui mmhub_v2_3_update_medium_grain_light_sleep(adev, 5770bb6d3dbSZhen Lei state == AMD_CG_STATE_GATE); 5784d8d75a4SHuang Rui 5794d8d75a4SHuang Rui return 0; 5804d8d75a4SHuang Rui } 5814d8d75a4SHuang Rui 58225faeddcSEvan Quan static void mmhub_v2_3_get_clockgating(struct amdgpu_device *adev, u64 *flags) 5834d8d75a4SHuang Rui { 5843c9a7b7dSAaron Liu int data, data1, data2, data3; 5854d8d75a4SHuang Rui 5864d8d75a4SHuang Rui if (amdgpu_sriov_vf(adev)) 5874d8d75a4SHuang Rui *flags = 0; 5884d8d75a4SHuang Rui 5893c9a7b7dSAaron Liu data = RREG32_SOC15(MMHUB, 0, mmDAGB0_CNTL_MISC2); 5903c9a7b7dSAaron Liu data1 = RREG32_SOC15(MMHUB, 0, mmMM_ATC_L2_CGTT_CLK_CTRL); 5913c9a7b7dSAaron Liu data2 = RREG32_SOC15(MMHUB, 0, mmDAGB0_WR_CGTT_CLK_CTRL); 5923c9a7b7dSAaron Liu data3 = RREG32_SOC15(MMHUB, 0, mmDAGB0_RD_CGTT_CLK_CTRL); 5934d8d75a4SHuang Rui 5944d8d75a4SHuang Rui /* AMD_CG_SUPPORT_MC_MGCG */ 5953c9a7b7dSAaron Liu if (!(data & (DAGB0_CNTL_MISC2__DISABLE_WRREQ_CG_MASK | 5964d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_WRRET_CG_MASK | 5974d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_RDREQ_CG_MASK | 5984d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_RDRET_CG_MASK | 5994d8d75a4SHuang Rui DAGB0_CNTL_MISC2__DISABLE_TLBWR_CG_MASK | 6003c9a7b7dSAaron Liu DAGB0_CNTL_MISC2__DISABLE_TLBRD_CG_MASK)) 6013c9a7b7dSAaron Liu && !(data1 & MM_ATC_L2_CGTT_CLK_CTRL__SOFT_OVERRIDE_MASK)) { 6024d8d75a4SHuang Rui *flags |= AMD_CG_SUPPORT_MC_MGCG; 6033c9a7b7dSAaron Liu } 6044d8d75a4SHuang Rui 6054d8d75a4SHuang Rui /* AMD_CG_SUPPORT_MC_LS */ 6063c9a7b7dSAaron Liu if (!(data1 & MM_ATC_L2_CGTT_CLK_CTRL__MGLS_OVERRIDE_MASK) 6073c9a7b7dSAaron Liu && !(data2 & (DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | 6083c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | 6093c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | 6103c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | 6113c9a7b7dSAaron Liu DAGB0_WR_CGTT_CLK_CTRL__LS_OVERRIDE_REGISTER_MASK)) 6123c9a7b7dSAaron Liu && !(data3 & (DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_MASK | 6133c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_WRITE_MASK | 6143c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_READ_MASK | 6153c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_RETURN_MASK | 6163c9a7b7dSAaron Liu DAGB0_RD_CGTT_CLK_CTRL__LS_OVERRIDE_REGISTER_MASK))) 6174d8d75a4SHuang Rui *flags |= AMD_CG_SUPPORT_MC_LS; 6184d8d75a4SHuang Rui } 6194d8d75a4SHuang Rui 6204d8d75a4SHuang Rui const struct amdgpu_mmhub_funcs mmhub_v2_3_funcs = { 6214d8d75a4SHuang Rui .init = mmhub_v2_3_init, 6224d8d75a4SHuang Rui .gart_enable = mmhub_v2_3_gart_enable, 6234d8d75a4SHuang Rui .set_fault_enable_default = mmhub_v2_3_set_fault_enable_default, 6244d8d75a4SHuang Rui .gart_disable = mmhub_v2_3_gart_disable, 6254d8d75a4SHuang Rui .set_clockgating = mmhub_v2_3_set_clockgating, 6264d8d75a4SHuang Rui .get_clockgating = mmhub_v2_3_get_clockgating, 6274d8d75a4SHuang Rui .setup_vm_pt_regs = mmhub_v2_3_setup_vm_pt_regs, 6284d8d75a4SHuang Rui }; 629