1af01d47dSLikun Gao /*
2af01d47dSLikun Gao * Copyright 2019 Advanced Micro Devices, Inc.
3af01d47dSLikun Gao *
4af01d47dSLikun Gao * Permission is hereby granted, free of charge, to any person obtaining a
5af01d47dSLikun Gao * copy of this software and associated documentation files (the "Software"),
6af01d47dSLikun Gao * to deal in the Software without restriction, including without limitation
7af01d47dSLikun Gao * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8af01d47dSLikun Gao * and/or sell copies of the Software, and to permit persons to whom the
9af01d47dSLikun Gao * Software is furnished to do so, subject to the following conditions:
10af01d47dSLikun Gao *
11af01d47dSLikun Gao * The above copyright notice and this permission notice shall be included in
12af01d47dSLikun Gao * all copies or substantial portions of the Software.
13af01d47dSLikun Gao *
14af01d47dSLikun Gao * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15af01d47dSLikun Gao * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16af01d47dSLikun Gao * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17af01d47dSLikun Gao * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18af01d47dSLikun Gao * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19af01d47dSLikun Gao * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20af01d47dSLikun Gao * OTHER DEALINGS IN THE SOFTWARE.
21af01d47dSLikun Gao *
22af01d47dSLikun Gao */
23af01d47dSLikun Gao
24af01d47dSLikun Gao #include "amdgpu.h"
25af01d47dSLikun Gao #include "gfxhub_v2_1.h"
26af01d47dSLikun Gao
27af01d47dSLikun Gao #include "gc/gc_10_3_0_offset.h"
28af01d47dSLikun Gao #include "gc/gc_10_3_0_sh_mask.h"
29af01d47dSLikun Gao #include "gc/gc_10_3_0_default.h"
30af01d47dSLikun Gao #include "navi10_enum.h"
31af01d47dSLikun Gao
32af01d47dSLikun Gao #include "soc15_common.h"
33af01d47dSLikun Gao
34b3accd6fSXiaomeng Hou #define mmGCUTCL2_HARVEST_BYPASS_GROUPS_YELLOW_CARP 0x16f8
35b3accd6fSXiaomeng Hou #define mmGCUTCL2_HARVEST_BYPASS_GROUPS_YELLOW_CARP_BASE_IDX 0
36b3accd6fSXiaomeng Hou
37*f51f2088SSrinivasan Shanmugam static const char * const gfxhub_client_ids[] = {
3893fabd84SAlex Deucher "CB/DB",
3993fabd84SAlex Deucher "Reserved",
4093fabd84SAlex Deucher "GE1",
4193fabd84SAlex Deucher "GE2",
4293fabd84SAlex Deucher "CPF",
4393fabd84SAlex Deucher "CPC",
4493fabd84SAlex Deucher "CPG",
4593fabd84SAlex Deucher "RLC",
4693fabd84SAlex Deucher "TCP",
4793fabd84SAlex Deucher "SQC (inst)",
4893fabd84SAlex Deucher "SQC (data)",
4993fabd84SAlex Deucher "SQG",
5093fabd84SAlex Deucher "Reserved",
5193fabd84SAlex Deucher "SDMA0",
5293fabd84SAlex Deucher "SDMA1",
5393fabd84SAlex Deucher "GCR",
5493fabd84SAlex Deucher "SDMA2",
5593fabd84SAlex Deucher "SDMA3",
5693fabd84SAlex Deucher };
5793fabd84SAlex Deucher
gfxhub_v2_1_get_invalidate_req(unsigned int vmid,uint32_t flush_type)58caa9f483SHuang Rui static uint32_t gfxhub_v2_1_get_invalidate_req(unsigned int vmid,
59caa9f483SHuang Rui uint32_t flush_type)
60caa9f483SHuang Rui {
61caa9f483SHuang Rui u32 req = 0;
62caa9f483SHuang Rui
63caa9f483SHuang Rui /* invalidate using legacy mode on vmid*/
64caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
65caa9f483SHuang Rui PER_VMID_INVALIDATE_REQ, 1 << vmid);
66caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type);
67caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1);
68caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1);
69caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1);
70caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1);
71caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1);
72caa9f483SHuang Rui req = REG_SET_FIELD(req, GCVM_INVALIDATE_ENG0_REQ,
73caa9f483SHuang Rui CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0);
74caa9f483SHuang Rui
75caa9f483SHuang Rui return req;
76caa9f483SHuang Rui }
77caa9f483SHuang Rui
782577db91SHuang Rui static void
gfxhub_v2_1_print_l2_protection_fault_status(struct amdgpu_device * adev,uint32_t status)792577db91SHuang Rui gfxhub_v2_1_print_l2_protection_fault_status(struct amdgpu_device *adev,
802577db91SHuang Rui uint32_t status)
812577db91SHuang Rui {
8293fabd84SAlex Deucher u32 cid = REG_GET_FIELD(status,
8393fabd84SAlex Deucher GCVM_L2_PROTECTION_FAULT_STATUS, CID);
8493fabd84SAlex Deucher
852577db91SHuang Rui dev_err(adev->dev,
862577db91SHuang Rui "GCVM_L2_PROTECTION_FAULT_STATUS:0x%08X\n",
872577db91SHuang Rui status);
8893fabd84SAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n",
8993fabd84SAlex Deucher cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid],
9093fabd84SAlex Deucher cid);
912577db91SHuang Rui dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n",
922577db91SHuang Rui REG_GET_FIELD(status,
932577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS));
942577db91SHuang Rui dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n",
952577db91SHuang Rui REG_GET_FIELD(status,
962577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR));
972577db91SHuang Rui dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n",
982577db91SHuang Rui REG_GET_FIELD(status,
992577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS));
1002577db91SHuang Rui dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n",
1012577db91SHuang Rui REG_GET_FIELD(status,
1022577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR));
1032577db91SHuang Rui dev_err(adev->dev, "\t RW: 0x%lx\n",
1042577db91SHuang Rui REG_GET_FIELD(status,
1052577db91SHuang Rui GCVM_L2_PROTECTION_FAULT_STATUS, RW));
1062577db91SHuang Rui }
1072577db91SHuang Rui
gfxhub_v2_1_get_fb_location(struct amdgpu_device * adev)1088ffff9b4SOak Zeng static u64 gfxhub_v2_1_get_fb_location(struct amdgpu_device *adev)
109af01d47dSLikun Gao {
110af01d47dSLikun Gao u64 base = RREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE);
111af01d47dSLikun Gao
112af01d47dSLikun Gao base &= GCMC_VM_FB_LOCATION_BASE__FB_BASE_MASK;
113af01d47dSLikun Gao base <<= 24;
114af01d47dSLikun Gao
115af01d47dSLikun Gao return base;
116af01d47dSLikun Gao }
117af01d47dSLikun Gao
gfxhub_v2_1_get_mc_fb_offset(struct amdgpu_device * adev)1188ffff9b4SOak Zeng static u64 gfxhub_v2_1_get_mc_fb_offset(struct amdgpu_device *adev)
119af01d47dSLikun Gao {
120af01d47dSLikun Gao return (u64)RREG32_SOC15(GC, 0, mmGCMC_VM_FB_OFFSET) << 24;
121af01d47dSLikun Gao }
122af01d47dSLikun Gao
gfxhub_v2_1_setup_vm_pt_regs(struct amdgpu_device * adev,uint32_t vmid,uint64_t page_table_base)1238ffff9b4SOak Zeng static void gfxhub_v2_1_setup_vm_pt_regs(struct amdgpu_device *adev, uint32_t vmid,
124af01d47dSLikun Gao uint64_t page_table_base)
125af01d47dSLikun Gao {
126f4caf584SHawking Zhang struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
127af01d47dSLikun Gao
128af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
12913ae12d9SHuang Rui hub->ctx_addr_distance * vmid,
13013ae12d9SHuang Rui lower_32_bits(page_table_base));
131af01d47dSLikun Gao
132af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
13313ae12d9SHuang Rui hub->ctx_addr_distance * vmid,
13413ae12d9SHuang Rui upper_32_bits(page_table_base));
135af01d47dSLikun Gao }
136af01d47dSLikun Gao
gfxhub_v2_1_init_gart_aperture_regs(struct amdgpu_device * adev)137af01d47dSLikun Gao static void gfxhub_v2_1_init_gart_aperture_regs(struct amdgpu_device *adev)
138af01d47dSLikun Gao {
139af01d47dSLikun Gao uint64_t pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
140af01d47dSLikun Gao
141af01d47dSLikun Gao gfxhub_v2_1_setup_vm_pt_regs(adev, 0, pt_base);
142af01d47dSLikun Gao
143af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
144af01d47dSLikun Gao (u32)(adev->gmc.gart_start >> 12));
145af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
146af01d47dSLikun Gao (u32)(adev->gmc.gart_start >> 44));
147af01d47dSLikun Gao
148af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
149af01d47dSLikun Gao (u32)(adev->gmc.gart_end >> 12));
150af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
151af01d47dSLikun Gao (u32)(adev->gmc.gart_end >> 44));
152af01d47dSLikun Gao }
153af01d47dSLikun Gao
gfxhub_v2_1_init_system_aperture_regs(struct amdgpu_device * adev)154af01d47dSLikun Gao static void gfxhub_v2_1_init_system_aperture_regs(struct amdgpu_device *adev)
155af01d47dSLikun Gao {
156af01d47dSLikun Gao uint64_t value;
157af01d47dSLikun Gao
15899698b51SAlex Deucher /* Program the AGP BAR */
159af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BASE, 0);
16099698b51SAlex Deucher WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
16199698b51SAlex Deucher WREG32_SOC15(GC, 0, mmGCMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
162af01d47dSLikun Gao
163af01d47dSLikun Gao /* Program the system aperture low logical page number. */
164af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_LOW_ADDR,
16599698b51SAlex Deucher min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
166af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
16799698b51SAlex Deucher max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
168af01d47dSLikun Gao
169af01d47dSLikun Gao /* Set default page address. */
1707ccfd79fSChristian König value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
171af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
172af01d47dSLikun Gao (u32)(value >> 12));
173af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
174af01d47dSLikun Gao (u32)(value >> 44));
175af01d47dSLikun Gao
176af01d47dSLikun Gao /* Program "protection fault". */
177af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
178af01d47dSLikun Gao (u32)(adev->dummy_page_addr >> 12));
179af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
180af01d47dSLikun Gao (u32)((u64)adev->dummy_page_addr >> 44));
181af01d47dSLikun Gao
182af01d47dSLikun Gao WREG32_FIELD15(GC, 0, GCVM_L2_PROTECTION_FAULT_CNTL2,
183af01d47dSLikun Gao ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
184af01d47dSLikun Gao }
185af01d47dSLikun Gao
186af01d47dSLikun Gao
gfxhub_v2_1_init_tlb_regs(struct amdgpu_device * adev)187af01d47dSLikun Gao static void gfxhub_v2_1_init_tlb_regs(struct amdgpu_device *adev)
188af01d47dSLikun Gao {
189af01d47dSLikun Gao uint32_t tmp;
190af01d47dSLikun Gao
191af01d47dSLikun Gao /* Setup TLB control */
192af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL);
193af01d47dSLikun Gao
194af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 1);
195af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE, 3);
196af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
197af01d47dSLikun Gao ENABLE_ADVANCED_DRIVER_MODEL, 1);
198af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
199af01d47dSLikun Gao SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
200af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
201af01d47dSLikun Gao MTYPE, MTYPE_UC); /* UC, uncached */
202af01d47dSLikun Gao
203af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp);
204af01d47dSLikun Gao }
205af01d47dSLikun Gao
gfxhub_v2_1_init_cache_regs(struct amdgpu_device * adev)206af01d47dSLikun Gao static void gfxhub_v2_1_init_cache_regs(struct amdgpu_device *adev)
207af01d47dSLikun Gao {
208af01d47dSLikun Gao uint32_t tmp;
209af01d47dSLikun Gao
2101d447326SLiu ChengZhe /* These registers are not accessible to VF-SRIOV.
2111d447326SLiu ChengZhe * The PF will program them instead.
2121d447326SLiu ChengZhe */
2131d447326SLiu ChengZhe if (amdgpu_sriov_vf(adev))
2141d447326SLiu ChengZhe return;
2151d447326SLiu ChengZhe
216af01d47dSLikun Gao /* Setup L2 cache */
217af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL);
218af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_CACHE, 1);
219af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 0);
220af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
221af01d47dSLikun Gao ENABLE_DEFAULT_PAGE_OUT_TO_SYSTEM_MEMORY, 1);
222af01d47dSLikun Gao /* XXX for emulation, Refer to closed source code.*/
223af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL,
224af01d47dSLikun Gao L2_PDE0_CACHE_TAG_GENERATION_MODE, 0);
225af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
226af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
227af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
228af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL, tmp);
229af01d47dSLikun Gao
230af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2);
231af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
232af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
233af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2, tmp);
234af01d47dSLikun Gao
235af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL3_DEFAULT;
236af01d47dSLikun Gao if (adev->gmc.translate_further) {
237af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 12);
238af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3,
239af01d47dSLikun Gao L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
240af01d47dSLikun Gao } else {
241af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3, BANK_SELECT, 9);
242af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL3,
243af01d47dSLikun Gao L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
244af01d47dSLikun Gao }
245af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, tmp);
246af01d47dSLikun Gao
247af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL4_DEFAULT;
248af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
249af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
250af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL4, tmp);
251af01d47dSLikun Gao
252af01d47dSLikun Gao tmp = mmGCVM_L2_CNTL5_DEFAULT;
253af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_CNTL5, L2_CACHE_SMALLK_FRAGMENT_SIZE, 0);
254af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL5, tmp);
255af01d47dSLikun Gao }
256af01d47dSLikun Gao
gfxhub_v2_1_enable_system_domain(struct amdgpu_device * adev)257af01d47dSLikun Gao static void gfxhub_v2_1_enable_system_domain(struct amdgpu_device *adev)
258af01d47dSLikun Gao {
259af01d47dSLikun Gao uint32_t tmp;
260af01d47dSLikun Gao
261af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL);
262af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
263af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH, 0);
264af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT0_CNTL,
265af01d47dSLikun Gao RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
266af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_CONTEXT0_CNTL, tmp);
267af01d47dSLikun Gao }
268af01d47dSLikun Gao
gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device * adev)269af01d47dSLikun Gao static void gfxhub_v2_1_disable_identity_aperture(struct amdgpu_device *adev)
270af01d47dSLikun Gao {
2711d447326SLiu ChengZhe /* These registers are not accessible to VF-SRIOV.
2721d447326SLiu ChengZhe * The PF will program them instead.
2731d447326SLiu ChengZhe */
2741d447326SLiu ChengZhe if (amdgpu_sriov_vf(adev))
2751d447326SLiu ChengZhe return;
2761d447326SLiu ChengZhe
277af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
278af01d47dSLikun Gao 0xFFFFFFFF);
279af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
280af01d47dSLikun Gao 0x0000000F);
281af01d47dSLikun Gao
282af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
283af01d47dSLikun Gao 0);
284af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
285af01d47dSLikun Gao 0);
286af01d47dSLikun Gao
287af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
288af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
289af01d47dSLikun Gao
290af01d47dSLikun Gao }
291af01d47dSLikun Gao
gfxhub_v2_1_setup_vmid_config(struct amdgpu_device * adev)292af01d47dSLikun Gao static void gfxhub_v2_1_setup_vmid_config(struct amdgpu_device *adev)
293af01d47dSLikun Gao {
294f4caf584SHawking Zhang struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
295af01d47dSLikun Gao int i;
296af01d47dSLikun Gao uint32_t tmp;
297af01d47dSLikun Gao
298af01d47dSLikun Gao for (i = 0; i <= 14; i++) {
299af01d47dSLikun Gao tmp = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL, i);
300af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
301af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
302af01d47dSLikun Gao adev->vm_manager.num_level);
303af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
304af01d47dSLikun Gao RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
305af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
306af01d47dSLikun Gao DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
307af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
308af01d47dSLikun Gao PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
309af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
310af01d47dSLikun Gao VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
311af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
312af01d47dSLikun Gao READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
313af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
314af01d47dSLikun Gao WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
315af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
316af01d47dSLikun Gao EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
317af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
318af01d47dSLikun Gao PAGE_TABLE_BLOCK_SIZE,
319af01d47dSLikun Gao adev->vm_manager.block_size - 9);
320af01d47dSLikun Gao /* Send no-retry XNACK on fault to suppress VM fault storm. */
321af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_CONTEXT1_CNTL,
322af01d47dSLikun Gao RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
3239b498efaSAlex Deucher !adev->gmc.noretry);
32413ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_CNTL,
32513ae12d9SHuang Rui i * hub->ctx_distance, tmp);
32613ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
32713ae12d9SHuang Rui i * hub->ctx_addr_distance, 0);
32813ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
32913ae12d9SHuang Rui i * hub->ctx_addr_distance, 0);
33013ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
33113ae12d9SHuang Rui i * hub->ctx_addr_distance,
332af01d47dSLikun Gao lower_32_bits(adev->vm_manager.max_pfn - 1));
33313ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
33413ae12d9SHuang Rui i * hub->ctx_addr_distance,
335af01d47dSLikun Gao upper_32_bits(adev->vm_manager.max_pfn - 1));
336af01d47dSLikun Gao }
337d7dab4fcSJack Xiao
338d7dab4fcSJack Xiao hub->vm_cntx_cntl = tmp;
339af01d47dSLikun Gao }
340af01d47dSLikun Gao
gfxhub_v2_1_program_invalidation(struct amdgpu_device * adev)341af01d47dSLikun Gao static void gfxhub_v2_1_program_invalidation(struct amdgpu_device *adev)
342af01d47dSLikun Gao {
343f4caf584SHawking Zhang struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
344*f51f2088SSrinivasan Shanmugam unsigned int i;
345af01d47dSLikun Gao
346af01d47dSLikun Gao for (i = 0 ; i < 18; ++i) {
347af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
34813ae12d9SHuang Rui i * hub->eng_addr_distance, 0xffffffff);
349af01d47dSLikun Gao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
35013ae12d9SHuang Rui i * hub->eng_addr_distance, 0x1f);
351af01d47dSLikun Gao }
352af01d47dSLikun Gao }
353af01d47dSLikun Gao
gfxhub_v2_1_gart_enable(struct amdgpu_device * adev)3548ffff9b4SOak Zeng static int gfxhub_v2_1_gart_enable(struct amdgpu_device *adev)
355af01d47dSLikun Gao {
356af01d47dSLikun Gao if (amdgpu_sriov_vf(adev)) {
357af01d47dSLikun Gao /*
358af01d47dSLikun Gao * GCMC_VM_FB_LOCATION_BASE/TOP is NULL for VF, becuase they are
359af01d47dSLikun Gao * VF copy registers so vbios post doesn't program them, for
360af01d47dSLikun Gao * SRIOV driver need to program them
361af01d47dSLikun Gao */
362af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE,
363af01d47dSLikun Gao adev->gmc.vram_start >> 24);
364af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_TOP,
365af01d47dSLikun Gao adev->gmc.vram_end >> 24);
366af01d47dSLikun Gao }
367af01d47dSLikun Gao
368af01d47dSLikun Gao /* GART Enable. */
369af01d47dSLikun Gao gfxhub_v2_1_init_gart_aperture_regs(adev);
370af01d47dSLikun Gao gfxhub_v2_1_init_system_aperture_regs(adev);
371af01d47dSLikun Gao gfxhub_v2_1_init_tlb_regs(adev);
372af01d47dSLikun Gao gfxhub_v2_1_init_cache_regs(adev);
373af01d47dSLikun Gao
374af01d47dSLikun Gao gfxhub_v2_1_enable_system_domain(adev);
375af01d47dSLikun Gao gfxhub_v2_1_disable_identity_aperture(adev);
376af01d47dSLikun Gao gfxhub_v2_1_setup_vmid_config(adev);
377af01d47dSLikun Gao gfxhub_v2_1_program_invalidation(adev);
378af01d47dSLikun Gao
379af01d47dSLikun Gao return 0;
380af01d47dSLikun Gao }
381af01d47dSLikun Gao
gfxhub_v2_1_gart_disable(struct amdgpu_device * adev)3828ffff9b4SOak Zeng static void gfxhub_v2_1_gart_disable(struct amdgpu_device *adev)
383af01d47dSLikun Gao {
384f4caf584SHawking Zhang struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
385af01d47dSLikun Gao u32 tmp;
386af01d47dSLikun Gao u32 i;
387af01d47dSLikun Gao
388af01d47dSLikun Gao /* Disable all tables */
389af01d47dSLikun Gao for (i = 0; i < 16; i++)
39013ae12d9SHuang Rui WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL,
39113ae12d9SHuang Rui i * hub->ctx_distance, 0);
392af01d47dSLikun Gao
393af01d47dSLikun Gao /* Setup TLB control */
394af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL);
395af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
396af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCMC_VM_MX_L1_TLB_CNTL,
397af01d47dSLikun Gao ENABLE_ADVANCED_DRIVER_MODEL, 0);
398af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, tmp);
399af01d47dSLikun Gao
400f8638ad7SPeng Ju Zhou if (amdgpu_sriov_vf(adev))
401f8638ad7SPeng Ju Zhou return;
402f8638ad7SPeng Ju Zhou
403af01d47dSLikun Gao /* Setup L2 cache */
404af01d47dSLikun Gao WREG32_FIELD15(GC, 0, GCVM_L2_CNTL, ENABLE_L2_CACHE, 0);
405af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL3, 0);
406af01d47dSLikun Gao }
407af01d47dSLikun Gao
408af01d47dSLikun Gao /**
409af01d47dSLikun Gao * gfxhub_v2_1_set_fault_enable_default - update GART/VM fault handling
410af01d47dSLikun Gao *
411af01d47dSLikun Gao * @adev: amdgpu_device pointer
412af01d47dSLikun Gao * @value: true redirects VM faults to the default page
413af01d47dSLikun Gao */
gfxhub_v2_1_set_fault_enable_default(struct amdgpu_device * adev,bool value)4148ffff9b4SOak Zeng static void gfxhub_v2_1_set_fault_enable_default(struct amdgpu_device *adev,
415af01d47dSLikun Gao bool value)
416af01d47dSLikun Gao {
417af01d47dSLikun Gao u32 tmp;
4181d447326SLiu ChengZhe
4191d447326SLiu ChengZhe /* These registers are not accessible to VF-SRIOV.
4201d447326SLiu ChengZhe * The PF will program them instead.
4211d447326SLiu ChengZhe */
4221d447326SLiu ChengZhe if (amdgpu_sriov_vf(adev))
4231d447326SLiu ChengZhe return;
4241d447326SLiu ChengZhe
425af01d47dSLikun Gao tmp = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL);
426af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
427af01d47dSLikun Gao RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
428af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
429af01d47dSLikun Gao PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
430af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
431af01d47dSLikun Gao PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
432af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
433af01d47dSLikun Gao PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
434af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
435af01d47dSLikun Gao TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
436af01d47dSLikun Gao value);
437af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
438af01d47dSLikun Gao NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
439af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
440af01d47dSLikun Gao DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
441af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
442af01d47dSLikun Gao VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
443af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
444af01d47dSLikun Gao READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
445af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
446af01d47dSLikun Gao WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
447af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
448af01d47dSLikun Gao EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
449af01d47dSLikun Gao if (!value) {
450af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
451af01d47dSLikun Gao CRASH_ON_NO_RETRY_FAULT, 1);
452af01d47dSLikun Gao tmp = REG_SET_FIELD(tmp, GCVM_L2_PROTECTION_FAULT_CNTL,
453af01d47dSLikun Gao CRASH_ON_RETRY_FAULT, 1);
454af01d47dSLikun Gao }
455af01d47dSLikun Gao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, tmp);
456af01d47dSLikun Gao }
457af01d47dSLikun Gao
4582577db91SHuang Rui static const struct amdgpu_vmhub_funcs gfxhub_v2_1_vmhub_funcs = {
4592577db91SHuang Rui .print_l2_protection_fault_status = gfxhub_v2_1_print_l2_protection_fault_status,
460caa9f483SHuang Rui .get_invalidate_req = gfxhub_v2_1_get_invalidate_req,
4612577db91SHuang Rui };
4622577db91SHuang Rui
gfxhub_v2_1_init(struct amdgpu_device * adev)4638ffff9b4SOak Zeng static void gfxhub_v2_1_init(struct amdgpu_device *adev)
464af01d47dSLikun Gao {
465f4caf584SHawking Zhang struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
466af01d47dSLikun Gao
467af01d47dSLikun Gao hub->ctx0_ptb_addr_lo32 =
468af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0,
469af01d47dSLikun Gao mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
470af01d47dSLikun Gao hub->ctx0_ptb_addr_hi32 =
471af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0,
472af01d47dSLikun Gao mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
473af01d47dSLikun Gao hub->vm_inv_eng0_sem =
474af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_SEM);
475af01d47dSLikun Gao hub->vm_inv_eng0_req =
476af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_REQ);
477af01d47dSLikun Gao hub->vm_inv_eng0_ack =
478af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_INVALIDATE_ENG0_ACK);
479af01d47dSLikun Gao hub->vm_context0_cntl =
480af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL);
481af01d47dSLikun Gao hub->vm_l2_pro_fault_status =
482af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_STATUS);
483af01d47dSLikun Gao hub->vm_l2_pro_fault_cntl =
484af01d47dSLikun Gao SOC15_REG_OFFSET(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL);
4851f9d56c3SHuang Rui
4861f9d56c3SHuang Rui hub->ctx_distance = mmGCVM_CONTEXT1_CNTL - mmGCVM_CONTEXT0_CNTL;
4871f9d56c3SHuang Rui hub->ctx_addr_distance = mmGCVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
4881f9d56c3SHuang Rui mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
4891f9d56c3SHuang Rui hub->eng_distance = mmGCVM_INVALIDATE_ENG1_REQ -
4901f9d56c3SHuang Rui mmGCVM_INVALIDATE_ENG0_REQ;
4911f9d56c3SHuang Rui hub->eng_addr_distance = mmGCVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
4921f9d56c3SHuang Rui mmGCVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
4935befb6fcSHuang Rui
4945befb6fcSHuang Rui hub->vm_cntx_cntl_vm_fault = GCVM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
4955befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
4965befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
4975befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
4985befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
4995befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK |
5005befb6fcSHuang Rui GCVM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK;
5012577db91SHuang Rui
5022577db91SHuang Rui hub->vmhub_funcs = &gfxhub_v2_1_vmhub_funcs;
503af01d47dSLikun Gao }
504fdb8483bSJohn Clements
gfxhub_v2_1_get_xgmi_info(struct amdgpu_device * adev)5058ffff9b4SOak Zeng static int gfxhub_v2_1_get_xgmi_info(struct amdgpu_device *adev)
506fdb8483bSJohn Clements {
507fdb8483bSJohn Clements u32 xgmi_lfb_cntl = RREG32_SOC15(GC, 0, mmGCMC_VM_XGMI_LFB_CNTL);
508fdb8483bSJohn Clements u32 max_region =
509fdb8483bSJohn Clements REG_GET_FIELD(xgmi_lfb_cntl, GCMC_VM_XGMI_LFB_CNTL, PF_MAX_REGION);
510fdb8483bSJohn Clements u32 max_num_physical_nodes = 0;
511fdb8483bSJohn Clements u32 max_physical_node_id = 0;
512fdb8483bSJohn Clements
5131d789535SAlex Deucher switch (adev->ip_versions[XGMI_HWIP][0]) {
514fac17723SAlex Deucher case IP_VERSION(4, 8, 0):
515fdb8483bSJohn Clements max_num_physical_nodes = 4;
516fdb8483bSJohn Clements max_physical_node_id = 3;
517fdb8483bSJohn Clements break;
518fdb8483bSJohn Clements default:
519fdb8483bSJohn Clements return -EINVAL;
520fdb8483bSJohn Clements }
521fdb8483bSJohn Clements
522fdb8483bSJohn Clements /* PF_MAX_REGION=0 means xgmi is disabled */
523fdb8483bSJohn Clements if (max_region) {
524fdb8483bSJohn Clements adev->gmc.xgmi.num_physical_nodes = max_region + 1;
525fdb8483bSJohn Clements if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes)
526fdb8483bSJohn Clements return -EINVAL;
527fdb8483bSJohn Clements
528fdb8483bSJohn Clements adev->gmc.xgmi.physical_node_id =
529fdb8483bSJohn Clements REG_GET_FIELD(xgmi_lfb_cntl, GCMC_VM_XGMI_LFB_CNTL, PF_LFB_REGION);
530fdb8483bSJohn Clements if (adev->gmc.xgmi.physical_node_id > max_physical_node_id)
531fdb8483bSJohn Clements return -EINVAL;
532fdb8483bSJohn Clements
533fdb8483bSJohn Clements adev->gmc.xgmi.node_segment_size = REG_GET_FIELD(
534fdb8483bSJohn Clements RREG32_SOC15(GC, 0, mmGCMC_VM_XGMI_LFB_SIZE),
535fdb8483bSJohn Clements GCMC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24;
536fdb8483bSJohn Clements }
537fdb8483bSJohn Clements
538fdb8483bSJohn Clements return 0;
539fdb8483bSJohn Clements }
5408ffff9b4SOak Zeng
gfxhub_v2_1_utcl2_harvest(struct amdgpu_device * adev)541b3accd6fSXiaomeng Hou static void gfxhub_v2_1_utcl2_harvest(struct amdgpu_device *adev)
542b3accd6fSXiaomeng Hou {
543b3accd6fSXiaomeng Hou int i;
544b3accd6fSXiaomeng Hou u32 tmp = 0, disabled_sa = 0;
545b3accd6fSXiaomeng Hou u32 efuse_setting, vbios_setting;
546b3accd6fSXiaomeng Hou
547b3accd6fSXiaomeng Hou u32 max_sa_mask = amdgpu_gfx_create_bitmask(
548b3accd6fSXiaomeng Hou adev->gfx.config.max_sh_per_se *
549b3accd6fSXiaomeng Hou adev->gfx.config.max_shader_engines);
550b3accd6fSXiaomeng Hou
5516d53b115SAaron Liu switch (adev->ip_versions[GC_HWIP][0]) {
5526d53b115SAaron Liu case IP_VERSION(10, 3, 1):
5536d53b115SAaron Liu case IP_VERSION(10, 3, 3):
554b3accd6fSXiaomeng Hou /* Get SA disabled bitmap from eFuse setting */
555b3accd6fSXiaomeng Hou efuse_setting = RREG32_SOC15(GC, 0, mmCC_GC_SA_UNIT_DISABLE);
556b3accd6fSXiaomeng Hou efuse_setting &= CC_GC_SA_UNIT_DISABLE__SA_DISABLE_MASK;
557b3accd6fSXiaomeng Hou efuse_setting >>= CC_GC_SA_UNIT_DISABLE__SA_DISABLE__SHIFT;
558b3accd6fSXiaomeng Hou
559b3accd6fSXiaomeng Hou /* Get SA disabled bitmap from VBIOS setting */
560b3accd6fSXiaomeng Hou vbios_setting = RREG32_SOC15(GC, 0, mmGC_USER_SA_UNIT_DISABLE);
561b3accd6fSXiaomeng Hou vbios_setting &= GC_USER_SA_UNIT_DISABLE__SA_DISABLE_MASK;
562b3accd6fSXiaomeng Hou vbios_setting >>= GC_USER_SA_UNIT_DISABLE__SA_DISABLE__SHIFT;
563b3accd6fSXiaomeng Hou
564b3accd6fSXiaomeng Hou disabled_sa |= efuse_setting | vbios_setting;
565b3accd6fSXiaomeng Hou /* Make sure not to report harvested SAs beyond the max SA count */
566b3accd6fSXiaomeng Hou disabled_sa &= max_sa_mask;
567b3accd6fSXiaomeng Hou
568b3accd6fSXiaomeng Hou for (i = 0; disabled_sa > 0; i++) {
569b3accd6fSXiaomeng Hou if (disabled_sa & 1)
570b3accd6fSXiaomeng Hou tmp |= 0x3 << (i * 2);
571b3accd6fSXiaomeng Hou disabled_sa >>= 1;
572b3accd6fSXiaomeng Hou }
573b3accd6fSXiaomeng Hou disabled_sa = tmp;
574b3accd6fSXiaomeng Hou
575b3accd6fSXiaomeng Hou WREG32_SOC15(GC, 0, mmGCUTCL2_HARVEST_BYPASS_GROUPS_YELLOW_CARP, disabled_sa);
5766d53b115SAaron Liu break;
5776d53b115SAaron Liu default:
5786d53b115SAaron Liu break;
579b3accd6fSXiaomeng Hou }
580b3accd6fSXiaomeng Hou }
581b3accd6fSXiaomeng Hou
gfxhub_v2_1_save_regs(struct amdgpu_device * adev)582bfaced6eSVictor Zhao static void gfxhub_v2_1_save_regs(struct amdgpu_device *adev)
583bfaced6eSVictor Zhao {
584bfaced6eSVictor Zhao int i;
585*f51f2088SSrinivasan Shanmugam
586bfaced6eSVictor Zhao adev->gmc.VM_L2_CNTL = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL);
587bfaced6eSVictor Zhao adev->gmc.VM_L2_CNTL2 = RREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2);
588bfaced6eSVictor Zhao adev->gmc.VM_DUMMY_PAGE_FAULT_CNTL = RREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_CNTL);
589bfaced6eSVictor Zhao adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_LO32 = RREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_ADDR_LO32);
590bfaced6eSVictor Zhao adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_HI32 = RREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_ADDR_HI32);
591bfaced6eSVictor Zhao adev->gmc.VM_L2_PROTECTION_FAULT_CNTL = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL);
592bfaced6eSVictor Zhao adev->gmc.VM_L2_PROTECTION_FAULT_CNTL2 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL2);
593bfaced6eSVictor Zhao adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL3 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_MM_CNTL3);
594bfaced6eSVictor Zhao adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL4 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_MM_CNTL4);
595bfaced6eSVictor Zhao adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_LO32 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_ADDR_LO32);
596bfaced6eSVictor Zhao adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_HI32 = RREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_ADDR_HI32);
597bfaced6eSVictor Zhao adev->gmc.VM_DEBUG = RREG32_SOC15(GC, 0, mmGCVM_DEBUG);
598bfaced6eSVictor Zhao adev->gmc.VM_L2_MM_GROUP_RT_CLASSES = RREG32_SOC15(GC, 0, mmGCVM_L2_MM_GROUP_RT_CLASSES);
599bfaced6eSVictor Zhao adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID = RREG32_SOC15(GC, 0, mmGCVM_L2_BANK_SELECT_RESERVED_CID);
600bfaced6eSVictor Zhao adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID2 = RREG32_SOC15(GC, 0, mmGCVM_L2_BANK_SELECT_RESERVED_CID2);
601bfaced6eSVictor Zhao adev->gmc.VM_L2_CACHE_PARITY_CNTL = RREG32_SOC15(GC, 0, mmGCVM_L2_CACHE_PARITY_CNTL);
602bfaced6eSVictor Zhao adev->gmc.VM_L2_IH_LOG_CNTL = RREG32_SOC15(GC, 0, mmGCVM_L2_IH_LOG_CNTL);
603bfaced6eSVictor Zhao
604bfaced6eSVictor Zhao for (i = 0; i <= 15; i++) {
605bfaced6eSVictor Zhao adev->gmc.VM_CONTEXT_CNTL[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL, i);
606bfaced6eSVictor Zhao adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, i * 2);
607bfaced6eSVictor Zhao adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, i * 2);
608bfaced6eSVictor Zhao adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, i * 2);
609bfaced6eSVictor Zhao adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, i * 2);
610bfaced6eSVictor Zhao adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, i * 2);
611bfaced6eSVictor Zhao adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[i] = RREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, i * 2);
612bfaced6eSVictor Zhao }
613bfaced6eSVictor Zhao
614bfaced6eSVictor Zhao adev->gmc.MC_VM_MX_L1_TLB_CNTL = RREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL);
615bfaced6eSVictor Zhao }
616bfaced6eSVictor Zhao
gfxhub_v2_1_restore_regs(struct amdgpu_device * adev)617bfaced6eSVictor Zhao static void gfxhub_v2_1_restore_regs(struct amdgpu_device *adev)
618bfaced6eSVictor Zhao {
619bfaced6eSVictor Zhao int i;
620*f51f2088SSrinivasan Shanmugam
621bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL, adev->gmc.VM_L2_CNTL);
622bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_CNTL2, adev->gmc.VM_L2_CNTL2);
623bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_CNTL, adev->gmc.VM_DUMMY_PAGE_FAULT_CNTL);
624bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_ADDR_LO32, adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_LO32);
625bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_DUMMY_PAGE_FAULT_ADDR_HI32, adev->gmc.VM_DUMMY_PAGE_FAULT_ADDR_HI32);
626bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL, adev->gmc.VM_L2_PROTECTION_FAULT_CNTL);
627bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_CNTL2, adev->gmc.VM_L2_PROTECTION_FAULT_CNTL2);
628bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_MM_CNTL3, adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL3);
629bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_MM_CNTL4, adev->gmc.VM_L2_PROTECTION_FAULT_MM_CNTL4);
630bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_ADDR_LO32, adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_LO32);
631bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_PROTECTION_FAULT_ADDR_HI32, adev->gmc.VM_L2_PROTECTION_FAULT_ADDR_HI32);
632bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_DEBUG, adev->gmc.VM_DEBUG);
633bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_MM_GROUP_RT_CLASSES, adev->gmc.VM_L2_MM_GROUP_RT_CLASSES);
634bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_BANK_SELECT_RESERVED_CID, adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID);
635bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_BANK_SELECT_RESERVED_CID2, adev->gmc.VM_L2_BANK_SELECT_RESERVED_CID2);
636bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_CACHE_PARITY_CNTL, adev->gmc.VM_L2_CACHE_PARITY_CNTL);
637bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCVM_L2_IH_LOG_CNTL, adev->gmc.VM_L2_IH_LOG_CNTL);
638bfaced6eSVictor Zhao
639bfaced6eSVictor Zhao for (i = 0; i <= 15; i++) {
640bfaced6eSVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_CNTL, i, adev->gmc.VM_CONTEXT_CNTL[i]);
641bfaced6eSVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_LO32[i]);
642bfaced6eSVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_BASE_ADDR_HI32[i]);
643bfaced6eSVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_LO32[i]);
644bfaced6eSVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_START_ADDR_HI32[i]);
645bfaced6eSVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_LO32[i]);
646bfaced6eSVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32, i * 2, adev->gmc.VM_CONTEXT_PAGE_TABLE_END_ADDR_HI32[i]);
647bfaced6eSVictor Zhao }
648bfaced6eSVictor Zhao
649bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_BASE, adev->gmc.vram_start >> 24);
650bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCMC_VM_FB_LOCATION_TOP, adev->gmc.vram_end >> 24);
651bfaced6eSVictor Zhao WREG32_SOC15(GC, 0, mmGCMC_VM_MX_L1_TLB_CNTL, adev->gmc.MC_VM_MX_L1_TLB_CNTL);
652bfaced6eSVictor Zhao }
653bfaced6eSVictor Zhao
gfxhub_v2_1_halt(struct amdgpu_device * adev)65472fadb13SVictor Zhao static void gfxhub_v2_1_halt(struct amdgpu_device *adev)
65572fadb13SVictor Zhao {
656f4caf584SHawking Zhang struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
65772fadb13SVictor Zhao int i;
65872fadb13SVictor Zhao uint32_t tmp;
65972fadb13SVictor Zhao int time = 1000;
66072fadb13SVictor Zhao
66172fadb13SVictor Zhao gfxhub_v2_1_set_fault_enable_default(adev, false);
66272fadb13SVictor Zhao
66372fadb13SVictor Zhao for (i = 0; i <= 14; i++) {
66472fadb13SVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
66572fadb13SVictor Zhao i * hub->ctx_addr_distance, ~0);
66672fadb13SVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
66772fadb13SVictor Zhao i * hub->ctx_addr_distance, ~0);
66872fadb13SVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
66972fadb13SVictor Zhao i * hub->ctx_addr_distance,
67072fadb13SVictor Zhao 0);
67172fadb13SVictor Zhao WREG32_SOC15_OFFSET(GC, 0, mmGCVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
67272fadb13SVictor Zhao i * hub->ctx_addr_distance,
67372fadb13SVictor Zhao 0);
67472fadb13SVictor Zhao }
67572fadb13SVictor Zhao tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
67672fadb13SVictor Zhao while ((tmp & (GRBM_STATUS2__EA_BUSY_MASK |
67772fadb13SVictor Zhao GRBM_STATUS2__EA_LINK_BUSY_MASK)) != 0 &&
67872fadb13SVictor Zhao time) {
67972fadb13SVictor Zhao udelay(100);
68072fadb13SVictor Zhao time--;
68172fadb13SVictor Zhao tmp = RREG32_SOC15(GC, 0, mmGRBM_STATUS2);
68272fadb13SVictor Zhao }
68372fadb13SVictor Zhao
684*f51f2088SSrinivasan Shanmugam if (!time)
68572fadb13SVictor Zhao DRM_WARN("failed to wait for GRBM(EA) idle\n");
68672fadb13SVictor Zhao }
68772fadb13SVictor Zhao
6888ffff9b4SOak Zeng const struct amdgpu_gfxhub_funcs gfxhub_v2_1_funcs = {
6898ffff9b4SOak Zeng .get_fb_location = gfxhub_v2_1_get_fb_location,
6908ffff9b4SOak Zeng .get_mc_fb_offset = gfxhub_v2_1_get_mc_fb_offset,
6918ffff9b4SOak Zeng .setup_vm_pt_regs = gfxhub_v2_1_setup_vm_pt_regs,
6928ffff9b4SOak Zeng .gart_enable = gfxhub_v2_1_gart_enable,
6938ffff9b4SOak Zeng .gart_disable = gfxhub_v2_1_gart_disable,
6948ffff9b4SOak Zeng .set_fault_enable_default = gfxhub_v2_1_set_fault_enable_default,
6958ffff9b4SOak Zeng .init = gfxhub_v2_1_init,
6968ffff9b4SOak Zeng .get_xgmi_info = gfxhub_v2_1_get_xgmi_info,
697b3accd6fSXiaomeng Hou .utcl2_harvest = gfxhub_v2_1_utcl2_harvest,
698bfaced6eSVictor Zhao .mode2_save_regs = gfxhub_v2_1_save_regs,
699bfaced6eSVictor Zhao .mode2_restore_regs = gfxhub_v2_1_restore_regs,
70072fadb13SVictor Zhao .halt = gfxhub_v2_1_halt,
7018ffff9b4SOak Zeng };
702