16d4496bcSLe Ma /*
26d4496bcSLe Ma * Copyright 2022 Advanced Micro Devices, Inc.
36d4496bcSLe Ma *
46d4496bcSLe Ma * Permission is hereby granted, free of charge, to any person obtaining a
56d4496bcSLe Ma * copy of this software and associated documentation files (the "Software"),
66d4496bcSLe Ma * to deal in the Software without restriction, including without limitation
76d4496bcSLe Ma * the rights to use, copy, modify, merge, publish, distribute, sublicense,
86d4496bcSLe Ma * and/or sell copies of the Software, and to permit persons to whom the
96d4496bcSLe Ma * Software is furnished to do so, subject to the following conditions:
106d4496bcSLe Ma *
116d4496bcSLe Ma * The above copyright notice and this permission notice shall be included in
126d4496bcSLe Ma * all copies or substantial portions of the Software.
136d4496bcSLe Ma *
146d4496bcSLe Ma * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
156d4496bcSLe Ma * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
166d4496bcSLe Ma * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
176d4496bcSLe Ma * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
186d4496bcSLe Ma * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
196d4496bcSLe Ma * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
206d4496bcSLe Ma * OTHER DEALINGS IN THE SOFTWARE.
216d4496bcSLe Ma *
226d4496bcSLe Ma */
236d4496bcSLe Ma #include "amdgpu.h"
24dfdd6f89SLijo Lazar #include "amdgpu_xcp.h"
256d4496bcSLe Ma #include "gfxhub_v1_2.h"
266d4496bcSLe Ma #include "gfxhub_v1_1.h"
276d4496bcSLe Ma
286d4496bcSLe Ma #include "gc/gc_9_4_3_offset.h"
296d4496bcSLe Ma #include "gc/gc_9_4_3_sh_mask.h"
306d4496bcSLe Ma #include "vega10_enum.h"
316d4496bcSLe Ma
326d4496bcSLe Ma #include "soc15_common.h"
336d4496bcSLe Ma
346d4496bcSLe Ma #define regVM_L2_CNTL3_DEFAULT 0x80100007
356d4496bcSLe Ma #define regVM_L2_CNTL4_DEFAULT 0x000000c1
366d4496bcSLe Ma
gfxhub_v1_2_get_mc_fb_offset(struct amdgpu_device * adev)376d4496bcSLe Ma static u64 gfxhub_v1_2_get_mc_fb_offset(struct amdgpu_device *adev)
386d4496bcSLe Ma {
39659a4ab8SLijo Lazar return (u64)RREG32_SOC15(GC, GET_INST(GC, 0), regMC_VM_FB_OFFSET) << 24;
406d4496bcSLe Ma }
416d4496bcSLe Ma
gfxhub_v1_2_xcc_setup_vm_pt_regs(struct amdgpu_device * adev,uint32_t vmid,uint64_t page_table_base,uint32_t xcc_mask)42dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_setup_vm_pt_regs(struct amdgpu_device *adev,
436d4496bcSLe Ma uint32_t vmid,
44dd1a02e2SLijo Lazar uint64_t page_table_base,
45dd1a02e2SLijo Lazar uint32_t xcc_mask)
466d4496bcSLe Ma {
47ed42f2ccSLe Ma struct amdgpu_vmhub *hub;
48dd1a02e2SLijo Lazar int i;
496d4496bcSLe Ma
50dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
51ed42f2ccSLe Ma hub = &adev->vmhub[AMDGPU_GFXHUB(i)];
52659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, i),
534667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32,
546d4496bcSLe Ma hub->ctx_addr_distance * vmid,
556d4496bcSLe Ma lower_32_bits(page_table_base));
566d4496bcSLe Ma
57659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, i),
584667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32,
596d4496bcSLe Ma hub->ctx_addr_distance * vmid,
606d4496bcSLe Ma upper_32_bits(page_table_base));
616d4496bcSLe Ma }
624667fbe2SLe Ma }
636d4496bcSLe Ma
gfxhub_v1_2_setup_vm_pt_regs(struct amdgpu_device * adev,uint32_t vmid,uint64_t page_table_base)64dd1a02e2SLijo Lazar static void gfxhub_v1_2_setup_vm_pt_regs(struct amdgpu_device *adev,
65dd1a02e2SLijo Lazar uint32_t vmid,
66dd1a02e2SLijo Lazar uint64_t page_table_base)
67dd1a02e2SLijo Lazar {
68dd1a02e2SLijo Lazar uint32_t xcc_mask;
69dd1a02e2SLijo Lazar
70dd1a02e2SLijo Lazar xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0);
71dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_setup_vm_pt_regs(adev, vmid, page_table_base, xcc_mask);
72dd1a02e2SLijo Lazar }
73dd1a02e2SLijo Lazar
gfxhub_v1_2_xcc_init_gart_aperture_regs(struct amdgpu_device * adev,uint32_t xcc_mask)74dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_init_gart_aperture_regs(struct amdgpu_device *adev,
75dd1a02e2SLijo Lazar uint32_t xcc_mask)
766d4496bcSLe Ma {
776d4496bcSLe Ma uint64_t pt_base;
78dd1a02e2SLijo Lazar int i;
796d4496bcSLe Ma
806d4496bcSLe Ma if (adev->gmc.pdb0_bo)
816d4496bcSLe Ma pt_base = amdgpu_gmc_pd_addr(adev->gmc.pdb0_bo);
826d4496bcSLe Ma else
836d4496bcSLe Ma pt_base = amdgpu_gmc_pd_addr(adev->gart.bo);
846d4496bcSLe Ma
85dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_setup_vm_pt_regs(adev, 0, pt_base, xcc_mask);
866d4496bcSLe Ma
876d4496bcSLe Ma /* If use GART for FB translation, vmid0 page table covers both
886d4496bcSLe Ma * vram and system memory (gart)
896d4496bcSLe Ma */
90dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
916d4496bcSLe Ma if (adev->gmc.pdb0_bo) {
92659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
934667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
946d4496bcSLe Ma (u32)(adev->gmc.fb_start >> 12));
95659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
964667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
976d4496bcSLe Ma (u32)(adev->gmc.fb_start >> 44));
986d4496bcSLe Ma
99659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
1004667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
1016d4496bcSLe Ma (u32)(adev->gmc.gart_end >> 12));
102659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
1034667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
1046d4496bcSLe Ma (u32)(adev->gmc.gart_end >> 44));
1056d4496bcSLe Ma } else {
106659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
1074667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_START_ADDR_LO32,
1086d4496bcSLe Ma (u32)(adev->gmc.gart_start >> 12));
109659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
1104667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_START_ADDR_HI32,
1116d4496bcSLe Ma (u32)(adev->gmc.gart_start >> 44));
1126d4496bcSLe Ma
113659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
1144667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_END_ADDR_LO32,
1156d4496bcSLe Ma (u32)(adev->gmc.gart_end >> 12));
116659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
1174667fbe2SLe Ma regVM_CONTEXT0_PAGE_TABLE_END_ADDR_HI32,
1186d4496bcSLe Ma (u32)(adev->gmc.gart_end >> 44));
1196d4496bcSLe Ma }
1206d4496bcSLe Ma }
1214667fbe2SLe Ma }
1226d4496bcSLe Ma
123dd1a02e2SLijo Lazar static void
gfxhub_v1_2_xcc_init_system_aperture_regs(struct amdgpu_device * adev,uint32_t xcc_mask)124dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_init_system_aperture_regs(struct amdgpu_device *adev,
125dd1a02e2SLijo Lazar uint32_t xcc_mask)
1266d4496bcSLe Ma {
1276d4496bcSLe Ma uint64_t value;
1286d4496bcSLe Ma uint32_t tmp;
129dd1a02e2SLijo Lazar int i;
1306d4496bcSLe Ma
131dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
1326d4496bcSLe Ma /* Program the AGP BAR */
133659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regMC_VM_AGP_BASE, 0);
134659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regMC_VM_AGP_BOT, adev->gmc.agp_start >> 24);
135659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regMC_VM_AGP_TOP, adev->gmc.agp_end >> 24);
1366d4496bcSLe Ma
1376d4496bcSLe Ma if (!amdgpu_sriov_vf(adev) || adev->asic_type <= CHIP_VEGA10) {
1386d4496bcSLe Ma /* Program the system aperture low logical page number. */
139659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regMC_VM_SYSTEM_APERTURE_LOW_ADDR,
1406d4496bcSLe Ma min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18);
1416d4496bcSLe Ma
142*08dde830SAlex Deucher if (adev->apu_flags & (AMD_APU_IS_RAVEN2 |
143*08dde830SAlex Deucher AMD_APU_IS_RENOIR |
144*08dde830SAlex Deucher AMD_APU_IS_GREEN_SARDINE))
1456d4496bcSLe Ma /*
1466d4496bcSLe Ma * Raven2 has a HW issue that it is unable to use the
1476d4496bcSLe Ma * vram which is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR.
1486d4496bcSLe Ma * So here is the workaround that increase system
1496d4496bcSLe Ma * aperture high address (add 1) to get rid of the VM
1506d4496bcSLe Ma * fault and hardware hang.
1516d4496bcSLe Ma */
152659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i),
1536d4496bcSLe Ma regMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1546d4496bcSLe Ma max((adev->gmc.fb_end >> 18) + 0x1,
1556d4496bcSLe Ma adev->gmc.agp_end >> 18));
1566d4496bcSLe Ma else
157659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i),
1586d4496bcSLe Ma regMC_VM_SYSTEM_APERTURE_HIGH_ADDR,
1596d4496bcSLe Ma max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18);
1606d4496bcSLe Ma
1616d4496bcSLe Ma /* Set default page address. */
1626d4496bcSLe Ma value = amdgpu_gmc_vram_mc2pa(adev, adev->mem_scratch.gpu_addr);
163659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB,
1646d4496bcSLe Ma (u32)(value >> 12));
165659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_SYSTEM_APERTURE_DEFAULT_ADDR_MSB,
1666d4496bcSLe Ma (u32)(value >> 44));
1676d4496bcSLe Ma
1686d4496bcSLe Ma /* Program "protection fault". */
169659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_LO32,
1706d4496bcSLe Ma (u32)(adev->dummy_page_addr >> 12));
171659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_DEFAULT_ADDR_HI32,
1726d4496bcSLe Ma (u32)((u64)adev->dummy_page_addr >> 44));
1736d4496bcSLe Ma
174659a4ab8SLijo Lazar tmp = RREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_CNTL2);
1756d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL2,
1766d4496bcSLe Ma ACTIVE_PAGE_MIGRATION_PTE_READ_RETRY, 1);
177659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_CNTL2, tmp);
1786d4496bcSLe Ma }
1796d4496bcSLe Ma
1806d4496bcSLe Ma /* In the case squeezing vram into GART aperture, we don't use
1816d4496bcSLe Ma * FB aperture and AGP aperture. Disable them.
1826d4496bcSLe Ma */
1836d4496bcSLe Ma if (adev->gmc.pdb0_bo) {
184659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_FB_LOCATION_TOP, 0);
185659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_FB_LOCATION_BASE, 0x00FFFFFF);
186659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_AGP_TOP, 0);
187659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_AGP_BOT, 0xFFFFFF);
188659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_SYSTEM_APERTURE_LOW_ADDR, 0x3FFFFFFF);
189659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_SYSTEM_APERTURE_HIGH_ADDR, 0);
1904667fbe2SLe Ma }
1916d4496bcSLe Ma }
1926d4496bcSLe Ma }
1936d4496bcSLe Ma
gfxhub_v1_2_xcc_init_tlb_regs(struct amdgpu_device * adev,uint32_t xcc_mask)194dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_init_tlb_regs(struct amdgpu_device *adev,
195dd1a02e2SLijo Lazar uint32_t xcc_mask)
1966d4496bcSLe Ma {
1976d4496bcSLe Ma uint32_t tmp;
198dd1a02e2SLijo Lazar int i;
1996d4496bcSLe Ma
200dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
2016d4496bcSLe Ma /* Setup TLB control */
202659a4ab8SLijo Lazar tmp = RREG32_SOC15(GC, GET_INST(GC, i), regMC_VM_MX_L1_TLB_CNTL);
2036d4496bcSLe Ma
2044667fbe2SLe Ma tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
2054667fbe2SLe Ma ENABLE_L1_TLB, 1);
2064667fbe2SLe Ma tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
2074667fbe2SLe Ma SYSTEM_ACCESS_MODE, 3);
2086d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
2096d4496bcSLe Ma ENABLE_ADVANCED_DRIVER_MODEL, 1);
2106d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
2116d4496bcSLe Ma SYSTEM_APERTURE_UNMAPPED_ACCESS, 0);
2126d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL,
2136d4496bcSLe Ma MTYPE, MTYPE_UC);/* XXX for emulation. */
2146d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ATC_EN, 1);
2156d4496bcSLe Ma
216659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regMC_VM_MX_L1_TLB_CNTL, tmp);
2174667fbe2SLe Ma }
2186d4496bcSLe Ma }
2196d4496bcSLe Ma
gfxhub_v1_2_xcc_init_cache_regs(struct amdgpu_device * adev,uint32_t xcc_mask)220dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_init_cache_regs(struct amdgpu_device *adev,
221dd1a02e2SLijo Lazar uint32_t xcc_mask)
2226d4496bcSLe Ma {
2236d4496bcSLe Ma uint32_t tmp;
224dd1a02e2SLijo Lazar int i;
2256d4496bcSLe Ma
226dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
2276d4496bcSLe Ma /* Setup L2 cache */
228659a4ab8SLijo Lazar tmp = RREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_CNTL);
2296d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 1);
2306d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING, 1);
2316d4496bcSLe Ma /* XXX for emulation, Refer to closed source code.*/
2326d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, L2_PDE0_CACHE_TAG_GENERATION_MODE,
2336d4496bcSLe Ma 0);
2346d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, PDE_FAULT_CLASSIFICATION, 0);
2356d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, CONTEXT1_IDENTITY_ACCESS_MODE, 1);
2366d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, IDENTITY_MODE_FRAGMENT_SIZE, 0);
237659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regVM_L2_CNTL, tmp);
2386d4496bcSLe Ma
239659a4ab8SLijo Lazar tmp = RREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_CNTL2);
2406d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS, 1);
2416d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL2, INVALIDATE_L2_CACHE, 1);
242659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regVM_L2_CNTL2, tmp);
2436d4496bcSLe Ma
2446d4496bcSLe Ma tmp = regVM_L2_CNTL3_DEFAULT;
2456d4496bcSLe Ma if (adev->gmc.translate_further) {
2466d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 12);
2476d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
2486d4496bcSLe Ma L2_CACHE_BIGK_FRAGMENT_SIZE, 9);
2496d4496bcSLe Ma } else {
2506d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3, BANK_SELECT, 9);
2516d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL3,
2526d4496bcSLe Ma L2_CACHE_BIGK_FRAGMENT_SIZE, 6);
2536d4496bcSLe Ma }
254659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regVM_L2_CNTL3, tmp);
2556d4496bcSLe Ma
2566d4496bcSLe Ma tmp = regVM_L2_CNTL4_DEFAULT;
2577a7aaab0SRajneesh Bhardwaj /* For AMD APP APUs setup WC memory */
2587a7aaab0SRajneesh Bhardwaj if (adev->gmc.xgmi.connected_to_cpu || adev->gmc.is_app_apu) {
2596d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 1);
2606d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 1);
2616d4496bcSLe Ma } else {
2626d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PDE_REQUEST_PHYSICAL, 0);
2636d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL4, VMC_TAP_PTE_REQUEST_PHYSICAL, 0);
2646d4496bcSLe Ma }
265659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, i), regVM_L2_CNTL4, tmp);
2664667fbe2SLe Ma }
2676d4496bcSLe Ma }
2686d4496bcSLe Ma
gfxhub_v1_2_xcc_enable_system_domain(struct amdgpu_device * adev,uint32_t xcc_mask)269dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_enable_system_domain(struct amdgpu_device *adev,
270dd1a02e2SLijo Lazar uint32_t xcc_mask)
2716d4496bcSLe Ma {
2726d4496bcSLe Ma uint32_t tmp;
273dd1a02e2SLijo Lazar int i;
2746d4496bcSLe Ma
275dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
276659a4ab8SLijo Lazar tmp = RREG32_SOC15(GC, GET_INST(GC, i), regVM_CONTEXT0_CNTL);
2776d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, ENABLE_CONTEXT, 1);
2786d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_DEPTH,
2796d4496bcSLe Ma adev->gmc.vmid0_page_table_depth);
2806d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL, PAGE_TABLE_BLOCK_SIZE,
2816d4496bcSLe Ma adev->gmc.vmid0_page_table_block_size);
2826d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT0_CNTL,
2836d4496bcSLe Ma RETRY_PERMISSION_OR_INVALID_PAGE_FAULT, 0);
284659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regVM_CONTEXT0_CNTL, tmp);
2854667fbe2SLe Ma }
2866d4496bcSLe Ma }
2876d4496bcSLe Ma
288dd1a02e2SLijo Lazar static void
gfxhub_v1_2_xcc_disable_identity_aperture(struct amdgpu_device * adev,uint32_t xcc_mask)289dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_disable_identity_aperture(struct amdgpu_device *adev,
290dd1a02e2SLijo Lazar uint32_t xcc_mask)
2916d4496bcSLe Ma {
292dd1a02e2SLijo Lazar int i;
2934667fbe2SLe Ma
294dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
295659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
2964667fbe2SLe Ma regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_LO32,
2976d4496bcSLe Ma 0XFFFFFFFF);
298659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
2994667fbe2SLe Ma regVM_L2_CONTEXT1_IDENTITY_APERTURE_LOW_ADDR_HI32,
3006d4496bcSLe Ma 0x0000000F);
3016d4496bcSLe Ma
302659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
3034667fbe2SLe Ma regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_LO32,
3046d4496bcSLe Ma 0);
305659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
3064667fbe2SLe Ma regVM_L2_CONTEXT1_IDENTITY_APERTURE_HIGH_ADDR_HI32,
3076d4496bcSLe Ma 0);
3086d4496bcSLe Ma
309659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
3104667fbe2SLe Ma regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_LO32, 0);
311659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i),
3124667fbe2SLe Ma regVM_L2_CONTEXT_IDENTITY_PHYSICAL_OFFSET_HI32, 0);
3134667fbe2SLe Ma }
3146d4496bcSLe Ma }
3156d4496bcSLe Ma
gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device * adev,uint32_t xcc_mask)316dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_setup_vmid_config(struct amdgpu_device *adev,
317dd1a02e2SLijo Lazar uint32_t xcc_mask)
3186d4496bcSLe Ma {
319ed42f2ccSLe Ma struct amdgpu_vmhub *hub;
32062e6771aSSrinivasan Shanmugam unsigned int num_level, block_size;
3216d4496bcSLe Ma uint32_t tmp;
322dd1a02e2SLijo Lazar int i, j;
3236d4496bcSLe Ma
3246d4496bcSLe Ma num_level = adev->vm_manager.num_level;
3256d4496bcSLe Ma block_size = adev->vm_manager.block_size;
3266d4496bcSLe Ma if (adev->gmc.translate_further)
3276d4496bcSLe Ma num_level -= 1;
3286d4496bcSLe Ma else
3296d4496bcSLe Ma block_size -= 9;
3306d4496bcSLe Ma
331dd1a02e2SLijo Lazar for_each_inst(j, xcc_mask) {
332ed42f2ccSLe Ma hub = &adev->vmhub[AMDGPU_GFXHUB(j)];
3336d4496bcSLe Ma for (i = 0; i <= 14; i++) {
334659a4ab8SLijo Lazar tmp = RREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regVM_CONTEXT1_CNTL, i);
3356d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, ENABLE_CONTEXT, 1);
3366d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL, PAGE_TABLE_DEPTH,
3376d4496bcSLe Ma num_level);
3386d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3396d4496bcSLe Ma RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
3406d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3416d4496bcSLe Ma DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT,
3426d4496bcSLe Ma 1);
3436d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3446d4496bcSLe Ma PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
3456d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3466d4496bcSLe Ma VALID_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
3476d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3486d4496bcSLe Ma READ_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
3496d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3506d4496bcSLe Ma WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
3516d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3526d4496bcSLe Ma EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, 1);
3536d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3546d4496bcSLe Ma PAGE_TABLE_BLOCK_SIZE,
3556d4496bcSLe Ma block_size);
3566d4496bcSLe Ma /* Send no-retry XNACK on fault to suppress VM fault storm.
357d2555586SAmber Lin * On 9.4.2 and 9.4.3, XNACK can be enabled in
358d2555586SAmber Lin * the SQ per-process.
3596d4496bcSLe Ma * Retry faults need to be enabled for that to work.
3606d4496bcSLe Ma */
3616d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
3626d4496bcSLe Ma RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
3636d4496bcSLe Ma !adev->gmc.noretry ||
364d2555586SAmber Lin adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
365d2555586SAmber Lin adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3));
366659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regVM_CONTEXT1_CNTL,
3676d4496bcSLe Ma i * hub->ctx_distance, tmp);
368659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
3696d4496bcSLe Ma regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
3706d4496bcSLe Ma i * hub->ctx_addr_distance, 0);
371659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
3726d4496bcSLe Ma regVM_CONTEXT1_PAGE_TABLE_START_ADDR_HI32,
3736d4496bcSLe Ma i * hub->ctx_addr_distance, 0);
374659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
3756d4496bcSLe Ma regVM_CONTEXT1_PAGE_TABLE_END_ADDR_LO32,
3766d4496bcSLe Ma i * hub->ctx_addr_distance,
3776d4496bcSLe Ma lower_32_bits(adev->vm_manager.max_pfn - 1));
378659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, j),
3796d4496bcSLe Ma regVM_CONTEXT1_PAGE_TABLE_END_ADDR_HI32,
3806d4496bcSLe Ma i * hub->ctx_addr_distance,
3816d4496bcSLe Ma upper_32_bits(adev->vm_manager.max_pfn - 1));
3826d4496bcSLe Ma }
3836d4496bcSLe Ma }
3844667fbe2SLe Ma }
3856d4496bcSLe Ma
gfxhub_v1_2_xcc_program_invalidation(struct amdgpu_device * adev,uint32_t xcc_mask)386dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_program_invalidation(struct amdgpu_device *adev,
387dd1a02e2SLijo Lazar uint32_t xcc_mask)
3886d4496bcSLe Ma {
389ed42f2ccSLe Ma struct amdgpu_vmhub *hub;
390dd1a02e2SLijo Lazar unsigned int i, j;
3916d4496bcSLe Ma
392dd1a02e2SLijo Lazar for_each_inst(j, xcc_mask) {
393ed42f2ccSLe Ma hub = &adev->vmhub[AMDGPU_GFXHUB(j)];
3948078f1c6SLijo Lazar
3956d4496bcSLe Ma for (i = 0 ; i < 18; ++i) {
396659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32,
3976d4496bcSLe Ma i * hub->eng_addr_distance, 0xffffffff);
398659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regVM_INVALIDATE_ENG0_ADDR_RANGE_HI32,
3996d4496bcSLe Ma i * hub->eng_addr_distance, 0x1f);
4006d4496bcSLe Ma }
4016d4496bcSLe Ma }
4024667fbe2SLe Ma }
4036d4496bcSLe Ma
gfxhub_v1_2_xcc_gart_enable(struct amdgpu_device * adev,uint32_t xcc_mask)404dd1a02e2SLijo Lazar static int gfxhub_v1_2_xcc_gart_enable(struct amdgpu_device *adev,
405dd1a02e2SLijo Lazar uint32_t xcc_mask)
4066d4496bcSLe Ma {
4076d4496bcSLe Ma /* GART Enable. */
408dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_init_gart_aperture_regs(adev, xcc_mask);
409dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_init_system_aperture_regs(adev, xcc_mask);
410dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_init_tlb_regs(adev, xcc_mask);
4116d4496bcSLe Ma if (!amdgpu_sriov_vf(adev))
412dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_init_cache_regs(adev, xcc_mask);
4136d4496bcSLe Ma
414dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_enable_system_domain(adev, xcc_mask);
4156d4496bcSLe Ma if (!amdgpu_sriov_vf(adev))
416dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_disable_identity_aperture(adev, xcc_mask);
417dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_setup_vmid_config(adev, xcc_mask);
418dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_program_invalidation(adev, xcc_mask);
4196d4496bcSLe Ma
4206d4496bcSLe Ma return 0;
4216d4496bcSLe Ma }
4226d4496bcSLe Ma
gfxhub_v1_2_gart_enable(struct amdgpu_device * adev)423dd1a02e2SLijo Lazar static int gfxhub_v1_2_gart_enable(struct amdgpu_device *adev)
424dd1a02e2SLijo Lazar {
425dd1a02e2SLijo Lazar uint32_t xcc_mask;
426dd1a02e2SLijo Lazar
427dd1a02e2SLijo Lazar xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0);
428dd1a02e2SLijo Lazar return gfxhub_v1_2_xcc_gart_enable(adev, xcc_mask);
429dd1a02e2SLijo Lazar }
430dd1a02e2SLijo Lazar
gfxhub_v1_2_xcc_gart_disable(struct amdgpu_device * adev,uint32_t xcc_mask)431dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_gart_disable(struct amdgpu_device *adev,
432dd1a02e2SLijo Lazar uint32_t xcc_mask)
4336d4496bcSLe Ma {
434ed42f2ccSLe Ma struct amdgpu_vmhub *hub;
4356d4496bcSLe Ma u32 tmp;
436dd1a02e2SLijo Lazar u32 i, j;
4376d4496bcSLe Ma
438dd1a02e2SLijo Lazar for_each_inst(j, xcc_mask) {
439ed42f2ccSLe Ma hub = &adev->vmhub[AMDGPU_GFXHUB(j)];
4406d4496bcSLe Ma /* Disable all tables */
4416d4496bcSLe Ma for (i = 0; i < 16; i++)
442659a4ab8SLijo Lazar WREG32_SOC15_OFFSET(GC, GET_INST(GC, j), regVM_CONTEXT0_CNTL,
4436d4496bcSLe Ma i * hub->ctx_distance, 0);
4446d4496bcSLe Ma
4456d4496bcSLe Ma /* Setup TLB control */
446659a4ab8SLijo Lazar tmp = RREG32_SOC15(GC, GET_INST(GC, j), regMC_VM_MX_L1_TLB_CNTL);
4476d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, MC_VM_MX_L1_TLB_CNTL, ENABLE_L1_TLB, 0);
4486d4496bcSLe Ma tmp = REG_SET_FIELD(tmp,
4496d4496bcSLe Ma MC_VM_MX_L1_TLB_CNTL,
4506d4496bcSLe Ma ENABLE_ADVANCED_DRIVER_MODEL,
4516d4496bcSLe Ma 0);
452659a4ab8SLijo Lazar WREG32_SOC15_RLC(GC, GET_INST(GC, j), regMC_VM_MX_L1_TLB_CNTL, tmp);
4536d4496bcSLe Ma
4546d4496bcSLe Ma /* Setup L2 cache */
455659a4ab8SLijo Lazar tmp = RREG32_SOC15(GC, GET_INST(GC, j), regVM_L2_CNTL);
4566d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_CNTL, ENABLE_L2_CACHE, 0);
457659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, j), regVM_L2_CNTL, tmp);
458659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, j), regVM_L2_CNTL3, 0);
4594667fbe2SLe Ma }
4606d4496bcSLe Ma }
4616d4496bcSLe Ma
gfxhub_v1_2_gart_disable(struct amdgpu_device * adev)462dd1a02e2SLijo Lazar static void gfxhub_v1_2_gart_disable(struct amdgpu_device *adev)
463dd1a02e2SLijo Lazar {
464dd1a02e2SLijo Lazar uint32_t xcc_mask;
465dd1a02e2SLijo Lazar
466dd1a02e2SLijo Lazar xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0);
467dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_gart_disable(adev, xcc_mask);
468dd1a02e2SLijo Lazar }
469dd1a02e2SLijo Lazar
gfxhub_v1_2_xcc_set_fault_enable_default(struct amdgpu_device * adev,bool value,uint32_t xcc_mask)470dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_set_fault_enable_default(struct amdgpu_device *adev,
471dd1a02e2SLijo Lazar bool value,
472dd1a02e2SLijo Lazar uint32_t xcc_mask)
4736d4496bcSLe Ma {
4746d4496bcSLe Ma u32 tmp;
475dd1a02e2SLijo Lazar int i;
4764667fbe2SLe Ma
477dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
478659a4ab8SLijo Lazar tmp = RREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_CNTL);
4796d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
4806d4496bcSLe Ma RANGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4816d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
4826d4496bcSLe Ma PDE0_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4836d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
4846d4496bcSLe Ma PDE1_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4856d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
4866d4496bcSLe Ma PDE2_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4876d4496bcSLe Ma tmp = REG_SET_FIELD(tmp,
4886d4496bcSLe Ma VM_L2_PROTECTION_FAULT_CNTL,
4896d4496bcSLe Ma TRANSLATE_FURTHER_PROTECTION_FAULT_ENABLE_DEFAULT,
4906d4496bcSLe Ma value);
4916d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
4926d4496bcSLe Ma NACK_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4936d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
4946d4496bcSLe Ma DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4956d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
4966d4496bcSLe Ma VALID_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4976d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
4986d4496bcSLe Ma READ_PROTECTION_FAULT_ENABLE_DEFAULT, value);
4996d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
5006d4496bcSLe Ma WRITE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
5016d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
5026d4496bcSLe Ma EXECUTE_PROTECTION_FAULT_ENABLE_DEFAULT, value);
5036d4496bcSLe Ma if (!value) {
5046d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
5056d4496bcSLe Ma CRASH_ON_NO_RETRY_FAULT, 1);
5066d4496bcSLe Ma tmp = REG_SET_FIELD(tmp, VM_L2_PROTECTION_FAULT_CNTL,
5076d4496bcSLe Ma CRASH_ON_RETRY_FAULT, 1);
5086d4496bcSLe Ma }
509659a4ab8SLijo Lazar WREG32_SOC15(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_CNTL, tmp);
5104667fbe2SLe Ma }
5116d4496bcSLe Ma }
5126d4496bcSLe Ma
513dd1a02e2SLijo Lazar /**
514dd1a02e2SLijo Lazar * gfxhub_v1_2_set_fault_enable_default - update GART/VM fault handling
515dd1a02e2SLijo Lazar *
516dd1a02e2SLijo Lazar * @adev: amdgpu_device pointer
517dd1a02e2SLijo Lazar * @value: true redirects VM faults to the default page
518dd1a02e2SLijo Lazar */
gfxhub_v1_2_set_fault_enable_default(struct amdgpu_device * adev,bool value)519dd1a02e2SLijo Lazar static void gfxhub_v1_2_set_fault_enable_default(struct amdgpu_device *adev,
520dd1a02e2SLijo Lazar bool value)
521dd1a02e2SLijo Lazar {
522dd1a02e2SLijo Lazar uint32_t xcc_mask;
523dd1a02e2SLijo Lazar
524dd1a02e2SLijo Lazar xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0);
525dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_set_fault_enable_default(adev, value, xcc_mask);
526dd1a02e2SLijo Lazar }
527dd1a02e2SLijo Lazar
gfxhub_v1_2_xcc_init(struct amdgpu_device * adev,uint32_t xcc_mask)528dd1a02e2SLijo Lazar static void gfxhub_v1_2_xcc_init(struct amdgpu_device *adev, uint32_t xcc_mask)
5296d4496bcSLe Ma {
530b35ce49aSLe Ma struct amdgpu_vmhub *hub;
531dd1a02e2SLijo Lazar int i;
532b35ce49aSLe Ma
533dd1a02e2SLijo Lazar for_each_inst(i, xcc_mask) {
534b35ce49aSLe Ma hub = &adev->vmhub[AMDGPU_GFXHUB(i)];
5356d4496bcSLe Ma
5366d4496bcSLe Ma hub->ctx0_ptb_addr_lo32 =
537659a4ab8SLijo Lazar SOC15_REG_OFFSET(GC, GET_INST(GC, i),
5386d4496bcSLe Ma regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32);
5396d4496bcSLe Ma hub->ctx0_ptb_addr_hi32 =
540659a4ab8SLijo Lazar SOC15_REG_OFFSET(GC, GET_INST(GC, i),
5416d4496bcSLe Ma regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_HI32);
5426d4496bcSLe Ma hub->vm_inv_eng0_sem =
543659a4ab8SLijo Lazar SOC15_REG_OFFSET(GC, GET_INST(GC, i), regVM_INVALIDATE_ENG0_SEM);
5446d4496bcSLe Ma hub->vm_inv_eng0_req =
545659a4ab8SLijo Lazar SOC15_REG_OFFSET(GC, GET_INST(GC, i), regVM_INVALIDATE_ENG0_REQ);
5466d4496bcSLe Ma hub->vm_inv_eng0_ack =
547659a4ab8SLijo Lazar SOC15_REG_OFFSET(GC, GET_INST(GC, i), regVM_INVALIDATE_ENG0_ACK);
5486d4496bcSLe Ma hub->vm_context0_cntl =
549659a4ab8SLijo Lazar SOC15_REG_OFFSET(GC, GET_INST(GC, i), regVM_CONTEXT0_CNTL);
5506d4496bcSLe Ma hub->vm_l2_pro_fault_status =
551659a4ab8SLijo Lazar SOC15_REG_OFFSET(GC, GET_INST(GC, i),
552b35ce49aSLe Ma regVM_L2_PROTECTION_FAULT_STATUS);
5536d4496bcSLe Ma hub->vm_l2_pro_fault_cntl =
554659a4ab8SLijo Lazar SOC15_REG_OFFSET(GC, GET_INST(GC, i), regVM_L2_PROTECTION_FAULT_CNTL);
5556d4496bcSLe Ma
556b35ce49aSLe Ma hub->ctx_distance = regVM_CONTEXT1_CNTL -
557b35ce49aSLe Ma regVM_CONTEXT0_CNTL;
558b35ce49aSLe Ma hub->ctx_addr_distance =
559b35ce49aSLe Ma regVM_CONTEXT1_PAGE_TABLE_BASE_ADDR_LO32 -
5606d4496bcSLe Ma regVM_CONTEXT0_PAGE_TABLE_BASE_ADDR_LO32;
561b35ce49aSLe Ma hub->eng_distance = regVM_INVALIDATE_ENG1_REQ -
562b35ce49aSLe Ma regVM_INVALIDATE_ENG0_REQ;
563b35ce49aSLe Ma hub->eng_addr_distance =
564b35ce49aSLe Ma regVM_INVALIDATE_ENG1_ADDR_RANGE_LO32 -
5656d4496bcSLe Ma regVM_INVALIDATE_ENG0_ADDR_RANGE_LO32;
5666d4496bcSLe Ma }
567b35ce49aSLe Ma }
5686d4496bcSLe Ma
gfxhub_v1_2_init(struct amdgpu_device * adev)569dd1a02e2SLijo Lazar static void gfxhub_v1_2_init(struct amdgpu_device *adev)
570dd1a02e2SLijo Lazar {
571dd1a02e2SLijo Lazar uint32_t xcc_mask;
572dd1a02e2SLijo Lazar
573dd1a02e2SLijo Lazar xcc_mask = GENMASK(NUM_XCC(adev->gfx.xcc_mask) - 1, 0);
574dd1a02e2SLijo Lazar gfxhub_v1_2_xcc_init(adev, xcc_mask);
575dd1a02e2SLijo Lazar }
576dd1a02e2SLijo Lazar
gfxhub_v1_2_get_xgmi_info(struct amdgpu_device * adev)5779eb7681fSShiwu Zhang static int gfxhub_v1_2_get_xgmi_info(struct amdgpu_device *adev)
5789eb7681fSShiwu Zhang {
5799eb7681fSShiwu Zhang u32 max_num_physical_nodes;
5809eb7681fSShiwu Zhang u32 max_physical_node_id;
5819eb7681fSShiwu Zhang u32 xgmi_lfb_cntl;
5829eb7681fSShiwu Zhang u32 max_region;
5839eb7681fSShiwu Zhang u64 seg_size;
5849eb7681fSShiwu Zhang
585659a4ab8SLijo Lazar xgmi_lfb_cntl = RREG32_SOC15(GC, GET_INST(GC, 0), regMC_VM_XGMI_LFB_CNTL);
5869eb7681fSShiwu Zhang seg_size = REG_GET_FIELD(
587659a4ab8SLijo Lazar RREG32_SOC15(GC, GET_INST(GC, 0), regMC_VM_XGMI_LFB_SIZE),
5889eb7681fSShiwu Zhang MC_VM_XGMI_LFB_SIZE, PF_LFB_SIZE) << 24;
5899eb7681fSShiwu Zhang max_region =
5909eb7681fSShiwu Zhang REG_GET_FIELD(xgmi_lfb_cntl, MC_VM_XGMI_LFB_CNTL, PF_MAX_REGION);
5919eb7681fSShiwu Zhang
5929eb7681fSShiwu Zhang
5939eb7681fSShiwu Zhang
5949eb7681fSShiwu Zhang max_num_physical_nodes = 8;
5959eb7681fSShiwu Zhang max_physical_node_id = 7;
5969eb7681fSShiwu Zhang
5979eb7681fSShiwu Zhang /* PF_MAX_REGION=0 means xgmi is disabled */
5989eb7681fSShiwu Zhang if (max_region || adev->gmc.xgmi.connected_to_cpu) {
5999eb7681fSShiwu Zhang adev->gmc.xgmi.num_physical_nodes = max_region + 1;
6009eb7681fSShiwu Zhang
6019eb7681fSShiwu Zhang if (adev->gmc.xgmi.num_physical_nodes > max_num_physical_nodes)
6029eb7681fSShiwu Zhang return -EINVAL;
6039eb7681fSShiwu Zhang
6049eb7681fSShiwu Zhang adev->gmc.xgmi.physical_node_id =
6059eb7681fSShiwu Zhang REG_GET_FIELD(xgmi_lfb_cntl, MC_VM_XGMI_LFB_CNTL,
6069eb7681fSShiwu Zhang PF_LFB_REGION);
6079eb7681fSShiwu Zhang
6089eb7681fSShiwu Zhang if (adev->gmc.xgmi.physical_node_id > max_physical_node_id)
6099eb7681fSShiwu Zhang return -EINVAL;
6109eb7681fSShiwu Zhang
6119eb7681fSShiwu Zhang adev->gmc.xgmi.node_segment_size = seg_size;
6129eb7681fSShiwu Zhang }
6139eb7681fSShiwu Zhang
6149eb7681fSShiwu Zhang return 0;
6159eb7681fSShiwu Zhang }
6166d4496bcSLe Ma
6176d4496bcSLe Ma const struct amdgpu_gfxhub_funcs gfxhub_v1_2_funcs = {
6186d4496bcSLe Ma .get_mc_fb_offset = gfxhub_v1_2_get_mc_fb_offset,
6196d4496bcSLe Ma .setup_vm_pt_regs = gfxhub_v1_2_setup_vm_pt_regs,
6206d4496bcSLe Ma .gart_enable = gfxhub_v1_2_gart_enable,
6216d4496bcSLe Ma .gart_disable = gfxhub_v1_2_gart_disable,
6226d4496bcSLe Ma .set_fault_enable_default = gfxhub_v1_2_set_fault_enable_default,
6236d4496bcSLe Ma .init = gfxhub_v1_2_init,
6249eb7681fSShiwu Zhang .get_xgmi_info = gfxhub_v1_2_get_xgmi_info,
6256d4496bcSLe Ma };
626dfdd6f89SLijo Lazar
gfxhub_v1_2_xcp_resume(void * handle,uint32_t inst_mask)627dfdd6f89SLijo Lazar static int gfxhub_v1_2_xcp_resume(void *handle, uint32_t inst_mask)
628dfdd6f89SLijo Lazar {
629dfdd6f89SLijo Lazar struct amdgpu_device *adev = (struct amdgpu_device *)handle;
630dfdd6f89SLijo Lazar bool value;
631dfdd6f89SLijo Lazar
632dfdd6f89SLijo Lazar if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS)
633dfdd6f89SLijo Lazar value = false;
634dfdd6f89SLijo Lazar else
635dfdd6f89SLijo Lazar value = true;
636dfdd6f89SLijo Lazar
637dfdd6f89SLijo Lazar gfxhub_v1_2_xcc_set_fault_enable_default(adev, value, inst_mask);
638dfdd6f89SLijo Lazar
639dfdd6f89SLijo Lazar if (!amdgpu_sriov_vf(adev))
6401893549aSSrinivasan Shanmugam return gfxhub_v1_2_xcc_gart_enable(adev, inst_mask);
641dfdd6f89SLijo Lazar
6421893549aSSrinivasan Shanmugam return 0;
643dfdd6f89SLijo Lazar }
644dfdd6f89SLijo Lazar
gfxhub_v1_2_xcp_suspend(void * handle,uint32_t inst_mask)645dfdd6f89SLijo Lazar static int gfxhub_v1_2_xcp_suspend(void *handle, uint32_t inst_mask)
646dfdd6f89SLijo Lazar {
647dfdd6f89SLijo Lazar struct amdgpu_device *adev = (struct amdgpu_device *)handle;
648dfdd6f89SLijo Lazar
649dfdd6f89SLijo Lazar if (!amdgpu_sriov_vf(adev))
650dfdd6f89SLijo Lazar gfxhub_v1_2_xcc_gart_disable(adev, inst_mask);
651dfdd6f89SLijo Lazar
652dfdd6f89SLijo Lazar return 0;
653dfdd6f89SLijo Lazar }
654dfdd6f89SLijo Lazar
655dfdd6f89SLijo Lazar struct amdgpu_xcp_ip_funcs gfxhub_v1_2_xcp_funcs = {
656dfdd6f89SLijo Lazar .suspend = &gfxhub_v1_2_xcp_suspend,
657dfdd6f89SLijo Lazar .resume = &gfxhub_v1_2_xcp_resume
658dfdd6f89SLijo Lazar };
659