1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23e60f8db5SAlex Xie #include <linux/firmware.h> 24fd5fd480SChunming Zhou #include <drm/drm_cache.h> 25e60f8db5SAlex Xie #include "amdgpu.h" 26e60f8db5SAlex Xie #include "gmc_v9_0.h" 278d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 282cddc50eSHuang Rui #include "amdgpu_gem.h" 29e60f8db5SAlex Xie 3075199b8cSFeifei Xu #include "hdp/hdp_4_0_offset.h" 3175199b8cSFeifei Xu #include "hdp/hdp_4_0_sh_mask.h" 32cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 33135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 34135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 35fb960bd2SFeifei Xu #include "vega10_enum.h" 3665417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 376ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 38250b4228SChristian König #include "oss/osssys_4_0_offset.h" 39e60f8db5SAlex Xie 40946a4d5bSShaoyun Liu #include "soc15.h" 41e60f8db5SAlex Xie #include "soc15_common.h" 4290c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 43e60f8db5SAlex Xie 44e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 45e60f8db5SAlex Xie #include "mmhub_v1_0.h" 46bf0a60b7SAlex Deucher #include "gfxhub_v1_1.h" 47e60f8db5SAlex Xie 4844a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 4944a99b65SAndrey Grodzovsky 50791c4769Sxinhui pan #include "amdgpu_ras.h" 51791c4769Sxinhui pan 52ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 53ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 54ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 55ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 56ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 57ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 58ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 59ebdef28eSAlex Deucher 60e60f8db5SAlex Xie /* XXX Move this macro to VEGA10 header file, which is like vid.h for VI.*/ 61e60f8db5SAlex Xie #define AMDGPU_NUM_OF_VMIDS 8 62e60f8db5SAlex Xie 63e60f8db5SAlex Xie static const u32 golden_settings_vega10_hdp[] = 64e60f8db5SAlex Xie { 65e60f8db5SAlex Xie 0xf64, 0x0fffffff, 0x00000000, 66e60f8db5SAlex Xie 0xf65, 0x0fffffff, 0x00000000, 67e60f8db5SAlex Xie 0xf66, 0x0fffffff, 0x00000000, 68e60f8db5SAlex Xie 0xf67, 0x0fffffff, 0x00000000, 69e60f8db5SAlex Xie 0xf68, 0x0fffffff, 0x00000000, 70e60f8db5SAlex Xie 0xf6a, 0x0fffffff, 0x00000000, 71e60f8db5SAlex Xie 0xf6b, 0x0fffffff, 0x00000000, 72e60f8db5SAlex Xie 0xf6c, 0x0fffffff, 0x00000000, 73e60f8db5SAlex Xie 0xf6d, 0x0fffffff, 0x00000000, 74e60f8db5SAlex Xie 0xf6e, 0x0fffffff, 0x00000000, 75e60f8db5SAlex Xie }; 76e60f8db5SAlex Xie 77946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 785c583018SEvan Quan { 79946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 80946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 815c583018SEvan Quan }; 825c583018SEvan Quan 83946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 845c583018SEvan Quan { 85946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 86946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 875c583018SEvan Quan }; 885c583018SEvan Quan 89791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 90791c4769Sxinhui pan (0x000143c0 + 0x00000000), 91791c4769Sxinhui pan (0x000143c0 + 0x00000800), 92791c4769Sxinhui pan (0x000143c0 + 0x00001000), 93791c4769Sxinhui pan (0x000143c0 + 0x00001800), 94791c4769Sxinhui pan (0x000543c0 + 0x00000000), 95791c4769Sxinhui pan (0x000543c0 + 0x00000800), 96791c4769Sxinhui pan (0x000543c0 + 0x00001000), 97791c4769Sxinhui pan (0x000543c0 + 0x00001800), 98791c4769Sxinhui pan (0x000943c0 + 0x00000000), 99791c4769Sxinhui pan (0x000943c0 + 0x00000800), 100791c4769Sxinhui pan (0x000943c0 + 0x00001000), 101791c4769Sxinhui pan (0x000943c0 + 0x00001800), 102791c4769Sxinhui pan (0x000d43c0 + 0x00000000), 103791c4769Sxinhui pan (0x000d43c0 + 0x00000800), 104791c4769Sxinhui pan (0x000d43c0 + 0x00001000), 105791c4769Sxinhui pan (0x000d43c0 + 0x00001800), 106791c4769Sxinhui pan (0x001143c0 + 0x00000000), 107791c4769Sxinhui pan (0x001143c0 + 0x00000800), 108791c4769Sxinhui pan (0x001143c0 + 0x00001000), 109791c4769Sxinhui pan (0x001143c0 + 0x00001800), 110791c4769Sxinhui pan (0x001543c0 + 0x00000000), 111791c4769Sxinhui pan (0x001543c0 + 0x00000800), 112791c4769Sxinhui pan (0x001543c0 + 0x00001000), 113791c4769Sxinhui pan (0x001543c0 + 0x00001800), 114791c4769Sxinhui pan (0x001943c0 + 0x00000000), 115791c4769Sxinhui pan (0x001943c0 + 0x00000800), 116791c4769Sxinhui pan (0x001943c0 + 0x00001000), 117791c4769Sxinhui pan (0x001943c0 + 0x00001800), 118791c4769Sxinhui pan (0x001d43c0 + 0x00000000), 119791c4769Sxinhui pan (0x001d43c0 + 0x00000800), 120791c4769Sxinhui pan (0x001d43c0 + 0x00001000), 121791c4769Sxinhui pan (0x001d43c0 + 0x00001800), 12202bab923SDavid Panariti }; 12302bab923SDavid Panariti 124791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 125791c4769Sxinhui pan (0x000143e0 + 0x00000000), 126791c4769Sxinhui pan (0x000143e0 + 0x00000800), 127791c4769Sxinhui pan (0x000143e0 + 0x00001000), 128791c4769Sxinhui pan (0x000143e0 + 0x00001800), 129791c4769Sxinhui pan (0x000543e0 + 0x00000000), 130791c4769Sxinhui pan (0x000543e0 + 0x00000800), 131791c4769Sxinhui pan (0x000543e0 + 0x00001000), 132791c4769Sxinhui pan (0x000543e0 + 0x00001800), 133791c4769Sxinhui pan (0x000943e0 + 0x00000000), 134791c4769Sxinhui pan (0x000943e0 + 0x00000800), 135791c4769Sxinhui pan (0x000943e0 + 0x00001000), 136791c4769Sxinhui pan (0x000943e0 + 0x00001800), 137791c4769Sxinhui pan (0x000d43e0 + 0x00000000), 138791c4769Sxinhui pan (0x000d43e0 + 0x00000800), 139791c4769Sxinhui pan (0x000d43e0 + 0x00001000), 140791c4769Sxinhui pan (0x000d43e0 + 0x00001800), 141791c4769Sxinhui pan (0x001143e0 + 0x00000000), 142791c4769Sxinhui pan (0x001143e0 + 0x00000800), 143791c4769Sxinhui pan (0x001143e0 + 0x00001000), 144791c4769Sxinhui pan (0x001143e0 + 0x00001800), 145791c4769Sxinhui pan (0x001543e0 + 0x00000000), 146791c4769Sxinhui pan (0x001543e0 + 0x00000800), 147791c4769Sxinhui pan (0x001543e0 + 0x00001000), 148791c4769Sxinhui pan (0x001543e0 + 0x00001800), 149791c4769Sxinhui pan (0x001943e0 + 0x00000000), 150791c4769Sxinhui pan (0x001943e0 + 0x00000800), 151791c4769Sxinhui pan (0x001943e0 + 0x00001000), 152791c4769Sxinhui pan (0x001943e0 + 0x00001800), 153791c4769Sxinhui pan (0x001d43e0 + 0x00000000), 154791c4769Sxinhui pan (0x001d43e0 + 0x00000800), 155791c4769Sxinhui pan (0x001d43e0 + 0x00001000), 156791c4769Sxinhui pan (0x001d43e0 + 0x00001800), 15702bab923SDavid Panariti }; 15802bab923SDavid Panariti 159791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_status_addrs[] = { 160791c4769Sxinhui pan (0x000143c2 + 0x00000000), 161791c4769Sxinhui pan (0x000143c2 + 0x00000800), 162791c4769Sxinhui pan (0x000143c2 + 0x00001000), 163791c4769Sxinhui pan (0x000143c2 + 0x00001800), 164791c4769Sxinhui pan (0x000543c2 + 0x00000000), 165791c4769Sxinhui pan (0x000543c2 + 0x00000800), 166791c4769Sxinhui pan (0x000543c2 + 0x00001000), 167791c4769Sxinhui pan (0x000543c2 + 0x00001800), 168791c4769Sxinhui pan (0x000943c2 + 0x00000000), 169791c4769Sxinhui pan (0x000943c2 + 0x00000800), 170791c4769Sxinhui pan (0x000943c2 + 0x00001000), 171791c4769Sxinhui pan (0x000943c2 + 0x00001800), 172791c4769Sxinhui pan (0x000d43c2 + 0x00000000), 173791c4769Sxinhui pan (0x000d43c2 + 0x00000800), 174791c4769Sxinhui pan (0x000d43c2 + 0x00001000), 175791c4769Sxinhui pan (0x000d43c2 + 0x00001800), 176791c4769Sxinhui pan (0x001143c2 + 0x00000000), 177791c4769Sxinhui pan (0x001143c2 + 0x00000800), 178791c4769Sxinhui pan (0x001143c2 + 0x00001000), 179791c4769Sxinhui pan (0x001143c2 + 0x00001800), 180791c4769Sxinhui pan (0x001543c2 + 0x00000000), 181791c4769Sxinhui pan (0x001543c2 + 0x00000800), 182791c4769Sxinhui pan (0x001543c2 + 0x00001000), 183791c4769Sxinhui pan (0x001543c2 + 0x00001800), 184791c4769Sxinhui pan (0x001943c2 + 0x00000000), 185791c4769Sxinhui pan (0x001943c2 + 0x00000800), 186791c4769Sxinhui pan (0x001943c2 + 0x00001000), 187791c4769Sxinhui pan (0x001943c2 + 0x00001800), 188791c4769Sxinhui pan (0x001d43c2 + 0x00000000), 189791c4769Sxinhui pan (0x001d43c2 + 0x00000800), 190791c4769Sxinhui pan (0x001d43c2 + 0x00001000), 191791c4769Sxinhui pan (0x001d43c2 + 0x00001800), 19202bab923SDavid Panariti }; 19302bab923SDavid Panariti 194791c4769Sxinhui pan static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 195791c4769Sxinhui pan struct amdgpu_irq_src *src, 196791c4769Sxinhui pan unsigned type, 197791c4769Sxinhui pan enum amdgpu_interrupt_state state) 198791c4769Sxinhui pan { 199791c4769Sxinhui pan u32 bits, i, tmp, reg; 200791c4769Sxinhui pan 201791c4769Sxinhui pan bits = 0x7f; 202791c4769Sxinhui pan 203791c4769Sxinhui pan switch (state) { 204791c4769Sxinhui pan case AMDGPU_IRQ_STATE_DISABLE: 205791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 206791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 207791c4769Sxinhui pan tmp = RREG32(reg); 208791c4769Sxinhui pan tmp &= ~bits; 209791c4769Sxinhui pan WREG32(reg, tmp); 210791c4769Sxinhui pan } 211791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 212791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 213791c4769Sxinhui pan tmp = RREG32(reg); 214791c4769Sxinhui pan tmp &= ~bits; 215791c4769Sxinhui pan WREG32(reg, tmp); 216791c4769Sxinhui pan } 217791c4769Sxinhui pan break; 218791c4769Sxinhui pan case AMDGPU_IRQ_STATE_ENABLE: 219791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 220791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 221791c4769Sxinhui pan tmp = RREG32(reg); 222791c4769Sxinhui pan tmp |= bits; 223791c4769Sxinhui pan WREG32(reg, tmp); 224791c4769Sxinhui pan } 225791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 226791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 227791c4769Sxinhui pan tmp = RREG32(reg); 228791c4769Sxinhui pan tmp |= bits; 229791c4769Sxinhui pan WREG32(reg, tmp); 230791c4769Sxinhui pan } 231791c4769Sxinhui pan break; 232791c4769Sxinhui pan default: 233791c4769Sxinhui pan break; 234791c4769Sxinhui pan } 235791c4769Sxinhui pan 236791c4769Sxinhui pan return 0; 237791c4769Sxinhui pan } 238791c4769Sxinhui pan 239791c4769Sxinhui pan static int gmc_v9_0_process_ras_data_cb(struct amdgpu_device *adev, 240791c4769Sxinhui pan struct amdgpu_iv_entry *entry) 241791c4769Sxinhui pan { 2429b54d201SEric Huang kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); 243791c4769Sxinhui pan amdgpu_ras_reset_gpu(adev, 0); 244791c4769Sxinhui pan return AMDGPU_RAS_UE; 245791c4769Sxinhui pan } 246791c4769Sxinhui pan 247791c4769Sxinhui pan static int gmc_v9_0_process_ecc_irq(struct amdgpu_device *adev, 248791c4769Sxinhui pan struct amdgpu_irq_src *source, 249791c4769Sxinhui pan struct amdgpu_iv_entry *entry) 250791c4769Sxinhui pan { 25114cfde84Sxinhui pan struct ras_common_if *ras_if = adev->gmc.ras_if; 252791c4769Sxinhui pan struct ras_dispatch_if ih_data = { 253791c4769Sxinhui pan .entry = entry, 254791c4769Sxinhui pan }; 25514cfde84Sxinhui pan 25614cfde84Sxinhui pan if (!ras_if) 25714cfde84Sxinhui pan return 0; 25814cfde84Sxinhui pan 25914cfde84Sxinhui pan ih_data.head = *ras_if; 26014cfde84Sxinhui pan 261791c4769Sxinhui pan amdgpu_ras_interrupt_dispatch(adev, &ih_data); 262791c4769Sxinhui pan return 0; 263791c4769Sxinhui pan } 264791c4769Sxinhui pan 265e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 266e60f8db5SAlex Xie struct amdgpu_irq_src *src, 267e60f8db5SAlex Xie unsigned type, 268e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 269e60f8db5SAlex Xie { 270e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 271ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 272e60f8db5SAlex Xie 27311250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 27411250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 27511250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 27611250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 27711250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 27811250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 27911250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 28011250164SChristian König 281e60f8db5SAlex Xie switch (state) { 282e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 283ae6d1416STom St Denis for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 284ae6d1416STom St Denis hub = &adev->vmhub[j]; 285e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 286e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 287e60f8db5SAlex Xie tmp = RREG32(reg); 288e60f8db5SAlex Xie tmp &= ~bits; 289e60f8db5SAlex Xie WREG32(reg, tmp); 290e60f8db5SAlex Xie } 291e60f8db5SAlex Xie } 292e60f8db5SAlex Xie break; 293e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 294ae6d1416STom St Denis for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 295ae6d1416STom St Denis hub = &adev->vmhub[j]; 296e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 297e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 298e60f8db5SAlex Xie tmp = RREG32(reg); 299e60f8db5SAlex Xie tmp |= bits; 300e60f8db5SAlex Xie WREG32(reg, tmp); 301e60f8db5SAlex Xie } 302e60f8db5SAlex Xie } 303e60f8db5SAlex Xie default: 304e60f8db5SAlex Xie break; 305e60f8db5SAlex Xie } 306e60f8db5SAlex Xie 307e60f8db5SAlex Xie return 0; 308e60f8db5SAlex Xie } 309e60f8db5SAlex Xie 310e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 311e60f8db5SAlex Xie struct amdgpu_irq_src *source, 312e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 313e60f8db5SAlex Xie { 314c4f46f22SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src]; 315c468f9e2SChristian König bool retry_fault = !!(entry->src_data[1] & 0x80); 3164d6cbde3SFelix Kuehling uint32_t status = 0; 317e60f8db5SAlex Xie u64 addr; 318e60f8db5SAlex Xie 319e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 320e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 321e60f8db5SAlex Xie 322c1a8abd9SChristian König if (retry_fault && amdgpu_gmc_filter_faults(adev, addr, entry->pasid, 323c1a8abd9SChristian König entry->timestamp)) 32422666cc1SChristian König return 1; /* This also prevents sending it to KFD */ 32522666cc1SChristian König 326c1a8abd9SChristian König /* If it's the first fault for this address, process it normally */ 32779a0c465SMonk Liu if (!amdgpu_sriov_vf(adev)) { 3285a9b8e8aSChristian König status = RREG32(hub->vm_l2_pro_fault_status); 3295a9b8e8aSChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 3304d6cbde3SFelix Kuehling } 331e60f8db5SAlex Xie 3324d6cbde3SFelix Kuehling if (printk_ratelimit()) { 33305794effSShirish S struct amdgpu_task_info task_info; 334efaa9646SAndrey Grodzovsky 33505794effSShirish S memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 336efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 337efaa9646SAndrey Grodzovsky 3384d6cbde3SFelix Kuehling dev_err(adev->dev, 339c468f9e2SChristian König "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 340c468f9e2SChristian König "pasid:%u, for process %s pid %d thread %s pid %d)\n", 341c4f46f22SChristian König entry->vmid_src ? "mmhub" : "gfxhub", 342c468f9e2SChristian König retry_fault ? "retry" : "no-retry", 343c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 344efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 345efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 3467d0aa376SAndrey Grodzovsky dev_err(adev->dev, " in page starting at address 0x%016llx from %d\n", 34779a0c465SMonk Liu addr, entry->client_id); 3484d6cbde3SFelix Kuehling if (!amdgpu_sriov_vf(adev)) 3494d6cbde3SFelix Kuehling dev_err(adev->dev, 3504d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 3514d6cbde3SFelix Kuehling status); 35279a0c465SMonk Liu } 353e60f8db5SAlex Xie 354e60f8db5SAlex Xie return 0; 355e60f8db5SAlex Xie } 356e60f8db5SAlex Xie 357e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 358e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 359e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 360e60f8db5SAlex Xie }; 361e60f8db5SAlex Xie 362791c4769Sxinhui pan 363791c4769Sxinhui pan static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 364791c4769Sxinhui pan .set = gmc_v9_0_ecc_interrupt_state, 365791c4769Sxinhui pan .process = gmc_v9_0_process_ecc_irq, 366791c4769Sxinhui pan }; 367791c4769Sxinhui pan 368e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 369e60f8db5SAlex Xie { 370770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 371770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 372791c4769Sxinhui pan 373791c4769Sxinhui pan adev->gmc.ecc_irq.num_types = 1; 374791c4769Sxinhui pan adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 375e60f8db5SAlex Xie } 376e60f8db5SAlex Xie 3772a79d868SYong Zhao static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 3782a79d868SYong Zhao uint32_t flush_type) 37903f89febSChristian König { 38003f89febSChristian König u32 req = 0; 38103f89febSChristian König 38203f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 383c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 3842a79d868SYong Zhao req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 38503f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 38603f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 38703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 38803f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 38903f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 39003f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 39103f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 39203f89febSChristian König 39303f89febSChristian König return req; 39403f89febSChristian König } 39503f89febSChristian König 396e60f8db5SAlex Xie /* 397e60f8db5SAlex Xie * GART 398e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 399e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 400e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 401e60f8db5SAlex Xie */ 402e60f8db5SAlex Xie 403e60f8db5SAlex Xie /** 4042a79d868SYong Zhao * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 405e60f8db5SAlex Xie * 406e60f8db5SAlex Xie * @adev: amdgpu_device pointer 407e60f8db5SAlex Xie * @vmid: vm instance to flush 4082a79d868SYong Zhao * @flush_type: the flush type 409e60f8db5SAlex Xie * 4102a79d868SYong Zhao * Flush the TLB for the requested page table using certain type. 411e60f8db5SAlex Xie */ 412132f34e4SChristian König static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, 4132a79d868SYong Zhao uint32_t vmid, uint32_t flush_type) 414e60f8db5SAlex Xie { 415e60f8db5SAlex Xie const unsigned eng = 17; 416e60f8db5SAlex Xie unsigned i, j; 417e60f8db5SAlex Xie 418e60f8db5SAlex Xie for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 419e60f8db5SAlex Xie struct amdgpu_vmhub *hub = &adev->vmhub[i]; 4202a79d868SYong Zhao u32 tmp = gmc_v9_0_get_invalidate_req(vmid, flush_type); 421e60f8db5SAlex Xie 42282d1a1b1SChengming Gui /* This is necessary for a HW workaround under SRIOV as well 42382d1a1b1SChengming Gui * as GFXOFF under bare metal 42482d1a1b1SChengming Gui */ 42582d1a1b1SChengming Gui if (adev->gfx.kiq.ring.sched.ready && 42682d1a1b1SChengming Gui (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 42782d1a1b1SChengming Gui !adev->in_gpu_reset) { 428af5fe1e9SChristian König uint32_t req = hub->vm_inv_eng0_req + eng; 429af5fe1e9SChristian König uint32_t ack = hub->vm_inv_eng0_ack + eng; 430af5fe1e9SChristian König 431af5fe1e9SChristian König amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, tmp, 432af5fe1e9SChristian König 1 << vmid); 4333890d111SEmily Deng continue; 434fc0faf04SEmily Deng } 4353890d111SEmily Deng 4363890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 437c7a7266bSXiangliang Yu WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp); 438e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 439c7a7266bSXiangliang Yu tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng); 440396557b0SChristian König if (tmp & (1 << vmid)) 441e60f8db5SAlex Xie break; 442e60f8db5SAlex Xie udelay(1); 443e60f8db5SAlex Xie } 4443890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 445396557b0SChristian König if (j < adev->usec_timeout) 446e60f8db5SAlex Xie continue; 447396557b0SChristian König 448e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 449e60f8db5SAlex Xie } 450e60f8db5SAlex Xie } 451e60f8db5SAlex Xie 4529096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 453c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 4549096d6e5SChristian König { 455250b4228SChristian König struct amdgpu_device *adev = ring->adev; 456250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 4572a79d868SYong Zhao uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 4589096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 4599096d6e5SChristian König 4609096d6e5SChristian König amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid), 4619096d6e5SChristian König lower_32_bits(pd_addr)); 4629096d6e5SChristian König 4639096d6e5SChristian König amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid), 4649096d6e5SChristian König upper_32_bits(pd_addr)); 4659096d6e5SChristian König 466f8bc9037SAlex Deucher amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng, 467f8bc9037SAlex Deucher hub->vm_inv_eng0_ack + eng, 468f8bc9037SAlex Deucher req, 1 << vmid); 469f732b6b3SChristian König 4709096d6e5SChristian König return pd_addr; 4719096d6e5SChristian König } 4729096d6e5SChristian König 473c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 474c633c00bSChristian König unsigned pasid) 475c633c00bSChristian König { 476c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 477c633c00bSChristian König uint32_t reg; 478c633c00bSChristian König 479c633c00bSChristian König if (ring->funcs->vmhub == AMDGPU_GFXHUB) 480c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 481c633c00bSChristian König else 482c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 483c633c00bSChristian König 484c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 485c633c00bSChristian König } 486c633c00bSChristian König 487e60f8db5SAlex Xie /* 488e60f8db5SAlex Xie * PTE format on VEGA 10: 489e60f8db5SAlex Xie * 63:59 reserved 490e60f8db5SAlex Xie * 58:57 mtype 491e60f8db5SAlex Xie * 56 F 492e60f8db5SAlex Xie * 55 L 493e60f8db5SAlex Xie * 54 P 494e60f8db5SAlex Xie * 53 SW 495e60f8db5SAlex Xie * 52 T 496e60f8db5SAlex Xie * 50:48 reserved 497e60f8db5SAlex Xie * 47:12 4k physical page base address 498e60f8db5SAlex Xie * 11:7 fragment 499e60f8db5SAlex Xie * 6 write 500e60f8db5SAlex Xie * 5 read 501e60f8db5SAlex Xie * 4 exe 502e60f8db5SAlex Xie * 3 Z 503e60f8db5SAlex Xie * 2 snooped 504e60f8db5SAlex Xie * 1 system 505e60f8db5SAlex Xie * 0 valid 506e60f8db5SAlex Xie * 507e60f8db5SAlex Xie * PDE format on VEGA 10: 508e60f8db5SAlex Xie * 63:59 block fragment size 509e60f8db5SAlex Xie * 58:55 reserved 510e60f8db5SAlex Xie * 54 P 511e60f8db5SAlex Xie * 53:48 reserved 512e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 513e60f8db5SAlex Xie * 5:3 reserved 514e60f8db5SAlex Xie * 2 C 515e60f8db5SAlex Xie * 1 system 516e60f8db5SAlex Xie * 0 valid 517e60f8db5SAlex Xie */ 518e60f8db5SAlex Xie 519e60f8db5SAlex Xie static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev, 520e60f8db5SAlex Xie uint32_t flags) 521e60f8db5SAlex Xie 522e60f8db5SAlex Xie { 523e60f8db5SAlex Xie uint64_t pte_flag = 0; 524e60f8db5SAlex Xie 525e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_EXECUTABLE) 526e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_EXECUTABLE; 527e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_READABLE) 528e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_READABLE; 529e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_WRITEABLE) 530e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_WRITEABLE; 531e60f8db5SAlex Xie 532e60f8db5SAlex Xie switch (flags & AMDGPU_VM_MTYPE_MASK) { 533e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 534e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 535e60f8db5SAlex Xie break; 536e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 537e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 538e60f8db5SAlex Xie break; 539e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 540e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC); 541e60f8db5SAlex Xie break; 542e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 543e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC); 544e60f8db5SAlex Xie break; 545e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 546e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC); 547e60f8db5SAlex Xie break; 548e60f8db5SAlex Xie default: 549e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 550e60f8db5SAlex Xie break; 551e60f8db5SAlex Xie } 552e60f8db5SAlex Xie 553e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_PRT) 554e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_PRT; 555e60f8db5SAlex Xie 556e60f8db5SAlex Xie return pte_flag; 557e60f8db5SAlex Xie } 558e60f8db5SAlex Xie 5593de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 5603de676d8SChristian König uint64_t *addr, uint64_t *flags) 561f75e237cSChristian König { 562bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 5633de676d8SChristian König *addr = adev->vm_manager.vram_base_offset + *addr - 564770d13b1SChristian König adev->gmc.vram_start; 5653de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 5666a42fd6fSChristian König 567770d13b1SChristian König if (!adev->gmc.translate_further) 5686a42fd6fSChristian König return; 5696a42fd6fSChristian König 5706a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 5716a42fd6fSChristian König /* Set the block fragment size */ 5726a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 5736a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 5746a42fd6fSChristian König 5756a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 5766a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 5776a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 5786a42fd6fSChristian König else 5796a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 5806a42fd6fSChristian König } 581f75e237cSChristian König } 582f75e237cSChristian König 583132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 584132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 5859096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 586c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 587b1166325SChristian König .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags, 588b1166325SChristian König .get_vm_pde = gmc_v9_0_get_vm_pde 589e60f8db5SAlex Xie }; 590e60f8db5SAlex Xie 591132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 592e60f8db5SAlex Xie { 593132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 594e60f8db5SAlex Xie } 595e60f8db5SAlex Xie 596e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 597e60f8db5SAlex Xie { 598e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 599e60f8db5SAlex Xie 600132f34e4SChristian König gmc_v9_0_set_gmc_funcs(adev); 601e60f8db5SAlex Xie gmc_v9_0_set_irq_funcs(adev); 602e60f8db5SAlex Xie 603770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 604770d13b1SChristian König adev->gmc.shared_aperture_end = 605770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 606bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 607770d13b1SChristian König adev->gmc.private_aperture_end = 608770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 609a7ea6548SAlex Deucher 610e60f8db5SAlex Xie return 0; 611e60f8db5SAlex Xie } 612e60f8db5SAlex Xie 613cd2b5623SAlex Deucher static bool gmc_v9_0_keep_stolen_memory(struct amdgpu_device *adev) 614cd2b5623SAlex Deucher { 615cd2b5623SAlex Deucher 616cd2b5623SAlex Deucher /* 617cd2b5623SAlex Deucher * TODO: 618cd2b5623SAlex Deucher * Currently there is a bug where some memory client outside 619cd2b5623SAlex Deucher * of the driver writes to first 8M of VRAM on S3 resume, 620cd2b5623SAlex Deucher * this overrides GART which by default gets placed in first 8M and 621cd2b5623SAlex Deucher * causes VM_FAULTS once GTT is accessed. 622cd2b5623SAlex Deucher * Keep the stolen memory reservation until the while this is not solved. 623cd2b5623SAlex Deucher * Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init 624cd2b5623SAlex Deucher */ 625cd2b5623SAlex Deucher switch (adev->asic_type) { 62695010ba7SAlex Deucher case CHIP_VEGA10: 6276abc0c8fSAlex Deucher return true; 6286abc0c8fSAlex Deucher case CHIP_RAVEN: 62937910935SFlora Cui return (adev->pdev->device == 0x15d8); 6306abc0c8fSAlex Deucher case CHIP_VEGA12: 631cd2b5623SAlex Deucher case CHIP_VEGA20: 632cd2b5623SAlex Deucher default: 6336abc0c8fSAlex Deucher return false; 634cd2b5623SAlex Deucher } 635cd2b5623SAlex Deucher } 636cd2b5623SAlex Deucher 637c713a461SEvan Quan static int gmc_v9_0_allocate_vm_inv_eng(struct amdgpu_device *adev) 638c713a461SEvan Quan { 639c713a461SEvan Quan struct amdgpu_ring *ring; 640c713a461SEvan Quan unsigned vm_inv_engs[AMDGPU_MAX_VMHUBS] = 641c713a461SEvan Quan {GFXHUB_FREE_VM_INV_ENGS_BITMAP, MMHUB_FREE_VM_INV_ENGS_BITMAP}; 642c713a461SEvan Quan unsigned i; 643c713a461SEvan Quan unsigned vmhub, inv_eng; 644c713a461SEvan Quan 645c713a461SEvan Quan for (i = 0; i < adev->num_rings; ++i) { 646c713a461SEvan Quan ring = adev->rings[i]; 647c713a461SEvan Quan vmhub = ring->funcs->vmhub; 648c713a461SEvan Quan 649c713a461SEvan Quan inv_eng = ffs(vm_inv_engs[vmhub]); 650c713a461SEvan Quan if (!inv_eng) { 651c713a461SEvan Quan dev_err(adev->dev, "no VM inv eng for ring %s\n", 652c713a461SEvan Quan ring->name); 653c713a461SEvan Quan return -EINVAL; 654c713a461SEvan Quan } 655c713a461SEvan Quan 656c713a461SEvan Quan ring->vm_inv_eng = inv_eng - 1; 65772464382SChristian König vm_inv_engs[vmhub] &= ~(1 << ring->vm_inv_eng); 658c713a461SEvan Quan 659c713a461SEvan Quan dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n", 660c713a461SEvan Quan ring->name, ring->vm_inv_eng, ring->funcs->vmhub); 661c713a461SEvan Quan } 662c713a461SEvan Quan 663c713a461SEvan Quan return 0; 664c713a461SEvan Quan } 665c713a461SEvan Quan 666791c4769Sxinhui pan static int gmc_v9_0_ecc_late_init(void *handle) 667791c4769Sxinhui pan { 668791c4769Sxinhui pan struct amdgpu_device *adev = (struct amdgpu_device *)handle; 669791c4769Sxinhui pan struct ras_common_if **ras_if = &adev->gmc.ras_if; 670791c4769Sxinhui pan struct ras_ih_if ih_info = { 671791c4769Sxinhui pan .cb = gmc_v9_0_process_ras_data_cb, 672791c4769Sxinhui pan }; 673791c4769Sxinhui pan struct ras_fs_if fs_info = { 674791c4769Sxinhui pan .sysfs_name = "umc_err_count", 675791c4769Sxinhui pan .debugfs_name = "umc_err_inject", 676791c4769Sxinhui pan }; 677791c4769Sxinhui pan struct ras_common_if ras_block = { 678791c4769Sxinhui pan .block = AMDGPU_RAS_BLOCK__UMC, 679791c4769Sxinhui pan .type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, 680791c4769Sxinhui pan .sub_block_index = 0, 681791c4769Sxinhui pan .name = "umc", 682791c4769Sxinhui pan }; 683791c4769Sxinhui pan int r; 684791c4769Sxinhui pan 685791c4769Sxinhui pan if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC)) { 68653d65054Sxinhui pan amdgpu_ras_feature_enable_on_boot(adev, &ras_block, 0); 687791c4769Sxinhui pan return 0; 688791c4769Sxinhui pan } 689acbbee01Sxinhui pan /* handle resume path. */ 690acbbee01Sxinhui pan if (*ras_if) 691acbbee01Sxinhui pan goto resume; 692791c4769Sxinhui pan 693791c4769Sxinhui pan *ras_if = kmalloc(sizeof(**ras_if), GFP_KERNEL); 694791c4769Sxinhui pan if (!*ras_if) 695791c4769Sxinhui pan return -ENOMEM; 696791c4769Sxinhui pan 697791c4769Sxinhui pan **ras_if = ras_block; 698791c4769Sxinhui pan 69953d65054Sxinhui pan r = amdgpu_ras_feature_enable_on_boot(adev, *ras_if, 1); 700791c4769Sxinhui pan if (r) 701791c4769Sxinhui pan goto feature; 702791c4769Sxinhui pan 703791c4769Sxinhui pan ih_info.head = **ras_if; 704791c4769Sxinhui pan fs_info.head = **ras_if; 705791c4769Sxinhui pan 706791c4769Sxinhui pan r = amdgpu_ras_interrupt_add_handler(adev, &ih_info); 707791c4769Sxinhui pan if (r) 708791c4769Sxinhui pan goto interrupt; 709791c4769Sxinhui pan 710791c4769Sxinhui pan r = amdgpu_ras_debugfs_create(adev, &fs_info); 711791c4769Sxinhui pan if (r) 712791c4769Sxinhui pan goto debugfs; 713791c4769Sxinhui pan 714791c4769Sxinhui pan r = amdgpu_ras_sysfs_create(adev, &fs_info); 715791c4769Sxinhui pan if (r) 716791c4769Sxinhui pan goto sysfs; 717acbbee01Sxinhui pan resume: 718791c4769Sxinhui pan r = amdgpu_irq_get(adev, &adev->gmc.ecc_irq, 0); 719791c4769Sxinhui pan if (r) 720791c4769Sxinhui pan goto irq; 721791c4769Sxinhui pan 722791c4769Sxinhui pan return 0; 723791c4769Sxinhui pan irq: 724791c4769Sxinhui pan amdgpu_ras_sysfs_remove(adev, *ras_if); 725791c4769Sxinhui pan sysfs: 726791c4769Sxinhui pan amdgpu_ras_debugfs_remove(adev, *ras_if); 727791c4769Sxinhui pan debugfs: 728791c4769Sxinhui pan amdgpu_ras_interrupt_remove_handler(adev, &ih_info); 729791c4769Sxinhui pan interrupt: 730791c4769Sxinhui pan amdgpu_ras_feature_enable(adev, *ras_if, 0); 731791c4769Sxinhui pan feature: 732791c4769Sxinhui pan kfree(*ras_if); 733791c4769Sxinhui pan *ras_if = NULL; 734791c4769Sxinhui pan return -EINVAL; 735791c4769Sxinhui pan } 736791c4769Sxinhui pan 737791c4769Sxinhui pan 738e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 739e60f8db5SAlex Xie { 740e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 741f49ea9f8SHawking Zhang bool r; 7424789c463SChristian König 743cd2b5623SAlex Deucher if (!gmc_v9_0_keep_stolen_memory(adev)) 744cd2b5623SAlex Deucher amdgpu_bo_late_init(adev); 7456f752ec2SAndrey Grodzovsky 746c713a461SEvan Quan r = gmc_v9_0_allocate_vm_inv_eng(adev); 747c713a461SEvan Quan if (r) 748c713a461SEvan Quan return r; 749f49ea9f8SHawking Zhang /* Check if ecc is available */ 750f49ea9f8SHawking Zhang if (!amdgpu_sriov_vf(adev)) { 751f49ea9f8SHawking Zhang switch (adev->asic_type) { 752f49ea9f8SHawking Zhang case CHIP_VEGA10: 753f49ea9f8SHawking Zhang case CHIP_VEGA20: 754f49ea9f8SHawking Zhang r = amdgpu_atomfirmware_mem_ecc_supported(adev); 755f49ea9f8SHawking Zhang if (!r) { 75602bab923SDavid Panariti DRM_INFO("ECC is not present.\n"); 757f49ea9f8SHawking Zhang if (adev->df_funcs->enable_ecc_force_par_wr_rmw) 758e1d1a772SAlex Deucher adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false); 75902bab923SDavid Panariti } else { 760f49ea9f8SHawking Zhang DRM_INFO("ECC is active.\n"); 761f49ea9f8SHawking Zhang } 762f49ea9f8SHawking Zhang 763f49ea9f8SHawking Zhang r = amdgpu_atomfirmware_sram_ecc_supported(adev); 764f49ea9f8SHawking Zhang if (!r) { 765f49ea9f8SHawking Zhang DRM_INFO("SRAM ECC is not present.\n"); 766f49ea9f8SHawking Zhang } else { 767f49ea9f8SHawking Zhang DRM_INFO("SRAM ECC is active.\n"); 768f49ea9f8SHawking Zhang } 769f49ea9f8SHawking Zhang break; 770f49ea9f8SHawking Zhang default: 771f49ea9f8SHawking Zhang break; 77202bab923SDavid Panariti } 7735ba4fa35SAlex Deucher } 77402bab923SDavid Panariti 775791c4769Sxinhui pan r = gmc_v9_0_ecc_late_init(handle); 776791c4769Sxinhui pan if (r) 777e60f8db5SAlex Xie return r; 778e60f8db5SAlex Xie 779770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 780e60f8db5SAlex Xie } 781e60f8db5SAlex Xie 782e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 783770d13b1SChristian König struct amdgpu_gmc *mc) 784e60f8db5SAlex Xie { 785e60f8db5SAlex Xie u64 base = 0; 786e60f8db5SAlex Xie if (!amdgpu_sriov_vf(adev)) 787e60f8db5SAlex Xie base = mmhub_v1_0_get_fb_location(adev); 7886fdd68b1SAlex Deucher /* add the xgmi offset of the physical node */ 7896fdd68b1SAlex Deucher base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 79083afe835SOak Zeng amdgpu_gmc_vram_location(adev, mc, base); 791961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 792c3e1b43cSChristian König if (!amdgpu_sriov_vf(adev)) 793c3e1b43cSChristian König amdgpu_gmc_agp_location(adev, mc); 794e60f8db5SAlex Xie /* base offset of vram pages */ 795e60f8db5SAlex Xie adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev); 7966fdd68b1SAlex Deucher 7976fdd68b1SAlex Deucher /* XXX: add the xgmi offset of the physical node? */ 7986fdd68b1SAlex Deucher adev->vm_manager.vram_base_offset += 7996fdd68b1SAlex Deucher adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 800e60f8db5SAlex Xie } 801e60f8db5SAlex Xie 802e60f8db5SAlex Xie /** 803e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 804e60f8db5SAlex Xie * 805e60f8db5SAlex Xie * @adev: amdgpu_device pointer 806e60f8db5SAlex Xie * 807e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 808e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 809e60f8db5SAlex Xie * Returns 0 for success. 810e60f8db5SAlex Xie */ 811e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 812e60f8db5SAlex Xie { 813e60f8db5SAlex Xie int chansize, numchan; 814e60f8db5SAlex Xie int r; 815e60f8db5SAlex Xie 8163d918c0eSShaoyun Liu if (amdgpu_emu_mode != 1) 817770d13b1SChristian König adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev); 818770d13b1SChristian König if (!adev->gmc.vram_width) { 819e60f8db5SAlex Xie /* hbm memory channel size */ 820585b7f16STom St Denis if (adev->flags & AMD_IS_APU) 821585b7f16STom St Denis chansize = 64; 822585b7f16STom St Denis else 823e60f8db5SAlex Xie chansize = 128; 824e60f8db5SAlex Xie 825070706c0SHawking Zhang numchan = adev->df_funcs->get_hbm_channel_number(adev); 826770d13b1SChristian König adev->gmc.vram_width = numchan * chansize; 827e60f8db5SAlex Xie } 828e60f8db5SAlex Xie 829e60f8db5SAlex Xie /* size in MB on si */ 830770d13b1SChristian König adev->gmc.mc_vram_size = 831bf383fb6SAlex Deucher adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL; 832770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 833e60f8db5SAlex Xie 834e60f8db5SAlex Xie if (!(adev->flags & AMD_IS_APU)) { 835e60f8db5SAlex Xie r = amdgpu_device_resize_fb_bar(adev); 836e60f8db5SAlex Xie if (r) 837e60f8db5SAlex Xie return r; 838e60f8db5SAlex Xie } 839770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 840770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 841e60f8db5SAlex Xie 842156a81beSChunming Zhou #ifdef CONFIG_X86_64 843156a81beSChunming Zhou if (adev->flags & AMD_IS_APU) { 844156a81beSChunming Zhou adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev); 845156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 846156a81beSChunming Zhou } 847156a81beSChunming Zhou #endif 848e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 849770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 850770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 851770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 852e60f8db5SAlex Xie 853e60f8db5SAlex Xie /* set the gart size */ 854e60f8db5SAlex Xie if (amdgpu_gart_size == -1) { 855e60f8db5SAlex Xie switch (adev->asic_type) { 856e60f8db5SAlex Xie case CHIP_VEGA10: /* all engines support GPUVM */ 857273a14cdSAlex Deucher case CHIP_VEGA12: /* all engines support GPUVM */ 858d96b428cSFeifei Xu case CHIP_VEGA20: 859e60f8db5SAlex Xie default: 860fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 861e60f8db5SAlex Xie break; 862e60f8db5SAlex Xie case CHIP_RAVEN: /* DCE SG support */ 863770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 864e60f8db5SAlex Xie break; 865e60f8db5SAlex Xie } 866e60f8db5SAlex Xie } else { 867770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 868e60f8db5SAlex Xie } 869e60f8db5SAlex Xie 870770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 871e60f8db5SAlex Xie 872e60f8db5SAlex Xie return 0; 873e60f8db5SAlex Xie } 874e60f8db5SAlex Xie 875e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 876e60f8db5SAlex Xie { 877e60f8db5SAlex Xie int r; 878e60f8db5SAlex Xie 8791123b989SChristian König if (adev->gart.bo) { 880e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 881e60f8db5SAlex Xie return 0; 882e60f8db5SAlex Xie } 883e60f8db5SAlex Xie /* Initialize common gart structure */ 884e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 885e60f8db5SAlex Xie if (r) 886e60f8db5SAlex Xie return r; 887e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 888e60f8db5SAlex Xie adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) | 889e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 890e60f8db5SAlex Xie return amdgpu_gart_table_vram_alloc(adev); 891e60f8db5SAlex Xie } 892e60f8db5SAlex Xie 893ebdef28eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 894ebdef28eSAlex Deucher { 895ebdef28eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 896ebdef28eSAlex Deucher unsigned size; 897ebdef28eSAlex Deucher 8986f752ec2SAndrey Grodzovsky /* 8996f752ec2SAndrey Grodzovsky * TODO Remove once GART corruption is resolved 9006f752ec2SAndrey Grodzovsky * Check related code in gmc_v9_0_sw_fini 9016f752ec2SAndrey Grodzovsky * */ 902cd2b5623SAlex Deucher if (gmc_v9_0_keep_stolen_memory(adev)) 903cd2b5623SAlex Deucher return 9 * 1024 * 1024; 9046f752ec2SAndrey Grodzovsky 905ebdef28eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 906ebdef28eSAlex Deucher size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ 907ebdef28eSAlex Deucher } else { 908ebdef28eSAlex Deucher u32 viewport; 909ebdef28eSAlex Deucher 910ebdef28eSAlex Deucher switch (adev->asic_type) { 911ebdef28eSAlex Deucher case CHIP_RAVEN: 912ebdef28eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 913ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, 914ebdef28eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 915ebdef28eSAlex Deucher REG_GET_FIELD(viewport, 916ebdef28eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 917ebdef28eSAlex Deucher 4); 918ebdef28eSAlex Deucher break; 919ebdef28eSAlex Deucher case CHIP_VEGA10: 920ebdef28eSAlex Deucher case CHIP_VEGA12: 921cd2b5623SAlex Deucher case CHIP_VEGA20: 922ebdef28eSAlex Deucher default: 923ebdef28eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 924ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 925ebdef28eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 926ebdef28eSAlex Deucher 4); 927ebdef28eSAlex Deucher break; 928ebdef28eSAlex Deucher } 929ebdef28eSAlex Deucher } 930ebdef28eSAlex Deucher /* return 0 if the pre-OS buffer uses up most of vram */ 931ebdef28eSAlex Deucher if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 932ebdef28eSAlex Deucher return 0; 9336f752ec2SAndrey Grodzovsky 934ebdef28eSAlex Deucher return size; 935ebdef28eSAlex Deucher } 936ebdef28eSAlex Deucher 937e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 938e60f8db5SAlex Xie { 939e60f8db5SAlex Xie int r; 940e60f8db5SAlex Xie int dma_bits; 941e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 942e60f8db5SAlex Xie 943e60f8db5SAlex Xie gfxhub_v1_0_init(adev); 944e60f8db5SAlex Xie mmhub_v1_0_init(adev); 945e60f8db5SAlex Xie 946770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 947e60f8db5SAlex Xie 9481e09b053SHawking Zhang adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev); 949e60f8db5SAlex Xie switch (adev->asic_type) { 950e60f8db5SAlex Xie case CHIP_RAVEN: 9516a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 952f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 9536a42fd6fSChristian König } else { 9546a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 9556a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 956770d13b1SChristian König adev->gmc.translate_further = 9576a42fd6fSChristian König adev->vm_manager.num_level > 1; 9586a42fd6fSChristian König } 959e60f8db5SAlex Xie break; 960e60f8db5SAlex Xie case CHIP_VEGA10: 961273a14cdSAlex Deucher case CHIP_VEGA12: 962d96b428cSFeifei Xu case CHIP_VEGA20: 963e60f8db5SAlex Xie /* 964e60f8db5SAlex Xie * To fulfill 4-level page support, 965e60f8db5SAlex Xie * vm size is 256TB (48bit), maximum size of Vega10, 966e60f8db5SAlex Xie * block size 512 (9bit) 967e60f8db5SAlex Xie */ 968cdba61daSwentalou /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 969cdba61daSwentalou if (amdgpu_sriov_vf(adev)) 970cdba61daSwentalou amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 971cdba61daSwentalou else 972f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 973e60f8db5SAlex Xie break; 974e60f8db5SAlex Xie default: 975e60f8db5SAlex Xie break; 976e60f8db5SAlex Xie } 977e60f8db5SAlex Xie 978e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 97944a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 980770d13b1SChristian König &adev->gmc.vm_fault); 98130da7bb1SChristian König if (r) 98230da7bb1SChristian König return r; 98330da7bb1SChristian König 98444a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 985770d13b1SChristian König &adev->gmc.vm_fault); 986e60f8db5SAlex Xie 987e60f8db5SAlex Xie if (r) 988e60f8db5SAlex Xie return r; 989e60f8db5SAlex Xie 990791c4769Sxinhui pan /* interrupt sent to DF. */ 991791c4769Sxinhui pan r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 992791c4769Sxinhui pan &adev->gmc.ecc_irq); 993791c4769Sxinhui pan if (r) 994791c4769Sxinhui pan return r; 995791c4769Sxinhui pan 996e60f8db5SAlex Xie /* Set the internal MC address mask 997e60f8db5SAlex Xie * This is the max address of the GPU's 998e60f8db5SAlex Xie * internal address space. 999e60f8db5SAlex Xie */ 1000770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 1001e60f8db5SAlex Xie 1002e60f8db5SAlex Xie /* set DMA mask + need_dma32 flags. 1003e60f8db5SAlex Xie * PCIE - can handle 44-bits. 1004e60f8db5SAlex Xie * IGP - can handle 44-bits 1005e60f8db5SAlex Xie * PCI - dma32 for legacy pci gart, 44 bits on vega10 1006e60f8db5SAlex Xie */ 1007e60f8db5SAlex Xie adev->need_dma32 = false; 1008e60f8db5SAlex Xie dma_bits = adev->need_dma32 ? 32 : 44; 1009e60f8db5SAlex Xie r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 1010e60f8db5SAlex Xie if (r) { 1011e60f8db5SAlex Xie adev->need_dma32 = true; 1012e60f8db5SAlex Xie dma_bits = 32; 1013e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 1014e60f8db5SAlex Xie } 1015e60f8db5SAlex Xie r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 1016e60f8db5SAlex Xie if (r) { 1017e60f8db5SAlex Xie pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32)); 1018e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No coherent DMA available.\n"); 1019e60f8db5SAlex Xie } 1020913b2cb7SMichael D Labriola adev->need_swiotlb = drm_need_swiotlb(dma_bits); 1021e60f8db5SAlex Xie 102247622ba0SAlex Deucher if (adev->gmc.xgmi.supported) { 1023bf0a60b7SAlex Deucher r = gfxhub_v1_1_get_xgmi_info(adev); 1024bf0a60b7SAlex Deucher if (r) 1025bf0a60b7SAlex Deucher return r; 1026bf0a60b7SAlex Deucher } 1027bf0a60b7SAlex Deucher 1028e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 1029e60f8db5SAlex Xie if (r) 1030e60f8db5SAlex Xie return r; 1031e60f8db5SAlex Xie 1032ebdef28eSAlex Deucher adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev); 1033ebdef28eSAlex Deucher 1034e60f8db5SAlex Xie /* Memory manager */ 1035e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 1036e60f8db5SAlex Xie if (r) 1037e60f8db5SAlex Xie return r; 1038e60f8db5SAlex Xie 1039e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 1040e60f8db5SAlex Xie if (r) 1041e60f8db5SAlex Xie return r; 1042e60f8db5SAlex Xie 104305ec3edaSChristian König /* 104405ec3edaSChristian König * number of VMs 104505ec3edaSChristian König * VMID 0 is reserved for System 104605ec3edaSChristian König * amdgpu graphics/compute will use VMIDs 1-7 104705ec3edaSChristian König * amdkfd will use VMIDs 8-15 104805ec3edaSChristian König */ 104905ec3edaSChristian König adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 105005ec3edaSChristian König adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 105105ec3edaSChristian König 105205ec3edaSChristian König amdgpu_vm_manager_init(adev); 105305ec3edaSChristian König 105405ec3edaSChristian König return 0; 1055e60f8db5SAlex Xie } 1056e60f8db5SAlex Xie 1057e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 1058e60f8db5SAlex Xie { 1059e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1060e60f8db5SAlex Xie 1061791c4769Sxinhui pan if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__UMC) && 1062791c4769Sxinhui pan adev->gmc.ras_if) { 1063791c4769Sxinhui pan struct ras_common_if *ras_if = adev->gmc.ras_if; 1064791c4769Sxinhui pan struct ras_ih_if ih_info = { 1065791c4769Sxinhui pan .head = *ras_if, 1066791c4769Sxinhui pan }; 1067791c4769Sxinhui pan 1068791c4769Sxinhui pan /*remove fs first*/ 1069791c4769Sxinhui pan amdgpu_ras_debugfs_remove(adev, ras_if); 1070791c4769Sxinhui pan amdgpu_ras_sysfs_remove(adev, ras_if); 1071791c4769Sxinhui pan /*remove the IH*/ 1072791c4769Sxinhui pan amdgpu_ras_interrupt_remove_handler(adev, &ih_info); 1073791c4769Sxinhui pan amdgpu_ras_feature_enable(adev, ras_if, 0); 1074791c4769Sxinhui pan kfree(ras_if); 1075791c4769Sxinhui pan } 1076791c4769Sxinhui pan 1077f59548c8SMonk Liu amdgpu_gem_force_release(adev); 1078e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 10796f752ec2SAndrey Grodzovsky 1080cd2b5623SAlex Deucher if (gmc_v9_0_keep_stolen_memory(adev)) 10816f752ec2SAndrey Grodzovsky amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); 10826f752ec2SAndrey Grodzovsky 1083a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 1084e60f8db5SAlex Xie amdgpu_bo_fini(adev); 1085a3d9103eSAndrey Grodzovsky amdgpu_gart_fini(adev); 1086e60f8db5SAlex Xie 1087e60f8db5SAlex Xie return 0; 1088e60f8db5SAlex Xie } 1089e60f8db5SAlex Xie 1090e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1091e60f8db5SAlex Xie { 1092946a4d5bSShaoyun Liu 1093e60f8db5SAlex Xie switch (adev->asic_type) { 1094e60f8db5SAlex Xie case CHIP_VEGA10: 1095d96b428cSFeifei Xu case CHIP_VEGA20: 1096946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 10975c583018SEvan Quan golden_settings_mmhub_1_0_0, 1098c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1099946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 11005c583018SEvan Quan golden_settings_athub_1_0_0, 1101c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1102e60f8db5SAlex Xie break; 1103273a14cdSAlex Deucher case CHIP_VEGA12: 1104273a14cdSAlex Deucher break; 1105e4f3abaaSChunming Zhou case CHIP_RAVEN: 1106946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 11075c583018SEvan Quan golden_settings_athub_1_0_0, 1108c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1109e4f3abaaSChunming Zhou break; 1110e60f8db5SAlex Xie default: 1111e60f8db5SAlex Xie break; 1112e60f8db5SAlex Xie } 1113e60f8db5SAlex Xie } 1114e60f8db5SAlex Xie 1115e60f8db5SAlex Xie /** 1116e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1117e60f8db5SAlex Xie * 1118e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1119e60f8db5SAlex Xie */ 1120e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1121e60f8db5SAlex Xie { 1122e60f8db5SAlex Xie int r; 1123e60f8db5SAlex Xie bool value; 1124e60f8db5SAlex Xie u32 tmp; 1125e60f8db5SAlex Xie 11269c3f2b54SAlex Deucher amdgpu_device_program_register_sequence(adev, 1127e60f8db5SAlex Xie golden_settings_vega10_hdp, 1128c47b41a7SChristian König ARRAY_SIZE(golden_settings_vega10_hdp)); 1129e60f8db5SAlex Xie 11301123b989SChristian König if (adev->gart.bo == NULL) { 1131e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1132e60f8db5SAlex Xie return -EINVAL; 1133e60f8db5SAlex Xie } 1134ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1135ce1b1b66SMonk Liu if (r) 1136ce1b1b66SMonk Liu return r; 1137e60f8db5SAlex Xie 11382fcd43ceSHawking Zhang switch (adev->asic_type) { 11392fcd43ceSHawking Zhang case CHIP_RAVEN: 1140f8386b35SHawking Zhang mmhub_v1_0_update_power_gating(adev, true); 11412fcd43ceSHawking Zhang break; 11422fcd43ceSHawking Zhang default: 11432fcd43ceSHawking Zhang break; 11442fcd43ceSHawking Zhang } 11452fcd43ceSHawking Zhang 1146e60f8db5SAlex Xie r = gfxhub_v1_0_gart_enable(adev); 1147e60f8db5SAlex Xie if (r) 1148e60f8db5SAlex Xie return r; 1149e60f8db5SAlex Xie 1150e60f8db5SAlex Xie r = mmhub_v1_0_gart_enable(adev); 1151e60f8db5SAlex Xie if (r) 1152e60f8db5SAlex Xie return r; 1153e60f8db5SAlex Xie 1154846347c9STom St Denis WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1155e60f8db5SAlex Xie 1156b9509c80SHuang Rui tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1157b9509c80SHuang Rui WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1158e60f8db5SAlex Xie 11591d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 116069882565SChristian König adev->nbio_funcs->hdp_flush(adev, NULL); 11611d4e0a8cSMonk Liu 1162e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1163e60f8db5SAlex Xie value = false; 1164e60f8db5SAlex Xie else 1165e60f8db5SAlex Xie value = true; 1166e60f8db5SAlex Xie 1167e60f8db5SAlex Xie gfxhub_v1_0_set_fault_enable_default(adev, value); 1168e60f8db5SAlex Xie mmhub_v1_0_set_fault_enable_default(adev, value); 11692a79d868SYong Zhao gmc_v9_0_flush_gpu_tlb(adev, 0, 0); 1170e60f8db5SAlex Xie 1171e60f8db5SAlex Xie DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1172770d13b1SChristian König (unsigned)(adev->gmc.gart_size >> 20), 11734e830fb1SChristian König (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1174e60f8db5SAlex Xie adev->gart.ready = true; 1175e60f8db5SAlex Xie return 0; 1176e60f8db5SAlex Xie } 1177e60f8db5SAlex Xie 1178e60f8db5SAlex Xie static int gmc_v9_0_hw_init(void *handle) 1179e60f8db5SAlex Xie { 1180e60f8db5SAlex Xie int r; 1181e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1182e60f8db5SAlex Xie 1183e60f8db5SAlex Xie /* The sequence of these two function calls matters.*/ 1184e60f8db5SAlex Xie gmc_v9_0_init_golden_registers(adev); 1185e60f8db5SAlex Xie 1186edca2d05SAlex Deucher if (adev->mode_info.num_crtc) { 1187edca2d05SAlex Deucher /* Lockout access through VGA aperture*/ 11884d9c333aSTom St Denis WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1189edca2d05SAlex Deucher 1190edca2d05SAlex Deucher /* disable VGA render */ 11914d9c333aSTom St Denis WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1192edca2d05SAlex Deucher } 1193edca2d05SAlex Deucher 1194e60f8db5SAlex Xie r = gmc_v9_0_gart_enable(adev); 1195e60f8db5SAlex Xie 1196e60f8db5SAlex Xie return r; 1197e60f8db5SAlex Xie } 1198e60f8db5SAlex Xie 1199e60f8db5SAlex Xie /** 1200e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1201e60f8db5SAlex Xie * 1202e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1203e60f8db5SAlex Xie * 1204e60f8db5SAlex Xie * This disables all VM page table. 1205e60f8db5SAlex Xie */ 1206e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1207e60f8db5SAlex Xie { 1208e60f8db5SAlex Xie gfxhub_v1_0_gart_disable(adev); 1209e60f8db5SAlex Xie mmhub_v1_0_gart_disable(adev); 1210ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1211e60f8db5SAlex Xie } 1212e60f8db5SAlex Xie 1213e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1214e60f8db5SAlex Xie { 1215e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1216e60f8db5SAlex Xie 12175dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 12185dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 12195dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 12205dd696aeSTrigger Huang return 0; 12215dd696aeSTrigger Huang } 12225dd696aeSTrigger Huang 1223791c4769Sxinhui pan amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1224770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1225e60f8db5SAlex Xie gmc_v9_0_gart_disable(adev); 1226e60f8db5SAlex Xie 1227e60f8db5SAlex Xie return 0; 1228e60f8db5SAlex Xie } 1229e60f8db5SAlex Xie 1230e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1231e60f8db5SAlex Xie { 1232e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1233e60f8db5SAlex Xie 1234f053cd47STom St Denis return gmc_v9_0_hw_fini(adev); 1235e60f8db5SAlex Xie } 1236e60f8db5SAlex Xie 1237e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1238e60f8db5SAlex Xie { 1239e60f8db5SAlex Xie int r; 1240e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1241e60f8db5SAlex Xie 1242e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1243e60f8db5SAlex Xie if (r) 1244e60f8db5SAlex Xie return r; 1245e60f8db5SAlex Xie 1246620f774fSChristian König amdgpu_vmid_reset_all(adev); 1247e60f8db5SAlex Xie 124832601d48SChristian König return 0; 1249e60f8db5SAlex Xie } 1250e60f8db5SAlex Xie 1251e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1252e60f8db5SAlex Xie { 1253e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1254e60f8db5SAlex Xie return true; 1255e60f8db5SAlex Xie } 1256e60f8db5SAlex Xie 1257e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1258e60f8db5SAlex Xie { 1259e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1260e60f8db5SAlex Xie return 0; 1261e60f8db5SAlex Xie } 1262e60f8db5SAlex Xie 1263e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1264e60f8db5SAlex Xie { 1265e60f8db5SAlex Xie /* XXX for emulation.*/ 1266e60f8db5SAlex Xie return 0; 1267e60f8db5SAlex Xie } 1268e60f8db5SAlex Xie 1269e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1270e60f8db5SAlex Xie enum amd_clockgating_state state) 1271e60f8db5SAlex Xie { 1272d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1273d5583d4fSHuang Rui 1274d5583d4fSHuang Rui return mmhub_v1_0_set_clockgating(adev, state); 1275e60f8db5SAlex Xie } 1276e60f8db5SAlex Xie 127713052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 127813052be5SHuang Rui { 127913052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 128013052be5SHuang Rui 128113052be5SHuang Rui mmhub_v1_0_get_clockgating(adev, flags); 128213052be5SHuang Rui } 128313052be5SHuang Rui 1284e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1285e60f8db5SAlex Xie enum amd_powergating_state state) 1286e60f8db5SAlex Xie { 1287e60f8db5SAlex Xie return 0; 1288e60f8db5SAlex Xie } 1289e60f8db5SAlex Xie 1290e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1291e60f8db5SAlex Xie .name = "gmc_v9_0", 1292e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1293e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1294e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1295e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1296e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1297e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1298e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1299e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1300e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1301e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1302e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1303e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1304e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 130513052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1306e60f8db5SAlex Xie }; 1307e60f8db5SAlex Xie 1308e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1309e60f8db5SAlex Xie { 1310e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1311e60f8db5SAlex Xie .major = 9, 1312e60f8db5SAlex Xie .minor = 0, 1313e60f8db5SAlex Xie .rev = 0, 1314e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1315e60f8db5SAlex Xie }; 1316