1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23f867723bSSam Ravnborg 24e60f8db5SAlex Xie #include <linux/firmware.h> 25f867723bSSam Ravnborg #include <linux/pci.h> 26f867723bSSam Ravnborg 27fd5fd480SChunming Zhou #include <drm/drm_cache.h> 28f867723bSSam Ravnborg 29e60f8db5SAlex Xie #include "amdgpu.h" 30e60f8db5SAlex Xie #include "gmc_v9_0.h" 318d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 322cddc50eSHuang Rui #include "amdgpu_gem.h" 33e60f8db5SAlex Xie 3475199b8cSFeifei Xu #include "hdp/hdp_4_0_offset.h" 3575199b8cSFeifei Xu #include "hdp/hdp_4_0_sh_mask.h" 36cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 37135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 38135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 39fb960bd2SFeifei Xu #include "vega10_enum.h" 4065417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 41ea930000SAlex Sierra #include "athub/athub_1_0_sh_mask.h" 426ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 43250b4228SChristian König #include "oss/osssys_4_0_offset.h" 44e60f8db5SAlex Xie 45946a4d5bSShaoyun Liu #include "soc15.h" 46ea930000SAlex Sierra #include "soc15d.h" 47e60f8db5SAlex Xie #include "soc15_common.h" 4890c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 49e60f8db5SAlex Xie 50e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 51e60f8db5SAlex Xie #include "mmhub_v1_0.h" 52bee7b51aSLe Ma #include "athub_v1_0.h" 53bf0a60b7SAlex Deucher #include "gfxhub_v1_1.h" 5451cce480SLe Ma #include "mmhub_v9_4.h" 555b6b35aaSHawking Zhang #include "umc_v6_1.h" 56e7da754bSMonk Liu #include "umc_v6_0.h" 57e60f8db5SAlex Xie 5844a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 5944a99b65SAndrey Grodzovsky 60791c4769Sxinhui pan #include "amdgpu_ras.h" 61029fbd43SHawking Zhang #include "amdgpu_xgmi.h" 62791c4769Sxinhui pan 63ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 64ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 65ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 66ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 67ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 68ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 69ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 70f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d 71f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 72f8646661SAlex Deucher 73ebdef28eSAlex Deucher 74e60f8db5SAlex Xie static const u32 golden_settings_vega10_hdp[] = 75e60f8db5SAlex Xie { 76e60f8db5SAlex Xie 0xf64, 0x0fffffff, 0x00000000, 77e60f8db5SAlex Xie 0xf65, 0x0fffffff, 0x00000000, 78e60f8db5SAlex Xie 0xf66, 0x0fffffff, 0x00000000, 79e60f8db5SAlex Xie 0xf67, 0x0fffffff, 0x00000000, 80e60f8db5SAlex Xie 0xf68, 0x0fffffff, 0x00000000, 81e60f8db5SAlex Xie 0xf6a, 0x0fffffff, 0x00000000, 82e60f8db5SAlex Xie 0xf6b, 0x0fffffff, 0x00000000, 83e60f8db5SAlex Xie 0xf6c, 0x0fffffff, 0x00000000, 84e60f8db5SAlex Xie 0xf6d, 0x0fffffff, 0x00000000, 85e60f8db5SAlex Xie 0xf6e, 0x0fffffff, 0x00000000, 86e60f8db5SAlex Xie }; 87e60f8db5SAlex Xie 88946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 895c583018SEvan Quan { 90946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 91946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 925c583018SEvan Quan }; 935c583018SEvan Quan 94946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 955c583018SEvan Quan { 96946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 97946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 985c583018SEvan Quan }; 995c583018SEvan Quan 100791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 101791c4769Sxinhui pan (0x000143c0 + 0x00000000), 102791c4769Sxinhui pan (0x000143c0 + 0x00000800), 103791c4769Sxinhui pan (0x000143c0 + 0x00001000), 104791c4769Sxinhui pan (0x000143c0 + 0x00001800), 105791c4769Sxinhui pan (0x000543c0 + 0x00000000), 106791c4769Sxinhui pan (0x000543c0 + 0x00000800), 107791c4769Sxinhui pan (0x000543c0 + 0x00001000), 108791c4769Sxinhui pan (0x000543c0 + 0x00001800), 109791c4769Sxinhui pan (0x000943c0 + 0x00000000), 110791c4769Sxinhui pan (0x000943c0 + 0x00000800), 111791c4769Sxinhui pan (0x000943c0 + 0x00001000), 112791c4769Sxinhui pan (0x000943c0 + 0x00001800), 113791c4769Sxinhui pan (0x000d43c0 + 0x00000000), 114791c4769Sxinhui pan (0x000d43c0 + 0x00000800), 115791c4769Sxinhui pan (0x000d43c0 + 0x00001000), 116791c4769Sxinhui pan (0x000d43c0 + 0x00001800), 117791c4769Sxinhui pan (0x001143c0 + 0x00000000), 118791c4769Sxinhui pan (0x001143c0 + 0x00000800), 119791c4769Sxinhui pan (0x001143c0 + 0x00001000), 120791c4769Sxinhui pan (0x001143c0 + 0x00001800), 121791c4769Sxinhui pan (0x001543c0 + 0x00000000), 122791c4769Sxinhui pan (0x001543c0 + 0x00000800), 123791c4769Sxinhui pan (0x001543c0 + 0x00001000), 124791c4769Sxinhui pan (0x001543c0 + 0x00001800), 125791c4769Sxinhui pan (0x001943c0 + 0x00000000), 126791c4769Sxinhui pan (0x001943c0 + 0x00000800), 127791c4769Sxinhui pan (0x001943c0 + 0x00001000), 128791c4769Sxinhui pan (0x001943c0 + 0x00001800), 129791c4769Sxinhui pan (0x001d43c0 + 0x00000000), 130791c4769Sxinhui pan (0x001d43c0 + 0x00000800), 131791c4769Sxinhui pan (0x001d43c0 + 0x00001000), 132791c4769Sxinhui pan (0x001d43c0 + 0x00001800), 13302bab923SDavid Panariti }; 13402bab923SDavid Panariti 135791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 136791c4769Sxinhui pan (0x000143e0 + 0x00000000), 137791c4769Sxinhui pan (0x000143e0 + 0x00000800), 138791c4769Sxinhui pan (0x000143e0 + 0x00001000), 139791c4769Sxinhui pan (0x000143e0 + 0x00001800), 140791c4769Sxinhui pan (0x000543e0 + 0x00000000), 141791c4769Sxinhui pan (0x000543e0 + 0x00000800), 142791c4769Sxinhui pan (0x000543e0 + 0x00001000), 143791c4769Sxinhui pan (0x000543e0 + 0x00001800), 144791c4769Sxinhui pan (0x000943e0 + 0x00000000), 145791c4769Sxinhui pan (0x000943e0 + 0x00000800), 146791c4769Sxinhui pan (0x000943e0 + 0x00001000), 147791c4769Sxinhui pan (0x000943e0 + 0x00001800), 148791c4769Sxinhui pan (0x000d43e0 + 0x00000000), 149791c4769Sxinhui pan (0x000d43e0 + 0x00000800), 150791c4769Sxinhui pan (0x000d43e0 + 0x00001000), 151791c4769Sxinhui pan (0x000d43e0 + 0x00001800), 152791c4769Sxinhui pan (0x001143e0 + 0x00000000), 153791c4769Sxinhui pan (0x001143e0 + 0x00000800), 154791c4769Sxinhui pan (0x001143e0 + 0x00001000), 155791c4769Sxinhui pan (0x001143e0 + 0x00001800), 156791c4769Sxinhui pan (0x001543e0 + 0x00000000), 157791c4769Sxinhui pan (0x001543e0 + 0x00000800), 158791c4769Sxinhui pan (0x001543e0 + 0x00001000), 159791c4769Sxinhui pan (0x001543e0 + 0x00001800), 160791c4769Sxinhui pan (0x001943e0 + 0x00000000), 161791c4769Sxinhui pan (0x001943e0 + 0x00000800), 162791c4769Sxinhui pan (0x001943e0 + 0x00001000), 163791c4769Sxinhui pan (0x001943e0 + 0x00001800), 164791c4769Sxinhui pan (0x001d43e0 + 0x00000000), 165791c4769Sxinhui pan (0x001d43e0 + 0x00000800), 166791c4769Sxinhui pan (0x001d43e0 + 0x00001000), 167791c4769Sxinhui pan (0x001d43e0 + 0x00001800), 16802bab923SDavid Panariti }; 16902bab923SDavid Panariti 170791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_status_addrs[] = { 171791c4769Sxinhui pan (0x000143c2 + 0x00000000), 172791c4769Sxinhui pan (0x000143c2 + 0x00000800), 173791c4769Sxinhui pan (0x000143c2 + 0x00001000), 174791c4769Sxinhui pan (0x000143c2 + 0x00001800), 175791c4769Sxinhui pan (0x000543c2 + 0x00000000), 176791c4769Sxinhui pan (0x000543c2 + 0x00000800), 177791c4769Sxinhui pan (0x000543c2 + 0x00001000), 178791c4769Sxinhui pan (0x000543c2 + 0x00001800), 179791c4769Sxinhui pan (0x000943c2 + 0x00000000), 180791c4769Sxinhui pan (0x000943c2 + 0x00000800), 181791c4769Sxinhui pan (0x000943c2 + 0x00001000), 182791c4769Sxinhui pan (0x000943c2 + 0x00001800), 183791c4769Sxinhui pan (0x000d43c2 + 0x00000000), 184791c4769Sxinhui pan (0x000d43c2 + 0x00000800), 185791c4769Sxinhui pan (0x000d43c2 + 0x00001000), 186791c4769Sxinhui pan (0x000d43c2 + 0x00001800), 187791c4769Sxinhui pan (0x001143c2 + 0x00000000), 188791c4769Sxinhui pan (0x001143c2 + 0x00000800), 189791c4769Sxinhui pan (0x001143c2 + 0x00001000), 190791c4769Sxinhui pan (0x001143c2 + 0x00001800), 191791c4769Sxinhui pan (0x001543c2 + 0x00000000), 192791c4769Sxinhui pan (0x001543c2 + 0x00000800), 193791c4769Sxinhui pan (0x001543c2 + 0x00001000), 194791c4769Sxinhui pan (0x001543c2 + 0x00001800), 195791c4769Sxinhui pan (0x001943c2 + 0x00000000), 196791c4769Sxinhui pan (0x001943c2 + 0x00000800), 197791c4769Sxinhui pan (0x001943c2 + 0x00001000), 198791c4769Sxinhui pan (0x001943c2 + 0x00001800), 199791c4769Sxinhui pan (0x001d43c2 + 0x00000000), 200791c4769Sxinhui pan (0x001d43c2 + 0x00000800), 201791c4769Sxinhui pan (0x001d43c2 + 0x00001000), 202791c4769Sxinhui pan (0x001d43c2 + 0x00001800), 20302bab923SDavid Panariti }; 20402bab923SDavid Panariti 205791c4769Sxinhui pan static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 206791c4769Sxinhui pan struct amdgpu_irq_src *src, 207791c4769Sxinhui pan unsigned type, 208791c4769Sxinhui pan enum amdgpu_interrupt_state state) 209791c4769Sxinhui pan { 210791c4769Sxinhui pan u32 bits, i, tmp, reg; 211791c4769Sxinhui pan 2121e2c6d55SJohn Clements /* Devices newer then VEGA10/12 shall have these programming 2131e2c6d55SJohn Clements sequences performed by PSP BL */ 2141e2c6d55SJohn Clements if (adev->asic_type >= CHIP_VEGA20) 2151e2c6d55SJohn Clements return 0; 2161e2c6d55SJohn Clements 217791c4769Sxinhui pan bits = 0x7f; 218791c4769Sxinhui pan 219791c4769Sxinhui pan switch (state) { 220791c4769Sxinhui pan case AMDGPU_IRQ_STATE_DISABLE: 221791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 222791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 223791c4769Sxinhui pan tmp = RREG32(reg); 224791c4769Sxinhui pan tmp &= ~bits; 225791c4769Sxinhui pan WREG32(reg, tmp); 226791c4769Sxinhui pan } 227791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 228791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 229791c4769Sxinhui pan tmp = RREG32(reg); 230791c4769Sxinhui pan tmp &= ~bits; 231791c4769Sxinhui pan WREG32(reg, tmp); 232791c4769Sxinhui pan } 233791c4769Sxinhui pan break; 234791c4769Sxinhui pan case AMDGPU_IRQ_STATE_ENABLE: 235791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 236791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 237791c4769Sxinhui pan tmp = RREG32(reg); 238791c4769Sxinhui pan tmp |= bits; 239791c4769Sxinhui pan WREG32(reg, tmp); 240791c4769Sxinhui pan } 241791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 242791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 243791c4769Sxinhui pan tmp = RREG32(reg); 244791c4769Sxinhui pan tmp |= bits; 245791c4769Sxinhui pan WREG32(reg, tmp); 246791c4769Sxinhui pan } 247791c4769Sxinhui pan break; 248791c4769Sxinhui pan default: 249791c4769Sxinhui pan break; 250791c4769Sxinhui pan } 251791c4769Sxinhui pan 252791c4769Sxinhui pan return 0; 253791c4769Sxinhui pan } 254791c4769Sxinhui pan 255e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 256e60f8db5SAlex Xie struct amdgpu_irq_src *src, 257e60f8db5SAlex Xie unsigned type, 258e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 259e60f8db5SAlex Xie { 260e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 261ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 262e60f8db5SAlex Xie 26311250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26411250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26511250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26611250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26711250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26811250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26911250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 27011250164SChristian König 271e60f8db5SAlex Xie switch (state) { 272e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 2731daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 274ae6d1416STom St Denis hub = &adev->vmhub[j]; 275e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 276e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 277e60f8db5SAlex Xie tmp = RREG32(reg); 278e60f8db5SAlex Xie tmp &= ~bits; 279e60f8db5SAlex Xie WREG32(reg, tmp); 280e60f8db5SAlex Xie } 281e60f8db5SAlex Xie } 282e60f8db5SAlex Xie break; 283e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 2841daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 285ae6d1416STom St Denis hub = &adev->vmhub[j]; 286e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 287e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 288e60f8db5SAlex Xie tmp = RREG32(reg); 289e60f8db5SAlex Xie tmp |= bits; 290e60f8db5SAlex Xie WREG32(reg, tmp); 291e60f8db5SAlex Xie } 292e60f8db5SAlex Xie } 293e60f8db5SAlex Xie default: 294e60f8db5SAlex Xie break; 295e60f8db5SAlex Xie } 296e60f8db5SAlex Xie 297e60f8db5SAlex Xie return 0; 298e60f8db5SAlex Xie } 299e60f8db5SAlex Xie 300e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 301e60f8db5SAlex Xie struct amdgpu_irq_src *source, 302e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 303e60f8db5SAlex Xie { 30451c60898SLe Ma struct amdgpu_vmhub *hub; 305c468f9e2SChristian König bool retry_fault = !!(entry->src_data[1] & 0x80); 3064d6cbde3SFelix Kuehling uint32_t status = 0; 307e60f8db5SAlex Xie u64 addr; 30851c60898SLe Ma char hub_name[10]; 309e60f8db5SAlex Xie 310e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 311e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 312e60f8db5SAlex Xie 313c1a8abd9SChristian König if (retry_fault && amdgpu_gmc_filter_faults(adev, addr, entry->pasid, 314c1a8abd9SChristian König entry->timestamp)) 31522666cc1SChristian König return 1; /* This also prevents sending it to KFD */ 31622666cc1SChristian König 31751c60898SLe Ma if (entry->client_id == SOC15_IH_CLIENTID_VMC) { 31851c60898SLe Ma snprintf(hub_name, sizeof(hub_name), "mmhub0"); 31951c60898SLe Ma hub = &adev->vmhub[AMDGPU_MMHUB_0]; 32051c60898SLe Ma } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { 32151c60898SLe Ma snprintf(hub_name, sizeof(hub_name), "mmhub1"); 32251c60898SLe Ma hub = &adev->vmhub[AMDGPU_MMHUB_1]; 32351c60898SLe Ma } else { 32451c60898SLe Ma snprintf(hub_name, sizeof(hub_name), "gfxhub0"); 32551c60898SLe Ma hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 32651c60898SLe Ma } 32751c60898SLe Ma 328c1a8abd9SChristian König /* If it's the first fault for this address, process it normally */ 329ec671737SChristian König if (retry_fault && !in_interrupt() && 330ec671737SChristian König amdgpu_vm_handle_fault(adev, entry->pasid, addr)) 331ec671737SChristian König return 1; /* This also prevents sending it to KFD */ 332ec671737SChristian König 33379a0c465SMonk Liu if (!amdgpu_sriov_vf(adev)) { 33453499173SXiaojie Yuan /* 33553499173SXiaojie Yuan * Issue a dummy read to wait for the status register to 33653499173SXiaojie Yuan * be updated to avoid reading an incorrect value due to 33753499173SXiaojie Yuan * the new fast GRBM interface. 33853499173SXiaojie Yuan */ 33953499173SXiaojie Yuan if (entry->vmid_src == AMDGPU_GFXHUB_0) 34053499173SXiaojie Yuan RREG32(hub->vm_l2_pro_fault_status); 34153499173SXiaojie Yuan 3425a9b8e8aSChristian König status = RREG32(hub->vm_l2_pro_fault_status); 3435a9b8e8aSChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 3444d6cbde3SFelix Kuehling } 345e60f8db5SAlex Xie 3464d6cbde3SFelix Kuehling if (printk_ratelimit()) { 34705794effSShirish S struct amdgpu_task_info task_info; 348efaa9646SAndrey Grodzovsky 34905794effSShirish S memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 350efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 351efaa9646SAndrey Grodzovsky 3524d6cbde3SFelix Kuehling dev_err(adev->dev, 353c468f9e2SChristian König "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 354c468f9e2SChristian König "pasid:%u, for process %s pid %d thread %s pid %d)\n", 35551c60898SLe Ma hub_name, retry_fault ? "retry" : "no-retry", 356c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 357efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 358efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 3595ddd4a9aSYong Zhao dev_err(adev->dev, " in page starting at address 0x%016llx from client %d\n", 36079a0c465SMonk Liu addr, entry->client_id); 3615ddd4a9aSYong Zhao if (!amdgpu_sriov_vf(adev)) { 3624d6cbde3SFelix Kuehling dev_err(adev->dev, 3634d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 3644d6cbde3SFelix Kuehling status); 365d2155a71SOak Zeng dev_err(adev->dev, "\t Faulty UTCL2 client ID: 0x%lx\n", 366d2155a71SOak Zeng REG_GET_FIELD(status, 367d2155a71SOak Zeng VM_L2_PROTECTION_FAULT_STATUS, CID)); 3685ddd4a9aSYong Zhao dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 3695ddd4a9aSYong Zhao REG_GET_FIELD(status, 3705ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 3715ddd4a9aSYong Zhao dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 3725ddd4a9aSYong Zhao REG_GET_FIELD(status, 3735ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 3745ddd4a9aSYong Zhao dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 3755ddd4a9aSYong Zhao REG_GET_FIELD(status, 3765ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 3775ddd4a9aSYong Zhao dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 3785ddd4a9aSYong Zhao REG_GET_FIELD(status, 3795ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 3804e0ae5e2SYong Zhao dev_err(adev->dev, "\t RW: 0x%lx\n", 3814e0ae5e2SYong Zhao REG_GET_FIELD(status, 3824e0ae5e2SYong Zhao VM_L2_PROTECTION_FAULT_STATUS, RW)); 3835ddd4a9aSYong Zhao 3845ddd4a9aSYong Zhao } 38579a0c465SMonk Liu } 386e60f8db5SAlex Xie 387e60f8db5SAlex Xie return 0; 388e60f8db5SAlex Xie } 389e60f8db5SAlex Xie 390e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 391e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 392e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 393e60f8db5SAlex Xie }; 394e60f8db5SAlex Xie 395791c4769Sxinhui pan 396791c4769Sxinhui pan static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 397791c4769Sxinhui pan .set = gmc_v9_0_ecc_interrupt_state, 39834cc4fd9STao Zhou .process = amdgpu_umc_process_ecc_irq, 399791c4769Sxinhui pan }; 400791c4769Sxinhui pan 401e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 402e60f8db5SAlex Xie { 403770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 404770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 405791c4769Sxinhui pan 4062ee9403eSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 407791c4769Sxinhui pan adev->gmc.ecc_irq.num_types = 1; 408791c4769Sxinhui pan adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 409e60f8db5SAlex Xie } 4102ee9403eSZhigang Luo } 411e60f8db5SAlex Xie 4122a79d868SYong Zhao static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 4132a79d868SYong Zhao uint32_t flush_type) 41403f89febSChristian König { 41503f89febSChristian König u32 req = 0; 41603f89febSChristian König 41703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 418c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 4192a79d868SYong Zhao req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 42003f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 42103f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 42203f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 42303f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 42403f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 42503f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 42603f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 42703f89febSChristian König 42803f89febSChristian König return req; 42903f89febSChristian König } 43003f89febSChristian König 43190f6452cSchangzhu /** 43290f6452cSchangzhu * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore 43390f6452cSchangzhu * 43490f6452cSchangzhu * @adev: amdgpu_device pointer 43590f6452cSchangzhu * @vmhub: vmhub type 43690f6452cSchangzhu * 43790f6452cSchangzhu */ 43890f6452cSchangzhu static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, 43990f6452cSchangzhu uint32_t vmhub) 44090f6452cSchangzhu { 44190f6452cSchangzhu return ((vmhub == AMDGPU_MMHUB_0 || 44290f6452cSchangzhu vmhub == AMDGPU_MMHUB_1) && 44390f6452cSchangzhu (!amdgpu_sriov_vf(adev)) && 44454f78a76SAlex Deucher (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && 44554f78a76SAlex Deucher (adev->apu_flags & AMD_APU_IS_PICASSO)))); 44690f6452cSchangzhu } 44790f6452cSchangzhu 448ea930000SAlex Sierra static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, 449ea930000SAlex Sierra uint8_t vmid, uint16_t *p_pasid) 450ea930000SAlex Sierra { 451ea930000SAlex Sierra uint32_t value; 452ea930000SAlex Sierra 453ea930000SAlex Sierra value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 454ea930000SAlex Sierra + vmid); 455ea930000SAlex Sierra *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 456ea930000SAlex Sierra 457ea930000SAlex Sierra return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 458ea930000SAlex Sierra } 459ea930000SAlex Sierra 460e60f8db5SAlex Xie /* 461e60f8db5SAlex Xie * GART 462e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 463e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 464e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 465e60f8db5SAlex Xie */ 466e60f8db5SAlex Xie 467e60f8db5SAlex Xie /** 4682a79d868SYong Zhao * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 469e60f8db5SAlex Xie * 470e60f8db5SAlex Xie * @adev: amdgpu_device pointer 471e60f8db5SAlex Xie * @vmid: vm instance to flush 4722a79d868SYong Zhao * @flush_type: the flush type 473e60f8db5SAlex Xie * 4742a79d868SYong Zhao * Flush the TLB for the requested page table using certain type. 475e60f8db5SAlex Xie */ 4763ff98548SOak Zeng static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 4773ff98548SOak Zeng uint32_t vmhub, uint32_t flush_type) 478e60f8db5SAlex Xie { 47990f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); 480e60f8db5SAlex Xie const unsigned eng = 17; 481b80cd524SFelix Kuehling u32 j, inv_req, inv_req2, tmp; 4823ff98548SOak Zeng struct amdgpu_vmhub *hub; 483e60f8db5SAlex Xie 4843ff98548SOak Zeng BUG_ON(vmhub >= adev->num_vmhubs); 4853ff98548SOak Zeng 4863ff98548SOak Zeng hub = &adev->vmhub[vmhub]; 487b80cd524SFelix Kuehling if (adev->gmc.xgmi.num_physical_nodes && 488b80cd524SFelix Kuehling adev->asic_type == CHIP_VEGA20) { 489b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 490b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 491b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 492b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 493b80cd524SFelix Kuehling * still need a second TLB flush after this. 494b80cd524SFelix Kuehling */ 495b80cd524SFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); 496b80cd524SFelix Kuehling inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 497b80cd524SFelix Kuehling } else { 49837c58ddfSFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); 499b80cd524SFelix Kuehling inv_req2 = 0; 500b80cd524SFelix Kuehling } 501e60f8db5SAlex Xie 50282d1a1b1SChengming Gui /* This is necessary for a HW workaround under SRIOV as well 50382d1a1b1SChengming Gui * as GFXOFF under bare metal 50482d1a1b1SChengming Gui */ 50582d1a1b1SChengming Gui if (adev->gfx.kiq.ring.sched.ready && 50682d1a1b1SChengming Gui (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 50753b3f8f4SDennis Li !amdgpu_in_reset(adev)) { 508148f597dSHuang Rui uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 509148f597dSHuang Rui uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 510af5fe1e9SChristian König 51137c58ddfSFelix Kuehling amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 512af5fe1e9SChristian König 1 << vmid); 5133ff98548SOak Zeng return; 514fc0faf04SEmily Deng } 5153890d111SEmily Deng 5163890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 517f920d1bbSchangzhu 518f920d1bbSchangzhu /* 519f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 520f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 521f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 522f920d1bbSchangzhu * to WA the Issue 523f920d1bbSchangzhu */ 524f920d1bbSchangzhu 525f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 52690f6452cSchangzhu if (use_semaphore) { 527f920d1bbSchangzhu for (j = 0; j < adev->usec_timeout; j++) { 528f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 529148f597dSHuang Rui tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + 530148f597dSHuang Rui hub->eng_distance * eng); 531f920d1bbSchangzhu if (tmp & 0x1) 532f920d1bbSchangzhu break; 533f920d1bbSchangzhu udelay(1); 534f920d1bbSchangzhu } 535f920d1bbSchangzhu 536f920d1bbSchangzhu if (j >= adev->usec_timeout) 537f920d1bbSchangzhu DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 538f920d1bbSchangzhu } 539f920d1bbSchangzhu 540b80cd524SFelix Kuehling do { 541148f597dSHuang Rui WREG32_NO_KIQ(hub->vm_inv_eng0_req + 542148f597dSHuang Rui hub->eng_distance * eng, inv_req); 54353499173SXiaojie Yuan 54453499173SXiaojie Yuan /* 545b80cd524SFelix Kuehling * Issue a dummy read to wait for the ACK register to 546b80cd524SFelix Kuehling * be cleared to avoid a false ACK due to the new fast 547b80cd524SFelix Kuehling * GRBM interface. 54853499173SXiaojie Yuan */ 54953499173SXiaojie Yuan if (vmhub == AMDGPU_GFXHUB_0) 550148f597dSHuang Rui RREG32_NO_KIQ(hub->vm_inv_eng0_req + 551148f597dSHuang Rui hub->eng_distance * eng); 55253499173SXiaojie Yuan 553e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 554148f597dSHuang Rui tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + 555148f597dSHuang Rui hub->eng_distance * eng); 556396557b0SChristian König if (tmp & (1 << vmid)) 557e60f8db5SAlex Xie break; 558e60f8db5SAlex Xie udelay(1); 559e60f8db5SAlex Xie } 560f920d1bbSchangzhu 561b80cd524SFelix Kuehling inv_req = inv_req2; 562b80cd524SFelix Kuehling inv_req2 = 0; 563b80cd524SFelix Kuehling } while (inv_req); 564b80cd524SFelix Kuehling 565f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 56690f6452cSchangzhu if (use_semaphore) 567f920d1bbSchangzhu /* 568f920d1bbSchangzhu * add semaphore release after invalidation, 569f920d1bbSchangzhu * write with 0 means semaphore release 570f920d1bbSchangzhu */ 571148f597dSHuang Rui WREG32_NO_KIQ(hub->vm_inv_eng0_sem + 572148f597dSHuang Rui hub->eng_distance * eng, 0); 573f920d1bbSchangzhu 5743890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 575f920d1bbSchangzhu 576396557b0SChristian König if (j < adev->usec_timeout) 5773ff98548SOak Zeng return; 578396557b0SChristian König 579e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 580e60f8db5SAlex Xie } 581e60f8db5SAlex Xie 582ea930000SAlex Sierra /** 583ea930000SAlex Sierra * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid 584ea930000SAlex Sierra * 585ea930000SAlex Sierra * @adev: amdgpu_device pointer 586ea930000SAlex Sierra * @pasid: pasid to be flush 587ea930000SAlex Sierra * 588ea930000SAlex Sierra * Flush the TLB for the requested pasid. 589ea930000SAlex Sierra */ 590ea930000SAlex Sierra static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 591ea930000SAlex Sierra uint16_t pasid, uint32_t flush_type, 592ea930000SAlex Sierra bool all_hub) 593ea930000SAlex Sierra { 594ea930000SAlex Sierra int vmid, i; 595ea930000SAlex Sierra signed long r; 596ea930000SAlex Sierra uint32_t seq; 597ea930000SAlex Sierra uint16_t queried_pasid; 598ea930000SAlex Sierra bool ret; 599ea930000SAlex Sierra struct amdgpu_ring *ring = &adev->gfx.kiq.ring; 600ea930000SAlex Sierra struct amdgpu_kiq *kiq = &adev->gfx.kiq; 601ea930000SAlex Sierra 60253b3f8f4SDennis Li if (amdgpu_in_reset(adev)) 603ea930000SAlex Sierra return -EIO; 604ea930000SAlex Sierra 605ea930000SAlex Sierra if (ring->sched.ready) { 606b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 607b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 608b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 609b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 610b80cd524SFelix Kuehling * still need a second TLB flush after this. 611b80cd524SFelix Kuehling */ 612b80cd524SFelix Kuehling bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && 613b80cd524SFelix Kuehling adev->asic_type == CHIP_VEGA20); 614b80cd524SFelix Kuehling /* 2 dwords flush + 8 dwords fence */ 615b80cd524SFelix Kuehling unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; 616b80cd524SFelix Kuehling 617b80cd524SFelix Kuehling if (vega20_xgmi_wa) 618b80cd524SFelix Kuehling ndw += kiq->pmf->invalidate_tlbs_size; 619b80cd524SFelix Kuehling 620ea930000SAlex Sierra spin_lock(&adev->gfx.kiq.ring_lock); 62136a1707aSAlex Sierra /* 2 dwords flush + 8 dwords fence */ 622b80cd524SFelix Kuehling amdgpu_ring_alloc(ring, ndw); 623b80cd524SFelix Kuehling if (vega20_xgmi_wa) 624b80cd524SFelix Kuehling kiq->pmf->kiq_invalidate_tlbs(ring, 625b80cd524SFelix Kuehling pasid, 2, all_hub); 626ea930000SAlex Sierra kiq->pmf->kiq_invalidate_tlbs(ring, 627ea930000SAlex Sierra pasid, flush_type, all_hub); 62804e4e2e9SYintian Tao r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 62904e4e2e9SYintian Tao if (r) { 63004e4e2e9SYintian Tao amdgpu_ring_undo(ring); 631abb17b1eSColin Ian King spin_unlock(&adev->gfx.kiq.ring_lock); 63204e4e2e9SYintian Tao return -ETIME; 63304e4e2e9SYintian Tao } 63404e4e2e9SYintian Tao 635ea930000SAlex Sierra amdgpu_ring_commit(ring); 636ea930000SAlex Sierra spin_unlock(&adev->gfx.kiq.ring_lock); 637ea930000SAlex Sierra r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); 638ea930000SAlex Sierra if (r < 1) { 639aac89168SDennis Li dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); 640ea930000SAlex Sierra return -ETIME; 641ea930000SAlex Sierra } 642ea930000SAlex Sierra 643ea930000SAlex Sierra return 0; 644ea930000SAlex Sierra } 645ea930000SAlex Sierra 646ea930000SAlex Sierra for (vmid = 1; vmid < 16; vmid++) { 647ea930000SAlex Sierra 648ea930000SAlex Sierra ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 649ea930000SAlex Sierra &queried_pasid); 650ea930000SAlex Sierra if (ret && queried_pasid == pasid) { 651ea930000SAlex Sierra if (all_hub) { 652ea930000SAlex Sierra for (i = 0; i < adev->num_vmhubs; i++) 653ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 654fa34edbeSFelix Kuehling i, flush_type); 655ea930000SAlex Sierra } else { 656ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 657fa34edbeSFelix Kuehling AMDGPU_GFXHUB_0, flush_type); 658ea930000SAlex Sierra } 659ea930000SAlex Sierra break; 660ea930000SAlex Sierra } 661ea930000SAlex Sierra } 662ea930000SAlex Sierra 663ea930000SAlex Sierra return 0; 664ea930000SAlex Sierra 665ea930000SAlex Sierra } 666ea930000SAlex Sierra 6679096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 668c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 6699096d6e5SChristian König { 67090f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); 671250b4228SChristian König struct amdgpu_device *adev = ring->adev; 672250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 6732a79d868SYong Zhao uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 6749096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 6759096d6e5SChristian König 676f920d1bbSchangzhu /* 677f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 678f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 679f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 680f920d1bbSchangzhu * to WA the Issue 681f920d1bbSchangzhu */ 682f920d1bbSchangzhu 683f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 68490f6452cSchangzhu if (use_semaphore) 685f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 686f920d1bbSchangzhu amdgpu_ring_emit_reg_wait(ring, 687148f597dSHuang Rui hub->vm_inv_eng0_sem + 688148f597dSHuang Rui hub->eng_distance * eng, 0x1, 0x1); 689f920d1bbSchangzhu 690148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 691148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 6929096d6e5SChristian König lower_32_bits(pd_addr)); 6939096d6e5SChristian König 694148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 695148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 6969096d6e5SChristian König upper_32_bits(pd_addr)); 6979096d6e5SChristian König 698148f597dSHuang Rui amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 699148f597dSHuang Rui hub->eng_distance * eng, 700148f597dSHuang Rui hub->vm_inv_eng0_ack + 701148f597dSHuang Rui hub->eng_distance * eng, 702f8bc9037SAlex Deucher req, 1 << vmid); 703f732b6b3SChristian König 704f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 70590f6452cSchangzhu if (use_semaphore) 706f920d1bbSchangzhu /* 707f920d1bbSchangzhu * add semaphore release after invalidation, 708f920d1bbSchangzhu * write with 0 means semaphore release 709f920d1bbSchangzhu */ 710148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 711148f597dSHuang Rui hub->eng_distance * eng, 0); 712f920d1bbSchangzhu 7139096d6e5SChristian König return pd_addr; 7149096d6e5SChristian König } 7159096d6e5SChristian König 716c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 717c633c00bSChristian König unsigned pasid) 718c633c00bSChristian König { 719c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 720c633c00bSChristian König uint32_t reg; 721c633c00bSChristian König 722f2d66571SLe Ma /* Do nothing because there's no lut register for mmhub1. */ 723f2d66571SLe Ma if (ring->funcs->vmhub == AMDGPU_MMHUB_1) 724f2d66571SLe Ma return; 725f2d66571SLe Ma 726a2d15ed7SLe Ma if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) 727c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 728c633c00bSChristian König else 729c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 730c633c00bSChristian König 731c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 732c633c00bSChristian König } 733c633c00bSChristian König 734e60f8db5SAlex Xie /* 735e60f8db5SAlex Xie * PTE format on VEGA 10: 736e60f8db5SAlex Xie * 63:59 reserved 737e60f8db5SAlex Xie * 58:57 mtype 738e60f8db5SAlex Xie * 56 F 739e60f8db5SAlex Xie * 55 L 740e60f8db5SAlex Xie * 54 P 741e60f8db5SAlex Xie * 53 SW 742e60f8db5SAlex Xie * 52 T 743e60f8db5SAlex Xie * 50:48 reserved 744e60f8db5SAlex Xie * 47:12 4k physical page base address 745e60f8db5SAlex Xie * 11:7 fragment 746e60f8db5SAlex Xie * 6 write 747e60f8db5SAlex Xie * 5 read 748e60f8db5SAlex Xie * 4 exe 749e60f8db5SAlex Xie * 3 Z 750e60f8db5SAlex Xie * 2 snooped 751e60f8db5SAlex Xie * 1 system 752e60f8db5SAlex Xie * 0 valid 753e60f8db5SAlex Xie * 754e60f8db5SAlex Xie * PDE format on VEGA 10: 755e60f8db5SAlex Xie * 63:59 block fragment size 756e60f8db5SAlex Xie * 58:55 reserved 757e60f8db5SAlex Xie * 54 P 758e60f8db5SAlex Xie * 53:48 reserved 759e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 760e60f8db5SAlex Xie * 5:3 reserved 761e60f8db5SAlex Xie * 2 C 762e60f8db5SAlex Xie * 1 system 763e60f8db5SAlex Xie * 0 valid 764e60f8db5SAlex Xie */ 765e60f8db5SAlex Xie 76671776b6dSChristian König static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) 767e60f8db5SAlex Xie 768e60f8db5SAlex Xie { 76971776b6dSChristian König switch (flags) { 770e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 77171776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 772e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 77371776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 774e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 77571776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); 776093e48c0SOak Zeng case AMDGPU_VM_MTYPE_RW: 77771776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); 778e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 77971776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); 780e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 78171776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); 782e60f8db5SAlex Xie default: 78371776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 784e60f8db5SAlex Xie } 785e60f8db5SAlex Xie } 786e60f8db5SAlex Xie 7873de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 7883de676d8SChristian König uint64_t *addr, uint64_t *flags) 789f75e237cSChristian König { 790bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 7913de676d8SChristian König *addr = adev->vm_manager.vram_base_offset + *addr - 792770d13b1SChristian König adev->gmc.vram_start; 7933de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 7946a42fd6fSChristian König 795770d13b1SChristian König if (!adev->gmc.translate_further) 7966a42fd6fSChristian König return; 7976a42fd6fSChristian König 7986a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 7996a42fd6fSChristian König /* Set the block fragment size */ 8006a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 8016a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 8026a42fd6fSChristian König 8036a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 8046a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 8056a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 8066a42fd6fSChristian König else 8076a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 8086a42fd6fSChristian König } 809f75e237cSChristian König } 810f75e237cSChristian König 811cbfae36cSChristian König static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, 812cbfae36cSChristian König struct amdgpu_bo_va_mapping *mapping, 813cbfae36cSChristian König uint64_t *flags) 814cbfae36cSChristian König { 815cbfae36cSChristian König *flags &= ~AMDGPU_PTE_EXECUTABLE; 816cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 817cbfae36cSChristian König 818cbfae36cSChristian König *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 819cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; 820cbfae36cSChristian König 821cbfae36cSChristian König if (mapping->flags & AMDGPU_PTE_PRT) { 822cbfae36cSChristian König *flags |= AMDGPU_PTE_PRT; 823cbfae36cSChristian König *flags &= ~AMDGPU_PTE_VALID; 824cbfae36cSChristian König } 825cbfae36cSChristian König 826cbfae36cSChristian König if (adev->asic_type == CHIP_ARCTURUS && 827cbfae36cSChristian König !(*flags & AMDGPU_PTE_SYSTEM) && 828cbfae36cSChristian König mapping->bo_va->is_xgmi) 829cbfae36cSChristian König *flags |= AMDGPU_PTE_SNOOPED; 830cbfae36cSChristian König } 831cbfae36cSChristian König 8327b885f0eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 8337b885f0eSAlex Deucher { 8347b885f0eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 8357b885f0eSAlex Deucher unsigned size; 8367b885f0eSAlex Deucher 8377b885f0eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 8387b885f0eSAlex Deucher size = AMDGPU_VBIOS_VGA_ALLOCATION; 8397b885f0eSAlex Deucher } else { 8407b885f0eSAlex Deucher u32 viewport; 8417b885f0eSAlex Deucher 8427b885f0eSAlex Deucher switch (adev->asic_type) { 8437b885f0eSAlex Deucher case CHIP_RAVEN: 8447b885f0eSAlex Deucher case CHIP_RENOIR: 8457b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 8467b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, 8477b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 8487b885f0eSAlex Deucher REG_GET_FIELD(viewport, 8497b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 8507b885f0eSAlex Deucher 4); 8517b885f0eSAlex Deucher break; 8527b885f0eSAlex Deucher case CHIP_VEGA10: 8537b885f0eSAlex Deucher case CHIP_VEGA12: 8547b885f0eSAlex Deucher case CHIP_VEGA20: 8557b885f0eSAlex Deucher default: 8567b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 8577b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 8587b885f0eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 8597b885f0eSAlex Deucher 4); 8607b885f0eSAlex Deucher break; 8617b885f0eSAlex Deucher } 8627b885f0eSAlex Deucher } 8637b885f0eSAlex Deucher 8647b885f0eSAlex Deucher return size; 8657b885f0eSAlex Deucher } 8667b885f0eSAlex Deucher 867132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 868132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 869ea930000SAlex Sierra .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, 8709096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 871c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 87271776b6dSChristian König .map_mtype = gmc_v9_0_map_mtype, 873cbfae36cSChristian König .get_vm_pde = gmc_v9_0_get_vm_pde, 8747b885f0eSAlex Deucher .get_vm_pte = gmc_v9_0_get_vm_pte, 8757b885f0eSAlex Deucher .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, 876e60f8db5SAlex Xie }; 877e60f8db5SAlex Xie 878132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 879e60f8db5SAlex Xie { 880132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 881e60f8db5SAlex Xie } 882e60f8db5SAlex Xie 8835b6b35aaSHawking Zhang static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) 8845b6b35aaSHawking Zhang { 8855b6b35aaSHawking Zhang switch (adev->asic_type) { 886e7da754bSMonk Liu case CHIP_VEGA10: 887e7da754bSMonk Liu adev->umc.funcs = &umc_v6_0_funcs; 888e7da754bSMonk Liu break; 8895b6b35aaSHawking Zhang case CHIP_VEGA20: 8903aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 8913aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 8923aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 8934cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; 8944cf781c2SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 8954cf781c2SJohn Clements adev->umc.funcs = &umc_v6_1_funcs; 8964cf781c2SJohn Clements break; 8979e612c11SHawking Zhang case CHIP_ARCTURUS: 8983aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 8993aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 9003aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 9014cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; 9023aacf4eaSTao Zhou adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 903045c0216STao Zhou adev->umc.funcs = &umc_v6_1_funcs; 9045b6b35aaSHawking Zhang break; 9055b6b35aaSHawking Zhang default: 9065b6b35aaSHawking Zhang break; 9075b6b35aaSHawking Zhang } 9085b6b35aaSHawking Zhang } 9095b6b35aaSHawking Zhang 9103d093da0STao Zhou static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) 9113d093da0STao Zhou { 9123d093da0STao Zhou switch (adev->asic_type) { 913f6c3623bSDennis Li case CHIP_ARCTURUS: 914f6c3623bSDennis Li adev->mmhub.funcs = &mmhub_v9_4_funcs; 915f6c3623bSDennis Li break; 9163d093da0STao Zhou default: 9179fb1506eSOak Zeng adev->mmhub.funcs = &mmhub_v1_0_funcs; 9183d093da0STao Zhou break; 9193d093da0STao Zhou } 9203d093da0STao Zhou } 9213d093da0STao Zhou 922e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 923e60f8db5SAlex Xie { 924e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 925e60f8db5SAlex Xie 926132f34e4SChristian König gmc_v9_0_set_gmc_funcs(adev); 927e60f8db5SAlex Xie gmc_v9_0_set_irq_funcs(adev); 9285b6b35aaSHawking Zhang gmc_v9_0_set_umc_funcs(adev); 9293d093da0STao Zhou gmc_v9_0_set_mmhub_funcs(adev); 930e60f8db5SAlex Xie 931770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 932770d13b1SChristian König adev->gmc.shared_aperture_end = 933770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 934bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 935770d13b1SChristian König adev->gmc.private_aperture_end = 936770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 937a7ea6548SAlex Deucher 938e60f8db5SAlex Xie return 0; 939e60f8db5SAlex Xie } 940e60f8db5SAlex Xie 941e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 942e60f8db5SAlex Xie { 943e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 944c5b6e585STao Zhou int r; 9454789c463SChristian König 946cd2b5623SAlex Deucher amdgpu_bo_late_init(adev); 9476f752ec2SAndrey Grodzovsky 948bdbe90f0SAlex Deucher r = amdgpu_gmc_allocate_vm_inv_eng(adev); 949c713a461SEvan Quan if (r) 950c713a461SEvan Quan return r; 951f49ea9f8SHawking Zhang /* Check if ecc is available */ 95288474ccaSGuchun Chen if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) { 953f49ea9f8SHawking Zhang r = amdgpu_atomfirmware_mem_ecc_supported(adev); 954f49ea9f8SHawking Zhang if (!r) { 95502bab923SDavid Panariti DRM_INFO("ECC is not present.\n"); 956bdf84a80SJoseph Greathouse if (adev->df.funcs->enable_ecc_force_par_wr_rmw) 957bdf84a80SJoseph Greathouse adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); 95888474ccaSGuchun Chen } else 959f49ea9f8SHawking Zhang DRM_INFO("ECC is active.\n"); 960f49ea9f8SHawking Zhang 961f49ea9f8SHawking Zhang r = amdgpu_atomfirmware_sram_ecc_supported(adev); 96288474ccaSGuchun Chen if (!r) 963f49ea9f8SHawking Zhang DRM_INFO("SRAM ECC is not present.\n"); 96488474ccaSGuchun Chen else 965f49ea9f8SHawking Zhang DRM_INFO("SRAM ECC is active.\n"); 966f49ea9f8SHawking Zhang } 96702bab923SDavid Panariti 968fe5211f1SHawking Zhang if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count) 969fe5211f1SHawking Zhang adev->mmhub.funcs->reset_ras_error_count(adev); 970fe5211f1SHawking Zhang 971ba083492STao Zhou r = amdgpu_gmc_ras_late_init(adev); 972791c4769Sxinhui pan if (r) 973e60f8db5SAlex Xie return r; 974e60f8db5SAlex Xie 975770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 976e60f8db5SAlex Xie } 977e60f8db5SAlex Xie 978e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 979770d13b1SChristian König struct amdgpu_gmc *mc) 980e60f8db5SAlex Xie { 981e60f8db5SAlex Xie u64 base = 0; 9829d4f837aSFrank.Min 9839fb1506eSOak Zeng if (!amdgpu_sriov_vf(adev)) 9849fb1506eSOak Zeng base = adev->mmhub.funcs->get_fb_location(adev); 9859d4f837aSFrank.Min 9866fdd68b1SAlex Deucher /* add the xgmi offset of the physical node */ 9876fdd68b1SAlex Deucher base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 98883afe835SOak Zeng amdgpu_gmc_vram_location(adev, mc, base); 989961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 990c3e1b43cSChristian König amdgpu_gmc_agp_location(adev, mc); 991e60f8db5SAlex Xie /* base offset of vram pages */ 992e60f8db5SAlex Xie adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev); 9936fdd68b1SAlex Deucher 9946fdd68b1SAlex Deucher /* XXX: add the xgmi offset of the physical node? */ 9956fdd68b1SAlex Deucher adev->vm_manager.vram_base_offset += 9966fdd68b1SAlex Deucher adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 997e60f8db5SAlex Xie } 998e60f8db5SAlex Xie 999e60f8db5SAlex Xie /** 1000e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 1001e60f8db5SAlex Xie * 1002e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1003e60f8db5SAlex Xie * 1004e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 1005e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 1006e60f8db5SAlex Xie * Returns 0 for success. 1007e60f8db5SAlex Xie */ 1008e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 1009e60f8db5SAlex Xie { 1010e60f8db5SAlex Xie int r; 1011e60f8db5SAlex Xie 1012e60f8db5SAlex Xie /* size in MB on si */ 1013770d13b1SChristian König adev->gmc.mc_vram_size = 1014bebc0762SHawking Zhang adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 1015770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 1016e60f8db5SAlex Xie 1017e60f8db5SAlex Xie if (!(adev->flags & AMD_IS_APU)) { 1018e60f8db5SAlex Xie r = amdgpu_device_resize_fb_bar(adev); 1019e60f8db5SAlex Xie if (r) 1020e60f8db5SAlex Xie return r; 1021e60f8db5SAlex Xie } 1022770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 1023770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 1024e60f8db5SAlex Xie 1025156a81beSChunming Zhou #ifdef CONFIG_X86_64 1026156a81beSChunming Zhou if (adev->flags & AMD_IS_APU) { 1027156a81beSChunming Zhou adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev); 1028156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 1029156a81beSChunming Zhou } 1030156a81beSChunming Zhou #endif 1031e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 1032770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 1033770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 1034770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 1035e60f8db5SAlex Xie 1036e60f8db5SAlex Xie /* set the gart size */ 1037e60f8db5SAlex Xie if (amdgpu_gart_size == -1) { 1038e60f8db5SAlex Xie switch (adev->asic_type) { 1039e60f8db5SAlex Xie case CHIP_VEGA10: /* all engines support GPUVM */ 1040273a14cdSAlex Deucher case CHIP_VEGA12: /* all engines support GPUVM */ 1041d96b428cSFeifei Xu case CHIP_VEGA20: 10423de2ff5dSLe Ma case CHIP_ARCTURUS: 1043e60f8db5SAlex Xie default: 1044fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 1045e60f8db5SAlex Xie break; 1046e60f8db5SAlex Xie case CHIP_RAVEN: /* DCE SG support */ 10478787ee01SHuang Rui case CHIP_RENOIR: 1048770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 1049e60f8db5SAlex Xie break; 1050e60f8db5SAlex Xie } 1051e60f8db5SAlex Xie } else { 1052770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 1053e60f8db5SAlex Xie } 1054e60f8db5SAlex Xie 1055770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 1056e60f8db5SAlex Xie 1057e60f8db5SAlex Xie return 0; 1058e60f8db5SAlex Xie } 1059e60f8db5SAlex Xie 1060e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 1061e60f8db5SAlex Xie { 1062e60f8db5SAlex Xie int r; 1063e60f8db5SAlex Xie 10641123b989SChristian König if (adev->gart.bo) { 1065e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 1066e60f8db5SAlex Xie return 0; 1067e60f8db5SAlex Xie } 1068e60f8db5SAlex Xie /* Initialize common gart structure */ 1069e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 1070e60f8db5SAlex Xie if (r) 1071e60f8db5SAlex Xie return r; 1072e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 10737596ab68SHawking Zhang adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | 1074e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 1075e60f8db5SAlex Xie return amdgpu_gart_table_vram_alloc(adev); 1076e60f8db5SAlex Xie } 1077e60f8db5SAlex Xie 1078b0a2db9bSAlex Deucher /** 1079b0a2db9bSAlex Deucher * gmc_v9_0_save_registers - saves regs 1080b0a2db9bSAlex Deucher * 1081b0a2db9bSAlex Deucher * @adev: amdgpu_device pointer 1082b0a2db9bSAlex Deucher * 1083b0a2db9bSAlex Deucher * This saves potential register values that should be 1084b0a2db9bSAlex Deucher * restored upon resume 1085b0a2db9bSAlex Deucher */ 1086b0a2db9bSAlex Deucher static void gmc_v9_0_save_registers(struct amdgpu_device *adev) 1087b0a2db9bSAlex Deucher { 1088b0a2db9bSAlex Deucher if (adev->asic_type == CHIP_RAVEN) 1089b0a2db9bSAlex Deucher adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); 1090b0a2db9bSAlex Deucher } 1091b0a2db9bSAlex Deucher 1092e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 1093e60f8db5SAlex Xie { 1094ad02e08eSOri Messinger int r, vram_width = 0, vram_type = 0, vram_vendor = 0; 1095e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1096e60f8db5SAlex Xie 1097e60f8db5SAlex Xie gfxhub_v1_0_init(adev); 10989fb1506eSOak Zeng 10999fb1506eSOak Zeng adev->mmhub.funcs->init(adev); 1100e60f8db5SAlex Xie 1101770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 1102e60f8db5SAlex Xie 1103ad02e08eSOri Messinger r = amdgpu_atomfirmware_get_vram_info(adev, 1104ad02e08eSOri Messinger &vram_width, &vram_type, &vram_vendor); 1105631cdbd2SAlex Deucher if (amdgpu_sriov_vf(adev)) 1106631cdbd2SAlex Deucher /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, 1107631cdbd2SAlex Deucher * and DF related registers is not readable, seems hardcord is the 1108631cdbd2SAlex Deucher * only way to set the correct vram_width 1109631cdbd2SAlex Deucher */ 1110631cdbd2SAlex Deucher adev->gmc.vram_width = 2048; 1111631cdbd2SAlex Deucher else if (amdgpu_emu_mode != 1) 1112631cdbd2SAlex Deucher adev->gmc.vram_width = vram_width; 1113631cdbd2SAlex Deucher 1114631cdbd2SAlex Deucher if (!adev->gmc.vram_width) { 1115631cdbd2SAlex Deucher int chansize, numchan; 1116631cdbd2SAlex Deucher 1117631cdbd2SAlex Deucher /* hbm memory channel size */ 1118631cdbd2SAlex Deucher if (adev->flags & AMD_IS_APU) 1119631cdbd2SAlex Deucher chansize = 64; 1120631cdbd2SAlex Deucher else 1121631cdbd2SAlex Deucher chansize = 128; 1122631cdbd2SAlex Deucher 1123bdf84a80SJoseph Greathouse numchan = adev->df.funcs->get_hbm_channel_number(adev); 1124631cdbd2SAlex Deucher adev->gmc.vram_width = numchan * chansize; 1125631cdbd2SAlex Deucher } 1126631cdbd2SAlex Deucher 1127631cdbd2SAlex Deucher adev->gmc.vram_type = vram_type; 1128ad02e08eSOri Messinger adev->gmc.vram_vendor = vram_vendor; 1129e60f8db5SAlex Xie switch (adev->asic_type) { 1130e60f8db5SAlex Xie case CHIP_RAVEN: 11311daa2bfaSLe Ma adev->num_vmhubs = 2; 11321daa2bfaSLe Ma 11336a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 1134f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 11356a42fd6fSChristian König } else { 11366a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 11376a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 1138770d13b1SChristian König adev->gmc.translate_further = 11396a42fd6fSChristian König adev->vm_manager.num_level > 1; 11406a42fd6fSChristian König } 1141e60f8db5SAlex Xie break; 1142e60f8db5SAlex Xie case CHIP_VEGA10: 1143273a14cdSAlex Deucher case CHIP_VEGA12: 1144d96b428cSFeifei Xu case CHIP_VEGA20: 11458787ee01SHuang Rui case CHIP_RENOIR: 11461daa2bfaSLe Ma adev->num_vmhubs = 2; 11471daa2bfaSLe Ma 11488787ee01SHuang Rui 1149e60f8db5SAlex Xie /* 1150e60f8db5SAlex Xie * To fulfill 4-level page support, 1151e60f8db5SAlex Xie * vm size is 256TB (48bit), maximum size of Vega10, 1152e60f8db5SAlex Xie * block size 512 (9bit) 1153e60f8db5SAlex Xie */ 1154cdba61daSwentalou /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 1155cdba61daSwentalou if (amdgpu_sriov_vf(adev)) 1156cdba61daSwentalou amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 1157cdba61daSwentalou else 1158f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1159e60f8db5SAlex Xie break; 11603de2ff5dSLe Ma case CHIP_ARCTURUS: 1161c8a6e2a3SLe Ma adev->num_vmhubs = 3; 1162c8a6e2a3SLe Ma 11633de2ff5dSLe Ma /* Keep the vm size same with Vega20 */ 11643de2ff5dSLe Ma amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 11653de2ff5dSLe Ma break; 1166e60f8db5SAlex Xie default: 1167e60f8db5SAlex Xie break; 1168e60f8db5SAlex Xie } 1169e60f8db5SAlex Xie 1170e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 117144a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 1172770d13b1SChristian König &adev->gmc.vm_fault); 117330da7bb1SChristian König if (r) 117430da7bb1SChristian König return r; 117530da7bb1SChristian König 11767d19b15fSLe Ma if (adev->asic_type == CHIP_ARCTURUS) { 11777d19b15fSLe Ma r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, 11787d19b15fSLe Ma &adev->gmc.vm_fault); 11797d19b15fSLe Ma if (r) 11807d19b15fSLe Ma return r; 11817d19b15fSLe Ma } 11827d19b15fSLe Ma 118344a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 1184770d13b1SChristian König &adev->gmc.vm_fault); 1185e60f8db5SAlex Xie 1186e60f8db5SAlex Xie if (r) 1187e60f8db5SAlex Xie return r; 1188e60f8db5SAlex Xie 11892ee9403eSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 1190791c4769Sxinhui pan /* interrupt sent to DF. */ 1191791c4769Sxinhui pan r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 1192791c4769Sxinhui pan &adev->gmc.ecc_irq); 1193791c4769Sxinhui pan if (r) 1194791c4769Sxinhui pan return r; 11952ee9403eSZhigang Luo } 1196791c4769Sxinhui pan 1197e60f8db5SAlex Xie /* Set the internal MC address mask 1198e60f8db5SAlex Xie * This is the max address of the GPU's 1199e60f8db5SAlex Xie * internal address space. 1200e60f8db5SAlex Xie */ 1201770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 1202e60f8db5SAlex Xie 1203244511f3SChristoph Hellwig r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); 1204e60f8db5SAlex Xie if (r) { 1205e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 1206244511f3SChristoph Hellwig return r; 1207e60f8db5SAlex Xie } 1208244511f3SChristoph Hellwig adev->need_swiotlb = drm_need_swiotlb(44); 1209e60f8db5SAlex Xie 121047622ba0SAlex Deucher if (adev->gmc.xgmi.supported) { 1211bf0a60b7SAlex Deucher r = gfxhub_v1_1_get_xgmi_info(adev); 1212bf0a60b7SAlex Deucher if (r) 1213bf0a60b7SAlex Deucher return r; 1214bf0a60b7SAlex Deucher } 1215bf0a60b7SAlex Deucher 1216e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 1217e60f8db5SAlex Xie if (r) 1218e60f8db5SAlex Xie return r; 1219e60f8db5SAlex Xie 12207b885f0eSAlex Deucher amdgpu_gmc_get_vbios_allocations(adev); 1221ebdef28eSAlex Deucher 1222e60f8db5SAlex Xie /* Memory manager */ 1223e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 1224e60f8db5SAlex Xie if (r) 1225e60f8db5SAlex Xie return r; 1226e60f8db5SAlex Xie 1227e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 1228e60f8db5SAlex Xie if (r) 1229e60f8db5SAlex Xie return r; 1230e60f8db5SAlex Xie 123105ec3edaSChristian König /* 123205ec3edaSChristian König * number of VMs 123305ec3edaSChristian König * VMID 0 is reserved for System 123481659b20SFelix Kuehling * amdgpu graphics/compute will use VMIDs 1..n-1 123581659b20SFelix Kuehling * amdkfd will use VMIDs n..15 123681659b20SFelix Kuehling * 123781659b20SFelix Kuehling * The first KFD VMID is 8 for GPUs with graphics, 3 for 123881659b20SFelix Kuehling * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs 123981659b20SFelix Kuehling * for video processing. 124005ec3edaSChristian König */ 124181659b20SFelix Kuehling adev->vm_manager.first_kfd_vmid = 124281659b20SFelix Kuehling adev->asic_type == CHIP_ARCTURUS ? 3 : 8; 124305ec3edaSChristian König 124405ec3edaSChristian König amdgpu_vm_manager_init(adev); 124505ec3edaSChristian König 1246b0a2db9bSAlex Deucher gmc_v9_0_save_registers(adev); 1247b0a2db9bSAlex Deucher 124805ec3edaSChristian König return 0; 1249e60f8db5SAlex Xie } 1250e60f8db5SAlex Xie 1251e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 1252e60f8db5SAlex Xie { 1253e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1254e60f8db5SAlex Xie 12552adf1344STao Zhou amdgpu_gmc_ras_fini(adev); 1256f59548c8SMonk Liu amdgpu_gem_force_release(adev); 1257e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 1258a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 1259e60f8db5SAlex Xie amdgpu_bo_fini(adev); 1260a3d9103eSAndrey Grodzovsky amdgpu_gart_fini(adev); 1261e60f8db5SAlex Xie 1262e60f8db5SAlex Xie return 0; 1263e60f8db5SAlex Xie } 1264e60f8db5SAlex Xie 1265e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1266e60f8db5SAlex Xie { 1267946a4d5bSShaoyun Liu 1268e60f8db5SAlex Xie switch (adev->asic_type) { 1269e60f8db5SAlex Xie case CHIP_VEGA10: 12704cd4c5c0SMonk Liu if (amdgpu_sriov_vf(adev)) 127198cad2deSTrigger Huang break; 127298cad2deSTrigger Huang /* fall through */ 1273d96b428cSFeifei Xu case CHIP_VEGA20: 1274946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 12755c583018SEvan Quan golden_settings_mmhub_1_0_0, 1276c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1277946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 12785c583018SEvan Quan golden_settings_athub_1_0_0, 1279c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1280e60f8db5SAlex Xie break; 1281273a14cdSAlex Deucher case CHIP_VEGA12: 1282273a14cdSAlex Deucher break; 1283e4f3abaaSChunming Zhou case CHIP_RAVEN: 12848787ee01SHuang Rui /* TODO for renoir */ 1285946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 12865c583018SEvan Quan golden_settings_athub_1_0_0, 1287c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1288e4f3abaaSChunming Zhou break; 1289e60f8db5SAlex Xie default: 1290e60f8db5SAlex Xie break; 1291e60f8db5SAlex Xie } 1292e60f8db5SAlex Xie } 1293e60f8db5SAlex Xie 1294e60f8db5SAlex Xie /** 1295c2ecd79bSShirish S * gmc_v9_0_restore_registers - restores regs 1296c2ecd79bSShirish S * 1297c2ecd79bSShirish S * @adev: amdgpu_device pointer 1298c2ecd79bSShirish S * 1299c2ecd79bSShirish S * This restores register values, saved at suspend. 1300c2ecd79bSShirish S */ 1301b0a2db9bSAlex Deucher void gmc_v9_0_restore_registers(struct amdgpu_device *adev) 1302c2ecd79bSShirish S { 1303c2ecd79bSShirish S if (adev->asic_type == CHIP_RAVEN) 1304f8646661SAlex Deucher WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); 1305c2ecd79bSShirish S } 1306c2ecd79bSShirish S 1307c2ecd79bSShirish S /** 1308e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1309e60f8db5SAlex Xie * 1310e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1311e60f8db5SAlex Xie */ 1312e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1313e60f8db5SAlex Xie { 1314cb1545f7SOak Zeng int r; 1315e60f8db5SAlex Xie 13161123b989SChristian König if (adev->gart.bo == NULL) { 1317e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1318e60f8db5SAlex Xie return -EINVAL; 1319e60f8db5SAlex Xie } 1320ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1321ce1b1b66SMonk Liu if (r) 1322ce1b1b66SMonk Liu return r; 1323e60f8db5SAlex Xie 1324e60f8db5SAlex Xie r = gfxhub_v1_0_gart_enable(adev); 1325e60f8db5SAlex Xie if (r) 1326e60f8db5SAlex Xie return r; 1327e60f8db5SAlex Xie 13289fb1506eSOak Zeng r = adev->mmhub.funcs->gart_enable(adev); 1329e60f8db5SAlex Xie if (r) 1330e60f8db5SAlex Xie return r; 1331e60f8db5SAlex Xie 1332cb1545f7SOak Zeng DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1333cb1545f7SOak Zeng (unsigned)(adev->gmc.gart_size >> 20), 1334cb1545f7SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1335cb1545f7SOak Zeng adev->gart.ready = true; 1336cb1545f7SOak Zeng return 0; 1337cb1545f7SOak Zeng } 1338cb1545f7SOak Zeng 1339cb1545f7SOak Zeng static int gmc_v9_0_hw_init(void *handle) 1340cb1545f7SOak Zeng { 1341cb1545f7SOak Zeng struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1342cb1545f7SOak Zeng bool value; 1343cb1545f7SOak Zeng int r, i; 1344cb1545f7SOak Zeng u32 tmp; 1345cb1545f7SOak Zeng 1346cb1545f7SOak Zeng /* The sequence of these two function calls matters.*/ 1347cb1545f7SOak Zeng gmc_v9_0_init_golden_registers(adev); 1348cb1545f7SOak Zeng 1349cb1545f7SOak Zeng if (adev->mode_info.num_crtc) { 1350cb1545f7SOak Zeng if (adev->asic_type != CHIP_ARCTURUS) { 1351cb1545f7SOak Zeng /* Lockout access through VGA aperture*/ 1352cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1353cb1545f7SOak Zeng 1354cb1545f7SOak Zeng /* disable VGA render */ 1355cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1356cb1545f7SOak Zeng } 1357cb1545f7SOak Zeng } 1358cb1545f7SOak Zeng 1359cb1545f7SOak Zeng amdgpu_device_program_register_sequence(adev, 1360cb1545f7SOak Zeng golden_settings_vega10_hdp, 1361cb1545f7SOak Zeng ARRAY_SIZE(golden_settings_vega10_hdp)); 1362cb1545f7SOak Zeng 13639fb1506eSOak Zeng if (adev->mmhub.funcs->update_power_gating) 13649fb1506eSOak Zeng adev->mmhub.funcs->update_power_gating(adev, true); 13659fb1506eSOak Zeng 1366cb1545f7SOak Zeng switch (adev->asic_type) { 1367f81b86a0SOak Zeng case CHIP_ARCTURUS: 1368f81b86a0SOak Zeng WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); 1369f81b86a0SOak Zeng break; 1370cb1545f7SOak Zeng default: 1371cb1545f7SOak Zeng break; 1372cb1545f7SOak Zeng } 1373cb1545f7SOak Zeng 1374846347c9STom St Denis WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1375e60f8db5SAlex Xie 1376b9509c80SHuang Rui tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1377b9509c80SHuang Rui WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1378e60f8db5SAlex Xie 1379fe2b5323STiecheng Zhou WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); 1380fe2b5323STiecheng Zhou WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); 1381fe2b5323STiecheng Zhou 13821d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 1383bebc0762SHawking Zhang adev->nbio.funcs->hdp_flush(adev, NULL); 13841d4e0a8cSMonk Liu 1385e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1386e60f8db5SAlex Xie value = false; 1387e60f8db5SAlex Xie else 1388e60f8db5SAlex Xie value = true; 1389e60f8db5SAlex Xie 139020bf2f6fSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 139108546895SZhigang Luo gfxhub_v1_0_set_fault_enable_default(adev, value); 13929fb1506eSOak Zeng adev->mmhub.funcs->set_fault_enable_default(adev, value); 139320bf2f6fSZhigang Luo } 13943ff98548SOak Zeng for (i = 0; i < adev->num_vmhubs; ++i) 13953ff98548SOak Zeng gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); 1396e60f8db5SAlex Xie 1397e7da754bSMonk Liu if (adev->umc.funcs && adev->umc.funcs->init_registers) 1398e7da754bSMonk Liu adev->umc.funcs->init_registers(adev); 1399e7da754bSMonk Liu 1400e60f8db5SAlex Xie r = gmc_v9_0_gart_enable(adev); 1401e60f8db5SAlex Xie 1402e60f8db5SAlex Xie return r; 1403e60f8db5SAlex Xie } 1404e60f8db5SAlex Xie 1405e60f8db5SAlex Xie /** 1406e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1407e60f8db5SAlex Xie * 1408e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1409e60f8db5SAlex Xie * 1410e60f8db5SAlex Xie * This disables all VM page table. 1411e60f8db5SAlex Xie */ 1412e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1413e60f8db5SAlex Xie { 1414e60f8db5SAlex Xie gfxhub_v1_0_gart_disable(adev); 14159fb1506eSOak Zeng adev->mmhub.funcs->gart_disable(adev); 1416ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1417e60f8db5SAlex Xie } 1418e60f8db5SAlex Xie 1419e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1420e60f8db5SAlex Xie { 1421e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1422e60f8db5SAlex Xie 14235dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 14245dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 14255dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 14265dd696aeSTrigger Huang return 0; 14275dd696aeSTrigger Huang } 14285dd696aeSTrigger Huang 1429791c4769Sxinhui pan amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1430770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1431e60f8db5SAlex Xie gmc_v9_0_gart_disable(adev); 1432e60f8db5SAlex Xie 1433e60f8db5SAlex Xie return 0; 1434e60f8db5SAlex Xie } 1435e60f8db5SAlex Xie 1436e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1437e60f8db5SAlex Xie { 1438c2ecd79bSShirish S int r; 1439e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1440e60f8db5SAlex Xie 1441c2ecd79bSShirish S r = gmc_v9_0_hw_fini(adev); 1442c2ecd79bSShirish S if (r) 1443c2ecd79bSShirish S return r; 1444c2ecd79bSShirish S 1445c2ecd79bSShirish S return 0; 1446e60f8db5SAlex Xie } 1447e60f8db5SAlex Xie 1448e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1449e60f8db5SAlex Xie { 1450e60f8db5SAlex Xie int r; 1451e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1452e60f8db5SAlex Xie 1453e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1454e60f8db5SAlex Xie if (r) 1455e60f8db5SAlex Xie return r; 1456e60f8db5SAlex Xie 1457620f774fSChristian König amdgpu_vmid_reset_all(adev); 1458e60f8db5SAlex Xie 145932601d48SChristian König return 0; 1460e60f8db5SAlex Xie } 1461e60f8db5SAlex Xie 1462e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1463e60f8db5SAlex Xie { 1464e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1465e60f8db5SAlex Xie return true; 1466e60f8db5SAlex Xie } 1467e60f8db5SAlex Xie 1468e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1469e60f8db5SAlex Xie { 1470e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1471e60f8db5SAlex Xie return 0; 1472e60f8db5SAlex Xie } 1473e60f8db5SAlex Xie 1474e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1475e60f8db5SAlex Xie { 1476e60f8db5SAlex Xie /* XXX for emulation.*/ 1477e60f8db5SAlex Xie return 0; 1478e60f8db5SAlex Xie } 1479e60f8db5SAlex Xie 1480e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1481e60f8db5SAlex Xie enum amd_clockgating_state state) 1482e60f8db5SAlex Xie { 1483d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1484d5583d4fSHuang Rui 14859fb1506eSOak Zeng adev->mmhub.funcs->set_clockgating(adev, state); 1486bee7b51aSLe Ma 1487bee7b51aSLe Ma athub_v1_0_set_clockgating(adev, state); 1488bee7b51aSLe Ma 1489bee7b51aSLe Ma return 0; 1490e60f8db5SAlex Xie } 1491e60f8db5SAlex Xie 149213052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 149313052be5SHuang Rui { 149413052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 149513052be5SHuang Rui 14969fb1506eSOak Zeng adev->mmhub.funcs->get_clockgating(adev, flags); 1497bee7b51aSLe Ma 1498bee7b51aSLe Ma athub_v1_0_get_clockgating(adev, flags); 149913052be5SHuang Rui } 150013052be5SHuang Rui 1501e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1502e60f8db5SAlex Xie enum amd_powergating_state state) 1503e60f8db5SAlex Xie { 1504e60f8db5SAlex Xie return 0; 1505e60f8db5SAlex Xie } 1506e60f8db5SAlex Xie 1507e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1508e60f8db5SAlex Xie .name = "gmc_v9_0", 1509e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1510e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1511e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1512e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1513e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1514e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1515e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1516e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1517e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1518e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1519e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1520e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1521e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 152213052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1523e60f8db5SAlex Xie }; 1524e60f8db5SAlex Xie 1525e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1526e60f8db5SAlex Xie { 1527e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1528e60f8db5SAlex Xie .major = 9, 1529e60f8db5SAlex Xie .minor = 0, 1530e60f8db5SAlex Xie .rev = 0, 1531e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1532e60f8db5SAlex Xie }; 1533