1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23f867723bSSam Ravnborg 24e60f8db5SAlex Xie #include <linux/firmware.h> 25f867723bSSam Ravnborg #include <linux/pci.h> 26f867723bSSam Ravnborg 27fd5fd480SChunming Zhou #include <drm/drm_cache.h> 28f867723bSSam Ravnborg 29e60f8db5SAlex Xie #include "amdgpu.h" 30e60f8db5SAlex Xie #include "gmc_v9_0.h" 318d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 322cddc50eSHuang Rui #include "amdgpu_gem.h" 33e60f8db5SAlex Xie 3475199b8cSFeifei Xu #include "hdp/hdp_4_0_offset.h" 3575199b8cSFeifei Xu #include "hdp/hdp_4_0_sh_mask.h" 36cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 37135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 38135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 39fb960bd2SFeifei Xu #include "vega10_enum.h" 4065417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 41ea930000SAlex Sierra #include "athub/athub_1_0_sh_mask.h" 426ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 43250b4228SChristian König #include "oss/osssys_4_0_offset.h" 44e60f8db5SAlex Xie 45946a4d5bSShaoyun Liu #include "soc15.h" 46ea930000SAlex Sierra #include "soc15d.h" 47e60f8db5SAlex Xie #include "soc15_common.h" 4890c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 49e60f8db5SAlex Xie 50e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 51e60f8db5SAlex Xie #include "mmhub_v1_0.h" 52bee7b51aSLe Ma #include "athub_v1_0.h" 53bf0a60b7SAlex Deucher #include "gfxhub_v1_1.h" 5451cce480SLe Ma #include "mmhub_v9_4.h" 555b6b35aaSHawking Zhang #include "umc_v6_1.h" 56e7da754bSMonk Liu #include "umc_v6_0.h" 57e60f8db5SAlex Xie 5844a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 5944a99b65SAndrey Grodzovsky 60791c4769Sxinhui pan #include "amdgpu_ras.h" 61029fbd43SHawking Zhang #include "amdgpu_xgmi.h" 62791c4769Sxinhui pan 63ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 64ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 65ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 66ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 67ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 68ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 69ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 70ebdef28eSAlex Deucher 71e60f8db5SAlex Xie static const u32 golden_settings_vega10_hdp[] = 72e60f8db5SAlex Xie { 73e60f8db5SAlex Xie 0xf64, 0x0fffffff, 0x00000000, 74e60f8db5SAlex Xie 0xf65, 0x0fffffff, 0x00000000, 75e60f8db5SAlex Xie 0xf66, 0x0fffffff, 0x00000000, 76e60f8db5SAlex Xie 0xf67, 0x0fffffff, 0x00000000, 77e60f8db5SAlex Xie 0xf68, 0x0fffffff, 0x00000000, 78e60f8db5SAlex Xie 0xf6a, 0x0fffffff, 0x00000000, 79e60f8db5SAlex Xie 0xf6b, 0x0fffffff, 0x00000000, 80e60f8db5SAlex Xie 0xf6c, 0x0fffffff, 0x00000000, 81e60f8db5SAlex Xie 0xf6d, 0x0fffffff, 0x00000000, 82e60f8db5SAlex Xie 0xf6e, 0x0fffffff, 0x00000000, 83e60f8db5SAlex Xie }; 84e60f8db5SAlex Xie 85946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 865c583018SEvan Quan { 87946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 88946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 895c583018SEvan Quan }; 905c583018SEvan Quan 91946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 925c583018SEvan Quan { 93946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 94946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 955c583018SEvan Quan }; 965c583018SEvan Quan 97791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 98791c4769Sxinhui pan (0x000143c0 + 0x00000000), 99791c4769Sxinhui pan (0x000143c0 + 0x00000800), 100791c4769Sxinhui pan (0x000143c0 + 0x00001000), 101791c4769Sxinhui pan (0x000143c0 + 0x00001800), 102791c4769Sxinhui pan (0x000543c0 + 0x00000000), 103791c4769Sxinhui pan (0x000543c0 + 0x00000800), 104791c4769Sxinhui pan (0x000543c0 + 0x00001000), 105791c4769Sxinhui pan (0x000543c0 + 0x00001800), 106791c4769Sxinhui pan (0x000943c0 + 0x00000000), 107791c4769Sxinhui pan (0x000943c0 + 0x00000800), 108791c4769Sxinhui pan (0x000943c0 + 0x00001000), 109791c4769Sxinhui pan (0x000943c0 + 0x00001800), 110791c4769Sxinhui pan (0x000d43c0 + 0x00000000), 111791c4769Sxinhui pan (0x000d43c0 + 0x00000800), 112791c4769Sxinhui pan (0x000d43c0 + 0x00001000), 113791c4769Sxinhui pan (0x000d43c0 + 0x00001800), 114791c4769Sxinhui pan (0x001143c0 + 0x00000000), 115791c4769Sxinhui pan (0x001143c0 + 0x00000800), 116791c4769Sxinhui pan (0x001143c0 + 0x00001000), 117791c4769Sxinhui pan (0x001143c0 + 0x00001800), 118791c4769Sxinhui pan (0x001543c0 + 0x00000000), 119791c4769Sxinhui pan (0x001543c0 + 0x00000800), 120791c4769Sxinhui pan (0x001543c0 + 0x00001000), 121791c4769Sxinhui pan (0x001543c0 + 0x00001800), 122791c4769Sxinhui pan (0x001943c0 + 0x00000000), 123791c4769Sxinhui pan (0x001943c0 + 0x00000800), 124791c4769Sxinhui pan (0x001943c0 + 0x00001000), 125791c4769Sxinhui pan (0x001943c0 + 0x00001800), 126791c4769Sxinhui pan (0x001d43c0 + 0x00000000), 127791c4769Sxinhui pan (0x001d43c0 + 0x00000800), 128791c4769Sxinhui pan (0x001d43c0 + 0x00001000), 129791c4769Sxinhui pan (0x001d43c0 + 0x00001800), 13002bab923SDavid Panariti }; 13102bab923SDavid Panariti 132791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 133791c4769Sxinhui pan (0x000143e0 + 0x00000000), 134791c4769Sxinhui pan (0x000143e0 + 0x00000800), 135791c4769Sxinhui pan (0x000143e0 + 0x00001000), 136791c4769Sxinhui pan (0x000143e0 + 0x00001800), 137791c4769Sxinhui pan (0x000543e0 + 0x00000000), 138791c4769Sxinhui pan (0x000543e0 + 0x00000800), 139791c4769Sxinhui pan (0x000543e0 + 0x00001000), 140791c4769Sxinhui pan (0x000543e0 + 0x00001800), 141791c4769Sxinhui pan (0x000943e0 + 0x00000000), 142791c4769Sxinhui pan (0x000943e0 + 0x00000800), 143791c4769Sxinhui pan (0x000943e0 + 0x00001000), 144791c4769Sxinhui pan (0x000943e0 + 0x00001800), 145791c4769Sxinhui pan (0x000d43e0 + 0x00000000), 146791c4769Sxinhui pan (0x000d43e0 + 0x00000800), 147791c4769Sxinhui pan (0x000d43e0 + 0x00001000), 148791c4769Sxinhui pan (0x000d43e0 + 0x00001800), 149791c4769Sxinhui pan (0x001143e0 + 0x00000000), 150791c4769Sxinhui pan (0x001143e0 + 0x00000800), 151791c4769Sxinhui pan (0x001143e0 + 0x00001000), 152791c4769Sxinhui pan (0x001143e0 + 0x00001800), 153791c4769Sxinhui pan (0x001543e0 + 0x00000000), 154791c4769Sxinhui pan (0x001543e0 + 0x00000800), 155791c4769Sxinhui pan (0x001543e0 + 0x00001000), 156791c4769Sxinhui pan (0x001543e0 + 0x00001800), 157791c4769Sxinhui pan (0x001943e0 + 0x00000000), 158791c4769Sxinhui pan (0x001943e0 + 0x00000800), 159791c4769Sxinhui pan (0x001943e0 + 0x00001000), 160791c4769Sxinhui pan (0x001943e0 + 0x00001800), 161791c4769Sxinhui pan (0x001d43e0 + 0x00000000), 162791c4769Sxinhui pan (0x001d43e0 + 0x00000800), 163791c4769Sxinhui pan (0x001d43e0 + 0x00001000), 164791c4769Sxinhui pan (0x001d43e0 + 0x00001800), 16502bab923SDavid Panariti }; 16602bab923SDavid Panariti 167791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_status_addrs[] = { 168791c4769Sxinhui pan (0x000143c2 + 0x00000000), 169791c4769Sxinhui pan (0x000143c2 + 0x00000800), 170791c4769Sxinhui pan (0x000143c2 + 0x00001000), 171791c4769Sxinhui pan (0x000143c2 + 0x00001800), 172791c4769Sxinhui pan (0x000543c2 + 0x00000000), 173791c4769Sxinhui pan (0x000543c2 + 0x00000800), 174791c4769Sxinhui pan (0x000543c2 + 0x00001000), 175791c4769Sxinhui pan (0x000543c2 + 0x00001800), 176791c4769Sxinhui pan (0x000943c2 + 0x00000000), 177791c4769Sxinhui pan (0x000943c2 + 0x00000800), 178791c4769Sxinhui pan (0x000943c2 + 0x00001000), 179791c4769Sxinhui pan (0x000943c2 + 0x00001800), 180791c4769Sxinhui pan (0x000d43c2 + 0x00000000), 181791c4769Sxinhui pan (0x000d43c2 + 0x00000800), 182791c4769Sxinhui pan (0x000d43c2 + 0x00001000), 183791c4769Sxinhui pan (0x000d43c2 + 0x00001800), 184791c4769Sxinhui pan (0x001143c2 + 0x00000000), 185791c4769Sxinhui pan (0x001143c2 + 0x00000800), 186791c4769Sxinhui pan (0x001143c2 + 0x00001000), 187791c4769Sxinhui pan (0x001143c2 + 0x00001800), 188791c4769Sxinhui pan (0x001543c2 + 0x00000000), 189791c4769Sxinhui pan (0x001543c2 + 0x00000800), 190791c4769Sxinhui pan (0x001543c2 + 0x00001000), 191791c4769Sxinhui pan (0x001543c2 + 0x00001800), 192791c4769Sxinhui pan (0x001943c2 + 0x00000000), 193791c4769Sxinhui pan (0x001943c2 + 0x00000800), 194791c4769Sxinhui pan (0x001943c2 + 0x00001000), 195791c4769Sxinhui pan (0x001943c2 + 0x00001800), 196791c4769Sxinhui pan (0x001d43c2 + 0x00000000), 197791c4769Sxinhui pan (0x001d43c2 + 0x00000800), 198791c4769Sxinhui pan (0x001d43c2 + 0x00001000), 199791c4769Sxinhui pan (0x001d43c2 + 0x00001800), 20002bab923SDavid Panariti }; 20102bab923SDavid Panariti 202791c4769Sxinhui pan static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 203791c4769Sxinhui pan struct amdgpu_irq_src *src, 204791c4769Sxinhui pan unsigned type, 205791c4769Sxinhui pan enum amdgpu_interrupt_state state) 206791c4769Sxinhui pan { 207791c4769Sxinhui pan u32 bits, i, tmp, reg; 208791c4769Sxinhui pan 2091e2c6d55SJohn Clements /* Devices newer then VEGA10/12 shall have these programming 2101e2c6d55SJohn Clements sequences performed by PSP BL */ 2111e2c6d55SJohn Clements if (adev->asic_type >= CHIP_VEGA20) 2121e2c6d55SJohn Clements return 0; 2131e2c6d55SJohn Clements 214791c4769Sxinhui pan bits = 0x7f; 215791c4769Sxinhui pan 216791c4769Sxinhui pan switch (state) { 217791c4769Sxinhui pan case AMDGPU_IRQ_STATE_DISABLE: 218791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 219791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 220791c4769Sxinhui pan tmp = RREG32(reg); 221791c4769Sxinhui pan tmp &= ~bits; 222791c4769Sxinhui pan WREG32(reg, tmp); 223791c4769Sxinhui pan } 224791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 225791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 226791c4769Sxinhui pan tmp = RREG32(reg); 227791c4769Sxinhui pan tmp &= ~bits; 228791c4769Sxinhui pan WREG32(reg, tmp); 229791c4769Sxinhui pan } 230791c4769Sxinhui pan break; 231791c4769Sxinhui pan case AMDGPU_IRQ_STATE_ENABLE: 232791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 233791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 234791c4769Sxinhui pan tmp = RREG32(reg); 235791c4769Sxinhui pan tmp |= bits; 236791c4769Sxinhui pan WREG32(reg, tmp); 237791c4769Sxinhui pan } 238791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 239791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 240791c4769Sxinhui pan tmp = RREG32(reg); 241791c4769Sxinhui pan tmp |= bits; 242791c4769Sxinhui pan WREG32(reg, tmp); 243791c4769Sxinhui pan } 244791c4769Sxinhui pan break; 245791c4769Sxinhui pan default: 246791c4769Sxinhui pan break; 247791c4769Sxinhui pan } 248791c4769Sxinhui pan 249791c4769Sxinhui pan return 0; 250791c4769Sxinhui pan } 251791c4769Sxinhui pan 252e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 253e60f8db5SAlex Xie struct amdgpu_irq_src *src, 254e60f8db5SAlex Xie unsigned type, 255e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 256e60f8db5SAlex Xie { 257e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 258ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 259e60f8db5SAlex Xie 26011250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26111250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26211250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26311250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26411250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26511250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 26611250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 26711250164SChristian König 268e60f8db5SAlex Xie switch (state) { 269e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 2701daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 271ae6d1416STom St Denis hub = &adev->vmhub[j]; 272e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 273e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 274e60f8db5SAlex Xie tmp = RREG32(reg); 275e60f8db5SAlex Xie tmp &= ~bits; 276e60f8db5SAlex Xie WREG32(reg, tmp); 277e60f8db5SAlex Xie } 278e60f8db5SAlex Xie } 279e60f8db5SAlex Xie break; 280e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 2811daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 282ae6d1416STom St Denis hub = &adev->vmhub[j]; 283e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 284e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 285e60f8db5SAlex Xie tmp = RREG32(reg); 286e60f8db5SAlex Xie tmp |= bits; 287e60f8db5SAlex Xie WREG32(reg, tmp); 288e60f8db5SAlex Xie } 289e60f8db5SAlex Xie } 290e60f8db5SAlex Xie default: 291e60f8db5SAlex Xie break; 292e60f8db5SAlex Xie } 293e60f8db5SAlex Xie 294e60f8db5SAlex Xie return 0; 295e60f8db5SAlex Xie } 296e60f8db5SAlex Xie 297e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 298e60f8db5SAlex Xie struct amdgpu_irq_src *source, 299e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 300e60f8db5SAlex Xie { 30151c60898SLe Ma struct amdgpu_vmhub *hub; 302c468f9e2SChristian König bool retry_fault = !!(entry->src_data[1] & 0x80); 3034d6cbde3SFelix Kuehling uint32_t status = 0; 304e60f8db5SAlex Xie u64 addr; 30551c60898SLe Ma char hub_name[10]; 306e60f8db5SAlex Xie 307e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 308e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 309e60f8db5SAlex Xie 310c1a8abd9SChristian König if (retry_fault && amdgpu_gmc_filter_faults(adev, addr, entry->pasid, 311c1a8abd9SChristian König entry->timestamp)) 31222666cc1SChristian König return 1; /* This also prevents sending it to KFD */ 31322666cc1SChristian König 31451c60898SLe Ma if (entry->client_id == SOC15_IH_CLIENTID_VMC) { 31551c60898SLe Ma snprintf(hub_name, sizeof(hub_name), "mmhub0"); 31651c60898SLe Ma hub = &adev->vmhub[AMDGPU_MMHUB_0]; 31751c60898SLe Ma } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { 31851c60898SLe Ma snprintf(hub_name, sizeof(hub_name), "mmhub1"); 31951c60898SLe Ma hub = &adev->vmhub[AMDGPU_MMHUB_1]; 32051c60898SLe Ma } else { 32151c60898SLe Ma snprintf(hub_name, sizeof(hub_name), "gfxhub0"); 32251c60898SLe Ma hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 32351c60898SLe Ma } 32451c60898SLe Ma 325c1a8abd9SChristian König /* If it's the first fault for this address, process it normally */ 326ec671737SChristian König if (retry_fault && !in_interrupt() && 327ec671737SChristian König amdgpu_vm_handle_fault(adev, entry->pasid, addr)) 328ec671737SChristian König return 1; /* This also prevents sending it to KFD */ 329ec671737SChristian König 33079a0c465SMonk Liu if (!amdgpu_sriov_vf(adev)) { 33153499173SXiaojie Yuan /* 33253499173SXiaojie Yuan * Issue a dummy read to wait for the status register to 33353499173SXiaojie Yuan * be updated to avoid reading an incorrect value due to 33453499173SXiaojie Yuan * the new fast GRBM interface. 33553499173SXiaojie Yuan */ 33653499173SXiaojie Yuan if (entry->vmid_src == AMDGPU_GFXHUB_0) 33753499173SXiaojie Yuan RREG32(hub->vm_l2_pro_fault_status); 33853499173SXiaojie Yuan 3395a9b8e8aSChristian König status = RREG32(hub->vm_l2_pro_fault_status); 3405a9b8e8aSChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 3414d6cbde3SFelix Kuehling } 342e60f8db5SAlex Xie 3434d6cbde3SFelix Kuehling if (printk_ratelimit()) { 34405794effSShirish S struct amdgpu_task_info task_info; 345efaa9646SAndrey Grodzovsky 34605794effSShirish S memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 347efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 348efaa9646SAndrey Grodzovsky 3494d6cbde3SFelix Kuehling dev_err(adev->dev, 350c468f9e2SChristian König "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 351c468f9e2SChristian König "pasid:%u, for process %s pid %d thread %s pid %d)\n", 35251c60898SLe Ma hub_name, retry_fault ? "retry" : "no-retry", 353c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 354efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 355efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 3565ddd4a9aSYong Zhao dev_err(adev->dev, " in page starting at address 0x%016llx from client %d\n", 35779a0c465SMonk Liu addr, entry->client_id); 3585ddd4a9aSYong Zhao if (!amdgpu_sriov_vf(adev)) { 3594d6cbde3SFelix Kuehling dev_err(adev->dev, 3604d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 3614d6cbde3SFelix Kuehling status); 362d2155a71SOak Zeng dev_err(adev->dev, "\t Faulty UTCL2 client ID: 0x%lx\n", 363d2155a71SOak Zeng REG_GET_FIELD(status, 364d2155a71SOak Zeng VM_L2_PROTECTION_FAULT_STATUS, CID)); 3655ddd4a9aSYong Zhao dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 3665ddd4a9aSYong Zhao REG_GET_FIELD(status, 3675ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 3685ddd4a9aSYong Zhao dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 3695ddd4a9aSYong Zhao REG_GET_FIELD(status, 3705ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 3715ddd4a9aSYong Zhao dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 3725ddd4a9aSYong Zhao REG_GET_FIELD(status, 3735ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 3745ddd4a9aSYong Zhao dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 3755ddd4a9aSYong Zhao REG_GET_FIELD(status, 3765ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 3774e0ae5e2SYong Zhao dev_err(adev->dev, "\t RW: 0x%lx\n", 3784e0ae5e2SYong Zhao REG_GET_FIELD(status, 3794e0ae5e2SYong Zhao VM_L2_PROTECTION_FAULT_STATUS, RW)); 3805ddd4a9aSYong Zhao 3815ddd4a9aSYong Zhao } 38279a0c465SMonk Liu } 383e60f8db5SAlex Xie 384e60f8db5SAlex Xie return 0; 385e60f8db5SAlex Xie } 386e60f8db5SAlex Xie 387e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 388e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 389e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 390e60f8db5SAlex Xie }; 391e60f8db5SAlex Xie 392791c4769Sxinhui pan 393791c4769Sxinhui pan static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 394791c4769Sxinhui pan .set = gmc_v9_0_ecc_interrupt_state, 39534cc4fd9STao Zhou .process = amdgpu_umc_process_ecc_irq, 396791c4769Sxinhui pan }; 397791c4769Sxinhui pan 398e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 399e60f8db5SAlex Xie { 400770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 401770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 402791c4769Sxinhui pan 4032ee9403eSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 404791c4769Sxinhui pan adev->gmc.ecc_irq.num_types = 1; 405791c4769Sxinhui pan adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 406e60f8db5SAlex Xie } 4072ee9403eSZhigang Luo } 408e60f8db5SAlex Xie 4092a79d868SYong Zhao static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 4102a79d868SYong Zhao uint32_t flush_type) 41103f89febSChristian König { 41203f89febSChristian König u32 req = 0; 41303f89febSChristian König 41403f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 415c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 4162a79d868SYong Zhao req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 41703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 41803f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 41903f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 42003f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 42103f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 42203f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 42303f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 42403f89febSChristian König 42503f89febSChristian König return req; 42603f89febSChristian König } 42703f89febSChristian König 42890f6452cSchangzhu /** 42990f6452cSchangzhu * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore 43090f6452cSchangzhu * 43190f6452cSchangzhu * @adev: amdgpu_device pointer 43290f6452cSchangzhu * @vmhub: vmhub type 43390f6452cSchangzhu * 43490f6452cSchangzhu */ 43590f6452cSchangzhu static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, 43690f6452cSchangzhu uint32_t vmhub) 43790f6452cSchangzhu { 43890f6452cSchangzhu return ((vmhub == AMDGPU_MMHUB_0 || 43990f6452cSchangzhu vmhub == AMDGPU_MMHUB_1) && 44090f6452cSchangzhu (!amdgpu_sriov_vf(adev)) && 44154f78a76SAlex Deucher (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && 44254f78a76SAlex Deucher (adev->apu_flags & AMD_APU_IS_PICASSO)))); 44390f6452cSchangzhu } 44490f6452cSchangzhu 445ea930000SAlex Sierra static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, 446ea930000SAlex Sierra uint8_t vmid, uint16_t *p_pasid) 447ea930000SAlex Sierra { 448ea930000SAlex Sierra uint32_t value; 449ea930000SAlex Sierra 450ea930000SAlex Sierra value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 451ea930000SAlex Sierra + vmid); 452ea930000SAlex Sierra *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 453ea930000SAlex Sierra 454ea930000SAlex Sierra return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 455ea930000SAlex Sierra } 456ea930000SAlex Sierra 457e60f8db5SAlex Xie /* 458e60f8db5SAlex Xie * GART 459e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 460e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 461e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 462e60f8db5SAlex Xie */ 463e60f8db5SAlex Xie 464e60f8db5SAlex Xie /** 4652a79d868SYong Zhao * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 466e60f8db5SAlex Xie * 467e60f8db5SAlex Xie * @adev: amdgpu_device pointer 468e60f8db5SAlex Xie * @vmid: vm instance to flush 4692a79d868SYong Zhao * @flush_type: the flush type 470e60f8db5SAlex Xie * 4712a79d868SYong Zhao * Flush the TLB for the requested page table using certain type. 472e60f8db5SAlex Xie */ 4733ff98548SOak Zeng static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 4743ff98548SOak Zeng uint32_t vmhub, uint32_t flush_type) 475e60f8db5SAlex Xie { 47690f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); 477e60f8db5SAlex Xie const unsigned eng = 17; 478b80cd524SFelix Kuehling u32 j, inv_req, inv_req2, tmp; 4793ff98548SOak Zeng struct amdgpu_vmhub *hub; 480e60f8db5SAlex Xie 4813ff98548SOak Zeng BUG_ON(vmhub >= adev->num_vmhubs); 4823ff98548SOak Zeng 4833ff98548SOak Zeng hub = &adev->vmhub[vmhub]; 484b80cd524SFelix Kuehling if (adev->gmc.xgmi.num_physical_nodes && 485b80cd524SFelix Kuehling adev->asic_type == CHIP_VEGA20) { 486b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 487b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 488b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 489b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 490b80cd524SFelix Kuehling * still need a second TLB flush after this. 491b80cd524SFelix Kuehling */ 492b80cd524SFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); 493b80cd524SFelix Kuehling inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 494b80cd524SFelix Kuehling } else { 49537c58ddfSFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); 496b80cd524SFelix Kuehling inv_req2 = 0; 497b80cd524SFelix Kuehling } 498e60f8db5SAlex Xie 49982d1a1b1SChengming Gui /* This is necessary for a HW workaround under SRIOV as well 50082d1a1b1SChengming Gui * as GFXOFF under bare metal 50182d1a1b1SChengming Gui */ 50282d1a1b1SChengming Gui if (adev->gfx.kiq.ring.sched.ready && 50382d1a1b1SChengming Gui (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 50453b3f8f4SDennis Li !amdgpu_in_reset(adev)) { 505148f597dSHuang Rui uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 506148f597dSHuang Rui uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 507af5fe1e9SChristian König 50837c58ddfSFelix Kuehling amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 509af5fe1e9SChristian König 1 << vmid); 5103ff98548SOak Zeng return; 511fc0faf04SEmily Deng } 5123890d111SEmily Deng 5133890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 514f920d1bbSchangzhu 515f920d1bbSchangzhu /* 516f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 517f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 518f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 519f920d1bbSchangzhu * to WA the Issue 520f920d1bbSchangzhu */ 521f920d1bbSchangzhu 522f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 52390f6452cSchangzhu if (use_semaphore) { 524f920d1bbSchangzhu for (j = 0; j < adev->usec_timeout; j++) { 525f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 526148f597dSHuang Rui tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + 527148f597dSHuang Rui hub->eng_distance * eng); 528f920d1bbSchangzhu if (tmp & 0x1) 529f920d1bbSchangzhu break; 530f920d1bbSchangzhu udelay(1); 531f920d1bbSchangzhu } 532f920d1bbSchangzhu 533f920d1bbSchangzhu if (j >= adev->usec_timeout) 534f920d1bbSchangzhu DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 535f920d1bbSchangzhu } 536f920d1bbSchangzhu 537b80cd524SFelix Kuehling do { 538148f597dSHuang Rui WREG32_NO_KIQ(hub->vm_inv_eng0_req + 539148f597dSHuang Rui hub->eng_distance * eng, inv_req); 54053499173SXiaojie Yuan 54153499173SXiaojie Yuan /* 542b80cd524SFelix Kuehling * Issue a dummy read to wait for the ACK register to 543b80cd524SFelix Kuehling * be cleared to avoid a false ACK due to the new fast 544b80cd524SFelix Kuehling * GRBM interface. 54553499173SXiaojie Yuan */ 54653499173SXiaojie Yuan if (vmhub == AMDGPU_GFXHUB_0) 547148f597dSHuang Rui RREG32_NO_KIQ(hub->vm_inv_eng0_req + 548148f597dSHuang Rui hub->eng_distance * eng); 54953499173SXiaojie Yuan 550e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 551148f597dSHuang Rui tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + 552148f597dSHuang Rui hub->eng_distance * eng); 553396557b0SChristian König if (tmp & (1 << vmid)) 554e60f8db5SAlex Xie break; 555e60f8db5SAlex Xie udelay(1); 556e60f8db5SAlex Xie } 557f920d1bbSchangzhu 558b80cd524SFelix Kuehling inv_req = inv_req2; 559b80cd524SFelix Kuehling inv_req2 = 0; 560b80cd524SFelix Kuehling } while (inv_req); 561b80cd524SFelix Kuehling 562f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 56390f6452cSchangzhu if (use_semaphore) 564f920d1bbSchangzhu /* 565f920d1bbSchangzhu * add semaphore release after invalidation, 566f920d1bbSchangzhu * write with 0 means semaphore release 567f920d1bbSchangzhu */ 568148f597dSHuang Rui WREG32_NO_KIQ(hub->vm_inv_eng0_sem + 569148f597dSHuang Rui hub->eng_distance * eng, 0); 570f920d1bbSchangzhu 5713890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 572f920d1bbSchangzhu 573396557b0SChristian König if (j < adev->usec_timeout) 5743ff98548SOak Zeng return; 575396557b0SChristian König 576e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 577e60f8db5SAlex Xie } 578e60f8db5SAlex Xie 579ea930000SAlex Sierra /** 580ea930000SAlex Sierra * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid 581ea930000SAlex Sierra * 582ea930000SAlex Sierra * @adev: amdgpu_device pointer 583ea930000SAlex Sierra * @pasid: pasid to be flush 584ea930000SAlex Sierra * 585ea930000SAlex Sierra * Flush the TLB for the requested pasid. 586ea930000SAlex Sierra */ 587ea930000SAlex Sierra static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 588ea930000SAlex Sierra uint16_t pasid, uint32_t flush_type, 589ea930000SAlex Sierra bool all_hub) 590ea930000SAlex Sierra { 591ea930000SAlex Sierra int vmid, i; 592ea930000SAlex Sierra signed long r; 593ea930000SAlex Sierra uint32_t seq; 594ea930000SAlex Sierra uint16_t queried_pasid; 595ea930000SAlex Sierra bool ret; 596ea930000SAlex Sierra struct amdgpu_ring *ring = &adev->gfx.kiq.ring; 597ea930000SAlex Sierra struct amdgpu_kiq *kiq = &adev->gfx.kiq; 598ea930000SAlex Sierra 59953b3f8f4SDennis Li if (amdgpu_in_reset(adev)) 600ea930000SAlex Sierra return -EIO; 601ea930000SAlex Sierra 602ea930000SAlex Sierra if (ring->sched.ready) { 603b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 604b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 605b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 606b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 607b80cd524SFelix Kuehling * still need a second TLB flush after this. 608b80cd524SFelix Kuehling */ 609b80cd524SFelix Kuehling bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && 610b80cd524SFelix Kuehling adev->asic_type == CHIP_VEGA20); 611b80cd524SFelix Kuehling /* 2 dwords flush + 8 dwords fence */ 612b80cd524SFelix Kuehling unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; 613b80cd524SFelix Kuehling 614b80cd524SFelix Kuehling if (vega20_xgmi_wa) 615b80cd524SFelix Kuehling ndw += kiq->pmf->invalidate_tlbs_size; 616b80cd524SFelix Kuehling 617ea930000SAlex Sierra spin_lock(&adev->gfx.kiq.ring_lock); 61836a1707aSAlex Sierra /* 2 dwords flush + 8 dwords fence */ 619b80cd524SFelix Kuehling amdgpu_ring_alloc(ring, ndw); 620b80cd524SFelix Kuehling if (vega20_xgmi_wa) 621b80cd524SFelix Kuehling kiq->pmf->kiq_invalidate_tlbs(ring, 622b80cd524SFelix Kuehling pasid, 2, all_hub); 623ea930000SAlex Sierra kiq->pmf->kiq_invalidate_tlbs(ring, 624ea930000SAlex Sierra pasid, flush_type, all_hub); 62504e4e2e9SYintian Tao r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 62604e4e2e9SYintian Tao if (r) { 62704e4e2e9SYintian Tao amdgpu_ring_undo(ring); 628abb17b1eSColin Ian King spin_unlock(&adev->gfx.kiq.ring_lock); 62904e4e2e9SYintian Tao return -ETIME; 63004e4e2e9SYintian Tao } 63104e4e2e9SYintian Tao 632ea930000SAlex Sierra amdgpu_ring_commit(ring); 633ea930000SAlex Sierra spin_unlock(&adev->gfx.kiq.ring_lock); 634ea930000SAlex Sierra r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); 635ea930000SAlex Sierra if (r < 1) { 636f1403342SChristian König DRM_ERROR("wait for kiq fence error: %ld.\n", r); 637ea930000SAlex Sierra return -ETIME; 638ea930000SAlex Sierra } 639ea930000SAlex Sierra 640ea930000SAlex Sierra return 0; 641ea930000SAlex Sierra } 642ea930000SAlex Sierra 643ea930000SAlex Sierra for (vmid = 1; vmid < 16; vmid++) { 644ea930000SAlex Sierra 645ea930000SAlex Sierra ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 646ea930000SAlex Sierra &queried_pasid); 647ea930000SAlex Sierra if (ret && queried_pasid == pasid) { 648ea930000SAlex Sierra if (all_hub) { 649ea930000SAlex Sierra for (i = 0; i < adev->num_vmhubs; i++) 650ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 651fa34edbeSFelix Kuehling i, flush_type); 652ea930000SAlex Sierra } else { 653ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 654fa34edbeSFelix Kuehling AMDGPU_GFXHUB_0, flush_type); 655ea930000SAlex Sierra } 656ea930000SAlex Sierra break; 657ea930000SAlex Sierra } 658ea930000SAlex Sierra } 659ea930000SAlex Sierra 660ea930000SAlex Sierra return 0; 661ea930000SAlex Sierra 662ea930000SAlex Sierra } 663ea930000SAlex Sierra 6649096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 665c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 6669096d6e5SChristian König { 66790f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); 668250b4228SChristian König struct amdgpu_device *adev = ring->adev; 669250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 6702a79d868SYong Zhao uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 6719096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 6729096d6e5SChristian König 673f920d1bbSchangzhu /* 674f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 675f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 676f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 677f920d1bbSchangzhu * to WA the Issue 678f920d1bbSchangzhu */ 679f920d1bbSchangzhu 680f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 68190f6452cSchangzhu if (use_semaphore) 682f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 683f920d1bbSchangzhu amdgpu_ring_emit_reg_wait(ring, 684148f597dSHuang Rui hub->vm_inv_eng0_sem + 685148f597dSHuang Rui hub->eng_distance * eng, 0x1, 0x1); 686f920d1bbSchangzhu 687148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 688148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 6899096d6e5SChristian König lower_32_bits(pd_addr)); 6909096d6e5SChristian König 691148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 692148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 6939096d6e5SChristian König upper_32_bits(pd_addr)); 6949096d6e5SChristian König 695148f597dSHuang Rui amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 696148f597dSHuang Rui hub->eng_distance * eng, 697148f597dSHuang Rui hub->vm_inv_eng0_ack + 698148f597dSHuang Rui hub->eng_distance * eng, 699f8bc9037SAlex Deucher req, 1 << vmid); 700f732b6b3SChristian König 701f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 70290f6452cSchangzhu if (use_semaphore) 703f920d1bbSchangzhu /* 704f920d1bbSchangzhu * add semaphore release after invalidation, 705f920d1bbSchangzhu * write with 0 means semaphore release 706f920d1bbSchangzhu */ 707148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 708148f597dSHuang Rui hub->eng_distance * eng, 0); 709f920d1bbSchangzhu 7109096d6e5SChristian König return pd_addr; 7119096d6e5SChristian König } 7129096d6e5SChristian König 713c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 714c633c00bSChristian König unsigned pasid) 715c633c00bSChristian König { 716c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 717c633c00bSChristian König uint32_t reg; 718c633c00bSChristian König 719f2d66571SLe Ma /* Do nothing because there's no lut register for mmhub1. */ 720f2d66571SLe Ma if (ring->funcs->vmhub == AMDGPU_MMHUB_1) 721f2d66571SLe Ma return; 722f2d66571SLe Ma 723a2d15ed7SLe Ma if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) 724c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 725c633c00bSChristian König else 726c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 727c633c00bSChristian König 728c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 729c633c00bSChristian König } 730c633c00bSChristian König 731e60f8db5SAlex Xie /* 732e60f8db5SAlex Xie * PTE format on VEGA 10: 733e60f8db5SAlex Xie * 63:59 reserved 734e60f8db5SAlex Xie * 58:57 mtype 735e60f8db5SAlex Xie * 56 F 736e60f8db5SAlex Xie * 55 L 737e60f8db5SAlex Xie * 54 P 738e60f8db5SAlex Xie * 53 SW 739e60f8db5SAlex Xie * 52 T 740e60f8db5SAlex Xie * 50:48 reserved 741e60f8db5SAlex Xie * 47:12 4k physical page base address 742e60f8db5SAlex Xie * 11:7 fragment 743e60f8db5SAlex Xie * 6 write 744e60f8db5SAlex Xie * 5 read 745e60f8db5SAlex Xie * 4 exe 746e60f8db5SAlex Xie * 3 Z 747e60f8db5SAlex Xie * 2 snooped 748e60f8db5SAlex Xie * 1 system 749e60f8db5SAlex Xie * 0 valid 750e60f8db5SAlex Xie * 751e60f8db5SAlex Xie * PDE format on VEGA 10: 752e60f8db5SAlex Xie * 63:59 block fragment size 753e60f8db5SAlex Xie * 58:55 reserved 754e60f8db5SAlex Xie * 54 P 755e60f8db5SAlex Xie * 53:48 reserved 756e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 757e60f8db5SAlex Xie * 5:3 reserved 758e60f8db5SAlex Xie * 2 C 759e60f8db5SAlex Xie * 1 system 760e60f8db5SAlex Xie * 0 valid 761e60f8db5SAlex Xie */ 762e60f8db5SAlex Xie 76371776b6dSChristian König static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) 764e60f8db5SAlex Xie 765e60f8db5SAlex Xie { 76671776b6dSChristian König switch (flags) { 767e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 76871776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 769e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 77071776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 771e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 77271776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); 773093e48c0SOak Zeng case AMDGPU_VM_MTYPE_RW: 77471776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); 775e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 77671776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); 777e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 77871776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); 779e60f8db5SAlex Xie default: 78071776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 781e60f8db5SAlex Xie } 782e60f8db5SAlex Xie } 783e60f8db5SAlex Xie 7843de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 7853de676d8SChristian König uint64_t *addr, uint64_t *flags) 786f75e237cSChristian König { 787bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 7883de676d8SChristian König *addr = adev->vm_manager.vram_base_offset + *addr - 789770d13b1SChristian König adev->gmc.vram_start; 7903de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 7916a42fd6fSChristian König 792770d13b1SChristian König if (!adev->gmc.translate_further) 7936a42fd6fSChristian König return; 7946a42fd6fSChristian König 7956a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 7966a42fd6fSChristian König /* Set the block fragment size */ 7976a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 7986a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 7996a42fd6fSChristian König 8006a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 8016a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 8026a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 8036a42fd6fSChristian König else 8046a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 8056a42fd6fSChristian König } 806f75e237cSChristian König } 807f75e237cSChristian König 808cbfae36cSChristian König static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, 809cbfae36cSChristian König struct amdgpu_bo_va_mapping *mapping, 810cbfae36cSChristian König uint64_t *flags) 811cbfae36cSChristian König { 812cbfae36cSChristian König *flags &= ~AMDGPU_PTE_EXECUTABLE; 813cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 814cbfae36cSChristian König 815cbfae36cSChristian König *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 816cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; 817cbfae36cSChristian König 818cbfae36cSChristian König if (mapping->flags & AMDGPU_PTE_PRT) { 819cbfae36cSChristian König *flags |= AMDGPU_PTE_PRT; 820cbfae36cSChristian König *flags &= ~AMDGPU_PTE_VALID; 821cbfae36cSChristian König } 822cbfae36cSChristian König 823cbfae36cSChristian König if (adev->asic_type == CHIP_ARCTURUS && 824cbfae36cSChristian König !(*flags & AMDGPU_PTE_SYSTEM) && 825cbfae36cSChristian König mapping->bo_va->is_xgmi) 826cbfae36cSChristian König *flags |= AMDGPU_PTE_SNOOPED; 827cbfae36cSChristian König } 828cbfae36cSChristian König 8297b885f0eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 8307b885f0eSAlex Deucher { 8317b885f0eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 8327b885f0eSAlex Deucher unsigned size; 8337b885f0eSAlex Deucher 8347b885f0eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 8357b885f0eSAlex Deucher size = AMDGPU_VBIOS_VGA_ALLOCATION; 8367b885f0eSAlex Deucher } else { 8377b885f0eSAlex Deucher u32 viewport; 8387b885f0eSAlex Deucher 8397b885f0eSAlex Deucher switch (adev->asic_type) { 8407b885f0eSAlex Deucher case CHIP_RAVEN: 8417b885f0eSAlex Deucher case CHIP_RENOIR: 8427b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 8437b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, 8447b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 8457b885f0eSAlex Deucher REG_GET_FIELD(viewport, 8467b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 8477b885f0eSAlex Deucher 4); 8487b885f0eSAlex Deucher break; 8497b885f0eSAlex Deucher case CHIP_VEGA10: 8507b885f0eSAlex Deucher case CHIP_VEGA12: 8517b885f0eSAlex Deucher case CHIP_VEGA20: 8527b885f0eSAlex Deucher default: 8537b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 8547b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 8557b885f0eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 8567b885f0eSAlex Deucher 4); 8577b885f0eSAlex Deucher break; 8587b885f0eSAlex Deucher } 8597b885f0eSAlex Deucher } 8607b885f0eSAlex Deucher 8617b885f0eSAlex Deucher return size; 8627b885f0eSAlex Deucher } 8637b885f0eSAlex Deucher 864132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 865132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 866ea930000SAlex Sierra .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, 8679096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 868c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 86971776b6dSChristian König .map_mtype = gmc_v9_0_map_mtype, 870cbfae36cSChristian König .get_vm_pde = gmc_v9_0_get_vm_pde, 8717b885f0eSAlex Deucher .get_vm_pte = gmc_v9_0_get_vm_pte, 8727b885f0eSAlex Deucher .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, 873e60f8db5SAlex Xie }; 874e60f8db5SAlex Xie 875132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 876e60f8db5SAlex Xie { 877132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 878e60f8db5SAlex Xie } 879e60f8db5SAlex Xie 8805b6b35aaSHawking Zhang static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) 8815b6b35aaSHawking Zhang { 8825b6b35aaSHawking Zhang switch (adev->asic_type) { 883e7da754bSMonk Liu case CHIP_VEGA10: 884e7da754bSMonk Liu adev->umc.funcs = &umc_v6_0_funcs; 885e7da754bSMonk Liu break; 8865b6b35aaSHawking Zhang case CHIP_VEGA20: 8873aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 8883aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 8893aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 8904cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; 8914cf781c2SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 8924cf781c2SJohn Clements adev->umc.funcs = &umc_v6_1_funcs; 8934cf781c2SJohn Clements break; 8949e612c11SHawking Zhang case CHIP_ARCTURUS: 8953aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 8963aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 8973aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 8984cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; 8993aacf4eaSTao Zhou adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 900045c0216STao Zhou adev->umc.funcs = &umc_v6_1_funcs; 9015b6b35aaSHawking Zhang break; 9025b6b35aaSHawking Zhang default: 9035b6b35aaSHawking Zhang break; 9045b6b35aaSHawking Zhang } 9055b6b35aaSHawking Zhang } 9065b6b35aaSHawking Zhang 9073d093da0STao Zhou static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) 9083d093da0STao Zhou { 9093d093da0STao Zhou switch (adev->asic_type) { 910f6c3623bSDennis Li case CHIP_ARCTURUS: 911f6c3623bSDennis Li adev->mmhub.funcs = &mmhub_v9_4_funcs; 912f6c3623bSDennis Li break; 9133d093da0STao Zhou default: 9149fb1506eSOak Zeng adev->mmhub.funcs = &mmhub_v1_0_funcs; 9153d093da0STao Zhou break; 9163d093da0STao Zhou } 9173d093da0STao Zhou } 9183d093da0STao Zhou 919e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 920e60f8db5SAlex Xie { 921e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 922e60f8db5SAlex Xie 923132f34e4SChristian König gmc_v9_0_set_gmc_funcs(adev); 924e60f8db5SAlex Xie gmc_v9_0_set_irq_funcs(adev); 9255b6b35aaSHawking Zhang gmc_v9_0_set_umc_funcs(adev); 9263d093da0STao Zhou gmc_v9_0_set_mmhub_funcs(adev); 927e60f8db5SAlex Xie 928770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 929770d13b1SChristian König adev->gmc.shared_aperture_end = 930770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 931bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 932770d13b1SChristian König adev->gmc.private_aperture_end = 933770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 934a7ea6548SAlex Deucher 935e60f8db5SAlex Xie return 0; 936e60f8db5SAlex Xie } 937e60f8db5SAlex Xie 938e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 939e60f8db5SAlex Xie { 940e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 941c5b6e585STao Zhou int r; 9424789c463SChristian König 943cd2b5623SAlex Deucher amdgpu_bo_late_init(adev); 9446f752ec2SAndrey Grodzovsky 945bdbe90f0SAlex Deucher r = amdgpu_gmc_allocate_vm_inv_eng(adev); 946c713a461SEvan Quan if (r) 947c713a461SEvan Quan return r; 948f49ea9f8SHawking Zhang /* Check if ecc is available */ 94988474ccaSGuchun Chen if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) { 950f49ea9f8SHawking Zhang r = amdgpu_atomfirmware_mem_ecc_supported(adev); 951f49ea9f8SHawking Zhang if (!r) { 95202bab923SDavid Panariti DRM_INFO("ECC is not present.\n"); 953bdf84a80SJoseph Greathouse if (adev->df.funcs->enable_ecc_force_par_wr_rmw) 954bdf84a80SJoseph Greathouse adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); 95588474ccaSGuchun Chen } else 956f49ea9f8SHawking Zhang DRM_INFO("ECC is active.\n"); 957f49ea9f8SHawking Zhang 958f49ea9f8SHawking Zhang r = amdgpu_atomfirmware_sram_ecc_supported(adev); 95988474ccaSGuchun Chen if (!r) 960f49ea9f8SHawking Zhang DRM_INFO("SRAM ECC is not present.\n"); 96188474ccaSGuchun Chen else 962f49ea9f8SHawking Zhang DRM_INFO("SRAM ECC is active.\n"); 963f49ea9f8SHawking Zhang } 96402bab923SDavid Panariti 965fe5211f1SHawking Zhang if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count) 966fe5211f1SHawking Zhang adev->mmhub.funcs->reset_ras_error_count(adev); 967fe5211f1SHawking Zhang 968ba083492STao Zhou r = amdgpu_gmc_ras_late_init(adev); 969791c4769Sxinhui pan if (r) 970e60f8db5SAlex Xie return r; 971e60f8db5SAlex Xie 972770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 973e60f8db5SAlex Xie } 974e60f8db5SAlex Xie 975e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 976770d13b1SChristian König struct amdgpu_gmc *mc) 977e60f8db5SAlex Xie { 978e60f8db5SAlex Xie u64 base = 0; 9799d4f837aSFrank.Min 9809fb1506eSOak Zeng if (!amdgpu_sriov_vf(adev)) 9819fb1506eSOak Zeng base = adev->mmhub.funcs->get_fb_location(adev); 9829d4f837aSFrank.Min 9836fdd68b1SAlex Deucher /* add the xgmi offset of the physical node */ 9846fdd68b1SAlex Deucher base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 98583afe835SOak Zeng amdgpu_gmc_vram_location(adev, mc, base); 986961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 987c3e1b43cSChristian König amdgpu_gmc_agp_location(adev, mc); 988e60f8db5SAlex Xie /* base offset of vram pages */ 989e60f8db5SAlex Xie adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev); 9906fdd68b1SAlex Deucher 9916fdd68b1SAlex Deucher /* XXX: add the xgmi offset of the physical node? */ 9926fdd68b1SAlex Deucher adev->vm_manager.vram_base_offset += 9936fdd68b1SAlex Deucher adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 994e60f8db5SAlex Xie } 995e60f8db5SAlex Xie 996e60f8db5SAlex Xie /** 997e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 998e60f8db5SAlex Xie * 999e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1000e60f8db5SAlex Xie * 1001e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 1002e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 1003e60f8db5SAlex Xie * Returns 0 for success. 1004e60f8db5SAlex Xie */ 1005e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 1006e60f8db5SAlex Xie { 1007e60f8db5SAlex Xie int r; 1008e60f8db5SAlex Xie 1009e60f8db5SAlex Xie /* size in MB on si */ 1010770d13b1SChristian König adev->gmc.mc_vram_size = 1011bebc0762SHawking Zhang adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 1012770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 1013e60f8db5SAlex Xie 1014e60f8db5SAlex Xie if (!(adev->flags & AMD_IS_APU)) { 1015e60f8db5SAlex Xie r = amdgpu_device_resize_fb_bar(adev); 1016e60f8db5SAlex Xie if (r) 1017e60f8db5SAlex Xie return r; 1018e60f8db5SAlex Xie } 1019770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 1020770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 1021e60f8db5SAlex Xie 1022156a81beSChunming Zhou #ifdef CONFIG_X86_64 1023156a81beSChunming Zhou if (adev->flags & AMD_IS_APU) { 1024156a81beSChunming Zhou adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev); 1025156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 1026156a81beSChunming Zhou } 1027156a81beSChunming Zhou #endif 1028e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 1029770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 1030770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 1031770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 1032e60f8db5SAlex Xie 1033e60f8db5SAlex Xie /* set the gart size */ 1034e60f8db5SAlex Xie if (amdgpu_gart_size == -1) { 1035e60f8db5SAlex Xie switch (adev->asic_type) { 1036e60f8db5SAlex Xie case CHIP_VEGA10: /* all engines support GPUVM */ 1037273a14cdSAlex Deucher case CHIP_VEGA12: /* all engines support GPUVM */ 1038d96b428cSFeifei Xu case CHIP_VEGA20: 10393de2ff5dSLe Ma case CHIP_ARCTURUS: 1040e60f8db5SAlex Xie default: 1041fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 1042e60f8db5SAlex Xie break; 1043e60f8db5SAlex Xie case CHIP_RAVEN: /* DCE SG support */ 10448787ee01SHuang Rui case CHIP_RENOIR: 1045770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 1046e60f8db5SAlex Xie break; 1047e60f8db5SAlex Xie } 1048e60f8db5SAlex Xie } else { 1049770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 1050e60f8db5SAlex Xie } 1051e60f8db5SAlex Xie 1052770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 1053e60f8db5SAlex Xie 1054e60f8db5SAlex Xie return 0; 1055e60f8db5SAlex Xie } 1056e60f8db5SAlex Xie 1057e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 1058e60f8db5SAlex Xie { 1059e60f8db5SAlex Xie int r; 1060e60f8db5SAlex Xie 10611123b989SChristian König if (adev->gart.bo) { 1062e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 1063e60f8db5SAlex Xie return 0; 1064e60f8db5SAlex Xie } 1065e60f8db5SAlex Xie /* Initialize common gart structure */ 1066e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 1067e60f8db5SAlex Xie if (r) 1068e60f8db5SAlex Xie return r; 1069e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 10707596ab68SHawking Zhang adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | 1071e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 1072e60f8db5SAlex Xie return amdgpu_gart_table_vram_alloc(adev); 1073e60f8db5SAlex Xie } 1074e60f8db5SAlex Xie 1075e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 1076e60f8db5SAlex Xie { 1077ad02e08eSOri Messinger int r, vram_width = 0, vram_type = 0, vram_vendor = 0; 1078e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1079e60f8db5SAlex Xie 1080e60f8db5SAlex Xie gfxhub_v1_0_init(adev); 10819fb1506eSOak Zeng 10829fb1506eSOak Zeng adev->mmhub.funcs->init(adev); 1083e60f8db5SAlex Xie 1084770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 1085e60f8db5SAlex Xie 1086ad02e08eSOri Messinger r = amdgpu_atomfirmware_get_vram_info(adev, 1087ad02e08eSOri Messinger &vram_width, &vram_type, &vram_vendor); 1088631cdbd2SAlex Deucher if (amdgpu_sriov_vf(adev)) 1089631cdbd2SAlex Deucher /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, 1090631cdbd2SAlex Deucher * and DF related registers is not readable, seems hardcord is the 1091631cdbd2SAlex Deucher * only way to set the correct vram_width 1092631cdbd2SAlex Deucher */ 1093631cdbd2SAlex Deucher adev->gmc.vram_width = 2048; 1094631cdbd2SAlex Deucher else if (amdgpu_emu_mode != 1) 1095631cdbd2SAlex Deucher adev->gmc.vram_width = vram_width; 1096631cdbd2SAlex Deucher 1097631cdbd2SAlex Deucher if (!adev->gmc.vram_width) { 1098631cdbd2SAlex Deucher int chansize, numchan; 1099631cdbd2SAlex Deucher 1100631cdbd2SAlex Deucher /* hbm memory channel size */ 1101631cdbd2SAlex Deucher if (adev->flags & AMD_IS_APU) 1102631cdbd2SAlex Deucher chansize = 64; 1103631cdbd2SAlex Deucher else 1104631cdbd2SAlex Deucher chansize = 128; 1105631cdbd2SAlex Deucher 1106bdf84a80SJoseph Greathouse numchan = adev->df.funcs->get_hbm_channel_number(adev); 1107631cdbd2SAlex Deucher adev->gmc.vram_width = numchan * chansize; 1108631cdbd2SAlex Deucher } 1109631cdbd2SAlex Deucher 1110631cdbd2SAlex Deucher adev->gmc.vram_type = vram_type; 1111ad02e08eSOri Messinger adev->gmc.vram_vendor = vram_vendor; 1112e60f8db5SAlex Xie switch (adev->asic_type) { 1113e60f8db5SAlex Xie case CHIP_RAVEN: 11141daa2bfaSLe Ma adev->num_vmhubs = 2; 11151daa2bfaSLe Ma 11166a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 1117f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 11186a42fd6fSChristian König } else { 11196a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 11206a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 1121770d13b1SChristian König adev->gmc.translate_further = 11226a42fd6fSChristian König adev->vm_manager.num_level > 1; 11236a42fd6fSChristian König } 1124e60f8db5SAlex Xie break; 1125e60f8db5SAlex Xie case CHIP_VEGA10: 1126273a14cdSAlex Deucher case CHIP_VEGA12: 1127d96b428cSFeifei Xu case CHIP_VEGA20: 11288787ee01SHuang Rui case CHIP_RENOIR: 11291daa2bfaSLe Ma adev->num_vmhubs = 2; 11301daa2bfaSLe Ma 11318787ee01SHuang Rui 1132e60f8db5SAlex Xie /* 1133e60f8db5SAlex Xie * To fulfill 4-level page support, 1134e60f8db5SAlex Xie * vm size is 256TB (48bit), maximum size of Vega10, 1135e60f8db5SAlex Xie * block size 512 (9bit) 1136e60f8db5SAlex Xie */ 1137cdba61daSwentalou /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 1138cdba61daSwentalou if (amdgpu_sriov_vf(adev)) 1139cdba61daSwentalou amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 1140cdba61daSwentalou else 1141f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1142e60f8db5SAlex Xie break; 11433de2ff5dSLe Ma case CHIP_ARCTURUS: 1144c8a6e2a3SLe Ma adev->num_vmhubs = 3; 1145c8a6e2a3SLe Ma 11463de2ff5dSLe Ma /* Keep the vm size same with Vega20 */ 11473de2ff5dSLe Ma amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 11483de2ff5dSLe Ma break; 1149e60f8db5SAlex Xie default: 1150e60f8db5SAlex Xie break; 1151e60f8db5SAlex Xie } 1152e60f8db5SAlex Xie 1153e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 115444a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 1155770d13b1SChristian König &adev->gmc.vm_fault); 115630da7bb1SChristian König if (r) 115730da7bb1SChristian König return r; 115830da7bb1SChristian König 11597d19b15fSLe Ma if (adev->asic_type == CHIP_ARCTURUS) { 11607d19b15fSLe Ma r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, 11617d19b15fSLe Ma &adev->gmc.vm_fault); 11627d19b15fSLe Ma if (r) 11637d19b15fSLe Ma return r; 11647d19b15fSLe Ma } 11657d19b15fSLe Ma 116644a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 1167770d13b1SChristian König &adev->gmc.vm_fault); 1168e60f8db5SAlex Xie 1169e60f8db5SAlex Xie if (r) 1170e60f8db5SAlex Xie return r; 1171e60f8db5SAlex Xie 11722ee9403eSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 1173791c4769Sxinhui pan /* interrupt sent to DF. */ 1174791c4769Sxinhui pan r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 1175791c4769Sxinhui pan &adev->gmc.ecc_irq); 1176791c4769Sxinhui pan if (r) 1177791c4769Sxinhui pan return r; 11782ee9403eSZhigang Luo } 1179791c4769Sxinhui pan 1180e60f8db5SAlex Xie /* Set the internal MC address mask 1181e60f8db5SAlex Xie * This is the max address of the GPU's 1182e60f8db5SAlex Xie * internal address space. 1183e60f8db5SAlex Xie */ 1184770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 1185e60f8db5SAlex Xie 1186244511f3SChristoph Hellwig r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); 1187e60f8db5SAlex Xie if (r) { 1188e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 1189244511f3SChristoph Hellwig return r; 1190e60f8db5SAlex Xie } 1191244511f3SChristoph Hellwig adev->need_swiotlb = drm_need_swiotlb(44); 1192e60f8db5SAlex Xie 119347622ba0SAlex Deucher if (adev->gmc.xgmi.supported) { 1194bf0a60b7SAlex Deucher r = gfxhub_v1_1_get_xgmi_info(adev); 1195bf0a60b7SAlex Deucher if (r) 1196bf0a60b7SAlex Deucher return r; 1197bf0a60b7SAlex Deucher } 1198bf0a60b7SAlex Deucher 1199e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 1200e60f8db5SAlex Xie if (r) 1201e60f8db5SAlex Xie return r; 1202e60f8db5SAlex Xie 12037b885f0eSAlex Deucher amdgpu_gmc_get_vbios_allocations(adev); 1204ebdef28eSAlex Deucher 1205e60f8db5SAlex Xie /* Memory manager */ 1206e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 1207e60f8db5SAlex Xie if (r) 1208e60f8db5SAlex Xie return r; 1209e60f8db5SAlex Xie 1210e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 1211e60f8db5SAlex Xie if (r) 1212e60f8db5SAlex Xie return r; 1213e60f8db5SAlex Xie 121405ec3edaSChristian König /* 121505ec3edaSChristian König * number of VMs 121605ec3edaSChristian König * VMID 0 is reserved for System 121781659b20SFelix Kuehling * amdgpu graphics/compute will use VMIDs 1..n-1 121881659b20SFelix Kuehling * amdkfd will use VMIDs n..15 121981659b20SFelix Kuehling * 122081659b20SFelix Kuehling * The first KFD VMID is 8 for GPUs with graphics, 3 for 122181659b20SFelix Kuehling * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs 122281659b20SFelix Kuehling * for video processing. 122305ec3edaSChristian König */ 122481659b20SFelix Kuehling adev->vm_manager.first_kfd_vmid = 122581659b20SFelix Kuehling adev->asic_type == CHIP_ARCTURUS ? 3 : 8; 122605ec3edaSChristian König 122705ec3edaSChristian König amdgpu_vm_manager_init(adev); 122805ec3edaSChristian König 122905ec3edaSChristian König return 0; 1230e60f8db5SAlex Xie } 1231e60f8db5SAlex Xie 1232e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 1233e60f8db5SAlex Xie { 1234e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1235e60f8db5SAlex Xie 12362adf1344STao Zhou amdgpu_gmc_ras_fini(adev); 1237f59548c8SMonk Liu amdgpu_gem_force_release(adev); 1238e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 1239a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 1240e60f8db5SAlex Xie amdgpu_bo_fini(adev); 1241a3d9103eSAndrey Grodzovsky amdgpu_gart_fini(adev); 1242e60f8db5SAlex Xie 1243e60f8db5SAlex Xie return 0; 1244e60f8db5SAlex Xie } 1245e60f8db5SAlex Xie 1246e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1247e60f8db5SAlex Xie { 1248946a4d5bSShaoyun Liu 1249e60f8db5SAlex Xie switch (adev->asic_type) { 1250e60f8db5SAlex Xie case CHIP_VEGA10: 12514cd4c5c0SMonk Liu if (amdgpu_sriov_vf(adev)) 125298cad2deSTrigger Huang break; 125398cad2deSTrigger Huang /* fall through */ 1254d96b428cSFeifei Xu case CHIP_VEGA20: 1255946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 12565c583018SEvan Quan golden_settings_mmhub_1_0_0, 1257c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1258946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 12595c583018SEvan Quan golden_settings_athub_1_0_0, 1260c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1261e60f8db5SAlex Xie break; 1262273a14cdSAlex Deucher case CHIP_VEGA12: 1263273a14cdSAlex Deucher break; 1264e4f3abaaSChunming Zhou case CHIP_RAVEN: 12658787ee01SHuang Rui /* TODO for renoir */ 1266946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 12675c583018SEvan Quan golden_settings_athub_1_0_0, 1268c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1269e4f3abaaSChunming Zhou break; 1270e60f8db5SAlex Xie default: 1271e60f8db5SAlex Xie break; 1272e60f8db5SAlex Xie } 1273e60f8db5SAlex Xie } 1274e60f8db5SAlex Xie 1275e60f8db5SAlex Xie /** 1276c2ecd79bSShirish S * gmc_v9_0_restore_registers - restores regs 1277c2ecd79bSShirish S * 1278c2ecd79bSShirish S * @adev: amdgpu_device pointer 1279c2ecd79bSShirish S * 1280c2ecd79bSShirish S * This restores register values, saved at suspend. 1281c2ecd79bSShirish S */ 1282c2ecd79bSShirish S static void gmc_v9_0_restore_registers(struct amdgpu_device *adev) 1283c2ecd79bSShirish S { 1284c2ecd79bSShirish S if (adev->asic_type == CHIP_RAVEN) 1285c2ecd79bSShirish S WREG32(mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); 1286c2ecd79bSShirish S } 1287c2ecd79bSShirish S 1288c2ecd79bSShirish S /** 1289e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1290e60f8db5SAlex Xie * 1291e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1292e60f8db5SAlex Xie */ 1293e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1294e60f8db5SAlex Xie { 1295cb1545f7SOak Zeng int r; 1296e60f8db5SAlex Xie 12971123b989SChristian König if (adev->gart.bo == NULL) { 1298e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1299e60f8db5SAlex Xie return -EINVAL; 1300e60f8db5SAlex Xie } 1301ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1302ce1b1b66SMonk Liu if (r) 1303ce1b1b66SMonk Liu return r; 1304e60f8db5SAlex Xie 1305e60f8db5SAlex Xie r = gfxhub_v1_0_gart_enable(adev); 1306e60f8db5SAlex Xie if (r) 1307e60f8db5SAlex Xie return r; 1308e60f8db5SAlex Xie 13099fb1506eSOak Zeng r = adev->mmhub.funcs->gart_enable(adev); 1310e60f8db5SAlex Xie if (r) 1311e60f8db5SAlex Xie return r; 1312e60f8db5SAlex Xie 1313cb1545f7SOak Zeng DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1314cb1545f7SOak Zeng (unsigned)(adev->gmc.gart_size >> 20), 1315cb1545f7SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1316cb1545f7SOak Zeng adev->gart.ready = true; 1317cb1545f7SOak Zeng return 0; 1318cb1545f7SOak Zeng } 1319cb1545f7SOak Zeng 1320cb1545f7SOak Zeng static int gmc_v9_0_hw_init(void *handle) 1321cb1545f7SOak Zeng { 1322cb1545f7SOak Zeng struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1323cb1545f7SOak Zeng bool value; 1324cb1545f7SOak Zeng int r, i; 1325cb1545f7SOak Zeng u32 tmp; 1326cb1545f7SOak Zeng 1327cb1545f7SOak Zeng /* The sequence of these two function calls matters.*/ 1328cb1545f7SOak Zeng gmc_v9_0_init_golden_registers(adev); 1329cb1545f7SOak Zeng 1330cb1545f7SOak Zeng if (adev->mode_info.num_crtc) { 1331cb1545f7SOak Zeng if (adev->asic_type != CHIP_ARCTURUS) { 1332cb1545f7SOak Zeng /* Lockout access through VGA aperture*/ 1333cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1334cb1545f7SOak Zeng 1335cb1545f7SOak Zeng /* disable VGA render */ 1336cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1337cb1545f7SOak Zeng } 1338cb1545f7SOak Zeng } 1339cb1545f7SOak Zeng 1340cb1545f7SOak Zeng amdgpu_device_program_register_sequence(adev, 1341cb1545f7SOak Zeng golden_settings_vega10_hdp, 1342cb1545f7SOak Zeng ARRAY_SIZE(golden_settings_vega10_hdp)); 1343cb1545f7SOak Zeng 13449fb1506eSOak Zeng if (adev->mmhub.funcs->update_power_gating) 13459fb1506eSOak Zeng adev->mmhub.funcs->update_power_gating(adev, true); 13469fb1506eSOak Zeng 1347cb1545f7SOak Zeng switch (adev->asic_type) { 1348f81b86a0SOak Zeng case CHIP_ARCTURUS: 1349f81b86a0SOak Zeng WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); 1350f81b86a0SOak Zeng break; 1351cb1545f7SOak Zeng default: 1352cb1545f7SOak Zeng break; 1353cb1545f7SOak Zeng } 1354cb1545f7SOak Zeng 1355846347c9STom St Denis WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1356e60f8db5SAlex Xie 1357b9509c80SHuang Rui tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1358b9509c80SHuang Rui WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1359e60f8db5SAlex Xie 1360fe2b5323STiecheng Zhou WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); 1361fe2b5323STiecheng Zhou WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); 1362fe2b5323STiecheng Zhou 13631d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 1364bebc0762SHawking Zhang adev->nbio.funcs->hdp_flush(adev, NULL); 13651d4e0a8cSMonk Liu 1366e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1367e60f8db5SAlex Xie value = false; 1368e60f8db5SAlex Xie else 1369e60f8db5SAlex Xie value = true; 1370e60f8db5SAlex Xie 137120bf2f6fSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 137208546895SZhigang Luo gfxhub_v1_0_set_fault_enable_default(adev, value); 13739fb1506eSOak Zeng adev->mmhub.funcs->set_fault_enable_default(adev, value); 137420bf2f6fSZhigang Luo } 13753ff98548SOak Zeng for (i = 0; i < adev->num_vmhubs; ++i) 13763ff98548SOak Zeng gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); 1377e60f8db5SAlex Xie 1378e7da754bSMonk Liu if (adev->umc.funcs && adev->umc.funcs->init_registers) 1379e7da754bSMonk Liu adev->umc.funcs->init_registers(adev); 1380e7da754bSMonk Liu 1381e60f8db5SAlex Xie r = gmc_v9_0_gart_enable(adev); 1382e60f8db5SAlex Xie 1383e60f8db5SAlex Xie return r; 1384e60f8db5SAlex Xie } 1385e60f8db5SAlex Xie 1386e60f8db5SAlex Xie /** 1387c2ecd79bSShirish S * gmc_v9_0_save_registers - saves regs 1388c2ecd79bSShirish S * 1389c2ecd79bSShirish S * @adev: amdgpu_device pointer 1390c2ecd79bSShirish S * 1391c2ecd79bSShirish S * This saves potential register values that should be 1392c2ecd79bSShirish S * restored upon resume 1393c2ecd79bSShirish S */ 1394c2ecd79bSShirish S static void gmc_v9_0_save_registers(struct amdgpu_device *adev) 1395c2ecd79bSShirish S { 1396c2ecd79bSShirish S if (adev->asic_type == CHIP_RAVEN) 1397c2ecd79bSShirish S adev->gmc.sdpif_register = RREG32(mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); 1398c2ecd79bSShirish S } 1399c2ecd79bSShirish S 1400c2ecd79bSShirish S /** 1401e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1402e60f8db5SAlex Xie * 1403e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1404e60f8db5SAlex Xie * 1405e60f8db5SAlex Xie * This disables all VM page table. 1406e60f8db5SAlex Xie */ 1407e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1408e60f8db5SAlex Xie { 1409e60f8db5SAlex Xie gfxhub_v1_0_gart_disable(adev); 14109fb1506eSOak Zeng adev->mmhub.funcs->gart_disable(adev); 1411ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1412e60f8db5SAlex Xie } 1413e60f8db5SAlex Xie 1414e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1415e60f8db5SAlex Xie { 1416e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1417e60f8db5SAlex Xie 14185dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 14195dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 14205dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 14215dd696aeSTrigger Huang return 0; 14225dd696aeSTrigger Huang } 14235dd696aeSTrigger Huang 1424791c4769Sxinhui pan amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1425770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1426e60f8db5SAlex Xie gmc_v9_0_gart_disable(adev); 1427e60f8db5SAlex Xie 1428e60f8db5SAlex Xie return 0; 1429e60f8db5SAlex Xie } 1430e60f8db5SAlex Xie 1431e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1432e60f8db5SAlex Xie { 1433c2ecd79bSShirish S int r; 1434e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1435e60f8db5SAlex Xie 1436c2ecd79bSShirish S r = gmc_v9_0_hw_fini(adev); 1437c2ecd79bSShirish S if (r) 1438c2ecd79bSShirish S return r; 1439c2ecd79bSShirish S 1440c2ecd79bSShirish S gmc_v9_0_save_registers(adev); 1441c2ecd79bSShirish S 1442c2ecd79bSShirish S return 0; 1443e60f8db5SAlex Xie } 1444e60f8db5SAlex Xie 1445e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1446e60f8db5SAlex Xie { 1447e60f8db5SAlex Xie int r; 1448e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1449e60f8db5SAlex Xie 1450c2ecd79bSShirish S gmc_v9_0_restore_registers(adev); 1451e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1452e60f8db5SAlex Xie if (r) 1453e60f8db5SAlex Xie return r; 1454e60f8db5SAlex Xie 1455620f774fSChristian König amdgpu_vmid_reset_all(adev); 1456e60f8db5SAlex Xie 145732601d48SChristian König return 0; 1458e60f8db5SAlex Xie } 1459e60f8db5SAlex Xie 1460e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1461e60f8db5SAlex Xie { 1462e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1463e60f8db5SAlex Xie return true; 1464e60f8db5SAlex Xie } 1465e60f8db5SAlex Xie 1466e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1467e60f8db5SAlex Xie { 1468e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1469e60f8db5SAlex Xie return 0; 1470e60f8db5SAlex Xie } 1471e60f8db5SAlex Xie 1472e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1473e60f8db5SAlex Xie { 1474e60f8db5SAlex Xie /* XXX for emulation.*/ 1475e60f8db5SAlex Xie return 0; 1476e60f8db5SAlex Xie } 1477e60f8db5SAlex Xie 1478e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1479e60f8db5SAlex Xie enum amd_clockgating_state state) 1480e60f8db5SAlex Xie { 1481d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1482d5583d4fSHuang Rui 14839fb1506eSOak Zeng adev->mmhub.funcs->set_clockgating(adev, state); 1484bee7b51aSLe Ma 1485bee7b51aSLe Ma athub_v1_0_set_clockgating(adev, state); 1486bee7b51aSLe Ma 1487bee7b51aSLe Ma return 0; 1488e60f8db5SAlex Xie } 1489e60f8db5SAlex Xie 149013052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 149113052be5SHuang Rui { 149213052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 149313052be5SHuang Rui 14949fb1506eSOak Zeng adev->mmhub.funcs->get_clockgating(adev, flags); 1495bee7b51aSLe Ma 1496bee7b51aSLe Ma athub_v1_0_get_clockgating(adev, flags); 149713052be5SHuang Rui } 149813052be5SHuang Rui 1499e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1500e60f8db5SAlex Xie enum amd_powergating_state state) 1501e60f8db5SAlex Xie { 1502e60f8db5SAlex Xie return 0; 1503e60f8db5SAlex Xie } 1504e60f8db5SAlex Xie 1505e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1506e60f8db5SAlex Xie .name = "gmc_v9_0", 1507e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1508e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1509e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1510e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1511e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1512e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1513e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1514e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1515e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1516e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1517e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1518e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1519e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 152013052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1521e60f8db5SAlex Xie }; 1522e60f8db5SAlex Xie 1523e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1524e60f8db5SAlex Xie { 1525e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1526e60f8db5SAlex Xie .major = 9, 1527e60f8db5SAlex Xie .minor = 0, 1528e60f8db5SAlex Xie .rev = 0, 1529e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1530e60f8db5SAlex Xie }; 1531