1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23e60f8db5SAlex Xie #include <linux/firmware.h> 24fd5fd480SChunming Zhou #include <drm/drm_cache.h> 25e60f8db5SAlex Xie #include "amdgpu.h" 26e60f8db5SAlex Xie #include "gmc_v9_0.h" 278d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 282cddc50eSHuang Rui #include "amdgpu_gem.h" 29e60f8db5SAlex Xie 3075199b8cSFeifei Xu #include "hdp/hdp_4_0_offset.h" 3175199b8cSFeifei Xu #include "hdp/hdp_4_0_sh_mask.h" 32cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 33135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 34135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 35fb960bd2SFeifei Xu #include "vega10_enum.h" 3665417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 376ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 38250b4228SChristian König #include "oss/osssys_4_0_offset.h" 39e60f8db5SAlex Xie 40946a4d5bSShaoyun Liu #include "soc15.h" 41e60f8db5SAlex Xie #include "soc15_common.h" 4290c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 43e60f8db5SAlex Xie 44e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 45e60f8db5SAlex Xie #include "mmhub_v1_0.h" 46bf0a60b7SAlex Deucher #include "gfxhub_v1_1.h" 47e60f8db5SAlex Xie 4844a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 4944a99b65SAndrey Grodzovsky 50ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 51ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 52ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 53ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 54ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 55ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 56ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 57ebdef28eSAlex Deucher 58e60f8db5SAlex Xie /* XXX Move this macro to VEGA10 header file, which is like vid.h for VI.*/ 59e60f8db5SAlex Xie #define AMDGPU_NUM_OF_VMIDS 8 60e60f8db5SAlex Xie 61e60f8db5SAlex Xie static const u32 golden_settings_vega10_hdp[] = 62e60f8db5SAlex Xie { 63e60f8db5SAlex Xie 0xf64, 0x0fffffff, 0x00000000, 64e60f8db5SAlex Xie 0xf65, 0x0fffffff, 0x00000000, 65e60f8db5SAlex Xie 0xf66, 0x0fffffff, 0x00000000, 66e60f8db5SAlex Xie 0xf67, 0x0fffffff, 0x00000000, 67e60f8db5SAlex Xie 0xf68, 0x0fffffff, 0x00000000, 68e60f8db5SAlex Xie 0xf6a, 0x0fffffff, 0x00000000, 69e60f8db5SAlex Xie 0xf6b, 0x0fffffff, 0x00000000, 70e60f8db5SAlex Xie 0xf6c, 0x0fffffff, 0x00000000, 71e60f8db5SAlex Xie 0xf6d, 0x0fffffff, 0x00000000, 72e60f8db5SAlex Xie 0xf6e, 0x0fffffff, 0x00000000, 73e60f8db5SAlex Xie }; 74e60f8db5SAlex Xie 75946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 765c583018SEvan Quan { 77946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 78946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 795c583018SEvan Quan }; 805c583018SEvan Quan 81946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 825c583018SEvan Quan { 83946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 84946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 855c583018SEvan Quan }; 865c583018SEvan Quan 8702bab923SDavid Panariti /* Ecc related register addresses, (BASE + reg offset) */ 8802bab923SDavid Panariti /* Universal Memory Controller caps (may be fused). */ 8902bab923SDavid Panariti /* UMCCH:UmcLocalCap */ 9002bab923SDavid Panariti #define UMCLOCALCAPS_ADDR0 (0x00014306 + 0x00000000) 9102bab923SDavid Panariti #define UMCLOCALCAPS_ADDR1 (0x00014306 + 0x00000800) 9202bab923SDavid Panariti #define UMCLOCALCAPS_ADDR2 (0x00014306 + 0x00001000) 9302bab923SDavid Panariti #define UMCLOCALCAPS_ADDR3 (0x00014306 + 0x00001800) 9402bab923SDavid Panariti #define UMCLOCALCAPS_ADDR4 (0x00054306 + 0x00000000) 9502bab923SDavid Panariti #define UMCLOCALCAPS_ADDR5 (0x00054306 + 0x00000800) 9602bab923SDavid Panariti #define UMCLOCALCAPS_ADDR6 (0x00054306 + 0x00001000) 9702bab923SDavid Panariti #define UMCLOCALCAPS_ADDR7 (0x00054306 + 0x00001800) 9802bab923SDavid Panariti #define UMCLOCALCAPS_ADDR8 (0x00094306 + 0x00000000) 9902bab923SDavid Panariti #define UMCLOCALCAPS_ADDR9 (0x00094306 + 0x00000800) 10002bab923SDavid Panariti #define UMCLOCALCAPS_ADDR10 (0x00094306 + 0x00001000) 10102bab923SDavid Panariti #define UMCLOCALCAPS_ADDR11 (0x00094306 + 0x00001800) 10202bab923SDavid Panariti #define UMCLOCALCAPS_ADDR12 (0x000d4306 + 0x00000000) 10302bab923SDavid Panariti #define UMCLOCALCAPS_ADDR13 (0x000d4306 + 0x00000800) 10402bab923SDavid Panariti #define UMCLOCALCAPS_ADDR14 (0x000d4306 + 0x00001000) 10502bab923SDavid Panariti #define UMCLOCALCAPS_ADDR15 (0x000d4306 + 0x00001800) 10602bab923SDavid Panariti 10702bab923SDavid Panariti /* Universal Memory Controller Channel config. */ 10802bab923SDavid Panariti /* UMCCH:UMC_CONFIG */ 10902bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR0 (0x00014040 + 0x00000000) 11002bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR1 (0x00014040 + 0x00000800) 11102bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR2 (0x00014040 + 0x00001000) 11202bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR3 (0x00014040 + 0x00001800) 11302bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR4 (0x00054040 + 0x00000000) 11402bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR5 (0x00054040 + 0x00000800) 11502bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR6 (0x00054040 + 0x00001000) 11602bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR7 (0x00054040 + 0x00001800) 11702bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR8 (0x00094040 + 0x00000000) 11802bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR9 (0x00094040 + 0x00000800) 11902bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR10 (0x00094040 + 0x00001000) 12002bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR11 (0x00094040 + 0x00001800) 12102bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR12 (0x000d4040 + 0x00000000) 12202bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR13 (0x000d4040 + 0x00000800) 12302bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR14 (0x000d4040 + 0x00001000) 12402bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR15 (0x000d4040 + 0x00001800) 12502bab923SDavid Panariti 12602bab923SDavid Panariti /* Universal Memory Controller Channel Ecc config. */ 12702bab923SDavid Panariti /* UMCCH:EccCtrl */ 12802bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR0 (0x00014053 + 0x00000000) 12902bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR1 (0x00014053 + 0x00000800) 13002bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR2 (0x00014053 + 0x00001000) 13102bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR3 (0x00014053 + 0x00001800) 13202bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR4 (0x00054053 + 0x00000000) 13302bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR5 (0x00054053 + 0x00000800) 13402bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR6 (0x00054053 + 0x00001000) 13502bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR7 (0x00054053 + 0x00001800) 13602bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR8 (0x00094053 + 0x00000000) 13702bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR9 (0x00094053 + 0x00000800) 13802bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR10 (0x00094053 + 0x00001000) 13902bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR11 (0x00094053 + 0x00001800) 14002bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR12 (0x000d4053 + 0x00000000) 14102bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR13 (0x000d4053 + 0x00000800) 14202bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR14 (0x000d4053 + 0x00001000) 14302bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR15 (0x000d4053 + 0x00001800) 14402bab923SDavid Panariti 14502bab923SDavid Panariti static const uint32_t ecc_umclocalcap_addrs[] = { 14602bab923SDavid Panariti UMCLOCALCAPS_ADDR0, 14702bab923SDavid Panariti UMCLOCALCAPS_ADDR1, 14802bab923SDavid Panariti UMCLOCALCAPS_ADDR2, 14902bab923SDavid Panariti UMCLOCALCAPS_ADDR3, 15002bab923SDavid Panariti UMCLOCALCAPS_ADDR4, 15102bab923SDavid Panariti UMCLOCALCAPS_ADDR5, 15202bab923SDavid Panariti UMCLOCALCAPS_ADDR6, 15302bab923SDavid Panariti UMCLOCALCAPS_ADDR7, 15402bab923SDavid Panariti UMCLOCALCAPS_ADDR8, 15502bab923SDavid Panariti UMCLOCALCAPS_ADDR9, 15602bab923SDavid Panariti UMCLOCALCAPS_ADDR10, 15702bab923SDavid Panariti UMCLOCALCAPS_ADDR11, 15802bab923SDavid Panariti UMCLOCALCAPS_ADDR12, 15902bab923SDavid Panariti UMCLOCALCAPS_ADDR13, 16002bab923SDavid Panariti UMCLOCALCAPS_ADDR14, 16102bab923SDavid Panariti UMCLOCALCAPS_ADDR15, 16202bab923SDavid Panariti }; 16302bab923SDavid Panariti 16402bab923SDavid Panariti static const uint32_t ecc_umcch_umc_config_addrs[] = { 16502bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR0, 16602bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR1, 16702bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR2, 16802bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR3, 16902bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR4, 17002bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR5, 17102bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR6, 17202bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR7, 17302bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR8, 17402bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR9, 17502bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR10, 17602bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR11, 17702bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR12, 17802bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR13, 17902bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR14, 18002bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR15, 18102bab923SDavid Panariti }; 18202bab923SDavid Panariti 18302bab923SDavid Panariti static const uint32_t ecc_umcch_eccctrl_addrs[] = { 18402bab923SDavid Panariti UMCCH_ECCCTRL_ADDR0, 18502bab923SDavid Panariti UMCCH_ECCCTRL_ADDR1, 18602bab923SDavid Panariti UMCCH_ECCCTRL_ADDR2, 18702bab923SDavid Panariti UMCCH_ECCCTRL_ADDR3, 18802bab923SDavid Panariti UMCCH_ECCCTRL_ADDR4, 18902bab923SDavid Panariti UMCCH_ECCCTRL_ADDR5, 19002bab923SDavid Panariti UMCCH_ECCCTRL_ADDR6, 19102bab923SDavid Panariti UMCCH_ECCCTRL_ADDR7, 19202bab923SDavid Panariti UMCCH_ECCCTRL_ADDR8, 19302bab923SDavid Panariti UMCCH_ECCCTRL_ADDR9, 19402bab923SDavid Panariti UMCCH_ECCCTRL_ADDR10, 19502bab923SDavid Panariti UMCCH_ECCCTRL_ADDR11, 19602bab923SDavid Panariti UMCCH_ECCCTRL_ADDR12, 19702bab923SDavid Panariti UMCCH_ECCCTRL_ADDR13, 19802bab923SDavid Panariti UMCCH_ECCCTRL_ADDR14, 19902bab923SDavid Panariti UMCCH_ECCCTRL_ADDR15, 20002bab923SDavid Panariti }; 20102bab923SDavid Panariti 202e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 203e60f8db5SAlex Xie struct amdgpu_irq_src *src, 204e60f8db5SAlex Xie unsigned type, 205e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 206e60f8db5SAlex Xie { 207e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 208ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 209e60f8db5SAlex Xie 21011250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21111250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21211250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21311250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21411250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21511250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21611250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 21711250164SChristian König 218e60f8db5SAlex Xie switch (state) { 219e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 220ae6d1416STom St Denis for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 221ae6d1416STom St Denis hub = &adev->vmhub[j]; 222e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 223e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 224e60f8db5SAlex Xie tmp = RREG32(reg); 225e60f8db5SAlex Xie tmp &= ~bits; 226e60f8db5SAlex Xie WREG32(reg, tmp); 227e60f8db5SAlex Xie } 228e60f8db5SAlex Xie } 229e60f8db5SAlex Xie break; 230e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 231ae6d1416STom St Denis for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 232ae6d1416STom St Denis hub = &adev->vmhub[j]; 233e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 234e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 235e60f8db5SAlex Xie tmp = RREG32(reg); 236e60f8db5SAlex Xie tmp |= bits; 237e60f8db5SAlex Xie WREG32(reg, tmp); 238e60f8db5SAlex Xie } 239e60f8db5SAlex Xie } 240e60f8db5SAlex Xie default: 241e60f8db5SAlex Xie break; 242e60f8db5SAlex Xie } 243e60f8db5SAlex Xie 244e60f8db5SAlex Xie return 0; 245e60f8db5SAlex Xie } 246e60f8db5SAlex Xie 247e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 248e60f8db5SAlex Xie struct amdgpu_irq_src *source, 249e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 250e60f8db5SAlex Xie { 251c4f46f22SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src]; 2524d6cbde3SFelix Kuehling uint32_t status = 0; 253e60f8db5SAlex Xie u64 addr; 254e60f8db5SAlex Xie 255e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 256e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 257e60f8db5SAlex Xie 25879a0c465SMonk Liu if (!amdgpu_sriov_vf(adev)) { 2595a9b8e8aSChristian König status = RREG32(hub->vm_l2_pro_fault_status); 2605a9b8e8aSChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 2614d6cbde3SFelix Kuehling } 262e60f8db5SAlex Xie 2634d6cbde3SFelix Kuehling if (printk_ratelimit()) { 264efaa9646SAndrey Grodzovsky struct amdgpu_task_info task_info = { 0 }; 265efaa9646SAndrey Grodzovsky 266efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 267efaa9646SAndrey Grodzovsky 2684d6cbde3SFelix Kuehling dev_err(adev->dev, 2690c79c0bbSChristian König "[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n", 270c4f46f22SChristian König entry->vmid_src ? "mmhub" : "gfxhub", 271c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 272efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 273efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 2747d0aa376SAndrey Grodzovsky dev_err(adev->dev, " in page starting at address 0x%016llx from %d\n", 27579a0c465SMonk Liu addr, entry->client_id); 2764d6cbde3SFelix Kuehling if (!amdgpu_sriov_vf(adev)) 2774d6cbde3SFelix Kuehling dev_err(adev->dev, 2784d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 2794d6cbde3SFelix Kuehling status); 28079a0c465SMonk Liu } 281e60f8db5SAlex Xie 282e60f8db5SAlex Xie return 0; 283e60f8db5SAlex Xie } 284e60f8db5SAlex Xie 285e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 286e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 287e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 288e60f8db5SAlex Xie }; 289e60f8db5SAlex Xie 290e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 291e60f8db5SAlex Xie { 292770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 293770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 294e60f8db5SAlex Xie } 295e60f8db5SAlex Xie 2962a79d868SYong Zhao static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 2972a79d868SYong Zhao uint32_t flush_type) 29803f89febSChristian König { 29903f89febSChristian König u32 req = 0; 30003f89febSChristian König 30103f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 302c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 3032a79d868SYong Zhao req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 30403f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 30503f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 30603f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 30703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 30803f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 30903f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 31003f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 31103f89febSChristian König 31203f89febSChristian König return req; 31303f89febSChristian König } 31403f89febSChristian König 315e60f8db5SAlex Xie /* 316e60f8db5SAlex Xie * GART 317e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 318e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 319e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 320e60f8db5SAlex Xie */ 321e60f8db5SAlex Xie 322e60f8db5SAlex Xie /** 3232a79d868SYong Zhao * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 324e60f8db5SAlex Xie * 325e60f8db5SAlex Xie * @adev: amdgpu_device pointer 326e60f8db5SAlex Xie * @vmid: vm instance to flush 3272a79d868SYong Zhao * @flush_type: the flush type 328e60f8db5SAlex Xie * 3292a79d868SYong Zhao * Flush the TLB for the requested page table using certain type. 330e60f8db5SAlex Xie */ 331132f34e4SChristian König static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, 3322a79d868SYong Zhao uint32_t vmid, uint32_t flush_type) 333e60f8db5SAlex Xie { 334e60f8db5SAlex Xie const unsigned eng = 17; 335e60f8db5SAlex Xie unsigned i, j; 336e60f8db5SAlex Xie 337e60f8db5SAlex Xie for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 338e60f8db5SAlex Xie struct amdgpu_vmhub *hub = &adev->vmhub[i]; 3392a79d868SYong Zhao u32 tmp = gmc_v9_0_get_invalidate_req(vmid, flush_type); 340e60f8db5SAlex Xie 34182d1a1b1SChengming Gui /* This is necessary for a HW workaround under SRIOV as well 34282d1a1b1SChengming Gui * as GFXOFF under bare metal 34382d1a1b1SChengming Gui */ 34482d1a1b1SChengming Gui if (adev->gfx.kiq.ring.sched.ready && 34582d1a1b1SChengming Gui (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 34682d1a1b1SChengming Gui !adev->in_gpu_reset) { 347af5fe1e9SChristian König uint32_t req = hub->vm_inv_eng0_req + eng; 348af5fe1e9SChristian König uint32_t ack = hub->vm_inv_eng0_ack + eng; 349af5fe1e9SChristian König 350af5fe1e9SChristian König amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, tmp, 351af5fe1e9SChristian König 1 << vmid); 3523890d111SEmily Deng continue; 353fc0faf04SEmily Deng } 3543890d111SEmily Deng 3553890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 356c7a7266bSXiangliang Yu WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp); 357e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 358c7a7266bSXiangliang Yu tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng); 359396557b0SChristian König if (tmp & (1 << vmid)) 360e60f8db5SAlex Xie break; 361e60f8db5SAlex Xie udelay(1); 362e60f8db5SAlex Xie } 3633890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 364396557b0SChristian König if (j < adev->usec_timeout) 365e60f8db5SAlex Xie continue; 366396557b0SChristian König 367e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 368e60f8db5SAlex Xie } 369e60f8db5SAlex Xie } 370e60f8db5SAlex Xie 3719096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 372c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 3739096d6e5SChristian König { 374250b4228SChristian König struct amdgpu_device *adev = ring->adev; 375250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 3762a79d868SYong Zhao uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 3779096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 3789096d6e5SChristian König 3799096d6e5SChristian König amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid), 3809096d6e5SChristian König lower_32_bits(pd_addr)); 3819096d6e5SChristian König 3829096d6e5SChristian König amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid), 3839096d6e5SChristian König upper_32_bits(pd_addr)); 3849096d6e5SChristian König 385f8bc9037SAlex Deucher amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng, 386f8bc9037SAlex Deucher hub->vm_inv_eng0_ack + eng, 387f8bc9037SAlex Deucher req, 1 << vmid); 388f732b6b3SChristian König 3899096d6e5SChristian König return pd_addr; 3909096d6e5SChristian König } 3919096d6e5SChristian König 392c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 393c633c00bSChristian König unsigned pasid) 394c633c00bSChristian König { 395c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 396c633c00bSChristian König uint32_t reg; 397c633c00bSChristian König 398c633c00bSChristian König if (ring->funcs->vmhub == AMDGPU_GFXHUB) 399c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 400c633c00bSChristian König else 401c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 402c633c00bSChristian König 403c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 404c633c00bSChristian König } 405c633c00bSChristian König 406e60f8db5SAlex Xie /** 407132f34e4SChristian König * gmc_v9_0_set_pte_pde - update the page tables using MMIO 408e60f8db5SAlex Xie * 409e60f8db5SAlex Xie * @adev: amdgpu_device pointer 410e60f8db5SAlex Xie * @cpu_pt_addr: cpu address of the page table 411e60f8db5SAlex Xie * @gpu_page_idx: entry in the page table to update 412e60f8db5SAlex Xie * @addr: dst addr to write into pte/pde 413e60f8db5SAlex Xie * @flags: access flags 414e60f8db5SAlex Xie * 415e60f8db5SAlex Xie * Update the page tables using the CPU. 416e60f8db5SAlex Xie */ 417132f34e4SChristian König static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, 418132f34e4SChristian König uint32_t gpu_page_idx, uint64_t addr, 419e60f8db5SAlex Xie uint64_t flags) 420e60f8db5SAlex Xie { 421e60f8db5SAlex Xie void __iomem *ptr = (void *)cpu_pt_addr; 422e60f8db5SAlex Xie uint64_t value; 423e60f8db5SAlex Xie 424e60f8db5SAlex Xie /* 425e60f8db5SAlex Xie * PTE format on VEGA 10: 426e60f8db5SAlex Xie * 63:59 reserved 427e60f8db5SAlex Xie * 58:57 mtype 428e60f8db5SAlex Xie * 56 F 429e60f8db5SAlex Xie * 55 L 430e60f8db5SAlex Xie * 54 P 431e60f8db5SAlex Xie * 53 SW 432e60f8db5SAlex Xie * 52 T 433e60f8db5SAlex Xie * 50:48 reserved 434e60f8db5SAlex Xie * 47:12 4k physical page base address 435e60f8db5SAlex Xie * 11:7 fragment 436e60f8db5SAlex Xie * 6 write 437e60f8db5SAlex Xie * 5 read 438e60f8db5SAlex Xie * 4 exe 439e60f8db5SAlex Xie * 3 Z 440e60f8db5SAlex Xie * 2 snooped 441e60f8db5SAlex Xie * 1 system 442e60f8db5SAlex Xie * 0 valid 443e60f8db5SAlex Xie * 444e60f8db5SAlex Xie * PDE format on VEGA 10: 445e60f8db5SAlex Xie * 63:59 block fragment size 446e60f8db5SAlex Xie * 58:55 reserved 447e60f8db5SAlex Xie * 54 P 448e60f8db5SAlex Xie * 53:48 reserved 449e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 450e60f8db5SAlex Xie * 5:3 reserved 451e60f8db5SAlex Xie * 2 C 452e60f8db5SAlex Xie * 1 system 453e60f8db5SAlex Xie * 0 valid 454e60f8db5SAlex Xie */ 455e60f8db5SAlex Xie 456e60f8db5SAlex Xie /* 457e60f8db5SAlex Xie * The following is for PTE only. GART does not have PDEs. 458e60f8db5SAlex Xie */ 459e60f8db5SAlex Xie value = addr & 0x0000FFFFFFFFF000ULL; 460e60f8db5SAlex Xie value |= flags; 461e60f8db5SAlex Xie writeq(value, ptr + (gpu_page_idx * 8)); 462e60f8db5SAlex Xie return 0; 463e60f8db5SAlex Xie } 464e60f8db5SAlex Xie 465e60f8db5SAlex Xie static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev, 466e60f8db5SAlex Xie uint32_t flags) 467e60f8db5SAlex Xie 468e60f8db5SAlex Xie { 469e60f8db5SAlex Xie uint64_t pte_flag = 0; 470e60f8db5SAlex Xie 471e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_EXECUTABLE) 472e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_EXECUTABLE; 473e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_READABLE) 474e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_READABLE; 475e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_WRITEABLE) 476e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_WRITEABLE; 477e60f8db5SAlex Xie 478e60f8db5SAlex Xie switch (flags & AMDGPU_VM_MTYPE_MASK) { 479e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 480e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 481e60f8db5SAlex Xie break; 482e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 483e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 484e60f8db5SAlex Xie break; 485e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 486e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC); 487e60f8db5SAlex Xie break; 488e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 489e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC); 490e60f8db5SAlex Xie break; 491e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 492e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC); 493e60f8db5SAlex Xie break; 494e60f8db5SAlex Xie default: 495e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 496e60f8db5SAlex Xie break; 497e60f8db5SAlex Xie } 498e60f8db5SAlex Xie 499e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_PRT) 500e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_PRT; 501e60f8db5SAlex Xie 502e60f8db5SAlex Xie return pte_flag; 503e60f8db5SAlex Xie } 504e60f8db5SAlex Xie 5053de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 5063de676d8SChristian König uint64_t *addr, uint64_t *flags) 507f75e237cSChristian König { 508bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 5093de676d8SChristian König *addr = adev->vm_manager.vram_base_offset + *addr - 510770d13b1SChristian König adev->gmc.vram_start; 5113de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 5126a42fd6fSChristian König 513770d13b1SChristian König if (!adev->gmc.translate_further) 5146a42fd6fSChristian König return; 5156a42fd6fSChristian König 5166a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 5176a42fd6fSChristian König /* Set the block fragment size */ 5186a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 5196a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 5206a42fd6fSChristian König 5216a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 5226a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 5236a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 5246a42fd6fSChristian König else 5256a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 5266a42fd6fSChristian König } 527f75e237cSChristian König } 528f75e237cSChristian König 529132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 530132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 5319096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 532c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 533132f34e4SChristian König .set_pte_pde = gmc_v9_0_set_pte_pde, 534b1166325SChristian König .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags, 535b1166325SChristian König .get_vm_pde = gmc_v9_0_get_vm_pde 536e60f8db5SAlex Xie }; 537e60f8db5SAlex Xie 538132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 539e60f8db5SAlex Xie { 540132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 541e60f8db5SAlex Xie } 542e60f8db5SAlex Xie 543e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 544e60f8db5SAlex Xie { 545e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 546e60f8db5SAlex Xie 547132f34e4SChristian König gmc_v9_0_set_gmc_funcs(adev); 548e60f8db5SAlex Xie gmc_v9_0_set_irq_funcs(adev); 549e60f8db5SAlex Xie 550770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 551770d13b1SChristian König adev->gmc.shared_aperture_end = 552770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 553bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 554770d13b1SChristian König adev->gmc.private_aperture_end = 555770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 556a7ea6548SAlex Deucher 557e60f8db5SAlex Xie return 0; 558e60f8db5SAlex Xie } 559e60f8db5SAlex Xie 56002bab923SDavid Panariti static int gmc_v9_0_ecc_available(struct amdgpu_device *adev) 56102bab923SDavid Panariti { 56202bab923SDavid Panariti uint32_t reg_val; 56302bab923SDavid Panariti uint32_t reg_addr; 56402bab923SDavid Panariti uint32_t field_val; 56502bab923SDavid Panariti size_t i; 56602bab923SDavid Panariti uint32_t fv2; 56702bab923SDavid Panariti size_t lost_sheep; 56802bab923SDavid Panariti 56902bab923SDavid Panariti DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n"); 57002bab923SDavid Panariti 57102bab923SDavid Panariti lost_sheep = 0; 57202bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) { 57302bab923SDavid Panariti reg_addr = ecc_umclocalcap_addrs[i]; 57402bab923SDavid Panariti DRM_DEBUG("ecc: " 57502bab923SDavid Panariti "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n", 57602bab923SDavid Panariti i, reg_addr); 57702bab923SDavid Panariti reg_val = RREG32(reg_addr); 57802bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap, 57902bab923SDavid Panariti EccDis); 58002bab923SDavid Panariti DRM_DEBUG("ecc: " 58102bab923SDavid Panariti "reg_val: 0x%08x, " 58202bab923SDavid Panariti "EccDis: 0x%08x, ", 58302bab923SDavid Panariti reg_val, field_val); 58402bab923SDavid Panariti if (field_val) { 58502bab923SDavid Panariti DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n"); 58602bab923SDavid Panariti ++lost_sheep; 58702bab923SDavid Panariti } 58802bab923SDavid Panariti } 58902bab923SDavid Panariti 59002bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) { 59102bab923SDavid Panariti reg_addr = ecc_umcch_umc_config_addrs[i]; 59202bab923SDavid Panariti DRM_DEBUG("ecc: " 59302bab923SDavid Panariti "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x", 59402bab923SDavid Panariti i, reg_addr); 59502bab923SDavid Panariti reg_val = RREG32(reg_addr); 59602bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG, 59702bab923SDavid Panariti DramReady); 59802bab923SDavid Panariti DRM_DEBUG("ecc: " 59902bab923SDavid Panariti "reg_val: 0x%08x, " 60002bab923SDavid Panariti "DramReady: 0x%08x\n", 60102bab923SDavid Panariti reg_val, field_val); 60202bab923SDavid Panariti 60302bab923SDavid Panariti if (!field_val) { 60402bab923SDavid Panariti DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n"); 60502bab923SDavid Panariti ++lost_sheep; 60602bab923SDavid Panariti } 60702bab923SDavid Panariti } 60802bab923SDavid Panariti 60902bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) { 61002bab923SDavid Panariti reg_addr = ecc_umcch_eccctrl_addrs[i]; 61102bab923SDavid Panariti DRM_DEBUG("ecc: " 61202bab923SDavid Panariti "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ", 61302bab923SDavid Panariti i, reg_addr); 61402bab923SDavid Panariti reg_val = RREG32(reg_addr); 61502bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl, 61602bab923SDavid Panariti WrEccEn); 61702bab923SDavid Panariti fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl, 61802bab923SDavid Panariti RdEccEn); 61902bab923SDavid Panariti DRM_DEBUG("ecc: " 62002bab923SDavid Panariti "reg_val: 0x%08x, " 62102bab923SDavid Panariti "WrEccEn: 0x%08x, " 62202bab923SDavid Panariti "RdEccEn: 0x%08x\n", 62302bab923SDavid Panariti reg_val, field_val, fv2); 62402bab923SDavid Panariti 62502bab923SDavid Panariti if (!field_val) { 6265a16008fSAlex Deucher DRM_DEBUG("ecc: WrEccEn is not set\n"); 62702bab923SDavid Panariti ++lost_sheep; 62802bab923SDavid Panariti } 62902bab923SDavid Panariti if (!fv2) { 6305a16008fSAlex Deucher DRM_DEBUG("ecc: RdEccEn is not set\n"); 63102bab923SDavid Panariti ++lost_sheep; 63202bab923SDavid Panariti } 63302bab923SDavid Panariti } 63402bab923SDavid Panariti 63502bab923SDavid Panariti DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep); 63602bab923SDavid Panariti return lost_sheep == 0; 63702bab923SDavid Panariti } 63802bab923SDavid Panariti 639cd2b5623SAlex Deucher static bool gmc_v9_0_keep_stolen_memory(struct amdgpu_device *adev) 640cd2b5623SAlex Deucher { 641cd2b5623SAlex Deucher 642cd2b5623SAlex Deucher /* 643cd2b5623SAlex Deucher * TODO: 644cd2b5623SAlex Deucher * Currently there is a bug where some memory client outside 645cd2b5623SAlex Deucher * of the driver writes to first 8M of VRAM on S3 resume, 646cd2b5623SAlex Deucher * this overrides GART which by default gets placed in first 8M and 647cd2b5623SAlex Deucher * causes VM_FAULTS once GTT is accessed. 648cd2b5623SAlex Deucher * Keep the stolen memory reservation until the while this is not solved. 649cd2b5623SAlex Deucher * Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init 650cd2b5623SAlex Deucher */ 651cd2b5623SAlex Deucher switch (adev->asic_type) { 65295010ba7SAlex Deucher case CHIP_VEGA10: 6536abc0c8fSAlex Deucher return true; 6546abc0c8fSAlex Deucher case CHIP_RAVEN: 6556abc0c8fSAlex Deucher case CHIP_VEGA12: 656cd2b5623SAlex Deucher case CHIP_VEGA20: 657cd2b5623SAlex Deucher default: 6586abc0c8fSAlex Deucher return false; 659cd2b5623SAlex Deucher } 660cd2b5623SAlex Deucher } 661cd2b5623SAlex Deucher 662e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 663e60f8db5SAlex Xie { 664e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 665c5066129Sozeng /* 666c5066129Sozeng * The latest engine allocation on gfx9 is: 667c5066129Sozeng * Engine 0, 1: idle 668c5066129Sozeng * Engine 2, 3: firmware 669c5066129Sozeng * Engine 4~13: amdgpu ring, subject to change when ring number changes 670c5066129Sozeng * Engine 14~15: idle 671c5066129Sozeng * Engine 16: kfd tlb invalidation 672c5066129Sozeng * Engine 17: Gart flushes 673c5066129Sozeng */ 674c5066129Sozeng unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 }; 6754789c463SChristian König unsigned i; 67602bab923SDavid Panariti int r; 6774789c463SChristian König 678cd2b5623SAlex Deucher if (!gmc_v9_0_keep_stolen_memory(adev)) 679cd2b5623SAlex Deucher amdgpu_bo_late_init(adev); 6806f752ec2SAndrey Grodzovsky 6814789c463SChristian König for(i = 0; i < adev->num_rings; ++i) { 6824789c463SChristian König struct amdgpu_ring *ring = adev->rings[i]; 6834789c463SChristian König unsigned vmhub = ring->funcs->vmhub; 6844789c463SChristian König 6854789c463SChristian König ring->vm_inv_eng = vm_inv_eng[vmhub]++; 6866e82c6e0SChristian König dev_info(adev->dev, "ring %s uses VM inv eng %u on hub %u\n", 6876e82c6e0SChristian König ring->name, ring->vm_inv_eng, ring->funcs->vmhub); 6884789c463SChristian König } 6894789c463SChristian König 690c5066129Sozeng /* Engine 16 is used for KFD and 17 for GART flushes */ 6914789c463SChristian König for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i) 692c5066129Sozeng BUG_ON(vm_inv_eng[i] > 16); 6934789c463SChristian König 6947b6cbae2SMonk Liu if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) { 69502bab923SDavid Panariti r = gmc_v9_0_ecc_available(adev); 69602bab923SDavid Panariti if (r == 1) { 69702bab923SDavid Panariti DRM_INFO("ECC is active.\n"); 69802bab923SDavid Panariti } else if (r == 0) { 69902bab923SDavid Panariti DRM_INFO("ECC is not present.\n"); 700e1d1a772SAlex Deucher adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false); 70102bab923SDavid Panariti } else { 70202bab923SDavid Panariti DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r); 70302bab923SDavid Panariti return r; 70402bab923SDavid Panariti } 7055ba4fa35SAlex Deucher } 70602bab923SDavid Panariti 707770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 708e60f8db5SAlex Xie } 709e60f8db5SAlex Xie 710e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 711770d13b1SChristian König struct amdgpu_gmc *mc) 712e60f8db5SAlex Xie { 713eeb2487dSMonk Liu u64 base = 0; 714eeb2487dSMonk Liu if (!amdgpu_sriov_vf(adev)) 715eeb2487dSMonk Liu base = mmhub_v1_0_get_fb_location(adev); 7166fdd68b1SAlex Deucher /* add the xgmi offset of the physical node */ 7176fdd68b1SAlex Deucher base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 718961c75cfSChristian König amdgpu_gmc_vram_location(adev, &adev->gmc, base); 719961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 720c3e1b43cSChristian König if (!amdgpu_sriov_vf(adev)) 721c3e1b43cSChristian König amdgpu_gmc_agp_location(adev, mc); 722bc099ee9SChunming Zhou /* base offset of vram pages */ 723bc099ee9SChunming Zhou adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev); 7246fdd68b1SAlex Deucher 7256fdd68b1SAlex Deucher /* XXX: add the xgmi offset of the physical node? */ 7266fdd68b1SAlex Deucher adev->vm_manager.vram_base_offset += 7276fdd68b1SAlex Deucher adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 728e60f8db5SAlex Xie } 729e60f8db5SAlex Xie 730e60f8db5SAlex Xie /** 731e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 732e60f8db5SAlex Xie * 733e60f8db5SAlex Xie * @adev: amdgpu_device pointer 734e60f8db5SAlex Xie * 735e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 736e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 737e60f8db5SAlex Xie * Returns 0 for success. 738e60f8db5SAlex Xie */ 739e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 740e60f8db5SAlex Xie { 741e60f8db5SAlex Xie int chansize, numchan; 742d6895ad3SChristian König int r; 743e60f8db5SAlex Xie 7443d918c0eSShaoyun Liu if (amdgpu_emu_mode != 1) 745770d13b1SChristian König adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev); 746770d13b1SChristian König if (!adev->gmc.vram_width) { 747e60f8db5SAlex Xie /* hbm memory channel size */ 748585b7f16STom St Denis if (adev->flags & AMD_IS_APU) 749585b7f16STom St Denis chansize = 64; 750585b7f16STom St Denis else 751e60f8db5SAlex Xie chansize = 128; 752e60f8db5SAlex Xie 753070706c0SHawking Zhang numchan = adev->df_funcs->get_hbm_channel_number(adev); 754770d13b1SChristian König adev->gmc.vram_width = numchan * chansize; 7558d6a5230SAlex Deucher } 756e60f8db5SAlex Xie 757e60f8db5SAlex Xie /* size in MB on si */ 758770d13b1SChristian König adev->gmc.mc_vram_size = 759bf383fb6SAlex Deucher adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL; 760770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 761d6895ad3SChristian König 762d6895ad3SChristian König if (!(adev->flags & AMD_IS_APU)) { 763d6895ad3SChristian König r = amdgpu_device_resize_fb_bar(adev); 764d6895ad3SChristian König if (r) 765d6895ad3SChristian König return r; 766d6895ad3SChristian König } 767770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 768770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 769e60f8db5SAlex Xie 770156a81beSChunming Zhou #ifdef CONFIG_X86_64 771156a81beSChunming Zhou if (adev->flags & AMD_IS_APU) { 772156a81beSChunming Zhou adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev); 773156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 774156a81beSChunming Zhou } 775156a81beSChunming Zhou #endif 776e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 777770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 778770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 779770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 780e60f8db5SAlex Xie 781c3db7b5aSAlex Deucher /* set the gart size */ 782c3db7b5aSAlex Deucher if (amdgpu_gart_size == -1) { 783c3db7b5aSAlex Deucher switch (adev->asic_type) { 784c3db7b5aSAlex Deucher case CHIP_VEGA10: /* all engines support GPUVM */ 785273a14cdSAlex Deucher case CHIP_VEGA12: /* all engines support GPUVM */ 786d96b428cSFeifei Xu case CHIP_VEGA20: 787c3db7b5aSAlex Deucher default: 788fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 789c3db7b5aSAlex Deucher break; 790c3db7b5aSAlex Deucher case CHIP_RAVEN: /* DCE SG support */ 791770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 792c3db7b5aSAlex Deucher break; 793c3db7b5aSAlex Deucher } 794c3db7b5aSAlex Deucher } else { 795770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 796c3db7b5aSAlex Deucher } 797c3db7b5aSAlex Deucher 798770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 799e60f8db5SAlex Xie 800e60f8db5SAlex Xie return 0; 801e60f8db5SAlex Xie } 802e60f8db5SAlex Xie 803e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 804e60f8db5SAlex Xie { 805e60f8db5SAlex Xie int r; 806e60f8db5SAlex Xie 8071123b989SChristian König if (adev->gart.bo) { 808e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 809e60f8db5SAlex Xie return 0; 810e60f8db5SAlex Xie } 811e60f8db5SAlex Xie /* Initialize common gart structure */ 812e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 813e60f8db5SAlex Xie if (r) 814e60f8db5SAlex Xie return r; 815e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 816e60f8db5SAlex Xie adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) | 817e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 818e60f8db5SAlex Xie return amdgpu_gart_table_vram_alloc(adev); 819e60f8db5SAlex Xie } 820e60f8db5SAlex Xie 821ebdef28eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 822ebdef28eSAlex Deucher { 823ebdef28eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 824ebdef28eSAlex Deucher unsigned size; 825ebdef28eSAlex Deucher 8266f752ec2SAndrey Grodzovsky /* 8276f752ec2SAndrey Grodzovsky * TODO Remove once GART corruption is resolved 8286f752ec2SAndrey Grodzovsky * Check related code in gmc_v9_0_sw_fini 8296f752ec2SAndrey Grodzovsky * */ 830cd2b5623SAlex Deucher if (gmc_v9_0_keep_stolen_memory(adev)) 831cd2b5623SAlex Deucher return 9 * 1024 * 1024; 8326f752ec2SAndrey Grodzovsky 833ebdef28eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 834ebdef28eSAlex Deucher size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ 835ebdef28eSAlex Deucher } else { 836ebdef28eSAlex Deucher u32 viewport; 837ebdef28eSAlex Deucher 838ebdef28eSAlex Deucher switch (adev->asic_type) { 839ebdef28eSAlex Deucher case CHIP_RAVEN: 840ebdef28eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 841ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, 842ebdef28eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 843ebdef28eSAlex Deucher REG_GET_FIELD(viewport, 844ebdef28eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 845ebdef28eSAlex Deucher 4); 846ebdef28eSAlex Deucher break; 847ebdef28eSAlex Deucher case CHIP_VEGA10: 848ebdef28eSAlex Deucher case CHIP_VEGA12: 849cd2b5623SAlex Deucher case CHIP_VEGA20: 850ebdef28eSAlex Deucher default: 851ebdef28eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 852ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 853ebdef28eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 854ebdef28eSAlex Deucher 4); 855ebdef28eSAlex Deucher break; 856ebdef28eSAlex Deucher } 857ebdef28eSAlex Deucher } 858ebdef28eSAlex Deucher /* return 0 if the pre-OS buffer uses up most of vram */ 859ebdef28eSAlex Deucher if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 860ebdef28eSAlex Deucher return 0; 8616f752ec2SAndrey Grodzovsky 862ebdef28eSAlex Deucher return size; 863ebdef28eSAlex Deucher } 864ebdef28eSAlex Deucher 865e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 866e60f8db5SAlex Xie { 867e60f8db5SAlex Xie int r; 868e60f8db5SAlex Xie int dma_bits; 869e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 870e60f8db5SAlex Xie 8710c8c0847SHuang Rui gfxhub_v1_0_init(adev); 87277f6c763SHuang Rui mmhub_v1_0_init(adev); 8730c8c0847SHuang Rui 874770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 875e60f8db5SAlex Xie 8761e09b053SHawking Zhang adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev); 877fd66560bSHawking Zhang switch (adev->asic_type) { 878fd66560bSHawking Zhang case CHIP_RAVEN: 8796a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 880f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 8816a42fd6fSChristian König } else { 8826a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 8836a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 884770d13b1SChristian König adev->gmc.translate_further = 8856a42fd6fSChristian König adev->vm_manager.num_level > 1; 8866a42fd6fSChristian König } 887fd66560bSHawking Zhang break; 888fd66560bSHawking Zhang case CHIP_VEGA10: 889273a14cdSAlex Deucher case CHIP_VEGA12: 890d96b428cSFeifei Xu case CHIP_VEGA20: 89136b32a68SZhang, Jerry /* 89236b32a68SZhang, Jerry * To fulfill 4-level page support, 89336b32a68SZhang, Jerry * vm size is 256TB (48bit), maximum size of Vega10, 89436b32a68SZhang, Jerry * block size 512 (9bit) 89536b32a68SZhang, Jerry */ 896f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 897fd66560bSHawking Zhang break; 898fd66560bSHawking Zhang default: 899fd66560bSHawking Zhang break; 900fd66560bSHawking Zhang } 901fd66560bSHawking Zhang 902e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 90344a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 904770d13b1SChristian König &adev->gmc.vm_fault); 90544a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 906770d13b1SChristian König &adev->gmc.vm_fault); 907e60f8db5SAlex Xie 908e60f8db5SAlex Xie if (r) 909e60f8db5SAlex Xie return r; 910e60f8db5SAlex Xie 911e60f8db5SAlex Xie /* Set the internal MC address mask 912e60f8db5SAlex Xie * This is the max address of the GPU's 913e60f8db5SAlex Xie * internal address space. 914e60f8db5SAlex Xie */ 915770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 916e60f8db5SAlex Xie 917e60f8db5SAlex Xie /* set DMA mask + need_dma32 flags. 918e60f8db5SAlex Xie * PCIE - can handle 44-bits. 919e60f8db5SAlex Xie * IGP - can handle 44-bits 920e60f8db5SAlex Xie * PCI - dma32 for legacy pci gart, 44 bits on vega10 921e60f8db5SAlex Xie */ 922e60f8db5SAlex Xie adev->need_dma32 = false; 923e60f8db5SAlex Xie dma_bits = adev->need_dma32 ? 32 : 44; 924e60f8db5SAlex Xie r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 925e60f8db5SAlex Xie if (r) { 926e60f8db5SAlex Xie adev->need_dma32 = true; 927e60f8db5SAlex Xie dma_bits = 32; 928e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 929e60f8db5SAlex Xie } 930e60f8db5SAlex Xie r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 931e60f8db5SAlex Xie if (r) { 932e60f8db5SAlex Xie pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32)); 933e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No coherent DMA available.\n"); 934e60f8db5SAlex Xie } 935fd5fd480SChunming Zhou adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits); 936e60f8db5SAlex Xie 937bf0a60b7SAlex Deucher if (adev->asic_type == CHIP_VEGA20) { 938bf0a60b7SAlex Deucher r = gfxhub_v1_1_get_xgmi_info(adev); 939bf0a60b7SAlex Deucher if (r) 940bf0a60b7SAlex Deucher return r; 941bf0a60b7SAlex Deucher } 942bf0a60b7SAlex Deucher 943e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 944e60f8db5SAlex Xie if (r) 945e60f8db5SAlex Xie return r; 946e60f8db5SAlex Xie 947ebdef28eSAlex Deucher adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev); 948ebdef28eSAlex Deucher 949e60f8db5SAlex Xie /* Memory manager */ 950e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 951e60f8db5SAlex Xie if (r) 952e60f8db5SAlex Xie return r; 953e60f8db5SAlex Xie 954e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 955e60f8db5SAlex Xie if (r) 956e60f8db5SAlex Xie return r; 957e60f8db5SAlex Xie 95805ec3edaSChristian König /* 95905ec3edaSChristian König * number of VMs 96005ec3edaSChristian König * VMID 0 is reserved for System 96105ec3edaSChristian König * amdgpu graphics/compute will use VMIDs 1-7 96205ec3edaSChristian König * amdkfd will use VMIDs 8-15 96305ec3edaSChristian König */ 96405ec3edaSChristian König adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 96505ec3edaSChristian König adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 96605ec3edaSChristian König 96705ec3edaSChristian König amdgpu_vm_manager_init(adev); 96805ec3edaSChristian König 96905ec3edaSChristian König return 0; 970e60f8db5SAlex Xie } 971e60f8db5SAlex Xie 972e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 973e60f8db5SAlex Xie { 974e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 975e60f8db5SAlex Xie 976f59548c8SMonk Liu amdgpu_gem_force_release(adev); 977e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 9786f752ec2SAndrey Grodzovsky 979cd2b5623SAlex Deucher if (gmc_v9_0_keep_stolen_memory(adev)) 9806f752ec2SAndrey Grodzovsky amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); 9816f752ec2SAndrey Grodzovsky 982a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 983e60f8db5SAlex Xie amdgpu_bo_fini(adev); 984a3d9103eSAndrey Grodzovsky amdgpu_gart_fini(adev); 985e60f8db5SAlex Xie 986e60f8db5SAlex Xie return 0; 987e60f8db5SAlex Xie } 988e60f8db5SAlex Xie 989e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 990e60f8db5SAlex Xie { 991946a4d5bSShaoyun Liu 992e60f8db5SAlex Xie switch (adev->asic_type) { 993e60f8db5SAlex Xie case CHIP_VEGA10: 994d96b428cSFeifei Xu case CHIP_VEGA20: 995946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 9965c583018SEvan Quan golden_settings_mmhub_1_0_0, 997c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 998946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 9995c583018SEvan Quan golden_settings_athub_1_0_0, 1000c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1001e60f8db5SAlex Xie break; 1002273a14cdSAlex Deucher case CHIP_VEGA12: 1003273a14cdSAlex Deucher break; 1004e4f3abaaSChunming Zhou case CHIP_RAVEN: 1005946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 10065c583018SEvan Quan golden_settings_athub_1_0_0, 1007c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1008e4f3abaaSChunming Zhou break; 1009e60f8db5SAlex Xie default: 1010e60f8db5SAlex Xie break; 1011e60f8db5SAlex Xie } 1012e60f8db5SAlex Xie } 1013e60f8db5SAlex Xie 1014e60f8db5SAlex Xie /** 1015e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1016e60f8db5SAlex Xie * 1017e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1018e60f8db5SAlex Xie */ 1019e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1020e60f8db5SAlex Xie { 1021e60f8db5SAlex Xie int r; 1022e60f8db5SAlex Xie bool value; 1023e60f8db5SAlex Xie u32 tmp; 1024e60f8db5SAlex Xie 10259c3f2b54SAlex Deucher amdgpu_device_program_register_sequence(adev, 1026e60f8db5SAlex Xie golden_settings_vega10_hdp, 1027c47b41a7SChristian König ARRAY_SIZE(golden_settings_vega10_hdp)); 1028e60f8db5SAlex Xie 10291123b989SChristian König if (adev->gart.bo == NULL) { 1030e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1031e60f8db5SAlex Xie return -EINVAL; 1032e60f8db5SAlex Xie } 1033ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1034ce1b1b66SMonk Liu if (r) 1035ce1b1b66SMonk Liu return r; 1036e60f8db5SAlex Xie 10372fcd43ceSHawking Zhang switch (adev->asic_type) { 10382fcd43ceSHawking Zhang case CHIP_RAVEN: 1039f8386b35SHawking Zhang mmhub_v1_0_update_power_gating(adev, true); 10402fcd43ceSHawking Zhang break; 10412fcd43ceSHawking Zhang default: 10422fcd43ceSHawking Zhang break; 10432fcd43ceSHawking Zhang } 10442fcd43ceSHawking Zhang 1045e60f8db5SAlex Xie r = gfxhub_v1_0_gart_enable(adev); 1046e60f8db5SAlex Xie if (r) 1047e60f8db5SAlex Xie return r; 1048e60f8db5SAlex Xie 1049e60f8db5SAlex Xie r = mmhub_v1_0_gart_enable(adev); 1050e60f8db5SAlex Xie if (r) 1051e60f8db5SAlex Xie return r; 1052e60f8db5SAlex Xie 1053846347c9STom St Denis WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1054e60f8db5SAlex Xie 1055b9509c80SHuang Rui tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1056b9509c80SHuang Rui WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1057e60f8db5SAlex Xie 10581d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 105969882565SChristian König adev->nbio_funcs->hdp_flush(adev, NULL); 10601d4e0a8cSMonk Liu 1061e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1062e60f8db5SAlex Xie value = false; 1063e60f8db5SAlex Xie else 1064e60f8db5SAlex Xie value = true; 1065e60f8db5SAlex Xie 1066e60f8db5SAlex Xie gfxhub_v1_0_set_fault_enable_default(adev, value); 1067e60f8db5SAlex Xie mmhub_v1_0_set_fault_enable_default(adev, value); 10682a79d868SYong Zhao gmc_v9_0_flush_gpu_tlb(adev, 0, 0); 1069e60f8db5SAlex Xie 1070e60f8db5SAlex Xie DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1071770d13b1SChristian König (unsigned)(adev->gmc.gart_size >> 20), 10724e830fb1SChristian König (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1073e60f8db5SAlex Xie adev->gart.ready = true; 1074e60f8db5SAlex Xie return 0; 1075e60f8db5SAlex Xie } 1076e60f8db5SAlex Xie 1077e60f8db5SAlex Xie static int gmc_v9_0_hw_init(void *handle) 1078e60f8db5SAlex Xie { 1079e60f8db5SAlex Xie int r; 1080e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1081e60f8db5SAlex Xie 1082e60f8db5SAlex Xie /* The sequence of these two function calls matters.*/ 1083e60f8db5SAlex Xie gmc_v9_0_init_golden_registers(adev); 1084e60f8db5SAlex Xie 1085edca2d05SAlex Deucher if (adev->mode_info.num_crtc) { 1086edca2d05SAlex Deucher /* Lockout access through VGA aperture*/ 10874d9c333aSTom St Denis WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1088edca2d05SAlex Deucher 1089edca2d05SAlex Deucher /* disable VGA render */ 10904d9c333aSTom St Denis WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1091edca2d05SAlex Deucher } 1092edca2d05SAlex Deucher 1093e60f8db5SAlex Xie r = gmc_v9_0_gart_enable(adev); 1094e60f8db5SAlex Xie 1095e60f8db5SAlex Xie return r; 1096e60f8db5SAlex Xie } 1097e60f8db5SAlex Xie 1098e60f8db5SAlex Xie /** 1099e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1100e60f8db5SAlex Xie * 1101e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1102e60f8db5SAlex Xie * 1103e60f8db5SAlex Xie * This disables all VM page table. 1104e60f8db5SAlex Xie */ 1105e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1106e60f8db5SAlex Xie { 1107e60f8db5SAlex Xie gfxhub_v1_0_gart_disable(adev); 1108e60f8db5SAlex Xie mmhub_v1_0_gart_disable(adev); 1109ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1110e60f8db5SAlex Xie } 1111e60f8db5SAlex Xie 1112e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1113e60f8db5SAlex Xie { 1114e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1115e60f8db5SAlex Xie 11165dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 11175dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 11185dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 11195dd696aeSTrigger Huang return 0; 11205dd696aeSTrigger Huang } 11215dd696aeSTrigger Huang 1122770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1123e60f8db5SAlex Xie gmc_v9_0_gart_disable(adev); 1124e60f8db5SAlex Xie 1125e60f8db5SAlex Xie return 0; 1126e60f8db5SAlex Xie } 1127e60f8db5SAlex Xie 1128e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1129e60f8db5SAlex Xie { 1130e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1131e60f8db5SAlex Xie 1132f053cd47STom St Denis return gmc_v9_0_hw_fini(adev); 1133e60f8db5SAlex Xie } 1134e60f8db5SAlex Xie 1135e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1136e60f8db5SAlex Xie { 1137e60f8db5SAlex Xie int r; 1138e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1139e60f8db5SAlex Xie 1140e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1141e60f8db5SAlex Xie if (r) 1142e60f8db5SAlex Xie return r; 1143e60f8db5SAlex Xie 1144620f774fSChristian König amdgpu_vmid_reset_all(adev); 1145e60f8db5SAlex Xie 114632601d48SChristian König return 0; 1147e60f8db5SAlex Xie } 1148e60f8db5SAlex Xie 1149e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1150e60f8db5SAlex Xie { 1151e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1152e60f8db5SAlex Xie return true; 1153e60f8db5SAlex Xie } 1154e60f8db5SAlex Xie 1155e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1156e60f8db5SAlex Xie { 1157e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1158e60f8db5SAlex Xie return 0; 1159e60f8db5SAlex Xie } 1160e60f8db5SAlex Xie 1161e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1162e60f8db5SAlex Xie { 1163e60f8db5SAlex Xie /* XXX for emulation.*/ 1164e60f8db5SAlex Xie return 0; 1165e60f8db5SAlex Xie } 1166e60f8db5SAlex Xie 1167e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1168e60f8db5SAlex Xie enum amd_clockgating_state state) 1169e60f8db5SAlex Xie { 1170d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1171d5583d4fSHuang Rui 1172d5583d4fSHuang Rui return mmhub_v1_0_set_clockgating(adev, state); 1173e60f8db5SAlex Xie } 1174e60f8db5SAlex Xie 117513052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 117613052be5SHuang Rui { 117713052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 117813052be5SHuang Rui 117913052be5SHuang Rui mmhub_v1_0_get_clockgating(adev, flags); 118013052be5SHuang Rui } 118113052be5SHuang Rui 1182e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1183e60f8db5SAlex Xie enum amd_powergating_state state) 1184e60f8db5SAlex Xie { 1185e60f8db5SAlex Xie return 0; 1186e60f8db5SAlex Xie } 1187e60f8db5SAlex Xie 1188e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1189e60f8db5SAlex Xie .name = "gmc_v9_0", 1190e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1191e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1192e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1193e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1194e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1195e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1196e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1197e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1198e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1199e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1200e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1201e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1202e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 120313052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1204e60f8db5SAlex Xie }; 1205e60f8db5SAlex Xie 1206e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1207e60f8db5SAlex Xie { 1208e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1209e60f8db5SAlex Xie .major = 9, 1210e60f8db5SAlex Xie .minor = 0, 1211e60f8db5SAlex Xie .rev = 0, 1212e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1213e60f8db5SAlex Xie }; 1214