1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23e60f8db5SAlex Xie #include <linux/firmware.h> 24fd5fd480SChunming Zhou #include <drm/drm_cache.h> 25e60f8db5SAlex Xie #include "amdgpu.h" 26e60f8db5SAlex Xie #include "gmc_v9_0.h" 278d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 282cddc50eSHuang Rui #include "amdgpu_gem.h" 29e60f8db5SAlex Xie 3075199b8cSFeifei Xu #include "hdp/hdp_4_0_offset.h" 3175199b8cSFeifei Xu #include "hdp/hdp_4_0_sh_mask.h" 32cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 33135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 34135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 35fb960bd2SFeifei Xu #include "vega10_enum.h" 3665417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 376ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 38250b4228SChristian König #include "oss/osssys_4_0_offset.h" 39e60f8db5SAlex Xie 40946a4d5bSShaoyun Liu #include "soc15.h" 41e60f8db5SAlex Xie #include "soc15_common.h" 4290c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 43e60f8db5SAlex Xie 44e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 45e60f8db5SAlex Xie #include "mmhub_v1_0.h" 46e60f8db5SAlex Xie 4744a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 4844a99b65SAndrey Grodzovsky 49ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 50ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 51ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 52ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 53ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 54ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 55ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 56ebdef28eSAlex Deucher 57e60f8db5SAlex Xie /* XXX Move this macro to VEGA10 header file, which is like vid.h for VI.*/ 58e60f8db5SAlex Xie #define AMDGPU_NUM_OF_VMIDS 8 59e60f8db5SAlex Xie 60e60f8db5SAlex Xie static const u32 golden_settings_vega10_hdp[] = 61e60f8db5SAlex Xie { 62e60f8db5SAlex Xie 0xf64, 0x0fffffff, 0x00000000, 63e60f8db5SAlex Xie 0xf65, 0x0fffffff, 0x00000000, 64e60f8db5SAlex Xie 0xf66, 0x0fffffff, 0x00000000, 65e60f8db5SAlex Xie 0xf67, 0x0fffffff, 0x00000000, 66e60f8db5SAlex Xie 0xf68, 0x0fffffff, 0x00000000, 67e60f8db5SAlex Xie 0xf6a, 0x0fffffff, 0x00000000, 68e60f8db5SAlex Xie 0xf6b, 0x0fffffff, 0x00000000, 69e60f8db5SAlex Xie 0xf6c, 0x0fffffff, 0x00000000, 70e60f8db5SAlex Xie 0xf6d, 0x0fffffff, 0x00000000, 71e60f8db5SAlex Xie 0xf6e, 0x0fffffff, 0x00000000, 72e60f8db5SAlex Xie }; 73e60f8db5SAlex Xie 74946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 755c583018SEvan Quan { 76946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 77946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 785c583018SEvan Quan }; 795c583018SEvan Quan 80946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 815c583018SEvan Quan { 82946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 83946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 845c583018SEvan Quan }; 855c583018SEvan Quan 8602bab923SDavid Panariti /* Ecc related register addresses, (BASE + reg offset) */ 8702bab923SDavid Panariti /* Universal Memory Controller caps (may be fused). */ 8802bab923SDavid Panariti /* UMCCH:UmcLocalCap */ 8902bab923SDavid Panariti #define UMCLOCALCAPS_ADDR0 (0x00014306 + 0x00000000) 9002bab923SDavid Panariti #define UMCLOCALCAPS_ADDR1 (0x00014306 + 0x00000800) 9102bab923SDavid Panariti #define UMCLOCALCAPS_ADDR2 (0x00014306 + 0x00001000) 9202bab923SDavid Panariti #define UMCLOCALCAPS_ADDR3 (0x00014306 + 0x00001800) 9302bab923SDavid Panariti #define UMCLOCALCAPS_ADDR4 (0x00054306 + 0x00000000) 9402bab923SDavid Panariti #define UMCLOCALCAPS_ADDR5 (0x00054306 + 0x00000800) 9502bab923SDavid Panariti #define UMCLOCALCAPS_ADDR6 (0x00054306 + 0x00001000) 9602bab923SDavid Panariti #define UMCLOCALCAPS_ADDR7 (0x00054306 + 0x00001800) 9702bab923SDavid Panariti #define UMCLOCALCAPS_ADDR8 (0x00094306 + 0x00000000) 9802bab923SDavid Panariti #define UMCLOCALCAPS_ADDR9 (0x00094306 + 0x00000800) 9902bab923SDavid Panariti #define UMCLOCALCAPS_ADDR10 (0x00094306 + 0x00001000) 10002bab923SDavid Panariti #define UMCLOCALCAPS_ADDR11 (0x00094306 + 0x00001800) 10102bab923SDavid Panariti #define UMCLOCALCAPS_ADDR12 (0x000d4306 + 0x00000000) 10202bab923SDavid Panariti #define UMCLOCALCAPS_ADDR13 (0x000d4306 + 0x00000800) 10302bab923SDavid Panariti #define UMCLOCALCAPS_ADDR14 (0x000d4306 + 0x00001000) 10402bab923SDavid Panariti #define UMCLOCALCAPS_ADDR15 (0x000d4306 + 0x00001800) 10502bab923SDavid Panariti 10602bab923SDavid Panariti /* Universal Memory Controller Channel config. */ 10702bab923SDavid Panariti /* UMCCH:UMC_CONFIG */ 10802bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR0 (0x00014040 + 0x00000000) 10902bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR1 (0x00014040 + 0x00000800) 11002bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR2 (0x00014040 + 0x00001000) 11102bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR3 (0x00014040 + 0x00001800) 11202bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR4 (0x00054040 + 0x00000000) 11302bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR5 (0x00054040 + 0x00000800) 11402bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR6 (0x00054040 + 0x00001000) 11502bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR7 (0x00054040 + 0x00001800) 11602bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR8 (0x00094040 + 0x00000000) 11702bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR9 (0x00094040 + 0x00000800) 11802bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR10 (0x00094040 + 0x00001000) 11902bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR11 (0x00094040 + 0x00001800) 12002bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR12 (0x000d4040 + 0x00000000) 12102bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR13 (0x000d4040 + 0x00000800) 12202bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR14 (0x000d4040 + 0x00001000) 12302bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR15 (0x000d4040 + 0x00001800) 12402bab923SDavid Panariti 12502bab923SDavid Panariti /* Universal Memory Controller Channel Ecc config. */ 12602bab923SDavid Panariti /* UMCCH:EccCtrl */ 12702bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR0 (0x00014053 + 0x00000000) 12802bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR1 (0x00014053 + 0x00000800) 12902bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR2 (0x00014053 + 0x00001000) 13002bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR3 (0x00014053 + 0x00001800) 13102bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR4 (0x00054053 + 0x00000000) 13202bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR5 (0x00054053 + 0x00000800) 13302bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR6 (0x00054053 + 0x00001000) 13402bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR7 (0x00054053 + 0x00001800) 13502bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR8 (0x00094053 + 0x00000000) 13602bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR9 (0x00094053 + 0x00000800) 13702bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR10 (0x00094053 + 0x00001000) 13802bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR11 (0x00094053 + 0x00001800) 13902bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR12 (0x000d4053 + 0x00000000) 14002bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR13 (0x000d4053 + 0x00000800) 14102bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR14 (0x000d4053 + 0x00001000) 14202bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR15 (0x000d4053 + 0x00001800) 14302bab923SDavid Panariti 14402bab923SDavid Panariti static const uint32_t ecc_umclocalcap_addrs[] = { 14502bab923SDavid Panariti UMCLOCALCAPS_ADDR0, 14602bab923SDavid Panariti UMCLOCALCAPS_ADDR1, 14702bab923SDavid Panariti UMCLOCALCAPS_ADDR2, 14802bab923SDavid Panariti UMCLOCALCAPS_ADDR3, 14902bab923SDavid Panariti UMCLOCALCAPS_ADDR4, 15002bab923SDavid Panariti UMCLOCALCAPS_ADDR5, 15102bab923SDavid Panariti UMCLOCALCAPS_ADDR6, 15202bab923SDavid Panariti UMCLOCALCAPS_ADDR7, 15302bab923SDavid Panariti UMCLOCALCAPS_ADDR8, 15402bab923SDavid Panariti UMCLOCALCAPS_ADDR9, 15502bab923SDavid Panariti UMCLOCALCAPS_ADDR10, 15602bab923SDavid Panariti UMCLOCALCAPS_ADDR11, 15702bab923SDavid Panariti UMCLOCALCAPS_ADDR12, 15802bab923SDavid Panariti UMCLOCALCAPS_ADDR13, 15902bab923SDavid Panariti UMCLOCALCAPS_ADDR14, 16002bab923SDavid Panariti UMCLOCALCAPS_ADDR15, 16102bab923SDavid Panariti }; 16202bab923SDavid Panariti 16302bab923SDavid Panariti static const uint32_t ecc_umcch_umc_config_addrs[] = { 16402bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR0, 16502bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR1, 16602bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR2, 16702bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR3, 16802bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR4, 16902bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR5, 17002bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR6, 17102bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR7, 17202bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR8, 17302bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR9, 17402bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR10, 17502bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR11, 17602bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR12, 17702bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR13, 17802bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR14, 17902bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR15, 18002bab923SDavid Panariti }; 18102bab923SDavid Panariti 18202bab923SDavid Panariti static const uint32_t ecc_umcch_eccctrl_addrs[] = { 18302bab923SDavid Panariti UMCCH_ECCCTRL_ADDR0, 18402bab923SDavid Panariti UMCCH_ECCCTRL_ADDR1, 18502bab923SDavid Panariti UMCCH_ECCCTRL_ADDR2, 18602bab923SDavid Panariti UMCCH_ECCCTRL_ADDR3, 18702bab923SDavid Panariti UMCCH_ECCCTRL_ADDR4, 18802bab923SDavid Panariti UMCCH_ECCCTRL_ADDR5, 18902bab923SDavid Panariti UMCCH_ECCCTRL_ADDR6, 19002bab923SDavid Panariti UMCCH_ECCCTRL_ADDR7, 19102bab923SDavid Panariti UMCCH_ECCCTRL_ADDR8, 19202bab923SDavid Panariti UMCCH_ECCCTRL_ADDR9, 19302bab923SDavid Panariti UMCCH_ECCCTRL_ADDR10, 19402bab923SDavid Panariti UMCCH_ECCCTRL_ADDR11, 19502bab923SDavid Panariti UMCCH_ECCCTRL_ADDR12, 19602bab923SDavid Panariti UMCCH_ECCCTRL_ADDR13, 19702bab923SDavid Panariti UMCCH_ECCCTRL_ADDR14, 19802bab923SDavid Panariti UMCCH_ECCCTRL_ADDR15, 19902bab923SDavid Panariti }; 20002bab923SDavid Panariti 201e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 202e60f8db5SAlex Xie struct amdgpu_irq_src *src, 203e60f8db5SAlex Xie unsigned type, 204e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 205e60f8db5SAlex Xie { 206e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 207ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 208e60f8db5SAlex Xie 20911250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21011250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21111250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21211250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21311250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21411250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21511250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 21611250164SChristian König 217e60f8db5SAlex Xie switch (state) { 218e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 219ae6d1416STom St Denis for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 220ae6d1416STom St Denis hub = &adev->vmhub[j]; 221e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 222e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 223e60f8db5SAlex Xie tmp = RREG32(reg); 224e60f8db5SAlex Xie tmp &= ~bits; 225e60f8db5SAlex Xie WREG32(reg, tmp); 226e60f8db5SAlex Xie } 227e60f8db5SAlex Xie } 228e60f8db5SAlex Xie break; 229e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 230ae6d1416STom St Denis for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 231ae6d1416STom St Denis hub = &adev->vmhub[j]; 232e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 233e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 234e60f8db5SAlex Xie tmp = RREG32(reg); 235e60f8db5SAlex Xie tmp |= bits; 236e60f8db5SAlex Xie WREG32(reg, tmp); 237e60f8db5SAlex Xie } 238e60f8db5SAlex Xie } 239e60f8db5SAlex Xie default: 240e60f8db5SAlex Xie break; 241e60f8db5SAlex Xie } 242e60f8db5SAlex Xie 243e60f8db5SAlex Xie return 0; 244e60f8db5SAlex Xie } 245e60f8db5SAlex Xie 246e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 247e60f8db5SAlex Xie struct amdgpu_irq_src *source, 248e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 249e60f8db5SAlex Xie { 250c4f46f22SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src]; 2514d6cbde3SFelix Kuehling uint32_t status = 0; 252e60f8db5SAlex Xie u64 addr; 253e60f8db5SAlex Xie 254e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 255e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 256e60f8db5SAlex Xie 25779a0c465SMonk Liu if (!amdgpu_sriov_vf(adev)) { 2585a9b8e8aSChristian König status = RREG32(hub->vm_l2_pro_fault_status); 2595a9b8e8aSChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 2604d6cbde3SFelix Kuehling } 261e60f8db5SAlex Xie 2624d6cbde3SFelix Kuehling if (printk_ratelimit()) { 263efaa9646SAndrey Grodzovsky struct amdgpu_task_info task_info = { 0 }; 264efaa9646SAndrey Grodzovsky 265efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 266efaa9646SAndrey Grodzovsky 2674d6cbde3SFelix Kuehling dev_err(adev->dev, 2680c79c0bbSChristian König "[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n", 269c4f46f22SChristian König entry->vmid_src ? "mmhub" : "gfxhub", 270c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 271efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 272efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 2737d0aa376SAndrey Grodzovsky dev_err(adev->dev, " in page starting at address 0x%016llx from %d\n", 27479a0c465SMonk Liu addr, entry->client_id); 2754d6cbde3SFelix Kuehling if (!amdgpu_sriov_vf(adev)) 2764d6cbde3SFelix Kuehling dev_err(adev->dev, 2774d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 2784d6cbde3SFelix Kuehling status); 27979a0c465SMonk Liu } 280e60f8db5SAlex Xie 281e60f8db5SAlex Xie return 0; 282e60f8db5SAlex Xie } 283e60f8db5SAlex Xie 284e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 285e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 286e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 287e60f8db5SAlex Xie }; 288e60f8db5SAlex Xie 289e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 290e60f8db5SAlex Xie { 291770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 292770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 293e60f8db5SAlex Xie } 294e60f8db5SAlex Xie 295c4f46f22SChristian König static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid) 29603f89febSChristian König { 29703f89febSChristian König u32 req = 0; 29803f89febSChristian König 299c4f46f22SChristian König /* invalidate using legacy mode on vmid*/ 30003f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 301c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 30203f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0); 30303f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 30403f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 30503f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 30603f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 30703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 30803f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 30903f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 31003f89febSChristian König 31103f89febSChristian König return req; 31203f89febSChristian König } 31303f89febSChristian König 3141849e737Skbuild test robot static signed long amdgpu_kiq_reg_write_reg_wait(struct amdgpu_device *adev, 3153890d111SEmily Deng uint32_t reg0, uint32_t reg1, 3163890d111SEmily Deng uint32_t ref, uint32_t mask) 3173890d111SEmily Deng { 3183890d111SEmily Deng signed long r, cnt = 0; 3193890d111SEmily Deng unsigned long flags; 3203890d111SEmily Deng uint32_t seq; 3213890d111SEmily Deng struct amdgpu_kiq *kiq = &adev->gfx.kiq; 3223890d111SEmily Deng struct amdgpu_ring *ring = &kiq->ring; 3233890d111SEmily Deng 3243890d111SEmily Deng spin_lock_irqsave(&kiq->ring_lock, flags); 3253890d111SEmily Deng 3263890d111SEmily Deng amdgpu_ring_alloc(ring, 32); 3273890d111SEmily Deng amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, 3283890d111SEmily Deng ref, mask); 3293890d111SEmily Deng amdgpu_fence_emit_polling(ring, &seq); 3303890d111SEmily Deng amdgpu_ring_commit(ring); 3313890d111SEmily Deng spin_unlock_irqrestore(&kiq->ring_lock, flags); 3323890d111SEmily Deng 3333890d111SEmily Deng r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 3343890d111SEmily Deng 335ae74da3eSEmily Deng /* don't wait anymore for IRQ context */ 336ae74da3eSEmily Deng if (r < 1 && in_interrupt()) 3373890d111SEmily Deng goto failed_kiq; 3383890d111SEmily Deng 3393890d111SEmily Deng might_sleep(); 3403890d111SEmily Deng 3413890d111SEmily Deng while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 3423890d111SEmily Deng msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 3433890d111SEmily Deng r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 3443890d111SEmily Deng } 3453890d111SEmily Deng 3463890d111SEmily Deng if (cnt > MAX_KIQ_REG_TRY) 3473890d111SEmily Deng goto failed_kiq; 3483890d111SEmily Deng 3493890d111SEmily Deng return 0; 3503890d111SEmily Deng 3513890d111SEmily Deng failed_kiq: 3523890d111SEmily Deng pr_err("failed to invalidate tlb with kiq\n"); 3533890d111SEmily Deng return r; 3543890d111SEmily Deng } 3553890d111SEmily Deng 356e60f8db5SAlex Xie /* 357e60f8db5SAlex Xie * GART 358e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 359e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 360e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 361e60f8db5SAlex Xie */ 362e60f8db5SAlex Xie 363e60f8db5SAlex Xie /** 364132f34e4SChristian König * gmc_v9_0_flush_gpu_tlb - gart tlb flush callback 365e60f8db5SAlex Xie * 366e60f8db5SAlex Xie * @adev: amdgpu_device pointer 367e60f8db5SAlex Xie * @vmid: vm instance to flush 368e60f8db5SAlex Xie * 369e60f8db5SAlex Xie * Flush the TLB for the requested page table. 370e60f8db5SAlex Xie */ 371132f34e4SChristian König static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, 372e60f8db5SAlex Xie uint32_t vmid) 373e60f8db5SAlex Xie { 374e60f8db5SAlex Xie /* Use register 17 for GART */ 375e60f8db5SAlex Xie const unsigned eng = 17; 376e60f8db5SAlex Xie unsigned i, j; 3773890d111SEmily Deng int r; 378e60f8db5SAlex Xie 379e60f8db5SAlex Xie for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 380e60f8db5SAlex Xie struct amdgpu_vmhub *hub = &adev->vmhub[i]; 38103f89febSChristian König u32 tmp = gmc_v9_0_get_invalidate_req(vmid); 382e60f8db5SAlex Xie 383fc0faf04SEmily Deng if (adev->gfx.kiq.ring.ready && 384ae74da3eSEmily Deng (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 385ae74da3eSEmily Deng !adev->in_gpu_reset) { 3863890d111SEmily Deng r = amdgpu_kiq_reg_write_reg_wait(adev, hub->vm_inv_eng0_req + eng, 3873890d111SEmily Deng hub->vm_inv_eng0_ack + eng, tmp, 1 << vmid); 3883890d111SEmily Deng if (!r) 3893890d111SEmily Deng continue; 390fc0faf04SEmily Deng } 3913890d111SEmily Deng 3923890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 3933890d111SEmily Deng 394c7a7266bSXiangliang Yu WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp); 395e60f8db5SAlex Xie 396e60f8db5SAlex Xie /* Busy wait for ACK.*/ 397e60f8db5SAlex Xie for (j = 0; j < 100; j++) { 398c7a7266bSXiangliang Yu tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng); 399e60f8db5SAlex Xie tmp &= 1 << vmid; 400e60f8db5SAlex Xie if (tmp) 401e60f8db5SAlex Xie break; 402e60f8db5SAlex Xie cpu_relax(); 403e60f8db5SAlex Xie } 4043890d111SEmily Deng if (j < 100) { 4053890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 406e60f8db5SAlex Xie continue; 4073890d111SEmily Deng } 408e60f8db5SAlex Xie 409e60f8db5SAlex Xie /* Wait for ACK with a delay.*/ 410e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 411c7a7266bSXiangliang Yu tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng); 412e60f8db5SAlex Xie tmp &= 1 << vmid; 413e60f8db5SAlex Xie if (tmp) 414e60f8db5SAlex Xie break; 415e60f8db5SAlex Xie udelay(1); 416e60f8db5SAlex Xie } 4173890d111SEmily Deng if (j < adev->usec_timeout) { 4183890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 419e60f8db5SAlex Xie continue; 4203890d111SEmily Deng } 4213890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 422e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 423e60f8db5SAlex Xie } 424e60f8db5SAlex Xie } 425e60f8db5SAlex Xie 4269096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 427c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 4289096d6e5SChristian König { 429250b4228SChristian König struct amdgpu_device *adev = ring->adev; 430250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 4319096d6e5SChristian König uint32_t req = gmc_v9_0_get_invalidate_req(vmid); 4329096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 4339096d6e5SChristian König 4349096d6e5SChristian König amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid), 4359096d6e5SChristian König lower_32_bits(pd_addr)); 4369096d6e5SChristian König 4379096d6e5SChristian König amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid), 4389096d6e5SChristian König upper_32_bits(pd_addr)); 4399096d6e5SChristian König 440f8bc9037SAlex Deucher amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng, 441f8bc9037SAlex Deucher hub->vm_inv_eng0_ack + eng, 442f8bc9037SAlex Deucher req, 1 << vmid); 443f732b6b3SChristian König 4449096d6e5SChristian König return pd_addr; 4459096d6e5SChristian König } 4469096d6e5SChristian König 447c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 448c633c00bSChristian König unsigned pasid) 449c633c00bSChristian König { 450c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 451c633c00bSChristian König uint32_t reg; 452c633c00bSChristian König 453c633c00bSChristian König if (ring->funcs->vmhub == AMDGPU_GFXHUB) 454c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 455c633c00bSChristian König else 456c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 457c633c00bSChristian König 458c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 459c633c00bSChristian König } 460c633c00bSChristian König 461e60f8db5SAlex Xie /** 462132f34e4SChristian König * gmc_v9_0_set_pte_pde - update the page tables using MMIO 463e60f8db5SAlex Xie * 464e60f8db5SAlex Xie * @adev: amdgpu_device pointer 465e60f8db5SAlex Xie * @cpu_pt_addr: cpu address of the page table 466e60f8db5SAlex Xie * @gpu_page_idx: entry in the page table to update 467e60f8db5SAlex Xie * @addr: dst addr to write into pte/pde 468e60f8db5SAlex Xie * @flags: access flags 469e60f8db5SAlex Xie * 470e60f8db5SAlex Xie * Update the page tables using the CPU. 471e60f8db5SAlex Xie */ 472132f34e4SChristian König static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, 473132f34e4SChristian König uint32_t gpu_page_idx, uint64_t addr, 474e60f8db5SAlex Xie uint64_t flags) 475e60f8db5SAlex Xie { 476e60f8db5SAlex Xie void __iomem *ptr = (void *)cpu_pt_addr; 477e60f8db5SAlex Xie uint64_t value; 478e60f8db5SAlex Xie 479e60f8db5SAlex Xie /* 480e60f8db5SAlex Xie * PTE format on VEGA 10: 481e60f8db5SAlex Xie * 63:59 reserved 482e60f8db5SAlex Xie * 58:57 mtype 483e60f8db5SAlex Xie * 56 F 484e60f8db5SAlex Xie * 55 L 485e60f8db5SAlex Xie * 54 P 486e60f8db5SAlex Xie * 53 SW 487e60f8db5SAlex Xie * 52 T 488e60f8db5SAlex Xie * 50:48 reserved 489e60f8db5SAlex Xie * 47:12 4k physical page base address 490e60f8db5SAlex Xie * 11:7 fragment 491e60f8db5SAlex Xie * 6 write 492e60f8db5SAlex Xie * 5 read 493e60f8db5SAlex Xie * 4 exe 494e60f8db5SAlex Xie * 3 Z 495e60f8db5SAlex Xie * 2 snooped 496e60f8db5SAlex Xie * 1 system 497e60f8db5SAlex Xie * 0 valid 498e60f8db5SAlex Xie * 499e60f8db5SAlex Xie * PDE format on VEGA 10: 500e60f8db5SAlex Xie * 63:59 block fragment size 501e60f8db5SAlex Xie * 58:55 reserved 502e60f8db5SAlex Xie * 54 P 503e60f8db5SAlex Xie * 53:48 reserved 504e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 505e60f8db5SAlex Xie * 5:3 reserved 506e60f8db5SAlex Xie * 2 C 507e60f8db5SAlex Xie * 1 system 508e60f8db5SAlex Xie * 0 valid 509e60f8db5SAlex Xie */ 510e60f8db5SAlex Xie 511e60f8db5SAlex Xie /* 512e60f8db5SAlex Xie * The following is for PTE only. GART does not have PDEs. 513e60f8db5SAlex Xie */ 514e60f8db5SAlex Xie value = addr & 0x0000FFFFFFFFF000ULL; 515e60f8db5SAlex Xie value |= flags; 516e60f8db5SAlex Xie writeq(value, ptr + (gpu_page_idx * 8)); 517e60f8db5SAlex Xie return 0; 518e60f8db5SAlex Xie } 519e60f8db5SAlex Xie 520e60f8db5SAlex Xie static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev, 521e60f8db5SAlex Xie uint32_t flags) 522e60f8db5SAlex Xie 523e60f8db5SAlex Xie { 524e60f8db5SAlex Xie uint64_t pte_flag = 0; 525e60f8db5SAlex Xie 526e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_EXECUTABLE) 527e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_EXECUTABLE; 528e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_READABLE) 529e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_READABLE; 530e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_WRITEABLE) 531e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_WRITEABLE; 532e60f8db5SAlex Xie 533e60f8db5SAlex Xie switch (flags & AMDGPU_VM_MTYPE_MASK) { 534e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 535e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 536e60f8db5SAlex Xie break; 537e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 538e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 539e60f8db5SAlex Xie break; 540e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 541e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC); 542e60f8db5SAlex Xie break; 543e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 544e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC); 545e60f8db5SAlex Xie break; 546e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 547e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC); 548e60f8db5SAlex Xie break; 549e60f8db5SAlex Xie default: 550e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 551e60f8db5SAlex Xie break; 552e60f8db5SAlex Xie } 553e60f8db5SAlex Xie 554e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_PRT) 555e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_PRT; 556e60f8db5SAlex Xie 557e60f8db5SAlex Xie return pte_flag; 558e60f8db5SAlex Xie } 559e60f8db5SAlex Xie 5603de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 5613de676d8SChristian König uint64_t *addr, uint64_t *flags) 562f75e237cSChristian König { 563bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 5643de676d8SChristian König *addr = adev->vm_manager.vram_base_offset + *addr - 565770d13b1SChristian König adev->gmc.vram_start; 5663de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 5676a42fd6fSChristian König 568770d13b1SChristian König if (!adev->gmc.translate_further) 5696a42fd6fSChristian König return; 5706a42fd6fSChristian König 5716a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 5726a42fd6fSChristian König /* Set the block fragment size */ 5736a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 5746a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 5756a42fd6fSChristian König 5766a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 5776a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 5786a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 5796a42fd6fSChristian König else 5806a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 5816a42fd6fSChristian König } 582f75e237cSChristian König } 583f75e237cSChristian König 584132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 585132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 5869096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 587c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 588132f34e4SChristian König .set_pte_pde = gmc_v9_0_set_pte_pde, 589b1166325SChristian König .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags, 590b1166325SChristian König .get_vm_pde = gmc_v9_0_get_vm_pde 591e60f8db5SAlex Xie }; 592e60f8db5SAlex Xie 593132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 594e60f8db5SAlex Xie { 595132f34e4SChristian König if (adev->gmc.gmc_funcs == NULL) 596132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 597e60f8db5SAlex Xie } 598e60f8db5SAlex Xie 599e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 600e60f8db5SAlex Xie { 601e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 602e60f8db5SAlex Xie 603132f34e4SChristian König gmc_v9_0_set_gmc_funcs(adev); 604e60f8db5SAlex Xie gmc_v9_0_set_irq_funcs(adev); 605e60f8db5SAlex Xie 606770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 607770d13b1SChristian König adev->gmc.shared_aperture_end = 608770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 609bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 610770d13b1SChristian König adev->gmc.private_aperture_end = 611770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 612a7ea6548SAlex Deucher 613e60f8db5SAlex Xie return 0; 614e60f8db5SAlex Xie } 615e60f8db5SAlex Xie 61602bab923SDavid Panariti static int gmc_v9_0_ecc_available(struct amdgpu_device *adev) 61702bab923SDavid Panariti { 61802bab923SDavid Panariti uint32_t reg_val; 61902bab923SDavid Panariti uint32_t reg_addr; 62002bab923SDavid Panariti uint32_t field_val; 62102bab923SDavid Panariti size_t i; 62202bab923SDavid Panariti uint32_t fv2; 62302bab923SDavid Panariti size_t lost_sheep; 62402bab923SDavid Panariti 62502bab923SDavid Panariti DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n"); 62602bab923SDavid Panariti 62702bab923SDavid Panariti lost_sheep = 0; 62802bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) { 62902bab923SDavid Panariti reg_addr = ecc_umclocalcap_addrs[i]; 63002bab923SDavid Panariti DRM_DEBUG("ecc: " 63102bab923SDavid Panariti "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n", 63202bab923SDavid Panariti i, reg_addr); 63302bab923SDavid Panariti reg_val = RREG32(reg_addr); 63402bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap, 63502bab923SDavid Panariti EccDis); 63602bab923SDavid Panariti DRM_DEBUG("ecc: " 63702bab923SDavid Panariti "reg_val: 0x%08x, " 63802bab923SDavid Panariti "EccDis: 0x%08x, ", 63902bab923SDavid Panariti reg_val, field_val); 64002bab923SDavid Panariti if (field_val) { 64102bab923SDavid Panariti DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n"); 64202bab923SDavid Panariti ++lost_sheep; 64302bab923SDavid Panariti } 64402bab923SDavid Panariti } 64502bab923SDavid Panariti 64602bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) { 64702bab923SDavid Panariti reg_addr = ecc_umcch_umc_config_addrs[i]; 64802bab923SDavid Panariti DRM_DEBUG("ecc: " 64902bab923SDavid Panariti "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x", 65002bab923SDavid Panariti i, reg_addr); 65102bab923SDavid Panariti reg_val = RREG32(reg_addr); 65202bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG, 65302bab923SDavid Panariti DramReady); 65402bab923SDavid Panariti DRM_DEBUG("ecc: " 65502bab923SDavid Panariti "reg_val: 0x%08x, " 65602bab923SDavid Panariti "DramReady: 0x%08x\n", 65702bab923SDavid Panariti reg_val, field_val); 65802bab923SDavid Panariti 65902bab923SDavid Panariti if (!field_val) { 66002bab923SDavid Panariti DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n"); 66102bab923SDavid Panariti ++lost_sheep; 66202bab923SDavid Panariti } 66302bab923SDavid Panariti } 66402bab923SDavid Panariti 66502bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) { 66602bab923SDavid Panariti reg_addr = ecc_umcch_eccctrl_addrs[i]; 66702bab923SDavid Panariti DRM_DEBUG("ecc: " 66802bab923SDavid Panariti "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ", 66902bab923SDavid Panariti i, reg_addr); 67002bab923SDavid Panariti reg_val = RREG32(reg_addr); 67102bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl, 67202bab923SDavid Panariti WrEccEn); 67302bab923SDavid Panariti fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl, 67402bab923SDavid Panariti RdEccEn); 67502bab923SDavid Panariti DRM_DEBUG("ecc: " 67602bab923SDavid Panariti "reg_val: 0x%08x, " 67702bab923SDavid Panariti "WrEccEn: 0x%08x, " 67802bab923SDavid Panariti "RdEccEn: 0x%08x\n", 67902bab923SDavid Panariti reg_val, field_val, fv2); 68002bab923SDavid Panariti 68102bab923SDavid Panariti if (!field_val) { 6825a16008fSAlex Deucher DRM_DEBUG("ecc: WrEccEn is not set\n"); 68302bab923SDavid Panariti ++lost_sheep; 68402bab923SDavid Panariti } 68502bab923SDavid Panariti if (!fv2) { 6865a16008fSAlex Deucher DRM_DEBUG("ecc: RdEccEn is not set\n"); 68702bab923SDavid Panariti ++lost_sheep; 68802bab923SDavid Panariti } 68902bab923SDavid Panariti } 69002bab923SDavid Panariti 69102bab923SDavid Panariti DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep); 69202bab923SDavid Panariti return lost_sheep == 0; 69302bab923SDavid Panariti } 69402bab923SDavid Panariti 695cd2b5623SAlex Deucher static bool gmc_v9_0_keep_stolen_memory(struct amdgpu_device *adev) 696cd2b5623SAlex Deucher { 697cd2b5623SAlex Deucher 698cd2b5623SAlex Deucher /* 699cd2b5623SAlex Deucher * TODO: 700cd2b5623SAlex Deucher * Currently there is a bug where some memory client outside 701cd2b5623SAlex Deucher * of the driver writes to first 8M of VRAM on S3 resume, 702cd2b5623SAlex Deucher * this overrides GART which by default gets placed in first 8M and 703cd2b5623SAlex Deucher * causes VM_FAULTS once GTT is accessed. 704cd2b5623SAlex Deucher * Keep the stolen memory reservation until the while this is not solved. 705cd2b5623SAlex Deucher * Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init 706cd2b5623SAlex Deucher */ 707cd2b5623SAlex Deucher switch (adev->asic_type) { 708cd2b5623SAlex Deucher case CHIP_RAVEN: 7096fb81375SAlex Deucher return false; 710cd2b5623SAlex Deucher case CHIP_VEGA10: 711cd2b5623SAlex Deucher case CHIP_VEGA12: 712cd2b5623SAlex Deucher case CHIP_VEGA20: 713cd2b5623SAlex Deucher default: 714cd2b5623SAlex Deucher return true; 715cd2b5623SAlex Deucher } 716cd2b5623SAlex Deucher } 717cd2b5623SAlex Deucher 718e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 719e60f8db5SAlex Xie { 720e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 721c5066129Sozeng /* 722c5066129Sozeng * The latest engine allocation on gfx9 is: 723c5066129Sozeng * Engine 0, 1: idle 724c5066129Sozeng * Engine 2, 3: firmware 725c5066129Sozeng * Engine 4~13: amdgpu ring, subject to change when ring number changes 726c5066129Sozeng * Engine 14~15: idle 727c5066129Sozeng * Engine 16: kfd tlb invalidation 728c5066129Sozeng * Engine 17: Gart flushes 729c5066129Sozeng */ 730c5066129Sozeng unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 }; 7314789c463SChristian König unsigned i; 73202bab923SDavid Panariti int r; 7334789c463SChristian König 734cd2b5623SAlex Deucher if (!gmc_v9_0_keep_stolen_memory(adev)) 735cd2b5623SAlex Deucher amdgpu_bo_late_init(adev); 7366f752ec2SAndrey Grodzovsky 7374789c463SChristian König for(i = 0; i < adev->num_rings; ++i) { 7384789c463SChristian König struct amdgpu_ring *ring = adev->rings[i]; 7394789c463SChristian König unsigned vmhub = ring->funcs->vmhub; 7404789c463SChristian König 7414789c463SChristian König ring->vm_inv_eng = vm_inv_eng[vmhub]++; 742775f55f1STom St Denis dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n", 743775f55f1STom St Denis ring->idx, ring->name, ring->vm_inv_eng, 744775f55f1STom St Denis ring->funcs->vmhub); 7454789c463SChristian König } 7464789c463SChristian König 747c5066129Sozeng /* Engine 16 is used for KFD and 17 for GART flushes */ 7484789c463SChristian König for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i) 749c5066129Sozeng BUG_ON(vm_inv_eng[i] > 16); 7504789c463SChristian König 7517b6cbae2SMonk Liu if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) { 75202bab923SDavid Panariti r = gmc_v9_0_ecc_available(adev); 75302bab923SDavid Panariti if (r == 1) { 75402bab923SDavid Panariti DRM_INFO("ECC is active.\n"); 75502bab923SDavid Panariti } else if (r == 0) { 75602bab923SDavid Panariti DRM_INFO("ECC is not present.\n"); 757e1d1a772SAlex Deucher adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false); 75802bab923SDavid Panariti } else { 75902bab923SDavid Panariti DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r); 76002bab923SDavid Panariti return r; 76102bab923SDavid Panariti } 7625ba4fa35SAlex Deucher } 76302bab923SDavid Panariti 764770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 765e60f8db5SAlex Xie } 766e60f8db5SAlex Xie 767e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 768770d13b1SChristian König struct amdgpu_gmc *mc) 769e60f8db5SAlex Xie { 770eeb2487dSMonk Liu u64 base = 0; 771eeb2487dSMonk Liu if (!amdgpu_sriov_vf(adev)) 772eeb2487dSMonk Liu base = mmhub_v1_0_get_fb_location(adev); 773961c75cfSChristian König amdgpu_gmc_vram_location(adev, &adev->gmc, base); 774961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 775bc099ee9SChunming Zhou /* base offset of vram pages */ 776bc099ee9SChunming Zhou adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev); 777e60f8db5SAlex Xie } 778e60f8db5SAlex Xie 779e60f8db5SAlex Xie /** 780e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 781e60f8db5SAlex Xie * 782e60f8db5SAlex Xie * @adev: amdgpu_device pointer 783e60f8db5SAlex Xie * 784e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 785e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 786e60f8db5SAlex Xie * Returns 0 for success. 787e60f8db5SAlex Xie */ 788e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 789e60f8db5SAlex Xie { 790e60f8db5SAlex Xie int chansize, numchan; 791d6895ad3SChristian König int r; 792e60f8db5SAlex Xie 7933d918c0eSShaoyun Liu if (amdgpu_emu_mode != 1) 794770d13b1SChristian König adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev); 795770d13b1SChristian König if (!adev->gmc.vram_width) { 796e60f8db5SAlex Xie /* hbm memory channel size */ 797585b7f16STom St Denis if (adev->flags & AMD_IS_APU) 798585b7f16STom St Denis chansize = 64; 799585b7f16STom St Denis else 800e60f8db5SAlex Xie chansize = 128; 801e60f8db5SAlex Xie 802070706c0SHawking Zhang numchan = adev->df_funcs->get_hbm_channel_number(adev); 803770d13b1SChristian König adev->gmc.vram_width = numchan * chansize; 8048d6a5230SAlex Deucher } 805e60f8db5SAlex Xie 806e60f8db5SAlex Xie /* size in MB on si */ 807770d13b1SChristian König adev->gmc.mc_vram_size = 808bf383fb6SAlex Deucher adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL; 809770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 810d6895ad3SChristian König 811d6895ad3SChristian König if (!(adev->flags & AMD_IS_APU)) { 812d6895ad3SChristian König r = amdgpu_device_resize_fb_bar(adev); 813d6895ad3SChristian König if (r) 814d6895ad3SChristian König return r; 815d6895ad3SChristian König } 816770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 817770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 818e60f8db5SAlex Xie 819156a81beSChunming Zhou #ifdef CONFIG_X86_64 820156a81beSChunming Zhou if (adev->flags & AMD_IS_APU) { 821156a81beSChunming Zhou adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev); 822156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 823156a81beSChunming Zhou } 824156a81beSChunming Zhou #endif 825e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 826770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 827770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 828770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 829e60f8db5SAlex Xie 830c3db7b5aSAlex Deucher /* set the gart size */ 831c3db7b5aSAlex Deucher if (amdgpu_gart_size == -1) { 832c3db7b5aSAlex Deucher switch (adev->asic_type) { 833c3db7b5aSAlex Deucher case CHIP_VEGA10: /* all engines support GPUVM */ 834273a14cdSAlex Deucher case CHIP_VEGA12: /* all engines support GPUVM */ 835d96b428cSFeifei Xu case CHIP_VEGA20: 836c3db7b5aSAlex Deucher default: 837fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 838c3db7b5aSAlex Deucher break; 839c3db7b5aSAlex Deucher case CHIP_RAVEN: /* DCE SG support */ 840770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 841c3db7b5aSAlex Deucher break; 842c3db7b5aSAlex Deucher } 843c3db7b5aSAlex Deucher } else { 844770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 845c3db7b5aSAlex Deucher } 846c3db7b5aSAlex Deucher 847770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 848e60f8db5SAlex Xie 849e60f8db5SAlex Xie return 0; 850e60f8db5SAlex Xie } 851e60f8db5SAlex Xie 852e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 853e60f8db5SAlex Xie { 854e60f8db5SAlex Xie int r; 855e60f8db5SAlex Xie 8561123b989SChristian König if (adev->gart.bo) { 857e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 858e60f8db5SAlex Xie return 0; 859e60f8db5SAlex Xie } 860e60f8db5SAlex Xie /* Initialize common gart structure */ 861e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 862e60f8db5SAlex Xie if (r) 863e60f8db5SAlex Xie return r; 864e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 865e60f8db5SAlex Xie adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) | 866e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 867e60f8db5SAlex Xie return amdgpu_gart_table_vram_alloc(adev); 868e60f8db5SAlex Xie } 869e60f8db5SAlex Xie 870ebdef28eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 871ebdef28eSAlex Deucher { 872ebdef28eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 873ebdef28eSAlex Deucher unsigned size; 874ebdef28eSAlex Deucher 8756f752ec2SAndrey Grodzovsky /* 8766f752ec2SAndrey Grodzovsky * TODO Remove once GART corruption is resolved 8776f752ec2SAndrey Grodzovsky * Check related code in gmc_v9_0_sw_fini 8786f752ec2SAndrey Grodzovsky * */ 879cd2b5623SAlex Deucher if (gmc_v9_0_keep_stolen_memory(adev)) 880cd2b5623SAlex Deucher return 9 * 1024 * 1024; 8816f752ec2SAndrey Grodzovsky 882ebdef28eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 883ebdef28eSAlex Deucher size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ 884ebdef28eSAlex Deucher } else { 885ebdef28eSAlex Deucher u32 viewport; 886ebdef28eSAlex Deucher 887ebdef28eSAlex Deucher switch (adev->asic_type) { 888ebdef28eSAlex Deucher case CHIP_RAVEN: 889ebdef28eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 890ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, 891ebdef28eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 892ebdef28eSAlex Deucher REG_GET_FIELD(viewport, 893ebdef28eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 894ebdef28eSAlex Deucher 4); 895ebdef28eSAlex Deucher break; 896ebdef28eSAlex Deucher case CHIP_VEGA10: 897ebdef28eSAlex Deucher case CHIP_VEGA12: 898cd2b5623SAlex Deucher case CHIP_VEGA20: 899ebdef28eSAlex Deucher default: 900ebdef28eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 901ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 902ebdef28eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 903ebdef28eSAlex Deucher 4); 904ebdef28eSAlex Deucher break; 905ebdef28eSAlex Deucher } 906ebdef28eSAlex Deucher } 907ebdef28eSAlex Deucher /* return 0 if the pre-OS buffer uses up most of vram */ 908ebdef28eSAlex Deucher if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 909ebdef28eSAlex Deucher return 0; 9106f752ec2SAndrey Grodzovsky 911ebdef28eSAlex Deucher return size; 912ebdef28eSAlex Deucher } 913ebdef28eSAlex Deucher 914e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 915e60f8db5SAlex Xie { 916e60f8db5SAlex Xie int r; 917e60f8db5SAlex Xie int dma_bits; 918e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 919e60f8db5SAlex Xie 9200c8c0847SHuang Rui gfxhub_v1_0_init(adev); 92177f6c763SHuang Rui mmhub_v1_0_init(adev); 9220c8c0847SHuang Rui 923770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 924e60f8db5SAlex Xie 9251e09b053SHawking Zhang adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev); 926fd66560bSHawking Zhang switch (adev->asic_type) { 927fd66560bSHawking Zhang case CHIP_RAVEN: 9286a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 929f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 9306a42fd6fSChristian König } else { 9316a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 9326a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 933770d13b1SChristian König adev->gmc.translate_further = 9346a42fd6fSChristian König adev->vm_manager.num_level > 1; 9356a42fd6fSChristian König } 936fd66560bSHawking Zhang break; 937fd66560bSHawking Zhang case CHIP_VEGA10: 938273a14cdSAlex Deucher case CHIP_VEGA12: 939d96b428cSFeifei Xu case CHIP_VEGA20: 94036b32a68SZhang, Jerry /* 94136b32a68SZhang, Jerry * To fulfill 4-level page support, 94236b32a68SZhang, Jerry * vm size is 256TB (48bit), maximum size of Vega10, 94336b32a68SZhang, Jerry * block size 512 (9bit) 94436b32a68SZhang, Jerry */ 945f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 946fd66560bSHawking Zhang break; 947fd66560bSHawking Zhang default: 948fd66560bSHawking Zhang break; 949fd66560bSHawking Zhang } 950fd66560bSHawking Zhang 951e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 95244a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 953770d13b1SChristian König &adev->gmc.vm_fault); 95444a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 955770d13b1SChristian König &adev->gmc.vm_fault); 956e60f8db5SAlex Xie 957e60f8db5SAlex Xie if (r) 958e60f8db5SAlex Xie return r; 959e60f8db5SAlex Xie 960e60f8db5SAlex Xie /* Set the internal MC address mask 961e60f8db5SAlex Xie * This is the max address of the GPU's 962e60f8db5SAlex Xie * internal address space. 963e60f8db5SAlex Xie */ 964770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 965e60f8db5SAlex Xie 966e60f8db5SAlex Xie /* set DMA mask + need_dma32 flags. 967e60f8db5SAlex Xie * PCIE - can handle 44-bits. 968e60f8db5SAlex Xie * IGP - can handle 44-bits 969e60f8db5SAlex Xie * PCI - dma32 for legacy pci gart, 44 bits on vega10 970e60f8db5SAlex Xie */ 971e60f8db5SAlex Xie adev->need_dma32 = false; 972e60f8db5SAlex Xie dma_bits = adev->need_dma32 ? 32 : 44; 973e60f8db5SAlex Xie r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 974e60f8db5SAlex Xie if (r) { 975e60f8db5SAlex Xie adev->need_dma32 = true; 976e60f8db5SAlex Xie dma_bits = 32; 977e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 978e60f8db5SAlex Xie } 979e60f8db5SAlex Xie r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 980e60f8db5SAlex Xie if (r) { 981e60f8db5SAlex Xie pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32)); 982e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No coherent DMA available.\n"); 983e60f8db5SAlex Xie } 984fd5fd480SChunming Zhou adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits); 985e60f8db5SAlex Xie 986e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 987e60f8db5SAlex Xie if (r) 988e60f8db5SAlex Xie return r; 989e60f8db5SAlex Xie 990ebdef28eSAlex Deucher adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev); 991ebdef28eSAlex Deucher 992e60f8db5SAlex Xie /* Memory manager */ 993e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 994e60f8db5SAlex Xie if (r) 995e60f8db5SAlex Xie return r; 996e60f8db5SAlex Xie 997e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 998e60f8db5SAlex Xie if (r) 999e60f8db5SAlex Xie return r; 1000e60f8db5SAlex Xie 100105ec3edaSChristian König /* 100205ec3edaSChristian König * number of VMs 100305ec3edaSChristian König * VMID 0 is reserved for System 100405ec3edaSChristian König * amdgpu graphics/compute will use VMIDs 1-7 100505ec3edaSChristian König * amdkfd will use VMIDs 8-15 100605ec3edaSChristian König */ 100705ec3edaSChristian König adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 100805ec3edaSChristian König adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 100905ec3edaSChristian König 101005ec3edaSChristian König amdgpu_vm_manager_init(adev); 101105ec3edaSChristian König 101205ec3edaSChristian König return 0; 1013e60f8db5SAlex Xie } 1014e60f8db5SAlex Xie 1015e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 1016e60f8db5SAlex Xie { 1017e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1018e60f8db5SAlex Xie 1019f59548c8SMonk Liu amdgpu_gem_force_release(adev); 1020e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 10216f752ec2SAndrey Grodzovsky 1022cd2b5623SAlex Deucher if (gmc_v9_0_keep_stolen_memory(adev)) 10236f752ec2SAndrey Grodzovsky amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); 10246f752ec2SAndrey Grodzovsky 1025a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 1026e60f8db5SAlex Xie amdgpu_bo_fini(adev); 1027a3d9103eSAndrey Grodzovsky amdgpu_gart_fini(adev); 1028e60f8db5SAlex Xie 1029e60f8db5SAlex Xie return 0; 1030e60f8db5SAlex Xie } 1031e60f8db5SAlex Xie 1032e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1033e60f8db5SAlex Xie { 1034946a4d5bSShaoyun Liu 1035e60f8db5SAlex Xie switch (adev->asic_type) { 1036e60f8db5SAlex Xie case CHIP_VEGA10: 1037d96b428cSFeifei Xu case CHIP_VEGA20: 1038946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 10395c583018SEvan Quan golden_settings_mmhub_1_0_0, 1040c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1041946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 10425c583018SEvan Quan golden_settings_athub_1_0_0, 1043c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1044e60f8db5SAlex Xie break; 1045273a14cdSAlex Deucher case CHIP_VEGA12: 1046273a14cdSAlex Deucher break; 1047e4f3abaaSChunming Zhou case CHIP_RAVEN: 1048946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 10495c583018SEvan Quan golden_settings_athub_1_0_0, 1050c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1051e4f3abaaSChunming Zhou break; 1052e60f8db5SAlex Xie default: 1053e60f8db5SAlex Xie break; 1054e60f8db5SAlex Xie } 1055e60f8db5SAlex Xie } 1056e60f8db5SAlex Xie 1057e60f8db5SAlex Xie /** 1058e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1059e60f8db5SAlex Xie * 1060e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1061e60f8db5SAlex Xie */ 1062e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1063e60f8db5SAlex Xie { 1064e60f8db5SAlex Xie int r; 1065e60f8db5SAlex Xie bool value; 1066e60f8db5SAlex Xie u32 tmp; 1067e60f8db5SAlex Xie 10689c3f2b54SAlex Deucher amdgpu_device_program_register_sequence(adev, 1069e60f8db5SAlex Xie golden_settings_vega10_hdp, 1070c47b41a7SChristian König ARRAY_SIZE(golden_settings_vega10_hdp)); 1071e60f8db5SAlex Xie 10721123b989SChristian König if (adev->gart.bo == NULL) { 1073e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1074e60f8db5SAlex Xie return -EINVAL; 1075e60f8db5SAlex Xie } 1076ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1077ce1b1b66SMonk Liu if (r) 1078ce1b1b66SMonk Liu return r; 1079e60f8db5SAlex Xie 10802fcd43ceSHawking Zhang switch (adev->asic_type) { 10812fcd43ceSHawking Zhang case CHIP_RAVEN: 10822fcd43ceSHawking Zhang mmhub_v1_0_initialize_power_gating(adev); 1083f8386b35SHawking Zhang mmhub_v1_0_update_power_gating(adev, true); 10842fcd43ceSHawking Zhang break; 10852fcd43ceSHawking Zhang default: 10862fcd43ceSHawking Zhang break; 10872fcd43ceSHawking Zhang } 10882fcd43ceSHawking Zhang 1089e60f8db5SAlex Xie r = gfxhub_v1_0_gart_enable(adev); 1090e60f8db5SAlex Xie if (r) 1091e60f8db5SAlex Xie return r; 1092e60f8db5SAlex Xie 1093e60f8db5SAlex Xie r = mmhub_v1_0_gart_enable(adev); 1094e60f8db5SAlex Xie if (r) 1095e60f8db5SAlex Xie return r; 1096e60f8db5SAlex Xie 1097846347c9STom St Denis WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1098e60f8db5SAlex Xie 1099b9509c80SHuang Rui tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1100b9509c80SHuang Rui WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1101e60f8db5SAlex Xie 11021d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 110369882565SChristian König adev->nbio_funcs->hdp_flush(adev, NULL); 11041d4e0a8cSMonk Liu 1105e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1106e60f8db5SAlex Xie value = false; 1107e60f8db5SAlex Xie else 1108e60f8db5SAlex Xie value = true; 1109e60f8db5SAlex Xie 1110e60f8db5SAlex Xie gfxhub_v1_0_set_fault_enable_default(adev, value); 1111e60f8db5SAlex Xie mmhub_v1_0_set_fault_enable_default(adev, value); 1112132f34e4SChristian König gmc_v9_0_flush_gpu_tlb(adev, 0); 1113e60f8db5SAlex Xie 1114e60f8db5SAlex Xie DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1115770d13b1SChristian König (unsigned)(adev->gmc.gart_size >> 20), 11164e830fb1SChristian König (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1117e60f8db5SAlex Xie adev->gart.ready = true; 1118e60f8db5SAlex Xie return 0; 1119e60f8db5SAlex Xie } 1120e60f8db5SAlex Xie 1121e60f8db5SAlex Xie static int gmc_v9_0_hw_init(void *handle) 1122e60f8db5SAlex Xie { 1123e60f8db5SAlex Xie int r; 1124e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1125e60f8db5SAlex Xie 1126e60f8db5SAlex Xie /* The sequence of these two function calls matters.*/ 1127e60f8db5SAlex Xie gmc_v9_0_init_golden_registers(adev); 1128e60f8db5SAlex Xie 1129edca2d05SAlex Deucher if (adev->mode_info.num_crtc) { 1130edca2d05SAlex Deucher /* Lockout access through VGA aperture*/ 11314d9c333aSTom St Denis WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1132edca2d05SAlex Deucher 1133edca2d05SAlex Deucher /* disable VGA render */ 11344d9c333aSTom St Denis WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1135edca2d05SAlex Deucher } 1136edca2d05SAlex Deucher 1137e60f8db5SAlex Xie r = gmc_v9_0_gart_enable(adev); 1138e60f8db5SAlex Xie 1139e60f8db5SAlex Xie return r; 1140e60f8db5SAlex Xie } 1141e60f8db5SAlex Xie 1142e60f8db5SAlex Xie /** 1143e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1144e60f8db5SAlex Xie * 1145e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1146e60f8db5SAlex Xie * 1147e60f8db5SAlex Xie * This disables all VM page table. 1148e60f8db5SAlex Xie */ 1149e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1150e60f8db5SAlex Xie { 1151e60f8db5SAlex Xie gfxhub_v1_0_gart_disable(adev); 1152e60f8db5SAlex Xie mmhub_v1_0_gart_disable(adev); 1153ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1154e60f8db5SAlex Xie } 1155e60f8db5SAlex Xie 1156e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1157e60f8db5SAlex Xie { 1158e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1159e60f8db5SAlex Xie 11605dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 11615dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 11625dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 11635dd696aeSTrigger Huang return 0; 11645dd696aeSTrigger Huang } 11655dd696aeSTrigger Huang 1166770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1167e60f8db5SAlex Xie gmc_v9_0_gart_disable(adev); 1168e60f8db5SAlex Xie 1169e60f8db5SAlex Xie return 0; 1170e60f8db5SAlex Xie } 1171e60f8db5SAlex Xie 1172e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1173e60f8db5SAlex Xie { 1174e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1175e60f8db5SAlex Xie 1176f053cd47STom St Denis return gmc_v9_0_hw_fini(adev); 1177e60f8db5SAlex Xie } 1178e60f8db5SAlex Xie 1179e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1180e60f8db5SAlex Xie { 1181e60f8db5SAlex Xie int r; 1182e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1183e60f8db5SAlex Xie 1184e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1185e60f8db5SAlex Xie if (r) 1186e60f8db5SAlex Xie return r; 1187e60f8db5SAlex Xie 1188620f774fSChristian König amdgpu_vmid_reset_all(adev); 1189e60f8db5SAlex Xie 119032601d48SChristian König return 0; 1191e60f8db5SAlex Xie } 1192e60f8db5SAlex Xie 1193e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1194e60f8db5SAlex Xie { 1195e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1196e60f8db5SAlex Xie return true; 1197e60f8db5SAlex Xie } 1198e60f8db5SAlex Xie 1199e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1200e60f8db5SAlex Xie { 1201e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1202e60f8db5SAlex Xie return 0; 1203e60f8db5SAlex Xie } 1204e60f8db5SAlex Xie 1205e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1206e60f8db5SAlex Xie { 1207e60f8db5SAlex Xie /* XXX for emulation.*/ 1208e60f8db5SAlex Xie return 0; 1209e60f8db5SAlex Xie } 1210e60f8db5SAlex Xie 1211e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1212e60f8db5SAlex Xie enum amd_clockgating_state state) 1213e60f8db5SAlex Xie { 1214d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1215d5583d4fSHuang Rui 1216d5583d4fSHuang Rui return mmhub_v1_0_set_clockgating(adev, state); 1217e60f8db5SAlex Xie } 1218e60f8db5SAlex Xie 121913052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 122013052be5SHuang Rui { 122113052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 122213052be5SHuang Rui 122313052be5SHuang Rui mmhub_v1_0_get_clockgating(adev, flags); 122413052be5SHuang Rui } 122513052be5SHuang Rui 1226e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1227e60f8db5SAlex Xie enum amd_powergating_state state) 1228e60f8db5SAlex Xie { 1229e60f8db5SAlex Xie return 0; 1230e60f8db5SAlex Xie } 1231e60f8db5SAlex Xie 1232e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1233e60f8db5SAlex Xie .name = "gmc_v9_0", 1234e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1235e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1236e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1237e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1238e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1239e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1240e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1241e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1242e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1243e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1244e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1245e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1246e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 124713052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1248e60f8db5SAlex Xie }; 1249e60f8db5SAlex Xie 1250e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1251e60f8db5SAlex Xie { 1252e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1253e60f8db5SAlex Xie .major = 9, 1254e60f8db5SAlex Xie .minor = 0, 1255e60f8db5SAlex Xie .rev = 0, 1256e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1257e60f8db5SAlex Xie }; 1258