1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23e60f8db5SAlex Xie #include <linux/firmware.h> 24fd5fd480SChunming Zhou #include <drm/drm_cache.h> 25e60f8db5SAlex Xie #include "amdgpu.h" 26e60f8db5SAlex Xie #include "gmc_v9_0.h" 278d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 282cddc50eSHuang Rui #include "amdgpu_gem.h" 29e60f8db5SAlex Xie 3075199b8cSFeifei Xu #include "hdp/hdp_4_0_offset.h" 3175199b8cSFeifei Xu #include "hdp/hdp_4_0_sh_mask.h" 32cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 33135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 34135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 35fb960bd2SFeifei Xu #include "vega10_enum.h" 3665417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 376ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 38250b4228SChristian König #include "oss/osssys_4_0_offset.h" 39e60f8db5SAlex Xie 40946a4d5bSShaoyun Liu #include "soc15.h" 41e60f8db5SAlex Xie #include "soc15_common.h" 4290c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 43e60f8db5SAlex Xie 44e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 45e60f8db5SAlex Xie #include "mmhub_v1_0.h" 46e60f8db5SAlex Xie 4744a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 4844a99b65SAndrey Grodzovsky 49ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 50ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 51ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 52ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 53ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 54ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 55ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 56ebdef28eSAlex Deucher 57e60f8db5SAlex Xie /* XXX Move this macro to VEGA10 header file, which is like vid.h for VI.*/ 58e60f8db5SAlex Xie #define AMDGPU_NUM_OF_VMIDS 8 59e60f8db5SAlex Xie 60e60f8db5SAlex Xie static const u32 golden_settings_vega10_hdp[] = 61e60f8db5SAlex Xie { 62e60f8db5SAlex Xie 0xf64, 0x0fffffff, 0x00000000, 63e60f8db5SAlex Xie 0xf65, 0x0fffffff, 0x00000000, 64e60f8db5SAlex Xie 0xf66, 0x0fffffff, 0x00000000, 65e60f8db5SAlex Xie 0xf67, 0x0fffffff, 0x00000000, 66e60f8db5SAlex Xie 0xf68, 0x0fffffff, 0x00000000, 67e60f8db5SAlex Xie 0xf6a, 0x0fffffff, 0x00000000, 68e60f8db5SAlex Xie 0xf6b, 0x0fffffff, 0x00000000, 69e60f8db5SAlex Xie 0xf6c, 0x0fffffff, 0x00000000, 70e60f8db5SAlex Xie 0xf6d, 0x0fffffff, 0x00000000, 71e60f8db5SAlex Xie 0xf6e, 0x0fffffff, 0x00000000, 72e60f8db5SAlex Xie }; 73e60f8db5SAlex Xie 74946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 755c583018SEvan Quan { 76946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 77946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 785c583018SEvan Quan }; 795c583018SEvan Quan 80946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 815c583018SEvan Quan { 82946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 83946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 845c583018SEvan Quan }; 855c583018SEvan Quan 8602bab923SDavid Panariti /* Ecc related register addresses, (BASE + reg offset) */ 8702bab923SDavid Panariti /* Universal Memory Controller caps (may be fused). */ 8802bab923SDavid Panariti /* UMCCH:UmcLocalCap */ 8902bab923SDavid Panariti #define UMCLOCALCAPS_ADDR0 (0x00014306 + 0x00000000) 9002bab923SDavid Panariti #define UMCLOCALCAPS_ADDR1 (0x00014306 + 0x00000800) 9102bab923SDavid Panariti #define UMCLOCALCAPS_ADDR2 (0x00014306 + 0x00001000) 9202bab923SDavid Panariti #define UMCLOCALCAPS_ADDR3 (0x00014306 + 0x00001800) 9302bab923SDavid Panariti #define UMCLOCALCAPS_ADDR4 (0x00054306 + 0x00000000) 9402bab923SDavid Panariti #define UMCLOCALCAPS_ADDR5 (0x00054306 + 0x00000800) 9502bab923SDavid Panariti #define UMCLOCALCAPS_ADDR6 (0x00054306 + 0x00001000) 9602bab923SDavid Panariti #define UMCLOCALCAPS_ADDR7 (0x00054306 + 0x00001800) 9702bab923SDavid Panariti #define UMCLOCALCAPS_ADDR8 (0x00094306 + 0x00000000) 9802bab923SDavid Panariti #define UMCLOCALCAPS_ADDR9 (0x00094306 + 0x00000800) 9902bab923SDavid Panariti #define UMCLOCALCAPS_ADDR10 (0x00094306 + 0x00001000) 10002bab923SDavid Panariti #define UMCLOCALCAPS_ADDR11 (0x00094306 + 0x00001800) 10102bab923SDavid Panariti #define UMCLOCALCAPS_ADDR12 (0x000d4306 + 0x00000000) 10202bab923SDavid Panariti #define UMCLOCALCAPS_ADDR13 (0x000d4306 + 0x00000800) 10302bab923SDavid Panariti #define UMCLOCALCAPS_ADDR14 (0x000d4306 + 0x00001000) 10402bab923SDavid Panariti #define UMCLOCALCAPS_ADDR15 (0x000d4306 + 0x00001800) 10502bab923SDavid Panariti 10602bab923SDavid Panariti /* Universal Memory Controller Channel config. */ 10702bab923SDavid Panariti /* UMCCH:UMC_CONFIG */ 10802bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR0 (0x00014040 + 0x00000000) 10902bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR1 (0x00014040 + 0x00000800) 11002bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR2 (0x00014040 + 0x00001000) 11102bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR3 (0x00014040 + 0x00001800) 11202bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR4 (0x00054040 + 0x00000000) 11302bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR5 (0x00054040 + 0x00000800) 11402bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR6 (0x00054040 + 0x00001000) 11502bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR7 (0x00054040 + 0x00001800) 11602bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR8 (0x00094040 + 0x00000000) 11702bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR9 (0x00094040 + 0x00000800) 11802bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR10 (0x00094040 + 0x00001000) 11902bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR11 (0x00094040 + 0x00001800) 12002bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR12 (0x000d4040 + 0x00000000) 12102bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR13 (0x000d4040 + 0x00000800) 12202bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR14 (0x000d4040 + 0x00001000) 12302bab923SDavid Panariti #define UMCCH_UMC_CONFIG_ADDR15 (0x000d4040 + 0x00001800) 12402bab923SDavid Panariti 12502bab923SDavid Panariti /* Universal Memory Controller Channel Ecc config. */ 12602bab923SDavid Panariti /* UMCCH:EccCtrl */ 12702bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR0 (0x00014053 + 0x00000000) 12802bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR1 (0x00014053 + 0x00000800) 12902bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR2 (0x00014053 + 0x00001000) 13002bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR3 (0x00014053 + 0x00001800) 13102bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR4 (0x00054053 + 0x00000000) 13202bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR5 (0x00054053 + 0x00000800) 13302bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR6 (0x00054053 + 0x00001000) 13402bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR7 (0x00054053 + 0x00001800) 13502bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR8 (0x00094053 + 0x00000000) 13602bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR9 (0x00094053 + 0x00000800) 13702bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR10 (0x00094053 + 0x00001000) 13802bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR11 (0x00094053 + 0x00001800) 13902bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR12 (0x000d4053 + 0x00000000) 14002bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR13 (0x000d4053 + 0x00000800) 14102bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR14 (0x000d4053 + 0x00001000) 14202bab923SDavid Panariti #define UMCCH_ECCCTRL_ADDR15 (0x000d4053 + 0x00001800) 14302bab923SDavid Panariti 14402bab923SDavid Panariti static const uint32_t ecc_umclocalcap_addrs[] = { 14502bab923SDavid Panariti UMCLOCALCAPS_ADDR0, 14602bab923SDavid Panariti UMCLOCALCAPS_ADDR1, 14702bab923SDavid Panariti UMCLOCALCAPS_ADDR2, 14802bab923SDavid Panariti UMCLOCALCAPS_ADDR3, 14902bab923SDavid Panariti UMCLOCALCAPS_ADDR4, 15002bab923SDavid Panariti UMCLOCALCAPS_ADDR5, 15102bab923SDavid Panariti UMCLOCALCAPS_ADDR6, 15202bab923SDavid Panariti UMCLOCALCAPS_ADDR7, 15302bab923SDavid Panariti UMCLOCALCAPS_ADDR8, 15402bab923SDavid Panariti UMCLOCALCAPS_ADDR9, 15502bab923SDavid Panariti UMCLOCALCAPS_ADDR10, 15602bab923SDavid Panariti UMCLOCALCAPS_ADDR11, 15702bab923SDavid Panariti UMCLOCALCAPS_ADDR12, 15802bab923SDavid Panariti UMCLOCALCAPS_ADDR13, 15902bab923SDavid Panariti UMCLOCALCAPS_ADDR14, 16002bab923SDavid Panariti UMCLOCALCAPS_ADDR15, 16102bab923SDavid Panariti }; 16202bab923SDavid Panariti 16302bab923SDavid Panariti static const uint32_t ecc_umcch_umc_config_addrs[] = { 16402bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR0, 16502bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR1, 16602bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR2, 16702bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR3, 16802bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR4, 16902bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR5, 17002bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR6, 17102bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR7, 17202bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR8, 17302bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR9, 17402bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR10, 17502bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR11, 17602bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR12, 17702bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR13, 17802bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR14, 17902bab923SDavid Panariti UMCCH_UMC_CONFIG_ADDR15, 18002bab923SDavid Panariti }; 18102bab923SDavid Panariti 18202bab923SDavid Panariti static const uint32_t ecc_umcch_eccctrl_addrs[] = { 18302bab923SDavid Panariti UMCCH_ECCCTRL_ADDR0, 18402bab923SDavid Panariti UMCCH_ECCCTRL_ADDR1, 18502bab923SDavid Panariti UMCCH_ECCCTRL_ADDR2, 18602bab923SDavid Panariti UMCCH_ECCCTRL_ADDR3, 18702bab923SDavid Panariti UMCCH_ECCCTRL_ADDR4, 18802bab923SDavid Panariti UMCCH_ECCCTRL_ADDR5, 18902bab923SDavid Panariti UMCCH_ECCCTRL_ADDR6, 19002bab923SDavid Panariti UMCCH_ECCCTRL_ADDR7, 19102bab923SDavid Panariti UMCCH_ECCCTRL_ADDR8, 19202bab923SDavid Panariti UMCCH_ECCCTRL_ADDR9, 19302bab923SDavid Panariti UMCCH_ECCCTRL_ADDR10, 19402bab923SDavid Panariti UMCCH_ECCCTRL_ADDR11, 19502bab923SDavid Panariti UMCCH_ECCCTRL_ADDR12, 19602bab923SDavid Panariti UMCCH_ECCCTRL_ADDR13, 19702bab923SDavid Panariti UMCCH_ECCCTRL_ADDR14, 19802bab923SDavid Panariti UMCCH_ECCCTRL_ADDR15, 19902bab923SDavid Panariti }; 20002bab923SDavid Panariti 201e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 202e60f8db5SAlex Xie struct amdgpu_irq_src *src, 203e60f8db5SAlex Xie unsigned type, 204e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 205e60f8db5SAlex Xie { 206e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 207ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 208e60f8db5SAlex Xie 20911250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21011250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21111250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21211250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21311250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21411250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 21511250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 21611250164SChristian König 217e60f8db5SAlex Xie switch (state) { 218e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 219ae6d1416STom St Denis for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 220ae6d1416STom St Denis hub = &adev->vmhub[j]; 221e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 222e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 223e60f8db5SAlex Xie tmp = RREG32(reg); 224e60f8db5SAlex Xie tmp &= ~bits; 225e60f8db5SAlex Xie WREG32(reg, tmp); 226e60f8db5SAlex Xie } 227e60f8db5SAlex Xie } 228e60f8db5SAlex Xie break; 229e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 230ae6d1416STom St Denis for (j = 0; j < AMDGPU_MAX_VMHUBS; j++) { 231ae6d1416STom St Denis hub = &adev->vmhub[j]; 232e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 233e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 234e60f8db5SAlex Xie tmp = RREG32(reg); 235e60f8db5SAlex Xie tmp |= bits; 236e60f8db5SAlex Xie WREG32(reg, tmp); 237e60f8db5SAlex Xie } 238e60f8db5SAlex Xie } 239e60f8db5SAlex Xie default: 240e60f8db5SAlex Xie break; 241e60f8db5SAlex Xie } 242e60f8db5SAlex Xie 243e60f8db5SAlex Xie return 0; 244e60f8db5SAlex Xie } 245e60f8db5SAlex Xie 246e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 247e60f8db5SAlex Xie struct amdgpu_irq_src *source, 248e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 249e60f8db5SAlex Xie { 250c4f46f22SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[entry->vmid_src]; 2514d6cbde3SFelix Kuehling uint32_t status = 0; 252e60f8db5SAlex Xie u64 addr; 253e60f8db5SAlex Xie 254e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 255e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 256e60f8db5SAlex Xie 25779a0c465SMonk Liu if (!amdgpu_sriov_vf(adev)) { 2585a9b8e8aSChristian König status = RREG32(hub->vm_l2_pro_fault_status); 2595a9b8e8aSChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 2604d6cbde3SFelix Kuehling } 261e60f8db5SAlex Xie 2624d6cbde3SFelix Kuehling if (printk_ratelimit()) { 263efaa9646SAndrey Grodzovsky struct amdgpu_task_info task_info = { 0 }; 264efaa9646SAndrey Grodzovsky 265efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 266efaa9646SAndrey Grodzovsky 2674d6cbde3SFelix Kuehling dev_err(adev->dev, 2680c79c0bbSChristian König "[%s] VMC page fault (src_id:%u ring:%u vmid:%u pasid:%u, for process %s pid %d thread %s pid %d)\n", 269c4f46f22SChristian König entry->vmid_src ? "mmhub" : "gfxhub", 270c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 271efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 272efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 2737d0aa376SAndrey Grodzovsky dev_err(adev->dev, " in page starting at address 0x%016llx from %d\n", 27479a0c465SMonk Liu addr, entry->client_id); 2754d6cbde3SFelix Kuehling if (!amdgpu_sriov_vf(adev)) 2764d6cbde3SFelix Kuehling dev_err(adev->dev, 2774d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 2784d6cbde3SFelix Kuehling status); 27979a0c465SMonk Liu } 280e60f8db5SAlex Xie 281e60f8db5SAlex Xie return 0; 282e60f8db5SAlex Xie } 283e60f8db5SAlex Xie 284e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 285e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 286e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 287e60f8db5SAlex Xie }; 288e60f8db5SAlex Xie 289e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 290e60f8db5SAlex Xie { 291770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 292770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 293e60f8db5SAlex Xie } 294e60f8db5SAlex Xie 295c4f46f22SChristian König static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid) 29603f89febSChristian König { 29703f89febSChristian König u32 req = 0; 29803f89febSChristian König 299c4f46f22SChristian König /* invalidate using legacy mode on vmid*/ 30003f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 301c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 30203f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, 0); 30303f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 30403f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 30503f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 30603f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 30703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 30803f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 30903f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 31003f89febSChristian König 31103f89febSChristian König return req; 31203f89febSChristian König } 31303f89febSChristian König 3141849e737Skbuild test robot static signed long amdgpu_kiq_reg_write_reg_wait(struct amdgpu_device *adev, 3153890d111SEmily Deng uint32_t reg0, uint32_t reg1, 3163890d111SEmily Deng uint32_t ref, uint32_t mask) 3173890d111SEmily Deng { 3183890d111SEmily Deng signed long r, cnt = 0; 3193890d111SEmily Deng unsigned long flags; 3203890d111SEmily Deng uint32_t seq; 3213890d111SEmily Deng struct amdgpu_kiq *kiq = &adev->gfx.kiq; 3223890d111SEmily Deng struct amdgpu_ring *ring = &kiq->ring; 3233890d111SEmily Deng 3243890d111SEmily Deng spin_lock_irqsave(&kiq->ring_lock, flags); 3253890d111SEmily Deng 3263890d111SEmily Deng amdgpu_ring_alloc(ring, 32); 3273890d111SEmily Deng amdgpu_ring_emit_reg_write_reg_wait(ring, reg0, reg1, 3283890d111SEmily Deng ref, mask); 3293890d111SEmily Deng amdgpu_fence_emit_polling(ring, &seq); 3303890d111SEmily Deng amdgpu_ring_commit(ring); 3313890d111SEmily Deng spin_unlock_irqrestore(&kiq->ring_lock, flags); 3323890d111SEmily Deng 3333890d111SEmily Deng r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 3343890d111SEmily Deng 335ae74da3eSEmily Deng /* don't wait anymore for IRQ context */ 336ae74da3eSEmily Deng if (r < 1 && in_interrupt()) 3373890d111SEmily Deng goto failed_kiq; 3383890d111SEmily Deng 3393890d111SEmily Deng might_sleep(); 3403890d111SEmily Deng 3413890d111SEmily Deng while (r < 1 && cnt++ < MAX_KIQ_REG_TRY) { 3423890d111SEmily Deng msleep(MAX_KIQ_REG_BAILOUT_INTERVAL); 3433890d111SEmily Deng r = amdgpu_fence_wait_polling(ring, seq, MAX_KIQ_REG_WAIT); 3443890d111SEmily Deng } 3453890d111SEmily Deng 3463890d111SEmily Deng if (cnt > MAX_KIQ_REG_TRY) 3473890d111SEmily Deng goto failed_kiq; 3483890d111SEmily Deng 3493890d111SEmily Deng return 0; 3503890d111SEmily Deng 3513890d111SEmily Deng failed_kiq: 3523890d111SEmily Deng pr_err("failed to invalidate tlb with kiq\n"); 3533890d111SEmily Deng return r; 3543890d111SEmily Deng } 3553890d111SEmily Deng 356e60f8db5SAlex Xie /* 357e60f8db5SAlex Xie * GART 358e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 359e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 360e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 361e60f8db5SAlex Xie */ 362e60f8db5SAlex Xie 363e60f8db5SAlex Xie /** 364132f34e4SChristian König * gmc_v9_0_flush_gpu_tlb - gart tlb flush callback 365e60f8db5SAlex Xie * 366e60f8db5SAlex Xie * @adev: amdgpu_device pointer 367e60f8db5SAlex Xie * @vmid: vm instance to flush 368e60f8db5SAlex Xie * 369e60f8db5SAlex Xie * Flush the TLB for the requested page table. 370e60f8db5SAlex Xie */ 371132f34e4SChristian König static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, 372e60f8db5SAlex Xie uint32_t vmid) 373e60f8db5SAlex Xie { 374e60f8db5SAlex Xie /* Use register 17 for GART */ 375e60f8db5SAlex Xie const unsigned eng = 17; 376e60f8db5SAlex Xie unsigned i, j; 3773890d111SEmily Deng int r; 378e60f8db5SAlex Xie 379e60f8db5SAlex Xie for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { 380e60f8db5SAlex Xie struct amdgpu_vmhub *hub = &adev->vmhub[i]; 38103f89febSChristian König u32 tmp = gmc_v9_0_get_invalidate_req(vmid); 382e60f8db5SAlex Xie 383fc0faf04SEmily Deng if (adev->gfx.kiq.ring.ready && 384ae74da3eSEmily Deng (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 385ae74da3eSEmily Deng !adev->in_gpu_reset) { 3863890d111SEmily Deng r = amdgpu_kiq_reg_write_reg_wait(adev, hub->vm_inv_eng0_req + eng, 3873890d111SEmily Deng hub->vm_inv_eng0_ack + eng, tmp, 1 << vmid); 3883890d111SEmily Deng if (!r) 3893890d111SEmily Deng continue; 390fc0faf04SEmily Deng } 3913890d111SEmily Deng 3923890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 3933890d111SEmily Deng 394c7a7266bSXiangliang Yu WREG32_NO_KIQ(hub->vm_inv_eng0_req + eng, tmp); 395e60f8db5SAlex Xie 396e60f8db5SAlex Xie /* Busy wait for ACK.*/ 397e60f8db5SAlex Xie for (j = 0; j < 100; j++) { 398c7a7266bSXiangliang Yu tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng); 399e60f8db5SAlex Xie tmp &= 1 << vmid; 400e60f8db5SAlex Xie if (tmp) 401e60f8db5SAlex Xie break; 402e60f8db5SAlex Xie cpu_relax(); 403e60f8db5SAlex Xie } 4043890d111SEmily Deng if (j < 100) { 4053890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 406e60f8db5SAlex Xie continue; 4073890d111SEmily Deng } 408e60f8db5SAlex Xie 409e60f8db5SAlex Xie /* Wait for ACK with a delay.*/ 410e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 411c7a7266bSXiangliang Yu tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + eng); 412e60f8db5SAlex Xie tmp &= 1 << vmid; 413e60f8db5SAlex Xie if (tmp) 414e60f8db5SAlex Xie break; 415e60f8db5SAlex Xie udelay(1); 416e60f8db5SAlex Xie } 4173890d111SEmily Deng if (j < adev->usec_timeout) { 4183890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 419e60f8db5SAlex Xie continue; 4203890d111SEmily Deng } 4213890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 422e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 423e60f8db5SAlex Xie } 424e60f8db5SAlex Xie } 425e60f8db5SAlex Xie 4269096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 427c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 4289096d6e5SChristian König { 429250b4228SChristian König struct amdgpu_device *adev = ring->adev; 430250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 4319096d6e5SChristian König uint32_t req = gmc_v9_0_get_invalidate_req(vmid); 4329096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 4339096d6e5SChristian König 4349096d6e5SChristian König amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + (2 * vmid), 4359096d6e5SChristian König lower_32_bits(pd_addr)); 4369096d6e5SChristian König 4379096d6e5SChristian König amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + (2 * vmid), 4389096d6e5SChristian König upper_32_bits(pd_addr)); 4399096d6e5SChristian König 440f8bc9037SAlex Deucher amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + eng, 441f8bc9037SAlex Deucher hub->vm_inv_eng0_ack + eng, 442f8bc9037SAlex Deucher req, 1 << vmid); 443f732b6b3SChristian König 4449096d6e5SChristian König return pd_addr; 4459096d6e5SChristian König } 4469096d6e5SChristian König 447c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 448c633c00bSChristian König unsigned pasid) 449c633c00bSChristian König { 450c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 451c633c00bSChristian König uint32_t reg; 452c633c00bSChristian König 453c633c00bSChristian König if (ring->funcs->vmhub == AMDGPU_GFXHUB) 454c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 455c633c00bSChristian König else 456c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 457c633c00bSChristian König 458c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 459c633c00bSChristian König } 460c633c00bSChristian König 461e60f8db5SAlex Xie /** 462132f34e4SChristian König * gmc_v9_0_set_pte_pde - update the page tables using MMIO 463e60f8db5SAlex Xie * 464e60f8db5SAlex Xie * @adev: amdgpu_device pointer 465e60f8db5SAlex Xie * @cpu_pt_addr: cpu address of the page table 466e60f8db5SAlex Xie * @gpu_page_idx: entry in the page table to update 467e60f8db5SAlex Xie * @addr: dst addr to write into pte/pde 468e60f8db5SAlex Xie * @flags: access flags 469e60f8db5SAlex Xie * 470e60f8db5SAlex Xie * Update the page tables using the CPU. 471e60f8db5SAlex Xie */ 472132f34e4SChristian König static int gmc_v9_0_set_pte_pde(struct amdgpu_device *adev, void *cpu_pt_addr, 473132f34e4SChristian König uint32_t gpu_page_idx, uint64_t addr, 474e60f8db5SAlex Xie uint64_t flags) 475e60f8db5SAlex Xie { 476e60f8db5SAlex Xie void __iomem *ptr = (void *)cpu_pt_addr; 477e60f8db5SAlex Xie uint64_t value; 478e60f8db5SAlex Xie 479e60f8db5SAlex Xie /* 480e60f8db5SAlex Xie * PTE format on VEGA 10: 481e60f8db5SAlex Xie * 63:59 reserved 482e60f8db5SAlex Xie * 58:57 mtype 483e60f8db5SAlex Xie * 56 F 484e60f8db5SAlex Xie * 55 L 485e60f8db5SAlex Xie * 54 P 486e60f8db5SAlex Xie * 53 SW 487e60f8db5SAlex Xie * 52 T 488e60f8db5SAlex Xie * 50:48 reserved 489e60f8db5SAlex Xie * 47:12 4k physical page base address 490e60f8db5SAlex Xie * 11:7 fragment 491e60f8db5SAlex Xie * 6 write 492e60f8db5SAlex Xie * 5 read 493e60f8db5SAlex Xie * 4 exe 494e60f8db5SAlex Xie * 3 Z 495e60f8db5SAlex Xie * 2 snooped 496e60f8db5SAlex Xie * 1 system 497e60f8db5SAlex Xie * 0 valid 498e60f8db5SAlex Xie * 499e60f8db5SAlex Xie * PDE format on VEGA 10: 500e60f8db5SAlex Xie * 63:59 block fragment size 501e60f8db5SAlex Xie * 58:55 reserved 502e60f8db5SAlex Xie * 54 P 503e60f8db5SAlex Xie * 53:48 reserved 504e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 505e60f8db5SAlex Xie * 5:3 reserved 506e60f8db5SAlex Xie * 2 C 507e60f8db5SAlex Xie * 1 system 508e60f8db5SAlex Xie * 0 valid 509e60f8db5SAlex Xie */ 510e60f8db5SAlex Xie 511e60f8db5SAlex Xie /* 512e60f8db5SAlex Xie * The following is for PTE only. GART does not have PDEs. 513e60f8db5SAlex Xie */ 514e60f8db5SAlex Xie value = addr & 0x0000FFFFFFFFF000ULL; 515e60f8db5SAlex Xie value |= flags; 516e60f8db5SAlex Xie writeq(value, ptr + (gpu_page_idx * 8)); 517e60f8db5SAlex Xie return 0; 518e60f8db5SAlex Xie } 519e60f8db5SAlex Xie 520e60f8db5SAlex Xie static uint64_t gmc_v9_0_get_vm_pte_flags(struct amdgpu_device *adev, 521e60f8db5SAlex Xie uint32_t flags) 522e60f8db5SAlex Xie 523e60f8db5SAlex Xie { 524e60f8db5SAlex Xie uint64_t pte_flag = 0; 525e60f8db5SAlex Xie 526e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_EXECUTABLE) 527e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_EXECUTABLE; 528e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_READABLE) 529e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_READABLE; 530e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_WRITEABLE) 531e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_WRITEABLE; 532e60f8db5SAlex Xie 533e60f8db5SAlex Xie switch (flags & AMDGPU_VM_MTYPE_MASK) { 534e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 535e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 536e60f8db5SAlex Xie break; 537e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 538e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 539e60f8db5SAlex Xie break; 540e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 541e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_WC); 542e60f8db5SAlex Xie break; 543e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 544e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_CC); 545e60f8db5SAlex Xie break; 546e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 547e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_UC); 548e60f8db5SAlex Xie break; 549e60f8db5SAlex Xie default: 550e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_MTYPE(MTYPE_NC); 551e60f8db5SAlex Xie break; 552e60f8db5SAlex Xie } 553e60f8db5SAlex Xie 554e60f8db5SAlex Xie if (flags & AMDGPU_VM_PAGE_PRT) 555e60f8db5SAlex Xie pte_flag |= AMDGPU_PTE_PRT; 556e60f8db5SAlex Xie 557e60f8db5SAlex Xie return pte_flag; 558e60f8db5SAlex Xie } 559e60f8db5SAlex Xie 5603de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 5613de676d8SChristian König uint64_t *addr, uint64_t *flags) 562f75e237cSChristian König { 563bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 5643de676d8SChristian König *addr = adev->vm_manager.vram_base_offset + *addr - 565770d13b1SChristian König adev->gmc.vram_start; 5663de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 5676a42fd6fSChristian König 568770d13b1SChristian König if (!adev->gmc.translate_further) 5696a42fd6fSChristian König return; 5706a42fd6fSChristian König 5716a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 5726a42fd6fSChristian König /* Set the block fragment size */ 5736a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 5746a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 5756a42fd6fSChristian König 5766a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 5776a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 5786a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 5796a42fd6fSChristian König else 5806a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 5816a42fd6fSChristian König } 582f75e237cSChristian König } 583f75e237cSChristian König 584132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 585132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 5869096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 587c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 588132f34e4SChristian König .set_pte_pde = gmc_v9_0_set_pte_pde, 589b1166325SChristian König .get_vm_pte_flags = gmc_v9_0_get_vm_pte_flags, 590b1166325SChristian König .get_vm_pde = gmc_v9_0_get_vm_pde 591e60f8db5SAlex Xie }; 592e60f8db5SAlex Xie 593132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 594e60f8db5SAlex Xie { 595132f34e4SChristian König if (adev->gmc.gmc_funcs == NULL) 596132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 597e60f8db5SAlex Xie } 598e60f8db5SAlex Xie 599e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 600e60f8db5SAlex Xie { 601e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 602e60f8db5SAlex Xie 603132f34e4SChristian König gmc_v9_0_set_gmc_funcs(adev); 604e60f8db5SAlex Xie gmc_v9_0_set_irq_funcs(adev); 605e60f8db5SAlex Xie 606770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 607770d13b1SChristian König adev->gmc.shared_aperture_end = 608770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 609bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 610770d13b1SChristian König adev->gmc.private_aperture_end = 611770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 612a7ea6548SAlex Deucher 613e60f8db5SAlex Xie return 0; 614e60f8db5SAlex Xie } 615e60f8db5SAlex Xie 61602bab923SDavid Panariti static int gmc_v9_0_ecc_available(struct amdgpu_device *adev) 61702bab923SDavid Panariti { 61802bab923SDavid Panariti uint32_t reg_val; 61902bab923SDavid Panariti uint32_t reg_addr; 62002bab923SDavid Panariti uint32_t field_val; 62102bab923SDavid Panariti size_t i; 62202bab923SDavid Panariti uint32_t fv2; 62302bab923SDavid Panariti size_t lost_sheep; 62402bab923SDavid Panariti 62502bab923SDavid Panariti DRM_DEBUG("ecc: gmc_v9_0_ecc_available()\n"); 62602bab923SDavid Panariti 62702bab923SDavid Panariti lost_sheep = 0; 62802bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umclocalcap_addrs); ++i) { 62902bab923SDavid Panariti reg_addr = ecc_umclocalcap_addrs[i]; 63002bab923SDavid Panariti DRM_DEBUG("ecc: " 63102bab923SDavid Panariti "UMCCH_UmcLocalCap[%zu]: reg_addr: 0x%08x\n", 63202bab923SDavid Panariti i, reg_addr); 63302bab923SDavid Panariti reg_val = RREG32(reg_addr); 63402bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UmcLocalCap, 63502bab923SDavid Panariti EccDis); 63602bab923SDavid Panariti DRM_DEBUG("ecc: " 63702bab923SDavid Panariti "reg_val: 0x%08x, " 63802bab923SDavid Panariti "EccDis: 0x%08x, ", 63902bab923SDavid Panariti reg_val, field_val); 64002bab923SDavid Panariti if (field_val) { 64102bab923SDavid Panariti DRM_ERROR("ecc: UmcLocalCap:EccDis is set.\n"); 64202bab923SDavid Panariti ++lost_sheep; 64302bab923SDavid Panariti } 64402bab923SDavid Panariti } 64502bab923SDavid Panariti 64602bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umcch_umc_config_addrs); ++i) { 64702bab923SDavid Panariti reg_addr = ecc_umcch_umc_config_addrs[i]; 64802bab923SDavid Panariti DRM_DEBUG("ecc: " 64902bab923SDavid Panariti "UMCCH0_0_UMC_CONFIG[%zu]: reg_addr: 0x%08x", 65002bab923SDavid Panariti i, reg_addr); 65102bab923SDavid Panariti reg_val = RREG32(reg_addr); 65202bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_UMC_CONFIG, 65302bab923SDavid Panariti DramReady); 65402bab923SDavid Panariti DRM_DEBUG("ecc: " 65502bab923SDavid Panariti "reg_val: 0x%08x, " 65602bab923SDavid Panariti "DramReady: 0x%08x\n", 65702bab923SDavid Panariti reg_val, field_val); 65802bab923SDavid Panariti 65902bab923SDavid Panariti if (!field_val) { 66002bab923SDavid Panariti DRM_ERROR("ecc: UMC_CONFIG:DramReady is not set.\n"); 66102bab923SDavid Panariti ++lost_sheep; 66202bab923SDavid Panariti } 66302bab923SDavid Panariti } 66402bab923SDavid Panariti 66502bab923SDavid Panariti for (i = 0; i < ARRAY_SIZE(ecc_umcch_eccctrl_addrs); ++i) { 66602bab923SDavid Panariti reg_addr = ecc_umcch_eccctrl_addrs[i]; 66702bab923SDavid Panariti DRM_DEBUG("ecc: " 66802bab923SDavid Panariti "UMCCH_EccCtrl[%zu]: reg_addr: 0x%08x, ", 66902bab923SDavid Panariti i, reg_addr); 67002bab923SDavid Panariti reg_val = RREG32(reg_addr); 67102bab923SDavid Panariti field_val = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl, 67202bab923SDavid Panariti WrEccEn); 67302bab923SDavid Panariti fv2 = REG_GET_FIELD(reg_val, UMCCH0_0_EccCtrl, 67402bab923SDavid Panariti RdEccEn); 67502bab923SDavid Panariti DRM_DEBUG("ecc: " 67602bab923SDavid Panariti "reg_val: 0x%08x, " 67702bab923SDavid Panariti "WrEccEn: 0x%08x, " 67802bab923SDavid Panariti "RdEccEn: 0x%08x\n", 67902bab923SDavid Panariti reg_val, field_val, fv2); 68002bab923SDavid Panariti 68102bab923SDavid Panariti if (!field_val) { 6825a16008fSAlex Deucher DRM_DEBUG("ecc: WrEccEn is not set\n"); 68302bab923SDavid Panariti ++lost_sheep; 68402bab923SDavid Panariti } 68502bab923SDavid Panariti if (!fv2) { 6865a16008fSAlex Deucher DRM_DEBUG("ecc: RdEccEn is not set\n"); 68702bab923SDavid Panariti ++lost_sheep; 68802bab923SDavid Panariti } 68902bab923SDavid Panariti } 69002bab923SDavid Panariti 69102bab923SDavid Panariti DRM_DEBUG("ecc: lost_sheep: %zu\n", lost_sheep); 69202bab923SDavid Panariti return lost_sheep == 0; 69302bab923SDavid Panariti } 69402bab923SDavid Panariti 695cd2b5623SAlex Deucher static bool gmc_v9_0_keep_stolen_memory(struct amdgpu_device *adev) 696cd2b5623SAlex Deucher { 697cd2b5623SAlex Deucher 698cd2b5623SAlex Deucher /* 699cd2b5623SAlex Deucher * TODO: 700cd2b5623SAlex Deucher * Currently there is a bug where some memory client outside 701cd2b5623SAlex Deucher * of the driver writes to first 8M of VRAM on S3 resume, 702cd2b5623SAlex Deucher * this overrides GART which by default gets placed in first 8M and 703cd2b5623SAlex Deucher * causes VM_FAULTS once GTT is accessed. 704cd2b5623SAlex Deucher * Keep the stolen memory reservation until the while this is not solved. 705cd2b5623SAlex Deucher * Also check code in gmc_v9_0_get_vbios_fb_size and gmc_v9_0_late_init 706cd2b5623SAlex Deucher */ 707cd2b5623SAlex Deucher switch (adev->asic_type) { 708cd2b5623SAlex Deucher case CHIP_RAVEN: 7096fb81375SAlex Deucher return false; 710cd2b5623SAlex Deucher case CHIP_VEGA12: 71195010ba7SAlex Deucher return false; 71295010ba7SAlex Deucher case CHIP_VEGA10: 713cd2b5623SAlex Deucher case CHIP_VEGA20: 714cd2b5623SAlex Deucher default: 715cd2b5623SAlex Deucher return true; 716cd2b5623SAlex Deucher } 717cd2b5623SAlex Deucher } 718cd2b5623SAlex Deucher 719e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 720e60f8db5SAlex Xie { 721e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 722c5066129Sozeng /* 723c5066129Sozeng * The latest engine allocation on gfx9 is: 724c5066129Sozeng * Engine 0, 1: idle 725c5066129Sozeng * Engine 2, 3: firmware 726c5066129Sozeng * Engine 4~13: amdgpu ring, subject to change when ring number changes 727c5066129Sozeng * Engine 14~15: idle 728c5066129Sozeng * Engine 16: kfd tlb invalidation 729c5066129Sozeng * Engine 17: Gart flushes 730c5066129Sozeng */ 731c5066129Sozeng unsigned vm_inv_eng[AMDGPU_MAX_VMHUBS] = { 4, 4 }; 7324789c463SChristian König unsigned i; 73302bab923SDavid Panariti int r; 7344789c463SChristian König 735cd2b5623SAlex Deucher if (!gmc_v9_0_keep_stolen_memory(adev)) 736cd2b5623SAlex Deucher amdgpu_bo_late_init(adev); 7376f752ec2SAndrey Grodzovsky 7384789c463SChristian König for(i = 0; i < adev->num_rings; ++i) { 7394789c463SChristian König struct amdgpu_ring *ring = adev->rings[i]; 7404789c463SChristian König unsigned vmhub = ring->funcs->vmhub; 7414789c463SChristian König 7424789c463SChristian König ring->vm_inv_eng = vm_inv_eng[vmhub]++; 743775f55f1STom St Denis dev_info(adev->dev, "ring %u(%s) uses VM inv eng %u on hub %u\n", 744775f55f1STom St Denis ring->idx, ring->name, ring->vm_inv_eng, 745775f55f1STom St Denis ring->funcs->vmhub); 7464789c463SChristian König } 7474789c463SChristian König 748c5066129Sozeng /* Engine 16 is used for KFD and 17 for GART flushes */ 7494789c463SChristian König for(i = 0; i < AMDGPU_MAX_VMHUBS; ++i) 750c5066129Sozeng BUG_ON(vm_inv_eng[i] > 16); 7514789c463SChristian König 7527b6cbae2SMonk Liu if (adev->asic_type == CHIP_VEGA10 && !amdgpu_sriov_vf(adev)) { 75302bab923SDavid Panariti r = gmc_v9_0_ecc_available(adev); 75402bab923SDavid Panariti if (r == 1) { 75502bab923SDavid Panariti DRM_INFO("ECC is active.\n"); 75602bab923SDavid Panariti } else if (r == 0) { 75702bab923SDavid Panariti DRM_INFO("ECC is not present.\n"); 758e1d1a772SAlex Deucher adev->df_funcs->enable_ecc_force_par_wr_rmw(adev, false); 75902bab923SDavid Panariti } else { 76002bab923SDavid Panariti DRM_ERROR("gmc_v9_0_ecc_available() failed. r: %d\n", r); 76102bab923SDavid Panariti return r; 76202bab923SDavid Panariti } 7635ba4fa35SAlex Deucher } 76402bab923SDavid Panariti 765770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 766e60f8db5SAlex Xie } 767e60f8db5SAlex Xie 768e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 769770d13b1SChristian König struct amdgpu_gmc *mc) 770e60f8db5SAlex Xie { 771eeb2487dSMonk Liu u64 base = 0; 772eeb2487dSMonk Liu if (!amdgpu_sriov_vf(adev)) 773eeb2487dSMonk Liu base = mmhub_v1_0_get_fb_location(adev); 774961c75cfSChristian König amdgpu_gmc_vram_location(adev, &adev->gmc, base); 775961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 776bc099ee9SChunming Zhou /* base offset of vram pages */ 777bc099ee9SChunming Zhou adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev); 778e60f8db5SAlex Xie } 779e60f8db5SAlex Xie 780e60f8db5SAlex Xie /** 781e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 782e60f8db5SAlex Xie * 783e60f8db5SAlex Xie * @adev: amdgpu_device pointer 784e60f8db5SAlex Xie * 785e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 786e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 787e60f8db5SAlex Xie * Returns 0 for success. 788e60f8db5SAlex Xie */ 789e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 790e60f8db5SAlex Xie { 791e60f8db5SAlex Xie int chansize, numchan; 792d6895ad3SChristian König int r; 793e60f8db5SAlex Xie 7943d918c0eSShaoyun Liu if (amdgpu_emu_mode != 1) 795770d13b1SChristian König adev->gmc.vram_width = amdgpu_atomfirmware_get_vram_width(adev); 796770d13b1SChristian König if (!adev->gmc.vram_width) { 797e60f8db5SAlex Xie /* hbm memory channel size */ 798585b7f16STom St Denis if (adev->flags & AMD_IS_APU) 799585b7f16STom St Denis chansize = 64; 800585b7f16STom St Denis else 801e60f8db5SAlex Xie chansize = 128; 802e60f8db5SAlex Xie 803070706c0SHawking Zhang numchan = adev->df_funcs->get_hbm_channel_number(adev); 804770d13b1SChristian König adev->gmc.vram_width = numchan * chansize; 8058d6a5230SAlex Deucher } 806e60f8db5SAlex Xie 807e60f8db5SAlex Xie /* size in MB on si */ 808770d13b1SChristian König adev->gmc.mc_vram_size = 809bf383fb6SAlex Deucher adev->nbio_funcs->get_memsize(adev) * 1024ULL * 1024ULL; 810770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 811d6895ad3SChristian König 812d6895ad3SChristian König if (!(adev->flags & AMD_IS_APU)) { 813d6895ad3SChristian König r = amdgpu_device_resize_fb_bar(adev); 814d6895ad3SChristian König if (r) 815d6895ad3SChristian König return r; 816d6895ad3SChristian König } 817770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 818770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 819e60f8db5SAlex Xie 820156a81beSChunming Zhou #ifdef CONFIG_X86_64 821156a81beSChunming Zhou if (adev->flags & AMD_IS_APU) { 822156a81beSChunming Zhou adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev); 823156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 824156a81beSChunming Zhou } 825156a81beSChunming Zhou #endif 826e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 827770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 828770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 829770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 830e60f8db5SAlex Xie 831c3db7b5aSAlex Deucher /* set the gart size */ 832c3db7b5aSAlex Deucher if (amdgpu_gart_size == -1) { 833c3db7b5aSAlex Deucher switch (adev->asic_type) { 834c3db7b5aSAlex Deucher case CHIP_VEGA10: /* all engines support GPUVM */ 835273a14cdSAlex Deucher case CHIP_VEGA12: /* all engines support GPUVM */ 836d96b428cSFeifei Xu case CHIP_VEGA20: 837c3db7b5aSAlex Deucher default: 838fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 839c3db7b5aSAlex Deucher break; 840c3db7b5aSAlex Deucher case CHIP_RAVEN: /* DCE SG support */ 841770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 842c3db7b5aSAlex Deucher break; 843c3db7b5aSAlex Deucher } 844c3db7b5aSAlex Deucher } else { 845770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 846c3db7b5aSAlex Deucher } 847c3db7b5aSAlex Deucher 848770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 849e60f8db5SAlex Xie 850e60f8db5SAlex Xie return 0; 851e60f8db5SAlex Xie } 852e60f8db5SAlex Xie 853e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 854e60f8db5SAlex Xie { 855e60f8db5SAlex Xie int r; 856e60f8db5SAlex Xie 8571123b989SChristian König if (adev->gart.bo) { 858e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 859e60f8db5SAlex Xie return 0; 860e60f8db5SAlex Xie } 861e60f8db5SAlex Xie /* Initialize common gart structure */ 862e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 863e60f8db5SAlex Xie if (r) 864e60f8db5SAlex Xie return r; 865e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 866e60f8db5SAlex Xie adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE(MTYPE_UC) | 867e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 868e60f8db5SAlex Xie return amdgpu_gart_table_vram_alloc(adev); 869e60f8db5SAlex Xie } 870e60f8db5SAlex Xie 871ebdef28eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 872ebdef28eSAlex Deucher { 873ebdef28eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 874ebdef28eSAlex Deucher unsigned size; 875ebdef28eSAlex Deucher 8766f752ec2SAndrey Grodzovsky /* 8776f752ec2SAndrey Grodzovsky * TODO Remove once GART corruption is resolved 8786f752ec2SAndrey Grodzovsky * Check related code in gmc_v9_0_sw_fini 8796f752ec2SAndrey Grodzovsky * */ 880cd2b5623SAlex Deucher if (gmc_v9_0_keep_stolen_memory(adev)) 881cd2b5623SAlex Deucher return 9 * 1024 * 1024; 8826f752ec2SAndrey Grodzovsky 883ebdef28eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 884ebdef28eSAlex Deucher size = 9 * 1024 * 1024; /* reserve 8MB for vga emulator and 1 MB for FB */ 885ebdef28eSAlex Deucher } else { 886ebdef28eSAlex Deucher u32 viewport; 887ebdef28eSAlex Deucher 888ebdef28eSAlex Deucher switch (adev->asic_type) { 889ebdef28eSAlex Deucher case CHIP_RAVEN: 890ebdef28eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 891ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, 892ebdef28eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 893ebdef28eSAlex Deucher REG_GET_FIELD(viewport, 894ebdef28eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 895ebdef28eSAlex Deucher 4); 896ebdef28eSAlex Deucher break; 897ebdef28eSAlex Deucher case CHIP_VEGA10: 898ebdef28eSAlex Deucher case CHIP_VEGA12: 899cd2b5623SAlex Deucher case CHIP_VEGA20: 900ebdef28eSAlex Deucher default: 901ebdef28eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 902ebdef28eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 903ebdef28eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 904ebdef28eSAlex Deucher 4); 905ebdef28eSAlex Deucher break; 906ebdef28eSAlex Deucher } 907ebdef28eSAlex Deucher } 908ebdef28eSAlex Deucher /* return 0 if the pre-OS buffer uses up most of vram */ 909ebdef28eSAlex Deucher if ((adev->gmc.real_vram_size - size) < (8 * 1024 * 1024)) 910ebdef28eSAlex Deucher return 0; 9116f752ec2SAndrey Grodzovsky 912ebdef28eSAlex Deucher return size; 913ebdef28eSAlex Deucher } 914ebdef28eSAlex Deucher 915e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 916e60f8db5SAlex Xie { 917e60f8db5SAlex Xie int r; 918e60f8db5SAlex Xie int dma_bits; 919e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 920e60f8db5SAlex Xie 9210c8c0847SHuang Rui gfxhub_v1_0_init(adev); 92277f6c763SHuang Rui mmhub_v1_0_init(adev); 9230c8c0847SHuang Rui 924770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 925e60f8db5SAlex Xie 9261e09b053SHawking Zhang adev->gmc.vram_type = amdgpu_atomfirmware_get_vram_type(adev); 927fd66560bSHawking Zhang switch (adev->asic_type) { 928fd66560bSHawking Zhang case CHIP_RAVEN: 9296a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 930f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 9316a42fd6fSChristian König } else { 9326a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 9336a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 934770d13b1SChristian König adev->gmc.translate_further = 9356a42fd6fSChristian König adev->vm_manager.num_level > 1; 9366a42fd6fSChristian König } 937fd66560bSHawking Zhang break; 938fd66560bSHawking Zhang case CHIP_VEGA10: 939273a14cdSAlex Deucher case CHIP_VEGA12: 940d96b428cSFeifei Xu case CHIP_VEGA20: 94136b32a68SZhang, Jerry /* 94236b32a68SZhang, Jerry * To fulfill 4-level page support, 94336b32a68SZhang, Jerry * vm size is 256TB (48bit), maximum size of Vega10, 94436b32a68SZhang, Jerry * block size 512 (9bit) 94536b32a68SZhang, Jerry */ 946f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 947fd66560bSHawking Zhang break; 948fd66560bSHawking Zhang default: 949fd66560bSHawking Zhang break; 950fd66560bSHawking Zhang } 951fd66560bSHawking Zhang 952e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 95344a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 954770d13b1SChristian König &adev->gmc.vm_fault); 95544a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 956770d13b1SChristian König &adev->gmc.vm_fault); 957e60f8db5SAlex Xie 958e60f8db5SAlex Xie if (r) 959e60f8db5SAlex Xie return r; 960e60f8db5SAlex Xie 961e60f8db5SAlex Xie /* Set the internal MC address mask 962e60f8db5SAlex Xie * This is the max address of the GPU's 963e60f8db5SAlex Xie * internal address space. 964e60f8db5SAlex Xie */ 965770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 966e60f8db5SAlex Xie 967e60f8db5SAlex Xie /* set DMA mask + need_dma32 flags. 968e60f8db5SAlex Xie * PCIE - can handle 44-bits. 969e60f8db5SAlex Xie * IGP - can handle 44-bits 970e60f8db5SAlex Xie * PCI - dma32 for legacy pci gart, 44 bits on vega10 971e60f8db5SAlex Xie */ 972e60f8db5SAlex Xie adev->need_dma32 = false; 973e60f8db5SAlex Xie dma_bits = adev->need_dma32 ? 32 : 44; 974e60f8db5SAlex Xie r = pci_set_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 975e60f8db5SAlex Xie if (r) { 976e60f8db5SAlex Xie adev->need_dma32 = true; 977e60f8db5SAlex Xie dma_bits = 32; 978e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 979e60f8db5SAlex Xie } 980e60f8db5SAlex Xie r = pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(dma_bits)); 981e60f8db5SAlex Xie if (r) { 982e60f8db5SAlex Xie pci_set_consistent_dma_mask(adev->pdev, DMA_BIT_MASK(32)); 983e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No coherent DMA available.\n"); 984e60f8db5SAlex Xie } 985fd5fd480SChunming Zhou adev->need_swiotlb = drm_get_max_iomem() > ((u64)1 << dma_bits); 986e60f8db5SAlex Xie 987e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 988e60f8db5SAlex Xie if (r) 989e60f8db5SAlex Xie return r; 990e60f8db5SAlex Xie 991ebdef28eSAlex Deucher adev->gmc.stolen_size = gmc_v9_0_get_vbios_fb_size(adev); 992ebdef28eSAlex Deucher 993e60f8db5SAlex Xie /* Memory manager */ 994e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 995e60f8db5SAlex Xie if (r) 996e60f8db5SAlex Xie return r; 997e60f8db5SAlex Xie 998e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 999e60f8db5SAlex Xie if (r) 1000e60f8db5SAlex Xie return r; 1001e60f8db5SAlex Xie 100205ec3edaSChristian König /* 100305ec3edaSChristian König * number of VMs 100405ec3edaSChristian König * VMID 0 is reserved for System 100505ec3edaSChristian König * amdgpu graphics/compute will use VMIDs 1-7 100605ec3edaSChristian König * amdkfd will use VMIDs 8-15 100705ec3edaSChristian König */ 100805ec3edaSChristian König adev->vm_manager.id_mgr[AMDGPU_GFXHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 100905ec3edaSChristian König adev->vm_manager.id_mgr[AMDGPU_MMHUB].num_ids = AMDGPU_NUM_OF_VMIDS; 101005ec3edaSChristian König 101105ec3edaSChristian König amdgpu_vm_manager_init(adev); 101205ec3edaSChristian König 101305ec3edaSChristian König return 0; 1014e60f8db5SAlex Xie } 1015e60f8db5SAlex Xie 1016e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 1017e60f8db5SAlex Xie { 1018e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1019e60f8db5SAlex Xie 1020f59548c8SMonk Liu amdgpu_gem_force_release(adev); 1021e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 10226f752ec2SAndrey Grodzovsky 1023cd2b5623SAlex Deucher if (gmc_v9_0_keep_stolen_memory(adev)) 10246f752ec2SAndrey Grodzovsky amdgpu_bo_free_kernel(&adev->stolen_vga_memory, NULL, NULL); 10256f752ec2SAndrey Grodzovsky 1026a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 1027e60f8db5SAlex Xie amdgpu_bo_fini(adev); 1028a3d9103eSAndrey Grodzovsky amdgpu_gart_fini(adev); 1029e60f8db5SAlex Xie 1030e60f8db5SAlex Xie return 0; 1031e60f8db5SAlex Xie } 1032e60f8db5SAlex Xie 1033e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1034e60f8db5SAlex Xie { 1035946a4d5bSShaoyun Liu 1036e60f8db5SAlex Xie switch (adev->asic_type) { 1037e60f8db5SAlex Xie case CHIP_VEGA10: 1038d96b428cSFeifei Xu case CHIP_VEGA20: 1039946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 10405c583018SEvan Quan golden_settings_mmhub_1_0_0, 1041c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1042946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 10435c583018SEvan Quan golden_settings_athub_1_0_0, 1044c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1045e60f8db5SAlex Xie break; 1046273a14cdSAlex Deucher case CHIP_VEGA12: 1047273a14cdSAlex Deucher break; 1048e4f3abaaSChunming Zhou case CHIP_RAVEN: 1049946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 10505c583018SEvan Quan golden_settings_athub_1_0_0, 1051c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1052e4f3abaaSChunming Zhou break; 1053e60f8db5SAlex Xie default: 1054e60f8db5SAlex Xie break; 1055e60f8db5SAlex Xie } 1056e60f8db5SAlex Xie } 1057e60f8db5SAlex Xie 1058e60f8db5SAlex Xie /** 1059e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1060e60f8db5SAlex Xie * 1061e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1062e60f8db5SAlex Xie */ 1063e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1064e60f8db5SAlex Xie { 1065e60f8db5SAlex Xie int r; 1066e60f8db5SAlex Xie bool value; 1067e60f8db5SAlex Xie u32 tmp; 1068e60f8db5SAlex Xie 10699c3f2b54SAlex Deucher amdgpu_device_program_register_sequence(adev, 1070e60f8db5SAlex Xie golden_settings_vega10_hdp, 1071c47b41a7SChristian König ARRAY_SIZE(golden_settings_vega10_hdp)); 1072e60f8db5SAlex Xie 10731123b989SChristian König if (adev->gart.bo == NULL) { 1074e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1075e60f8db5SAlex Xie return -EINVAL; 1076e60f8db5SAlex Xie } 1077ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1078ce1b1b66SMonk Liu if (r) 1079ce1b1b66SMonk Liu return r; 1080e60f8db5SAlex Xie 10812fcd43ceSHawking Zhang switch (adev->asic_type) { 10822fcd43ceSHawking Zhang case CHIP_RAVEN: 10832fcd43ceSHawking Zhang mmhub_v1_0_initialize_power_gating(adev); 1084f8386b35SHawking Zhang mmhub_v1_0_update_power_gating(adev, true); 10852fcd43ceSHawking Zhang break; 10862fcd43ceSHawking Zhang default: 10872fcd43ceSHawking Zhang break; 10882fcd43ceSHawking Zhang } 10892fcd43ceSHawking Zhang 1090e60f8db5SAlex Xie r = gfxhub_v1_0_gart_enable(adev); 1091e60f8db5SAlex Xie if (r) 1092e60f8db5SAlex Xie return r; 1093e60f8db5SAlex Xie 1094e60f8db5SAlex Xie r = mmhub_v1_0_gart_enable(adev); 1095e60f8db5SAlex Xie if (r) 1096e60f8db5SAlex Xie return r; 1097e60f8db5SAlex Xie 1098846347c9STom St Denis WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1099e60f8db5SAlex Xie 1100b9509c80SHuang Rui tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1101b9509c80SHuang Rui WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1102e60f8db5SAlex Xie 11031d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 110469882565SChristian König adev->nbio_funcs->hdp_flush(adev, NULL); 11051d4e0a8cSMonk Liu 1106e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1107e60f8db5SAlex Xie value = false; 1108e60f8db5SAlex Xie else 1109e60f8db5SAlex Xie value = true; 1110e60f8db5SAlex Xie 1111e60f8db5SAlex Xie gfxhub_v1_0_set_fault_enable_default(adev, value); 1112e60f8db5SAlex Xie mmhub_v1_0_set_fault_enable_default(adev, value); 1113132f34e4SChristian König gmc_v9_0_flush_gpu_tlb(adev, 0); 1114e60f8db5SAlex Xie 1115e60f8db5SAlex Xie DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1116770d13b1SChristian König (unsigned)(adev->gmc.gart_size >> 20), 11174e830fb1SChristian König (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1118e60f8db5SAlex Xie adev->gart.ready = true; 1119e60f8db5SAlex Xie return 0; 1120e60f8db5SAlex Xie } 1121e60f8db5SAlex Xie 1122e60f8db5SAlex Xie static int gmc_v9_0_hw_init(void *handle) 1123e60f8db5SAlex Xie { 1124e60f8db5SAlex Xie int r; 1125e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1126e60f8db5SAlex Xie 1127e60f8db5SAlex Xie /* The sequence of these two function calls matters.*/ 1128e60f8db5SAlex Xie gmc_v9_0_init_golden_registers(adev); 1129e60f8db5SAlex Xie 1130edca2d05SAlex Deucher if (adev->mode_info.num_crtc) { 1131edca2d05SAlex Deucher /* Lockout access through VGA aperture*/ 11324d9c333aSTom St Denis WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1133edca2d05SAlex Deucher 1134edca2d05SAlex Deucher /* disable VGA render */ 11354d9c333aSTom St Denis WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1136edca2d05SAlex Deucher } 1137edca2d05SAlex Deucher 1138e60f8db5SAlex Xie r = gmc_v9_0_gart_enable(adev); 1139e60f8db5SAlex Xie 1140e60f8db5SAlex Xie return r; 1141e60f8db5SAlex Xie } 1142e60f8db5SAlex Xie 1143e60f8db5SAlex Xie /** 1144e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1145e60f8db5SAlex Xie * 1146e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1147e60f8db5SAlex Xie * 1148e60f8db5SAlex Xie * This disables all VM page table. 1149e60f8db5SAlex Xie */ 1150e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1151e60f8db5SAlex Xie { 1152e60f8db5SAlex Xie gfxhub_v1_0_gart_disable(adev); 1153e60f8db5SAlex Xie mmhub_v1_0_gart_disable(adev); 1154ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1155e60f8db5SAlex Xie } 1156e60f8db5SAlex Xie 1157e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1158e60f8db5SAlex Xie { 1159e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1160e60f8db5SAlex Xie 11615dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 11625dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 11635dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 11645dd696aeSTrigger Huang return 0; 11655dd696aeSTrigger Huang } 11665dd696aeSTrigger Huang 1167770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1168e60f8db5SAlex Xie gmc_v9_0_gart_disable(adev); 1169e60f8db5SAlex Xie 1170e60f8db5SAlex Xie return 0; 1171e60f8db5SAlex Xie } 1172e60f8db5SAlex Xie 1173e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1174e60f8db5SAlex Xie { 1175e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1176e60f8db5SAlex Xie 1177f053cd47STom St Denis return gmc_v9_0_hw_fini(adev); 1178e60f8db5SAlex Xie } 1179e60f8db5SAlex Xie 1180e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1181e60f8db5SAlex Xie { 1182e60f8db5SAlex Xie int r; 1183e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1184e60f8db5SAlex Xie 1185e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1186e60f8db5SAlex Xie if (r) 1187e60f8db5SAlex Xie return r; 1188e60f8db5SAlex Xie 1189620f774fSChristian König amdgpu_vmid_reset_all(adev); 1190e60f8db5SAlex Xie 119132601d48SChristian König return 0; 1192e60f8db5SAlex Xie } 1193e60f8db5SAlex Xie 1194e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1195e60f8db5SAlex Xie { 1196e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1197e60f8db5SAlex Xie return true; 1198e60f8db5SAlex Xie } 1199e60f8db5SAlex Xie 1200e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1201e60f8db5SAlex Xie { 1202e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1203e60f8db5SAlex Xie return 0; 1204e60f8db5SAlex Xie } 1205e60f8db5SAlex Xie 1206e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1207e60f8db5SAlex Xie { 1208e60f8db5SAlex Xie /* XXX for emulation.*/ 1209e60f8db5SAlex Xie return 0; 1210e60f8db5SAlex Xie } 1211e60f8db5SAlex Xie 1212e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1213e60f8db5SAlex Xie enum amd_clockgating_state state) 1214e60f8db5SAlex Xie { 1215d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1216d5583d4fSHuang Rui 1217d5583d4fSHuang Rui return mmhub_v1_0_set_clockgating(adev, state); 1218e60f8db5SAlex Xie } 1219e60f8db5SAlex Xie 122013052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 122113052be5SHuang Rui { 122213052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 122313052be5SHuang Rui 122413052be5SHuang Rui mmhub_v1_0_get_clockgating(adev, flags); 122513052be5SHuang Rui } 122613052be5SHuang Rui 1227e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1228e60f8db5SAlex Xie enum amd_powergating_state state) 1229e60f8db5SAlex Xie { 1230e60f8db5SAlex Xie return 0; 1231e60f8db5SAlex Xie } 1232e60f8db5SAlex Xie 1233e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1234e60f8db5SAlex Xie .name = "gmc_v9_0", 1235e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1236e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1237e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1238e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1239e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1240e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1241e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1242e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1243e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1244e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1245e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1246e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1247e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 124813052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1249e60f8db5SAlex Xie }; 1250e60f8db5SAlex Xie 1251e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1252e60f8db5SAlex Xie { 1253e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1254e60f8db5SAlex Xie .major = 9, 1255e60f8db5SAlex Xie .minor = 0, 1256e60f8db5SAlex Xie .rev = 0, 1257e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1258e60f8db5SAlex Xie }; 1259