1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23f867723bSSam Ravnborg 24e60f8db5SAlex Xie #include <linux/firmware.h> 25f867723bSSam Ravnborg #include <linux/pci.h> 26f867723bSSam Ravnborg 27fd5fd480SChunming Zhou #include <drm/drm_cache.h> 28f867723bSSam Ravnborg 29e60f8db5SAlex Xie #include "amdgpu.h" 30e60f8db5SAlex Xie #include "gmc_v9_0.h" 318d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 322cddc50eSHuang Rui #include "amdgpu_gem.h" 33e60f8db5SAlex Xie 34cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 35135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 36135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 37fb960bd2SFeifei Xu #include "vega10_enum.h" 3865417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 39ea930000SAlex Sierra #include "athub/athub_1_0_sh_mask.h" 406ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 41250b4228SChristian König #include "oss/osssys_4_0_offset.h" 42e60f8db5SAlex Xie 43946a4d5bSShaoyun Liu #include "soc15.h" 44ea930000SAlex Sierra #include "soc15d.h" 45e60f8db5SAlex Xie #include "soc15_common.h" 4690c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 47e60f8db5SAlex Xie 48e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 49e60f8db5SAlex Xie #include "mmhub_v1_0.h" 50bee7b51aSLe Ma #include "athub_v1_0.h" 51bf0a60b7SAlex Deucher #include "gfxhub_v1_1.h" 5251cce480SLe Ma #include "mmhub_v9_4.h" 5385e39550SLe Ma #include "mmhub_v1_7.h" 545b6b35aaSHawking Zhang #include "umc_v6_1.h" 55e7da754bSMonk Liu #include "umc_v6_0.h" 56*6f12507fSHawking Zhang #include "hdp_v4_0.h" 57e60f8db5SAlex Xie 5844a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 5944a99b65SAndrey Grodzovsky 60791c4769Sxinhui pan #include "amdgpu_ras.h" 61029fbd43SHawking Zhang #include "amdgpu_xgmi.h" 62791c4769Sxinhui pan 63ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 64ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 65ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 66ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 67ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 68ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 69ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 70f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d 71f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 72f8646661SAlex Deucher 73ebdef28eSAlex Deucher 74be99ecbfSAlex Deucher static const char *gfxhub_client_ids[] = { 75be99ecbfSAlex Deucher "CB", 76be99ecbfSAlex Deucher "DB", 77be99ecbfSAlex Deucher "IA", 78be99ecbfSAlex Deucher "WD", 79be99ecbfSAlex Deucher "CPF", 80be99ecbfSAlex Deucher "CPC", 81be99ecbfSAlex Deucher "CPG", 82be99ecbfSAlex Deucher "RLC", 83be99ecbfSAlex Deucher "TCP", 84be99ecbfSAlex Deucher "SQC (inst)", 85be99ecbfSAlex Deucher "SQC (data)", 86be99ecbfSAlex Deucher "SQG", 87be99ecbfSAlex Deucher "PA", 88be99ecbfSAlex Deucher }; 89be99ecbfSAlex Deucher 9002f23f5fSAlex Deucher static const char *mmhub_client_ids_raven[][2] = { 9102f23f5fSAlex Deucher [0][0] = "MP1", 9202f23f5fSAlex Deucher [1][0] = "MP0", 9302f23f5fSAlex Deucher [2][0] = "VCN", 9402f23f5fSAlex Deucher [3][0] = "VCNU", 9502f23f5fSAlex Deucher [4][0] = "HDP", 9602f23f5fSAlex Deucher [5][0] = "DCE", 9702f23f5fSAlex Deucher [13][0] = "UTCL2", 9802f23f5fSAlex Deucher [19][0] = "TLS", 9902f23f5fSAlex Deucher [26][0] = "OSS", 10002f23f5fSAlex Deucher [27][0] = "SDMA0", 10102f23f5fSAlex Deucher [0][1] = "MP1", 10202f23f5fSAlex Deucher [1][1] = "MP0", 10302f23f5fSAlex Deucher [2][1] = "VCN", 10402f23f5fSAlex Deucher [3][1] = "VCNU", 10502f23f5fSAlex Deucher [4][1] = "HDP", 10602f23f5fSAlex Deucher [5][1] = "XDP", 10702f23f5fSAlex Deucher [6][1] = "DBGU0", 10802f23f5fSAlex Deucher [7][1] = "DCE", 10902f23f5fSAlex Deucher [8][1] = "DCEDWB0", 11002f23f5fSAlex Deucher [9][1] = "DCEDWB1", 11102f23f5fSAlex Deucher [26][1] = "OSS", 11202f23f5fSAlex Deucher [27][1] = "SDMA0", 11302f23f5fSAlex Deucher }; 11402f23f5fSAlex Deucher 11502f23f5fSAlex Deucher static const char *mmhub_client_ids_renoir[][2] = { 11602f23f5fSAlex Deucher [0][0] = "MP1", 11702f23f5fSAlex Deucher [1][0] = "MP0", 11802f23f5fSAlex Deucher [2][0] = "HDP", 11902f23f5fSAlex Deucher [4][0] = "DCEDMC", 12002f23f5fSAlex Deucher [5][0] = "DCEVGA", 12102f23f5fSAlex Deucher [13][0] = "UTCL2", 12202f23f5fSAlex Deucher [19][0] = "TLS", 12302f23f5fSAlex Deucher [26][0] = "OSS", 12402f23f5fSAlex Deucher [27][0] = "SDMA0", 12502f23f5fSAlex Deucher [28][0] = "VCN", 12602f23f5fSAlex Deucher [29][0] = "VCNU", 12702f23f5fSAlex Deucher [30][0] = "JPEG", 12802f23f5fSAlex Deucher [0][1] = "MP1", 12902f23f5fSAlex Deucher [1][1] = "MP0", 13002f23f5fSAlex Deucher [2][1] = "HDP", 13102f23f5fSAlex Deucher [3][1] = "XDP", 13202f23f5fSAlex Deucher [6][1] = "DBGU0", 13302f23f5fSAlex Deucher [7][1] = "DCEDMC", 13402f23f5fSAlex Deucher [8][1] = "DCEVGA", 13502f23f5fSAlex Deucher [9][1] = "DCEDWB", 13602f23f5fSAlex Deucher [26][1] = "OSS", 13702f23f5fSAlex Deucher [27][1] = "SDMA0", 13802f23f5fSAlex Deucher [28][1] = "VCN", 13902f23f5fSAlex Deucher [29][1] = "VCNU", 14002f23f5fSAlex Deucher [30][1] = "JPEG", 14102f23f5fSAlex Deucher }; 14202f23f5fSAlex Deucher 14302f23f5fSAlex Deucher static const char *mmhub_client_ids_vega10[][2] = { 14402f23f5fSAlex Deucher [0][0] = "MP0", 14502f23f5fSAlex Deucher [1][0] = "UVD", 14602f23f5fSAlex Deucher [2][0] = "UVDU", 14702f23f5fSAlex Deucher [3][0] = "HDP", 14802f23f5fSAlex Deucher [13][0] = "UTCL2", 14902f23f5fSAlex Deucher [14][0] = "OSS", 15002f23f5fSAlex Deucher [15][0] = "SDMA1", 15102f23f5fSAlex Deucher [32+0][0] = "VCE0", 15202f23f5fSAlex Deucher [32+1][0] = "VCE0U", 15302f23f5fSAlex Deucher [32+2][0] = "XDMA", 15402f23f5fSAlex Deucher [32+3][0] = "DCE", 15502f23f5fSAlex Deucher [32+4][0] = "MP1", 15602f23f5fSAlex Deucher [32+14][0] = "SDMA0", 15702f23f5fSAlex Deucher [0][1] = "MP0", 15802f23f5fSAlex Deucher [1][1] = "UVD", 15902f23f5fSAlex Deucher [2][1] = "UVDU", 16002f23f5fSAlex Deucher [3][1] = "DBGU0", 16102f23f5fSAlex Deucher [4][1] = "HDP", 16202f23f5fSAlex Deucher [5][1] = "XDP", 16302f23f5fSAlex Deucher [14][1] = "OSS", 16402f23f5fSAlex Deucher [15][1] = "SDMA0", 16502f23f5fSAlex Deucher [32+0][1] = "VCE0", 16602f23f5fSAlex Deucher [32+1][1] = "VCE0U", 16702f23f5fSAlex Deucher [32+2][1] = "XDMA", 16802f23f5fSAlex Deucher [32+3][1] = "DCE", 16902f23f5fSAlex Deucher [32+4][1] = "DCEDWB", 17002f23f5fSAlex Deucher [32+5][1] = "MP1", 17102f23f5fSAlex Deucher [32+6][1] = "DBGU1", 17202f23f5fSAlex Deucher [32+14][1] = "SDMA1", 17302f23f5fSAlex Deucher }; 17402f23f5fSAlex Deucher 17502f23f5fSAlex Deucher static const char *mmhub_client_ids_vega12[][2] = { 17602f23f5fSAlex Deucher [0][0] = "MP0", 17702f23f5fSAlex Deucher [1][0] = "VCE0", 17802f23f5fSAlex Deucher [2][0] = "VCE0U", 17902f23f5fSAlex Deucher [3][0] = "HDP", 18002f23f5fSAlex Deucher [13][0] = "UTCL2", 18102f23f5fSAlex Deucher [14][0] = "OSS", 18202f23f5fSAlex Deucher [15][0] = "SDMA1", 18302f23f5fSAlex Deucher [32+0][0] = "DCE", 18402f23f5fSAlex Deucher [32+1][0] = "XDMA", 18502f23f5fSAlex Deucher [32+2][0] = "UVD", 18602f23f5fSAlex Deucher [32+3][0] = "UVDU", 18702f23f5fSAlex Deucher [32+4][0] = "MP1", 18802f23f5fSAlex Deucher [32+15][0] = "SDMA0", 18902f23f5fSAlex Deucher [0][1] = "MP0", 19002f23f5fSAlex Deucher [1][1] = "VCE0", 19102f23f5fSAlex Deucher [2][1] = "VCE0U", 19202f23f5fSAlex Deucher [3][1] = "DBGU0", 19302f23f5fSAlex Deucher [4][1] = "HDP", 19402f23f5fSAlex Deucher [5][1] = "XDP", 19502f23f5fSAlex Deucher [14][1] = "OSS", 19602f23f5fSAlex Deucher [15][1] = "SDMA0", 19702f23f5fSAlex Deucher [32+0][1] = "DCE", 19802f23f5fSAlex Deucher [32+1][1] = "DCEDWB", 19902f23f5fSAlex Deucher [32+2][1] = "XDMA", 20002f23f5fSAlex Deucher [32+3][1] = "UVD", 20102f23f5fSAlex Deucher [32+4][1] = "UVDU", 20202f23f5fSAlex Deucher [32+5][1] = "MP1", 20302f23f5fSAlex Deucher [32+6][1] = "DBGU1", 20402f23f5fSAlex Deucher [32+15][1] = "SDMA1", 20502f23f5fSAlex Deucher }; 20602f23f5fSAlex Deucher 20702f23f5fSAlex Deucher static const char *mmhub_client_ids_vega20[][2] = { 20802f23f5fSAlex Deucher [0][0] = "XDMA", 20902f23f5fSAlex Deucher [1][0] = "DCE", 21002f23f5fSAlex Deucher [2][0] = "VCE0", 21102f23f5fSAlex Deucher [3][0] = "VCE0U", 21202f23f5fSAlex Deucher [4][0] = "UVD", 21302f23f5fSAlex Deucher [5][0] = "UVD1U", 21402f23f5fSAlex Deucher [13][0] = "OSS", 21502f23f5fSAlex Deucher [14][0] = "HDP", 21602f23f5fSAlex Deucher [15][0] = "SDMA0", 21702f23f5fSAlex Deucher [32+0][0] = "UVD", 21802f23f5fSAlex Deucher [32+1][0] = "UVDU", 21902f23f5fSAlex Deucher [32+2][0] = "MP1", 22002f23f5fSAlex Deucher [32+3][0] = "MP0", 22102f23f5fSAlex Deucher [32+12][0] = "UTCL2", 22202f23f5fSAlex Deucher [32+14][0] = "SDMA1", 22302f23f5fSAlex Deucher [0][1] = "XDMA", 22402f23f5fSAlex Deucher [1][1] = "DCE", 22502f23f5fSAlex Deucher [2][1] = "DCEDWB", 22602f23f5fSAlex Deucher [3][1] = "VCE0", 22702f23f5fSAlex Deucher [4][1] = "VCE0U", 22802f23f5fSAlex Deucher [5][1] = "UVD1", 22902f23f5fSAlex Deucher [6][1] = "UVD1U", 23002f23f5fSAlex Deucher [7][1] = "DBGU0", 23102f23f5fSAlex Deucher [8][1] = "XDP", 23202f23f5fSAlex Deucher [13][1] = "OSS", 23302f23f5fSAlex Deucher [14][1] = "HDP", 23402f23f5fSAlex Deucher [15][1] = "SDMA0", 23502f23f5fSAlex Deucher [32+0][1] = "UVD", 23602f23f5fSAlex Deucher [32+1][1] = "UVDU", 23702f23f5fSAlex Deucher [32+2][1] = "DBGU1", 23802f23f5fSAlex Deucher [32+3][1] = "MP1", 23902f23f5fSAlex Deucher [32+4][1] = "MP0", 24002f23f5fSAlex Deucher [32+14][1] = "SDMA1", 24102f23f5fSAlex Deucher }; 24202f23f5fSAlex Deucher 24302f23f5fSAlex Deucher static const char *mmhub_client_ids_arcturus[][2] = { 244e83db774SAlex Deucher [0][0] = "DBGU1", 245e83db774SAlex Deucher [1][0] = "XDP", 24602f23f5fSAlex Deucher [2][0] = "MP1", 24702f23f5fSAlex Deucher [14][0] = "HDP", 248e83db774SAlex Deucher [171][0] = "JPEG", 249e83db774SAlex Deucher [172][0] = "VCN", 250e83db774SAlex Deucher [173][0] = "VCNU", 251e83db774SAlex Deucher [203][0] = "JPEG1", 252e83db774SAlex Deucher [204][0] = "VCN1", 253e83db774SAlex Deucher [205][0] = "VCN1U", 254e83db774SAlex Deucher [256][0] = "SDMA0", 255e83db774SAlex Deucher [257][0] = "SDMA1", 256e83db774SAlex Deucher [258][0] = "SDMA2", 257e83db774SAlex Deucher [259][0] = "SDMA3", 258e83db774SAlex Deucher [260][0] = "SDMA4", 259e83db774SAlex Deucher [261][0] = "SDMA5", 260e83db774SAlex Deucher [262][0] = "SDMA6", 261e83db774SAlex Deucher [263][0] = "SDMA7", 262e83db774SAlex Deucher [384][0] = "OSS", 26302f23f5fSAlex Deucher [0][1] = "DBGU1", 26402f23f5fSAlex Deucher [1][1] = "XDP", 26502f23f5fSAlex Deucher [2][1] = "MP1", 26602f23f5fSAlex Deucher [14][1] = "HDP", 267e83db774SAlex Deucher [171][1] = "JPEG", 268e83db774SAlex Deucher [172][1] = "VCN", 269e83db774SAlex Deucher [173][1] = "VCNU", 270e83db774SAlex Deucher [203][1] = "JPEG1", 271e83db774SAlex Deucher [204][1] = "VCN1", 272e83db774SAlex Deucher [205][1] = "VCN1U", 273e83db774SAlex Deucher [256][1] = "SDMA0", 274e83db774SAlex Deucher [257][1] = "SDMA1", 275e83db774SAlex Deucher [258][1] = "SDMA2", 276e83db774SAlex Deucher [259][1] = "SDMA3", 277e83db774SAlex Deucher [260][1] = "SDMA4", 278e83db774SAlex Deucher [261][1] = "SDMA5", 279e83db774SAlex Deucher [262][1] = "SDMA6", 280e83db774SAlex Deucher [263][1] = "SDMA7", 281e83db774SAlex Deucher [384][1] = "OSS", 28202f23f5fSAlex Deucher }; 283e60f8db5SAlex Xie 284e844cd99SAlex Deucher static const char *mmhub_client_ids_aldebaran[][2] = { 285e844cd99SAlex Deucher [2][0] = "MP1", 286e844cd99SAlex Deucher [3][0] = "MP0", 287f4ec3e50SAlex Sierra [32+1][0] = "DBGU_IO0", 288f4ec3e50SAlex Sierra [32+2][0] = "DBGU_IO2", 289e844cd99SAlex Deucher [32+4][0] = "MPIO", 290e844cd99SAlex Deucher [96+11][0] = "JPEG0", 291e844cd99SAlex Deucher [96+12][0] = "VCN0", 292e844cd99SAlex Deucher [96+13][0] = "VCNU0", 293e844cd99SAlex Deucher [128+11][0] = "JPEG1", 294e844cd99SAlex Deucher [128+12][0] = "VCN1", 295e844cd99SAlex Deucher [128+13][0] = "VCNU1", 296f4ec3e50SAlex Sierra [160+1][0] = "XDP", 297e844cd99SAlex Deucher [160+14][0] = "HDP", 298f4ec3e50SAlex Sierra [256+0][0] = "SDMA0", 299f4ec3e50SAlex Sierra [256+1][0] = "SDMA1", 300f4ec3e50SAlex Sierra [256+2][0] = "SDMA2", 301f4ec3e50SAlex Sierra [256+3][0] = "SDMA3", 302f4ec3e50SAlex Sierra [256+4][0] = "SDMA4", 303f4ec3e50SAlex Sierra [384+0][0] = "OSS", 304e844cd99SAlex Deucher [2][1] = "MP1", 305e844cd99SAlex Deucher [3][1] = "MP0", 306e844cd99SAlex Deucher [32+1][1] = "DBGU_IO0", 307e844cd99SAlex Deucher [32+2][1] = "DBGU_IO2", 308e844cd99SAlex Deucher [32+4][1] = "MPIO", 309e844cd99SAlex Deucher [96+11][1] = "JPEG0", 310e844cd99SAlex Deucher [96+12][1] = "VCN0", 311e844cd99SAlex Deucher [96+13][1] = "VCNU0", 312e844cd99SAlex Deucher [128+11][1] = "JPEG1", 313e844cd99SAlex Deucher [128+12][1] = "VCN1", 314e844cd99SAlex Deucher [128+13][1] = "VCNU1", 315f4ec3e50SAlex Sierra [160+1][1] = "XDP", 316e844cd99SAlex Deucher [160+14][1] = "HDP", 317f4ec3e50SAlex Sierra [256+0][1] = "SDMA0", 318f4ec3e50SAlex Sierra [256+1][1] = "SDMA1", 319f4ec3e50SAlex Sierra [256+2][1] = "SDMA2", 320f4ec3e50SAlex Sierra [256+3][1] = "SDMA3", 321f4ec3e50SAlex Sierra [256+4][1] = "SDMA4", 322f4ec3e50SAlex Sierra [384+0][1] = "OSS", 323e844cd99SAlex Deucher }; 324e844cd99SAlex Deucher 325946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 3265c583018SEvan Quan { 327946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 328946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 3295c583018SEvan Quan }; 3305c583018SEvan Quan 331946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 3325c583018SEvan Quan { 333946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 334946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 3355c583018SEvan Quan }; 3365c583018SEvan Quan 337791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 338791c4769Sxinhui pan (0x000143c0 + 0x00000000), 339791c4769Sxinhui pan (0x000143c0 + 0x00000800), 340791c4769Sxinhui pan (0x000143c0 + 0x00001000), 341791c4769Sxinhui pan (0x000143c0 + 0x00001800), 342791c4769Sxinhui pan (0x000543c0 + 0x00000000), 343791c4769Sxinhui pan (0x000543c0 + 0x00000800), 344791c4769Sxinhui pan (0x000543c0 + 0x00001000), 345791c4769Sxinhui pan (0x000543c0 + 0x00001800), 346791c4769Sxinhui pan (0x000943c0 + 0x00000000), 347791c4769Sxinhui pan (0x000943c0 + 0x00000800), 348791c4769Sxinhui pan (0x000943c0 + 0x00001000), 349791c4769Sxinhui pan (0x000943c0 + 0x00001800), 350791c4769Sxinhui pan (0x000d43c0 + 0x00000000), 351791c4769Sxinhui pan (0x000d43c0 + 0x00000800), 352791c4769Sxinhui pan (0x000d43c0 + 0x00001000), 353791c4769Sxinhui pan (0x000d43c0 + 0x00001800), 354791c4769Sxinhui pan (0x001143c0 + 0x00000000), 355791c4769Sxinhui pan (0x001143c0 + 0x00000800), 356791c4769Sxinhui pan (0x001143c0 + 0x00001000), 357791c4769Sxinhui pan (0x001143c0 + 0x00001800), 358791c4769Sxinhui pan (0x001543c0 + 0x00000000), 359791c4769Sxinhui pan (0x001543c0 + 0x00000800), 360791c4769Sxinhui pan (0x001543c0 + 0x00001000), 361791c4769Sxinhui pan (0x001543c0 + 0x00001800), 362791c4769Sxinhui pan (0x001943c0 + 0x00000000), 363791c4769Sxinhui pan (0x001943c0 + 0x00000800), 364791c4769Sxinhui pan (0x001943c0 + 0x00001000), 365791c4769Sxinhui pan (0x001943c0 + 0x00001800), 366791c4769Sxinhui pan (0x001d43c0 + 0x00000000), 367791c4769Sxinhui pan (0x001d43c0 + 0x00000800), 368791c4769Sxinhui pan (0x001d43c0 + 0x00001000), 369791c4769Sxinhui pan (0x001d43c0 + 0x00001800), 37002bab923SDavid Panariti }; 37102bab923SDavid Panariti 372791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 373791c4769Sxinhui pan (0x000143e0 + 0x00000000), 374791c4769Sxinhui pan (0x000143e0 + 0x00000800), 375791c4769Sxinhui pan (0x000143e0 + 0x00001000), 376791c4769Sxinhui pan (0x000143e0 + 0x00001800), 377791c4769Sxinhui pan (0x000543e0 + 0x00000000), 378791c4769Sxinhui pan (0x000543e0 + 0x00000800), 379791c4769Sxinhui pan (0x000543e0 + 0x00001000), 380791c4769Sxinhui pan (0x000543e0 + 0x00001800), 381791c4769Sxinhui pan (0x000943e0 + 0x00000000), 382791c4769Sxinhui pan (0x000943e0 + 0x00000800), 383791c4769Sxinhui pan (0x000943e0 + 0x00001000), 384791c4769Sxinhui pan (0x000943e0 + 0x00001800), 385791c4769Sxinhui pan (0x000d43e0 + 0x00000000), 386791c4769Sxinhui pan (0x000d43e0 + 0x00000800), 387791c4769Sxinhui pan (0x000d43e0 + 0x00001000), 388791c4769Sxinhui pan (0x000d43e0 + 0x00001800), 389791c4769Sxinhui pan (0x001143e0 + 0x00000000), 390791c4769Sxinhui pan (0x001143e0 + 0x00000800), 391791c4769Sxinhui pan (0x001143e0 + 0x00001000), 392791c4769Sxinhui pan (0x001143e0 + 0x00001800), 393791c4769Sxinhui pan (0x001543e0 + 0x00000000), 394791c4769Sxinhui pan (0x001543e0 + 0x00000800), 395791c4769Sxinhui pan (0x001543e0 + 0x00001000), 396791c4769Sxinhui pan (0x001543e0 + 0x00001800), 397791c4769Sxinhui pan (0x001943e0 + 0x00000000), 398791c4769Sxinhui pan (0x001943e0 + 0x00000800), 399791c4769Sxinhui pan (0x001943e0 + 0x00001000), 400791c4769Sxinhui pan (0x001943e0 + 0x00001800), 401791c4769Sxinhui pan (0x001d43e0 + 0x00000000), 402791c4769Sxinhui pan (0x001d43e0 + 0x00000800), 403791c4769Sxinhui pan (0x001d43e0 + 0x00001000), 404791c4769Sxinhui pan (0x001d43e0 + 0x00001800), 40502bab923SDavid Panariti }; 40602bab923SDavid Panariti 407791c4769Sxinhui pan static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 408791c4769Sxinhui pan struct amdgpu_irq_src *src, 409791c4769Sxinhui pan unsigned type, 410791c4769Sxinhui pan enum amdgpu_interrupt_state state) 411791c4769Sxinhui pan { 412791c4769Sxinhui pan u32 bits, i, tmp, reg; 413791c4769Sxinhui pan 4141e2c6d55SJohn Clements /* Devices newer then VEGA10/12 shall have these programming 4151e2c6d55SJohn Clements sequences performed by PSP BL */ 4161e2c6d55SJohn Clements if (adev->asic_type >= CHIP_VEGA20) 4171e2c6d55SJohn Clements return 0; 4181e2c6d55SJohn Clements 419791c4769Sxinhui pan bits = 0x7f; 420791c4769Sxinhui pan 421791c4769Sxinhui pan switch (state) { 422791c4769Sxinhui pan case AMDGPU_IRQ_STATE_DISABLE: 423791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 424791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 425791c4769Sxinhui pan tmp = RREG32(reg); 426791c4769Sxinhui pan tmp &= ~bits; 427791c4769Sxinhui pan WREG32(reg, tmp); 428791c4769Sxinhui pan } 429791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 430791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 431791c4769Sxinhui pan tmp = RREG32(reg); 432791c4769Sxinhui pan tmp &= ~bits; 433791c4769Sxinhui pan WREG32(reg, tmp); 434791c4769Sxinhui pan } 435791c4769Sxinhui pan break; 436791c4769Sxinhui pan case AMDGPU_IRQ_STATE_ENABLE: 437791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 438791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 439791c4769Sxinhui pan tmp = RREG32(reg); 440791c4769Sxinhui pan tmp |= bits; 441791c4769Sxinhui pan WREG32(reg, tmp); 442791c4769Sxinhui pan } 443791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 444791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 445791c4769Sxinhui pan tmp = RREG32(reg); 446791c4769Sxinhui pan tmp |= bits; 447791c4769Sxinhui pan WREG32(reg, tmp); 448791c4769Sxinhui pan } 449791c4769Sxinhui pan break; 450791c4769Sxinhui pan default: 451791c4769Sxinhui pan break; 452791c4769Sxinhui pan } 453791c4769Sxinhui pan 454791c4769Sxinhui pan return 0; 455791c4769Sxinhui pan } 456791c4769Sxinhui pan 457e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 458e60f8db5SAlex Xie struct amdgpu_irq_src *src, 459e60f8db5SAlex Xie unsigned type, 460e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 461e60f8db5SAlex Xie { 462e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 463ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 464e60f8db5SAlex Xie 46511250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46611250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46711250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46811250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46911250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47011250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47111250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 47211250164SChristian König 473e60f8db5SAlex Xie switch (state) { 474e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 4751daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 476ae6d1416STom St Denis hub = &adev->vmhub[j]; 477e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 478e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 479e60f8db5SAlex Xie tmp = RREG32(reg); 480e60f8db5SAlex Xie tmp &= ~bits; 481e60f8db5SAlex Xie WREG32(reg, tmp); 482e60f8db5SAlex Xie } 483e60f8db5SAlex Xie } 484e60f8db5SAlex Xie break; 485e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 4861daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 487ae6d1416STom St Denis hub = &adev->vmhub[j]; 488e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 489e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 490e60f8db5SAlex Xie tmp = RREG32(reg); 491e60f8db5SAlex Xie tmp |= bits; 492e60f8db5SAlex Xie WREG32(reg, tmp); 493e60f8db5SAlex Xie } 494e60f8db5SAlex Xie } 4959304ca4dSGustavo A. R. Silva break; 496e60f8db5SAlex Xie default: 497e60f8db5SAlex Xie break; 498e60f8db5SAlex Xie } 499e60f8db5SAlex Xie 500e60f8db5SAlex Xie return 0; 501e60f8db5SAlex Xie } 502e60f8db5SAlex Xie 503e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 504e60f8db5SAlex Xie struct amdgpu_irq_src *source, 505e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 506e60f8db5SAlex Xie { 507c468f9e2SChristian König bool retry_fault = !!(entry->src_data[1] & 0x80); 50802f23f5fSAlex Deucher uint32_t status = 0, cid = 0, rw = 0; 509e3898719SChristian König struct amdgpu_task_info task_info; 510e3898719SChristian König struct amdgpu_vmhub *hub; 51102f23f5fSAlex Deucher const char *mmhub_cid; 512e3898719SChristian König const char *hub_name; 513e3898719SChristian König u64 addr; 514e60f8db5SAlex Xie 515e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 516e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 517e60f8db5SAlex Xie 5180291150dSChristian König if (retry_fault) { 5190291150dSChristian König /* Returning 1 here also prevents sending the IV to the KFD */ 52022666cc1SChristian König 5210291150dSChristian König /* Process it onyl if it's the first fault for this address */ 5220291150dSChristian König if (entry->ih != &adev->irq.ih_soft && 5230291150dSChristian König amdgpu_gmc_filter_faults(adev, addr, entry->pasid, 5240291150dSChristian König entry->timestamp)) 5250291150dSChristian König return 1; 5260291150dSChristian König 5270291150dSChristian König /* Delegate it to a different ring if the hardware hasn't 5280291150dSChristian König * already done it. 5290291150dSChristian König */ 53058df0d71SSebastian Andrzej Siewior if (entry->ih == &adev->irq.ih) { 5310291150dSChristian König amdgpu_irq_delegate(adev, entry, 8); 5320291150dSChristian König return 1; 5330291150dSChristian König } 5340291150dSChristian König 5350291150dSChristian König /* Try to handle the recoverable page faults by filling page 5360291150dSChristian König * tables 5370291150dSChristian König */ 5380291150dSChristian König if (amdgpu_vm_handle_fault(adev, entry->pasid, addr)) 5390291150dSChristian König return 1; 5400291150dSChristian König } 541ec671737SChristian König 542e3898719SChristian König if (!printk_ratelimit()) 543e3898719SChristian König return 0; 54453499173SXiaojie Yuan 545e3898719SChristian König if (entry->client_id == SOC15_IH_CLIENTID_VMC) { 546e3898719SChristian König hub_name = "mmhub0"; 547e3898719SChristian König hub = &adev->vmhub[AMDGPU_MMHUB_0]; 548e3898719SChristian König } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { 549e3898719SChristian König hub_name = "mmhub1"; 550e3898719SChristian König hub = &adev->vmhub[AMDGPU_MMHUB_1]; 551e3898719SChristian König } else { 552e3898719SChristian König hub_name = "gfxhub0"; 553e3898719SChristian König hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 5544d6cbde3SFelix Kuehling } 555e60f8db5SAlex Xie 55605794effSShirish S memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 557efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 558efaa9646SAndrey Grodzovsky 5594d6cbde3SFelix Kuehling dev_err(adev->dev, 560c468f9e2SChristian König "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 561c468f9e2SChristian König "pasid:%u, for process %s pid %d thread %s pid %d)\n", 56251c60898SLe Ma hub_name, retry_fault ? "retry" : "no-retry", 563c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 564efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 565efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 566be14729aSYong Zhao dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n", 567be14729aSYong Zhao addr, entry->client_id, 568be14729aSYong Zhao soc15_ih_clientid_name[entry->client_id]); 569e3898719SChristian König 570e3898719SChristian König if (amdgpu_sriov_vf(adev)) 571e3898719SChristian König return 0; 572e3898719SChristian König 573e3898719SChristian König /* 574e3898719SChristian König * Issue a dummy read to wait for the status register to 575e3898719SChristian König * be updated to avoid reading an incorrect value due to 576e3898719SChristian König * the new fast GRBM interface. 577e3898719SChristian König */ 5787845d80dSAlex Deucher if ((entry->vmid_src == AMDGPU_GFXHUB_0) && 5797845d80dSAlex Deucher (adev->asic_type < CHIP_ALDEBARAN)) 580e3898719SChristian König RREG32(hub->vm_l2_pro_fault_status); 581e3898719SChristian König 582e3898719SChristian König status = RREG32(hub->vm_l2_pro_fault_status); 583e3898719SChristian König cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID); 584e3898719SChristian König rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW); 585e3898719SChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 586e3898719SChristian König 587e3898719SChristian König 5884d6cbde3SFelix Kuehling dev_err(adev->dev, 5894d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 5904d6cbde3SFelix Kuehling status); 59102f23f5fSAlex Deucher if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) { 592be99ecbfSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 593e3898719SChristian König cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : 594e3898719SChristian König gfxhub_client_ids[cid], 595be99ecbfSAlex Deucher cid); 59602f23f5fSAlex Deucher } else { 59702f23f5fSAlex Deucher switch (adev->asic_type) { 59802f23f5fSAlex Deucher case CHIP_VEGA10: 59902f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega10[cid][rw]; 60002f23f5fSAlex Deucher break; 60102f23f5fSAlex Deucher case CHIP_VEGA12: 60202f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega12[cid][rw]; 60302f23f5fSAlex Deucher break; 60402f23f5fSAlex Deucher case CHIP_VEGA20: 60502f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega20[cid][rw]; 60602f23f5fSAlex Deucher break; 60702f23f5fSAlex Deucher case CHIP_ARCTURUS: 60802f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; 60902f23f5fSAlex Deucher break; 61002f23f5fSAlex Deucher case CHIP_RAVEN: 61102f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_raven[cid][rw]; 61202f23f5fSAlex Deucher break; 61302f23f5fSAlex Deucher case CHIP_RENOIR: 61402f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_renoir[cid][rw]; 61502f23f5fSAlex Deucher break; 616e844cd99SAlex Deucher case CHIP_ALDEBARAN: 617e844cd99SAlex Deucher mmhub_cid = mmhub_client_ids_aldebaran[cid][rw]; 618e844cd99SAlex Deucher break; 61902f23f5fSAlex Deucher default: 62002f23f5fSAlex Deucher mmhub_cid = NULL; 62102f23f5fSAlex Deucher break; 62202f23f5fSAlex Deucher } 62302f23f5fSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 62402f23f5fSAlex Deucher mmhub_cid ? mmhub_cid : "unknown", cid); 62502f23f5fSAlex Deucher } 6265ddd4a9aSYong Zhao dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 6275ddd4a9aSYong Zhao REG_GET_FIELD(status, 6285ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 6295ddd4a9aSYong Zhao dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 6305ddd4a9aSYong Zhao REG_GET_FIELD(status, 6315ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 6325ddd4a9aSYong Zhao dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 6335ddd4a9aSYong Zhao REG_GET_FIELD(status, 6345ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 6355ddd4a9aSYong Zhao dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 6365ddd4a9aSYong Zhao REG_GET_FIELD(status, 6375ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 63802f23f5fSAlex Deucher dev_err(adev->dev, "\t RW: 0x%x\n", rw); 639e60f8db5SAlex Xie return 0; 640e60f8db5SAlex Xie } 641e60f8db5SAlex Xie 642e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 643e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 644e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 645e60f8db5SAlex Xie }; 646e60f8db5SAlex Xie 647791c4769Sxinhui pan 648791c4769Sxinhui pan static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 649791c4769Sxinhui pan .set = gmc_v9_0_ecc_interrupt_state, 65034cc4fd9STao Zhou .process = amdgpu_umc_process_ecc_irq, 651791c4769Sxinhui pan }; 652791c4769Sxinhui pan 653e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 654e60f8db5SAlex Xie { 655770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 656770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 657791c4769Sxinhui pan 65868d705ddSHawking Zhang if (!amdgpu_sriov_vf(adev) && 65968d705ddSHawking Zhang !adev->gmc.xgmi.connected_to_cpu) { 660791c4769Sxinhui pan adev->gmc.ecc_irq.num_types = 1; 661791c4769Sxinhui pan adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 662e60f8db5SAlex Xie } 6632ee9403eSZhigang Luo } 664e60f8db5SAlex Xie 6652a79d868SYong Zhao static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 6662a79d868SYong Zhao uint32_t flush_type) 66703f89febSChristian König { 66803f89febSChristian König u32 req = 0; 66903f89febSChristian König 67003f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 671c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 6722a79d868SYong Zhao req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 67303f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 67403f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 67503f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 67603f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 67703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 67803f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 67903f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 68003f89febSChristian König 68103f89febSChristian König return req; 68203f89febSChristian König } 68303f89febSChristian König 68490f6452cSchangzhu /** 68590f6452cSchangzhu * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore 68690f6452cSchangzhu * 68790f6452cSchangzhu * @adev: amdgpu_device pointer 68890f6452cSchangzhu * @vmhub: vmhub type 68990f6452cSchangzhu * 69090f6452cSchangzhu */ 69190f6452cSchangzhu static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, 69290f6452cSchangzhu uint32_t vmhub) 69390f6452cSchangzhu { 694b7daed1bSAmber Lin if (adev->asic_type == CHIP_ALDEBARAN) 695d477c5aaSHawking Zhang return false; 696d477c5aaSHawking Zhang 69790f6452cSchangzhu return ((vmhub == AMDGPU_MMHUB_0 || 69890f6452cSchangzhu vmhub == AMDGPU_MMHUB_1) && 69990f6452cSchangzhu (!amdgpu_sriov_vf(adev)) && 70054f78a76SAlex Deucher (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && 70154f78a76SAlex Deucher (adev->apu_flags & AMD_APU_IS_PICASSO)))); 70290f6452cSchangzhu } 70390f6452cSchangzhu 704ea930000SAlex Sierra static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, 705ea930000SAlex Sierra uint8_t vmid, uint16_t *p_pasid) 706ea930000SAlex Sierra { 707ea930000SAlex Sierra uint32_t value; 708ea930000SAlex Sierra 709ea930000SAlex Sierra value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 710ea930000SAlex Sierra + vmid); 711ea930000SAlex Sierra *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 712ea930000SAlex Sierra 713ea930000SAlex Sierra return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 714ea930000SAlex Sierra } 715ea930000SAlex Sierra 716e60f8db5SAlex Xie /* 717e60f8db5SAlex Xie * GART 718e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 719e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 720e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 721e60f8db5SAlex Xie */ 722e60f8db5SAlex Xie 723e60f8db5SAlex Xie /** 7242a79d868SYong Zhao * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 725e60f8db5SAlex Xie * 726e60f8db5SAlex Xie * @adev: amdgpu_device pointer 727e60f8db5SAlex Xie * @vmid: vm instance to flush 728bf0df09cSLee Jones * @vmhub: which hub to flush 7292a79d868SYong Zhao * @flush_type: the flush type 730e60f8db5SAlex Xie * 7312a79d868SYong Zhao * Flush the TLB for the requested page table using certain type. 732e60f8db5SAlex Xie */ 7333ff98548SOak Zeng static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 7343ff98548SOak Zeng uint32_t vmhub, uint32_t flush_type) 735e60f8db5SAlex Xie { 73690f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); 737e60f8db5SAlex Xie const unsigned eng = 17; 738b80cd524SFelix Kuehling u32 j, inv_req, inv_req2, tmp; 7393ff98548SOak Zeng struct amdgpu_vmhub *hub; 740e60f8db5SAlex Xie 7413ff98548SOak Zeng BUG_ON(vmhub >= adev->num_vmhubs); 7423ff98548SOak Zeng 7433ff98548SOak Zeng hub = &adev->vmhub[vmhub]; 744b80cd524SFelix Kuehling if (adev->gmc.xgmi.num_physical_nodes && 745b80cd524SFelix Kuehling adev->asic_type == CHIP_VEGA20) { 746b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 747b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 748b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 749b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 750b80cd524SFelix Kuehling * still need a second TLB flush after this. 751b80cd524SFelix Kuehling */ 752b80cd524SFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); 753b80cd524SFelix Kuehling inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 754b80cd524SFelix Kuehling } else { 75537c58ddfSFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); 756b80cd524SFelix Kuehling inv_req2 = 0; 757b80cd524SFelix Kuehling } 758e60f8db5SAlex Xie 75982d1a1b1SChengming Gui /* This is necessary for a HW workaround under SRIOV as well 76082d1a1b1SChengming Gui * as GFXOFF under bare metal 76182d1a1b1SChengming Gui */ 76282d1a1b1SChengming Gui if (adev->gfx.kiq.ring.sched.ready && 76382d1a1b1SChengming Gui (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 76481202807SDennis Li down_read_trylock(&adev->reset_sem)) { 765148f597dSHuang Rui uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 766148f597dSHuang Rui uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 767af5fe1e9SChristian König 76837c58ddfSFelix Kuehling amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 769af5fe1e9SChristian König 1 << vmid); 77081202807SDennis Li up_read(&adev->reset_sem); 7713ff98548SOak Zeng return; 772fc0faf04SEmily Deng } 7733890d111SEmily Deng 7743890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 775f920d1bbSchangzhu 776f920d1bbSchangzhu /* 777f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 778f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 779f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 780f920d1bbSchangzhu * to WA the Issue 781f920d1bbSchangzhu */ 782f920d1bbSchangzhu 783f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 78490f6452cSchangzhu if (use_semaphore) { 785f920d1bbSchangzhu for (j = 0; j < adev->usec_timeout; j++) { 786f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 787148f597dSHuang Rui tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + 788148f597dSHuang Rui hub->eng_distance * eng); 789f920d1bbSchangzhu if (tmp & 0x1) 790f920d1bbSchangzhu break; 791f920d1bbSchangzhu udelay(1); 792f920d1bbSchangzhu } 793f920d1bbSchangzhu 794f920d1bbSchangzhu if (j >= adev->usec_timeout) 795f920d1bbSchangzhu DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 796f920d1bbSchangzhu } 797f920d1bbSchangzhu 798b80cd524SFelix Kuehling do { 799148f597dSHuang Rui WREG32_NO_KIQ(hub->vm_inv_eng0_req + 800148f597dSHuang Rui hub->eng_distance * eng, inv_req); 80153499173SXiaojie Yuan 80253499173SXiaojie Yuan /* 803b80cd524SFelix Kuehling * Issue a dummy read to wait for the ACK register to 804b80cd524SFelix Kuehling * be cleared to avoid a false ACK due to the new fast 805b80cd524SFelix Kuehling * GRBM interface. 80653499173SXiaojie Yuan */ 8077845d80dSAlex Deucher if ((vmhub == AMDGPU_GFXHUB_0) && 8087845d80dSAlex Deucher (adev->asic_type < CHIP_ALDEBARAN)) 809148f597dSHuang Rui RREG32_NO_KIQ(hub->vm_inv_eng0_req + 810148f597dSHuang Rui hub->eng_distance * eng); 81153499173SXiaojie Yuan 812e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 813148f597dSHuang Rui tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + 814148f597dSHuang Rui hub->eng_distance * eng); 815396557b0SChristian König if (tmp & (1 << vmid)) 816e60f8db5SAlex Xie break; 817e60f8db5SAlex Xie udelay(1); 818e60f8db5SAlex Xie } 819f920d1bbSchangzhu 820b80cd524SFelix Kuehling inv_req = inv_req2; 821b80cd524SFelix Kuehling inv_req2 = 0; 822b80cd524SFelix Kuehling } while (inv_req); 823b80cd524SFelix Kuehling 824f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 82590f6452cSchangzhu if (use_semaphore) 826f920d1bbSchangzhu /* 827f920d1bbSchangzhu * add semaphore release after invalidation, 828f920d1bbSchangzhu * write with 0 means semaphore release 829f920d1bbSchangzhu */ 830148f597dSHuang Rui WREG32_NO_KIQ(hub->vm_inv_eng0_sem + 831148f597dSHuang Rui hub->eng_distance * eng, 0); 832f920d1bbSchangzhu 8333890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 834f920d1bbSchangzhu 835396557b0SChristian König if (j < adev->usec_timeout) 8363ff98548SOak Zeng return; 837396557b0SChristian König 838e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 839e60f8db5SAlex Xie } 840e60f8db5SAlex Xie 841ea930000SAlex Sierra /** 842ea930000SAlex Sierra * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid 843ea930000SAlex Sierra * 844ea930000SAlex Sierra * @adev: amdgpu_device pointer 845ea930000SAlex Sierra * @pasid: pasid to be flush 846bf0df09cSLee Jones * @flush_type: the flush type 847bf0df09cSLee Jones * @all_hub: flush all hubs 848ea930000SAlex Sierra * 849ea930000SAlex Sierra * Flush the TLB for the requested pasid. 850ea930000SAlex Sierra */ 851ea930000SAlex Sierra static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 852ea930000SAlex Sierra uint16_t pasid, uint32_t flush_type, 853ea930000SAlex Sierra bool all_hub) 854ea930000SAlex Sierra { 855ea930000SAlex Sierra int vmid, i; 856ea930000SAlex Sierra signed long r; 857ea930000SAlex Sierra uint32_t seq; 858ea930000SAlex Sierra uint16_t queried_pasid; 859ea930000SAlex Sierra bool ret; 860ea930000SAlex Sierra struct amdgpu_ring *ring = &adev->gfx.kiq.ring; 861ea930000SAlex Sierra struct amdgpu_kiq *kiq = &adev->gfx.kiq; 862ea930000SAlex Sierra 86353b3f8f4SDennis Li if (amdgpu_in_reset(adev)) 864ea930000SAlex Sierra return -EIO; 865ea930000SAlex Sierra 86681202807SDennis Li if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) { 867b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 868b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 869b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 870b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 871b80cd524SFelix Kuehling * still need a second TLB flush after this. 872b80cd524SFelix Kuehling */ 873b80cd524SFelix Kuehling bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && 874b80cd524SFelix Kuehling adev->asic_type == CHIP_VEGA20); 875b80cd524SFelix Kuehling /* 2 dwords flush + 8 dwords fence */ 876b80cd524SFelix Kuehling unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; 877b80cd524SFelix Kuehling 878b80cd524SFelix Kuehling if (vega20_xgmi_wa) 879b80cd524SFelix Kuehling ndw += kiq->pmf->invalidate_tlbs_size; 880b80cd524SFelix Kuehling 881ea930000SAlex Sierra spin_lock(&adev->gfx.kiq.ring_lock); 88236a1707aSAlex Sierra /* 2 dwords flush + 8 dwords fence */ 883b80cd524SFelix Kuehling amdgpu_ring_alloc(ring, ndw); 884b80cd524SFelix Kuehling if (vega20_xgmi_wa) 885b80cd524SFelix Kuehling kiq->pmf->kiq_invalidate_tlbs(ring, 886b80cd524SFelix Kuehling pasid, 2, all_hub); 887ea930000SAlex Sierra kiq->pmf->kiq_invalidate_tlbs(ring, 888ea930000SAlex Sierra pasid, flush_type, all_hub); 88904e4e2e9SYintian Tao r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 89004e4e2e9SYintian Tao if (r) { 89104e4e2e9SYintian Tao amdgpu_ring_undo(ring); 892abb17b1eSColin Ian King spin_unlock(&adev->gfx.kiq.ring_lock); 89381202807SDennis Li up_read(&adev->reset_sem); 89404e4e2e9SYintian Tao return -ETIME; 89504e4e2e9SYintian Tao } 89604e4e2e9SYintian Tao 897ea930000SAlex Sierra amdgpu_ring_commit(ring); 898ea930000SAlex Sierra spin_unlock(&adev->gfx.kiq.ring_lock); 899ea930000SAlex Sierra r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); 900ea930000SAlex Sierra if (r < 1) { 901aac89168SDennis Li dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); 90281202807SDennis Li up_read(&adev->reset_sem); 903ea930000SAlex Sierra return -ETIME; 904ea930000SAlex Sierra } 90581202807SDennis Li up_read(&adev->reset_sem); 906ea930000SAlex Sierra return 0; 907ea930000SAlex Sierra } 908ea930000SAlex Sierra 909ea930000SAlex Sierra for (vmid = 1; vmid < 16; vmid++) { 910ea930000SAlex Sierra 911ea930000SAlex Sierra ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 912ea930000SAlex Sierra &queried_pasid); 913ea930000SAlex Sierra if (ret && queried_pasid == pasid) { 914ea930000SAlex Sierra if (all_hub) { 915ea930000SAlex Sierra for (i = 0; i < adev->num_vmhubs; i++) 916ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 917fa34edbeSFelix Kuehling i, flush_type); 918ea930000SAlex Sierra } else { 919ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 920fa34edbeSFelix Kuehling AMDGPU_GFXHUB_0, flush_type); 921ea930000SAlex Sierra } 922ea930000SAlex Sierra break; 923ea930000SAlex Sierra } 924ea930000SAlex Sierra } 925ea930000SAlex Sierra 926ea930000SAlex Sierra return 0; 927ea930000SAlex Sierra 928ea930000SAlex Sierra } 929ea930000SAlex Sierra 9309096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 931c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 9329096d6e5SChristian König { 93390f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); 934250b4228SChristian König struct amdgpu_device *adev = ring->adev; 935250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 9362a79d868SYong Zhao uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 9379096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 9389096d6e5SChristian König 939f920d1bbSchangzhu /* 940f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 941f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 942f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 943f920d1bbSchangzhu * to WA the Issue 944f920d1bbSchangzhu */ 945f920d1bbSchangzhu 946f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 94790f6452cSchangzhu if (use_semaphore) 948f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 949f920d1bbSchangzhu amdgpu_ring_emit_reg_wait(ring, 950148f597dSHuang Rui hub->vm_inv_eng0_sem + 951148f597dSHuang Rui hub->eng_distance * eng, 0x1, 0x1); 952f920d1bbSchangzhu 953148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 954148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 9559096d6e5SChristian König lower_32_bits(pd_addr)); 9569096d6e5SChristian König 957148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 958148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 9599096d6e5SChristian König upper_32_bits(pd_addr)); 9609096d6e5SChristian König 961148f597dSHuang Rui amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 962148f597dSHuang Rui hub->eng_distance * eng, 963148f597dSHuang Rui hub->vm_inv_eng0_ack + 964148f597dSHuang Rui hub->eng_distance * eng, 965f8bc9037SAlex Deucher req, 1 << vmid); 966f732b6b3SChristian König 967f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 96890f6452cSchangzhu if (use_semaphore) 969f920d1bbSchangzhu /* 970f920d1bbSchangzhu * add semaphore release after invalidation, 971f920d1bbSchangzhu * write with 0 means semaphore release 972f920d1bbSchangzhu */ 973148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 974148f597dSHuang Rui hub->eng_distance * eng, 0); 975f920d1bbSchangzhu 9769096d6e5SChristian König return pd_addr; 9779096d6e5SChristian König } 9789096d6e5SChristian König 979c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 980c633c00bSChristian König unsigned pasid) 981c633c00bSChristian König { 982c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 983c633c00bSChristian König uint32_t reg; 984c633c00bSChristian König 985f2d66571SLe Ma /* Do nothing because there's no lut register for mmhub1. */ 986f2d66571SLe Ma if (ring->funcs->vmhub == AMDGPU_MMHUB_1) 987f2d66571SLe Ma return; 988f2d66571SLe Ma 989a2d15ed7SLe Ma if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) 990c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 991c633c00bSChristian König else 992c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 993c633c00bSChristian König 994c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 995c633c00bSChristian König } 996c633c00bSChristian König 997e60f8db5SAlex Xie /* 998e60f8db5SAlex Xie * PTE format on VEGA 10: 999e60f8db5SAlex Xie * 63:59 reserved 1000e60f8db5SAlex Xie * 58:57 mtype 1001e60f8db5SAlex Xie * 56 F 1002e60f8db5SAlex Xie * 55 L 1003e60f8db5SAlex Xie * 54 P 1004e60f8db5SAlex Xie * 53 SW 1005e60f8db5SAlex Xie * 52 T 1006e60f8db5SAlex Xie * 50:48 reserved 1007e60f8db5SAlex Xie * 47:12 4k physical page base address 1008e60f8db5SAlex Xie * 11:7 fragment 1009e60f8db5SAlex Xie * 6 write 1010e60f8db5SAlex Xie * 5 read 1011e60f8db5SAlex Xie * 4 exe 1012e60f8db5SAlex Xie * 3 Z 1013e60f8db5SAlex Xie * 2 snooped 1014e60f8db5SAlex Xie * 1 system 1015e60f8db5SAlex Xie * 0 valid 1016e60f8db5SAlex Xie * 1017e60f8db5SAlex Xie * PDE format on VEGA 10: 1018e60f8db5SAlex Xie * 63:59 block fragment size 1019e60f8db5SAlex Xie * 58:55 reserved 1020e60f8db5SAlex Xie * 54 P 1021e60f8db5SAlex Xie * 53:48 reserved 1022e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 1023e60f8db5SAlex Xie * 5:3 reserved 1024e60f8db5SAlex Xie * 2 C 1025e60f8db5SAlex Xie * 1 system 1026e60f8db5SAlex Xie * 0 valid 1027e60f8db5SAlex Xie */ 1028e60f8db5SAlex Xie 102971776b6dSChristian König static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) 1030e60f8db5SAlex Xie 1031e60f8db5SAlex Xie { 103271776b6dSChristian König switch (flags) { 1033e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 103471776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1035e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 103671776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1037e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 103871776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); 1039093e48c0SOak Zeng case AMDGPU_VM_MTYPE_RW: 104071776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); 1041e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 104271776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); 1043e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 104471776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); 1045e60f8db5SAlex Xie default: 104671776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1047e60f8db5SAlex Xie } 1048e60f8db5SAlex Xie } 1049e60f8db5SAlex Xie 10503de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 10513de676d8SChristian König uint64_t *addr, uint64_t *flags) 1052f75e237cSChristian König { 1053bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 10540ca565abSOak Zeng *addr = amdgpu_gmc_vram_mc2pa(adev, *addr); 10553de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 10566a42fd6fSChristian König 1057770d13b1SChristian König if (!adev->gmc.translate_further) 10586a42fd6fSChristian König return; 10596a42fd6fSChristian König 10606a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 10616a42fd6fSChristian König /* Set the block fragment size */ 10626a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 10636a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 10646a42fd6fSChristian König 10656a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 10666a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 10676a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 10686a42fd6fSChristian König else 10696a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 10706a42fd6fSChristian König } 1071f75e237cSChristian König } 1072f75e237cSChristian König 1073cbfae36cSChristian König static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, 1074cbfae36cSChristian König struct amdgpu_bo_va_mapping *mapping, 1075cbfae36cSChristian König uint64_t *flags) 1076cbfae36cSChristian König { 1077cbfae36cSChristian König *flags &= ~AMDGPU_PTE_EXECUTABLE; 1078cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1079cbfae36cSChristian König 1080cbfae36cSChristian König *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1081cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; 1082cbfae36cSChristian König 1083cbfae36cSChristian König if (mapping->flags & AMDGPU_PTE_PRT) { 1084cbfae36cSChristian König *flags |= AMDGPU_PTE_PRT; 1085cbfae36cSChristian König *flags &= ~AMDGPU_PTE_VALID; 1086cbfae36cSChristian König } 1087cbfae36cSChristian König 10887ffe7238SYong Zhao if ((adev->asic_type == CHIP_ARCTURUS || 10897ffe7238SYong Zhao adev->asic_type == CHIP_ALDEBARAN) && 1090cbfae36cSChristian König !(*flags & AMDGPU_PTE_SYSTEM) && 1091cbfae36cSChristian König mapping->bo_va->is_xgmi) 1092cbfae36cSChristian König *flags |= AMDGPU_PTE_SNOOPED; 109372b4db0fSEric Huang 109472b4db0fSEric Huang if (adev->asic_type == CHIP_ALDEBARAN) 109572b4db0fSEric Huang *flags |= mapping->flags & AMDGPU_PTE_SNOOPED; 1096cbfae36cSChristian König } 1097cbfae36cSChristian König 10987b885f0eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 10997b885f0eSAlex Deucher { 11007b885f0eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 11017b885f0eSAlex Deucher unsigned size; 11027b885f0eSAlex Deucher 11037b885f0eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 11047b885f0eSAlex Deucher size = AMDGPU_VBIOS_VGA_ALLOCATION; 11057b885f0eSAlex Deucher } else { 11067b885f0eSAlex Deucher u32 viewport; 11077b885f0eSAlex Deucher 11087b885f0eSAlex Deucher switch (adev->asic_type) { 11097b885f0eSAlex Deucher case CHIP_RAVEN: 11107b885f0eSAlex Deucher case CHIP_RENOIR: 11117b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 11127b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, 11137b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 11147b885f0eSAlex Deucher REG_GET_FIELD(viewport, 11157b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 11167b885f0eSAlex Deucher 4); 11177b885f0eSAlex Deucher break; 11187b885f0eSAlex Deucher case CHIP_VEGA10: 11197b885f0eSAlex Deucher case CHIP_VEGA12: 11207b885f0eSAlex Deucher case CHIP_VEGA20: 11217b885f0eSAlex Deucher default: 11227b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 11237b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 11247b885f0eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 11257b885f0eSAlex Deucher 4); 11267b885f0eSAlex Deucher break; 11277b885f0eSAlex Deucher } 11287b885f0eSAlex Deucher } 11297b885f0eSAlex Deucher 11307b885f0eSAlex Deucher return size; 11317b885f0eSAlex Deucher } 11327b885f0eSAlex Deucher 1133132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 1134132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 1135ea930000SAlex Sierra .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, 11369096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 1137c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 113871776b6dSChristian König .map_mtype = gmc_v9_0_map_mtype, 1139cbfae36cSChristian König .get_vm_pde = gmc_v9_0_get_vm_pde, 11407b885f0eSAlex Deucher .get_vm_pte = gmc_v9_0_get_vm_pte, 11417b885f0eSAlex Deucher .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, 1142e60f8db5SAlex Xie }; 1143e60f8db5SAlex Xie 1144132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 1145e60f8db5SAlex Xie { 1146132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 1147e60f8db5SAlex Xie } 1148e60f8db5SAlex Xie 11495b6b35aaSHawking Zhang static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) 11505b6b35aaSHawking Zhang { 11515b6b35aaSHawking Zhang switch (adev->asic_type) { 1152e7da754bSMonk Liu case CHIP_VEGA10: 1153e7da754bSMonk Liu adev->umc.funcs = &umc_v6_0_funcs; 1154e7da754bSMonk Liu break; 11555b6b35aaSHawking Zhang case CHIP_VEGA20: 11563aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 11573aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 11583aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 11594cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; 11604cf781c2SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 116149070c4eSHawking Zhang adev->umc.ras_funcs = &umc_v6_1_ras_funcs; 11624cf781c2SJohn Clements break; 11639e612c11SHawking Zhang case CHIP_ARCTURUS: 11643aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 11653aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 11663aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 11674cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; 11683aacf4eaSTao Zhou adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 116949070c4eSHawking Zhang adev->umc.ras_funcs = &umc_v6_1_ras_funcs; 11705b6b35aaSHawking Zhang break; 11715b6b35aaSHawking Zhang default: 11725b6b35aaSHawking Zhang break; 11735b6b35aaSHawking Zhang } 11745b6b35aaSHawking Zhang } 11755b6b35aaSHawking Zhang 11763d093da0STao Zhou static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) 11773d093da0STao Zhou { 11783d093da0STao Zhou switch (adev->asic_type) { 1179f6c3623bSDennis Li case CHIP_ARCTURUS: 1180f6c3623bSDennis Li adev->mmhub.funcs = &mmhub_v9_4_funcs; 1181f6c3623bSDennis Li break; 11824da999cdSOak Zeng case CHIP_ALDEBARAN: 11834da999cdSOak Zeng adev->mmhub.funcs = &mmhub_v1_7_funcs; 11844da999cdSOak Zeng break; 11853d093da0STao Zhou default: 11869fb1506eSOak Zeng adev->mmhub.funcs = &mmhub_v1_0_funcs; 11873d093da0STao Zhou break; 11883d093da0STao Zhou } 11893d093da0STao Zhou } 11903d093da0STao Zhou 1191d844c6d7SHawking Zhang static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev) 1192d844c6d7SHawking Zhang { 1193d844c6d7SHawking Zhang switch (adev->asic_type) { 1194d844c6d7SHawking Zhang case CHIP_VEGA20: 1195d844c6d7SHawking Zhang adev->mmhub.ras_funcs = &mmhub_v1_0_ras_funcs; 1196d844c6d7SHawking Zhang break; 1197d844c6d7SHawking Zhang case CHIP_ARCTURUS: 1198d844c6d7SHawking Zhang adev->mmhub.ras_funcs = &mmhub_v9_4_ras_funcs; 1199d844c6d7SHawking Zhang break; 1200d844c6d7SHawking Zhang case CHIP_ALDEBARAN: 1201d844c6d7SHawking Zhang adev->mmhub.ras_funcs = &mmhub_v1_7_ras_funcs; 1202d844c6d7SHawking Zhang break; 1203d844c6d7SHawking Zhang default: 1204d844c6d7SHawking Zhang /* mmhub ras is not available */ 1205d844c6d7SHawking Zhang break; 1206d844c6d7SHawking Zhang } 1207d844c6d7SHawking Zhang } 1208d844c6d7SHawking Zhang 12098ffff9b4SOak Zeng static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev) 12108ffff9b4SOak Zeng { 12118ffff9b4SOak Zeng adev->gfxhub.funcs = &gfxhub_v1_0_funcs; 12128ffff9b4SOak Zeng } 12138ffff9b4SOak Zeng 1214*6f12507fSHawking Zhang static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev) 1215*6f12507fSHawking Zhang { 1216*6f12507fSHawking Zhang adev->hdp.ras_funcs = &hdp_v4_0_ras_funcs; 1217*6f12507fSHawking Zhang } 1218*6f12507fSHawking Zhang 1219e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 1220e60f8db5SAlex Xie { 1221e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1222e60f8db5SAlex Xie 122331691b8dSRajneesh Bhardwaj if (adev->asic_type == CHIP_VEGA20 || 122431691b8dSRajneesh Bhardwaj adev->asic_type == CHIP_ARCTURUS) 122531691b8dSRajneesh Bhardwaj adev->gmc.xgmi.supported = true; 122631691b8dSRajneesh Bhardwaj 122731691b8dSRajneesh Bhardwaj if (adev->asic_type == CHIP_ALDEBARAN) { 122831691b8dSRajneesh Bhardwaj adev->gmc.xgmi.supported = true; 122931691b8dSRajneesh Bhardwaj adev->gmc.xgmi.connected_to_cpu = 123031691b8dSRajneesh Bhardwaj adev->smuio.funcs->is_host_gpu_xgmi_supported(adev); 123131691b8dSRajneesh Bhardwaj } 123231691b8dSRajneesh Bhardwaj 123349070c4eSHawking Zhang gmc_v9_0_set_gmc_funcs(adev); 123449070c4eSHawking Zhang gmc_v9_0_set_irq_funcs(adev); 123549070c4eSHawking Zhang gmc_v9_0_set_umc_funcs(adev); 123649070c4eSHawking Zhang gmc_v9_0_set_mmhub_funcs(adev); 1237d844c6d7SHawking Zhang gmc_v9_0_set_mmhub_ras_funcs(adev); 123849070c4eSHawking Zhang gmc_v9_0_set_gfxhub_funcs(adev); 1239*6f12507fSHawking Zhang gmc_v9_0_set_hdp_ras_funcs(adev); 124049070c4eSHawking Zhang 1241770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 1242770d13b1SChristian König adev->gmc.shared_aperture_end = 1243770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 1244bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 1245770d13b1SChristian König adev->gmc.private_aperture_end = 1246770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 1247a7ea6548SAlex Deucher 1248e60f8db5SAlex Xie return 0; 1249e60f8db5SAlex Xie } 1250e60f8db5SAlex Xie 1251e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 1252e60f8db5SAlex Xie { 1253e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1254c5b6e585STao Zhou int r; 12554789c463SChristian König 1256bdbe90f0SAlex Deucher r = amdgpu_gmc_allocate_vm_inv_eng(adev); 1257c713a461SEvan Quan if (r) 1258c713a461SEvan Quan return r; 12594a20300bSGuchun Chen 12604a20300bSGuchun Chen /* 12614a20300bSGuchun Chen * Workaround performance drop issue with VBIOS enables partial 12624a20300bSGuchun Chen * writes, while disables HBM ECC for vega10. 12634a20300bSGuchun Chen */ 126488474ccaSGuchun Chen if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) { 12654a20300bSGuchun Chen if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) { 1266bdf84a80SJoseph Greathouse if (adev->df.funcs->enable_ecc_force_par_wr_rmw) 1267bdf84a80SJoseph Greathouse adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); 12684a20300bSGuchun Chen } 1269f49ea9f8SHawking Zhang } 127002bab923SDavid Panariti 12718bc7b360SHawking Zhang if (adev->mmhub.ras_funcs && 12728bc7b360SHawking Zhang adev->mmhub.ras_funcs->reset_ras_error_count) 12738bc7b360SHawking Zhang adev->mmhub.ras_funcs->reset_ras_error_count(adev); 1274fe5211f1SHawking Zhang 1275ba083492STao Zhou r = amdgpu_gmc_ras_late_init(adev); 1276791c4769Sxinhui pan if (r) 1277e60f8db5SAlex Xie return r; 1278e60f8db5SAlex Xie 1279770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 1280e60f8db5SAlex Xie } 1281e60f8db5SAlex Xie 1282e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 1283770d13b1SChristian König struct amdgpu_gmc *mc) 1284e60f8db5SAlex Xie { 1285e60f8db5SAlex Xie u64 base = 0; 12869d4f837aSFrank.Min 12879fb1506eSOak Zeng if (!amdgpu_sriov_vf(adev)) 12889fb1506eSOak Zeng base = adev->mmhub.funcs->get_fb_location(adev); 12899d4f837aSFrank.Min 12906fdd68b1SAlex Deucher /* add the xgmi offset of the physical node */ 12916fdd68b1SAlex Deucher base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1292f527f310SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 1293f527f310SOak Zeng amdgpu_gmc_sysvm_location(adev, mc); 1294f527f310SOak Zeng } else { 129583afe835SOak Zeng amdgpu_gmc_vram_location(adev, mc, base); 1296961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 1297c3e1b43cSChristian König amdgpu_gmc_agp_location(adev, mc); 1298f527f310SOak Zeng } 1299e60f8db5SAlex Xie /* base offset of vram pages */ 13008ffff9b4SOak Zeng adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); 13016fdd68b1SAlex Deucher 13026fdd68b1SAlex Deucher /* XXX: add the xgmi offset of the physical node? */ 13036fdd68b1SAlex Deucher adev->vm_manager.vram_base_offset += 13046fdd68b1SAlex Deucher adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1305e60f8db5SAlex Xie } 1306e60f8db5SAlex Xie 1307e60f8db5SAlex Xie /** 1308e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 1309e60f8db5SAlex Xie * 1310e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1311e60f8db5SAlex Xie * 1312e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 1313e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 1314e60f8db5SAlex Xie * Returns 0 for success. 1315e60f8db5SAlex Xie */ 1316e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 1317e60f8db5SAlex Xie { 1318e60f8db5SAlex Xie int r; 1319e60f8db5SAlex Xie 1320e60f8db5SAlex Xie /* size in MB on si */ 1321770d13b1SChristian König adev->gmc.mc_vram_size = 1322bebc0762SHawking Zhang adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 1323770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 1324e60f8db5SAlex Xie 1325be566196SOak Zeng if (!(adev->flags & AMD_IS_APU) && 1326be566196SOak Zeng !adev->gmc.xgmi.connected_to_cpu) { 1327e60f8db5SAlex Xie r = amdgpu_device_resize_fb_bar(adev); 1328e60f8db5SAlex Xie if (r) 1329e60f8db5SAlex Xie return r; 1330e60f8db5SAlex Xie } 1331770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 1332770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 1333e60f8db5SAlex Xie 1334156a81beSChunming Zhou #ifdef CONFIG_X86_64 133531691b8dSRajneesh Bhardwaj /* 133631691b8dSRajneesh Bhardwaj * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi 133731691b8dSRajneesh Bhardwaj * interface can use VRAM through here as it appears system reserved 133831691b8dSRajneesh Bhardwaj * memory in host address space. 133931691b8dSRajneesh Bhardwaj * 134031691b8dSRajneesh Bhardwaj * For APUs, VRAM is just the stolen system memory and can be accessed 134131691b8dSRajneesh Bhardwaj * directly. 134231691b8dSRajneesh Bhardwaj * 134331691b8dSRajneesh Bhardwaj * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR. 134431691b8dSRajneesh Bhardwaj */ 134531691b8dSRajneesh Bhardwaj 134631691b8dSRajneesh Bhardwaj /* check whether both host-gpu and gpu-gpu xgmi links exist */ 13473de60d96SHawking Zhang if ((adev->flags & AMD_IS_APU) || 13483de60d96SHawking Zhang (adev->gmc.xgmi.supported && 13493de60d96SHawking Zhang adev->gmc.xgmi.connected_to_cpu)) { 13503de60d96SHawking Zhang adev->gmc.aper_base = 13513de60d96SHawking Zhang adev->gfxhub.funcs->get_mc_fb_offset(adev) + 13523de60d96SHawking Zhang adev->gmc.xgmi.physical_node_id * 135331691b8dSRajneesh Bhardwaj adev->gmc.xgmi.node_segment_size; 1354156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 1355156a81beSChunming Zhou } 135631691b8dSRajneesh Bhardwaj 1357156a81beSChunming Zhou #endif 1358e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 1359770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 1360770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 1361770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 1362e60f8db5SAlex Xie 1363e60f8db5SAlex Xie /* set the gart size */ 1364e60f8db5SAlex Xie if (amdgpu_gart_size == -1) { 1365e60f8db5SAlex Xie switch (adev->asic_type) { 1366e60f8db5SAlex Xie case CHIP_VEGA10: /* all engines support GPUVM */ 1367273a14cdSAlex Deucher case CHIP_VEGA12: /* all engines support GPUVM */ 1368d96b428cSFeifei Xu case CHIP_VEGA20: 13693de2ff5dSLe Ma case CHIP_ARCTURUS: 137085e39550SLe Ma case CHIP_ALDEBARAN: 1371e60f8db5SAlex Xie default: 1372fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 1373e60f8db5SAlex Xie break; 1374e60f8db5SAlex Xie case CHIP_RAVEN: /* DCE SG support */ 13758787ee01SHuang Rui case CHIP_RENOIR: 1376770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 1377e60f8db5SAlex Xie break; 1378e60f8db5SAlex Xie } 1379e60f8db5SAlex Xie } else { 1380770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 1381e60f8db5SAlex Xie } 1382e60f8db5SAlex Xie 1383f1dc12caSOak Zeng adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; 1384f1dc12caSOak Zeng 1385770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 1386e60f8db5SAlex Xie 1387e60f8db5SAlex Xie return 0; 1388e60f8db5SAlex Xie } 1389e60f8db5SAlex Xie 1390e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 1391e60f8db5SAlex Xie { 1392e60f8db5SAlex Xie int r; 1393e60f8db5SAlex Xie 13941123b989SChristian König if (adev->gart.bo) { 1395e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 1396e60f8db5SAlex Xie return 0; 1397e60f8db5SAlex Xie } 13987b454b3aSOak Zeng 13997b454b3aSOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 14007b454b3aSOak Zeng adev->gmc.vmid0_page_table_depth = 1; 14017b454b3aSOak Zeng adev->gmc.vmid0_page_table_block_size = 12; 14027b454b3aSOak Zeng } else { 14037b454b3aSOak Zeng adev->gmc.vmid0_page_table_depth = 0; 14047b454b3aSOak Zeng adev->gmc.vmid0_page_table_block_size = 0; 14057b454b3aSOak Zeng } 14067b454b3aSOak Zeng 1407e60f8db5SAlex Xie /* Initialize common gart structure */ 1408e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 1409e60f8db5SAlex Xie if (r) 1410e60f8db5SAlex Xie return r; 1411e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 14127596ab68SHawking Zhang adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | 1413e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 1414522510a6SOak Zeng 1415522510a6SOak Zeng r = amdgpu_gart_table_vram_alloc(adev); 1416522510a6SOak Zeng if (r) 1417522510a6SOak Zeng return r; 1418522510a6SOak Zeng 1419522510a6SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 1420522510a6SOak Zeng r = amdgpu_gmc_pdb0_alloc(adev); 1421522510a6SOak Zeng } 1422522510a6SOak Zeng 1423522510a6SOak Zeng return r; 1424e60f8db5SAlex Xie } 1425e60f8db5SAlex Xie 1426b0a2db9bSAlex Deucher /** 1427b0a2db9bSAlex Deucher * gmc_v9_0_save_registers - saves regs 1428b0a2db9bSAlex Deucher * 1429b0a2db9bSAlex Deucher * @adev: amdgpu_device pointer 1430b0a2db9bSAlex Deucher * 1431b0a2db9bSAlex Deucher * This saves potential register values that should be 1432b0a2db9bSAlex Deucher * restored upon resume 1433b0a2db9bSAlex Deucher */ 1434b0a2db9bSAlex Deucher static void gmc_v9_0_save_registers(struct amdgpu_device *adev) 1435ebdef28eSAlex Deucher { 1436b0a2db9bSAlex Deucher if (adev->asic_type == CHIP_RAVEN) 1437b0a2db9bSAlex Deucher adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); 1438ebdef28eSAlex Deucher } 1439ebdef28eSAlex Deucher 1440e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 1441e60f8db5SAlex Xie { 1442ad02e08eSOri Messinger int r, vram_width = 0, vram_type = 0, vram_vendor = 0; 1443e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1444e60f8db5SAlex Xie 14458ffff9b4SOak Zeng adev->gfxhub.funcs->init(adev); 14469fb1506eSOak Zeng 14479fb1506eSOak Zeng adev->mmhub.funcs->init(adev); 1448e60f8db5SAlex Xie 1449770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 1450e60f8db5SAlex Xie 1451ad02e08eSOri Messinger r = amdgpu_atomfirmware_get_vram_info(adev, 1452ad02e08eSOri Messinger &vram_width, &vram_type, &vram_vendor); 1453631cdbd2SAlex Deucher if (amdgpu_sriov_vf(adev)) 1454631cdbd2SAlex Deucher /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, 1455631cdbd2SAlex Deucher * and DF related registers is not readable, seems hardcord is the 1456631cdbd2SAlex Deucher * only way to set the correct vram_width 1457631cdbd2SAlex Deucher */ 1458631cdbd2SAlex Deucher adev->gmc.vram_width = 2048; 1459631cdbd2SAlex Deucher else if (amdgpu_emu_mode != 1) 1460631cdbd2SAlex Deucher adev->gmc.vram_width = vram_width; 1461631cdbd2SAlex Deucher 1462631cdbd2SAlex Deucher if (!adev->gmc.vram_width) { 1463631cdbd2SAlex Deucher int chansize, numchan; 1464631cdbd2SAlex Deucher 1465631cdbd2SAlex Deucher /* hbm memory channel size */ 1466631cdbd2SAlex Deucher if (adev->flags & AMD_IS_APU) 1467631cdbd2SAlex Deucher chansize = 64; 1468631cdbd2SAlex Deucher else 1469631cdbd2SAlex Deucher chansize = 128; 1470631cdbd2SAlex Deucher 1471bdf84a80SJoseph Greathouse numchan = adev->df.funcs->get_hbm_channel_number(adev); 1472631cdbd2SAlex Deucher adev->gmc.vram_width = numchan * chansize; 1473631cdbd2SAlex Deucher } 1474631cdbd2SAlex Deucher 1475631cdbd2SAlex Deucher adev->gmc.vram_type = vram_type; 1476ad02e08eSOri Messinger adev->gmc.vram_vendor = vram_vendor; 1477e60f8db5SAlex Xie switch (adev->asic_type) { 1478e60f8db5SAlex Xie case CHIP_RAVEN: 14791daa2bfaSLe Ma adev->num_vmhubs = 2; 14801daa2bfaSLe Ma 14816a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 1482f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 14836a42fd6fSChristian König } else { 14846a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 14856a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 1486770d13b1SChristian König adev->gmc.translate_further = 14876a42fd6fSChristian König adev->vm_manager.num_level > 1; 14886a42fd6fSChristian König } 1489e60f8db5SAlex Xie break; 1490e60f8db5SAlex Xie case CHIP_VEGA10: 1491273a14cdSAlex Deucher case CHIP_VEGA12: 1492d96b428cSFeifei Xu case CHIP_VEGA20: 14938787ee01SHuang Rui case CHIP_RENOIR: 149485e39550SLe Ma case CHIP_ALDEBARAN: 14951daa2bfaSLe Ma adev->num_vmhubs = 2; 14961daa2bfaSLe Ma 14978787ee01SHuang Rui 1498e60f8db5SAlex Xie /* 1499e60f8db5SAlex Xie * To fulfill 4-level page support, 1500e60f8db5SAlex Xie * vm size is 256TB (48bit), maximum size of Vega10, 1501e60f8db5SAlex Xie * block size 512 (9bit) 1502e60f8db5SAlex Xie */ 1503cdba61daSwentalou /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 1504cdba61daSwentalou if (amdgpu_sriov_vf(adev)) 1505cdba61daSwentalou amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 1506cdba61daSwentalou else 1507f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1508e60f8db5SAlex Xie break; 15093de2ff5dSLe Ma case CHIP_ARCTURUS: 1510c8a6e2a3SLe Ma adev->num_vmhubs = 3; 1511c8a6e2a3SLe Ma 15123de2ff5dSLe Ma /* Keep the vm size same with Vega20 */ 15133de2ff5dSLe Ma amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 15143de2ff5dSLe Ma break; 1515e60f8db5SAlex Xie default: 1516e60f8db5SAlex Xie break; 1517e60f8db5SAlex Xie } 1518e60f8db5SAlex Xie 1519e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 152044a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 1521770d13b1SChristian König &adev->gmc.vm_fault); 152230da7bb1SChristian König if (r) 152330da7bb1SChristian König return r; 152430da7bb1SChristian König 15257d19b15fSLe Ma if (adev->asic_type == CHIP_ARCTURUS) { 15267d19b15fSLe Ma r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, 15277d19b15fSLe Ma &adev->gmc.vm_fault); 15287d19b15fSLe Ma if (r) 15297d19b15fSLe Ma return r; 15307d19b15fSLe Ma } 15317d19b15fSLe Ma 153244a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 1533770d13b1SChristian König &adev->gmc.vm_fault); 1534e60f8db5SAlex Xie 1535e60f8db5SAlex Xie if (r) 1536e60f8db5SAlex Xie return r; 1537e60f8db5SAlex Xie 153868d705ddSHawking Zhang if (!amdgpu_sriov_vf(adev) && 153968d705ddSHawking Zhang !adev->gmc.xgmi.connected_to_cpu) { 1540791c4769Sxinhui pan /* interrupt sent to DF. */ 1541791c4769Sxinhui pan r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 1542791c4769Sxinhui pan &adev->gmc.ecc_irq); 1543791c4769Sxinhui pan if (r) 1544791c4769Sxinhui pan return r; 15452ee9403eSZhigang Luo } 1546791c4769Sxinhui pan 1547e60f8db5SAlex Xie /* Set the internal MC address mask 1548e60f8db5SAlex Xie * This is the max address of the GPU's 1549e60f8db5SAlex Xie * internal address space. 1550e60f8db5SAlex Xie */ 1551770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 1552e60f8db5SAlex Xie 1553244511f3SChristoph Hellwig r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); 1554e60f8db5SAlex Xie if (r) { 1555e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 1556244511f3SChristoph Hellwig return r; 1557e60f8db5SAlex Xie } 1558244511f3SChristoph Hellwig adev->need_swiotlb = drm_need_swiotlb(44); 1559e60f8db5SAlex Xie 156047622ba0SAlex Deucher if (adev->gmc.xgmi.supported) { 15618ffff9b4SOak Zeng r = adev->gfxhub.funcs->get_xgmi_info(adev); 1562bf0a60b7SAlex Deucher if (r) 1563bf0a60b7SAlex Deucher return r; 1564bf0a60b7SAlex Deucher } 1565bf0a60b7SAlex Deucher 1566e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 1567e60f8db5SAlex Xie if (r) 1568e60f8db5SAlex Xie return r; 1569e60f8db5SAlex Xie 15707b885f0eSAlex Deucher amdgpu_gmc_get_vbios_allocations(adev); 1571ebdef28eSAlex Deucher 1572e60f8db5SAlex Xie /* Memory manager */ 1573e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 1574e60f8db5SAlex Xie if (r) 1575e60f8db5SAlex Xie return r; 1576e60f8db5SAlex Xie 1577e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 1578e60f8db5SAlex Xie if (r) 1579e60f8db5SAlex Xie return r; 1580e60f8db5SAlex Xie 158105ec3edaSChristian König /* 158205ec3edaSChristian König * number of VMs 158305ec3edaSChristian König * VMID 0 is reserved for System 158481659b20SFelix Kuehling * amdgpu graphics/compute will use VMIDs 1..n-1 158581659b20SFelix Kuehling * amdkfd will use VMIDs n..15 158681659b20SFelix Kuehling * 158781659b20SFelix Kuehling * The first KFD VMID is 8 for GPUs with graphics, 3 for 158881659b20SFelix Kuehling * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs 158981659b20SFelix Kuehling * for video processing. 159005ec3edaSChristian König */ 159181659b20SFelix Kuehling adev->vm_manager.first_kfd_vmid = 15926dce50b1SFelix Kuehling (adev->asic_type == CHIP_ARCTURUS || 15936dce50b1SFelix Kuehling adev->asic_type == CHIP_ALDEBARAN) ? 3 : 8; 159405ec3edaSChristian König 159505ec3edaSChristian König amdgpu_vm_manager_init(adev); 159605ec3edaSChristian König 1597b0a2db9bSAlex Deucher gmc_v9_0_save_registers(adev); 1598b0a2db9bSAlex Deucher 159905ec3edaSChristian König return 0; 1600e60f8db5SAlex Xie } 1601e60f8db5SAlex Xie 1602e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 1603e60f8db5SAlex Xie { 1604e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1605e60f8db5SAlex Xie 16062adf1344STao Zhou amdgpu_gmc_ras_fini(adev); 1607f59548c8SMonk Liu amdgpu_gem_force_release(adev); 1608e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 1609a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 161037c49dedSLijo Lazar amdgpu_bo_unref(&adev->gmc.pdb0_bo); 1611e60f8db5SAlex Xie amdgpu_bo_fini(adev); 1612a3d9103eSAndrey Grodzovsky amdgpu_gart_fini(adev); 1613e60f8db5SAlex Xie 1614e60f8db5SAlex Xie return 0; 1615e60f8db5SAlex Xie } 1616e60f8db5SAlex Xie 1617e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1618e60f8db5SAlex Xie { 1619946a4d5bSShaoyun Liu 1620e60f8db5SAlex Xie switch (adev->asic_type) { 1621e60f8db5SAlex Xie case CHIP_VEGA10: 16224cd4c5c0SMonk Liu if (amdgpu_sriov_vf(adev)) 162398cad2deSTrigger Huang break; 1624df561f66SGustavo A. R. Silva fallthrough; 1625d96b428cSFeifei Xu case CHIP_VEGA20: 1626946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 16275c583018SEvan Quan golden_settings_mmhub_1_0_0, 1628c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1629946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 16305c583018SEvan Quan golden_settings_athub_1_0_0, 1631c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1632e60f8db5SAlex Xie break; 1633273a14cdSAlex Deucher case CHIP_VEGA12: 1634273a14cdSAlex Deucher break; 1635e4f3abaaSChunming Zhou case CHIP_RAVEN: 16368787ee01SHuang Rui /* TODO for renoir */ 1637946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 16385c583018SEvan Quan golden_settings_athub_1_0_0, 1639c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1640e4f3abaaSChunming Zhou break; 1641e60f8db5SAlex Xie default: 1642e60f8db5SAlex Xie break; 1643e60f8db5SAlex Xie } 1644e60f8db5SAlex Xie } 1645e60f8db5SAlex Xie 1646e60f8db5SAlex Xie /** 1647c2ecd79bSShirish S * gmc_v9_0_restore_registers - restores regs 1648c2ecd79bSShirish S * 1649c2ecd79bSShirish S * @adev: amdgpu_device pointer 1650c2ecd79bSShirish S * 1651c2ecd79bSShirish S * This restores register values, saved at suspend. 1652c2ecd79bSShirish S */ 1653b0a2db9bSAlex Deucher void gmc_v9_0_restore_registers(struct amdgpu_device *adev) 1654c2ecd79bSShirish S { 16550eaa8012SShirish S if (adev->asic_type == CHIP_RAVEN) { 1656f8646661SAlex Deucher WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); 16570eaa8012SShirish S WARN_ON(adev->gmc.sdpif_register != 16580eaa8012SShirish S RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0)); 16590eaa8012SShirish S } 1660c2ecd79bSShirish S } 1661c2ecd79bSShirish S 1662c2ecd79bSShirish S /** 1663e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1664e60f8db5SAlex Xie * 1665e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1666e60f8db5SAlex Xie */ 1667e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1668e60f8db5SAlex Xie { 1669cb1545f7SOak Zeng int r; 1670e60f8db5SAlex Xie 1671522510a6SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) 1672522510a6SOak Zeng amdgpu_gmc_init_pdb0(adev); 1673522510a6SOak Zeng 16741123b989SChristian König if (adev->gart.bo == NULL) { 1675e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1676e60f8db5SAlex Xie return -EINVAL; 1677e60f8db5SAlex Xie } 1678522510a6SOak Zeng 1679ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1680ce1b1b66SMonk Liu if (r) 1681ce1b1b66SMonk Liu return r; 1682e60f8db5SAlex Xie 16838ffff9b4SOak Zeng r = adev->gfxhub.funcs->gart_enable(adev); 1684e60f8db5SAlex Xie if (r) 1685e60f8db5SAlex Xie return r; 1686e60f8db5SAlex Xie 16879fb1506eSOak Zeng r = adev->mmhub.funcs->gart_enable(adev); 1688e60f8db5SAlex Xie if (r) 1689e60f8db5SAlex Xie return r; 1690e60f8db5SAlex Xie 1691522510a6SOak Zeng DRM_INFO("PCIE GART of %uM enabled.\n", 1692522510a6SOak Zeng (unsigned)(adev->gmc.gart_size >> 20)); 1693522510a6SOak Zeng if (adev->gmc.pdb0_bo) 1694522510a6SOak Zeng DRM_INFO("PDB0 located at 0x%016llX\n", 1695522510a6SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo)); 1696522510a6SOak Zeng DRM_INFO("PTB located at 0x%016llX\n", 1697cb1545f7SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1698522510a6SOak Zeng 1699cb1545f7SOak Zeng adev->gart.ready = true; 1700cb1545f7SOak Zeng return 0; 1701cb1545f7SOak Zeng } 1702cb1545f7SOak Zeng 1703cb1545f7SOak Zeng static int gmc_v9_0_hw_init(void *handle) 1704cb1545f7SOak Zeng { 1705cb1545f7SOak Zeng struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1706cb1545f7SOak Zeng bool value; 1707cb1545f7SOak Zeng int r, i; 1708cb1545f7SOak Zeng 1709cb1545f7SOak Zeng /* The sequence of these two function calls matters.*/ 1710cb1545f7SOak Zeng gmc_v9_0_init_golden_registers(adev); 1711cb1545f7SOak Zeng 1712cb1545f7SOak Zeng if (adev->mode_info.num_crtc) { 1713cb1545f7SOak Zeng /* Lockout access through VGA aperture*/ 1714cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1715cb1545f7SOak Zeng /* disable VGA render */ 1716cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1717cb1545f7SOak Zeng } 1718cb1545f7SOak Zeng 17199fb1506eSOak Zeng if (adev->mmhub.funcs->update_power_gating) 17209fb1506eSOak Zeng adev->mmhub.funcs->update_power_gating(adev, true); 17219fb1506eSOak Zeng 1722455d40c9SLikun Gao adev->hdp.funcs->init_registers(adev); 1723fe2b5323STiecheng Zhou 17241d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 1725455d40c9SLikun Gao adev->hdp.funcs->flush_hdp(adev, NULL); 17261d4e0a8cSMonk Liu 1727e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1728e60f8db5SAlex Xie value = false; 1729e60f8db5SAlex Xie else 1730e60f8db5SAlex Xie value = true; 1731e60f8db5SAlex Xie 173220bf2f6fSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 17338ffff9b4SOak Zeng adev->gfxhub.funcs->set_fault_enable_default(adev, value); 17349fb1506eSOak Zeng adev->mmhub.funcs->set_fault_enable_default(adev, value); 173520bf2f6fSZhigang Luo } 17363ff98548SOak Zeng for (i = 0; i < adev->num_vmhubs; ++i) 17373ff98548SOak Zeng gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); 1738e60f8db5SAlex Xie 1739e7da754bSMonk Liu if (adev->umc.funcs && adev->umc.funcs->init_registers) 1740e7da754bSMonk Liu adev->umc.funcs->init_registers(adev); 1741e7da754bSMonk Liu 1742e60f8db5SAlex Xie r = gmc_v9_0_gart_enable(adev); 1743e60f8db5SAlex Xie 1744e60f8db5SAlex Xie return r; 1745e60f8db5SAlex Xie } 1746e60f8db5SAlex Xie 1747e60f8db5SAlex Xie /** 1748e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1749e60f8db5SAlex Xie * 1750e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1751e60f8db5SAlex Xie * 1752e60f8db5SAlex Xie * This disables all VM page table. 1753e60f8db5SAlex Xie */ 1754e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1755e60f8db5SAlex Xie { 17568ffff9b4SOak Zeng adev->gfxhub.funcs->gart_disable(adev); 17579fb1506eSOak Zeng adev->mmhub.funcs->gart_disable(adev); 1758ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1759e60f8db5SAlex Xie } 1760e60f8db5SAlex Xie 1761e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1762e60f8db5SAlex Xie { 1763e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1764e60f8db5SAlex Xie 17655dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 17665dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 17675dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 17685dd696aeSTrigger Huang return 0; 17695dd696aeSTrigger Huang } 17705dd696aeSTrigger Huang 1771791c4769Sxinhui pan amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1772770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1773e60f8db5SAlex Xie gmc_v9_0_gart_disable(adev); 1774e60f8db5SAlex Xie 1775e60f8db5SAlex Xie return 0; 1776e60f8db5SAlex Xie } 1777e60f8db5SAlex Xie 1778e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1779e60f8db5SAlex Xie { 1780e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1781e60f8db5SAlex Xie 1782c24a3c05SLiu Shixin return gmc_v9_0_hw_fini(adev); 1783e60f8db5SAlex Xie } 1784e60f8db5SAlex Xie 1785e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1786e60f8db5SAlex Xie { 1787e60f8db5SAlex Xie int r; 1788e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1789e60f8db5SAlex Xie 1790e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1791e60f8db5SAlex Xie if (r) 1792e60f8db5SAlex Xie return r; 1793e60f8db5SAlex Xie 1794620f774fSChristian König amdgpu_vmid_reset_all(adev); 1795e60f8db5SAlex Xie 179632601d48SChristian König return 0; 1797e60f8db5SAlex Xie } 1798e60f8db5SAlex Xie 1799e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1800e60f8db5SAlex Xie { 1801e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1802e60f8db5SAlex Xie return true; 1803e60f8db5SAlex Xie } 1804e60f8db5SAlex Xie 1805e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1806e60f8db5SAlex Xie { 1807e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1808e60f8db5SAlex Xie return 0; 1809e60f8db5SAlex Xie } 1810e60f8db5SAlex Xie 1811e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1812e60f8db5SAlex Xie { 1813e60f8db5SAlex Xie /* XXX for emulation.*/ 1814e60f8db5SAlex Xie return 0; 1815e60f8db5SAlex Xie } 1816e60f8db5SAlex Xie 1817e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1818e60f8db5SAlex Xie enum amd_clockgating_state state) 1819e60f8db5SAlex Xie { 1820d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1821d5583d4fSHuang Rui 18229fb1506eSOak Zeng adev->mmhub.funcs->set_clockgating(adev, state); 1823bee7b51aSLe Ma 1824bee7b51aSLe Ma athub_v1_0_set_clockgating(adev, state); 1825bee7b51aSLe Ma 1826bee7b51aSLe Ma return 0; 1827e60f8db5SAlex Xie } 1828e60f8db5SAlex Xie 182913052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 183013052be5SHuang Rui { 183113052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 183213052be5SHuang Rui 18339fb1506eSOak Zeng adev->mmhub.funcs->get_clockgating(adev, flags); 1834bee7b51aSLe Ma 1835bee7b51aSLe Ma athub_v1_0_get_clockgating(adev, flags); 183613052be5SHuang Rui } 183713052be5SHuang Rui 1838e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1839e60f8db5SAlex Xie enum amd_powergating_state state) 1840e60f8db5SAlex Xie { 1841e60f8db5SAlex Xie return 0; 1842e60f8db5SAlex Xie } 1843e60f8db5SAlex Xie 1844e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1845e60f8db5SAlex Xie .name = "gmc_v9_0", 1846e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1847e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1848e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1849e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1850e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1851e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1852e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1853e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1854e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1855e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1856e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1857e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1858e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 185913052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1860e60f8db5SAlex Xie }; 1861e60f8db5SAlex Xie 1862e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1863e60f8db5SAlex Xie { 1864e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1865e60f8db5SAlex Xie .major = 9, 1866e60f8db5SAlex Xie .minor = 0, 1867e60f8db5SAlex Xie .rev = 0, 1868e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1869e60f8db5SAlex Xie }; 1870