1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23f867723bSSam Ravnborg 24e60f8db5SAlex Xie #include <linux/firmware.h> 25f867723bSSam Ravnborg #include <linux/pci.h> 26f867723bSSam Ravnborg 27fd5fd480SChunming Zhou #include <drm/drm_cache.h> 28f867723bSSam Ravnborg 29e60f8db5SAlex Xie #include "amdgpu.h" 30e60f8db5SAlex Xie #include "gmc_v9_0.h" 318d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 322cddc50eSHuang Rui #include "amdgpu_gem.h" 33e60f8db5SAlex Xie 34cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 35135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 36135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 37fb960bd2SFeifei Xu #include "vega10_enum.h" 3865417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 39ea930000SAlex Sierra #include "athub/athub_1_0_sh_mask.h" 406ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 41250b4228SChristian König #include "oss/osssys_4_0_offset.h" 42e60f8db5SAlex Xie 43946a4d5bSShaoyun Liu #include "soc15.h" 44ea930000SAlex Sierra #include "soc15d.h" 45e60f8db5SAlex Xie #include "soc15_common.h" 4690c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 47e60f8db5SAlex Xie 48e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 49e60f8db5SAlex Xie #include "mmhub_v1_0.h" 50bee7b51aSLe Ma #include "athub_v1_0.h" 51bf0a60b7SAlex Deucher #include "gfxhub_v1_1.h" 5251cce480SLe Ma #include "mmhub_v9_4.h" 5385e39550SLe Ma #include "mmhub_v1_7.h" 545b6b35aaSHawking Zhang #include "umc_v6_1.h" 55e7da754bSMonk Liu #include "umc_v6_0.h" 56186c8a85SJohn Clements #include "umc_v6_7.h" 576f12507fSHawking Zhang #include "hdp_v4_0.h" 583907c492SJohn Clements #include "mca_v3_0.h" 59e60f8db5SAlex Xie 6044a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 6144a99b65SAndrey Grodzovsky 62791c4769Sxinhui pan #include "amdgpu_ras.h" 63029fbd43SHawking Zhang #include "amdgpu_xgmi.h" 64791c4769Sxinhui pan 65ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 66ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 67ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 68ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 69ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 70ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 71ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 72f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d 73f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 74f8646661SAlex Deucher 75ebdef28eSAlex Deucher 76be99ecbfSAlex Deucher static const char *gfxhub_client_ids[] = { 77be99ecbfSAlex Deucher "CB", 78be99ecbfSAlex Deucher "DB", 79be99ecbfSAlex Deucher "IA", 80be99ecbfSAlex Deucher "WD", 81be99ecbfSAlex Deucher "CPF", 82be99ecbfSAlex Deucher "CPC", 83be99ecbfSAlex Deucher "CPG", 84be99ecbfSAlex Deucher "RLC", 85be99ecbfSAlex Deucher "TCP", 86be99ecbfSAlex Deucher "SQC (inst)", 87be99ecbfSAlex Deucher "SQC (data)", 88be99ecbfSAlex Deucher "SQG", 89be99ecbfSAlex Deucher "PA", 90be99ecbfSAlex Deucher }; 91be99ecbfSAlex Deucher 9202f23f5fSAlex Deucher static const char *mmhub_client_ids_raven[][2] = { 9302f23f5fSAlex Deucher [0][0] = "MP1", 9402f23f5fSAlex Deucher [1][0] = "MP0", 9502f23f5fSAlex Deucher [2][0] = "VCN", 9602f23f5fSAlex Deucher [3][0] = "VCNU", 9702f23f5fSAlex Deucher [4][0] = "HDP", 9802f23f5fSAlex Deucher [5][0] = "DCE", 9902f23f5fSAlex Deucher [13][0] = "UTCL2", 10002f23f5fSAlex Deucher [19][0] = "TLS", 10102f23f5fSAlex Deucher [26][0] = "OSS", 10202f23f5fSAlex Deucher [27][0] = "SDMA0", 10302f23f5fSAlex Deucher [0][1] = "MP1", 10402f23f5fSAlex Deucher [1][1] = "MP0", 10502f23f5fSAlex Deucher [2][1] = "VCN", 10602f23f5fSAlex Deucher [3][1] = "VCNU", 10702f23f5fSAlex Deucher [4][1] = "HDP", 10802f23f5fSAlex Deucher [5][1] = "XDP", 10902f23f5fSAlex Deucher [6][1] = "DBGU0", 11002f23f5fSAlex Deucher [7][1] = "DCE", 11102f23f5fSAlex Deucher [8][1] = "DCEDWB0", 11202f23f5fSAlex Deucher [9][1] = "DCEDWB1", 11302f23f5fSAlex Deucher [26][1] = "OSS", 11402f23f5fSAlex Deucher [27][1] = "SDMA0", 11502f23f5fSAlex Deucher }; 11602f23f5fSAlex Deucher 11702f23f5fSAlex Deucher static const char *mmhub_client_ids_renoir[][2] = { 11802f23f5fSAlex Deucher [0][0] = "MP1", 11902f23f5fSAlex Deucher [1][0] = "MP0", 12002f23f5fSAlex Deucher [2][0] = "HDP", 12102f23f5fSAlex Deucher [4][0] = "DCEDMC", 12202f23f5fSAlex Deucher [5][0] = "DCEVGA", 12302f23f5fSAlex Deucher [13][0] = "UTCL2", 12402f23f5fSAlex Deucher [19][0] = "TLS", 12502f23f5fSAlex Deucher [26][0] = "OSS", 12602f23f5fSAlex Deucher [27][0] = "SDMA0", 12702f23f5fSAlex Deucher [28][0] = "VCN", 12802f23f5fSAlex Deucher [29][0] = "VCNU", 12902f23f5fSAlex Deucher [30][0] = "JPEG", 13002f23f5fSAlex Deucher [0][1] = "MP1", 13102f23f5fSAlex Deucher [1][1] = "MP0", 13202f23f5fSAlex Deucher [2][1] = "HDP", 13302f23f5fSAlex Deucher [3][1] = "XDP", 13402f23f5fSAlex Deucher [6][1] = "DBGU0", 13502f23f5fSAlex Deucher [7][1] = "DCEDMC", 13602f23f5fSAlex Deucher [8][1] = "DCEVGA", 13702f23f5fSAlex Deucher [9][1] = "DCEDWB", 13802f23f5fSAlex Deucher [26][1] = "OSS", 13902f23f5fSAlex Deucher [27][1] = "SDMA0", 14002f23f5fSAlex Deucher [28][1] = "VCN", 14102f23f5fSAlex Deucher [29][1] = "VCNU", 14202f23f5fSAlex Deucher [30][1] = "JPEG", 14302f23f5fSAlex Deucher }; 14402f23f5fSAlex Deucher 14502f23f5fSAlex Deucher static const char *mmhub_client_ids_vega10[][2] = { 14602f23f5fSAlex Deucher [0][0] = "MP0", 14702f23f5fSAlex Deucher [1][0] = "UVD", 14802f23f5fSAlex Deucher [2][0] = "UVDU", 14902f23f5fSAlex Deucher [3][0] = "HDP", 15002f23f5fSAlex Deucher [13][0] = "UTCL2", 15102f23f5fSAlex Deucher [14][0] = "OSS", 15202f23f5fSAlex Deucher [15][0] = "SDMA1", 15302f23f5fSAlex Deucher [32+0][0] = "VCE0", 15402f23f5fSAlex Deucher [32+1][0] = "VCE0U", 15502f23f5fSAlex Deucher [32+2][0] = "XDMA", 15602f23f5fSAlex Deucher [32+3][0] = "DCE", 15702f23f5fSAlex Deucher [32+4][0] = "MP1", 15802f23f5fSAlex Deucher [32+14][0] = "SDMA0", 15902f23f5fSAlex Deucher [0][1] = "MP0", 16002f23f5fSAlex Deucher [1][1] = "UVD", 16102f23f5fSAlex Deucher [2][1] = "UVDU", 16202f23f5fSAlex Deucher [3][1] = "DBGU0", 16302f23f5fSAlex Deucher [4][1] = "HDP", 16402f23f5fSAlex Deucher [5][1] = "XDP", 16502f23f5fSAlex Deucher [14][1] = "OSS", 16602f23f5fSAlex Deucher [15][1] = "SDMA0", 16702f23f5fSAlex Deucher [32+0][1] = "VCE0", 16802f23f5fSAlex Deucher [32+1][1] = "VCE0U", 16902f23f5fSAlex Deucher [32+2][1] = "XDMA", 17002f23f5fSAlex Deucher [32+3][1] = "DCE", 17102f23f5fSAlex Deucher [32+4][1] = "DCEDWB", 17202f23f5fSAlex Deucher [32+5][1] = "MP1", 17302f23f5fSAlex Deucher [32+6][1] = "DBGU1", 17402f23f5fSAlex Deucher [32+14][1] = "SDMA1", 17502f23f5fSAlex Deucher }; 17602f23f5fSAlex Deucher 17702f23f5fSAlex Deucher static const char *mmhub_client_ids_vega12[][2] = { 17802f23f5fSAlex Deucher [0][0] = "MP0", 17902f23f5fSAlex Deucher [1][0] = "VCE0", 18002f23f5fSAlex Deucher [2][0] = "VCE0U", 18102f23f5fSAlex Deucher [3][0] = "HDP", 18202f23f5fSAlex Deucher [13][0] = "UTCL2", 18302f23f5fSAlex Deucher [14][0] = "OSS", 18402f23f5fSAlex Deucher [15][0] = "SDMA1", 18502f23f5fSAlex Deucher [32+0][0] = "DCE", 18602f23f5fSAlex Deucher [32+1][0] = "XDMA", 18702f23f5fSAlex Deucher [32+2][0] = "UVD", 18802f23f5fSAlex Deucher [32+3][0] = "UVDU", 18902f23f5fSAlex Deucher [32+4][0] = "MP1", 19002f23f5fSAlex Deucher [32+15][0] = "SDMA0", 19102f23f5fSAlex Deucher [0][1] = "MP0", 19202f23f5fSAlex Deucher [1][1] = "VCE0", 19302f23f5fSAlex Deucher [2][1] = "VCE0U", 19402f23f5fSAlex Deucher [3][1] = "DBGU0", 19502f23f5fSAlex Deucher [4][1] = "HDP", 19602f23f5fSAlex Deucher [5][1] = "XDP", 19702f23f5fSAlex Deucher [14][1] = "OSS", 19802f23f5fSAlex Deucher [15][1] = "SDMA0", 19902f23f5fSAlex Deucher [32+0][1] = "DCE", 20002f23f5fSAlex Deucher [32+1][1] = "DCEDWB", 20102f23f5fSAlex Deucher [32+2][1] = "XDMA", 20202f23f5fSAlex Deucher [32+3][1] = "UVD", 20302f23f5fSAlex Deucher [32+4][1] = "UVDU", 20402f23f5fSAlex Deucher [32+5][1] = "MP1", 20502f23f5fSAlex Deucher [32+6][1] = "DBGU1", 20602f23f5fSAlex Deucher [32+15][1] = "SDMA1", 20702f23f5fSAlex Deucher }; 20802f23f5fSAlex Deucher 20902f23f5fSAlex Deucher static const char *mmhub_client_ids_vega20[][2] = { 21002f23f5fSAlex Deucher [0][0] = "XDMA", 21102f23f5fSAlex Deucher [1][0] = "DCE", 21202f23f5fSAlex Deucher [2][0] = "VCE0", 21302f23f5fSAlex Deucher [3][0] = "VCE0U", 21402f23f5fSAlex Deucher [4][0] = "UVD", 21502f23f5fSAlex Deucher [5][0] = "UVD1U", 21602f23f5fSAlex Deucher [13][0] = "OSS", 21702f23f5fSAlex Deucher [14][0] = "HDP", 21802f23f5fSAlex Deucher [15][0] = "SDMA0", 21902f23f5fSAlex Deucher [32+0][0] = "UVD", 22002f23f5fSAlex Deucher [32+1][0] = "UVDU", 22102f23f5fSAlex Deucher [32+2][0] = "MP1", 22202f23f5fSAlex Deucher [32+3][0] = "MP0", 22302f23f5fSAlex Deucher [32+12][0] = "UTCL2", 22402f23f5fSAlex Deucher [32+14][0] = "SDMA1", 22502f23f5fSAlex Deucher [0][1] = "XDMA", 22602f23f5fSAlex Deucher [1][1] = "DCE", 22702f23f5fSAlex Deucher [2][1] = "DCEDWB", 22802f23f5fSAlex Deucher [3][1] = "VCE0", 22902f23f5fSAlex Deucher [4][1] = "VCE0U", 23002f23f5fSAlex Deucher [5][1] = "UVD1", 23102f23f5fSAlex Deucher [6][1] = "UVD1U", 23202f23f5fSAlex Deucher [7][1] = "DBGU0", 23302f23f5fSAlex Deucher [8][1] = "XDP", 23402f23f5fSAlex Deucher [13][1] = "OSS", 23502f23f5fSAlex Deucher [14][1] = "HDP", 23602f23f5fSAlex Deucher [15][1] = "SDMA0", 23702f23f5fSAlex Deucher [32+0][1] = "UVD", 23802f23f5fSAlex Deucher [32+1][1] = "UVDU", 23902f23f5fSAlex Deucher [32+2][1] = "DBGU1", 24002f23f5fSAlex Deucher [32+3][1] = "MP1", 24102f23f5fSAlex Deucher [32+4][1] = "MP0", 24202f23f5fSAlex Deucher [32+14][1] = "SDMA1", 24302f23f5fSAlex Deucher }; 24402f23f5fSAlex Deucher 24502f23f5fSAlex Deucher static const char *mmhub_client_ids_arcturus[][2] = { 246e83db774SAlex Deucher [0][0] = "DBGU1", 247e83db774SAlex Deucher [1][0] = "XDP", 24802f23f5fSAlex Deucher [2][0] = "MP1", 24902f23f5fSAlex Deucher [14][0] = "HDP", 250e83db774SAlex Deucher [171][0] = "JPEG", 251e83db774SAlex Deucher [172][0] = "VCN", 252e83db774SAlex Deucher [173][0] = "VCNU", 253e83db774SAlex Deucher [203][0] = "JPEG1", 254e83db774SAlex Deucher [204][0] = "VCN1", 255e83db774SAlex Deucher [205][0] = "VCN1U", 256e83db774SAlex Deucher [256][0] = "SDMA0", 257e83db774SAlex Deucher [257][0] = "SDMA1", 258e83db774SAlex Deucher [258][0] = "SDMA2", 259e83db774SAlex Deucher [259][0] = "SDMA3", 260e83db774SAlex Deucher [260][0] = "SDMA4", 261e83db774SAlex Deucher [261][0] = "SDMA5", 262e83db774SAlex Deucher [262][0] = "SDMA6", 263e83db774SAlex Deucher [263][0] = "SDMA7", 264e83db774SAlex Deucher [384][0] = "OSS", 26502f23f5fSAlex Deucher [0][1] = "DBGU1", 26602f23f5fSAlex Deucher [1][1] = "XDP", 26702f23f5fSAlex Deucher [2][1] = "MP1", 26802f23f5fSAlex Deucher [14][1] = "HDP", 269e83db774SAlex Deucher [171][1] = "JPEG", 270e83db774SAlex Deucher [172][1] = "VCN", 271e83db774SAlex Deucher [173][1] = "VCNU", 272e83db774SAlex Deucher [203][1] = "JPEG1", 273e83db774SAlex Deucher [204][1] = "VCN1", 274e83db774SAlex Deucher [205][1] = "VCN1U", 275e83db774SAlex Deucher [256][1] = "SDMA0", 276e83db774SAlex Deucher [257][1] = "SDMA1", 277e83db774SAlex Deucher [258][1] = "SDMA2", 278e83db774SAlex Deucher [259][1] = "SDMA3", 279e83db774SAlex Deucher [260][1] = "SDMA4", 280e83db774SAlex Deucher [261][1] = "SDMA5", 281e83db774SAlex Deucher [262][1] = "SDMA6", 282e83db774SAlex Deucher [263][1] = "SDMA7", 283e83db774SAlex Deucher [384][1] = "OSS", 28402f23f5fSAlex Deucher }; 285e60f8db5SAlex Xie 286e844cd99SAlex Deucher static const char *mmhub_client_ids_aldebaran[][2] = { 287e844cd99SAlex Deucher [2][0] = "MP1", 288e844cd99SAlex Deucher [3][0] = "MP0", 289f4ec3e50SAlex Sierra [32+1][0] = "DBGU_IO0", 290f4ec3e50SAlex Sierra [32+2][0] = "DBGU_IO2", 291e844cd99SAlex Deucher [32+4][0] = "MPIO", 292e844cd99SAlex Deucher [96+11][0] = "JPEG0", 293e844cd99SAlex Deucher [96+12][0] = "VCN0", 294e844cd99SAlex Deucher [96+13][0] = "VCNU0", 295e844cd99SAlex Deucher [128+11][0] = "JPEG1", 296e844cd99SAlex Deucher [128+12][0] = "VCN1", 297e844cd99SAlex Deucher [128+13][0] = "VCNU1", 298f4ec3e50SAlex Sierra [160+1][0] = "XDP", 299e844cd99SAlex Deucher [160+14][0] = "HDP", 300f4ec3e50SAlex Sierra [256+0][0] = "SDMA0", 301f4ec3e50SAlex Sierra [256+1][0] = "SDMA1", 302f4ec3e50SAlex Sierra [256+2][0] = "SDMA2", 303f4ec3e50SAlex Sierra [256+3][0] = "SDMA3", 304f4ec3e50SAlex Sierra [256+4][0] = "SDMA4", 305f4ec3e50SAlex Sierra [384+0][0] = "OSS", 306e844cd99SAlex Deucher [2][1] = "MP1", 307e844cd99SAlex Deucher [3][1] = "MP0", 308e844cd99SAlex Deucher [32+1][1] = "DBGU_IO0", 309e844cd99SAlex Deucher [32+2][1] = "DBGU_IO2", 310e844cd99SAlex Deucher [32+4][1] = "MPIO", 311e844cd99SAlex Deucher [96+11][1] = "JPEG0", 312e844cd99SAlex Deucher [96+12][1] = "VCN0", 313e844cd99SAlex Deucher [96+13][1] = "VCNU0", 314e844cd99SAlex Deucher [128+11][1] = "JPEG1", 315e844cd99SAlex Deucher [128+12][1] = "VCN1", 316e844cd99SAlex Deucher [128+13][1] = "VCNU1", 317f4ec3e50SAlex Sierra [160+1][1] = "XDP", 318e844cd99SAlex Deucher [160+14][1] = "HDP", 319f4ec3e50SAlex Sierra [256+0][1] = "SDMA0", 320f4ec3e50SAlex Sierra [256+1][1] = "SDMA1", 321f4ec3e50SAlex Sierra [256+2][1] = "SDMA2", 322f4ec3e50SAlex Sierra [256+3][1] = "SDMA3", 323f4ec3e50SAlex Sierra [256+4][1] = "SDMA4", 324f4ec3e50SAlex Sierra [384+0][1] = "OSS", 325e844cd99SAlex Deucher }; 326e844cd99SAlex Deucher 327946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 3285c583018SEvan Quan { 329946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 330946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 3315c583018SEvan Quan }; 3325c583018SEvan Quan 333946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 3345c583018SEvan Quan { 335946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 336946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 3375c583018SEvan Quan }; 3385c583018SEvan Quan 339791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 340791c4769Sxinhui pan (0x000143c0 + 0x00000000), 341791c4769Sxinhui pan (0x000143c0 + 0x00000800), 342791c4769Sxinhui pan (0x000143c0 + 0x00001000), 343791c4769Sxinhui pan (0x000143c0 + 0x00001800), 344791c4769Sxinhui pan (0x000543c0 + 0x00000000), 345791c4769Sxinhui pan (0x000543c0 + 0x00000800), 346791c4769Sxinhui pan (0x000543c0 + 0x00001000), 347791c4769Sxinhui pan (0x000543c0 + 0x00001800), 348791c4769Sxinhui pan (0x000943c0 + 0x00000000), 349791c4769Sxinhui pan (0x000943c0 + 0x00000800), 350791c4769Sxinhui pan (0x000943c0 + 0x00001000), 351791c4769Sxinhui pan (0x000943c0 + 0x00001800), 352791c4769Sxinhui pan (0x000d43c0 + 0x00000000), 353791c4769Sxinhui pan (0x000d43c0 + 0x00000800), 354791c4769Sxinhui pan (0x000d43c0 + 0x00001000), 355791c4769Sxinhui pan (0x000d43c0 + 0x00001800), 356791c4769Sxinhui pan (0x001143c0 + 0x00000000), 357791c4769Sxinhui pan (0x001143c0 + 0x00000800), 358791c4769Sxinhui pan (0x001143c0 + 0x00001000), 359791c4769Sxinhui pan (0x001143c0 + 0x00001800), 360791c4769Sxinhui pan (0x001543c0 + 0x00000000), 361791c4769Sxinhui pan (0x001543c0 + 0x00000800), 362791c4769Sxinhui pan (0x001543c0 + 0x00001000), 363791c4769Sxinhui pan (0x001543c0 + 0x00001800), 364791c4769Sxinhui pan (0x001943c0 + 0x00000000), 365791c4769Sxinhui pan (0x001943c0 + 0x00000800), 366791c4769Sxinhui pan (0x001943c0 + 0x00001000), 367791c4769Sxinhui pan (0x001943c0 + 0x00001800), 368791c4769Sxinhui pan (0x001d43c0 + 0x00000000), 369791c4769Sxinhui pan (0x001d43c0 + 0x00000800), 370791c4769Sxinhui pan (0x001d43c0 + 0x00001000), 371791c4769Sxinhui pan (0x001d43c0 + 0x00001800), 37202bab923SDavid Panariti }; 37302bab923SDavid Panariti 374791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 375791c4769Sxinhui pan (0x000143e0 + 0x00000000), 376791c4769Sxinhui pan (0x000143e0 + 0x00000800), 377791c4769Sxinhui pan (0x000143e0 + 0x00001000), 378791c4769Sxinhui pan (0x000143e0 + 0x00001800), 379791c4769Sxinhui pan (0x000543e0 + 0x00000000), 380791c4769Sxinhui pan (0x000543e0 + 0x00000800), 381791c4769Sxinhui pan (0x000543e0 + 0x00001000), 382791c4769Sxinhui pan (0x000543e0 + 0x00001800), 383791c4769Sxinhui pan (0x000943e0 + 0x00000000), 384791c4769Sxinhui pan (0x000943e0 + 0x00000800), 385791c4769Sxinhui pan (0x000943e0 + 0x00001000), 386791c4769Sxinhui pan (0x000943e0 + 0x00001800), 387791c4769Sxinhui pan (0x000d43e0 + 0x00000000), 388791c4769Sxinhui pan (0x000d43e0 + 0x00000800), 389791c4769Sxinhui pan (0x000d43e0 + 0x00001000), 390791c4769Sxinhui pan (0x000d43e0 + 0x00001800), 391791c4769Sxinhui pan (0x001143e0 + 0x00000000), 392791c4769Sxinhui pan (0x001143e0 + 0x00000800), 393791c4769Sxinhui pan (0x001143e0 + 0x00001000), 394791c4769Sxinhui pan (0x001143e0 + 0x00001800), 395791c4769Sxinhui pan (0x001543e0 + 0x00000000), 396791c4769Sxinhui pan (0x001543e0 + 0x00000800), 397791c4769Sxinhui pan (0x001543e0 + 0x00001000), 398791c4769Sxinhui pan (0x001543e0 + 0x00001800), 399791c4769Sxinhui pan (0x001943e0 + 0x00000000), 400791c4769Sxinhui pan (0x001943e0 + 0x00000800), 401791c4769Sxinhui pan (0x001943e0 + 0x00001000), 402791c4769Sxinhui pan (0x001943e0 + 0x00001800), 403791c4769Sxinhui pan (0x001d43e0 + 0x00000000), 404791c4769Sxinhui pan (0x001d43e0 + 0x00000800), 405791c4769Sxinhui pan (0x001d43e0 + 0x00001000), 406791c4769Sxinhui pan (0x001d43e0 + 0x00001800), 40702bab923SDavid Panariti }; 40802bab923SDavid Panariti 409791c4769Sxinhui pan static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 410791c4769Sxinhui pan struct amdgpu_irq_src *src, 411791c4769Sxinhui pan unsigned type, 412791c4769Sxinhui pan enum amdgpu_interrupt_state state) 413791c4769Sxinhui pan { 414791c4769Sxinhui pan u32 bits, i, tmp, reg; 415791c4769Sxinhui pan 4161e2c6d55SJohn Clements /* Devices newer then VEGA10/12 shall have these programming 4171e2c6d55SJohn Clements sequences performed by PSP BL */ 4181e2c6d55SJohn Clements if (adev->asic_type >= CHIP_VEGA20) 4191e2c6d55SJohn Clements return 0; 4201e2c6d55SJohn Clements 421791c4769Sxinhui pan bits = 0x7f; 422791c4769Sxinhui pan 423791c4769Sxinhui pan switch (state) { 424791c4769Sxinhui pan case AMDGPU_IRQ_STATE_DISABLE: 425791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 426791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 427791c4769Sxinhui pan tmp = RREG32(reg); 428791c4769Sxinhui pan tmp &= ~bits; 429791c4769Sxinhui pan WREG32(reg, tmp); 430791c4769Sxinhui pan } 431791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 432791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 433791c4769Sxinhui pan tmp = RREG32(reg); 434791c4769Sxinhui pan tmp &= ~bits; 435791c4769Sxinhui pan WREG32(reg, tmp); 436791c4769Sxinhui pan } 437791c4769Sxinhui pan break; 438791c4769Sxinhui pan case AMDGPU_IRQ_STATE_ENABLE: 439791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 440791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 441791c4769Sxinhui pan tmp = RREG32(reg); 442791c4769Sxinhui pan tmp |= bits; 443791c4769Sxinhui pan WREG32(reg, tmp); 444791c4769Sxinhui pan } 445791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 446791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 447791c4769Sxinhui pan tmp = RREG32(reg); 448791c4769Sxinhui pan tmp |= bits; 449791c4769Sxinhui pan WREG32(reg, tmp); 450791c4769Sxinhui pan } 451791c4769Sxinhui pan break; 452791c4769Sxinhui pan default: 453791c4769Sxinhui pan break; 454791c4769Sxinhui pan } 455791c4769Sxinhui pan 456791c4769Sxinhui pan return 0; 457791c4769Sxinhui pan } 458791c4769Sxinhui pan 459e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 460e60f8db5SAlex Xie struct amdgpu_irq_src *src, 461e60f8db5SAlex Xie unsigned type, 462e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 463e60f8db5SAlex Xie { 464e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 465ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 466e60f8db5SAlex Xie 46711250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46811250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46911250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47011250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47111250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47211250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47311250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 47411250164SChristian König 475e60f8db5SAlex Xie switch (state) { 476e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 4771daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 478ae6d1416STom St Denis hub = &adev->vmhub[j]; 479e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 480e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 481*92f153bbSVictor Skvortsov 482*92f153bbSVictor Skvortsov if (j == AMDGPU_GFXHUB_0) 483*92f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP(GC, reg); 484*92f153bbSVictor Skvortsov else 485*92f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP(MMHUB, reg); 486*92f153bbSVictor Skvortsov 487e60f8db5SAlex Xie tmp &= ~bits; 488*92f153bbSVictor Skvortsov 489*92f153bbSVictor Skvortsov if (j == AMDGPU_GFXHUB_0) 490*92f153bbSVictor Skvortsov WREG32_SOC15_IP(GC, reg, tmp); 491*92f153bbSVictor Skvortsov else 492*92f153bbSVictor Skvortsov WREG32_SOC15_IP(MMHUB, reg, tmp); 493e60f8db5SAlex Xie } 494e60f8db5SAlex Xie } 495e60f8db5SAlex Xie break; 496e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 4971daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 498ae6d1416STom St Denis hub = &adev->vmhub[j]; 499e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 500e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 501*92f153bbSVictor Skvortsov 502*92f153bbSVictor Skvortsov if (j == AMDGPU_GFXHUB_0) 503*92f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP(GC, reg); 504*92f153bbSVictor Skvortsov else 505*92f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP(MMHUB, reg); 506*92f153bbSVictor Skvortsov 507e60f8db5SAlex Xie tmp |= bits; 508*92f153bbSVictor Skvortsov 509*92f153bbSVictor Skvortsov if (j == AMDGPU_GFXHUB_0) 510*92f153bbSVictor Skvortsov WREG32_SOC15_IP(GC, reg, tmp); 511*92f153bbSVictor Skvortsov else 512*92f153bbSVictor Skvortsov WREG32_SOC15_IP(MMHUB, reg, tmp); 513e60f8db5SAlex Xie } 514e60f8db5SAlex Xie } 5159304ca4dSGustavo A. R. Silva break; 516e60f8db5SAlex Xie default: 517e60f8db5SAlex Xie break; 518e60f8db5SAlex Xie } 519e60f8db5SAlex Xie 520e60f8db5SAlex Xie return 0; 521e60f8db5SAlex Xie } 522e60f8db5SAlex Xie 523e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 524e60f8db5SAlex Xie struct amdgpu_irq_src *source, 525e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 526e60f8db5SAlex Xie { 527c468f9e2SChristian König bool retry_fault = !!(entry->src_data[1] & 0x80); 528ff891a2eSPhilip Yang bool write_fault = !!(entry->src_data[1] & 0x20); 52902f23f5fSAlex Deucher uint32_t status = 0, cid = 0, rw = 0; 530e3898719SChristian König struct amdgpu_task_info task_info; 531e3898719SChristian König struct amdgpu_vmhub *hub; 53202f23f5fSAlex Deucher const char *mmhub_cid; 533e3898719SChristian König const char *hub_name; 534e3898719SChristian König u64 addr; 535e60f8db5SAlex Xie 536e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 537e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 538e60f8db5SAlex Xie 5390291150dSChristian König if (retry_fault) { 5400291150dSChristian König /* Returning 1 here also prevents sending the IV to the KFD */ 54122666cc1SChristian König 5420291150dSChristian König /* Process it onyl if it's the first fault for this address */ 5430291150dSChristian König if (entry->ih != &adev->irq.ih_soft && 5443c2d6ea2SPhilip Yang amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, 5450291150dSChristian König entry->timestamp)) 5460291150dSChristian König return 1; 5470291150dSChristian König 5480291150dSChristian König /* Delegate it to a different ring if the hardware hasn't 5490291150dSChristian König * already done it. 5500291150dSChristian König */ 55158df0d71SSebastian Andrzej Siewior if (entry->ih == &adev->irq.ih) { 5520291150dSChristian König amdgpu_irq_delegate(adev, entry, 8); 5530291150dSChristian König return 1; 5540291150dSChristian König } 5550291150dSChristian König 5560291150dSChristian König /* Try to handle the recoverable page faults by filling page 5570291150dSChristian König * tables 5580291150dSChristian König */ 559ff891a2eSPhilip Yang if (amdgpu_vm_handle_fault(adev, entry->pasid, addr, write_fault)) 5600291150dSChristian König return 1; 5610291150dSChristian König } 562ec671737SChristian König 563e3898719SChristian König if (!printk_ratelimit()) 564e3898719SChristian König return 0; 56553499173SXiaojie Yuan 566e3898719SChristian König if (entry->client_id == SOC15_IH_CLIENTID_VMC) { 567e3898719SChristian König hub_name = "mmhub0"; 568e3898719SChristian König hub = &adev->vmhub[AMDGPU_MMHUB_0]; 569e3898719SChristian König } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { 570e3898719SChristian König hub_name = "mmhub1"; 571e3898719SChristian König hub = &adev->vmhub[AMDGPU_MMHUB_1]; 572e3898719SChristian König } else { 573e3898719SChristian König hub_name = "gfxhub0"; 574e3898719SChristian König hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 5754d6cbde3SFelix Kuehling } 576e60f8db5SAlex Xie 57705794effSShirish S memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 578efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 579efaa9646SAndrey Grodzovsky 5804d6cbde3SFelix Kuehling dev_err(adev->dev, 581c468f9e2SChristian König "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 582c468f9e2SChristian König "pasid:%u, for process %s pid %d thread %s pid %d)\n", 58351c60898SLe Ma hub_name, retry_fault ? "retry" : "no-retry", 584c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 585efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 586efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 587be14729aSYong Zhao dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n", 588be14729aSYong Zhao addr, entry->client_id, 589be14729aSYong Zhao soc15_ih_clientid_name[entry->client_id]); 590e3898719SChristian König 591e3898719SChristian König if (amdgpu_sriov_vf(adev)) 592e3898719SChristian König return 0; 593e3898719SChristian König 594e3898719SChristian König /* 595e3898719SChristian König * Issue a dummy read to wait for the status register to 596e3898719SChristian König * be updated to avoid reading an incorrect value due to 597e3898719SChristian König * the new fast GRBM interface. 598e3898719SChristian König */ 5997845d80dSAlex Deucher if ((entry->vmid_src == AMDGPU_GFXHUB_0) && 600630e959fSAlex Deucher (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2))) 601e3898719SChristian König RREG32(hub->vm_l2_pro_fault_status); 602e3898719SChristian König 603e3898719SChristian König status = RREG32(hub->vm_l2_pro_fault_status); 604e3898719SChristian König cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID); 605e3898719SChristian König rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW); 606e3898719SChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 607e3898719SChristian König 608e3898719SChristian König 6094d6cbde3SFelix Kuehling dev_err(adev->dev, 6104d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 6114d6cbde3SFelix Kuehling status); 61202f23f5fSAlex Deucher if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) { 613be99ecbfSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 614e3898719SChristian König cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : 615e3898719SChristian König gfxhub_client_ids[cid], 616be99ecbfSAlex Deucher cid); 61702f23f5fSAlex Deucher } else { 618630e959fSAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 619630e959fSAlex Deucher case IP_VERSION(9, 0, 0): 62002f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega10[cid][rw]; 62102f23f5fSAlex Deucher break; 622630e959fSAlex Deucher case IP_VERSION(9, 3, 0): 62302f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega12[cid][rw]; 62402f23f5fSAlex Deucher break; 625630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 62602f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega20[cid][rw]; 62702f23f5fSAlex Deucher break; 628630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 62902f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; 63002f23f5fSAlex Deucher break; 631630e959fSAlex Deucher case IP_VERSION(9, 1, 0): 632630e959fSAlex Deucher case IP_VERSION(9, 2, 0): 63302f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_raven[cid][rw]; 63402f23f5fSAlex Deucher break; 635630e959fSAlex Deucher case IP_VERSION(1, 5, 0): 636630e959fSAlex Deucher case IP_VERSION(2, 4, 0): 63702f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_renoir[cid][rw]; 63802f23f5fSAlex Deucher break; 639630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 640e844cd99SAlex Deucher mmhub_cid = mmhub_client_ids_aldebaran[cid][rw]; 641e844cd99SAlex Deucher break; 64202f23f5fSAlex Deucher default: 64302f23f5fSAlex Deucher mmhub_cid = NULL; 64402f23f5fSAlex Deucher break; 64502f23f5fSAlex Deucher } 64602f23f5fSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 64702f23f5fSAlex Deucher mmhub_cid ? mmhub_cid : "unknown", cid); 64802f23f5fSAlex Deucher } 6495ddd4a9aSYong Zhao dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 6505ddd4a9aSYong Zhao REG_GET_FIELD(status, 6515ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 6525ddd4a9aSYong Zhao dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 6535ddd4a9aSYong Zhao REG_GET_FIELD(status, 6545ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 6555ddd4a9aSYong Zhao dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 6565ddd4a9aSYong Zhao REG_GET_FIELD(status, 6575ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 6585ddd4a9aSYong Zhao dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 6595ddd4a9aSYong Zhao REG_GET_FIELD(status, 6605ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 66102f23f5fSAlex Deucher dev_err(adev->dev, "\t RW: 0x%x\n", rw); 662e60f8db5SAlex Xie return 0; 663e60f8db5SAlex Xie } 664e60f8db5SAlex Xie 665e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 666e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 667e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 668e60f8db5SAlex Xie }; 669e60f8db5SAlex Xie 670791c4769Sxinhui pan 671791c4769Sxinhui pan static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 672791c4769Sxinhui pan .set = gmc_v9_0_ecc_interrupt_state, 67334cc4fd9STao Zhou .process = amdgpu_umc_process_ecc_irq, 674791c4769Sxinhui pan }; 675791c4769Sxinhui pan 676e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 677e60f8db5SAlex Xie { 678770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 679770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 680791c4769Sxinhui pan 68168d705ddSHawking Zhang if (!amdgpu_sriov_vf(adev) && 68268d705ddSHawking Zhang !adev->gmc.xgmi.connected_to_cpu) { 683791c4769Sxinhui pan adev->gmc.ecc_irq.num_types = 1; 684791c4769Sxinhui pan adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 685e60f8db5SAlex Xie } 6862ee9403eSZhigang Luo } 687e60f8db5SAlex Xie 6882a79d868SYong Zhao static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 6892a79d868SYong Zhao uint32_t flush_type) 69003f89febSChristian König { 69103f89febSChristian König u32 req = 0; 69203f89febSChristian König 69303f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 694c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 6952a79d868SYong Zhao req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 69603f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 69703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 69803f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 69903f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 70003f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 70103f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 70203f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 70303f89febSChristian König 70403f89febSChristian König return req; 70503f89febSChristian König } 70603f89febSChristian König 70790f6452cSchangzhu /** 70890f6452cSchangzhu * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore 70990f6452cSchangzhu * 71090f6452cSchangzhu * @adev: amdgpu_device pointer 71190f6452cSchangzhu * @vmhub: vmhub type 71290f6452cSchangzhu * 71390f6452cSchangzhu */ 71490f6452cSchangzhu static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, 71590f6452cSchangzhu uint32_t vmhub) 71690f6452cSchangzhu { 717630e959fSAlex Deucher if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 718d477c5aaSHawking Zhang return false; 719d477c5aaSHawking Zhang 72090f6452cSchangzhu return ((vmhub == AMDGPU_MMHUB_0 || 72190f6452cSchangzhu vmhub == AMDGPU_MMHUB_1) && 72290f6452cSchangzhu (!amdgpu_sriov_vf(adev)) && 72354f78a76SAlex Deucher (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && 72454f78a76SAlex Deucher (adev->apu_flags & AMD_APU_IS_PICASSO)))); 72590f6452cSchangzhu } 72690f6452cSchangzhu 727ea930000SAlex Sierra static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, 728ea930000SAlex Sierra uint8_t vmid, uint16_t *p_pasid) 729ea930000SAlex Sierra { 730ea930000SAlex Sierra uint32_t value; 731ea930000SAlex Sierra 732ea930000SAlex Sierra value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 733ea930000SAlex Sierra + vmid); 734ea930000SAlex Sierra *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 735ea930000SAlex Sierra 736ea930000SAlex Sierra return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 737ea930000SAlex Sierra } 738ea930000SAlex Sierra 739e60f8db5SAlex Xie /* 740e60f8db5SAlex Xie * GART 741e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 742e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 743e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 744e60f8db5SAlex Xie */ 745e60f8db5SAlex Xie 746e60f8db5SAlex Xie /** 7472a79d868SYong Zhao * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 748e60f8db5SAlex Xie * 749e60f8db5SAlex Xie * @adev: amdgpu_device pointer 750e60f8db5SAlex Xie * @vmid: vm instance to flush 751bf0df09cSLee Jones * @vmhub: which hub to flush 7522a79d868SYong Zhao * @flush_type: the flush type 753e60f8db5SAlex Xie * 7542a79d868SYong Zhao * Flush the TLB for the requested page table using certain type. 755e60f8db5SAlex Xie */ 7563ff98548SOak Zeng static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 7573ff98548SOak Zeng uint32_t vmhub, uint32_t flush_type) 758e60f8db5SAlex Xie { 75990f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); 760e60f8db5SAlex Xie const unsigned eng = 17; 761b80cd524SFelix Kuehling u32 j, inv_req, inv_req2, tmp; 7623ff98548SOak Zeng struct amdgpu_vmhub *hub; 763e60f8db5SAlex Xie 7643ff98548SOak Zeng BUG_ON(vmhub >= adev->num_vmhubs); 7653ff98548SOak Zeng 7663ff98548SOak Zeng hub = &adev->vmhub[vmhub]; 767b80cd524SFelix Kuehling if (adev->gmc.xgmi.num_physical_nodes && 768630e959fSAlex Deucher adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0)) { 769b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 770b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 771b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 772b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 773b80cd524SFelix Kuehling * still need a second TLB flush after this. 774b80cd524SFelix Kuehling */ 775b80cd524SFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); 776b80cd524SFelix Kuehling inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 777b80cd524SFelix Kuehling } else { 77837c58ddfSFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); 779b80cd524SFelix Kuehling inv_req2 = 0; 780b80cd524SFelix Kuehling } 781e60f8db5SAlex Xie 78282d1a1b1SChengming Gui /* This is necessary for a HW workaround under SRIOV as well 78382d1a1b1SChengming Gui * as GFXOFF under bare metal 78482d1a1b1SChengming Gui */ 78582d1a1b1SChengming Gui if (adev->gfx.kiq.ring.sched.ready && 78682d1a1b1SChengming Gui (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 78781202807SDennis Li down_read_trylock(&adev->reset_sem)) { 788148f597dSHuang Rui uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 789148f597dSHuang Rui uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 790af5fe1e9SChristian König 79137c58ddfSFelix Kuehling amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 792af5fe1e9SChristian König 1 << vmid); 79381202807SDennis Li up_read(&adev->reset_sem); 7943ff98548SOak Zeng return; 795fc0faf04SEmily Deng } 7963890d111SEmily Deng 7973890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 798f920d1bbSchangzhu 799f920d1bbSchangzhu /* 800f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 801f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 802f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 803f920d1bbSchangzhu * to WA the Issue 804f920d1bbSchangzhu */ 805f920d1bbSchangzhu 806f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 80790f6452cSchangzhu if (use_semaphore) { 808f920d1bbSchangzhu for (j = 0; j < adev->usec_timeout; j++) { 809*92f153bbSVictor Skvortsov /* a read return value of 1 means semaphore acquire */ 810*92f153bbSVictor Skvortsov if (vmhub == AMDGPU_GFXHUB_0) 811*92f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng); 812*92f153bbSVictor Skvortsov else 813*92f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng); 814*92f153bbSVictor Skvortsov 815f920d1bbSchangzhu if (tmp & 0x1) 816f920d1bbSchangzhu break; 817f920d1bbSchangzhu udelay(1); 818f920d1bbSchangzhu } 819f920d1bbSchangzhu 820f920d1bbSchangzhu if (j >= adev->usec_timeout) 821f920d1bbSchangzhu DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 822f920d1bbSchangzhu } 823f920d1bbSchangzhu 824b80cd524SFelix Kuehling do { 825*92f153bbSVictor Skvortsov if (vmhub == AMDGPU_GFXHUB_0) 826*92f153bbSVictor Skvortsov WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req); 827*92f153bbSVictor Skvortsov else 828*92f153bbSVictor Skvortsov WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req); 82953499173SXiaojie Yuan 83053499173SXiaojie Yuan /* 831b80cd524SFelix Kuehling * Issue a dummy read to wait for the ACK register to 832b80cd524SFelix Kuehling * be cleared to avoid a false ACK due to the new fast 833b80cd524SFelix Kuehling * GRBM interface. 83453499173SXiaojie Yuan */ 8357845d80dSAlex Deucher if ((vmhub == AMDGPU_GFXHUB_0) && 836630e959fSAlex Deucher (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2))) 837148f597dSHuang Rui RREG32_NO_KIQ(hub->vm_inv_eng0_req + 838148f597dSHuang Rui hub->eng_distance * eng); 83953499173SXiaojie Yuan 840e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 841*92f153bbSVictor Skvortsov if (vmhub == AMDGPU_GFXHUB_0) 842*92f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_ack + hub->eng_distance * eng); 843*92f153bbSVictor Skvortsov else 844*92f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_ack + hub->eng_distance * eng); 845*92f153bbSVictor Skvortsov 846396557b0SChristian König if (tmp & (1 << vmid)) 847e60f8db5SAlex Xie break; 848e60f8db5SAlex Xie udelay(1); 849e60f8db5SAlex Xie } 850f920d1bbSchangzhu 851b80cd524SFelix Kuehling inv_req = inv_req2; 852b80cd524SFelix Kuehling inv_req2 = 0; 853b80cd524SFelix Kuehling } while (inv_req); 854b80cd524SFelix Kuehling 855f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 856*92f153bbSVictor Skvortsov if (use_semaphore) { 857f920d1bbSchangzhu /* 858f920d1bbSchangzhu * add semaphore release after invalidation, 859f920d1bbSchangzhu * write with 0 means semaphore release 860f920d1bbSchangzhu */ 861*92f153bbSVictor Skvortsov if (vmhub == AMDGPU_GFXHUB_0) 862*92f153bbSVictor Skvortsov WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0); 863*92f153bbSVictor Skvortsov else 864*92f153bbSVictor Skvortsov WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0); 865*92f153bbSVictor Skvortsov } 866f920d1bbSchangzhu 8673890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 868f920d1bbSchangzhu 869396557b0SChristian König if (j < adev->usec_timeout) 8703ff98548SOak Zeng return; 871396557b0SChristian König 872e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 873e60f8db5SAlex Xie } 874e60f8db5SAlex Xie 875ea930000SAlex Sierra /** 876ea930000SAlex Sierra * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid 877ea930000SAlex Sierra * 878ea930000SAlex Sierra * @adev: amdgpu_device pointer 879ea930000SAlex Sierra * @pasid: pasid to be flush 880bf0df09cSLee Jones * @flush_type: the flush type 881bf0df09cSLee Jones * @all_hub: flush all hubs 882ea930000SAlex Sierra * 883ea930000SAlex Sierra * Flush the TLB for the requested pasid. 884ea930000SAlex Sierra */ 885ea930000SAlex Sierra static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 886ea930000SAlex Sierra uint16_t pasid, uint32_t flush_type, 887ea930000SAlex Sierra bool all_hub) 888ea930000SAlex Sierra { 889ea930000SAlex Sierra int vmid, i; 890ea930000SAlex Sierra signed long r; 891ea930000SAlex Sierra uint32_t seq; 892ea930000SAlex Sierra uint16_t queried_pasid; 893ea930000SAlex Sierra bool ret; 894ea930000SAlex Sierra struct amdgpu_ring *ring = &adev->gfx.kiq.ring; 895ea930000SAlex Sierra struct amdgpu_kiq *kiq = &adev->gfx.kiq; 896ea930000SAlex Sierra 89753b3f8f4SDennis Li if (amdgpu_in_reset(adev)) 898ea930000SAlex Sierra return -EIO; 899ea930000SAlex Sierra 90081202807SDennis Li if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) { 901b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 902b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 903b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 904b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 905b80cd524SFelix Kuehling * still need a second TLB flush after this. 906b80cd524SFelix Kuehling */ 907b80cd524SFelix Kuehling bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && 908630e959fSAlex Deucher adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0)); 909b80cd524SFelix Kuehling /* 2 dwords flush + 8 dwords fence */ 910b80cd524SFelix Kuehling unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; 911b80cd524SFelix Kuehling 912b80cd524SFelix Kuehling if (vega20_xgmi_wa) 913b80cd524SFelix Kuehling ndw += kiq->pmf->invalidate_tlbs_size; 914b80cd524SFelix Kuehling 915ea930000SAlex Sierra spin_lock(&adev->gfx.kiq.ring_lock); 91636a1707aSAlex Sierra /* 2 dwords flush + 8 dwords fence */ 917b80cd524SFelix Kuehling amdgpu_ring_alloc(ring, ndw); 918b80cd524SFelix Kuehling if (vega20_xgmi_wa) 919b80cd524SFelix Kuehling kiq->pmf->kiq_invalidate_tlbs(ring, 920b80cd524SFelix Kuehling pasid, 2, all_hub); 921ea930000SAlex Sierra kiq->pmf->kiq_invalidate_tlbs(ring, 922ea930000SAlex Sierra pasid, flush_type, all_hub); 92304e4e2e9SYintian Tao r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 92404e4e2e9SYintian Tao if (r) { 92504e4e2e9SYintian Tao amdgpu_ring_undo(ring); 926abb17b1eSColin Ian King spin_unlock(&adev->gfx.kiq.ring_lock); 92781202807SDennis Li up_read(&adev->reset_sem); 92804e4e2e9SYintian Tao return -ETIME; 92904e4e2e9SYintian Tao } 93004e4e2e9SYintian Tao 931ea930000SAlex Sierra amdgpu_ring_commit(ring); 932ea930000SAlex Sierra spin_unlock(&adev->gfx.kiq.ring_lock); 933ea930000SAlex Sierra r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); 934ea930000SAlex Sierra if (r < 1) { 935aac89168SDennis Li dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); 93681202807SDennis Li up_read(&adev->reset_sem); 937ea930000SAlex Sierra return -ETIME; 938ea930000SAlex Sierra } 93981202807SDennis Li up_read(&adev->reset_sem); 940ea930000SAlex Sierra return 0; 941ea930000SAlex Sierra } 942ea930000SAlex Sierra 943ea930000SAlex Sierra for (vmid = 1; vmid < 16; vmid++) { 944ea930000SAlex Sierra 945ea930000SAlex Sierra ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 946ea930000SAlex Sierra &queried_pasid); 947ea930000SAlex Sierra if (ret && queried_pasid == pasid) { 948ea930000SAlex Sierra if (all_hub) { 949ea930000SAlex Sierra for (i = 0; i < adev->num_vmhubs; i++) 950ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 951fa34edbeSFelix Kuehling i, flush_type); 952ea930000SAlex Sierra } else { 953ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 954fa34edbeSFelix Kuehling AMDGPU_GFXHUB_0, flush_type); 955ea930000SAlex Sierra } 956ea930000SAlex Sierra break; 957ea930000SAlex Sierra } 958ea930000SAlex Sierra } 959ea930000SAlex Sierra 960ea930000SAlex Sierra return 0; 961ea930000SAlex Sierra 962ea930000SAlex Sierra } 963ea930000SAlex Sierra 9649096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 965c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 9669096d6e5SChristian König { 96790f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); 968250b4228SChristian König struct amdgpu_device *adev = ring->adev; 969250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 9702a79d868SYong Zhao uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 9719096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 9729096d6e5SChristian König 973f920d1bbSchangzhu /* 974f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 975f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 976f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 977f920d1bbSchangzhu * to WA the Issue 978f920d1bbSchangzhu */ 979f920d1bbSchangzhu 980f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 98190f6452cSchangzhu if (use_semaphore) 982f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 983f920d1bbSchangzhu amdgpu_ring_emit_reg_wait(ring, 984148f597dSHuang Rui hub->vm_inv_eng0_sem + 985148f597dSHuang Rui hub->eng_distance * eng, 0x1, 0x1); 986f920d1bbSchangzhu 987148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 988148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 9899096d6e5SChristian König lower_32_bits(pd_addr)); 9909096d6e5SChristian König 991148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 992148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 9939096d6e5SChristian König upper_32_bits(pd_addr)); 9949096d6e5SChristian König 995148f597dSHuang Rui amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 996148f597dSHuang Rui hub->eng_distance * eng, 997148f597dSHuang Rui hub->vm_inv_eng0_ack + 998148f597dSHuang Rui hub->eng_distance * eng, 999f8bc9037SAlex Deucher req, 1 << vmid); 1000f732b6b3SChristian König 1001f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 100290f6452cSchangzhu if (use_semaphore) 1003f920d1bbSchangzhu /* 1004f920d1bbSchangzhu * add semaphore release after invalidation, 1005f920d1bbSchangzhu * write with 0 means semaphore release 1006f920d1bbSchangzhu */ 1007148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 1008148f597dSHuang Rui hub->eng_distance * eng, 0); 1009f920d1bbSchangzhu 10109096d6e5SChristian König return pd_addr; 10119096d6e5SChristian König } 10129096d6e5SChristian König 1013c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 1014c633c00bSChristian König unsigned pasid) 1015c633c00bSChristian König { 1016c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 1017c633c00bSChristian König uint32_t reg; 1018c633c00bSChristian König 1019f2d66571SLe Ma /* Do nothing because there's no lut register for mmhub1. */ 1020f2d66571SLe Ma if (ring->funcs->vmhub == AMDGPU_MMHUB_1) 1021f2d66571SLe Ma return; 1022f2d66571SLe Ma 1023a2d15ed7SLe Ma if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) 1024c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 1025c633c00bSChristian König else 1026c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 1027c633c00bSChristian König 1028c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 1029c633c00bSChristian König } 1030c633c00bSChristian König 1031e60f8db5SAlex Xie /* 1032e60f8db5SAlex Xie * PTE format on VEGA 10: 1033e60f8db5SAlex Xie * 63:59 reserved 1034e60f8db5SAlex Xie * 58:57 mtype 1035e60f8db5SAlex Xie * 56 F 1036e60f8db5SAlex Xie * 55 L 1037e60f8db5SAlex Xie * 54 P 1038e60f8db5SAlex Xie * 53 SW 1039e60f8db5SAlex Xie * 52 T 1040e60f8db5SAlex Xie * 50:48 reserved 1041e60f8db5SAlex Xie * 47:12 4k physical page base address 1042e60f8db5SAlex Xie * 11:7 fragment 1043e60f8db5SAlex Xie * 6 write 1044e60f8db5SAlex Xie * 5 read 1045e60f8db5SAlex Xie * 4 exe 1046e60f8db5SAlex Xie * 3 Z 1047e60f8db5SAlex Xie * 2 snooped 1048e60f8db5SAlex Xie * 1 system 1049e60f8db5SAlex Xie * 0 valid 1050e60f8db5SAlex Xie * 1051e60f8db5SAlex Xie * PDE format on VEGA 10: 1052e60f8db5SAlex Xie * 63:59 block fragment size 1053e60f8db5SAlex Xie * 58:55 reserved 1054e60f8db5SAlex Xie * 54 P 1055e60f8db5SAlex Xie * 53:48 reserved 1056e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 1057e60f8db5SAlex Xie * 5:3 reserved 1058e60f8db5SAlex Xie * 2 C 1059e60f8db5SAlex Xie * 1 system 1060e60f8db5SAlex Xie * 0 valid 1061e60f8db5SAlex Xie */ 1062e60f8db5SAlex Xie 106371776b6dSChristian König static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) 1064e60f8db5SAlex Xie 1065e60f8db5SAlex Xie { 106671776b6dSChristian König switch (flags) { 1067e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 106871776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1069e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 107071776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1071e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 107271776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); 1073093e48c0SOak Zeng case AMDGPU_VM_MTYPE_RW: 107471776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); 1075e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 107671776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); 1077e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 107871776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); 1079e60f8db5SAlex Xie default: 108071776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1081e60f8db5SAlex Xie } 1082e60f8db5SAlex Xie } 1083e60f8db5SAlex Xie 10843de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 10853de676d8SChristian König uint64_t *addr, uint64_t *flags) 1086f75e237cSChristian König { 1087bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 10880ca565abSOak Zeng *addr = amdgpu_gmc_vram_mc2pa(adev, *addr); 10893de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 10906a42fd6fSChristian König 1091770d13b1SChristian König if (!adev->gmc.translate_further) 10926a42fd6fSChristian König return; 10936a42fd6fSChristian König 10946a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 10956a42fd6fSChristian König /* Set the block fragment size */ 10966a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 10976a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 10986a42fd6fSChristian König 10996a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 11006a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 11016a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 11026a42fd6fSChristian König else 11036a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 11046a42fd6fSChristian König } 1105f75e237cSChristian König } 1106f75e237cSChristian König 1107cbfae36cSChristian König static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, 1108cbfae36cSChristian König struct amdgpu_bo_va_mapping *mapping, 1109cbfae36cSChristian König uint64_t *flags) 1110cbfae36cSChristian König { 1111cbfae36cSChristian König *flags &= ~AMDGPU_PTE_EXECUTABLE; 1112cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1113cbfae36cSChristian König 1114cbfae36cSChristian König *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1115cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; 1116cbfae36cSChristian König 1117cbfae36cSChristian König if (mapping->flags & AMDGPU_PTE_PRT) { 1118cbfae36cSChristian König *flags |= AMDGPU_PTE_PRT; 1119cbfae36cSChristian König *flags &= ~AMDGPU_PTE_VALID; 1120cbfae36cSChristian König } 1121cbfae36cSChristian König 1122630e959fSAlex Deucher if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || 1123630e959fSAlex Deucher adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) && 1124cbfae36cSChristian König !(*flags & AMDGPU_PTE_SYSTEM) && 1125cbfae36cSChristian König mapping->bo_va->is_xgmi) 1126cbfae36cSChristian König *flags |= AMDGPU_PTE_SNOOPED; 112772b4db0fSEric Huang 1128630e959fSAlex Deucher if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 112972b4db0fSEric Huang *flags |= mapping->flags & AMDGPU_PTE_SNOOPED; 1130cbfae36cSChristian König } 1131cbfae36cSChristian König 11327b885f0eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 11337b885f0eSAlex Deucher { 11347b885f0eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 11357b885f0eSAlex Deucher unsigned size; 11367b885f0eSAlex Deucher 11377b885f0eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 11387b885f0eSAlex Deucher size = AMDGPU_VBIOS_VGA_ALLOCATION; 11397b885f0eSAlex Deucher } else { 11407b885f0eSAlex Deucher u32 viewport; 11417b885f0eSAlex Deucher 1142630e959fSAlex Deucher switch (adev->ip_versions[DCE_HWIP][0]) { 1143630e959fSAlex Deucher case IP_VERSION(1, 0, 0): 1144630e959fSAlex Deucher case IP_VERSION(1, 0, 1): 1145630e959fSAlex Deucher case IP_VERSION(2, 1, 0): 11467b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 11477b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, 11487b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 11497b885f0eSAlex Deucher REG_GET_FIELD(viewport, 11507b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 11517b885f0eSAlex Deucher 4); 11527b885f0eSAlex Deucher break; 11537b885f0eSAlex Deucher default: 11547b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 11557b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 11567b885f0eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 11577b885f0eSAlex Deucher 4); 11587b885f0eSAlex Deucher break; 11597b885f0eSAlex Deucher } 11607b885f0eSAlex Deucher } 11617b885f0eSAlex Deucher 11627b885f0eSAlex Deucher return size; 11637b885f0eSAlex Deucher } 11647b885f0eSAlex Deucher 1165132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 1166132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 1167ea930000SAlex Sierra .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, 11689096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 1169c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 117071776b6dSChristian König .map_mtype = gmc_v9_0_map_mtype, 1171cbfae36cSChristian König .get_vm_pde = gmc_v9_0_get_vm_pde, 11727b885f0eSAlex Deucher .get_vm_pte = gmc_v9_0_get_vm_pte, 11737b885f0eSAlex Deucher .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, 1174e60f8db5SAlex Xie }; 1175e60f8db5SAlex Xie 1176132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 1177e60f8db5SAlex Xie { 1178132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 1179e60f8db5SAlex Xie } 1180e60f8db5SAlex Xie 11815b6b35aaSHawking Zhang static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) 11825b6b35aaSHawking Zhang { 1183630e959fSAlex Deucher switch (adev->ip_versions[UMC_HWIP][0]) { 1184630e959fSAlex Deucher case IP_VERSION(6, 0, 0): 1185e7da754bSMonk Liu adev->umc.funcs = &umc_v6_0_funcs; 1186e7da754bSMonk Liu break; 1187630e959fSAlex Deucher case IP_VERSION(6, 1, 1): 11883aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 11893aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 11903aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 11914cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; 11924cf781c2SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 119349070c4eSHawking Zhang adev->umc.ras_funcs = &umc_v6_1_ras_funcs; 11944cf781c2SJohn Clements break; 1195630e959fSAlex Deucher case IP_VERSION(6, 1, 2): 11963aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 11973aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 11983aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 11994cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; 12003aacf4eaSTao Zhou adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 120149070c4eSHawking Zhang adev->umc.ras_funcs = &umc_v6_1_ras_funcs; 12025b6b35aaSHawking Zhang break; 1203630e959fSAlex Deucher case IP_VERSION(6, 7, 0): 1204186c8a85SJohn Clements adev->umc.max_ras_err_cnt_per_query = UMC_V6_7_TOTAL_CHANNEL_NUM; 1205719e433eSMukul Joshi adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM; 1206719e433eSMukul Joshi adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM; 1207186c8a85SJohn Clements adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET; 1208186c8a85SJohn Clements if (!adev->gmc.xgmi.connected_to_cpu) 1209186c8a85SJohn Clements adev->umc.ras_funcs = &umc_v6_7_ras_funcs; 1210186c8a85SJohn Clements if (1 & adev->smuio.funcs->get_die_id(adev)) 1211186c8a85SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0]; 1212186c8a85SJohn Clements else 1213186c8a85SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0]; 1214186c8a85SJohn Clements break; 12155b6b35aaSHawking Zhang default: 12165b6b35aaSHawking Zhang break; 12175b6b35aaSHawking Zhang } 12185b6b35aaSHawking Zhang } 12195b6b35aaSHawking Zhang 12203d093da0STao Zhou static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) 12213d093da0STao Zhou { 1222630e959fSAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 1223630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 1224f6c3623bSDennis Li adev->mmhub.funcs = &mmhub_v9_4_funcs; 1225f6c3623bSDennis Li break; 1226630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 12274da999cdSOak Zeng adev->mmhub.funcs = &mmhub_v1_7_funcs; 12284da999cdSOak Zeng break; 12293d093da0STao Zhou default: 12309fb1506eSOak Zeng adev->mmhub.funcs = &mmhub_v1_0_funcs; 12313d093da0STao Zhou break; 12323d093da0STao Zhou } 12333d093da0STao Zhou } 12343d093da0STao Zhou 1235d844c6d7SHawking Zhang static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev) 1236d844c6d7SHawking Zhang { 1237630e959fSAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 1238630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 1239d844c6d7SHawking Zhang adev->mmhub.ras_funcs = &mmhub_v1_0_ras_funcs; 1240d844c6d7SHawking Zhang break; 1241630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 1242d844c6d7SHawking Zhang adev->mmhub.ras_funcs = &mmhub_v9_4_ras_funcs; 1243d844c6d7SHawking Zhang break; 1244630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 1245d844c6d7SHawking Zhang adev->mmhub.ras_funcs = &mmhub_v1_7_ras_funcs; 1246d844c6d7SHawking Zhang break; 1247d844c6d7SHawking Zhang default: 1248d844c6d7SHawking Zhang /* mmhub ras is not available */ 1249d844c6d7SHawking Zhang break; 1250d844c6d7SHawking Zhang } 1251d844c6d7SHawking Zhang } 1252d844c6d7SHawking Zhang 12538ffff9b4SOak Zeng static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev) 12548ffff9b4SOak Zeng { 12558ffff9b4SOak Zeng adev->gfxhub.funcs = &gfxhub_v1_0_funcs; 12568ffff9b4SOak Zeng } 12578ffff9b4SOak Zeng 12586f12507fSHawking Zhang static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev) 12596f12507fSHawking Zhang { 12606f12507fSHawking Zhang adev->hdp.ras_funcs = &hdp_v4_0_ras_funcs; 12616f12507fSHawking Zhang } 12626f12507fSHawking Zhang 12633907c492SJohn Clements static void gmc_v9_0_set_mca_funcs(struct amdgpu_device *adev) 12643907c492SJohn Clements { 1265630e959fSAlex Deucher /* is UMC the right IP to check for MCA? Maybe DF? */ 1266630e959fSAlex Deucher switch (adev->ip_versions[UMC_HWIP][0]) { 1267630e959fSAlex Deucher case IP_VERSION(6, 7, 0): 12683907c492SJohn Clements if (!adev->gmc.xgmi.connected_to_cpu) 12693907c492SJohn Clements adev->mca.funcs = &mca_v3_0_funcs; 12703907c492SJohn Clements break; 12713907c492SJohn Clements default: 12723907c492SJohn Clements break; 12733907c492SJohn Clements } 12743907c492SJohn Clements } 12753907c492SJohn Clements 1276e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 1277e60f8db5SAlex Xie { 1278e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1279e60f8db5SAlex Xie 1280630e959fSAlex Deucher /* ARCT and VEGA20 don't have XGMI defined in their IP discovery tables */ 128131691b8dSRajneesh Bhardwaj if (adev->asic_type == CHIP_VEGA20 || 128231691b8dSRajneesh Bhardwaj adev->asic_type == CHIP_ARCTURUS) 128331691b8dSRajneesh Bhardwaj adev->gmc.xgmi.supported = true; 128431691b8dSRajneesh Bhardwaj 1285630e959fSAlex Deucher if (adev->ip_versions[XGMI_HWIP][0] == IP_VERSION(6, 1, 0)) { 128631691b8dSRajneesh Bhardwaj adev->gmc.xgmi.supported = true; 128731691b8dSRajneesh Bhardwaj adev->gmc.xgmi.connected_to_cpu = 128831691b8dSRajneesh Bhardwaj adev->smuio.funcs->is_host_gpu_xgmi_supported(adev); 128931691b8dSRajneesh Bhardwaj } 129031691b8dSRajneesh Bhardwaj 129149070c4eSHawking Zhang gmc_v9_0_set_gmc_funcs(adev); 129249070c4eSHawking Zhang gmc_v9_0_set_irq_funcs(adev); 129349070c4eSHawking Zhang gmc_v9_0_set_umc_funcs(adev); 129449070c4eSHawking Zhang gmc_v9_0_set_mmhub_funcs(adev); 1295d844c6d7SHawking Zhang gmc_v9_0_set_mmhub_ras_funcs(adev); 129649070c4eSHawking Zhang gmc_v9_0_set_gfxhub_funcs(adev); 12976f12507fSHawking Zhang gmc_v9_0_set_hdp_ras_funcs(adev); 12983907c492SJohn Clements gmc_v9_0_set_mca_funcs(adev); 129949070c4eSHawking Zhang 1300770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 1301770d13b1SChristian König adev->gmc.shared_aperture_end = 1302770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 1303bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 1304770d13b1SChristian König adev->gmc.private_aperture_end = 1305770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 1306a7ea6548SAlex Deucher 1307e60f8db5SAlex Xie return 0; 1308e60f8db5SAlex Xie } 1309e60f8db5SAlex Xie 1310e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 1311e60f8db5SAlex Xie { 1312e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1313c5b6e585STao Zhou int r; 13144789c463SChristian König 1315bdbe90f0SAlex Deucher r = amdgpu_gmc_allocate_vm_inv_eng(adev); 1316c713a461SEvan Quan if (r) 1317c713a461SEvan Quan return r; 13184a20300bSGuchun Chen 13194a20300bSGuchun Chen /* 13204a20300bSGuchun Chen * Workaround performance drop issue with VBIOS enables partial 13214a20300bSGuchun Chen * writes, while disables HBM ECC for vega10. 13224a20300bSGuchun Chen */ 1323630e959fSAlex Deucher if (!amdgpu_sriov_vf(adev) && 1324630e959fSAlex Deucher (adev->ip_versions[UMC_HWIP][0] == IP_VERSION(6, 0, 0))) { 13258ab0d6f0SLuben Tuikov if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) { 1326cace4bffSHawking Zhang if (adev->df.funcs && 1327cace4bffSHawking Zhang adev->df.funcs->enable_ecc_force_par_wr_rmw) 1328bdf84a80SJoseph Greathouse adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); 13294a20300bSGuchun Chen } 1330f49ea9f8SHawking Zhang } 133102bab923SDavid Panariti 13328f6368a9SJohn Clements if (!amdgpu_persistent_edc_harvesting_supported(adev)) { 13338bc7b360SHawking Zhang if (adev->mmhub.ras_funcs && 13348bc7b360SHawking Zhang adev->mmhub.ras_funcs->reset_ras_error_count) 13358bc7b360SHawking Zhang adev->mmhub.ras_funcs->reset_ras_error_count(adev); 1336fe5211f1SHawking Zhang 133778871b6cSHawking Zhang if (adev->hdp.ras_funcs && 133878871b6cSHawking Zhang adev->hdp.ras_funcs->reset_ras_error_count) 133978871b6cSHawking Zhang adev->hdp.ras_funcs->reset_ras_error_count(adev); 13408f6368a9SJohn Clements } 134178871b6cSHawking Zhang 1342ba083492STao Zhou r = amdgpu_gmc_ras_late_init(adev); 1343791c4769Sxinhui pan if (r) 1344e60f8db5SAlex Xie return r; 1345e60f8db5SAlex Xie 1346770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 1347e60f8db5SAlex Xie } 1348e60f8db5SAlex Xie 1349e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 1350770d13b1SChristian König struct amdgpu_gmc *mc) 1351e60f8db5SAlex Xie { 1352adbe2e3dSZhigang Luo u64 base = adev->mmhub.funcs->get_fb_location(adev); 13539d4f837aSFrank.Min 13546fdd68b1SAlex Deucher /* add the xgmi offset of the physical node */ 13556fdd68b1SAlex Deucher base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1356f527f310SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 1357f527f310SOak Zeng amdgpu_gmc_sysvm_location(adev, mc); 1358f527f310SOak Zeng } else { 135983afe835SOak Zeng amdgpu_gmc_vram_location(adev, mc, base); 1360961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 1361c3e1b43cSChristian König amdgpu_gmc_agp_location(adev, mc); 1362f527f310SOak Zeng } 1363e60f8db5SAlex Xie /* base offset of vram pages */ 13648ffff9b4SOak Zeng adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); 13656fdd68b1SAlex Deucher 13666fdd68b1SAlex Deucher /* XXX: add the xgmi offset of the physical node? */ 13676fdd68b1SAlex Deucher adev->vm_manager.vram_base_offset += 13686fdd68b1SAlex Deucher adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1369e60f8db5SAlex Xie } 1370e60f8db5SAlex Xie 1371e60f8db5SAlex Xie /** 1372e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 1373e60f8db5SAlex Xie * 1374e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1375e60f8db5SAlex Xie * 1376e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 1377e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 1378e60f8db5SAlex Xie * Returns 0 for success. 1379e60f8db5SAlex Xie */ 1380e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 1381e60f8db5SAlex Xie { 1382e60f8db5SAlex Xie int r; 1383e60f8db5SAlex Xie 1384e60f8db5SAlex Xie /* size in MB on si */ 1385770d13b1SChristian König adev->gmc.mc_vram_size = 1386bebc0762SHawking Zhang adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 1387770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 1388e60f8db5SAlex Xie 1389be566196SOak Zeng if (!(adev->flags & AMD_IS_APU) && 1390be566196SOak Zeng !adev->gmc.xgmi.connected_to_cpu) { 1391e60f8db5SAlex Xie r = amdgpu_device_resize_fb_bar(adev); 1392e60f8db5SAlex Xie if (r) 1393e60f8db5SAlex Xie return r; 1394e60f8db5SAlex Xie } 1395770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 1396770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 1397e60f8db5SAlex Xie 1398156a81beSChunming Zhou #ifdef CONFIG_X86_64 139931691b8dSRajneesh Bhardwaj /* 140031691b8dSRajneesh Bhardwaj * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi 140131691b8dSRajneesh Bhardwaj * interface can use VRAM through here as it appears system reserved 140231691b8dSRajneesh Bhardwaj * memory in host address space. 140331691b8dSRajneesh Bhardwaj * 140431691b8dSRajneesh Bhardwaj * For APUs, VRAM is just the stolen system memory and can be accessed 140531691b8dSRajneesh Bhardwaj * directly. 140631691b8dSRajneesh Bhardwaj * 140731691b8dSRajneesh Bhardwaj * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR. 140831691b8dSRajneesh Bhardwaj */ 140931691b8dSRajneesh Bhardwaj 141031691b8dSRajneesh Bhardwaj /* check whether both host-gpu and gpu-gpu xgmi links exist */ 14113de60d96SHawking Zhang if ((adev->flags & AMD_IS_APU) || 14123de60d96SHawking Zhang (adev->gmc.xgmi.supported && 14133de60d96SHawking Zhang adev->gmc.xgmi.connected_to_cpu)) { 14143de60d96SHawking Zhang adev->gmc.aper_base = 14153de60d96SHawking Zhang adev->gfxhub.funcs->get_mc_fb_offset(adev) + 14163de60d96SHawking Zhang adev->gmc.xgmi.physical_node_id * 141731691b8dSRajneesh Bhardwaj adev->gmc.xgmi.node_segment_size; 1418156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 1419156a81beSChunming Zhou } 142031691b8dSRajneesh Bhardwaj 1421156a81beSChunming Zhou #endif 1422e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 1423770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 1424770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 1425770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 1426e60f8db5SAlex Xie 1427e60f8db5SAlex Xie /* set the gart size */ 1428e60f8db5SAlex Xie if (amdgpu_gart_size == -1) { 1429630e959fSAlex Deucher switch (adev->ip_versions[GC_HWIP][0]) { 1430630e959fSAlex Deucher case IP_VERSION(9, 0, 1): /* all engines support GPUVM */ 1431630e959fSAlex Deucher case IP_VERSION(9, 2, 1): /* all engines support GPUVM */ 1432630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 1433630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 1434630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 1435e60f8db5SAlex Xie default: 1436fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 1437e60f8db5SAlex Xie break; 1438630e959fSAlex Deucher case IP_VERSION(9, 1, 0): /* DCE SG support */ 1439630e959fSAlex Deucher case IP_VERSION(9, 2, 2): /* DCE SG support */ 1440630e959fSAlex Deucher case IP_VERSION(9, 3, 0): 1441770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 1442e60f8db5SAlex Xie break; 1443e60f8db5SAlex Xie } 1444e60f8db5SAlex Xie } else { 1445770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 1446e60f8db5SAlex Xie } 1447e60f8db5SAlex Xie 1448f1dc12caSOak Zeng adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; 1449f1dc12caSOak Zeng 1450770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 1451e60f8db5SAlex Xie 1452e60f8db5SAlex Xie return 0; 1453e60f8db5SAlex Xie } 1454e60f8db5SAlex Xie 1455e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 1456e60f8db5SAlex Xie { 1457e60f8db5SAlex Xie int r; 1458e60f8db5SAlex Xie 14591123b989SChristian König if (adev->gart.bo) { 1460e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 1461e60f8db5SAlex Xie return 0; 1462e60f8db5SAlex Xie } 14637b454b3aSOak Zeng 14647b454b3aSOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 14657b454b3aSOak Zeng adev->gmc.vmid0_page_table_depth = 1; 14667b454b3aSOak Zeng adev->gmc.vmid0_page_table_block_size = 12; 14677b454b3aSOak Zeng } else { 14687b454b3aSOak Zeng adev->gmc.vmid0_page_table_depth = 0; 14697b454b3aSOak Zeng adev->gmc.vmid0_page_table_block_size = 0; 14707b454b3aSOak Zeng } 14717b454b3aSOak Zeng 1472e60f8db5SAlex Xie /* Initialize common gart structure */ 1473e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 1474e60f8db5SAlex Xie if (r) 1475e60f8db5SAlex Xie return r; 1476e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 14777596ab68SHawking Zhang adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | 1478e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 1479522510a6SOak Zeng 1480522510a6SOak Zeng r = amdgpu_gart_table_vram_alloc(adev); 1481522510a6SOak Zeng if (r) 1482522510a6SOak Zeng return r; 1483522510a6SOak Zeng 1484522510a6SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 1485522510a6SOak Zeng r = amdgpu_gmc_pdb0_alloc(adev); 1486522510a6SOak Zeng } 1487522510a6SOak Zeng 1488522510a6SOak Zeng return r; 1489e60f8db5SAlex Xie } 1490e60f8db5SAlex Xie 1491b0a2db9bSAlex Deucher /** 1492b0a2db9bSAlex Deucher * gmc_v9_0_save_registers - saves regs 1493b0a2db9bSAlex Deucher * 1494b0a2db9bSAlex Deucher * @adev: amdgpu_device pointer 1495b0a2db9bSAlex Deucher * 1496b0a2db9bSAlex Deucher * This saves potential register values that should be 1497b0a2db9bSAlex Deucher * restored upon resume 1498b0a2db9bSAlex Deucher */ 1499b0a2db9bSAlex Deucher static void gmc_v9_0_save_registers(struct amdgpu_device *adev) 1500ebdef28eSAlex Deucher { 1501630e959fSAlex Deucher if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) || 1502630e959fSAlex Deucher (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1))) 1503b0a2db9bSAlex Deucher adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); 1504ebdef28eSAlex Deucher } 1505ebdef28eSAlex Deucher 1506e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 1507e60f8db5SAlex Xie { 1508ad02e08eSOri Messinger int r, vram_width = 0, vram_type = 0, vram_vendor = 0; 1509e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1510e60f8db5SAlex Xie 15118ffff9b4SOak Zeng adev->gfxhub.funcs->init(adev); 15129fb1506eSOak Zeng 15139fb1506eSOak Zeng adev->mmhub.funcs->init(adev); 15143907c492SJohn Clements if (adev->mca.funcs) 15153907c492SJohn Clements adev->mca.funcs->init(adev); 1516e60f8db5SAlex Xie 1517770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 1518e60f8db5SAlex Xie 1519ad02e08eSOri Messinger r = amdgpu_atomfirmware_get_vram_info(adev, 1520ad02e08eSOri Messinger &vram_width, &vram_type, &vram_vendor); 1521631cdbd2SAlex Deucher if (amdgpu_sriov_vf(adev)) 1522631cdbd2SAlex Deucher /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, 1523631cdbd2SAlex Deucher * and DF related registers is not readable, seems hardcord is the 1524631cdbd2SAlex Deucher * only way to set the correct vram_width 1525631cdbd2SAlex Deucher */ 1526631cdbd2SAlex Deucher adev->gmc.vram_width = 2048; 1527631cdbd2SAlex Deucher else if (amdgpu_emu_mode != 1) 1528631cdbd2SAlex Deucher adev->gmc.vram_width = vram_width; 1529631cdbd2SAlex Deucher 1530631cdbd2SAlex Deucher if (!adev->gmc.vram_width) { 1531631cdbd2SAlex Deucher int chansize, numchan; 1532631cdbd2SAlex Deucher 1533631cdbd2SAlex Deucher /* hbm memory channel size */ 1534631cdbd2SAlex Deucher if (adev->flags & AMD_IS_APU) 1535631cdbd2SAlex Deucher chansize = 64; 1536631cdbd2SAlex Deucher else 1537631cdbd2SAlex Deucher chansize = 128; 1538cace4bffSHawking Zhang if (adev->df.funcs && 1539cace4bffSHawking Zhang adev->df.funcs->get_hbm_channel_number) { 1540bdf84a80SJoseph Greathouse numchan = adev->df.funcs->get_hbm_channel_number(adev); 1541631cdbd2SAlex Deucher adev->gmc.vram_width = numchan * chansize; 1542631cdbd2SAlex Deucher } 1543cace4bffSHawking Zhang } 1544631cdbd2SAlex Deucher 1545631cdbd2SAlex Deucher adev->gmc.vram_type = vram_type; 1546ad02e08eSOri Messinger adev->gmc.vram_vendor = vram_vendor; 1547630e959fSAlex Deucher switch (adev->ip_versions[GC_HWIP][0]) { 1548630e959fSAlex Deucher case IP_VERSION(9, 1, 0): 1549630e959fSAlex Deucher case IP_VERSION(9, 2, 2): 15501daa2bfaSLe Ma adev->num_vmhubs = 2; 15511daa2bfaSLe Ma 15526a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 1553f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 15546a42fd6fSChristian König } else { 15556a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 15566a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 1557770d13b1SChristian König adev->gmc.translate_further = 15586a42fd6fSChristian König adev->vm_manager.num_level > 1; 15596a42fd6fSChristian König } 1560e60f8db5SAlex Xie break; 1561630e959fSAlex Deucher case IP_VERSION(9, 0, 1): 1562630e959fSAlex Deucher case IP_VERSION(9, 2, 1): 1563630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 1564630e959fSAlex Deucher case IP_VERSION(9, 3, 0): 1565630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 15661daa2bfaSLe Ma adev->num_vmhubs = 2; 15671daa2bfaSLe Ma 15688787ee01SHuang Rui 1569e60f8db5SAlex Xie /* 1570e60f8db5SAlex Xie * To fulfill 4-level page support, 1571e60f8db5SAlex Xie * vm size is 256TB (48bit), maximum size of Vega10, 1572e60f8db5SAlex Xie * block size 512 (9bit) 1573e60f8db5SAlex Xie */ 1574cdba61daSwentalou /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 1575cdba61daSwentalou if (amdgpu_sriov_vf(adev)) 1576cdba61daSwentalou amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 1577cdba61daSwentalou else 1578f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1579e60f8db5SAlex Xie break; 1580630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 1581c8a6e2a3SLe Ma adev->num_vmhubs = 3; 1582c8a6e2a3SLe Ma 15833de2ff5dSLe Ma /* Keep the vm size same with Vega20 */ 15843de2ff5dSLe Ma amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 15853de2ff5dSLe Ma break; 1586e60f8db5SAlex Xie default: 1587e60f8db5SAlex Xie break; 1588e60f8db5SAlex Xie } 1589e60f8db5SAlex Xie 1590e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 159144a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 1592770d13b1SChristian König &adev->gmc.vm_fault); 159330da7bb1SChristian König if (r) 159430da7bb1SChristian König return r; 159530da7bb1SChristian König 1596630e959fSAlex Deucher if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) { 15977d19b15fSLe Ma r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, 15987d19b15fSLe Ma &adev->gmc.vm_fault); 15997d19b15fSLe Ma if (r) 16007d19b15fSLe Ma return r; 16017d19b15fSLe Ma } 16027d19b15fSLe Ma 160344a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 1604770d13b1SChristian König &adev->gmc.vm_fault); 1605e60f8db5SAlex Xie 1606e60f8db5SAlex Xie if (r) 1607e60f8db5SAlex Xie return r; 1608e60f8db5SAlex Xie 160968d705ddSHawking Zhang if (!amdgpu_sriov_vf(adev) && 161068d705ddSHawking Zhang !adev->gmc.xgmi.connected_to_cpu) { 1611791c4769Sxinhui pan /* interrupt sent to DF. */ 1612791c4769Sxinhui pan r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 1613791c4769Sxinhui pan &adev->gmc.ecc_irq); 1614791c4769Sxinhui pan if (r) 1615791c4769Sxinhui pan return r; 16162ee9403eSZhigang Luo } 1617791c4769Sxinhui pan 1618e60f8db5SAlex Xie /* Set the internal MC address mask 1619e60f8db5SAlex Xie * This is the max address of the GPU's 1620e60f8db5SAlex Xie * internal address space. 1621e60f8db5SAlex Xie */ 1622770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 1623e60f8db5SAlex Xie 1624244511f3SChristoph Hellwig r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); 1625e60f8db5SAlex Xie if (r) { 1626e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 1627244511f3SChristoph Hellwig return r; 1628e60f8db5SAlex Xie } 1629244511f3SChristoph Hellwig adev->need_swiotlb = drm_need_swiotlb(44); 1630e60f8db5SAlex Xie 163147622ba0SAlex Deucher if (adev->gmc.xgmi.supported) { 16328ffff9b4SOak Zeng r = adev->gfxhub.funcs->get_xgmi_info(adev); 1633bf0a60b7SAlex Deucher if (r) 1634bf0a60b7SAlex Deucher return r; 1635bf0a60b7SAlex Deucher } 1636bf0a60b7SAlex Deucher 1637e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 1638e60f8db5SAlex Xie if (r) 1639e60f8db5SAlex Xie return r; 1640e60f8db5SAlex Xie 16417b885f0eSAlex Deucher amdgpu_gmc_get_vbios_allocations(adev); 1642ebdef28eSAlex Deucher 1643e60f8db5SAlex Xie /* Memory manager */ 1644e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 1645e60f8db5SAlex Xie if (r) 1646e60f8db5SAlex Xie return r; 1647e60f8db5SAlex Xie 1648e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 1649e60f8db5SAlex Xie if (r) 1650e60f8db5SAlex Xie return r; 1651e60f8db5SAlex Xie 165205ec3edaSChristian König /* 165305ec3edaSChristian König * number of VMs 165405ec3edaSChristian König * VMID 0 is reserved for System 165581659b20SFelix Kuehling * amdgpu graphics/compute will use VMIDs 1..n-1 165681659b20SFelix Kuehling * amdkfd will use VMIDs n..15 165781659b20SFelix Kuehling * 165881659b20SFelix Kuehling * The first KFD VMID is 8 for GPUs with graphics, 3 for 165981659b20SFelix Kuehling * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs 166081659b20SFelix Kuehling * for video processing. 166105ec3edaSChristian König */ 166281659b20SFelix Kuehling adev->vm_manager.first_kfd_vmid = 1663630e959fSAlex Deucher (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || 1664630e959fSAlex Deucher adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) ? 3 : 8; 166505ec3edaSChristian König 166605ec3edaSChristian König amdgpu_vm_manager_init(adev); 166705ec3edaSChristian König 1668b0a2db9bSAlex Deucher gmc_v9_0_save_registers(adev); 1669b0a2db9bSAlex Deucher 167005ec3edaSChristian König return 0; 1671e60f8db5SAlex Xie } 1672e60f8db5SAlex Xie 1673e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 1674e60f8db5SAlex Xie { 1675e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1676e60f8db5SAlex Xie 16772adf1344STao Zhou amdgpu_gmc_ras_fini(adev); 1678f59548c8SMonk Liu amdgpu_gem_force_release(adev); 1679e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 1680a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 168137c49dedSLijo Lazar amdgpu_bo_unref(&adev->gmc.pdb0_bo); 1682e60f8db5SAlex Xie amdgpu_bo_fini(adev); 1683e60f8db5SAlex Xie 1684e60f8db5SAlex Xie return 0; 1685e60f8db5SAlex Xie } 1686e60f8db5SAlex Xie 1687e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1688e60f8db5SAlex Xie { 1689946a4d5bSShaoyun Liu 1690630e959fSAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 1691630e959fSAlex Deucher case IP_VERSION(9, 0, 0): 16924cd4c5c0SMonk Liu if (amdgpu_sriov_vf(adev)) 169398cad2deSTrigger Huang break; 1694df561f66SGustavo A. R. Silva fallthrough; 1695630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 1696946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 16975c583018SEvan Quan golden_settings_mmhub_1_0_0, 1698c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1699946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 17005c583018SEvan Quan golden_settings_athub_1_0_0, 1701c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1702e60f8db5SAlex Xie break; 1703630e959fSAlex Deucher case IP_VERSION(9, 1, 0): 1704630e959fSAlex Deucher case IP_VERSION(9, 2, 0): 17058787ee01SHuang Rui /* TODO for renoir */ 1706946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 17075c583018SEvan Quan golden_settings_athub_1_0_0, 1708c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1709e4f3abaaSChunming Zhou break; 1710e60f8db5SAlex Xie default: 1711e60f8db5SAlex Xie break; 1712e60f8db5SAlex Xie } 1713e60f8db5SAlex Xie } 1714e60f8db5SAlex Xie 1715e60f8db5SAlex Xie /** 1716c2ecd79bSShirish S * gmc_v9_0_restore_registers - restores regs 1717c2ecd79bSShirish S * 1718c2ecd79bSShirish S * @adev: amdgpu_device pointer 1719c2ecd79bSShirish S * 1720c2ecd79bSShirish S * This restores register values, saved at suspend. 1721c2ecd79bSShirish S */ 1722b0a2db9bSAlex Deucher void gmc_v9_0_restore_registers(struct amdgpu_device *adev) 1723c2ecd79bSShirish S { 1724630e959fSAlex Deucher if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) || 1725630e959fSAlex Deucher (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1))) { 1726f8646661SAlex Deucher WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); 17270eaa8012SShirish S WARN_ON(adev->gmc.sdpif_register != 17280eaa8012SShirish S RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0)); 17290eaa8012SShirish S } 1730c2ecd79bSShirish S } 1731c2ecd79bSShirish S 1732c2ecd79bSShirish S /** 1733e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1734e60f8db5SAlex Xie * 1735e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1736e60f8db5SAlex Xie */ 1737e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1738e60f8db5SAlex Xie { 1739cb1545f7SOak Zeng int r; 1740e60f8db5SAlex Xie 1741522510a6SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) 1742522510a6SOak Zeng amdgpu_gmc_init_pdb0(adev); 1743522510a6SOak Zeng 17441123b989SChristian König if (adev->gart.bo == NULL) { 1745e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1746e60f8db5SAlex Xie return -EINVAL; 1747e60f8db5SAlex Xie } 1748522510a6SOak Zeng 1749948e7ce0SJingwen Chen if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) 1750948e7ce0SJingwen Chen goto skip_pin_bo; 1751948e7ce0SJingwen Chen 1752ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1753ce1b1b66SMonk Liu if (r) 1754ce1b1b66SMonk Liu return r; 1755e60f8db5SAlex Xie 1756948e7ce0SJingwen Chen skip_pin_bo: 17578ffff9b4SOak Zeng r = adev->gfxhub.funcs->gart_enable(adev); 1758e60f8db5SAlex Xie if (r) 1759e60f8db5SAlex Xie return r; 1760e60f8db5SAlex Xie 17619fb1506eSOak Zeng r = adev->mmhub.funcs->gart_enable(adev); 1762e60f8db5SAlex Xie if (r) 1763e60f8db5SAlex Xie return r; 1764e60f8db5SAlex Xie 1765522510a6SOak Zeng DRM_INFO("PCIE GART of %uM enabled.\n", 1766522510a6SOak Zeng (unsigned)(adev->gmc.gart_size >> 20)); 1767522510a6SOak Zeng if (adev->gmc.pdb0_bo) 1768522510a6SOak Zeng DRM_INFO("PDB0 located at 0x%016llX\n", 1769522510a6SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo)); 1770522510a6SOak Zeng DRM_INFO("PTB located at 0x%016llX\n", 1771cb1545f7SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1772522510a6SOak Zeng 1773cb1545f7SOak Zeng adev->gart.ready = true; 1774cb1545f7SOak Zeng return 0; 1775cb1545f7SOak Zeng } 1776cb1545f7SOak Zeng 1777cb1545f7SOak Zeng static int gmc_v9_0_hw_init(void *handle) 1778cb1545f7SOak Zeng { 1779cb1545f7SOak Zeng struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1780cb1545f7SOak Zeng bool value; 178147d9c6faSchiminghao int i; 1782cb1545f7SOak Zeng 1783cb1545f7SOak Zeng /* The sequence of these two function calls matters.*/ 1784cb1545f7SOak Zeng gmc_v9_0_init_golden_registers(adev); 1785cb1545f7SOak Zeng 1786cb1545f7SOak Zeng if (adev->mode_info.num_crtc) { 1787cb1545f7SOak Zeng /* Lockout access through VGA aperture*/ 1788cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1789cb1545f7SOak Zeng /* disable VGA render */ 1790cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1791cb1545f7SOak Zeng } 1792cb1545f7SOak Zeng 17939fb1506eSOak Zeng if (adev->mmhub.funcs->update_power_gating) 17949fb1506eSOak Zeng adev->mmhub.funcs->update_power_gating(adev, true); 17959fb1506eSOak Zeng 1796455d40c9SLikun Gao adev->hdp.funcs->init_registers(adev); 1797fe2b5323STiecheng Zhou 17981d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 1799455d40c9SLikun Gao adev->hdp.funcs->flush_hdp(adev, NULL); 18001d4e0a8cSMonk Liu 1801e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1802e60f8db5SAlex Xie value = false; 1803e60f8db5SAlex Xie else 1804e60f8db5SAlex Xie value = true; 1805e60f8db5SAlex Xie 180620bf2f6fSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 18078ffff9b4SOak Zeng adev->gfxhub.funcs->set_fault_enable_default(adev, value); 18089fb1506eSOak Zeng adev->mmhub.funcs->set_fault_enable_default(adev, value); 180920bf2f6fSZhigang Luo } 18103ff98548SOak Zeng for (i = 0; i < adev->num_vmhubs; ++i) 18113ff98548SOak Zeng gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); 1812e60f8db5SAlex Xie 1813e7da754bSMonk Liu if (adev->umc.funcs && adev->umc.funcs->init_registers) 1814e7da754bSMonk Liu adev->umc.funcs->init_registers(adev); 1815e7da754bSMonk Liu 181647d9c6faSchiminghao return gmc_v9_0_gart_enable(adev); 1817e60f8db5SAlex Xie } 1818e60f8db5SAlex Xie 1819e60f8db5SAlex Xie /** 1820e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1821e60f8db5SAlex Xie * 1822e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1823e60f8db5SAlex Xie * 1824e60f8db5SAlex Xie * This disables all VM page table. 1825e60f8db5SAlex Xie */ 1826e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1827e60f8db5SAlex Xie { 18288ffff9b4SOak Zeng adev->gfxhub.funcs->gart_disable(adev); 18299fb1506eSOak Zeng adev->mmhub.funcs->gart_disable(adev); 1830ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1831e60f8db5SAlex Xie } 1832e60f8db5SAlex Xie 1833e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1834e60f8db5SAlex Xie { 1835e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1836e60f8db5SAlex Xie 183771cf9e72SLeslie Shi gmc_v9_0_gart_disable(adev); 183871cf9e72SLeslie Shi 18395dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 18405dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 18415dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 18425dd696aeSTrigger Huang return 0; 18435dd696aeSTrigger Huang } 18445dd696aeSTrigger Huang 184517252701SEvan Quan /* 184617252701SEvan Quan * Pair the operations did in gmc_v9_0_hw_init and thus maintain 184717252701SEvan Quan * a correct cached state for GMC. Otherwise, the "gate" again 184817252701SEvan Quan * operation on S3 resuming will fail due to wrong cached state. 184917252701SEvan Quan */ 185017252701SEvan Quan if (adev->mmhub.funcs->update_power_gating) 185117252701SEvan Quan adev->mmhub.funcs->update_power_gating(adev, false); 185217252701SEvan Quan 1853791c4769Sxinhui pan amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1854770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1855e60f8db5SAlex Xie 1856e60f8db5SAlex Xie return 0; 1857e60f8db5SAlex Xie } 1858e60f8db5SAlex Xie 1859e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1860e60f8db5SAlex Xie { 1861e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1862e60f8db5SAlex Xie 1863c24a3c05SLiu Shixin return gmc_v9_0_hw_fini(adev); 1864e60f8db5SAlex Xie } 1865e60f8db5SAlex Xie 1866e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1867e60f8db5SAlex Xie { 1868e60f8db5SAlex Xie int r; 1869e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1870e60f8db5SAlex Xie 1871e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1872e60f8db5SAlex Xie if (r) 1873e60f8db5SAlex Xie return r; 1874e60f8db5SAlex Xie 1875620f774fSChristian König amdgpu_vmid_reset_all(adev); 1876e60f8db5SAlex Xie 187732601d48SChristian König return 0; 1878e60f8db5SAlex Xie } 1879e60f8db5SAlex Xie 1880e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1881e60f8db5SAlex Xie { 1882e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1883e60f8db5SAlex Xie return true; 1884e60f8db5SAlex Xie } 1885e60f8db5SAlex Xie 1886e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1887e60f8db5SAlex Xie { 1888e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1889e60f8db5SAlex Xie return 0; 1890e60f8db5SAlex Xie } 1891e60f8db5SAlex Xie 1892e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1893e60f8db5SAlex Xie { 1894e60f8db5SAlex Xie /* XXX for emulation.*/ 1895e60f8db5SAlex Xie return 0; 1896e60f8db5SAlex Xie } 1897e60f8db5SAlex Xie 1898e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1899e60f8db5SAlex Xie enum amd_clockgating_state state) 1900e60f8db5SAlex Xie { 1901d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1902d5583d4fSHuang Rui 19039fb1506eSOak Zeng adev->mmhub.funcs->set_clockgating(adev, state); 1904bee7b51aSLe Ma 1905bee7b51aSLe Ma athub_v1_0_set_clockgating(adev, state); 1906bee7b51aSLe Ma 1907bee7b51aSLe Ma return 0; 1908e60f8db5SAlex Xie } 1909e60f8db5SAlex Xie 191013052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 191113052be5SHuang Rui { 191213052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 191313052be5SHuang Rui 19149fb1506eSOak Zeng adev->mmhub.funcs->get_clockgating(adev, flags); 1915bee7b51aSLe Ma 1916bee7b51aSLe Ma athub_v1_0_get_clockgating(adev, flags); 191713052be5SHuang Rui } 191813052be5SHuang Rui 1919e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1920e60f8db5SAlex Xie enum amd_powergating_state state) 1921e60f8db5SAlex Xie { 1922e60f8db5SAlex Xie return 0; 1923e60f8db5SAlex Xie } 1924e60f8db5SAlex Xie 1925e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1926e60f8db5SAlex Xie .name = "gmc_v9_0", 1927e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1928e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1929e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1930e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1931e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1932e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1933e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1934e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1935e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1936e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1937e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1938e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1939e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 194013052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1941e60f8db5SAlex Xie }; 1942e60f8db5SAlex Xie 1943e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1944e60f8db5SAlex Xie { 1945e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1946e60f8db5SAlex Xie .major = 9, 1947e60f8db5SAlex Xie .minor = 0, 1948e60f8db5SAlex Xie .rev = 0, 1949e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1950e60f8db5SAlex Xie }; 1951