1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23f867723bSSam Ravnborg 24e60f8db5SAlex Xie #include <linux/firmware.h> 25f867723bSSam Ravnborg #include <linux/pci.h> 26f867723bSSam Ravnborg 27fd5fd480SChunming Zhou #include <drm/drm_cache.h> 28f867723bSSam Ravnborg 29e60f8db5SAlex Xie #include "amdgpu.h" 30e60f8db5SAlex Xie #include "gmc_v9_0.h" 318d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 322cddc50eSHuang Rui #include "amdgpu_gem.h" 33e60f8db5SAlex Xie 34cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 35135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 36135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 37fb960bd2SFeifei Xu #include "vega10_enum.h" 3865417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 39ea930000SAlex Sierra #include "athub/athub_1_0_sh_mask.h" 406ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 41250b4228SChristian König #include "oss/osssys_4_0_offset.h" 42e60f8db5SAlex Xie 43946a4d5bSShaoyun Liu #include "soc15.h" 44ea930000SAlex Sierra #include "soc15d.h" 45e60f8db5SAlex Xie #include "soc15_common.h" 4690c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 47e60f8db5SAlex Xie 48e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 49e60f8db5SAlex Xie #include "mmhub_v1_0.h" 50bee7b51aSLe Ma #include "athub_v1_0.h" 51bf0a60b7SAlex Deucher #include "gfxhub_v1_1.h" 5251cce480SLe Ma #include "mmhub_v9_4.h" 5385e39550SLe Ma #include "mmhub_v1_7.h" 545b6b35aaSHawking Zhang #include "umc_v6_1.h" 55e7da754bSMonk Liu #include "umc_v6_0.h" 56e60f8db5SAlex Xie 5744a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 5844a99b65SAndrey Grodzovsky 59791c4769Sxinhui pan #include "amdgpu_ras.h" 60029fbd43SHawking Zhang #include "amdgpu_xgmi.h" 61791c4769Sxinhui pan 62ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 63ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 64ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 65ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 66ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 67ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 68ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 69f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d 70f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 71f8646661SAlex Deucher 72ebdef28eSAlex Deucher 73be99ecbfSAlex Deucher static const char *gfxhub_client_ids[] = { 74be99ecbfSAlex Deucher "CB", 75be99ecbfSAlex Deucher "DB", 76be99ecbfSAlex Deucher "IA", 77be99ecbfSAlex Deucher "WD", 78be99ecbfSAlex Deucher "CPF", 79be99ecbfSAlex Deucher "CPC", 80be99ecbfSAlex Deucher "CPG", 81be99ecbfSAlex Deucher "RLC", 82be99ecbfSAlex Deucher "TCP", 83be99ecbfSAlex Deucher "SQC (inst)", 84be99ecbfSAlex Deucher "SQC (data)", 85be99ecbfSAlex Deucher "SQG", 86be99ecbfSAlex Deucher "PA", 87be99ecbfSAlex Deucher }; 88be99ecbfSAlex Deucher 8902f23f5fSAlex Deucher static const char *mmhub_client_ids_raven[][2] = { 9002f23f5fSAlex Deucher [0][0] = "MP1", 9102f23f5fSAlex Deucher [1][0] = "MP0", 9202f23f5fSAlex Deucher [2][0] = "VCN", 9302f23f5fSAlex Deucher [3][0] = "VCNU", 9402f23f5fSAlex Deucher [4][0] = "HDP", 9502f23f5fSAlex Deucher [5][0] = "DCE", 9602f23f5fSAlex Deucher [13][0] = "UTCL2", 9702f23f5fSAlex Deucher [19][0] = "TLS", 9802f23f5fSAlex Deucher [26][0] = "OSS", 9902f23f5fSAlex Deucher [27][0] = "SDMA0", 10002f23f5fSAlex Deucher [0][1] = "MP1", 10102f23f5fSAlex Deucher [1][1] = "MP0", 10202f23f5fSAlex Deucher [2][1] = "VCN", 10302f23f5fSAlex Deucher [3][1] = "VCNU", 10402f23f5fSAlex Deucher [4][1] = "HDP", 10502f23f5fSAlex Deucher [5][1] = "XDP", 10602f23f5fSAlex Deucher [6][1] = "DBGU0", 10702f23f5fSAlex Deucher [7][1] = "DCE", 10802f23f5fSAlex Deucher [8][1] = "DCEDWB0", 10902f23f5fSAlex Deucher [9][1] = "DCEDWB1", 11002f23f5fSAlex Deucher [26][1] = "OSS", 11102f23f5fSAlex Deucher [27][1] = "SDMA0", 11202f23f5fSAlex Deucher }; 11302f23f5fSAlex Deucher 11402f23f5fSAlex Deucher static const char *mmhub_client_ids_renoir[][2] = { 11502f23f5fSAlex Deucher [0][0] = "MP1", 11602f23f5fSAlex Deucher [1][0] = "MP0", 11702f23f5fSAlex Deucher [2][0] = "HDP", 11802f23f5fSAlex Deucher [4][0] = "DCEDMC", 11902f23f5fSAlex Deucher [5][0] = "DCEVGA", 12002f23f5fSAlex Deucher [13][0] = "UTCL2", 12102f23f5fSAlex Deucher [19][0] = "TLS", 12202f23f5fSAlex Deucher [26][0] = "OSS", 12302f23f5fSAlex Deucher [27][0] = "SDMA0", 12402f23f5fSAlex Deucher [28][0] = "VCN", 12502f23f5fSAlex Deucher [29][0] = "VCNU", 12602f23f5fSAlex Deucher [30][0] = "JPEG", 12702f23f5fSAlex Deucher [0][1] = "MP1", 12802f23f5fSAlex Deucher [1][1] = "MP0", 12902f23f5fSAlex Deucher [2][1] = "HDP", 13002f23f5fSAlex Deucher [3][1] = "XDP", 13102f23f5fSAlex Deucher [6][1] = "DBGU0", 13202f23f5fSAlex Deucher [7][1] = "DCEDMC", 13302f23f5fSAlex Deucher [8][1] = "DCEVGA", 13402f23f5fSAlex Deucher [9][1] = "DCEDWB", 13502f23f5fSAlex Deucher [26][1] = "OSS", 13602f23f5fSAlex Deucher [27][1] = "SDMA0", 13702f23f5fSAlex Deucher [28][1] = "VCN", 13802f23f5fSAlex Deucher [29][1] = "VCNU", 13902f23f5fSAlex Deucher [30][1] = "JPEG", 14002f23f5fSAlex Deucher }; 14102f23f5fSAlex Deucher 14202f23f5fSAlex Deucher static const char *mmhub_client_ids_vega10[][2] = { 14302f23f5fSAlex Deucher [0][0] = "MP0", 14402f23f5fSAlex Deucher [1][0] = "UVD", 14502f23f5fSAlex Deucher [2][0] = "UVDU", 14602f23f5fSAlex Deucher [3][0] = "HDP", 14702f23f5fSAlex Deucher [13][0] = "UTCL2", 14802f23f5fSAlex Deucher [14][0] = "OSS", 14902f23f5fSAlex Deucher [15][0] = "SDMA1", 15002f23f5fSAlex Deucher [32+0][0] = "VCE0", 15102f23f5fSAlex Deucher [32+1][0] = "VCE0U", 15202f23f5fSAlex Deucher [32+2][0] = "XDMA", 15302f23f5fSAlex Deucher [32+3][0] = "DCE", 15402f23f5fSAlex Deucher [32+4][0] = "MP1", 15502f23f5fSAlex Deucher [32+14][0] = "SDMA0", 15602f23f5fSAlex Deucher [0][1] = "MP0", 15702f23f5fSAlex Deucher [1][1] = "UVD", 15802f23f5fSAlex Deucher [2][1] = "UVDU", 15902f23f5fSAlex Deucher [3][1] = "DBGU0", 16002f23f5fSAlex Deucher [4][1] = "HDP", 16102f23f5fSAlex Deucher [5][1] = "XDP", 16202f23f5fSAlex Deucher [14][1] = "OSS", 16302f23f5fSAlex Deucher [15][1] = "SDMA0", 16402f23f5fSAlex Deucher [32+0][1] = "VCE0", 16502f23f5fSAlex Deucher [32+1][1] = "VCE0U", 16602f23f5fSAlex Deucher [32+2][1] = "XDMA", 16702f23f5fSAlex Deucher [32+3][1] = "DCE", 16802f23f5fSAlex Deucher [32+4][1] = "DCEDWB", 16902f23f5fSAlex Deucher [32+5][1] = "MP1", 17002f23f5fSAlex Deucher [32+6][1] = "DBGU1", 17102f23f5fSAlex Deucher [32+14][1] = "SDMA1", 17202f23f5fSAlex Deucher }; 17302f23f5fSAlex Deucher 17402f23f5fSAlex Deucher static const char *mmhub_client_ids_vega12[][2] = { 17502f23f5fSAlex Deucher [0][0] = "MP0", 17602f23f5fSAlex Deucher [1][0] = "VCE0", 17702f23f5fSAlex Deucher [2][0] = "VCE0U", 17802f23f5fSAlex Deucher [3][0] = "HDP", 17902f23f5fSAlex Deucher [13][0] = "UTCL2", 18002f23f5fSAlex Deucher [14][0] = "OSS", 18102f23f5fSAlex Deucher [15][0] = "SDMA1", 18202f23f5fSAlex Deucher [32+0][0] = "DCE", 18302f23f5fSAlex Deucher [32+1][0] = "XDMA", 18402f23f5fSAlex Deucher [32+2][0] = "UVD", 18502f23f5fSAlex Deucher [32+3][0] = "UVDU", 18602f23f5fSAlex Deucher [32+4][0] = "MP1", 18702f23f5fSAlex Deucher [32+15][0] = "SDMA0", 18802f23f5fSAlex Deucher [0][1] = "MP0", 18902f23f5fSAlex Deucher [1][1] = "VCE0", 19002f23f5fSAlex Deucher [2][1] = "VCE0U", 19102f23f5fSAlex Deucher [3][1] = "DBGU0", 19202f23f5fSAlex Deucher [4][1] = "HDP", 19302f23f5fSAlex Deucher [5][1] = "XDP", 19402f23f5fSAlex Deucher [14][1] = "OSS", 19502f23f5fSAlex Deucher [15][1] = "SDMA0", 19602f23f5fSAlex Deucher [32+0][1] = "DCE", 19702f23f5fSAlex Deucher [32+1][1] = "DCEDWB", 19802f23f5fSAlex Deucher [32+2][1] = "XDMA", 19902f23f5fSAlex Deucher [32+3][1] = "UVD", 20002f23f5fSAlex Deucher [32+4][1] = "UVDU", 20102f23f5fSAlex Deucher [32+5][1] = "MP1", 20202f23f5fSAlex Deucher [32+6][1] = "DBGU1", 20302f23f5fSAlex Deucher [32+15][1] = "SDMA1", 20402f23f5fSAlex Deucher }; 20502f23f5fSAlex Deucher 20602f23f5fSAlex Deucher static const char *mmhub_client_ids_vega20[][2] = { 20702f23f5fSAlex Deucher [0][0] = "XDMA", 20802f23f5fSAlex Deucher [1][0] = "DCE", 20902f23f5fSAlex Deucher [2][0] = "VCE0", 21002f23f5fSAlex Deucher [3][0] = "VCE0U", 21102f23f5fSAlex Deucher [4][0] = "UVD", 21202f23f5fSAlex Deucher [5][0] = "UVD1U", 21302f23f5fSAlex Deucher [13][0] = "OSS", 21402f23f5fSAlex Deucher [14][0] = "HDP", 21502f23f5fSAlex Deucher [15][0] = "SDMA0", 21602f23f5fSAlex Deucher [32+0][0] = "UVD", 21702f23f5fSAlex Deucher [32+1][0] = "UVDU", 21802f23f5fSAlex Deucher [32+2][0] = "MP1", 21902f23f5fSAlex Deucher [32+3][0] = "MP0", 22002f23f5fSAlex Deucher [32+12][0] = "UTCL2", 22102f23f5fSAlex Deucher [32+14][0] = "SDMA1", 22202f23f5fSAlex Deucher [0][1] = "XDMA", 22302f23f5fSAlex Deucher [1][1] = "DCE", 22402f23f5fSAlex Deucher [2][1] = "DCEDWB", 22502f23f5fSAlex Deucher [3][1] = "VCE0", 22602f23f5fSAlex Deucher [4][1] = "VCE0U", 22702f23f5fSAlex Deucher [5][1] = "UVD1", 22802f23f5fSAlex Deucher [6][1] = "UVD1U", 22902f23f5fSAlex Deucher [7][1] = "DBGU0", 23002f23f5fSAlex Deucher [8][1] = "XDP", 23102f23f5fSAlex Deucher [13][1] = "OSS", 23202f23f5fSAlex Deucher [14][1] = "HDP", 23302f23f5fSAlex Deucher [15][1] = "SDMA0", 23402f23f5fSAlex Deucher [32+0][1] = "UVD", 23502f23f5fSAlex Deucher [32+1][1] = "UVDU", 23602f23f5fSAlex Deucher [32+2][1] = "DBGU1", 23702f23f5fSAlex Deucher [32+3][1] = "MP1", 23802f23f5fSAlex Deucher [32+4][1] = "MP0", 23902f23f5fSAlex Deucher [32+14][1] = "SDMA1", 24002f23f5fSAlex Deucher }; 24102f23f5fSAlex Deucher 24202f23f5fSAlex Deucher static const char *mmhub_client_ids_arcturus[][2] = { 243e83db774SAlex Deucher [0][0] = "DBGU1", 244e83db774SAlex Deucher [1][0] = "XDP", 24502f23f5fSAlex Deucher [2][0] = "MP1", 24602f23f5fSAlex Deucher [14][0] = "HDP", 247e83db774SAlex Deucher [171][0] = "JPEG", 248e83db774SAlex Deucher [172][0] = "VCN", 249e83db774SAlex Deucher [173][0] = "VCNU", 250e83db774SAlex Deucher [203][0] = "JPEG1", 251e83db774SAlex Deucher [204][0] = "VCN1", 252e83db774SAlex Deucher [205][0] = "VCN1U", 253e83db774SAlex Deucher [256][0] = "SDMA0", 254e83db774SAlex Deucher [257][0] = "SDMA1", 255e83db774SAlex Deucher [258][0] = "SDMA2", 256e83db774SAlex Deucher [259][0] = "SDMA3", 257e83db774SAlex Deucher [260][0] = "SDMA4", 258e83db774SAlex Deucher [261][0] = "SDMA5", 259e83db774SAlex Deucher [262][0] = "SDMA6", 260e83db774SAlex Deucher [263][0] = "SDMA7", 261e83db774SAlex Deucher [384][0] = "OSS", 26202f23f5fSAlex Deucher [0][1] = "DBGU1", 26302f23f5fSAlex Deucher [1][1] = "XDP", 26402f23f5fSAlex Deucher [2][1] = "MP1", 26502f23f5fSAlex Deucher [14][1] = "HDP", 266e83db774SAlex Deucher [171][1] = "JPEG", 267e83db774SAlex Deucher [172][1] = "VCN", 268e83db774SAlex Deucher [173][1] = "VCNU", 269e83db774SAlex Deucher [203][1] = "JPEG1", 270e83db774SAlex Deucher [204][1] = "VCN1", 271e83db774SAlex Deucher [205][1] = "VCN1U", 272e83db774SAlex Deucher [256][1] = "SDMA0", 273e83db774SAlex Deucher [257][1] = "SDMA1", 274e83db774SAlex Deucher [258][1] = "SDMA2", 275e83db774SAlex Deucher [259][1] = "SDMA3", 276e83db774SAlex Deucher [260][1] = "SDMA4", 277e83db774SAlex Deucher [261][1] = "SDMA5", 278e83db774SAlex Deucher [262][1] = "SDMA6", 279e83db774SAlex Deucher [263][1] = "SDMA7", 280e83db774SAlex Deucher [384][1] = "OSS", 28102f23f5fSAlex Deucher }; 282e60f8db5SAlex Xie 283e844cd99SAlex Deucher static const char *mmhub_client_ids_aldebaran[][2] = { 284e844cd99SAlex Deucher [2][0] = "MP1", 285e844cd99SAlex Deucher [3][0] = "MP0", 286f4ec3e50SAlex Sierra [32+1][0] = "DBGU_IO0", 287f4ec3e50SAlex Sierra [32+2][0] = "DBGU_IO2", 288e844cd99SAlex Deucher [32+4][0] = "MPIO", 289e844cd99SAlex Deucher [96+11][0] = "JPEG0", 290e844cd99SAlex Deucher [96+12][0] = "VCN0", 291e844cd99SAlex Deucher [96+13][0] = "VCNU0", 292e844cd99SAlex Deucher [128+11][0] = "JPEG1", 293e844cd99SAlex Deucher [128+12][0] = "VCN1", 294e844cd99SAlex Deucher [128+13][0] = "VCNU1", 295f4ec3e50SAlex Sierra [160+1][0] = "XDP", 296e844cd99SAlex Deucher [160+14][0] = "HDP", 297f4ec3e50SAlex Sierra [256+0][0] = "SDMA0", 298f4ec3e50SAlex Sierra [256+1][0] = "SDMA1", 299f4ec3e50SAlex Sierra [256+2][0] = "SDMA2", 300f4ec3e50SAlex Sierra [256+3][0] = "SDMA3", 301f4ec3e50SAlex Sierra [256+4][0] = "SDMA4", 302f4ec3e50SAlex Sierra [384+0][0] = "OSS", 303e844cd99SAlex Deucher [2][1] = "MP1", 304e844cd99SAlex Deucher [3][1] = "MP0", 305e844cd99SAlex Deucher [32+1][1] = "DBGU_IO0", 306e844cd99SAlex Deucher [32+2][1] = "DBGU_IO2", 307e844cd99SAlex Deucher [32+4][1] = "MPIO", 308e844cd99SAlex Deucher [96+11][1] = "JPEG0", 309e844cd99SAlex Deucher [96+12][1] = "VCN0", 310e844cd99SAlex Deucher [96+13][1] = "VCNU0", 311e844cd99SAlex Deucher [128+11][1] = "JPEG1", 312e844cd99SAlex Deucher [128+12][1] = "VCN1", 313e844cd99SAlex Deucher [128+13][1] = "VCNU1", 314f4ec3e50SAlex Sierra [160+1][1] = "XDP", 315e844cd99SAlex Deucher [160+14][1] = "HDP", 316f4ec3e50SAlex Sierra [256+0][1] = "SDMA0", 317f4ec3e50SAlex Sierra [256+1][1] = "SDMA1", 318f4ec3e50SAlex Sierra [256+2][1] = "SDMA2", 319f4ec3e50SAlex Sierra [256+3][1] = "SDMA3", 320f4ec3e50SAlex Sierra [256+4][1] = "SDMA4", 321f4ec3e50SAlex Sierra [384+0][1] = "OSS", 322e844cd99SAlex Deucher }; 323e844cd99SAlex Deucher 324946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 3255c583018SEvan Quan { 326946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 327946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 3285c583018SEvan Quan }; 3295c583018SEvan Quan 330946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 3315c583018SEvan Quan { 332946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 333946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 3345c583018SEvan Quan }; 3355c583018SEvan Quan 336791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 337791c4769Sxinhui pan (0x000143c0 + 0x00000000), 338791c4769Sxinhui pan (0x000143c0 + 0x00000800), 339791c4769Sxinhui pan (0x000143c0 + 0x00001000), 340791c4769Sxinhui pan (0x000143c0 + 0x00001800), 341791c4769Sxinhui pan (0x000543c0 + 0x00000000), 342791c4769Sxinhui pan (0x000543c0 + 0x00000800), 343791c4769Sxinhui pan (0x000543c0 + 0x00001000), 344791c4769Sxinhui pan (0x000543c0 + 0x00001800), 345791c4769Sxinhui pan (0x000943c0 + 0x00000000), 346791c4769Sxinhui pan (0x000943c0 + 0x00000800), 347791c4769Sxinhui pan (0x000943c0 + 0x00001000), 348791c4769Sxinhui pan (0x000943c0 + 0x00001800), 349791c4769Sxinhui pan (0x000d43c0 + 0x00000000), 350791c4769Sxinhui pan (0x000d43c0 + 0x00000800), 351791c4769Sxinhui pan (0x000d43c0 + 0x00001000), 352791c4769Sxinhui pan (0x000d43c0 + 0x00001800), 353791c4769Sxinhui pan (0x001143c0 + 0x00000000), 354791c4769Sxinhui pan (0x001143c0 + 0x00000800), 355791c4769Sxinhui pan (0x001143c0 + 0x00001000), 356791c4769Sxinhui pan (0x001143c0 + 0x00001800), 357791c4769Sxinhui pan (0x001543c0 + 0x00000000), 358791c4769Sxinhui pan (0x001543c0 + 0x00000800), 359791c4769Sxinhui pan (0x001543c0 + 0x00001000), 360791c4769Sxinhui pan (0x001543c0 + 0x00001800), 361791c4769Sxinhui pan (0x001943c0 + 0x00000000), 362791c4769Sxinhui pan (0x001943c0 + 0x00000800), 363791c4769Sxinhui pan (0x001943c0 + 0x00001000), 364791c4769Sxinhui pan (0x001943c0 + 0x00001800), 365791c4769Sxinhui pan (0x001d43c0 + 0x00000000), 366791c4769Sxinhui pan (0x001d43c0 + 0x00000800), 367791c4769Sxinhui pan (0x001d43c0 + 0x00001000), 368791c4769Sxinhui pan (0x001d43c0 + 0x00001800), 36902bab923SDavid Panariti }; 37002bab923SDavid Panariti 371791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 372791c4769Sxinhui pan (0x000143e0 + 0x00000000), 373791c4769Sxinhui pan (0x000143e0 + 0x00000800), 374791c4769Sxinhui pan (0x000143e0 + 0x00001000), 375791c4769Sxinhui pan (0x000143e0 + 0x00001800), 376791c4769Sxinhui pan (0x000543e0 + 0x00000000), 377791c4769Sxinhui pan (0x000543e0 + 0x00000800), 378791c4769Sxinhui pan (0x000543e0 + 0x00001000), 379791c4769Sxinhui pan (0x000543e0 + 0x00001800), 380791c4769Sxinhui pan (0x000943e0 + 0x00000000), 381791c4769Sxinhui pan (0x000943e0 + 0x00000800), 382791c4769Sxinhui pan (0x000943e0 + 0x00001000), 383791c4769Sxinhui pan (0x000943e0 + 0x00001800), 384791c4769Sxinhui pan (0x000d43e0 + 0x00000000), 385791c4769Sxinhui pan (0x000d43e0 + 0x00000800), 386791c4769Sxinhui pan (0x000d43e0 + 0x00001000), 387791c4769Sxinhui pan (0x000d43e0 + 0x00001800), 388791c4769Sxinhui pan (0x001143e0 + 0x00000000), 389791c4769Sxinhui pan (0x001143e0 + 0x00000800), 390791c4769Sxinhui pan (0x001143e0 + 0x00001000), 391791c4769Sxinhui pan (0x001143e0 + 0x00001800), 392791c4769Sxinhui pan (0x001543e0 + 0x00000000), 393791c4769Sxinhui pan (0x001543e0 + 0x00000800), 394791c4769Sxinhui pan (0x001543e0 + 0x00001000), 395791c4769Sxinhui pan (0x001543e0 + 0x00001800), 396791c4769Sxinhui pan (0x001943e0 + 0x00000000), 397791c4769Sxinhui pan (0x001943e0 + 0x00000800), 398791c4769Sxinhui pan (0x001943e0 + 0x00001000), 399791c4769Sxinhui pan (0x001943e0 + 0x00001800), 400791c4769Sxinhui pan (0x001d43e0 + 0x00000000), 401791c4769Sxinhui pan (0x001d43e0 + 0x00000800), 402791c4769Sxinhui pan (0x001d43e0 + 0x00001000), 403791c4769Sxinhui pan (0x001d43e0 + 0x00001800), 40402bab923SDavid Panariti }; 40502bab923SDavid Panariti 406791c4769Sxinhui pan static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 407791c4769Sxinhui pan struct amdgpu_irq_src *src, 408791c4769Sxinhui pan unsigned type, 409791c4769Sxinhui pan enum amdgpu_interrupt_state state) 410791c4769Sxinhui pan { 411791c4769Sxinhui pan u32 bits, i, tmp, reg; 412791c4769Sxinhui pan 4131e2c6d55SJohn Clements /* Devices newer then VEGA10/12 shall have these programming 4141e2c6d55SJohn Clements sequences performed by PSP BL */ 4151e2c6d55SJohn Clements if (adev->asic_type >= CHIP_VEGA20) 4161e2c6d55SJohn Clements return 0; 4171e2c6d55SJohn Clements 418791c4769Sxinhui pan bits = 0x7f; 419791c4769Sxinhui pan 420791c4769Sxinhui pan switch (state) { 421791c4769Sxinhui pan case AMDGPU_IRQ_STATE_DISABLE: 422791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 423791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 424791c4769Sxinhui pan tmp = RREG32(reg); 425791c4769Sxinhui pan tmp &= ~bits; 426791c4769Sxinhui pan WREG32(reg, tmp); 427791c4769Sxinhui pan } 428791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 429791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 430791c4769Sxinhui pan tmp = RREG32(reg); 431791c4769Sxinhui pan tmp &= ~bits; 432791c4769Sxinhui pan WREG32(reg, tmp); 433791c4769Sxinhui pan } 434791c4769Sxinhui pan break; 435791c4769Sxinhui pan case AMDGPU_IRQ_STATE_ENABLE: 436791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 437791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 438791c4769Sxinhui pan tmp = RREG32(reg); 439791c4769Sxinhui pan tmp |= bits; 440791c4769Sxinhui pan WREG32(reg, tmp); 441791c4769Sxinhui pan } 442791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 443791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 444791c4769Sxinhui pan tmp = RREG32(reg); 445791c4769Sxinhui pan tmp |= bits; 446791c4769Sxinhui pan WREG32(reg, tmp); 447791c4769Sxinhui pan } 448791c4769Sxinhui pan break; 449791c4769Sxinhui pan default: 450791c4769Sxinhui pan break; 451791c4769Sxinhui pan } 452791c4769Sxinhui pan 453791c4769Sxinhui pan return 0; 454791c4769Sxinhui pan } 455791c4769Sxinhui pan 456e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 457e60f8db5SAlex Xie struct amdgpu_irq_src *src, 458e60f8db5SAlex Xie unsigned type, 459e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 460e60f8db5SAlex Xie { 461e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 462ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 463e60f8db5SAlex Xie 46411250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46511250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46611250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46711250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46811250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 46911250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47011250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 47111250164SChristian König 472e60f8db5SAlex Xie switch (state) { 473e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 4741daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 475ae6d1416STom St Denis hub = &adev->vmhub[j]; 476e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 477e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 478e60f8db5SAlex Xie tmp = RREG32(reg); 479e60f8db5SAlex Xie tmp &= ~bits; 480e60f8db5SAlex Xie WREG32(reg, tmp); 481e60f8db5SAlex Xie } 482e60f8db5SAlex Xie } 483e60f8db5SAlex Xie break; 484e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 4851daa2bfaSLe Ma for (j = 0; j < adev->num_vmhubs; j++) { 486ae6d1416STom St Denis hub = &adev->vmhub[j]; 487e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 488e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 489e60f8db5SAlex Xie tmp = RREG32(reg); 490e60f8db5SAlex Xie tmp |= bits; 491e60f8db5SAlex Xie WREG32(reg, tmp); 492e60f8db5SAlex Xie } 493e60f8db5SAlex Xie } 4949304ca4dSGustavo A. R. Silva break; 495e60f8db5SAlex Xie default: 496e60f8db5SAlex Xie break; 497e60f8db5SAlex Xie } 498e60f8db5SAlex Xie 499e60f8db5SAlex Xie return 0; 500e60f8db5SAlex Xie } 501e60f8db5SAlex Xie 502e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 503e60f8db5SAlex Xie struct amdgpu_irq_src *source, 504e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 505e60f8db5SAlex Xie { 506c468f9e2SChristian König bool retry_fault = !!(entry->src_data[1] & 0x80); 50702f23f5fSAlex Deucher uint32_t status = 0, cid = 0, rw = 0; 508e3898719SChristian König struct amdgpu_task_info task_info; 509e3898719SChristian König struct amdgpu_vmhub *hub; 51002f23f5fSAlex Deucher const char *mmhub_cid; 511e3898719SChristian König const char *hub_name; 512e3898719SChristian König u64 addr; 513e60f8db5SAlex Xie 514e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 515e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 516e60f8db5SAlex Xie 5170291150dSChristian König if (retry_fault) { 5180291150dSChristian König /* Returning 1 here also prevents sending the IV to the KFD */ 51922666cc1SChristian König 5200291150dSChristian König /* Process it onyl if it's the first fault for this address */ 5210291150dSChristian König if (entry->ih != &adev->irq.ih_soft && 5220291150dSChristian König amdgpu_gmc_filter_faults(adev, addr, entry->pasid, 5230291150dSChristian König entry->timestamp)) 5240291150dSChristian König return 1; 5250291150dSChristian König 5260291150dSChristian König /* Delegate it to a different ring if the hardware hasn't 5270291150dSChristian König * already done it. 5280291150dSChristian König */ 52958df0d71SSebastian Andrzej Siewior if (entry->ih == &adev->irq.ih) { 5300291150dSChristian König amdgpu_irq_delegate(adev, entry, 8); 5310291150dSChristian König return 1; 5320291150dSChristian König } 5330291150dSChristian König 5340291150dSChristian König /* Try to handle the recoverable page faults by filling page 5350291150dSChristian König * tables 5360291150dSChristian König */ 5370291150dSChristian König if (amdgpu_vm_handle_fault(adev, entry->pasid, addr)) 5380291150dSChristian König return 1; 5390291150dSChristian König } 540ec671737SChristian König 541e3898719SChristian König if (!printk_ratelimit()) 542e3898719SChristian König return 0; 54353499173SXiaojie Yuan 544e3898719SChristian König if (entry->client_id == SOC15_IH_CLIENTID_VMC) { 545e3898719SChristian König hub_name = "mmhub0"; 546e3898719SChristian König hub = &adev->vmhub[AMDGPU_MMHUB_0]; 547e3898719SChristian König } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { 548e3898719SChristian König hub_name = "mmhub1"; 549e3898719SChristian König hub = &adev->vmhub[AMDGPU_MMHUB_1]; 550e3898719SChristian König } else { 551e3898719SChristian König hub_name = "gfxhub0"; 552e3898719SChristian König hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 5534d6cbde3SFelix Kuehling } 554e60f8db5SAlex Xie 55505794effSShirish S memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 556efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 557efaa9646SAndrey Grodzovsky 5584d6cbde3SFelix Kuehling dev_err(adev->dev, 559c468f9e2SChristian König "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 560c468f9e2SChristian König "pasid:%u, for process %s pid %d thread %s pid %d)\n", 56151c60898SLe Ma hub_name, retry_fault ? "retry" : "no-retry", 562c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 563efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 564efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 565be14729aSYong Zhao dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n", 566be14729aSYong Zhao addr, entry->client_id, 567be14729aSYong Zhao soc15_ih_clientid_name[entry->client_id]); 568e3898719SChristian König 569e3898719SChristian König if (amdgpu_sriov_vf(adev)) 570e3898719SChristian König return 0; 571e3898719SChristian König 572e3898719SChristian König /* 573e3898719SChristian König * Issue a dummy read to wait for the status register to 574e3898719SChristian König * be updated to avoid reading an incorrect value due to 575e3898719SChristian König * the new fast GRBM interface. 576e3898719SChristian König */ 577e3898719SChristian König if (entry->vmid_src == AMDGPU_GFXHUB_0) 578e3898719SChristian König RREG32(hub->vm_l2_pro_fault_status); 579e3898719SChristian König 580e3898719SChristian König status = RREG32(hub->vm_l2_pro_fault_status); 581e3898719SChristian König cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID); 582e3898719SChristian König rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW); 583e3898719SChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 584e3898719SChristian König 585e3898719SChristian König 5864d6cbde3SFelix Kuehling dev_err(adev->dev, 5874d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 5884d6cbde3SFelix Kuehling status); 58902f23f5fSAlex Deucher if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) { 590be99ecbfSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 591e3898719SChristian König cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : 592e3898719SChristian König gfxhub_client_ids[cid], 593be99ecbfSAlex Deucher cid); 59402f23f5fSAlex Deucher } else { 59502f23f5fSAlex Deucher switch (adev->asic_type) { 59602f23f5fSAlex Deucher case CHIP_VEGA10: 59702f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega10[cid][rw]; 59802f23f5fSAlex Deucher break; 59902f23f5fSAlex Deucher case CHIP_VEGA12: 60002f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega12[cid][rw]; 60102f23f5fSAlex Deucher break; 60202f23f5fSAlex Deucher case CHIP_VEGA20: 60302f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega20[cid][rw]; 60402f23f5fSAlex Deucher break; 60502f23f5fSAlex Deucher case CHIP_ARCTURUS: 60602f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; 60702f23f5fSAlex Deucher break; 60802f23f5fSAlex Deucher case CHIP_RAVEN: 60902f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_raven[cid][rw]; 61002f23f5fSAlex Deucher break; 61102f23f5fSAlex Deucher case CHIP_RENOIR: 61202f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_renoir[cid][rw]; 61302f23f5fSAlex Deucher break; 614e844cd99SAlex Deucher case CHIP_ALDEBARAN: 615e844cd99SAlex Deucher mmhub_cid = mmhub_client_ids_aldebaran[cid][rw]; 616e844cd99SAlex Deucher break; 61702f23f5fSAlex Deucher default: 61802f23f5fSAlex Deucher mmhub_cid = NULL; 61902f23f5fSAlex Deucher break; 62002f23f5fSAlex Deucher } 62102f23f5fSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 62202f23f5fSAlex Deucher mmhub_cid ? mmhub_cid : "unknown", cid); 62302f23f5fSAlex Deucher } 6245ddd4a9aSYong Zhao dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 6255ddd4a9aSYong Zhao REG_GET_FIELD(status, 6265ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 6275ddd4a9aSYong Zhao dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 6285ddd4a9aSYong Zhao REG_GET_FIELD(status, 6295ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 6305ddd4a9aSYong Zhao dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 6315ddd4a9aSYong Zhao REG_GET_FIELD(status, 6325ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 6335ddd4a9aSYong Zhao dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 6345ddd4a9aSYong Zhao REG_GET_FIELD(status, 6355ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 63602f23f5fSAlex Deucher dev_err(adev->dev, "\t RW: 0x%x\n", rw); 637e60f8db5SAlex Xie return 0; 638e60f8db5SAlex Xie } 639e60f8db5SAlex Xie 640e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 641e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 642e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 643e60f8db5SAlex Xie }; 644e60f8db5SAlex Xie 645791c4769Sxinhui pan 646791c4769Sxinhui pan static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 647791c4769Sxinhui pan .set = gmc_v9_0_ecc_interrupt_state, 64834cc4fd9STao Zhou .process = amdgpu_umc_process_ecc_irq, 649791c4769Sxinhui pan }; 650791c4769Sxinhui pan 651e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 652e60f8db5SAlex Xie { 653770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 654770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 655791c4769Sxinhui pan 6562ee9403eSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 657791c4769Sxinhui pan adev->gmc.ecc_irq.num_types = 1; 658791c4769Sxinhui pan adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 659e60f8db5SAlex Xie } 6602ee9403eSZhigang Luo } 661e60f8db5SAlex Xie 6622a79d868SYong Zhao static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 6632a79d868SYong Zhao uint32_t flush_type) 66403f89febSChristian König { 66503f89febSChristian König u32 req = 0; 66603f89febSChristian König 66703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 668c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 6692a79d868SYong Zhao req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 67003f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 67103f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 67203f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 67303f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 67403f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 67503f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 67603f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 67703f89febSChristian König 67803f89febSChristian König return req; 67903f89febSChristian König } 68003f89febSChristian König 68190f6452cSchangzhu /** 68290f6452cSchangzhu * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore 68390f6452cSchangzhu * 68490f6452cSchangzhu * @adev: amdgpu_device pointer 68590f6452cSchangzhu * @vmhub: vmhub type 68690f6452cSchangzhu * 68790f6452cSchangzhu */ 68890f6452cSchangzhu static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, 68990f6452cSchangzhu uint32_t vmhub) 69090f6452cSchangzhu { 691b7daed1bSAmber Lin if (adev->asic_type == CHIP_ALDEBARAN) 692d477c5aaSHawking Zhang return false; 693d477c5aaSHawking Zhang 69490f6452cSchangzhu return ((vmhub == AMDGPU_MMHUB_0 || 69590f6452cSchangzhu vmhub == AMDGPU_MMHUB_1) && 69690f6452cSchangzhu (!amdgpu_sriov_vf(adev)) && 69754f78a76SAlex Deucher (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && 69854f78a76SAlex Deucher (adev->apu_flags & AMD_APU_IS_PICASSO)))); 69990f6452cSchangzhu } 70090f6452cSchangzhu 701ea930000SAlex Sierra static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, 702ea930000SAlex Sierra uint8_t vmid, uint16_t *p_pasid) 703ea930000SAlex Sierra { 704ea930000SAlex Sierra uint32_t value; 705ea930000SAlex Sierra 706ea930000SAlex Sierra value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 707ea930000SAlex Sierra + vmid); 708ea930000SAlex Sierra *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 709ea930000SAlex Sierra 710ea930000SAlex Sierra return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 711ea930000SAlex Sierra } 712ea930000SAlex Sierra 713e60f8db5SAlex Xie /* 714e60f8db5SAlex Xie * GART 715e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 716e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 717e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 718e60f8db5SAlex Xie */ 719e60f8db5SAlex Xie 720e60f8db5SAlex Xie /** 7212a79d868SYong Zhao * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 722e60f8db5SAlex Xie * 723e60f8db5SAlex Xie * @adev: amdgpu_device pointer 724e60f8db5SAlex Xie * @vmid: vm instance to flush 725bf0df09cSLee Jones * @vmhub: which hub to flush 7262a79d868SYong Zhao * @flush_type: the flush type 727e60f8db5SAlex Xie * 7282a79d868SYong Zhao * Flush the TLB for the requested page table using certain type. 729e60f8db5SAlex Xie */ 7303ff98548SOak Zeng static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 7313ff98548SOak Zeng uint32_t vmhub, uint32_t flush_type) 732e60f8db5SAlex Xie { 73390f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); 734e60f8db5SAlex Xie const unsigned eng = 17; 735b80cd524SFelix Kuehling u32 j, inv_req, inv_req2, tmp; 7363ff98548SOak Zeng struct amdgpu_vmhub *hub; 737e60f8db5SAlex Xie 7383ff98548SOak Zeng BUG_ON(vmhub >= adev->num_vmhubs); 7393ff98548SOak Zeng 7403ff98548SOak Zeng hub = &adev->vmhub[vmhub]; 741b80cd524SFelix Kuehling if (adev->gmc.xgmi.num_physical_nodes && 742b80cd524SFelix Kuehling adev->asic_type == CHIP_VEGA20) { 743b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 744b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 745b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 746b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 747b80cd524SFelix Kuehling * still need a second TLB flush after this. 748b80cd524SFelix Kuehling */ 749b80cd524SFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); 750b80cd524SFelix Kuehling inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 751b80cd524SFelix Kuehling } else { 75237c58ddfSFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); 753b80cd524SFelix Kuehling inv_req2 = 0; 754b80cd524SFelix Kuehling } 755e60f8db5SAlex Xie 75682d1a1b1SChengming Gui /* This is necessary for a HW workaround under SRIOV as well 75782d1a1b1SChengming Gui * as GFXOFF under bare metal 75882d1a1b1SChengming Gui */ 75982d1a1b1SChengming Gui if (adev->gfx.kiq.ring.sched.ready && 76082d1a1b1SChengming Gui (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 76181202807SDennis Li down_read_trylock(&adev->reset_sem)) { 762148f597dSHuang Rui uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 763148f597dSHuang Rui uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 764af5fe1e9SChristian König 76537c58ddfSFelix Kuehling amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 766af5fe1e9SChristian König 1 << vmid); 76781202807SDennis Li up_read(&adev->reset_sem); 7683ff98548SOak Zeng return; 769fc0faf04SEmily Deng } 7703890d111SEmily Deng 7713890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 772f920d1bbSchangzhu 773f920d1bbSchangzhu /* 774f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 775f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 776f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 777f920d1bbSchangzhu * to WA the Issue 778f920d1bbSchangzhu */ 779f920d1bbSchangzhu 780f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 78190f6452cSchangzhu if (use_semaphore) { 782f920d1bbSchangzhu for (j = 0; j < adev->usec_timeout; j++) { 783f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 784148f597dSHuang Rui tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + 785148f597dSHuang Rui hub->eng_distance * eng); 786f920d1bbSchangzhu if (tmp & 0x1) 787f920d1bbSchangzhu break; 788f920d1bbSchangzhu udelay(1); 789f920d1bbSchangzhu } 790f920d1bbSchangzhu 791f920d1bbSchangzhu if (j >= adev->usec_timeout) 792f920d1bbSchangzhu DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 793f920d1bbSchangzhu } 794f920d1bbSchangzhu 795b80cd524SFelix Kuehling do { 796148f597dSHuang Rui WREG32_NO_KIQ(hub->vm_inv_eng0_req + 797148f597dSHuang Rui hub->eng_distance * eng, inv_req); 79853499173SXiaojie Yuan 79953499173SXiaojie Yuan /* 800b80cd524SFelix Kuehling * Issue a dummy read to wait for the ACK register to 801b80cd524SFelix Kuehling * be cleared to avoid a false ACK due to the new fast 802b80cd524SFelix Kuehling * GRBM interface. 80353499173SXiaojie Yuan */ 80453499173SXiaojie Yuan if (vmhub == AMDGPU_GFXHUB_0) 805148f597dSHuang Rui RREG32_NO_KIQ(hub->vm_inv_eng0_req + 806148f597dSHuang Rui hub->eng_distance * eng); 80753499173SXiaojie Yuan 808e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 809148f597dSHuang Rui tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + 810148f597dSHuang Rui hub->eng_distance * eng); 811396557b0SChristian König if (tmp & (1 << vmid)) 812e60f8db5SAlex Xie break; 813e60f8db5SAlex Xie udelay(1); 814e60f8db5SAlex Xie } 815f920d1bbSchangzhu 816b80cd524SFelix Kuehling inv_req = inv_req2; 817b80cd524SFelix Kuehling inv_req2 = 0; 818b80cd524SFelix Kuehling } while (inv_req); 819b80cd524SFelix Kuehling 820f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 82190f6452cSchangzhu if (use_semaphore) 822f920d1bbSchangzhu /* 823f920d1bbSchangzhu * add semaphore release after invalidation, 824f920d1bbSchangzhu * write with 0 means semaphore release 825f920d1bbSchangzhu */ 826148f597dSHuang Rui WREG32_NO_KIQ(hub->vm_inv_eng0_sem + 827148f597dSHuang Rui hub->eng_distance * eng, 0); 828f920d1bbSchangzhu 8293890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 830f920d1bbSchangzhu 831396557b0SChristian König if (j < adev->usec_timeout) 8323ff98548SOak Zeng return; 833396557b0SChristian König 834e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 835e60f8db5SAlex Xie } 836e60f8db5SAlex Xie 837ea930000SAlex Sierra /** 838ea930000SAlex Sierra * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid 839ea930000SAlex Sierra * 840ea930000SAlex Sierra * @adev: amdgpu_device pointer 841ea930000SAlex Sierra * @pasid: pasid to be flush 842bf0df09cSLee Jones * @flush_type: the flush type 843bf0df09cSLee Jones * @all_hub: flush all hubs 844ea930000SAlex Sierra * 845ea930000SAlex Sierra * Flush the TLB for the requested pasid. 846ea930000SAlex Sierra */ 847ea930000SAlex Sierra static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 848ea930000SAlex Sierra uint16_t pasid, uint32_t flush_type, 849ea930000SAlex Sierra bool all_hub) 850ea930000SAlex Sierra { 851ea930000SAlex Sierra int vmid, i; 852ea930000SAlex Sierra signed long r; 853ea930000SAlex Sierra uint32_t seq; 854ea930000SAlex Sierra uint16_t queried_pasid; 855ea930000SAlex Sierra bool ret; 856ea930000SAlex Sierra struct amdgpu_ring *ring = &adev->gfx.kiq.ring; 857ea930000SAlex Sierra struct amdgpu_kiq *kiq = &adev->gfx.kiq; 858ea930000SAlex Sierra 85953b3f8f4SDennis Li if (amdgpu_in_reset(adev)) 860ea930000SAlex Sierra return -EIO; 861ea930000SAlex Sierra 86281202807SDennis Li if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) { 863b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 864b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 865b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 866b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 867b80cd524SFelix Kuehling * still need a second TLB flush after this. 868b80cd524SFelix Kuehling */ 869b80cd524SFelix Kuehling bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && 870b80cd524SFelix Kuehling adev->asic_type == CHIP_VEGA20); 871b80cd524SFelix Kuehling /* 2 dwords flush + 8 dwords fence */ 872b80cd524SFelix Kuehling unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; 873b80cd524SFelix Kuehling 874b80cd524SFelix Kuehling if (vega20_xgmi_wa) 875b80cd524SFelix Kuehling ndw += kiq->pmf->invalidate_tlbs_size; 876b80cd524SFelix Kuehling 877ea930000SAlex Sierra spin_lock(&adev->gfx.kiq.ring_lock); 87836a1707aSAlex Sierra /* 2 dwords flush + 8 dwords fence */ 879b80cd524SFelix Kuehling amdgpu_ring_alloc(ring, ndw); 880b80cd524SFelix Kuehling if (vega20_xgmi_wa) 881b80cd524SFelix Kuehling kiq->pmf->kiq_invalidate_tlbs(ring, 882b80cd524SFelix Kuehling pasid, 2, all_hub); 883ea930000SAlex Sierra kiq->pmf->kiq_invalidate_tlbs(ring, 884ea930000SAlex Sierra pasid, flush_type, all_hub); 88504e4e2e9SYintian Tao r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 88604e4e2e9SYintian Tao if (r) { 88704e4e2e9SYintian Tao amdgpu_ring_undo(ring); 888abb17b1eSColin Ian King spin_unlock(&adev->gfx.kiq.ring_lock); 88981202807SDennis Li up_read(&adev->reset_sem); 89004e4e2e9SYintian Tao return -ETIME; 89104e4e2e9SYintian Tao } 89204e4e2e9SYintian Tao 893ea930000SAlex Sierra amdgpu_ring_commit(ring); 894ea930000SAlex Sierra spin_unlock(&adev->gfx.kiq.ring_lock); 895ea930000SAlex Sierra r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); 896ea930000SAlex Sierra if (r < 1) { 897aac89168SDennis Li dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); 89881202807SDennis Li up_read(&adev->reset_sem); 899ea930000SAlex Sierra return -ETIME; 900ea930000SAlex Sierra } 90181202807SDennis Li up_read(&adev->reset_sem); 902ea930000SAlex Sierra return 0; 903ea930000SAlex Sierra } 904ea930000SAlex Sierra 905ea930000SAlex Sierra for (vmid = 1; vmid < 16; vmid++) { 906ea930000SAlex Sierra 907ea930000SAlex Sierra ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 908ea930000SAlex Sierra &queried_pasid); 909ea930000SAlex Sierra if (ret && queried_pasid == pasid) { 910ea930000SAlex Sierra if (all_hub) { 911ea930000SAlex Sierra for (i = 0; i < adev->num_vmhubs; i++) 912ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 913fa34edbeSFelix Kuehling i, flush_type); 914ea930000SAlex Sierra } else { 915ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 916fa34edbeSFelix Kuehling AMDGPU_GFXHUB_0, flush_type); 917ea930000SAlex Sierra } 918ea930000SAlex Sierra break; 919ea930000SAlex Sierra } 920ea930000SAlex Sierra } 921ea930000SAlex Sierra 922ea930000SAlex Sierra return 0; 923ea930000SAlex Sierra 924ea930000SAlex Sierra } 925ea930000SAlex Sierra 9269096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 927c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 9289096d6e5SChristian König { 92990f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); 930250b4228SChristian König struct amdgpu_device *adev = ring->adev; 931250b4228SChristian König struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 9322a79d868SYong Zhao uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 9339096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 9349096d6e5SChristian König 935f920d1bbSchangzhu /* 936f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 937f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 938f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 939f920d1bbSchangzhu * to WA the Issue 940f920d1bbSchangzhu */ 941f920d1bbSchangzhu 942f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 94390f6452cSchangzhu if (use_semaphore) 944f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 945f920d1bbSchangzhu amdgpu_ring_emit_reg_wait(ring, 946148f597dSHuang Rui hub->vm_inv_eng0_sem + 947148f597dSHuang Rui hub->eng_distance * eng, 0x1, 0x1); 948f920d1bbSchangzhu 949148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 950148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 9519096d6e5SChristian König lower_32_bits(pd_addr)); 9529096d6e5SChristian König 953148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 954148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 9559096d6e5SChristian König upper_32_bits(pd_addr)); 9569096d6e5SChristian König 957148f597dSHuang Rui amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 958148f597dSHuang Rui hub->eng_distance * eng, 959148f597dSHuang Rui hub->vm_inv_eng0_ack + 960148f597dSHuang Rui hub->eng_distance * eng, 961f8bc9037SAlex Deucher req, 1 << vmid); 962f732b6b3SChristian König 963f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 96490f6452cSchangzhu if (use_semaphore) 965f920d1bbSchangzhu /* 966f920d1bbSchangzhu * add semaphore release after invalidation, 967f920d1bbSchangzhu * write with 0 means semaphore release 968f920d1bbSchangzhu */ 969148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 970148f597dSHuang Rui hub->eng_distance * eng, 0); 971f920d1bbSchangzhu 9729096d6e5SChristian König return pd_addr; 9739096d6e5SChristian König } 9749096d6e5SChristian König 975c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 976c633c00bSChristian König unsigned pasid) 977c633c00bSChristian König { 978c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 979c633c00bSChristian König uint32_t reg; 980c633c00bSChristian König 981f2d66571SLe Ma /* Do nothing because there's no lut register for mmhub1. */ 982f2d66571SLe Ma if (ring->funcs->vmhub == AMDGPU_MMHUB_1) 983f2d66571SLe Ma return; 984f2d66571SLe Ma 985a2d15ed7SLe Ma if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) 986c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 987c633c00bSChristian König else 988c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 989c633c00bSChristian König 990c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 991c633c00bSChristian König } 992c633c00bSChristian König 993e60f8db5SAlex Xie /* 994e60f8db5SAlex Xie * PTE format on VEGA 10: 995e60f8db5SAlex Xie * 63:59 reserved 996e60f8db5SAlex Xie * 58:57 mtype 997e60f8db5SAlex Xie * 56 F 998e60f8db5SAlex Xie * 55 L 999e60f8db5SAlex Xie * 54 P 1000e60f8db5SAlex Xie * 53 SW 1001e60f8db5SAlex Xie * 52 T 1002e60f8db5SAlex Xie * 50:48 reserved 1003e60f8db5SAlex Xie * 47:12 4k physical page base address 1004e60f8db5SAlex Xie * 11:7 fragment 1005e60f8db5SAlex Xie * 6 write 1006e60f8db5SAlex Xie * 5 read 1007e60f8db5SAlex Xie * 4 exe 1008e60f8db5SAlex Xie * 3 Z 1009e60f8db5SAlex Xie * 2 snooped 1010e60f8db5SAlex Xie * 1 system 1011e60f8db5SAlex Xie * 0 valid 1012e60f8db5SAlex Xie * 1013e60f8db5SAlex Xie * PDE format on VEGA 10: 1014e60f8db5SAlex Xie * 63:59 block fragment size 1015e60f8db5SAlex Xie * 58:55 reserved 1016e60f8db5SAlex Xie * 54 P 1017e60f8db5SAlex Xie * 53:48 reserved 1018e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 1019e60f8db5SAlex Xie * 5:3 reserved 1020e60f8db5SAlex Xie * 2 C 1021e60f8db5SAlex Xie * 1 system 1022e60f8db5SAlex Xie * 0 valid 1023e60f8db5SAlex Xie */ 1024e60f8db5SAlex Xie 102571776b6dSChristian König static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) 1026e60f8db5SAlex Xie 1027e60f8db5SAlex Xie { 102871776b6dSChristian König switch (flags) { 1029e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 103071776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1031e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 103271776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1033e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 103471776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); 1035093e48c0SOak Zeng case AMDGPU_VM_MTYPE_RW: 103671776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); 1037e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 103871776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); 1039e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 104071776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); 1041e60f8db5SAlex Xie default: 104271776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1043e60f8db5SAlex Xie } 1044e60f8db5SAlex Xie } 1045e60f8db5SAlex Xie 10463de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 10473de676d8SChristian König uint64_t *addr, uint64_t *flags) 1048f75e237cSChristian König { 1049bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 10503de676d8SChristian König *addr = adev->vm_manager.vram_base_offset + *addr - 1051770d13b1SChristian König adev->gmc.vram_start; 10523de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 10536a42fd6fSChristian König 1054770d13b1SChristian König if (!adev->gmc.translate_further) 10556a42fd6fSChristian König return; 10566a42fd6fSChristian König 10576a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 10586a42fd6fSChristian König /* Set the block fragment size */ 10596a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 10606a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 10616a42fd6fSChristian König 10626a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 10636a42fd6fSChristian König if (*flags & AMDGPU_PDE_PTE) 10646a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 10656a42fd6fSChristian König else 10666a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 10676a42fd6fSChristian König } 1068f75e237cSChristian König } 1069f75e237cSChristian König 1070cbfae36cSChristian König static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, 1071cbfae36cSChristian König struct amdgpu_bo_va_mapping *mapping, 1072cbfae36cSChristian König uint64_t *flags) 1073cbfae36cSChristian König { 1074cbfae36cSChristian König *flags &= ~AMDGPU_PTE_EXECUTABLE; 1075cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1076cbfae36cSChristian König 1077cbfae36cSChristian König *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1078cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; 1079cbfae36cSChristian König 1080cbfae36cSChristian König if (mapping->flags & AMDGPU_PTE_PRT) { 1081cbfae36cSChristian König *flags |= AMDGPU_PTE_PRT; 1082cbfae36cSChristian König *flags &= ~AMDGPU_PTE_VALID; 1083cbfae36cSChristian König } 1084cbfae36cSChristian König 10857ffe7238SYong Zhao if ((adev->asic_type == CHIP_ARCTURUS || 10867ffe7238SYong Zhao adev->asic_type == CHIP_ALDEBARAN) && 1087cbfae36cSChristian König !(*flags & AMDGPU_PTE_SYSTEM) && 1088cbfae36cSChristian König mapping->bo_va->is_xgmi) 1089cbfae36cSChristian König *flags |= AMDGPU_PTE_SNOOPED; 109072b4db0fSEric Huang 109172b4db0fSEric Huang if (adev->asic_type == CHIP_ALDEBARAN) 109272b4db0fSEric Huang *flags |= mapping->flags & AMDGPU_PTE_SNOOPED; 1093cbfae36cSChristian König } 1094cbfae36cSChristian König 10957b885f0eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 10967b885f0eSAlex Deucher { 10977b885f0eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 10987b885f0eSAlex Deucher unsigned size; 10997b885f0eSAlex Deucher 11007b885f0eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 11017b885f0eSAlex Deucher size = AMDGPU_VBIOS_VGA_ALLOCATION; 11027b885f0eSAlex Deucher } else { 11037b885f0eSAlex Deucher u32 viewport; 11047b885f0eSAlex Deucher 11057b885f0eSAlex Deucher switch (adev->asic_type) { 11067b885f0eSAlex Deucher case CHIP_RAVEN: 11077b885f0eSAlex Deucher case CHIP_RENOIR: 11087b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 11097b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, 11107b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 11117b885f0eSAlex Deucher REG_GET_FIELD(viewport, 11127b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 11137b885f0eSAlex Deucher 4); 11147b885f0eSAlex Deucher break; 11157b885f0eSAlex Deucher case CHIP_VEGA10: 11167b885f0eSAlex Deucher case CHIP_VEGA12: 11177b885f0eSAlex Deucher case CHIP_VEGA20: 11187b885f0eSAlex Deucher default: 11197b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 11207b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 11217b885f0eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 11227b885f0eSAlex Deucher 4); 11237b885f0eSAlex Deucher break; 11247b885f0eSAlex Deucher } 11257b885f0eSAlex Deucher } 11267b885f0eSAlex Deucher 11277b885f0eSAlex Deucher return size; 11287b885f0eSAlex Deucher } 11297b885f0eSAlex Deucher 1130132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 1131132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 1132ea930000SAlex Sierra .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, 11339096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 1134c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 113571776b6dSChristian König .map_mtype = gmc_v9_0_map_mtype, 1136cbfae36cSChristian König .get_vm_pde = gmc_v9_0_get_vm_pde, 11377b885f0eSAlex Deucher .get_vm_pte = gmc_v9_0_get_vm_pte, 11387b885f0eSAlex Deucher .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, 1139e60f8db5SAlex Xie }; 1140e60f8db5SAlex Xie 1141132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 1142e60f8db5SAlex Xie { 1143132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 1144e60f8db5SAlex Xie } 1145e60f8db5SAlex Xie 11465b6b35aaSHawking Zhang static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) 11475b6b35aaSHawking Zhang { 11485b6b35aaSHawking Zhang switch (adev->asic_type) { 1149e7da754bSMonk Liu case CHIP_VEGA10: 1150e7da754bSMonk Liu adev->umc.funcs = &umc_v6_0_funcs; 1151e7da754bSMonk Liu break; 11525b6b35aaSHawking Zhang case CHIP_VEGA20: 11533aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 11543aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 11553aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 11564cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; 11574cf781c2SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 1158*49070c4eSHawking Zhang adev->umc.ras_funcs = &umc_v6_1_ras_funcs; 11594cf781c2SJohn Clements break; 11609e612c11SHawking Zhang case CHIP_ARCTURUS: 11613aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 11623aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 11633aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 11644cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; 11653aacf4eaSTao Zhou adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 1166*49070c4eSHawking Zhang adev->umc.ras_funcs = &umc_v6_1_ras_funcs; 11675b6b35aaSHawking Zhang break; 11685b6b35aaSHawking Zhang default: 11695b6b35aaSHawking Zhang break; 11705b6b35aaSHawking Zhang } 11715b6b35aaSHawking Zhang } 11725b6b35aaSHawking Zhang 11733d093da0STao Zhou static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) 11743d093da0STao Zhou { 11753d093da0STao Zhou switch (adev->asic_type) { 1176f6c3623bSDennis Li case CHIP_ARCTURUS: 1177f6c3623bSDennis Li adev->mmhub.funcs = &mmhub_v9_4_funcs; 1178f6c3623bSDennis Li break; 11794da999cdSOak Zeng case CHIP_ALDEBARAN: 11804da999cdSOak Zeng adev->mmhub.funcs = &mmhub_v1_7_funcs; 11814da999cdSOak Zeng break; 11823d093da0STao Zhou default: 11839fb1506eSOak Zeng adev->mmhub.funcs = &mmhub_v1_0_funcs; 11843d093da0STao Zhou break; 11853d093da0STao Zhou } 11863d093da0STao Zhou } 11873d093da0STao Zhou 11888ffff9b4SOak Zeng static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev) 11898ffff9b4SOak Zeng { 11908ffff9b4SOak Zeng adev->gfxhub.funcs = &gfxhub_v1_0_funcs; 11918ffff9b4SOak Zeng } 11928ffff9b4SOak Zeng 1193e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 1194e60f8db5SAlex Xie { 1195e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1196e60f8db5SAlex Xie 119731691b8dSRajneesh Bhardwaj if (adev->asic_type == CHIP_VEGA20 || 119831691b8dSRajneesh Bhardwaj adev->asic_type == CHIP_ARCTURUS) 119931691b8dSRajneesh Bhardwaj adev->gmc.xgmi.supported = true; 120031691b8dSRajneesh Bhardwaj 120131691b8dSRajneesh Bhardwaj if (adev->asic_type == CHIP_ALDEBARAN) { 120231691b8dSRajneesh Bhardwaj adev->gmc.xgmi.supported = true; 120331691b8dSRajneesh Bhardwaj adev->gmc.xgmi.connected_to_cpu = 120431691b8dSRajneesh Bhardwaj adev->smuio.funcs->is_host_gpu_xgmi_supported(adev); 120531691b8dSRajneesh Bhardwaj } 120631691b8dSRajneesh Bhardwaj 1207*49070c4eSHawking Zhang gmc_v9_0_set_gmc_funcs(adev); 1208*49070c4eSHawking Zhang gmc_v9_0_set_irq_funcs(adev); 1209*49070c4eSHawking Zhang gmc_v9_0_set_umc_funcs(adev); 1210*49070c4eSHawking Zhang gmc_v9_0_set_mmhub_funcs(adev); 1211*49070c4eSHawking Zhang gmc_v9_0_set_gfxhub_funcs(adev); 1212*49070c4eSHawking Zhang 1213770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 1214770d13b1SChristian König adev->gmc.shared_aperture_end = 1215770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 1216bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 1217770d13b1SChristian König adev->gmc.private_aperture_end = 1218770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 1219a7ea6548SAlex Deucher 1220e60f8db5SAlex Xie return 0; 1221e60f8db5SAlex Xie } 1222e60f8db5SAlex Xie 1223e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 1224e60f8db5SAlex Xie { 1225e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1226c5b6e585STao Zhou int r; 12274789c463SChristian König 1228bdbe90f0SAlex Deucher r = amdgpu_gmc_allocate_vm_inv_eng(adev); 1229c713a461SEvan Quan if (r) 1230c713a461SEvan Quan return r; 12314a20300bSGuchun Chen 12324a20300bSGuchun Chen /* 12334a20300bSGuchun Chen * Workaround performance drop issue with VBIOS enables partial 12344a20300bSGuchun Chen * writes, while disables HBM ECC for vega10. 12354a20300bSGuchun Chen */ 123688474ccaSGuchun Chen if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) { 12374a20300bSGuchun Chen if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) { 1238bdf84a80SJoseph Greathouse if (adev->df.funcs->enable_ecc_force_par_wr_rmw) 1239bdf84a80SJoseph Greathouse adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); 12404a20300bSGuchun Chen } 1241f49ea9f8SHawking Zhang } 124202bab923SDavid Panariti 1243fe5211f1SHawking Zhang if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count) 1244fe5211f1SHawking Zhang adev->mmhub.funcs->reset_ras_error_count(adev); 1245fe5211f1SHawking Zhang 1246ba083492STao Zhou r = amdgpu_gmc_ras_late_init(adev); 1247791c4769Sxinhui pan if (r) 1248e60f8db5SAlex Xie return r; 1249e60f8db5SAlex Xie 1250770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 1251e60f8db5SAlex Xie } 1252e60f8db5SAlex Xie 1253e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 1254770d13b1SChristian König struct amdgpu_gmc *mc) 1255e60f8db5SAlex Xie { 1256e60f8db5SAlex Xie u64 base = 0; 12579d4f837aSFrank.Min 12589fb1506eSOak Zeng if (!amdgpu_sriov_vf(adev)) 12599fb1506eSOak Zeng base = adev->mmhub.funcs->get_fb_location(adev); 12609d4f837aSFrank.Min 12616fdd68b1SAlex Deucher /* add the xgmi offset of the physical node */ 12626fdd68b1SAlex Deucher base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1263f527f310SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 1264f527f310SOak Zeng amdgpu_gmc_sysvm_location(adev, mc); 1265f527f310SOak Zeng } else { 126683afe835SOak Zeng amdgpu_gmc_vram_location(adev, mc, base); 1267961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 1268c3e1b43cSChristian König amdgpu_gmc_agp_location(adev, mc); 1269f527f310SOak Zeng } 1270e60f8db5SAlex Xie /* base offset of vram pages */ 12718ffff9b4SOak Zeng adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); 12726fdd68b1SAlex Deucher 12736fdd68b1SAlex Deucher /* XXX: add the xgmi offset of the physical node? */ 12746fdd68b1SAlex Deucher adev->vm_manager.vram_base_offset += 12756fdd68b1SAlex Deucher adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1276e60f8db5SAlex Xie } 1277e60f8db5SAlex Xie 1278e60f8db5SAlex Xie /** 1279e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 1280e60f8db5SAlex Xie * 1281e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1282e60f8db5SAlex Xie * 1283e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 1284e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 1285e60f8db5SAlex Xie * Returns 0 for success. 1286e60f8db5SAlex Xie */ 1287e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 1288e60f8db5SAlex Xie { 1289e60f8db5SAlex Xie int r; 1290e60f8db5SAlex Xie 1291e60f8db5SAlex Xie /* size in MB on si */ 1292770d13b1SChristian König adev->gmc.mc_vram_size = 1293bebc0762SHawking Zhang adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 1294770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 1295e60f8db5SAlex Xie 1296be566196SOak Zeng if (!(adev->flags & AMD_IS_APU) && 1297be566196SOak Zeng !adev->gmc.xgmi.connected_to_cpu) { 1298e60f8db5SAlex Xie r = amdgpu_device_resize_fb_bar(adev); 1299e60f8db5SAlex Xie if (r) 1300e60f8db5SAlex Xie return r; 1301e60f8db5SAlex Xie } 1302770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 1303770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 1304e60f8db5SAlex Xie 1305156a81beSChunming Zhou #ifdef CONFIG_X86_64 130631691b8dSRajneesh Bhardwaj /* 130731691b8dSRajneesh Bhardwaj * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi 130831691b8dSRajneesh Bhardwaj * interface can use VRAM through here as it appears system reserved 130931691b8dSRajneesh Bhardwaj * memory in host address space. 131031691b8dSRajneesh Bhardwaj * 131131691b8dSRajneesh Bhardwaj * For APUs, VRAM is just the stolen system memory and can be accessed 131231691b8dSRajneesh Bhardwaj * directly. 131331691b8dSRajneesh Bhardwaj * 131431691b8dSRajneesh Bhardwaj * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR. 131531691b8dSRajneesh Bhardwaj */ 131631691b8dSRajneesh Bhardwaj 131731691b8dSRajneesh Bhardwaj /* check whether both host-gpu and gpu-gpu xgmi links exist */ 13183de60d96SHawking Zhang if ((adev->flags & AMD_IS_APU) || 13193de60d96SHawking Zhang (adev->gmc.xgmi.supported && 13203de60d96SHawking Zhang adev->gmc.xgmi.connected_to_cpu)) { 13213de60d96SHawking Zhang adev->gmc.aper_base = 13223de60d96SHawking Zhang adev->gfxhub.funcs->get_mc_fb_offset(adev) + 13233de60d96SHawking Zhang adev->gmc.xgmi.physical_node_id * 132431691b8dSRajneesh Bhardwaj adev->gmc.xgmi.node_segment_size; 1325156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 1326156a81beSChunming Zhou } 132731691b8dSRajneesh Bhardwaj 1328156a81beSChunming Zhou #endif 1329e60f8db5SAlex Xie /* In case the PCI BAR is larger than the actual amount of vram */ 1330770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 1331770d13b1SChristian König if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 1332770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 1333e60f8db5SAlex Xie 1334e60f8db5SAlex Xie /* set the gart size */ 1335e60f8db5SAlex Xie if (amdgpu_gart_size == -1) { 1336e60f8db5SAlex Xie switch (adev->asic_type) { 1337e60f8db5SAlex Xie case CHIP_VEGA10: /* all engines support GPUVM */ 1338273a14cdSAlex Deucher case CHIP_VEGA12: /* all engines support GPUVM */ 1339d96b428cSFeifei Xu case CHIP_VEGA20: 13403de2ff5dSLe Ma case CHIP_ARCTURUS: 134185e39550SLe Ma case CHIP_ALDEBARAN: 1342e60f8db5SAlex Xie default: 1343fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 1344e60f8db5SAlex Xie break; 1345e60f8db5SAlex Xie case CHIP_RAVEN: /* DCE SG support */ 13468787ee01SHuang Rui case CHIP_RENOIR: 1347770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 1348e60f8db5SAlex Xie break; 1349e60f8db5SAlex Xie } 1350e60f8db5SAlex Xie } else { 1351770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 1352e60f8db5SAlex Xie } 1353e60f8db5SAlex Xie 1354f1dc12caSOak Zeng adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; 1355f1dc12caSOak Zeng 1356770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 1357e60f8db5SAlex Xie 1358e60f8db5SAlex Xie return 0; 1359e60f8db5SAlex Xie } 1360e60f8db5SAlex Xie 1361e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 1362e60f8db5SAlex Xie { 1363e60f8db5SAlex Xie int r; 1364e60f8db5SAlex Xie 13651123b989SChristian König if (adev->gart.bo) { 1366e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 1367e60f8db5SAlex Xie return 0; 1368e60f8db5SAlex Xie } 13697b454b3aSOak Zeng 13707b454b3aSOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 13717b454b3aSOak Zeng adev->gmc.vmid0_page_table_depth = 1; 13727b454b3aSOak Zeng adev->gmc.vmid0_page_table_block_size = 12; 13737b454b3aSOak Zeng } else { 13747b454b3aSOak Zeng adev->gmc.vmid0_page_table_depth = 0; 13757b454b3aSOak Zeng adev->gmc.vmid0_page_table_block_size = 0; 13767b454b3aSOak Zeng } 13777b454b3aSOak Zeng 1378e60f8db5SAlex Xie /* Initialize common gart structure */ 1379e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 1380e60f8db5SAlex Xie if (r) 1381e60f8db5SAlex Xie return r; 1382e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 13837596ab68SHawking Zhang adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | 1384e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 1385522510a6SOak Zeng 1386522510a6SOak Zeng r = amdgpu_gart_table_vram_alloc(adev); 1387522510a6SOak Zeng if (r) 1388522510a6SOak Zeng return r; 1389522510a6SOak Zeng 1390522510a6SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 1391522510a6SOak Zeng r = amdgpu_gmc_pdb0_alloc(adev); 1392522510a6SOak Zeng } 1393522510a6SOak Zeng 1394522510a6SOak Zeng return r; 1395e60f8db5SAlex Xie } 1396e60f8db5SAlex Xie 1397b0a2db9bSAlex Deucher /** 1398b0a2db9bSAlex Deucher * gmc_v9_0_save_registers - saves regs 1399b0a2db9bSAlex Deucher * 1400b0a2db9bSAlex Deucher * @adev: amdgpu_device pointer 1401b0a2db9bSAlex Deucher * 1402b0a2db9bSAlex Deucher * This saves potential register values that should be 1403b0a2db9bSAlex Deucher * restored upon resume 1404b0a2db9bSAlex Deucher */ 1405b0a2db9bSAlex Deucher static void gmc_v9_0_save_registers(struct amdgpu_device *adev) 1406ebdef28eSAlex Deucher { 1407b0a2db9bSAlex Deucher if (adev->asic_type == CHIP_RAVEN) 1408b0a2db9bSAlex Deucher adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); 1409ebdef28eSAlex Deucher } 1410ebdef28eSAlex Deucher 1411e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 1412e60f8db5SAlex Xie { 1413ad02e08eSOri Messinger int r, vram_width = 0, vram_type = 0, vram_vendor = 0; 1414e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1415e60f8db5SAlex Xie 14168ffff9b4SOak Zeng adev->gfxhub.funcs->init(adev); 14179fb1506eSOak Zeng 14189fb1506eSOak Zeng adev->mmhub.funcs->init(adev); 1419e60f8db5SAlex Xie 1420770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 1421e60f8db5SAlex Xie 1422ad02e08eSOri Messinger r = amdgpu_atomfirmware_get_vram_info(adev, 1423ad02e08eSOri Messinger &vram_width, &vram_type, &vram_vendor); 1424631cdbd2SAlex Deucher if (amdgpu_sriov_vf(adev)) 1425631cdbd2SAlex Deucher /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, 1426631cdbd2SAlex Deucher * and DF related registers is not readable, seems hardcord is the 1427631cdbd2SAlex Deucher * only way to set the correct vram_width 1428631cdbd2SAlex Deucher */ 1429631cdbd2SAlex Deucher adev->gmc.vram_width = 2048; 1430631cdbd2SAlex Deucher else if (amdgpu_emu_mode != 1) 1431631cdbd2SAlex Deucher adev->gmc.vram_width = vram_width; 1432631cdbd2SAlex Deucher 1433631cdbd2SAlex Deucher if (!adev->gmc.vram_width) { 1434631cdbd2SAlex Deucher int chansize, numchan; 1435631cdbd2SAlex Deucher 1436631cdbd2SAlex Deucher /* hbm memory channel size */ 1437631cdbd2SAlex Deucher if (adev->flags & AMD_IS_APU) 1438631cdbd2SAlex Deucher chansize = 64; 1439631cdbd2SAlex Deucher else 1440631cdbd2SAlex Deucher chansize = 128; 1441631cdbd2SAlex Deucher 1442bdf84a80SJoseph Greathouse numchan = adev->df.funcs->get_hbm_channel_number(adev); 1443631cdbd2SAlex Deucher adev->gmc.vram_width = numchan * chansize; 1444631cdbd2SAlex Deucher } 1445631cdbd2SAlex Deucher 1446631cdbd2SAlex Deucher adev->gmc.vram_type = vram_type; 1447ad02e08eSOri Messinger adev->gmc.vram_vendor = vram_vendor; 1448e60f8db5SAlex Xie switch (adev->asic_type) { 1449e60f8db5SAlex Xie case CHIP_RAVEN: 14501daa2bfaSLe Ma adev->num_vmhubs = 2; 14511daa2bfaSLe Ma 14526a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 1453f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 14546a42fd6fSChristian König } else { 14556a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 14566a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 1457770d13b1SChristian König adev->gmc.translate_further = 14586a42fd6fSChristian König adev->vm_manager.num_level > 1; 14596a42fd6fSChristian König } 1460e60f8db5SAlex Xie break; 1461e60f8db5SAlex Xie case CHIP_VEGA10: 1462273a14cdSAlex Deucher case CHIP_VEGA12: 1463d96b428cSFeifei Xu case CHIP_VEGA20: 14648787ee01SHuang Rui case CHIP_RENOIR: 146585e39550SLe Ma case CHIP_ALDEBARAN: 14661daa2bfaSLe Ma adev->num_vmhubs = 2; 14671daa2bfaSLe Ma 14688787ee01SHuang Rui 1469e60f8db5SAlex Xie /* 1470e60f8db5SAlex Xie * To fulfill 4-level page support, 1471e60f8db5SAlex Xie * vm size is 256TB (48bit), maximum size of Vega10, 1472e60f8db5SAlex Xie * block size 512 (9bit) 1473e60f8db5SAlex Xie */ 1474cdba61daSwentalou /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 1475cdba61daSwentalou if (amdgpu_sriov_vf(adev)) 1476cdba61daSwentalou amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 1477cdba61daSwentalou else 1478f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1479e60f8db5SAlex Xie break; 14803de2ff5dSLe Ma case CHIP_ARCTURUS: 1481c8a6e2a3SLe Ma adev->num_vmhubs = 3; 1482c8a6e2a3SLe Ma 14833de2ff5dSLe Ma /* Keep the vm size same with Vega20 */ 14843de2ff5dSLe Ma amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 14853de2ff5dSLe Ma break; 1486e60f8db5SAlex Xie default: 1487e60f8db5SAlex Xie break; 1488e60f8db5SAlex Xie } 1489e60f8db5SAlex Xie 1490e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 149144a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 1492770d13b1SChristian König &adev->gmc.vm_fault); 149330da7bb1SChristian König if (r) 149430da7bb1SChristian König return r; 149530da7bb1SChristian König 14967d19b15fSLe Ma if (adev->asic_type == CHIP_ARCTURUS) { 14977d19b15fSLe Ma r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, 14987d19b15fSLe Ma &adev->gmc.vm_fault); 14997d19b15fSLe Ma if (r) 15007d19b15fSLe Ma return r; 15017d19b15fSLe Ma } 15027d19b15fSLe Ma 150344a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 1504770d13b1SChristian König &adev->gmc.vm_fault); 1505e60f8db5SAlex Xie 1506e60f8db5SAlex Xie if (r) 1507e60f8db5SAlex Xie return r; 1508e60f8db5SAlex Xie 15092ee9403eSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 1510791c4769Sxinhui pan /* interrupt sent to DF. */ 1511791c4769Sxinhui pan r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 1512791c4769Sxinhui pan &adev->gmc.ecc_irq); 1513791c4769Sxinhui pan if (r) 1514791c4769Sxinhui pan return r; 15152ee9403eSZhigang Luo } 1516791c4769Sxinhui pan 1517e60f8db5SAlex Xie /* Set the internal MC address mask 1518e60f8db5SAlex Xie * This is the max address of the GPU's 1519e60f8db5SAlex Xie * internal address space. 1520e60f8db5SAlex Xie */ 1521770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 1522e60f8db5SAlex Xie 1523244511f3SChristoph Hellwig r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); 1524e60f8db5SAlex Xie if (r) { 1525e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 1526244511f3SChristoph Hellwig return r; 1527e60f8db5SAlex Xie } 1528244511f3SChristoph Hellwig adev->need_swiotlb = drm_need_swiotlb(44); 1529e60f8db5SAlex Xie 153047622ba0SAlex Deucher if (adev->gmc.xgmi.supported) { 15318ffff9b4SOak Zeng r = adev->gfxhub.funcs->get_xgmi_info(adev); 1532bf0a60b7SAlex Deucher if (r) 1533bf0a60b7SAlex Deucher return r; 1534bf0a60b7SAlex Deucher } 1535bf0a60b7SAlex Deucher 1536e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 1537e60f8db5SAlex Xie if (r) 1538e60f8db5SAlex Xie return r; 1539e60f8db5SAlex Xie 15407b885f0eSAlex Deucher amdgpu_gmc_get_vbios_allocations(adev); 1541ebdef28eSAlex Deucher 1542e60f8db5SAlex Xie /* Memory manager */ 1543e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 1544e60f8db5SAlex Xie if (r) 1545e60f8db5SAlex Xie return r; 1546e60f8db5SAlex Xie 1547e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 1548e60f8db5SAlex Xie if (r) 1549e60f8db5SAlex Xie return r; 1550e60f8db5SAlex Xie 155105ec3edaSChristian König /* 155205ec3edaSChristian König * number of VMs 155305ec3edaSChristian König * VMID 0 is reserved for System 155481659b20SFelix Kuehling * amdgpu graphics/compute will use VMIDs 1..n-1 155581659b20SFelix Kuehling * amdkfd will use VMIDs n..15 155681659b20SFelix Kuehling * 155781659b20SFelix Kuehling * The first KFD VMID is 8 for GPUs with graphics, 3 for 155881659b20SFelix Kuehling * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs 155981659b20SFelix Kuehling * for video processing. 156005ec3edaSChristian König */ 156181659b20SFelix Kuehling adev->vm_manager.first_kfd_vmid = 15626dce50b1SFelix Kuehling (adev->asic_type == CHIP_ARCTURUS || 15636dce50b1SFelix Kuehling adev->asic_type == CHIP_ALDEBARAN) ? 3 : 8; 156405ec3edaSChristian König 156505ec3edaSChristian König amdgpu_vm_manager_init(adev); 156605ec3edaSChristian König 1567b0a2db9bSAlex Deucher gmc_v9_0_save_registers(adev); 1568b0a2db9bSAlex Deucher 156905ec3edaSChristian König return 0; 1570e60f8db5SAlex Xie } 1571e60f8db5SAlex Xie 1572e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 1573e60f8db5SAlex Xie { 1574e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1575e60f8db5SAlex Xie 15762adf1344STao Zhou amdgpu_gmc_ras_fini(adev); 1577f59548c8SMonk Liu amdgpu_gem_force_release(adev); 1578e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 1579a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 158037c49dedSLijo Lazar amdgpu_bo_unref(&adev->gmc.pdb0_bo); 1581e60f8db5SAlex Xie amdgpu_bo_fini(adev); 1582a3d9103eSAndrey Grodzovsky amdgpu_gart_fini(adev); 1583e60f8db5SAlex Xie 1584e60f8db5SAlex Xie return 0; 1585e60f8db5SAlex Xie } 1586e60f8db5SAlex Xie 1587e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1588e60f8db5SAlex Xie { 1589946a4d5bSShaoyun Liu 1590e60f8db5SAlex Xie switch (adev->asic_type) { 1591e60f8db5SAlex Xie case CHIP_VEGA10: 15924cd4c5c0SMonk Liu if (amdgpu_sriov_vf(adev)) 159398cad2deSTrigger Huang break; 1594df561f66SGustavo A. R. Silva fallthrough; 1595d96b428cSFeifei Xu case CHIP_VEGA20: 1596946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 15975c583018SEvan Quan golden_settings_mmhub_1_0_0, 1598c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1599946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 16005c583018SEvan Quan golden_settings_athub_1_0_0, 1601c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1602e60f8db5SAlex Xie break; 1603273a14cdSAlex Deucher case CHIP_VEGA12: 1604273a14cdSAlex Deucher break; 1605e4f3abaaSChunming Zhou case CHIP_RAVEN: 16068787ee01SHuang Rui /* TODO for renoir */ 1607946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 16085c583018SEvan Quan golden_settings_athub_1_0_0, 1609c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 1610e4f3abaaSChunming Zhou break; 1611e60f8db5SAlex Xie default: 1612e60f8db5SAlex Xie break; 1613e60f8db5SAlex Xie } 1614e60f8db5SAlex Xie } 1615e60f8db5SAlex Xie 1616e60f8db5SAlex Xie /** 1617c2ecd79bSShirish S * gmc_v9_0_restore_registers - restores regs 1618c2ecd79bSShirish S * 1619c2ecd79bSShirish S * @adev: amdgpu_device pointer 1620c2ecd79bSShirish S * 1621c2ecd79bSShirish S * This restores register values, saved at suspend. 1622c2ecd79bSShirish S */ 1623b0a2db9bSAlex Deucher void gmc_v9_0_restore_registers(struct amdgpu_device *adev) 1624c2ecd79bSShirish S { 16250eaa8012SShirish S if (adev->asic_type == CHIP_RAVEN) { 1626f8646661SAlex Deucher WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); 16270eaa8012SShirish S WARN_ON(adev->gmc.sdpif_register != 16280eaa8012SShirish S RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0)); 16290eaa8012SShirish S } 1630c2ecd79bSShirish S } 1631c2ecd79bSShirish S 1632c2ecd79bSShirish S /** 1633e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 1634e60f8db5SAlex Xie * 1635e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1636e60f8db5SAlex Xie */ 1637e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1638e60f8db5SAlex Xie { 1639cb1545f7SOak Zeng int r; 1640e60f8db5SAlex Xie 1641522510a6SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) 1642522510a6SOak Zeng amdgpu_gmc_init_pdb0(adev); 1643522510a6SOak Zeng 16441123b989SChristian König if (adev->gart.bo == NULL) { 1645e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1646e60f8db5SAlex Xie return -EINVAL; 1647e60f8db5SAlex Xie } 1648522510a6SOak Zeng 1649ce1b1b66SMonk Liu r = amdgpu_gart_table_vram_pin(adev); 1650ce1b1b66SMonk Liu if (r) 1651ce1b1b66SMonk Liu return r; 1652e60f8db5SAlex Xie 16538ffff9b4SOak Zeng r = adev->gfxhub.funcs->gart_enable(adev); 1654e60f8db5SAlex Xie if (r) 1655e60f8db5SAlex Xie return r; 1656e60f8db5SAlex Xie 16579fb1506eSOak Zeng r = adev->mmhub.funcs->gart_enable(adev); 1658e60f8db5SAlex Xie if (r) 1659e60f8db5SAlex Xie return r; 1660e60f8db5SAlex Xie 1661522510a6SOak Zeng DRM_INFO("PCIE GART of %uM enabled.\n", 1662522510a6SOak Zeng (unsigned)(adev->gmc.gart_size >> 20)); 1663522510a6SOak Zeng if (adev->gmc.pdb0_bo) 1664522510a6SOak Zeng DRM_INFO("PDB0 located at 0x%016llX\n", 1665522510a6SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo)); 1666522510a6SOak Zeng DRM_INFO("PTB located at 0x%016llX\n", 1667cb1545f7SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1668522510a6SOak Zeng 1669cb1545f7SOak Zeng adev->gart.ready = true; 1670cb1545f7SOak Zeng return 0; 1671cb1545f7SOak Zeng } 1672cb1545f7SOak Zeng 1673cb1545f7SOak Zeng static int gmc_v9_0_hw_init(void *handle) 1674cb1545f7SOak Zeng { 1675cb1545f7SOak Zeng struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1676cb1545f7SOak Zeng bool value; 1677cb1545f7SOak Zeng int r, i; 1678cb1545f7SOak Zeng 1679cb1545f7SOak Zeng /* The sequence of these two function calls matters.*/ 1680cb1545f7SOak Zeng gmc_v9_0_init_golden_registers(adev); 1681cb1545f7SOak Zeng 1682cb1545f7SOak Zeng if (adev->mode_info.num_crtc) { 1683cb1545f7SOak Zeng /* Lockout access through VGA aperture*/ 1684cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1685cb1545f7SOak Zeng /* disable VGA render */ 1686cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1687cb1545f7SOak Zeng } 1688cb1545f7SOak Zeng 16899fb1506eSOak Zeng if (adev->mmhub.funcs->update_power_gating) 16909fb1506eSOak Zeng adev->mmhub.funcs->update_power_gating(adev, true); 16919fb1506eSOak Zeng 1692455d40c9SLikun Gao adev->hdp.funcs->init_registers(adev); 1693fe2b5323STiecheng Zhou 16941d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 1695455d40c9SLikun Gao adev->hdp.funcs->flush_hdp(adev, NULL); 16961d4e0a8cSMonk Liu 1697e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1698e60f8db5SAlex Xie value = false; 1699e60f8db5SAlex Xie else 1700e60f8db5SAlex Xie value = true; 1701e60f8db5SAlex Xie 170220bf2f6fSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 17038ffff9b4SOak Zeng adev->gfxhub.funcs->set_fault_enable_default(adev, value); 17049fb1506eSOak Zeng adev->mmhub.funcs->set_fault_enable_default(adev, value); 170520bf2f6fSZhigang Luo } 17063ff98548SOak Zeng for (i = 0; i < adev->num_vmhubs; ++i) 17073ff98548SOak Zeng gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); 1708e60f8db5SAlex Xie 1709e7da754bSMonk Liu if (adev->umc.funcs && adev->umc.funcs->init_registers) 1710e7da754bSMonk Liu adev->umc.funcs->init_registers(adev); 1711e7da754bSMonk Liu 1712e60f8db5SAlex Xie r = gmc_v9_0_gart_enable(adev); 1713e60f8db5SAlex Xie 1714e60f8db5SAlex Xie return r; 1715e60f8db5SAlex Xie } 1716e60f8db5SAlex Xie 1717e60f8db5SAlex Xie /** 1718e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 1719e60f8db5SAlex Xie * 1720e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1721e60f8db5SAlex Xie * 1722e60f8db5SAlex Xie * This disables all VM page table. 1723e60f8db5SAlex Xie */ 1724e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1725e60f8db5SAlex Xie { 17268ffff9b4SOak Zeng adev->gfxhub.funcs->gart_disable(adev); 17279fb1506eSOak Zeng adev->mmhub.funcs->gart_disable(adev); 1728ce1b1b66SMonk Liu amdgpu_gart_table_vram_unpin(adev); 1729e60f8db5SAlex Xie } 1730e60f8db5SAlex Xie 1731e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 1732e60f8db5SAlex Xie { 1733e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1734e60f8db5SAlex Xie 17355dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 17365dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 17375dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 17385dd696aeSTrigger Huang return 0; 17395dd696aeSTrigger Huang } 17405dd696aeSTrigger Huang 1741791c4769Sxinhui pan amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1742770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1743e60f8db5SAlex Xie gmc_v9_0_gart_disable(adev); 1744e60f8db5SAlex Xie 1745e60f8db5SAlex Xie return 0; 1746e60f8db5SAlex Xie } 1747e60f8db5SAlex Xie 1748e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 1749e60f8db5SAlex Xie { 1750e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1751e60f8db5SAlex Xie 1752c24a3c05SLiu Shixin return gmc_v9_0_hw_fini(adev); 1753e60f8db5SAlex Xie } 1754e60f8db5SAlex Xie 1755e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 1756e60f8db5SAlex Xie { 1757e60f8db5SAlex Xie int r; 1758e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1759e60f8db5SAlex Xie 1760e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 1761e60f8db5SAlex Xie if (r) 1762e60f8db5SAlex Xie return r; 1763e60f8db5SAlex Xie 1764620f774fSChristian König amdgpu_vmid_reset_all(adev); 1765e60f8db5SAlex Xie 176632601d48SChristian König return 0; 1767e60f8db5SAlex Xie } 1768e60f8db5SAlex Xie 1769e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 1770e60f8db5SAlex Xie { 1771e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 1772e60f8db5SAlex Xie return true; 1773e60f8db5SAlex Xie } 1774e60f8db5SAlex Xie 1775e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 1776e60f8db5SAlex Xie { 1777e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 1778e60f8db5SAlex Xie return 0; 1779e60f8db5SAlex Xie } 1780e60f8db5SAlex Xie 1781e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 1782e60f8db5SAlex Xie { 1783e60f8db5SAlex Xie /* XXX for emulation.*/ 1784e60f8db5SAlex Xie return 0; 1785e60f8db5SAlex Xie } 1786e60f8db5SAlex Xie 1787e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 1788e60f8db5SAlex Xie enum amd_clockgating_state state) 1789e60f8db5SAlex Xie { 1790d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1791d5583d4fSHuang Rui 17929fb1506eSOak Zeng adev->mmhub.funcs->set_clockgating(adev, state); 1793bee7b51aSLe Ma 1794bee7b51aSLe Ma athub_v1_0_set_clockgating(adev, state); 1795bee7b51aSLe Ma 1796bee7b51aSLe Ma return 0; 1797e60f8db5SAlex Xie } 1798e60f8db5SAlex Xie 179913052be5SHuang Rui static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 180013052be5SHuang Rui { 180113052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 180213052be5SHuang Rui 18039fb1506eSOak Zeng adev->mmhub.funcs->get_clockgating(adev, flags); 1804bee7b51aSLe Ma 1805bee7b51aSLe Ma athub_v1_0_get_clockgating(adev, flags); 180613052be5SHuang Rui } 180713052be5SHuang Rui 1808e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 1809e60f8db5SAlex Xie enum amd_powergating_state state) 1810e60f8db5SAlex Xie { 1811e60f8db5SAlex Xie return 0; 1812e60f8db5SAlex Xie } 1813e60f8db5SAlex Xie 1814e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1815e60f8db5SAlex Xie .name = "gmc_v9_0", 1816e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 1817e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 1818e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 1819e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 1820e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 1821e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 1822e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 1823e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 1824e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 1825e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 1826e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 1827e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1828e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 182913052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1830e60f8db5SAlex Xie }; 1831e60f8db5SAlex Xie 1832e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1833e60f8db5SAlex Xie { 1834e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 1835e60f8db5SAlex Xie .major = 9, 1836e60f8db5SAlex Xie .minor = 0, 1837e60f8db5SAlex Xie .rev = 0, 1838e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 1839e60f8db5SAlex Xie }; 1840