1e60f8db5SAlex Xie /* 2e60f8db5SAlex Xie * Copyright 2016 Advanced Micro Devices, Inc. 3e60f8db5SAlex Xie * 4e60f8db5SAlex Xie * Permission is hereby granted, free of charge, to any person obtaining a 5e60f8db5SAlex Xie * copy of this software and associated documentation files (the "Software"), 6e60f8db5SAlex Xie * to deal in the Software without restriction, including without limitation 7e60f8db5SAlex Xie * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8e60f8db5SAlex Xie * and/or sell copies of the Software, and to permit persons to whom the 9e60f8db5SAlex Xie * Software is furnished to do so, subject to the following conditions: 10e60f8db5SAlex Xie * 11e60f8db5SAlex Xie * The above copyright notice and this permission notice shall be included in 12e60f8db5SAlex Xie * all copies or substantial portions of the Software. 13e60f8db5SAlex Xie * 14e60f8db5SAlex Xie * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15e60f8db5SAlex Xie * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16e60f8db5SAlex Xie * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17e60f8db5SAlex Xie * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18e60f8db5SAlex Xie * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19e60f8db5SAlex Xie * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20e60f8db5SAlex Xie * OTHER DEALINGS IN THE SOFTWARE. 21e60f8db5SAlex Xie * 22e60f8db5SAlex Xie */ 23f867723bSSam Ravnborg 24e60f8db5SAlex Xie #include <linux/firmware.h> 25f867723bSSam Ravnborg #include <linux/pci.h> 26f867723bSSam Ravnborg 27fd5fd480SChunming Zhou #include <drm/drm_cache.h> 28f867723bSSam Ravnborg 29e60f8db5SAlex Xie #include "amdgpu.h" 30e60f8db5SAlex Xie #include "gmc_v9_0.h" 318d6a5230SAlex Deucher #include "amdgpu_atomfirmware.h" 322cddc50eSHuang Rui #include "amdgpu_gem.h" 33e60f8db5SAlex Xie 34cde5c34fSFeifei Xu #include "gc/gc_9_0_sh_mask.h" 35135d4b10SFeifei Xu #include "dce/dce_12_0_offset.h" 36135d4b10SFeifei Xu #include "dce/dce_12_0_sh_mask.h" 37fb960bd2SFeifei Xu #include "vega10_enum.h" 3865417d9fSFeifei Xu #include "mmhub/mmhub_1_0_offset.h" 39ea930000SAlex Sierra #include "athub/athub_1_0_sh_mask.h" 406ce68225SFeifei Xu #include "athub/athub_1_0_offset.h" 41250b4228SChristian König #include "oss/osssys_4_0_offset.h" 42e60f8db5SAlex Xie 43946a4d5bSShaoyun Liu #include "soc15.h" 44ea930000SAlex Sierra #include "soc15d.h" 45e60f8db5SAlex Xie #include "soc15_common.h" 4690c7a935SFeifei Xu #include "umc/umc_6_0_sh_mask.h" 47e60f8db5SAlex Xie 48e60f8db5SAlex Xie #include "gfxhub_v1_0.h" 49e60f8db5SAlex Xie #include "mmhub_v1_0.h" 50bee7b51aSLe Ma #include "athub_v1_0.h" 51bf0a60b7SAlex Deucher #include "gfxhub_v1_1.h" 52018f7300SLe Ma #include "gfxhub_v1_2.h" 5351cce480SLe Ma #include "mmhub_v9_4.h" 5485e39550SLe Ma #include "mmhub_v1_7.h" 55018f7300SLe Ma #include "mmhub_v1_8.h" 565b6b35aaSHawking Zhang #include "umc_v6_1.h" 57e7da754bSMonk Liu #include "umc_v6_0.h" 58186c8a85SJohn Clements #include "umc_v6_7.h" 596f12507fSHawking Zhang #include "hdp_v4_0.h" 603907c492SJohn Clements #include "mca_v3_0.h" 61e60f8db5SAlex Xie 6244a99b65SAndrey Grodzovsky #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 6344a99b65SAndrey Grodzovsky 64791c4769Sxinhui pan #include "amdgpu_ras.h" 65029fbd43SHawking Zhang #include "amdgpu_xgmi.h" 66791c4769Sxinhui pan 67d0fb18b5SAndrey Grodzovsky #include "amdgpu_reset.h" 68d0fb18b5SAndrey Grodzovsky 69ebdef28eSAlex Deucher /* add these here since we already include dce12 headers and these are for DCN */ 70ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 71ebdef28eSAlex Deucher #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 72ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 73ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 74ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 75ebdef28eSAlex Deucher #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 76f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d 77f8646661SAlex Deucher #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 78f8646661SAlex Deucher 79dc5d4affSHarry Wentland #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2 0x05ea 80dc5d4affSHarry Wentland #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2_BASE_IDX 2 81dc5d4affSHarry Wentland 82a433f1f5SLijo Lazar #define MAX_MEM_RANGES 8 83ebdef28eSAlex Deucher 84be99ecbfSAlex Deucher static const char *gfxhub_client_ids[] = { 85be99ecbfSAlex Deucher "CB", 86be99ecbfSAlex Deucher "DB", 87be99ecbfSAlex Deucher "IA", 88be99ecbfSAlex Deucher "WD", 89be99ecbfSAlex Deucher "CPF", 90be99ecbfSAlex Deucher "CPC", 91be99ecbfSAlex Deucher "CPG", 92be99ecbfSAlex Deucher "RLC", 93be99ecbfSAlex Deucher "TCP", 94be99ecbfSAlex Deucher "SQC (inst)", 95be99ecbfSAlex Deucher "SQC (data)", 96be99ecbfSAlex Deucher "SQG", 97be99ecbfSAlex Deucher "PA", 98be99ecbfSAlex Deucher }; 99be99ecbfSAlex Deucher 10002f23f5fSAlex Deucher static const char *mmhub_client_ids_raven[][2] = { 10102f23f5fSAlex Deucher [0][0] = "MP1", 10202f23f5fSAlex Deucher [1][0] = "MP0", 10302f23f5fSAlex Deucher [2][0] = "VCN", 10402f23f5fSAlex Deucher [3][0] = "VCNU", 10502f23f5fSAlex Deucher [4][0] = "HDP", 10602f23f5fSAlex Deucher [5][0] = "DCE", 10702f23f5fSAlex Deucher [13][0] = "UTCL2", 10802f23f5fSAlex Deucher [19][0] = "TLS", 10902f23f5fSAlex Deucher [26][0] = "OSS", 11002f23f5fSAlex Deucher [27][0] = "SDMA0", 11102f23f5fSAlex Deucher [0][1] = "MP1", 11202f23f5fSAlex Deucher [1][1] = "MP0", 11302f23f5fSAlex Deucher [2][1] = "VCN", 11402f23f5fSAlex Deucher [3][1] = "VCNU", 11502f23f5fSAlex Deucher [4][1] = "HDP", 11602f23f5fSAlex Deucher [5][1] = "XDP", 11702f23f5fSAlex Deucher [6][1] = "DBGU0", 11802f23f5fSAlex Deucher [7][1] = "DCE", 11902f23f5fSAlex Deucher [8][1] = "DCEDWB0", 12002f23f5fSAlex Deucher [9][1] = "DCEDWB1", 12102f23f5fSAlex Deucher [26][1] = "OSS", 12202f23f5fSAlex Deucher [27][1] = "SDMA0", 12302f23f5fSAlex Deucher }; 12402f23f5fSAlex Deucher 12502f23f5fSAlex Deucher static const char *mmhub_client_ids_renoir[][2] = { 12602f23f5fSAlex Deucher [0][0] = "MP1", 12702f23f5fSAlex Deucher [1][0] = "MP0", 12802f23f5fSAlex Deucher [2][0] = "HDP", 12902f23f5fSAlex Deucher [4][0] = "DCEDMC", 13002f23f5fSAlex Deucher [5][0] = "DCEVGA", 13102f23f5fSAlex Deucher [13][0] = "UTCL2", 13202f23f5fSAlex Deucher [19][0] = "TLS", 13302f23f5fSAlex Deucher [26][0] = "OSS", 13402f23f5fSAlex Deucher [27][0] = "SDMA0", 13502f23f5fSAlex Deucher [28][0] = "VCN", 13602f23f5fSAlex Deucher [29][0] = "VCNU", 13702f23f5fSAlex Deucher [30][0] = "JPEG", 13802f23f5fSAlex Deucher [0][1] = "MP1", 13902f23f5fSAlex Deucher [1][1] = "MP0", 14002f23f5fSAlex Deucher [2][1] = "HDP", 14102f23f5fSAlex Deucher [3][1] = "XDP", 14202f23f5fSAlex Deucher [6][1] = "DBGU0", 14302f23f5fSAlex Deucher [7][1] = "DCEDMC", 14402f23f5fSAlex Deucher [8][1] = "DCEVGA", 14502f23f5fSAlex Deucher [9][1] = "DCEDWB", 14602f23f5fSAlex Deucher [26][1] = "OSS", 14702f23f5fSAlex Deucher [27][1] = "SDMA0", 14802f23f5fSAlex Deucher [28][1] = "VCN", 14902f23f5fSAlex Deucher [29][1] = "VCNU", 15002f23f5fSAlex Deucher [30][1] = "JPEG", 15102f23f5fSAlex Deucher }; 15202f23f5fSAlex Deucher 15302f23f5fSAlex Deucher static const char *mmhub_client_ids_vega10[][2] = { 15402f23f5fSAlex Deucher [0][0] = "MP0", 15502f23f5fSAlex Deucher [1][0] = "UVD", 15602f23f5fSAlex Deucher [2][0] = "UVDU", 15702f23f5fSAlex Deucher [3][0] = "HDP", 15802f23f5fSAlex Deucher [13][0] = "UTCL2", 15902f23f5fSAlex Deucher [14][0] = "OSS", 16002f23f5fSAlex Deucher [15][0] = "SDMA1", 16102f23f5fSAlex Deucher [32+0][0] = "VCE0", 16202f23f5fSAlex Deucher [32+1][0] = "VCE0U", 16302f23f5fSAlex Deucher [32+2][0] = "XDMA", 16402f23f5fSAlex Deucher [32+3][0] = "DCE", 16502f23f5fSAlex Deucher [32+4][0] = "MP1", 16602f23f5fSAlex Deucher [32+14][0] = "SDMA0", 16702f23f5fSAlex Deucher [0][1] = "MP0", 16802f23f5fSAlex Deucher [1][1] = "UVD", 16902f23f5fSAlex Deucher [2][1] = "UVDU", 17002f23f5fSAlex Deucher [3][1] = "DBGU0", 17102f23f5fSAlex Deucher [4][1] = "HDP", 17202f23f5fSAlex Deucher [5][1] = "XDP", 17302f23f5fSAlex Deucher [14][1] = "OSS", 17402f23f5fSAlex Deucher [15][1] = "SDMA0", 17502f23f5fSAlex Deucher [32+0][1] = "VCE0", 17602f23f5fSAlex Deucher [32+1][1] = "VCE0U", 17702f23f5fSAlex Deucher [32+2][1] = "XDMA", 17802f23f5fSAlex Deucher [32+3][1] = "DCE", 17902f23f5fSAlex Deucher [32+4][1] = "DCEDWB", 18002f23f5fSAlex Deucher [32+5][1] = "MP1", 18102f23f5fSAlex Deucher [32+6][1] = "DBGU1", 18202f23f5fSAlex Deucher [32+14][1] = "SDMA1", 18302f23f5fSAlex Deucher }; 18402f23f5fSAlex Deucher 18502f23f5fSAlex Deucher static const char *mmhub_client_ids_vega12[][2] = { 18602f23f5fSAlex Deucher [0][0] = "MP0", 18702f23f5fSAlex Deucher [1][0] = "VCE0", 18802f23f5fSAlex Deucher [2][0] = "VCE0U", 18902f23f5fSAlex Deucher [3][0] = "HDP", 19002f23f5fSAlex Deucher [13][0] = "UTCL2", 19102f23f5fSAlex Deucher [14][0] = "OSS", 19202f23f5fSAlex Deucher [15][0] = "SDMA1", 19302f23f5fSAlex Deucher [32+0][0] = "DCE", 19402f23f5fSAlex Deucher [32+1][0] = "XDMA", 19502f23f5fSAlex Deucher [32+2][0] = "UVD", 19602f23f5fSAlex Deucher [32+3][0] = "UVDU", 19702f23f5fSAlex Deucher [32+4][0] = "MP1", 19802f23f5fSAlex Deucher [32+15][0] = "SDMA0", 19902f23f5fSAlex Deucher [0][1] = "MP0", 20002f23f5fSAlex Deucher [1][1] = "VCE0", 20102f23f5fSAlex Deucher [2][1] = "VCE0U", 20202f23f5fSAlex Deucher [3][1] = "DBGU0", 20302f23f5fSAlex Deucher [4][1] = "HDP", 20402f23f5fSAlex Deucher [5][1] = "XDP", 20502f23f5fSAlex Deucher [14][1] = "OSS", 20602f23f5fSAlex Deucher [15][1] = "SDMA0", 20702f23f5fSAlex Deucher [32+0][1] = "DCE", 20802f23f5fSAlex Deucher [32+1][1] = "DCEDWB", 20902f23f5fSAlex Deucher [32+2][1] = "XDMA", 21002f23f5fSAlex Deucher [32+3][1] = "UVD", 21102f23f5fSAlex Deucher [32+4][1] = "UVDU", 21202f23f5fSAlex Deucher [32+5][1] = "MP1", 21302f23f5fSAlex Deucher [32+6][1] = "DBGU1", 21402f23f5fSAlex Deucher [32+15][1] = "SDMA1", 21502f23f5fSAlex Deucher }; 21602f23f5fSAlex Deucher 21702f23f5fSAlex Deucher static const char *mmhub_client_ids_vega20[][2] = { 21802f23f5fSAlex Deucher [0][0] = "XDMA", 21902f23f5fSAlex Deucher [1][0] = "DCE", 22002f23f5fSAlex Deucher [2][0] = "VCE0", 22102f23f5fSAlex Deucher [3][0] = "VCE0U", 22202f23f5fSAlex Deucher [4][0] = "UVD", 22302f23f5fSAlex Deucher [5][0] = "UVD1U", 22402f23f5fSAlex Deucher [13][0] = "OSS", 22502f23f5fSAlex Deucher [14][0] = "HDP", 22602f23f5fSAlex Deucher [15][0] = "SDMA0", 22702f23f5fSAlex Deucher [32+0][0] = "UVD", 22802f23f5fSAlex Deucher [32+1][0] = "UVDU", 22902f23f5fSAlex Deucher [32+2][0] = "MP1", 23002f23f5fSAlex Deucher [32+3][0] = "MP0", 23102f23f5fSAlex Deucher [32+12][0] = "UTCL2", 23202f23f5fSAlex Deucher [32+14][0] = "SDMA1", 23302f23f5fSAlex Deucher [0][1] = "XDMA", 23402f23f5fSAlex Deucher [1][1] = "DCE", 23502f23f5fSAlex Deucher [2][1] = "DCEDWB", 23602f23f5fSAlex Deucher [3][1] = "VCE0", 23702f23f5fSAlex Deucher [4][1] = "VCE0U", 23802f23f5fSAlex Deucher [5][1] = "UVD1", 23902f23f5fSAlex Deucher [6][1] = "UVD1U", 24002f23f5fSAlex Deucher [7][1] = "DBGU0", 24102f23f5fSAlex Deucher [8][1] = "XDP", 24202f23f5fSAlex Deucher [13][1] = "OSS", 24302f23f5fSAlex Deucher [14][1] = "HDP", 24402f23f5fSAlex Deucher [15][1] = "SDMA0", 24502f23f5fSAlex Deucher [32+0][1] = "UVD", 24602f23f5fSAlex Deucher [32+1][1] = "UVDU", 24702f23f5fSAlex Deucher [32+2][1] = "DBGU1", 24802f23f5fSAlex Deucher [32+3][1] = "MP1", 24902f23f5fSAlex Deucher [32+4][1] = "MP0", 25002f23f5fSAlex Deucher [32+14][1] = "SDMA1", 25102f23f5fSAlex Deucher }; 25202f23f5fSAlex Deucher 25302f23f5fSAlex Deucher static const char *mmhub_client_ids_arcturus[][2] = { 254e83db774SAlex Deucher [0][0] = "DBGU1", 255e83db774SAlex Deucher [1][0] = "XDP", 25602f23f5fSAlex Deucher [2][0] = "MP1", 25702f23f5fSAlex Deucher [14][0] = "HDP", 258e83db774SAlex Deucher [171][0] = "JPEG", 259e83db774SAlex Deucher [172][0] = "VCN", 260e83db774SAlex Deucher [173][0] = "VCNU", 261e83db774SAlex Deucher [203][0] = "JPEG1", 262e83db774SAlex Deucher [204][0] = "VCN1", 263e83db774SAlex Deucher [205][0] = "VCN1U", 264e83db774SAlex Deucher [256][0] = "SDMA0", 265e83db774SAlex Deucher [257][0] = "SDMA1", 266e83db774SAlex Deucher [258][0] = "SDMA2", 267e83db774SAlex Deucher [259][0] = "SDMA3", 268e83db774SAlex Deucher [260][0] = "SDMA4", 269e83db774SAlex Deucher [261][0] = "SDMA5", 270e83db774SAlex Deucher [262][0] = "SDMA6", 271e83db774SAlex Deucher [263][0] = "SDMA7", 272e83db774SAlex Deucher [384][0] = "OSS", 27302f23f5fSAlex Deucher [0][1] = "DBGU1", 27402f23f5fSAlex Deucher [1][1] = "XDP", 27502f23f5fSAlex Deucher [2][1] = "MP1", 27602f23f5fSAlex Deucher [14][1] = "HDP", 277e83db774SAlex Deucher [171][1] = "JPEG", 278e83db774SAlex Deucher [172][1] = "VCN", 279e83db774SAlex Deucher [173][1] = "VCNU", 280e83db774SAlex Deucher [203][1] = "JPEG1", 281e83db774SAlex Deucher [204][1] = "VCN1", 282e83db774SAlex Deucher [205][1] = "VCN1U", 283e83db774SAlex Deucher [256][1] = "SDMA0", 284e83db774SAlex Deucher [257][1] = "SDMA1", 285e83db774SAlex Deucher [258][1] = "SDMA2", 286e83db774SAlex Deucher [259][1] = "SDMA3", 287e83db774SAlex Deucher [260][1] = "SDMA4", 288e83db774SAlex Deucher [261][1] = "SDMA5", 289e83db774SAlex Deucher [262][1] = "SDMA6", 290e83db774SAlex Deucher [263][1] = "SDMA7", 291e83db774SAlex Deucher [384][1] = "OSS", 29202f23f5fSAlex Deucher }; 293e60f8db5SAlex Xie 294e844cd99SAlex Deucher static const char *mmhub_client_ids_aldebaran[][2] = { 295e844cd99SAlex Deucher [2][0] = "MP1", 296e844cd99SAlex Deucher [3][0] = "MP0", 297f4ec3e50SAlex Sierra [32+1][0] = "DBGU_IO0", 298f4ec3e50SAlex Sierra [32+2][0] = "DBGU_IO2", 299e844cd99SAlex Deucher [32+4][0] = "MPIO", 300e844cd99SAlex Deucher [96+11][0] = "JPEG0", 301e844cd99SAlex Deucher [96+12][0] = "VCN0", 302e844cd99SAlex Deucher [96+13][0] = "VCNU0", 303e844cd99SAlex Deucher [128+11][0] = "JPEG1", 304e844cd99SAlex Deucher [128+12][0] = "VCN1", 305e844cd99SAlex Deucher [128+13][0] = "VCNU1", 306f4ec3e50SAlex Sierra [160+1][0] = "XDP", 307e844cd99SAlex Deucher [160+14][0] = "HDP", 308f4ec3e50SAlex Sierra [256+0][0] = "SDMA0", 309f4ec3e50SAlex Sierra [256+1][0] = "SDMA1", 310f4ec3e50SAlex Sierra [256+2][0] = "SDMA2", 311f4ec3e50SAlex Sierra [256+3][0] = "SDMA3", 312f4ec3e50SAlex Sierra [256+4][0] = "SDMA4", 313f4ec3e50SAlex Sierra [384+0][0] = "OSS", 314e844cd99SAlex Deucher [2][1] = "MP1", 315e844cd99SAlex Deucher [3][1] = "MP0", 316e844cd99SAlex Deucher [32+1][1] = "DBGU_IO0", 317e844cd99SAlex Deucher [32+2][1] = "DBGU_IO2", 318e844cd99SAlex Deucher [32+4][1] = "MPIO", 319e844cd99SAlex Deucher [96+11][1] = "JPEG0", 320e844cd99SAlex Deucher [96+12][1] = "VCN0", 321e844cd99SAlex Deucher [96+13][1] = "VCNU0", 322e844cd99SAlex Deucher [128+11][1] = "JPEG1", 323e844cd99SAlex Deucher [128+12][1] = "VCN1", 324e844cd99SAlex Deucher [128+13][1] = "VCNU1", 325f4ec3e50SAlex Sierra [160+1][1] = "XDP", 326e844cd99SAlex Deucher [160+14][1] = "HDP", 327f4ec3e50SAlex Sierra [256+0][1] = "SDMA0", 328f4ec3e50SAlex Sierra [256+1][1] = "SDMA1", 329f4ec3e50SAlex Sierra [256+2][1] = "SDMA2", 330f4ec3e50SAlex Sierra [256+3][1] = "SDMA3", 331f4ec3e50SAlex Sierra [256+4][1] = "SDMA4", 332f4ec3e50SAlex Sierra [384+0][1] = "OSS", 333e844cd99SAlex Deucher }; 334e844cd99SAlex Deucher 335946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 3365c583018SEvan Quan { 337946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 338946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 3395c583018SEvan Quan }; 3405c583018SEvan Quan 341946a4d5bSShaoyun Liu static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 3425c583018SEvan Quan { 343946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 344946a4d5bSShaoyun Liu SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 3455c583018SEvan Quan }; 3465c583018SEvan Quan 347791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 348791c4769Sxinhui pan (0x000143c0 + 0x00000000), 349791c4769Sxinhui pan (0x000143c0 + 0x00000800), 350791c4769Sxinhui pan (0x000143c0 + 0x00001000), 351791c4769Sxinhui pan (0x000143c0 + 0x00001800), 352791c4769Sxinhui pan (0x000543c0 + 0x00000000), 353791c4769Sxinhui pan (0x000543c0 + 0x00000800), 354791c4769Sxinhui pan (0x000543c0 + 0x00001000), 355791c4769Sxinhui pan (0x000543c0 + 0x00001800), 356791c4769Sxinhui pan (0x000943c0 + 0x00000000), 357791c4769Sxinhui pan (0x000943c0 + 0x00000800), 358791c4769Sxinhui pan (0x000943c0 + 0x00001000), 359791c4769Sxinhui pan (0x000943c0 + 0x00001800), 360791c4769Sxinhui pan (0x000d43c0 + 0x00000000), 361791c4769Sxinhui pan (0x000d43c0 + 0x00000800), 362791c4769Sxinhui pan (0x000d43c0 + 0x00001000), 363791c4769Sxinhui pan (0x000d43c0 + 0x00001800), 364791c4769Sxinhui pan (0x001143c0 + 0x00000000), 365791c4769Sxinhui pan (0x001143c0 + 0x00000800), 366791c4769Sxinhui pan (0x001143c0 + 0x00001000), 367791c4769Sxinhui pan (0x001143c0 + 0x00001800), 368791c4769Sxinhui pan (0x001543c0 + 0x00000000), 369791c4769Sxinhui pan (0x001543c0 + 0x00000800), 370791c4769Sxinhui pan (0x001543c0 + 0x00001000), 371791c4769Sxinhui pan (0x001543c0 + 0x00001800), 372791c4769Sxinhui pan (0x001943c0 + 0x00000000), 373791c4769Sxinhui pan (0x001943c0 + 0x00000800), 374791c4769Sxinhui pan (0x001943c0 + 0x00001000), 375791c4769Sxinhui pan (0x001943c0 + 0x00001800), 376791c4769Sxinhui pan (0x001d43c0 + 0x00000000), 377791c4769Sxinhui pan (0x001d43c0 + 0x00000800), 378791c4769Sxinhui pan (0x001d43c0 + 0x00001000), 379791c4769Sxinhui pan (0x001d43c0 + 0x00001800), 38002bab923SDavid Panariti }; 38102bab923SDavid Panariti 382791c4769Sxinhui pan static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 383791c4769Sxinhui pan (0x000143e0 + 0x00000000), 384791c4769Sxinhui pan (0x000143e0 + 0x00000800), 385791c4769Sxinhui pan (0x000143e0 + 0x00001000), 386791c4769Sxinhui pan (0x000143e0 + 0x00001800), 387791c4769Sxinhui pan (0x000543e0 + 0x00000000), 388791c4769Sxinhui pan (0x000543e0 + 0x00000800), 389791c4769Sxinhui pan (0x000543e0 + 0x00001000), 390791c4769Sxinhui pan (0x000543e0 + 0x00001800), 391791c4769Sxinhui pan (0x000943e0 + 0x00000000), 392791c4769Sxinhui pan (0x000943e0 + 0x00000800), 393791c4769Sxinhui pan (0x000943e0 + 0x00001000), 394791c4769Sxinhui pan (0x000943e0 + 0x00001800), 395791c4769Sxinhui pan (0x000d43e0 + 0x00000000), 396791c4769Sxinhui pan (0x000d43e0 + 0x00000800), 397791c4769Sxinhui pan (0x000d43e0 + 0x00001000), 398791c4769Sxinhui pan (0x000d43e0 + 0x00001800), 399791c4769Sxinhui pan (0x001143e0 + 0x00000000), 400791c4769Sxinhui pan (0x001143e0 + 0x00000800), 401791c4769Sxinhui pan (0x001143e0 + 0x00001000), 402791c4769Sxinhui pan (0x001143e0 + 0x00001800), 403791c4769Sxinhui pan (0x001543e0 + 0x00000000), 404791c4769Sxinhui pan (0x001543e0 + 0x00000800), 405791c4769Sxinhui pan (0x001543e0 + 0x00001000), 406791c4769Sxinhui pan (0x001543e0 + 0x00001800), 407791c4769Sxinhui pan (0x001943e0 + 0x00000000), 408791c4769Sxinhui pan (0x001943e0 + 0x00000800), 409791c4769Sxinhui pan (0x001943e0 + 0x00001000), 410791c4769Sxinhui pan (0x001943e0 + 0x00001800), 411791c4769Sxinhui pan (0x001d43e0 + 0x00000000), 412791c4769Sxinhui pan (0x001d43e0 + 0x00000800), 413791c4769Sxinhui pan (0x001d43e0 + 0x00001000), 414791c4769Sxinhui pan (0x001d43e0 + 0x00001800), 41502bab923SDavid Panariti }; 41602bab923SDavid Panariti 417791c4769Sxinhui pan static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 418791c4769Sxinhui pan struct amdgpu_irq_src *src, 419791c4769Sxinhui pan unsigned type, 420791c4769Sxinhui pan enum amdgpu_interrupt_state state) 421791c4769Sxinhui pan { 422791c4769Sxinhui pan u32 bits, i, tmp, reg; 423791c4769Sxinhui pan 4241e2c6d55SJohn Clements /* Devices newer then VEGA10/12 shall have these programming 4251e2c6d55SJohn Clements sequences performed by PSP BL */ 4261e2c6d55SJohn Clements if (adev->asic_type >= CHIP_VEGA20) 4271e2c6d55SJohn Clements return 0; 4281e2c6d55SJohn Clements 429791c4769Sxinhui pan bits = 0x7f; 430791c4769Sxinhui pan 431791c4769Sxinhui pan switch (state) { 432791c4769Sxinhui pan case AMDGPU_IRQ_STATE_DISABLE: 433791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 434791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 435791c4769Sxinhui pan tmp = RREG32(reg); 436791c4769Sxinhui pan tmp &= ~bits; 437791c4769Sxinhui pan WREG32(reg, tmp); 438791c4769Sxinhui pan } 439791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 440791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 441791c4769Sxinhui pan tmp = RREG32(reg); 442791c4769Sxinhui pan tmp &= ~bits; 443791c4769Sxinhui pan WREG32(reg, tmp); 444791c4769Sxinhui pan } 445791c4769Sxinhui pan break; 446791c4769Sxinhui pan case AMDGPU_IRQ_STATE_ENABLE: 447791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 448791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_addrs[i]; 449791c4769Sxinhui pan tmp = RREG32(reg); 450791c4769Sxinhui pan tmp |= bits; 451791c4769Sxinhui pan WREG32(reg, tmp); 452791c4769Sxinhui pan } 453791c4769Sxinhui pan for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 454791c4769Sxinhui pan reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 455791c4769Sxinhui pan tmp = RREG32(reg); 456791c4769Sxinhui pan tmp |= bits; 457791c4769Sxinhui pan WREG32(reg, tmp); 458791c4769Sxinhui pan } 459791c4769Sxinhui pan break; 460791c4769Sxinhui pan default: 461791c4769Sxinhui pan break; 462791c4769Sxinhui pan } 463791c4769Sxinhui pan 464791c4769Sxinhui pan return 0; 465791c4769Sxinhui pan } 466791c4769Sxinhui pan 467e60f8db5SAlex Xie static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 468e60f8db5SAlex Xie struct amdgpu_irq_src *src, 469e60f8db5SAlex Xie unsigned type, 470e60f8db5SAlex Xie enum amdgpu_interrupt_state state) 471e60f8db5SAlex Xie { 472e60f8db5SAlex Xie struct amdgpu_vmhub *hub; 473ae6d1416STom St Denis u32 tmp, reg, bits, i, j; 474e60f8db5SAlex Xie 47511250164SChristian König bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47611250164SChristian König VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47711250164SChristian König VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47811250164SChristian König VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 47911250164SChristian König VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 48011250164SChristian König VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 48111250164SChristian König VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 48211250164SChristian König 483e60f8db5SAlex Xie switch (state) { 484e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_DISABLE: 485d9426c3dSLe Ma for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 486ae6d1416STom St Denis hub = &adev->vmhub[j]; 487e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 488e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 48992f153bbSVictor Skvortsov 490b93df61dSAlex Deucher /* This works because this interrupt is only 491b93df61dSAlex Deucher * enabled at init/resume and disabled in 492b93df61dSAlex Deucher * fini/suspend, so the overall state doesn't 493b93df61dSAlex Deucher * change over the course of suspend/resume. 494b93df61dSAlex Deucher */ 495f4caf584SHawking Zhang if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) 496b93df61dSAlex Deucher continue; 497b93df61dSAlex Deucher 498f4caf584SHawking Zhang if (j >= AMDGPU_MMHUB0(0)) 49992f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP(MMHUB, reg); 500f4caf584SHawking Zhang else 501f4caf584SHawking Zhang tmp = RREG32_SOC15_IP(GC, reg); 50292f153bbSVictor Skvortsov 503e60f8db5SAlex Xie tmp &= ~bits; 50492f153bbSVictor Skvortsov 505f4caf584SHawking Zhang if (j >= AMDGPU_MMHUB0(0)) 50692f153bbSVictor Skvortsov WREG32_SOC15_IP(MMHUB, reg, tmp); 507f4caf584SHawking Zhang else 508f4caf584SHawking Zhang WREG32_SOC15_IP(GC, reg, tmp); 509e60f8db5SAlex Xie } 510e60f8db5SAlex Xie } 511e60f8db5SAlex Xie break; 512e60f8db5SAlex Xie case AMDGPU_IRQ_STATE_ENABLE: 513d9426c3dSLe Ma for_each_set_bit(j, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 514ae6d1416STom St Denis hub = &adev->vmhub[j]; 515e60f8db5SAlex Xie for (i = 0; i < 16; i++) { 516e60f8db5SAlex Xie reg = hub->vm_context0_cntl + i; 51792f153bbSVictor Skvortsov 518b93df61dSAlex Deucher /* This works because this interrupt is only 519b93df61dSAlex Deucher * enabled at init/resume and disabled in 520b93df61dSAlex Deucher * fini/suspend, so the overall state doesn't 521b93df61dSAlex Deucher * change over the course of suspend/resume. 522b93df61dSAlex Deucher */ 523f4caf584SHawking Zhang if (adev->in_s0ix && (j == AMDGPU_GFXHUB(0))) 524b93df61dSAlex Deucher continue; 525b93df61dSAlex Deucher 526f4caf584SHawking Zhang if (j >= AMDGPU_MMHUB0(0)) 52792f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP(MMHUB, reg); 528f4caf584SHawking Zhang else 529f4caf584SHawking Zhang tmp = RREG32_SOC15_IP(GC, reg); 53092f153bbSVictor Skvortsov 531e60f8db5SAlex Xie tmp |= bits; 53292f153bbSVictor Skvortsov 533f4caf584SHawking Zhang if (j >= AMDGPU_MMHUB0(0)) 53492f153bbSVictor Skvortsov WREG32_SOC15_IP(MMHUB, reg, tmp); 535f4caf584SHawking Zhang else 536f4caf584SHawking Zhang WREG32_SOC15_IP(GC, reg, tmp); 537e60f8db5SAlex Xie } 538e60f8db5SAlex Xie } 5399304ca4dSGustavo A. R. Silva break; 540e60f8db5SAlex Xie default: 541e60f8db5SAlex Xie break; 542e60f8db5SAlex Xie } 543e60f8db5SAlex Xie 544e60f8db5SAlex Xie return 0; 545e60f8db5SAlex Xie } 546e60f8db5SAlex Xie 547e60f8db5SAlex Xie static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 548e60f8db5SAlex Xie struct amdgpu_irq_src *source, 549e60f8db5SAlex Xie struct amdgpu_iv_entry *entry) 550e60f8db5SAlex Xie { 551c468f9e2SChristian König bool retry_fault = !!(entry->src_data[1] & 0x80); 552ff891a2eSPhilip Yang bool write_fault = !!(entry->src_data[1] & 0x20); 55302f23f5fSAlex Deucher uint32_t status = 0, cid = 0, rw = 0; 554e3898719SChristian König struct amdgpu_task_info task_info; 555e3898719SChristian König struct amdgpu_vmhub *hub; 55602f23f5fSAlex Deucher const char *mmhub_cid; 557e3898719SChristian König const char *hub_name; 558e3898719SChristian König u64 addr; 559318e431bSMukul Joshi uint32_t cam_index = 0; 5606dabce86SHarshit Mogalapalli int ret, xcc_id = 0; 5616dabce86SHarshit Mogalapalli uint32_t node_id; 562eaae4beeSPhilip Yang 56398b2e9caSLe Ma node_id = entry->node_id; 564e60f8db5SAlex Xie 565e60f8db5SAlex Xie addr = (u64)entry->src_data[0] << 12; 566e60f8db5SAlex Xie addr |= ((u64)entry->src_data[1] & 0xf) << 44; 567e60f8db5SAlex Xie 5685fb34bd9SAlex Sierra if (entry->client_id == SOC15_IH_CLIENTID_VMC) { 5695fb34bd9SAlex Sierra hub_name = "mmhub0"; 57098b2e9caSLe Ma hub = &adev->vmhub[AMDGPU_MMHUB0(node_id / 4)]; 5715fb34bd9SAlex Sierra } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { 5725fb34bd9SAlex Sierra hub_name = "mmhub1"; 5735fb34bd9SAlex Sierra hub = &adev->vmhub[AMDGPU_MMHUB1(0)]; 5745fb34bd9SAlex Sierra } else { 5755fb34bd9SAlex Sierra hub_name = "gfxhub0"; 57698b2e9caSLe Ma if (adev->gfx.funcs->ih_node_to_logical_xcc) { 57798b2e9caSLe Ma xcc_id = adev->gfx.funcs->ih_node_to_logical_xcc(adev, 57898b2e9caSLe Ma node_id); 57998b2e9caSLe Ma if (xcc_id < 0) 58098b2e9caSLe Ma xcc_id = 0; 58198b2e9caSLe Ma } 58298b2e9caSLe Ma hub = &adev->vmhub[xcc_id]; 5835fb34bd9SAlex Sierra } 5845fb34bd9SAlex Sierra 5850291150dSChristian König if (retry_fault) { 586318e431bSMukul Joshi if (adev->irq.retry_cam_enabled) { 587318e431bSMukul Joshi /* Delegate it to a different ring if the hardware hasn't 588318e431bSMukul Joshi * already done it. 589318e431bSMukul Joshi */ 590318e431bSMukul Joshi if (entry->ih == &adev->irq.ih) { 591318e431bSMukul Joshi amdgpu_irq_delegate(adev, entry, 8); 592318e431bSMukul Joshi return 1; 593318e431bSMukul Joshi } 59422666cc1SChristian König 595318e431bSMukul Joshi cam_index = entry->src_data[2] & 0x3ff; 596318e431bSMukul Joshi 597f5fe7edfSMukul Joshi ret = amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 5985fb34bd9SAlex Sierra addr, write_fault); 599318e431bSMukul Joshi WDOORBELL32(adev->irq.retry_cam_doorbell_index, cam_index); 600318e431bSMukul Joshi if (ret) 601318e431bSMukul Joshi return 1; 602318e431bSMukul Joshi } else { 6030291150dSChristian König /* Process it onyl if it's the first fault for this address */ 6040291150dSChristian König if (entry->ih != &adev->irq.ih_soft && 6053c2d6ea2SPhilip Yang amdgpu_gmc_filter_faults(adev, entry->ih, addr, entry->pasid, 6060291150dSChristian König entry->timestamp)) 6070291150dSChristian König return 1; 6080291150dSChristian König 6090291150dSChristian König /* Delegate it to a different ring if the hardware hasn't 6100291150dSChristian König * already done it. 6110291150dSChristian König */ 61258df0d71SSebastian Andrzej Siewior if (entry->ih == &adev->irq.ih) { 6130291150dSChristian König amdgpu_irq_delegate(adev, entry, 8); 6140291150dSChristian König return 1; 6150291150dSChristian König } 6160291150dSChristian König 6170291150dSChristian König /* Try to handle the recoverable page faults by filling page 6180291150dSChristian König * tables 6190291150dSChristian König */ 620f5fe7edfSMukul Joshi if (amdgpu_vm_handle_fault(adev, entry->pasid, entry->vmid, node_id, 6215fb34bd9SAlex Sierra addr, write_fault)) 6220291150dSChristian König return 1; 6230291150dSChristian König } 624318e431bSMukul Joshi } 625ec671737SChristian König 626e3898719SChristian König if (!printk_ratelimit()) 627e3898719SChristian König return 0; 62853499173SXiaojie Yuan 629e60f8db5SAlex Xie 63005794effSShirish S memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 631efaa9646SAndrey Grodzovsky amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 632efaa9646SAndrey Grodzovsky 6334d6cbde3SFelix Kuehling dev_err(adev->dev, 634c468f9e2SChristian König "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 635c468f9e2SChristian König "pasid:%u, for process %s pid %d thread %s pid %d)\n", 63651c60898SLe Ma hub_name, retry_fault ? "retry" : "no-retry", 637c4f46f22SChristian König entry->src_id, entry->ring_id, entry->vmid, 638efaa9646SAndrey Grodzovsky entry->pasid, task_info.process_name, task_info.tgid, 639efaa9646SAndrey Grodzovsky task_info.task_name, task_info.pid); 640be14729aSYong Zhao dev_err(adev->dev, " in page starting at address 0x%016llx from IH client 0x%x (%s)\n", 641be14729aSYong Zhao addr, entry->client_id, 642be14729aSYong Zhao soc15_ih_clientid_name[entry->client_id]); 643e3898719SChristian König 644eaae4beeSPhilip Yang if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) 645eaae4beeSPhilip Yang dev_err(adev->dev, " cookie node_id %d fault from die %s%d%s\n", 646eaae4beeSPhilip Yang node_id, node_id % 4 == 3 ? "RSV" : "AID", node_id / 4, 647eaae4beeSPhilip Yang node_id % 4 == 1 ? ".XCD0" : node_id % 4 == 2 ? ".XCD1" : ""); 648eaae4beeSPhilip Yang 649e3898719SChristian König if (amdgpu_sriov_vf(adev)) 650e3898719SChristian König return 0; 651e3898719SChristian König 652e3898719SChristian König /* 653e3898719SChristian König * Issue a dummy read to wait for the status register to 654e3898719SChristian König * be updated to avoid reading an incorrect value due to 655e3898719SChristian König * the new fast GRBM interface. 656e3898719SChristian König */ 657f4caf584SHawking Zhang if ((entry->vmid_src == AMDGPU_GFXHUB(0)) && 658630e959fSAlex Deucher (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2))) 659e3898719SChristian König RREG32(hub->vm_l2_pro_fault_status); 660e3898719SChristian König 661e3898719SChristian König status = RREG32(hub->vm_l2_pro_fault_status); 662e3898719SChristian König cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID); 663e3898719SChristian König rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW); 664e3898719SChristian König WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 665e3898719SChristian König 6664d6cbde3SFelix Kuehling dev_err(adev->dev, 6674d6cbde3SFelix Kuehling "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 6684d6cbde3SFelix Kuehling status); 66921e1217bSMukul Joshi if (entry->vmid_src == AMDGPU_GFXHUB(0)) { 670be99ecbfSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 671e3898719SChristian König cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : 672e3898719SChristian König gfxhub_client_ids[cid], 673be99ecbfSAlex Deucher cid); 67402f23f5fSAlex Deucher } else { 675630e959fSAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 676630e959fSAlex Deucher case IP_VERSION(9, 0, 0): 67702f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega10[cid][rw]; 67802f23f5fSAlex Deucher break; 679630e959fSAlex Deucher case IP_VERSION(9, 3, 0): 68002f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega12[cid][rw]; 68102f23f5fSAlex Deucher break; 682630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 68302f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_vega20[cid][rw]; 68402f23f5fSAlex Deucher break; 685630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 68602f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; 68702f23f5fSAlex Deucher break; 688630e959fSAlex Deucher case IP_VERSION(9, 1, 0): 689630e959fSAlex Deucher case IP_VERSION(9, 2, 0): 69002f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_raven[cid][rw]; 69102f23f5fSAlex Deucher break; 692630e959fSAlex Deucher case IP_VERSION(1, 5, 0): 693630e959fSAlex Deucher case IP_VERSION(2, 4, 0): 69402f23f5fSAlex Deucher mmhub_cid = mmhub_client_ids_renoir[cid][rw]; 69502f23f5fSAlex Deucher break; 696ab1a157eSHawking Zhang case IP_VERSION(1, 8, 0): 697630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 698e844cd99SAlex Deucher mmhub_cid = mmhub_client_ids_aldebaran[cid][rw]; 699e844cd99SAlex Deucher break; 70002f23f5fSAlex Deucher default: 70102f23f5fSAlex Deucher mmhub_cid = NULL; 70202f23f5fSAlex Deucher break; 70302f23f5fSAlex Deucher } 70402f23f5fSAlex Deucher dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 70502f23f5fSAlex Deucher mmhub_cid ? mmhub_cid : "unknown", cid); 70602f23f5fSAlex Deucher } 7075ddd4a9aSYong Zhao dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 7085ddd4a9aSYong Zhao REG_GET_FIELD(status, 7095ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 7105ddd4a9aSYong Zhao dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 7115ddd4a9aSYong Zhao REG_GET_FIELD(status, 7125ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 7135ddd4a9aSYong Zhao dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 7145ddd4a9aSYong Zhao REG_GET_FIELD(status, 7155ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 7165ddd4a9aSYong Zhao dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 7175ddd4a9aSYong Zhao REG_GET_FIELD(status, 7185ddd4a9aSYong Zhao VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 71902f23f5fSAlex Deucher dev_err(adev->dev, "\t RW: 0x%x\n", rw); 720e60f8db5SAlex Xie return 0; 721e60f8db5SAlex Xie } 722e60f8db5SAlex Xie 723e60f8db5SAlex Xie static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 724e60f8db5SAlex Xie .set = gmc_v9_0_vm_fault_interrupt_state, 725e60f8db5SAlex Xie .process = gmc_v9_0_process_interrupt, 726e60f8db5SAlex Xie }; 727e60f8db5SAlex Xie 728791c4769Sxinhui pan 729791c4769Sxinhui pan static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 730791c4769Sxinhui pan .set = gmc_v9_0_ecc_interrupt_state, 73134cc4fd9STao Zhou .process = amdgpu_umc_process_ecc_irq, 732791c4769Sxinhui pan }; 733791c4769Sxinhui pan 734e60f8db5SAlex Xie static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 735e60f8db5SAlex Xie { 736770d13b1SChristian König adev->gmc.vm_fault.num_types = 1; 737770d13b1SChristian König adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 738791c4769Sxinhui pan 73968d705ddSHawking Zhang if (!amdgpu_sriov_vf(adev) && 74068d705ddSHawking Zhang !adev->gmc.xgmi.connected_to_cpu) { 741791c4769Sxinhui pan adev->gmc.ecc_irq.num_types = 1; 742791c4769Sxinhui pan adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 743e60f8db5SAlex Xie } 7442ee9403eSZhigang Luo } 745e60f8db5SAlex Xie 7462a79d868SYong Zhao static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 7472a79d868SYong Zhao uint32_t flush_type) 74803f89febSChristian König { 74903f89febSChristian König u32 req = 0; 75003f89febSChristian König 75103f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 752c4f46f22SChristian König PER_VMID_INVALIDATE_REQ, 1 << vmid); 7532a79d868SYong Zhao req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 75403f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 75503f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 75603f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 75703f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 75803f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 75903f89febSChristian König req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 76003f89febSChristian König CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 76103f89febSChristian König 76203f89febSChristian König return req; 76303f89febSChristian König } 76403f89febSChristian König 76590f6452cSchangzhu /** 76690f6452cSchangzhu * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore 76790f6452cSchangzhu * 76890f6452cSchangzhu * @adev: amdgpu_device pointer 76990f6452cSchangzhu * @vmhub: vmhub type 77090f6452cSchangzhu * 77190f6452cSchangzhu */ 77290f6452cSchangzhu static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, 77390f6452cSchangzhu uint32_t vmhub) 77490f6452cSchangzhu { 775ab1a157eSHawking Zhang if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) || 776ab1a157eSHawking Zhang adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) 777d477c5aaSHawking Zhang return false; 778d477c5aaSHawking Zhang 779f4caf584SHawking Zhang return ((vmhub == AMDGPU_MMHUB0(0) || 780f4caf584SHawking Zhang vmhub == AMDGPU_MMHUB1(0)) && 78190f6452cSchangzhu (!amdgpu_sriov_vf(adev)) && 78254f78a76SAlex Deucher (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && 78354f78a76SAlex Deucher (adev->apu_flags & AMD_APU_IS_PICASSO)))); 78490f6452cSchangzhu } 78590f6452cSchangzhu 786ea930000SAlex Sierra static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, 787ea930000SAlex Sierra uint8_t vmid, uint16_t *p_pasid) 788ea930000SAlex Sierra { 789ea930000SAlex Sierra uint32_t value; 790ea930000SAlex Sierra 791ea930000SAlex Sierra value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 792ea930000SAlex Sierra + vmid); 793ea930000SAlex Sierra *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 794ea930000SAlex Sierra 795ea930000SAlex Sierra return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 796ea930000SAlex Sierra } 797ea930000SAlex Sierra 798e60f8db5SAlex Xie /* 799e60f8db5SAlex Xie * GART 800e60f8db5SAlex Xie * VMID 0 is the physical GPU addresses as used by the kernel. 801e60f8db5SAlex Xie * VMIDs 1-15 are used for userspace clients and are handled 802e60f8db5SAlex Xie * by the amdgpu vm/hsa code. 803e60f8db5SAlex Xie */ 804e60f8db5SAlex Xie 805e60f8db5SAlex Xie /** 8062a79d868SYong Zhao * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 807e60f8db5SAlex Xie * 808e60f8db5SAlex Xie * @adev: amdgpu_device pointer 809e60f8db5SAlex Xie * @vmid: vm instance to flush 810bf0df09cSLee Jones * @vmhub: which hub to flush 8112a79d868SYong Zhao * @flush_type: the flush type 812e60f8db5SAlex Xie * 8132a79d868SYong Zhao * Flush the TLB for the requested page table using certain type. 814e60f8db5SAlex Xie */ 8153ff98548SOak Zeng static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 8163ff98548SOak Zeng uint32_t vmhub, uint32_t flush_type) 817e60f8db5SAlex Xie { 81890f6452cSchangzhu bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); 819e60f8db5SAlex Xie const unsigned eng = 17; 820b80cd524SFelix Kuehling u32 j, inv_req, inv_req2, tmp; 8213ff98548SOak Zeng struct amdgpu_vmhub *hub; 822e60f8db5SAlex Xie 823d9426c3dSLe Ma BUG_ON(vmhub >= AMDGPU_MAX_VMHUBS); 8243ff98548SOak Zeng 8253ff98548SOak Zeng hub = &adev->vmhub[vmhub]; 826b80cd524SFelix Kuehling if (adev->gmc.xgmi.num_physical_nodes && 827630e959fSAlex Deucher adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0)) { 828b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 829b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 830b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 831b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 832b80cd524SFelix Kuehling * still need a second TLB flush after this. 833b80cd524SFelix Kuehling */ 834b80cd524SFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); 835b80cd524SFelix Kuehling inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 8362e8cc5d3SGraham Sider } else if (flush_type == 2 && 8372e8cc5d3SGraham Sider adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3) && 8382e8cc5d3SGraham Sider adev->rev_id == 0) { 8392e8cc5d3SGraham Sider inv_req = gmc_v9_0_get_invalidate_req(vmid, 0); 8402e8cc5d3SGraham Sider inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 841b80cd524SFelix Kuehling } else { 84237c58ddfSFelix Kuehling inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); 843b80cd524SFelix Kuehling inv_req2 = 0; 844b80cd524SFelix Kuehling } 845e60f8db5SAlex Xie 84682d1a1b1SChengming Gui /* This is necessary for a HW workaround under SRIOV as well 84782d1a1b1SChengming Gui * as GFXOFF under bare metal 84882d1a1b1SChengming Gui */ 849277bd337SLe Ma if (adev->gfx.kiq[0].ring.sched.ready && 85082d1a1b1SChengming Gui (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 851d0fb18b5SAndrey Grodzovsky down_read_trylock(&adev->reset_domain->sem)) { 852148f597dSHuang Rui uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 853148f597dSHuang Rui uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 854af5fe1e9SChristian König 85537c58ddfSFelix Kuehling amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 856af5fe1e9SChristian König 1 << vmid); 857d0fb18b5SAndrey Grodzovsky up_read(&adev->reset_domain->sem); 8583ff98548SOak Zeng return; 859fc0faf04SEmily Deng } 8603890d111SEmily Deng 8613890d111SEmily Deng spin_lock(&adev->gmc.invalidate_lock); 862f920d1bbSchangzhu 863f920d1bbSchangzhu /* 864f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 865f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 866f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 867f920d1bbSchangzhu * to WA the Issue 868f920d1bbSchangzhu */ 869f920d1bbSchangzhu 870f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 87190f6452cSchangzhu if (use_semaphore) { 872f920d1bbSchangzhu for (j = 0; j < adev->usec_timeout; j++) { 87392f153bbSVictor Skvortsov /* a read return value of 1 means semaphore acquire */ 874f4caf584SHawking Zhang if (vmhub >= AMDGPU_MMHUB0(0)) 87592f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng); 876f4caf584SHawking Zhang else 877f4caf584SHawking Zhang tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng); 878f920d1bbSchangzhu if (tmp & 0x1) 879f920d1bbSchangzhu break; 880f920d1bbSchangzhu udelay(1); 881f920d1bbSchangzhu } 882f920d1bbSchangzhu 883f920d1bbSchangzhu if (j >= adev->usec_timeout) 884f920d1bbSchangzhu DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 885f920d1bbSchangzhu } 886f920d1bbSchangzhu 887b80cd524SFelix Kuehling do { 888f4caf584SHawking Zhang if (vmhub >= AMDGPU_MMHUB0(0)) 88992f153bbSVictor Skvortsov WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req); 890f4caf584SHawking Zhang else 891f4caf584SHawking Zhang WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_req + hub->eng_distance * eng, inv_req); 89253499173SXiaojie Yuan 89353499173SXiaojie Yuan /* 894b80cd524SFelix Kuehling * Issue a dummy read to wait for the ACK register to 895b80cd524SFelix Kuehling * be cleared to avoid a false ACK due to the new fast 896b80cd524SFelix Kuehling * GRBM interface. 89753499173SXiaojie Yuan */ 898f4caf584SHawking Zhang if ((vmhub == AMDGPU_GFXHUB(0)) && 899630e959fSAlex Deucher (adev->ip_versions[GC_HWIP][0] < IP_VERSION(9, 4, 2))) 900148f597dSHuang Rui RREG32_NO_KIQ(hub->vm_inv_eng0_req + 901148f597dSHuang Rui hub->eng_distance * eng); 90253499173SXiaojie Yuan 903e60f8db5SAlex Xie for (j = 0; j < adev->usec_timeout; j++) { 904f4caf584SHawking Zhang if (vmhub >= AMDGPU_MMHUB0(0)) 90592f153bbSVictor Skvortsov tmp = RREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_ack + hub->eng_distance * eng); 906f4caf584SHawking Zhang else 907f4caf584SHawking Zhang tmp = RREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_ack + hub->eng_distance * eng); 908396557b0SChristian König if (tmp & (1 << vmid)) 909e60f8db5SAlex Xie break; 910e60f8db5SAlex Xie udelay(1); 911e60f8db5SAlex Xie } 912f920d1bbSchangzhu 913b80cd524SFelix Kuehling inv_req = inv_req2; 914b80cd524SFelix Kuehling inv_req2 = 0; 915b80cd524SFelix Kuehling } while (inv_req); 916b80cd524SFelix Kuehling 917f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 91892f153bbSVictor Skvortsov if (use_semaphore) { 919f920d1bbSchangzhu /* 920f920d1bbSchangzhu * add semaphore release after invalidation, 921f920d1bbSchangzhu * write with 0 means semaphore release 922f920d1bbSchangzhu */ 923f4caf584SHawking Zhang if (vmhub >= AMDGPU_MMHUB0(0)) 924a0a0c69cSLijo Lazar WREG32_SOC15_IP_NO_KIQ(MMHUB, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0); 92592f153bbSVictor Skvortsov else 926a0a0c69cSLijo Lazar WREG32_SOC15_IP_NO_KIQ(GC, hub->vm_inv_eng0_sem + hub->eng_distance * eng, 0); 92792f153bbSVictor Skvortsov } 928f920d1bbSchangzhu 9293890d111SEmily Deng spin_unlock(&adev->gmc.invalidate_lock); 930f920d1bbSchangzhu 931396557b0SChristian König if (j < adev->usec_timeout) 9323ff98548SOak Zeng return; 933396557b0SChristian König 934e60f8db5SAlex Xie DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 935e60f8db5SAlex Xie } 936e60f8db5SAlex Xie 937ea930000SAlex Sierra /** 938ea930000SAlex Sierra * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid 939ea930000SAlex Sierra * 940ea930000SAlex Sierra * @adev: amdgpu_device pointer 941ea930000SAlex Sierra * @pasid: pasid to be flush 942bf0df09cSLee Jones * @flush_type: the flush type 943bf0df09cSLee Jones * @all_hub: flush all hubs 9441bae03aaSSrinivasan Shanmugam * @inst: is used to select which instance of KIQ to use for the invalidation 945ea930000SAlex Sierra * 946ea930000SAlex Sierra * Flush the TLB for the requested pasid. 947ea930000SAlex Sierra */ 948ea930000SAlex Sierra static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 949ea930000SAlex Sierra uint16_t pasid, uint32_t flush_type, 950f87f6864SMukul Joshi bool all_hub, uint32_t inst) 951ea930000SAlex Sierra { 952ea930000SAlex Sierra int vmid, i; 953ea930000SAlex Sierra signed long r; 954ea930000SAlex Sierra uint32_t seq; 955ea930000SAlex Sierra uint16_t queried_pasid; 956ea930000SAlex Sierra bool ret; 957373008bfSDusica Milinkovic u32 usec_timeout = amdgpu_sriov_vf(adev) ? SRIOV_USEC_TIMEOUT : adev->usec_timeout; 958f87f6864SMukul Joshi struct amdgpu_ring *ring = &adev->gfx.kiq[inst].ring; 959f87f6864SMukul Joshi struct amdgpu_kiq *kiq = &adev->gfx.kiq[inst]; 960ea930000SAlex Sierra 96153b3f8f4SDennis Li if (amdgpu_in_reset(adev)) 962ea930000SAlex Sierra return -EIO; 963ea930000SAlex Sierra 964d0fb18b5SAndrey Grodzovsky if (ring->sched.ready && down_read_trylock(&adev->reset_domain->sem)) { 965b80cd524SFelix Kuehling /* Vega20+XGMI caches PTEs in TC and TLB. Add a 966b80cd524SFelix Kuehling * heavy-weight TLB flush (type 2), which flushes 967b80cd524SFelix Kuehling * both. Due to a race condition with concurrent 968b80cd524SFelix Kuehling * memory accesses using the same TLB cache line, we 969b80cd524SFelix Kuehling * still need a second TLB flush after this. 970b80cd524SFelix Kuehling */ 971b80cd524SFelix Kuehling bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && 972630e959fSAlex Deucher adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0)); 973b80cd524SFelix Kuehling /* 2 dwords flush + 8 dwords fence */ 974b80cd524SFelix Kuehling unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; 975b80cd524SFelix Kuehling 976b80cd524SFelix Kuehling if (vega20_xgmi_wa) 977b80cd524SFelix Kuehling ndw += kiq->pmf->invalidate_tlbs_size; 978b80cd524SFelix Kuehling 979f87f6864SMukul Joshi spin_lock(&adev->gfx.kiq[inst].ring_lock); 98036a1707aSAlex Sierra /* 2 dwords flush + 8 dwords fence */ 981b80cd524SFelix Kuehling amdgpu_ring_alloc(ring, ndw); 982b80cd524SFelix Kuehling if (vega20_xgmi_wa) 983b80cd524SFelix Kuehling kiq->pmf->kiq_invalidate_tlbs(ring, 984b80cd524SFelix Kuehling pasid, 2, all_hub); 9852e8cc5d3SGraham Sider 9862e8cc5d3SGraham Sider if (flush_type == 2 && 9872e8cc5d3SGraham Sider adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3) && 9882e8cc5d3SGraham Sider adev->rev_id == 0) 9892e8cc5d3SGraham Sider kiq->pmf->kiq_invalidate_tlbs(ring, 9902e8cc5d3SGraham Sider pasid, 0, all_hub); 9912e8cc5d3SGraham Sider 992ea930000SAlex Sierra kiq->pmf->kiq_invalidate_tlbs(ring, 993ea930000SAlex Sierra pasid, flush_type, all_hub); 99404e4e2e9SYintian Tao r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 99504e4e2e9SYintian Tao if (r) { 99604e4e2e9SYintian Tao amdgpu_ring_undo(ring); 997f87f6864SMukul Joshi spin_unlock(&adev->gfx.kiq[inst].ring_lock); 998d0fb18b5SAndrey Grodzovsky up_read(&adev->reset_domain->sem); 99904e4e2e9SYintian Tao return -ETIME; 100004e4e2e9SYintian Tao } 100104e4e2e9SYintian Tao 1002ea930000SAlex Sierra amdgpu_ring_commit(ring); 1003f87f6864SMukul Joshi spin_unlock(&adev->gfx.kiq[inst].ring_lock); 1004373008bfSDusica Milinkovic r = amdgpu_fence_wait_polling(ring, seq, usec_timeout); 1005ea930000SAlex Sierra if (r < 1) { 1006aac89168SDennis Li dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); 1007d0fb18b5SAndrey Grodzovsky up_read(&adev->reset_domain->sem); 1008ea930000SAlex Sierra return -ETIME; 1009ea930000SAlex Sierra } 1010d0fb18b5SAndrey Grodzovsky up_read(&adev->reset_domain->sem); 1011ea930000SAlex Sierra return 0; 1012ea930000SAlex Sierra } 1013ea930000SAlex Sierra 1014ea930000SAlex Sierra for (vmid = 1; vmid < 16; vmid++) { 1015ea930000SAlex Sierra 1016ea930000SAlex Sierra ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 1017ea930000SAlex Sierra &queried_pasid); 1018ea930000SAlex Sierra if (ret && queried_pasid == pasid) { 1019ea930000SAlex Sierra if (all_hub) { 1020d9426c3dSLe Ma for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) 1021ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 1022fa34edbeSFelix Kuehling i, flush_type); 1023ea930000SAlex Sierra } else { 1024ea930000SAlex Sierra gmc_v9_0_flush_gpu_tlb(adev, vmid, 1025f4caf584SHawking Zhang AMDGPU_GFXHUB(0), flush_type); 1026ea930000SAlex Sierra } 1027ea930000SAlex Sierra break; 1028ea930000SAlex Sierra } 1029ea930000SAlex Sierra } 1030ea930000SAlex Sierra 1031ea930000SAlex Sierra return 0; 1032ea930000SAlex Sierra 1033ea930000SAlex Sierra } 1034ea930000SAlex Sierra 10359096d6e5SChristian König static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 1036c633c00bSChristian König unsigned vmid, uint64_t pd_addr) 10379096d6e5SChristian König { 10380530553bSLe Ma bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->vm_hub); 1039250b4228SChristian König struct amdgpu_device *adev = ring->adev; 10400530553bSLe Ma struct amdgpu_vmhub *hub = &adev->vmhub[ring->vm_hub]; 10412a79d868SYong Zhao uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 10429096d6e5SChristian König unsigned eng = ring->vm_inv_eng; 10439096d6e5SChristian König 1044f920d1bbSchangzhu /* 1045f920d1bbSchangzhu * It may lose gpuvm invalidate acknowldege state across power-gating 1046f920d1bbSchangzhu * off cycle, add semaphore acquire before invalidation and semaphore 1047f920d1bbSchangzhu * release after invalidation to avoid entering power gated state 1048f920d1bbSchangzhu * to WA the Issue 1049f920d1bbSchangzhu */ 1050f920d1bbSchangzhu 1051f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 105290f6452cSchangzhu if (use_semaphore) 1053f920d1bbSchangzhu /* a read return value of 1 means semaphore acuqire */ 1054f920d1bbSchangzhu amdgpu_ring_emit_reg_wait(ring, 1055148f597dSHuang Rui hub->vm_inv_eng0_sem + 1056148f597dSHuang Rui hub->eng_distance * eng, 0x1, 0x1); 1057f920d1bbSchangzhu 1058148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 1059148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 10609096d6e5SChristian König lower_32_bits(pd_addr)); 10619096d6e5SChristian König 1062148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 1063148f597dSHuang Rui (hub->ctx_addr_distance * vmid), 10649096d6e5SChristian König upper_32_bits(pd_addr)); 10659096d6e5SChristian König 1066148f597dSHuang Rui amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 1067148f597dSHuang Rui hub->eng_distance * eng, 1068148f597dSHuang Rui hub->vm_inv_eng0_ack + 1069148f597dSHuang Rui hub->eng_distance * eng, 1070f8bc9037SAlex Deucher req, 1 << vmid); 1071f732b6b3SChristian König 1072f920d1bbSchangzhu /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 107390f6452cSchangzhu if (use_semaphore) 1074f920d1bbSchangzhu /* 1075f920d1bbSchangzhu * add semaphore release after invalidation, 1076f920d1bbSchangzhu * write with 0 means semaphore release 1077f920d1bbSchangzhu */ 1078148f597dSHuang Rui amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 1079148f597dSHuang Rui hub->eng_distance * eng, 0); 1080f920d1bbSchangzhu 10819096d6e5SChristian König return pd_addr; 10829096d6e5SChristian König } 10839096d6e5SChristian König 1084c633c00bSChristian König static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 1085c633c00bSChristian König unsigned pasid) 1086c633c00bSChristian König { 1087c633c00bSChristian König struct amdgpu_device *adev = ring->adev; 1088c633c00bSChristian König uint32_t reg; 1089c633c00bSChristian König 1090f2d66571SLe Ma /* Do nothing because there's no lut register for mmhub1. */ 1091f4caf584SHawking Zhang if (ring->vm_hub == AMDGPU_MMHUB1(0)) 1092f2d66571SLe Ma return; 1093f2d66571SLe Ma 1094f4caf584SHawking Zhang if (ring->vm_hub == AMDGPU_GFXHUB(0)) 1095c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 1096c633c00bSChristian König else 1097c633c00bSChristian König reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 1098c633c00bSChristian König 1099c633c00bSChristian König amdgpu_ring_emit_wreg(ring, reg, pasid); 1100c633c00bSChristian König } 1101c633c00bSChristian König 1102e60f8db5SAlex Xie /* 1103e60f8db5SAlex Xie * PTE format on VEGA 10: 1104e60f8db5SAlex Xie * 63:59 reserved 1105e60f8db5SAlex Xie * 58:57 mtype 1106e60f8db5SAlex Xie * 56 F 1107e60f8db5SAlex Xie * 55 L 1108e60f8db5SAlex Xie * 54 P 1109e60f8db5SAlex Xie * 53 SW 1110e60f8db5SAlex Xie * 52 T 1111e60f8db5SAlex Xie * 50:48 reserved 1112e60f8db5SAlex Xie * 47:12 4k physical page base address 1113e60f8db5SAlex Xie * 11:7 fragment 1114e60f8db5SAlex Xie * 6 write 1115e60f8db5SAlex Xie * 5 read 1116e60f8db5SAlex Xie * 4 exe 1117e60f8db5SAlex Xie * 3 Z 1118e60f8db5SAlex Xie * 2 snooped 1119e60f8db5SAlex Xie * 1 system 1120e60f8db5SAlex Xie * 0 valid 1121e60f8db5SAlex Xie * 1122e60f8db5SAlex Xie * PDE format on VEGA 10: 1123e60f8db5SAlex Xie * 63:59 block fragment size 1124e60f8db5SAlex Xie * 58:55 reserved 1125e60f8db5SAlex Xie * 54 P 1126e60f8db5SAlex Xie * 53:48 reserved 1127e60f8db5SAlex Xie * 47:6 physical base address of PD or PTE 1128e60f8db5SAlex Xie * 5:3 reserved 1129e60f8db5SAlex Xie * 2 C 1130e60f8db5SAlex Xie * 1 system 1131e60f8db5SAlex Xie * 0 valid 1132e60f8db5SAlex Xie */ 1133e60f8db5SAlex Xie 113471776b6dSChristian König static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) 1135e60f8db5SAlex Xie 1136e60f8db5SAlex Xie { 113771776b6dSChristian König switch (flags) { 1138e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_DEFAULT: 113971776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1140e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_NC: 114171776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1142e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_WC: 114371776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); 1144093e48c0SOak Zeng case AMDGPU_VM_MTYPE_RW: 114571776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); 1146e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_CC: 114771776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); 1148e60f8db5SAlex Xie case AMDGPU_VM_MTYPE_UC: 114971776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); 1150e60f8db5SAlex Xie default: 115171776b6dSChristian König return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1152e60f8db5SAlex Xie } 1153e60f8db5SAlex Xie } 1154e60f8db5SAlex Xie 11553de676d8SChristian König static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 11563de676d8SChristian König uint64_t *addr, uint64_t *flags) 1157f75e237cSChristian König { 1158bbc9fb10SChristian König if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 11590ca565abSOak Zeng *addr = amdgpu_gmc_vram_mc2pa(adev, *addr); 11603de676d8SChristian König BUG_ON(*addr & 0xFFFF00000000003FULL); 11616a42fd6fSChristian König 1162770d13b1SChristian König if (!adev->gmc.translate_further) 11636a42fd6fSChristian König return; 11646a42fd6fSChristian König 11656a42fd6fSChristian König if (level == AMDGPU_VM_PDB1) { 11666a42fd6fSChristian König /* Set the block fragment size */ 11676a42fd6fSChristian König if (!(*flags & AMDGPU_PDE_PTE)) 11686a42fd6fSChristian König *flags |= AMDGPU_PDE_BFS(0x9); 11696a42fd6fSChristian König 11706a42fd6fSChristian König } else if (level == AMDGPU_VM_PDB0) { 117137a0bad6SMukul Joshi if (*flags & AMDGPU_PDE_PTE) { 11726a42fd6fSChristian König *flags &= ~AMDGPU_PDE_PTE; 117337a0bad6SMukul Joshi if (!(*flags & AMDGPU_PTE_VALID)) 117437a0bad6SMukul Joshi *addr |= 1 << PAGE_SHIFT; 117537a0bad6SMukul Joshi } else { 11766a42fd6fSChristian König *flags |= AMDGPU_PTE_TF; 11776a42fd6fSChristian König } 1178f75e237cSChristian König } 117937a0bad6SMukul Joshi } 1180f75e237cSChristian König 1181d1a372afSFelix Kuehling static void gmc_v9_0_get_coherence_flags(struct amdgpu_device *adev, 1182d1a372afSFelix Kuehling struct amdgpu_bo *bo, 1183d1a372afSFelix Kuehling struct amdgpu_bo_va_mapping *mapping, 1184d1a372afSFelix Kuehling uint64_t *flags) 1185d1a372afSFelix Kuehling { 1186d1a372afSFelix Kuehling struct amdgpu_device *bo_adev = amdgpu_ttm_adev(bo->tbo.bdev); 1187d1a372afSFelix Kuehling bool is_vram = bo->tbo.resource->mem_type == TTM_PL_VRAM; 1188d1a372afSFelix Kuehling bool coherent = bo->flags & AMDGPU_GEM_CREATE_COHERENT; 1189d1a372afSFelix Kuehling bool uncached = bo->flags & AMDGPU_GEM_CREATE_UNCACHED; 1190dc12f9edSPhilip Yang struct amdgpu_vm *vm = mapping->bo_va->base.vm; 11911e4a0033SFelix Kuehling unsigned int mtype_local, mtype; 1192d1a372afSFelix Kuehling bool snoop = false; 11931e4a0033SFelix Kuehling bool is_local; 1194d1a372afSFelix Kuehling 1195d1a372afSFelix Kuehling switch (adev->ip_versions[GC_HWIP][0]) { 1196d1a372afSFelix Kuehling case IP_VERSION(9, 4, 1): 1197d1a372afSFelix Kuehling case IP_VERSION(9, 4, 2): 1198d1a372afSFelix Kuehling if (is_vram) { 1199d1a372afSFelix Kuehling if (bo_adev == adev) { 1200d1a372afSFelix Kuehling if (uncached) 1201d1a372afSFelix Kuehling mtype = MTYPE_UC; 1202d1a372afSFelix Kuehling else if (coherent) 1203d1a372afSFelix Kuehling mtype = MTYPE_CC; 1204d1a372afSFelix Kuehling else 1205d1a372afSFelix Kuehling mtype = MTYPE_RW; 1206d1a372afSFelix Kuehling /* FIXME: is this still needed? Or does 1207d1a372afSFelix Kuehling * amdgpu_ttm_tt_pde_flags already handle this? 1208d1a372afSFelix Kuehling */ 1209ab1a157eSHawking Zhang if ((adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) || 1210ab1a157eSHawking Zhang adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) && 1211d1a372afSFelix Kuehling adev->gmc.xgmi.connected_to_cpu) 1212d1a372afSFelix Kuehling snoop = true; 1213d1a372afSFelix Kuehling } else { 1214d1a372afSFelix Kuehling if (uncached || coherent) 1215d1a372afSFelix Kuehling mtype = MTYPE_UC; 1216d1a372afSFelix Kuehling else 1217d1a372afSFelix Kuehling mtype = MTYPE_NC; 1218d1a372afSFelix Kuehling if (mapping->bo_va->is_xgmi) 1219d1a372afSFelix Kuehling snoop = true; 1220d1a372afSFelix Kuehling } 1221d1a372afSFelix Kuehling } else { 1222d1a372afSFelix Kuehling if (uncached || coherent) 1223d1a372afSFelix Kuehling mtype = MTYPE_UC; 1224d1a372afSFelix Kuehling else 1225d1a372afSFelix Kuehling mtype = MTYPE_NC; 1226d1a372afSFelix Kuehling /* FIXME: is this still needed? Or does 1227d1a372afSFelix Kuehling * amdgpu_ttm_tt_pde_flags already handle this? 1228d1a372afSFelix Kuehling */ 1229d1a372afSFelix Kuehling snoop = true; 1230d1a372afSFelix Kuehling } 1231d1a372afSFelix Kuehling break; 1232753b999aSRajneesh Bhardwaj case IP_VERSION(9, 4, 3): 12331e4a0033SFelix Kuehling /* Only local VRAM BOs or system memory on non-NUMA APUs 12341e4a0033SFelix Kuehling * can be assumed to be local in their entirety. Choose 12351e4a0033SFelix Kuehling * MTYPE_NC as safe fallback for all system memory BOs on 12361e4a0033SFelix Kuehling * NUMA systems. Their MTYPE can be overridden per-page in 12371e4a0033SFelix Kuehling * gmc_v9_0_override_vm_pte_flags. 1238753b999aSRajneesh Bhardwaj */ 1239b9cbd510SGraham Sider mtype_local = MTYPE_RW; 124076eb9c95SDavid Francis if (amdgpu_mtype_local == 1) { 124176eb9c95SDavid Francis DRM_INFO_ONCE("Using MTYPE_NC for local memory\n"); 124276eb9c95SDavid Francis mtype_local = MTYPE_NC; 124376eb9c95SDavid Francis } else if (amdgpu_mtype_local == 2) { 124476eb9c95SDavid Francis DRM_INFO_ONCE("Using MTYPE_CC for local memory\n"); 1245b9cbd510SGraham Sider mtype_local = MTYPE_CC; 1246b9cbd510SGraham Sider } else { 1247b9cbd510SGraham Sider DRM_INFO_ONCE("Using MTYPE_RW for local memory\n"); 124876eb9c95SDavid Francis } 12491e4a0033SFelix Kuehling is_local = (!is_vram && (adev->flags & AMD_IS_APU) && 12501e4a0033SFelix Kuehling num_possible_nodes() <= 1) || 1251dc12f9edSPhilip Yang (is_vram && adev == bo_adev && 12523ebfd221SPhilip Yang KFD_XCP_MEM_ID(adev, bo->xcp_id) == vm->mem_id); 1253753b999aSRajneesh Bhardwaj snoop = true; 1254753b999aSRajneesh Bhardwaj if (uncached) { 1255753b999aSRajneesh Bhardwaj mtype = MTYPE_UC; 1256753b999aSRajneesh Bhardwaj } else if (adev->flags & AMD_IS_APU) { 12571e4a0033SFelix Kuehling mtype = is_local ? mtype_local : MTYPE_NC; 1258753b999aSRajneesh Bhardwaj } else { 1259753b999aSRajneesh Bhardwaj /* dGPU */ 12601e4a0033SFelix Kuehling if (is_local) 12611e4a0033SFelix Kuehling mtype = mtype_local; 1262d839a158SGraham Sider else if (is_vram) 1263753b999aSRajneesh Bhardwaj mtype = MTYPE_NC; 1264d839a158SGraham Sider else 1265d839a158SGraham Sider mtype = MTYPE_UC; 1266753b999aSRajneesh Bhardwaj } 1267753b999aSRajneesh Bhardwaj 1268753b999aSRajneesh Bhardwaj break; 1269d1a372afSFelix Kuehling default: 1270d1a372afSFelix Kuehling if (uncached || coherent) 1271d1a372afSFelix Kuehling mtype = MTYPE_UC; 1272d1a372afSFelix Kuehling else 1273d1a372afSFelix Kuehling mtype = MTYPE_NC; 1274d1a372afSFelix Kuehling 1275d1a372afSFelix Kuehling /* FIXME: is this still needed? Or does 1276d1a372afSFelix Kuehling * amdgpu_ttm_tt_pde_flags already handle this? 1277d1a372afSFelix Kuehling */ 1278d1a372afSFelix Kuehling if (!is_vram) 1279d1a372afSFelix Kuehling snoop = true; 1280d1a372afSFelix Kuehling } 1281d1a372afSFelix Kuehling 1282d1a372afSFelix Kuehling if (mtype != MTYPE_NC) 1283d1a372afSFelix Kuehling *flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) | 1284d1a372afSFelix Kuehling AMDGPU_PTE_MTYPE_VG10(mtype); 1285d1a372afSFelix Kuehling *flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; 1286d1a372afSFelix Kuehling } 1287d1a372afSFelix Kuehling 1288cbfae36cSChristian König static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, 1289cbfae36cSChristian König struct amdgpu_bo_va_mapping *mapping, 1290cbfae36cSChristian König uint64_t *flags) 1291cbfae36cSChristian König { 12929c3db58bSChristian König struct amdgpu_bo *bo = mapping->bo_va->base.bo; 12939c3db58bSChristian König 1294cbfae36cSChristian König *flags &= ~AMDGPU_PTE_EXECUTABLE; 1295cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1296cbfae36cSChristian König 1297cbfae36cSChristian König *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1298cbfae36cSChristian König *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; 1299cbfae36cSChristian König 1300cbfae36cSChristian König if (mapping->flags & AMDGPU_PTE_PRT) { 1301cbfae36cSChristian König *flags |= AMDGPU_PTE_PRT; 1302cbfae36cSChristian König *flags &= ~AMDGPU_PTE_VALID; 1303cbfae36cSChristian König } 1304cbfae36cSChristian König 13059c3db58bSChristian König if (bo && bo->tbo.resource) 1306d1a372afSFelix Kuehling gmc_v9_0_get_coherence_flags(adev, mapping->bo_va->base.bo, 1307d1a372afSFelix Kuehling mapping, flags); 1308cbfae36cSChristian König } 1309cbfae36cSChristian König 1310352b919cSFelix Kuehling static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev, 1311352b919cSFelix Kuehling struct amdgpu_vm *vm, 1312352b919cSFelix Kuehling uint64_t addr, uint64_t *flags) 1313352b919cSFelix Kuehling { 1314352b919cSFelix Kuehling int local_node, nid; 1315352b919cSFelix Kuehling 1316352b919cSFelix Kuehling /* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system 1317352b919cSFelix Kuehling * memory can use more efficient MTYPEs. 1318352b919cSFelix Kuehling */ 1319352b919cSFelix Kuehling if (adev->ip_versions[GC_HWIP][0] != IP_VERSION(9, 4, 3)) 1320352b919cSFelix Kuehling return; 1321352b919cSFelix Kuehling 1322352b919cSFelix Kuehling /* Only direct-mapped memory allows us to determine the NUMA node from 1323352b919cSFelix Kuehling * the DMA address. 1324352b919cSFelix Kuehling */ 1325352b919cSFelix Kuehling if (!adev->ram_is_direct_mapped) { 1326352b919cSFelix Kuehling dev_dbg(adev->dev, "RAM is not direct mapped\n"); 1327352b919cSFelix Kuehling return; 1328352b919cSFelix Kuehling } 1329352b919cSFelix Kuehling 1330352b919cSFelix Kuehling /* Only override mappings with MTYPE_NC, which is the safe default for 1331352b919cSFelix Kuehling * cacheable memory. 1332352b919cSFelix Kuehling */ 1333352b919cSFelix Kuehling if ((*flags & AMDGPU_PTE_MTYPE_VG10_MASK) != 1334352b919cSFelix Kuehling AMDGPU_PTE_MTYPE_VG10(MTYPE_NC)) { 1335352b919cSFelix Kuehling dev_dbg(adev->dev, "MTYPE is not NC\n"); 1336352b919cSFelix Kuehling return; 1337352b919cSFelix Kuehling } 1338352b919cSFelix Kuehling 1339dc12f9edSPhilip Yang /* FIXME: Only supported on native mode for now. For carve-out, the 1340352b919cSFelix Kuehling * NUMA affinity of the GPU/VM needs to come from the PCI info because 1341352b919cSFelix Kuehling * memory partitions are not associated with different NUMA nodes. 1342352b919cSFelix Kuehling */ 1343dc12f9edSPhilip Yang if (adev->gmc.is_app_apu && vm->mem_id >= 0) { 1344dc12f9edSPhilip Yang local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node; 1345352b919cSFelix Kuehling } else { 1346352b919cSFelix Kuehling dev_dbg(adev->dev, "Only native mode APU is supported.\n"); 1347352b919cSFelix Kuehling return; 1348352b919cSFelix Kuehling } 1349352b919cSFelix Kuehling 1350352b919cSFelix Kuehling /* Only handle real RAM. Mappings of PCIe resources don't have struct 1351352b919cSFelix Kuehling * page or NUMA nodes. 1352352b919cSFelix Kuehling */ 1353352b919cSFelix Kuehling if (!page_is_ram(addr >> PAGE_SHIFT)) { 1354352b919cSFelix Kuehling dev_dbg(adev->dev, "Page is not RAM.\n"); 1355352b919cSFelix Kuehling return; 1356352b919cSFelix Kuehling } 1357352b919cSFelix Kuehling nid = pfn_to_nid(addr >> PAGE_SHIFT); 1358352b919cSFelix Kuehling dev_dbg(adev->dev, "vm->mem_id=%d, local_node=%d, nid=%d\n", 1359dc12f9edSPhilip Yang vm->mem_id, local_node, nid); 1360352b919cSFelix Kuehling if (nid == local_node) { 1361352b919cSFelix Kuehling uint64_t old_flags = *flags; 1362b9cbd510SGraham Sider unsigned int mtype_local = MTYPE_RW; 136376eb9c95SDavid Francis 136476eb9c95SDavid Francis if (amdgpu_mtype_local == 1) 136576eb9c95SDavid Francis mtype_local = MTYPE_NC; 136676eb9c95SDavid Francis else if (amdgpu_mtype_local == 2) 1367b9cbd510SGraham Sider mtype_local = MTYPE_CC; 1368352b919cSFelix Kuehling 1369352b919cSFelix Kuehling *flags = (*flags & ~AMDGPU_PTE_MTYPE_VG10_MASK) | 1370352b919cSFelix Kuehling AMDGPU_PTE_MTYPE_VG10(mtype_local); 1371352b919cSFelix Kuehling dev_dbg(adev->dev, "flags updated from %llx to %llx\n", 1372352b919cSFelix Kuehling old_flags, *flags); 1373352b919cSFelix Kuehling } 1374352b919cSFelix Kuehling } 1375352b919cSFelix Kuehling 13767b885f0eSAlex Deucher static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 13777b885f0eSAlex Deucher { 13787b885f0eSAlex Deucher u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 13797b885f0eSAlex Deucher unsigned size; 13807b885f0eSAlex Deucher 1381dc5d4affSHarry Wentland /* TODO move to DC so GMC doesn't need to hard-code DCN registers */ 1382dc5d4affSHarry Wentland 13837b885f0eSAlex Deucher if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 13847b885f0eSAlex Deucher size = AMDGPU_VBIOS_VGA_ALLOCATION; 13857b885f0eSAlex Deucher } else { 13867b885f0eSAlex Deucher u32 viewport; 13877b885f0eSAlex Deucher 1388630e959fSAlex Deucher switch (adev->ip_versions[DCE_HWIP][0]) { 1389630e959fSAlex Deucher case IP_VERSION(1, 0, 0): 1390630e959fSAlex Deucher case IP_VERSION(1, 0, 1): 13917b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 13927b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, 13937b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 13947b885f0eSAlex Deucher REG_GET_FIELD(viewport, 13957b885f0eSAlex Deucher HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 13967b885f0eSAlex Deucher 4); 13977b885f0eSAlex Deucher break; 1398dc5d4affSHarry Wentland case IP_VERSION(2, 1, 0): 1399dc5d4affSHarry Wentland viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_DCN2); 1400dc5d4affSHarry Wentland size = (REG_GET_FIELD(viewport, 1401dc5d4affSHarry Wentland HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 1402dc5d4affSHarry Wentland REG_GET_FIELD(viewport, 1403dc5d4affSHarry Wentland HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 1404dc5d4affSHarry Wentland 4); 1405dc5d4affSHarry Wentland break; 14067b885f0eSAlex Deucher default: 14077b885f0eSAlex Deucher viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 14087b885f0eSAlex Deucher size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 14097b885f0eSAlex Deucher REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 14107b885f0eSAlex Deucher 4); 14117b885f0eSAlex Deucher break; 14127b885f0eSAlex Deucher } 14137b885f0eSAlex Deucher } 14147b885f0eSAlex Deucher 14157b885f0eSAlex Deucher return size; 14167b885f0eSAlex Deucher } 14177b885f0eSAlex Deucher 1418b6f90baaSLijo Lazar static enum amdgpu_memory_partition 14190f2e1d62SLijo Lazar gmc_v9_0_get_memory_partition(struct amdgpu_device *adev, u32 *supp_modes) 1420b6f90baaSLijo Lazar { 1421b6f90baaSLijo Lazar enum amdgpu_memory_partition mode = UNKNOWN_MEMORY_PARTITION_MODE; 1422b6f90baaSLijo Lazar 1423b6f90baaSLijo Lazar if (adev->nbio.funcs->get_memory_partition_mode) 14240f2e1d62SLijo Lazar mode = adev->nbio.funcs->get_memory_partition_mode(adev, 14250f2e1d62SLijo Lazar supp_modes); 1426b6f90baaSLijo Lazar 1427b6f90baaSLijo Lazar return mode; 1428b6f90baaSLijo Lazar } 1429b6f90baaSLijo Lazar 14300f2e1d62SLijo Lazar static enum amdgpu_memory_partition 14310f2e1d62SLijo Lazar gmc_v9_0_query_memory_partition(struct amdgpu_device *adev) 14320f2e1d62SLijo Lazar { 143346f7b4deSGavin Wan if (amdgpu_sriov_vf(adev)) 143446f7b4deSGavin Wan return AMDGPU_NPS1_PARTITION_MODE; 143546f7b4deSGavin Wan 14360f2e1d62SLijo Lazar return gmc_v9_0_get_memory_partition(adev, NULL); 14370f2e1d62SLijo Lazar } 14380f2e1d62SLijo Lazar 1439132f34e4SChristian König static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 1440132f34e4SChristian König .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 1441ea930000SAlex Sierra .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, 14429096d6e5SChristian König .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 1443c633c00bSChristian König .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 144471776b6dSChristian König .map_mtype = gmc_v9_0_map_mtype, 1445cbfae36cSChristian König .get_vm_pde = gmc_v9_0_get_vm_pde, 14467b885f0eSAlex Deucher .get_vm_pte = gmc_v9_0_get_vm_pte, 1447352b919cSFelix Kuehling .override_vm_pte_flags = gmc_v9_0_override_vm_pte_flags, 14487b885f0eSAlex Deucher .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, 1449b6f90baaSLijo Lazar .query_mem_partition_mode = &gmc_v9_0_query_memory_partition, 1450e60f8db5SAlex Xie }; 1451e60f8db5SAlex Xie 1452132f34e4SChristian König static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 1453e60f8db5SAlex Xie { 1454132f34e4SChristian König adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 1455e60f8db5SAlex Xie } 1456e60f8db5SAlex Xie 14575b6b35aaSHawking Zhang static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) 14585b6b35aaSHawking Zhang { 1459630e959fSAlex Deucher switch (adev->ip_versions[UMC_HWIP][0]) { 1460630e959fSAlex Deucher case IP_VERSION(6, 0, 0): 1461e7da754bSMonk Liu adev->umc.funcs = &umc_v6_0_funcs; 1462e7da754bSMonk Liu break; 1463630e959fSAlex Deucher case IP_VERSION(6, 1, 1): 14643aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 14653aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 14663aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 14674cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; 1468e69c7857STao Zhou adev->umc.retire_unit = 1; 14694cf781c2SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 1470efe17d5aSyipechai adev->umc.ras = &umc_v6_1_ras; 14714cf781c2SJohn Clements break; 1472630e959fSAlex Deucher case IP_VERSION(6, 1, 2): 14733aacf4eaSTao Zhou adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 14743aacf4eaSTao Zhou adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 14753aacf4eaSTao Zhou adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 14764cf781c2SJohn Clements adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; 1477e69c7857STao Zhou adev->umc.retire_unit = 1; 14783aacf4eaSTao Zhou adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 1479efe17d5aSyipechai adev->umc.ras = &umc_v6_1_ras; 14805b6b35aaSHawking Zhang break; 1481630e959fSAlex Deucher case IP_VERSION(6, 7, 0): 1482498d46feSTao Zhou adev->umc.max_ras_err_cnt_per_query = 1483498d46feSTao Zhou UMC_V6_7_TOTAL_CHANNEL_NUM * UMC_V6_7_BAD_PAGE_NUM_PER_CHANNEL; 1484719e433eSMukul Joshi adev->umc.channel_inst_num = UMC_V6_7_CHANNEL_INSTANCE_NUM; 1485719e433eSMukul Joshi adev->umc.umc_inst_num = UMC_V6_7_UMC_INSTANCE_NUM; 1486186c8a85SJohn Clements adev->umc.channel_offs = UMC_V6_7_PER_CHANNEL_OFFSET; 1487e69c7857STao Zhou adev->umc.retire_unit = (UMC_V6_7_NA_MAP_PA_NUM * 2); 1488186c8a85SJohn Clements if (!adev->gmc.xgmi.connected_to_cpu) 1489efe17d5aSyipechai adev->umc.ras = &umc_v6_7_ras; 1490186c8a85SJohn Clements if (1 & adev->smuio.funcs->get_die_id(adev)) 1491186c8a85SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_first[0][0]; 1492186c8a85SJohn Clements else 1493186c8a85SJohn Clements adev->umc.channel_idx_tbl = &umc_v6_7_channel_idx_tbl_second[0][0]; 1494186c8a85SJohn Clements break; 14955b6b35aaSHawking Zhang default: 14965b6b35aaSHawking Zhang break; 14975b6b35aaSHawking Zhang } 14985b6b35aaSHawking Zhang } 14995b6b35aaSHawking Zhang 15003d093da0STao Zhou static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) 15013d093da0STao Zhou { 1502630e959fSAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 1503630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 1504f6c3623bSDennis Li adev->mmhub.funcs = &mmhub_v9_4_funcs; 1505f6c3623bSDennis Li break; 1506630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 15074da999cdSOak Zeng adev->mmhub.funcs = &mmhub_v1_7_funcs; 15084da999cdSOak Zeng break; 1509018f7300SLe Ma case IP_VERSION(1, 8, 0): 1510018f7300SLe Ma adev->mmhub.funcs = &mmhub_v1_8_funcs; 1511018f7300SLe Ma break; 15123d093da0STao Zhou default: 15139fb1506eSOak Zeng adev->mmhub.funcs = &mmhub_v1_0_funcs; 15143d093da0STao Zhou break; 15153d093da0STao Zhou } 15163d093da0STao Zhou } 15173d093da0STao Zhou 1518d844c6d7SHawking Zhang static void gmc_v9_0_set_mmhub_ras_funcs(struct amdgpu_device *adev) 1519d844c6d7SHawking Zhang { 1520630e959fSAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 1521630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 15225e67bba3Syipechai adev->mmhub.ras = &mmhub_v1_0_ras; 1523d844c6d7SHawking Zhang break; 1524630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 15255e67bba3Syipechai adev->mmhub.ras = &mmhub_v9_4_ras; 1526d844c6d7SHawking Zhang break; 1527630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 15285e67bba3Syipechai adev->mmhub.ras = &mmhub_v1_7_ras; 1529d844c6d7SHawking Zhang break; 153073c2b3fdSHawking Zhang case IP_VERSION(1, 8, 0): 153173c2b3fdSHawking Zhang adev->mmhub.ras = &mmhub_v1_8_ras; 153273c2b3fdSHawking Zhang break; 1533d844c6d7SHawking Zhang default: 1534d844c6d7SHawking Zhang /* mmhub ras is not available */ 1535d844c6d7SHawking Zhang break; 1536d844c6d7SHawking Zhang } 1537d844c6d7SHawking Zhang } 1538d844c6d7SHawking Zhang 15398ffff9b4SOak Zeng static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev) 15408ffff9b4SOak Zeng { 1541018f7300SLe Ma if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) 1542018f7300SLe Ma adev->gfxhub.funcs = &gfxhub_v1_2_funcs; 1543018f7300SLe Ma else 15448ffff9b4SOak Zeng adev->gfxhub.funcs = &gfxhub_v1_0_funcs; 15458ffff9b4SOak Zeng } 15468ffff9b4SOak Zeng 15476f12507fSHawking Zhang static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev) 15486f12507fSHawking Zhang { 15496d76e904Syipechai adev->hdp.ras = &hdp_v4_0_ras; 15506f12507fSHawking Zhang } 15516f12507fSHawking Zhang 15527f544c54SHawking Zhang static void gmc_v9_0_set_mca_ras_funcs(struct amdgpu_device *adev) 15533907c492SJohn Clements { 15547f544c54SHawking Zhang struct amdgpu_mca *mca = &adev->mca; 15557f544c54SHawking Zhang 1556630e959fSAlex Deucher /* is UMC the right IP to check for MCA? Maybe DF? */ 1557630e959fSAlex Deucher switch (adev->ip_versions[UMC_HWIP][0]) { 1558630e959fSAlex Deucher case IP_VERSION(6, 7, 0): 15597f544c54SHawking Zhang if (!adev->gmc.xgmi.connected_to_cpu) { 15607f544c54SHawking Zhang mca->mp0.ras = &mca_v3_0_mp0_ras; 15617f544c54SHawking Zhang mca->mp1.ras = &mca_v3_0_mp1_ras; 15627f544c54SHawking Zhang mca->mpio.ras = &mca_v3_0_mpio_ras; 15637f544c54SHawking Zhang } 15643907c492SJohn Clements break; 15653907c492SJohn Clements default: 15663907c492SJohn Clements break; 15673907c492SJohn Clements } 15683907c492SJohn Clements } 15693907c492SJohn Clements 1570da9d669eSHawking Zhang static void gmc_v9_0_set_xgmi_ras_funcs(struct amdgpu_device *adev) 1571da9d669eSHawking Zhang { 1572da9d669eSHawking Zhang if (!adev->gmc.xgmi.connected_to_cpu) 1573da9d669eSHawking Zhang adev->gmc.xgmi.ras = &xgmi_ras; 1574da9d669eSHawking Zhang } 1575da9d669eSHawking Zhang 1576e60f8db5SAlex Xie static int gmc_v9_0_early_init(void *handle) 1577e60f8db5SAlex Xie { 1578e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1579e60f8db5SAlex Xie 15809eb7681fSShiwu Zhang /* 15819eb7681fSShiwu Zhang * 9.4.0, 9.4.1 and 9.4.3 don't have XGMI defined 15829eb7681fSShiwu Zhang * in their IP discovery tables 15839eb7681fSShiwu Zhang */ 15849eb7681fSShiwu Zhang if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 0) || 15859eb7681fSShiwu Zhang adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || 15869eb7681fSShiwu Zhang adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) 158731691b8dSRajneesh Bhardwaj adev->gmc.xgmi.supported = true; 158831691b8dSRajneesh Bhardwaj 1589630e959fSAlex Deucher if (adev->ip_versions[XGMI_HWIP][0] == IP_VERSION(6, 1, 0)) { 159031691b8dSRajneesh Bhardwaj adev->gmc.xgmi.supported = true; 159131691b8dSRajneesh Bhardwaj adev->gmc.xgmi.connected_to_cpu = 159231691b8dSRajneesh Bhardwaj adev->smuio.funcs->is_host_gpu_xgmi_supported(adev); 159331691b8dSRajneesh Bhardwaj } 159431691b8dSRajneesh Bhardwaj 1595497db7eaSRajneesh Bhardwaj if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) { 1596497db7eaSRajneesh Bhardwaj enum amdgpu_pkg_type pkg_type = 1597497db7eaSRajneesh Bhardwaj adev->smuio.funcs->get_pkg_type(adev); 1598497db7eaSRajneesh Bhardwaj /* On GFXIP 9.4.3. APU, there is no physical VRAM domain present 1599497db7eaSRajneesh Bhardwaj * and the APU, can be in used two possible modes: 1600497db7eaSRajneesh Bhardwaj * - carveout mode 1601497db7eaSRajneesh Bhardwaj * - native APU mode 1602497db7eaSRajneesh Bhardwaj * "is_app_apu" can be used to identify the APU in the native 1603497db7eaSRajneesh Bhardwaj * mode. 1604497db7eaSRajneesh Bhardwaj */ 1605497db7eaSRajneesh Bhardwaj adev->gmc.is_app_apu = (pkg_type == AMDGPU_PKG_TYPE_APU && 1606497db7eaSRajneesh Bhardwaj !pci_resource_len(adev->pdev, 0)); 1607497db7eaSRajneesh Bhardwaj } 1608497db7eaSRajneesh Bhardwaj 160949070c4eSHawking Zhang gmc_v9_0_set_gmc_funcs(adev); 161049070c4eSHawking Zhang gmc_v9_0_set_irq_funcs(adev); 161149070c4eSHawking Zhang gmc_v9_0_set_umc_funcs(adev); 161249070c4eSHawking Zhang gmc_v9_0_set_mmhub_funcs(adev); 1613d844c6d7SHawking Zhang gmc_v9_0_set_mmhub_ras_funcs(adev); 161449070c4eSHawking Zhang gmc_v9_0_set_gfxhub_funcs(adev); 16156f12507fSHawking Zhang gmc_v9_0_set_hdp_ras_funcs(adev); 16167f544c54SHawking Zhang gmc_v9_0_set_mca_ras_funcs(adev); 1617da9d669eSHawking Zhang gmc_v9_0_set_xgmi_ras_funcs(adev); 161849070c4eSHawking Zhang 1619770d13b1SChristian König adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 1620770d13b1SChristian König adev->gmc.shared_aperture_end = 1621770d13b1SChristian König adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 1622bfa8eea2SFlora Cui adev->gmc.private_aperture_start = 0x1000000000000000ULL; 1623770d13b1SChristian König adev->gmc.private_aperture_end = 1624770d13b1SChristian König adev->gmc.private_aperture_start + (4ULL << 30) - 1; 1625*e77673d1SMukul Joshi adev->gmc.noretry_flags = AMDGPU_VM_NORETRY_FLAGS_TF; 1626a7ea6548SAlex Deucher 1627e60f8db5SAlex Xie return 0; 1628e60f8db5SAlex Xie } 1629e60f8db5SAlex Xie 1630e60f8db5SAlex Xie static int gmc_v9_0_late_init(void *handle) 1631e60f8db5SAlex Xie { 1632e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1633c5b6e585STao Zhou int r; 16344789c463SChristian König 1635bdbe90f0SAlex Deucher r = amdgpu_gmc_allocate_vm_inv_eng(adev); 1636c713a461SEvan Quan if (r) 1637c713a461SEvan Quan return r; 16384a20300bSGuchun Chen 16394a20300bSGuchun Chen /* 16404a20300bSGuchun Chen * Workaround performance drop issue with VBIOS enables partial 16414a20300bSGuchun Chen * writes, while disables HBM ECC for vega10. 16424a20300bSGuchun Chen */ 1643630e959fSAlex Deucher if (!amdgpu_sriov_vf(adev) && 1644630e959fSAlex Deucher (adev->ip_versions[UMC_HWIP][0] == IP_VERSION(6, 0, 0))) { 16458ab0d6f0SLuben Tuikov if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) { 1646cace4bffSHawking Zhang if (adev->df.funcs && 1647cace4bffSHawking Zhang adev->df.funcs->enable_ecc_force_par_wr_rmw) 1648bdf84a80SJoseph Greathouse adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); 16494a20300bSGuchun Chen } 1650f49ea9f8SHawking Zhang } 165102bab923SDavid Panariti 16528f6368a9SJohn Clements if (!amdgpu_persistent_edc_harvesting_supported(adev)) { 16535e67bba3Syipechai if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops && 16545e67bba3Syipechai adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count) 16555e67bba3Syipechai adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev); 1656fe5211f1SHawking Zhang 16576d76e904Syipechai if (adev->hdp.ras && adev->hdp.ras->ras_block.hw_ops && 16586d76e904Syipechai adev->hdp.ras->ras_block.hw_ops->reset_ras_error_count) 16596d76e904Syipechai adev->hdp.ras->ras_block.hw_ops->reset_ras_error_count(adev); 16608f6368a9SJohn Clements } 166178871b6cSHawking Zhang 1662ba083492STao Zhou r = amdgpu_gmc_ras_late_init(adev); 1663791c4769Sxinhui pan if (r) 1664e60f8db5SAlex Xie return r; 1665e60f8db5SAlex Xie 1666770d13b1SChristian König return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 1667e60f8db5SAlex Xie } 1668e60f8db5SAlex Xie 1669e60f8db5SAlex Xie static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 1670770d13b1SChristian König struct amdgpu_gmc *mc) 1671e60f8db5SAlex Xie { 1672adbe2e3dSZhigang Luo u64 base = adev->mmhub.funcs->get_fb_location(adev); 16739d4f837aSFrank.Min 16746fdd68b1SAlex Deucher /* add the xgmi offset of the physical node */ 16756fdd68b1SAlex Deucher base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1676f527f310SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 1677f527f310SOak Zeng amdgpu_gmc_sysvm_location(adev, mc); 1678f527f310SOak Zeng } else { 167983afe835SOak Zeng amdgpu_gmc_vram_location(adev, mc, base); 1680961c75cfSChristian König amdgpu_gmc_gart_location(adev, mc); 1681c3e1b43cSChristian König amdgpu_gmc_agp_location(adev, mc); 1682f527f310SOak Zeng } 1683e60f8db5SAlex Xie /* base offset of vram pages */ 16848ffff9b4SOak Zeng adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); 16856fdd68b1SAlex Deucher 16866fdd68b1SAlex Deucher /* XXX: add the xgmi offset of the physical node? */ 16876fdd68b1SAlex Deucher adev->vm_manager.vram_base_offset += 16886fdd68b1SAlex Deucher adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1689e60f8db5SAlex Xie } 1690e60f8db5SAlex Xie 1691e60f8db5SAlex Xie /** 1692e60f8db5SAlex Xie * gmc_v9_0_mc_init - initialize the memory controller driver params 1693e60f8db5SAlex Xie * 1694e60f8db5SAlex Xie * @adev: amdgpu_device pointer 1695e60f8db5SAlex Xie * 1696e60f8db5SAlex Xie * Look up the amount of vram, vram width, and decide how to place 1697e60f8db5SAlex Xie * vram and gart within the GPU's physical address space. 1698e60f8db5SAlex Xie * Returns 0 for success. 1699e60f8db5SAlex Xie */ 1700e60f8db5SAlex Xie static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 1701e60f8db5SAlex Xie { 1702e60f8db5SAlex Xie int r; 1703e60f8db5SAlex Xie 1704e60f8db5SAlex Xie /* size in MB on si */ 1705228ce176SRajneesh Bhardwaj if (!adev->gmc.is_app_apu) { 1706770d13b1SChristian König adev->gmc.mc_vram_size = 1707bebc0762SHawking Zhang adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 1708228ce176SRajneesh Bhardwaj } else { 1709228ce176SRajneesh Bhardwaj DRM_DEBUG("Set mc_vram_size = 0 for APP APU\n"); 1710228ce176SRajneesh Bhardwaj adev->gmc.mc_vram_size = 0; 1711228ce176SRajneesh Bhardwaj } 1712770d13b1SChristian König adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 1713e60f8db5SAlex Xie 1714be566196SOak Zeng if (!(adev->flags & AMD_IS_APU) && 1715be566196SOak Zeng !adev->gmc.xgmi.connected_to_cpu) { 1716e60f8db5SAlex Xie r = amdgpu_device_resize_fb_bar(adev); 1717e60f8db5SAlex Xie if (r) 1718e60f8db5SAlex Xie return r; 1719e60f8db5SAlex Xie } 1720770d13b1SChristian König adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 1721770d13b1SChristian König adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 1722e60f8db5SAlex Xie 1723156a81beSChunming Zhou #ifdef CONFIG_X86_64 172431691b8dSRajneesh Bhardwaj /* 172531691b8dSRajneesh Bhardwaj * AMD Accelerated Processing Platform (APP) supporting GPU-HOST xgmi 172631691b8dSRajneesh Bhardwaj * interface can use VRAM through here as it appears system reserved 172731691b8dSRajneesh Bhardwaj * memory in host address space. 172831691b8dSRajneesh Bhardwaj * 172931691b8dSRajneesh Bhardwaj * For APUs, VRAM is just the stolen system memory and can be accessed 173031691b8dSRajneesh Bhardwaj * directly. 173131691b8dSRajneesh Bhardwaj * 173231691b8dSRajneesh Bhardwaj * Otherwise, use the legacy Host Data Path (HDP) through PCIe BAR. 173331691b8dSRajneesh Bhardwaj */ 173431691b8dSRajneesh Bhardwaj 173531691b8dSRajneesh Bhardwaj /* check whether both host-gpu and gpu-gpu xgmi links exist */ 1736b0a3bbf9SGavin Wan if ((!amdgpu_sriov_vf(adev) && 1737b0a3bbf9SGavin Wan (adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) || 17383de60d96SHawking Zhang (adev->gmc.xgmi.supported && 17393de60d96SHawking Zhang adev->gmc.xgmi.connected_to_cpu)) { 17403de60d96SHawking Zhang adev->gmc.aper_base = 17413de60d96SHawking Zhang adev->gfxhub.funcs->get_mc_fb_offset(adev) + 17423de60d96SHawking Zhang adev->gmc.xgmi.physical_node_id * 174331691b8dSRajneesh Bhardwaj adev->gmc.xgmi.node_segment_size; 1744156a81beSChunming Zhou adev->gmc.aper_size = adev->gmc.real_vram_size; 1745156a81beSChunming Zhou } 174631691b8dSRajneesh Bhardwaj 1747156a81beSChunming Zhou #endif 1748770d13b1SChristian König adev->gmc.visible_vram_size = adev->gmc.aper_size; 1749e60f8db5SAlex Xie 1750e60f8db5SAlex Xie /* set the gart size */ 1751e60f8db5SAlex Xie if (amdgpu_gart_size == -1) { 1752630e959fSAlex Deucher switch (adev->ip_versions[GC_HWIP][0]) { 1753630e959fSAlex Deucher case IP_VERSION(9, 0, 1): /* all engines support GPUVM */ 1754630e959fSAlex Deucher case IP_VERSION(9, 2, 1): /* all engines support GPUVM */ 1755630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 1756630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 1757630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 1758ab1a157eSHawking Zhang case IP_VERSION(9, 4, 3): 1759e60f8db5SAlex Xie default: 1760fe19b862SMonk Liu adev->gmc.gart_size = 512ULL << 20; 1761e60f8db5SAlex Xie break; 1762630e959fSAlex Deucher case IP_VERSION(9, 1, 0): /* DCE SG support */ 1763630e959fSAlex Deucher case IP_VERSION(9, 2, 2): /* DCE SG support */ 1764630e959fSAlex Deucher case IP_VERSION(9, 3, 0): 1765770d13b1SChristian König adev->gmc.gart_size = 1024ULL << 20; 1766e60f8db5SAlex Xie break; 1767e60f8db5SAlex Xie } 1768e60f8db5SAlex Xie } else { 1769770d13b1SChristian König adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 1770e60f8db5SAlex Xie } 1771e60f8db5SAlex Xie 1772f1dc12caSOak Zeng adev->gmc.gart_size += adev->pm.smu_prv_buffer_size; 1773f1dc12caSOak Zeng 1774770d13b1SChristian König gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 1775e60f8db5SAlex Xie 1776e60f8db5SAlex Xie return 0; 1777e60f8db5SAlex Xie } 1778e60f8db5SAlex Xie 1779e60f8db5SAlex Xie static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 1780e60f8db5SAlex Xie { 1781e60f8db5SAlex Xie int r; 1782e60f8db5SAlex Xie 17831123b989SChristian König if (adev->gart.bo) { 1784e60f8db5SAlex Xie WARN(1, "VEGA10 PCIE GART already initialized\n"); 1785e60f8db5SAlex Xie return 0; 1786e60f8db5SAlex Xie } 17877b454b3aSOak Zeng 17887b454b3aSOak Zeng if (adev->gmc.xgmi.connected_to_cpu) { 17897b454b3aSOak Zeng adev->gmc.vmid0_page_table_depth = 1; 17907b454b3aSOak Zeng adev->gmc.vmid0_page_table_block_size = 12; 17917b454b3aSOak Zeng } else { 17927b454b3aSOak Zeng adev->gmc.vmid0_page_table_depth = 0; 17937b454b3aSOak Zeng adev->gmc.vmid0_page_table_block_size = 0; 17947b454b3aSOak Zeng } 17957b454b3aSOak Zeng 1796e60f8db5SAlex Xie /* Initialize common gart structure */ 1797e60f8db5SAlex Xie r = amdgpu_gart_init(adev); 1798e60f8db5SAlex Xie if (r) 1799e60f8db5SAlex Xie return r; 1800e60f8db5SAlex Xie adev->gart.table_size = adev->gart.num_gpu_pages * 8; 18017596ab68SHawking Zhang adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | 1802e60f8db5SAlex Xie AMDGPU_PTE_EXECUTABLE; 1803522510a6SOak Zeng 1804c9a502e9SFelix Kuehling if (!adev->gmc.real_vram_size) { 1805c9a502e9SFelix Kuehling dev_info(adev->dev, "Put GART in system memory for APU\n"); 1806c9a502e9SFelix Kuehling r = amdgpu_gart_table_ram_alloc(adev); 1807c9a502e9SFelix Kuehling if (r) 1808c9a502e9SFelix Kuehling dev_err(adev->dev, "Failed to allocate GART in system memory\n"); 1809c9a502e9SFelix Kuehling } else { 1810522510a6SOak Zeng r = amdgpu_gart_table_vram_alloc(adev); 1811522510a6SOak Zeng if (r) 1812522510a6SOak Zeng return r; 1813522510a6SOak Zeng 1814c9a502e9SFelix Kuehling if (adev->gmc.xgmi.connected_to_cpu) 1815522510a6SOak Zeng r = amdgpu_gmc_pdb0_alloc(adev); 1816522510a6SOak Zeng } 1817522510a6SOak Zeng 1818522510a6SOak Zeng return r; 1819e60f8db5SAlex Xie } 1820e60f8db5SAlex Xie 1821b0a2db9bSAlex Deucher /** 1822b0a2db9bSAlex Deucher * gmc_v9_0_save_registers - saves regs 1823b0a2db9bSAlex Deucher * 1824b0a2db9bSAlex Deucher * @adev: amdgpu_device pointer 1825b0a2db9bSAlex Deucher * 1826b0a2db9bSAlex Deucher * This saves potential register values that should be 1827b0a2db9bSAlex Deucher * restored upon resume 1828b0a2db9bSAlex Deucher */ 1829b0a2db9bSAlex Deucher static void gmc_v9_0_save_registers(struct amdgpu_device *adev) 1830ebdef28eSAlex Deucher { 1831630e959fSAlex Deucher if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) || 1832630e959fSAlex Deucher (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1))) 1833b0a2db9bSAlex Deucher adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); 1834ebdef28eSAlex Deucher } 1835ebdef28eSAlex Deucher 1836a433f1f5SLijo Lazar static bool gmc_v9_0_validate_partition_info(struct amdgpu_device *adev) 1837a433f1f5SLijo Lazar { 1838a433f1f5SLijo Lazar enum amdgpu_memory_partition mode; 1839a433f1f5SLijo Lazar u32 supp_modes; 1840a433f1f5SLijo Lazar bool valid; 1841a433f1f5SLijo Lazar 1842a433f1f5SLijo Lazar mode = gmc_v9_0_get_memory_partition(adev, &supp_modes); 1843a433f1f5SLijo Lazar 1844a433f1f5SLijo Lazar /* Mode detected by hardware not present in supported modes */ 1845a433f1f5SLijo Lazar if ((mode != UNKNOWN_MEMORY_PARTITION_MODE) && 1846a433f1f5SLijo Lazar !(BIT(mode - 1) & supp_modes)) 1847a433f1f5SLijo Lazar return false; 1848a433f1f5SLijo Lazar 1849a433f1f5SLijo Lazar switch (mode) { 1850a433f1f5SLijo Lazar case UNKNOWN_MEMORY_PARTITION_MODE: 1851a433f1f5SLijo Lazar case AMDGPU_NPS1_PARTITION_MODE: 1852a433f1f5SLijo Lazar valid = (adev->gmc.num_mem_partitions == 1); 1853a433f1f5SLijo Lazar break; 1854a433f1f5SLijo Lazar case AMDGPU_NPS2_PARTITION_MODE: 1855a433f1f5SLijo Lazar valid = (adev->gmc.num_mem_partitions == 2); 1856a433f1f5SLijo Lazar break; 1857a433f1f5SLijo Lazar case AMDGPU_NPS4_PARTITION_MODE: 1858a433f1f5SLijo Lazar valid = (adev->gmc.num_mem_partitions == 3 || 1859a433f1f5SLijo Lazar adev->gmc.num_mem_partitions == 4); 1860a433f1f5SLijo Lazar break; 1861a433f1f5SLijo Lazar default: 1862a433f1f5SLijo Lazar valid = false; 1863a433f1f5SLijo Lazar } 1864a433f1f5SLijo Lazar 1865a433f1f5SLijo Lazar return valid; 1866a433f1f5SLijo Lazar } 1867a433f1f5SLijo Lazar 1868a433f1f5SLijo Lazar static bool gmc_v9_0_is_node_present(int *node_ids, int num_ids, int nid) 1869a433f1f5SLijo Lazar { 1870a433f1f5SLijo Lazar int i; 1871a433f1f5SLijo Lazar 1872a433f1f5SLijo Lazar /* Check if node with id 'nid' is present in 'node_ids' array */ 1873a433f1f5SLijo Lazar for (i = 0; i < num_ids; ++i) 1874a433f1f5SLijo Lazar if (node_ids[i] == nid) 1875a433f1f5SLijo Lazar return true; 1876a433f1f5SLijo Lazar 1877a433f1f5SLijo Lazar return false; 1878a433f1f5SLijo Lazar } 1879a433f1f5SLijo Lazar 1880a433f1f5SLijo Lazar static void 1881a433f1f5SLijo Lazar gmc_v9_0_init_acpi_mem_ranges(struct amdgpu_device *adev, 1882a433f1f5SLijo Lazar struct amdgpu_mem_partition_info *mem_ranges) 1883a433f1f5SLijo Lazar { 1884a433f1f5SLijo Lazar int num_ranges = 0, ret, mem_groups; 1885a433f1f5SLijo Lazar struct amdgpu_numa_info numa_info; 1886a433f1f5SLijo Lazar int node_ids[MAX_MEM_RANGES]; 1887a433f1f5SLijo Lazar int num_xcc, xcc_id; 1888a433f1f5SLijo Lazar uint32_t xcc_mask; 1889a433f1f5SLijo Lazar 1890a433f1f5SLijo Lazar num_xcc = NUM_XCC(adev->gfx.xcc_mask); 1891a433f1f5SLijo Lazar xcc_mask = (1U << num_xcc) - 1; 1892a433f1f5SLijo Lazar mem_groups = hweight32(adev->aid_mask); 1893a433f1f5SLijo Lazar 1894a433f1f5SLijo Lazar for_each_inst(xcc_id, xcc_mask) { 1895a433f1f5SLijo Lazar ret = amdgpu_acpi_get_mem_info(adev, xcc_id, &numa_info); 1896a433f1f5SLijo Lazar if (ret) 1897a433f1f5SLijo Lazar continue; 1898a433f1f5SLijo Lazar 1899a433f1f5SLijo Lazar if (numa_info.nid == NUMA_NO_NODE) { 1900a433f1f5SLijo Lazar mem_ranges[0].size = numa_info.size; 1901a433f1f5SLijo Lazar mem_ranges[0].numa.node = numa_info.nid; 1902a433f1f5SLijo Lazar num_ranges = 1; 1903a433f1f5SLijo Lazar break; 1904a433f1f5SLijo Lazar } 1905a433f1f5SLijo Lazar 1906a433f1f5SLijo Lazar if (gmc_v9_0_is_node_present(node_ids, num_ranges, 1907a433f1f5SLijo Lazar numa_info.nid)) 1908a433f1f5SLijo Lazar continue; 1909a433f1f5SLijo Lazar 1910a433f1f5SLijo Lazar node_ids[num_ranges] = numa_info.nid; 1911a433f1f5SLijo Lazar mem_ranges[num_ranges].numa.node = numa_info.nid; 1912a433f1f5SLijo Lazar mem_ranges[num_ranges].size = numa_info.size; 1913a433f1f5SLijo Lazar ++num_ranges; 1914a433f1f5SLijo Lazar } 1915a433f1f5SLijo Lazar 1916a433f1f5SLijo Lazar adev->gmc.num_mem_partitions = num_ranges; 1917a433f1f5SLijo Lazar 1918a433f1f5SLijo Lazar /* If there is only partition, don't use entire size */ 191945b3a914SAlex Deucher if (adev->gmc.num_mem_partitions == 1) { 192045b3a914SAlex Deucher mem_ranges[0].size = mem_ranges[0].size * (mem_groups - 1); 192145b3a914SAlex Deucher do_div(mem_ranges[0].size, mem_groups); 192245b3a914SAlex Deucher } 1923a433f1f5SLijo Lazar } 1924a433f1f5SLijo Lazar 1925a433f1f5SLijo Lazar static void 1926a433f1f5SLijo Lazar gmc_v9_0_init_sw_mem_ranges(struct amdgpu_device *adev, 1927a433f1f5SLijo Lazar struct amdgpu_mem_partition_info *mem_ranges) 1928a433f1f5SLijo Lazar { 1929a433f1f5SLijo Lazar enum amdgpu_memory_partition mode; 1930a433f1f5SLijo Lazar u32 start_addr = 0, size; 1931a433f1f5SLijo Lazar int i; 1932a433f1f5SLijo Lazar 1933a433f1f5SLijo Lazar mode = gmc_v9_0_query_memory_partition(adev); 1934a433f1f5SLijo Lazar 1935a433f1f5SLijo Lazar switch (mode) { 1936a433f1f5SLijo Lazar case UNKNOWN_MEMORY_PARTITION_MODE: 1937a433f1f5SLijo Lazar case AMDGPU_NPS1_PARTITION_MODE: 1938a433f1f5SLijo Lazar adev->gmc.num_mem_partitions = 1; 1939a433f1f5SLijo Lazar break; 1940a433f1f5SLijo Lazar case AMDGPU_NPS2_PARTITION_MODE: 1941a433f1f5SLijo Lazar adev->gmc.num_mem_partitions = 2; 1942a433f1f5SLijo Lazar break; 1943a433f1f5SLijo Lazar case AMDGPU_NPS4_PARTITION_MODE: 1944a433f1f5SLijo Lazar if (adev->flags & AMD_IS_APU) 1945a433f1f5SLijo Lazar adev->gmc.num_mem_partitions = 3; 1946a433f1f5SLijo Lazar else 1947a433f1f5SLijo Lazar adev->gmc.num_mem_partitions = 4; 1948a433f1f5SLijo Lazar break; 1949a433f1f5SLijo Lazar default: 1950a433f1f5SLijo Lazar adev->gmc.num_mem_partitions = 1; 1951a433f1f5SLijo Lazar break; 1952a433f1f5SLijo Lazar } 1953a433f1f5SLijo Lazar 195445b3a914SAlex Deucher size = adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT; 195545b3a914SAlex Deucher size /= adev->gmc.num_mem_partitions; 1956a433f1f5SLijo Lazar 1957a433f1f5SLijo Lazar for (i = 0; i < adev->gmc.num_mem_partitions; ++i) { 1958a433f1f5SLijo Lazar mem_ranges[i].range.fpfn = start_addr; 1959a433f1f5SLijo Lazar mem_ranges[i].size = ((u64)size << AMDGPU_GPU_PAGE_SHIFT); 1960a433f1f5SLijo Lazar mem_ranges[i].range.lpfn = start_addr + size - 1; 1961a433f1f5SLijo Lazar start_addr += size; 1962a433f1f5SLijo Lazar } 1963a433f1f5SLijo Lazar 1964a433f1f5SLijo Lazar /* Adjust the last one */ 1965a433f1f5SLijo Lazar mem_ranges[adev->gmc.num_mem_partitions - 1].range.lpfn = 1966a433f1f5SLijo Lazar (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT) - 1; 1967a433f1f5SLijo Lazar mem_ranges[adev->gmc.num_mem_partitions - 1].size = 1968a433f1f5SLijo Lazar adev->gmc.real_vram_size - 1969a433f1f5SLijo Lazar ((u64)mem_ranges[adev->gmc.num_mem_partitions - 1].range.fpfn 1970a433f1f5SLijo Lazar << AMDGPU_GPU_PAGE_SHIFT); 1971a433f1f5SLijo Lazar } 1972a433f1f5SLijo Lazar 1973a433f1f5SLijo Lazar static int gmc_v9_0_init_mem_ranges(struct amdgpu_device *adev) 1974a433f1f5SLijo Lazar { 1975a433f1f5SLijo Lazar bool valid; 1976a433f1f5SLijo Lazar 1977a433f1f5SLijo Lazar adev->gmc.mem_partitions = kzalloc( 1978a433f1f5SLijo Lazar MAX_MEM_RANGES * sizeof(struct amdgpu_mem_partition_info), 1979a433f1f5SLijo Lazar GFP_KERNEL); 1980a433f1f5SLijo Lazar 1981a433f1f5SLijo Lazar if (!adev->gmc.mem_partitions) 1982a433f1f5SLijo Lazar return -ENOMEM; 1983a433f1f5SLijo Lazar 1984a433f1f5SLijo Lazar /* TODO : Get the range from PSP/Discovery for dGPU */ 1985a433f1f5SLijo Lazar if (adev->gmc.is_app_apu) 1986a433f1f5SLijo Lazar gmc_v9_0_init_acpi_mem_ranges(adev, adev->gmc.mem_partitions); 1987a433f1f5SLijo Lazar else 1988a433f1f5SLijo Lazar gmc_v9_0_init_sw_mem_ranges(adev, adev->gmc.mem_partitions); 1989a433f1f5SLijo Lazar 199046f7b4deSGavin Wan if (amdgpu_sriov_vf(adev)) 199146f7b4deSGavin Wan valid = true; 199246f7b4deSGavin Wan else 1993a433f1f5SLijo Lazar valid = gmc_v9_0_validate_partition_info(adev); 1994a433f1f5SLijo Lazar if (!valid) { 1995a433f1f5SLijo Lazar /* TODO: handle invalid case */ 1996a433f1f5SLijo Lazar dev_WARN(adev->dev, 1997a433f1f5SLijo Lazar "Mem ranges not matching with hardware config"); 1998a433f1f5SLijo Lazar } 1999a433f1f5SLijo Lazar 2000a433f1f5SLijo Lazar return 0; 2001a433f1f5SLijo Lazar } 2002a433f1f5SLijo Lazar 2003e60f8db5SAlex Xie static int gmc_v9_0_sw_init(void *handle) 2004e60f8db5SAlex Xie { 200524bf9fd1SHarish Kasiviswanathan int r, vram_width = 0, vram_type = 0, vram_vendor = 0, dma_addr_bits; 2006e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 20077a1efad0SLijo Lazar unsigned long inst_mask = adev->aid_mask; 2008e60f8db5SAlex Xie 20098ffff9b4SOak Zeng adev->gfxhub.funcs->init(adev); 20109fb1506eSOak Zeng 20119fb1506eSOak Zeng adev->mmhub.funcs->init(adev); 2012e60f8db5SAlex Xie 2013770d13b1SChristian König spin_lock_init(&adev->gmc.invalidate_lock); 2014e60f8db5SAlex Xie 20159535a86aSShiwu Zhang if (!(adev->bios) || adev->gmc.is_app_apu) { 20169535a86aSShiwu Zhang if (adev->flags & AMD_IS_APU) { 20179535a86aSShiwu Zhang if (adev->gmc.is_app_apu) { 20189535a86aSShiwu Zhang adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM; 20199535a86aSShiwu Zhang adev->gmc.vram_width = 128 * 64; 20209535a86aSShiwu Zhang } else { 20219535a86aSShiwu Zhang adev->gmc.vram_type = AMDGPU_VRAM_TYPE_DDR4; 20229535a86aSShiwu Zhang adev->gmc.vram_width = 64 * 64; 20239535a86aSShiwu Zhang } 20249535a86aSShiwu Zhang } else { 20259535a86aSShiwu Zhang adev->gmc.vram_type = AMDGPU_VRAM_TYPE_HBM; 20269535a86aSShiwu Zhang adev->gmc.vram_width = 128 * 64; 20279535a86aSShiwu Zhang } 20289535a86aSShiwu Zhang } else { 2029ad02e08eSOri Messinger r = amdgpu_atomfirmware_get_vram_info(adev, 2030ad02e08eSOri Messinger &vram_width, &vram_type, &vram_vendor); 2031631cdbd2SAlex Deucher if (amdgpu_sriov_vf(adev)) 2032631cdbd2SAlex Deucher /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, 2033631cdbd2SAlex Deucher * and DF related registers is not readable, seems hardcord is the 2034631cdbd2SAlex Deucher * only way to set the correct vram_width 2035631cdbd2SAlex Deucher */ 2036631cdbd2SAlex Deucher adev->gmc.vram_width = 2048; 2037631cdbd2SAlex Deucher else if (amdgpu_emu_mode != 1) 2038631cdbd2SAlex Deucher adev->gmc.vram_width = vram_width; 2039631cdbd2SAlex Deucher 2040631cdbd2SAlex Deucher if (!adev->gmc.vram_width) { 2041631cdbd2SAlex Deucher int chansize, numchan; 2042631cdbd2SAlex Deucher 2043631cdbd2SAlex Deucher /* hbm memory channel size */ 2044631cdbd2SAlex Deucher if (adev->flags & AMD_IS_APU) 2045631cdbd2SAlex Deucher chansize = 64; 2046631cdbd2SAlex Deucher else 2047631cdbd2SAlex Deucher chansize = 128; 2048cace4bffSHawking Zhang if (adev->df.funcs && 2049cace4bffSHawking Zhang adev->df.funcs->get_hbm_channel_number) { 2050bdf84a80SJoseph Greathouse numchan = adev->df.funcs->get_hbm_channel_number(adev); 2051631cdbd2SAlex Deucher adev->gmc.vram_width = numchan * chansize; 2052631cdbd2SAlex Deucher } 2053cace4bffSHawking Zhang } 2054631cdbd2SAlex Deucher 2055631cdbd2SAlex Deucher adev->gmc.vram_type = vram_type; 2056ad02e08eSOri Messinger adev->gmc.vram_vendor = vram_vendor; 20579535a86aSShiwu Zhang } 2058630e959fSAlex Deucher switch (adev->ip_versions[GC_HWIP][0]) { 2059630e959fSAlex Deucher case IP_VERSION(9, 1, 0): 2060630e959fSAlex Deucher case IP_VERSION(9, 2, 2): 2061d9426c3dSLe Ma set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask); 2062d9426c3dSLe Ma set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask); 20631daa2bfaSLe Ma 20646a42fd6fSChristian König if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 2065f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 20666a42fd6fSChristian König } else { 20676a42fd6fSChristian König /* vm_size is 128TB + 512GB for legacy 3-level page support */ 20686a42fd6fSChristian König amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 2069770d13b1SChristian König adev->gmc.translate_further = 20706a42fd6fSChristian König adev->vm_manager.num_level > 1; 20716a42fd6fSChristian König } 2072e60f8db5SAlex Xie break; 2073630e959fSAlex Deucher case IP_VERSION(9, 0, 1): 2074630e959fSAlex Deucher case IP_VERSION(9, 2, 1): 2075630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 2076630e959fSAlex Deucher case IP_VERSION(9, 3, 0): 2077630e959fSAlex Deucher case IP_VERSION(9, 4, 2): 2078d9426c3dSLe Ma set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask); 2079d9426c3dSLe Ma set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask); 20808787ee01SHuang Rui 2081e60f8db5SAlex Xie /* 2082e60f8db5SAlex Xie * To fulfill 4-level page support, 2083e60f8db5SAlex Xie * vm size is 256TB (48bit), maximum size of Vega10, 2084e60f8db5SAlex Xie * block size 512 (9bit) 2085e60f8db5SAlex Xie */ 2086cdba61daSwentalou /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 2087cdba61daSwentalou if (amdgpu_sriov_vf(adev)) 2088cdba61daSwentalou amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 2089cdba61daSwentalou else 2090f3368128SChristian König amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 2091352e683bSJoseph Greathouse if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 2092352e683bSJoseph Greathouse adev->gmc.translate_further = adev->vm_manager.num_level > 1; 2093e60f8db5SAlex Xie break; 2094630e959fSAlex Deucher case IP_VERSION(9, 4, 1): 2095d9426c3dSLe Ma set_bit(AMDGPU_GFXHUB(0), adev->vmhubs_mask); 2096d9426c3dSLe Ma set_bit(AMDGPU_MMHUB0(0), adev->vmhubs_mask); 2097d9426c3dSLe Ma set_bit(AMDGPU_MMHUB1(0), adev->vmhubs_mask); 2098c8a6e2a3SLe Ma 20993de2ff5dSLe Ma /* Keep the vm size same with Vega20 */ 21003de2ff5dSLe Ma amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 2101352e683bSJoseph Greathouse adev->gmc.translate_further = adev->vm_manager.num_level > 1; 21023de2ff5dSLe Ma break; 2103ce8a12a5SLe Ma case IP_VERSION(9, 4, 3): 21048078f1c6SLijo Lazar bitmap_set(adev->vmhubs_mask, AMDGPU_GFXHUB(0), 21058078f1c6SLijo Lazar NUM_XCC(adev->gfx.xcc_mask)); 21067a1efad0SLijo Lazar 21077a1efad0SLijo Lazar inst_mask <<= AMDGPU_MMHUB0(0); 21087a1efad0SLijo Lazar bitmap_or(adev->vmhubs_mask, adev->vmhubs_mask, &inst_mask, 32); 2109ce8a12a5SLe Ma 2110ce8a12a5SLe Ma amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 2111d728eda3SPhilip Yang adev->gmc.translate_further = adev->vm_manager.num_level > 1; 2112ce8a12a5SLe Ma break; 2113e60f8db5SAlex Xie default: 2114e60f8db5SAlex Xie break; 2115e60f8db5SAlex Xie } 2116e60f8db5SAlex Xie 2117e60f8db5SAlex Xie /* This interrupt is VMC page fault.*/ 211844a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 2119770d13b1SChristian König &adev->gmc.vm_fault); 212030da7bb1SChristian König if (r) 212130da7bb1SChristian König return r; 212230da7bb1SChristian König 2123630e959fSAlex Deucher if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1)) { 21247d19b15fSLe Ma r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, 21257d19b15fSLe Ma &adev->gmc.vm_fault); 21267d19b15fSLe Ma if (r) 21277d19b15fSLe Ma return r; 21287d19b15fSLe Ma } 21297d19b15fSLe Ma 213044a99b65SAndrey Grodzovsky r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 2131770d13b1SChristian König &adev->gmc.vm_fault); 2132e60f8db5SAlex Xie 2133e60f8db5SAlex Xie if (r) 2134e60f8db5SAlex Xie return r; 2135e60f8db5SAlex Xie 213668d705ddSHawking Zhang if (!amdgpu_sriov_vf(adev) && 213768d705ddSHawking Zhang !adev->gmc.xgmi.connected_to_cpu) { 2138791c4769Sxinhui pan /* interrupt sent to DF. */ 2139791c4769Sxinhui pan r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 2140791c4769Sxinhui pan &adev->gmc.ecc_irq); 2141791c4769Sxinhui pan if (r) 2142791c4769Sxinhui pan return r; 21432ee9403eSZhigang Luo } 2144791c4769Sxinhui pan 2145e60f8db5SAlex Xie /* Set the internal MC address mask 2146e60f8db5SAlex Xie * This is the max address of the GPU's 2147e60f8db5SAlex Xie * internal address space. 2148e60f8db5SAlex Xie */ 2149770d13b1SChristian König adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 2150e60f8db5SAlex Xie 215112c4d7edSLijo Lazar dma_addr_bits = adev->ip_versions[GC_HWIP][0] >= IP_VERSION(9, 4, 2) ? 48:44; 215224bf9fd1SHarish Kasiviswanathan r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(dma_addr_bits)); 2153e60f8db5SAlex Xie if (r) { 2154e60f8db5SAlex Xie printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 2155244511f3SChristoph Hellwig return r; 2156e60f8db5SAlex Xie } 215724bf9fd1SHarish Kasiviswanathan adev->need_swiotlb = drm_need_swiotlb(dma_addr_bits); 2158e60f8db5SAlex Xie 2159e60f8db5SAlex Xie r = gmc_v9_0_mc_init(adev); 2160e60f8db5SAlex Xie if (r) 2161e60f8db5SAlex Xie return r; 2162e60f8db5SAlex Xie 21637b885f0eSAlex Deucher amdgpu_gmc_get_vbios_allocations(adev); 2164ebdef28eSAlex Deucher 2165a433f1f5SLijo Lazar if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) { 2166a433f1f5SLijo Lazar r = gmc_v9_0_init_mem_ranges(adev); 2167a433f1f5SLijo Lazar if (r) 2168a433f1f5SLijo Lazar return r; 2169a433f1f5SLijo Lazar } 2170a433f1f5SLijo Lazar 2171e60f8db5SAlex Xie /* Memory manager */ 2172e60f8db5SAlex Xie r = amdgpu_bo_init(adev); 2173e60f8db5SAlex Xie if (r) 2174e60f8db5SAlex Xie return r; 2175e60f8db5SAlex Xie 2176e60f8db5SAlex Xie r = gmc_v9_0_gart_init(adev); 2177e60f8db5SAlex Xie if (r) 2178e60f8db5SAlex Xie return r; 2179e60f8db5SAlex Xie 218005ec3edaSChristian König /* 218105ec3edaSChristian König * number of VMs 218205ec3edaSChristian König * VMID 0 is reserved for System 218381659b20SFelix Kuehling * amdgpu graphics/compute will use VMIDs 1..n-1 218481659b20SFelix Kuehling * amdkfd will use VMIDs n..15 218581659b20SFelix Kuehling * 218681659b20SFelix Kuehling * The first KFD VMID is 8 for GPUs with graphics, 3 for 218781659b20SFelix Kuehling * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs 218881659b20SFelix Kuehling * for video processing. 218905ec3edaSChristian König */ 219081659b20SFelix Kuehling adev->vm_manager.first_kfd_vmid = 2191630e959fSAlex Deucher (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 1) || 2192ab1a157eSHawking Zhang adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) || 2193ab1a157eSHawking Zhang adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) ? 3 : 8; 219405ec3edaSChristian König 219505ec3edaSChristian König amdgpu_vm_manager_init(adev); 219605ec3edaSChristian König 2197b0a2db9bSAlex Deucher gmc_v9_0_save_registers(adev); 2198b0a2db9bSAlex Deucher 2199a6dcf9a7SHawking Zhang r = amdgpu_gmc_ras_sw_init(adev); 2200a6dcf9a7SHawking Zhang if (r) 2201a6dcf9a7SHawking Zhang return r; 2202a6dcf9a7SHawking Zhang 2203b6f90baaSLijo Lazar if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) 2204b6f90baaSLijo Lazar amdgpu_gmc_sysfs_init(adev); 2205b6f90baaSLijo Lazar 220605ec3edaSChristian König return 0; 2207e60f8db5SAlex Xie } 2208e60f8db5SAlex Xie 2209e60f8db5SAlex Xie static int gmc_v9_0_sw_fini(void *handle) 2210e60f8db5SAlex Xie { 2211e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2212e60f8db5SAlex Xie 2213b6f90baaSLijo Lazar if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 3)) 2214b6f90baaSLijo Lazar amdgpu_gmc_sysfs_fini(adev); 2215a433f1f5SLijo Lazar adev->gmc.num_mem_partitions = 0; 2216a433f1f5SLijo Lazar kfree(adev->gmc.mem_partitions); 2217b6f90baaSLijo Lazar 22182adf1344STao Zhou amdgpu_gmc_ras_fini(adev); 2219f59548c8SMonk Liu amdgpu_gem_force_release(adev); 2220e60f8db5SAlex Xie amdgpu_vm_manager_fini(adev); 2221c9a502e9SFelix Kuehling if (!adev->gmc.real_vram_size) { 2222c9a502e9SFelix Kuehling dev_info(adev->dev, "Put GART in system memory for APU free\n"); 2223c9a502e9SFelix Kuehling amdgpu_gart_table_ram_free(adev); 2224c9a502e9SFelix Kuehling } else { 2225a3d9103eSAndrey Grodzovsky amdgpu_gart_table_vram_free(adev); 2226c9a502e9SFelix Kuehling } 22272d505453SGuchun Chen amdgpu_bo_free_kernel(&adev->gmc.pdb0_bo, NULL, &adev->gmc.ptr_pdb0); 2228e60f8db5SAlex Xie amdgpu_bo_fini(adev); 2229e60f8db5SAlex Xie 2230e60f8db5SAlex Xie return 0; 2231e60f8db5SAlex Xie } 2232e60f8db5SAlex Xie 2233e60f8db5SAlex Xie static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 2234e60f8db5SAlex Xie { 2235946a4d5bSShaoyun Liu 2236630e959fSAlex Deucher switch (adev->ip_versions[MMHUB_HWIP][0]) { 2237630e959fSAlex Deucher case IP_VERSION(9, 0, 0): 22384cd4c5c0SMonk Liu if (amdgpu_sriov_vf(adev)) 223998cad2deSTrigger Huang break; 2240df561f66SGustavo A. R. Silva fallthrough; 2241630e959fSAlex Deucher case IP_VERSION(9, 4, 0): 2242946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 22435c583018SEvan Quan golden_settings_mmhub_1_0_0, 2244c47b41a7SChristian König ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 2245946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 22465c583018SEvan Quan golden_settings_athub_1_0_0, 2247c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 2248e60f8db5SAlex Xie break; 2249630e959fSAlex Deucher case IP_VERSION(9, 1, 0): 2250630e959fSAlex Deucher case IP_VERSION(9, 2, 0): 22518787ee01SHuang Rui /* TODO for renoir */ 2252946a4d5bSShaoyun Liu soc15_program_register_sequence(adev, 22535c583018SEvan Quan golden_settings_athub_1_0_0, 2254c47b41a7SChristian König ARRAY_SIZE(golden_settings_athub_1_0_0)); 2255e4f3abaaSChunming Zhou break; 2256e60f8db5SAlex Xie default: 2257e60f8db5SAlex Xie break; 2258e60f8db5SAlex Xie } 2259e60f8db5SAlex Xie } 2260e60f8db5SAlex Xie 2261e60f8db5SAlex Xie /** 2262c2ecd79bSShirish S * gmc_v9_0_restore_registers - restores regs 2263c2ecd79bSShirish S * 2264c2ecd79bSShirish S * @adev: amdgpu_device pointer 2265c2ecd79bSShirish S * 2266c2ecd79bSShirish S * This restores register values, saved at suspend. 2267c2ecd79bSShirish S */ 2268b0a2db9bSAlex Deucher void gmc_v9_0_restore_registers(struct amdgpu_device *adev) 2269c2ecd79bSShirish S { 2270630e959fSAlex Deucher if ((adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 0)) || 2271630e959fSAlex Deucher (adev->ip_versions[DCE_HWIP][0] == IP_VERSION(1, 0, 1))) { 2272f8646661SAlex Deucher WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); 22730eaa8012SShirish S WARN_ON(adev->gmc.sdpif_register != 22740eaa8012SShirish S RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0)); 22750eaa8012SShirish S } 2276c2ecd79bSShirish S } 2277c2ecd79bSShirish S 2278c2ecd79bSShirish S /** 2279e60f8db5SAlex Xie * gmc_v9_0_gart_enable - gart enable 2280e60f8db5SAlex Xie * 2281e60f8db5SAlex Xie * @adev: amdgpu_device pointer 2282e60f8db5SAlex Xie */ 2283e60f8db5SAlex Xie static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 2284e60f8db5SAlex Xie { 2285cb1545f7SOak Zeng int r; 2286e60f8db5SAlex Xie 2287522510a6SOak Zeng if (adev->gmc.xgmi.connected_to_cpu) 2288522510a6SOak Zeng amdgpu_gmc_init_pdb0(adev); 2289522510a6SOak Zeng 22901123b989SChristian König if (adev->gart.bo == NULL) { 2291e60f8db5SAlex Xie dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 2292e60f8db5SAlex Xie return -EINVAL; 2293e60f8db5SAlex Xie } 2294522510a6SOak Zeng 22951b08dfb8SChristian König amdgpu_gtt_mgr_recover(&adev->mman.gtt_mgr); 2296b93df61dSAlex Deucher 2297b93df61dSAlex Deucher if (!adev->in_s0ix) { 22988ffff9b4SOak Zeng r = adev->gfxhub.funcs->gart_enable(adev); 2299e60f8db5SAlex Xie if (r) 2300e60f8db5SAlex Xie return r; 2301b93df61dSAlex Deucher } 2302e60f8db5SAlex Xie 23039fb1506eSOak Zeng r = adev->mmhub.funcs->gart_enable(adev); 2304e60f8db5SAlex Xie if (r) 2305e60f8db5SAlex Xie return r; 2306e60f8db5SAlex Xie 2307522510a6SOak Zeng DRM_INFO("PCIE GART of %uM enabled.\n", 2308522510a6SOak Zeng (unsigned)(adev->gmc.gart_size >> 20)); 2309522510a6SOak Zeng if (adev->gmc.pdb0_bo) 2310522510a6SOak Zeng DRM_INFO("PDB0 located at 0x%016llX\n", 2311522510a6SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gmc.pdb0_bo)); 2312522510a6SOak Zeng DRM_INFO("PTB located at 0x%016llX\n", 2313cb1545f7SOak Zeng (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 2314522510a6SOak Zeng 2315cb1545f7SOak Zeng return 0; 2316cb1545f7SOak Zeng } 2317cb1545f7SOak Zeng 2318cb1545f7SOak Zeng static int gmc_v9_0_hw_init(void *handle) 2319cb1545f7SOak Zeng { 2320cb1545f7SOak Zeng struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2321cb1545f7SOak Zeng bool value; 2322479e3b02SXiaojian Du int i, r; 2323cb1545f7SOak Zeng 2324cb1545f7SOak Zeng /* The sequence of these two function calls matters.*/ 2325cb1545f7SOak Zeng gmc_v9_0_init_golden_registers(adev); 2326cb1545f7SOak Zeng 2327cb1545f7SOak Zeng if (adev->mode_info.num_crtc) { 2328cb1545f7SOak Zeng /* Lockout access through VGA aperture*/ 2329cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 2330cb1545f7SOak Zeng /* disable VGA render */ 2331cb1545f7SOak Zeng WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 2332cb1545f7SOak Zeng } 2333cb1545f7SOak Zeng 23349fb1506eSOak Zeng if (adev->mmhub.funcs->update_power_gating) 23359fb1506eSOak Zeng adev->mmhub.funcs->update_power_gating(adev, true); 23369fb1506eSOak Zeng 2337455d40c9SLikun Gao adev->hdp.funcs->init_registers(adev); 2338fe2b5323STiecheng Zhou 23391d4e0a8cSMonk Liu /* After HDP is initialized, flush HDP.*/ 2340455d40c9SLikun Gao adev->hdp.funcs->flush_hdp(adev, NULL); 23411d4e0a8cSMonk Liu 2342e60f8db5SAlex Xie if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 2343e60f8db5SAlex Xie value = false; 2344e60f8db5SAlex Xie else 2345e60f8db5SAlex Xie value = true; 2346e60f8db5SAlex Xie 234720bf2f6fSZhigang Luo if (!amdgpu_sriov_vf(adev)) { 2348b93df61dSAlex Deucher if (!adev->in_s0ix) 23498ffff9b4SOak Zeng adev->gfxhub.funcs->set_fault_enable_default(adev, value); 23509fb1506eSOak Zeng adev->mmhub.funcs->set_fault_enable_default(adev, value); 235120bf2f6fSZhigang Luo } 2352d9426c3dSLe Ma for_each_set_bit(i, adev->vmhubs_mask, AMDGPU_MAX_VMHUBS) { 2353f4caf584SHawking Zhang if (adev->in_s0ix && (i == AMDGPU_GFXHUB(0))) 2354b93df61dSAlex Deucher continue; 23553ff98548SOak Zeng gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); 2356b93df61dSAlex Deucher } 2357e60f8db5SAlex Xie 2358e7da754bSMonk Liu if (adev->umc.funcs && adev->umc.funcs->init_registers) 2359e7da754bSMonk Liu adev->umc.funcs->init_registers(adev); 2360e7da754bSMonk Liu 2361479e3b02SXiaojian Du r = gmc_v9_0_gart_enable(adev); 2362479e3b02SXiaojian Du if (r) 2363479e3b02SXiaojian Du return r; 2364479e3b02SXiaojian Du 2365479e3b02SXiaojian Du if (amdgpu_emu_mode == 1) 2366479e3b02SXiaojian Du return amdgpu_gmc_vram_checking(adev); 2367479e3b02SXiaojian Du else 2368479e3b02SXiaojian Du return r; 2369e60f8db5SAlex Xie } 2370e60f8db5SAlex Xie 2371e60f8db5SAlex Xie /** 2372e60f8db5SAlex Xie * gmc_v9_0_gart_disable - gart disable 2373e60f8db5SAlex Xie * 2374e60f8db5SAlex Xie * @adev: amdgpu_device pointer 2375e60f8db5SAlex Xie * 2376e60f8db5SAlex Xie * This disables all VM page table. 2377e60f8db5SAlex Xie */ 2378e60f8db5SAlex Xie static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 2379e60f8db5SAlex Xie { 2380b93df61dSAlex Deucher if (!adev->in_s0ix) 23818ffff9b4SOak Zeng adev->gfxhub.funcs->gart_disable(adev); 23829fb1506eSOak Zeng adev->mmhub.funcs->gart_disable(adev); 2383e60f8db5SAlex Xie } 2384e60f8db5SAlex Xie 2385e60f8db5SAlex Xie static int gmc_v9_0_hw_fini(void *handle) 2386e60f8db5SAlex Xie { 2387e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2388e60f8db5SAlex Xie 238971cf9e72SLeslie Shi gmc_v9_0_gart_disable(adev); 239071cf9e72SLeslie Shi 23915dd696aeSTrigger Huang if (amdgpu_sriov_vf(adev)) { 23925dd696aeSTrigger Huang /* full access mode, so don't touch any GMC register */ 23935dd696aeSTrigger Huang DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 23945dd696aeSTrigger Huang return 0; 23955dd696aeSTrigger Huang } 23965dd696aeSTrigger Huang 239717252701SEvan Quan /* 239817252701SEvan Quan * Pair the operations did in gmc_v9_0_hw_init and thus maintain 239917252701SEvan Quan * a correct cached state for GMC. Otherwise, the "gate" again 240017252701SEvan Quan * operation on S3 resuming will fail due to wrong cached state. 240117252701SEvan Quan */ 240217252701SEvan Quan if (adev->mmhub.funcs->update_power_gating) 240317252701SEvan Quan adev->mmhub.funcs->update_power_gating(adev, false); 240417252701SEvan Quan 2405770d13b1SChristian König amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 2406e60f8db5SAlex Xie 2407e60f8db5SAlex Xie return 0; 2408e60f8db5SAlex Xie } 2409e60f8db5SAlex Xie 2410e60f8db5SAlex Xie static int gmc_v9_0_suspend(void *handle) 2411e60f8db5SAlex Xie { 2412e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2413e60f8db5SAlex Xie 2414c24a3c05SLiu Shixin return gmc_v9_0_hw_fini(adev); 2415e60f8db5SAlex Xie } 2416e60f8db5SAlex Xie 2417e60f8db5SAlex Xie static int gmc_v9_0_resume(void *handle) 2418e60f8db5SAlex Xie { 2419e60f8db5SAlex Xie int r; 2420e60f8db5SAlex Xie struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2421e60f8db5SAlex Xie 2422e60f8db5SAlex Xie r = gmc_v9_0_hw_init(adev); 2423e60f8db5SAlex Xie if (r) 2424e60f8db5SAlex Xie return r; 2425e60f8db5SAlex Xie 2426620f774fSChristian König amdgpu_vmid_reset_all(adev); 2427e60f8db5SAlex Xie 242832601d48SChristian König return 0; 2429e60f8db5SAlex Xie } 2430e60f8db5SAlex Xie 2431e60f8db5SAlex Xie static bool gmc_v9_0_is_idle(void *handle) 2432e60f8db5SAlex Xie { 2433e60f8db5SAlex Xie /* MC is always ready in GMC v9.*/ 2434e60f8db5SAlex Xie return true; 2435e60f8db5SAlex Xie } 2436e60f8db5SAlex Xie 2437e60f8db5SAlex Xie static int gmc_v9_0_wait_for_idle(void *handle) 2438e60f8db5SAlex Xie { 2439e60f8db5SAlex Xie /* There is no need to wait for MC idle in GMC v9.*/ 2440e60f8db5SAlex Xie return 0; 2441e60f8db5SAlex Xie } 2442e60f8db5SAlex Xie 2443e60f8db5SAlex Xie static int gmc_v9_0_soft_reset(void *handle) 2444e60f8db5SAlex Xie { 2445e60f8db5SAlex Xie /* XXX for emulation.*/ 2446e60f8db5SAlex Xie return 0; 2447e60f8db5SAlex Xie } 2448e60f8db5SAlex Xie 2449e60f8db5SAlex Xie static int gmc_v9_0_set_clockgating_state(void *handle, 2450e60f8db5SAlex Xie enum amd_clockgating_state state) 2451e60f8db5SAlex Xie { 2452d5583d4fSHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 2453d5583d4fSHuang Rui 24549fb1506eSOak Zeng adev->mmhub.funcs->set_clockgating(adev, state); 2455bee7b51aSLe Ma 2456bee7b51aSLe Ma athub_v1_0_set_clockgating(adev, state); 2457bee7b51aSLe Ma 2458bee7b51aSLe Ma return 0; 2459e60f8db5SAlex Xie } 2460e60f8db5SAlex Xie 246125faeddcSEvan Quan static void gmc_v9_0_get_clockgating_state(void *handle, u64 *flags) 246213052be5SHuang Rui { 246313052be5SHuang Rui struct amdgpu_device *adev = (struct amdgpu_device *)handle; 246413052be5SHuang Rui 24659fb1506eSOak Zeng adev->mmhub.funcs->get_clockgating(adev, flags); 2466bee7b51aSLe Ma 2467bee7b51aSLe Ma athub_v1_0_get_clockgating(adev, flags); 246813052be5SHuang Rui } 246913052be5SHuang Rui 2470e60f8db5SAlex Xie static int gmc_v9_0_set_powergating_state(void *handle, 2471e60f8db5SAlex Xie enum amd_powergating_state state) 2472e60f8db5SAlex Xie { 2473e60f8db5SAlex Xie return 0; 2474e60f8db5SAlex Xie } 2475e60f8db5SAlex Xie 2476e60f8db5SAlex Xie const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 2477e60f8db5SAlex Xie .name = "gmc_v9_0", 2478e60f8db5SAlex Xie .early_init = gmc_v9_0_early_init, 2479e60f8db5SAlex Xie .late_init = gmc_v9_0_late_init, 2480e60f8db5SAlex Xie .sw_init = gmc_v9_0_sw_init, 2481e60f8db5SAlex Xie .sw_fini = gmc_v9_0_sw_fini, 2482e60f8db5SAlex Xie .hw_init = gmc_v9_0_hw_init, 2483e60f8db5SAlex Xie .hw_fini = gmc_v9_0_hw_fini, 2484e60f8db5SAlex Xie .suspend = gmc_v9_0_suspend, 2485e60f8db5SAlex Xie .resume = gmc_v9_0_resume, 2486e60f8db5SAlex Xie .is_idle = gmc_v9_0_is_idle, 2487e60f8db5SAlex Xie .wait_for_idle = gmc_v9_0_wait_for_idle, 2488e60f8db5SAlex Xie .soft_reset = gmc_v9_0_soft_reset, 2489e60f8db5SAlex Xie .set_clockgating_state = gmc_v9_0_set_clockgating_state, 2490e60f8db5SAlex Xie .set_powergating_state = gmc_v9_0_set_powergating_state, 249113052be5SHuang Rui .get_clockgating_state = gmc_v9_0_get_clockgating_state, 2492e60f8db5SAlex Xie }; 2493e60f8db5SAlex Xie 2494e60f8db5SAlex Xie const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 2495e60f8db5SAlex Xie { 2496e60f8db5SAlex Xie .type = AMD_IP_BLOCK_TYPE_GMC, 2497e60f8db5SAlex Xie .major = 9, 2498e60f8db5SAlex Xie .minor = 0, 2499e60f8db5SAlex Xie .rev = 0, 2500e60f8db5SAlex Xie .funcs = &gmc_v9_0_ip_funcs, 2501e60f8db5SAlex Xie }; 2502