1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <linux/pci.h> 26 27 #include <drm/drm_cache.h> 28 29 #include "amdgpu.h" 30 #include "gmc_v9_0.h" 31 #include "amdgpu_atomfirmware.h" 32 #include "amdgpu_gem.h" 33 34 #include "hdp/hdp_4_0_offset.h" 35 #include "hdp/hdp_4_0_sh_mask.h" 36 #include "gc/gc_9_0_sh_mask.h" 37 #include "dce/dce_12_0_offset.h" 38 #include "dce/dce_12_0_sh_mask.h" 39 #include "vega10_enum.h" 40 #include "mmhub/mmhub_1_0_offset.h" 41 #include "athub/athub_1_0_sh_mask.h" 42 #include "athub/athub_1_0_offset.h" 43 #include "oss/osssys_4_0_offset.h" 44 45 #include "soc15.h" 46 #include "soc15d.h" 47 #include "soc15_common.h" 48 #include "umc/umc_6_0_sh_mask.h" 49 50 #include "gfxhub_v1_0.h" 51 #include "mmhub_v1_0.h" 52 #include "athub_v1_0.h" 53 #include "gfxhub_v1_1.h" 54 #include "mmhub_v9_4.h" 55 #include "umc_v6_1.h" 56 #include "umc_v6_0.h" 57 58 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 59 60 #include "amdgpu_ras.h" 61 #include "amdgpu_xgmi.h" 62 63 /* add these here since we already include dce12 headers and these are for DCN */ 64 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 65 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 66 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 67 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 68 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 69 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 70 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d 71 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 72 73 74 static const char *gfxhub_client_ids[] = { 75 "CB", 76 "DB", 77 "IA", 78 "WD", 79 "CPF", 80 "CPC", 81 "CPG", 82 "RLC", 83 "TCP", 84 "SQC (inst)", 85 "SQC (data)", 86 "SQG", 87 "PA", 88 }; 89 90 static const char *mmhub_client_ids_raven[][2] = { 91 [0][0] = "MP1", 92 [1][0] = "MP0", 93 [2][0] = "VCN", 94 [3][0] = "VCNU", 95 [4][0] = "HDP", 96 [5][0] = "DCE", 97 [13][0] = "UTCL2", 98 [19][0] = "TLS", 99 [26][0] = "OSS", 100 [27][0] = "SDMA0", 101 [0][1] = "MP1", 102 [1][1] = "MP0", 103 [2][1] = "VCN", 104 [3][1] = "VCNU", 105 [4][1] = "HDP", 106 [5][1] = "XDP", 107 [6][1] = "DBGU0", 108 [7][1] = "DCE", 109 [8][1] = "DCEDWB0", 110 [9][1] = "DCEDWB1", 111 [26][1] = "OSS", 112 [27][1] = "SDMA0", 113 }; 114 115 static const char *mmhub_client_ids_renoir[][2] = { 116 [0][0] = "MP1", 117 [1][0] = "MP0", 118 [2][0] = "HDP", 119 [4][0] = "DCEDMC", 120 [5][0] = "DCEVGA", 121 [13][0] = "UTCL2", 122 [19][0] = "TLS", 123 [26][0] = "OSS", 124 [27][0] = "SDMA0", 125 [28][0] = "VCN", 126 [29][0] = "VCNU", 127 [30][0] = "JPEG", 128 [0][1] = "MP1", 129 [1][1] = "MP0", 130 [2][1] = "HDP", 131 [3][1] = "XDP", 132 [6][1] = "DBGU0", 133 [7][1] = "DCEDMC", 134 [8][1] = "DCEVGA", 135 [9][1] = "DCEDWB", 136 [26][1] = "OSS", 137 [27][1] = "SDMA0", 138 [28][1] = "VCN", 139 [29][1] = "VCNU", 140 [30][1] = "JPEG", 141 }; 142 143 static const char *mmhub_client_ids_vega10[][2] = { 144 [0][0] = "MP0", 145 [1][0] = "UVD", 146 [2][0] = "UVDU", 147 [3][0] = "HDP", 148 [13][0] = "UTCL2", 149 [14][0] = "OSS", 150 [15][0] = "SDMA1", 151 [32+0][0] = "VCE0", 152 [32+1][0] = "VCE0U", 153 [32+2][0] = "XDMA", 154 [32+3][0] = "DCE", 155 [32+4][0] = "MP1", 156 [32+14][0] = "SDMA0", 157 [0][1] = "MP0", 158 [1][1] = "UVD", 159 [2][1] = "UVDU", 160 [3][1] = "DBGU0", 161 [4][1] = "HDP", 162 [5][1] = "XDP", 163 [14][1] = "OSS", 164 [15][1] = "SDMA0", 165 [32+0][1] = "VCE0", 166 [32+1][1] = "VCE0U", 167 [32+2][1] = "XDMA", 168 [32+3][1] = "DCE", 169 [32+4][1] = "DCEDWB", 170 [32+5][1] = "MP1", 171 [32+6][1] = "DBGU1", 172 [32+14][1] = "SDMA1", 173 }; 174 175 static const char *mmhub_client_ids_vega12[][2] = { 176 [0][0] = "MP0", 177 [1][0] = "VCE0", 178 [2][0] = "VCE0U", 179 [3][0] = "HDP", 180 [13][0] = "UTCL2", 181 [14][0] = "OSS", 182 [15][0] = "SDMA1", 183 [32+0][0] = "DCE", 184 [32+1][0] = "XDMA", 185 [32+2][0] = "UVD", 186 [32+3][0] = "UVDU", 187 [32+4][0] = "MP1", 188 [32+15][0] = "SDMA0", 189 [0][1] = "MP0", 190 [1][1] = "VCE0", 191 [2][1] = "VCE0U", 192 [3][1] = "DBGU0", 193 [4][1] = "HDP", 194 [5][1] = "XDP", 195 [14][1] = "OSS", 196 [15][1] = "SDMA0", 197 [32+0][1] = "DCE", 198 [32+1][1] = "DCEDWB", 199 [32+2][1] = "XDMA", 200 [32+3][1] = "UVD", 201 [32+4][1] = "UVDU", 202 [32+5][1] = "MP1", 203 [32+6][1] = "DBGU1", 204 [32+15][1] = "SDMA1", 205 }; 206 207 static const char *mmhub_client_ids_vega20[][2] = { 208 [0][0] = "XDMA", 209 [1][0] = "DCE", 210 [2][0] = "VCE0", 211 [3][0] = "VCE0U", 212 [4][0] = "UVD", 213 [5][0] = "UVD1U", 214 [13][0] = "OSS", 215 [14][0] = "HDP", 216 [15][0] = "SDMA0", 217 [32+0][0] = "UVD", 218 [32+1][0] = "UVDU", 219 [32+2][0] = "MP1", 220 [32+3][0] = "MP0", 221 [32+12][0] = "UTCL2", 222 [32+14][0] = "SDMA1", 223 [0][1] = "XDMA", 224 [1][1] = "DCE", 225 [2][1] = "DCEDWB", 226 [3][1] = "VCE0", 227 [4][1] = "VCE0U", 228 [5][1] = "UVD1", 229 [6][1] = "UVD1U", 230 [7][1] = "DBGU0", 231 [8][1] = "XDP", 232 [13][1] = "OSS", 233 [14][1] = "HDP", 234 [15][1] = "SDMA0", 235 [32+0][1] = "UVD", 236 [32+1][1] = "UVDU", 237 [32+2][1] = "DBGU1", 238 [32+3][1] = "MP1", 239 [32+4][1] = "MP0", 240 [32+14][1] = "SDMA1", 241 }; 242 243 static const char *mmhub_client_ids_arcturus[][2] = { 244 [2][0] = "MP1", 245 [3][0] = "MP0", 246 [10][0] = "UTCL2", 247 [13][0] = "OSS", 248 [14][0] = "HDP", 249 [15][0] = "SDMA0", 250 [32+15][0] = "SDMA1", 251 [64+15][0] = "SDMA2", 252 [96+15][0] = "SDMA3", 253 [128+15][0] = "SDMA4", 254 [160+11][0] = "JPEG", 255 [160+12][0] = "VCN", 256 [160+13][0] = "VCNU", 257 [160+15][0] = "SDMA5", 258 [192+10][0] = "UTCL2", 259 [192+11][0] = "JPEG1", 260 [192+12][0] = "VCN1", 261 [192+13][0] = "VCN1U", 262 [192+15][0] = "SDMA6", 263 [224+15][0] = "SDMA7", 264 [0][1] = "DBGU1", 265 [1][1] = "XDP", 266 [2][1] = "MP1", 267 [3][1] = "MP0", 268 [13][1] = "OSS", 269 [14][1] = "HDP", 270 [15][1] = "SDMA0", 271 [32+15][1] = "SDMA1", 272 [32+15][1] = "SDMA1", 273 [64+15][1] = "SDMA2", 274 [96+15][1] = "SDMA3", 275 [128+15][1] = "SDMA4", 276 [160+11][1] = "JPEG", 277 [160+12][1] = "VCN", 278 [160+13][1] = "VCNU", 279 [160+15][1] = "SDMA5", 280 [192+11][1] = "JPEG1", 281 [192+12][1] = "VCN1", 282 [192+13][1] = "VCN1U", 283 [192+15][1] = "SDMA6", 284 [224+15][1] = "SDMA7", 285 }; 286 287 static const u32 golden_settings_vega10_hdp[] = 288 { 289 0xf64, 0x0fffffff, 0x00000000, 290 0xf65, 0x0fffffff, 0x00000000, 291 0xf66, 0x0fffffff, 0x00000000, 292 0xf67, 0x0fffffff, 0x00000000, 293 0xf68, 0x0fffffff, 0x00000000, 294 0xf6a, 0x0fffffff, 0x00000000, 295 0xf6b, 0x0fffffff, 0x00000000, 296 0xf6c, 0x0fffffff, 0x00000000, 297 0xf6d, 0x0fffffff, 0x00000000, 298 0xf6e, 0x0fffffff, 0x00000000, 299 }; 300 301 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 302 { 303 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 304 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 305 }; 306 307 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 308 { 309 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 310 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 311 }; 312 313 static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 314 (0x000143c0 + 0x00000000), 315 (0x000143c0 + 0x00000800), 316 (0x000143c0 + 0x00001000), 317 (0x000143c0 + 0x00001800), 318 (0x000543c0 + 0x00000000), 319 (0x000543c0 + 0x00000800), 320 (0x000543c0 + 0x00001000), 321 (0x000543c0 + 0x00001800), 322 (0x000943c0 + 0x00000000), 323 (0x000943c0 + 0x00000800), 324 (0x000943c0 + 0x00001000), 325 (0x000943c0 + 0x00001800), 326 (0x000d43c0 + 0x00000000), 327 (0x000d43c0 + 0x00000800), 328 (0x000d43c0 + 0x00001000), 329 (0x000d43c0 + 0x00001800), 330 (0x001143c0 + 0x00000000), 331 (0x001143c0 + 0x00000800), 332 (0x001143c0 + 0x00001000), 333 (0x001143c0 + 0x00001800), 334 (0x001543c0 + 0x00000000), 335 (0x001543c0 + 0x00000800), 336 (0x001543c0 + 0x00001000), 337 (0x001543c0 + 0x00001800), 338 (0x001943c0 + 0x00000000), 339 (0x001943c0 + 0x00000800), 340 (0x001943c0 + 0x00001000), 341 (0x001943c0 + 0x00001800), 342 (0x001d43c0 + 0x00000000), 343 (0x001d43c0 + 0x00000800), 344 (0x001d43c0 + 0x00001000), 345 (0x001d43c0 + 0x00001800), 346 }; 347 348 static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 349 (0x000143e0 + 0x00000000), 350 (0x000143e0 + 0x00000800), 351 (0x000143e0 + 0x00001000), 352 (0x000143e0 + 0x00001800), 353 (0x000543e0 + 0x00000000), 354 (0x000543e0 + 0x00000800), 355 (0x000543e0 + 0x00001000), 356 (0x000543e0 + 0x00001800), 357 (0x000943e0 + 0x00000000), 358 (0x000943e0 + 0x00000800), 359 (0x000943e0 + 0x00001000), 360 (0x000943e0 + 0x00001800), 361 (0x000d43e0 + 0x00000000), 362 (0x000d43e0 + 0x00000800), 363 (0x000d43e0 + 0x00001000), 364 (0x000d43e0 + 0x00001800), 365 (0x001143e0 + 0x00000000), 366 (0x001143e0 + 0x00000800), 367 (0x001143e0 + 0x00001000), 368 (0x001143e0 + 0x00001800), 369 (0x001543e0 + 0x00000000), 370 (0x001543e0 + 0x00000800), 371 (0x001543e0 + 0x00001000), 372 (0x001543e0 + 0x00001800), 373 (0x001943e0 + 0x00000000), 374 (0x001943e0 + 0x00000800), 375 (0x001943e0 + 0x00001000), 376 (0x001943e0 + 0x00001800), 377 (0x001d43e0 + 0x00000000), 378 (0x001d43e0 + 0x00000800), 379 (0x001d43e0 + 0x00001000), 380 (0x001d43e0 + 0x00001800), 381 }; 382 383 static const uint32_t ecc_umc_mcumc_status_addrs[] = { 384 (0x000143c2 + 0x00000000), 385 (0x000143c2 + 0x00000800), 386 (0x000143c2 + 0x00001000), 387 (0x000143c2 + 0x00001800), 388 (0x000543c2 + 0x00000000), 389 (0x000543c2 + 0x00000800), 390 (0x000543c2 + 0x00001000), 391 (0x000543c2 + 0x00001800), 392 (0x000943c2 + 0x00000000), 393 (0x000943c2 + 0x00000800), 394 (0x000943c2 + 0x00001000), 395 (0x000943c2 + 0x00001800), 396 (0x000d43c2 + 0x00000000), 397 (0x000d43c2 + 0x00000800), 398 (0x000d43c2 + 0x00001000), 399 (0x000d43c2 + 0x00001800), 400 (0x001143c2 + 0x00000000), 401 (0x001143c2 + 0x00000800), 402 (0x001143c2 + 0x00001000), 403 (0x001143c2 + 0x00001800), 404 (0x001543c2 + 0x00000000), 405 (0x001543c2 + 0x00000800), 406 (0x001543c2 + 0x00001000), 407 (0x001543c2 + 0x00001800), 408 (0x001943c2 + 0x00000000), 409 (0x001943c2 + 0x00000800), 410 (0x001943c2 + 0x00001000), 411 (0x001943c2 + 0x00001800), 412 (0x001d43c2 + 0x00000000), 413 (0x001d43c2 + 0x00000800), 414 (0x001d43c2 + 0x00001000), 415 (0x001d43c2 + 0x00001800), 416 }; 417 418 static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 419 struct amdgpu_irq_src *src, 420 unsigned type, 421 enum amdgpu_interrupt_state state) 422 { 423 u32 bits, i, tmp, reg; 424 425 /* Devices newer then VEGA10/12 shall have these programming 426 sequences performed by PSP BL */ 427 if (adev->asic_type >= CHIP_VEGA20) 428 return 0; 429 430 bits = 0x7f; 431 432 switch (state) { 433 case AMDGPU_IRQ_STATE_DISABLE: 434 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 435 reg = ecc_umc_mcumc_ctrl_addrs[i]; 436 tmp = RREG32(reg); 437 tmp &= ~bits; 438 WREG32(reg, tmp); 439 } 440 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 441 reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 442 tmp = RREG32(reg); 443 tmp &= ~bits; 444 WREG32(reg, tmp); 445 } 446 break; 447 case AMDGPU_IRQ_STATE_ENABLE: 448 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 449 reg = ecc_umc_mcumc_ctrl_addrs[i]; 450 tmp = RREG32(reg); 451 tmp |= bits; 452 WREG32(reg, tmp); 453 } 454 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 455 reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 456 tmp = RREG32(reg); 457 tmp |= bits; 458 WREG32(reg, tmp); 459 } 460 break; 461 default: 462 break; 463 } 464 465 return 0; 466 } 467 468 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 469 struct amdgpu_irq_src *src, 470 unsigned type, 471 enum amdgpu_interrupt_state state) 472 { 473 struct amdgpu_vmhub *hub; 474 u32 tmp, reg, bits, i, j; 475 476 bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 477 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 478 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 479 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 480 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 481 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 482 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 483 484 switch (state) { 485 case AMDGPU_IRQ_STATE_DISABLE: 486 for (j = 0; j < adev->num_vmhubs; j++) { 487 hub = &adev->vmhub[j]; 488 for (i = 0; i < 16; i++) { 489 reg = hub->vm_context0_cntl + i; 490 tmp = RREG32(reg); 491 tmp &= ~bits; 492 WREG32(reg, tmp); 493 } 494 } 495 break; 496 case AMDGPU_IRQ_STATE_ENABLE: 497 for (j = 0; j < adev->num_vmhubs; j++) { 498 hub = &adev->vmhub[j]; 499 for (i = 0; i < 16; i++) { 500 reg = hub->vm_context0_cntl + i; 501 tmp = RREG32(reg); 502 tmp |= bits; 503 WREG32(reg, tmp); 504 } 505 } 506 default: 507 break; 508 } 509 510 return 0; 511 } 512 513 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 514 struct amdgpu_irq_src *source, 515 struct amdgpu_iv_entry *entry) 516 { 517 struct amdgpu_vmhub *hub; 518 bool retry_fault = !!(entry->src_data[1] & 0x80); 519 uint32_t status = 0, cid = 0, rw = 0; 520 u64 addr; 521 char hub_name[10]; 522 const char *mmhub_cid; 523 524 addr = (u64)entry->src_data[0] << 12; 525 addr |= ((u64)entry->src_data[1] & 0xf) << 44; 526 527 if (retry_fault && amdgpu_gmc_filter_faults(adev, addr, entry->pasid, 528 entry->timestamp)) 529 return 1; /* This also prevents sending it to KFD */ 530 531 if (entry->client_id == SOC15_IH_CLIENTID_VMC) { 532 snprintf(hub_name, sizeof(hub_name), "mmhub0"); 533 hub = &adev->vmhub[AMDGPU_MMHUB_0]; 534 } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { 535 snprintf(hub_name, sizeof(hub_name), "mmhub1"); 536 hub = &adev->vmhub[AMDGPU_MMHUB_1]; 537 } else { 538 snprintf(hub_name, sizeof(hub_name), "gfxhub0"); 539 hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 540 } 541 542 /* If it's the first fault for this address, process it normally */ 543 if (retry_fault && !in_interrupt() && 544 amdgpu_vm_handle_fault(adev, entry->pasid, addr)) 545 return 1; /* This also prevents sending it to KFD */ 546 547 if (!amdgpu_sriov_vf(adev)) { 548 /* 549 * Issue a dummy read to wait for the status register to 550 * be updated to avoid reading an incorrect value due to 551 * the new fast GRBM interface. 552 */ 553 if (entry->vmid_src == AMDGPU_GFXHUB_0) 554 RREG32(hub->vm_l2_pro_fault_status); 555 556 status = RREG32(hub->vm_l2_pro_fault_status); 557 cid = REG_GET_FIELD(status, 558 VM_L2_PROTECTION_FAULT_STATUS, CID); 559 rw = REG_GET_FIELD(status, 560 VM_L2_PROTECTION_FAULT_STATUS, RW); 561 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 562 } 563 564 if (printk_ratelimit()) { 565 struct amdgpu_task_info task_info; 566 567 memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 568 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 569 570 dev_err(adev->dev, 571 "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 572 "pasid:%u, for process %s pid %d thread %s pid %d)\n", 573 hub_name, retry_fault ? "retry" : "no-retry", 574 entry->src_id, entry->ring_id, entry->vmid, 575 entry->pasid, task_info.process_name, task_info.tgid, 576 task_info.task_name, task_info.pid); 577 dev_err(adev->dev, " in page starting at address 0x%016llx from client %d\n", 578 addr, entry->client_id); 579 if (!amdgpu_sriov_vf(adev)) { 580 dev_err(adev->dev, 581 "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 582 status); 583 if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) { 584 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 585 cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : gfxhub_client_ids[cid], 586 cid); 587 } else { 588 switch (adev->asic_type) { 589 case CHIP_VEGA10: 590 mmhub_cid = mmhub_client_ids_vega10[cid][rw]; 591 break; 592 case CHIP_VEGA12: 593 mmhub_cid = mmhub_client_ids_vega12[cid][rw]; 594 break; 595 case CHIP_VEGA20: 596 mmhub_cid = mmhub_client_ids_vega20[cid][rw]; 597 break; 598 case CHIP_ARCTURUS: 599 mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; 600 break; 601 case CHIP_RAVEN: 602 mmhub_cid = mmhub_client_ids_raven[cid][rw]; 603 break; 604 case CHIP_RENOIR: 605 mmhub_cid = mmhub_client_ids_renoir[cid][rw]; 606 break; 607 default: 608 mmhub_cid = NULL; 609 break; 610 } 611 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 612 mmhub_cid ? mmhub_cid : "unknown", cid); 613 } 614 dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 615 REG_GET_FIELD(status, 616 VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 617 dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 618 REG_GET_FIELD(status, 619 VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 620 dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 621 REG_GET_FIELD(status, 622 VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 623 dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 624 REG_GET_FIELD(status, 625 VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 626 dev_err(adev->dev, "\t RW: 0x%x\n", rw); 627 } 628 } 629 630 return 0; 631 } 632 633 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 634 .set = gmc_v9_0_vm_fault_interrupt_state, 635 .process = gmc_v9_0_process_interrupt, 636 }; 637 638 639 static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 640 .set = gmc_v9_0_ecc_interrupt_state, 641 .process = amdgpu_umc_process_ecc_irq, 642 }; 643 644 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 645 { 646 adev->gmc.vm_fault.num_types = 1; 647 adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 648 649 if (!amdgpu_sriov_vf(adev)) { 650 adev->gmc.ecc_irq.num_types = 1; 651 adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 652 } 653 } 654 655 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 656 uint32_t flush_type) 657 { 658 u32 req = 0; 659 660 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 661 PER_VMID_INVALIDATE_REQ, 1 << vmid); 662 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 663 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 664 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 665 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 666 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 667 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 668 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 669 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 670 671 return req; 672 } 673 674 /** 675 * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore 676 * 677 * @adev: amdgpu_device pointer 678 * @vmhub: vmhub type 679 * 680 */ 681 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, 682 uint32_t vmhub) 683 { 684 return ((vmhub == AMDGPU_MMHUB_0 || 685 vmhub == AMDGPU_MMHUB_1) && 686 (!amdgpu_sriov_vf(adev)) && 687 (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && 688 (adev->apu_flags & AMD_APU_IS_PICASSO)))); 689 } 690 691 static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, 692 uint8_t vmid, uint16_t *p_pasid) 693 { 694 uint32_t value; 695 696 value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 697 + vmid); 698 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 699 700 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 701 } 702 703 /* 704 * GART 705 * VMID 0 is the physical GPU addresses as used by the kernel. 706 * VMIDs 1-15 are used for userspace clients and are handled 707 * by the amdgpu vm/hsa code. 708 */ 709 710 /** 711 * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 712 * 713 * @adev: amdgpu_device pointer 714 * @vmid: vm instance to flush 715 * @flush_type: the flush type 716 * 717 * Flush the TLB for the requested page table using certain type. 718 */ 719 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 720 uint32_t vmhub, uint32_t flush_type) 721 { 722 bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); 723 const unsigned eng = 17; 724 u32 j, inv_req, inv_req2, tmp; 725 struct amdgpu_vmhub *hub; 726 727 BUG_ON(vmhub >= adev->num_vmhubs); 728 729 hub = &adev->vmhub[vmhub]; 730 if (adev->gmc.xgmi.num_physical_nodes && 731 adev->asic_type == CHIP_VEGA20) { 732 /* Vega20+XGMI caches PTEs in TC and TLB. Add a 733 * heavy-weight TLB flush (type 2), which flushes 734 * both. Due to a race condition with concurrent 735 * memory accesses using the same TLB cache line, we 736 * still need a second TLB flush after this. 737 */ 738 inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); 739 inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 740 } else { 741 inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); 742 inv_req2 = 0; 743 } 744 745 /* This is necessary for a HW workaround under SRIOV as well 746 * as GFXOFF under bare metal 747 */ 748 if (adev->gfx.kiq.ring.sched.ready && 749 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 750 down_read_trylock(&adev->reset_sem)) { 751 uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 752 uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 753 754 amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 755 1 << vmid); 756 up_read(&adev->reset_sem); 757 return; 758 } 759 760 spin_lock(&adev->gmc.invalidate_lock); 761 762 /* 763 * It may lose gpuvm invalidate acknowldege state across power-gating 764 * off cycle, add semaphore acquire before invalidation and semaphore 765 * release after invalidation to avoid entering power gated state 766 * to WA the Issue 767 */ 768 769 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 770 if (use_semaphore) { 771 for (j = 0; j < adev->usec_timeout; j++) { 772 /* a read return value of 1 means semaphore acuqire */ 773 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + 774 hub->eng_distance * eng); 775 if (tmp & 0x1) 776 break; 777 udelay(1); 778 } 779 780 if (j >= adev->usec_timeout) 781 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 782 } 783 784 do { 785 WREG32_NO_KIQ(hub->vm_inv_eng0_req + 786 hub->eng_distance * eng, inv_req); 787 788 /* 789 * Issue a dummy read to wait for the ACK register to 790 * be cleared to avoid a false ACK due to the new fast 791 * GRBM interface. 792 */ 793 if (vmhub == AMDGPU_GFXHUB_0) 794 RREG32_NO_KIQ(hub->vm_inv_eng0_req + 795 hub->eng_distance * eng); 796 797 for (j = 0; j < adev->usec_timeout; j++) { 798 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + 799 hub->eng_distance * eng); 800 if (tmp & (1 << vmid)) 801 break; 802 udelay(1); 803 } 804 805 inv_req = inv_req2; 806 inv_req2 = 0; 807 } while (inv_req); 808 809 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 810 if (use_semaphore) 811 /* 812 * add semaphore release after invalidation, 813 * write with 0 means semaphore release 814 */ 815 WREG32_NO_KIQ(hub->vm_inv_eng0_sem + 816 hub->eng_distance * eng, 0); 817 818 spin_unlock(&adev->gmc.invalidate_lock); 819 820 if (j < adev->usec_timeout) 821 return; 822 823 DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 824 } 825 826 /** 827 * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid 828 * 829 * @adev: amdgpu_device pointer 830 * @pasid: pasid to be flush 831 * 832 * Flush the TLB for the requested pasid. 833 */ 834 static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 835 uint16_t pasid, uint32_t flush_type, 836 bool all_hub) 837 { 838 int vmid, i; 839 signed long r; 840 uint32_t seq; 841 uint16_t queried_pasid; 842 bool ret; 843 struct amdgpu_ring *ring = &adev->gfx.kiq.ring; 844 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 845 846 if (amdgpu_in_reset(adev)) 847 return -EIO; 848 849 if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) { 850 /* Vega20+XGMI caches PTEs in TC and TLB. Add a 851 * heavy-weight TLB flush (type 2), which flushes 852 * both. Due to a race condition with concurrent 853 * memory accesses using the same TLB cache line, we 854 * still need a second TLB flush after this. 855 */ 856 bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && 857 adev->asic_type == CHIP_VEGA20); 858 /* 2 dwords flush + 8 dwords fence */ 859 unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; 860 861 if (vega20_xgmi_wa) 862 ndw += kiq->pmf->invalidate_tlbs_size; 863 864 spin_lock(&adev->gfx.kiq.ring_lock); 865 /* 2 dwords flush + 8 dwords fence */ 866 amdgpu_ring_alloc(ring, ndw); 867 if (vega20_xgmi_wa) 868 kiq->pmf->kiq_invalidate_tlbs(ring, 869 pasid, 2, all_hub); 870 kiq->pmf->kiq_invalidate_tlbs(ring, 871 pasid, flush_type, all_hub); 872 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 873 if (r) { 874 amdgpu_ring_undo(ring); 875 spin_unlock(&adev->gfx.kiq.ring_lock); 876 up_read(&adev->reset_sem); 877 return -ETIME; 878 } 879 880 amdgpu_ring_commit(ring); 881 spin_unlock(&adev->gfx.kiq.ring_lock); 882 r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); 883 if (r < 1) { 884 dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); 885 up_read(&adev->reset_sem); 886 return -ETIME; 887 } 888 up_read(&adev->reset_sem); 889 return 0; 890 } 891 892 for (vmid = 1; vmid < 16; vmid++) { 893 894 ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 895 &queried_pasid); 896 if (ret && queried_pasid == pasid) { 897 if (all_hub) { 898 for (i = 0; i < adev->num_vmhubs; i++) 899 gmc_v9_0_flush_gpu_tlb(adev, vmid, 900 i, flush_type); 901 } else { 902 gmc_v9_0_flush_gpu_tlb(adev, vmid, 903 AMDGPU_GFXHUB_0, flush_type); 904 } 905 break; 906 } 907 } 908 909 return 0; 910 911 } 912 913 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 914 unsigned vmid, uint64_t pd_addr) 915 { 916 bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); 917 struct amdgpu_device *adev = ring->adev; 918 struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 919 uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 920 unsigned eng = ring->vm_inv_eng; 921 922 /* 923 * It may lose gpuvm invalidate acknowldege state across power-gating 924 * off cycle, add semaphore acquire before invalidation and semaphore 925 * release after invalidation to avoid entering power gated state 926 * to WA the Issue 927 */ 928 929 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 930 if (use_semaphore) 931 /* a read return value of 1 means semaphore acuqire */ 932 amdgpu_ring_emit_reg_wait(ring, 933 hub->vm_inv_eng0_sem + 934 hub->eng_distance * eng, 0x1, 0x1); 935 936 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 937 (hub->ctx_addr_distance * vmid), 938 lower_32_bits(pd_addr)); 939 940 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 941 (hub->ctx_addr_distance * vmid), 942 upper_32_bits(pd_addr)); 943 944 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 945 hub->eng_distance * eng, 946 hub->vm_inv_eng0_ack + 947 hub->eng_distance * eng, 948 req, 1 << vmid); 949 950 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 951 if (use_semaphore) 952 /* 953 * add semaphore release after invalidation, 954 * write with 0 means semaphore release 955 */ 956 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 957 hub->eng_distance * eng, 0); 958 959 return pd_addr; 960 } 961 962 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 963 unsigned pasid) 964 { 965 struct amdgpu_device *adev = ring->adev; 966 uint32_t reg; 967 968 /* Do nothing because there's no lut register for mmhub1. */ 969 if (ring->funcs->vmhub == AMDGPU_MMHUB_1) 970 return; 971 972 if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) 973 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 974 else 975 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 976 977 amdgpu_ring_emit_wreg(ring, reg, pasid); 978 } 979 980 /* 981 * PTE format on VEGA 10: 982 * 63:59 reserved 983 * 58:57 mtype 984 * 56 F 985 * 55 L 986 * 54 P 987 * 53 SW 988 * 52 T 989 * 50:48 reserved 990 * 47:12 4k physical page base address 991 * 11:7 fragment 992 * 6 write 993 * 5 read 994 * 4 exe 995 * 3 Z 996 * 2 snooped 997 * 1 system 998 * 0 valid 999 * 1000 * PDE format on VEGA 10: 1001 * 63:59 block fragment size 1002 * 58:55 reserved 1003 * 54 P 1004 * 53:48 reserved 1005 * 47:6 physical base address of PD or PTE 1006 * 5:3 reserved 1007 * 2 C 1008 * 1 system 1009 * 0 valid 1010 */ 1011 1012 static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) 1013 1014 { 1015 switch (flags) { 1016 case AMDGPU_VM_MTYPE_DEFAULT: 1017 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1018 case AMDGPU_VM_MTYPE_NC: 1019 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1020 case AMDGPU_VM_MTYPE_WC: 1021 return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); 1022 case AMDGPU_VM_MTYPE_RW: 1023 return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); 1024 case AMDGPU_VM_MTYPE_CC: 1025 return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); 1026 case AMDGPU_VM_MTYPE_UC: 1027 return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); 1028 default: 1029 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1030 } 1031 } 1032 1033 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 1034 uint64_t *addr, uint64_t *flags) 1035 { 1036 if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 1037 *addr = adev->vm_manager.vram_base_offset + *addr - 1038 adev->gmc.vram_start; 1039 BUG_ON(*addr & 0xFFFF00000000003FULL); 1040 1041 if (!adev->gmc.translate_further) 1042 return; 1043 1044 if (level == AMDGPU_VM_PDB1) { 1045 /* Set the block fragment size */ 1046 if (!(*flags & AMDGPU_PDE_PTE)) 1047 *flags |= AMDGPU_PDE_BFS(0x9); 1048 1049 } else if (level == AMDGPU_VM_PDB0) { 1050 if (*flags & AMDGPU_PDE_PTE) 1051 *flags &= ~AMDGPU_PDE_PTE; 1052 else 1053 *flags |= AMDGPU_PTE_TF; 1054 } 1055 } 1056 1057 static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, 1058 struct amdgpu_bo_va_mapping *mapping, 1059 uint64_t *flags) 1060 { 1061 *flags &= ~AMDGPU_PTE_EXECUTABLE; 1062 *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1063 1064 *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1065 *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; 1066 1067 if (mapping->flags & AMDGPU_PTE_PRT) { 1068 *flags |= AMDGPU_PTE_PRT; 1069 *flags &= ~AMDGPU_PTE_VALID; 1070 } 1071 1072 if (adev->asic_type == CHIP_ARCTURUS && 1073 !(*flags & AMDGPU_PTE_SYSTEM) && 1074 mapping->bo_va->is_xgmi) 1075 *flags |= AMDGPU_PTE_SNOOPED; 1076 } 1077 1078 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 1079 { 1080 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 1081 unsigned size; 1082 1083 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 1084 size = AMDGPU_VBIOS_VGA_ALLOCATION; 1085 } else { 1086 u32 viewport; 1087 1088 switch (adev->asic_type) { 1089 case CHIP_RAVEN: 1090 case CHIP_RENOIR: 1091 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 1092 size = (REG_GET_FIELD(viewport, 1093 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 1094 REG_GET_FIELD(viewport, 1095 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 1096 4); 1097 break; 1098 case CHIP_VEGA10: 1099 case CHIP_VEGA12: 1100 case CHIP_VEGA20: 1101 default: 1102 viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 1103 size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 1104 REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 1105 4); 1106 break; 1107 } 1108 } 1109 1110 return size; 1111 } 1112 1113 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 1114 .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 1115 .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, 1116 .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 1117 .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 1118 .map_mtype = gmc_v9_0_map_mtype, 1119 .get_vm_pde = gmc_v9_0_get_vm_pde, 1120 .get_vm_pte = gmc_v9_0_get_vm_pte, 1121 .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, 1122 }; 1123 1124 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 1125 { 1126 adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 1127 } 1128 1129 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) 1130 { 1131 switch (adev->asic_type) { 1132 case CHIP_VEGA10: 1133 adev->umc.funcs = &umc_v6_0_funcs; 1134 break; 1135 case CHIP_VEGA20: 1136 adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 1137 adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 1138 adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 1139 adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; 1140 adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 1141 adev->umc.funcs = &umc_v6_1_funcs; 1142 break; 1143 case CHIP_ARCTURUS: 1144 adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 1145 adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 1146 adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 1147 adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; 1148 adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 1149 adev->umc.funcs = &umc_v6_1_funcs; 1150 break; 1151 default: 1152 break; 1153 } 1154 } 1155 1156 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) 1157 { 1158 switch (adev->asic_type) { 1159 case CHIP_ARCTURUS: 1160 adev->mmhub.funcs = &mmhub_v9_4_funcs; 1161 break; 1162 default: 1163 adev->mmhub.funcs = &mmhub_v1_0_funcs; 1164 break; 1165 } 1166 } 1167 1168 static int gmc_v9_0_early_init(void *handle) 1169 { 1170 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1171 1172 gmc_v9_0_set_gmc_funcs(adev); 1173 gmc_v9_0_set_irq_funcs(adev); 1174 gmc_v9_0_set_umc_funcs(adev); 1175 gmc_v9_0_set_mmhub_funcs(adev); 1176 1177 adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 1178 adev->gmc.shared_aperture_end = 1179 adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 1180 adev->gmc.private_aperture_start = 0x1000000000000000ULL; 1181 adev->gmc.private_aperture_end = 1182 adev->gmc.private_aperture_start + (4ULL << 30) - 1; 1183 1184 return 0; 1185 } 1186 1187 static int gmc_v9_0_late_init(void *handle) 1188 { 1189 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1190 int r; 1191 1192 amdgpu_bo_late_init(adev); 1193 1194 r = amdgpu_gmc_allocate_vm_inv_eng(adev); 1195 if (r) 1196 return r; 1197 /* Check if ecc is available */ 1198 if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) { 1199 r = amdgpu_atomfirmware_mem_ecc_supported(adev); 1200 if (!r) { 1201 DRM_INFO("ECC is not present.\n"); 1202 if (adev->df.funcs->enable_ecc_force_par_wr_rmw) 1203 adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); 1204 } else 1205 DRM_INFO("ECC is active.\n"); 1206 1207 r = amdgpu_atomfirmware_sram_ecc_supported(adev); 1208 if (!r) 1209 DRM_INFO("SRAM ECC is not present.\n"); 1210 else 1211 DRM_INFO("SRAM ECC is active.\n"); 1212 } 1213 1214 if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count) 1215 adev->mmhub.funcs->reset_ras_error_count(adev); 1216 1217 r = amdgpu_gmc_ras_late_init(adev); 1218 if (r) 1219 return r; 1220 1221 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 1222 } 1223 1224 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 1225 struct amdgpu_gmc *mc) 1226 { 1227 u64 base = 0; 1228 1229 if (!amdgpu_sriov_vf(adev)) 1230 base = adev->mmhub.funcs->get_fb_location(adev); 1231 1232 /* add the xgmi offset of the physical node */ 1233 base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1234 amdgpu_gmc_vram_location(adev, mc, base); 1235 amdgpu_gmc_gart_location(adev, mc); 1236 amdgpu_gmc_agp_location(adev, mc); 1237 /* base offset of vram pages */ 1238 adev->vm_manager.vram_base_offset = gfxhub_v1_0_get_mc_fb_offset(adev); 1239 1240 /* XXX: add the xgmi offset of the physical node? */ 1241 adev->vm_manager.vram_base_offset += 1242 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1243 } 1244 1245 /** 1246 * gmc_v9_0_mc_init - initialize the memory controller driver params 1247 * 1248 * @adev: amdgpu_device pointer 1249 * 1250 * Look up the amount of vram, vram width, and decide how to place 1251 * vram and gart within the GPU's physical address space. 1252 * Returns 0 for success. 1253 */ 1254 static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 1255 { 1256 int r; 1257 1258 /* size in MB on si */ 1259 adev->gmc.mc_vram_size = 1260 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 1261 adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 1262 1263 if (!(adev->flags & AMD_IS_APU)) { 1264 r = amdgpu_device_resize_fb_bar(adev); 1265 if (r) 1266 return r; 1267 } 1268 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 1269 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 1270 1271 #ifdef CONFIG_X86_64 1272 if (adev->flags & AMD_IS_APU) { 1273 adev->gmc.aper_base = gfxhub_v1_0_get_mc_fb_offset(adev); 1274 adev->gmc.aper_size = adev->gmc.real_vram_size; 1275 } 1276 #endif 1277 /* In case the PCI BAR is larger than the actual amount of vram */ 1278 adev->gmc.visible_vram_size = adev->gmc.aper_size; 1279 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 1280 adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 1281 1282 /* set the gart size */ 1283 if (amdgpu_gart_size == -1) { 1284 switch (adev->asic_type) { 1285 case CHIP_VEGA10: /* all engines support GPUVM */ 1286 case CHIP_VEGA12: /* all engines support GPUVM */ 1287 case CHIP_VEGA20: 1288 case CHIP_ARCTURUS: 1289 default: 1290 adev->gmc.gart_size = 512ULL << 20; 1291 break; 1292 case CHIP_RAVEN: /* DCE SG support */ 1293 case CHIP_RENOIR: 1294 adev->gmc.gart_size = 1024ULL << 20; 1295 break; 1296 } 1297 } else { 1298 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 1299 } 1300 1301 gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 1302 1303 return 0; 1304 } 1305 1306 static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 1307 { 1308 int r; 1309 1310 if (adev->gart.bo) { 1311 WARN(1, "VEGA10 PCIE GART already initialized\n"); 1312 return 0; 1313 } 1314 /* Initialize common gart structure */ 1315 r = amdgpu_gart_init(adev); 1316 if (r) 1317 return r; 1318 adev->gart.table_size = adev->gart.num_gpu_pages * 8; 1319 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | 1320 AMDGPU_PTE_EXECUTABLE; 1321 return amdgpu_gart_table_vram_alloc(adev); 1322 } 1323 1324 /** 1325 * gmc_v9_0_save_registers - saves regs 1326 * 1327 * @adev: amdgpu_device pointer 1328 * 1329 * This saves potential register values that should be 1330 * restored upon resume 1331 */ 1332 static void gmc_v9_0_save_registers(struct amdgpu_device *adev) 1333 { 1334 if (adev->asic_type == CHIP_RAVEN) 1335 adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); 1336 } 1337 1338 static int gmc_v9_0_sw_init(void *handle) 1339 { 1340 int r, vram_width = 0, vram_type = 0, vram_vendor = 0; 1341 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1342 1343 gfxhub_v1_0_init(adev); 1344 1345 adev->mmhub.funcs->init(adev); 1346 1347 spin_lock_init(&adev->gmc.invalidate_lock); 1348 1349 r = amdgpu_atomfirmware_get_vram_info(adev, 1350 &vram_width, &vram_type, &vram_vendor); 1351 if (amdgpu_sriov_vf(adev)) 1352 /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, 1353 * and DF related registers is not readable, seems hardcord is the 1354 * only way to set the correct vram_width 1355 */ 1356 adev->gmc.vram_width = 2048; 1357 else if (amdgpu_emu_mode != 1) 1358 adev->gmc.vram_width = vram_width; 1359 1360 if (!adev->gmc.vram_width) { 1361 int chansize, numchan; 1362 1363 /* hbm memory channel size */ 1364 if (adev->flags & AMD_IS_APU) 1365 chansize = 64; 1366 else 1367 chansize = 128; 1368 1369 numchan = adev->df.funcs->get_hbm_channel_number(adev); 1370 adev->gmc.vram_width = numchan * chansize; 1371 } 1372 1373 adev->gmc.vram_type = vram_type; 1374 adev->gmc.vram_vendor = vram_vendor; 1375 switch (adev->asic_type) { 1376 case CHIP_RAVEN: 1377 adev->num_vmhubs = 2; 1378 1379 if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 1380 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1381 } else { 1382 /* vm_size is 128TB + 512GB for legacy 3-level page support */ 1383 amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 1384 adev->gmc.translate_further = 1385 adev->vm_manager.num_level > 1; 1386 } 1387 break; 1388 case CHIP_VEGA10: 1389 case CHIP_VEGA12: 1390 case CHIP_VEGA20: 1391 case CHIP_RENOIR: 1392 adev->num_vmhubs = 2; 1393 1394 1395 /* 1396 * To fulfill 4-level page support, 1397 * vm size is 256TB (48bit), maximum size of Vega10, 1398 * block size 512 (9bit) 1399 */ 1400 /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 1401 if (amdgpu_sriov_vf(adev)) 1402 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 1403 else 1404 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1405 break; 1406 case CHIP_ARCTURUS: 1407 adev->num_vmhubs = 3; 1408 1409 /* Keep the vm size same with Vega20 */ 1410 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1411 break; 1412 default: 1413 break; 1414 } 1415 1416 /* This interrupt is VMC page fault.*/ 1417 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 1418 &adev->gmc.vm_fault); 1419 if (r) 1420 return r; 1421 1422 if (adev->asic_type == CHIP_ARCTURUS) { 1423 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, 1424 &adev->gmc.vm_fault); 1425 if (r) 1426 return r; 1427 } 1428 1429 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 1430 &adev->gmc.vm_fault); 1431 1432 if (r) 1433 return r; 1434 1435 if (!amdgpu_sriov_vf(adev)) { 1436 /* interrupt sent to DF. */ 1437 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 1438 &adev->gmc.ecc_irq); 1439 if (r) 1440 return r; 1441 } 1442 1443 /* Set the internal MC address mask 1444 * This is the max address of the GPU's 1445 * internal address space. 1446 */ 1447 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 1448 1449 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); 1450 if (r) { 1451 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 1452 return r; 1453 } 1454 adev->need_swiotlb = drm_need_swiotlb(44); 1455 1456 if (adev->gmc.xgmi.supported) { 1457 r = gfxhub_v1_1_get_xgmi_info(adev); 1458 if (r) 1459 return r; 1460 } 1461 1462 r = gmc_v9_0_mc_init(adev); 1463 if (r) 1464 return r; 1465 1466 amdgpu_gmc_get_vbios_allocations(adev); 1467 1468 /* Memory manager */ 1469 r = amdgpu_bo_init(adev); 1470 if (r) 1471 return r; 1472 1473 r = gmc_v9_0_gart_init(adev); 1474 if (r) 1475 return r; 1476 1477 /* 1478 * number of VMs 1479 * VMID 0 is reserved for System 1480 * amdgpu graphics/compute will use VMIDs 1..n-1 1481 * amdkfd will use VMIDs n..15 1482 * 1483 * The first KFD VMID is 8 for GPUs with graphics, 3 for 1484 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs 1485 * for video processing. 1486 */ 1487 adev->vm_manager.first_kfd_vmid = 1488 adev->asic_type == CHIP_ARCTURUS ? 3 : 8; 1489 1490 amdgpu_vm_manager_init(adev); 1491 1492 gmc_v9_0_save_registers(adev); 1493 1494 return 0; 1495 } 1496 1497 static int gmc_v9_0_sw_fini(void *handle) 1498 { 1499 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1500 1501 amdgpu_gmc_ras_fini(adev); 1502 amdgpu_gem_force_release(adev); 1503 amdgpu_vm_manager_fini(adev); 1504 amdgpu_gart_table_vram_free(adev); 1505 amdgpu_bo_fini(adev); 1506 amdgpu_gart_fini(adev); 1507 1508 return 0; 1509 } 1510 1511 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1512 { 1513 1514 switch (adev->asic_type) { 1515 case CHIP_VEGA10: 1516 if (amdgpu_sriov_vf(adev)) 1517 break; 1518 fallthrough; 1519 case CHIP_VEGA20: 1520 soc15_program_register_sequence(adev, 1521 golden_settings_mmhub_1_0_0, 1522 ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1523 soc15_program_register_sequence(adev, 1524 golden_settings_athub_1_0_0, 1525 ARRAY_SIZE(golden_settings_athub_1_0_0)); 1526 break; 1527 case CHIP_VEGA12: 1528 break; 1529 case CHIP_RAVEN: 1530 /* TODO for renoir */ 1531 soc15_program_register_sequence(adev, 1532 golden_settings_athub_1_0_0, 1533 ARRAY_SIZE(golden_settings_athub_1_0_0)); 1534 break; 1535 default: 1536 break; 1537 } 1538 } 1539 1540 /** 1541 * gmc_v9_0_restore_registers - restores regs 1542 * 1543 * @adev: amdgpu_device pointer 1544 * 1545 * This restores register values, saved at suspend. 1546 */ 1547 void gmc_v9_0_restore_registers(struct amdgpu_device *adev) 1548 { 1549 if (adev->asic_type == CHIP_RAVEN) 1550 WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); 1551 } 1552 1553 /** 1554 * gmc_v9_0_gart_enable - gart enable 1555 * 1556 * @adev: amdgpu_device pointer 1557 */ 1558 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1559 { 1560 int r; 1561 1562 if (adev->gart.bo == NULL) { 1563 dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1564 return -EINVAL; 1565 } 1566 r = amdgpu_gart_table_vram_pin(adev); 1567 if (r) 1568 return r; 1569 1570 r = gfxhub_v1_0_gart_enable(adev); 1571 if (r) 1572 return r; 1573 1574 r = adev->mmhub.funcs->gart_enable(adev); 1575 if (r) 1576 return r; 1577 1578 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1579 (unsigned)(adev->gmc.gart_size >> 20), 1580 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1581 adev->gart.ready = true; 1582 return 0; 1583 } 1584 1585 static int gmc_v9_0_hw_init(void *handle) 1586 { 1587 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1588 bool value; 1589 int r, i; 1590 u32 tmp; 1591 1592 /* The sequence of these two function calls matters.*/ 1593 gmc_v9_0_init_golden_registers(adev); 1594 1595 if (adev->mode_info.num_crtc) { 1596 if (adev->asic_type != CHIP_ARCTURUS) { 1597 /* Lockout access through VGA aperture*/ 1598 WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1599 1600 /* disable VGA render */ 1601 WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1602 } 1603 } 1604 1605 amdgpu_device_program_register_sequence(adev, 1606 golden_settings_vega10_hdp, 1607 ARRAY_SIZE(golden_settings_vega10_hdp)); 1608 1609 if (adev->mmhub.funcs->update_power_gating) 1610 adev->mmhub.funcs->update_power_gating(adev, true); 1611 1612 switch (adev->asic_type) { 1613 case CHIP_ARCTURUS: 1614 WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); 1615 break; 1616 default: 1617 break; 1618 } 1619 1620 WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1621 1622 tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1623 WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1624 1625 WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); 1626 WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); 1627 1628 /* After HDP is initialized, flush HDP.*/ 1629 adev->nbio.funcs->hdp_flush(adev, NULL); 1630 1631 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1632 value = false; 1633 else 1634 value = true; 1635 1636 if (!amdgpu_sriov_vf(adev)) { 1637 gfxhub_v1_0_set_fault_enable_default(adev, value); 1638 adev->mmhub.funcs->set_fault_enable_default(adev, value); 1639 } 1640 for (i = 0; i < adev->num_vmhubs; ++i) 1641 gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); 1642 1643 if (adev->umc.funcs && adev->umc.funcs->init_registers) 1644 adev->umc.funcs->init_registers(adev); 1645 1646 r = gmc_v9_0_gart_enable(adev); 1647 1648 return r; 1649 } 1650 1651 /** 1652 * gmc_v9_0_gart_disable - gart disable 1653 * 1654 * @adev: amdgpu_device pointer 1655 * 1656 * This disables all VM page table. 1657 */ 1658 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1659 { 1660 gfxhub_v1_0_gart_disable(adev); 1661 adev->mmhub.funcs->gart_disable(adev); 1662 amdgpu_gart_table_vram_unpin(adev); 1663 } 1664 1665 static int gmc_v9_0_hw_fini(void *handle) 1666 { 1667 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1668 1669 if (amdgpu_sriov_vf(adev)) { 1670 /* full access mode, so don't touch any GMC register */ 1671 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 1672 return 0; 1673 } 1674 1675 amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1676 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1677 gmc_v9_0_gart_disable(adev); 1678 1679 return 0; 1680 } 1681 1682 static int gmc_v9_0_suspend(void *handle) 1683 { 1684 int r; 1685 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1686 1687 r = gmc_v9_0_hw_fini(adev); 1688 if (r) 1689 return r; 1690 1691 return 0; 1692 } 1693 1694 static int gmc_v9_0_resume(void *handle) 1695 { 1696 int r; 1697 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1698 1699 r = gmc_v9_0_hw_init(adev); 1700 if (r) 1701 return r; 1702 1703 amdgpu_vmid_reset_all(adev); 1704 1705 return 0; 1706 } 1707 1708 static bool gmc_v9_0_is_idle(void *handle) 1709 { 1710 /* MC is always ready in GMC v9.*/ 1711 return true; 1712 } 1713 1714 static int gmc_v9_0_wait_for_idle(void *handle) 1715 { 1716 /* There is no need to wait for MC idle in GMC v9.*/ 1717 return 0; 1718 } 1719 1720 static int gmc_v9_0_soft_reset(void *handle) 1721 { 1722 /* XXX for emulation.*/ 1723 return 0; 1724 } 1725 1726 static int gmc_v9_0_set_clockgating_state(void *handle, 1727 enum amd_clockgating_state state) 1728 { 1729 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1730 1731 adev->mmhub.funcs->set_clockgating(adev, state); 1732 1733 athub_v1_0_set_clockgating(adev, state); 1734 1735 return 0; 1736 } 1737 1738 static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 1739 { 1740 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1741 1742 adev->mmhub.funcs->get_clockgating(adev, flags); 1743 1744 athub_v1_0_get_clockgating(adev, flags); 1745 } 1746 1747 static int gmc_v9_0_set_powergating_state(void *handle, 1748 enum amd_powergating_state state) 1749 { 1750 return 0; 1751 } 1752 1753 const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1754 .name = "gmc_v9_0", 1755 .early_init = gmc_v9_0_early_init, 1756 .late_init = gmc_v9_0_late_init, 1757 .sw_init = gmc_v9_0_sw_init, 1758 .sw_fini = gmc_v9_0_sw_fini, 1759 .hw_init = gmc_v9_0_hw_init, 1760 .hw_fini = gmc_v9_0_hw_fini, 1761 .suspend = gmc_v9_0_suspend, 1762 .resume = gmc_v9_0_resume, 1763 .is_idle = gmc_v9_0_is_idle, 1764 .wait_for_idle = gmc_v9_0_wait_for_idle, 1765 .soft_reset = gmc_v9_0_soft_reset, 1766 .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1767 .set_powergating_state = gmc_v9_0_set_powergating_state, 1768 .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1769 }; 1770 1771 const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1772 { 1773 .type = AMD_IP_BLOCK_TYPE_GMC, 1774 .major = 9, 1775 .minor = 0, 1776 .rev = 0, 1777 .funcs = &gmc_v9_0_ip_funcs, 1778 }; 1779