1 /* 2 * Copyright 2016 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/firmware.h> 25 #include <linux/pci.h> 26 27 #include <drm/drm_cache.h> 28 29 #include "amdgpu.h" 30 #include "gmc_v9_0.h" 31 #include "amdgpu_atomfirmware.h" 32 #include "amdgpu_gem.h" 33 34 #include "hdp/hdp_4_0_offset.h" 35 #include "hdp/hdp_4_0_sh_mask.h" 36 #include "gc/gc_9_0_sh_mask.h" 37 #include "dce/dce_12_0_offset.h" 38 #include "dce/dce_12_0_sh_mask.h" 39 #include "vega10_enum.h" 40 #include "mmhub/mmhub_1_0_offset.h" 41 #include "athub/athub_1_0_sh_mask.h" 42 #include "athub/athub_1_0_offset.h" 43 #include "oss/osssys_4_0_offset.h" 44 45 #include "soc15.h" 46 #include "soc15d.h" 47 #include "soc15_common.h" 48 #include "umc/umc_6_0_sh_mask.h" 49 50 #include "gfxhub_v1_0.h" 51 #include "mmhub_v1_0.h" 52 #include "athub_v1_0.h" 53 #include "gfxhub_v1_1.h" 54 #include "mmhub_v9_4.h" 55 #include "umc_v6_1.h" 56 #include "umc_v6_0.h" 57 58 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h" 59 60 #include "amdgpu_ras.h" 61 #include "amdgpu_xgmi.h" 62 63 /* add these here since we already include dce12 headers and these are for DCN */ 64 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION 0x055d 65 #define mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION_BASE_IDX 2 66 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH__SHIFT 0x0 67 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT__SHIFT 0x10 68 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_WIDTH_MASK 0x00003FFFL 69 #define HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION__PRI_VIEWPORT_HEIGHT_MASK 0x3FFF0000L 70 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0 0x049d 71 #define mmDCHUBBUB_SDPIF_MMIO_CNTRL_0_BASE_IDX 2 72 73 74 static const char *gfxhub_client_ids[] = { 75 "CB", 76 "DB", 77 "IA", 78 "WD", 79 "CPF", 80 "CPC", 81 "CPG", 82 "RLC", 83 "TCP", 84 "SQC (inst)", 85 "SQC (data)", 86 "SQG", 87 "PA", 88 }; 89 90 static const char *mmhub_client_ids_raven[][2] = { 91 [0][0] = "MP1", 92 [1][0] = "MP0", 93 [2][0] = "VCN", 94 [3][0] = "VCNU", 95 [4][0] = "HDP", 96 [5][0] = "DCE", 97 [13][0] = "UTCL2", 98 [19][0] = "TLS", 99 [26][0] = "OSS", 100 [27][0] = "SDMA0", 101 [0][1] = "MP1", 102 [1][1] = "MP0", 103 [2][1] = "VCN", 104 [3][1] = "VCNU", 105 [4][1] = "HDP", 106 [5][1] = "XDP", 107 [6][1] = "DBGU0", 108 [7][1] = "DCE", 109 [8][1] = "DCEDWB0", 110 [9][1] = "DCEDWB1", 111 [26][1] = "OSS", 112 [27][1] = "SDMA0", 113 }; 114 115 static const char *mmhub_client_ids_renoir[][2] = { 116 [0][0] = "MP1", 117 [1][0] = "MP0", 118 [2][0] = "HDP", 119 [4][0] = "DCEDMC", 120 [5][0] = "DCEVGA", 121 [13][0] = "UTCL2", 122 [19][0] = "TLS", 123 [26][0] = "OSS", 124 [27][0] = "SDMA0", 125 [28][0] = "VCN", 126 [29][0] = "VCNU", 127 [30][0] = "JPEG", 128 [0][1] = "MP1", 129 [1][1] = "MP0", 130 [2][1] = "HDP", 131 [3][1] = "XDP", 132 [6][1] = "DBGU0", 133 [7][1] = "DCEDMC", 134 [8][1] = "DCEVGA", 135 [9][1] = "DCEDWB", 136 [26][1] = "OSS", 137 [27][1] = "SDMA0", 138 [28][1] = "VCN", 139 [29][1] = "VCNU", 140 [30][1] = "JPEG", 141 }; 142 143 static const char *mmhub_client_ids_vega10[][2] = { 144 [0][0] = "MP0", 145 [1][0] = "UVD", 146 [2][0] = "UVDU", 147 [3][0] = "HDP", 148 [13][0] = "UTCL2", 149 [14][0] = "OSS", 150 [15][0] = "SDMA1", 151 [32+0][0] = "VCE0", 152 [32+1][0] = "VCE0U", 153 [32+2][0] = "XDMA", 154 [32+3][0] = "DCE", 155 [32+4][0] = "MP1", 156 [32+14][0] = "SDMA0", 157 [0][1] = "MP0", 158 [1][1] = "UVD", 159 [2][1] = "UVDU", 160 [3][1] = "DBGU0", 161 [4][1] = "HDP", 162 [5][1] = "XDP", 163 [14][1] = "OSS", 164 [15][1] = "SDMA0", 165 [32+0][1] = "VCE0", 166 [32+1][1] = "VCE0U", 167 [32+2][1] = "XDMA", 168 [32+3][1] = "DCE", 169 [32+4][1] = "DCEDWB", 170 [32+5][1] = "MP1", 171 [32+6][1] = "DBGU1", 172 [32+14][1] = "SDMA1", 173 }; 174 175 static const char *mmhub_client_ids_vega12[][2] = { 176 [0][0] = "MP0", 177 [1][0] = "VCE0", 178 [2][0] = "VCE0U", 179 [3][0] = "HDP", 180 [13][0] = "UTCL2", 181 [14][0] = "OSS", 182 [15][0] = "SDMA1", 183 [32+0][0] = "DCE", 184 [32+1][0] = "XDMA", 185 [32+2][0] = "UVD", 186 [32+3][0] = "UVDU", 187 [32+4][0] = "MP1", 188 [32+15][0] = "SDMA0", 189 [0][1] = "MP0", 190 [1][1] = "VCE0", 191 [2][1] = "VCE0U", 192 [3][1] = "DBGU0", 193 [4][1] = "HDP", 194 [5][1] = "XDP", 195 [14][1] = "OSS", 196 [15][1] = "SDMA0", 197 [32+0][1] = "DCE", 198 [32+1][1] = "DCEDWB", 199 [32+2][1] = "XDMA", 200 [32+3][1] = "UVD", 201 [32+4][1] = "UVDU", 202 [32+5][1] = "MP1", 203 [32+6][1] = "DBGU1", 204 [32+15][1] = "SDMA1", 205 }; 206 207 static const char *mmhub_client_ids_vega20[][2] = { 208 [0][0] = "XDMA", 209 [1][0] = "DCE", 210 [2][0] = "VCE0", 211 [3][0] = "VCE0U", 212 [4][0] = "UVD", 213 [5][0] = "UVD1U", 214 [13][0] = "OSS", 215 [14][0] = "HDP", 216 [15][0] = "SDMA0", 217 [32+0][0] = "UVD", 218 [32+1][0] = "UVDU", 219 [32+2][0] = "MP1", 220 [32+3][0] = "MP0", 221 [32+12][0] = "UTCL2", 222 [32+14][0] = "SDMA1", 223 [0][1] = "XDMA", 224 [1][1] = "DCE", 225 [2][1] = "DCEDWB", 226 [3][1] = "VCE0", 227 [4][1] = "VCE0U", 228 [5][1] = "UVD1", 229 [6][1] = "UVD1U", 230 [7][1] = "DBGU0", 231 [8][1] = "XDP", 232 [13][1] = "OSS", 233 [14][1] = "HDP", 234 [15][1] = "SDMA0", 235 [32+0][1] = "UVD", 236 [32+1][1] = "UVDU", 237 [32+2][1] = "DBGU1", 238 [32+3][1] = "MP1", 239 [32+4][1] = "MP0", 240 [32+14][1] = "SDMA1", 241 }; 242 243 static const char *mmhub_client_ids_arcturus[][2] = { 244 [2][0] = "MP1", 245 [3][0] = "MP0", 246 [10][0] = "UTCL2", 247 [13][0] = "OSS", 248 [14][0] = "HDP", 249 [15][0] = "SDMA0", 250 [32+15][0] = "SDMA1", 251 [64+15][0] = "SDMA2", 252 [96+15][0] = "SDMA3", 253 [128+15][0] = "SDMA4", 254 [160+11][0] = "JPEG", 255 [160+12][0] = "VCN", 256 [160+13][0] = "VCNU", 257 [160+15][0] = "SDMA5", 258 [192+10][0] = "UTCL2", 259 [192+11][0] = "JPEG1", 260 [192+12][0] = "VCN1", 261 [192+13][0] = "VCN1U", 262 [192+15][0] = "SDMA6", 263 [224+15][0] = "SDMA7", 264 [0][1] = "DBGU1", 265 [1][1] = "XDP", 266 [2][1] = "MP1", 267 [3][1] = "MP0", 268 [13][1] = "OSS", 269 [14][1] = "HDP", 270 [15][1] = "SDMA0", 271 [32+15][1] = "SDMA1", 272 [64+15][1] = "SDMA2", 273 [96+15][1] = "SDMA3", 274 [128+15][1] = "SDMA4", 275 [160+11][1] = "JPEG", 276 [160+12][1] = "VCN", 277 [160+13][1] = "VCNU", 278 [160+15][1] = "SDMA5", 279 [192+11][1] = "JPEG1", 280 [192+12][1] = "VCN1", 281 [192+13][1] = "VCN1U", 282 [192+15][1] = "SDMA6", 283 [224+15][1] = "SDMA7", 284 }; 285 286 static const u32 golden_settings_vega10_hdp[] = 287 { 288 0xf64, 0x0fffffff, 0x00000000, 289 0xf65, 0x0fffffff, 0x00000000, 290 0xf66, 0x0fffffff, 0x00000000, 291 0xf67, 0x0fffffff, 0x00000000, 292 0xf68, 0x0fffffff, 0x00000000, 293 0xf6a, 0x0fffffff, 0x00000000, 294 0xf6b, 0x0fffffff, 0x00000000, 295 0xf6c, 0x0fffffff, 0x00000000, 296 0xf6d, 0x0fffffff, 0x00000000, 297 0xf6e, 0x0fffffff, 0x00000000, 298 }; 299 300 static const struct soc15_reg_golden golden_settings_mmhub_1_0_0[] = 301 { 302 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmDAGB1_WRCLI2, 0x00000007, 0xfe5fe0fa), 303 SOC15_REG_GOLDEN_VALUE(MMHUB, 0, mmMMEA1_DRAM_WR_CLI2GRP_MAP0, 0x00000030, 0x55555565) 304 }; 305 306 static const struct soc15_reg_golden golden_settings_athub_1_0_0[] = 307 { 308 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL, 0x0000ff00, 0x00000800), 309 SOC15_REG_GOLDEN_VALUE(ATHUB, 0, mmRPB_ARB_CNTL2, 0x00ff00ff, 0x00080008) 310 }; 311 312 static const uint32_t ecc_umc_mcumc_ctrl_addrs[] = { 313 (0x000143c0 + 0x00000000), 314 (0x000143c0 + 0x00000800), 315 (0x000143c0 + 0x00001000), 316 (0x000143c0 + 0x00001800), 317 (0x000543c0 + 0x00000000), 318 (0x000543c0 + 0x00000800), 319 (0x000543c0 + 0x00001000), 320 (0x000543c0 + 0x00001800), 321 (0x000943c0 + 0x00000000), 322 (0x000943c0 + 0x00000800), 323 (0x000943c0 + 0x00001000), 324 (0x000943c0 + 0x00001800), 325 (0x000d43c0 + 0x00000000), 326 (0x000d43c0 + 0x00000800), 327 (0x000d43c0 + 0x00001000), 328 (0x000d43c0 + 0x00001800), 329 (0x001143c0 + 0x00000000), 330 (0x001143c0 + 0x00000800), 331 (0x001143c0 + 0x00001000), 332 (0x001143c0 + 0x00001800), 333 (0x001543c0 + 0x00000000), 334 (0x001543c0 + 0x00000800), 335 (0x001543c0 + 0x00001000), 336 (0x001543c0 + 0x00001800), 337 (0x001943c0 + 0x00000000), 338 (0x001943c0 + 0x00000800), 339 (0x001943c0 + 0x00001000), 340 (0x001943c0 + 0x00001800), 341 (0x001d43c0 + 0x00000000), 342 (0x001d43c0 + 0x00000800), 343 (0x001d43c0 + 0x00001000), 344 (0x001d43c0 + 0x00001800), 345 }; 346 347 static const uint32_t ecc_umc_mcumc_ctrl_mask_addrs[] = { 348 (0x000143e0 + 0x00000000), 349 (0x000143e0 + 0x00000800), 350 (0x000143e0 + 0x00001000), 351 (0x000143e0 + 0x00001800), 352 (0x000543e0 + 0x00000000), 353 (0x000543e0 + 0x00000800), 354 (0x000543e0 + 0x00001000), 355 (0x000543e0 + 0x00001800), 356 (0x000943e0 + 0x00000000), 357 (0x000943e0 + 0x00000800), 358 (0x000943e0 + 0x00001000), 359 (0x000943e0 + 0x00001800), 360 (0x000d43e0 + 0x00000000), 361 (0x000d43e0 + 0x00000800), 362 (0x000d43e0 + 0x00001000), 363 (0x000d43e0 + 0x00001800), 364 (0x001143e0 + 0x00000000), 365 (0x001143e0 + 0x00000800), 366 (0x001143e0 + 0x00001000), 367 (0x001143e0 + 0x00001800), 368 (0x001543e0 + 0x00000000), 369 (0x001543e0 + 0x00000800), 370 (0x001543e0 + 0x00001000), 371 (0x001543e0 + 0x00001800), 372 (0x001943e0 + 0x00000000), 373 (0x001943e0 + 0x00000800), 374 (0x001943e0 + 0x00001000), 375 (0x001943e0 + 0x00001800), 376 (0x001d43e0 + 0x00000000), 377 (0x001d43e0 + 0x00000800), 378 (0x001d43e0 + 0x00001000), 379 (0x001d43e0 + 0x00001800), 380 }; 381 382 static int gmc_v9_0_ecc_interrupt_state(struct amdgpu_device *adev, 383 struct amdgpu_irq_src *src, 384 unsigned type, 385 enum amdgpu_interrupt_state state) 386 { 387 u32 bits, i, tmp, reg; 388 389 /* Devices newer then VEGA10/12 shall have these programming 390 sequences performed by PSP BL */ 391 if (adev->asic_type >= CHIP_VEGA20) 392 return 0; 393 394 bits = 0x7f; 395 396 switch (state) { 397 case AMDGPU_IRQ_STATE_DISABLE: 398 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 399 reg = ecc_umc_mcumc_ctrl_addrs[i]; 400 tmp = RREG32(reg); 401 tmp &= ~bits; 402 WREG32(reg, tmp); 403 } 404 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 405 reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 406 tmp = RREG32(reg); 407 tmp &= ~bits; 408 WREG32(reg, tmp); 409 } 410 break; 411 case AMDGPU_IRQ_STATE_ENABLE: 412 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_addrs); i++) { 413 reg = ecc_umc_mcumc_ctrl_addrs[i]; 414 tmp = RREG32(reg); 415 tmp |= bits; 416 WREG32(reg, tmp); 417 } 418 for (i = 0; i < ARRAY_SIZE(ecc_umc_mcumc_ctrl_mask_addrs); i++) { 419 reg = ecc_umc_mcumc_ctrl_mask_addrs[i]; 420 tmp = RREG32(reg); 421 tmp |= bits; 422 WREG32(reg, tmp); 423 } 424 break; 425 default: 426 break; 427 } 428 429 return 0; 430 } 431 432 static int gmc_v9_0_vm_fault_interrupt_state(struct amdgpu_device *adev, 433 struct amdgpu_irq_src *src, 434 unsigned type, 435 enum amdgpu_interrupt_state state) 436 { 437 struct amdgpu_vmhub *hub; 438 u32 tmp, reg, bits, i, j; 439 440 bits = VM_CONTEXT1_CNTL__RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 441 VM_CONTEXT1_CNTL__DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 442 VM_CONTEXT1_CNTL__PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 443 VM_CONTEXT1_CNTL__VALID_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 444 VM_CONTEXT1_CNTL__READ_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 445 VM_CONTEXT1_CNTL__WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK | 446 VM_CONTEXT1_CNTL__EXECUTE_PROTECTION_FAULT_ENABLE_INTERRUPT_MASK; 447 448 switch (state) { 449 case AMDGPU_IRQ_STATE_DISABLE: 450 for (j = 0; j < adev->num_vmhubs; j++) { 451 hub = &adev->vmhub[j]; 452 for (i = 0; i < 16; i++) { 453 reg = hub->vm_context0_cntl + i; 454 tmp = RREG32(reg); 455 tmp &= ~bits; 456 WREG32(reg, tmp); 457 } 458 } 459 break; 460 case AMDGPU_IRQ_STATE_ENABLE: 461 for (j = 0; j < adev->num_vmhubs; j++) { 462 hub = &adev->vmhub[j]; 463 for (i = 0; i < 16; i++) { 464 reg = hub->vm_context0_cntl + i; 465 tmp = RREG32(reg); 466 tmp |= bits; 467 WREG32(reg, tmp); 468 } 469 } 470 break; 471 default: 472 break; 473 } 474 475 return 0; 476 } 477 478 static int gmc_v9_0_process_interrupt(struct amdgpu_device *adev, 479 struct amdgpu_irq_src *source, 480 struct amdgpu_iv_entry *entry) 481 { 482 bool retry_fault = !!(entry->src_data[1] & 0x80); 483 uint32_t status = 0, cid = 0, rw = 0; 484 struct amdgpu_task_info task_info; 485 struct amdgpu_vmhub *hub; 486 const char *mmhub_cid; 487 const char *hub_name; 488 u64 addr; 489 490 addr = (u64)entry->src_data[0] << 12; 491 addr |= ((u64)entry->src_data[1] & 0xf) << 44; 492 493 if (retry_fault) { 494 /* Returning 1 here also prevents sending the IV to the KFD */ 495 496 /* Process it onyl if it's the first fault for this address */ 497 if (entry->ih != &adev->irq.ih_soft && 498 amdgpu_gmc_filter_faults(adev, addr, entry->pasid, 499 entry->timestamp)) 500 return 1; 501 502 /* Delegate it to a different ring if the hardware hasn't 503 * already done it. 504 */ 505 if (in_interrupt()) { 506 amdgpu_irq_delegate(adev, entry, 8); 507 return 1; 508 } 509 510 /* Try to handle the recoverable page faults by filling page 511 * tables 512 */ 513 if (amdgpu_vm_handle_fault(adev, entry->pasid, addr)) 514 return 1; 515 } 516 517 if (!printk_ratelimit()) 518 return 0; 519 520 if (entry->client_id == SOC15_IH_CLIENTID_VMC) { 521 hub_name = "mmhub0"; 522 hub = &adev->vmhub[AMDGPU_MMHUB_0]; 523 } else if (entry->client_id == SOC15_IH_CLIENTID_VMC1) { 524 hub_name = "mmhub1"; 525 hub = &adev->vmhub[AMDGPU_MMHUB_1]; 526 } else { 527 hub_name = "gfxhub0"; 528 hub = &adev->vmhub[AMDGPU_GFXHUB_0]; 529 } 530 531 memset(&task_info, 0, sizeof(struct amdgpu_task_info)); 532 amdgpu_vm_get_task_info(adev, entry->pasid, &task_info); 533 534 dev_err(adev->dev, 535 "[%s] %s page fault (src_id:%u ring:%u vmid:%u " 536 "pasid:%u, for process %s pid %d thread %s pid %d)\n", 537 hub_name, retry_fault ? "retry" : "no-retry", 538 entry->src_id, entry->ring_id, entry->vmid, 539 entry->pasid, task_info.process_name, task_info.tgid, 540 task_info.task_name, task_info.pid); 541 dev_err(adev->dev, " in page starting at address 0x%012llx from client %d\n", 542 addr, entry->client_id); 543 544 if (amdgpu_sriov_vf(adev)) 545 return 0; 546 547 /* 548 * Issue a dummy read to wait for the status register to 549 * be updated to avoid reading an incorrect value due to 550 * the new fast GRBM interface. 551 */ 552 if (entry->vmid_src == AMDGPU_GFXHUB_0) 553 RREG32(hub->vm_l2_pro_fault_status); 554 555 status = RREG32(hub->vm_l2_pro_fault_status); 556 cid = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, CID); 557 rw = REG_GET_FIELD(status, VM_L2_PROTECTION_FAULT_STATUS, RW); 558 WREG32_P(hub->vm_l2_pro_fault_cntl, 1, ~1); 559 560 561 dev_err(adev->dev, 562 "VM_L2_PROTECTION_FAULT_STATUS:0x%08X\n", 563 status); 564 if (hub == &adev->vmhub[AMDGPU_GFXHUB_0]) { 565 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 566 cid >= ARRAY_SIZE(gfxhub_client_ids) ? "unknown" : 567 gfxhub_client_ids[cid], 568 cid); 569 } else { 570 switch (adev->asic_type) { 571 case CHIP_VEGA10: 572 mmhub_cid = mmhub_client_ids_vega10[cid][rw]; 573 break; 574 case CHIP_VEGA12: 575 mmhub_cid = mmhub_client_ids_vega12[cid][rw]; 576 break; 577 case CHIP_VEGA20: 578 mmhub_cid = mmhub_client_ids_vega20[cid][rw]; 579 break; 580 case CHIP_ARCTURUS: 581 mmhub_cid = mmhub_client_ids_arcturus[cid][rw]; 582 break; 583 case CHIP_RAVEN: 584 mmhub_cid = mmhub_client_ids_raven[cid][rw]; 585 break; 586 case CHIP_RENOIR: 587 mmhub_cid = mmhub_client_ids_renoir[cid][rw]; 588 break; 589 default: 590 mmhub_cid = NULL; 591 break; 592 } 593 dev_err(adev->dev, "\t Faulty UTCL2 client ID: %s (0x%x)\n", 594 mmhub_cid ? mmhub_cid : "unknown", cid); 595 } 596 dev_err(adev->dev, "\t MORE_FAULTS: 0x%lx\n", 597 REG_GET_FIELD(status, 598 VM_L2_PROTECTION_FAULT_STATUS, MORE_FAULTS)); 599 dev_err(adev->dev, "\t WALKER_ERROR: 0x%lx\n", 600 REG_GET_FIELD(status, 601 VM_L2_PROTECTION_FAULT_STATUS, WALKER_ERROR)); 602 dev_err(adev->dev, "\t PERMISSION_FAULTS: 0x%lx\n", 603 REG_GET_FIELD(status, 604 VM_L2_PROTECTION_FAULT_STATUS, PERMISSION_FAULTS)); 605 dev_err(adev->dev, "\t MAPPING_ERROR: 0x%lx\n", 606 REG_GET_FIELD(status, 607 VM_L2_PROTECTION_FAULT_STATUS, MAPPING_ERROR)); 608 dev_err(adev->dev, "\t RW: 0x%x\n", rw); 609 return 0; 610 } 611 612 static const struct amdgpu_irq_src_funcs gmc_v9_0_irq_funcs = { 613 .set = gmc_v9_0_vm_fault_interrupt_state, 614 .process = gmc_v9_0_process_interrupt, 615 }; 616 617 618 static const struct amdgpu_irq_src_funcs gmc_v9_0_ecc_funcs = { 619 .set = gmc_v9_0_ecc_interrupt_state, 620 .process = amdgpu_umc_process_ecc_irq, 621 }; 622 623 static void gmc_v9_0_set_irq_funcs(struct amdgpu_device *adev) 624 { 625 adev->gmc.vm_fault.num_types = 1; 626 adev->gmc.vm_fault.funcs = &gmc_v9_0_irq_funcs; 627 628 if (!amdgpu_sriov_vf(adev)) { 629 adev->gmc.ecc_irq.num_types = 1; 630 adev->gmc.ecc_irq.funcs = &gmc_v9_0_ecc_funcs; 631 } 632 } 633 634 static uint32_t gmc_v9_0_get_invalidate_req(unsigned int vmid, 635 uint32_t flush_type) 636 { 637 u32 req = 0; 638 639 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 640 PER_VMID_INVALIDATE_REQ, 1 << vmid); 641 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, FLUSH_TYPE, flush_type); 642 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PTES, 1); 643 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE0, 1); 644 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE1, 1); 645 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L2_PDE2, 1); 646 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, INVALIDATE_L1_PTES, 1); 647 req = REG_SET_FIELD(req, VM_INVALIDATE_ENG0_REQ, 648 CLEAR_PROTECTION_FAULT_STATUS_ADDR, 0); 649 650 return req; 651 } 652 653 /** 654 * gmc_v9_0_use_invalidate_semaphore - judge whether to use semaphore 655 * 656 * @adev: amdgpu_device pointer 657 * @vmhub: vmhub type 658 * 659 */ 660 static bool gmc_v9_0_use_invalidate_semaphore(struct amdgpu_device *adev, 661 uint32_t vmhub) 662 { 663 return ((vmhub == AMDGPU_MMHUB_0 || 664 vmhub == AMDGPU_MMHUB_1) && 665 (!amdgpu_sriov_vf(adev)) && 666 (!(!(adev->apu_flags & AMD_APU_IS_RAVEN2) && 667 (adev->apu_flags & AMD_APU_IS_PICASSO)))); 668 } 669 670 static bool gmc_v9_0_get_atc_vmid_pasid_mapping_info(struct amdgpu_device *adev, 671 uint8_t vmid, uint16_t *p_pasid) 672 { 673 uint32_t value; 674 675 value = RREG32(SOC15_REG_OFFSET(ATHUB, 0, mmATC_VMID0_PASID_MAPPING) 676 + vmid); 677 *p_pasid = value & ATC_VMID0_PASID_MAPPING__PASID_MASK; 678 679 return !!(value & ATC_VMID0_PASID_MAPPING__VALID_MASK); 680 } 681 682 /* 683 * GART 684 * VMID 0 is the physical GPU addresses as used by the kernel. 685 * VMIDs 1-15 are used for userspace clients and are handled 686 * by the amdgpu vm/hsa code. 687 */ 688 689 /** 690 * gmc_v9_0_flush_gpu_tlb - tlb flush with certain type 691 * 692 * @adev: amdgpu_device pointer 693 * @vmid: vm instance to flush 694 * @vmhub: which hub to flush 695 * @flush_type: the flush type 696 * 697 * Flush the TLB for the requested page table using certain type. 698 */ 699 static void gmc_v9_0_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid, 700 uint32_t vmhub, uint32_t flush_type) 701 { 702 bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(adev, vmhub); 703 const unsigned eng = 17; 704 u32 j, inv_req, inv_req2, tmp; 705 struct amdgpu_vmhub *hub; 706 707 BUG_ON(vmhub >= adev->num_vmhubs); 708 709 hub = &adev->vmhub[vmhub]; 710 if (adev->gmc.xgmi.num_physical_nodes && 711 adev->asic_type == CHIP_VEGA20) { 712 /* Vega20+XGMI caches PTEs in TC and TLB. Add a 713 * heavy-weight TLB flush (type 2), which flushes 714 * both. Due to a race condition with concurrent 715 * memory accesses using the same TLB cache line, we 716 * still need a second TLB flush after this. 717 */ 718 inv_req = gmc_v9_0_get_invalidate_req(vmid, 2); 719 inv_req2 = gmc_v9_0_get_invalidate_req(vmid, flush_type); 720 } else { 721 inv_req = gmc_v9_0_get_invalidate_req(vmid, flush_type); 722 inv_req2 = 0; 723 } 724 725 /* This is necessary for a HW workaround under SRIOV as well 726 * as GFXOFF under bare metal 727 */ 728 if (adev->gfx.kiq.ring.sched.ready && 729 (amdgpu_sriov_runtime(adev) || !amdgpu_sriov_vf(adev)) && 730 down_read_trylock(&adev->reset_sem)) { 731 uint32_t req = hub->vm_inv_eng0_req + hub->eng_distance * eng; 732 uint32_t ack = hub->vm_inv_eng0_ack + hub->eng_distance * eng; 733 734 amdgpu_virt_kiq_reg_write_reg_wait(adev, req, ack, inv_req, 735 1 << vmid); 736 up_read(&adev->reset_sem); 737 return; 738 } 739 740 spin_lock(&adev->gmc.invalidate_lock); 741 742 /* 743 * It may lose gpuvm invalidate acknowldege state across power-gating 744 * off cycle, add semaphore acquire before invalidation and semaphore 745 * release after invalidation to avoid entering power gated state 746 * to WA the Issue 747 */ 748 749 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 750 if (use_semaphore) { 751 for (j = 0; j < adev->usec_timeout; j++) { 752 /* a read return value of 1 means semaphore acuqire */ 753 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_sem + 754 hub->eng_distance * eng); 755 if (tmp & 0x1) 756 break; 757 udelay(1); 758 } 759 760 if (j >= adev->usec_timeout) 761 DRM_ERROR("Timeout waiting for sem acquire in VM flush!\n"); 762 } 763 764 do { 765 WREG32_NO_KIQ(hub->vm_inv_eng0_req + 766 hub->eng_distance * eng, inv_req); 767 768 /* 769 * Issue a dummy read to wait for the ACK register to 770 * be cleared to avoid a false ACK due to the new fast 771 * GRBM interface. 772 */ 773 if (vmhub == AMDGPU_GFXHUB_0) 774 RREG32_NO_KIQ(hub->vm_inv_eng0_req + 775 hub->eng_distance * eng); 776 777 for (j = 0; j < adev->usec_timeout; j++) { 778 tmp = RREG32_NO_KIQ(hub->vm_inv_eng0_ack + 779 hub->eng_distance * eng); 780 if (tmp & (1 << vmid)) 781 break; 782 udelay(1); 783 } 784 785 inv_req = inv_req2; 786 inv_req2 = 0; 787 } while (inv_req); 788 789 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 790 if (use_semaphore) 791 /* 792 * add semaphore release after invalidation, 793 * write with 0 means semaphore release 794 */ 795 WREG32_NO_KIQ(hub->vm_inv_eng0_sem + 796 hub->eng_distance * eng, 0); 797 798 spin_unlock(&adev->gmc.invalidate_lock); 799 800 if (j < adev->usec_timeout) 801 return; 802 803 DRM_ERROR("Timeout waiting for VM flush ACK!\n"); 804 } 805 806 /** 807 * gmc_v9_0_flush_gpu_tlb_pasid - tlb flush via pasid 808 * 809 * @adev: amdgpu_device pointer 810 * @pasid: pasid to be flush 811 * @flush_type: the flush type 812 * @all_hub: flush all hubs 813 * 814 * Flush the TLB for the requested pasid. 815 */ 816 static int gmc_v9_0_flush_gpu_tlb_pasid(struct amdgpu_device *adev, 817 uint16_t pasid, uint32_t flush_type, 818 bool all_hub) 819 { 820 int vmid, i; 821 signed long r; 822 uint32_t seq; 823 uint16_t queried_pasid; 824 bool ret; 825 struct amdgpu_ring *ring = &adev->gfx.kiq.ring; 826 struct amdgpu_kiq *kiq = &adev->gfx.kiq; 827 828 if (amdgpu_in_reset(adev)) 829 return -EIO; 830 831 if (ring->sched.ready && down_read_trylock(&adev->reset_sem)) { 832 /* Vega20+XGMI caches PTEs in TC and TLB. Add a 833 * heavy-weight TLB flush (type 2), which flushes 834 * both. Due to a race condition with concurrent 835 * memory accesses using the same TLB cache line, we 836 * still need a second TLB flush after this. 837 */ 838 bool vega20_xgmi_wa = (adev->gmc.xgmi.num_physical_nodes && 839 adev->asic_type == CHIP_VEGA20); 840 /* 2 dwords flush + 8 dwords fence */ 841 unsigned int ndw = kiq->pmf->invalidate_tlbs_size + 8; 842 843 if (vega20_xgmi_wa) 844 ndw += kiq->pmf->invalidate_tlbs_size; 845 846 spin_lock(&adev->gfx.kiq.ring_lock); 847 /* 2 dwords flush + 8 dwords fence */ 848 amdgpu_ring_alloc(ring, ndw); 849 if (vega20_xgmi_wa) 850 kiq->pmf->kiq_invalidate_tlbs(ring, 851 pasid, 2, all_hub); 852 kiq->pmf->kiq_invalidate_tlbs(ring, 853 pasid, flush_type, all_hub); 854 r = amdgpu_fence_emit_polling(ring, &seq, MAX_KIQ_REG_WAIT); 855 if (r) { 856 amdgpu_ring_undo(ring); 857 spin_unlock(&adev->gfx.kiq.ring_lock); 858 up_read(&adev->reset_sem); 859 return -ETIME; 860 } 861 862 amdgpu_ring_commit(ring); 863 spin_unlock(&adev->gfx.kiq.ring_lock); 864 r = amdgpu_fence_wait_polling(ring, seq, adev->usec_timeout); 865 if (r < 1) { 866 dev_err(adev->dev, "wait for kiq fence error: %ld.\n", r); 867 up_read(&adev->reset_sem); 868 return -ETIME; 869 } 870 up_read(&adev->reset_sem); 871 return 0; 872 } 873 874 for (vmid = 1; vmid < 16; vmid++) { 875 876 ret = gmc_v9_0_get_atc_vmid_pasid_mapping_info(adev, vmid, 877 &queried_pasid); 878 if (ret && queried_pasid == pasid) { 879 if (all_hub) { 880 for (i = 0; i < adev->num_vmhubs; i++) 881 gmc_v9_0_flush_gpu_tlb(adev, vmid, 882 i, flush_type); 883 } else { 884 gmc_v9_0_flush_gpu_tlb(adev, vmid, 885 AMDGPU_GFXHUB_0, flush_type); 886 } 887 break; 888 } 889 } 890 891 return 0; 892 893 } 894 895 static uint64_t gmc_v9_0_emit_flush_gpu_tlb(struct amdgpu_ring *ring, 896 unsigned vmid, uint64_t pd_addr) 897 { 898 bool use_semaphore = gmc_v9_0_use_invalidate_semaphore(ring->adev, ring->funcs->vmhub); 899 struct amdgpu_device *adev = ring->adev; 900 struct amdgpu_vmhub *hub = &adev->vmhub[ring->funcs->vmhub]; 901 uint32_t req = gmc_v9_0_get_invalidate_req(vmid, 0); 902 unsigned eng = ring->vm_inv_eng; 903 904 /* 905 * It may lose gpuvm invalidate acknowldege state across power-gating 906 * off cycle, add semaphore acquire before invalidation and semaphore 907 * release after invalidation to avoid entering power gated state 908 * to WA the Issue 909 */ 910 911 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 912 if (use_semaphore) 913 /* a read return value of 1 means semaphore acuqire */ 914 amdgpu_ring_emit_reg_wait(ring, 915 hub->vm_inv_eng0_sem + 916 hub->eng_distance * eng, 0x1, 0x1); 917 918 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_lo32 + 919 (hub->ctx_addr_distance * vmid), 920 lower_32_bits(pd_addr)); 921 922 amdgpu_ring_emit_wreg(ring, hub->ctx0_ptb_addr_hi32 + 923 (hub->ctx_addr_distance * vmid), 924 upper_32_bits(pd_addr)); 925 926 amdgpu_ring_emit_reg_write_reg_wait(ring, hub->vm_inv_eng0_req + 927 hub->eng_distance * eng, 928 hub->vm_inv_eng0_ack + 929 hub->eng_distance * eng, 930 req, 1 << vmid); 931 932 /* TODO: It needs to continue working on debugging with semaphore for GFXHUB as well. */ 933 if (use_semaphore) 934 /* 935 * add semaphore release after invalidation, 936 * write with 0 means semaphore release 937 */ 938 amdgpu_ring_emit_wreg(ring, hub->vm_inv_eng0_sem + 939 hub->eng_distance * eng, 0); 940 941 return pd_addr; 942 } 943 944 static void gmc_v9_0_emit_pasid_mapping(struct amdgpu_ring *ring, unsigned vmid, 945 unsigned pasid) 946 { 947 struct amdgpu_device *adev = ring->adev; 948 uint32_t reg; 949 950 /* Do nothing because there's no lut register for mmhub1. */ 951 if (ring->funcs->vmhub == AMDGPU_MMHUB_1) 952 return; 953 954 if (ring->funcs->vmhub == AMDGPU_GFXHUB_0) 955 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT) + vmid; 956 else 957 reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_VMID_0_LUT_MM) + vmid; 958 959 amdgpu_ring_emit_wreg(ring, reg, pasid); 960 } 961 962 /* 963 * PTE format on VEGA 10: 964 * 63:59 reserved 965 * 58:57 mtype 966 * 56 F 967 * 55 L 968 * 54 P 969 * 53 SW 970 * 52 T 971 * 50:48 reserved 972 * 47:12 4k physical page base address 973 * 11:7 fragment 974 * 6 write 975 * 5 read 976 * 4 exe 977 * 3 Z 978 * 2 snooped 979 * 1 system 980 * 0 valid 981 * 982 * PDE format on VEGA 10: 983 * 63:59 block fragment size 984 * 58:55 reserved 985 * 54 P 986 * 53:48 reserved 987 * 47:6 physical base address of PD or PTE 988 * 5:3 reserved 989 * 2 C 990 * 1 system 991 * 0 valid 992 */ 993 994 static uint64_t gmc_v9_0_map_mtype(struct amdgpu_device *adev, uint32_t flags) 995 996 { 997 switch (flags) { 998 case AMDGPU_VM_MTYPE_DEFAULT: 999 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1000 case AMDGPU_VM_MTYPE_NC: 1001 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1002 case AMDGPU_VM_MTYPE_WC: 1003 return AMDGPU_PTE_MTYPE_VG10(MTYPE_WC); 1004 case AMDGPU_VM_MTYPE_RW: 1005 return AMDGPU_PTE_MTYPE_VG10(MTYPE_RW); 1006 case AMDGPU_VM_MTYPE_CC: 1007 return AMDGPU_PTE_MTYPE_VG10(MTYPE_CC); 1008 case AMDGPU_VM_MTYPE_UC: 1009 return AMDGPU_PTE_MTYPE_VG10(MTYPE_UC); 1010 default: 1011 return AMDGPU_PTE_MTYPE_VG10(MTYPE_NC); 1012 } 1013 } 1014 1015 static void gmc_v9_0_get_vm_pde(struct amdgpu_device *adev, int level, 1016 uint64_t *addr, uint64_t *flags) 1017 { 1018 if (!(*flags & AMDGPU_PDE_PTE) && !(*flags & AMDGPU_PTE_SYSTEM)) 1019 *addr = adev->vm_manager.vram_base_offset + *addr - 1020 adev->gmc.vram_start; 1021 BUG_ON(*addr & 0xFFFF00000000003FULL); 1022 1023 if (!adev->gmc.translate_further) 1024 return; 1025 1026 if (level == AMDGPU_VM_PDB1) { 1027 /* Set the block fragment size */ 1028 if (!(*flags & AMDGPU_PDE_PTE)) 1029 *flags |= AMDGPU_PDE_BFS(0x9); 1030 1031 } else if (level == AMDGPU_VM_PDB0) { 1032 if (*flags & AMDGPU_PDE_PTE) 1033 *flags &= ~AMDGPU_PDE_PTE; 1034 else 1035 *flags |= AMDGPU_PTE_TF; 1036 } 1037 } 1038 1039 static void gmc_v9_0_get_vm_pte(struct amdgpu_device *adev, 1040 struct amdgpu_bo_va_mapping *mapping, 1041 uint64_t *flags) 1042 { 1043 *flags &= ~AMDGPU_PTE_EXECUTABLE; 1044 *flags |= mapping->flags & AMDGPU_PTE_EXECUTABLE; 1045 1046 *flags &= ~AMDGPU_PTE_MTYPE_VG10_MASK; 1047 *flags |= mapping->flags & AMDGPU_PTE_MTYPE_VG10_MASK; 1048 1049 if (mapping->flags & AMDGPU_PTE_PRT) { 1050 *flags |= AMDGPU_PTE_PRT; 1051 *flags &= ~AMDGPU_PTE_VALID; 1052 } 1053 1054 if (adev->asic_type == CHIP_ARCTURUS && 1055 !(*flags & AMDGPU_PTE_SYSTEM) && 1056 mapping->bo_va->is_xgmi) 1057 *flags |= AMDGPU_PTE_SNOOPED; 1058 } 1059 1060 static unsigned gmc_v9_0_get_vbios_fb_size(struct amdgpu_device *adev) 1061 { 1062 u32 d1vga_control = RREG32_SOC15(DCE, 0, mmD1VGA_CONTROL); 1063 unsigned size; 1064 1065 if (REG_GET_FIELD(d1vga_control, D1VGA_CONTROL, D1VGA_MODE_ENABLE)) { 1066 size = AMDGPU_VBIOS_VGA_ALLOCATION; 1067 } else { 1068 u32 viewport; 1069 1070 switch (adev->asic_type) { 1071 case CHIP_RAVEN: 1072 case CHIP_RENOIR: 1073 viewport = RREG32_SOC15(DCE, 0, mmHUBP0_DCSURF_PRI_VIEWPORT_DIMENSION); 1074 size = (REG_GET_FIELD(viewport, 1075 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_HEIGHT) * 1076 REG_GET_FIELD(viewport, 1077 HUBP0_DCSURF_PRI_VIEWPORT_DIMENSION, PRI_VIEWPORT_WIDTH) * 1078 4); 1079 break; 1080 case CHIP_VEGA10: 1081 case CHIP_VEGA12: 1082 case CHIP_VEGA20: 1083 default: 1084 viewport = RREG32_SOC15(DCE, 0, mmSCL0_VIEWPORT_SIZE); 1085 size = (REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_HEIGHT) * 1086 REG_GET_FIELD(viewport, SCL0_VIEWPORT_SIZE, VIEWPORT_WIDTH) * 1087 4); 1088 break; 1089 } 1090 } 1091 1092 return size; 1093 } 1094 1095 static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = { 1096 .flush_gpu_tlb = gmc_v9_0_flush_gpu_tlb, 1097 .flush_gpu_tlb_pasid = gmc_v9_0_flush_gpu_tlb_pasid, 1098 .emit_flush_gpu_tlb = gmc_v9_0_emit_flush_gpu_tlb, 1099 .emit_pasid_mapping = gmc_v9_0_emit_pasid_mapping, 1100 .map_mtype = gmc_v9_0_map_mtype, 1101 .get_vm_pde = gmc_v9_0_get_vm_pde, 1102 .get_vm_pte = gmc_v9_0_get_vm_pte, 1103 .get_vbios_fb_size = gmc_v9_0_get_vbios_fb_size, 1104 }; 1105 1106 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev) 1107 { 1108 adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs; 1109 } 1110 1111 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) 1112 { 1113 switch (adev->asic_type) { 1114 case CHIP_VEGA10: 1115 adev->umc.funcs = &umc_v6_0_funcs; 1116 break; 1117 case CHIP_VEGA20: 1118 adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 1119 adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 1120 adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 1121 adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_VG20; 1122 adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 1123 adev->umc.funcs = &umc_v6_1_funcs; 1124 break; 1125 case CHIP_ARCTURUS: 1126 adev->umc.max_ras_err_cnt_per_query = UMC_V6_1_TOTAL_CHANNEL_NUM; 1127 adev->umc.channel_inst_num = UMC_V6_1_CHANNEL_INSTANCE_NUM; 1128 adev->umc.umc_inst_num = UMC_V6_1_UMC_INSTANCE_NUM; 1129 adev->umc.channel_offs = UMC_V6_1_PER_CHANNEL_OFFSET_ARCT; 1130 adev->umc.channel_idx_tbl = &umc_v6_1_channel_idx_tbl[0][0]; 1131 adev->umc.funcs = &umc_v6_1_funcs; 1132 break; 1133 default: 1134 break; 1135 } 1136 } 1137 1138 static void gmc_v9_0_set_mmhub_funcs(struct amdgpu_device *adev) 1139 { 1140 switch (adev->asic_type) { 1141 case CHIP_ARCTURUS: 1142 adev->mmhub.funcs = &mmhub_v9_4_funcs; 1143 break; 1144 default: 1145 adev->mmhub.funcs = &mmhub_v1_0_funcs; 1146 break; 1147 } 1148 } 1149 1150 static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev) 1151 { 1152 adev->gfxhub.funcs = &gfxhub_v1_0_funcs; 1153 } 1154 1155 static int gmc_v9_0_early_init(void *handle) 1156 { 1157 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1158 1159 gmc_v9_0_set_gmc_funcs(adev); 1160 gmc_v9_0_set_irq_funcs(adev); 1161 gmc_v9_0_set_umc_funcs(adev); 1162 gmc_v9_0_set_mmhub_funcs(adev); 1163 gmc_v9_0_set_gfxhub_funcs(adev); 1164 1165 adev->gmc.shared_aperture_start = 0x2000000000000000ULL; 1166 adev->gmc.shared_aperture_end = 1167 adev->gmc.shared_aperture_start + (4ULL << 30) - 1; 1168 adev->gmc.private_aperture_start = 0x1000000000000000ULL; 1169 adev->gmc.private_aperture_end = 1170 adev->gmc.private_aperture_start + (4ULL << 30) - 1; 1171 1172 return 0; 1173 } 1174 1175 static int gmc_v9_0_late_init(void *handle) 1176 { 1177 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1178 int r; 1179 1180 amdgpu_bo_late_init(adev); 1181 1182 r = amdgpu_gmc_allocate_vm_inv_eng(adev); 1183 if (r) 1184 return r; 1185 1186 /* 1187 * Workaround performance drop issue with VBIOS enables partial 1188 * writes, while disables HBM ECC for vega10. 1189 */ 1190 if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) { 1191 if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) { 1192 if (adev->df.funcs->enable_ecc_force_par_wr_rmw) 1193 adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false); 1194 } 1195 } 1196 1197 if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count) 1198 adev->mmhub.funcs->reset_ras_error_count(adev); 1199 1200 r = amdgpu_gmc_ras_late_init(adev); 1201 if (r) 1202 return r; 1203 1204 return amdgpu_irq_get(adev, &adev->gmc.vm_fault, 0); 1205 } 1206 1207 static void gmc_v9_0_vram_gtt_location(struct amdgpu_device *adev, 1208 struct amdgpu_gmc *mc) 1209 { 1210 u64 base = 0; 1211 1212 if (!amdgpu_sriov_vf(adev)) 1213 base = adev->mmhub.funcs->get_fb_location(adev); 1214 1215 /* add the xgmi offset of the physical node */ 1216 base += adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1217 amdgpu_gmc_vram_location(adev, mc, base); 1218 amdgpu_gmc_gart_location(adev, mc); 1219 amdgpu_gmc_agp_location(adev, mc); 1220 /* base offset of vram pages */ 1221 adev->vm_manager.vram_base_offset = adev->gfxhub.funcs->get_mc_fb_offset(adev); 1222 1223 /* XXX: add the xgmi offset of the physical node? */ 1224 adev->vm_manager.vram_base_offset += 1225 adev->gmc.xgmi.physical_node_id * adev->gmc.xgmi.node_segment_size; 1226 } 1227 1228 /** 1229 * gmc_v9_0_mc_init - initialize the memory controller driver params 1230 * 1231 * @adev: amdgpu_device pointer 1232 * 1233 * Look up the amount of vram, vram width, and decide how to place 1234 * vram and gart within the GPU's physical address space. 1235 * Returns 0 for success. 1236 */ 1237 static int gmc_v9_0_mc_init(struct amdgpu_device *adev) 1238 { 1239 int r; 1240 1241 /* size in MB on si */ 1242 adev->gmc.mc_vram_size = 1243 adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL; 1244 adev->gmc.real_vram_size = adev->gmc.mc_vram_size; 1245 1246 if (!(adev->flags & AMD_IS_APU)) { 1247 r = amdgpu_device_resize_fb_bar(adev); 1248 if (r) 1249 return r; 1250 } 1251 adev->gmc.aper_base = pci_resource_start(adev->pdev, 0); 1252 adev->gmc.aper_size = pci_resource_len(adev->pdev, 0); 1253 1254 #ifdef CONFIG_X86_64 1255 if (adev->flags & AMD_IS_APU) { 1256 adev->gmc.aper_base = adev->gfxhub.funcs->get_mc_fb_offset(adev); 1257 adev->gmc.aper_size = adev->gmc.real_vram_size; 1258 } 1259 #endif 1260 /* In case the PCI BAR is larger than the actual amount of vram */ 1261 adev->gmc.visible_vram_size = adev->gmc.aper_size; 1262 if (adev->gmc.visible_vram_size > adev->gmc.real_vram_size) 1263 adev->gmc.visible_vram_size = adev->gmc.real_vram_size; 1264 1265 /* set the gart size */ 1266 if (amdgpu_gart_size == -1) { 1267 switch (adev->asic_type) { 1268 case CHIP_VEGA10: /* all engines support GPUVM */ 1269 case CHIP_VEGA12: /* all engines support GPUVM */ 1270 case CHIP_VEGA20: 1271 case CHIP_ARCTURUS: 1272 default: 1273 adev->gmc.gart_size = 512ULL << 20; 1274 break; 1275 case CHIP_RAVEN: /* DCE SG support */ 1276 case CHIP_RENOIR: 1277 adev->gmc.gart_size = 1024ULL << 20; 1278 break; 1279 } 1280 } else { 1281 adev->gmc.gart_size = (u64)amdgpu_gart_size << 20; 1282 } 1283 1284 gmc_v9_0_vram_gtt_location(adev, &adev->gmc); 1285 1286 return 0; 1287 } 1288 1289 static int gmc_v9_0_gart_init(struct amdgpu_device *adev) 1290 { 1291 int r; 1292 1293 if (adev->gart.bo) { 1294 WARN(1, "VEGA10 PCIE GART already initialized\n"); 1295 return 0; 1296 } 1297 /* Initialize common gart structure */ 1298 r = amdgpu_gart_init(adev); 1299 if (r) 1300 return r; 1301 adev->gart.table_size = adev->gart.num_gpu_pages * 8; 1302 adev->gart.gart_pte_flags = AMDGPU_PTE_MTYPE_VG10(MTYPE_UC) | 1303 AMDGPU_PTE_EXECUTABLE; 1304 return amdgpu_gart_table_vram_alloc(adev); 1305 } 1306 1307 /** 1308 * gmc_v9_0_save_registers - saves regs 1309 * 1310 * @adev: amdgpu_device pointer 1311 * 1312 * This saves potential register values that should be 1313 * restored upon resume 1314 */ 1315 static void gmc_v9_0_save_registers(struct amdgpu_device *adev) 1316 { 1317 if (adev->asic_type == CHIP_RAVEN) 1318 adev->gmc.sdpif_register = RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0); 1319 } 1320 1321 static int gmc_v9_0_sw_init(void *handle) 1322 { 1323 int r, vram_width = 0, vram_type = 0, vram_vendor = 0; 1324 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1325 1326 adev->gfxhub.funcs->init(adev); 1327 1328 adev->mmhub.funcs->init(adev); 1329 1330 spin_lock_init(&adev->gmc.invalidate_lock); 1331 1332 r = amdgpu_atomfirmware_get_vram_info(adev, 1333 &vram_width, &vram_type, &vram_vendor); 1334 if (amdgpu_sriov_vf(adev)) 1335 /* For Vega10 SR-IOV, vram_width can't be read from ATOM as RAVEN, 1336 * and DF related registers is not readable, seems hardcord is the 1337 * only way to set the correct vram_width 1338 */ 1339 adev->gmc.vram_width = 2048; 1340 else if (amdgpu_emu_mode != 1) 1341 adev->gmc.vram_width = vram_width; 1342 1343 if (!adev->gmc.vram_width) { 1344 int chansize, numchan; 1345 1346 /* hbm memory channel size */ 1347 if (adev->flags & AMD_IS_APU) 1348 chansize = 64; 1349 else 1350 chansize = 128; 1351 1352 numchan = adev->df.funcs->get_hbm_channel_number(adev); 1353 adev->gmc.vram_width = numchan * chansize; 1354 } 1355 1356 adev->gmc.vram_type = vram_type; 1357 adev->gmc.vram_vendor = vram_vendor; 1358 switch (adev->asic_type) { 1359 case CHIP_RAVEN: 1360 adev->num_vmhubs = 2; 1361 1362 if (adev->rev_id == 0x0 || adev->rev_id == 0x1) { 1363 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1364 } else { 1365 /* vm_size is 128TB + 512GB for legacy 3-level page support */ 1366 amdgpu_vm_adjust_size(adev, 128 * 1024 + 512, 9, 2, 48); 1367 adev->gmc.translate_further = 1368 adev->vm_manager.num_level > 1; 1369 } 1370 break; 1371 case CHIP_VEGA10: 1372 case CHIP_VEGA12: 1373 case CHIP_VEGA20: 1374 case CHIP_RENOIR: 1375 adev->num_vmhubs = 2; 1376 1377 1378 /* 1379 * To fulfill 4-level page support, 1380 * vm size is 256TB (48bit), maximum size of Vega10, 1381 * block size 512 (9bit) 1382 */ 1383 /* sriov restrict max_pfn below AMDGPU_GMC_HOLE */ 1384 if (amdgpu_sriov_vf(adev)) 1385 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 47); 1386 else 1387 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1388 break; 1389 case CHIP_ARCTURUS: 1390 adev->num_vmhubs = 3; 1391 1392 /* Keep the vm size same with Vega20 */ 1393 amdgpu_vm_adjust_size(adev, 256 * 1024, 9, 3, 48); 1394 break; 1395 default: 1396 break; 1397 } 1398 1399 /* This interrupt is VMC page fault.*/ 1400 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC, VMC_1_0__SRCID__VM_FAULT, 1401 &adev->gmc.vm_fault); 1402 if (r) 1403 return r; 1404 1405 if (adev->asic_type == CHIP_ARCTURUS) { 1406 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_VMC1, VMC_1_0__SRCID__VM_FAULT, 1407 &adev->gmc.vm_fault); 1408 if (r) 1409 return r; 1410 } 1411 1412 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_UTCL2, UTCL2_1_0__SRCID__FAULT, 1413 &adev->gmc.vm_fault); 1414 1415 if (r) 1416 return r; 1417 1418 if (!amdgpu_sriov_vf(adev)) { 1419 /* interrupt sent to DF. */ 1420 r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DF, 0, 1421 &adev->gmc.ecc_irq); 1422 if (r) 1423 return r; 1424 } 1425 1426 /* Set the internal MC address mask 1427 * This is the max address of the GPU's 1428 * internal address space. 1429 */ 1430 adev->gmc.mc_mask = 0xffffffffffffULL; /* 48 bit MC */ 1431 1432 r = dma_set_mask_and_coherent(adev->dev, DMA_BIT_MASK(44)); 1433 if (r) { 1434 printk(KERN_WARNING "amdgpu: No suitable DMA available.\n"); 1435 return r; 1436 } 1437 adev->need_swiotlb = drm_need_swiotlb(44); 1438 1439 if (adev->gmc.xgmi.supported) { 1440 r = adev->gfxhub.funcs->get_xgmi_info(adev); 1441 if (r) 1442 return r; 1443 } 1444 1445 r = gmc_v9_0_mc_init(adev); 1446 if (r) 1447 return r; 1448 1449 amdgpu_gmc_get_vbios_allocations(adev); 1450 1451 /* Memory manager */ 1452 r = amdgpu_bo_init(adev); 1453 if (r) 1454 return r; 1455 1456 r = gmc_v9_0_gart_init(adev); 1457 if (r) 1458 return r; 1459 1460 /* 1461 * number of VMs 1462 * VMID 0 is reserved for System 1463 * amdgpu graphics/compute will use VMIDs 1..n-1 1464 * amdkfd will use VMIDs n..15 1465 * 1466 * The first KFD VMID is 8 for GPUs with graphics, 3 for 1467 * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs 1468 * for video processing. 1469 */ 1470 adev->vm_manager.first_kfd_vmid = 1471 adev->asic_type == CHIP_ARCTURUS ? 3 : 8; 1472 1473 amdgpu_vm_manager_init(adev); 1474 1475 gmc_v9_0_save_registers(adev); 1476 1477 return 0; 1478 } 1479 1480 static int gmc_v9_0_sw_fini(void *handle) 1481 { 1482 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1483 1484 amdgpu_gmc_ras_fini(adev); 1485 amdgpu_gem_force_release(adev); 1486 amdgpu_vm_manager_fini(adev); 1487 amdgpu_gart_table_vram_free(adev); 1488 amdgpu_bo_fini(adev); 1489 amdgpu_gart_fini(adev); 1490 1491 return 0; 1492 } 1493 1494 static void gmc_v9_0_init_golden_registers(struct amdgpu_device *adev) 1495 { 1496 1497 switch (adev->asic_type) { 1498 case CHIP_VEGA10: 1499 if (amdgpu_sriov_vf(adev)) 1500 break; 1501 fallthrough; 1502 case CHIP_VEGA20: 1503 soc15_program_register_sequence(adev, 1504 golden_settings_mmhub_1_0_0, 1505 ARRAY_SIZE(golden_settings_mmhub_1_0_0)); 1506 soc15_program_register_sequence(adev, 1507 golden_settings_athub_1_0_0, 1508 ARRAY_SIZE(golden_settings_athub_1_0_0)); 1509 break; 1510 case CHIP_VEGA12: 1511 break; 1512 case CHIP_RAVEN: 1513 /* TODO for renoir */ 1514 soc15_program_register_sequence(adev, 1515 golden_settings_athub_1_0_0, 1516 ARRAY_SIZE(golden_settings_athub_1_0_0)); 1517 break; 1518 default: 1519 break; 1520 } 1521 } 1522 1523 /** 1524 * gmc_v9_0_restore_registers - restores regs 1525 * 1526 * @adev: amdgpu_device pointer 1527 * 1528 * This restores register values, saved at suspend. 1529 */ 1530 void gmc_v9_0_restore_registers(struct amdgpu_device *adev) 1531 { 1532 if (adev->asic_type == CHIP_RAVEN) { 1533 WREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0, adev->gmc.sdpif_register); 1534 WARN_ON(adev->gmc.sdpif_register != 1535 RREG32_SOC15(DCE, 0, mmDCHUBBUB_SDPIF_MMIO_CNTRL_0)); 1536 } 1537 } 1538 1539 /** 1540 * gmc_v9_0_gart_enable - gart enable 1541 * 1542 * @adev: amdgpu_device pointer 1543 */ 1544 static int gmc_v9_0_gart_enable(struct amdgpu_device *adev) 1545 { 1546 int r; 1547 1548 if (adev->gart.bo == NULL) { 1549 dev_err(adev->dev, "No VRAM object for PCIE GART.\n"); 1550 return -EINVAL; 1551 } 1552 r = amdgpu_gart_table_vram_pin(adev); 1553 if (r) 1554 return r; 1555 1556 r = adev->gfxhub.funcs->gart_enable(adev); 1557 if (r) 1558 return r; 1559 1560 r = adev->mmhub.funcs->gart_enable(adev); 1561 if (r) 1562 return r; 1563 1564 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n", 1565 (unsigned)(adev->gmc.gart_size >> 20), 1566 (unsigned long long)amdgpu_bo_gpu_offset(adev->gart.bo)); 1567 adev->gart.ready = true; 1568 return 0; 1569 } 1570 1571 static int gmc_v9_0_hw_init(void *handle) 1572 { 1573 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1574 bool value; 1575 int r, i; 1576 u32 tmp; 1577 1578 /* The sequence of these two function calls matters.*/ 1579 gmc_v9_0_init_golden_registers(adev); 1580 1581 if (adev->mode_info.num_crtc) { 1582 if (adev->asic_type != CHIP_ARCTURUS) { 1583 /* Lockout access through VGA aperture*/ 1584 WREG32_FIELD15(DCE, 0, VGA_HDP_CONTROL, VGA_MEMORY_DISABLE, 1); 1585 1586 /* disable VGA render */ 1587 WREG32_FIELD15(DCE, 0, VGA_RENDER_CONTROL, VGA_VSTATUS_CNTL, 0); 1588 } 1589 } 1590 1591 amdgpu_device_program_register_sequence(adev, 1592 golden_settings_vega10_hdp, 1593 ARRAY_SIZE(golden_settings_vega10_hdp)); 1594 1595 if (adev->mmhub.funcs->update_power_gating) 1596 adev->mmhub.funcs->update_power_gating(adev, true); 1597 1598 switch (adev->asic_type) { 1599 case CHIP_ARCTURUS: 1600 WREG32_FIELD15(HDP, 0, HDP_MMHUB_CNTL, HDP_MMHUB_GCC, 1); 1601 break; 1602 default: 1603 break; 1604 } 1605 1606 WREG32_FIELD15(HDP, 0, HDP_MISC_CNTL, FLUSH_INVALIDATE_CACHE, 1); 1607 1608 tmp = RREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL); 1609 WREG32_SOC15(HDP, 0, mmHDP_HOST_PATH_CNTL, tmp); 1610 1611 WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE, (adev->gmc.vram_start >> 8)); 1612 WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40)); 1613 1614 /* After HDP is initialized, flush HDP.*/ 1615 adev->nbio.funcs->hdp_flush(adev, NULL); 1616 1617 if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_ALWAYS) 1618 value = false; 1619 else 1620 value = true; 1621 1622 if (!amdgpu_sriov_vf(adev)) { 1623 adev->gfxhub.funcs->set_fault_enable_default(adev, value); 1624 adev->mmhub.funcs->set_fault_enable_default(adev, value); 1625 } 1626 for (i = 0; i < adev->num_vmhubs; ++i) 1627 gmc_v9_0_flush_gpu_tlb(adev, 0, i, 0); 1628 1629 if (adev->umc.funcs && adev->umc.funcs->init_registers) 1630 adev->umc.funcs->init_registers(adev); 1631 1632 r = gmc_v9_0_gart_enable(adev); 1633 1634 return r; 1635 } 1636 1637 /** 1638 * gmc_v9_0_gart_disable - gart disable 1639 * 1640 * @adev: amdgpu_device pointer 1641 * 1642 * This disables all VM page table. 1643 */ 1644 static void gmc_v9_0_gart_disable(struct amdgpu_device *adev) 1645 { 1646 adev->gfxhub.funcs->gart_disable(adev); 1647 adev->mmhub.funcs->gart_disable(adev); 1648 amdgpu_gart_table_vram_unpin(adev); 1649 } 1650 1651 static int gmc_v9_0_hw_fini(void *handle) 1652 { 1653 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1654 1655 if (amdgpu_sriov_vf(adev)) { 1656 /* full access mode, so don't touch any GMC register */ 1657 DRM_DEBUG("For SRIOV client, shouldn't do anything.\n"); 1658 return 0; 1659 } 1660 1661 amdgpu_irq_put(adev, &adev->gmc.ecc_irq, 0); 1662 amdgpu_irq_put(adev, &adev->gmc.vm_fault, 0); 1663 gmc_v9_0_gart_disable(adev); 1664 1665 return 0; 1666 } 1667 1668 static int gmc_v9_0_suspend(void *handle) 1669 { 1670 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1671 1672 return gmc_v9_0_hw_fini(adev); 1673 } 1674 1675 static int gmc_v9_0_resume(void *handle) 1676 { 1677 int r; 1678 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1679 1680 r = gmc_v9_0_hw_init(adev); 1681 if (r) 1682 return r; 1683 1684 amdgpu_vmid_reset_all(adev); 1685 1686 return 0; 1687 } 1688 1689 static bool gmc_v9_0_is_idle(void *handle) 1690 { 1691 /* MC is always ready in GMC v9.*/ 1692 return true; 1693 } 1694 1695 static int gmc_v9_0_wait_for_idle(void *handle) 1696 { 1697 /* There is no need to wait for MC idle in GMC v9.*/ 1698 return 0; 1699 } 1700 1701 static int gmc_v9_0_soft_reset(void *handle) 1702 { 1703 /* XXX for emulation.*/ 1704 return 0; 1705 } 1706 1707 static int gmc_v9_0_set_clockgating_state(void *handle, 1708 enum amd_clockgating_state state) 1709 { 1710 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1711 1712 adev->mmhub.funcs->set_clockgating(adev, state); 1713 1714 athub_v1_0_set_clockgating(adev, state); 1715 1716 return 0; 1717 } 1718 1719 static void gmc_v9_0_get_clockgating_state(void *handle, u32 *flags) 1720 { 1721 struct amdgpu_device *adev = (struct amdgpu_device *)handle; 1722 1723 adev->mmhub.funcs->get_clockgating(adev, flags); 1724 1725 athub_v1_0_get_clockgating(adev, flags); 1726 } 1727 1728 static int gmc_v9_0_set_powergating_state(void *handle, 1729 enum amd_powergating_state state) 1730 { 1731 return 0; 1732 } 1733 1734 const struct amd_ip_funcs gmc_v9_0_ip_funcs = { 1735 .name = "gmc_v9_0", 1736 .early_init = gmc_v9_0_early_init, 1737 .late_init = gmc_v9_0_late_init, 1738 .sw_init = gmc_v9_0_sw_init, 1739 .sw_fini = gmc_v9_0_sw_fini, 1740 .hw_init = gmc_v9_0_hw_init, 1741 .hw_fini = gmc_v9_0_hw_fini, 1742 .suspend = gmc_v9_0_suspend, 1743 .resume = gmc_v9_0_resume, 1744 .is_idle = gmc_v9_0_is_idle, 1745 .wait_for_idle = gmc_v9_0_wait_for_idle, 1746 .soft_reset = gmc_v9_0_soft_reset, 1747 .set_clockgating_state = gmc_v9_0_set_clockgating_state, 1748 .set_powergating_state = gmc_v9_0_set_powergating_state, 1749 .get_clockgating_state = gmc_v9_0_get_clockgating_state, 1750 }; 1751 1752 const struct amdgpu_ip_block_version gmc_v9_0_ip_block = 1753 { 1754 .type = AMD_IP_BLOCK_TYPE_GMC, 1755 .major = 9, 1756 .minor = 0, 1757 .rev = 0, 1758 .funcs = &gmc_v9_0_ip_funcs, 1759 }; 1760