1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/power_supply.h> 29 #include <linux/kthread.h> 30 #include <linux/console.h> 31 #include <linux/slab.h> 32 #include <drm/drmP.h> 33 #include <drm/drm_crtc_helper.h> 34 #include <drm/drm_atomic_helper.h> 35 #include <drm/amdgpu_drm.h> 36 #include <linux/vgaarb.h> 37 #include <linux/vga_switcheroo.h> 38 #include <linux/efi.h> 39 #include "amdgpu.h" 40 #include "amdgpu_trace.h" 41 #include "amdgpu_i2c.h" 42 #include "atom.h" 43 #include "amdgpu_atombios.h" 44 #include "amdgpu_atomfirmware.h" 45 #include "amd_pcie.h" 46 #ifdef CONFIG_DRM_AMDGPU_SI 47 #include "si.h" 48 #endif 49 #ifdef CONFIG_DRM_AMDGPU_CIK 50 #include "cik.h" 51 #endif 52 #include "vi.h" 53 #include "soc15.h" 54 #include "bif/bif_4_1_d.h" 55 #include <linux/pci.h> 56 #include <linux/firmware.h> 57 #include "amdgpu_vf_error.h" 58 59 #include "amdgpu_amdkfd.h" 60 #include "amdgpu_pm.h" 61 62 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 63 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); 64 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 65 66 #define AMDGPU_RESUME_MS 2000 67 68 static const char *amdgpu_asic_name[] = { 69 "TAHITI", 70 "PITCAIRN", 71 "VERDE", 72 "OLAND", 73 "HAINAN", 74 "BONAIRE", 75 "KAVERI", 76 "KABINI", 77 "HAWAII", 78 "MULLINS", 79 "TOPAZ", 80 "TONGA", 81 "FIJI", 82 "CARRIZO", 83 "STONEY", 84 "POLARIS10", 85 "POLARIS11", 86 "POLARIS12", 87 "VEGAM", 88 "VEGA10", 89 "VEGA12", 90 "VEGA20", 91 "RAVEN", 92 "LAST", 93 }; 94 95 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); 96 97 /** 98 * amdgpu_device_is_px - Is the device is a dGPU with HG/PX power control 99 * 100 * @dev: drm_device pointer 101 * 102 * Returns true if the device is a dGPU with HG/PX power control, 103 * otherwise return false. 104 */ 105 bool amdgpu_device_is_px(struct drm_device *dev) 106 { 107 struct amdgpu_device *adev = dev->dev_private; 108 109 if (adev->flags & AMD_IS_PX) 110 return true; 111 return false; 112 } 113 114 /* 115 * MMIO register access helper functions. 116 */ 117 /** 118 * amdgpu_mm_rreg - read a memory mapped IO register 119 * 120 * @adev: amdgpu_device pointer 121 * @reg: dword aligned register offset 122 * @acc_flags: access flags which require special behavior 123 * 124 * Returns the 32 bit value from the offset specified. 125 */ 126 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg, 127 uint32_t acc_flags) 128 { 129 uint32_t ret; 130 131 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) 132 return amdgpu_virt_kiq_rreg(adev, reg); 133 134 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) 135 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 136 else { 137 unsigned long flags; 138 139 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 140 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); 141 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); 142 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 143 } 144 trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret); 145 return ret; 146 } 147 148 /* 149 * MMIO register read with bytes helper functions 150 * @offset:bytes offset from MMIO start 151 * 152 */ 153 154 /** 155 * amdgpu_mm_rreg8 - read a memory mapped IO register 156 * 157 * @adev: amdgpu_device pointer 158 * @offset: byte aligned register offset 159 * 160 * Returns the 8 bit value from the offset specified. 161 */ 162 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) { 163 if (offset < adev->rmmio_size) 164 return (readb(adev->rmmio + offset)); 165 BUG(); 166 } 167 168 /* 169 * MMIO register write with bytes helper functions 170 * @offset:bytes offset from MMIO start 171 * @value: the value want to be written to the register 172 * 173 */ 174 /** 175 * amdgpu_mm_wreg8 - read a memory mapped IO register 176 * 177 * @adev: amdgpu_device pointer 178 * @offset: byte aligned register offset 179 * @value: 8 bit value to write 180 * 181 * Writes the value specified to the offset specified. 182 */ 183 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) { 184 if (offset < adev->rmmio_size) 185 writeb(value, adev->rmmio + offset); 186 else 187 BUG(); 188 } 189 190 /** 191 * amdgpu_mm_wreg - write to a memory mapped IO register 192 * 193 * @adev: amdgpu_device pointer 194 * @reg: dword aligned register offset 195 * @v: 32 bit value to write to the register 196 * @acc_flags: access flags which require special behavior 197 * 198 * Writes the value specified to the offset specified. 199 */ 200 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v, 201 uint32_t acc_flags) 202 { 203 trace_amdgpu_mm_wreg(adev->pdev->device, reg, v); 204 205 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { 206 adev->last_mm_index = v; 207 } 208 209 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && amdgpu_sriov_runtime(adev)) 210 return amdgpu_virt_kiq_wreg(adev, reg, v); 211 212 if ((reg * 4) < adev->rmmio_size && !(acc_flags & AMDGPU_REGS_IDX)) 213 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 214 else { 215 unsigned long flags; 216 217 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 218 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4)); 219 writel(v, ((void __iomem *)adev->rmmio) + (mmMM_DATA * 4)); 220 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 221 } 222 223 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) { 224 udelay(500); 225 } 226 } 227 228 /** 229 * amdgpu_io_rreg - read an IO register 230 * 231 * @adev: amdgpu_device pointer 232 * @reg: dword aligned register offset 233 * 234 * Returns the 32 bit value from the offset specified. 235 */ 236 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg) 237 { 238 if ((reg * 4) < adev->rio_mem_size) 239 return ioread32(adev->rio_mem + (reg * 4)); 240 else { 241 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); 242 return ioread32(adev->rio_mem + (mmMM_DATA * 4)); 243 } 244 } 245 246 /** 247 * amdgpu_io_wreg - write to an IO register 248 * 249 * @adev: amdgpu_device pointer 250 * @reg: dword aligned register offset 251 * @v: 32 bit value to write to the register 252 * 253 * Writes the value specified to the offset specified. 254 */ 255 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 256 { 257 if (adev->asic_type >= CHIP_VEGA10 && reg == 0) { 258 adev->last_mm_index = v; 259 } 260 261 if ((reg * 4) < adev->rio_mem_size) 262 iowrite32(v, adev->rio_mem + (reg * 4)); 263 else { 264 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); 265 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4)); 266 } 267 268 if (adev->asic_type >= CHIP_VEGA10 && reg == 1 && adev->last_mm_index == 0x5702C) { 269 udelay(500); 270 } 271 } 272 273 /** 274 * amdgpu_mm_rdoorbell - read a doorbell dword 275 * 276 * @adev: amdgpu_device pointer 277 * @index: doorbell index 278 * 279 * Returns the value in the doorbell aperture at the 280 * requested doorbell index (CIK). 281 */ 282 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) 283 { 284 if (index < adev->doorbell.num_doorbells) { 285 return readl(adev->doorbell.ptr + index); 286 } else { 287 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 288 return 0; 289 } 290 } 291 292 /** 293 * amdgpu_mm_wdoorbell - write a doorbell dword 294 * 295 * @adev: amdgpu_device pointer 296 * @index: doorbell index 297 * @v: value to write 298 * 299 * Writes @v to the doorbell aperture at the 300 * requested doorbell index (CIK). 301 */ 302 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) 303 { 304 if (index < adev->doorbell.num_doorbells) { 305 writel(v, adev->doorbell.ptr + index); 306 } else { 307 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 308 } 309 } 310 311 /** 312 * amdgpu_mm_rdoorbell64 - read a doorbell Qword 313 * 314 * @adev: amdgpu_device pointer 315 * @index: doorbell index 316 * 317 * Returns the value in the doorbell aperture at the 318 * requested doorbell index (VEGA10+). 319 */ 320 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) 321 { 322 if (index < adev->doorbell.num_doorbells) { 323 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index)); 324 } else { 325 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 326 return 0; 327 } 328 } 329 330 /** 331 * amdgpu_mm_wdoorbell64 - write a doorbell Qword 332 * 333 * @adev: amdgpu_device pointer 334 * @index: doorbell index 335 * @v: value to write 336 * 337 * Writes @v to the doorbell aperture at the 338 * requested doorbell index (VEGA10+). 339 */ 340 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) 341 { 342 if (index < adev->doorbell.num_doorbells) { 343 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v); 344 } else { 345 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 346 } 347 } 348 349 /** 350 * amdgpu_invalid_rreg - dummy reg read function 351 * 352 * @adev: amdgpu device pointer 353 * @reg: offset of register 354 * 355 * Dummy register read function. Used for register blocks 356 * that certain asics don't have (all asics). 357 * Returns the value in the register. 358 */ 359 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) 360 { 361 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg); 362 BUG(); 363 return 0; 364 } 365 366 /** 367 * amdgpu_invalid_wreg - dummy reg write function 368 * 369 * @adev: amdgpu device pointer 370 * @reg: offset of register 371 * @v: value to write to the register 372 * 373 * Dummy register read function. Used for register blocks 374 * that certain asics don't have (all asics). 375 */ 376 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 377 { 378 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n", 379 reg, v); 380 BUG(); 381 } 382 383 /** 384 * amdgpu_block_invalid_rreg - dummy reg read function 385 * 386 * @adev: amdgpu device pointer 387 * @block: offset of instance 388 * @reg: offset of register 389 * 390 * Dummy register read function. Used for register blocks 391 * that certain asics don't have (all asics). 392 * Returns the value in the register. 393 */ 394 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, 395 uint32_t block, uint32_t reg) 396 { 397 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n", 398 reg, block); 399 BUG(); 400 return 0; 401 } 402 403 /** 404 * amdgpu_block_invalid_wreg - dummy reg write function 405 * 406 * @adev: amdgpu device pointer 407 * @block: offset of instance 408 * @reg: offset of register 409 * @v: value to write to the register 410 * 411 * Dummy register read function. Used for register blocks 412 * that certain asics don't have (all asics). 413 */ 414 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, 415 uint32_t block, 416 uint32_t reg, uint32_t v) 417 { 418 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n", 419 reg, block, v); 420 BUG(); 421 } 422 423 /** 424 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page 425 * 426 * @adev: amdgpu device pointer 427 * 428 * Allocates a scratch page of VRAM for use by various things in the 429 * driver. 430 */ 431 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) 432 { 433 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, 434 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 435 &adev->vram_scratch.robj, 436 &adev->vram_scratch.gpu_addr, 437 (void **)&adev->vram_scratch.ptr); 438 } 439 440 /** 441 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page 442 * 443 * @adev: amdgpu device pointer 444 * 445 * Frees the VRAM scratch page. 446 */ 447 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev) 448 { 449 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL); 450 } 451 452 /** 453 * amdgpu_device_program_register_sequence - program an array of registers. 454 * 455 * @adev: amdgpu_device pointer 456 * @registers: pointer to the register array 457 * @array_size: size of the register array 458 * 459 * Programs an array or registers with and and or masks. 460 * This is a helper for setting golden registers. 461 */ 462 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, 463 const u32 *registers, 464 const u32 array_size) 465 { 466 u32 tmp, reg, and_mask, or_mask; 467 int i; 468 469 if (array_size % 3) 470 return; 471 472 for (i = 0; i < array_size; i +=3) { 473 reg = registers[i + 0]; 474 and_mask = registers[i + 1]; 475 or_mask = registers[i + 2]; 476 477 if (and_mask == 0xffffffff) { 478 tmp = or_mask; 479 } else { 480 tmp = RREG32(reg); 481 tmp &= ~and_mask; 482 tmp |= or_mask; 483 } 484 WREG32(reg, tmp); 485 } 486 } 487 488 /** 489 * amdgpu_device_pci_config_reset - reset the GPU 490 * 491 * @adev: amdgpu_device pointer 492 * 493 * Resets the GPU using the pci config reset sequence. 494 * Only applicable to asics prior to vega10. 495 */ 496 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) 497 { 498 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); 499 } 500 501 /* 502 * GPU doorbell aperture helpers function. 503 */ 504 /** 505 * amdgpu_device_doorbell_init - Init doorbell driver information. 506 * 507 * @adev: amdgpu_device pointer 508 * 509 * Init doorbell driver information (CIK) 510 * Returns 0 on success, error on failure. 511 */ 512 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev) 513 { 514 /* No doorbell on SI hardware generation */ 515 if (adev->asic_type < CHIP_BONAIRE) { 516 adev->doorbell.base = 0; 517 adev->doorbell.size = 0; 518 adev->doorbell.num_doorbells = 0; 519 adev->doorbell.ptr = NULL; 520 return 0; 521 } 522 523 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) 524 return -EINVAL; 525 526 /* doorbell bar mapping */ 527 adev->doorbell.base = pci_resource_start(adev->pdev, 2); 528 adev->doorbell.size = pci_resource_len(adev->pdev, 2); 529 530 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32), 531 AMDGPU_DOORBELL_MAX_ASSIGNMENT+1); 532 if (adev->doorbell.num_doorbells == 0) 533 return -EINVAL; 534 535 adev->doorbell.ptr = ioremap(adev->doorbell.base, 536 adev->doorbell.num_doorbells * 537 sizeof(u32)); 538 if (adev->doorbell.ptr == NULL) 539 return -ENOMEM; 540 541 return 0; 542 } 543 544 /** 545 * amdgpu_device_doorbell_fini - Tear down doorbell driver information. 546 * 547 * @adev: amdgpu_device pointer 548 * 549 * Tear down doorbell driver information (CIK) 550 */ 551 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev) 552 { 553 iounmap(adev->doorbell.ptr); 554 adev->doorbell.ptr = NULL; 555 } 556 557 558 559 /* 560 * amdgpu_device_wb_*() 561 * Writeback is the method by which the GPU updates special pages in memory 562 * with the status of certain GPU events (fences, ring pointers,etc.). 563 */ 564 565 /** 566 * amdgpu_device_wb_fini - Disable Writeback and free memory 567 * 568 * @adev: amdgpu_device pointer 569 * 570 * Disables Writeback and frees the Writeback memory (all asics). 571 * Used at driver shutdown. 572 */ 573 static void amdgpu_device_wb_fini(struct amdgpu_device *adev) 574 { 575 if (adev->wb.wb_obj) { 576 amdgpu_bo_free_kernel(&adev->wb.wb_obj, 577 &adev->wb.gpu_addr, 578 (void **)&adev->wb.wb); 579 adev->wb.wb_obj = NULL; 580 } 581 } 582 583 /** 584 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory 585 * 586 * @adev: amdgpu_device pointer 587 * 588 * Initializes writeback and allocates writeback memory (all asics). 589 * Used at driver startup. 590 * Returns 0 on success or an -error on failure. 591 */ 592 static int amdgpu_device_wb_init(struct amdgpu_device *adev) 593 { 594 int r; 595 596 if (adev->wb.wb_obj == NULL) { 597 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */ 598 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, 599 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 600 &adev->wb.wb_obj, &adev->wb.gpu_addr, 601 (void **)&adev->wb.wb); 602 if (r) { 603 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); 604 return r; 605 } 606 607 adev->wb.num_wb = AMDGPU_MAX_WB; 608 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); 609 610 /* clear wb memory */ 611 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); 612 } 613 614 return 0; 615 } 616 617 /** 618 * amdgpu_device_wb_get - Allocate a wb entry 619 * 620 * @adev: amdgpu_device pointer 621 * @wb: wb index 622 * 623 * Allocate a wb slot for use by the driver (all asics). 624 * Returns 0 on success or -EINVAL on failure. 625 */ 626 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) 627 { 628 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); 629 630 if (offset < adev->wb.num_wb) { 631 __set_bit(offset, adev->wb.used); 632 *wb = offset << 3; /* convert to dw offset */ 633 return 0; 634 } else { 635 return -EINVAL; 636 } 637 } 638 639 /** 640 * amdgpu_device_wb_free - Free a wb entry 641 * 642 * @adev: amdgpu_device pointer 643 * @wb: wb index 644 * 645 * Free a wb slot allocated for use by the driver (all asics) 646 */ 647 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) 648 { 649 wb >>= 3; 650 if (wb < adev->wb.num_wb) 651 __clear_bit(wb, adev->wb.used); 652 } 653 654 /** 655 * amdgpu_device_vram_location - try to find VRAM location 656 * 657 * @adev: amdgpu device structure holding all necessary informations 658 * @mc: memory controller structure holding memory informations 659 * @base: base address at which to put VRAM 660 * 661 * Function will try to place VRAM at base address provided 662 * as parameter. 663 */ 664 void amdgpu_device_vram_location(struct amdgpu_device *adev, 665 struct amdgpu_gmc *mc, u64 base) 666 { 667 uint64_t limit = (uint64_t)amdgpu_vram_limit << 20; 668 669 mc->vram_start = base; 670 mc->vram_end = mc->vram_start + mc->mc_vram_size - 1; 671 if (limit && limit < mc->real_vram_size) 672 mc->real_vram_size = limit; 673 dev_info(adev->dev, "VRAM: %lluM 0x%016llX - 0x%016llX (%lluM used)\n", 674 mc->mc_vram_size >> 20, mc->vram_start, 675 mc->vram_end, mc->real_vram_size >> 20); 676 } 677 678 /** 679 * amdgpu_device_gart_location - try to find GART location 680 * 681 * @adev: amdgpu device structure holding all necessary informations 682 * @mc: memory controller structure holding memory informations 683 * 684 * Function will place try to place GART before or after VRAM. 685 * 686 * If GART size is bigger than space left then we ajust GART size. 687 * Thus function will never fails. 688 */ 689 void amdgpu_device_gart_location(struct amdgpu_device *adev, 690 struct amdgpu_gmc *mc) 691 { 692 u64 size_af, size_bf; 693 694 mc->gart_size += adev->pm.smu_prv_buffer_size; 695 696 size_af = adev->gmc.mc_mask - mc->vram_end; 697 size_bf = mc->vram_start; 698 if (size_bf > size_af) { 699 if (mc->gart_size > size_bf) { 700 dev_warn(adev->dev, "limiting GART\n"); 701 mc->gart_size = size_bf; 702 } 703 mc->gart_start = 0; 704 } else { 705 if (mc->gart_size > size_af) { 706 dev_warn(adev->dev, "limiting GART\n"); 707 mc->gart_size = size_af; 708 } 709 /* VCE doesn't like it when BOs cross a 4GB segment, so align 710 * the GART base on a 4GB boundary as well. 711 */ 712 mc->gart_start = ALIGN(mc->vram_end + 1, 0x100000000ULL); 713 } 714 mc->gart_end = mc->gart_start + mc->gart_size - 1; 715 dev_info(adev->dev, "GART: %lluM 0x%016llX - 0x%016llX\n", 716 mc->gart_size >> 20, mc->gart_start, mc->gart_end); 717 } 718 719 /** 720 * amdgpu_device_resize_fb_bar - try to resize FB BAR 721 * 722 * @adev: amdgpu_device pointer 723 * 724 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not 725 * to fail, but if any of the BARs is not accessible after the size we abort 726 * driver loading by returning -ENODEV. 727 */ 728 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) 729 { 730 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size); 731 u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1; 732 struct pci_bus *root; 733 struct resource *res; 734 unsigned i; 735 u16 cmd; 736 int r; 737 738 /* Bypass for VF */ 739 if (amdgpu_sriov_vf(adev)) 740 return 0; 741 742 /* Check if the root BUS has 64bit memory resources */ 743 root = adev->pdev->bus; 744 while (root->parent) 745 root = root->parent; 746 747 pci_bus_for_each_resource(root, res, i) { 748 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 749 res->start > 0x100000000ull) 750 break; 751 } 752 753 /* Trying to resize is pointless without a root hub window above 4GB */ 754 if (!res) 755 return 0; 756 757 /* Disable memory decoding while we change the BAR addresses and size */ 758 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); 759 pci_write_config_word(adev->pdev, PCI_COMMAND, 760 cmd & ~PCI_COMMAND_MEMORY); 761 762 /* Free the VRAM and doorbell BAR, we most likely need to move both. */ 763 amdgpu_device_doorbell_fini(adev); 764 if (adev->asic_type >= CHIP_BONAIRE) 765 pci_release_resource(adev->pdev, 2); 766 767 pci_release_resource(adev->pdev, 0); 768 769 r = pci_resize_resource(adev->pdev, 0, rbar_size); 770 if (r == -ENOSPC) 771 DRM_INFO("Not enough PCI address space for a large BAR."); 772 else if (r && r != -ENOTSUPP) 773 DRM_ERROR("Problem resizing BAR0 (%d).", r); 774 775 pci_assign_unassigned_bus_resources(adev->pdev->bus); 776 777 /* When the doorbell or fb BAR isn't available we have no chance of 778 * using the device. 779 */ 780 r = amdgpu_device_doorbell_init(adev); 781 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) 782 return -ENODEV; 783 784 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); 785 786 return 0; 787 } 788 789 /* 790 * GPU helpers function. 791 */ 792 /** 793 * amdgpu_device_need_post - check if the hw need post or not 794 * 795 * @adev: amdgpu_device pointer 796 * 797 * Check if the asic has been initialized (all asics) at driver startup 798 * or post is needed if hw reset is performed. 799 * Returns true if need or false if not. 800 */ 801 bool amdgpu_device_need_post(struct amdgpu_device *adev) 802 { 803 uint32_t reg; 804 805 if (amdgpu_sriov_vf(adev)) 806 return false; 807 808 if (amdgpu_passthrough(adev)) { 809 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot 810 * some old smc fw still need driver do vPost otherwise gpu hang, while 811 * those smc fw version above 22.15 doesn't have this flaw, so we force 812 * vpost executed for smc version below 22.15 813 */ 814 if (adev->asic_type == CHIP_FIJI) { 815 int err; 816 uint32_t fw_ver; 817 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); 818 /* force vPost if error occured */ 819 if (err) 820 return true; 821 822 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); 823 if (fw_ver < 0x00160e00) 824 return true; 825 } 826 } 827 828 if (adev->has_hw_reset) { 829 adev->has_hw_reset = false; 830 return true; 831 } 832 833 /* bios scratch used on CIK+ */ 834 if (adev->asic_type >= CHIP_BONAIRE) 835 return amdgpu_atombios_scratch_need_asic_init(adev); 836 837 /* check MEM_SIZE for older asics */ 838 reg = amdgpu_asic_get_config_memsize(adev); 839 840 if ((reg != 0) && (reg != 0xffffffff)) 841 return false; 842 843 return true; 844 } 845 846 /* if we get transitioned to only one device, take VGA back */ 847 /** 848 * amdgpu_device_vga_set_decode - enable/disable vga decode 849 * 850 * @cookie: amdgpu_device pointer 851 * @state: enable/disable vga decode 852 * 853 * Enable/disable vga decode (all asics). 854 * Returns VGA resource flags. 855 */ 856 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state) 857 { 858 struct amdgpu_device *adev = cookie; 859 amdgpu_asic_set_vga_state(adev, state); 860 if (state) 861 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 862 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 863 else 864 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 865 } 866 867 /** 868 * amdgpu_device_check_block_size - validate the vm block size 869 * 870 * @adev: amdgpu_device pointer 871 * 872 * Validates the vm block size specified via module parameter. 873 * The vm block size defines number of bits in page table versus page directory, 874 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 875 * page table and the remaining bits are in the page directory. 876 */ 877 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) 878 { 879 /* defines number of bits in page table versus page directory, 880 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 881 * page table and the remaining bits are in the page directory */ 882 if (amdgpu_vm_block_size == -1) 883 return; 884 885 if (amdgpu_vm_block_size < 9) { 886 dev_warn(adev->dev, "VM page table size (%d) too small\n", 887 amdgpu_vm_block_size); 888 amdgpu_vm_block_size = -1; 889 } 890 } 891 892 /** 893 * amdgpu_device_check_vm_size - validate the vm size 894 * 895 * @adev: amdgpu_device pointer 896 * 897 * Validates the vm size in GB specified via module parameter. 898 * The VM size is the size of the GPU virtual memory space in GB. 899 */ 900 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) 901 { 902 /* no need to check the default value */ 903 if (amdgpu_vm_size == -1) 904 return; 905 906 if (amdgpu_vm_size < 1) { 907 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 908 amdgpu_vm_size); 909 amdgpu_vm_size = -1; 910 } 911 } 912 913 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) 914 { 915 struct sysinfo si; 916 bool is_os_64 = (sizeof(void *) == 8) ? true : false; 917 uint64_t total_memory; 918 uint64_t dram_size_seven_GB = 0x1B8000000; 919 uint64_t dram_size_three_GB = 0xB8000000; 920 921 if (amdgpu_smu_memory_pool_size == 0) 922 return; 923 924 if (!is_os_64) { 925 DRM_WARN("Not 64-bit OS, feature not supported\n"); 926 goto def_value; 927 } 928 si_meminfo(&si); 929 total_memory = (uint64_t)si.totalram * si.mem_unit; 930 931 if ((amdgpu_smu_memory_pool_size == 1) || 932 (amdgpu_smu_memory_pool_size == 2)) { 933 if (total_memory < dram_size_three_GB) 934 goto def_value1; 935 } else if ((amdgpu_smu_memory_pool_size == 4) || 936 (amdgpu_smu_memory_pool_size == 8)) { 937 if (total_memory < dram_size_seven_GB) 938 goto def_value1; 939 } else { 940 DRM_WARN("Smu memory pool size not supported\n"); 941 goto def_value; 942 } 943 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; 944 945 return; 946 947 def_value1: 948 DRM_WARN("No enough system memory\n"); 949 def_value: 950 adev->pm.smu_prv_buffer_size = 0; 951 } 952 953 /** 954 * amdgpu_device_check_arguments - validate module params 955 * 956 * @adev: amdgpu_device pointer 957 * 958 * Validates certain module parameters and updates 959 * the associated values used by the driver (all asics). 960 */ 961 static void amdgpu_device_check_arguments(struct amdgpu_device *adev) 962 { 963 if (amdgpu_sched_jobs < 4) { 964 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", 965 amdgpu_sched_jobs); 966 amdgpu_sched_jobs = 4; 967 } else if (!is_power_of_2(amdgpu_sched_jobs)){ 968 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", 969 amdgpu_sched_jobs); 970 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); 971 } 972 973 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { 974 /* gart size must be greater or equal to 32M */ 975 dev_warn(adev->dev, "gart size (%d) too small\n", 976 amdgpu_gart_size); 977 amdgpu_gart_size = -1; 978 } 979 980 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { 981 /* gtt size must be greater or equal to 32M */ 982 dev_warn(adev->dev, "gtt size (%d) too small\n", 983 amdgpu_gtt_size); 984 amdgpu_gtt_size = -1; 985 } 986 987 /* valid range is between 4 and 9 inclusive */ 988 if (amdgpu_vm_fragment_size != -1 && 989 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { 990 dev_warn(adev->dev, "valid range is between 4 and 9\n"); 991 amdgpu_vm_fragment_size = -1; 992 } 993 994 amdgpu_device_check_smu_prv_buffer_size(adev); 995 996 amdgpu_device_check_vm_size(adev); 997 998 amdgpu_device_check_block_size(adev); 999 1000 if (amdgpu_vram_page_split != -1 && (amdgpu_vram_page_split < 16 || 1001 !is_power_of_2(amdgpu_vram_page_split))) { 1002 dev_warn(adev->dev, "invalid VRAM page split (%d)\n", 1003 amdgpu_vram_page_split); 1004 amdgpu_vram_page_split = 1024; 1005 } 1006 1007 if (amdgpu_lockup_timeout == 0) { 1008 dev_warn(adev->dev, "lockup_timeout msut be > 0, adjusting to 10000\n"); 1009 amdgpu_lockup_timeout = 10000; 1010 } 1011 1012 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); 1013 } 1014 1015 /** 1016 * amdgpu_switcheroo_set_state - set switcheroo state 1017 * 1018 * @pdev: pci dev pointer 1019 * @state: vga_switcheroo state 1020 * 1021 * Callback for the switcheroo driver. Suspends or resumes the 1022 * the asics before or after it is powered up using ACPI methods. 1023 */ 1024 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state) 1025 { 1026 struct drm_device *dev = pci_get_drvdata(pdev); 1027 1028 if (amdgpu_device_is_px(dev) && state == VGA_SWITCHEROO_OFF) 1029 return; 1030 1031 if (state == VGA_SWITCHEROO_ON) { 1032 pr_info("amdgpu: switched on\n"); 1033 /* don't suspend or resume card normally */ 1034 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1035 1036 amdgpu_device_resume(dev, true, true); 1037 1038 dev->switch_power_state = DRM_SWITCH_POWER_ON; 1039 drm_kms_helper_poll_enable(dev); 1040 } else { 1041 pr_info("amdgpu: switched off\n"); 1042 drm_kms_helper_poll_disable(dev); 1043 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1044 amdgpu_device_suspend(dev, true, true); 1045 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 1046 } 1047 } 1048 1049 /** 1050 * amdgpu_switcheroo_can_switch - see if switcheroo state can change 1051 * 1052 * @pdev: pci dev pointer 1053 * 1054 * Callback for the switcheroo driver. Check of the switcheroo 1055 * state can be changed. 1056 * Returns true if the state can be changed, false if not. 1057 */ 1058 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) 1059 { 1060 struct drm_device *dev = pci_get_drvdata(pdev); 1061 1062 /* 1063 * FIXME: open_count is protected by drm_global_mutex but that would lead to 1064 * locking inversion with the driver load path. And the access here is 1065 * completely racy anyway. So don't bother with locking for now. 1066 */ 1067 return dev->open_count == 0; 1068 } 1069 1070 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { 1071 .set_gpu_state = amdgpu_switcheroo_set_state, 1072 .reprobe = NULL, 1073 .can_switch = amdgpu_switcheroo_can_switch, 1074 }; 1075 1076 /** 1077 * amdgpu_device_ip_set_clockgating_state - set the CG state 1078 * 1079 * @dev: amdgpu_device pointer 1080 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1081 * @state: clockgating state (gate or ungate) 1082 * 1083 * Sets the requested clockgating state for all instances of 1084 * the hardware IP specified. 1085 * Returns the error code from the last instance. 1086 */ 1087 int amdgpu_device_ip_set_clockgating_state(void *dev, 1088 enum amd_ip_block_type block_type, 1089 enum amd_clockgating_state state) 1090 { 1091 struct amdgpu_device *adev = dev; 1092 int i, r = 0; 1093 1094 for (i = 0; i < adev->num_ip_blocks; i++) { 1095 if (!adev->ip_blocks[i].status.valid) 1096 continue; 1097 if (adev->ip_blocks[i].version->type != block_type) 1098 continue; 1099 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) 1100 continue; 1101 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( 1102 (void *)adev, state); 1103 if (r) 1104 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n", 1105 adev->ip_blocks[i].version->funcs->name, r); 1106 } 1107 return r; 1108 } 1109 1110 /** 1111 * amdgpu_device_ip_set_powergating_state - set the PG state 1112 * 1113 * @dev: amdgpu_device pointer 1114 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1115 * @state: powergating state (gate or ungate) 1116 * 1117 * Sets the requested powergating state for all instances of 1118 * the hardware IP specified. 1119 * Returns the error code from the last instance. 1120 */ 1121 int amdgpu_device_ip_set_powergating_state(void *dev, 1122 enum amd_ip_block_type block_type, 1123 enum amd_powergating_state state) 1124 { 1125 struct amdgpu_device *adev = dev; 1126 int i, r = 0; 1127 1128 for (i = 0; i < adev->num_ip_blocks; i++) { 1129 if (!adev->ip_blocks[i].status.valid) 1130 continue; 1131 if (adev->ip_blocks[i].version->type != block_type) 1132 continue; 1133 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) 1134 continue; 1135 r = adev->ip_blocks[i].version->funcs->set_powergating_state( 1136 (void *)adev, state); 1137 if (r) 1138 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n", 1139 adev->ip_blocks[i].version->funcs->name, r); 1140 } 1141 return r; 1142 } 1143 1144 /** 1145 * amdgpu_device_ip_get_clockgating_state - get the CG state 1146 * 1147 * @adev: amdgpu_device pointer 1148 * @flags: clockgating feature flags 1149 * 1150 * Walks the list of IPs on the device and updates the clockgating 1151 * flags for each IP. 1152 * Updates @flags with the feature flags for each hardware IP where 1153 * clockgating is enabled. 1154 */ 1155 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, 1156 u32 *flags) 1157 { 1158 int i; 1159 1160 for (i = 0; i < adev->num_ip_blocks; i++) { 1161 if (!adev->ip_blocks[i].status.valid) 1162 continue; 1163 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) 1164 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); 1165 } 1166 } 1167 1168 /** 1169 * amdgpu_device_ip_wait_for_idle - wait for idle 1170 * 1171 * @adev: amdgpu_device pointer 1172 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1173 * 1174 * Waits for the request hardware IP to be idle. 1175 * Returns 0 for success or a negative error code on failure. 1176 */ 1177 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, 1178 enum amd_ip_block_type block_type) 1179 { 1180 int i, r; 1181 1182 for (i = 0; i < adev->num_ip_blocks; i++) { 1183 if (!adev->ip_blocks[i].status.valid) 1184 continue; 1185 if (adev->ip_blocks[i].version->type == block_type) { 1186 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); 1187 if (r) 1188 return r; 1189 break; 1190 } 1191 } 1192 return 0; 1193 1194 } 1195 1196 /** 1197 * amdgpu_device_ip_is_idle - is the hardware IP idle 1198 * 1199 * @adev: amdgpu_device pointer 1200 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1201 * 1202 * Check if the hardware IP is idle or not. 1203 * Returns true if it the IP is idle, false if not. 1204 */ 1205 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, 1206 enum amd_ip_block_type block_type) 1207 { 1208 int i; 1209 1210 for (i = 0; i < adev->num_ip_blocks; i++) { 1211 if (!adev->ip_blocks[i].status.valid) 1212 continue; 1213 if (adev->ip_blocks[i].version->type == block_type) 1214 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); 1215 } 1216 return true; 1217 1218 } 1219 1220 /** 1221 * amdgpu_device_ip_get_ip_block - get a hw IP pointer 1222 * 1223 * @adev: amdgpu_device pointer 1224 * @type: Type of hardware IP (SMU, GFX, UVD, etc.) 1225 * 1226 * Returns a pointer to the hardware IP block structure 1227 * if it exists for the asic, otherwise NULL. 1228 */ 1229 struct amdgpu_ip_block * 1230 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, 1231 enum amd_ip_block_type type) 1232 { 1233 int i; 1234 1235 for (i = 0; i < adev->num_ip_blocks; i++) 1236 if (adev->ip_blocks[i].version->type == type) 1237 return &adev->ip_blocks[i]; 1238 1239 return NULL; 1240 } 1241 1242 /** 1243 * amdgpu_device_ip_block_version_cmp 1244 * 1245 * @adev: amdgpu_device pointer 1246 * @type: enum amd_ip_block_type 1247 * @major: major version 1248 * @minor: minor version 1249 * 1250 * return 0 if equal or greater 1251 * return 1 if smaller or the ip_block doesn't exist 1252 */ 1253 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, 1254 enum amd_ip_block_type type, 1255 u32 major, u32 minor) 1256 { 1257 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); 1258 1259 if (ip_block && ((ip_block->version->major > major) || 1260 ((ip_block->version->major == major) && 1261 (ip_block->version->minor >= minor)))) 1262 return 0; 1263 1264 return 1; 1265 } 1266 1267 /** 1268 * amdgpu_device_ip_block_add 1269 * 1270 * @adev: amdgpu_device pointer 1271 * @ip_block_version: pointer to the IP to add 1272 * 1273 * Adds the IP block driver information to the collection of IPs 1274 * on the asic. 1275 */ 1276 int amdgpu_device_ip_block_add(struct amdgpu_device *adev, 1277 const struct amdgpu_ip_block_version *ip_block_version) 1278 { 1279 if (!ip_block_version) 1280 return -EINVAL; 1281 1282 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, 1283 ip_block_version->funcs->name); 1284 1285 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; 1286 1287 return 0; 1288 } 1289 1290 /** 1291 * amdgpu_device_enable_virtual_display - enable virtual display feature 1292 * 1293 * @adev: amdgpu_device pointer 1294 * 1295 * Enabled the virtual display feature if the user has enabled it via 1296 * the module parameter virtual_display. This feature provides a virtual 1297 * display hardware on headless boards or in virtualized environments. 1298 * This function parses and validates the configuration string specified by 1299 * the user and configues the virtual display configuration (number of 1300 * virtual connectors, crtcs, etc.) specified. 1301 */ 1302 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) 1303 { 1304 adev->enable_virtual_display = false; 1305 1306 if (amdgpu_virtual_display) { 1307 struct drm_device *ddev = adev->ddev; 1308 const char *pci_address_name = pci_name(ddev->pdev); 1309 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; 1310 1311 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); 1312 pciaddstr_tmp = pciaddstr; 1313 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { 1314 pciaddname = strsep(&pciaddname_tmp, ","); 1315 if (!strcmp("all", pciaddname) 1316 || !strcmp(pci_address_name, pciaddname)) { 1317 long num_crtc; 1318 int res = -1; 1319 1320 adev->enable_virtual_display = true; 1321 1322 if (pciaddname_tmp) 1323 res = kstrtol(pciaddname_tmp, 10, 1324 &num_crtc); 1325 1326 if (!res) { 1327 if (num_crtc < 1) 1328 num_crtc = 1; 1329 if (num_crtc > 6) 1330 num_crtc = 6; 1331 adev->mode_info.num_crtc = num_crtc; 1332 } else { 1333 adev->mode_info.num_crtc = 1; 1334 } 1335 break; 1336 } 1337 } 1338 1339 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", 1340 amdgpu_virtual_display, pci_address_name, 1341 adev->enable_virtual_display, adev->mode_info.num_crtc); 1342 1343 kfree(pciaddstr); 1344 } 1345 } 1346 1347 /** 1348 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware 1349 * 1350 * @adev: amdgpu_device pointer 1351 * 1352 * Parses the asic configuration parameters specified in the gpu info 1353 * firmware and makes them availale to the driver for use in configuring 1354 * the asic. 1355 * Returns 0 on success, -EINVAL on failure. 1356 */ 1357 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) 1358 { 1359 const char *chip_name; 1360 char fw_name[30]; 1361 int err; 1362 const struct gpu_info_firmware_header_v1_0 *hdr; 1363 1364 adev->firmware.gpu_info_fw = NULL; 1365 1366 switch (adev->asic_type) { 1367 case CHIP_TOPAZ: 1368 case CHIP_TONGA: 1369 case CHIP_FIJI: 1370 case CHIP_POLARIS10: 1371 case CHIP_POLARIS11: 1372 case CHIP_POLARIS12: 1373 case CHIP_VEGAM: 1374 case CHIP_CARRIZO: 1375 case CHIP_STONEY: 1376 #ifdef CONFIG_DRM_AMDGPU_SI 1377 case CHIP_VERDE: 1378 case CHIP_TAHITI: 1379 case CHIP_PITCAIRN: 1380 case CHIP_OLAND: 1381 case CHIP_HAINAN: 1382 #endif 1383 #ifdef CONFIG_DRM_AMDGPU_CIK 1384 case CHIP_BONAIRE: 1385 case CHIP_HAWAII: 1386 case CHIP_KAVERI: 1387 case CHIP_KABINI: 1388 case CHIP_MULLINS: 1389 #endif 1390 case CHIP_VEGA20: 1391 default: 1392 return 0; 1393 case CHIP_VEGA10: 1394 chip_name = "vega10"; 1395 break; 1396 case CHIP_VEGA12: 1397 chip_name = "vega12"; 1398 break; 1399 case CHIP_RAVEN: 1400 chip_name = "raven"; 1401 break; 1402 } 1403 1404 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name); 1405 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev); 1406 if (err) { 1407 dev_err(adev->dev, 1408 "Failed to load gpu_info firmware \"%s\"\n", 1409 fw_name); 1410 goto out; 1411 } 1412 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw); 1413 if (err) { 1414 dev_err(adev->dev, 1415 "Failed to validate gpu_info firmware \"%s\"\n", 1416 fw_name); 1417 goto out; 1418 } 1419 1420 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; 1421 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); 1422 1423 switch (hdr->version_major) { 1424 case 1: 1425 { 1426 const struct gpu_info_firmware_v1_0 *gpu_info_fw = 1427 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + 1428 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1429 1430 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); 1431 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); 1432 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); 1433 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); 1434 adev->gfx.config.max_texture_channel_caches = 1435 le32_to_cpu(gpu_info_fw->gc_num_tccs); 1436 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); 1437 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); 1438 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); 1439 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); 1440 adev->gfx.config.double_offchip_lds_buf = 1441 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); 1442 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); 1443 adev->gfx.cu_info.max_waves_per_simd = 1444 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); 1445 adev->gfx.cu_info.max_scratch_slots_per_cu = 1446 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); 1447 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); 1448 break; 1449 } 1450 default: 1451 dev_err(adev->dev, 1452 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); 1453 err = -EINVAL; 1454 goto out; 1455 } 1456 out: 1457 return err; 1458 } 1459 1460 /** 1461 * amdgpu_device_ip_early_init - run early init for hardware IPs 1462 * 1463 * @adev: amdgpu_device pointer 1464 * 1465 * Early initialization pass for hardware IPs. The hardware IPs that make 1466 * up each asic are discovered each IP's early_init callback is run. This 1467 * is the first stage in initializing the asic. 1468 * Returns 0 on success, negative error code on failure. 1469 */ 1470 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) 1471 { 1472 int i, r; 1473 1474 amdgpu_device_enable_virtual_display(adev); 1475 1476 switch (adev->asic_type) { 1477 case CHIP_TOPAZ: 1478 case CHIP_TONGA: 1479 case CHIP_FIJI: 1480 case CHIP_POLARIS10: 1481 case CHIP_POLARIS11: 1482 case CHIP_POLARIS12: 1483 case CHIP_VEGAM: 1484 case CHIP_CARRIZO: 1485 case CHIP_STONEY: 1486 if (adev->asic_type == CHIP_CARRIZO || adev->asic_type == CHIP_STONEY) 1487 adev->family = AMDGPU_FAMILY_CZ; 1488 else 1489 adev->family = AMDGPU_FAMILY_VI; 1490 1491 r = vi_set_ip_blocks(adev); 1492 if (r) 1493 return r; 1494 break; 1495 #ifdef CONFIG_DRM_AMDGPU_SI 1496 case CHIP_VERDE: 1497 case CHIP_TAHITI: 1498 case CHIP_PITCAIRN: 1499 case CHIP_OLAND: 1500 case CHIP_HAINAN: 1501 adev->family = AMDGPU_FAMILY_SI; 1502 r = si_set_ip_blocks(adev); 1503 if (r) 1504 return r; 1505 break; 1506 #endif 1507 #ifdef CONFIG_DRM_AMDGPU_CIK 1508 case CHIP_BONAIRE: 1509 case CHIP_HAWAII: 1510 case CHIP_KAVERI: 1511 case CHIP_KABINI: 1512 case CHIP_MULLINS: 1513 if ((adev->asic_type == CHIP_BONAIRE) || (adev->asic_type == CHIP_HAWAII)) 1514 adev->family = AMDGPU_FAMILY_CI; 1515 else 1516 adev->family = AMDGPU_FAMILY_KV; 1517 1518 r = cik_set_ip_blocks(adev); 1519 if (r) 1520 return r; 1521 break; 1522 #endif 1523 case CHIP_VEGA10: 1524 case CHIP_VEGA12: 1525 case CHIP_VEGA20: 1526 case CHIP_RAVEN: 1527 if (adev->asic_type == CHIP_RAVEN) 1528 adev->family = AMDGPU_FAMILY_RV; 1529 else 1530 adev->family = AMDGPU_FAMILY_AI; 1531 1532 r = soc15_set_ip_blocks(adev); 1533 if (r) 1534 return r; 1535 break; 1536 default: 1537 /* FIXME: not supported yet */ 1538 return -EINVAL; 1539 } 1540 1541 r = amdgpu_device_parse_gpu_info_fw(adev); 1542 if (r) 1543 return r; 1544 1545 amdgpu_amdkfd_device_probe(adev); 1546 1547 if (amdgpu_sriov_vf(adev)) { 1548 r = amdgpu_virt_request_full_gpu(adev, true); 1549 if (r) 1550 return -EAGAIN; 1551 } 1552 1553 adev->powerplay.pp_feature = amdgpu_pp_feature_mask; 1554 1555 for (i = 0; i < adev->num_ip_blocks; i++) { 1556 if ((amdgpu_ip_block_mask & (1 << i)) == 0) { 1557 DRM_ERROR("disabled ip block: %d <%s>\n", 1558 i, adev->ip_blocks[i].version->funcs->name); 1559 adev->ip_blocks[i].status.valid = false; 1560 } else { 1561 if (adev->ip_blocks[i].version->funcs->early_init) { 1562 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); 1563 if (r == -ENOENT) { 1564 adev->ip_blocks[i].status.valid = false; 1565 } else if (r) { 1566 DRM_ERROR("early_init of IP block <%s> failed %d\n", 1567 adev->ip_blocks[i].version->funcs->name, r); 1568 return r; 1569 } else { 1570 adev->ip_blocks[i].status.valid = true; 1571 } 1572 } else { 1573 adev->ip_blocks[i].status.valid = true; 1574 } 1575 } 1576 } 1577 1578 adev->cg_flags &= amdgpu_cg_mask; 1579 adev->pg_flags &= amdgpu_pg_mask; 1580 1581 return 0; 1582 } 1583 1584 /** 1585 * amdgpu_device_ip_init - run init for hardware IPs 1586 * 1587 * @adev: amdgpu_device pointer 1588 * 1589 * Main initialization pass for hardware IPs. The list of all the hardware 1590 * IPs that make up the asic is walked and the sw_init and hw_init callbacks 1591 * are run. sw_init initializes the software state associated with each IP 1592 * and hw_init initializes the hardware associated with each IP. 1593 * Returns 0 on success, negative error code on failure. 1594 */ 1595 static int amdgpu_device_ip_init(struct amdgpu_device *adev) 1596 { 1597 int i, r; 1598 1599 for (i = 0; i < adev->num_ip_blocks; i++) { 1600 if (!adev->ip_blocks[i].status.valid) 1601 continue; 1602 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); 1603 if (r) { 1604 DRM_ERROR("sw_init of IP block <%s> failed %d\n", 1605 adev->ip_blocks[i].version->funcs->name, r); 1606 return r; 1607 } 1608 adev->ip_blocks[i].status.sw = true; 1609 1610 /* need to do gmc hw init early so we can allocate gpu mem */ 1611 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 1612 r = amdgpu_device_vram_scratch_init(adev); 1613 if (r) { 1614 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r); 1615 return r; 1616 } 1617 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); 1618 if (r) { 1619 DRM_ERROR("hw_init %d failed %d\n", i, r); 1620 return r; 1621 } 1622 r = amdgpu_device_wb_init(adev); 1623 if (r) { 1624 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r); 1625 return r; 1626 } 1627 adev->ip_blocks[i].status.hw = true; 1628 1629 /* right after GMC hw init, we create CSA */ 1630 if (amdgpu_sriov_vf(adev)) { 1631 r = amdgpu_allocate_static_csa(adev); 1632 if (r) { 1633 DRM_ERROR("allocate CSA failed %d\n", r); 1634 return r; 1635 } 1636 } 1637 } 1638 } 1639 1640 for (i = 0; i < adev->num_ip_blocks; i++) { 1641 if (!adev->ip_blocks[i].status.sw) 1642 continue; 1643 if (adev->ip_blocks[i].status.hw) 1644 continue; 1645 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); 1646 if (r) { 1647 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 1648 adev->ip_blocks[i].version->funcs->name, r); 1649 return r; 1650 } 1651 adev->ip_blocks[i].status.hw = true; 1652 } 1653 1654 amdgpu_amdkfd_device_init(adev); 1655 1656 if (amdgpu_sriov_vf(adev)) 1657 amdgpu_virt_release_full_gpu(adev, true); 1658 1659 return 0; 1660 } 1661 1662 /** 1663 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer 1664 * 1665 * @adev: amdgpu_device pointer 1666 * 1667 * Writes a reset magic value to the gart pointer in VRAM. The driver calls 1668 * this function before a GPU reset. If the value is retained after a 1669 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents. 1670 */ 1671 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) 1672 { 1673 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); 1674 } 1675 1676 /** 1677 * amdgpu_device_check_vram_lost - check if vram is valid 1678 * 1679 * @adev: amdgpu_device pointer 1680 * 1681 * Checks the reset magic value written to the gart pointer in VRAM. 1682 * The driver calls this after a GPU reset to see if the contents of 1683 * VRAM is lost or now. 1684 * returns true if vram is lost, false if not. 1685 */ 1686 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) 1687 { 1688 return !!memcmp(adev->gart.ptr, adev->reset_magic, 1689 AMDGPU_RESET_MAGIC_NUM); 1690 } 1691 1692 /** 1693 * amdgpu_device_ip_late_set_cg_state - late init for clockgating 1694 * 1695 * @adev: amdgpu_device pointer 1696 * 1697 * Late initialization pass enabling clockgating for hardware IPs. 1698 * The list of all the hardware IPs that make up the asic is walked and the 1699 * set_clockgating_state callbacks are run. This stage is run late 1700 * in the init process. 1701 * Returns 0 on success, negative error code on failure. 1702 */ 1703 static int amdgpu_device_ip_late_set_cg_state(struct amdgpu_device *adev) 1704 { 1705 int i = 0, r; 1706 1707 if (amdgpu_emu_mode == 1) 1708 return 0; 1709 1710 for (i = 0; i < adev->num_ip_blocks; i++) { 1711 if (!adev->ip_blocks[i].status.valid) 1712 continue; 1713 /* skip CG for VCE/UVD, it's handled specially */ 1714 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 1715 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 1716 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 1717 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 1718 /* enable clockgating to save power */ 1719 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, 1720 AMD_CG_STATE_GATE); 1721 if (r) { 1722 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n", 1723 adev->ip_blocks[i].version->funcs->name, r); 1724 return r; 1725 } 1726 } 1727 } 1728 1729 return 0; 1730 } 1731 1732 static int amdgpu_device_ip_late_set_pg_state(struct amdgpu_device *adev) 1733 { 1734 int i = 0, r; 1735 1736 if (amdgpu_emu_mode == 1) 1737 return 0; 1738 1739 for (i = 0; i < adev->num_ip_blocks; i++) { 1740 if (!adev->ip_blocks[i].status.valid) 1741 continue; 1742 /* skip CG for VCE/UVD, it's handled specially */ 1743 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 1744 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 1745 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 1746 adev->ip_blocks[i].version->funcs->set_powergating_state) { 1747 /* enable powergating to save power */ 1748 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, 1749 AMD_PG_STATE_GATE); 1750 if (r) { 1751 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n", 1752 adev->ip_blocks[i].version->funcs->name, r); 1753 return r; 1754 } 1755 } 1756 } 1757 return 0; 1758 } 1759 1760 /** 1761 * amdgpu_device_ip_late_init - run late init for hardware IPs 1762 * 1763 * @adev: amdgpu_device pointer 1764 * 1765 * Late initialization pass for hardware IPs. The list of all the hardware 1766 * IPs that make up the asic is walked and the late_init callbacks are run. 1767 * late_init covers any special initialization that an IP requires 1768 * after all of the have been initialized or something that needs to happen 1769 * late in the init process. 1770 * Returns 0 on success, negative error code on failure. 1771 */ 1772 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) 1773 { 1774 int i = 0, r; 1775 1776 for (i = 0; i < adev->num_ip_blocks; i++) { 1777 if (!adev->ip_blocks[i].status.valid) 1778 continue; 1779 if (adev->ip_blocks[i].version->funcs->late_init) { 1780 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); 1781 if (r) { 1782 DRM_ERROR("late_init of IP block <%s> failed %d\n", 1783 adev->ip_blocks[i].version->funcs->name, r); 1784 return r; 1785 } 1786 adev->ip_blocks[i].status.late_initialized = true; 1787 } 1788 } 1789 1790 amdgpu_device_ip_late_set_cg_state(adev); 1791 amdgpu_device_ip_late_set_pg_state(adev); 1792 1793 queue_delayed_work(system_wq, &adev->late_init_work, 1794 msecs_to_jiffies(AMDGPU_RESUME_MS)); 1795 1796 amdgpu_device_fill_reset_magic(adev); 1797 1798 return 0; 1799 } 1800 1801 /** 1802 * amdgpu_device_ip_fini - run fini for hardware IPs 1803 * 1804 * @adev: amdgpu_device pointer 1805 * 1806 * Main teardown pass for hardware IPs. The list of all the hardware 1807 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks 1808 * are run. hw_fini tears down the hardware associated with each IP 1809 * and sw_fini tears down any software state associated with each IP. 1810 * Returns 0 on success, negative error code on failure. 1811 */ 1812 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) 1813 { 1814 int i, r; 1815 1816 amdgpu_amdkfd_device_fini(adev); 1817 /* need to disable SMC first */ 1818 for (i = 0; i < adev->num_ip_blocks; i++) { 1819 if (!adev->ip_blocks[i].status.hw) 1820 continue; 1821 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC && 1822 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 1823 /* ungate blocks before hw fini so that we can shutdown the blocks safely */ 1824 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, 1825 AMD_CG_STATE_UNGATE); 1826 if (r) { 1827 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n", 1828 adev->ip_blocks[i].version->funcs->name, r); 1829 return r; 1830 } 1831 if (adev->powerplay.pp_funcs->set_powergating_by_smu) 1832 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false); 1833 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); 1834 /* XXX handle errors */ 1835 if (r) { 1836 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 1837 adev->ip_blocks[i].version->funcs->name, r); 1838 } 1839 adev->ip_blocks[i].status.hw = false; 1840 break; 1841 } 1842 } 1843 1844 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 1845 if (!adev->ip_blocks[i].status.hw) 1846 continue; 1847 1848 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 1849 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 1850 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 1851 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 1852 /* ungate blocks before hw fini so that we can shutdown the blocks safely */ 1853 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, 1854 AMD_CG_STATE_UNGATE); 1855 if (r) { 1856 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n", 1857 adev->ip_blocks[i].version->funcs->name, r); 1858 return r; 1859 } 1860 } 1861 1862 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); 1863 /* XXX handle errors */ 1864 if (r) { 1865 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 1866 adev->ip_blocks[i].version->funcs->name, r); 1867 } 1868 1869 adev->ip_blocks[i].status.hw = false; 1870 } 1871 1872 1873 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 1874 if (!adev->ip_blocks[i].status.sw) 1875 continue; 1876 1877 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 1878 amdgpu_free_static_csa(adev); 1879 amdgpu_device_wb_fini(adev); 1880 amdgpu_device_vram_scratch_fini(adev); 1881 } 1882 1883 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); 1884 /* XXX handle errors */ 1885 if (r) { 1886 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n", 1887 adev->ip_blocks[i].version->funcs->name, r); 1888 } 1889 adev->ip_blocks[i].status.sw = false; 1890 adev->ip_blocks[i].status.valid = false; 1891 } 1892 1893 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 1894 if (!adev->ip_blocks[i].status.late_initialized) 1895 continue; 1896 if (adev->ip_blocks[i].version->funcs->late_fini) 1897 adev->ip_blocks[i].version->funcs->late_fini((void *)adev); 1898 adev->ip_blocks[i].status.late_initialized = false; 1899 } 1900 1901 if (amdgpu_sriov_vf(adev)) 1902 if (amdgpu_virt_release_full_gpu(adev, false)) 1903 DRM_ERROR("failed to release exclusive mode on fini\n"); 1904 1905 return 0; 1906 } 1907 1908 /** 1909 * amdgpu_device_ip_late_init_func_handler - work handler for clockgating 1910 * 1911 * @work: work_struct 1912 * 1913 * Work handler for amdgpu_device_ip_late_set_cg_state. We put the 1914 * clockgating setup into a worker thread to speed up driver init and 1915 * resume from suspend. 1916 */ 1917 static void amdgpu_device_ip_late_init_func_handler(struct work_struct *work) 1918 { 1919 struct amdgpu_device *adev = 1920 container_of(work, struct amdgpu_device, late_init_work.work); 1921 int r; 1922 1923 r = amdgpu_ib_ring_tests(adev); 1924 if (r) 1925 DRM_ERROR("ib ring test failed (%d).\n", r); 1926 } 1927 1928 /** 1929 * amdgpu_device_ip_suspend - run suspend for hardware IPs 1930 * 1931 * @adev: amdgpu_device pointer 1932 * 1933 * Main suspend function for hardware IPs. The list of all the hardware 1934 * IPs that make up the asic is walked, clockgating is disabled and the 1935 * suspend callbacks are run. suspend puts the hardware and software state 1936 * in each IP into a state suitable for suspend. 1937 * Returns 0 on success, negative error code on failure. 1938 */ 1939 int amdgpu_device_ip_suspend(struct amdgpu_device *adev) 1940 { 1941 int i, r; 1942 1943 if (amdgpu_sriov_vf(adev)) 1944 amdgpu_virt_request_full_gpu(adev, false); 1945 1946 /* ungate SMC block first */ 1947 r = amdgpu_device_ip_set_clockgating_state(adev, AMD_IP_BLOCK_TYPE_SMC, 1948 AMD_CG_STATE_UNGATE); 1949 if (r) { 1950 DRM_ERROR("set_clockgating_state(ungate) SMC failed %d\n", r); 1951 } 1952 1953 /* call smu to disable gfx off feature first when suspend */ 1954 if (adev->powerplay.pp_funcs->set_powergating_by_smu) 1955 amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, false); 1956 1957 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 1958 if (!adev->ip_blocks[i].status.valid) 1959 continue; 1960 /* ungate blocks so that suspend can properly shut them down */ 1961 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_SMC && 1962 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 1963 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, 1964 AMD_CG_STATE_UNGATE); 1965 if (r) { 1966 DRM_ERROR("set_clockgating_state(ungate) of IP block <%s> failed %d\n", 1967 adev->ip_blocks[i].version->funcs->name, r); 1968 } 1969 } 1970 /* XXX handle errors */ 1971 r = adev->ip_blocks[i].version->funcs->suspend(adev); 1972 /* XXX handle errors */ 1973 if (r) { 1974 DRM_ERROR("suspend of IP block <%s> failed %d\n", 1975 adev->ip_blocks[i].version->funcs->name, r); 1976 } 1977 } 1978 1979 if (amdgpu_sriov_vf(adev)) 1980 amdgpu_virt_release_full_gpu(adev, false); 1981 1982 return 0; 1983 } 1984 1985 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) 1986 { 1987 int i, r; 1988 1989 static enum amd_ip_block_type ip_order[] = { 1990 AMD_IP_BLOCK_TYPE_GMC, 1991 AMD_IP_BLOCK_TYPE_COMMON, 1992 AMD_IP_BLOCK_TYPE_IH, 1993 }; 1994 1995 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 1996 int j; 1997 struct amdgpu_ip_block *block; 1998 1999 for (j = 0; j < adev->num_ip_blocks; j++) { 2000 block = &adev->ip_blocks[j]; 2001 2002 if (block->version->type != ip_order[i] || 2003 !block->status.valid) 2004 continue; 2005 2006 r = block->version->funcs->hw_init(adev); 2007 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed"); 2008 if (r) 2009 return r; 2010 } 2011 } 2012 2013 return 0; 2014 } 2015 2016 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) 2017 { 2018 int i, r; 2019 2020 static enum amd_ip_block_type ip_order[] = { 2021 AMD_IP_BLOCK_TYPE_SMC, 2022 AMD_IP_BLOCK_TYPE_PSP, 2023 AMD_IP_BLOCK_TYPE_DCE, 2024 AMD_IP_BLOCK_TYPE_GFX, 2025 AMD_IP_BLOCK_TYPE_SDMA, 2026 AMD_IP_BLOCK_TYPE_UVD, 2027 AMD_IP_BLOCK_TYPE_VCE 2028 }; 2029 2030 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 2031 int j; 2032 struct amdgpu_ip_block *block; 2033 2034 for (j = 0; j < adev->num_ip_blocks; j++) { 2035 block = &adev->ip_blocks[j]; 2036 2037 if (block->version->type != ip_order[i] || 2038 !block->status.valid) 2039 continue; 2040 2041 r = block->version->funcs->hw_init(adev); 2042 DRM_INFO("RE-INIT: %s %s\n", block->version->funcs->name, r?"failed":"successed"); 2043 if (r) 2044 return r; 2045 } 2046 } 2047 2048 return 0; 2049 } 2050 2051 /** 2052 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs 2053 * 2054 * @adev: amdgpu_device pointer 2055 * 2056 * First resume function for hardware IPs. The list of all the hardware 2057 * IPs that make up the asic is walked and the resume callbacks are run for 2058 * COMMON, GMC, and IH. resume puts the hardware into a functional state 2059 * after a suspend and updates the software state as necessary. This 2060 * function is also used for restoring the GPU after a GPU reset. 2061 * Returns 0 on success, negative error code on failure. 2062 */ 2063 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) 2064 { 2065 int i, r; 2066 2067 for (i = 0; i < adev->num_ip_blocks; i++) { 2068 if (!adev->ip_blocks[i].status.valid) 2069 continue; 2070 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2071 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 2072 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2073 r = adev->ip_blocks[i].version->funcs->resume(adev); 2074 if (r) { 2075 DRM_ERROR("resume of IP block <%s> failed %d\n", 2076 adev->ip_blocks[i].version->funcs->name, r); 2077 return r; 2078 } 2079 } 2080 } 2081 2082 return 0; 2083 } 2084 2085 /** 2086 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs 2087 * 2088 * @adev: amdgpu_device pointer 2089 * 2090 * First resume function for hardware IPs. The list of all the hardware 2091 * IPs that make up the asic is walked and the resume callbacks are run for 2092 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a 2093 * functional state after a suspend and updates the software state as 2094 * necessary. This function is also used for restoring the GPU after a GPU 2095 * reset. 2096 * Returns 0 on success, negative error code on failure. 2097 */ 2098 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) 2099 { 2100 int i, r; 2101 2102 for (i = 0; i < adev->num_ip_blocks; i++) { 2103 if (!adev->ip_blocks[i].status.valid) 2104 continue; 2105 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2106 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 2107 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) 2108 continue; 2109 r = adev->ip_blocks[i].version->funcs->resume(adev); 2110 if (r) { 2111 DRM_ERROR("resume of IP block <%s> failed %d\n", 2112 adev->ip_blocks[i].version->funcs->name, r); 2113 return r; 2114 } 2115 } 2116 2117 return 0; 2118 } 2119 2120 /** 2121 * amdgpu_device_ip_resume - run resume for hardware IPs 2122 * 2123 * @adev: amdgpu_device pointer 2124 * 2125 * Main resume function for hardware IPs. The hardware IPs 2126 * are split into two resume functions because they are 2127 * are also used in in recovering from a GPU reset and some additional 2128 * steps need to be take between them. In this case (S3/S4) they are 2129 * run sequentially. 2130 * Returns 0 on success, negative error code on failure. 2131 */ 2132 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) 2133 { 2134 int r; 2135 2136 r = amdgpu_device_ip_resume_phase1(adev); 2137 if (r) 2138 return r; 2139 r = amdgpu_device_ip_resume_phase2(adev); 2140 2141 return r; 2142 } 2143 2144 /** 2145 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV 2146 * 2147 * @adev: amdgpu_device pointer 2148 * 2149 * Query the VBIOS data tables to determine if the board supports SR-IOV. 2150 */ 2151 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) 2152 { 2153 if (amdgpu_sriov_vf(adev)) { 2154 if (adev->is_atom_fw) { 2155 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev)) 2156 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 2157 } else { 2158 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) 2159 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 2160 } 2161 2162 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) 2163 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 2164 } 2165 } 2166 2167 /** 2168 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic 2169 * 2170 * @asic_type: AMD asic type 2171 * 2172 * Check if there is DC (new modesetting infrastructre) support for an asic. 2173 * returns true if DC has support, false if not. 2174 */ 2175 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) 2176 { 2177 switch (asic_type) { 2178 #if defined(CONFIG_DRM_AMD_DC) 2179 case CHIP_BONAIRE: 2180 case CHIP_KAVERI: 2181 case CHIP_KABINI: 2182 case CHIP_MULLINS: 2183 /* 2184 * We have systems in the wild with these ASICs that require 2185 * LVDS and VGA support which is not supported with DC. 2186 * 2187 * Fallback to the non-DC driver here by default so as not to 2188 * cause regressions. 2189 */ 2190 return amdgpu_dc > 0; 2191 case CHIP_HAWAII: 2192 case CHIP_CARRIZO: 2193 case CHIP_STONEY: 2194 case CHIP_POLARIS10: 2195 case CHIP_POLARIS11: 2196 case CHIP_POLARIS12: 2197 case CHIP_VEGAM: 2198 case CHIP_TONGA: 2199 case CHIP_FIJI: 2200 case CHIP_VEGA10: 2201 case CHIP_VEGA12: 2202 case CHIP_VEGA20: 2203 #if defined(CONFIG_DRM_AMD_DC_DCN1_0) 2204 case CHIP_RAVEN: 2205 #endif 2206 return amdgpu_dc != 0; 2207 #endif 2208 default: 2209 return false; 2210 } 2211 } 2212 2213 /** 2214 * amdgpu_device_has_dc_support - check if dc is supported 2215 * 2216 * @adev: amdgpu_device_pointer 2217 * 2218 * Returns true for supported, false for not supported 2219 */ 2220 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) 2221 { 2222 if (amdgpu_sriov_vf(adev)) 2223 return false; 2224 2225 return amdgpu_device_asic_has_dc_support(adev->asic_type); 2226 } 2227 2228 /** 2229 * amdgpu_device_init - initialize the driver 2230 * 2231 * @adev: amdgpu_device pointer 2232 * @ddev: drm dev pointer 2233 * @pdev: pci dev pointer 2234 * @flags: driver flags 2235 * 2236 * Initializes the driver info and hw (all asics). 2237 * Returns 0 for success or an error on failure. 2238 * Called at driver startup. 2239 */ 2240 int amdgpu_device_init(struct amdgpu_device *adev, 2241 struct drm_device *ddev, 2242 struct pci_dev *pdev, 2243 uint32_t flags) 2244 { 2245 int r, i; 2246 bool runtime = false; 2247 u32 max_MBps; 2248 2249 adev->shutdown = false; 2250 adev->dev = &pdev->dev; 2251 adev->ddev = ddev; 2252 adev->pdev = pdev; 2253 adev->flags = flags; 2254 adev->asic_type = flags & AMD_ASIC_MASK; 2255 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; 2256 if (amdgpu_emu_mode == 1) 2257 adev->usec_timeout *= 2; 2258 adev->gmc.gart_size = 512 * 1024 * 1024; 2259 adev->accel_working = false; 2260 adev->num_rings = 0; 2261 adev->mman.buffer_funcs = NULL; 2262 adev->mman.buffer_funcs_ring = NULL; 2263 adev->vm_manager.vm_pte_funcs = NULL; 2264 adev->vm_manager.vm_pte_num_rings = 0; 2265 adev->gmc.gmc_funcs = NULL; 2266 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); 2267 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 2268 2269 adev->smc_rreg = &amdgpu_invalid_rreg; 2270 adev->smc_wreg = &amdgpu_invalid_wreg; 2271 adev->pcie_rreg = &amdgpu_invalid_rreg; 2272 adev->pcie_wreg = &amdgpu_invalid_wreg; 2273 adev->pciep_rreg = &amdgpu_invalid_rreg; 2274 adev->pciep_wreg = &amdgpu_invalid_wreg; 2275 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; 2276 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; 2277 adev->didt_rreg = &amdgpu_invalid_rreg; 2278 adev->didt_wreg = &amdgpu_invalid_wreg; 2279 adev->gc_cac_rreg = &amdgpu_invalid_rreg; 2280 adev->gc_cac_wreg = &amdgpu_invalid_wreg; 2281 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; 2282 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; 2283 2284 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", 2285 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, 2286 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); 2287 2288 /* mutex initialization are all done here so we 2289 * can recall function without having locking issues */ 2290 atomic_set(&adev->irq.ih.lock, 0); 2291 mutex_init(&adev->firmware.mutex); 2292 mutex_init(&adev->pm.mutex); 2293 mutex_init(&adev->gfx.gpu_clock_mutex); 2294 mutex_init(&adev->srbm_mutex); 2295 mutex_init(&adev->gfx.pipe_reserve_mutex); 2296 mutex_init(&adev->grbm_idx_mutex); 2297 mutex_init(&adev->mn_lock); 2298 mutex_init(&adev->virt.vf_errors.lock); 2299 hash_init(adev->mn_hash); 2300 mutex_init(&adev->lock_reset); 2301 2302 amdgpu_device_check_arguments(adev); 2303 2304 spin_lock_init(&adev->mmio_idx_lock); 2305 spin_lock_init(&adev->smc_idx_lock); 2306 spin_lock_init(&adev->pcie_idx_lock); 2307 spin_lock_init(&adev->uvd_ctx_idx_lock); 2308 spin_lock_init(&adev->didt_idx_lock); 2309 spin_lock_init(&adev->gc_cac_idx_lock); 2310 spin_lock_init(&adev->se_cac_idx_lock); 2311 spin_lock_init(&adev->audio_endpt_idx_lock); 2312 spin_lock_init(&adev->mm_stats.lock); 2313 2314 INIT_LIST_HEAD(&adev->shadow_list); 2315 mutex_init(&adev->shadow_list_lock); 2316 2317 INIT_LIST_HEAD(&adev->ring_lru_list); 2318 spin_lock_init(&adev->ring_lru_list_lock); 2319 2320 INIT_DELAYED_WORK(&adev->late_init_work, 2321 amdgpu_device_ip_late_init_func_handler); 2322 2323 adev->pm.ac_power = power_supply_is_system_supplied() > 0 ? true : false; 2324 2325 /* Registers mapping */ 2326 /* TODO: block userspace mapping of io register */ 2327 if (adev->asic_type >= CHIP_BONAIRE) { 2328 adev->rmmio_base = pci_resource_start(adev->pdev, 5); 2329 adev->rmmio_size = pci_resource_len(adev->pdev, 5); 2330 } else { 2331 adev->rmmio_base = pci_resource_start(adev->pdev, 2); 2332 adev->rmmio_size = pci_resource_len(adev->pdev, 2); 2333 } 2334 2335 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); 2336 if (adev->rmmio == NULL) { 2337 return -ENOMEM; 2338 } 2339 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); 2340 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size); 2341 2342 /* doorbell bar mapping */ 2343 amdgpu_device_doorbell_init(adev); 2344 2345 /* io port mapping */ 2346 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { 2347 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) { 2348 adev->rio_mem_size = pci_resource_len(adev->pdev, i); 2349 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size); 2350 break; 2351 } 2352 } 2353 if (adev->rio_mem == NULL) 2354 DRM_INFO("PCI I/O BAR is not found.\n"); 2355 2356 amdgpu_device_get_pcie_info(adev); 2357 2358 /* early init functions */ 2359 r = amdgpu_device_ip_early_init(adev); 2360 if (r) 2361 return r; 2362 2363 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ 2364 /* this will fail for cards that aren't VGA class devices, just 2365 * ignore it */ 2366 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode); 2367 2368 if (amdgpu_device_is_px(ddev)) 2369 runtime = true; 2370 if (!pci_is_thunderbolt_attached(adev->pdev)) 2371 vga_switcheroo_register_client(adev->pdev, 2372 &amdgpu_switcheroo_ops, runtime); 2373 if (runtime) 2374 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); 2375 2376 if (amdgpu_emu_mode == 1) { 2377 /* post the asic on emulation mode */ 2378 emu_soc_asic_init(adev); 2379 goto fence_driver_init; 2380 } 2381 2382 /* Read BIOS */ 2383 if (!amdgpu_get_bios(adev)) { 2384 r = -EINVAL; 2385 goto failed; 2386 } 2387 2388 r = amdgpu_atombios_init(adev); 2389 if (r) { 2390 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); 2391 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0); 2392 goto failed; 2393 } 2394 2395 /* detect if we are with an SRIOV vbios */ 2396 amdgpu_device_detect_sriov_bios(adev); 2397 2398 /* Post card if necessary */ 2399 if (amdgpu_device_need_post(adev)) { 2400 if (!adev->bios) { 2401 dev_err(adev->dev, "no vBIOS found\n"); 2402 r = -EINVAL; 2403 goto failed; 2404 } 2405 DRM_INFO("GPU posting now...\n"); 2406 r = amdgpu_atom_asic_init(adev->mode_info.atom_context); 2407 if (r) { 2408 dev_err(adev->dev, "gpu post error!\n"); 2409 goto failed; 2410 } 2411 } 2412 2413 if (adev->is_atom_fw) { 2414 /* Initialize clocks */ 2415 r = amdgpu_atomfirmware_get_clock_info(adev); 2416 if (r) { 2417 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 2418 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 2419 goto failed; 2420 } 2421 } else { 2422 /* Initialize clocks */ 2423 r = amdgpu_atombios_get_clock_info(adev); 2424 if (r) { 2425 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 2426 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 2427 goto failed; 2428 } 2429 /* init i2c buses */ 2430 if (!amdgpu_device_has_dc_support(adev)) 2431 amdgpu_atombios_i2c_init(adev); 2432 } 2433 2434 fence_driver_init: 2435 /* Fence driver */ 2436 r = amdgpu_fence_driver_init(adev); 2437 if (r) { 2438 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n"); 2439 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 2440 goto failed; 2441 } 2442 2443 /* init the mode config */ 2444 drm_mode_config_init(adev->ddev); 2445 2446 r = amdgpu_device_ip_init(adev); 2447 if (r) { 2448 /* failed in exclusive mode due to timeout */ 2449 if (amdgpu_sriov_vf(adev) && 2450 !amdgpu_sriov_runtime(adev) && 2451 amdgpu_virt_mmio_blocked(adev) && 2452 !amdgpu_virt_wait_reset(adev)) { 2453 dev_err(adev->dev, "VF exclusive mode timeout\n"); 2454 /* Don't send request since VF is inactive. */ 2455 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 2456 adev->virt.ops = NULL; 2457 r = -EAGAIN; 2458 goto failed; 2459 } 2460 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); 2461 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 2462 goto failed; 2463 } 2464 2465 adev->accel_working = true; 2466 2467 amdgpu_vm_check_compute_bug(adev); 2468 2469 /* Initialize the buffer migration limit. */ 2470 if (amdgpu_moverate >= 0) 2471 max_MBps = amdgpu_moverate; 2472 else 2473 max_MBps = 8; /* Allow 8 MB/s. */ 2474 /* Get a log2 for easy divisions. */ 2475 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); 2476 2477 r = amdgpu_ib_pool_init(adev); 2478 if (r) { 2479 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 2480 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 2481 goto failed; 2482 } 2483 2484 if (amdgpu_sriov_vf(adev)) 2485 amdgpu_virt_init_data_exchange(adev); 2486 2487 amdgpu_fbdev_init(adev); 2488 2489 r = amdgpu_pm_sysfs_init(adev); 2490 if (r) 2491 DRM_ERROR("registering pm debugfs failed (%d).\n", r); 2492 2493 r = amdgpu_debugfs_gem_init(adev); 2494 if (r) 2495 DRM_ERROR("registering gem debugfs failed (%d).\n", r); 2496 2497 r = amdgpu_debugfs_regs_init(adev); 2498 if (r) 2499 DRM_ERROR("registering register debugfs failed (%d).\n", r); 2500 2501 r = amdgpu_debugfs_firmware_init(adev); 2502 if (r) 2503 DRM_ERROR("registering firmware debugfs failed (%d).\n", r); 2504 2505 r = amdgpu_debugfs_init(adev); 2506 if (r) 2507 DRM_ERROR("Creating debugfs files failed (%d).\n", r); 2508 2509 if ((amdgpu_testing & 1)) { 2510 if (adev->accel_working) 2511 amdgpu_test_moves(adev); 2512 else 2513 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n"); 2514 } 2515 if (amdgpu_benchmarking) { 2516 if (adev->accel_working) 2517 amdgpu_benchmark(adev, amdgpu_benchmarking); 2518 else 2519 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n"); 2520 } 2521 2522 /* enable clockgating, etc. after ib tests, etc. since some blocks require 2523 * explicit gating rather than handling it automatically. 2524 */ 2525 r = amdgpu_device_ip_late_init(adev); 2526 if (r) { 2527 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); 2528 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 2529 goto failed; 2530 } 2531 2532 return 0; 2533 2534 failed: 2535 amdgpu_vf_error_trans_all(adev); 2536 if (runtime) 2537 vga_switcheroo_fini_domain_pm_ops(adev->dev); 2538 2539 return r; 2540 } 2541 2542 /** 2543 * amdgpu_device_fini - tear down the driver 2544 * 2545 * @adev: amdgpu_device pointer 2546 * 2547 * Tear down the driver info (all asics). 2548 * Called at driver shutdown. 2549 */ 2550 void amdgpu_device_fini(struct amdgpu_device *adev) 2551 { 2552 int r; 2553 2554 DRM_INFO("amdgpu: finishing device.\n"); 2555 adev->shutdown = true; 2556 /* disable all interrupts */ 2557 amdgpu_irq_disable_all(adev); 2558 if (adev->mode_info.mode_config_initialized){ 2559 if (!amdgpu_device_has_dc_support(adev)) 2560 drm_crtc_force_disable_all(adev->ddev); 2561 else 2562 drm_atomic_helper_shutdown(adev->ddev); 2563 } 2564 amdgpu_ib_pool_fini(adev); 2565 amdgpu_fence_driver_fini(adev); 2566 amdgpu_pm_sysfs_fini(adev); 2567 amdgpu_fbdev_fini(adev); 2568 r = amdgpu_device_ip_fini(adev); 2569 if (adev->firmware.gpu_info_fw) { 2570 release_firmware(adev->firmware.gpu_info_fw); 2571 adev->firmware.gpu_info_fw = NULL; 2572 } 2573 adev->accel_working = false; 2574 cancel_delayed_work_sync(&adev->late_init_work); 2575 /* free i2c buses */ 2576 if (!amdgpu_device_has_dc_support(adev)) 2577 amdgpu_i2c_fini(adev); 2578 2579 if (amdgpu_emu_mode != 1) 2580 amdgpu_atombios_fini(adev); 2581 2582 kfree(adev->bios); 2583 adev->bios = NULL; 2584 if (!pci_is_thunderbolt_attached(adev->pdev)) 2585 vga_switcheroo_unregister_client(adev->pdev); 2586 if (adev->flags & AMD_IS_PX) 2587 vga_switcheroo_fini_domain_pm_ops(adev->dev); 2588 vga_client_register(adev->pdev, NULL, NULL, NULL); 2589 if (adev->rio_mem) 2590 pci_iounmap(adev->pdev, adev->rio_mem); 2591 adev->rio_mem = NULL; 2592 iounmap(adev->rmmio); 2593 adev->rmmio = NULL; 2594 amdgpu_device_doorbell_fini(adev); 2595 amdgpu_debugfs_regs_cleanup(adev); 2596 } 2597 2598 2599 /* 2600 * Suspend & resume. 2601 */ 2602 /** 2603 * amdgpu_device_suspend - initiate device suspend 2604 * 2605 * @dev: drm dev pointer 2606 * @suspend: suspend state 2607 * @fbcon : notify the fbdev of suspend 2608 * 2609 * Puts the hw in the suspend state (all asics). 2610 * Returns 0 for success or an error on failure. 2611 * Called at driver suspend. 2612 */ 2613 int amdgpu_device_suspend(struct drm_device *dev, bool suspend, bool fbcon) 2614 { 2615 struct amdgpu_device *adev; 2616 struct drm_crtc *crtc; 2617 struct drm_connector *connector; 2618 int r; 2619 2620 if (dev == NULL || dev->dev_private == NULL) { 2621 return -ENODEV; 2622 } 2623 2624 adev = dev->dev_private; 2625 2626 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 2627 return 0; 2628 2629 drm_kms_helper_poll_disable(dev); 2630 2631 if (!amdgpu_device_has_dc_support(adev)) { 2632 /* turn off display hw */ 2633 drm_modeset_lock_all(dev); 2634 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 2635 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_OFF); 2636 } 2637 drm_modeset_unlock_all(dev); 2638 } 2639 2640 amdgpu_amdkfd_suspend(adev); 2641 2642 /* unpin the front buffers and cursors */ 2643 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { 2644 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); 2645 struct drm_framebuffer *fb = crtc->primary->fb; 2646 struct amdgpu_bo *robj; 2647 2648 if (amdgpu_crtc->cursor_bo) { 2649 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 2650 r = amdgpu_bo_reserve(aobj, true); 2651 if (r == 0) { 2652 amdgpu_bo_unpin(aobj); 2653 amdgpu_bo_unreserve(aobj); 2654 } 2655 } 2656 2657 if (fb == NULL || fb->obj[0] == NULL) { 2658 continue; 2659 } 2660 robj = gem_to_amdgpu_bo(fb->obj[0]); 2661 /* don't unpin kernel fb objects */ 2662 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) { 2663 r = amdgpu_bo_reserve(robj, true); 2664 if (r == 0) { 2665 amdgpu_bo_unpin(robj); 2666 amdgpu_bo_unreserve(robj); 2667 } 2668 } 2669 } 2670 /* evict vram memory */ 2671 amdgpu_bo_evict_vram(adev); 2672 2673 amdgpu_fence_driver_suspend(adev); 2674 2675 r = amdgpu_device_ip_suspend(adev); 2676 2677 /* evict remaining vram memory 2678 * This second call to evict vram is to evict the gart page table 2679 * using the CPU. 2680 */ 2681 amdgpu_bo_evict_vram(adev); 2682 2683 pci_save_state(dev->pdev); 2684 if (suspend) { 2685 /* Shut down the device */ 2686 pci_disable_device(dev->pdev); 2687 pci_set_power_state(dev->pdev, PCI_D3hot); 2688 } else { 2689 r = amdgpu_asic_reset(adev); 2690 if (r) 2691 DRM_ERROR("amdgpu asic reset failed\n"); 2692 } 2693 2694 if (fbcon) { 2695 console_lock(); 2696 amdgpu_fbdev_set_suspend(adev, 1); 2697 console_unlock(); 2698 } 2699 return 0; 2700 } 2701 2702 /** 2703 * amdgpu_device_resume - initiate device resume 2704 * 2705 * @dev: drm dev pointer 2706 * @resume: resume state 2707 * @fbcon : notify the fbdev of resume 2708 * 2709 * Bring the hw back to operating state (all asics). 2710 * Returns 0 for success or an error on failure. 2711 * Called at driver resume. 2712 */ 2713 int amdgpu_device_resume(struct drm_device *dev, bool resume, bool fbcon) 2714 { 2715 struct drm_connector *connector; 2716 struct amdgpu_device *adev = dev->dev_private; 2717 struct drm_crtc *crtc; 2718 int r = 0; 2719 2720 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 2721 return 0; 2722 2723 if (fbcon) 2724 console_lock(); 2725 2726 if (resume) { 2727 pci_set_power_state(dev->pdev, PCI_D0); 2728 pci_restore_state(dev->pdev); 2729 r = pci_enable_device(dev->pdev); 2730 if (r) 2731 goto unlock; 2732 } 2733 2734 /* post card */ 2735 if (amdgpu_device_need_post(adev)) { 2736 r = amdgpu_atom_asic_init(adev->mode_info.atom_context); 2737 if (r) 2738 DRM_ERROR("amdgpu asic init failed\n"); 2739 } 2740 2741 r = amdgpu_device_ip_resume(adev); 2742 if (r) { 2743 DRM_ERROR("amdgpu_device_ip_resume failed (%d).\n", r); 2744 goto unlock; 2745 } 2746 amdgpu_fence_driver_resume(adev); 2747 2748 2749 r = amdgpu_device_ip_late_init(adev); 2750 if (r) 2751 goto unlock; 2752 2753 /* pin cursors */ 2754 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { 2755 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); 2756 2757 if (amdgpu_crtc->cursor_bo) { 2758 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 2759 r = amdgpu_bo_reserve(aobj, true); 2760 if (r == 0) { 2761 r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM); 2762 if (r != 0) 2763 DRM_ERROR("Failed to pin cursor BO (%d)\n", r); 2764 amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj); 2765 amdgpu_bo_unreserve(aobj); 2766 } 2767 } 2768 } 2769 r = amdgpu_amdkfd_resume(adev); 2770 if (r) 2771 return r; 2772 2773 /* blat the mode back in */ 2774 if (fbcon) { 2775 if (!amdgpu_device_has_dc_support(adev)) { 2776 /* pre DCE11 */ 2777 drm_helper_resume_force_mode(dev); 2778 2779 /* turn on display hw */ 2780 drm_modeset_lock_all(dev); 2781 list_for_each_entry(connector, &dev->mode_config.connector_list, head) { 2782 drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON); 2783 } 2784 drm_modeset_unlock_all(dev); 2785 } 2786 } 2787 2788 drm_kms_helper_poll_enable(dev); 2789 2790 /* 2791 * Most of the connector probing functions try to acquire runtime pm 2792 * refs to ensure that the GPU is powered on when connector polling is 2793 * performed. Since we're calling this from a runtime PM callback, 2794 * trying to acquire rpm refs will cause us to deadlock. 2795 * 2796 * Since we're guaranteed to be holding the rpm lock, it's safe to 2797 * temporarily disable the rpm helpers so this doesn't deadlock us. 2798 */ 2799 #ifdef CONFIG_PM 2800 dev->dev->power.disable_depth++; 2801 #endif 2802 if (!amdgpu_device_has_dc_support(adev)) 2803 drm_helper_hpd_irq_event(dev); 2804 else 2805 drm_kms_helper_hotplug_event(dev); 2806 #ifdef CONFIG_PM 2807 dev->dev->power.disable_depth--; 2808 #endif 2809 2810 if (fbcon) 2811 amdgpu_fbdev_set_suspend(adev, 0); 2812 2813 unlock: 2814 if (fbcon) 2815 console_unlock(); 2816 2817 return r; 2818 } 2819 2820 /** 2821 * amdgpu_device_ip_check_soft_reset - did soft reset succeed 2822 * 2823 * @adev: amdgpu_device pointer 2824 * 2825 * The list of all the hardware IPs that make up the asic is walked and 2826 * the check_soft_reset callbacks are run. check_soft_reset determines 2827 * if the asic is still hung or not. 2828 * Returns true if any of the IPs are still in a hung state, false if not. 2829 */ 2830 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) 2831 { 2832 int i; 2833 bool asic_hang = false; 2834 2835 if (amdgpu_sriov_vf(adev)) 2836 return true; 2837 2838 if (amdgpu_asic_need_full_reset(adev)) 2839 return true; 2840 2841 for (i = 0; i < adev->num_ip_blocks; i++) { 2842 if (!adev->ip_blocks[i].status.valid) 2843 continue; 2844 if (adev->ip_blocks[i].version->funcs->check_soft_reset) 2845 adev->ip_blocks[i].status.hang = 2846 adev->ip_blocks[i].version->funcs->check_soft_reset(adev); 2847 if (adev->ip_blocks[i].status.hang) { 2848 DRM_INFO("IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); 2849 asic_hang = true; 2850 } 2851 } 2852 return asic_hang; 2853 } 2854 2855 /** 2856 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset 2857 * 2858 * @adev: amdgpu_device pointer 2859 * 2860 * The list of all the hardware IPs that make up the asic is walked and the 2861 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset 2862 * handles any IP specific hardware or software state changes that are 2863 * necessary for a soft reset to succeed. 2864 * Returns 0 on success, negative error code on failure. 2865 */ 2866 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) 2867 { 2868 int i, r = 0; 2869 2870 for (i = 0; i < adev->num_ip_blocks; i++) { 2871 if (!adev->ip_blocks[i].status.valid) 2872 continue; 2873 if (adev->ip_blocks[i].status.hang && 2874 adev->ip_blocks[i].version->funcs->pre_soft_reset) { 2875 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); 2876 if (r) 2877 return r; 2878 } 2879 } 2880 2881 return 0; 2882 } 2883 2884 /** 2885 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed 2886 * 2887 * @adev: amdgpu_device pointer 2888 * 2889 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu 2890 * reset is necessary to recover. 2891 * Returns true if a full asic reset is required, false if not. 2892 */ 2893 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) 2894 { 2895 int i; 2896 2897 if (amdgpu_asic_need_full_reset(adev)) 2898 return true; 2899 2900 for (i = 0; i < adev->num_ip_blocks; i++) { 2901 if (!adev->ip_blocks[i].status.valid) 2902 continue; 2903 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || 2904 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || 2905 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || 2906 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || 2907 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 2908 if (adev->ip_blocks[i].status.hang) { 2909 DRM_INFO("Some block need full reset!\n"); 2910 return true; 2911 } 2912 } 2913 } 2914 return false; 2915 } 2916 2917 /** 2918 * amdgpu_device_ip_soft_reset - do a soft reset 2919 * 2920 * @adev: amdgpu_device pointer 2921 * 2922 * The list of all the hardware IPs that make up the asic is walked and the 2923 * soft_reset callbacks are run if the block is hung. soft_reset handles any 2924 * IP specific hardware or software state changes that are necessary to soft 2925 * reset the IP. 2926 * Returns 0 on success, negative error code on failure. 2927 */ 2928 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) 2929 { 2930 int i, r = 0; 2931 2932 for (i = 0; i < adev->num_ip_blocks; i++) { 2933 if (!adev->ip_blocks[i].status.valid) 2934 continue; 2935 if (adev->ip_blocks[i].status.hang && 2936 adev->ip_blocks[i].version->funcs->soft_reset) { 2937 r = adev->ip_blocks[i].version->funcs->soft_reset(adev); 2938 if (r) 2939 return r; 2940 } 2941 } 2942 2943 return 0; 2944 } 2945 2946 /** 2947 * amdgpu_device_ip_post_soft_reset - clean up from soft reset 2948 * 2949 * @adev: amdgpu_device pointer 2950 * 2951 * The list of all the hardware IPs that make up the asic is walked and the 2952 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset 2953 * handles any IP specific hardware or software state changes that are 2954 * necessary after the IP has been soft reset. 2955 * Returns 0 on success, negative error code on failure. 2956 */ 2957 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) 2958 { 2959 int i, r = 0; 2960 2961 for (i = 0; i < adev->num_ip_blocks; i++) { 2962 if (!adev->ip_blocks[i].status.valid) 2963 continue; 2964 if (adev->ip_blocks[i].status.hang && 2965 adev->ip_blocks[i].version->funcs->post_soft_reset) 2966 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); 2967 if (r) 2968 return r; 2969 } 2970 2971 return 0; 2972 } 2973 2974 /** 2975 * amdgpu_device_recover_vram_from_shadow - restore shadowed VRAM buffers 2976 * 2977 * @adev: amdgpu_device pointer 2978 * @ring: amdgpu_ring for the engine handling the buffer operations 2979 * @bo: amdgpu_bo buffer whose shadow is being restored 2980 * @fence: dma_fence associated with the operation 2981 * 2982 * Restores the VRAM buffer contents from the shadow in GTT. Used to 2983 * restore things like GPUVM page tables after a GPU reset where 2984 * the contents of VRAM might be lost. 2985 * Returns 0 on success, negative error code on failure. 2986 */ 2987 static int amdgpu_device_recover_vram_from_shadow(struct amdgpu_device *adev, 2988 struct amdgpu_ring *ring, 2989 struct amdgpu_bo *bo, 2990 struct dma_fence **fence) 2991 { 2992 uint32_t domain; 2993 int r; 2994 2995 if (!bo->shadow) 2996 return 0; 2997 2998 r = amdgpu_bo_reserve(bo, true); 2999 if (r) 3000 return r; 3001 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); 3002 /* if bo has been evicted, then no need to recover */ 3003 if (domain == AMDGPU_GEM_DOMAIN_VRAM) { 3004 r = amdgpu_bo_validate(bo->shadow); 3005 if (r) { 3006 DRM_ERROR("bo validate failed!\n"); 3007 goto err; 3008 } 3009 3010 r = amdgpu_bo_restore_from_shadow(adev, ring, bo, 3011 NULL, fence, true); 3012 if (r) { 3013 DRM_ERROR("recover page table failed!\n"); 3014 goto err; 3015 } 3016 } 3017 err: 3018 amdgpu_bo_unreserve(bo); 3019 return r; 3020 } 3021 3022 /** 3023 * amdgpu_device_handle_vram_lost - Handle the loss of VRAM contents 3024 * 3025 * @adev: amdgpu_device pointer 3026 * 3027 * Restores the contents of VRAM buffers from the shadows in GTT. Used to 3028 * restore things like GPUVM page tables after a GPU reset where 3029 * the contents of VRAM might be lost. 3030 * Returns 0 on success, 1 on failure. 3031 */ 3032 static int amdgpu_device_handle_vram_lost(struct amdgpu_device *adev) 3033 { 3034 struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring; 3035 struct amdgpu_bo *bo, *tmp; 3036 struct dma_fence *fence = NULL, *next = NULL; 3037 long r = 1; 3038 int i = 0; 3039 long tmo; 3040 3041 if (amdgpu_sriov_runtime(adev)) 3042 tmo = msecs_to_jiffies(amdgpu_lockup_timeout); 3043 else 3044 tmo = msecs_to_jiffies(100); 3045 3046 DRM_INFO("recover vram bo from shadow start\n"); 3047 mutex_lock(&adev->shadow_list_lock); 3048 list_for_each_entry_safe(bo, tmp, &adev->shadow_list, shadow_list) { 3049 next = NULL; 3050 amdgpu_device_recover_vram_from_shadow(adev, ring, bo, &next); 3051 if (fence) { 3052 r = dma_fence_wait_timeout(fence, false, tmo); 3053 if (r == 0) 3054 pr_err("wait fence %p[%d] timeout\n", fence, i); 3055 else if (r < 0) 3056 pr_err("wait fence %p[%d] interrupted\n", fence, i); 3057 if (r < 1) { 3058 dma_fence_put(fence); 3059 fence = next; 3060 break; 3061 } 3062 i++; 3063 } 3064 3065 dma_fence_put(fence); 3066 fence = next; 3067 } 3068 mutex_unlock(&adev->shadow_list_lock); 3069 3070 if (fence) { 3071 r = dma_fence_wait_timeout(fence, false, tmo); 3072 if (r == 0) 3073 pr_err("wait fence %p[%d] timeout\n", fence, i); 3074 else if (r < 0) 3075 pr_err("wait fence %p[%d] interrupted\n", fence, i); 3076 3077 } 3078 dma_fence_put(fence); 3079 3080 if (r > 0) 3081 DRM_INFO("recover vram bo from shadow done\n"); 3082 else 3083 DRM_ERROR("recover vram bo from shadow failed\n"); 3084 3085 return (r > 0) ? 0 : 1; 3086 } 3087 3088 /** 3089 * amdgpu_device_reset - reset ASIC/GPU for bare-metal or passthrough 3090 * 3091 * @adev: amdgpu device pointer 3092 * 3093 * attempt to do soft-reset or full-reset and reinitialize Asic 3094 * return 0 means successed otherwise failed 3095 */ 3096 static int amdgpu_device_reset(struct amdgpu_device *adev) 3097 { 3098 bool need_full_reset, vram_lost = 0; 3099 int r; 3100 3101 need_full_reset = amdgpu_device_ip_need_full_reset(adev); 3102 3103 if (!need_full_reset) { 3104 amdgpu_device_ip_pre_soft_reset(adev); 3105 r = amdgpu_device_ip_soft_reset(adev); 3106 amdgpu_device_ip_post_soft_reset(adev); 3107 if (r || amdgpu_device_ip_check_soft_reset(adev)) { 3108 DRM_INFO("soft reset failed, will fallback to full reset!\n"); 3109 need_full_reset = true; 3110 } 3111 } 3112 3113 if (need_full_reset) { 3114 r = amdgpu_device_ip_suspend(adev); 3115 3116 retry: 3117 r = amdgpu_asic_reset(adev); 3118 /* post card */ 3119 amdgpu_atom_asic_init(adev->mode_info.atom_context); 3120 3121 if (!r) { 3122 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n"); 3123 r = amdgpu_device_ip_resume_phase1(adev); 3124 if (r) 3125 goto out; 3126 3127 vram_lost = amdgpu_device_check_vram_lost(adev); 3128 if (vram_lost) { 3129 DRM_ERROR("VRAM is lost!\n"); 3130 atomic_inc(&adev->vram_lost_counter); 3131 } 3132 3133 r = amdgpu_gtt_mgr_recover( 3134 &adev->mman.bdev.man[TTM_PL_TT]); 3135 if (r) 3136 goto out; 3137 3138 r = amdgpu_device_ip_resume_phase2(adev); 3139 if (r) 3140 goto out; 3141 3142 if (vram_lost) 3143 amdgpu_device_fill_reset_magic(adev); 3144 } 3145 } 3146 3147 out: 3148 if (!r) { 3149 amdgpu_irq_gpu_reset_resume_helper(adev); 3150 r = amdgpu_ib_ring_tests(adev); 3151 if (r) { 3152 dev_err(adev->dev, "ib ring test failed (%d).\n", r); 3153 r = amdgpu_device_ip_suspend(adev); 3154 need_full_reset = true; 3155 goto retry; 3156 } 3157 } 3158 3159 if (!r && ((need_full_reset && !(adev->flags & AMD_IS_APU)) || vram_lost)) 3160 r = amdgpu_device_handle_vram_lost(adev); 3161 3162 return r; 3163 } 3164 3165 /** 3166 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf 3167 * 3168 * @adev: amdgpu device pointer 3169 * @from_hypervisor: request from hypervisor 3170 * 3171 * do VF FLR and reinitialize Asic 3172 * return 0 means successed otherwise failed 3173 */ 3174 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, 3175 bool from_hypervisor) 3176 { 3177 int r; 3178 3179 if (from_hypervisor) 3180 r = amdgpu_virt_request_full_gpu(adev, true); 3181 else 3182 r = amdgpu_virt_reset_gpu(adev); 3183 if (r) 3184 return r; 3185 3186 /* Resume IP prior to SMC */ 3187 r = amdgpu_device_ip_reinit_early_sriov(adev); 3188 if (r) 3189 goto error; 3190 3191 /* we need recover gart prior to run SMC/CP/SDMA resume */ 3192 amdgpu_gtt_mgr_recover(&adev->mman.bdev.man[TTM_PL_TT]); 3193 3194 /* now we are okay to resume SMC/CP/SDMA */ 3195 r = amdgpu_device_ip_reinit_late_sriov(adev); 3196 if (r) 3197 goto error; 3198 3199 amdgpu_irq_gpu_reset_resume_helper(adev); 3200 r = amdgpu_ib_ring_tests(adev); 3201 3202 error: 3203 amdgpu_virt_release_full_gpu(adev, true); 3204 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { 3205 atomic_inc(&adev->vram_lost_counter); 3206 r = amdgpu_device_handle_vram_lost(adev); 3207 } 3208 3209 return r; 3210 } 3211 3212 /** 3213 * amdgpu_device_gpu_recover - reset the asic and recover scheduler 3214 * 3215 * @adev: amdgpu device pointer 3216 * @job: which job trigger hang 3217 * @force: forces reset regardless of amdgpu_gpu_recovery 3218 * 3219 * Attempt to reset the GPU if it has hung (all asics). 3220 * Returns 0 for success or an error on failure. 3221 */ 3222 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, 3223 struct amdgpu_job *job, bool force) 3224 { 3225 int i, r, resched; 3226 3227 if (!force && !amdgpu_device_ip_check_soft_reset(adev)) { 3228 DRM_INFO("No hardware hang detected. Did some blocks stall?\n"); 3229 return 0; 3230 } 3231 3232 if (!force && (amdgpu_gpu_recovery == 0 || 3233 (amdgpu_gpu_recovery == -1 && !amdgpu_sriov_vf(adev)))) { 3234 DRM_INFO("GPU recovery disabled.\n"); 3235 return 0; 3236 } 3237 3238 dev_info(adev->dev, "GPU reset begin!\n"); 3239 3240 mutex_lock(&adev->lock_reset); 3241 atomic_inc(&adev->gpu_reset_counter); 3242 adev->in_gpu_reset = 1; 3243 3244 /* block TTM */ 3245 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); 3246 3247 /* block all schedulers and reset given job's ring */ 3248 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 3249 struct amdgpu_ring *ring = adev->rings[i]; 3250 3251 if (!ring || !ring->sched.thread) 3252 continue; 3253 3254 kthread_park(ring->sched.thread); 3255 3256 if (job && job->ring->idx != i) 3257 continue; 3258 3259 drm_sched_hw_job_reset(&ring->sched, &job->base); 3260 3261 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 3262 amdgpu_fence_driver_force_completion(ring); 3263 } 3264 3265 if (amdgpu_sriov_vf(adev)) 3266 r = amdgpu_device_reset_sriov(adev, job ? false : true); 3267 else 3268 r = amdgpu_device_reset(adev); 3269 3270 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 3271 struct amdgpu_ring *ring = adev->rings[i]; 3272 3273 if (!ring || !ring->sched.thread) 3274 continue; 3275 3276 /* only need recovery sched of the given job's ring 3277 * or all rings (in the case @job is NULL) 3278 * after above amdgpu_reset accomplished 3279 */ 3280 if ((!job || job->ring->idx == i) && !r) 3281 drm_sched_job_recovery(&ring->sched); 3282 3283 kthread_unpark(ring->sched.thread); 3284 } 3285 3286 if (!amdgpu_device_has_dc_support(adev)) { 3287 drm_helper_resume_force_mode(adev->ddev); 3288 } 3289 3290 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched); 3291 3292 if (r) { 3293 /* bad news, how to tell it to userspace ? */ 3294 dev_info(adev->dev, "GPU reset(%d) failed\n", atomic_read(&adev->gpu_reset_counter)); 3295 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); 3296 } else { 3297 dev_info(adev->dev, "GPU reset(%d) successed!\n",atomic_read(&adev->gpu_reset_counter)); 3298 } 3299 3300 amdgpu_vf_error_trans_all(adev); 3301 adev->in_gpu_reset = 0; 3302 mutex_unlock(&adev->lock_reset); 3303 return r; 3304 } 3305 3306 /** 3307 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot 3308 * 3309 * @adev: amdgpu_device pointer 3310 * 3311 * Fetchs and stores in the driver the PCIE capabilities (gen speed 3312 * and lanes) of the slot the device is in. Handles APUs and 3313 * virtualized environments where PCIE config space may not be available. 3314 */ 3315 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) 3316 { 3317 struct pci_dev *pdev; 3318 enum pci_bus_speed speed_cap; 3319 enum pcie_link_width link_width; 3320 3321 if (amdgpu_pcie_gen_cap) 3322 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; 3323 3324 if (amdgpu_pcie_lane_cap) 3325 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; 3326 3327 /* covers APUs as well */ 3328 if (pci_is_root_bus(adev->pdev->bus)) { 3329 if (adev->pm.pcie_gen_mask == 0) 3330 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; 3331 if (adev->pm.pcie_mlw_mask == 0) 3332 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; 3333 return; 3334 } 3335 3336 if (adev->pm.pcie_gen_mask == 0) { 3337 /* asic caps */ 3338 pdev = adev->pdev; 3339 speed_cap = pcie_get_speed_cap(pdev); 3340 if (speed_cap == PCI_SPEED_UNKNOWN) { 3341 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 3342 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 3343 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 3344 } else { 3345 if (speed_cap == PCIE_SPEED_16_0GT) 3346 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 3347 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 3348 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 3349 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); 3350 else if (speed_cap == PCIE_SPEED_8_0GT) 3351 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 3352 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 3353 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 3354 else if (speed_cap == PCIE_SPEED_5_0GT) 3355 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 3356 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); 3357 else 3358 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; 3359 } 3360 /* platform caps */ 3361 pdev = adev->ddev->pdev->bus->self; 3362 speed_cap = pcie_get_speed_cap(pdev); 3363 if (speed_cap == PCI_SPEED_UNKNOWN) { 3364 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 3365 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 3366 } else { 3367 if (speed_cap == PCIE_SPEED_16_0GT) 3368 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 3369 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 3370 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 3371 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); 3372 else if (speed_cap == PCIE_SPEED_8_0GT) 3373 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 3374 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 3375 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); 3376 else if (speed_cap == PCIE_SPEED_5_0GT) 3377 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 3378 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 3379 else 3380 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; 3381 3382 } 3383 } 3384 if (adev->pm.pcie_mlw_mask == 0) { 3385 pdev = adev->ddev->pdev->bus->self; 3386 link_width = pcie_get_width_cap(pdev); 3387 if (link_width == PCIE_LNK_WIDTH_UNKNOWN) { 3388 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; 3389 } else { 3390 switch (link_width) { 3391 case PCIE_LNK_X32: 3392 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | 3393 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 3394 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 3395 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 3396 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 3397 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 3398 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 3399 break; 3400 case PCIE_LNK_X16: 3401 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 3402 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 3403 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 3404 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 3405 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 3406 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 3407 break; 3408 case PCIE_LNK_X12: 3409 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 3410 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 3411 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 3412 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 3413 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 3414 break; 3415 case PCIE_LNK_X8: 3416 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 3417 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 3418 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 3419 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 3420 break; 3421 case PCIE_LNK_X4: 3422 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 3423 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 3424 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 3425 break; 3426 case PCIE_LNK_X2: 3427 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 3428 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 3429 break; 3430 case PCIE_LNK_X1: 3431 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; 3432 break; 3433 default: 3434 break; 3435 } 3436 } 3437 } 3438 } 3439 3440