1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/power_supply.h> 29 #include <linux/kthread.h> 30 #include <linux/module.h> 31 #include <linux/console.h> 32 #include <linux/slab.h> 33 #include <linux/iommu.h> 34 #include <linux/pci.h> 35 36 #include <drm/drm_atomic_helper.h> 37 #include <drm/drm_probe_helper.h> 38 #include <drm/amdgpu_drm.h> 39 #include <linux/vgaarb.h> 40 #include <linux/vga_switcheroo.h> 41 #include <linux/efi.h> 42 #include "amdgpu.h" 43 #include "amdgpu_trace.h" 44 #include "amdgpu_i2c.h" 45 #include "atom.h" 46 #include "amdgpu_atombios.h" 47 #include "amdgpu_atomfirmware.h" 48 #include "amd_pcie.h" 49 #ifdef CONFIG_DRM_AMDGPU_SI 50 #include "si.h" 51 #endif 52 #ifdef CONFIG_DRM_AMDGPU_CIK 53 #include "cik.h" 54 #endif 55 #include "vi.h" 56 #include "soc15.h" 57 #include "nv.h" 58 #include "bif/bif_4_1_d.h" 59 #include <linux/firmware.h> 60 #include "amdgpu_vf_error.h" 61 62 #include "amdgpu_amdkfd.h" 63 #include "amdgpu_pm.h" 64 65 #include "amdgpu_xgmi.h" 66 #include "amdgpu_ras.h" 67 #include "amdgpu_pmu.h" 68 #include "amdgpu_fru_eeprom.h" 69 #include "amdgpu_reset.h" 70 71 #include <linux/suspend.h> 72 #include <drm/task_barrier.h> 73 #include <linux/pm_runtime.h> 74 75 #include <drm/drm_drv.h> 76 77 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 78 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); 79 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 80 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); 81 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); 82 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin"); 83 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin"); 84 85 #define AMDGPU_RESUME_MS 2000 86 #define AMDGPU_MAX_RETRY_LIMIT 2 87 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL) 88 89 const char *amdgpu_asic_name[] = { 90 "TAHITI", 91 "PITCAIRN", 92 "VERDE", 93 "OLAND", 94 "HAINAN", 95 "BONAIRE", 96 "KAVERI", 97 "KABINI", 98 "HAWAII", 99 "MULLINS", 100 "TOPAZ", 101 "TONGA", 102 "FIJI", 103 "CARRIZO", 104 "STONEY", 105 "POLARIS10", 106 "POLARIS11", 107 "POLARIS12", 108 "VEGAM", 109 "VEGA10", 110 "VEGA12", 111 "VEGA20", 112 "RAVEN", 113 "ARCTURUS", 114 "RENOIR", 115 "ALDEBARAN", 116 "NAVI10", 117 "CYAN_SKILLFISH", 118 "NAVI14", 119 "NAVI12", 120 "SIENNA_CICHLID", 121 "NAVY_FLOUNDER", 122 "VANGOGH", 123 "DIMGREY_CAVEFISH", 124 "BEIGE_GOBY", 125 "YELLOW_CARP", 126 "IP DISCOVERY", 127 "LAST", 128 }; 129 130 /** 131 * DOC: pcie_replay_count 132 * 133 * The amdgpu driver provides a sysfs API for reporting the total number 134 * of PCIe replays (NAKs) 135 * The file pcie_replay_count is used for this and returns the total 136 * number of replays as a sum of the NAKs generated and NAKs received 137 */ 138 139 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, 140 struct device_attribute *attr, char *buf) 141 { 142 struct drm_device *ddev = dev_get_drvdata(dev); 143 struct amdgpu_device *adev = drm_to_adev(ddev); 144 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); 145 146 return sysfs_emit(buf, "%llu\n", cnt); 147 } 148 149 static DEVICE_ATTR(pcie_replay_count, S_IRUGO, 150 amdgpu_device_get_pcie_replay_count, NULL); 151 152 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); 153 154 /** 155 * DOC: product_name 156 * 157 * The amdgpu driver provides a sysfs API for reporting the product name 158 * for the device 159 * The file serial_number is used for this and returns the product name 160 * as returned from the FRU. 161 * NOTE: This is only available for certain server cards 162 */ 163 164 static ssize_t amdgpu_device_get_product_name(struct device *dev, 165 struct device_attribute *attr, char *buf) 166 { 167 struct drm_device *ddev = dev_get_drvdata(dev); 168 struct amdgpu_device *adev = drm_to_adev(ddev); 169 170 return sysfs_emit(buf, "%s\n", adev->product_name); 171 } 172 173 static DEVICE_ATTR(product_name, S_IRUGO, 174 amdgpu_device_get_product_name, NULL); 175 176 /** 177 * DOC: product_number 178 * 179 * The amdgpu driver provides a sysfs API for reporting the part number 180 * for the device 181 * The file serial_number is used for this and returns the part number 182 * as returned from the FRU. 183 * NOTE: This is only available for certain server cards 184 */ 185 186 static ssize_t amdgpu_device_get_product_number(struct device *dev, 187 struct device_attribute *attr, char *buf) 188 { 189 struct drm_device *ddev = dev_get_drvdata(dev); 190 struct amdgpu_device *adev = drm_to_adev(ddev); 191 192 return sysfs_emit(buf, "%s\n", adev->product_number); 193 } 194 195 static DEVICE_ATTR(product_number, S_IRUGO, 196 amdgpu_device_get_product_number, NULL); 197 198 /** 199 * DOC: serial_number 200 * 201 * The amdgpu driver provides a sysfs API for reporting the serial number 202 * for the device 203 * The file serial_number is used for this and returns the serial number 204 * as returned from the FRU. 205 * NOTE: This is only available for certain server cards 206 */ 207 208 static ssize_t amdgpu_device_get_serial_number(struct device *dev, 209 struct device_attribute *attr, char *buf) 210 { 211 struct drm_device *ddev = dev_get_drvdata(dev); 212 struct amdgpu_device *adev = drm_to_adev(ddev); 213 214 return sysfs_emit(buf, "%s\n", adev->serial); 215 } 216 217 static DEVICE_ATTR(serial_number, S_IRUGO, 218 amdgpu_device_get_serial_number, NULL); 219 220 /** 221 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control 222 * 223 * @dev: drm_device pointer 224 * 225 * Returns true if the device is a dGPU with ATPX power control, 226 * otherwise return false. 227 */ 228 bool amdgpu_device_supports_px(struct drm_device *dev) 229 { 230 struct amdgpu_device *adev = drm_to_adev(dev); 231 232 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) 233 return true; 234 return false; 235 } 236 237 /** 238 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources 239 * 240 * @dev: drm_device pointer 241 * 242 * Returns true if the device is a dGPU with ACPI power control, 243 * otherwise return false. 244 */ 245 bool amdgpu_device_supports_boco(struct drm_device *dev) 246 { 247 struct amdgpu_device *adev = drm_to_adev(dev); 248 249 if (adev->has_pr3 || 250 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) 251 return true; 252 return false; 253 } 254 255 /** 256 * amdgpu_device_supports_baco - Does the device support BACO 257 * 258 * @dev: drm_device pointer 259 * 260 * Returns true if the device supporte BACO, 261 * otherwise return false. 262 */ 263 bool amdgpu_device_supports_baco(struct drm_device *dev) 264 { 265 struct amdgpu_device *adev = drm_to_adev(dev); 266 267 return amdgpu_asic_supports_baco(adev); 268 } 269 270 /** 271 * amdgpu_device_supports_smart_shift - Is the device dGPU with 272 * smart shift support 273 * 274 * @dev: drm_device pointer 275 * 276 * Returns true if the device is a dGPU with Smart Shift support, 277 * otherwise returns false. 278 */ 279 bool amdgpu_device_supports_smart_shift(struct drm_device *dev) 280 { 281 return (amdgpu_device_supports_boco(dev) && 282 amdgpu_acpi_is_power_shift_control_supported()); 283 } 284 285 /* 286 * VRAM access helper functions 287 */ 288 289 /** 290 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA 291 * 292 * @adev: amdgpu_device pointer 293 * @pos: offset of the buffer in vram 294 * @buf: virtual address of the buffer in system memory 295 * @size: read/write size, sizeof(@buf) must > @size 296 * @write: true - write to vram, otherwise - read from vram 297 */ 298 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos, 299 void *buf, size_t size, bool write) 300 { 301 unsigned long flags; 302 uint32_t hi = ~0, tmp = 0; 303 uint32_t *data = buf; 304 uint64_t last; 305 int idx; 306 307 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 308 return; 309 310 BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)); 311 312 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 313 for (last = pos + size; pos < last; pos += 4) { 314 tmp = pos >> 31; 315 316 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000); 317 if (tmp != hi) { 318 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp); 319 hi = tmp; 320 } 321 if (write) 322 WREG32_NO_KIQ(mmMM_DATA, *data++); 323 else 324 *data++ = RREG32_NO_KIQ(mmMM_DATA); 325 } 326 327 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 328 drm_dev_exit(idx); 329 } 330 331 /** 332 * amdgpu_device_aper_access - access vram by vram aperature 333 * 334 * @adev: amdgpu_device pointer 335 * @pos: offset of the buffer in vram 336 * @buf: virtual address of the buffer in system memory 337 * @size: read/write size, sizeof(@buf) must > @size 338 * @write: true - write to vram, otherwise - read from vram 339 * 340 * The return value means how many bytes have been transferred. 341 */ 342 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos, 343 void *buf, size_t size, bool write) 344 { 345 #ifdef CONFIG_64BIT 346 void __iomem *addr; 347 size_t count = 0; 348 uint64_t last; 349 350 if (!adev->mman.aper_base_kaddr) 351 return 0; 352 353 last = min(pos + size, adev->gmc.visible_vram_size); 354 if (last > pos) { 355 addr = adev->mman.aper_base_kaddr + pos; 356 count = last - pos; 357 358 if (write) { 359 memcpy_toio(addr, buf, count); 360 mb(); 361 amdgpu_device_flush_hdp(adev, NULL); 362 } else { 363 amdgpu_device_invalidate_hdp(adev, NULL); 364 mb(); 365 memcpy_fromio(buf, addr, count); 366 } 367 368 } 369 370 return count; 371 #else 372 return 0; 373 #endif 374 } 375 376 /** 377 * amdgpu_device_vram_access - read/write a buffer in vram 378 * 379 * @adev: amdgpu_device pointer 380 * @pos: offset of the buffer in vram 381 * @buf: virtual address of the buffer in system memory 382 * @size: read/write size, sizeof(@buf) must > @size 383 * @write: true - write to vram, otherwise - read from vram 384 */ 385 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, 386 void *buf, size_t size, bool write) 387 { 388 size_t count; 389 390 /* try to using vram apreature to access vram first */ 391 count = amdgpu_device_aper_access(adev, pos, buf, size, write); 392 size -= count; 393 if (size) { 394 /* using MM to access rest vram */ 395 pos += count; 396 buf += count; 397 amdgpu_device_mm_access(adev, pos, buf, size, write); 398 } 399 } 400 401 /* 402 * register access helper functions. 403 */ 404 405 /* Check if hw access should be skipped because of hotplug or device error */ 406 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev) 407 { 408 if (adev->no_hw_access) 409 return true; 410 411 #ifdef CONFIG_LOCKDEP 412 /* 413 * This is a bit complicated to understand, so worth a comment. What we assert 414 * here is that the GPU reset is not running on another thread in parallel. 415 * 416 * For this we trylock the read side of the reset semaphore, if that succeeds 417 * we know that the reset is not running in paralell. 418 * 419 * If the trylock fails we assert that we are either already holding the read 420 * side of the lock or are the reset thread itself and hold the write side of 421 * the lock. 422 */ 423 if (in_task()) { 424 if (down_read_trylock(&adev->reset_domain->sem)) 425 up_read(&adev->reset_domain->sem); 426 else 427 lockdep_assert_held(&adev->reset_domain->sem); 428 } 429 #endif 430 return false; 431 } 432 433 /** 434 * amdgpu_device_rreg - read a memory mapped IO or indirect register 435 * 436 * @adev: amdgpu_device pointer 437 * @reg: dword aligned register offset 438 * @acc_flags: access flags which require special behavior 439 * 440 * Returns the 32 bit value from the offset specified. 441 */ 442 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, 443 uint32_t reg, uint32_t acc_flags) 444 { 445 uint32_t ret; 446 447 if (amdgpu_device_skip_hw_access(adev)) 448 return 0; 449 450 if ((reg * 4) < adev->rmmio_size) { 451 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 452 amdgpu_sriov_runtime(adev) && 453 down_read_trylock(&adev->reset_domain->sem)) { 454 ret = amdgpu_kiq_rreg(adev, reg); 455 up_read(&adev->reset_domain->sem); 456 } else { 457 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 458 } 459 } else { 460 ret = adev->pcie_rreg(adev, reg * 4); 461 } 462 463 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); 464 465 return ret; 466 } 467 468 /* 469 * MMIO register read with bytes helper functions 470 * @offset:bytes offset from MMIO start 471 * 472 */ 473 474 /** 475 * amdgpu_mm_rreg8 - read a memory mapped IO register 476 * 477 * @adev: amdgpu_device pointer 478 * @offset: byte aligned register offset 479 * 480 * Returns the 8 bit value from the offset specified. 481 */ 482 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) 483 { 484 if (amdgpu_device_skip_hw_access(adev)) 485 return 0; 486 487 if (offset < adev->rmmio_size) 488 return (readb(adev->rmmio + offset)); 489 BUG(); 490 } 491 492 /* 493 * MMIO register write with bytes helper functions 494 * @offset:bytes offset from MMIO start 495 * @value: the value want to be written to the register 496 * 497 */ 498 /** 499 * amdgpu_mm_wreg8 - read a memory mapped IO register 500 * 501 * @adev: amdgpu_device pointer 502 * @offset: byte aligned register offset 503 * @value: 8 bit value to write 504 * 505 * Writes the value specified to the offset specified. 506 */ 507 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) 508 { 509 if (amdgpu_device_skip_hw_access(adev)) 510 return; 511 512 if (offset < adev->rmmio_size) 513 writeb(value, adev->rmmio + offset); 514 else 515 BUG(); 516 } 517 518 /** 519 * amdgpu_device_wreg - write to a memory mapped IO or indirect register 520 * 521 * @adev: amdgpu_device pointer 522 * @reg: dword aligned register offset 523 * @v: 32 bit value to write to the register 524 * @acc_flags: access flags which require special behavior 525 * 526 * Writes the value specified to the offset specified. 527 */ 528 void amdgpu_device_wreg(struct amdgpu_device *adev, 529 uint32_t reg, uint32_t v, 530 uint32_t acc_flags) 531 { 532 if (amdgpu_device_skip_hw_access(adev)) 533 return; 534 535 if ((reg * 4) < adev->rmmio_size) { 536 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 537 amdgpu_sriov_runtime(adev) && 538 down_read_trylock(&adev->reset_domain->sem)) { 539 amdgpu_kiq_wreg(adev, reg, v); 540 up_read(&adev->reset_domain->sem); 541 } else { 542 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 543 } 544 } else { 545 adev->pcie_wreg(adev, reg * 4, v); 546 } 547 548 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); 549 } 550 551 /** 552 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range 553 * 554 * @adev: amdgpu_device pointer 555 * @reg: mmio/rlc register 556 * @v: value to write 557 * 558 * this function is invoked only for the debugfs register access 559 */ 560 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, 561 uint32_t reg, uint32_t v) 562 { 563 if (amdgpu_device_skip_hw_access(adev)) 564 return; 565 566 if (amdgpu_sriov_fullaccess(adev) && 567 adev->gfx.rlc.funcs && 568 adev->gfx.rlc.funcs->is_rlcg_access_range) { 569 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) 570 return amdgpu_sriov_wreg(adev, reg, v, 0, 0); 571 } else if ((reg * 4) >= adev->rmmio_size) { 572 adev->pcie_wreg(adev, reg * 4, v); 573 } else { 574 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 575 } 576 } 577 578 /** 579 * amdgpu_mm_rdoorbell - read a doorbell dword 580 * 581 * @adev: amdgpu_device pointer 582 * @index: doorbell index 583 * 584 * Returns the value in the doorbell aperture at the 585 * requested doorbell index (CIK). 586 */ 587 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) 588 { 589 if (amdgpu_device_skip_hw_access(adev)) 590 return 0; 591 592 if (index < adev->doorbell.num_doorbells) { 593 return readl(adev->doorbell.ptr + index); 594 } else { 595 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 596 return 0; 597 } 598 } 599 600 /** 601 * amdgpu_mm_wdoorbell - write a doorbell dword 602 * 603 * @adev: amdgpu_device pointer 604 * @index: doorbell index 605 * @v: value to write 606 * 607 * Writes @v to the doorbell aperture at the 608 * requested doorbell index (CIK). 609 */ 610 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) 611 { 612 if (amdgpu_device_skip_hw_access(adev)) 613 return; 614 615 if (index < adev->doorbell.num_doorbells) { 616 writel(v, adev->doorbell.ptr + index); 617 } else { 618 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 619 } 620 } 621 622 /** 623 * amdgpu_mm_rdoorbell64 - read a doorbell Qword 624 * 625 * @adev: amdgpu_device pointer 626 * @index: doorbell index 627 * 628 * Returns the value in the doorbell aperture at the 629 * requested doorbell index (VEGA10+). 630 */ 631 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) 632 { 633 if (amdgpu_device_skip_hw_access(adev)) 634 return 0; 635 636 if (index < adev->doorbell.num_doorbells) { 637 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index)); 638 } else { 639 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 640 return 0; 641 } 642 } 643 644 /** 645 * amdgpu_mm_wdoorbell64 - write a doorbell Qword 646 * 647 * @adev: amdgpu_device pointer 648 * @index: doorbell index 649 * @v: value to write 650 * 651 * Writes @v to the doorbell aperture at the 652 * requested doorbell index (VEGA10+). 653 */ 654 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) 655 { 656 if (amdgpu_device_skip_hw_access(adev)) 657 return; 658 659 if (index < adev->doorbell.num_doorbells) { 660 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v); 661 } else { 662 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 663 } 664 } 665 666 /** 667 * amdgpu_device_indirect_rreg - read an indirect register 668 * 669 * @adev: amdgpu_device pointer 670 * @pcie_index: mmio register offset 671 * @pcie_data: mmio register offset 672 * @reg_addr: indirect register address to read from 673 * 674 * Returns the value of indirect register @reg_addr 675 */ 676 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, 677 u32 pcie_index, u32 pcie_data, 678 u32 reg_addr) 679 { 680 unsigned long flags; 681 u32 r; 682 void __iomem *pcie_index_offset; 683 void __iomem *pcie_data_offset; 684 685 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 686 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 687 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 688 689 writel(reg_addr, pcie_index_offset); 690 readl(pcie_index_offset); 691 r = readl(pcie_data_offset); 692 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 693 694 return r; 695 } 696 697 /** 698 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register 699 * 700 * @adev: amdgpu_device pointer 701 * @pcie_index: mmio register offset 702 * @pcie_data: mmio register offset 703 * @reg_addr: indirect register address to read from 704 * 705 * Returns the value of indirect register @reg_addr 706 */ 707 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, 708 u32 pcie_index, u32 pcie_data, 709 u32 reg_addr) 710 { 711 unsigned long flags; 712 u64 r; 713 void __iomem *pcie_index_offset; 714 void __iomem *pcie_data_offset; 715 716 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 717 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 718 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 719 720 /* read low 32 bits */ 721 writel(reg_addr, pcie_index_offset); 722 readl(pcie_index_offset); 723 r = readl(pcie_data_offset); 724 /* read high 32 bits */ 725 writel(reg_addr + 4, pcie_index_offset); 726 readl(pcie_index_offset); 727 r |= ((u64)readl(pcie_data_offset) << 32); 728 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 729 730 return r; 731 } 732 733 /** 734 * amdgpu_device_indirect_wreg - write an indirect register address 735 * 736 * @adev: amdgpu_device pointer 737 * @pcie_index: mmio register offset 738 * @pcie_data: mmio register offset 739 * @reg_addr: indirect register offset 740 * @reg_data: indirect register data 741 * 742 */ 743 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, 744 u32 pcie_index, u32 pcie_data, 745 u32 reg_addr, u32 reg_data) 746 { 747 unsigned long flags; 748 void __iomem *pcie_index_offset; 749 void __iomem *pcie_data_offset; 750 751 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 752 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 753 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 754 755 writel(reg_addr, pcie_index_offset); 756 readl(pcie_index_offset); 757 writel(reg_data, pcie_data_offset); 758 readl(pcie_data_offset); 759 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 760 } 761 762 /** 763 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address 764 * 765 * @adev: amdgpu_device pointer 766 * @pcie_index: mmio register offset 767 * @pcie_data: mmio register offset 768 * @reg_addr: indirect register offset 769 * @reg_data: indirect register data 770 * 771 */ 772 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, 773 u32 pcie_index, u32 pcie_data, 774 u32 reg_addr, u64 reg_data) 775 { 776 unsigned long flags; 777 void __iomem *pcie_index_offset; 778 void __iomem *pcie_data_offset; 779 780 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 781 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 782 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 783 784 /* write low 32 bits */ 785 writel(reg_addr, pcie_index_offset); 786 readl(pcie_index_offset); 787 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 788 readl(pcie_data_offset); 789 /* write high 32 bits */ 790 writel(reg_addr + 4, pcie_index_offset); 791 readl(pcie_index_offset); 792 writel((u32)(reg_data >> 32), pcie_data_offset); 793 readl(pcie_data_offset); 794 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 795 } 796 797 /** 798 * amdgpu_invalid_rreg - dummy reg read function 799 * 800 * @adev: amdgpu_device pointer 801 * @reg: offset of register 802 * 803 * Dummy register read function. Used for register blocks 804 * that certain asics don't have (all asics). 805 * Returns the value in the register. 806 */ 807 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) 808 { 809 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg); 810 BUG(); 811 return 0; 812 } 813 814 /** 815 * amdgpu_invalid_wreg - dummy reg write function 816 * 817 * @adev: amdgpu_device pointer 818 * @reg: offset of register 819 * @v: value to write to the register 820 * 821 * Dummy register read function. Used for register blocks 822 * that certain asics don't have (all asics). 823 */ 824 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 825 { 826 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n", 827 reg, v); 828 BUG(); 829 } 830 831 /** 832 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function 833 * 834 * @adev: amdgpu_device pointer 835 * @reg: offset of register 836 * 837 * Dummy register read function. Used for register blocks 838 * that certain asics don't have (all asics). 839 * Returns the value in the register. 840 */ 841 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg) 842 { 843 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg); 844 BUG(); 845 return 0; 846 } 847 848 /** 849 * amdgpu_invalid_wreg64 - dummy reg write function 850 * 851 * @adev: amdgpu_device pointer 852 * @reg: offset of register 853 * @v: value to write to the register 854 * 855 * Dummy register read function. Used for register blocks 856 * that certain asics don't have (all asics). 857 */ 858 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v) 859 { 860 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n", 861 reg, v); 862 BUG(); 863 } 864 865 /** 866 * amdgpu_block_invalid_rreg - dummy reg read function 867 * 868 * @adev: amdgpu_device pointer 869 * @block: offset of instance 870 * @reg: offset of register 871 * 872 * Dummy register read function. Used for register blocks 873 * that certain asics don't have (all asics). 874 * Returns the value in the register. 875 */ 876 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, 877 uint32_t block, uint32_t reg) 878 { 879 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n", 880 reg, block); 881 BUG(); 882 return 0; 883 } 884 885 /** 886 * amdgpu_block_invalid_wreg - dummy reg write function 887 * 888 * @adev: amdgpu_device pointer 889 * @block: offset of instance 890 * @reg: offset of register 891 * @v: value to write to the register 892 * 893 * Dummy register read function. Used for register blocks 894 * that certain asics don't have (all asics). 895 */ 896 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, 897 uint32_t block, 898 uint32_t reg, uint32_t v) 899 { 900 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n", 901 reg, block, v); 902 BUG(); 903 } 904 905 /** 906 * amdgpu_device_asic_init - Wrapper for atom asic_init 907 * 908 * @adev: amdgpu_device pointer 909 * 910 * Does any asic specific work and then calls atom asic init. 911 */ 912 static int amdgpu_device_asic_init(struct amdgpu_device *adev) 913 { 914 amdgpu_asic_pre_asic_init(adev); 915 916 if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)) 917 return amdgpu_atomfirmware_asic_init(adev, true); 918 else 919 return amdgpu_atom_asic_init(adev->mode_info.atom_context); 920 } 921 922 /** 923 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page 924 * 925 * @adev: amdgpu_device pointer 926 * 927 * Allocates a scratch page of VRAM for use by various things in the 928 * driver. 929 */ 930 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) 931 { 932 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, 933 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 934 &adev->vram_scratch.robj, 935 &adev->vram_scratch.gpu_addr, 936 (void **)&adev->vram_scratch.ptr); 937 } 938 939 /** 940 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page 941 * 942 * @adev: amdgpu_device pointer 943 * 944 * Frees the VRAM scratch page. 945 */ 946 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev) 947 { 948 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL); 949 } 950 951 /** 952 * amdgpu_device_program_register_sequence - program an array of registers. 953 * 954 * @adev: amdgpu_device pointer 955 * @registers: pointer to the register array 956 * @array_size: size of the register array 957 * 958 * Programs an array or registers with and and or masks. 959 * This is a helper for setting golden registers. 960 */ 961 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, 962 const u32 *registers, 963 const u32 array_size) 964 { 965 u32 tmp, reg, and_mask, or_mask; 966 int i; 967 968 if (array_size % 3) 969 return; 970 971 for (i = 0; i < array_size; i +=3) { 972 reg = registers[i + 0]; 973 and_mask = registers[i + 1]; 974 or_mask = registers[i + 2]; 975 976 if (and_mask == 0xffffffff) { 977 tmp = or_mask; 978 } else { 979 tmp = RREG32(reg); 980 tmp &= ~and_mask; 981 if (adev->family >= AMDGPU_FAMILY_AI) 982 tmp |= (or_mask & and_mask); 983 else 984 tmp |= or_mask; 985 } 986 WREG32(reg, tmp); 987 } 988 } 989 990 /** 991 * amdgpu_device_pci_config_reset - reset the GPU 992 * 993 * @adev: amdgpu_device pointer 994 * 995 * Resets the GPU using the pci config reset sequence. 996 * Only applicable to asics prior to vega10. 997 */ 998 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) 999 { 1000 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); 1001 } 1002 1003 /** 1004 * amdgpu_device_pci_reset - reset the GPU using generic PCI means 1005 * 1006 * @adev: amdgpu_device pointer 1007 * 1008 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.). 1009 */ 1010 int amdgpu_device_pci_reset(struct amdgpu_device *adev) 1011 { 1012 return pci_reset_function(adev->pdev); 1013 } 1014 1015 /* 1016 * GPU doorbell aperture helpers function. 1017 */ 1018 /** 1019 * amdgpu_device_doorbell_init - Init doorbell driver information. 1020 * 1021 * @adev: amdgpu_device pointer 1022 * 1023 * Init doorbell driver information (CIK) 1024 * Returns 0 on success, error on failure. 1025 */ 1026 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev) 1027 { 1028 1029 /* No doorbell on SI hardware generation */ 1030 if (adev->asic_type < CHIP_BONAIRE) { 1031 adev->doorbell.base = 0; 1032 adev->doorbell.size = 0; 1033 adev->doorbell.num_doorbells = 0; 1034 adev->doorbell.ptr = NULL; 1035 return 0; 1036 } 1037 1038 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) 1039 return -EINVAL; 1040 1041 amdgpu_asic_init_doorbell_index(adev); 1042 1043 /* doorbell bar mapping */ 1044 adev->doorbell.base = pci_resource_start(adev->pdev, 2); 1045 adev->doorbell.size = pci_resource_len(adev->pdev, 2); 1046 1047 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32), 1048 adev->doorbell_index.max_assignment+1); 1049 if (adev->doorbell.num_doorbells == 0) 1050 return -EINVAL; 1051 1052 /* For Vega, reserve and map two pages on doorbell BAR since SDMA 1053 * paging queue doorbell use the second page. The 1054 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the 1055 * doorbells are in the first page. So with paging queue enabled, 1056 * the max num_doorbells should + 1 page (0x400 in dword) 1057 */ 1058 if (adev->asic_type >= CHIP_VEGA10) 1059 adev->doorbell.num_doorbells += 0x400; 1060 1061 adev->doorbell.ptr = ioremap(adev->doorbell.base, 1062 adev->doorbell.num_doorbells * 1063 sizeof(u32)); 1064 if (adev->doorbell.ptr == NULL) 1065 return -ENOMEM; 1066 1067 return 0; 1068 } 1069 1070 /** 1071 * amdgpu_device_doorbell_fini - Tear down doorbell driver information. 1072 * 1073 * @adev: amdgpu_device pointer 1074 * 1075 * Tear down doorbell driver information (CIK) 1076 */ 1077 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev) 1078 { 1079 iounmap(adev->doorbell.ptr); 1080 adev->doorbell.ptr = NULL; 1081 } 1082 1083 1084 1085 /* 1086 * amdgpu_device_wb_*() 1087 * Writeback is the method by which the GPU updates special pages in memory 1088 * with the status of certain GPU events (fences, ring pointers,etc.). 1089 */ 1090 1091 /** 1092 * amdgpu_device_wb_fini - Disable Writeback and free memory 1093 * 1094 * @adev: amdgpu_device pointer 1095 * 1096 * Disables Writeback and frees the Writeback memory (all asics). 1097 * Used at driver shutdown. 1098 */ 1099 static void amdgpu_device_wb_fini(struct amdgpu_device *adev) 1100 { 1101 if (adev->wb.wb_obj) { 1102 amdgpu_bo_free_kernel(&adev->wb.wb_obj, 1103 &adev->wb.gpu_addr, 1104 (void **)&adev->wb.wb); 1105 adev->wb.wb_obj = NULL; 1106 } 1107 } 1108 1109 /** 1110 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory 1111 * 1112 * @adev: amdgpu_device pointer 1113 * 1114 * Initializes writeback and allocates writeback memory (all asics). 1115 * Used at driver startup. 1116 * Returns 0 on success or an -error on failure. 1117 */ 1118 static int amdgpu_device_wb_init(struct amdgpu_device *adev) 1119 { 1120 int r; 1121 1122 if (adev->wb.wb_obj == NULL) { 1123 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */ 1124 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, 1125 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1126 &adev->wb.wb_obj, &adev->wb.gpu_addr, 1127 (void **)&adev->wb.wb); 1128 if (r) { 1129 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); 1130 return r; 1131 } 1132 1133 adev->wb.num_wb = AMDGPU_MAX_WB; 1134 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); 1135 1136 /* clear wb memory */ 1137 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); 1138 } 1139 1140 return 0; 1141 } 1142 1143 /** 1144 * amdgpu_device_wb_get - Allocate a wb entry 1145 * 1146 * @adev: amdgpu_device pointer 1147 * @wb: wb index 1148 * 1149 * Allocate a wb slot for use by the driver (all asics). 1150 * Returns 0 on success or -EINVAL on failure. 1151 */ 1152 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) 1153 { 1154 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); 1155 1156 if (offset < adev->wb.num_wb) { 1157 __set_bit(offset, adev->wb.used); 1158 *wb = offset << 3; /* convert to dw offset */ 1159 return 0; 1160 } else { 1161 return -EINVAL; 1162 } 1163 } 1164 1165 /** 1166 * amdgpu_device_wb_free - Free a wb entry 1167 * 1168 * @adev: amdgpu_device pointer 1169 * @wb: wb index 1170 * 1171 * Free a wb slot allocated for use by the driver (all asics) 1172 */ 1173 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) 1174 { 1175 wb >>= 3; 1176 if (wb < adev->wb.num_wb) 1177 __clear_bit(wb, adev->wb.used); 1178 } 1179 1180 /** 1181 * amdgpu_device_resize_fb_bar - try to resize FB BAR 1182 * 1183 * @adev: amdgpu_device pointer 1184 * 1185 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not 1186 * to fail, but if any of the BARs is not accessible after the size we abort 1187 * driver loading by returning -ENODEV. 1188 */ 1189 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) 1190 { 1191 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); 1192 struct pci_bus *root; 1193 struct resource *res; 1194 unsigned i; 1195 u16 cmd; 1196 int r; 1197 1198 /* Bypass for VF */ 1199 if (amdgpu_sriov_vf(adev)) 1200 return 0; 1201 1202 /* skip if the bios has already enabled large BAR */ 1203 if (adev->gmc.real_vram_size && 1204 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) 1205 return 0; 1206 1207 /* Check if the root BUS has 64bit memory resources */ 1208 root = adev->pdev->bus; 1209 while (root->parent) 1210 root = root->parent; 1211 1212 pci_bus_for_each_resource(root, res, i) { 1213 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 1214 res->start > 0x100000000ull) 1215 break; 1216 } 1217 1218 /* Trying to resize is pointless without a root hub window above 4GB */ 1219 if (!res) 1220 return 0; 1221 1222 /* Limit the BAR size to what is available */ 1223 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1, 1224 rbar_size); 1225 1226 /* Disable memory decoding while we change the BAR addresses and size */ 1227 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); 1228 pci_write_config_word(adev->pdev, PCI_COMMAND, 1229 cmd & ~PCI_COMMAND_MEMORY); 1230 1231 /* Free the VRAM and doorbell BAR, we most likely need to move both. */ 1232 amdgpu_device_doorbell_fini(adev); 1233 if (adev->asic_type >= CHIP_BONAIRE) 1234 pci_release_resource(adev->pdev, 2); 1235 1236 pci_release_resource(adev->pdev, 0); 1237 1238 r = pci_resize_resource(adev->pdev, 0, rbar_size); 1239 if (r == -ENOSPC) 1240 DRM_INFO("Not enough PCI address space for a large BAR."); 1241 else if (r && r != -ENOTSUPP) 1242 DRM_ERROR("Problem resizing BAR0 (%d).", r); 1243 1244 pci_assign_unassigned_bus_resources(adev->pdev->bus); 1245 1246 /* When the doorbell or fb BAR isn't available we have no chance of 1247 * using the device. 1248 */ 1249 r = amdgpu_device_doorbell_init(adev); 1250 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) 1251 return -ENODEV; 1252 1253 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); 1254 1255 return 0; 1256 } 1257 1258 /* 1259 * GPU helpers function. 1260 */ 1261 /** 1262 * amdgpu_device_need_post - check if the hw need post or not 1263 * 1264 * @adev: amdgpu_device pointer 1265 * 1266 * Check if the asic has been initialized (all asics) at driver startup 1267 * or post is needed if hw reset is performed. 1268 * Returns true if need or false if not. 1269 */ 1270 bool amdgpu_device_need_post(struct amdgpu_device *adev) 1271 { 1272 uint32_t reg; 1273 1274 if (amdgpu_sriov_vf(adev)) 1275 return false; 1276 1277 if (amdgpu_passthrough(adev)) { 1278 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot 1279 * some old smc fw still need driver do vPost otherwise gpu hang, while 1280 * those smc fw version above 22.15 doesn't have this flaw, so we force 1281 * vpost executed for smc version below 22.15 1282 */ 1283 if (adev->asic_type == CHIP_FIJI) { 1284 int err; 1285 uint32_t fw_ver; 1286 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); 1287 /* force vPost if error occured */ 1288 if (err) 1289 return true; 1290 1291 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); 1292 if (fw_ver < 0x00160e00) 1293 return true; 1294 } 1295 } 1296 1297 /* Don't post if we need to reset whole hive on init */ 1298 if (adev->gmc.xgmi.pending_reset) 1299 return false; 1300 1301 if (adev->has_hw_reset) { 1302 adev->has_hw_reset = false; 1303 return true; 1304 } 1305 1306 /* bios scratch used on CIK+ */ 1307 if (adev->asic_type >= CHIP_BONAIRE) 1308 return amdgpu_atombios_scratch_need_asic_init(adev); 1309 1310 /* check MEM_SIZE for older asics */ 1311 reg = amdgpu_asic_get_config_memsize(adev); 1312 1313 if ((reg != 0) && (reg != 0xffffffff)) 1314 return false; 1315 1316 return true; 1317 } 1318 1319 /** 1320 * amdgpu_device_should_use_aspm - check if the device should program ASPM 1321 * 1322 * @adev: amdgpu_device pointer 1323 * 1324 * Confirm whether the module parameter and pcie bridge agree that ASPM should 1325 * be set for this device. 1326 * 1327 * Returns true if it should be used or false if not. 1328 */ 1329 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) 1330 { 1331 switch (amdgpu_aspm) { 1332 case -1: 1333 break; 1334 case 0: 1335 return false; 1336 case 1: 1337 return true; 1338 default: 1339 return false; 1340 } 1341 return pcie_aspm_enabled(adev->pdev); 1342 } 1343 1344 /* if we get transitioned to only one device, take VGA back */ 1345 /** 1346 * amdgpu_device_vga_set_decode - enable/disable vga decode 1347 * 1348 * @pdev: PCI device pointer 1349 * @state: enable/disable vga decode 1350 * 1351 * Enable/disable vga decode (all asics). 1352 * Returns VGA resource flags. 1353 */ 1354 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev, 1355 bool state) 1356 { 1357 struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev)); 1358 amdgpu_asic_set_vga_state(adev, state); 1359 if (state) 1360 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 1361 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1362 else 1363 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1364 } 1365 1366 /** 1367 * amdgpu_device_check_block_size - validate the vm block size 1368 * 1369 * @adev: amdgpu_device pointer 1370 * 1371 * Validates the vm block size specified via module parameter. 1372 * The vm block size defines number of bits in page table versus page directory, 1373 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1374 * page table and the remaining bits are in the page directory. 1375 */ 1376 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) 1377 { 1378 /* defines number of bits in page table versus page directory, 1379 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1380 * page table and the remaining bits are in the page directory */ 1381 if (amdgpu_vm_block_size == -1) 1382 return; 1383 1384 if (amdgpu_vm_block_size < 9) { 1385 dev_warn(adev->dev, "VM page table size (%d) too small\n", 1386 amdgpu_vm_block_size); 1387 amdgpu_vm_block_size = -1; 1388 } 1389 } 1390 1391 /** 1392 * amdgpu_device_check_vm_size - validate the vm size 1393 * 1394 * @adev: amdgpu_device pointer 1395 * 1396 * Validates the vm size in GB specified via module parameter. 1397 * The VM size is the size of the GPU virtual memory space in GB. 1398 */ 1399 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) 1400 { 1401 /* no need to check the default value */ 1402 if (amdgpu_vm_size == -1) 1403 return; 1404 1405 if (amdgpu_vm_size < 1) { 1406 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 1407 amdgpu_vm_size); 1408 amdgpu_vm_size = -1; 1409 } 1410 } 1411 1412 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) 1413 { 1414 struct sysinfo si; 1415 bool is_os_64 = (sizeof(void *) == 8); 1416 uint64_t total_memory; 1417 uint64_t dram_size_seven_GB = 0x1B8000000; 1418 uint64_t dram_size_three_GB = 0xB8000000; 1419 1420 if (amdgpu_smu_memory_pool_size == 0) 1421 return; 1422 1423 if (!is_os_64) { 1424 DRM_WARN("Not 64-bit OS, feature not supported\n"); 1425 goto def_value; 1426 } 1427 si_meminfo(&si); 1428 total_memory = (uint64_t)si.totalram * si.mem_unit; 1429 1430 if ((amdgpu_smu_memory_pool_size == 1) || 1431 (amdgpu_smu_memory_pool_size == 2)) { 1432 if (total_memory < dram_size_three_GB) 1433 goto def_value1; 1434 } else if ((amdgpu_smu_memory_pool_size == 4) || 1435 (amdgpu_smu_memory_pool_size == 8)) { 1436 if (total_memory < dram_size_seven_GB) 1437 goto def_value1; 1438 } else { 1439 DRM_WARN("Smu memory pool size not supported\n"); 1440 goto def_value; 1441 } 1442 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; 1443 1444 return; 1445 1446 def_value1: 1447 DRM_WARN("No enough system memory\n"); 1448 def_value: 1449 adev->pm.smu_prv_buffer_size = 0; 1450 } 1451 1452 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev) 1453 { 1454 if (!(adev->flags & AMD_IS_APU) || 1455 adev->asic_type < CHIP_RAVEN) 1456 return 0; 1457 1458 switch (adev->asic_type) { 1459 case CHIP_RAVEN: 1460 if (adev->pdev->device == 0x15dd) 1461 adev->apu_flags |= AMD_APU_IS_RAVEN; 1462 if (adev->pdev->device == 0x15d8) 1463 adev->apu_flags |= AMD_APU_IS_PICASSO; 1464 break; 1465 case CHIP_RENOIR: 1466 if ((adev->pdev->device == 0x1636) || 1467 (adev->pdev->device == 0x164c)) 1468 adev->apu_flags |= AMD_APU_IS_RENOIR; 1469 else 1470 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; 1471 break; 1472 case CHIP_VANGOGH: 1473 adev->apu_flags |= AMD_APU_IS_VANGOGH; 1474 break; 1475 case CHIP_YELLOW_CARP: 1476 break; 1477 case CHIP_CYAN_SKILLFISH: 1478 if ((adev->pdev->device == 0x13FE) || 1479 (adev->pdev->device == 0x143F)) 1480 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; 1481 break; 1482 default: 1483 break; 1484 } 1485 1486 return 0; 1487 } 1488 1489 /** 1490 * amdgpu_device_check_arguments - validate module params 1491 * 1492 * @adev: amdgpu_device pointer 1493 * 1494 * Validates certain module parameters and updates 1495 * the associated values used by the driver (all asics). 1496 */ 1497 static int amdgpu_device_check_arguments(struct amdgpu_device *adev) 1498 { 1499 if (amdgpu_sched_jobs < 4) { 1500 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", 1501 amdgpu_sched_jobs); 1502 amdgpu_sched_jobs = 4; 1503 } else if (!is_power_of_2(amdgpu_sched_jobs)){ 1504 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", 1505 amdgpu_sched_jobs); 1506 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); 1507 } 1508 1509 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { 1510 /* gart size must be greater or equal to 32M */ 1511 dev_warn(adev->dev, "gart size (%d) too small\n", 1512 amdgpu_gart_size); 1513 amdgpu_gart_size = -1; 1514 } 1515 1516 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { 1517 /* gtt size must be greater or equal to 32M */ 1518 dev_warn(adev->dev, "gtt size (%d) too small\n", 1519 amdgpu_gtt_size); 1520 amdgpu_gtt_size = -1; 1521 } 1522 1523 /* valid range is between 4 and 9 inclusive */ 1524 if (amdgpu_vm_fragment_size != -1 && 1525 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { 1526 dev_warn(adev->dev, "valid range is between 4 and 9\n"); 1527 amdgpu_vm_fragment_size = -1; 1528 } 1529 1530 if (amdgpu_sched_hw_submission < 2) { 1531 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", 1532 amdgpu_sched_hw_submission); 1533 amdgpu_sched_hw_submission = 2; 1534 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) { 1535 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", 1536 amdgpu_sched_hw_submission); 1537 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission); 1538 } 1539 1540 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { 1541 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); 1542 amdgpu_reset_method = -1; 1543 } 1544 1545 amdgpu_device_check_smu_prv_buffer_size(adev); 1546 1547 amdgpu_device_check_vm_size(adev); 1548 1549 amdgpu_device_check_block_size(adev); 1550 1551 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); 1552 1553 amdgpu_gmc_tmz_set(adev); 1554 1555 1556 return 0; 1557 } 1558 1559 /** 1560 * amdgpu_switcheroo_set_state - set switcheroo state 1561 * 1562 * @pdev: pci dev pointer 1563 * @state: vga_switcheroo state 1564 * 1565 * Callback for the switcheroo driver. Suspends or resumes the 1566 * the asics before or after it is powered up using ACPI methods. 1567 */ 1568 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, 1569 enum vga_switcheroo_state state) 1570 { 1571 struct drm_device *dev = pci_get_drvdata(pdev); 1572 int r; 1573 1574 if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF) 1575 return; 1576 1577 if (state == VGA_SWITCHEROO_ON) { 1578 pr_info("switched on\n"); 1579 /* don't suspend or resume card normally */ 1580 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1581 1582 pci_set_power_state(pdev, PCI_D0); 1583 amdgpu_device_load_pci_state(pdev); 1584 r = pci_enable_device(pdev); 1585 if (r) 1586 DRM_WARN("pci_enable_device failed (%d)\n", r); 1587 amdgpu_device_resume(dev, true); 1588 1589 dev->switch_power_state = DRM_SWITCH_POWER_ON; 1590 } else { 1591 pr_info("switched off\n"); 1592 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1593 amdgpu_device_suspend(dev, true); 1594 amdgpu_device_cache_pci_state(pdev); 1595 /* Shut down the device */ 1596 pci_disable_device(pdev); 1597 pci_set_power_state(pdev, PCI_D3cold); 1598 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 1599 } 1600 } 1601 1602 /** 1603 * amdgpu_switcheroo_can_switch - see if switcheroo state can change 1604 * 1605 * @pdev: pci dev pointer 1606 * 1607 * Callback for the switcheroo driver. Check of the switcheroo 1608 * state can be changed. 1609 * Returns true if the state can be changed, false if not. 1610 */ 1611 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) 1612 { 1613 struct drm_device *dev = pci_get_drvdata(pdev); 1614 1615 /* 1616 * FIXME: open_count is protected by drm_global_mutex but that would lead to 1617 * locking inversion with the driver load path. And the access here is 1618 * completely racy anyway. So don't bother with locking for now. 1619 */ 1620 return atomic_read(&dev->open_count) == 0; 1621 } 1622 1623 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { 1624 .set_gpu_state = amdgpu_switcheroo_set_state, 1625 .reprobe = NULL, 1626 .can_switch = amdgpu_switcheroo_can_switch, 1627 }; 1628 1629 /** 1630 * amdgpu_device_ip_set_clockgating_state - set the CG state 1631 * 1632 * @dev: amdgpu_device pointer 1633 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1634 * @state: clockgating state (gate or ungate) 1635 * 1636 * Sets the requested clockgating state for all instances of 1637 * the hardware IP specified. 1638 * Returns the error code from the last instance. 1639 */ 1640 int amdgpu_device_ip_set_clockgating_state(void *dev, 1641 enum amd_ip_block_type block_type, 1642 enum amd_clockgating_state state) 1643 { 1644 struct amdgpu_device *adev = dev; 1645 int i, r = 0; 1646 1647 for (i = 0; i < adev->num_ip_blocks; i++) { 1648 if (!adev->ip_blocks[i].status.valid) 1649 continue; 1650 if (adev->ip_blocks[i].version->type != block_type) 1651 continue; 1652 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) 1653 continue; 1654 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( 1655 (void *)adev, state); 1656 if (r) 1657 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n", 1658 adev->ip_blocks[i].version->funcs->name, r); 1659 } 1660 return r; 1661 } 1662 1663 /** 1664 * amdgpu_device_ip_set_powergating_state - set the PG state 1665 * 1666 * @dev: amdgpu_device pointer 1667 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1668 * @state: powergating state (gate or ungate) 1669 * 1670 * Sets the requested powergating state for all instances of 1671 * the hardware IP specified. 1672 * Returns the error code from the last instance. 1673 */ 1674 int amdgpu_device_ip_set_powergating_state(void *dev, 1675 enum amd_ip_block_type block_type, 1676 enum amd_powergating_state state) 1677 { 1678 struct amdgpu_device *adev = dev; 1679 int i, r = 0; 1680 1681 for (i = 0; i < adev->num_ip_blocks; i++) { 1682 if (!adev->ip_blocks[i].status.valid) 1683 continue; 1684 if (adev->ip_blocks[i].version->type != block_type) 1685 continue; 1686 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) 1687 continue; 1688 r = adev->ip_blocks[i].version->funcs->set_powergating_state( 1689 (void *)adev, state); 1690 if (r) 1691 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n", 1692 adev->ip_blocks[i].version->funcs->name, r); 1693 } 1694 return r; 1695 } 1696 1697 /** 1698 * amdgpu_device_ip_get_clockgating_state - get the CG state 1699 * 1700 * @adev: amdgpu_device pointer 1701 * @flags: clockgating feature flags 1702 * 1703 * Walks the list of IPs on the device and updates the clockgating 1704 * flags for each IP. 1705 * Updates @flags with the feature flags for each hardware IP where 1706 * clockgating is enabled. 1707 */ 1708 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, 1709 u64 *flags) 1710 { 1711 int i; 1712 1713 for (i = 0; i < adev->num_ip_blocks; i++) { 1714 if (!adev->ip_blocks[i].status.valid) 1715 continue; 1716 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) 1717 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); 1718 } 1719 } 1720 1721 /** 1722 * amdgpu_device_ip_wait_for_idle - wait for idle 1723 * 1724 * @adev: amdgpu_device pointer 1725 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1726 * 1727 * Waits for the request hardware IP to be idle. 1728 * Returns 0 for success or a negative error code on failure. 1729 */ 1730 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, 1731 enum amd_ip_block_type block_type) 1732 { 1733 int i, r; 1734 1735 for (i = 0; i < adev->num_ip_blocks; i++) { 1736 if (!adev->ip_blocks[i].status.valid) 1737 continue; 1738 if (adev->ip_blocks[i].version->type == block_type) { 1739 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); 1740 if (r) 1741 return r; 1742 break; 1743 } 1744 } 1745 return 0; 1746 1747 } 1748 1749 /** 1750 * amdgpu_device_ip_is_idle - is the hardware IP idle 1751 * 1752 * @adev: amdgpu_device pointer 1753 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1754 * 1755 * Check if the hardware IP is idle or not. 1756 * Returns true if it the IP is idle, false if not. 1757 */ 1758 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, 1759 enum amd_ip_block_type block_type) 1760 { 1761 int i; 1762 1763 for (i = 0; i < adev->num_ip_blocks; i++) { 1764 if (!adev->ip_blocks[i].status.valid) 1765 continue; 1766 if (adev->ip_blocks[i].version->type == block_type) 1767 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); 1768 } 1769 return true; 1770 1771 } 1772 1773 /** 1774 * amdgpu_device_ip_get_ip_block - get a hw IP pointer 1775 * 1776 * @adev: amdgpu_device pointer 1777 * @type: Type of hardware IP (SMU, GFX, UVD, etc.) 1778 * 1779 * Returns a pointer to the hardware IP block structure 1780 * if it exists for the asic, otherwise NULL. 1781 */ 1782 struct amdgpu_ip_block * 1783 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, 1784 enum amd_ip_block_type type) 1785 { 1786 int i; 1787 1788 for (i = 0; i < adev->num_ip_blocks; i++) 1789 if (adev->ip_blocks[i].version->type == type) 1790 return &adev->ip_blocks[i]; 1791 1792 return NULL; 1793 } 1794 1795 /** 1796 * amdgpu_device_ip_block_version_cmp 1797 * 1798 * @adev: amdgpu_device pointer 1799 * @type: enum amd_ip_block_type 1800 * @major: major version 1801 * @minor: minor version 1802 * 1803 * return 0 if equal or greater 1804 * return 1 if smaller or the ip_block doesn't exist 1805 */ 1806 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, 1807 enum amd_ip_block_type type, 1808 u32 major, u32 minor) 1809 { 1810 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); 1811 1812 if (ip_block && ((ip_block->version->major > major) || 1813 ((ip_block->version->major == major) && 1814 (ip_block->version->minor >= minor)))) 1815 return 0; 1816 1817 return 1; 1818 } 1819 1820 /** 1821 * amdgpu_device_ip_block_add 1822 * 1823 * @adev: amdgpu_device pointer 1824 * @ip_block_version: pointer to the IP to add 1825 * 1826 * Adds the IP block driver information to the collection of IPs 1827 * on the asic. 1828 */ 1829 int amdgpu_device_ip_block_add(struct amdgpu_device *adev, 1830 const struct amdgpu_ip_block_version *ip_block_version) 1831 { 1832 if (!ip_block_version) 1833 return -EINVAL; 1834 1835 switch (ip_block_version->type) { 1836 case AMD_IP_BLOCK_TYPE_VCN: 1837 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK) 1838 return 0; 1839 break; 1840 case AMD_IP_BLOCK_TYPE_JPEG: 1841 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK) 1842 return 0; 1843 break; 1844 default: 1845 break; 1846 } 1847 1848 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, 1849 ip_block_version->funcs->name); 1850 1851 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; 1852 1853 return 0; 1854 } 1855 1856 /** 1857 * amdgpu_device_enable_virtual_display - enable virtual display feature 1858 * 1859 * @adev: amdgpu_device pointer 1860 * 1861 * Enabled the virtual display feature if the user has enabled it via 1862 * the module parameter virtual_display. This feature provides a virtual 1863 * display hardware on headless boards or in virtualized environments. 1864 * This function parses and validates the configuration string specified by 1865 * the user and configues the virtual display configuration (number of 1866 * virtual connectors, crtcs, etc.) specified. 1867 */ 1868 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) 1869 { 1870 adev->enable_virtual_display = false; 1871 1872 if (amdgpu_virtual_display) { 1873 const char *pci_address_name = pci_name(adev->pdev); 1874 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; 1875 1876 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); 1877 pciaddstr_tmp = pciaddstr; 1878 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { 1879 pciaddname = strsep(&pciaddname_tmp, ","); 1880 if (!strcmp("all", pciaddname) 1881 || !strcmp(pci_address_name, pciaddname)) { 1882 long num_crtc; 1883 int res = -1; 1884 1885 adev->enable_virtual_display = true; 1886 1887 if (pciaddname_tmp) 1888 res = kstrtol(pciaddname_tmp, 10, 1889 &num_crtc); 1890 1891 if (!res) { 1892 if (num_crtc < 1) 1893 num_crtc = 1; 1894 if (num_crtc > 6) 1895 num_crtc = 6; 1896 adev->mode_info.num_crtc = num_crtc; 1897 } else { 1898 adev->mode_info.num_crtc = 1; 1899 } 1900 break; 1901 } 1902 } 1903 1904 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", 1905 amdgpu_virtual_display, pci_address_name, 1906 adev->enable_virtual_display, adev->mode_info.num_crtc); 1907 1908 kfree(pciaddstr); 1909 } 1910 } 1911 1912 /** 1913 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware 1914 * 1915 * @adev: amdgpu_device pointer 1916 * 1917 * Parses the asic configuration parameters specified in the gpu info 1918 * firmware and makes them availale to the driver for use in configuring 1919 * the asic. 1920 * Returns 0 on success, -EINVAL on failure. 1921 */ 1922 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) 1923 { 1924 const char *chip_name; 1925 char fw_name[40]; 1926 int err; 1927 const struct gpu_info_firmware_header_v1_0 *hdr; 1928 1929 adev->firmware.gpu_info_fw = NULL; 1930 1931 if (adev->mman.discovery_bin) { 1932 /* 1933 * FIXME: The bounding box is still needed by Navi12, so 1934 * temporarily read it from gpu_info firmware. Should be dropped 1935 * when DAL no longer needs it. 1936 */ 1937 if (adev->asic_type != CHIP_NAVI12) 1938 return 0; 1939 } 1940 1941 switch (adev->asic_type) { 1942 #ifdef CONFIG_DRM_AMDGPU_SI 1943 case CHIP_VERDE: 1944 case CHIP_TAHITI: 1945 case CHIP_PITCAIRN: 1946 case CHIP_OLAND: 1947 case CHIP_HAINAN: 1948 #endif 1949 #ifdef CONFIG_DRM_AMDGPU_CIK 1950 case CHIP_BONAIRE: 1951 case CHIP_HAWAII: 1952 case CHIP_KAVERI: 1953 case CHIP_KABINI: 1954 case CHIP_MULLINS: 1955 #endif 1956 case CHIP_TOPAZ: 1957 case CHIP_TONGA: 1958 case CHIP_FIJI: 1959 case CHIP_POLARIS10: 1960 case CHIP_POLARIS11: 1961 case CHIP_POLARIS12: 1962 case CHIP_VEGAM: 1963 case CHIP_CARRIZO: 1964 case CHIP_STONEY: 1965 case CHIP_VEGA20: 1966 case CHIP_ALDEBARAN: 1967 case CHIP_SIENNA_CICHLID: 1968 case CHIP_NAVY_FLOUNDER: 1969 case CHIP_DIMGREY_CAVEFISH: 1970 case CHIP_BEIGE_GOBY: 1971 default: 1972 return 0; 1973 case CHIP_VEGA10: 1974 chip_name = "vega10"; 1975 break; 1976 case CHIP_VEGA12: 1977 chip_name = "vega12"; 1978 break; 1979 case CHIP_RAVEN: 1980 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1981 chip_name = "raven2"; 1982 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1983 chip_name = "picasso"; 1984 else 1985 chip_name = "raven"; 1986 break; 1987 case CHIP_ARCTURUS: 1988 chip_name = "arcturus"; 1989 break; 1990 case CHIP_NAVI12: 1991 chip_name = "navi12"; 1992 break; 1993 } 1994 1995 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name); 1996 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev); 1997 if (err) { 1998 dev_err(adev->dev, 1999 "Failed to load gpu_info firmware \"%s\"\n", 2000 fw_name); 2001 goto out; 2002 } 2003 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw); 2004 if (err) { 2005 dev_err(adev->dev, 2006 "Failed to validate gpu_info firmware \"%s\"\n", 2007 fw_name); 2008 goto out; 2009 } 2010 2011 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; 2012 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); 2013 2014 switch (hdr->version_major) { 2015 case 1: 2016 { 2017 const struct gpu_info_firmware_v1_0 *gpu_info_fw = 2018 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + 2019 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2020 2021 /* 2022 * Should be droped when DAL no longer needs it. 2023 */ 2024 if (adev->asic_type == CHIP_NAVI12) 2025 goto parse_soc_bounding_box; 2026 2027 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); 2028 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); 2029 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); 2030 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); 2031 adev->gfx.config.max_texture_channel_caches = 2032 le32_to_cpu(gpu_info_fw->gc_num_tccs); 2033 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); 2034 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); 2035 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); 2036 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); 2037 adev->gfx.config.double_offchip_lds_buf = 2038 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); 2039 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); 2040 adev->gfx.cu_info.max_waves_per_simd = 2041 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); 2042 adev->gfx.cu_info.max_scratch_slots_per_cu = 2043 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); 2044 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); 2045 if (hdr->version_minor >= 1) { 2046 const struct gpu_info_firmware_v1_1 *gpu_info_fw = 2047 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + 2048 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2049 adev->gfx.config.num_sc_per_sh = 2050 le32_to_cpu(gpu_info_fw->num_sc_per_sh); 2051 adev->gfx.config.num_packer_per_sc = 2052 le32_to_cpu(gpu_info_fw->num_packer_per_sc); 2053 } 2054 2055 parse_soc_bounding_box: 2056 /* 2057 * soc bounding box info is not integrated in disocovery table, 2058 * we always need to parse it from gpu info firmware if needed. 2059 */ 2060 if (hdr->version_minor == 2) { 2061 const struct gpu_info_firmware_v1_2 *gpu_info_fw = 2062 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + 2063 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2064 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; 2065 } 2066 break; 2067 } 2068 default: 2069 dev_err(adev->dev, 2070 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); 2071 err = -EINVAL; 2072 goto out; 2073 } 2074 out: 2075 return err; 2076 } 2077 2078 /** 2079 * amdgpu_device_ip_early_init - run early init for hardware IPs 2080 * 2081 * @adev: amdgpu_device pointer 2082 * 2083 * Early initialization pass for hardware IPs. The hardware IPs that make 2084 * up each asic are discovered each IP's early_init callback is run. This 2085 * is the first stage in initializing the asic. 2086 * Returns 0 on success, negative error code on failure. 2087 */ 2088 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) 2089 { 2090 struct drm_device *dev = adev_to_drm(adev); 2091 struct pci_dev *parent; 2092 int i, r; 2093 2094 amdgpu_device_enable_virtual_display(adev); 2095 2096 if (amdgpu_sriov_vf(adev)) { 2097 r = amdgpu_virt_request_full_gpu(adev, true); 2098 if (r) 2099 return r; 2100 } 2101 2102 switch (adev->asic_type) { 2103 #ifdef CONFIG_DRM_AMDGPU_SI 2104 case CHIP_VERDE: 2105 case CHIP_TAHITI: 2106 case CHIP_PITCAIRN: 2107 case CHIP_OLAND: 2108 case CHIP_HAINAN: 2109 adev->family = AMDGPU_FAMILY_SI; 2110 r = si_set_ip_blocks(adev); 2111 if (r) 2112 return r; 2113 break; 2114 #endif 2115 #ifdef CONFIG_DRM_AMDGPU_CIK 2116 case CHIP_BONAIRE: 2117 case CHIP_HAWAII: 2118 case CHIP_KAVERI: 2119 case CHIP_KABINI: 2120 case CHIP_MULLINS: 2121 if (adev->flags & AMD_IS_APU) 2122 adev->family = AMDGPU_FAMILY_KV; 2123 else 2124 adev->family = AMDGPU_FAMILY_CI; 2125 2126 r = cik_set_ip_blocks(adev); 2127 if (r) 2128 return r; 2129 break; 2130 #endif 2131 case CHIP_TOPAZ: 2132 case CHIP_TONGA: 2133 case CHIP_FIJI: 2134 case CHIP_POLARIS10: 2135 case CHIP_POLARIS11: 2136 case CHIP_POLARIS12: 2137 case CHIP_VEGAM: 2138 case CHIP_CARRIZO: 2139 case CHIP_STONEY: 2140 if (adev->flags & AMD_IS_APU) 2141 adev->family = AMDGPU_FAMILY_CZ; 2142 else 2143 adev->family = AMDGPU_FAMILY_VI; 2144 2145 r = vi_set_ip_blocks(adev); 2146 if (r) 2147 return r; 2148 break; 2149 default: 2150 r = amdgpu_discovery_set_ip_blocks(adev); 2151 if (r) 2152 return r; 2153 break; 2154 } 2155 2156 if (amdgpu_has_atpx() && 2157 (amdgpu_is_atpx_hybrid() || 2158 amdgpu_has_atpx_dgpu_power_cntl()) && 2159 ((adev->flags & AMD_IS_APU) == 0) && 2160 !pci_is_thunderbolt_attached(to_pci_dev(dev->dev))) 2161 adev->flags |= AMD_IS_PX; 2162 2163 if (!(adev->flags & AMD_IS_APU)) { 2164 parent = pci_upstream_bridge(adev->pdev); 2165 adev->has_pr3 = parent ? pci_pr3_present(parent) : false; 2166 } 2167 2168 amdgpu_amdkfd_device_probe(adev); 2169 2170 adev->pm.pp_feature = amdgpu_pp_feature_mask; 2171 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS) 2172 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 2173 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) 2174 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; 2175 2176 for (i = 0; i < adev->num_ip_blocks; i++) { 2177 if ((amdgpu_ip_block_mask & (1 << i)) == 0) { 2178 DRM_ERROR("disabled ip block: %d <%s>\n", 2179 i, adev->ip_blocks[i].version->funcs->name); 2180 adev->ip_blocks[i].status.valid = false; 2181 } else { 2182 if (adev->ip_blocks[i].version->funcs->early_init) { 2183 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); 2184 if (r == -ENOENT) { 2185 adev->ip_blocks[i].status.valid = false; 2186 } else if (r) { 2187 DRM_ERROR("early_init of IP block <%s> failed %d\n", 2188 adev->ip_blocks[i].version->funcs->name, r); 2189 return r; 2190 } else { 2191 adev->ip_blocks[i].status.valid = true; 2192 } 2193 } else { 2194 adev->ip_blocks[i].status.valid = true; 2195 } 2196 } 2197 /* get the vbios after the asic_funcs are set up */ 2198 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2199 r = amdgpu_device_parse_gpu_info_fw(adev); 2200 if (r) 2201 return r; 2202 2203 /* Read BIOS */ 2204 if (!amdgpu_get_bios(adev)) 2205 return -EINVAL; 2206 2207 r = amdgpu_atombios_init(adev); 2208 if (r) { 2209 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); 2210 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0); 2211 return r; 2212 } 2213 2214 /*get pf2vf msg info at it's earliest time*/ 2215 if (amdgpu_sriov_vf(adev)) 2216 amdgpu_virt_init_data_exchange(adev); 2217 2218 } 2219 } 2220 2221 adev->cg_flags &= amdgpu_cg_mask; 2222 adev->pg_flags &= amdgpu_pg_mask; 2223 2224 return 0; 2225 } 2226 2227 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) 2228 { 2229 int i, r; 2230 2231 for (i = 0; i < adev->num_ip_blocks; i++) { 2232 if (!adev->ip_blocks[i].status.sw) 2233 continue; 2234 if (adev->ip_blocks[i].status.hw) 2235 continue; 2236 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2237 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || 2238 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2239 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2240 if (r) { 2241 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2242 adev->ip_blocks[i].version->funcs->name, r); 2243 return r; 2244 } 2245 adev->ip_blocks[i].status.hw = true; 2246 } 2247 } 2248 2249 return 0; 2250 } 2251 2252 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) 2253 { 2254 int i, r; 2255 2256 for (i = 0; i < adev->num_ip_blocks; i++) { 2257 if (!adev->ip_blocks[i].status.sw) 2258 continue; 2259 if (adev->ip_blocks[i].status.hw) 2260 continue; 2261 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2262 if (r) { 2263 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2264 adev->ip_blocks[i].version->funcs->name, r); 2265 return r; 2266 } 2267 adev->ip_blocks[i].status.hw = true; 2268 } 2269 2270 return 0; 2271 } 2272 2273 static int amdgpu_device_fw_loading(struct amdgpu_device *adev) 2274 { 2275 int r = 0; 2276 int i; 2277 uint32_t smu_version; 2278 2279 if (adev->asic_type >= CHIP_VEGA10) { 2280 for (i = 0; i < adev->num_ip_blocks; i++) { 2281 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) 2282 continue; 2283 2284 if (!adev->ip_blocks[i].status.sw) 2285 continue; 2286 2287 /* no need to do the fw loading again if already done*/ 2288 if (adev->ip_blocks[i].status.hw == true) 2289 break; 2290 2291 if (amdgpu_in_reset(adev) || adev->in_suspend) { 2292 r = adev->ip_blocks[i].version->funcs->resume(adev); 2293 if (r) { 2294 DRM_ERROR("resume of IP block <%s> failed %d\n", 2295 adev->ip_blocks[i].version->funcs->name, r); 2296 return r; 2297 } 2298 } else { 2299 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2300 if (r) { 2301 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2302 adev->ip_blocks[i].version->funcs->name, r); 2303 return r; 2304 } 2305 } 2306 2307 adev->ip_blocks[i].status.hw = true; 2308 break; 2309 } 2310 } 2311 2312 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) 2313 r = amdgpu_pm_load_smu_firmware(adev, &smu_version); 2314 2315 return r; 2316 } 2317 2318 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) 2319 { 2320 long timeout; 2321 int r, i; 2322 2323 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2324 struct amdgpu_ring *ring = adev->rings[i]; 2325 2326 /* No need to setup the GPU scheduler for rings that don't need it */ 2327 if (!ring || ring->no_scheduler) 2328 continue; 2329 2330 switch (ring->funcs->type) { 2331 case AMDGPU_RING_TYPE_GFX: 2332 timeout = adev->gfx_timeout; 2333 break; 2334 case AMDGPU_RING_TYPE_COMPUTE: 2335 timeout = adev->compute_timeout; 2336 break; 2337 case AMDGPU_RING_TYPE_SDMA: 2338 timeout = adev->sdma_timeout; 2339 break; 2340 default: 2341 timeout = adev->video_timeout; 2342 break; 2343 } 2344 2345 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, 2346 ring->num_hw_submission, amdgpu_job_hang_limit, 2347 timeout, adev->reset_domain->wq, 2348 ring->sched_score, ring->name, 2349 adev->dev); 2350 if (r) { 2351 DRM_ERROR("Failed to create scheduler on ring %s.\n", 2352 ring->name); 2353 return r; 2354 } 2355 } 2356 2357 return 0; 2358 } 2359 2360 2361 /** 2362 * amdgpu_device_ip_init - run init for hardware IPs 2363 * 2364 * @adev: amdgpu_device pointer 2365 * 2366 * Main initialization pass for hardware IPs. The list of all the hardware 2367 * IPs that make up the asic is walked and the sw_init and hw_init callbacks 2368 * are run. sw_init initializes the software state associated with each IP 2369 * and hw_init initializes the hardware associated with each IP. 2370 * Returns 0 on success, negative error code on failure. 2371 */ 2372 static int amdgpu_device_ip_init(struct amdgpu_device *adev) 2373 { 2374 int i, r; 2375 2376 r = amdgpu_ras_init(adev); 2377 if (r) 2378 return r; 2379 2380 for (i = 0; i < adev->num_ip_blocks; i++) { 2381 if (!adev->ip_blocks[i].status.valid) 2382 continue; 2383 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); 2384 if (r) { 2385 DRM_ERROR("sw_init of IP block <%s> failed %d\n", 2386 adev->ip_blocks[i].version->funcs->name, r); 2387 goto init_failed; 2388 } 2389 adev->ip_blocks[i].status.sw = true; 2390 2391 /* need to do gmc hw init early so we can allocate gpu mem */ 2392 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2393 /* Try to reserve bad pages early */ 2394 if (amdgpu_sriov_vf(adev)) 2395 amdgpu_virt_exchange_data(adev); 2396 2397 r = amdgpu_device_vram_scratch_init(adev); 2398 if (r) { 2399 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r); 2400 goto init_failed; 2401 } 2402 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); 2403 if (r) { 2404 DRM_ERROR("hw_init %d failed %d\n", i, r); 2405 goto init_failed; 2406 } 2407 r = amdgpu_device_wb_init(adev); 2408 if (r) { 2409 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r); 2410 goto init_failed; 2411 } 2412 adev->ip_blocks[i].status.hw = true; 2413 2414 /* right after GMC hw init, we create CSA */ 2415 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { 2416 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, 2417 AMDGPU_GEM_DOMAIN_VRAM, 2418 AMDGPU_CSA_SIZE); 2419 if (r) { 2420 DRM_ERROR("allocate CSA failed %d\n", r); 2421 goto init_failed; 2422 } 2423 } 2424 } 2425 } 2426 2427 if (amdgpu_sriov_vf(adev)) 2428 amdgpu_virt_init_data_exchange(adev); 2429 2430 r = amdgpu_ib_pool_init(adev); 2431 if (r) { 2432 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 2433 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 2434 goto init_failed; 2435 } 2436 2437 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ 2438 if (r) 2439 goto init_failed; 2440 2441 r = amdgpu_device_ip_hw_init_phase1(adev); 2442 if (r) 2443 goto init_failed; 2444 2445 r = amdgpu_device_fw_loading(adev); 2446 if (r) 2447 goto init_failed; 2448 2449 r = amdgpu_device_ip_hw_init_phase2(adev); 2450 if (r) 2451 goto init_failed; 2452 2453 /* 2454 * retired pages will be loaded from eeprom and reserved here, 2455 * it should be called after amdgpu_device_ip_hw_init_phase2 since 2456 * for some ASICs the RAS EEPROM code relies on SMU fully functioning 2457 * for I2C communication which only true at this point. 2458 * 2459 * amdgpu_ras_recovery_init may fail, but the upper only cares the 2460 * failure from bad gpu situation and stop amdgpu init process 2461 * accordingly. For other failed cases, it will still release all 2462 * the resource and print error message, rather than returning one 2463 * negative value to upper level. 2464 * 2465 * Note: theoretically, this should be called before all vram allocations 2466 * to protect retired page from abusing 2467 */ 2468 r = amdgpu_ras_recovery_init(adev); 2469 if (r) 2470 goto init_failed; 2471 2472 /** 2473 * In case of XGMI grab extra reference for reset domain for this device 2474 */ 2475 if (adev->gmc.xgmi.num_physical_nodes > 1) { 2476 if (amdgpu_xgmi_add_device(adev) == 0) { 2477 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2478 2479 if (!hive->reset_domain || 2480 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { 2481 r = -ENOENT; 2482 goto init_failed; 2483 } 2484 2485 /* Drop the early temporary reset domain we created for device */ 2486 amdgpu_reset_put_reset_domain(adev->reset_domain); 2487 adev->reset_domain = hive->reset_domain; 2488 } 2489 } 2490 2491 r = amdgpu_device_init_schedulers(adev); 2492 if (r) 2493 goto init_failed; 2494 2495 /* Don't init kfd if whole hive need to be reset during init */ 2496 if (!adev->gmc.xgmi.pending_reset) 2497 amdgpu_amdkfd_device_init(adev); 2498 2499 amdgpu_fru_get_product_info(adev); 2500 2501 init_failed: 2502 if (amdgpu_sriov_vf(adev)) 2503 amdgpu_virt_release_full_gpu(adev, true); 2504 2505 return r; 2506 } 2507 2508 /** 2509 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer 2510 * 2511 * @adev: amdgpu_device pointer 2512 * 2513 * Writes a reset magic value to the gart pointer in VRAM. The driver calls 2514 * this function before a GPU reset. If the value is retained after a 2515 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents. 2516 */ 2517 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) 2518 { 2519 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); 2520 } 2521 2522 /** 2523 * amdgpu_device_check_vram_lost - check if vram is valid 2524 * 2525 * @adev: amdgpu_device pointer 2526 * 2527 * Checks the reset magic value written to the gart pointer in VRAM. 2528 * The driver calls this after a GPU reset to see if the contents of 2529 * VRAM is lost or now. 2530 * returns true if vram is lost, false if not. 2531 */ 2532 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) 2533 { 2534 if (memcmp(adev->gart.ptr, adev->reset_magic, 2535 AMDGPU_RESET_MAGIC_NUM)) 2536 return true; 2537 2538 if (!amdgpu_in_reset(adev)) 2539 return false; 2540 2541 /* 2542 * For all ASICs with baco/mode1 reset, the VRAM is 2543 * always assumed to be lost. 2544 */ 2545 switch (amdgpu_asic_reset_method(adev)) { 2546 case AMD_RESET_METHOD_BACO: 2547 case AMD_RESET_METHOD_MODE1: 2548 return true; 2549 default: 2550 return false; 2551 } 2552 } 2553 2554 /** 2555 * amdgpu_device_set_cg_state - set clockgating for amdgpu device 2556 * 2557 * @adev: amdgpu_device pointer 2558 * @state: clockgating state (gate or ungate) 2559 * 2560 * The list of all the hardware IPs that make up the asic is walked and the 2561 * set_clockgating_state callbacks are run. 2562 * Late initialization pass enabling clockgating for hardware IPs. 2563 * Fini or suspend, pass disabling clockgating for hardware IPs. 2564 * Returns 0 on success, negative error code on failure. 2565 */ 2566 2567 int amdgpu_device_set_cg_state(struct amdgpu_device *adev, 2568 enum amd_clockgating_state state) 2569 { 2570 int i, j, r; 2571 2572 if (amdgpu_emu_mode == 1) 2573 return 0; 2574 2575 for (j = 0; j < adev->num_ip_blocks; j++) { 2576 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 2577 if (!adev->ip_blocks[i].status.late_initialized) 2578 continue; 2579 /* skip CG for GFX on S0ix */ 2580 if (adev->in_s0ix && 2581 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX) 2582 continue; 2583 /* skip CG for VCE/UVD, it's handled specially */ 2584 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 2585 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 2586 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 2587 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 2588 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 2589 /* enable clockgating to save power */ 2590 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, 2591 state); 2592 if (r) { 2593 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n", 2594 adev->ip_blocks[i].version->funcs->name, r); 2595 return r; 2596 } 2597 } 2598 } 2599 2600 return 0; 2601 } 2602 2603 int amdgpu_device_set_pg_state(struct amdgpu_device *adev, 2604 enum amd_powergating_state state) 2605 { 2606 int i, j, r; 2607 2608 if (amdgpu_emu_mode == 1) 2609 return 0; 2610 2611 for (j = 0; j < adev->num_ip_blocks; j++) { 2612 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 2613 if (!adev->ip_blocks[i].status.late_initialized) 2614 continue; 2615 /* skip PG for GFX on S0ix */ 2616 if (adev->in_s0ix && 2617 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX) 2618 continue; 2619 /* skip CG for VCE/UVD, it's handled specially */ 2620 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 2621 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 2622 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 2623 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 2624 adev->ip_blocks[i].version->funcs->set_powergating_state) { 2625 /* enable powergating to save power */ 2626 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, 2627 state); 2628 if (r) { 2629 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n", 2630 adev->ip_blocks[i].version->funcs->name, r); 2631 return r; 2632 } 2633 } 2634 } 2635 return 0; 2636 } 2637 2638 static int amdgpu_device_enable_mgpu_fan_boost(void) 2639 { 2640 struct amdgpu_gpu_instance *gpu_ins; 2641 struct amdgpu_device *adev; 2642 int i, ret = 0; 2643 2644 mutex_lock(&mgpu_info.mutex); 2645 2646 /* 2647 * MGPU fan boost feature should be enabled 2648 * only when there are two or more dGPUs in 2649 * the system 2650 */ 2651 if (mgpu_info.num_dgpu < 2) 2652 goto out; 2653 2654 for (i = 0; i < mgpu_info.num_dgpu; i++) { 2655 gpu_ins = &(mgpu_info.gpu_ins[i]); 2656 adev = gpu_ins->adev; 2657 if (!(adev->flags & AMD_IS_APU) && 2658 !gpu_ins->mgpu_fan_enabled) { 2659 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); 2660 if (ret) 2661 break; 2662 2663 gpu_ins->mgpu_fan_enabled = 1; 2664 } 2665 } 2666 2667 out: 2668 mutex_unlock(&mgpu_info.mutex); 2669 2670 return ret; 2671 } 2672 2673 /** 2674 * amdgpu_device_ip_late_init - run late init for hardware IPs 2675 * 2676 * @adev: amdgpu_device pointer 2677 * 2678 * Late initialization pass for hardware IPs. The list of all the hardware 2679 * IPs that make up the asic is walked and the late_init callbacks are run. 2680 * late_init covers any special initialization that an IP requires 2681 * after all of the have been initialized or something that needs to happen 2682 * late in the init process. 2683 * Returns 0 on success, negative error code on failure. 2684 */ 2685 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) 2686 { 2687 struct amdgpu_gpu_instance *gpu_instance; 2688 int i = 0, r; 2689 2690 for (i = 0; i < adev->num_ip_blocks; i++) { 2691 if (!adev->ip_blocks[i].status.hw) 2692 continue; 2693 if (adev->ip_blocks[i].version->funcs->late_init) { 2694 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); 2695 if (r) { 2696 DRM_ERROR("late_init of IP block <%s> failed %d\n", 2697 adev->ip_blocks[i].version->funcs->name, r); 2698 return r; 2699 } 2700 } 2701 adev->ip_blocks[i].status.late_initialized = true; 2702 } 2703 2704 r = amdgpu_ras_late_init(adev); 2705 if (r) { 2706 DRM_ERROR("amdgpu_ras_late_init failed %d", r); 2707 return r; 2708 } 2709 2710 amdgpu_ras_set_error_query_ready(adev, true); 2711 2712 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 2713 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 2714 2715 amdgpu_device_fill_reset_magic(adev); 2716 2717 r = amdgpu_device_enable_mgpu_fan_boost(); 2718 if (r) 2719 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r); 2720 2721 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */ 2722 if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)|| 2723 adev->asic_type == CHIP_ALDEBARAN )) 2724 amdgpu_dpm_handle_passthrough_sbr(adev, true); 2725 2726 if (adev->gmc.xgmi.num_physical_nodes > 1) { 2727 mutex_lock(&mgpu_info.mutex); 2728 2729 /* 2730 * Reset device p-state to low as this was booted with high. 2731 * 2732 * This should be performed only after all devices from the same 2733 * hive get initialized. 2734 * 2735 * However, it's unknown how many device in the hive in advance. 2736 * As this is counted one by one during devices initializations. 2737 * 2738 * So, we wait for all XGMI interlinked devices initialized. 2739 * This may bring some delays as those devices may come from 2740 * different hives. But that should be OK. 2741 */ 2742 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { 2743 for (i = 0; i < mgpu_info.num_gpu; i++) { 2744 gpu_instance = &(mgpu_info.gpu_ins[i]); 2745 if (gpu_instance->adev->flags & AMD_IS_APU) 2746 continue; 2747 2748 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 2749 AMDGPU_XGMI_PSTATE_MIN); 2750 if (r) { 2751 DRM_ERROR("pstate setting failed (%d).\n", r); 2752 break; 2753 } 2754 } 2755 } 2756 2757 mutex_unlock(&mgpu_info.mutex); 2758 } 2759 2760 return 0; 2761 } 2762 2763 /** 2764 * amdgpu_device_smu_fini_early - smu hw_fini wrapper 2765 * 2766 * @adev: amdgpu_device pointer 2767 * 2768 * For ASICs need to disable SMC first 2769 */ 2770 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev) 2771 { 2772 int i, r; 2773 2774 if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0)) 2775 return; 2776 2777 for (i = 0; i < adev->num_ip_blocks; i++) { 2778 if (!adev->ip_blocks[i].status.hw) 2779 continue; 2780 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 2781 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); 2782 /* XXX handle errors */ 2783 if (r) { 2784 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 2785 adev->ip_blocks[i].version->funcs->name, r); 2786 } 2787 adev->ip_blocks[i].status.hw = false; 2788 break; 2789 } 2790 } 2791 } 2792 2793 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) 2794 { 2795 int i, r; 2796 2797 for (i = 0; i < adev->num_ip_blocks; i++) { 2798 if (!adev->ip_blocks[i].version->funcs->early_fini) 2799 continue; 2800 2801 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev); 2802 if (r) { 2803 DRM_DEBUG("early_fini of IP block <%s> failed %d\n", 2804 adev->ip_blocks[i].version->funcs->name, r); 2805 } 2806 } 2807 2808 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 2809 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 2810 2811 amdgpu_amdkfd_suspend(adev, false); 2812 2813 /* Workaroud for ASICs need to disable SMC first */ 2814 amdgpu_device_smu_fini_early(adev); 2815 2816 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2817 if (!adev->ip_blocks[i].status.hw) 2818 continue; 2819 2820 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); 2821 /* XXX handle errors */ 2822 if (r) { 2823 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 2824 adev->ip_blocks[i].version->funcs->name, r); 2825 } 2826 2827 adev->ip_blocks[i].status.hw = false; 2828 } 2829 2830 if (amdgpu_sriov_vf(adev)) { 2831 if (amdgpu_virt_release_full_gpu(adev, false)) 2832 DRM_ERROR("failed to release exclusive mode on fini\n"); 2833 } 2834 2835 return 0; 2836 } 2837 2838 /** 2839 * amdgpu_device_ip_fini - run fini for hardware IPs 2840 * 2841 * @adev: amdgpu_device pointer 2842 * 2843 * Main teardown pass for hardware IPs. The list of all the hardware 2844 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks 2845 * are run. hw_fini tears down the hardware associated with each IP 2846 * and sw_fini tears down any software state associated with each IP. 2847 * Returns 0 on success, negative error code on failure. 2848 */ 2849 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) 2850 { 2851 int i, r; 2852 2853 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) 2854 amdgpu_virt_release_ras_err_handler_data(adev); 2855 2856 if (adev->gmc.xgmi.num_physical_nodes > 1) 2857 amdgpu_xgmi_remove_device(adev); 2858 2859 amdgpu_amdkfd_device_fini_sw(adev); 2860 2861 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2862 if (!adev->ip_blocks[i].status.sw) 2863 continue; 2864 2865 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2866 amdgpu_ucode_free_bo(adev); 2867 amdgpu_free_static_csa(&adev->virt.csa_obj); 2868 amdgpu_device_wb_fini(adev); 2869 amdgpu_device_vram_scratch_fini(adev); 2870 amdgpu_ib_pool_fini(adev); 2871 } 2872 2873 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); 2874 /* XXX handle errors */ 2875 if (r) { 2876 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n", 2877 adev->ip_blocks[i].version->funcs->name, r); 2878 } 2879 adev->ip_blocks[i].status.sw = false; 2880 adev->ip_blocks[i].status.valid = false; 2881 } 2882 2883 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2884 if (!adev->ip_blocks[i].status.late_initialized) 2885 continue; 2886 if (adev->ip_blocks[i].version->funcs->late_fini) 2887 adev->ip_blocks[i].version->funcs->late_fini((void *)adev); 2888 adev->ip_blocks[i].status.late_initialized = false; 2889 } 2890 2891 amdgpu_ras_fini(adev); 2892 2893 return 0; 2894 } 2895 2896 /** 2897 * amdgpu_device_delayed_init_work_handler - work handler for IB tests 2898 * 2899 * @work: work_struct. 2900 */ 2901 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) 2902 { 2903 struct amdgpu_device *adev = 2904 container_of(work, struct amdgpu_device, delayed_init_work.work); 2905 int r; 2906 2907 r = amdgpu_ib_ring_tests(adev); 2908 if (r) 2909 DRM_ERROR("ib ring test failed (%d).\n", r); 2910 } 2911 2912 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) 2913 { 2914 struct amdgpu_device *adev = 2915 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); 2916 2917 WARN_ON_ONCE(adev->gfx.gfx_off_state); 2918 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); 2919 2920 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true)) 2921 adev->gfx.gfx_off_state = true; 2922 } 2923 2924 /** 2925 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) 2926 * 2927 * @adev: amdgpu_device pointer 2928 * 2929 * Main suspend function for hardware IPs. The list of all the hardware 2930 * IPs that make up the asic is walked, clockgating is disabled and the 2931 * suspend callbacks are run. suspend puts the hardware and software state 2932 * in each IP into a state suitable for suspend. 2933 * Returns 0 on success, negative error code on failure. 2934 */ 2935 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) 2936 { 2937 int i, r; 2938 2939 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 2940 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 2941 2942 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2943 if (!adev->ip_blocks[i].status.valid) 2944 continue; 2945 2946 /* displays are handled separately */ 2947 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) 2948 continue; 2949 2950 /* XXX handle errors */ 2951 r = adev->ip_blocks[i].version->funcs->suspend(adev); 2952 /* XXX handle errors */ 2953 if (r) { 2954 DRM_ERROR("suspend of IP block <%s> failed %d\n", 2955 adev->ip_blocks[i].version->funcs->name, r); 2956 return r; 2957 } 2958 2959 adev->ip_blocks[i].status.hw = false; 2960 } 2961 2962 return 0; 2963 } 2964 2965 /** 2966 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) 2967 * 2968 * @adev: amdgpu_device pointer 2969 * 2970 * Main suspend function for hardware IPs. The list of all the hardware 2971 * IPs that make up the asic is walked, clockgating is disabled and the 2972 * suspend callbacks are run. suspend puts the hardware and software state 2973 * in each IP into a state suitable for suspend. 2974 * Returns 0 on success, negative error code on failure. 2975 */ 2976 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) 2977 { 2978 int i, r; 2979 2980 if (adev->in_s0ix) 2981 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry); 2982 2983 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2984 if (!adev->ip_blocks[i].status.valid) 2985 continue; 2986 /* displays are handled in phase1 */ 2987 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) 2988 continue; 2989 /* PSP lost connection when err_event_athub occurs */ 2990 if (amdgpu_ras_intr_triggered() && 2991 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 2992 adev->ip_blocks[i].status.hw = false; 2993 continue; 2994 } 2995 2996 /* skip unnecessary suspend if we do not initialize them yet */ 2997 if (adev->gmc.xgmi.pending_reset && 2998 !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 2999 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC || 3000 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3001 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) { 3002 adev->ip_blocks[i].status.hw = false; 3003 continue; 3004 } 3005 3006 /* skip suspend of gfx and psp for S0ix 3007 * gfx is in gfxoff state, so on resume it will exit gfxoff just 3008 * like at runtime. PSP is also part of the always on hardware 3009 * so no need to suspend it. 3010 */ 3011 if (adev->in_s0ix && 3012 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || 3013 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)) 3014 continue; 3015 3016 /* XXX handle errors */ 3017 r = adev->ip_blocks[i].version->funcs->suspend(adev); 3018 /* XXX handle errors */ 3019 if (r) { 3020 DRM_ERROR("suspend of IP block <%s> failed %d\n", 3021 adev->ip_blocks[i].version->funcs->name, r); 3022 } 3023 adev->ip_blocks[i].status.hw = false; 3024 /* handle putting the SMC in the appropriate state */ 3025 if(!amdgpu_sriov_vf(adev)){ 3026 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3027 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); 3028 if (r) { 3029 DRM_ERROR("SMC failed to set mp1 state %d, %d\n", 3030 adev->mp1_state, r); 3031 return r; 3032 } 3033 } 3034 } 3035 } 3036 3037 return 0; 3038 } 3039 3040 /** 3041 * amdgpu_device_ip_suspend - run suspend for hardware IPs 3042 * 3043 * @adev: amdgpu_device pointer 3044 * 3045 * Main suspend function for hardware IPs. The list of all the hardware 3046 * IPs that make up the asic is walked, clockgating is disabled and the 3047 * suspend callbacks are run. suspend puts the hardware and software state 3048 * in each IP into a state suitable for suspend. 3049 * Returns 0 on success, negative error code on failure. 3050 */ 3051 int amdgpu_device_ip_suspend(struct amdgpu_device *adev) 3052 { 3053 int r; 3054 3055 if (amdgpu_sriov_vf(adev)) { 3056 amdgpu_virt_fini_data_exchange(adev); 3057 amdgpu_virt_request_full_gpu(adev, false); 3058 } 3059 3060 r = amdgpu_device_ip_suspend_phase1(adev); 3061 if (r) 3062 return r; 3063 r = amdgpu_device_ip_suspend_phase2(adev); 3064 3065 if (amdgpu_sriov_vf(adev)) 3066 amdgpu_virt_release_full_gpu(adev, false); 3067 3068 return r; 3069 } 3070 3071 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) 3072 { 3073 int i, r; 3074 3075 static enum amd_ip_block_type ip_order[] = { 3076 AMD_IP_BLOCK_TYPE_GMC, 3077 AMD_IP_BLOCK_TYPE_COMMON, 3078 AMD_IP_BLOCK_TYPE_PSP, 3079 AMD_IP_BLOCK_TYPE_IH, 3080 }; 3081 3082 for (i = 0; i < adev->num_ip_blocks; i++) { 3083 int j; 3084 struct amdgpu_ip_block *block; 3085 3086 block = &adev->ip_blocks[i]; 3087 block->status.hw = false; 3088 3089 for (j = 0; j < ARRAY_SIZE(ip_order); j++) { 3090 3091 if (block->version->type != ip_order[j] || 3092 !block->status.valid) 3093 continue; 3094 3095 r = block->version->funcs->hw_init(adev); 3096 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); 3097 if (r) 3098 return r; 3099 block->status.hw = true; 3100 } 3101 } 3102 3103 return 0; 3104 } 3105 3106 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) 3107 { 3108 int i, r; 3109 3110 static enum amd_ip_block_type ip_order[] = { 3111 AMD_IP_BLOCK_TYPE_SMC, 3112 AMD_IP_BLOCK_TYPE_DCE, 3113 AMD_IP_BLOCK_TYPE_GFX, 3114 AMD_IP_BLOCK_TYPE_SDMA, 3115 AMD_IP_BLOCK_TYPE_UVD, 3116 AMD_IP_BLOCK_TYPE_VCE, 3117 AMD_IP_BLOCK_TYPE_VCN 3118 }; 3119 3120 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 3121 int j; 3122 struct amdgpu_ip_block *block; 3123 3124 for (j = 0; j < adev->num_ip_blocks; j++) { 3125 block = &adev->ip_blocks[j]; 3126 3127 if (block->version->type != ip_order[i] || 3128 !block->status.valid || 3129 block->status.hw) 3130 continue; 3131 3132 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) 3133 r = block->version->funcs->resume(adev); 3134 else 3135 r = block->version->funcs->hw_init(adev); 3136 3137 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); 3138 if (r) 3139 return r; 3140 block->status.hw = true; 3141 } 3142 } 3143 3144 return 0; 3145 } 3146 3147 /** 3148 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs 3149 * 3150 * @adev: amdgpu_device pointer 3151 * 3152 * First resume function for hardware IPs. The list of all the hardware 3153 * IPs that make up the asic is walked and the resume callbacks are run for 3154 * COMMON, GMC, and IH. resume puts the hardware into a functional state 3155 * after a suspend and updates the software state as necessary. This 3156 * function is also used for restoring the GPU after a GPU reset. 3157 * Returns 0 on success, negative error code on failure. 3158 */ 3159 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) 3160 { 3161 int i, r; 3162 3163 for (i = 0; i < adev->num_ip_blocks; i++) { 3164 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3165 continue; 3166 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3167 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3168 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 3169 3170 r = adev->ip_blocks[i].version->funcs->resume(adev); 3171 if (r) { 3172 DRM_ERROR("resume of IP block <%s> failed %d\n", 3173 adev->ip_blocks[i].version->funcs->name, r); 3174 return r; 3175 } 3176 adev->ip_blocks[i].status.hw = true; 3177 } 3178 } 3179 3180 return 0; 3181 } 3182 3183 /** 3184 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs 3185 * 3186 * @adev: amdgpu_device pointer 3187 * 3188 * First resume function for hardware IPs. The list of all the hardware 3189 * IPs that make up the asic is walked and the resume callbacks are run for 3190 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a 3191 * functional state after a suspend and updates the software state as 3192 * necessary. This function is also used for restoring the GPU after a GPU 3193 * reset. 3194 * Returns 0 on success, negative error code on failure. 3195 */ 3196 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) 3197 { 3198 int i, r; 3199 3200 for (i = 0; i < adev->num_ip_blocks; i++) { 3201 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3202 continue; 3203 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3204 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3205 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3206 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3207 continue; 3208 r = adev->ip_blocks[i].version->funcs->resume(adev); 3209 if (r) { 3210 DRM_ERROR("resume of IP block <%s> failed %d\n", 3211 adev->ip_blocks[i].version->funcs->name, r); 3212 return r; 3213 } 3214 adev->ip_blocks[i].status.hw = true; 3215 } 3216 3217 return 0; 3218 } 3219 3220 /** 3221 * amdgpu_device_ip_resume - run resume for hardware IPs 3222 * 3223 * @adev: amdgpu_device pointer 3224 * 3225 * Main resume function for hardware IPs. The hardware IPs 3226 * are split into two resume functions because they are 3227 * are also used in in recovering from a GPU reset and some additional 3228 * steps need to be take between them. In this case (S3/S4) they are 3229 * run sequentially. 3230 * Returns 0 on success, negative error code on failure. 3231 */ 3232 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) 3233 { 3234 int r; 3235 3236 r = amdgpu_amdkfd_resume_iommu(adev); 3237 if (r) 3238 return r; 3239 3240 r = amdgpu_device_ip_resume_phase1(adev); 3241 if (r) 3242 return r; 3243 3244 r = amdgpu_device_fw_loading(adev); 3245 if (r) 3246 return r; 3247 3248 r = amdgpu_device_ip_resume_phase2(adev); 3249 3250 return r; 3251 } 3252 3253 /** 3254 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV 3255 * 3256 * @adev: amdgpu_device pointer 3257 * 3258 * Query the VBIOS data tables to determine if the board supports SR-IOV. 3259 */ 3260 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) 3261 { 3262 if (amdgpu_sriov_vf(adev)) { 3263 if (adev->is_atom_fw) { 3264 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev)) 3265 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3266 } else { 3267 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) 3268 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3269 } 3270 3271 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) 3272 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 3273 } 3274 } 3275 3276 /** 3277 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic 3278 * 3279 * @asic_type: AMD asic type 3280 * 3281 * Check if there is DC (new modesetting infrastructre) support for an asic. 3282 * returns true if DC has support, false if not. 3283 */ 3284 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) 3285 { 3286 switch (asic_type) { 3287 #ifdef CONFIG_DRM_AMDGPU_SI 3288 case CHIP_HAINAN: 3289 #endif 3290 case CHIP_TOPAZ: 3291 /* chips with no display hardware */ 3292 return false; 3293 #if defined(CONFIG_DRM_AMD_DC) 3294 case CHIP_TAHITI: 3295 case CHIP_PITCAIRN: 3296 case CHIP_VERDE: 3297 case CHIP_OLAND: 3298 /* 3299 * We have systems in the wild with these ASICs that require 3300 * LVDS and VGA support which is not supported with DC. 3301 * 3302 * Fallback to the non-DC driver here by default so as not to 3303 * cause regressions. 3304 */ 3305 #if defined(CONFIG_DRM_AMD_DC_SI) 3306 return amdgpu_dc > 0; 3307 #else 3308 return false; 3309 #endif 3310 case CHIP_BONAIRE: 3311 case CHIP_KAVERI: 3312 case CHIP_KABINI: 3313 case CHIP_MULLINS: 3314 /* 3315 * We have systems in the wild with these ASICs that require 3316 * LVDS and VGA support which is not supported with DC. 3317 * 3318 * Fallback to the non-DC driver here by default so as not to 3319 * cause regressions. 3320 */ 3321 return amdgpu_dc > 0; 3322 case CHIP_HAWAII: 3323 case CHIP_CARRIZO: 3324 case CHIP_STONEY: 3325 case CHIP_POLARIS10: 3326 case CHIP_POLARIS11: 3327 case CHIP_POLARIS12: 3328 case CHIP_VEGAM: 3329 case CHIP_TONGA: 3330 case CHIP_FIJI: 3331 case CHIP_VEGA10: 3332 case CHIP_VEGA12: 3333 case CHIP_VEGA20: 3334 #if defined(CONFIG_DRM_AMD_DC_DCN) 3335 case CHIP_RAVEN: 3336 case CHIP_NAVI10: 3337 case CHIP_NAVI14: 3338 case CHIP_NAVI12: 3339 case CHIP_RENOIR: 3340 case CHIP_CYAN_SKILLFISH: 3341 case CHIP_SIENNA_CICHLID: 3342 case CHIP_NAVY_FLOUNDER: 3343 case CHIP_DIMGREY_CAVEFISH: 3344 case CHIP_BEIGE_GOBY: 3345 case CHIP_VANGOGH: 3346 case CHIP_YELLOW_CARP: 3347 #endif 3348 default: 3349 return amdgpu_dc != 0; 3350 #else 3351 default: 3352 if (amdgpu_dc > 0) 3353 DRM_INFO_ONCE("Display Core has been requested via kernel parameter " 3354 "but isn't supported by ASIC, ignoring\n"); 3355 return false; 3356 #endif 3357 } 3358 } 3359 3360 /** 3361 * amdgpu_device_has_dc_support - check if dc is supported 3362 * 3363 * @adev: amdgpu_device pointer 3364 * 3365 * Returns true for supported, false for not supported 3366 */ 3367 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) 3368 { 3369 if (amdgpu_sriov_vf(adev) || 3370 adev->enable_virtual_display || 3371 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 3372 return false; 3373 3374 return amdgpu_device_asic_has_dc_support(adev->asic_type); 3375 } 3376 3377 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) 3378 { 3379 struct amdgpu_device *adev = 3380 container_of(__work, struct amdgpu_device, xgmi_reset_work); 3381 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 3382 3383 /* It's a bug to not have a hive within this function */ 3384 if (WARN_ON(!hive)) 3385 return; 3386 3387 /* 3388 * Use task barrier to synchronize all xgmi reset works across the 3389 * hive. task_barrier_enter and task_barrier_exit will block 3390 * until all the threads running the xgmi reset works reach 3391 * those points. task_barrier_full will do both blocks. 3392 */ 3393 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { 3394 3395 task_barrier_enter(&hive->tb); 3396 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev)); 3397 3398 if (adev->asic_reset_res) 3399 goto fail; 3400 3401 task_barrier_exit(&hive->tb); 3402 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev)); 3403 3404 if (adev->asic_reset_res) 3405 goto fail; 3406 3407 if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops && 3408 adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count) 3409 adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev); 3410 } else { 3411 3412 task_barrier_full(&hive->tb); 3413 adev->asic_reset_res = amdgpu_asic_reset(adev); 3414 } 3415 3416 fail: 3417 if (adev->asic_reset_res) 3418 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s", 3419 adev->asic_reset_res, adev_to_drm(adev)->unique); 3420 amdgpu_put_xgmi_hive(hive); 3421 } 3422 3423 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) 3424 { 3425 char *input = amdgpu_lockup_timeout; 3426 char *timeout_setting = NULL; 3427 int index = 0; 3428 long timeout; 3429 int ret = 0; 3430 3431 /* 3432 * By default timeout for non compute jobs is 10000 3433 * and 60000 for compute jobs. 3434 * In SR-IOV or passthrough mode, timeout for compute 3435 * jobs are 60000 by default. 3436 */ 3437 adev->gfx_timeout = msecs_to_jiffies(10000); 3438 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 3439 if (amdgpu_sriov_vf(adev)) 3440 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ? 3441 msecs_to_jiffies(60000) : msecs_to_jiffies(10000); 3442 else 3443 adev->compute_timeout = msecs_to_jiffies(60000); 3444 3445 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 3446 while ((timeout_setting = strsep(&input, ",")) && 3447 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 3448 ret = kstrtol(timeout_setting, 0, &timeout); 3449 if (ret) 3450 return ret; 3451 3452 if (timeout == 0) { 3453 index++; 3454 continue; 3455 } else if (timeout < 0) { 3456 timeout = MAX_SCHEDULE_TIMEOUT; 3457 dev_warn(adev->dev, "lockup timeout disabled"); 3458 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 3459 } else { 3460 timeout = msecs_to_jiffies(timeout); 3461 } 3462 3463 switch (index++) { 3464 case 0: 3465 adev->gfx_timeout = timeout; 3466 break; 3467 case 1: 3468 adev->compute_timeout = timeout; 3469 break; 3470 case 2: 3471 adev->sdma_timeout = timeout; 3472 break; 3473 case 3: 3474 adev->video_timeout = timeout; 3475 break; 3476 default: 3477 break; 3478 } 3479 } 3480 /* 3481 * There is only one value specified and 3482 * it should apply to all non-compute jobs. 3483 */ 3484 if (index == 1) { 3485 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 3486 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) 3487 adev->compute_timeout = adev->gfx_timeout; 3488 } 3489 } 3490 3491 return ret; 3492 } 3493 3494 /** 3495 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU 3496 * 3497 * @adev: amdgpu_device pointer 3498 * 3499 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode 3500 */ 3501 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev) 3502 { 3503 struct iommu_domain *domain; 3504 3505 domain = iommu_get_domain_for_dev(adev->dev); 3506 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) 3507 adev->ram_is_direct_mapped = true; 3508 } 3509 3510 static const struct attribute *amdgpu_dev_attributes[] = { 3511 &dev_attr_product_name.attr, 3512 &dev_attr_product_number.attr, 3513 &dev_attr_serial_number.attr, 3514 &dev_attr_pcie_replay_count.attr, 3515 NULL 3516 }; 3517 3518 /** 3519 * amdgpu_device_init - initialize the driver 3520 * 3521 * @adev: amdgpu_device pointer 3522 * @flags: driver flags 3523 * 3524 * Initializes the driver info and hw (all asics). 3525 * Returns 0 for success or an error on failure. 3526 * Called at driver startup. 3527 */ 3528 int amdgpu_device_init(struct amdgpu_device *adev, 3529 uint32_t flags) 3530 { 3531 struct drm_device *ddev = adev_to_drm(adev); 3532 struct pci_dev *pdev = adev->pdev; 3533 int r, i; 3534 bool px = false; 3535 u32 max_MBps; 3536 3537 adev->shutdown = false; 3538 adev->flags = flags; 3539 3540 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) 3541 adev->asic_type = amdgpu_force_asic_type; 3542 else 3543 adev->asic_type = flags & AMD_ASIC_MASK; 3544 3545 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; 3546 if (amdgpu_emu_mode == 1) 3547 adev->usec_timeout *= 10; 3548 adev->gmc.gart_size = 512 * 1024 * 1024; 3549 adev->accel_working = false; 3550 adev->num_rings = 0; 3551 adev->mman.buffer_funcs = NULL; 3552 adev->mman.buffer_funcs_ring = NULL; 3553 adev->vm_manager.vm_pte_funcs = NULL; 3554 adev->vm_manager.vm_pte_num_scheds = 0; 3555 adev->gmc.gmc_funcs = NULL; 3556 adev->harvest_ip_mask = 0x0; 3557 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); 3558 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 3559 3560 adev->smc_rreg = &amdgpu_invalid_rreg; 3561 adev->smc_wreg = &amdgpu_invalid_wreg; 3562 adev->pcie_rreg = &amdgpu_invalid_rreg; 3563 adev->pcie_wreg = &amdgpu_invalid_wreg; 3564 adev->pciep_rreg = &amdgpu_invalid_rreg; 3565 adev->pciep_wreg = &amdgpu_invalid_wreg; 3566 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; 3567 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; 3568 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; 3569 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; 3570 adev->didt_rreg = &amdgpu_invalid_rreg; 3571 adev->didt_wreg = &amdgpu_invalid_wreg; 3572 adev->gc_cac_rreg = &amdgpu_invalid_rreg; 3573 adev->gc_cac_wreg = &amdgpu_invalid_wreg; 3574 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; 3575 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; 3576 3577 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", 3578 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, 3579 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); 3580 3581 /* mutex initialization are all done here so we 3582 * can recall function without having locking issues */ 3583 mutex_init(&adev->firmware.mutex); 3584 mutex_init(&adev->pm.mutex); 3585 mutex_init(&adev->gfx.gpu_clock_mutex); 3586 mutex_init(&adev->srbm_mutex); 3587 mutex_init(&adev->gfx.pipe_reserve_mutex); 3588 mutex_init(&adev->gfx.gfx_off_mutex); 3589 mutex_init(&adev->grbm_idx_mutex); 3590 mutex_init(&adev->mn_lock); 3591 mutex_init(&adev->virt.vf_errors.lock); 3592 hash_init(adev->mn_hash); 3593 mutex_init(&adev->psp.mutex); 3594 mutex_init(&adev->notifier_lock); 3595 mutex_init(&adev->pm.stable_pstate_ctx_lock); 3596 mutex_init(&adev->benchmark_mutex); 3597 3598 amdgpu_device_init_apu_flags(adev); 3599 3600 r = amdgpu_device_check_arguments(adev); 3601 if (r) 3602 return r; 3603 3604 spin_lock_init(&adev->mmio_idx_lock); 3605 spin_lock_init(&adev->smc_idx_lock); 3606 spin_lock_init(&adev->pcie_idx_lock); 3607 spin_lock_init(&adev->uvd_ctx_idx_lock); 3608 spin_lock_init(&adev->didt_idx_lock); 3609 spin_lock_init(&adev->gc_cac_idx_lock); 3610 spin_lock_init(&adev->se_cac_idx_lock); 3611 spin_lock_init(&adev->audio_endpt_idx_lock); 3612 spin_lock_init(&adev->mm_stats.lock); 3613 3614 INIT_LIST_HEAD(&adev->shadow_list); 3615 mutex_init(&adev->shadow_list_lock); 3616 3617 INIT_LIST_HEAD(&adev->reset_list); 3618 3619 INIT_LIST_HEAD(&adev->ras_list); 3620 3621 INIT_DELAYED_WORK(&adev->delayed_init_work, 3622 amdgpu_device_delayed_init_work_handler); 3623 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, 3624 amdgpu_device_delay_enable_gfx_off); 3625 3626 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); 3627 3628 adev->gfx.gfx_off_req_count = 1; 3629 adev->pm.ac_power = power_supply_is_system_supplied() > 0; 3630 3631 atomic_set(&adev->throttling_logging_enabled, 1); 3632 /* 3633 * If throttling continues, logging will be performed every minute 3634 * to avoid log flooding. "-1" is subtracted since the thermal 3635 * throttling interrupt comes every second. Thus, the total logging 3636 * interval is 59 seconds(retelimited printk interval) + 1(waiting 3637 * for throttling interrupt) = 60 seconds. 3638 */ 3639 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); 3640 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); 3641 3642 /* Registers mapping */ 3643 /* TODO: block userspace mapping of io register */ 3644 if (adev->asic_type >= CHIP_BONAIRE) { 3645 adev->rmmio_base = pci_resource_start(adev->pdev, 5); 3646 adev->rmmio_size = pci_resource_len(adev->pdev, 5); 3647 } else { 3648 adev->rmmio_base = pci_resource_start(adev->pdev, 2); 3649 adev->rmmio_size = pci_resource_len(adev->pdev, 2); 3650 } 3651 3652 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++) 3653 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); 3654 3655 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); 3656 if (adev->rmmio == NULL) { 3657 return -ENOMEM; 3658 } 3659 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); 3660 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size); 3661 3662 amdgpu_device_get_pcie_info(adev); 3663 3664 if (amdgpu_mcbp) 3665 DRM_INFO("MCBP is enabled\n"); 3666 3667 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10) 3668 adev->enable_mes = true; 3669 3670 /* 3671 * Reset domain needs to be present early, before XGMI hive discovered 3672 * (if any) and intitialized to use reset sem and in_gpu reset flag 3673 * early on during init and before calling to RREG32. 3674 */ 3675 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); 3676 if (!adev->reset_domain) 3677 return -ENOMEM; 3678 3679 /* detect hw virtualization here */ 3680 amdgpu_detect_virtualization(adev); 3681 3682 r = amdgpu_device_get_job_timeout_settings(adev); 3683 if (r) { 3684 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); 3685 return r; 3686 } 3687 3688 /* early init functions */ 3689 r = amdgpu_device_ip_early_init(adev); 3690 if (r) 3691 return r; 3692 3693 amdgpu_gmc_noretry_set(adev); 3694 /* Need to get xgmi info early to decide the reset behavior*/ 3695 if (adev->gmc.xgmi.supported) { 3696 r = adev->gfxhub.funcs->get_xgmi_info(adev); 3697 if (r) 3698 return r; 3699 } 3700 3701 /* enable PCIE atomic ops */ 3702 if (amdgpu_sriov_vf(adev)) 3703 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *) 3704 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags == 3705 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64); 3706 else 3707 adev->have_atomics_support = 3708 !pci_enable_atomic_ops_to_root(adev->pdev, 3709 PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 3710 PCI_EXP_DEVCAP2_ATOMIC_COMP64); 3711 if (!adev->have_atomics_support) 3712 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); 3713 3714 /* doorbell bar mapping and doorbell index init*/ 3715 amdgpu_device_doorbell_init(adev); 3716 3717 if (amdgpu_emu_mode == 1) { 3718 /* post the asic on emulation mode */ 3719 emu_soc_asic_init(adev); 3720 goto fence_driver_init; 3721 } 3722 3723 amdgpu_reset_init(adev); 3724 3725 /* detect if we are with an SRIOV vbios */ 3726 amdgpu_device_detect_sriov_bios(adev); 3727 3728 /* check if we need to reset the asic 3729 * E.g., driver was not cleanly unloaded previously, etc. 3730 */ 3731 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { 3732 if (adev->gmc.xgmi.num_physical_nodes) { 3733 dev_info(adev->dev, "Pending hive reset.\n"); 3734 adev->gmc.xgmi.pending_reset = true; 3735 /* Only need to init necessary block for SMU to handle the reset */ 3736 for (i = 0; i < adev->num_ip_blocks; i++) { 3737 if (!adev->ip_blocks[i].status.valid) 3738 continue; 3739 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3740 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3741 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3742 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) { 3743 DRM_DEBUG("IP %s disabled for hw_init.\n", 3744 adev->ip_blocks[i].version->funcs->name); 3745 adev->ip_blocks[i].status.hw = true; 3746 } 3747 } 3748 } else { 3749 r = amdgpu_asic_reset(adev); 3750 if (r) { 3751 dev_err(adev->dev, "asic reset on init failed\n"); 3752 goto failed; 3753 } 3754 } 3755 } 3756 3757 pci_enable_pcie_error_reporting(adev->pdev); 3758 3759 /* Post card if necessary */ 3760 if (amdgpu_device_need_post(adev)) { 3761 if (!adev->bios) { 3762 dev_err(adev->dev, "no vBIOS found\n"); 3763 r = -EINVAL; 3764 goto failed; 3765 } 3766 DRM_INFO("GPU posting now...\n"); 3767 r = amdgpu_device_asic_init(adev); 3768 if (r) { 3769 dev_err(adev->dev, "gpu post error!\n"); 3770 goto failed; 3771 } 3772 } 3773 3774 if (adev->is_atom_fw) { 3775 /* Initialize clocks */ 3776 r = amdgpu_atomfirmware_get_clock_info(adev); 3777 if (r) { 3778 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 3779 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 3780 goto failed; 3781 } 3782 } else { 3783 /* Initialize clocks */ 3784 r = amdgpu_atombios_get_clock_info(adev); 3785 if (r) { 3786 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 3787 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 3788 goto failed; 3789 } 3790 /* init i2c buses */ 3791 if (!amdgpu_device_has_dc_support(adev)) 3792 amdgpu_atombios_i2c_init(adev); 3793 } 3794 3795 fence_driver_init: 3796 /* Fence driver */ 3797 r = amdgpu_fence_driver_sw_init(adev); 3798 if (r) { 3799 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); 3800 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 3801 goto failed; 3802 } 3803 3804 /* init the mode config */ 3805 drm_mode_config_init(adev_to_drm(adev)); 3806 3807 r = amdgpu_device_ip_init(adev); 3808 if (r) { 3809 /* failed in exclusive mode due to timeout */ 3810 if (amdgpu_sriov_vf(adev) && 3811 !amdgpu_sriov_runtime(adev) && 3812 amdgpu_virt_mmio_blocked(adev) && 3813 !amdgpu_virt_wait_reset(adev)) { 3814 dev_err(adev->dev, "VF exclusive mode timeout\n"); 3815 /* Don't send request since VF is inactive. */ 3816 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 3817 adev->virt.ops = NULL; 3818 r = -EAGAIN; 3819 goto release_ras_con; 3820 } 3821 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); 3822 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 3823 goto release_ras_con; 3824 } 3825 3826 amdgpu_fence_driver_hw_init(adev); 3827 3828 dev_info(adev->dev, 3829 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n", 3830 adev->gfx.config.max_shader_engines, 3831 adev->gfx.config.max_sh_per_se, 3832 adev->gfx.config.max_cu_per_sh, 3833 adev->gfx.cu_info.number); 3834 3835 adev->accel_working = true; 3836 3837 amdgpu_vm_check_compute_bug(adev); 3838 3839 /* Initialize the buffer migration limit. */ 3840 if (amdgpu_moverate >= 0) 3841 max_MBps = amdgpu_moverate; 3842 else 3843 max_MBps = 8; /* Allow 8 MB/s. */ 3844 /* Get a log2 for easy divisions. */ 3845 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); 3846 3847 r = amdgpu_pm_sysfs_init(adev); 3848 if (r) { 3849 adev->pm_sysfs_en = false; 3850 DRM_ERROR("registering pm debugfs failed (%d).\n", r); 3851 } else 3852 adev->pm_sysfs_en = true; 3853 3854 r = amdgpu_ucode_sysfs_init(adev); 3855 if (r) { 3856 adev->ucode_sysfs_en = false; 3857 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r); 3858 } else 3859 adev->ucode_sysfs_en = true; 3860 3861 /* 3862 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost. 3863 * Otherwise the mgpu fan boost feature will be skipped due to the 3864 * gpu instance is counted less. 3865 */ 3866 amdgpu_register_gpu_instance(adev); 3867 3868 /* enable clockgating, etc. after ib tests, etc. since some blocks require 3869 * explicit gating rather than handling it automatically. 3870 */ 3871 if (!adev->gmc.xgmi.pending_reset) { 3872 r = amdgpu_device_ip_late_init(adev); 3873 if (r) { 3874 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); 3875 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 3876 goto release_ras_con; 3877 } 3878 /* must succeed. */ 3879 amdgpu_ras_resume(adev); 3880 queue_delayed_work(system_wq, &adev->delayed_init_work, 3881 msecs_to_jiffies(AMDGPU_RESUME_MS)); 3882 } 3883 3884 if (amdgpu_sriov_vf(adev)) 3885 flush_delayed_work(&adev->delayed_init_work); 3886 3887 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes); 3888 if (r) 3889 dev_err(adev->dev, "Could not create amdgpu device attr\n"); 3890 3891 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 3892 r = amdgpu_pmu_init(adev); 3893 if (r) 3894 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); 3895 3896 /* Have stored pci confspace at hand for restore in sudden PCI error */ 3897 if (amdgpu_device_cache_pci_state(adev->pdev)) 3898 pci_restore_state(pdev); 3899 3900 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ 3901 /* this will fail for cards that aren't VGA class devices, just 3902 * ignore it */ 3903 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 3904 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); 3905 3906 if (amdgpu_device_supports_px(ddev)) { 3907 px = true; 3908 vga_switcheroo_register_client(adev->pdev, 3909 &amdgpu_switcheroo_ops, px); 3910 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); 3911 } 3912 3913 if (adev->gmc.xgmi.pending_reset) 3914 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work, 3915 msecs_to_jiffies(AMDGPU_RESUME_MS)); 3916 3917 amdgpu_device_check_iommu_direct_map(adev); 3918 3919 return 0; 3920 3921 release_ras_con: 3922 amdgpu_release_ras_context(adev); 3923 3924 failed: 3925 amdgpu_vf_error_trans_all(adev); 3926 3927 return r; 3928 } 3929 3930 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) 3931 { 3932 3933 /* Clear all CPU mappings pointing to this device */ 3934 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); 3935 3936 /* Unmap all mapped bars - Doorbell, registers and VRAM */ 3937 amdgpu_device_doorbell_fini(adev); 3938 3939 iounmap(adev->rmmio); 3940 adev->rmmio = NULL; 3941 if (adev->mman.aper_base_kaddr) 3942 iounmap(adev->mman.aper_base_kaddr); 3943 adev->mman.aper_base_kaddr = NULL; 3944 3945 /* Memory manager related */ 3946 if (!adev->gmc.xgmi.connected_to_cpu) { 3947 arch_phys_wc_del(adev->gmc.vram_mtrr); 3948 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 3949 } 3950 } 3951 3952 /** 3953 * amdgpu_device_fini_hw - tear down the driver 3954 * 3955 * @adev: amdgpu_device pointer 3956 * 3957 * Tear down the driver info (all asics). 3958 * Called at driver shutdown. 3959 */ 3960 void amdgpu_device_fini_hw(struct amdgpu_device *adev) 3961 { 3962 dev_info(adev->dev, "amdgpu: finishing device.\n"); 3963 flush_delayed_work(&adev->delayed_init_work); 3964 if (adev->mman.initialized) { 3965 flush_delayed_work(&adev->mman.bdev.wq); 3966 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); 3967 } 3968 adev->shutdown = true; 3969 3970 /* make sure IB test finished before entering exclusive mode 3971 * to avoid preemption on IB test 3972 * */ 3973 if (amdgpu_sriov_vf(adev)) { 3974 amdgpu_virt_request_full_gpu(adev, false); 3975 amdgpu_virt_fini_data_exchange(adev); 3976 } 3977 3978 /* disable all interrupts */ 3979 amdgpu_irq_disable_all(adev); 3980 if (adev->mode_info.mode_config_initialized){ 3981 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev))) 3982 drm_helper_force_disable_all(adev_to_drm(adev)); 3983 else 3984 drm_atomic_helper_shutdown(adev_to_drm(adev)); 3985 } 3986 amdgpu_fence_driver_hw_fini(adev); 3987 3988 if (adev->pm_sysfs_en) 3989 amdgpu_pm_sysfs_fini(adev); 3990 if (adev->ucode_sysfs_en) 3991 amdgpu_ucode_sysfs_fini(adev); 3992 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); 3993 3994 /* disable ras feature must before hw fini */ 3995 amdgpu_ras_pre_fini(adev); 3996 3997 amdgpu_device_ip_fini_early(adev); 3998 3999 amdgpu_irq_fini_hw(adev); 4000 4001 if (adev->mman.initialized) 4002 ttm_device_clear_dma_mappings(&adev->mman.bdev); 4003 4004 amdgpu_gart_dummy_page_fini(adev); 4005 4006 if (drm_dev_is_unplugged(adev_to_drm(adev))) 4007 amdgpu_device_unmap_mmio(adev); 4008 4009 } 4010 4011 void amdgpu_device_fini_sw(struct amdgpu_device *adev) 4012 { 4013 int idx; 4014 4015 amdgpu_fence_driver_sw_fini(adev); 4016 amdgpu_device_ip_fini(adev); 4017 release_firmware(adev->firmware.gpu_info_fw); 4018 adev->firmware.gpu_info_fw = NULL; 4019 adev->accel_working = false; 4020 4021 amdgpu_reset_fini(adev); 4022 4023 /* free i2c buses */ 4024 if (!amdgpu_device_has_dc_support(adev)) 4025 amdgpu_i2c_fini(adev); 4026 4027 if (amdgpu_emu_mode != 1) 4028 amdgpu_atombios_fini(adev); 4029 4030 kfree(adev->bios); 4031 adev->bios = NULL; 4032 if (amdgpu_device_supports_px(adev_to_drm(adev))) { 4033 vga_switcheroo_unregister_client(adev->pdev); 4034 vga_switcheroo_fini_domain_pm_ops(adev->dev); 4035 } 4036 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4037 vga_client_unregister(adev->pdev); 4038 4039 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 4040 4041 iounmap(adev->rmmio); 4042 adev->rmmio = NULL; 4043 amdgpu_device_doorbell_fini(adev); 4044 drm_dev_exit(idx); 4045 } 4046 4047 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4048 amdgpu_pmu_fini(adev); 4049 if (adev->mman.discovery_bin) 4050 amdgpu_discovery_fini(adev); 4051 4052 amdgpu_reset_put_reset_domain(adev->reset_domain); 4053 adev->reset_domain = NULL; 4054 4055 kfree(adev->pci_state); 4056 4057 } 4058 4059 /** 4060 * amdgpu_device_evict_resources - evict device resources 4061 * @adev: amdgpu device object 4062 * 4063 * Evicts all ttm device resources(vram BOs, gart table) from the lru list 4064 * of the vram memory type. Mainly used for evicting device resources 4065 * at suspend time. 4066 * 4067 */ 4068 static void amdgpu_device_evict_resources(struct amdgpu_device *adev) 4069 { 4070 /* No need to evict vram on APUs for suspend to ram or s2idle */ 4071 if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU)) 4072 return; 4073 4074 if (amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM)) 4075 DRM_WARN("evicting device resources failed\n"); 4076 4077 } 4078 4079 /* 4080 * Suspend & resume. 4081 */ 4082 /** 4083 * amdgpu_device_suspend - initiate device suspend 4084 * 4085 * @dev: drm dev pointer 4086 * @fbcon : notify the fbdev of suspend 4087 * 4088 * Puts the hw in the suspend state (all asics). 4089 * Returns 0 for success or an error on failure. 4090 * Called at driver suspend. 4091 */ 4092 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) 4093 { 4094 struct amdgpu_device *adev = drm_to_adev(dev); 4095 4096 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4097 return 0; 4098 4099 adev->in_suspend = true; 4100 4101 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3)) 4102 DRM_WARN("smart shift update failed\n"); 4103 4104 drm_kms_helper_poll_disable(dev); 4105 4106 if (fbcon) 4107 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true); 4108 4109 cancel_delayed_work_sync(&adev->delayed_init_work); 4110 4111 amdgpu_ras_suspend(adev); 4112 4113 amdgpu_device_ip_suspend_phase1(adev); 4114 4115 if (!adev->in_s0ix) 4116 amdgpu_amdkfd_suspend(adev, adev->in_runpm); 4117 4118 amdgpu_device_evict_resources(adev); 4119 4120 amdgpu_fence_driver_hw_fini(adev); 4121 4122 amdgpu_device_ip_suspend_phase2(adev); 4123 4124 return 0; 4125 } 4126 4127 /** 4128 * amdgpu_device_resume - initiate device resume 4129 * 4130 * @dev: drm dev pointer 4131 * @fbcon : notify the fbdev of resume 4132 * 4133 * Bring the hw back to operating state (all asics). 4134 * Returns 0 for success or an error on failure. 4135 * Called at driver resume. 4136 */ 4137 int amdgpu_device_resume(struct drm_device *dev, bool fbcon) 4138 { 4139 struct amdgpu_device *adev = drm_to_adev(dev); 4140 int r = 0; 4141 4142 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4143 return 0; 4144 4145 if (adev->in_s0ix) 4146 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); 4147 4148 /* post card */ 4149 if (amdgpu_device_need_post(adev)) { 4150 r = amdgpu_device_asic_init(adev); 4151 if (r) 4152 dev_err(adev->dev, "amdgpu asic init failed\n"); 4153 } 4154 4155 r = amdgpu_device_ip_resume(adev); 4156 if (r) { 4157 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); 4158 return r; 4159 } 4160 amdgpu_fence_driver_hw_init(adev); 4161 4162 r = amdgpu_device_ip_late_init(adev); 4163 if (r) 4164 return r; 4165 4166 queue_delayed_work(system_wq, &adev->delayed_init_work, 4167 msecs_to_jiffies(AMDGPU_RESUME_MS)); 4168 4169 if (!adev->in_s0ix) { 4170 r = amdgpu_amdkfd_resume(adev, adev->in_runpm); 4171 if (r) 4172 return r; 4173 } 4174 4175 /* Make sure IB tests flushed */ 4176 flush_delayed_work(&adev->delayed_init_work); 4177 4178 if (fbcon) 4179 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false); 4180 4181 drm_kms_helper_poll_enable(dev); 4182 4183 amdgpu_ras_resume(adev); 4184 4185 /* 4186 * Most of the connector probing functions try to acquire runtime pm 4187 * refs to ensure that the GPU is powered on when connector polling is 4188 * performed. Since we're calling this from a runtime PM callback, 4189 * trying to acquire rpm refs will cause us to deadlock. 4190 * 4191 * Since we're guaranteed to be holding the rpm lock, it's safe to 4192 * temporarily disable the rpm helpers so this doesn't deadlock us. 4193 */ 4194 #ifdef CONFIG_PM 4195 dev->dev->power.disable_depth++; 4196 #endif 4197 if (!amdgpu_device_has_dc_support(adev)) 4198 drm_helper_hpd_irq_event(dev); 4199 else 4200 drm_kms_helper_hotplug_event(dev); 4201 #ifdef CONFIG_PM 4202 dev->dev->power.disable_depth--; 4203 #endif 4204 adev->in_suspend = false; 4205 4206 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0)) 4207 DRM_WARN("smart shift update failed\n"); 4208 4209 return 0; 4210 } 4211 4212 /** 4213 * amdgpu_device_ip_check_soft_reset - did soft reset succeed 4214 * 4215 * @adev: amdgpu_device pointer 4216 * 4217 * The list of all the hardware IPs that make up the asic is walked and 4218 * the check_soft_reset callbacks are run. check_soft_reset determines 4219 * if the asic is still hung or not. 4220 * Returns true if any of the IPs are still in a hung state, false if not. 4221 */ 4222 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) 4223 { 4224 int i; 4225 bool asic_hang = false; 4226 4227 if (amdgpu_sriov_vf(adev)) 4228 return true; 4229 4230 if (amdgpu_asic_need_full_reset(adev)) 4231 return true; 4232 4233 for (i = 0; i < adev->num_ip_blocks; i++) { 4234 if (!adev->ip_blocks[i].status.valid) 4235 continue; 4236 if (adev->ip_blocks[i].version->funcs->check_soft_reset) 4237 adev->ip_blocks[i].status.hang = 4238 adev->ip_blocks[i].version->funcs->check_soft_reset(adev); 4239 if (adev->ip_blocks[i].status.hang) { 4240 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); 4241 asic_hang = true; 4242 } 4243 } 4244 return asic_hang; 4245 } 4246 4247 /** 4248 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset 4249 * 4250 * @adev: amdgpu_device pointer 4251 * 4252 * The list of all the hardware IPs that make up the asic is walked and the 4253 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset 4254 * handles any IP specific hardware or software state changes that are 4255 * necessary for a soft reset to succeed. 4256 * Returns 0 on success, negative error code on failure. 4257 */ 4258 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) 4259 { 4260 int i, r = 0; 4261 4262 for (i = 0; i < adev->num_ip_blocks; i++) { 4263 if (!adev->ip_blocks[i].status.valid) 4264 continue; 4265 if (adev->ip_blocks[i].status.hang && 4266 adev->ip_blocks[i].version->funcs->pre_soft_reset) { 4267 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); 4268 if (r) 4269 return r; 4270 } 4271 } 4272 4273 return 0; 4274 } 4275 4276 /** 4277 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed 4278 * 4279 * @adev: amdgpu_device pointer 4280 * 4281 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu 4282 * reset is necessary to recover. 4283 * Returns true if a full asic reset is required, false if not. 4284 */ 4285 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) 4286 { 4287 int i; 4288 4289 if (amdgpu_asic_need_full_reset(adev)) 4290 return true; 4291 4292 for (i = 0; i < adev->num_ip_blocks; i++) { 4293 if (!adev->ip_blocks[i].status.valid) 4294 continue; 4295 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || 4296 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || 4297 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || 4298 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || 4299 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 4300 if (adev->ip_blocks[i].status.hang) { 4301 dev_info(adev->dev, "Some block need full reset!\n"); 4302 return true; 4303 } 4304 } 4305 } 4306 return false; 4307 } 4308 4309 /** 4310 * amdgpu_device_ip_soft_reset - do a soft reset 4311 * 4312 * @adev: amdgpu_device pointer 4313 * 4314 * The list of all the hardware IPs that make up the asic is walked and the 4315 * soft_reset callbacks are run if the block is hung. soft_reset handles any 4316 * IP specific hardware or software state changes that are necessary to soft 4317 * reset the IP. 4318 * Returns 0 on success, negative error code on failure. 4319 */ 4320 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) 4321 { 4322 int i, r = 0; 4323 4324 for (i = 0; i < adev->num_ip_blocks; i++) { 4325 if (!adev->ip_blocks[i].status.valid) 4326 continue; 4327 if (adev->ip_blocks[i].status.hang && 4328 adev->ip_blocks[i].version->funcs->soft_reset) { 4329 r = adev->ip_blocks[i].version->funcs->soft_reset(adev); 4330 if (r) 4331 return r; 4332 } 4333 } 4334 4335 return 0; 4336 } 4337 4338 /** 4339 * amdgpu_device_ip_post_soft_reset - clean up from soft reset 4340 * 4341 * @adev: amdgpu_device pointer 4342 * 4343 * The list of all the hardware IPs that make up the asic is walked and the 4344 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset 4345 * handles any IP specific hardware or software state changes that are 4346 * necessary after the IP has been soft reset. 4347 * Returns 0 on success, negative error code on failure. 4348 */ 4349 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) 4350 { 4351 int i, r = 0; 4352 4353 for (i = 0; i < adev->num_ip_blocks; i++) { 4354 if (!adev->ip_blocks[i].status.valid) 4355 continue; 4356 if (adev->ip_blocks[i].status.hang && 4357 adev->ip_blocks[i].version->funcs->post_soft_reset) 4358 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); 4359 if (r) 4360 return r; 4361 } 4362 4363 return 0; 4364 } 4365 4366 /** 4367 * amdgpu_device_recover_vram - Recover some VRAM contents 4368 * 4369 * @adev: amdgpu_device pointer 4370 * 4371 * Restores the contents of VRAM buffers from the shadows in GTT. Used to 4372 * restore things like GPUVM page tables after a GPU reset where 4373 * the contents of VRAM might be lost. 4374 * 4375 * Returns: 4376 * 0 on success, negative error code on failure. 4377 */ 4378 static int amdgpu_device_recover_vram(struct amdgpu_device *adev) 4379 { 4380 struct dma_fence *fence = NULL, *next = NULL; 4381 struct amdgpu_bo *shadow; 4382 struct amdgpu_bo_vm *vmbo; 4383 long r = 1, tmo; 4384 4385 if (amdgpu_sriov_runtime(adev)) 4386 tmo = msecs_to_jiffies(8000); 4387 else 4388 tmo = msecs_to_jiffies(100); 4389 4390 dev_info(adev->dev, "recover vram bo from shadow start\n"); 4391 mutex_lock(&adev->shadow_list_lock); 4392 list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) { 4393 shadow = &vmbo->bo; 4394 /* No need to recover an evicted BO */ 4395 if (shadow->tbo.resource->mem_type != TTM_PL_TT || 4396 shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET || 4397 shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM) 4398 continue; 4399 4400 r = amdgpu_bo_restore_shadow(shadow, &next); 4401 if (r) 4402 break; 4403 4404 if (fence) { 4405 tmo = dma_fence_wait_timeout(fence, false, tmo); 4406 dma_fence_put(fence); 4407 fence = next; 4408 if (tmo == 0) { 4409 r = -ETIMEDOUT; 4410 break; 4411 } else if (tmo < 0) { 4412 r = tmo; 4413 break; 4414 } 4415 } else { 4416 fence = next; 4417 } 4418 } 4419 mutex_unlock(&adev->shadow_list_lock); 4420 4421 if (fence) 4422 tmo = dma_fence_wait_timeout(fence, false, tmo); 4423 dma_fence_put(fence); 4424 4425 if (r < 0 || tmo <= 0) { 4426 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo); 4427 return -EIO; 4428 } 4429 4430 dev_info(adev->dev, "recover vram bo from shadow done\n"); 4431 return 0; 4432 } 4433 4434 4435 /** 4436 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf 4437 * 4438 * @adev: amdgpu_device pointer 4439 * @from_hypervisor: request from hypervisor 4440 * 4441 * do VF FLR and reinitialize Asic 4442 * return 0 means succeeded otherwise failed 4443 */ 4444 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, 4445 bool from_hypervisor) 4446 { 4447 int r; 4448 struct amdgpu_hive_info *hive = NULL; 4449 int retry_limit = 0; 4450 4451 retry: 4452 amdgpu_amdkfd_pre_reset(adev); 4453 4454 amdgpu_amdkfd_pre_reset(adev); 4455 4456 if (from_hypervisor) 4457 r = amdgpu_virt_request_full_gpu(adev, true); 4458 else 4459 r = amdgpu_virt_reset_gpu(adev); 4460 if (r) 4461 return r; 4462 4463 /* Resume IP prior to SMC */ 4464 r = amdgpu_device_ip_reinit_early_sriov(adev); 4465 if (r) 4466 goto error; 4467 4468 amdgpu_virt_init_data_exchange(adev); 4469 4470 r = amdgpu_device_fw_loading(adev); 4471 if (r) 4472 return r; 4473 4474 /* now we are okay to resume SMC/CP/SDMA */ 4475 r = amdgpu_device_ip_reinit_late_sriov(adev); 4476 if (r) 4477 goto error; 4478 4479 hive = amdgpu_get_xgmi_hive(adev); 4480 /* Update PSP FW topology after reset */ 4481 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) 4482 r = amdgpu_xgmi_update_topology(hive, adev); 4483 4484 if (hive) 4485 amdgpu_put_xgmi_hive(hive); 4486 4487 if (!r) { 4488 amdgpu_irq_gpu_reset_resume_helper(adev); 4489 r = amdgpu_ib_ring_tests(adev); 4490 amdgpu_amdkfd_post_reset(adev); 4491 } 4492 4493 error: 4494 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { 4495 amdgpu_inc_vram_lost(adev); 4496 r = amdgpu_device_recover_vram(adev); 4497 } 4498 amdgpu_virt_release_full_gpu(adev, true); 4499 4500 if (AMDGPU_RETRY_SRIOV_RESET(r)) { 4501 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) { 4502 retry_limit++; 4503 goto retry; 4504 } else 4505 DRM_ERROR("GPU reset retry is beyond the retry limit\n"); 4506 } 4507 4508 return r; 4509 } 4510 4511 /** 4512 * amdgpu_device_has_job_running - check if there is any job in mirror list 4513 * 4514 * @adev: amdgpu_device pointer 4515 * 4516 * check if there is any job in mirror list 4517 */ 4518 bool amdgpu_device_has_job_running(struct amdgpu_device *adev) 4519 { 4520 int i; 4521 struct drm_sched_job *job; 4522 4523 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4524 struct amdgpu_ring *ring = adev->rings[i]; 4525 4526 if (!ring || !ring->sched.thread) 4527 continue; 4528 4529 spin_lock(&ring->sched.job_list_lock); 4530 job = list_first_entry_or_null(&ring->sched.pending_list, 4531 struct drm_sched_job, list); 4532 spin_unlock(&ring->sched.job_list_lock); 4533 if (job) 4534 return true; 4535 } 4536 return false; 4537 } 4538 4539 /** 4540 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery 4541 * 4542 * @adev: amdgpu_device pointer 4543 * 4544 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover 4545 * a hung GPU. 4546 */ 4547 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) 4548 { 4549 if (!amdgpu_device_ip_check_soft_reset(adev)) { 4550 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n"); 4551 return false; 4552 } 4553 4554 if (amdgpu_gpu_recovery == 0) 4555 goto disabled; 4556 4557 if (amdgpu_sriov_vf(adev)) 4558 return true; 4559 4560 if (amdgpu_gpu_recovery == -1) { 4561 switch (adev->asic_type) { 4562 #ifdef CONFIG_DRM_AMDGPU_SI 4563 case CHIP_VERDE: 4564 case CHIP_TAHITI: 4565 case CHIP_PITCAIRN: 4566 case CHIP_OLAND: 4567 case CHIP_HAINAN: 4568 #endif 4569 #ifdef CONFIG_DRM_AMDGPU_CIK 4570 case CHIP_KAVERI: 4571 case CHIP_KABINI: 4572 case CHIP_MULLINS: 4573 #endif 4574 case CHIP_CARRIZO: 4575 case CHIP_STONEY: 4576 case CHIP_CYAN_SKILLFISH: 4577 goto disabled; 4578 default: 4579 break; 4580 } 4581 } 4582 4583 return true; 4584 4585 disabled: 4586 dev_info(adev->dev, "GPU recovery disabled.\n"); 4587 return false; 4588 } 4589 4590 int amdgpu_device_mode1_reset(struct amdgpu_device *adev) 4591 { 4592 u32 i; 4593 int ret = 0; 4594 4595 amdgpu_atombios_scratch_regs_engine_hung(adev, true); 4596 4597 dev_info(adev->dev, "GPU mode1 reset\n"); 4598 4599 /* disable BM */ 4600 pci_clear_master(adev->pdev); 4601 4602 amdgpu_device_cache_pci_state(adev->pdev); 4603 4604 if (amdgpu_dpm_is_mode1_reset_supported(adev)) { 4605 dev_info(adev->dev, "GPU smu mode1 reset\n"); 4606 ret = amdgpu_dpm_mode1_reset(adev); 4607 } else { 4608 dev_info(adev->dev, "GPU psp mode1 reset\n"); 4609 ret = psp_gpu_reset(adev); 4610 } 4611 4612 if (ret) 4613 dev_err(adev->dev, "GPU mode1 reset failed\n"); 4614 4615 amdgpu_device_load_pci_state(adev->pdev); 4616 4617 /* wait for asic to come out of reset */ 4618 for (i = 0; i < adev->usec_timeout; i++) { 4619 u32 memsize = adev->nbio.funcs->get_memsize(adev); 4620 4621 if (memsize != 0xffffffff) 4622 break; 4623 udelay(1); 4624 } 4625 4626 amdgpu_atombios_scratch_regs_engine_hung(adev, false); 4627 return ret; 4628 } 4629 4630 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, 4631 struct amdgpu_reset_context *reset_context) 4632 { 4633 int i, r = 0; 4634 struct amdgpu_job *job = NULL; 4635 bool need_full_reset = 4636 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4637 4638 if (reset_context->reset_req_dev == adev) 4639 job = reset_context->job; 4640 4641 if (amdgpu_sriov_vf(adev)) { 4642 /* stop the data exchange thread */ 4643 amdgpu_virt_fini_data_exchange(adev); 4644 } 4645 4646 /* block all schedulers and reset given job's ring */ 4647 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4648 struct amdgpu_ring *ring = adev->rings[i]; 4649 4650 if (!ring || !ring->sched.thread) 4651 continue; 4652 4653 /*clear job fence from fence drv to avoid force_completion 4654 *leave NULL and vm flush fence in fence drv */ 4655 amdgpu_fence_driver_clear_job_fences(ring); 4656 4657 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 4658 amdgpu_fence_driver_force_completion(ring); 4659 } 4660 4661 if (job && job->vm) 4662 drm_sched_increase_karma(&job->base); 4663 4664 r = amdgpu_reset_prepare_hwcontext(adev, reset_context); 4665 /* If reset handler not implemented, continue; otherwise return */ 4666 if (r == -ENOSYS) 4667 r = 0; 4668 else 4669 return r; 4670 4671 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ 4672 if (!amdgpu_sriov_vf(adev)) { 4673 4674 if (!need_full_reset) 4675 need_full_reset = amdgpu_device_ip_need_full_reset(adev); 4676 4677 if (!need_full_reset) { 4678 amdgpu_device_ip_pre_soft_reset(adev); 4679 r = amdgpu_device_ip_soft_reset(adev); 4680 amdgpu_device_ip_post_soft_reset(adev); 4681 if (r || amdgpu_device_ip_check_soft_reset(adev)) { 4682 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); 4683 need_full_reset = true; 4684 } 4685 } 4686 4687 if (need_full_reset) 4688 r = amdgpu_device_ip_suspend(adev); 4689 if (need_full_reset) 4690 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4691 else 4692 clear_bit(AMDGPU_NEED_FULL_RESET, 4693 &reset_context->flags); 4694 } 4695 4696 return r; 4697 } 4698 4699 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev) 4700 { 4701 uint32_t reg_value; 4702 int i; 4703 4704 lockdep_assert_held(&adev->reset_domain->sem); 4705 dump_stack(); 4706 4707 for (i = 0; i < adev->num_regs; i++) { 4708 reg_value = RREG32(adev->reset_dump_reg_list[i]); 4709 trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i], reg_value); 4710 } 4711 4712 return 0; 4713 } 4714 4715 int amdgpu_do_asic_reset(struct list_head *device_list_handle, 4716 struct amdgpu_reset_context *reset_context) 4717 { 4718 struct amdgpu_device *tmp_adev = NULL; 4719 bool need_full_reset, skip_hw_reset, vram_lost = false; 4720 int r = 0; 4721 4722 /* Try reset handler method first */ 4723 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 4724 reset_list); 4725 amdgpu_reset_reg_dumps(tmp_adev); 4726 r = amdgpu_reset_perform_reset(tmp_adev, reset_context); 4727 /* If reset handler not implemented, continue; otherwise return */ 4728 if (r == -ENOSYS) 4729 r = 0; 4730 else 4731 return r; 4732 4733 /* Reset handler not implemented, use the default method */ 4734 need_full_reset = 4735 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4736 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); 4737 4738 /* 4739 * ASIC reset has to be done on all XGMI hive nodes ASAP 4740 * to allow proper links negotiation in FW (within 1 sec) 4741 */ 4742 if (!skip_hw_reset && need_full_reset) { 4743 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 4744 /* For XGMI run all resets in parallel to speed up the process */ 4745 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 4746 tmp_adev->gmc.xgmi.pending_reset = false; 4747 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work)) 4748 r = -EALREADY; 4749 } else 4750 r = amdgpu_asic_reset(tmp_adev); 4751 4752 if (r) { 4753 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s", 4754 r, adev_to_drm(tmp_adev)->unique); 4755 break; 4756 } 4757 } 4758 4759 /* For XGMI wait for all resets to complete before proceed */ 4760 if (!r) { 4761 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 4762 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 4763 flush_work(&tmp_adev->xgmi_reset_work); 4764 r = tmp_adev->asic_reset_res; 4765 if (r) 4766 break; 4767 } 4768 } 4769 } 4770 } 4771 4772 if (!r && amdgpu_ras_intr_triggered()) { 4773 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 4774 if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops && 4775 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count) 4776 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev); 4777 } 4778 4779 amdgpu_ras_intr_cleared(); 4780 } 4781 4782 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 4783 if (need_full_reset) { 4784 /* post card */ 4785 r = amdgpu_device_asic_init(tmp_adev); 4786 if (r) { 4787 dev_warn(tmp_adev->dev, "asic atom init failed!"); 4788 } else { 4789 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); 4790 r = amdgpu_amdkfd_resume_iommu(tmp_adev); 4791 if (r) 4792 goto out; 4793 4794 r = amdgpu_device_ip_resume_phase1(tmp_adev); 4795 if (r) 4796 goto out; 4797 4798 vram_lost = amdgpu_device_check_vram_lost(tmp_adev); 4799 if (vram_lost) { 4800 DRM_INFO("VRAM is lost due to GPU reset!\n"); 4801 amdgpu_inc_vram_lost(tmp_adev); 4802 } 4803 4804 r = amdgpu_device_fw_loading(tmp_adev); 4805 if (r) 4806 return r; 4807 4808 r = amdgpu_device_ip_resume_phase2(tmp_adev); 4809 if (r) 4810 goto out; 4811 4812 if (vram_lost) 4813 amdgpu_device_fill_reset_magic(tmp_adev); 4814 4815 /* 4816 * Add this ASIC as tracked as reset was already 4817 * complete successfully. 4818 */ 4819 amdgpu_register_gpu_instance(tmp_adev); 4820 4821 if (!reset_context->hive && 4822 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 4823 amdgpu_xgmi_add_device(tmp_adev); 4824 4825 r = amdgpu_device_ip_late_init(tmp_adev); 4826 if (r) 4827 goto out; 4828 4829 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false); 4830 4831 /* 4832 * The GPU enters bad state once faulty pages 4833 * by ECC has reached the threshold, and ras 4834 * recovery is scheduled next. So add one check 4835 * here to break recovery if it indeed exceeds 4836 * bad page threshold, and remind user to 4837 * retire this GPU or setting one bigger 4838 * bad_page_threshold value to fix this once 4839 * probing driver again. 4840 */ 4841 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) { 4842 /* must succeed. */ 4843 amdgpu_ras_resume(tmp_adev); 4844 } else { 4845 r = -EINVAL; 4846 goto out; 4847 } 4848 4849 /* Update PSP FW topology after reset */ 4850 if (reset_context->hive && 4851 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 4852 r = amdgpu_xgmi_update_topology( 4853 reset_context->hive, tmp_adev); 4854 } 4855 } 4856 4857 out: 4858 if (!r) { 4859 amdgpu_irq_gpu_reset_resume_helper(tmp_adev); 4860 r = amdgpu_ib_ring_tests(tmp_adev); 4861 if (r) { 4862 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); 4863 need_full_reset = true; 4864 r = -EAGAIN; 4865 goto end; 4866 } 4867 } 4868 4869 if (!r) 4870 r = amdgpu_device_recover_vram(tmp_adev); 4871 else 4872 tmp_adev->asic_reset_res = r; 4873 } 4874 4875 end: 4876 if (need_full_reset) 4877 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4878 else 4879 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4880 return r; 4881 } 4882 4883 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev) 4884 { 4885 4886 switch (amdgpu_asic_reset_method(adev)) { 4887 case AMD_RESET_METHOD_MODE1: 4888 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; 4889 break; 4890 case AMD_RESET_METHOD_MODE2: 4891 adev->mp1_state = PP_MP1_STATE_RESET; 4892 break; 4893 default: 4894 adev->mp1_state = PP_MP1_STATE_NONE; 4895 break; 4896 } 4897 } 4898 4899 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev) 4900 { 4901 amdgpu_vf_error_trans_all(adev); 4902 adev->mp1_state = PP_MP1_STATE_NONE; 4903 } 4904 4905 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) 4906 { 4907 struct pci_dev *p = NULL; 4908 4909 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 4910 adev->pdev->bus->number, 1); 4911 if (p) { 4912 pm_runtime_enable(&(p->dev)); 4913 pm_runtime_resume(&(p->dev)); 4914 } 4915 } 4916 4917 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) 4918 { 4919 enum amd_reset_method reset_method; 4920 struct pci_dev *p = NULL; 4921 u64 expires; 4922 4923 /* 4924 * For now, only BACO and mode1 reset are confirmed 4925 * to suffer the audio issue without proper suspended. 4926 */ 4927 reset_method = amdgpu_asic_reset_method(adev); 4928 if ((reset_method != AMD_RESET_METHOD_BACO) && 4929 (reset_method != AMD_RESET_METHOD_MODE1)) 4930 return -EINVAL; 4931 4932 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 4933 adev->pdev->bus->number, 1); 4934 if (!p) 4935 return -ENODEV; 4936 4937 expires = pm_runtime_autosuspend_expiration(&(p->dev)); 4938 if (!expires) 4939 /* 4940 * If we cannot get the audio device autosuspend delay, 4941 * a fixed 4S interval will be used. Considering 3S is 4942 * the audio controller default autosuspend delay setting. 4943 * 4S used here is guaranteed to cover that. 4944 */ 4945 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL; 4946 4947 while (!pm_runtime_status_suspended(&(p->dev))) { 4948 if (!pm_runtime_suspend(&(p->dev))) 4949 break; 4950 4951 if (expires < ktime_get_mono_fast_ns()) { 4952 dev_warn(adev->dev, "failed to suspend display audio\n"); 4953 /* TODO: abort the succeeding gpu reset? */ 4954 return -ETIMEDOUT; 4955 } 4956 } 4957 4958 pm_runtime_disable(&(p->dev)); 4959 4960 return 0; 4961 } 4962 4963 static void amdgpu_device_recheck_guilty_jobs( 4964 struct amdgpu_device *adev, struct list_head *device_list_handle, 4965 struct amdgpu_reset_context *reset_context) 4966 { 4967 int i, r = 0; 4968 4969 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4970 struct amdgpu_ring *ring = adev->rings[i]; 4971 int ret = 0; 4972 struct drm_sched_job *s_job; 4973 4974 if (!ring || !ring->sched.thread) 4975 continue; 4976 4977 s_job = list_first_entry_or_null(&ring->sched.pending_list, 4978 struct drm_sched_job, list); 4979 if (s_job == NULL) 4980 continue; 4981 4982 /* clear job's guilty and depend the folowing step to decide the real one */ 4983 drm_sched_reset_karma(s_job); 4984 /* for the real bad job, it will be resubmitted twice, adding a dma_fence_get 4985 * to make sure fence is balanced */ 4986 dma_fence_get(s_job->s_fence->parent); 4987 drm_sched_resubmit_jobs_ext(&ring->sched, 1); 4988 4989 ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout); 4990 if (ret == 0) { /* timeout */ 4991 DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n", 4992 ring->sched.name, s_job->id); 4993 4994 /* set guilty */ 4995 drm_sched_increase_karma(s_job); 4996 retry: 4997 /* do hw reset */ 4998 if (amdgpu_sriov_vf(adev)) { 4999 amdgpu_virt_fini_data_exchange(adev); 5000 r = amdgpu_device_reset_sriov(adev, false); 5001 if (r) 5002 adev->asic_reset_res = r; 5003 } else { 5004 clear_bit(AMDGPU_SKIP_HW_RESET, 5005 &reset_context->flags); 5006 r = amdgpu_do_asic_reset(device_list_handle, 5007 reset_context); 5008 if (r && r == -EAGAIN) 5009 goto retry; 5010 } 5011 5012 /* 5013 * add reset counter so that the following 5014 * resubmitted job could flush vmid 5015 */ 5016 atomic_inc(&adev->gpu_reset_counter); 5017 continue; 5018 } 5019 5020 /* got the hw fence, signal finished fence */ 5021 atomic_dec(ring->sched.score); 5022 dma_fence_put(s_job->s_fence->parent); 5023 dma_fence_get(&s_job->s_fence->finished); 5024 dma_fence_signal(&s_job->s_fence->finished); 5025 dma_fence_put(&s_job->s_fence->finished); 5026 5027 /* remove node from list and free the job */ 5028 spin_lock(&ring->sched.job_list_lock); 5029 list_del_init(&s_job->list); 5030 spin_unlock(&ring->sched.job_list_lock); 5031 ring->sched.ops->free_job(s_job); 5032 } 5033 } 5034 5035 /** 5036 * amdgpu_device_gpu_recover_imp - reset the asic and recover scheduler 5037 * 5038 * @adev: amdgpu_device pointer 5039 * @job: which job trigger hang 5040 * 5041 * Attempt to reset the GPU if it has hung (all asics). 5042 * Attempt to do soft-reset or full-reset and reinitialize Asic 5043 * Returns 0 for success or an error on failure. 5044 */ 5045 5046 int amdgpu_device_gpu_recover_imp(struct amdgpu_device *adev, 5047 struct amdgpu_job *job) 5048 { 5049 struct list_head device_list, *device_list_handle = NULL; 5050 bool job_signaled = false; 5051 struct amdgpu_hive_info *hive = NULL; 5052 struct amdgpu_device *tmp_adev = NULL; 5053 int i, r = 0; 5054 bool need_emergency_restart = false; 5055 bool audio_suspended = false; 5056 int tmp_vram_lost_counter; 5057 struct amdgpu_reset_context reset_context; 5058 5059 memset(&reset_context, 0, sizeof(reset_context)); 5060 5061 /* 5062 * Special case: RAS triggered and full reset isn't supported 5063 */ 5064 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev); 5065 5066 /* 5067 * Flush RAM to disk so that after reboot 5068 * the user can read log and see why the system rebooted. 5069 */ 5070 if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) { 5071 DRM_WARN("Emergency reboot."); 5072 5073 ksys_sync_helper(); 5074 emergency_restart(); 5075 } 5076 5077 dev_info(adev->dev, "GPU %s begin!\n", 5078 need_emergency_restart ? "jobs stop":"reset"); 5079 5080 if (!amdgpu_sriov_vf(adev)) 5081 hive = amdgpu_get_xgmi_hive(adev); 5082 if (hive) 5083 mutex_lock(&hive->hive_lock); 5084 5085 reset_context.method = AMD_RESET_METHOD_NONE; 5086 reset_context.reset_req_dev = adev; 5087 reset_context.job = job; 5088 reset_context.hive = hive; 5089 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 5090 5091 /* 5092 * Build list of devices to reset. 5093 * In case we are in XGMI hive mode, resort the device list 5094 * to put adev in the 1st position. 5095 */ 5096 INIT_LIST_HEAD(&device_list); 5097 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) { 5098 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) 5099 list_add_tail(&tmp_adev->reset_list, &device_list); 5100 if (!list_is_first(&adev->reset_list, &device_list)) 5101 list_rotate_to_front(&adev->reset_list, &device_list); 5102 device_list_handle = &device_list; 5103 } else { 5104 list_add_tail(&adev->reset_list, &device_list); 5105 device_list_handle = &device_list; 5106 } 5107 5108 /* We need to lock reset domain only once both for XGMI and single device */ 5109 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5110 reset_list); 5111 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); 5112 5113 /* block all schedulers and reset given job's ring */ 5114 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5115 5116 amdgpu_device_set_mp1_state(tmp_adev); 5117 5118 /* 5119 * Try to put the audio codec into suspend state 5120 * before gpu reset started. 5121 * 5122 * Due to the power domain of the graphics device 5123 * is shared with AZ power domain. Without this, 5124 * we may change the audio hardware from behind 5125 * the audio driver's back. That will trigger 5126 * some audio codec errors. 5127 */ 5128 if (!amdgpu_device_suspend_display_audio(tmp_adev)) 5129 audio_suspended = true; 5130 5131 amdgpu_ras_set_error_query_ready(tmp_adev, false); 5132 5133 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); 5134 5135 if (!amdgpu_sriov_vf(tmp_adev)) 5136 amdgpu_amdkfd_pre_reset(tmp_adev); 5137 5138 /* 5139 * Mark these ASICs to be reseted as untracked first 5140 * And add them back after reset completed 5141 */ 5142 amdgpu_unregister_gpu_instance(tmp_adev); 5143 5144 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true); 5145 5146 /* disable ras on ALL IPs */ 5147 if (!need_emergency_restart && 5148 amdgpu_device_ip_need_full_reset(tmp_adev)) 5149 amdgpu_ras_suspend(tmp_adev); 5150 5151 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5152 struct amdgpu_ring *ring = tmp_adev->rings[i]; 5153 5154 if (!ring || !ring->sched.thread) 5155 continue; 5156 5157 drm_sched_stop(&ring->sched, job ? &job->base : NULL); 5158 5159 if (need_emergency_restart) 5160 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); 5161 } 5162 atomic_inc(&tmp_adev->gpu_reset_counter); 5163 } 5164 5165 if (need_emergency_restart) 5166 goto skip_sched_resume; 5167 5168 /* 5169 * Must check guilty signal here since after this point all old 5170 * HW fences are force signaled. 5171 * 5172 * job->base holds a reference to parent fence 5173 */ 5174 if (job && job->base.s_fence->parent && 5175 dma_fence_is_signaled(job->base.s_fence->parent)) { 5176 job_signaled = true; 5177 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); 5178 goto skip_hw_reset; 5179 } 5180 5181 retry: /* Rest of adevs pre asic reset from XGMI hive. */ 5182 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5183 r = amdgpu_device_pre_asic_reset(tmp_adev, &reset_context); 5184 /*TODO Should we stop ?*/ 5185 if (r) { 5186 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", 5187 r, adev_to_drm(tmp_adev)->unique); 5188 tmp_adev->asic_reset_res = r; 5189 } 5190 } 5191 5192 tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter)); 5193 /* Actual ASIC resets if needed.*/ 5194 /* Host driver will handle XGMI hive reset for SRIOV */ 5195 if (amdgpu_sriov_vf(adev)) { 5196 r = amdgpu_device_reset_sriov(adev, job ? false : true); 5197 if (r) 5198 adev->asic_reset_res = r; 5199 } else { 5200 r = amdgpu_do_asic_reset(device_list_handle, &reset_context); 5201 if (r && r == -EAGAIN) 5202 goto retry; 5203 } 5204 5205 skip_hw_reset: 5206 5207 /* Post ASIC reset for all devs .*/ 5208 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5209 5210 /* 5211 * Sometimes a later bad compute job can block a good gfx job as gfx 5212 * and compute ring share internal GC HW mutually. We add an additional 5213 * guilty jobs recheck step to find the real guilty job, it synchronously 5214 * submits and pends for the first job being signaled. If it gets timeout, 5215 * we identify it as a real guilty job. 5216 */ 5217 if (amdgpu_gpu_recovery == 2 && 5218 !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter))) 5219 amdgpu_device_recheck_guilty_jobs( 5220 tmp_adev, device_list_handle, &reset_context); 5221 5222 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5223 struct amdgpu_ring *ring = tmp_adev->rings[i]; 5224 5225 if (!ring || !ring->sched.thread) 5226 continue; 5227 5228 /* No point to resubmit jobs if we didn't HW reset*/ 5229 if (!tmp_adev->asic_reset_res && !job_signaled) 5230 drm_sched_resubmit_jobs(&ring->sched); 5231 5232 drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res); 5233 } 5234 5235 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) { 5236 drm_helper_resume_force_mode(adev_to_drm(tmp_adev)); 5237 } 5238 5239 if (tmp_adev->asic_reset_res) 5240 r = tmp_adev->asic_reset_res; 5241 5242 tmp_adev->asic_reset_res = 0; 5243 5244 if (r) { 5245 /* bad news, how to tell it to userspace ? */ 5246 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter)); 5247 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); 5248 } else { 5249 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter)); 5250 if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0)) 5251 DRM_WARN("smart shift update failed\n"); 5252 } 5253 } 5254 5255 skip_sched_resume: 5256 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5257 /* unlock kfd: SRIOV would do it separately */ 5258 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev)) 5259 amdgpu_amdkfd_post_reset(tmp_adev); 5260 5261 /* kfd_post_reset will do nothing if kfd device is not initialized, 5262 * need to bring up kfd here if it's not be initialized before 5263 */ 5264 if (!adev->kfd.init_complete) 5265 amdgpu_amdkfd_device_init(adev); 5266 5267 if (audio_suspended) 5268 amdgpu_device_resume_display_audio(tmp_adev); 5269 5270 amdgpu_device_unset_mp1_state(tmp_adev); 5271 } 5272 5273 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5274 reset_list); 5275 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); 5276 5277 if (hive) { 5278 mutex_unlock(&hive->hive_lock); 5279 amdgpu_put_xgmi_hive(hive); 5280 } 5281 5282 if (r) 5283 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); 5284 return r; 5285 } 5286 5287 struct amdgpu_recover_work_struct { 5288 struct work_struct base; 5289 struct amdgpu_device *adev; 5290 struct amdgpu_job *job; 5291 int ret; 5292 }; 5293 5294 static void amdgpu_device_queue_gpu_recover_work(struct work_struct *work) 5295 { 5296 struct amdgpu_recover_work_struct *recover_work = container_of(work, struct amdgpu_recover_work_struct, base); 5297 5298 recover_work->ret = amdgpu_device_gpu_recover_imp(recover_work->adev, recover_work->job); 5299 } 5300 /* 5301 * Serialize gpu recover into reset domain single threaded wq 5302 */ 5303 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, 5304 struct amdgpu_job *job) 5305 { 5306 struct amdgpu_recover_work_struct work = {.adev = adev, .job = job}; 5307 5308 INIT_WORK(&work.base, amdgpu_device_queue_gpu_recover_work); 5309 5310 if (!amdgpu_reset_domain_schedule(adev->reset_domain, &work.base)) 5311 return -EAGAIN; 5312 5313 flush_work(&work.base); 5314 5315 return work.ret; 5316 } 5317 5318 /** 5319 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot 5320 * 5321 * @adev: amdgpu_device pointer 5322 * 5323 * Fetchs and stores in the driver the PCIE capabilities (gen speed 5324 * and lanes) of the slot the device is in. Handles APUs and 5325 * virtualized environments where PCIE config space may not be available. 5326 */ 5327 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) 5328 { 5329 struct pci_dev *pdev; 5330 enum pci_bus_speed speed_cap, platform_speed_cap; 5331 enum pcie_link_width platform_link_width; 5332 5333 if (amdgpu_pcie_gen_cap) 5334 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; 5335 5336 if (amdgpu_pcie_lane_cap) 5337 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; 5338 5339 /* covers APUs as well */ 5340 if (pci_is_root_bus(adev->pdev->bus)) { 5341 if (adev->pm.pcie_gen_mask == 0) 5342 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; 5343 if (adev->pm.pcie_mlw_mask == 0) 5344 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; 5345 return; 5346 } 5347 5348 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) 5349 return; 5350 5351 pcie_bandwidth_available(adev->pdev, NULL, 5352 &platform_speed_cap, &platform_link_width); 5353 5354 if (adev->pm.pcie_gen_mask == 0) { 5355 /* asic caps */ 5356 pdev = adev->pdev; 5357 speed_cap = pcie_get_speed_cap(pdev); 5358 if (speed_cap == PCI_SPEED_UNKNOWN) { 5359 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5360 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5361 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 5362 } else { 5363 if (speed_cap == PCIE_SPEED_32_0GT) 5364 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5365 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5366 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5367 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 | 5368 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5); 5369 else if (speed_cap == PCIE_SPEED_16_0GT) 5370 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5371 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5372 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5373 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); 5374 else if (speed_cap == PCIE_SPEED_8_0GT) 5375 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5376 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5377 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 5378 else if (speed_cap == PCIE_SPEED_5_0GT) 5379 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5380 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); 5381 else 5382 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; 5383 } 5384 /* platform caps */ 5385 if (platform_speed_cap == PCI_SPEED_UNKNOWN) { 5386 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5387 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 5388 } else { 5389 if (platform_speed_cap == PCIE_SPEED_32_0GT) 5390 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5391 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5392 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5393 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 | 5394 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5); 5395 else if (platform_speed_cap == PCIE_SPEED_16_0GT) 5396 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5397 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5398 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5399 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); 5400 else if (platform_speed_cap == PCIE_SPEED_8_0GT) 5401 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5402 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5403 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); 5404 else if (platform_speed_cap == PCIE_SPEED_5_0GT) 5405 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5406 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 5407 else 5408 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; 5409 5410 } 5411 } 5412 if (adev->pm.pcie_mlw_mask == 0) { 5413 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { 5414 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; 5415 } else { 5416 switch (platform_link_width) { 5417 case PCIE_LNK_X32: 5418 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | 5419 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 5420 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 5421 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 5422 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5423 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5424 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5425 break; 5426 case PCIE_LNK_X16: 5427 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 5428 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 5429 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 5430 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5431 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5432 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5433 break; 5434 case PCIE_LNK_X12: 5435 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 5436 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 5437 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5438 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5439 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5440 break; 5441 case PCIE_LNK_X8: 5442 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 5443 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5444 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5445 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5446 break; 5447 case PCIE_LNK_X4: 5448 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5449 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5450 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5451 break; 5452 case PCIE_LNK_X2: 5453 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5454 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5455 break; 5456 case PCIE_LNK_X1: 5457 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; 5458 break; 5459 default: 5460 break; 5461 } 5462 } 5463 } 5464 } 5465 5466 int amdgpu_device_baco_enter(struct drm_device *dev) 5467 { 5468 struct amdgpu_device *adev = drm_to_adev(dev); 5469 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5470 5471 if (!amdgpu_device_supports_baco(adev_to_drm(adev))) 5472 return -ENOTSUPP; 5473 5474 if (ras && adev->ras_enabled && 5475 adev->nbio.funcs->enable_doorbell_interrupt) 5476 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); 5477 5478 return amdgpu_dpm_baco_enter(adev); 5479 } 5480 5481 int amdgpu_device_baco_exit(struct drm_device *dev) 5482 { 5483 struct amdgpu_device *adev = drm_to_adev(dev); 5484 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5485 int ret = 0; 5486 5487 if (!amdgpu_device_supports_baco(adev_to_drm(adev))) 5488 return -ENOTSUPP; 5489 5490 ret = amdgpu_dpm_baco_exit(adev); 5491 if (ret) 5492 return ret; 5493 5494 if (ras && adev->ras_enabled && 5495 adev->nbio.funcs->enable_doorbell_interrupt) 5496 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); 5497 5498 if (amdgpu_passthrough(adev) && 5499 adev->nbio.funcs->clear_doorbell_interrupt) 5500 adev->nbio.funcs->clear_doorbell_interrupt(adev); 5501 5502 return 0; 5503 } 5504 5505 /** 5506 * amdgpu_pci_error_detected - Called when a PCI error is detected. 5507 * @pdev: PCI device struct 5508 * @state: PCI channel state 5509 * 5510 * Description: Called when a PCI error is detected. 5511 * 5512 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT. 5513 */ 5514 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) 5515 { 5516 struct drm_device *dev = pci_get_drvdata(pdev); 5517 struct amdgpu_device *adev = drm_to_adev(dev); 5518 int i; 5519 5520 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state); 5521 5522 if (adev->gmc.xgmi.num_physical_nodes > 1) { 5523 DRM_WARN("No support for XGMI hive yet..."); 5524 return PCI_ERS_RESULT_DISCONNECT; 5525 } 5526 5527 adev->pci_channel_state = state; 5528 5529 switch (state) { 5530 case pci_channel_io_normal: 5531 return PCI_ERS_RESULT_CAN_RECOVER; 5532 /* Fatal error, prepare for slot reset */ 5533 case pci_channel_io_frozen: 5534 /* 5535 * Locking adev->reset_domain->sem will prevent any external access 5536 * to GPU during PCI error recovery 5537 */ 5538 amdgpu_device_lock_reset_domain(adev->reset_domain); 5539 amdgpu_device_set_mp1_state(adev); 5540 5541 /* 5542 * Block any work scheduling as we do for regular GPU reset 5543 * for the duration of the recovery 5544 */ 5545 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5546 struct amdgpu_ring *ring = adev->rings[i]; 5547 5548 if (!ring || !ring->sched.thread) 5549 continue; 5550 5551 drm_sched_stop(&ring->sched, NULL); 5552 } 5553 atomic_inc(&adev->gpu_reset_counter); 5554 return PCI_ERS_RESULT_NEED_RESET; 5555 case pci_channel_io_perm_failure: 5556 /* Permanent error, prepare for device removal */ 5557 return PCI_ERS_RESULT_DISCONNECT; 5558 } 5559 5560 return PCI_ERS_RESULT_NEED_RESET; 5561 } 5562 5563 /** 5564 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers 5565 * @pdev: pointer to PCI device 5566 */ 5567 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev) 5568 { 5569 5570 DRM_INFO("PCI error: mmio enabled callback!!\n"); 5571 5572 /* TODO - dump whatever for debugging purposes */ 5573 5574 /* This called only if amdgpu_pci_error_detected returns 5575 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still 5576 * works, no need to reset slot. 5577 */ 5578 5579 return PCI_ERS_RESULT_RECOVERED; 5580 } 5581 5582 /** 5583 * amdgpu_pci_slot_reset - Called when PCI slot has been reset. 5584 * @pdev: PCI device struct 5585 * 5586 * Description: This routine is called by the pci error recovery 5587 * code after the PCI slot has been reset, just before we 5588 * should resume normal operations. 5589 */ 5590 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) 5591 { 5592 struct drm_device *dev = pci_get_drvdata(pdev); 5593 struct amdgpu_device *adev = drm_to_adev(dev); 5594 int r, i; 5595 struct amdgpu_reset_context reset_context; 5596 u32 memsize; 5597 struct list_head device_list; 5598 5599 DRM_INFO("PCI error: slot reset callback!!\n"); 5600 5601 memset(&reset_context, 0, sizeof(reset_context)); 5602 5603 INIT_LIST_HEAD(&device_list); 5604 list_add_tail(&adev->reset_list, &device_list); 5605 5606 /* wait for asic to come out of reset */ 5607 msleep(500); 5608 5609 /* Restore PCI confspace */ 5610 amdgpu_device_load_pci_state(pdev); 5611 5612 /* confirm ASIC came out of reset */ 5613 for (i = 0; i < adev->usec_timeout; i++) { 5614 memsize = amdgpu_asic_get_config_memsize(adev); 5615 5616 if (memsize != 0xffffffff) 5617 break; 5618 udelay(1); 5619 } 5620 if (memsize == 0xffffffff) { 5621 r = -ETIME; 5622 goto out; 5623 } 5624 5625 reset_context.method = AMD_RESET_METHOD_NONE; 5626 reset_context.reset_req_dev = adev; 5627 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 5628 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); 5629 5630 adev->no_hw_access = true; 5631 r = amdgpu_device_pre_asic_reset(adev, &reset_context); 5632 adev->no_hw_access = false; 5633 if (r) 5634 goto out; 5635 5636 r = amdgpu_do_asic_reset(&device_list, &reset_context); 5637 5638 out: 5639 if (!r) { 5640 if (amdgpu_device_cache_pci_state(adev->pdev)) 5641 pci_restore_state(adev->pdev); 5642 5643 DRM_INFO("PCIe error recovery succeeded\n"); 5644 } else { 5645 DRM_ERROR("PCIe error recovery failed, err:%d", r); 5646 amdgpu_device_unset_mp1_state(adev); 5647 amdgpu_device_unlock_reset_domain(adev->reset_domain); 5648 } 5649 5650 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 5651 } 5652 5653 /** 5654 * amdgpu_pci_resume() - resume normal ops after PCI reset 5655 * @pdev: pointer to PCI device 5656 * 5657 * Called when the error recovery driver tells us that its 5658 * OK to resume normal operation. 5659 */ 5660 void amdgpu_pci_resume(struct pci_dev *pdev) 5661 { 5662 struct drm_device *dev = pci_get_drvdata(pdev); 5663 struct amdgpu_device *adev = drm_to_adev(dev); 5664 int i; 5665 5666 5667 DRM_INFO("PCI error: resume callback!!\n"); 5668 5669 /* Only continue execution for the case of pci_channel_io_frozen */ 5670 if (adev->pci_channel_state != pci_channel_io_frozen) 5671 return; 5672 5673 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5674 struct amdgpu_ring *ring = adev->rings[i]; 5675 5676 if (!ring || !ring->sched.thread) 5677 continue; 5678 5679 5680 drm_sched_resubmit_jobs(&ring->sched); 5681 drm_sched_start(&ring->sched, true); 5682 } 5683 5684 amdgpu_device_unset_mp1_state(adev); 5685 amdgpu_device_unlock_reset_domain(adev->reset_domain); 5686 } 5687 5688 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev) 5689 { 5690 struct drm_device *dev = pci_get_drvdata(pdev); 5691 struct amdgpu_device *adev = drm_to_adev(dev); 5692 int r; 5693 5694 r = pci_save_state(pdev); 5695 if (!r) { 5696 kfree(adev->pci_state); 5697 5698 adev->pci_state = pci_store_saved_state(pdev); 5699 5700 if (!adev->pci_state) { 5701 DRM_ERROR("Failed to store PCI saved state"); 5702 return false; 5703 } 5704 } else { 5705 DRM_WARN("Failed to save PCI state, err:%d\n", r); 5706 return false; 5707 } 5708 5709 return true; 5710 } 5711 5712 bool amdgpu_device_load_pci_state(struct pci_dev *pdev) 5713 { 5714 struct drm_device *dev = pci_get_drvdata(pdev); 5715 struct amdgpu_device *adev = drm_to_adev(dev); 5716 int r; 5717 5718 if (!adev->pci_state) 5719 return false; 5720 5721 r = pci_load_saved_state(pdev, adev->pci_state); 5722 5723 if (!r) { 5724 pci_restore_state(pdev); 5725 } else { 5726 DRM_WARN("Failed to load PCI state, err:%d\n", r); 5727 return false; 5728 } 5729 5730 return true; 5731 } 5732 5733 void amdgpu_device_flush_hdp(struct amdgpu_device *adev, 5734 struct amdgpu_ring *ring) 5735 { 5736 #ifdef CONFIG_X86_64 5737 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 5738 return; 5739 #endif 5740 if (adev->gmc.xgmi.connected_to_cpu) 5741 return; 5742 5743 if (ring && ring->funcs->emit_hdp_flush) 5744 amdgpu_ring_emit_hdp_flush(ring); 5745 else 5746 amdgpu_asic_flush_hdp(adev, ring); 5747 } 5748 5749 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev, 5750 struct amdgpu_ring *ring) 5751 { 5752 #ifdef CONFIG_X86_64 5753 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 5754 return; 5755 #endif 5756 if (adev->gmc.xgmi.connected_to_cpu) 5757 return; 5758 5759 amdgpu_asic_invalidate_hdp(adev, ring); 5760 } 5761 5762 int amdgpu_in_reset(struct amdgpu_device *adev) 5763 { 5764 return atomic_read(&adev->reset_domain->in_gpu_reset); 5765 } 5766 5767 /** 5768 * amdgpu_device_halt() - bring hardware to some kind of halt state 5769 * 5770 * @adev: amdgpu_device pointer 5771 * 5772 * Bring hardware to some kind of halt state so that no one can touch it 5773 * any more. It will help to maintain error context when error occurred. 5774 * Compare to a simple hang, the system will keep stable at least for SSH 5775 * access. Then it should be trivial to inspect the hardware state and 5776 * see what's going on. Implemented as following: 5777 * 5778 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc), 5779 * clears all CPU mappings to device, disallows remappings through page faults 5780 * 2. amdgpu_irq_disable_all() disables all interrupts 5781 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences 5782 * 4. set adev->no_hw_access to avoid potential crashes after setp 5 5783 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings 5784 * 6. pci_disable_device() and pci_wait_for_pending_transaction() 5785 * flush any in flight DMA operations 5786 */ 5787 void amdgpu_device_halt(struct amdgpu_device *adev) 5788 { 5789 struct pci_dev *pdev = adev->pdev; 5790 struct drm_device *ddev = adev_to_drm(adev); 5791 5792 drm_dev_unplug(ddev); 5793 5794 amdgpu_irq_disable_all(adev); 5795 5796 amdgpu_fence_driver_hw_fini(adev); 5797 5798 adev->no_hw_access = true; 5799 5800 amdgpu_device_unmap_mmio(adev); 5801 5802 pci_disable_device(pdev); 5803 pci_wait_for_pending_transaction(pdev); 5804 } 5805 5806 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, 5807 u32 reg) 5808 { 5809 unsigned long flags, address, data; 5810 u32 r; 5811 5812 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 5813 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 5814 5815 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 5816 WREG32(address, reg * 4); 5817 (void)RREG32(address); 5818 r = RREG32(data); 5819 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 5820 return r; 5821 } 5822 5823 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, 5824 u32 reg, u32 v) 5825 { 5826 unsigned long flags, address, data; 5827 5828 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 5829 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 5830 5831 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 5832 WREG32(address, reg * 4); 5833 (void)RREG32(address); 5834 WREG32(data, v); 5835 (void)RREG32(data); 5836 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 5837 } 5838