1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/power_supply.h> 29 #include <linux/kthread.h> 30 #include <linux/module.h> 31 #include <linux/console.h> 32 #include <linux/slab.h> 33 #include <linux/iommu.h> 34 #include <linux/pci.h> 35 #include <linux/devcoredump.h> 36 #include <generated/utsrelease.h> 37 #include <linux/pci-p2pdma.h> 38 39 #include <drm/drm_atomic_helper.h> 40 #include <drm/drm_fb_helper.h> 41 #include <drm/drm_probe_helper.h> 42 #include <drm/amdgpu_drm.h> 43 #include <linux/vgaarb.h> 44 #include <linux/vga_switcheroo.h> 45 #include <linux/efi.h> 46 #include "amdgpu.h" 47 #include "amdgpu_trace.h" 48 #include "amdgpu_i2c.h" 49 #include "atom.h" 50 #include "amdgpu_atombios.h" 51 #include "amdgpu_atomfirmware.h" 52 #include "amd_pcie.h" 53 #ifdef CONFIG_DRM_AMDGPU_SI 54 #include "si.h" 55 #endif 56 #ifdef CONFIG_DRM_AMDGPU_CIK 57 #include "cik.h" 58 #endif 59 #include "vi.h" 60 #include "soc15.h" 61 #include "nv.h" 62 #include "bif/bif_4_1_d.h" 63 #include <linux/firmware.h> 64 #include "amdgpu_vf_error.h" 65 66 #include "amdgpu_amdkfd.h" 67 #include "amdgpu_pm.h" 68 69 #include "amdgpu_xgmi.h" 70 #include "amdgpu_ras.h" 71 #include "amdgpu_pmu.h" 72 #include "amdgpu_fru_eeprom.h" 73 #include "amdgpu_reset.h" 74 75 #include <linux/suspend.h> 76 #include <drm/task_barrier.h> 77 #include <linux/pm_runtime.h> 78 79 #include <drm/drm_drv.h> 80 81 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 82 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); 83 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 84 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); 85 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); 86 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin"); 87 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin"); 88 89 #define AMDGPU_RESUME_MS 2000 90 #define AMDGPU_MAX_RETRY_LIMIT 2 91 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL) 92 93 const char *amdgpu_asic_name[] = { 94 "TAHITI", 95 "PITCAIRN", 96 "VERDE", 97 "OLAND", 98 "HAINAN", 99 "BONAIRE", 100 "KAVERI", 101 "KABINI", 102 "HAWAII", 103 "MULLINS", 104 "TOPAZ", 105 "TONGA", 106 "FIJI", 107 "CARRIZO", 108 "STONEY", 109 "POLARIS10", 110 "POLARIS11", 111 "POLARIS12", 112 "VEGAM", 113 "VEGA10", 114 "VEGA12", 115 "VEGA20", 116 "RAVEN", 117 "ARCTURUS", 118 "RENOIR", 119 "ALDEBARAN", 120 "NAVI10", 121 "CYAN_SKILLFISH", 122 "NAVI14", 123 "NAVI12", 124 "SIENNA_CICHLID", 125 "NAVY_FLOUNDER", 126 "VANGOGH", 127 "DIMGREY_CAVEFISH", 128 "BEIGE_GOBY", 129 "YELLOW_CARP", 130 "IP DISCOVERY", 131 "LAST", 132 }; 133 134 /** 135 * DOC: pcie_replay_count 136 * 137 * The amdgpu driver provides a sysfs API for reporting the total number 138 * of PCIe replays (NAKs) 139 * The file pcie_replay_count is used for this and returns the total 140 * number of replays as a sum of the NAKs generated and NAKs received 141 */ 142 143 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, 144 struct device_attribute *attr, char *buf) 145 { 146 struct drm_device *ddev = dev_get_drvdata(dev); 147 struct amdgpu_device *adev = drm_to_adev(ddev); 148 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); 149 150 return sysfs_emit(buf, "%llu\n", cnt); 151 } 152 153 static DEVICE_ATTR(pcie_replay_count, S_IRUGO, 154 amdgpu_device_get_pcie_replay_count, NULL); 155 156 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); 157 158 /** 159 * DOC: product_name 160 * 161 * The amdgpu driver provides a sysfs API for reporting the product name 162 * for the device 163 * The file serial_number is used for this and returns the product name 164 * as returned from the FRU. 165 * NOTE: This is only available for certain server cards 166 */ 167 168 static ssize_t amdgpu_device_get_product_name(struct device *dev, 169 struct device_attribute *attr, char *buf) 170 { 171 struct drm_device *ddev = dev_get_drvdata(dev); 172 struct amdgpu_device *adev = drm_to_adev(ddev); 173 174 return sysfs_emit(buf, "%s\n", adev->product_name); 175 } 176 177 static DEVICE_ATTR(product_name, S_IRUGO, 178 amdgpu_device_get_product_name, NULL); 179 180 /** 181 * DOC: product_number 182 * 183 * The amdgpu driver provides a sysfs API for reporting the part number 184 * for the device 185 * The file serial_number is used for this and returns the part number 186 * as returned from the FRU. 187 * NOTE: This is only available for certain server cards 188 */ 189 190 static ssize_t amdgpu_device_get_product_number(struct device *dev, 191 struct device_attribute *attr, char *buf) 192 { 193 struct drm_device *ddev = dev_get_drvdata(dev); 194 struct amdgpu_device *adev = drm_to_adev(ddev); 195 196 return sysfs_emit(buf, "%s\n", adev->product_number); 197 } 198 199 static DEVICE_ATTR(product_number, S_IRUGO, 200 amdgpu_device_get_product_number, NULL); 201 202 /** 203 * DOC: serial_number 204 * 205 * The amdgpu driver provides a sysfs API for reporting the serial number 206 * for the device 207 * The file serial_number is used for this and returns the serial number 208 * as returned from the FRU. 209 * NOTE: This is only available for certain server cards 210 */ 211 212 static ssize_t amdgpu_device_get_serial_number(struct device *dev, 213 struct device_attribute *attr, char *buf) 214 { 215 struct drm_device *ddev = dev_get_drvdata(dev); 216 struct amdgpu_device *adev = drm_to_adev(ddev); 217 218 return sysfs_emit(buf, "%s\n", adev->serial); 219 } 220 221 static DEVICE_ATTR(serial_number, S_IRUGO, 222 amdgpu_device_get_serial_number, NULL); 223 224 /** 225 * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control 226 * 227 * @dev: drm_device pointer 228 * 229 * Returns true if the device is a dGPU with ATPX power control, 230 * otherwise return false. 231 */ 232 bool amdgpu_device_supports_px(struct drm_device *dev) 233 { 234 struct amdgpu_device *adev = drm_to_adev(dev); 235 236 if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid()) 237 return true; 238 return false; 239 } 240 241 /** 242 * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources 243 * 244 * @dev: drm_device pointer 245 * 246 * Returns true if the device is a dGPU with ACPI power control, 247 * otherwise return false. 248 */ 249 bool amdgpu_device_supports_boco(struct drm_device *dev) 250 { 251 struct amdgpu_device *adev = drm_to_adev(dev); 252 253 if (adev->has_pr3 || 254 ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid())) 255 return true; 256 return false; 257 } 258 259 /** 260 * amdgpu_device_supports_baco - Does the device support BACO 261 * 262 * @dev: drm_device pointer 263 * 264 * Returns true if the device supporte BACO, 265 * otherwise return false. 266 */ 267 bool amdgpu_device_supports_baco(struct drm_device *dev) 268 { 269 struct amdgpu_device *adev = drm_to_adev(dev); 270 271 return amdgpu_asic_supports_baco(adev); 272 } 273 274 /** 275 * amdgpu_device_supports_smart_shift - Is the device dGPU with 276 * smart shift support 277 * 278 * @dev: drm_device pointer 279 * 280 * Returns true if the device is a dGPU with Smart Shift support, 281 * otherwise returns false. 282 */ 283 bool amdgpu_device_supports_smart_shift(struct drm_device *dev) 284 { 285 return (amdgpu_device_supports_boco(dev) && 286 amdgpu_acpi_is_power_shift_control_supported()); 287 } 288 289 /* 290 * VRAM access helper functions 291 */ 292 293 /** 294 * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA 295 * 296 * @adev: amdgpu_device pointer 297 * @pos: offset of the buffer in vram 298 * @buf: virtual address of the buffer in system memory 299 * @size: read/write size, sizeof(@buf) must > @size 300 * @write: true - write to vram, otherwise - read from vram 301 */ 302 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos, 303 void *buf, size_t size, bool write) 304 { 305 unsigned long flags; 306 uint32_t hi = ~0, tmp = 0; 307 uint32_t *data = buf; 308 uint64_t last; 309 int idx; 310 311 if (!drm_dev_enter(adev_to_drm(adev), &idx)) 312 return; 313 314 BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)); 315 316 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 317 for (last = pos + size; pos < last; pos += 4) { 318 tmp = pos >> 31; 319 320 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000); 321 if (tmp != hi) { 322 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp); 323 hi = tmp; 324 } 325 if (write) 326 WREG32_NO_KIQ(mmMM_DATA, *data++); 327 else 328 *data++ = RREG32_NO_KIQ(mmMM_DATA); 329 } 330 331 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 332 drm_dev_exit(idx); 333 } 334 335 /** 336 * amdgpu_device_aper_access - access vram by vram aperature 337 * 338 * @adev: amdgpu_device pointer 339 * @pos: offset of the buffer in vram 340 * @buf: virtual address of the buffer in system memory 341 * @size: read/write size, sizeof(@buf) must > @size 342 * @write: true - write to vram, otherwise - read from vram 343 * 344 * The return value means how many bytes have been transferred. 345 */ 346 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos, 347 void *buf, size_t size, bool write) 348 { 349 #ifdef CONFIG_64BIT 350 void __iomem *addr; 351 size_t count = 0; 352 uint64_t last; 353 354 if (!adev->mman.aper_base_kaddr) 355 return 0; 356 357 last = min(pos + size, adev->gmc.visible_vram_size); 358 if (last > pos) { 359 addr = adev->mman.aper_base_kaddr + pos; 360 count = last - pos; 361 362 if (write) { 363 memcpy_toio(addr, buf, count); 364 mb(); 365 amdgpu_device_flush_hdp(adev, NULL); 366 } else { 367 amdgpu_device_invalidate_hdp(adev, NULL); 368 mb(); 369 memcpy_fromio(buf, addr, count); 370 } 371 372 } 373 374 return count; 375 #else 376 return 0; 377 #endif 378 } 379 380 /** 381 * amdgpu_device_vram_access - read/write a buffer in vram 382 * 383 * @adev: amdgpu_device pointer 384 * @pos: offset of the buffer in vram 385 * @buf: virtual address of the buffer in system memory 386 * @size: read/write size, sizeof(@buf) must > @size 387 * @write: true - write to vram, otherwise - read from vram 388 */ 389 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, 390 void *buf, size_t size, bool write) 391 { 392 size_t count; 393 394 /* try to using vram apreature to access vram first */ 395 count = amdgpu_device_aper_access(adev, pos, buf, size, write); 396 size -= count; 397 if (size) { 398 /* using MM to access rest vram */ 399 pos += count; 400 buf += count; 401 amdgpu_device_mm_access(adev, pos, buf, size, write); 402 } 403 } 404 405 /* 406 * register access helper functions. 407 */ 408 409 /* Check if hw access should be skipped because of hotplug or device error */ 410 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev) 411 { 412 if (adev->no_hw_access) 413 return true; 414 415 #ifdef CONFIG_LOCKDEP 416 /* 417 * This is a bit complicated to understand, so worth a comment. What we assert 418 * here is that the GPU reset is not running on another thread in parallel. 419 * 420 * For this we trylock the read side of the reset semaphore, if that succeeds 421 * we know that the reset is not running in paralell. 422 * 423 * If the trylock fails we assert that we are either already holding the read 424 * side of the lock or are the reset thread itself and hold the write side of 425 * the lock. 426 */ 427 if (in_task()) { 428 if (down_read_trylock(&adev->reset_domain->sem)) 429 up_read(&adev->reset_domain->sem); 430 else 431 lockdep_assert_held(&adev->reset_domain->sem); 432 } 433 #endif 434 return false; 435 } 436 437 /** 438 * amdgpu_device_rreg - read a memory mapped IO or indirect register 439 * 440 * @adev: amdgpu_device pointer 441 * @reg: dword aligned register offset 442 * @acc_flags: access flags which require special behavior 443 * 444 * Returns the 32 bit value from the offset specified. 445 */ 446 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, 447 uint32_t reg, uint32_t acc_flags) 448 { 449 uint32_t ret; 450 451 if (amdgpu_device_skip_hw_access(adev)) 452 return 0; 453 454 if ((reg * 4) < adev->rmmio_size) { 455 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 456 amdgpu_sriov_runtime(adev) && 457 down_read_trylock(&adev->reset_domain->sem)) { 458 ret = amdgpu_kiq_rreg(adev, reg); 459 up_read(&adev->reset_domain->sem); 460 } else { 461 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 462 } 463 } else { 464 ret = adev->pcie_rreg(adev, reg * 4); 465 } 466 467 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); 468 469 return ret; 470 } 471 472 /* 473 * MMIO register read with bytes helper functions 474 * @offset:bytes offset from MMIO start 475 * 476 */ 477 478 /** 479 * amdgpu_mm_rreg8 - read a memory mapped IO register 480 * 481 * @adev: amdgpu_device pointer 482 * @offset: byte aligned register offset 483 * 484 * Returns the 8 bit value from the offset specified. 485 */ 486 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) 487 { 488 if (amdgpu_device_skip_hw_access(adev)) 489 return 0; 490 491 if (offset < adev->rmmio_size) 492 return (readb(adev->rmmio + offset)); 493 BUG(); 494 } 495 496 /* 497 * MMIO register write with bytes helper functions 498 * @offset:bytes offset from MMIO start 499 * @value: the value want to be written to the register 500 * 501 */ 502 /** 503 * amdgpu_mm_wreg8 - read a memory mapped IO register 504 * 505 * @adev: amdgpu_device pointer 506 * @offset: byte aligned register offset 507 * @value: 8 bit value to write 508 * 509 * Writes the value specified to the offset specified. 510 */ 511 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) 512 { 513 if (amdgpu_device_skip_hw_access(adev)) 514 return; 515 516 if (offset < adev->rmmio_size) 517 writeb(value, adev->rmmio + offset); 518 else 519 BUG(); 520 } 521 522 /** 523 * amdgpu_device_wreg - write to a memory mapped IO or indirect register 524 * 525 * @adev: amdgpu_device pointer 526 * @reg: dword aligned register offset 527 * @v: 32 bit value to write to the register 528 * @acc_flags: access flags which require special behavior 529 * 530 * Writes the value specified to the offset specified. 531 */ 532 void amdgpu_device_wreg(struct amdgpu_device *adev, 533 uint32_t reg, uint32_t v, 534 uint32_t acc_flags) 535 { 536 if (amdgpu_device_skip_hw_access(adev)) 537 return; 538 539 if ((reg * 4) < adev->rmmio_size) { 540 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 541 amdgpu_sriov_runtime(adev) && 542 down_read_trylock(&adev->reset_domain->sem)) { 543 amdgpu_kiq_wreg(adev, reg, v); 544 up_read(&adev->reset_domain->sem); 545 } else { 546 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 547 } 548 } else { 549 adev->pcie_wreg(adev, reg * 4, v); 550 } 551 552 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); 553 } 554 555 /** 556 * amdgpu_mm_wreg_mmio_rlc - write register either with direct/indirect mmio or with RLC path if in range 557 * 558 * @adev: amdgpu_device pointer 559 * @reg: mmio/rlc register 560 * @v: value to write 561 * 562 * this function is invoked only for the debugfs register access 563 */ 564 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, 565 uint32_t reg, uint32_t v) 566 { 567 if (amdgpu_device_skip_hw_access(adev)) 568 return; 569 570 if (amdgpu_sriov_fullaccess(adev) && 571 adev->gfx.rlc.funcs && 572 adev->gfx.rlc.funcs->is_rlcg_access_range) { 573 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) 574 return amdgpu_sriov_wreg(adev, reg, v, 0, 0); 575 } else if ((reg * 4) >= adev->rmmio_size) { 576 adev->pcie_wreg(adev, reg * 4, v); 577 } else { 578 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 579 } 580 } 581 582 /** 583 * amdgpu_mm_rdoorbell - read a doorbell dword 584 * 585 * @adev: amdgpu_device pointer 586 * @index: doorbell index 587 * 588 * Returns the value in the doorbell aperture at the 589 * requested doorbell index (CIK). 590 */ 591 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) 592 { 593 if (amdgpu_device_skip_hw_access(adev)) 594 return 0; 595 596 if (index < adev->doorbell.num_doorbells) { 597 return readl(adev->doorbell.ptr + index); 598 } else { 599 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 600 return 0; 601 } 602 } 603 604 /** 605 * amdgpu_mm_wdoorbell - write a doorbell dword 606 * 607 * @adev: amdgpu_device pointer 608 * @index: doorbell index 609 * @v: value to write 610 * 611 * Writes @v to the doorbell aperture at the 612 * requested doorbell index (CIK). 613 */ 614 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) 615 { 616 if (amdgpu_device_skip_hw_access(adev)) 617 return; 618 619 if (index < adev->doorbell.num_doorbells) { 620 writel(v, adev->doorbell.ptr + index); 621 } else { 622 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 623 } 624 } 625 626 /** 627 * amdgpu_mm_rdoorbell64 - read a doorbell Qword 628 * 629 * @adev: amdgpu_device pointer 630 * @index: doorbell index 631 * 632 * Returns the value in the doorbell aperture at the 633 * requested doorbell index (VEGA10+). 634 */ 635 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) 636 { 637 if (amdgpu_device_skip_hw_access(adev)) 638 return 0; 639 640 if (index < adev->doorbell.num_doorbells) { 641 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index)); 642 } else { 643 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 644 return 0; 645 } 646 } 647 648 /** 649 * amdgpu_mm_wdoorbell64 - write a doorbell Qword 650 * 651 * @adev: amdgpu_device pointer 652 * @index: doorbell index 653 * @v: value to write 654 * 655 * Writes @v to the doorbell aperture at the 656 * requested doorbell index (VEGA10+). 657 */ 658 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) 659 { 660 if (amdgpu_device_skip_hw_access(adev)) 661 return; 662 663 if (index < adev->doorbell.num_doorbells) { 664 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v); 665 } else { 666 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 667 } 668 } 669 670 /** 671 * amdgpu_device_indirect_rreg - read an indirect register 672 * 673 * @adev: amdgpu_device pointer 674 * @pcie_index: mmio register offset 675 * @pcie_data: mmio register offset 676 * @reg_addr: indirect register address to read from 677 * 678 * Returns the value of indirect register @reg_addr 679 */ 680 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, 681 u32 pcie_index, u32 pcie_data, 682 u32 reg_addr) 683 { 684 unsigned long flags; 685 u32 r; 686 void __iomem *pcie_index_offset; 687 void __iomem *pcie_data_offset; 688 689 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 690 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 691 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 692 693 writel(reg_addr, pcie_index_offset); 694 readl(pcie_index_offset); 695 r = readl(pcie_data_offset); 696 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 697 698 return r; 699 } 700 701 /** 702 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register 703 * 704 * @adev: amdgpu_device pointer 705 * @pcie_index: mmio register offset 706 * @pcie_data: mmio register offset 707 * @reg_addr: indirect register address to read from 708 * 709 * Returns the value of indirect register @reg_addr 710 */ 711 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, 712 u32 pcie_index, u32 pcie_data, 713 u32 reg_addr) 714 { 715 unsigned long flags; 716 u64 r; 717 void __iomem *pcie_index_offset; 718 void __iomem *pcie_data_offset; 719 720 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 721 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 722 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 723 724 /* read low 32 bits */ 725 writel(reg_addr, pcie_index_offset); 726 readl(pcie_index_offset); 727 r = readl(pcie_data_offset); 728 /* read high 32 bits */ 729 writel(reg_addr + 4, pcie_index_offset); 730 readl(pcie_index_offset); 731 r |= ((u64)readl(pcie_data_offset) << 32); 732 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 733 734 return r; 735 } 736 737 /** 738 * amdgpu_device_indirect_wreg - write an indirect register address 739 * 740 * @adev: amdgpu_device pointer 741 * @pcie_index: mmio register offset 742 * @pcie_data: mmio register offset 743 * @reg_addr: indirect register offset 744 * @reg_data: indirect register data 745 * 746 */ 747 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, 748 u32 pcie_index, u32 pcie_data, 749 u32 reg_addr, u32 reg_data) 750 { 751 unsigned long flags; 752 void __iomem *pcie_index_offset; 753 void __iomem *pcie_data_offset; 754 755 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 756 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 757 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 758 759 writel(reg_addr, pcie_index_offset); 760 readl(pcie_index_offset); 761 writel(reg_data, pcie_data_offset); 762 readl(pcie_data_offset); 763 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 764 } 765 766 /** 767 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address 768 * 769 * @adev: amdgpu_device pointer 770 * @pcie_index: mmio register offset 771 * @pcie_data: mmio register offset 772 * @reg_addr: indirect register offset 773 * @reg_data: indirect register data 774 * 775 */ 776 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, 777 u32 pcie_index, u32 pcie_data, 778 u32 reg_addr, u64 reg_data) 779 { 780 unsigned long flags; 781 void __iomem *pcie_index_offset; 782 void __iomem *pcie_data_offset; 783 784 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 785 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 786 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 787 788 /* write low 32 bits */ 789 writel(reg_addr, pcie_index_offset); 790 readl(pcie_index_offset); 791 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 792 readl(pcie_data_offset); 793 /* write high 32 bits */ 794 writel(reg_addr + 4, pcie_index_offset); 795 readl(pcie_index_offset); 796 writel((u32)(reg_data >> 32), pcie_data_offset); 797 readl(pcie_data_offset); 798 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 799 } 800 801 /** 802 * amdgpu_invalid_rreg - dummy reg read function 803 * 804 * @adev: amdgpu_device pointer 805 * @reg: offset of register 806 * 807 * Dummy register read function. Used for register blocks 808 * that certain asics don't have (all asics). 809 * Returns the value in the register. 810 */ 811 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) 812 { 813 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg); 814 BUG(); 815 return 0; 816 } 817 818 /** 819 * amdgpu_invalid_wreg - dummy reg write function 820 * 821 * @adev: amdgpu_device pointer 822 * @reg: offset of register 823 * @v: value to write to the register 824 * 825 * Dummy register read function. Used for register blocks 826 * that certain asics don't have (all asics). 827 */ 828 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 829 { 830 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n", 831 reg, v); 832 BUG(); 833 } 834 835 /** 836 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function 837 * 838 * @adev: amdgpu_device pointer 839 * @reg: offset of register 840 * 841 * Dummy register read function. Used for register blocks 842 * that certain asics don't have (all asics). 843 * Returns the value in the register. 844 */ 845 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg) 846 { 847 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg); 848 BUG(); 849 return 0; 850 } 851 852 /** 853 * amdgpu_invalid_wreg64 - dummy reg write function 854 * 855 * @adev: amdgpu_device pointer 856 * @reg: offset of register 857 * @v: value to write to the register 858 * 859 * Dummy register read function. Used for register blocks 860 * that certain asics don't have (all asics). 861 */ 862 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v) 863 { 864 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n", 865 reg, v); 866 BUG(); 867 } 868 869 /** 870 * amdgpu_block_invalid_rreg - dummy reg read function 871 * 872 * @adev: amdgpu_device pointer 873 * @block: offset of instance 874 * @reg: offset of register 875 * 876 * Dummy register read function. Used for register blocks 877 * that certain asics don't have (all asics). 878 * Returns the value in the register. 879 */ 880 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, 881 uint32_t block, uint32_t reg) 882 { 883 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n", 884 reg, block); 885 BUG(); 886 return 0; 887 } 888 889 /** 890 * amdgpu_block_invalid_wreg - dummy reg write function 891 * 892 * @adev: amdgpu_device pointer 893 * @block: offset of instance 894 * @reg: offset of register 895 * @v: value to write to the register 896 * 897 * Dummy register read function. Used for register blocks 898 * that certain asics don't have (all asics). 899 */ 900 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, 901 uint32_t block, 902 uint32_t reg, uint32_t v) 903 { 904 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n", 905 reg, block, v); 906 BUG(); 907 } 908 909 /** 910 * amdgpu_device_asic_init - Wrapper for atom asic_init 911 * 912 * @adev: amdgpu_device pointer 913 * 914 * Does any asic specific work and then calls atom asic init. 915 */ 916 static int amdgpu_device_asic_init(struct amdgpu_device *adev) 917 { 918 amdgpu_asic_pre_asic_init(adev); 919 920 if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0)) 921 return amdgpu_atomfirmware_asic_init(adev, true); 922 else 923 return amdgpu_atom_asic_init(adev->mode_info.atom_context); 924 } 925 926 /** 927 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page 928 * 929 * @adev: amdgpu_device pointer 930 * 931 * Allocates a scratch page of VRAM for use by various things in the 932 * driver. 933 */ 934 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) 935 { 936 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, 937 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 938 &adev->vram_scratch.robj, 939 &adev->vram_scratch.gpu_addr, 940 (void **)&adev->vram_scratch.ptr); 941 } 942 943 /** 944 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page 945 * 946 * @adev: amdgpu_device pointer 947 * 948 * Frees the VRAM scratch page. 949 */ 950 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev) 951 { 952 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL); 953 } 954 955 /** 956 * amdgpu_device_program_register_sequence - program an array of registers. 957 * 958 * @adev: amdgpu_device pointer 959 * @registers: pointer to the register array 960 * @array_size: size of the register array 961 * 962 * Programs an array or registers with and and or masks. 963 * This is a helper for setting golden registers. 964 */ 965 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, 966 const u32 *registers, 967 const u32 array_size) 968 { 969 u32 tmp, reg, and_mask, or_mask; 970 int i; 971 972 if (array_size % 3) 973 return; 974 975 for (i = 0; i < array_size; i +=3) { 976 reg = registers[i + 0]; 977 and_mask = registers[i + 1]; 978 or_mask = registers[i + 2]; 979 980 if (and_mask == 0xffffffff) { 981 tmp = or_mask; 982 } else { 983 tmp = RREG32(reg); 984 tmp &= ~and_mask; 985 if (adev->family >= AMDGPU_FAMILY_AI) 986 tmp |= (or_mask & and_mask); 987 else 988 tmp |= or_mask; 989 } 990 WREG32(reg, tmp); 991 } 992 } 993 994 /** 995 * amdgpu_device_pci_config_reset - reset the GPU 996 * 997 * @adev: amdgpu_device pointer 998 * 999 * Resets the GPU using the pci config reset sequence. 1000 * Only applicable to asics prior to vega10. 1001 */ 1002 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) 1003 { 1004 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); 1005 } 1006 1007 /** 1008 * amdgpu_device_pci_reset - reset the GPU using generic PCI means 1009 * 1010 * @adev: amdgpu_device pointer 1011 * 1012 * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.). 1013 */ 1014 int amdgpu_device_pci_reset(struct amdgpu_device *adev) 1015 { 1016 return pci_reset_function(adev->pdev); 1017 } 1018 1019 /* 1020 * GPU doorbell aperture helpers function. 1021 */ 1022 /** 1023 * amdgpu_device_doorbell_init - Init doorbell driver information. 1024 * 1025 * @adev: amdgpu_device pointer 1026 * 1027 * Init doorbell driver information (CIK) 1028 * Returns 0 on success, error on failure. 1029 */ 1030 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev) 1031 { 1032 1033 /* No doorbell on SI hardware generation */ 1034 if (adev->asic_type < CHIP_BONAIRE) { 1035 adev->doorbell.base = 0; 1036 adev->doorbell.size = 0; 1037 adev->doorbell.num_doorbells = 0; 1038 adev->doorbell.ptr = NULL; 1039 return 0; 1040 } 1041 1042 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) 1043 return -EINVAL; 1044 1045 amdgpu_asic_init_doorbell_index(adev); 1046 1047 /* doorbell bar mapping */ 1048 adev->doorbell.base = pci_resource_start(adev->pdev, 2); 1049 adev->doorbell.size = pci_resource_len(adev->pdev, 2); 1050 1051 if (adev->enable_mes) { 1052 adev->doorbell.num_doorbells = 1053 adev->doorbell.size / sizeof(u32); 1054 } else { 1055 adev->doorbell.num_doorbells = 1056 min_t(u32, adev->doorbell.size / sizeof(u32), 1057 adev->doorbell_index.max_assignment+1); 1058 if (adev->doorbell.num_doorbells == 0) 1059 return -EINVAL; 1060 1061 /* For Vega, reserve and map two pages on doorbell BAR since SDMA 1062 * paging queue doorbell use the second page. The 1063 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the 1064 * doorbells are in the first page. So with paging queue enabled, 1065 * the max num_doorbells should + 1 page (0x400 in dword) 1066 */ 1067 if (adev->asic_type >= CHIP_VEGA10) 1068 adev->doorbell.num_doorbells += 0x400; 1069 } 1070 1071 adev->doorbell.ptr = ioremap(adev->doorbell.base, 1072 adev->doorbell.num_doorbells * 1073 sizeof(u32)); 1074 if (adev->doorbell.ptr == NULL) 1075 return -ENOMEM; 1076 1077 return 0; 1078 } 1079 1080 /** 1081 * amdgpu_device_doorbell_fini - Tear down doorbell driver information. 1082 * 1083 * @adev: amdgpu_device pointer 1084 * 1085 * Tear down doorbell driver information (CIK) 1086 */ 1087 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev) 1088 { 1089 iounmap(adev->doorbell.ptr); 1090 adev->doorbell.ptr = NULL; 1091 } 1092 1093 1094 1095 /* 1096 * amdgpu_device_wb_*() 1097 * Writeback is the method by which the GPU updates special pages in memory 1098 * with the status of certain GPU events (fences, ring pointers,etc.). 1099 */ 1100 1101 /** 1102 * amdgpu_device_wb_fini - Disable Writeback and free memory 1103 * 1104 * @adev: amdgpu_device pointer 1105 * 1106 * Disables Writeback and frees the Writeback memory (all asics). 1107 * Used at driver shutdown. 1108 */ 1109 static void amdgpu_device_wb_fini(struct amdgpu_device *adev) 1110 { 1111 if (adev->wb.wb_obj) { 1112 amdgpu_bo_free_kernel(&adev->wb.wb_obj, 1113 &adev->wb.gpu_addr, 1114 (void **)&adev->wb.wb); 1115 adev->wb.wb_obj = NULL; 1116 } 1117 } 1118 1119 /** 1120 * amdgpu_device_wb_init - Init Writeback driver info and allocate memory 1121 * 1122 * @adev: amdgpu_device pointer 1123 * 1124 * Initializes writeback and allocates writeback memory (all asics). 1125 * Used at driver startup. 1126 * Returns 0 on success or an -error on failure. 1127 */ 1128 static int amdgpu_device_wb_init(struct amdgpu_device *adev) 1129 { 1130 int r; 1131 1132 if (adev->wb.wb_obj == NULL) { 1133 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */ 1134 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, 1135 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1136 &adev->wb.wb_obj, &adev->wb.gpu_addr, 1137 (void **)&adev->wb.wb); 1138 if (r) { 1139 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); 1140 return r; 1141 } 1142 1143 adev->wb.num_wb = AMDGPU_MAX_WB; 1144 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); 1145 1146 /* clear wb memory */ 1147 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); 1148 } 1149 1150 return 0; 1151 } 1152 1153 /** 1154 * amdgpu_device_wb_get - Allocate a wb entry 1155 * 1156 * @adev: amdgpu_device pointer 1157 * @wb: wb index 1158 * 1159 * Allocate a wb slot for use by the driver (all asics). 1160 * Returns 0 on success or -EINVAL on failure. 1161 */ 1162 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) 1163 { 1164 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); 1165 1166 if (offset < adev->wb.num_wb) { 1167 __set_bit(offset, adev->wb.used); 1168 *wb = offset << 3; /* convert to dw offset */ 1169 return 0; 1170 } else { 1171 return -EINVAL; 1172 } 1173 } 1174 1175 /** 1176 * amdgpu_device_wb_free - Free a wb entry 1177 * 1178 * @adev: amdgpu_device pointer 1179 * @wb: wb index 1180 * 1181 * Free a wb slot allocated for use by the driver (all asics) 1182 */ 1183 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) 1184 { 1185 wb >>= 3; 1186 if (wb < adev->wb.num_wb) 1187 __clear_bit(wb, adev->wb.used); 1188 } 1189 1190 /** 1191 * amdgpu_device_resize_fb_bar - try to resize FB BAR 1192 * 1193 * @adev: amdgpu_device pointer 1194 * 1195 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not 1196 * to fail, but if any of the BARs is not accessible after the size we abort 1197 * driver loading by returning -ENODEV. 1198 */ 1199 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) 1200 { 1201 int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size); 1202 struct pci_bus *root; 1203 struct resource *res; 1204 unsigned i; 1205 u16 cmd; 1206 int r; 1207 1208 /* Bypass for VF */ 1209 if (amdgpu_sriov_vf(adev)) 1210 return 0; 1211 1212 /* skip if the bios has already enabled large BAR */ 1213 if (adev->gmc.real_vram_size && 1214 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) 1215 return 0; 1216 1217 /* Check if the root BUS has 64bit memory resources */ 1218 root = adev->pdev->bus; 1219 while (root->parent) 1220 root = root->parent; 1221 1222 pci_bus_for_each_resource(root, res, i) { 1223 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 1224 res->start > 0x100000000ull) 1225 break; 1226 } 1227 1228 /* Trying to resize is pointless without a root hub window above 4GB */ 1229 if (!res) 1230 return 0; 1231 1232 /* Limit the BAR size to what is available */ 1233 rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1, 1234 rbar_size); 1235 1236 /* Disable memory decoding while we change the BAR addresses and size */ 1237 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); 1238 pci_write_config_word(adev->pdev, PCI_COMMAND, 1239 cmd & ~PCI_COMMAND_MEMORY); 1240 1241 /* Free the VRAM and doorbell BAR, we most likely need to move both. */ 1242 amdgpu_device_doorbell_fini(adev); 1243 if (adev->asic_type >= CHIP_BONAIRE) 1244 pci_release_resource(adev->pdev, 2); 1245 1246 pci_release_resource(adev->pdev, 0); 1247 1248 r = pci_resize_resource(adev->pdev, 0, rbar_size); 1249 if (r == -ENOSPC) 1250 DRM_INFO("Not enough PCI address space for a large BAR."); 1251 else if (r && r != -ENOTSUPP) 1252 DRM_ERROR("Problem resizing BAR0 (%d).", r); 1253 1254 pci_assign_unassigned_bus_resources(adev->pdev->bus); 1255 1256 /* When the doorbell or fb BAR isn't available we have no chance of 1257 * using the device. 1258 */ 1259 r = amdgpu_device_doorbell_init(adev); 1260 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) 1261 return -ENODEV; 1262 1263 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); 1264 1265 return 0; 1266 } 1267 1268 /* 1269 * GPU helpers function. 1270 */ 1271 /** 1272 * amdgpu_device_need_post - check if the hw need post or not 1273 * 1274 * @adev: amdgpu_device pointer 1275 * 1276 * Check if the asic has been initialized (all asics) at driver startup 1277 * or post is needed if hw reset is performed. 1278 * Returns true if need or false if not. 1279 */ 1280 bool amdgpu_device_need_post(struct amdgpu_device *adev) 1281 { 1282 uint32_t reg; 1283 1284 if (amdgpu_sriov_vf(adev)) 1285 return false; 1286 1287 if (amdgpu_passthrough(adev)) { 1288 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot 1289 * some old smc fw still need driver do vPost otherwise gpu hang, while 1290 * those smc fw version above 22.15 doesn't have this flaw, so we force 1291 * vpost executed for smc version below 22.15 1292 */ 1293 if (adev->asic_type == CHIP_FIJI) { 1294 int err; 1295 uint32_t fw_ver; 1296 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); 1297 /* force vPost if error occured */ 1298 if (err) 1299 return true; 1300 1301 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); 1302 if (fw_ver < 0x00160e00) 1303 return true; 1304 } 1305 } 1306 1307 /* Don't post if we need to reset whole hive on init */ 1308 if (adev->gmc.xgmi.pending_reset) 1309 return false; 1310 1311 if (adev->has_hw_reset) { 1312 adev->has_hw_reset = false; 1313 return true; 1314 } 1315 1316 /* bios scratch used on CIK+ */ 1317 if (adev->asic_type >= CHIP_BONAIRE) 1318 return amdgpu_atombios_scratch_need_asic_init(adev); 1319 1320 /* check MEM_SIZE for older asics */ 1321 reg = amdgpu_asic_get_config_memsize(adev); 1322 1323 if ((reg != 0) && (reg != 0xffffffff)) 1324 return false; 1325 1326 return true; 1327 } 1328 1329 /** 1330 * amdgpu_device_should_use_aspm - check if the device should program ASPM 1331 * 1332 * @adev: amdgpu_device pointer 1333 * 1334 * Confirm whether the module parameter and pcie bridge agree that ASPM should 1335 * be set for this device. 1336 * 1337 * Returns true if it should be used or false if not. 1338 */ 1339 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev) 1340 { 1341 switch (amdgpu_aspm) { 1342 case -1: 1343 break; 1344 case 0: 1345 return false; 1346 case 1: 1347 return true; 1348 default: 1349 return false; 1350 } 1351 return pcie_aspm_enabled(adev->pdev); 1352 } 1353 1354 /* if we get transitioned to only one device, take VGA back */ 1355 /** 1356 * amdgpu_device_vga_set_decode - enable/disable vga decode 1357 * 1358 * @pdev: PCI device pointer 1359 * @state: enable/disable vga decode 1360 * 1361 * Enable/disable vga decode (all asics). 1362 * Returns VGA resource flags. 1363 */ 1364 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev, 1365 bool state) 1366 { 1367 struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev)); 1368 amdgpu_asic_set_vga_state(adev, state); 1369 if (state) 1370 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 1371 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1372 else 1373 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1374 } 1375 1376 /** 1377 * amdgpu_device_check_block_size - validate the vm block size 1378 * 1379 * @adev: amdgpu_device pointer 1380 * 1381 * Validates the vm block size specified via module parameter. 1382 * The vm block size defines number of bits in page table versus page directory, 1383 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1384 * page table and the remaining bits are in the page directory. 1385 */ 1386 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) 1387 { 1388 /* defines number of bits in page table versus page directory, 1389 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1390 * page table and the remaining bits are in the page directory */ 1391 if (amdgpu_vm_block_size == -1) 1392 return; 1393 1394 if (amdgpu_vm_block_size < 9) { 1395 dev_warn(adev->dev, "VM page table size (%d) too small\n", 1396 amdgpu_vm_block_size); 1397 amdgpu_vm_block_size = -1; 1398 } 1399 } 1400 1401 /** 1402 * amdgpu_device_check_vm_size - validate the vm size 1403 * 1404 * @adev: amdgpu_device pointer 1405 * 1406 * Validates the vm size in GB specified via module parameter. 1407 * The VM size is the size of the GPU virtual memory space in GB. 1408 */ 1409 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) 1410 { 1411 /* no need to check the default value */ 1412 if (amdgpu_vm_size == -1) 1413 return; 1414 1415 if (amdgpu_vm_size < 1) { 1416 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 1417 amdgpu_vm_size); 1418 amdgpu_vm_size = -1; 1419 } 1420 } 1421 1422 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) 1423 { 1424 struct sysinfo si; 1425 bool is_os_64 = (sizeof(void *) == 8); 1426 uint64_t total_memory; 1427 uint64_t dram_size_seven_GB = 0x1B8000000; 1428 uint64_t dram_size_three_GB = 0xB8000000; 1429 1430 if (amdgpu_smu_memory_pool_size == 0) 1431 return; 1432 1433 if (!is_os_64) { 1434 DRM_WARN("Not 64-bit OS, feature not supported\n"); 1435 goto def_value; 1436 } 1437 si_meminfo(&si); 1438 total_memory = (uint64_t)si.totalram * si.mem_unit; 1439 1440 if ((amdgpu_smu_memory_pool_size == 1) || 1441 (amdgpu_smu_memory_pool_size == 2)) { 1442 if (total_memory < dram_size_three_GB) 1443 goto def_value1; 1444 } else if ((amdgpu_smu_memory_pool_size == 4) || 1445 (amdgpu_smu_memory_pool_size == 8)) { 1446 if (total_memory < dram_size_seven_GB) 1447 goto def_value1; 1448 } else { 1449 DRM_WARN("Smu memory pool size not supported\n"); 1450 goto def_value; 1451 } 1452 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; 1453 1454 return; 1455 1456 def_value1: 1457 DRM_WARN("No enough system memory\n"); 1458 def_value: 1459 adev->pm.smu_prv_buffer_size = 0; 1460 } 1461 1462 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev) 1463 { 1464 if (!(adev->flags & AMD_IS_APU) || 1465 adev->asic_type < CHIP_RAVEN) 1466 return 0; 1467 1468 switch (adev->asic_type) { 1469 case CHIP_RAVEN: 1470 if (adev->pdev->device == 0x15dd) 1471 adev->apu_flags |= AMD_APU_IS_RAVEN; 1472 if (adev->pdev->device == 0x15d8) 1473 adev->apu_flags |= AMD_APU_IS_PICASSO; 1474 break; 1475 case CHIP_RENOIR: 1476 if ((adev->pdev->device == 0x1636) || 1477 (adev->pdev->device == 0x164c)) 1478 adev->apu_flags |= AMD_APU_IS_RENOIR; 1479 else 1480 adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE; 1481 break; 1482 case CHIP_VANGOGH: 1483 adev->apu_flags |= AMD_APU_IS_VANGOGH; 1484 break; 1485 case CHIP_YELLOW_CARP: 1486 break; 1487 case CHIP_CYAN_SKILLFISH: 1488 if ((adev->pdev->device == 0x13FE) || 1489 (adev->pdev->device == 0x143F)) 1490 adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2; 1491 break; 1492 default: 1493 break; 1494 } 1495 1496 return 0; 1497 } 1498 1499 /** 1500 * amdgpu_device_check_arguments - validate module params 1501 * 1502 * @adev: amdgpu_device pointer 1503 * 1504 * Validates certain module parameters and updates 1505 * the associated values used by the driver (all asics). 1506 */ 1507 static int amdgpu_device_check_arguments(struct amdgpu_device *adev) 1508 { 1509 if (amdgpu_sched_jobs < 4) { 1510 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", 1511 amdgpu_sched_jobs); 1512 amdgpu_sched_jobs = 4; 1513 } else if (!is_power_of_2(amdgpu_sched_jobs)){ 1514 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", 1515 amdgpu_sched_jobs); 1516 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); 1517 } 1518 1519 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { 1520 /* gart size must be greater or equal to 32M */ 1521 dev_warn(adev->dev, "gart size (%d) too small\n", 1522 amdgpu_gart_size); 1523 amdgpu_gart_size = -1; 1524 } 1525 1526 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { 1527 /* gtt size must be greater or equal to 32M */ 1528 dev_warn(adev->dev, "gtt size (%d) too small\n", 1529 amdgpu_gtt_size); 1530 amdgpu_gtt_size = -1; 1531 } 1532 1533 /* valid range is between 4 and 9 inclusive */ 1534 if (amdgpu_vm_fragment_size != -1 && 1535 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { 1536 dev_warn(adev->dev, "valid range is between 4 and 9\n"); 1537 amdgpu_vm_fragment_size = -1; 1538 } 1539 1540 if (amdgpu_sched_hw_submission < 2) { 1541 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", 1542 amdgpu_sched_hw_submission); 1543 amdgpu_sched_hw_submission = 2; 1544 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) { 1545 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", 1546 amdgpu_sched_hw_submission); 1547 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission); 1548 } 1549 1550 if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) { 1551 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n"); 1552 amdgpu_reset_method = -1; 1553 } 1554 1555 amdgpu_device_check_smu_prv_buffer_size(adev); 1556 1557 amdgpu_device_check_vm_size(adev); 1558 1559 amdgpu_device_check_block_size(adev); 1560 1561 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); 1562 1563 return 0; 1564 } 1565 1566 /** 1567 * amdgpu_switcheroo_set_state - set switcheroo state 1568 * 1569 * @pdev: pci dev pointer 1570 * @state: vga_switcheroo state 1571 * 1572 * Callback for the switcheroo driver. Suspends or resumes 1573 * the asics before or after it is powered up using ACPI methods. 1574 */ 1575 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, 1576 enum vga_switcheroo_state state) 1577 { 1578 struct drm_device *dev = pci_get_drvdata(pdev); 1579 int r; 1580 1581 if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF) 1582 return; 1583 1584 if (state == VGA_SWITCHEROO_ON) { 1585 pr_info("switched on\n"); 1586 /* don't suspend or resume card normally */ 1587 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1588 1589 pci_set_power_state(pdev, PCI_D0); 1590 amdgpu_device_load_pci_state(pdev); 1591 r = pci_enable_device(pdev); 1592 if (r) 1593 DRM_WARN("pci_enable_device failed (%d)\n", r); 1594 amdgpu_device_resume(dev, true); 1595 1596 dev->switch_power_state = DRM_SWITCH_POWER_ON; 1597 } else { 1598 pr_info("switched off\n"); 1599 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1600 amdgpu_device_suspend(dev, true); 1601 amdgpu_device_cache_pci_state(pdev); 1602 /* Shut down the device */ 1603 pci_disable_device(pdev); 1604 pci_set_power_state(pdev, PCI_D3cold); 1605 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 1606 } 1607 } 1608 1609 /** 1610 * amdgpu_switcheroo_can_switch - see if switcheroo state can change 1611 * 1612 * @pdev: pci dev pointer 1613 * 1614 * Callback for the switcheroo driver. Check of the switcheroo 1615 * state can be changed. 1616 * Returns true if the state can be changed, false if not. 1617 */ 1618 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) 1619 { 1620 struct drm_device *dev = pci_get_drvdata(pdev); 1621 1622 /* 1623 * FIXME: open_count is protected by drm_global_mutex but that would lead to 1624 * locking inversion with the driver load path. And the access here is 1625 * completely racy anyway. So don't bother with locking for now. 1626 */ 1627 return atomic_read(&dev->open_count) == 0; 1628 } 1629 1630 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { 1631 .set_gpu_state = amdgpu_switcheroo_set_state, 1632 .reprobe = NULL, 1633 .can_switch = amdgpu_switcheroo_can_switch, 1634 }; 1635 1636 /** 1637 * amdgpu_device_ip_set_clockgating_state - set the CG state 1638 * 1639 * @dev: amdgpu_device pointer 1640 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1641 * @state: clockgating state (gate or ungate) 1642 * 1643 * Sets the requested clockgating state for all instances of 1644 * the hardware IP specified. 1645 * Returns the error code from the last instance. 1646 */ 1647 int amdgpu_device_ip_set_clockgating_state(void *dev, 1648 enum amd_ip_block_type block_type, 1649 enum amd_clockgating_state state) 1650 { 1651 struct amdgpu_device *adev = dev; 1652 int i, r = 0; 1653 1654 for (i = 0; i < adev->num_ip_blocks; i++) { 1655 if (!adev->ip_blocks[i].status.valid) 1656 continue; 1657 if (adev->ip_blocks[i].version->type != block_type) 1658 continue; 1659 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) 1660 continue; 1661 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( 1662 (void *)adev, state); 1663 if (r) 1664 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n", 1665 adev->ip_blocks[i].version->funcs->name, r); 1666 } 1667 return r; 1668 } 1669 1670 /** 1671 * amdgpu_device_ip_set_powergating_state - set the PG state 1672 * 1673 * @dev: amdgpu_device pointer 1674 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1675 * @state: powergating state (gate or ungate) 1676 * 1677 * Sets the requested powergating state for all instances of 1678 * the hardware IP specified. 1679 * Returns the error code from the last instance. 1680 */ 1681 int amdgpu_device_ip_set_powergating_state(void *dev, 1682 enum amd_ip_block_type block_type, 1683 enum amd_powergating_state state) 1684 { 1685 struct amdgpu_device *adev = dev; 1686 int i, r = 0; 1687 1688 for (i = 0; i < adev->num_ip_blocks; i++) { 1689 if (!adev->ip_blocks[i].status.valid) 1690 continue; 1691 if (adev->ip_blocks[i].version->type != block_type) 1692 continue; 1693 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) 1694 continue; 1695 r = adev->ip_blocks[i].version->funcs->set_powergating_state( 1696 (void *)adev, state); 1697 if (r) 1698 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n", 1699 adev->ip_blocks[i].version->funcs->name, r); 1700 } 1701 return r; 1702 } 1703 1704 /** 1705 * amdgpu_device_ip_get_clockgating_state - get the CG state 1706 * 1707 * @adev: amdgpu_device pointer 1708 * @flags: clockgating feature flags 1709 * 1710 * Walks the list of IPs on the device and updates the clockgating 1711 * flags for each IP. 1712 * Updates @flags with the feature flags for each hardware IP where 1713 * clockgating is enabled. 1714 */ 1715 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, 1716 u64 *flags) 1717 { 1718 int i; 1719 1720 for (i = 0; i < adev->num_ip_blocks; i++) { 1721 if (!adev->ip_blocks[i].status.valid) 1722 continue; 1723 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) 1724 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); 1725 } 1726 } 1727 1728 /** 1729 * amdgpu_device_ip_wait_for_idle - wait for idle 1730 * 1731 * @adev: amdgpu_device pointer 1732 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1733 * 1734 * Waits for the request hardware IP to be idle. 1735 * Returns 0 for success or a negative error code on failure. 1736 */ 1737 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, 1738 enum amd_ip_block_type block_type) 1739 { 1740 int i, r; 1741 1742 for (i = 0; i < adev->num_ip_blocks; i++) { 1743 if (!adev->ip_blocks[i].status.valid) 1744 continue; 1745 if (adev->ip_blocks[i].version->type == block_type) { 1746 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); 1747 if (r) 1748 return r; 1749 break; 1750 } 1751 } 1752 return 0; 1753 1754 } 1755 1756 /** 1757 * amdgpu_device_ip_is_idle - is the hardware IP idle 1758 * 1759 * @adev: amdgpu_device pointer 1760 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1761 * 1762 * Check if the hardware IP is idle or not. 1763 * Returns true if it the IP is idle, false if not. 1764 */ 1765 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, 1766 enum amd_ip_block_type block_type) 1767 { 1768 int i; 1769 1770 for (i = 0; i < adev->num_ip_blocks; i++) { 1771 if (!adev->ip_blocks[i].status.valid) 1772 continue; 1773 if (adev->ip_blocks[i].version->type == block_type) 1774 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); 1775 } 1776 return true; 1777 1778 } 1779 1780 /** 1781 * amdgpu_device_ip_get_ip_block - get a hw IP pointer 1782 * 1783 * @adev: amdgpu_device pointer 1784 * @type: Type of hardware IP (SMU, GFX, UVD, etc.) 1785 * 1786 * Returns a pointer to the hardware IP block structure 1787 * if it exists for the asic, otherwise NULL. 1788 */ 1789 struct amdgpu_ip_block * 1790 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, 1791 enum amd_ip_block_type type) 1792 { 1793 int i; 1794 1795 for (i = 0; i < adev->num_ip_blocks; i++) 1796 if (adev->ip_blocks[i].version->type == type) 1797 return &adev->ip_blocks[i]; 1798 1799 return NULL; 1800 } 1801 1802 /** 1803 * amdgpu_device_ip_block_version_cmp 1804 * 1805 * @adev: amdgpu_device pointer 1806 * @type: enum amd_ip_block_type 1807 * @major: major version 1808 * @minor: minor version 1809 * 1810 * return 0 if equal or greater 1811 * return 1 if smaller or the ip_block doesn't exist 1812 */ 1813 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, 1814 enum amd_ip_block_type type, 1815 u32 major, u32 minor) 1816 { 1817 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); 1818 1819 if (ip_block && ((ip_block->version->major > major) || 1820 ((ip_block->version->major == major) && 1821 (ip_block->version->minor >= minor)))) 1822 return 0; 1823 1824 return 1; 1825 } 1826 1827 /** 1828 * amdgpu_device_ip_block_add 1829 * 1830 * @adev: amdgpu_device pointer 1831 * @ip_block_version: pointer to the IP to add 1832 * 1833 * Adds the IP block driver information to the collection of IPs 1834 * on the asic. 1835 */ 1836 int amdgpu_device_ip_block_add(struct amdgpu_device *adev, 1837 const struct amdgpu_ip_block_version *ip_block_version) 1838 { 1839 if (!ip_block_version) 1840 return -EINVAL; 1841 1842 switch (ip_block_version->type) { 1843 case AMD_IP_BLOCK_TYPE_VCN: 1844 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK) 1845 return 0; 1846 break; 1847 case AMD_IP_BLOCK_TYPE_JPEG: 1848 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK) 1849 return 0; 1850 break; 1851 default: 1852 break; 1853 } 1854 1855 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, 1856 ip_block_version->funcs->name); 1857 1858 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; 1859 1860 return 0; 1861 } 1862 1863 /** 1864 * amdgpu_device_enable_virtual_display - enable virtual display feature 1865 * 1866 * @adev: amdgpu_device pointer 1867 * 1868 * Enabled the virtual display feature if the user has enabled it via 1869 * the module parameter virtual_display. This feature provides a virtual 1870 * display hardware on headless boards or in virtualized environments. 1871 * This function parses and validates the configuration string specified by 1872 * the user and configues the virtual display configuration (number of 1873 * virtual connectors, crtcs, etc.) specified. 1874 */ 1875 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) 1876 { 1877 adev->enable_virtual_display = false; 1878 1879 if (amdgpu_virtual_display) { 1880 const char *pci_address_name = pci_name(adev->pdev); 1881 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; 1882 1883 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); 1884 pciaddstr_tmp = pciaddstr; 1885 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { 1886 pciaddname = strsep(&pciaddname_tmp, ","); 1887 if (!strcmp("all", pciaddname) 1888 || !strcmp(pci_address_name, pciaddname)) { 1889 long num_crtc; 1890 int res = -1; 1891 1892 adev->enable_virtual_display = true; 1893 1894 if (pciaddname_tmp) 1895 res = kstrtol(pciaddname_tmp, 10, 1896 &num_crtc); 1897 1898 if (!res) { 1899 if (num_crtc < 1) 1900 num_crtc = 1; 1901 if (num_crtc > 6) 1902 num_crtc = 6; 1903 adev->mode_info.num_crtc = num_crtc; 1904 } else { 1905 adev->mode_info.num_crtc = 1; 1906 } 1907 break; 1908 } 1909 } 1910 1911 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", 1912 amdgpu_virtual_display, pci_address_name, 1913 adev->enable_virtual_display, adev->mode_info.num_crtc); 1914 1915 kfree(pciaddstr); 1916 } 1917 } 1918 1919 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev) 1920 { 1921 if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) { 1922 adev->mode_info.num_crtc = 1; 1923 adev->enable_virtual_display = true; 1924 DRM_INFO("virtual_display:%d, num_crtc:%d\n", 1925 adev->enable_virtual_display, adev->mode_info.num_crtc); 1926 } 1927 } 1928 1929 /** 1930 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware 1931 * 1932 * @adev: amdgpu_device pointer 1933 * 1934 * Parses the asic configuration parameters specified in the gpu info 1935 * firmware and makes them availale to the driver for use in configuring 1936 * the asic. 1937 * Returns 0 on success, -EINVAL on failure. 1938 */ 1939 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) 1940 { 1941 const char *chip_name; 1942 char fw_name[40]; 1943 int err; 1944 const struct gpu_info_firmware_header_v1_0 *hdr; 1945 1946 adev->firmware.gpu_info_fw = NULL; 1947 1948 if (adev->mman.discovery_bin) { 1949 /* 1950 * FIXME: The bounding box is still needed by Navi12, so 1951 * temporarily read it from gpu_info firmware. Should be dropped 1952 * when DAL no longer needs it. 1953 */ 1954 if (adev->asic_type != CHIP_NAVI12) 1955 return 0; 1956 } 1957 1958 switch (adev->asic_type) { 1959 default: 1960 return 0; 1961 case CHIP_VEGA10: 1962 chip_name = "vega10"; 1963 break; 1964 case CHIP_VEGA12: 1965 chip_name = "vega12"; 1966 break; 1967 case CHIP_RAVEN: 1968 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1969 chip_name = "raven2"; 1970 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1971 chip_name = "picasso"; 1972 else 1973 chip_name = "raven"; 1974 break; 1975 case CHIP_ARCTURUS: 1976 chip_name = "arcturus"; 1977 break; 1978 case CHIP_NAVI12: 1979 chip_name = "navi12"; 1980 break; 1981 } 1982 1983 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name); 1984 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev); 1985 if (err) { 1986 dev_err(adev->dev, 1987 "Failed to load gpu_info firmware \"%s\"\n", 1988 fw_name); 1989 goto out; 1990 } 1991 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw); 1992 if (err) { 1993 dev_err(adev->dev, 1994 "Failed to validate gpu_info firmware \"%s\"\n", 1995 fw_name); 1996 goto out; 1997 } 1998 1999 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; 2000 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); 2001 2002 switch (hdr->version_major) { 2003 case 1: 2004 { 2005 const struct gpu_info_firmware_v1_0 *gpu_info_fw = 2006 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + 2007 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2008 2009 /* 2010 * Should be droped when DAL no longer needs it. 2011 */ 2012 if (adev->asic_type == CHIP_NAVI12) 2013 goto parse_soc_bounding_box; 2014 2015 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); 2016 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); 2017 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); 2018 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); 2019 adev->gfx.config.max_texture_channel_caches = 2020 le32_to_cpu(gpu_info_fw->gc_num_tccs); 2021 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); 2022 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); 2023 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); 2024 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); 2025 adev->gfx.config.double_offchip_lds_buf = 2026 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); 2027 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); 2028 adev->gfx.cu_info.max_waves_per_simd = 2029 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); 2030 adev->gfx.cu_info.max_scratch_slots_per_cu = 2031 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); 2032 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); 2033 if (hdr->version_minor >= 1) { 2034 const struct gpu_info_firmware_v1_1 *gpu_info_fw = 2035 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + 2036 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2037 adev->gfx.config.num_sc_per_sh = 2038 le32_to_cpu(gpu_info_fw->num_sc_per_sh); 2039 adev->gfx.config.num_packer_per_sc = 2040 le32_to_cpu(gpu_info_fw->num_packer_per_sc); 2041 } 2042 2043 parse_soc_bounding_box: 2044 /* 2045 * soc bounding box info is not integrated in disocovery table, 2046 * we always need to parse it from gpu info firmware if needed. 2047 */ 2048 if (hdr->version_minor == 2) { 2049 const struct gpu_info_firmware_v1_2 *gpu_info_fw = 2050 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + 2051 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 2052 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; 2053 } 2054 break; 2055 } 2056 default: 2057 dev_err(adev->dev, 2058 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); 2059 err = -EINVAL; 2060 goto out; 2061 } 2062 out: 2063 return err; 2064 } 2065 2066 /** 2067 * amdgpu_device_ip_early_init - run early init for hardware IPs 2068 * 2069 * @adev: amdgpu_device pointer 2070 * 2071 * Early initialization pass for hardware IPs. The hardware IPs that make 2072 * up each asic are discovered each IP's early_init callback is run. This 2073 * is the first stage in initializing the asic. 2074 * Returns 0 on success, negative error code on failure. 2075 */ 2076 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) 2077 { 2078 struct drm_device *dev = adev_to_drm(adev); 2079 struct pci_dev *parent; 2080 int i, r; 2081 2082 amdgpu_device_enable_virtual_display(adev); 2083 2084 if (amdgpu_sriov_vf(adev)) { 2085 r = amdgpu_virt_request_full_gpu(adev, true); 2086 if (r) 2087 return r; 2088 } 2089 2090 switch (adev->asic_type) { 2091 #ifdef CONFIG_DRM_AMDGPU_SI 2092 case CHIP_VERDE: 2093 case CHIP_TAHITI: 2094 case CHIP_PITCAIRN: 2095 case CHIP_OLAND: 2096 case CHIP_HAINAN: 2097 adev->family = AMDGPU_FAMILY_SI; 2098 r = si_set_ip_blocks(adev); 2099 if (r) 2100 return r; 2101 break; 2102 #endif 2103 #ifdef CONFIG_DRM_AMDGPU_CIK 2104 case CHIP_BONAIRE: 2105 case CHIP_HAWAII: 2106 case CHIP_KAVERI: 2107 case CHIP_KABINI: 2108 case CHIP_MULLINS: 2109 if (adev->flags & AMD_IS_APU) 2110 adev->family = AMDGPU_FAMILY_KV; 2111 else 2112 adev->family = AMDGPU_FAMILY_CI; 2113 2114 r = cik_set_ip_blocks(adev); 2115 if (r) 2116 return r; 2117 break; 2118 #endif 2119 case CHIP_TOPAZ: 2120 case CHIP_TONGA: 2121 case CHIP_FIJI: 2122 case CHIP_POLARIS10: 2123 case CHIP_POLARIS11: 2124 case CHIP_POLARIS12: 2125 case CHIP_VEGAM: 2126 case CHIP_CARRIZO: 2127 case CHIP_STONEY: 2128 if (adev->flags & AMD_IS_APU) 2129 adev->family = AMDGPU_FAMILY_CZ; 2130 else 2131 adev->family = AMDGPU_FAMILY_VI; 2132 2133 r = vi_set_ip_blocks(adev); 2134 if (r) 2135 return r; 2136 break; 2137 default: 2138 r = amdgpu_discovery_set_ip_blocks(adev); 2139 if (r) 2140 return r; 2141 break; 2142 } 2143 2144 if (amdgpu_has_atpx() && 2145 (amdgpu_is_atpx_hybrid() || 2146 amdgpu_has_atpx_dgpu_power_cntl()) && 2147 ((adev->flags & AMD_IS_APU) == 0) && 2148 !pci_is_thunderbolt_attached(to_pci_dev(dev->dev))) 2149 adev->flags |= AMD_IS_PX; 2150 2151 if (!(adev->flags & AMD_IS_APU)) { 2152 parent = pci_upstream_bridge(adev->pdev); 2153 adev->has_pr3 = parent ? pci_pr3_present(parent) : false; 2154 } 2155 2156 amdgpu_amdkfd_device_probe(adev); 2157 2158 adev->pm.pp_feature = amdgpu_pp_feature_mask; 2159 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS) 2160 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 2161 if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID) 2162 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK; 2163 2164 for (i = 0; i < adev->num_ip_blocks; i++) { 2165 if ((amdgpu_ip_block_mask & (1 << i)) == 0) { 2166 DRM_ERROR("disabled ip block: %d <%s>\n", 2167 i, adev->ip_blocks[i].version->funcs->name); 2168 adev->ip_blocks[i].status.valid = false; 2169 } else { 2170 if (adev->ip_blocks[i].version->funcs->early_init) { 2171 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); 2172 if (r == -ENOENT) { 2173 adev->ip_blocks[i].status.valid = false; 2174 } else if (r) { 2175 DRM_ERROR("early_init of IP block <%s> failed %d\n", 2176 adev->ip_blocks[i].version->funcs->name, r); 2177 return r; 2178 } else { 2179 adev->ip_blocks[i].status.valid = true; 2180 } 2181 } else { 2182 adev->ip_blocks[i].status.valid = true; 2183 } 2184 } 2185 /* get the vbios after the asic_funcs are set up */ 2186 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2187 r = amdgpu_device_parse_gpu_info_fw(adev); 2188 if (r) 2189 return r; 2190 2191 /* Read BIOS */ 2192 if (!amdgpu_get_bios(adev)) 2193 return -EINVAL; 2194 2195 r = amdgpu_atombios_init(adev); 2196 if (r) { 2197 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); 2198 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0); 2199 return r; 2200 } 2201 2202 /*get pf2vf msg info at it's earliest time*/ 2203 if (amdgpu_sriov_vf(adev)) 2204 amdgpu_virt_init_data_exchange(adev); 2205 2206 } 2207 } 2208 2209 adev->cg_flags &= amdgpu_cg_mask; 2210 adev->pg_flags &= amdgpu_pg_mask; 2211 2212 return 0; 2213 } 2214 2215 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) 2216 { 2217 int i, r; 2218 2219 for (i = 0; i < adev->num_ip_blocks; i++) { 2220 if (!adev->ip_blocks[i].status.sw) 2221 continue; 2222 if (adev->ip_blocks[i].status.hw) 2223 continue; 2224 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2225 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || 2226 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2227 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2228 if (r) { 2229 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2230 adev->ip_blocks[i].version->funcs->name, r); 2231 return r; 2232 } 2233 adev->ip_blocks[i].status.hw = true; 2234 } 2235 } 2236 2237 return 0; 2238 } 2239 2240 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) 2241 { 2242 int i, r; 2243 2244 for (i = 0; i < adev->num_ip_blocks; i++) { 2245 if (!adev->ip_blocks[i].status.sw) 2246 continue; 2247 if (adev->ip_blocks[i].status.hw) 2248 continue; 2249 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2250 if (r) { 2251 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2252 adev->ip_blocks[i].version->funcs->name, r); 2253 return r; 2254 } 2255 adev->ip_blocks[i].status.hw = true; 2256 } 2257 2258 return 0; 2259 } 2260 2261 static int amdgpu_device_fw_loading(struct amdgpu_device *adev) 2262 { 2263 int r = 0; 2264 int i; 2265 uint32_t smu_version; 2266 2267 if (adev->asic_type >= CHIP_VEGA10) { 2268 for (i = 0; i < adev->num_ip_blocks; i++) { 2269 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) 2270 continue; 2271 2272 if (!adev->ip_blocks[i].status.sw) 2273 continue; 2274 2275 /* no need to do the fw loading again if already done*/ 2276 if (adev->ip_blocks[i].status.hw == true) 2277 break; 2278 2279 if (amdgpu_in_reset(adev) || adev->in_suspend) { 2280 r = adev->ip_blocks[i].version->funcs->resume(adev); 2281 if (r) { 2282 DRM_ERROR("resume of IP block <%s> failed %d\n", 2283 adev->ip_blocks[i].version->funcs->name, r); 2284 return r; 2285 } 2286 } else { 2287 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2288 if (r) { 2289 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2290 adev->ip_blocks[i].version->funcs->name, r); 2291 return r; 2292 } 2293 } 2294 2295 adev->ip_blocks[i].status.hw = true; 2296 break; 2297 } 2298 } 2299 2300 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) 2301 r = amdgpu_pm_load_smu_firmware(adev, &smu_version); 2302 2303 return r; 2304 } 2305 2306 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) 2307 { 2308 long timeout; 2309 int r, i; 2310 2311 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 2312 struct amdgpu_ring *ring = adev->rings[i]; 2313 2314 /* No need to setup the GPU scheduler for rings that don't need it */ 2315 if (!ring || ring->no_scheduler) 2316 continue; 2317 2318 switch (ring->funcs->type) { 2319 case AMDGPU_RING_TYPE_GFX: 2320 timeout = adev->gfx_timeout; 2321 break; 2322 case AMDGPU_RING_TYPE_COMPUTE: 2323 timeout = adev->compute_timeout; 2324 break; 2325 case AMDGPU_RING_TYPE_SDMA: 2326 timeout = adev->sdma_timeout; 2327 break; 2328 default: 2329 timeout = adev->video_timeout; 2330 break; 2331 } 2332 2333 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, 2334 ring->num_hw_submission, amdgpu_job_hang_limit, 2335 timeout, adev->reset_domain->wq, 2336 ring->sched_score, ring->name, 2337 adev->dev); 2338 if (r) { 2339 DRM_ERROR("Failed to create scheduler on ring %s.\n", 2340 ring->name); 2341 return r; 2342 } 2343 } 2344 2345 return 0; 2346 } 2347 2348 2349 /** 2350 * amdgpu_device_ip_init - run init for hardware IPs 2351 * 2352 * @adev: amdgpu_device pointer 2353 * 2354 * Main initialization pass for hardware IPs. The list of all the hardware 2355 * IPs that make up the asic is walked and the sw_init and hw_init callbacks 2356 * are run. sw_init initializes the software state associated with each IP 2357 * and hw_init initializes the hardware associated with each IP. 2358 * Returns 0 on success, negative error code on failure. 2359 */ 2360 static int amdgpu_device_ip_init(struct amdgpu_device *adev) 2361 { 2362 int i, r; 2363 2364 r = amdgpu_ras_init(adev); 2365 if (r) 2366 return r; 2367 2368 for (i = 0; i < adev->num_ip_blocks; i++) { 2369 if (!adev->ip_blocks[i].status.valid) 2370 continue; 2371 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); 2372 if (r) { 2373 DRM_ERROR("sw_init of IP block <%s> failed %d\n", 2374 adev->ip_blocks[i].version->funcs->name, r); 2375 goto init_failed; 2376 } 2377 adev->ip_blocks[i].status.sw = true; 2378 2379 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2380 /* need to do common hw init early so everything is set up for gmc */ 2381 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); 2382 if (r) { 2383 DRM_ERROR("hw_init %d failed %d\n", i, r); 2384 goto init_failed; 2385 } 2386 adev->ip_blocks[i].status.hw = true; 2387 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2388 /* need to do gmc hw init early so we can allocate gpu mem */ 2389 /* Try to reserve bad pages early */ 2390 if (amdgpu_sriov_vf(adev)) 2391 amdgpu_virt_exchange_data(adev); 2392 2393 r = amdgpu_device_vram_scratch_init(adev); 2394 if (r) { 2395 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r); 2396 goto init_failed; 2397 } 2398 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); 2399 if (r) { 2400 DRM_ERROR("hw_init %d failed %d\n", i, r); 2401 goto init_failed; 2402 } 2403 r = amdgpu_device_wb_init(adev); 2404 if (r) { 2405 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r); 2406 goto init_failed; 2407 } 2408 adev->ip_blocks[i].status.hw = true; 2409 2410 /* right after GMC hw init, we create CSA */ 2411 if (amdgpu_mcbp) { 2412 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, 2413 AMDGPU_GEM_DOMAIN_VRAM, 2414 AMDGPU_CSA_SIZE); 2415 if (r) { 2416 DRM_ERROR("allocate CSA failed %d\n", r); 2417 goto init_failed; 2418 } 2419 } 2420 } 2421 } 2422 2423 if (amdgpu_sriov_vf(adev)) 2424 amdgpu_virt_init_data_exchange(adev); 2425 2426 r = amdgpu_ib_pool_init(adev); 2427 if (r) { 2428 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 2429 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 2430 goto init_failed; 2431 } 2432 2433 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ 2434 if (r) 2435 goto init_failed; 2436 2437 r = amdgpu_device_ip_hw_init_phase1(adev); 2438 if (r) 2439 goto init_failed; 2440 2441 r = amdgpu_device_fw_loading(adev); 2442 if (r) 2443 goto init_failed; 2444 2445 r = amdgpu_device_ip_hw_init_phase2(adev); 2446 if (r) 2447 goto init_failed; 2448 2449 /* 2450 * retired pages will be loaded from eeprom and reserved here, 2451 * it should be called after amdgpu_device_ip_hw_init_phase2 since 2452 * for some ASICs the RAS EEPROM code relies on SMU fully functioning 2453 * for I2C communication which only true at this point. 2454 * 2455 * amdgpu_ras_recovery_init may fail, but the upper only cares the 2456 * failure from bad gpu situation and stop amdgpu init process 2457 * accordingly. For other failed cases, it will still release all 2458 * the resource and print error message, rather than returning one 2459 * negative value to upper level. 2460 * 2461 * Note: theoretically, this should be called before all vram allocations 2462 * to protect retired page from abusing 2463 */ 2464 r = amdgpu_ras_recovery_init(adev); 2465 if (r) 2466 goto init_failed; 2467 2468 /** 2469 * In case of XGMI grab extra reference for reset domain for this device 2470 */ 2471 if (adev->gmc.xgmi.num_physical_nodes > 1) { 2472 if (amdgpu_xgmi_add_device(adev) == 0) { 2473 if (!amdgpu_sriov_vf(adev)) { 2474 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 2475 2476 if (WARN_ON(!hive)) { 2477 r = -ENOENT; 2478 goto init_failed; 2479 } 2480 2481 if (!hive->reset_domain || 2482 !amdgpu_reset_get_reset_domain(hive->reset_domain)) { 2483 r = -ENOENT; 2484 amdgpu_put_xgmi_hive(hive); 2485 goto init_failed; 2486 } 2487 2488 /* Drop the early temporary reset domain we created for device */ 2489 amdgpu_reset_put_reset_domain(adev->reset_domain); 2490 adev->reset_domain = hive->reset_domain; 2491 amdgpu_put_xgmi_hive(hive); 2492 } 2493 } 2494 } 2495 2496 r = amdgpu_device_init_schedulers(adev); 2497 if (r) 2498 goto init_failed; 2499 2500 /* Don't init kfd if whole hive need to be reset during init */ 2501 if (!adev->gmc.xgmi.pending_reset) 2502 amdgpu_amdkfd_device_init(adev); 2503 2504 amdgpu_fru_get_product_info(adev); 2505 2506 init_failed: 2507 if (amdgpu_sriov_vf(adev)) 2508 amdgpu_virt_release_full_gpu(adev, true); 2509 2510 return r; 2511 } 2512 2513 /** 2514 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer 2515 * 2516 * @adev: amdgpu_device pointer 2517 * 2518 * Writes a reset magic value to the gart pointer in VRAM. The driver calls 2519 * this function before a GPU reset. If the value is retained after a 2520 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents. 2521 */ 2522 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) 2523 { 2524 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); 2525 } 2526 2527 /** 2528 * amdgpu_device_check_vram_lost - check if vram is valid 2529 * 2530 * @adev: amdgpu_device pointer 2531 * 2532 * Checks the reset magic value written to the gart pointer in VRAM. 2533 * The driver calls this after a GPU reset to see if the contents of 2534 * VRAM is lost or now. 2535 * returns true if vram is lost, false if not. 2536 */ 2537 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) 2538 { 2539 if (memcmp(adev->gart.ptr, adev->reset_magic, 2540 AMDGPU_RESET_MAGIC_NUM)) 2541 return true; 2542 2543 if (!amdgpu_in_reset(adev)) 2544 return false; 2545 2546 /* 2547 * For all ASICs with baco/mode1 reset, the VRAM is 2548 * always assumed to be lost. 2549 */ 2550 switch (amdgpu_asic_reset_method(adev)) { 2551 case AMD_RESET_METHOD_BACO: 2552 case AMD_RESET_METHOD_MODE1: 2553 return true; 2554 default: 2555 return false; 2556 } 2557 } 2558 2559 /** 2560 * amdgpu_device_set_cg_state - set clockgating for amdgpu device 2561 * 2562 * @adev: amdgpu_device pointer 2563 * @state: clockgating state (gate or ungate) 2564 * 2565 * The list of all the hardware IPs that make up the asic is walked and the 2566 * set_clockgating_state callbacks are run. 2567 * Late initialization pass enabling clockgating for hardware IPs. 2568 * Fini or suspend, pass disabling clockgating for hardware IPs. 2569 * Returns 0 on success, negative error code on failure. 2570 */ 2571 2572 int amdgpu_device_set_cg_state(struct amdgpu_device *adev, 2573 enum amd_clockgating_state state) 2574 { 2575 int i, j, r; 2576 2577 if (amdgpu_emu_mode == 1) 2578 return 0; 2579 2580 for (j = 0; j < adev->num_ip_blocks; j++) { 2581 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 2582 if (!adev->ip_blocks[i].status.late_initialized) 2583 continue; 2584 /* skip CG for GFX on S0ix */ 2585 if (adev->in_s0ix && 2586 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX) 2587 continue; 2588 /* skip CG for VCE/UVD, it's handled specially */ 2589 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 2590 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 2591 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 2592 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 2593 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 2594 /* enable clockgating to save power */ 2595 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, 2596 state); 2597 if (r) { 2598 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n", 2599 adev->ip_blocks[i].version->funcs->name, r); 2600 return r; 2601 } 2602 } 2603 } 2604 2605 return 0; 2606 } 2607 2608 int amdgpu_device_set_pg_state(struct amdgpu_device *adev, 2609 enum amd_powergating_state state) 2610 { 2611 int i, j, r; 2612 2613 if (amdgpu_emu_mode == 1) 2614 return 0; 2615 2616 for (j = 0; j < adev->num_ip_blocks; j++) { 2617 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 2618 if (!adev->ip_blocks[i].status.late_initialized) 2619 continue; 2620 /* skip PG for GFX on S0ix */ 2621 if (adev->in_s0ix && 2622 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX) 2623 continue; 2624 /* skip CG for VCE/UVD, it's handled specially */ 2625 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 2626 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 2627 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 2628 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 2629 adev->ip_blocks[i].version->funcs->set_powergating_state) { 2630 /* enable powergating to save power */ 2631 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, 2632 state); 2633 if (r) { 2634 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n", 2635 adev->ip_blocks[i].version->funcs->name, r); 2636 return r; 2637 } 2638 } 2639 } 2640 return 0; 2641 } 2642 2643 static int amdgpu_device_enable_mgpu_fan_boost(void) 2644 { 2645 struct amdgpu_gpu_instance *gpu_ins; 2646 struct amdgpu_device *adev; 2647 int i, ret = 0; 2648 2649 mutex_lock(&mgpu_info.mutex); 2650 2651 /* 2652 * MGPU fan boost feature should be enabled 2653 * only when there are two or more dGPUs in 2654 * the system 2655 */ 2656 if (mgpu_info.num_dgpu < 2) 2657 goto out; 2658 2659 for (i = 0; i < mgpu_info.num_dgpu; i++) { 2660 gpu_ins = &(mgpu_info.gpu_ins[i]); 2661 adev = gpu_ins->adev; 2662 if (!(adev->flags & AMD_IS_APU) && 2663 !gpu_ins->mgpu_fan_enabled) { 2664 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); 2665 if (ret) 2666 break; 2667 2668 gpu_ins->mgpu_fan_enabled = 1; 2669 } 2670 } 2671 2672 out: 2673 mutex_unlock(&mgpu_info.mutex); 2674 2675 return ret; 2676 } 2677 2678 /** 2679 * amdgpu_device_ip_late_init - run late init for hardware IPs 2680 * 2681 * @adev: amdgpu_device pointer 2682 * 2683 * Late initialization pass for hardware IPs. The list of all the hardware 2684 * IPs that make up the asic is walked and the late_init callbacks are run. 2685 * late_init covers any special initialization that an IP requires 2686 * after all of the have been initialized or something that needs to happen 2687 * late in the init process. 2688 * Returns 0 on success, negative error code on failure. 2689 */ 2690 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) 2691 { 2692 struct amdgpu_gpu_instance *gpu_instance; 2693 int i = 0, r; 2694 2695 for (i = 0; i < adev->num_ip_blocks; i++) { 2696 if (!adev->ip_blocks[i].status.hw) 2697 continue; 2698 if (adev->ip_blocks[i].version->funcs->late_init) { 2699 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); 2700 if (r) { 2701 DRM_ERROR("late_init of IP block <%s> failed %d\n", 2702 adev->ip_blocks[i].version->funcs->name, r); 2703 return r; 2704 } 2705 } 2706 adev->ip_blocks[i].status.late_initialized = true; 2707 } 2708 2709 r = amdgpu_ras_late_init(adev); 2710 if (r) { 2711 DRM_ERROR("amdgpu_ras_late_init failed %d", r); 2712 return r; 2713 } 2714 2715 amdgpu_ras_set_error_query_ready(adev, true); 2716 2717 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 2718 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 2719 2720 amdgpu_device_fill_reset_magic(adev); 2721 2722 r = amdgpu_device_enable_mgpu_fan_boost(); 2723 if (r) 2724 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r); 2725 2726 /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */ 2727 if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)|| 2728 adev->asic_type == CHIP_ALDEBARAN )) 2729 amdgpu_dpm_handle_passthrough_sbr(adev, true); 2730 2731 if (adev->gmc.xgmi.num_physical_nodes > 1) { 2732 mutex_lock(&mgpu_info.mutex); 2733 2734 /* 2735 * Reset device p-state to low as this was booted with high. 2736 * 2737 * This should be performed only after all devices from the same 2738 * hive get initialized. 2739 * 2740 * However, it's unknown how many device in the hive in advance. 2741 * As this is counted one by one during devices initializations. 2742 * 2743 * So, we wait for all XGMI interlinked devices initialized. 2744 * This may bring some delays as those devices may come from 2745 * different hives. But that should be OK. 2746 */ 2747 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { 2748 for (i = 0; i < mgpu_info.num_gpu; i++) { 2749 gpu_instance = &(mgpu_info.gpu_ins[i]); 2750 if (gpu_instance->adev->flags & AMD_IS_APU) 2751 continue; 2752 2753 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 2754 AMDGPU_XGMI_PSTATE_MIN); 2755 if (r) { 2756 DRM_ERROR("pstate setting failed (%d).\n", r); 2757 break; 2758 } 2759 } 2760 } 2761 2762 mutex_unlock(&mgpu_info.mutex); 2763 } 2764 2765 return 0; 2766 } 2767 2768 /** 2769 * amdgpu_device_smu_fini_early - smu hw_fini wrapper 2770 * 2771 * @adev: amdgpu_device pointer 2772 * 2773 * For ASICs need to disable SMC first 2774 */ 2775 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev) 2776 { 2777 int i, r; 2778 2779 if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0)) 2780 return; 2781 2782 for (i = 0; i < adev->num_ip_blocks; i++) { 2783 if (!adev->ip_blocks[i].status.hw) 2784 continue; 2785 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 2786 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); 2787 /* XXX handle errors */ 2788 if (r) { 2789 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 2790 adev->ip_blocks[i].version->funcs->name, r); 2791 } 2792 adev->ip_blocks[i].status.hw = false; 2793 break; 2794 } 2795 } 2796 } 2797 2798 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev) 2799 { 2800 int i, r; 2801 2802 for (i = 0; i < adev->num_ip_blocks; i++) { 2803 if (!adev->ip_blocks[i].version->funcs->early_fini) 2804 continue; 2805 2806 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev); 2807 if (r) { 2808 DRM_DEBUG("early_fini of IP block <%s> failed %d\n", 2809 adev->ip_blocks[i].version->funcs->name, r); 2810 } 2811 } 2812 2813 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 2814 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 2815 2816 amdgpu_amdkfd_suspend(adev, false); 2817 2818 /* Workaroud for ASICs need to disable SMC first */ 2819 amdgpu_device_smu_fini_early(adev); 2820 2821 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2822 if (!adev->ip_blocks[i].status.hw) 2823 continue; 2824 2825 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); 2826 /* XXX handle errors */ 2827 if (r) { 2828 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 2829 adev->ip_blocks[i].version->funcs->name, r); 2830 } 2831 2832 adev->ip_blocks[i].status.hw = false; 2833 } 2834 2835 if (amdgpu_sriov_vf(adev)) { 2836 if (amdgpu_virt_release_full_gpu(adev, false)) 2837 DRM_ERROR("failed to release exclusive mode on fini\n"); 2838 } 2839 2840 return 0; 2841 } 2842 2843 /** 2844 * amdgpu_device_ip_fini - run fini for hardware IPs 2845 * 2846 * @adev: amdgpu_device pointer 2847 * 2848 * Main teardown pass for hardware IPs. The list of all the hardware 2849 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks 2850 * are run. hw_fini tears down the hardware associated with each IP 2851 * and sw_fini tears down any software state associated with each IP. 2852 * Returns 0 on success, negative error code on failure. 2853 */ 2854 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) 2855 { 2856 int i, r; 2857 2858 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) 2859 amdgpu_virt_release_ras_err_handler_data(adev); 2860 2861 if (adev->gmc.xgmi.num_physical_nodes > 1) 2862 amdgpu_xgmi_remove_device(adev); 2863 2864 amdgpu_amdkfd_device_fini_sw(adev); 2865 2866 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2867 if (!adev->ip_blocks[i].status.sw) 2868 continue; 2869 2870 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2871 amdgpu_ucode_free_bo(adev); 2872 amdgpu_free_static_csa(&adev->virt.csa_obj); 2873 amdgpu_device_wb_fini(adev); 2874 amdgpu_device_vram_scratch_fini(adev); 2875 amdgpu_ib_pool_fini(adev); 2876 } 2877 2878 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); 2879 /* XXX handle errors */ 2880 if (r) { 2881 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n", 2882 adev->ip_blocks[i].version->funcs->name, r); 2883 } 2884 adev->ip_blocks[i].status.sw = false; 2885 adev->ip_blocks[i].status.valid = false; 2886 } 2887 2888 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2889 if (!adev->ip_blocks[i].status.late_initialized) 2890 continue; 2891 if (adev->ip_blocks[i].version->funcs->late_fini) 2892 adev->ip_blocks[i].version->funcs->late_fini((void *)adev); 2893 adev->ip_blocks[i].status.late_initialized = false; 2894 } 2895 2896 amdgpu_ras_fini(adev); 2897 2898 return 0; 2899 } 2900 2901 /** 2902 * amdgpu_device_delayed_init_work_handler - work handler for IB tests 2903 * 2904 * @work: work_struct. 2905 */ 2906 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) 2907 { 2908 struct amdgpu_device *adev = 2909 container_of(work, struct amdgpu_device, delayed_init_work.work); 2910 int r; 2911 2912 r = amdgpu_ib_ring_tests(adev); 2913 if (r) 2914 DRM_ERROR("ib ring test failed (%d).\n", r); 2915 } 2916 2917 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) 2918 { 2919 struct amdgpu_device *adev = 2920 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); 2921 2922 WARN_ON_ONCE(adev->gfx.gfx_off_state); 2923 WARN_ON_ONCE(adev->gfx.gfx_off_req_count); 2924 2925 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true)) 2926 adev->gfx.gfx_off_state = true; 2927 } 2928 2929 /** 2930 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) 2931 * 2932 * @adev: amdgpu_device pointer 2933 * 2934 * Main suspend function for hardware IPs. The list of all the hardware 2935 * IPs that make up the asic is walked, clockgating is disabled and the 2936 * suspend callbacks are run. suspend puts the hardware and software state 2937 * in each IP into a state suitable for suspend. 2938 * Returns 0 on success, negative error code on failure. 2939 */ 2940 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) 2941 { 2942 int i, r; 2943 2944 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 2945 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 2946 2947 /* 2948 * Per PMFW team's suggestion, driver needs to handle gfxoff 2949 * and df cstate features disablement for gpu reset(e.g. Mode1Reset) 2950 * scenario. Add the missing df cstate disablement here. 2951 */ 2952 if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW)) 2953 dev_warn(adev->dev, "Failed to disallow df cstate"); 2954 2955 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2956 if (!adev->ip_blocks[i].status.valid) 2957 continue; 2958 2959 /* displays are handled separately */ 2960 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) 2961 continue; 2962 2963 /* XXX handle errors */ 2964 r = adev->ip_blocks[i].version->funcs->suspend(adev); 2965 /* XXX handle errors */ 2966 if (r) { 2967 DRM_ERROR("suspend of IP block <%s> failed %d\n", 2968 adev->ip_blocks[i].version->funcs->name, r); 2969 return r; 2970 } 2971 2972 adev->ip_blocks[i].status.hw = false; 2973 } 2974 2975 return 0; 2976 } 2977 2978 /** 2979 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) 2980 * 2981 * @adev: amdgpu_device pointer 2982 * 2983 * Main suspend function for hardware IPs. The list of all the hardware 2984 * IPs that make up the asic is walked, clockgating is disabled and the 2985 * suspend callbacks are run. suspend puts the hardware and software state 2986 * in each IP into a state suitable for suspend. 2987 * Returns 0 on success, negative error code on failure. 2988 */ 2989 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) 2990 { 2991 int i, r; 2992 2993 if (adev->in_s0ix) 2994 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry); 2995 2996 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2997 if (!adev->ip_blocks[i].status.valid) 2998 continue; 2999 /* displays are handled in phase1 */ 3000 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) 3001 continue; 3002 /* PSP lost connection when err_event_athub occurs */ 3003 if (amdgpu_ras_intr_triggered() && 3004 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 3005 adev->ip_blocks[i].status.hw = false; 3006 continue; 3007 } 3008 3009 /* skip unnecessary suspend if we do not initialize them yet */ 3010 if (adev->gmc.xgmi.pending_reset && 3011 !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3012 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC || 3013 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3014 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) { 3015 adev->ip_blocks[i].status.hw = false; 3016 continue; 3017 } 3018 3019 /* skip suspend of gfx and psp for S0ix 3020 * gfx is in gfxoff state, so on resume it will exit gfxoff just 3021 * like at runtime. PSP is also part of the always on hardware 3022 * so no need to suspend it. 3023 */ 3024 if (adev->in_s0ix && 3025 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP || 3026 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)) 3027 continue; 3028 3029 /* XXX handle errors */ 3030 r = adev->ip_blocks[i].version->funcs->suspend(adev); 3031 /* XXX handle errors */ 3032 if (r) { 3033 DRM_ERROR("suspend of IP block <%s> failed %d\n", 3034 adev->ip_blocks[i].version->funcs->name, r); 3035 } 3036 adev->ip_blocks[i].status.hw = false; 3037 /* handle putting the SMC in the appropriate state */ 3038 if(!amdgpu_sriov_vf(adev)){ 3039 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3040 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); 3041 if (r) { 3042 DRM_ERROR("SMC failed to set mp1 state %d, %d\n", 3043 adev->mp1_state, r); 3044 return r; 3045 } 3046 } 3047 } 3048 } 3049 3050 return 0; 3051 } 3052 3053 /** 3054 * amdgpu_device_ip_suspend - run suspend for hardware IPs 3055 * 3056 * @adev: amdgpu_device pointer 3057 * 3058 * Main suspend function for hardware IPs. The list of all the hardware 3059 * IPs that make up the asic is walked, clockgating is disabled and the 3060 * suspend callbacks are run. suspend puts the hardware and software state 3061 * in each IP into a state suitable for suspend. 3062 * Returns 0 on success, negative error code on failure. 3063 */ 3064 int amdgpu_device_ip_suspend(struct amdgpu_device *adev) 3065 { 3066 int r; 3067 3068 if (amdgpu_sriov_vf(adev)) { 3069 amdgpu_virt_fini_data_exchange(adev); 3070 amdgpu_virt_request_full_gpu(adev, false); 3071 } 3072 3073 r = amdgpu_device_ip_suspend_phase1(adev); 3074 if (r) 3075 return r; 3076 r = amdgpu_device_ip_suspend_phase2(adev); 3077 3078 if (amdgpu_sriov_vf(adev)) 3079 amdgpu_virt_release_full_gpu(adev, false); 3080 3081 return r; 3082 } 3083 3084 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) 3085 { 3086 int i, r; 3087 3088 static enum amd_ip_block_type ip_order[] = { 3089 AMD_IP_BLOCK_TYPE_COMMON, 3090 AMD_IP_BLOCK_TYPE_GMC, 3091 AMD_IP_BLOCK_TYPE_PSP, 3092 AMD_IP_BLOCK_TYPE_IH, 3093 }; 3094 3095 for (i = 0; i < adev->num_ip_blocks; i++) { 3096 int j; 3097 struct amdgpu_ip_block *block; 3098 3099 block = &adev->ip_blocks[i]; 3100 block->status.hw = false; 3101 3102 for (j = 0; j < ARRAY_SIZE(ip_order); j++) { 3103 3104 if (block->version->type != ip_order[j] || 3105 !block->status.valid) 3106 continue; 3107 3108 r = block->version->funcs->hw_init(adev); 3109 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); 3110 if (r) 3111 return r; 3112 block->status.hw = true; 3113 } 3114 } 3115 3116 return 0; 3117 } 3118 3119 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) 3120 { 3121 int i, r; 3122 3123 static enum amd_ip_block_type ip_order[] = { 3124 AMD_IP_BLOCK_TYPE_SMC, 3125 AMD_IP_BLOCK_TYPE_DCE, 3126 AMD_IP_BLOCK_TYPE_GFX, 3127 AMD_IP_BLOCK_TYPE_SDMA, 3128 AMD_IP_BLOCK_TYPE_UVD, 3129 AMD_IP_BLOCK_TYPE_VCE, 3130 AMD_IP_BLOCK_TYPE_VCN 3131 }; 3132 3133 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 3134 int j; 3135 struct amdgpu_ip_block *block; 3136 3137 for (j = 0; j < adev->num_ip_blocks; j++) { 3138 block = &adev->ip_blocks[j]; 3139 3140 if (block->version->type != ip_order[i] || 3141 !block->status.valid || 3142 block->status.hw) 3143 continue; 3144 3145 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) 3146 r = block->version->funcs->resume(adev); 3147 else 3148 r = block->version->funcs->hw_init(adev); 3149 3150 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); 3151 if (r) 3152 return r; 3153 block->status.hw = true; 3154 } 3155 } 3156 3157 return 0; 3158 } 3159 3160 /** 3161 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs 3162 * 3163 * @adev: amdgpu_device pointer 3164 * 3165 * First resume function for hardware IPs. The list of all the hardware 3166 * IPs that make up the asic is walked and the resume callbacks are run for 3167 * COMMON, GMC, and IH. resume puts the hardware into a functional state 3168 * after a suspend and updates the software state as necessary. This 3169 * function is also used for restoring the GPU after a GPU reset. 3170 * Returns 0 on success, negative error code on failure. 3171 */ 3172 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) 3173 { 3174 int i, r; 3175 3176 for (i = 0; i < adev->num_ip_blocks; i++) { 3177 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3178 continue; 3179 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3180 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3181 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3182 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) { 3183 3184 r = adev->ip_blocks[i].version->funcs->resume(adev); 3185 if (r) { 3186 DRM_ERROR("resume of IP block <%s> failed %d\n", 3187 adev->ip_blocks[i].version->funcs->name, r); 3188 return r; 3189 } 3190 adev->ip_blocks[i].status.hw = true; 3191 } 3192 } 3193 3194 return 0; 3195 } 3196 3197 /** 3198 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs 3199 * 3200 * @adev: amdgpu_device pointer 3201 * 3202 * First resume function for hardware IPs. The list of all the hardware 3203 * IPs that make up the asic is walked and the resume callbacks are run for 3204 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a 3205 * functional state after a suspend and updates the software state as 3206 * necessary. This function is also used for restoring the GPU after a GPU 3207 * reset. 3208 * Returns 0 on success, negative error code on failure. 3209 */ 3210 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) 3211 { 3212 int i, r; 3213 3214 for (i = 0; i < adev->num_ip_blocks; i++) { 3215 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 3216 continue; 3217 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3218 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3219 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3220 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 3221 continue; 3222 r = adev->ip_blocks[i].version->funcs->resume(adev); 3223 if (r) { 3224 DRM_ERROR("resume of IP block <%s> failed %d\n", 3225 adev->ip_blocks[i].version->funcs->name, r); 3226 return r; 3227 } 3228 adev->ip_blocks[i].status.hw = true; 3229 3230 if (adev->in_s0ix && adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 3231 /* disable gfxoff for IP resume. The gfxoff will be re-enabled in 3232 * amdgpu_device_resume() after IP resume. 3233 */ 3234 amdgpu_gfx_off_ctrl(adev, false); 3235 DRM_DEBUG("will disable gfxoff for re-initializing other blocks\n"); 3236 } 3237 3238 } 3239 3240 return 0; 3241 } 3242 3243 /** 3244 * amdgpu_device_ip_resume - run resume for hardware IPs 3245 * 3246 * @adev: amdgpu_device pointer 3247 * 3248 * Main resume function for hardware IPs. The hardware IPs 3249 * are split into two resume functions because they are 3250 * are also used in in recovering from a GPU reset and some additional 3251 * steps need to be take between them. In this case (S3/S4) they are 3252 * run sequentially. 3253 * Returns 0 on success, negative error code on failure. 3254 */ 3255 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) 3256 { 3257 int r; 3258 3259 r = amdgpu_amdkfd_resume_iommu(adev); 3260 if (r) 3261 return r; 3262 3263 r = amdgpu_device_ip_resume_phase1(adev); 3264 if (r) 3265 return r; 3266 3267 r = amdgpu_device_fw_loading(adev); 3268 if (r) 3269 return r; 3270 3271 r = amdgpu_device_ip_resume_phase2(adev); 3272 3273 return r; 3274 } 3275 3276 /** 3277 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV 3278 * 3279 * @adev: amdgpu_device pointer 3280 * 3281 * Query the VBIOS data tables to determine if the board supports SR-IOV. 3282 */ 3283 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) 3284 { 3285 if (amdgpu_sriov_vf(adev)) { 3286 if (adev->is_atom_fw) { 3287 if (amdgpu_atomfirmware_gpu_virtualization_supported(adev)) 3288 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3289 } else { 3290 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) 3291 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 3292 } 3293 3294 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) 3295 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 3296 } 3297 } 3298 3299 /** 3300 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic 3301 * 3302 * @asic_type: AMD asic type 3303 * 3304 * Check if there is DC (new modesetting infrastructre) support for an asic. 3305 * returns true if DC has support, false if not. 3306 */ 3307 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) 3308 { 3309 switch (asic_type) { 3310 #ifdef CONFIG_DRM_AMDGPU_SI 3311 case CHIP_HAINAN: 3312 #endif 3313 case CHIP_TOPAZ: 3314 /* chips with no display hardware */ 3315 return false; 3316 #if defined(CONFIG_DRM_AMD_DC) 3317 case CHIP_TAHITI: 3318 case CHIP_PITCAIRN: 3319 case CHIP_VERDE: 3320 case CHIP_OLAND: 3321 /* 3322 * We have systems in the wild with these ASICs that require 3323 * LVDS and VGA support which is not supported with DC. 3324 * 3325 * Fallback to the non-DC driver here by default so as not to 3326 * cause regressions. 3327 */ 3328 #if defined(CONFIG_DRM_AMD_DC_SI) 3329 return amdgpu_dc > 0; 3330 #else 3331 return false; 3332 #endif 3333 case CHIP_BONAIRE: 3334 case CHIP_KAVERI: 3335 case CHIP_KABINI: 3336 case CHIP_MULLINS: 3337 /* 3338 * We have systems in the wild with these ASICs that require 3339 * VGA support which is not supported with DC. 3340 * 3341 * Fallback to the non-DC driver here by default so as not to 3342 * cause regressions. 3343 */ 3344 return amdgpu_dc > 0; 3345 default: 3346 return amdgpu_dc != 0; 3347 #else 3348 default: 3349 if (amdgpu_dc > 0) 3350 DRM_INFO_ONCE("Display Core has been requested via kernel parameter " 3351 "but isn't supported by ASIC, ignoring\n"); 3352 return false; 3353 #endif 3354 } 3355 } 3356 3357 /** 3358 * amdgpu_device_has_dc_support - check if dc is supported 3359 * 3360 * @adev: amdgpu_device pointer 3361 * 3362 * Returns true for supported, false for not supported 3363 */ 3364 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) 3365 { 3366 if (adev->enable_virtual_display || 3367 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 3368 return false; 3369 3370 return amdgpu_device_asic_has_dc_support(adev->asic_type); 3371 } 3372 3373 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) 3374 { 3375 struct amdgpu_device *adev = 3376 container_of(__work, struct amdgpu_device, xgmi_reset_work); 3377 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 3378 3379 /* It's a bug to not have a hive within this function */ 3380 if (WARN_ON(!hive)) 3381 return; 3382 3383 /* 3384 * Use task barrier to synchronize all xgmi reset works across the 3385 * hive. task_barrier_enter and task_barrier_exit will block 3386 * until all the threads running the xgmi reset works reach 3387 * those points. task_barrier_full will do both blocks. 3388 */ 3389 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { 3390 3391 task_barrier_enter(&hive->tb); 3392 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev)); 3393 3394 if (adev->asic_reset_res) 3395 goto fail; 3396 3397 task_barrier_exit(&hive->tb); 3398 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev)); 3399 3400 if (adev->asic_reset_res) 3401 goto fail; 3402 3403 if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops && 3404 adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count) 3405 adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev); 3406 } else { 3407 3408 task_barrier_full(&hive->tb); 3409 adev->asic_reset_res = amdgpu_asic_reset(adev); 3410 } 3411 3412 fail: 3413 if (adev->asic_reset_res) 3414 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s", 3415 adev->asic_reset_res, adev_to_drm(adev)->unique); 3416 amdgpu_put_xgmi_hive(hive); 3417 } 3418 3419 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) 3420 { 3421 char *input = amdgpu_lockup_timeout; 3422 char *timeout_setting = NULL; 3423 int index = 0; 3424 long timeout; 3425 int ret = 0; 3426 3427 /* 3428 * By default timeout for non compute jobs is 10000 3429 * and 60000 for compute jobs. 3430 * In SR-IOV or passthrough mode, timeout for compute 3431 * jobs are 60000 by default. 3432 */ 3433 adev->gfx_timeout = msecs_to_jiffies(10000); 3434 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 3435 if (amdgpu_sriov_vf(adev)) 3436 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ? 3437 msecs_to_jiffies(60000) : msecs_to_jiffies(10000); 3438 else 3439 adev->compute_timeout = msecs_to_jiffies(60000); 3440 3441 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 3442 while ((timeout_setting = strsep(&input, ",")) && 3443 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 3444 ret = kstrtol(timeout_setting, 0, &timeout); 3445 if (ret) 3446 return ret; 3447 3448 if (timeout == 0) { 3449 index++; 3450 continue; 3451 } else if (timeout < 0) { 3452 timeout = MAX_SCHEDULE_TIMEOUT; 3453 dev_warn(adev->dev, "lockup timeout disabled"); 3454 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); 3455 } else { 3456 timeout = msecs_to_jiffies(timeout); 3457 } 3458 3459 switch (index++) { 3460 case 0: 3461 adev->gfx_timeout = timeout; 3462 break; 3463 case 1: 3464 adev->compute_timeout = timeout; 3465 break; 3466 case 2: 3467 adev->sdma_timeout = timeout; 3468 break; 3469 case 3: 3470 adev->video_timeout = timeout; 3471 break; 3472 default: 3473 break; 3474 } 3475 } 3476 /* 3477 * There is only one value specified and 3478 * it should apply to all non-compute jobs. 3479 */ 3480 if (index == 1) { 3481 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 3482 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) 3483 adev->compute_timeout = adev->gfx_timeout; 3484 } 3485 } 3486 3487 return ret; 3488 } 3489 3490 /** 3491 * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU 3492 * 3493 * @adev: amdgpu_device pointer 3494 * 3495 * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode 3496 */ 3497 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev) 3498 { 3499 struct iommu_domain *domain; 3500 3501 domain = iommu_get_domain_for_dev(adev->dev); 3502 if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY) 3503 adev->ram_is_direct_mapped = true; 3504 } 3505 3506 static const struct attribute *amdgpu_dev_attributes[] = { 3507 &dev_attr_product_name.attr, 3508 &dev_attr_product_number.attr, 3509 &dev_attr_serial_number.attr, 3510 &dev_attr_pcie_replay_count.attr, 3511 NULL 3512 }; 3513 3514 /** 3515 * amdgpu_device_init - initialize the driver 3516 * 3517 * @adev: amdgpu_device pointer 3518 * @flags: driver flags 3519 * 3520 * Initializes the driver info and hw (all asics). 3521 * Returns 0 for success or an error on failure. 3522 * Called at driver startup. 3523 */ 3524 int amdgpu_device_init(struct amdgpu_device *adev, 3525 uint32_t flags) 3526 { 3527 struct drm_device *ddev = adev_to_drm(adev); 3528 struct pci_dev *pdev = adev->pdev; 3529 int r, i; 3530 bool px = false; 3531 u32 max_MBps; 3532 3533 adev->shutdown = false; 3534 adev->flags = flags; 3535 3536 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) 3537 adev->asic_type = amdgpu_force_asic_type; 3538 else 3539 adev->asic_type = flags & AMD_ASIC_MASK; 3540 3541 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; 3542 if (amdgpu_emu_mode == 1) 3543 adev->usec_timeout *= 10; 3544 adev->gmc.gart_size = 512 * 1024 * 1024; 3545 adev->accel_working = false; 3546 adev->num_rings = 0; 3547 RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub()); 3548 adev->mman.buffer_funcs = NULL; 3549 adev->mman.buffer_funcs_ring = NULL; 3550 adev->vm_manager.vm_pte_funcs = NULL; 3551 adev->vm_manager.vm_pte_num_scheds = 0; 3552 adev->gmc.gmc_funcs = NULL; 3553 adev->harvest_ip_mask = 0x0; 3554 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); 3555 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 3556 3557 adev->smc_rreg = &amdgpu_invalid_rreg; 3558 adev->smc_wreg = &amdgpu_invalid_wreg; 3559 adev->pcie_rreg = &amdgpu_invalid_rreg; 3560 adev->pcie_wreg = &amdgpu_invalid_wreg; 3561 adev->pciep_rreg = &amdgpu_invalid_rreg; 3562 adev->pciep_wreg = &amdgpu_invalid_wreg; 3563 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; 3564 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; 3565 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; 3566 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; 3567 adev->didt_rreg = &amdgpu_invalid_rreg; 3568 adev->didt_wreg = &amdgpu_invalid_wreg; 3569 adev->gc_cac_rreg = &amdgpu_invalid_rreg; 3570 adev->gc_cac_wreg = &amdgpu_invalid_wreg; 3571 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; 3572 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; 3573 3574 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", 3575 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, 3576 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); 3577 3578 /* mutex initialization are all done here so we 3579 * can recall function without having locking issues */ 3580 mutex_init(&adev->firmware.mutex); 3581 mutex_init(&adev->pm.mutex); 3582 mutex_init(&adev->gfx.gpu_clock_mutex); 3583 mutex_init(&adev->srbm_mutex); 3584 mutex_init(&adev->gfx.pipe_reserve_mutex); 3585 mutex_init(&adev->gfx.gfx_off_mutex); 3586 mutex_init(&adev->grbm_idx_mutex); 3587 mutex_init(&adev->mn_lock); 3588 mutex_init(&adev->virt.vf_errors.lock); 3589 hash_init(adev->mn_hash); 3590 mutex_init(&adev->psp.mutex); 3591 mutex_init(&adev->notifier_lock); 3592 mutex_init(&adev->pm.stable_pstate_ctx_lock); 3593 mutex_init(&adev->benchmark_mutex); 3594 3595 amdgpu_device_init_apu_flags(adev); 3596 3597 r = amdgpu_device_check_arguments(adev); 3598 if (r) 3599 return r; 3600 3601 spin_lock_init(&adev->mmio_idx_lock); 3602 spin_lock_init(&adev->smc_idx_lock); 3603 spin_lock_init(&adev->pcie_idx_lock); 3604 spin_lock_init(&adev->uvd_ctx_idx_lock); 3605 spin_lock_init(&adev->didt_idx_lock); 3606 spin_lock_init(&adev->gc_cac_idx_lock); 3607 spin_lock_init(&adev->se_cac_idx_lock); 3608 spin_lock_init(&adev->audio_endpt_idx_lock); 3609 spin_lock_init(&adev->mm_stats.lock); 3610 3611 INIT_LIST_HEAD(&adev->shadow_list); 3612 mutex_init(&adev->shadow_list_lock); 3613 3614 INIT_LIST_HEAD(&adev->reset_list); 3615 3616 INIT_LIST_HEAD(&adev->ras_list); 3617 3618 INIT_DELAYED_WORK(&adev->delayed_init_work, 3619 amdgpu_device_delayed_init_work_handler); 3620 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, 3621 amdgpu_device_delay_enable_gfx_off); 3622 3623 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); 3624 3625 adev->gfx.gfx_off_req_count = 1; 3626 adev->gfx.gfx_off_residency = 0; 3627 adev->gfx.gfx_off_entrycount = 0; 3628 adev->pm.ac_power = power_supply_is_system_supplied() > 0; 3629 3630 atomic_set(&adev->throttling_logging_enabled, 1); 3631 /* 3632 * If throttling continues, logging will be performed every minute 3633 * to avoid log flooding. "-1" is subtracted since the thermal 3634 * throttling interrupt comes every second. Thus, the total logging 3635 * interval is 59 seconds(retelimited printk interval) + 1(waiting 3636 * for throttling interrupt) = 60 seconds. 3637 */ 3638 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); 3639 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); 3640 3641 /* Registers mapping */ 3642 /* TODO: block userspace mapping of io register */ 3643 if (adev->asic_type >= CHIP_BONAIRE) { 3644 adev->rmmio_base = pci_resource_start(adev->pdev, 5); 3645 adev->rmmio_size = pci_resource_len(adev->pdev, 5); 3646 } else { 3647 adev->rmmio_base = pci_resource_start(adev->pdev, 2); 3648 adev->rmmio_size = pci_resource_len(adev->pdev, 2); 3649 } 3650 3651 for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++) 3652 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN); 3653 3654 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); 3655 if (adev->rmmio == NULL) { 3656 return -ENOMEM; 3657 } 3658 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); 3659 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size); 3660 3661 amdgpu_device_get_pcie_info(adev); 3662 3663 if (amdgpu_mcbp) 3664 DRM_INFO("MCBP is enabled\n"); 3665 3666 /* 3667 * Reset domain needs to be present early, before XGMI hive discovered 3668 * (if any) and intitialized to use reset sem and in_gpu reset flag 3669 * early on during init and before calling to RREG32. 3670 */ 3671 adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev"); 3672 if (!adev->reset_domain) 3673 return -ENOMEM; 3674 3675 /* detect hw virtualization here */ 3676 amdgpu_detect_virtualization(adev); 3677 3678 r = amdgpu_device_get_job_timeout_settings(adev); 3679 if (r) { 3680 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); 3681 return r; 3682 } 3683 3684 /* early init functions */ 3685 r = amdgpu_device_ip_early_init(adev); 3686 if (r) 3687 return r; 3688 3689 /* Enable TMZ based on IP_VERSION */ 3690 amdgpu_gmc_tmz_set(adev); 3691 3692 amdgpu_gmc_noretry_set(adev); 3693 /* Need to get xgmi info early to decide the reset behavior*/ 3694 if (adev->gmc.xgmi.supported) { 3695 r = adev->gfxhub.funcs->get_xgmi_info(adev); 3696 if (r) 3697 return r; 3698 } 3699 3700 /* enable PCIE atomic ops */ 3701 if (amdgpu_sriov_vf(adev)) 3702 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *) 3703 adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags == 3704 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64); 3705 else 3706 adev->have_atomics_support = 3707 !pci_enable_atomic_ops_to_root(adev->pdev, 3708 PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 3709 PCI_EXP_DEVCAP2_ATOMIC_COMP64); 3710 if (!adev->have_atomics_support) 3711 dev_info(adev->dev, "PCIE atomic ops is not supported\n"); 3712 3713 /* doorbell bar mapping and doorbell index init*/ 3714 amdgpu_device_doorbell_init(adev); 3715 3716 if (amdgpu_emu_mode == 1) { 3717 /* post the asic on emulation mode */ 3718 emu_soc_asic_init(adev); 3719 goto fence_driver_init; 3720 } 3721 3722 amdgpu_reset_init(adev); 3723 3724 /* detect if we are with an SRIOV vbios */ 3725 amdgpu_device_detect_sriov_bios(adev); 3726 3727 /* check if we need to reset the asic 3728 * E.g., driver was not cleanly unloaded previously, etc. 3729 */ 3730 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { 3731 if (adev->gmc.xgmi.num_physical_nodes) { 3732 dev_info(adev->dev, "Pending hive reset.\n"); 3733 adev->gmc.xgmi.pending_reset = true; 3734 /* Only need to init necessary block for SMU to handle the reset */ 3735 for (i = 0; i < adev->num_ip_blocks; i++) { 3736 if (!adev->ip_blocks[i].status.valid) 3737 continue; 3738 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 3739 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 3740 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 3741 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) { 3742 DRM_DEBUG("IP %s disabled for hw_init.\n", 3743 adev->ip_blocks[i].version->funcs->name); 3744 adev->ip_blocks[i].status.hw = true; 3745 } 3746 } 3747 } else { 3748 r = amdgpu_asic_reset(adev); 3749 if (r) { 3750 dev_err(adev->dev, "asic reset on init failed\n"); 3751 goto failed; 3752 } 3753 } 3754 } 3755 3756 pci_enable_pcie_error_reporting(adev->pdev); 3757 3758 /* Post card if necessary */ 3759 if (amdgpu_device_need_post(adev)) { 3760 if (!adev->bios) { 3761 dev_err(adev->dev, "no vBIOS found\n"); 3762 r = -EINVAL; 3763 goto failed; 3764 } 3765 DRM_INFO("GPU posting now...\n"); 3766 r = amdgpu_device_asic_init(adev); 3767 if (r) { 3768 dev_err(adev->dev, "gpu post error!\n"); 3769 goto failed; 3770 } 3771 } 3772 3773 if (adev->is_atom_fw) { 3774 /* Initialize clocks */ 3775 r = amdgpu_atomfirmware_get_clock_info(adev); 3776 if (r) { 3777 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 3778 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 3779 goto failed; 3780 } 3781 } else { 3782 /* Initialize clocks */ 3783 r = amdgpu_atombios_get_clock_info(adev); 3784 if (r) { 3785 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 3786 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 3787 goto failed; 3788 } 3789 /* init i2c buses */ 3790 if (!amdgpu_device_has_dc_support(adev)) 3791 amdgpu_atombios_i2c_init(adev); 3792 } 3793 3794 fence_driver_init: 3795 /* Fence driver */ 3796 r = amdgpu_fence_driver_sw_init(adev); 3797 if (r) { 3798 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n"); 3799 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 3800 goto failed; 3801 } 3802 3803 /* init the mode config */ 3804 drm_mode_config_init(adev_to_drm(adev)); 3805 3806 r = amdgpu_device_ip_init(adev); 3807 if (r) { 3808 /* failed in exclusive mode due to timeout */ 3809 if (amdgpu_sriov_vf(adev) && 3810 !amdgpu_sriov_runtime(adev) && 3811 amdgpu_virt_mmio_blocked(adev) && 3812 !amdgpu_virt_wait_reset(adev)) { 3813 dev_err(adev->dev, "VF exclusive mode timeout\n"); 3814 /* Don't send request since VF is inactive. */ 3815 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 3816 adev->virt.ops = NULL; 3817 r = -EAGAIN; 3818 goto release_ras_con; 3819 } 3820 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); 3821 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 3822 goto release_ras_con; 3823 } 3824 3825 amdgpu_fence_driver_hw_init(adev); 3826 3827 dev_info(adev->dev, 3828 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n", 3829 adev->gfx.config.max_shader_engines, 3830 adev->gfx.config.max_sh_per_se, 3831 adev->gfx.config.max_cu_per_sh, 3832 adev->gfx.cu_info.number); 3833 3834 adev->accel_working = true; 3835 3836 amdgpu_vm_check_compute_bug(adev); 3837 3838 /* Initialize the buffer migration limit. */ 3839 if (amdgpu_moverate >= 0) 3840 max_MBps = amdgpu_moverate; 3841 else 3842 max_MBps = 8; /* Allow 8 MB/s. */ 3843 /* Get a log2 for easy divisions. */ 3844 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); 3845 3846 r = amdgpu_pm_sysfs_init(adev); 3847 if (r) { 3848 adev->pm_sysfs_en = false; 3849 DRM_ERROR("registering pm debugfs failed (%d).\n", r); 3850 } else 3851 adev->pm_sysfs_en = true; 3852 3853 r = amdgpu_ucode_sysfs_init(adev); 3854 if (r) { 3855 adev->ucode_sysfs_en = false; 3856 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r); 3857 } else 3858 adev->ucode_sysfs_en = true; 3859 3860 r = amdgpu_psp_sysfs_init(adev); 3861 if (r) { 3862 adev->psp_sysfs_en = false; 3863 if (!amdgpu_sriov_vf(adev)) 3864 DRM_ERROR("Creating psp sysfs failed\n"); 3865 } else 3866 adev->psp_sysfs_en = true; 3867 3868 /* 3869 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost. 3870 * Otherwise the mgpu fan boost feature will be skipped due to the 3871 * gpu instance is counted less. 3872 */ 3873 amdgpu_register_gpu_instance(adev); 3874 3875 /* enable clockgating, etc. after ib tests, etc. since some blocks require 3876 * explicit gating rather than handling it automatically. 3877 */ 3878 if (!adev->gmc.xgmi.pending_reset) { 3879 r = amdgpu_device_ip_late_init(adev); 3880 if (r) { 3881 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); 3882 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 3883 goto release_ras_con; 3884 } 3885 /* must succeed. */ 3886 amdgpu_ras_resume(adev); 3887 queue_delayed_work(system_wq, &adev->delayed_init_work, 3888 msecs_to_jiffies(AMDGPU_RESUME_MS)); 3889 } 3890 3891 if (amdgpu_sriov_vf(adev)) 3892 flush_delayed_work(&adev->delayed_init_work); 3893 3894 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes); 3895 if (r) 3896 dev_err(adev->dev, "Could not create amdgpu device attr\n"); 3897 3898 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 3899 r = amdgpu_pmu_init(adev); 3900 if (r) 3901 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); 3902 3903 /* Have stored pci confspace at hand for restore in sudden PCI error */ 3904 if (amdgpu_device_cache_pci_state(adev->pdev)) 3905 pci_restore_state(pdev); 3906 3907 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ 3908 /* this will fail for cards that aren't VGA class devices, just 3909 * ignore it */ 3910 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 3911 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode); 3912 3913 if (amdgpu_device_supports_px(ddev)) { 3914 px = true; 3915 vga_switcheroo_register_client(adev->pdev, 3916 &amdgpu_switcheroo_ops, px); 3917 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); 3918 } 3919 3920 if (adev->gmc.xgmi.pending_reset) 3921 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work, 3922 msecs_to_jiffies(AMDGPU_RESUME_MS)); 3923 3924 amdgpu_device_check_iommu_direct_map(adev); 3925 3926 return 0; 3927 3928 release_ras_con: 3929 amdgpu_release_ras_context(adev); 3930 3931 failed: 3932 amdgpu_vf_error_trans_all(adev); 3933 3934 return r; 3935 } 3936 3937 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev) 3938 { 3939 3940 /* Clear all CPU mappings pointing to this device */ 3941 unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1); 3942 3943 /* Unmap all mapped bars - Doorbell, registers and VRAM */ 3944 amdgpu_device_doorbell_fini(adev); 3945 3946 iounmap(adev->rmmio); 3947 adev->rmmio = NULL; 3948 if (adev->mman.aper_base_kaddr) 3949 iounmap(adev->mman.aper_base_kaddr); 3950 adev->mman.aper_base_kaddr = NULL; 3951 3952 /* Memory manager related */ 3953 if (!adev->gmc.xgmi.connected_to_cpu) { 3954 arch_phys_wc_del(adev->gmc.vram_mtrr); 3955 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size); 3956 } 3957 } 3958 3959 /** 3960 * amdgpu_device_fini_hw - tear down the driver 3961 * 3962 * @adev: amdgpu_device pointer 3963 * 3964 * Tear down the driver info (all asics). 3965 * Called at driver shutdown. 3966 */ 3967 void amdgpu_device_fini_hw(struct amdgpu_device *adev) 3968 { 3969 dev_info(adev->dev, "amdgpu: finishing device.\n"); 3970 flush_delayed_work(&adev->delayed_init_work); 3971 adev->shutdown = true; 3972 3973 /* make sure IB test finished before entering exclusive mode 3974 * to avoid preemption on IB test 3975 * */ 3976 if (amdgpu_sriov_vf(adev)) { 3977 amdgpu_virt_request_full_gpu(adev, false); 3978 amdgpu_virt_fini_data_exchange(adev); 3979 } 3980 3981 /* disable all interrupts */ 3982 amdgpu_irq_disable_all(adev); 3983 if (adev->mode_info.mode_config_initialized){ 3984 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev))) 3985 drm_helper_force_disable_all(adev_to_drm(adev)); 3986 else 3987 drm_atomic_helper_shutdown(adev_to_drm(adev)); 3988 } 3989 amdgpu_fence_driver_hw_fini(adev); 3990 3991 if (adev->mman.initialized) { 3992 flush_delayed_work(&adev->mman.bdev.wq); 3993 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); 3994 } 3995 3996 if (adev->pm_sysfs_en) 3997 amdgpu_pm_sysfs_fini(adev); 3998 if (adev->ucode_sysfs_en) 3999 amdgpu_ucode_sysfs_fini(adev); 4000 if (adev->psp_sysfs_en) 4001 amdgpu_psp_sysfs_fini(adev); 4002 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); 4003 4004 /* disable ras feature must before hw fini */ 4005 amdgpu_ras_pre_fini(adev); 4006 4007 amdgpu_device_ip_fini_early(adev); 4008 4009 amdgpu_irq_fini_hw(adev); 4010 4011 if (adev->mman.initialized) 4012 ttm_device_clear_dma_mappings(&adev->mman.bdev); 4013 4014 amdgpu_gart_dummy_page_fini(adev); 4015 4016 amdgpu_device_unmap_mmio(adev); 4017 4018 } 4019 4020 void amdgpu_device_fini_sw(struct amdgpu_device *adev) 4021 { 4022 int idx; 4023 4024 amdgpu_fence_driver_sw_fini(adev); 4025 amdgpu_device_ip_fini(adev); 4026 release_firmware(adev->firmware.gpu_info_fw); 4027 adev->firmware.gpu_info_fw = NULL; 4028 adev->accel_working = false; 4029 dma_fence_put(rcu_dereference_protected(adev->gang_submit, true)); 4030 4031 amdgpu_reset_fini(adev); 4032 4033 /* free i2c buses */ 4034 if (!amdgpu_device_has_dc_support(adev)) 4035 amdgpu_i2c_fini(adev); 4036 4037 if (amdgpu_emu_mode != 1) 4038 amdgpu_atombios_fini(adev); 4039 4040 kfree(adev->bios); 4041 adev->bios = NULL; 4042 if (amdgpu_device_supports_px(adev_to_drm(adev))) { 4043 vga_switcheroo_unregister_client(adev->pdev); 4044 vga_switcheroo_fini_domain_pm_ops(adev->dev); 4045 } 4046 if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA) 4047 vga_client_unregister(adev->pdev); 4048 4049 if (drm_dev_enter(adev_to_drm(adev), &idx)) { 4050 4051 iounmap(adev->rmmio); 4052 adev->rmmio = NULL; 4053 amdgpu_device_doorbell_fini(adev); 4054 drm_dev_exit(idx); 4055 } 4056 4057 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 4058 amdgpu_pmu_fini(adev); 4059 if (adev->mman.discovery_bin) 4060 amdgpu_discovery_fini(adev); 4061 4062 amdgpu_reset_put_reset_domain(adev->reset_domain); 4063 adev->reset_domain = NULL; 4064 4065 kfree(adev->pci_state); 4066 4067 } 4068 4069 /** 4070 * amdgpu_device_evict_resources - evict device resources 4071 * @adev: amdgpu device object 4072 * 4073 * Evicts all ttm device resources(vram BOs, gart table) from the lru list 4074 * of the vram memory type. Mainly used for evicting device resources 4075 * at suspend time. 4076 * 4077 */ 4078 static int amdgpu_device_evict_resources(struct amdgpu_device *adev) 4079 { 4080 int ret; 4081 4082 /* No need to evict vram on APUs for suspend to ram or s2idle */ 4083 if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU)) 4084 return 0; 4085 4086 ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM); 4087 if (ret) 4088 DRM_WARN("evicting device resources failed\n"); 4089 return ret; 4090 } 4091 4092 /* 4093 * Suspend & resume. 4094 */ 4095 /** 4096 * amdgpu_device_suspend - initiate device suspend 4097 * 4098 * @dev: drm dev pointer 4099 * @fbcon : notify the fbdev of suspend 4100 * 4101 * Puts the hw in the suspend state (all asics). 4102 * Returns 0 for success or an error on failure. 4103 * Called at driver suspend. 4104 */ 4105 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) 4106 { 4107 struct amdgpu_device *adev = drm_to_adev(dev); 4108 int r = 0; 4109 4110 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4111 return 0; 4112 4113 adev->in_suspend = true; 4114 4115 if (amdgpu_sriov_vf(adev)) { 4116 amdgpu_virt_fini_data_exchange(adev); 4117 r = amdgpu_virt_request_full_gpu(adev, false); 4118 if (r) 4119 return r; 4120 } 4121 4122 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3)) 4123 DRM_WARN("smart shift update failed\n"); 4124 4125 drm_kms_helper_poll_disable(dev); 4126 4127 if (fbcon) 4128 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true); 4129 4130 cancel_delayed_work_sync(&adev->delayed_init_work); 4131 4132 amdgpu_ras_suspend(adev); 4133 4134 amdgpu_device_ip_suspend_phase1(adev); 4135 4136 if (!adev->in_s0ix) 4137 amdgpu_amdkfd_suspend(adev, adev->in_runpm); 4138 4139 r = amdgpu_device_evict_resources(adev); 4140 if (r) 4141 return r; 4142 4143 amdgpu_fence_driver_hw_fini(adev); 4144 4145 amdgpu_device_ip_suspend_phase2(adev); 4146 4147 if (amdgpu_sriov_vf(adev)) 4148 amdgpu_virt_release_full_gpu(adev, false); 4149 4150 return 0; 4151 } 4152 4153 /** 4154 * amdgpu_device_resume - initiate device resume 4155 * 4156 * @dev: drm dev pointer 4157 * @fbcon : notify the fbdev of resume 4158 * 4159 * Bring the hw back to operating state (all asics). 4160 * Returns 0 for success or an error on failure. 4161 * Called at driver resume. 4162 */ 4163 int amdgpu_device_resume(struct drm_device *dev, bool fbcon) 4164 { 4165 struct amdgpu_device *adev = drm_to_adev(dev); 4166 int r = 0; 4167 4168 if (amdgpu_sriov_vf(adev)) { 4169 r = amdgpu_virt_request_full_gpu(adev, true); 4170 if (r) 4171 return r; 4172 } 4173 4174 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 4175 return 0; 4176 4177 if (adev->in_s0ix) 4178 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry); 4179 4180 /* post card */ 4181 if (amdgpu_device_need_post(adev)) { 4182 r = amdgpu_device_asic_init(adev); 4183 if (r) 4184 dev_err(adev->dev, "amdgpu asic init failed\n"); 4185 } 4186 4187 r = amdgpu_device_ip_resume(adev); 4188 4189 if (r) { 4190 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); 4191 goto exit; 4192 } 4193 amdgpu_fence_driver_hw_init(adev); 4194 4195 r = amdgpu_device_ip_late_init(adev); 4196 if (r) 4197 goto exit; 4198 4199 queue_delayed_work(system_wq, &adev->delayed_init_work, 4200 msecs_to_jiffies(AMDGPU_RESUME_MS)); 4201 4202 if (!adev->in_s0ix) { 4203 r = amdgpu_amdkfd_resume(adev, adev->in_runpm); 4204 if (r) 4205 goto exit; 4206 } 4207 4208 exit: 4209 if (amdgpu_sriov_vf(adev)) { 4210 amdgpu_virt_init_data_exchange(adev); 4211 amdgpu_virt_release_full_gpu(adev, true); 4212 } 4213 4214 if (r) 4215 return r; 4216 4217 /* Make sure IB tests flushed */ 4218 flush_delayed_work(&adev->delayed_init_work); 4219 4220 if (adev->in_s0ix) { 4221 /* re-enable gfxoff after IP resume. This re-enables gfxoff after 4222 * it was disabled for IP resume in amdgpu_device_ip_resume_phase2(). 4223 */ 4224 amdgpu_gfx_off_ctrl(adev, true); 4225 DRM_DEBUG("will enable gfxoff for the mission mode\n"); 4226 } 4227 if (fbcon) 4228 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false); 4229 4230 drm_kms_helper_poll_enable(dev); 4231 4232 amdgpu_ras_resume(adev); 4233 4234 if (adev->mode_info.num_crtc) { 4235 /* 4236 * Most of the connector probing functions try to acquire runtime pm 4237 * refs to ensure that the GPU is powered on when connector polling is 4238 * performed. Since we're calling this from a runtime PM callback, 4239 * trying to acquire rpm refs will cause us to deadlock. 4240 * 4241 * Since we're guaranteed to be holding the rpm lock, it's safe to 4242 * temporarily disable the rpm helpers so this doesn't deadlock us. 4243 */ 4244 #ifdef CONFIG_PM 4245 dev->dev->power.disable_depth++; 4246 #endif 4247 if (!adev->dc_enabled) 4248 drm_helper_hpd_irq_event(dev); 4249 else 4250 drm_kms_helper_hotplug_event(dev); 4251 #ifdef CONFIG_PM 4252 dev->dev->power.disable_depth--; 4253 #endif 4254 } 4255 adev->in_suspend = false; 4256 4257 if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0)) 4258 DRM_WARN("smart shift update failed\n"); 4259 4260 return 0; 4261 } 4262 4263 /** 4264 * amdgpu_device_ip_check_soft_reset - did soft reset succeed 4265 * 4266 * @adev: amdgpu_device pointer 4267 * 4268 * The list of all the hardware IPs that make up the asic is walked and 4269 * the check_soft_reset callbacks are run. check_soft_reset determines 4270 * if the asic is still hung or not. 4271 * Returns true if any of the IPs are still in a hung state, false if not. 4272 */ 4273 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) 4274 { 4275 int i; 4276 bool asic_hang = false; 4277 4278 if (amdgpu_sriov_vf(adev)) 4279 return true; 4280 4281 if (amdgpu_asic_need_full_reset(adev)) 4282 return true; 4283 4284 for (i = 0; i < adev->num_ip_blocks; i++) { 4285 if (!adev->ip_blocks[i].status.valid) 4286 continue; 4287 if (adev->ip_blocks[i].version->funcs->check_soft_reset) 4288 adev->ip_blocks[i].status.hang = 4289 adev->ip_blocks[i].version->funcs->check_soft_reset(adev); 4290 if (adev->ip_blocks[i].status.hang) { 4291 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); 4292 asic_hang = true; 4293 } 4294 } 4295 return asic_hang; 4296 } 4297 4298 /** 4299 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset 4300 * 4301 * @adev: amdgpu_device pointer 4302 * 4303 * The list of all the hardware IPs that make up the asic is walked and the 4304 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset 4305 * handles any IP specific hardware or software state changes that are 4306 * necessary for a soft reset to succeed. 4307 * Returns 0 on success, negative error code on failure. 4308 */ 4309 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) 4310 { 4311 int i, r = 0; 4312 4313 for (i = 0; i < adev->num_ip_blocks; i++) { 4314 if (!adev->ip_blocks[i].status.valid) 4315 continue; 4316 if (adev->ip_blocks[i].status.hang && 4317 adev->ip_blocks[i].version->funcs->pre_soft_reset) { 4318 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); 4319 if (r) 4320 return r; 4321 } 4322 } 4323 4324 return 0; 4325 } 4326 4327 /** 4328 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed 4329 * 4330 * @adev: amdgpu_device pointer 4331 * 4332 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu 4333 * reset is necessary to recover. 4334 * Returns true if a full asic reset is required, false if not. 4335 */ 4336 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) 4337 { 4338 int i; 4339 4340 if (amdgpu_asic_need_full_reset(adev)) 4341 return true; 4342 4343 for (i = 0; i < adev->num_ip_blocks; i++) { 4344 if (!adev->ip_blocks[i].status.valid) 4345 continue; 4346 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || 4347 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || 4348 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || 4349 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || 4350 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 4351 if (adev->ip_blocks[i].status.hang) { 4352 dev_info(adev->dev, "Some block need full reset!\n"); 4353 return true; 4354 } 4355 } 4356 } 4357 return false; 4358 } 4359 4360 /** 4361 * amdgpu_device_ip_soft_reset - do a soft reset 4362 * 4363 * @adev: amdgpu_device pointer 4364 * 4365 * The list of all the hardware IPs that make up the asic is walked and the 4366 * soft_reset callbacks are run if the block is hung. soft_reset handles any 4367 * IP specific hardware or software state changes that are necessary to soft 4368 * reset the IP. 4369 * Returns 0 on success, negative error code on failure. 4370 */ 4371 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) 4372 { 4373 int i, r = 0; 4374 4375 for (i = 0; i < adev->num_ip_blocks; i++) { 4376 if (!adev->ip_blocks[i].status.valid) 4377 continue; 4378 if (adev->ip_blocks[i].status.hang && 4379 adev->ip_blocks[i].version->funcs->soft_reset) { 4380 r = adev->ip_blocks[i].version->funcs->soft_reset(adev); 4381 if (r) 4382 return r; 4383 } 4384 } 4385 4386 return 0; 4387 } 4388 4389 /** 4390 * amdgpu_device_ip_post_soft_reset - clean up from soft reset 4391 * 4392 * @adev: amdgpu_device pointer 4393 * 4394 * The list of all the hardware IPs that make up the asic is walked and the 4395 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset 4396 * handles any IP specific hardware or software state changes that are 4397 * necessary after the IP has been soft reset. 4398 * Returns 0 on success, negative error code on failure. 4399 */ 4400 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) 4401 { 4402 int i, r = 0; 4403 4404 for (i = 0; i < adev->num_ip_blocks; i++) { 4405 if (!adev->ip_blocks[i].status.valid) 4406 continue; 4407 if (adev->ip_blocks[i].status.hang && 4408 adev->ip_blocks[i].version->funcs->post_soft_reset) 4409 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); 4410 if (r) 4411 return r; 4412 } 4413 4414 return 0; 4415 } 4416 4417 /** 4418 * amdgpu_device_recover_vram - Recover some VRAM contents 4419 * 4420 * @adev: amdgpu_device pointer 4421 * 4422 * Restores the contents of VRAM buffers from the shadows in GTT. Used to 4423 * restore things like GPUVM page tables after a GPU reset where 4424 * the contents of VRAM might be lost. 4425 * 4426 * Returns: 4427 * 0 on success, negative error code on failure. 4428 */ 4429 static int amdgpu_device_recover_vram(struct amdgpu_device *adev) 4430 { 4431 struct dma_fence *fence = NULL, *next = NULL; 4432 struct amdgpu_bo *shadow; 4433 struct amdgpu_bo_vm *vmbo; 4434 long r = 1, tmo; 4435 4436 if (amdgpu_sriov_runtime(adev)) 4437 tmo = msecs_to_jiffies(8000); 4438 else 4439 tmo = msecs_to_jiffies(100); 4440 4441 dev_info(adev->dev, "recover vram bo from shadow start\n"); 4442 mutex_lock(&adev->shadow_list_lock); 4443 list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) { 4444 shadow = &vmbo->bo; 4445 /* No need to recover an evicted BO */ 4446 if (shadow->tbo.resource->mem_type != TTM_PL_TT || 4447 shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET || 4448 shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM) 4449 continue; 4450 4451 r = amdgpu_bo_restore_shadow(shadow, &next); 4452 if (r) 4453 break; 4454 4455 if (fence) { 4456 tmo = dma_fence_wait_timeout(fence, false, tmo); 4457 dma_fence_put(fence); 4458 fence = next; 4459 if (tmo == 0) { 4460 r = -ETIMEDOUT; 4461 break; 4462 } else if (tmo < 0) { 4463 r = tmo; 4464 break; 4465 } 4466 } else { 4467 fence = next; 4468 } 4469 } 4470 mutex_unlock(&adev->shadow_list_lock); 4471 4472 if (fence) 4473 tmo = dma_fence_wait_timeout(fence, false, tmo); 4474 dma_fence_put(fence); 4475 4476 if (r < 0 || tmo <= 0) { 4477 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo); 4478 return -EIO; 4479 } 4480 4481 dev_info(adev->dev, "recover vram bo from shadow done\n"); 4482 return 0; 4483 } 4484 4485 4486 /** 4487 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf 4488 * 4489 * @adev: amdgpu_device pointer 4490 * @from_hypervisor: request from hypervisor 4491 * 4492 * do VF FLR and reinitialize Asic 4493 * return 0 means succeeded otherwise failed 4494 */ 4495 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, 4496 bool from_hypervisor) 4497 { 4498 int r; 4499 struct amdgpu_hive_info *hive = NULL; 4500 int retry_limit = 0; 4501 4502 retry: 4503 amdgpu_amdkfd_pre_reset(adev); 4504 4505 if (from_hypervisor) 4506 r = amdgpu_virt_request_full_gpu(adev, true); 4507 else 4508 r = amdgpu_virt_reset_gpu(adev); 4509 if (r) 4510 return r; 4511 4512 /* Resume IP prior to SMC */ 4513 r = amdgpu_device_ip_reinit_early_sriov(adev); 4514 if (r) 4515 goto error; 4516 4517 amdgpu_virt_init_data_exchange(adev); 4518 4519 r = amdgpu_device_fw_loading(adev); 4520 if (r) 4521 return r; 4522 4523 /* now we are okay to resume SMC/CP/SDMA */ 4524 r = amdgpu_device_ip_reinit_late_sriov(adev); 4525 if (r) 4526 goto error; 4527 4528 hive = amdgpu_get_xgmi_hive(adev); 4529 /* Update PSP FW topology after reset */ 4530 if (hive && adev->gmc.xgmi.num_physical_nodes > 1) 4531 r = amdgpu_xgmi_update_topology(hive, adev); 4532 4533 if (hive) 4534 amdgpu_put_xgmi_hive(hive); 4535 4536 if (!r) { 4537 amdgpu_irq_gpu_reset_resume_helper(adev); 4538 r = amdgpu_ib_ring_tests(adev); 4539 4540 amdgpu_amdkfd_post_reset(adev); 4541 } 4542 4543 error: 4544 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { 4545 amdgpu_inc_vram_lost(adev); 4546 r = amdgpu_device_recover_vram(adev); 4547 } 4548 amdgpu_virt_release_full_gpu(adev, true); 4549 4550 if (AMDGPU_RETRY_SRIOV_RESET(r)) { 4551 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) { 4552 retry_limit++; 4553 goto retry; 4554 } else 4555 DRM_ERROR("GPU reset retry is beyond the retry limit\n"); 4556 } 4557 4558 return r; 4559 } 4560 4561 /** 4562 * amdgpu_device_has_job_running - check if there is any job in mirror list 4563 * 4564 * @adev: amdgpu_device pointer 4565 * 4566 * check if there is any job in mirror list 4567 */ 4568 bool amdgpu_device_has_job_running(struct amdgpu_device *adev) 4569 { 4570 int i; 4571 struct drm_sched_job *job; 4572 4573 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4574 struct amdgpu_ring *ring = adev->rings[i]; 4575 4576 if (!ring || !ring->sched.thread) 4577 continue; 4578 4579 spin_lock(&ring->sched.job_list_lock); 4580 job = list_first_entry_or_null(&ring->sched.pending_list, 4581 struct drm_sched_job, list); 4582 spin_unlock(&ring->sched.job_list_lock); 4583 if (job) 4584 return true; 4585 } 4586 return false; 4587 } 4588 4589 /** 4590 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery 4591 * 4592 * @adev: amdgpu_device pointer 4593 * 4594 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover 4595 * a hung GPU. 4596 */ 4597 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) 4598 { 4599 4600 if (amdgpu_gpu_recovery == 0) 4601 goto disabled; 4602 4603 /* Skip soft reset check in fatal error mode */ 4604 if (!amdgpu_ras_is_poison_mode_supported(adev)) 4605 return true; 4606 4607 if (!amdgpu_device_ip_check_soft_reset(adev)) { 4608 dev_info(adev->dev,"Timeout, but no hardware hang detected.\n"); 4609 return false; 4610 } 4611 4612 if (amdgpu_sriov_vf(adev)) 4613 return true; 4614 4615 if (amdgpu_gpu_recovery == -1) { 4616 switch (adev->asic_type) { 4617 #ifdef CONFIG_DRM_AMDGPU_SI 4618 case CHIP_VERDE: 4619 case CHIP_TAHITI: 4620 case CHIP_PITCAIRN: 4621 case CHIP_OLAND: 4622 case CHIP_HAINAN: 4623 #endif 4624 #ifdef CONFIG_DRM_AMDGPU_CIK 4625 case CHIP_KAVERI: 4626 case CHIP_KABINI: 4627 case CHIP_MULLINS: 4628 #endif 4629 case CHIP_CARRIZO: 4630 case CHIP_STONEY: 4631 case CHIP_CYAN_SKILLFISH: 4632 goto disabled; 4633 default: 4634 break; 4635 } 4636 } 4637 4638 return true; 4639 4640 disabled: 4641 dev_info(adev->dev, "GPU recovery disabled.\n"); 4642 return false; 4643 } 4644 4645 int amdgpu_device_mode1_reset(struct amdgpu_device *adev) 4646 { 4647 u32 i; 4648 int ret = 0; 4649 4650 amdgpu_atombios_scratch_regs_engine_hung(adev, true); 4651 4652 dev_info(adev->dev, "GPU mode1 reset\n"); 4653 4654 /* disable BM */ 4655 pci_clear_master(adev->pdev); 4656 4657 amdgpu_device_cache_pci_state(adev->pdev); 4658 4659 if (amdgpu_dpm_is_mode1_reset_supported(adev)) { 4660 dev_info(adev->dev, "GPU smu mode1 reset\n"); 4661 ret = amdgpu_dpm_mode1_reset(adev); 4662 } else { 4663 dev_info(adev->dev, "GPU psp mode1 reset\n"); 4664 ret = psp_gpu_reset(adev); 4665 } 4666 4667 if (ret) 4668 dev_err(adev->dev, "GPU mode1 reset failed\n"); 4669 4670 amdgpu_device_load_pci_state(adev->pdev); 4671 4672 /* wait for asic to come out of reset */ 4673 for (i = 0; i < adev->usec_timeout; i++) { 4674 u32 memsize = adev->nbio.funcs->get_memsize(adev); 4675 4676 if (memsize != 0xffffffff) 4677 break; 4678 udelay(1); 4679 } 4680 4681 amdgpu_atombios_scratch_regs_engine_hung(adev, false); 4682 return ret; 4683 } 4684 4685 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, 4686 struct amdgpu_reset_context *reset_context) 4687 { 4688 int i, r = 0; 4689 struct amdgpu_job *job = NULL; 4690 bool need_full_reset = 4691 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4692 4693 if (reset_context->reset_req_dev == adev) 4694 job = reset_context->job; 4695 4696 if (amdgpu_sriov_vf(adev)) { 4697 /* stop the data exchange thread */ 4698 amdgpu_virt_fini_data_exchange(adev); 4699 } 4700 4701 amdgpu_fence_driver_isr_toggle(adev, true); 4702 4703 /* block all schedulers and reset given job's ring */ 4704 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4705 struct amdgpu_ring *ring = adev->rings[i]; 4706 4707 if (!ring || !ring->sched.thread) 4708 continue; 4709 4710 /*clear job fence from fence drv to avoid force_completion 4711 *leave NULL and vm flush fence in fence drv */ 4712 amdgpu_fence_driver_clear_job_fences(ring); 4713 4714 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 4715 amdgpu_fence_driver_force_completion(ring); 4716 } 4717 4718 amdgpu_fence_driver_isr_toggle(adev, false); 4719 4720 if (job && job->vm) 4721 drm_sched_increase_karma(&job->base); 4722 4723 r = amdgpu_reset_prepare_hwcontext(adev, reset_context); 4724 /* If reset handler not implemented, continue; otherwise return */ 4725 if (r == -ENOSYS) 4726 r = 0; 4727 else 4728 return r; 4729 4730 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ 4731 if (!amdgpu_sriov_vf(adev)) { 4732 4733 if (!need_full_reset) 4734 need_full_reset = amdgpu_device_ip_need_full_reset(adev); 4735 4736 if (!need_full_reset && amdgpu_gpu_recovery) { 4737 amdgpu_device_ip_pre_soft_reset(adev); 4738 r = amdgpu_device_ip_soft_reset(adev); 4739 amdgpu_device_ip_post_soft_reset(adev); 4740 if (r || amdgpu_device_ip_check_soft_reset(adev)) { 4741 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); 4742 need_full_reset = true; 4743 } 4744 } 4745 4746 if (need_full_reset) 4747 r = amdgpu_device_ip_suspend(adev); 4748 if (need_full_reset) 4749 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4750 else 4751 clear_bit(AMDGPU_NEED_FULL_RESET, 4752 &reset_context->flags); 4753 } 4754 4755 return r; 4756 } 4757 4758 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev) 4759 { 4760 int i; 4761 4762 lockdep_assert_held(&adev->reset_domain->sem); 4763 4764 for (i = 0; i < adev->num_regs; i++) { 4765 adev->reset_dump_reg_value[i] = RREG32(adev->reset_dump_reg_list[i]); 4766 trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i], 4767 adev->reset_dump_reg_value[i]); 4768 } 4769 4770 return 0; 4771 } 4772 4773 #ifdef CONFIG_DEV_COREDUMP 4774 static ssize_t amdgpu_devcoredump_read(char *buffer, loff_t offset, 4775 size_t count, void *data, size_t datalen) 4776 { 4777 struct drm_printer p; 4778 struct amdgpu_device *adev = data; 4779 struct drm_print_iterator iter; 4780 int i; 4781 4782 iter.data = buffer; 4783 iter.offset = 0; 4784 iter.start = offset; 4785 iter.remain = count; 4786 4787 p = drm_coredump_printer(&iter); 4788 4789 drm_printf(&p, "**** AMDGPU Device Coredump ****\n"); 4790 drm_printf(&p, "kernel: " UTS_RELEASE "\n"); 4791 drm_printf(&p, "module: " KBUILD_MODNAME "\n"); 4792 drm_printf(&p, "time: %lld.%09ld\n", adev->reset_time.tv_sec, adev->reset_time.tv_nsec); 4793 if (adev->reset_task_info.pid) 4794 drm_printf(&p, "process_name: %s PID: %d\n", 4795 adev->reset_task_info.process_name, 4796 adev->reset_task_info.pid); 4797 4798 if (adev->reset_vram_lost) 4799 drm_printf(&p, "VRAM is lost due to GPU reset!\n"); 4800 if (adev->num_regs) { 4801 drm_printf(&p, "AMDGPU register dumps:\nOffset: Value:\n"); 4802 4803 for (i = 0; i < adev->num_regs; i++) 4804 drm_printf(&p, "0x%08x: 0x%08x\n", 4805 adev->reset_dump_reg_list[i], 4806 adev->reset_dump_reg_value[i]); 4807 } 4808 4809 return count - iter.remain; 4810 } 4811 4812 static void amdgpu_devcoredump_free(void *data) 4813 { 4814 } 4815 4816 static void amdgpu_reset_capture_coredumpm(struct amdgpu_device *adev) 4817 { 4818 struct drm_device *dev = adev_to_drm(adev); 4819 4820 ktime_get_ts64(&adev->reset_time); 4821 dev_coredumpm(dev->dev, THIS_MODULE, adev, 0, GFP_KERNEL, 4822 amdgpu_devcoredump_read, amdgpu_devcoredump_free); 4823 } 4824 #endif 4825 4826 int amdgpu_do_asic_reset(struct list_head *device_list_handle, 4827 struct amdgpu_reset_context *reset_context) 4828 { 4829 struct amdgpu_device *tmp_adev = NULL; 4830 bool need_full_reset, skip_hw_reset, vram_lost = false; 4831 int r = 0; 4832 bool gpu_reset_for_dev_remove = 0; 4833 4834 /* Try reset handler method first */ 4835 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 4836 reset_list); 4837 amdgpu_reset_reg_dumps(tmp_adev); 4838 4839 reset_context->reset_device_list = device_list_handle; 4840 r = amdgpu_reset_perform_reset(tmp_adev, reset_context); 4841 /* If reset handler not implemented, continue; otherwise return */ 4842 if (r == -ENOSYS) 4843 r = 0; 4844 else 4845 return r; 4846 4847 /* Reset handler not implemented, use the default method */ 4848 need_full_reset = 4849 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4850 skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags); 4851 4852 gpu_reset_for_dev_remove = 4853 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) && 4854 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 4855 4856 /* 4857 * ASIC reset has to be done on all XGMI hive nodes ASAP 4858 * to allow proper links negotiation in FW (within 1 sec) 4859 */ 4860 if (!skip_hw_reset && need_full_reset) { 4861 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 4862 /* For XGMI run all resets in parallel to speed up the process */ 4863 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 4864 tmp_adev->gmc.xgmi.pending_reset = false; 4865 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work)) 4866 r = -EALREADY; 4867 } else 4868 r = amdgpu_asic_reset(tmp_adev); 4869 4870 if (r) { 4871 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s", 4872 r, adev_to_drm(tmp_adev)->unique); 4873 break; 4874 } 4875 } 4876 4877 /* For XGMI wait for all resets to complete before proceed */ 4878 if (!r) { 4879 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 4880 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 4881 flush_work(&tmp_adev->xgmi_reset_work); 4882 r = tmp_adev->asic_reset_res; 4883 if (r) 4884 break; 4885 } 4886 } 4887 } 4888 } 4889 4890 if (!r && amdgpu_ras_intr_triggered()) { 4891 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 4892 if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops && 4893 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count) 4894 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev); 4895 } 4896 4897 amdgpu_ras_intr_cleared(); 4898 } 4899 4900 /* Since the mode1 reset affects base ip blocks, the 4901 * phase1 ip blocks need to be resumed. Otherwise there 4902 * will be a BIOS signature error and the psp bootloader 4903 * can't load kdb on the next amdgpu install. 4904 */ 4905 if (gpu_reset_for_dev_remove) { 4906 list_for_each_entry(tmp_adev, device_list_handle, reset_list) 4907 amdgpu_device_ip_resume_phase1(tmp_adev); 4908 4909 goto end; 4910 } 4911 4912 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 4913 if (need_full_reset) { 4914 /* post card */ 4915 r = amdgpu_device_asic_init(tmp_adev); 4916 if (r) { 4917 dev_warn(tmp_adev->dev, "asic atom init failed!"); 4918 } else { 4919 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); 4920 r = amdgpu_amdkfd_resume_iommu(tmp_adev); 4921 if (r) 4922 goto out; 4923 4924 r = amdgpu_device_ip_resume_phase1(tmp_adev); 4925 if (r) 4926 goto out; 4927 4928 vram_lost = amdgpu_device_check_vram_lost(tmp_adev); 4929 #ifdef CONFIG_DEV_COREDUMP 4930 tmp_adev->reset_vram_lost = vram_lost; 4931 memset(&tmp_adev->reset_task_info, 0, 4932 sizeof(tmp_adev->reset_task_info)); 4933 if (reset_context->job && reset_context->job->vm) 4934 tmp_adev->reset_task_info = 4935 reset_context->job->vm->task_info; 4936 amdgpu_reset_capture_coredumpm(tmp_adev); 4937 #endif 4938 if (vram_lost) { 4939 DRM_INFO("VRAM is lost due to GPU reset!\n"); 4940 amdgpu_inc_vram_lost(tmp_adev); 4941 } 4942 4943 r = amdgpu_device_fw_loading(tmp_adev); 4944 if (r) 4945 return r; 4946 4947 r = amdgpu_device_ip_resume_phase2(tmp_adev); 4948 if (r) 4949 goto out; 4950 4951 if (vram_lost) 4952 amdgpu_device_fill_reset_magic(tmp_adev); 4953 4954 /* 4955 * Add this ASIC as tracked as reset was already 4956 * complete successfully. 4957 */ 4958 amdgpu_register_gpu_instance(tmp_adev); 4959 4960 if (!reset_context->hive && 4961 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 4962 amdgpu_xgmi_add_device(tmp_adev); 4963 4964 r = amdgpu_device_ip_late_init(tmp_adev); 4965 if (r) 4966 goto out; 4967 4968 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false); 4969 4970 /* 4971 * The GPU enters bad state once faulty pages 4972 * by ECC has reached the threshold, and ras 4973 * recovery is scheduled next. So add one check 4974 * here to break recovery if it indeed exceeds 4975 * bad page threshold, and remind user to 4976 * retire this GPU or setting one bigger 4977 * bad_page_threshold value to fix this once 4978 * probing driver again. 4979 */ 4980 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) { 4981 /* must succeed. */ 4982 amdgpu_ras_resume(tmp_adev); 4983 } else { 4984 r = -EINVAL; 4985 goto out; 4986 } 4987 4988 /* Update PSP FW topology after reset */ 4989 if (reset_context->hive && 4990 tmp_adev->gmc.xgmi.num_physical_nodes > 1) 4991 r = amdgpu_xgmi_update_topology( 4992 reset_context->hive, tmp_adev); 4993 } 4994 } 4995 4996 out: 4997 if (!r) { 4998 amdgpu_irq_gpu_reset_resume_helper(tmp_adev); 4999 r = amdgpu_ib_ring_tests(tmp_adev); 5000 if (r) { 5001 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); 5002 need_full_reset = true; 5003 r = -EAGAIN; 5004 goto end; 5005 } 5006 } 5007 5008 if (!r) 5009 r = amdgpu_device_recover_vram(tmp_adev); 5010 else 5011 tmp_adev->asic_reset_res = r; 5012 } 5013 5014 end: 5015 if (need_full_reset) 5016 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5017 else 5018 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5019 return r; 5020 } 5021 5022 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev) 5023 { 5024 5025 switch (amdgpu_asic_reset_method(adev)) { 5026 case AMD_RESET_METHOD_MODE1: 5027 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; 5028 break; 5029 case AMD_RESET_METHOD_MODE2: 5030 adev->mp1_state = PP_MP1_STATE_RESET; 5031 break; 5032 default: 5033 adev->mp1_state = PP_MP1_STATE_NONE; 5034 break; 5035 } 5036 } 5037 5038 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev) 5039 { 5040 amdgpu_vf_error_trans_all(adev); 5041 adev->mp1_state = PP_MP1_STATE_NONE; 5042 } 5043 5044 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) 5045 { 5046 struct pci_dev *p = NULL; 5047 5048 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 5049 adev->pdev->bus->number, 1); 5050 if (p) { 5051 pm_runtime_enable(&(p->dev)); 5052 pm_runtime_resume(&(p->dev)); 5053 } 5054 5055 pci_dev_put(p); 5056 } 5057 5058 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) 5059 { 5060 enum amd_reset_method reset_method; 5061 struct pci_dev *p = NULL; 5062 u64 expires; 5063 5064 /* 5065 * For now, only BACO and mode1 reset are confirmed 5066 * to suffer the audio issue without proper suspended. 5067 */ 5068 reset_method = amdgpu_asic_reset_method(adev); 5069 if ((reset_method != AMD_RESET_METHOD_BACO) && 5070 (reset_method != AMD_RESET_METHOD_MODE1)) 5071 return -EINVAL; 5072 5073 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 5074 adev->pdev->bus->number, 1); 5075 if (!p) 5076 return -ENODEV; 5077 5078 expires = pm_runtime_autosuspend_expiration(&(p->dev)); 5079 if (!expires) 5080 /* 5081 * If we cannot get the audio device autosuspend delay, 5082 * a fixed 4S interval will be used. Considering 3S is 5083 * the audio controller default autosuspend delay setting. 5084 * 4S used here is guaranteed to cover that. 5085 */ 5086 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL; 5087 5088 while (!pm_runtime_status_suspended(&(p->dev))) { 5089 if (!pm_runtime_suspend(&(p->dev))) 5090 break; 5091 5092 if (expires < ktime_get_mono_fast_ns()) { 5093 dev_warn(adev->dev, "failed to suspend display audio\n"); 5094 pci_dev_put(p); 5095 /* TODO: abort the succeeding gpu reset? */ 5096 return -ETIMEDOUT; 5097 } 5098 } 5099 5100 pm_runtime_disable(&(p->dev)); 5101 5102 pci_dev_put(p); 5103 return 0; 5104 } 5105 5106 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) 5107 { 5108 struct amdgpu_ras *con = amdgpu_ras_get_context(adev); 5109 5110 #if defined(CONFIG_DEBUG_FS) 5111 if (!amdgpu_sriov_vf(adev)) 5112 cancel_work(&adev->reset_work); 5113 #endif 5114 5115 if (adev->kfd.dev) 5116 cancel_work(&adev->kfd.reset_work); 5117 5118 if (amdgpu_sriov_vf(adev)) 5119 cancel_work(&adev->virt.flr_work); 5120 5121 if (con && adev->ras_enabled) 5122 cancel_work(&con->recovery_work); 5123 5124 } 5125 5126 /** 5127 * amdgpu_device_gpu_recover - reset the asic and recover scheduler 5128 * 5129 * @adev: amdgpu_device pointer 5130 * @job: which job trigger hang 5131 * 5132 * Attempt to reset the GPU if it has hung (all asics). 5133 * Attempt to do soft-reset or full-reset and reinitialize Asic 5134 * Returns 0 for success or an error on failure. 5135 */ 5136 5137 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, 5138 struct amdgpu_job *job, 5139 struct amdgpu_reset_context *reset_context) 5140 { 5141 struct list_head device_list, *device_list_handle = NULL; 5142 bool job_signaled = false; 5143 struct amdgpu_hive_info *hive = NULL; 5144 struct amdgpu_device *tmp_adev = NULL; 5145 int i, r = 0; 5146 bool need_emergency_restart = false; 5147 bool audio_suspended = false; 5148 bool gpu_reset_for_dev_remove = false; 5149 5150 gpu_reset_for_dev_remove = 5151 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) && 5152 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags); 5153 5154 /* 5155 * Special case: RAS triggered and full reset isn't supported 5156 */ 5157 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev); 5158 5159 /* 5160 * Flush RAM to disk so that after reboot 5161 * the user can read log and see why the system rebooted. 5162 */ 5163 if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) { 5164 DRM_WARN("Emergency reboot."); 5165 5166 ksys_sync_helper(); 5167 emergency_restart(); 5168 } 5169 5170 dev_info(adev->dev, "GPU %s begin!\n", 5171 need_emergency_restart ? "jobs stop":"reset"); 5172 5173 if (!amdgpu_sriov_vf(adev)) 5174 hive = amdgpu_get_xgmi_hive(adev); 5175 if (hive) 5176 mutex_lock(&hive->hive_lock); 5177 5178 reset_context->job = job; 5179 reset_context->hive = hive; 5180 /* 5181 * Build list of devices to reset. 5182 * In case we are in XGMI hive mode, resort the device list 5183 * to put adev in the 1st position. 5184 */ 5185 INIT_LIST_HEAD(&device_list); 5186 if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) { 5187 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 5188 list_add_tail(&tmp_adev->reset_list, &device_list); 5189 if (gpu_reset_for_dev_remove && adev->shutdown) 5190 tmp_adev->shutdown = true; 5191 } 5192 if (!list_is_first(&adev->reset_list, &device_list)) 5193 list_rotate_to_front(&adev->reset_list, &device_list); 5194 device_list_handle = &device_list; 5195 } else { 5196 list_add_tail(&adev->reset_list, &device_list); 5197 device_list_handle = &device_list; 5198 } 5199 5200 /* We need to lock reset domain only once both for XGMI and single device */ 5201 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5202 reset_list); 5203 amdgpu_device_lock_reset_domain(tmp_adev->reset_domain); 5204 5205 /* block all schedulers and reset given job's ring */ 5206 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5207 5208 amdgpu_device_set_mp1_state(tmp_adev); 5209 5210 /* 5211 * Try to put the audio codec into suspend state 5212 * before gpu reset started. 5213 * 5214 * Due to the power domain of the graphics device 5215 * is shared with AZ power domain. Without this, 5216 * we may change the audio hardware from behind 5217 * the audio driver's back. That will trigger 5218 * some audio codec errors. 5219 */ 5220 if (!amdgpu_device_suspend_display_audio(tmp_adev)) 5221 audio_suspended = true; 5222 5223 amdgpu_ras_set_error_query_ready(tmp_adev, false); 5224 5225 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); 5226 5227 if (!amdgpu_sriov_vf(tmp_adev)) 5228 amdgpu_amdkfd_pre_reset(tmp_adev); 5229 5230 /* 5231 * Mark these ASICs to be reseted as untracked first 5232 * And add them back after reset completed 5233 */ 5234 amdgpu_unregister_gpu_instance(tmp_adev); 5235 5236 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true); 5237 5238 /* disable ras on ALL IPs */ 5239 if (!need_emergency_restart && 5240 amdgpu_device_ip_need_full_reset(tmp_adev)) 5241 amdgpu_ras_suspend(tmp_adev); 5242 5243 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5244 struct amdgpu_ring *ring = tmp_adev->rings[i]; 5245 5246 if (!ring || !ring->sched.thread) 5247 continue; 5248 5249 drm_sched_stop(&ring->sched, job ? &job->base : NULL); 5250 5251 if (need_emergency_restart) 5252 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); 5253 } 5254 atomic_inc(&tmp_adev->gpu_reset_counter); 5255 } 5256 5257 if (need_emergency_restart) 5258 goto skip_sched_resume; 5259 5260 /* 5261 * Must check guilty signal here since after this point all old 5262 * HW fences are force signaled. 5263 * 5264 * job->base holds a reference to parent fence 5265 */ 5266 if (job && dma_fence_is_signaled(&job->hw_fence)) { 5267 job_signaled = true; 5268 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); 5269 goto skip_hw_reset; 5270 } 5271 5272 retry: /* Rest of adevs pre asic reset from XGMI hive. */ 5273 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5274 if (gpu_reset_for_dev_remove) { 5275 /* Workaroud for ASICs need to disable SMC first */ 5276 amdgpu_device_smu_fini_early(tmp_adev); 5277 } 5278 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context); 5279 /*TODO Should we stop ?*/ 5280 if (r) { 5281 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", 5282 r, adev_to_drm(tmp_adev)->unique); 5283 tmp_adev->asic_reset_res = r; 5284 } 5285 5286 /* 5287 * Drop all pending non scheduler resets. Scheduler resets 5288 * were already dropped during drm_sched_stop 5289 */ 5290 amdgpu_device_stop_pending_resets(tmp_adev); 5291 } 5292 5293 /* Actual ASIC resets if needed.*/ 5294 /* Host driver will handle XGMI hive reset for SRIOV */ 5295 if (amdgpu_sriov_vf(adev)) { 5296 r = amdgpu_device_reset_sriov(adev, job ? false : true); 5297 if (r) 5298 adev->asic_reset_res = r; 5299 5300 /* Aldebaran supports ras in SRIOV, so need resume ras during reset */ 5301 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2)) 5302 amdgpu_ras_resume(adev); 5303 } else { 5304 r = amdgpu_do_asic_reset(device_list_handle, reset_context); 5305 if (r && r == -EAGAIN) 5306 goto retry; 5307 5308 if (!r && gpu_reset_for_dev_remove) 5309 goto recover_end; 5310 } 5311 5312 skip_hw_reset: 5313 5314 /* Post ASIC reset for all devs .*/ 5315 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5316 5317 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5318 struct amdgpu_ring *ring = tmp_adev->rings[i]; 5319 5320 if (!ring || !ring->sched.thread) 5321 continue; 5322 5323 drm_sched_start(&ring->sched, true); 5324 } 5325 5326 if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3)) 5327 amdgpu_mes_self_test(tmp_adev); 5328 5329 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) { 5330 drm_helper_resume_force_mode(adev_to_drm(tmp_adev)); 5331 } 5332 5333 if (tmp_adev->asic_reset_res) 5334 r = tmp_adev->asic_reset_res; 5335 5336 tmp_adev->asic_reset_res = 0; 5337 5338 if (r) { 5339 /* bad news, how to tell it to userspace ? */ 5340 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter)); 5341 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); 5342 } else { 5343 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter)); 5344 if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0)) 5345 DRM_WARN("smart shift update failed\n"); 5346 } 5347 } 5348 5349 skip_sched_resume: 5350 list_for_each_entry(tmp_adev, device_list_handle, reset_list) { 5351 /* unlock kfd: SRIOV would do it separately */ 5352 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev)) 5353 amdgpu_amdkfd_post_reset(tmp_adev); 5354 5355 /* kfd_post_reset will do nothing if kfd device is not initialized, 5356 * need to bring up kfd here if it's not be initialized before 5357 */ 5358 if (!adev->kfd.init_complete) 5359 amdgpu_amdkfd_device_init(adev); 5360 5361 if (audio_suspended) 5362 amdgpu_device_resume_display_audio(tmp_adev); 5363 5364 amdgpu_device_unset_mp1_state(tmp_adev); 5365 5366 amdgpu_ras_set_error_query_ready(tmp_adev, true); 5367 } 5368 5369 recover_end: 5370 tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device, 5371 reset_list); 5372 amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain); 5373 5374 if (hive) { 5375 mutex_unlock(&hive->hive_lock); 5376 amdgpu_put_xgmi_hive(hive); 5377 } 5378 5379 if (r) 5380 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); 5381 5382 atomic_set(&adev->reset_domain->reset_res, r); 5383 return r; 5384 } 5385 5386 /** 5387 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot 5388 * 5389 * @adev: amdgpu_device pointer 5390 * 5391 * Fetchs and stores in the driver the PCIE capabilities (gen speed 5392 * and lanes) of the slot the device is in. Handles APUs and 5393 * virtualized environments where PCIE config space may not be available. 5394 */ 5395 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) 5396 { 5397 struct pci_dev *pdev; 5398 enum pci_bus_speed speed_cap, platform_speed_cap; 5399 enum pcie_link_width platform_link_width; 5400 5401 if (amdgpu_pcie_gen_cap) 5402 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; 5403 5404 if (amdgpu_pcie_lane_cap) 5405 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; 5406 5407 /* covers APUs as well */ 5408 if (pci_is_root_bus(adev->pdev->bus)) { 5409 if (adev->pm.pcie_gen_mask == 0) 5410 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; 5411 if (adev->pm.pcie_mlw_mask == 0) 5412 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; 5413 return; 5414 } 5415 5416 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) 5417 return; 5418 5419 pcie_bandwidth_available(adev->pdev, NULL, 5420 &platform_speed_cap, &platform_link_width); 5421 5422 if (adev->pm.pcie_gen_mask == 0) { 5423 /* asic caps */ 5424 pdev = adev->pdev; 5425 speed_cap = pcie_get_speed_cap(pdev); 5426 if (speed_cap == PCI_SPEED_UNKNOWN) { 5427 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5428 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5429 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 5430 } else { 5431 if (speed_cap == PCIE_SPEED_32_0GT) 5432 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5433 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5434 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5435 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 | 5436 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5); 5437 else if (speed_cap == PCIE_SPEED_16_0GT) 5438 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5439 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5440 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5441 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); 5442 else if (speed_cap == PCIE_SPEED_8_0GT) 5443 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5444 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5445 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 5446 else if (speed_cap == PCIE_SPEED_5_0GT) 5447 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5448 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); 5449 else 5450 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; 5451 } 5452 /* platform caps */ 5453 if (platform_speed_cap == PCI_SPEED_UNKNOWN) { 5454 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5455 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 5456 } else { 5457 if (platform_speed_cap == PCIE_SPEED_32_0GT) 5458 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5459 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5460 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5461 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 | 5462 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5); 5463 else if (platform_speed_cap == PCIE_SPEED_16_0GT) 5464 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5465 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5466 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 5467 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); 5468 else if (platform_speed_cap == PCIE_SPEED_8_0GT) 5469 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5470 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 5471 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); 5472 else if (platform_speed_cap == PCIE_SPEED_5_0GT) 5473 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 5474 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 5475 else 5476 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; 5477 5478 } 5479 } 5480 if (adev->pm.pcie_mlw_mask == 0) { 5481 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { 5482 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; 5483 } else { 5484 switch (platform_link_width) { 5485 case PCIE_LNK_X32: 5486 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | 5487 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 5488 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 5489 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 5490 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5491 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5492 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5493 break; 5494 case PCIE_LNK_X16: 5495 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 5496 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 5497 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 5498 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5499 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5500 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5501 break; 5502 case PCIE_LNK_X12: 5503 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 5504 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 5505 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5506 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5507 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5508 break; 5509 case PCIE_LNK_X8: 5510 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 5511 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5512 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5513 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5514 break; 5515 case PCIE_LNK_X4: 5516 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 5517 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5518 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5519 break; 5520 case PCIE_LNK_X2: 5521 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 5522 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 5523 break; 5524 case PCIE_LNK_X1: 5525 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; 5526 break; 5527 default: 5528 break; 5529 } 5530 } 5531 } 5532 } 5533 5534 /** 5535 * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR 5536 * 5537 * @adev: amdgpu_device pointer 5538 * @peer_adev: amdgpu_device pointer for peer device trying to access @adev 5539 * 5540 * Return true if @peer_adev can access (DMA) @adev through the PCIe 5541 * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of 5542 * @peer_adev. 5543 */ 5544 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev, 5545 struct amdgpu_device *peer_adev) 5546 { 5547 #ifdef CONFIG_HSA_AMD_P2P 5548 uint64_t address_mask = peer_adev->dev->dma_mask ? 5549 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1); 5550 resource_size_t aper_limit = 5551 adev->gmc.aper_base + adev->gmc.aper_size - 1; 5552 bool p2p_access = 5553 !adev->gmc.xgmi.connected_to_cpu && 5554 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0); 5555 5556 return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size && 5557 adev->gmc.real_vram_size == adev->gmc.visible_vram_size && 5558 !(adev->gmc.aper_base & address_mask || 5559 aper_limit & address_mask)); 5560 #else 5561 return false; 5562 #endif 5563 } 5564 5565 int amdgpu_device_baco_enter(struct drm_device *dev) 5566 { 5567 struct amdgpu_device *adev = drm_to_adev(dev); 5568 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5569 5570 if (!amdgpu_device_supports_baco(adev_to_drm(adev))) 5571 return -ENOTSUPP; 5572 5573 if (ras && adev->ras_enabled && 5574 adev->nbio.funcs->enable_doorbell_interrupt) 5575 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); 5576 5577 return amdgpu_dpm_baco_enter(adev); 5578 } 5579 5580 int amdgpu_device_baco_exit(struct drm_device *dev) 5581 { 5582 struct amdgpu_device *adev = drm_to_adev(dev); 5583 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 5584 int ret = 0; 5585 5586 if (!amdgpu_device_supports_baco(adev_to_drm(adev))) 5587 return -ENOTSUPP; 5588 5589 ret = amdgpu_dpm_baco_exit(adev); 5590 if (ret) 5591 return ret; 5592 5593 if (ras && adev->ras_enabled && 5594 adev->nbio.funcs->enable_doorbell_interrupt) 5595 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); 5596 5597 if (amdgpu_passthrough(adev) && 5598 adev->nbio.funcs->clear_doorbell_interrupt) 5599 adev->nbio.funcs->clear_doorbell_interrupt(adev); 5600 5601 return 0; 5602 } 5603 5604 /** 5605 * amdgpu_pci_error_detected - Called when a PCI error is detected. 5606 * @pdev: PCI device struct 5607 * @state: PCI channel state 5608 * 5609 * Description: Called when a PCI error is detected. 5610 * 5611 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT. 5612 */ 5613 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) 5614 { 5615 struct drm_device *dev = pci_get_drvdata(pdev); 5616 struct amdgpu_device *adev = drm_to_adev(dev); 5617 int i; 5618 5619 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state); 5620 5621 if (adev->gmc.xgmi.num_physical_nodes > 1) { 5622 DRM_WARN("No support for XGMI hive yet..."); 5623 return PCI_ERS_RESULT_DISCONNECT; 5624 } 5625 5626 adev->pci_channel_state = state; 5627 5628 switch (state) { 5629 case pci_channel_io_normal: 5630 return PCI_ERS_RESULT_CAN_RECOVER; 5631 /* Fatal error, prepare for slot reset */ 5632 case pci_channel_io_frozen: 5633 /* 5634 * Locking adev->reset_domain->sem will prevent any external access 5635 * to GPU during PCI error recovery 5636 */ 5637 amdgpu_device_lock_reset_domain(adev->reset_domain); 5638 amdgpu_device_set_mp1_state(adev); 5639 5640 /* 5641 * Block any work scheduling as we do for regular GPU reset 5642 * for the duration of the recovery 5643 */ 5644 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5645 struct amdgpu_ring *ring = adev->rings[i]; 5646 5647 if (!ring || !ring->sched.thread) 5648 continue; 5649 5650 drm_sched_stop(&ring->sched, NULL); 5651 } 5652 atomic_inc(&adev->gpu_reset_counter); 5653 return PCI_ERS_RESULT_NEED_RESET; 5654 case pci_channel_io_perm_failure: 5655 /* Permanent error, prepare for device removal */ 5656 return PCI_ERS_RESULT_DISCONNECT; 5657 } 5658 5659 return PCI_ERS_RESULT_NEED_RESET; 5660 } 5661 5662 /** 5663 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers 5664 * @pdev: pointer to PCI device 5665 */ 5666 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev) 5667 { 5668 5669 DRM_INFO("PCI error: mmio enabled callback!!\n"); 5670 5671 /* TODO - dump whatever for debugging purposes */ 5672 5673 /* This called only if amdgpu_pci_error_detected returns 5674 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still 5675 * works, no need to reset slot. 5676 */ 5677 5678 return PCI_ERS_RESULT_RECOVERED; 5679 } 5680 5681 /** 5682 * amdgpu_pci_slot_reset - Called when PCI slot has been reset. 5683 * @pdev: PCI device struct 5684 * 5685 * Description: This routine is called by the pci error recovery 5686 * code after the PCI slot has been reset, just before we 5687 * should resume normal operations. 5688 */ 5689 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) 5690 { 5691 struct drm_device *dev = pci_get_drvdata(pdev); 5692 struct amdgpu_device *adev = drm_to_adev(dev); 5693 int r, i; 5694 struct amdgpu_reset_context reset_context; 5695 u32 memsize; 5696 struct list_head device_list; 5697 5698 DRM_INFO("PCI error: slot reset callback!!\n"); 5699 5700 memset(&reset_context, 0, sizeof(reset_context)); 5701 5702 INIT_LIST_HEAD(&device_list); 5703 list_add_tail(&adev->reset_list, &device_list); 5704 5705 /* wait for asic to come out of reset */ 5706 msleep(500); 5707 5708 /* Restore PCI confspace */ 5709 amdgpu_device_load_pci_state(pdev); 5710 5711 /* confirm ASIC came out of reset */ 5712 for (i = 0; i < adev->usec_timeout; i++) { 5713 memsize = amdgpu_asic_get_config_memsize(adev); 5714 5715 if (memsize != 0xffffffff) 5716 break; 5717 udelay(1); 5718 } 5719 if (memsize == 0xffffffff) { 5720 r = -ETIME; 5721 goto out; 5722 } 5723 5724 reset_context.method = AMD_RESET_METHOD_NONE; 5725 reset_context.reset_req_dev = adev; 5726 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); 5727 set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags); 5728 5729 adev->no_hw_access = true; 5730 r = amdgpu_device_pre_asic_reset(adev, &reset_context); 5731 adev->no_hw_access = false; 5732 if (r) 5733 goto out; 5734 5735 r = amdgpu_do_asic_reset(&device_list, &reset_context); 5736 5737 out: 5738 if (!r) { 5739 if (amdgpu_device_cache_pci_state(adev->pdev)) 5740 pci_restore_state(adev->pdev); 5741 5742 DRM_INFO("PCIe error recovery succeeded\n"); 5743 } else { 5744 DRM_ERROR("PCIe error recovery failed, err:%d", r); 5745 amdgpu_device_unset_mp1_state(adev); 5746 amdgpu_device_unlock_reset_domain(adev->reset_domain); 5747 } 5748 5749 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 5750 } 5751 5752 /** 5753 * amdgpu_pci_resume() - resume normal ops after PCI reset 5754 * @pdev: pointer to PCI device 5755 * 5756 * Called when the error recovery driver tells us that its 5757 * OK to resume normal operation. 5758 */ 5759 void amdgpu_pci_resume(struct pci_dev *pdev) 5760 { 5761 struct drm_device *dev = pci_get_drvdata(pdev); 5762 struct amdgpu_device *adev = drm_to_adev(dev); 5763 int i; 5764 5765 5766 DRM_INFO("PCI error: resume callback!!\n"); 5767 5768 /* Only continue execution for the case of pci_channel_io_frozen */ 5769 if (adev->pci_channel_state != pci_channel_io_frozen) 5770 return; 5771 5772 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5773 struct amdgpu_ring *ring = adev->rings[i]; 5774 5775 if (!ring || !ring->sched.thread) 5776 continue; 5777 5778 drm_sched_start(&ring->sched, true); 5779 } 5780 5781 amdgpu_device_unset_mp1_state(adev); 5782 amdgpu_device_unlock_reset_domain(adev->reset_domain); 5783 } 5784 5785 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev) 5786 { 5787 struct drm_device *dev = pci_get_drvdata(pdev); 5788 struct amdgpu_device *adev = drm_to_adev(dev); 5789 int r; 5790 5791 r = pci_save_state(pdev); 5792 if (!r) { 5793 kfree(adev->pci_state); 5794 5795 adev->pci_state = pci_store_saved_state(pdev); 5796 5797 if (!adev->pci_state) { 5798 DRM_ERROR("Failed to store PCI saved state"); 5799 return false; 5800 } 5801 } else { 5802 DRM_WARN("Failed to save PCI state, err:%d\n", r); 5803 return false; 5804 } 5805 5806 return true; 5807 } 5808 5809 bool amdgpu_device_load_pci_state(struct pci_dev *pdev) 5810 { 5811 struct drm_device *dev = pci_get_drvdata(pdev); 5812 struct amdgpu_device *adev = drm_to_adev(dev); 5813 int r; 5814 5815 if (!adev->pci_state) 5816 return false; 5817 5818 r = pci_load_saved_state(pdev, adev->pci_state); 5819 5820 if (!r) { 5821 pci_restore_state(pdev); 5822 } else { 5823 DRM_WARN("Failed to load PCI state, err:%d\n", r); 5824 return false; 5825 } 5826 5827 return true; 5828 } 5829 5830 void amdgpu_device_flush_hdp(struct amdgpu_device *adev, 5831 struct amdgpu_ring *ring) 5832 { 5833 #ifdef CONFIG_X86_64 5834 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 5835 return; 5836 #endif 5837 if (adev->gmc.xgmi.connected_to_cpu) 5838 return; 5839 5840 if (ring && ring->funcs->emit_hdp_flush) 5841 amdgpu_ring_emit_hdp_flush(ring); 5842 else 5843 amdgpu_asic_flush_hdp(adev, ring); 5844 } 5845 5846 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev, 5847 struct amdgpu_ring *ring) 5848 { 5849 #ifdef CONFIG_X86_64 5850 if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev)) 5851 return; 5852 #endif 5853 if (adev->gmc.xgmi.connected_to_cpu) 5854 return; 5855 5856 amdgpu_asic_invalidate_hdp(adev, ring); 5857 } 5858 5859 int amdgpu_in_reset(struct amdgpu_device *adev) 5860 { 5861 return atomic_read(&adev->reset_domain->in_gpu_reset); 5862 } 5863 5864 /** 5865 * amdgpu_device_halt() - bring hardware to some kind of halt state 5866 * 5867 * @adev: amdgpu_device pointer 5868 * 5869 * Bring hardware to some kind of halt state so that no one can touch it 5870 * any more. It will help to maintain error context when error occurred. 5871 * Compare to a simple hang, the system will keep stable at least for SSH 5872 * access. Then it should be trivial to inspect the hardware state and 5873 * see what's going on. Implemented as following: 5874 * 5875 * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc), 5876 * clears all CPU mappings to device, disallows remappings through page faults 5877 * 2. amdgpu_irq_disable_all() disables all interrupts 5878 * 3. amdgpu_fence_driver_hw_fini() signals all HW fences 5879 * 4. set adev->no_hw_access to avoid potential crashes after setp 5 5880 * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings 5881 * 6. pci_disable_device() and pci_wait_for_pending_transaction() 5882 * flush any in flight DMA operations 5883 */ 5884 void amdgpu_device_halt(struct amdgpu_device *adev) 5885 { 5886 struct pci_dev *pdev = adev->pdev; 5887 struct drm_device *ddev = adev_to_drm(adev); 5888 5889 drm_dev_unplug(ddev); 5890 5891 amdgpu_irq_disable_all(adev); 5892 5893 amdgpu_fence_driver_hw_fini(adev); 5894 5895 adev->no_hw_access = true; 5896 5897 amdgpu_device_unmap_mmio(adev); 5898 5899 pci_disable_device(pdev); 5900 pci_wait_for_pending_transaction(pdev); 5901 } 5902 5903 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev, 5904 u32 reg) 5905 { 5906 unsigned long flags, address, data; 5907 u32 r; 5908 5909 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 5910 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 5911 5912 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 5913 WREG32(address, reg * 4); 5914 (void)RREG32(address); 5915 r = RREG32(data); 5916 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 5917 return r; 5918 } 5919 5920 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev, 5921 u32 reg, u32 v) 5922 { 5923 unsigned long flags, address, data; 5924 5925 address = adev->nbio.funcs->get_pcie_port_index_offset(adev); 5926 data = adev->nbio.funcs->get_pcie_port_data_offset(adev); 5927 5928 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 5929 WREG32(address, reg * 4); 5930 (void)RREG32(address); 5931 WREG32(data, v); 5932 (void)RREG32(data); 5933 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 5934 } 5935 5936 /** 5937 * amdgpu_device_switch_gang - switch to a new gang 5938 * @adev: amdgpu_device pointer 5939 * @gang: the gang to switch to 5940 * 5941 * Try to switch to a new gang. 5942 * Returns: NULL if we switched to the new gang or a reference to the current 5943 * gang leader. 5944 */ 5945 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev, 5946 struct dma_fence *gang) 5947 { 5948 struct dma_fence *old = NULL; 5949 5950 do { 5951 dma_fence_put(old); 5952 rcu_read_lock(); 5953 old = dma_fence_get_rcu_safe(&adev->gang_submit); 5954 rcu_read_unlock(); 5955 5956 if (old == gang) 5957 break; 5958 5959 if (!dma_fence_is_signaled(old)) 5960 return old; 5961 5962 } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit, 5963 old, gang) != old); 5964 5965 dma_fence_put(old); 5966 return NULL; 5967 } 5968 5969 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev) 5970 { 5971 switch (adev->asic_type) { 5972 #ifdef CONFIG_DRM_AMDGPU_SI 5973 case CHIP_HAINAN: 5974 #endif 5975 case CHIP_TOPAZ: 5976 /* chips with no display hardware */ 5977 return false; 5978 #ifdef CONFIG_DRM_AMDGPU_SI 5979 case CHIP_TAHITI: 5980 case CHIP_PITCAIRN: 5981 case CHIP_VERDE: 5982 case CHIP_OLAND: 5983 #endif 5984 #ifdef CONFIG_DRM_AMDGPU_CIK 5985 case CHIP_BONAIRE: 5986 case CHIP_HAWAII: 5987 case CHIP_KAVERI: 5988 case CHIP_KABINI: 5989 case CHIP_MULLINS: 5990 #endif 5991 case CHIP_TONGA: 5992 case CHIP_FIJI: 5993 case CHIP_POLARIS10: 5994 case CHIP_POLARIS11: 5995 case CHIP_POLARIS12: 5996 case CHIP_VEGAM: 5997 case CHIP_CARRIZO: 5998 case CHIP_STONEY: 5999 /* chips with display hardware */ 6000 return true; 6001 default: 6002 /* IP discovery */ 6003 if (!adev->ip_versions[DCE_HWIP][0] || 6004 (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK)) 6005 return false; 6006 return true; 6007 } 6008 } 6009