1 /* 2 * Copyright 2008 Advanced Micro Devices, Inc. 3 * Copyright 2008 Red Hat Inc. 4 * Copyright 2009 Jerome Glisse. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 * 24 * Authors: Dave Airlie 25 * Alex Deucher 26 * Jerome Glisse 27 */ 28 #include <linux/power_supply.h> 29 #include <linux/kthread.h> 30 #include <linux/module.h> 31 #include <linux/console.h> 32 #include <linux/slab.h> 33 34 #include <drm/drm_atomic_helper.h> 35 #include <drm/drm_probe_helper.h> 36 #include <drm/amdgpu_drm.h> 37 #include <linux/vgaarb.h> 38 #include <linux/vga_switcheroo.h> 39 #include <linux/efi.h> 40 #include "amdgpu.h" 41 #include "amdgpu_trace.h" 42 #include "amdgpu_i2c.h" 43 #include "atom.h" 44 #include "amdgpu_atombios.h" 45 #include "amdgpu_atomfirmware.h" 46 #include "amd_pcie.h" 47 #ifdef CONFIG_DRM_AMDGPU_SI 48 #include "si.h" 49 #endif 50 #ifdef CONFIG_DRM_AMDGPU_CIK 51 #include "cik.h" 52 #endif 53 #include "vi.h" 54 #include "soc15.h" 55 #include "nv.h" 56 #include "bif/bif_4_1_d.h" 57 #include <linux/pci.h> 58 #include <linux/firmware.h> 59 #include "amdgpu_vf_error.h" 60 61 #include "amdgpu_amdkfd.h" 62 #include "amdgpu_pm.h" 63 64 #include "amdgpu_xgmi.h" 65 #include "amdgpu_ras.h" 66 #include "amdgpu_pmu.h" 67 #include "amdgpu_fru_eeprom.h" 68 69 #include <linux/suspend.h> 70 #include <drm/task_barrier.h> 71 #include <linux/pm_runtime.h> 72 73 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin"); 74 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin"); 75 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin"); 76 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin"); 77 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin"); 78 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin"); 79 MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin"); 80 MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin"); 81 MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin"); 82 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin"); 83 MODULE_FIRMWARE("amdgpu/vangogh_gpu_info.bin"); 84 MODULE_FIRMWARE("amdgpu/green_sardine_gpu_info.bin"); 85 86 #define AMDGPU_RESUME_MS 2000 87 88 const char *amdgpu_asic_name[] = { 89 "TAHITI", 90 "PITCAIRN", 91 "VERDE", 92 "OLAND", 93 "HAINAN", 94 "BONAIRE", 95 "KAVERI", 96 "KABINI", 97 "HAWAII", 98 "MULLINS", 99 "TOPAZ", 100 "TONGA", 101 "FIJI", 102 "CARRIZO", 103 "STONEY", 104 "POLARIS10", 105 "POLARIS11", 106 "POLARIS12", 107 "VEGAM", 108 "VEGA10", 109 "VEGA12", 110 "VEGA20", 111 "RAVEN", 112 "ARCTURUS", 113 "RENOIR", 114 "NAVI10", 115 "NAVI14", 116 "NAVI12", 117 "SIENNA_CICHLID", 118 "NAVY_FLOUNDER", 119 "VANGOGH", 120 "DIMGREY_CAVEFISH", 121 "LAST", 122 }; 123 124 /** 125 * DOC: pcie_replay_count 126 * 127 * The amdgpu driver provides a sysfs API for reporting the total number 128 * of PCIe replays (NAKs) 129 * The file pcie_replay_count is used for this and returns the total 130 * number of replays as a sum of the NAKs generated and NAKs received 131 */ 132 133 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev, 134 struct device_attribute *attr, char *buf) 135 { 136 struct drm_device *ddev = dev_get_drvdata(dev); 137 struct amdgpu_device *adev = drm_to_adev(ddev); 138 uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev); 139 140 return snprintf(buf, PAGE_SIZE, "%llu\n", cnt); 141 } 142 143 static DEVICE_ATTR(pcie_replay_count, S_IRUGO, 144 amdgpu_device_get_pcie_replay_count, NULL); 145 146 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev); 147 148 /** 149 * DOC: product_name 150 * 151 * The amdgpu driver provides a sysfs API for reporting the product name 152 * for the device 153 * The file serial_number is used for this and returns the product name 154 * as returned from the FRU. 155 * NOTE: This is only available for certain server cards 156 */ 157 158 static ssize_t amdgpu_device_get_product_name(struct device *dev, 159 struct device_attribute *attr, char *buf) 160 { 161 struct drm_device *ddev = dev_get_drvdata(dev); 162 struct amdgpu_device *adev = drm_to_adev(ddev); 163 164 return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_name); 165 } 166 167 static DEVICE_ATTR(product_name, S_IRUGO, 168 amdgpu_device_get_product_name, NULL); 169 170 /** 171 * DOC: product_number 172 * 173 * The amdgpu driver provides a sysfs API for reporting the part number 174 * for the device 175 * The file serial_number is used for this and returns the part number 176 * as returned from the FRU. 177 * NOTE: This is only available for certain server cards 178 */ 179 180 static ssize_t amdgpu_device_get_product_number(struct device *dev, 181 struct device_attribute *attr, char *buf) 182 { 183 struct drm_device *ddev = dev_get_drvdata(dev); 184 struct amdgpu_device *adev = drm_to_adev(ddev); 185 186 return snprintf(buf, PAGE_SIZE, "%s\n", adev->product_number); 187 } 188 189 static DEVICE_ATTR(product_number, S_IRUGO, 190 amdgpu_device_get_product_number, NULL); 191 192 /** 193 * DOC: serial_number 194 * 195 * The amdgpu driver provides a sysfs API for reporting the serial number 196 * for the device 197 * The file serial_number is used for this and returns the serial number 198 * as returned from the FRU. 199 * NOTE: This is only available for certain server cards 200 */ 201 202 static ssize_t amdgpu_device_get_serial_number(struct device *dev, 203 struct device_attribute *attr, char *buf) 204 { 205 struct drm_device *ddev = dev_get_drvdata(dev); 206 struct amdgpu_device *adev = drm_to_adev(ddev); 207 208 return snprintf(buf, PAGE_SIZE, "%s\n", adev->serial); 209 } 210 211 static DEVICE_ATTR(serial_number, S_IRUGO, 212 amdgpu_device_get_serial_number, NULL); 213 214 /** 215 * amdgpu_device_supports_boco - Is the device a dGPU with HG/PX power control 216 * 217 * @dev: drm_device pointer 218 * 219 * Returns true if the device is a dGPU with HG/PX power control, 220 * otherwise return false. 221 */ 222 bool amdgpu_device_supports_boco(struct drm_device *dev) 223 { 224 struct amdgpu_device *adev = drm_to_adev(dev); 225 226 if (adev->flags & AMD_IS_PX) 227 return true; 228 return false; 229 } 230 231 /** 232 * amdgpu_device_supports_baco - Does the device support BACO 233 * 234 * @dev: drm_device pointer 235 * 236 * Returns true if the device supporte BACO, 237 * otherwise return false. 238 */ 239 bool amdgpu_device_supports_baco(struct drm_device *dev) 240 { 241 struct amdgpu_device *adev = drm_to_adev(dev); 242 243 return amdgpu_asic_supports_baco(adev); 244 } 245 246 /** 247 * VRAM access helper functions. 248 * 249 * amdgpu_device_vram_access - read/write a buffer in vram 250 * 251 * @adev: amdgpu_device pointer 252 * @pos: offset of the buffer in vram 253 * @buf: virtual address of the buffer in system memory 254 * @size: read/write size, sizeof(@buf) must > @size 255 * @write: true - write to vram, otherwise - read from vram 256 */ 257 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos, 258 uint32_t *buf, size_t size, bool write) 259 { 260 unsigned long flags; 261 uint32_t hi = ~0; 262 uint64_t last; 263 264 265 #ifdef CONFIG_64BIT 266 last = min(pos + size, adev->gmc.visible_vram_size); 267 if (last > pos) { 268 void __iomem *addr = adev->mman.aper_base_kaddr + pos; 269 size_t count = last - pos; 270 271 if (write) { 272 memcpy_toio(addr, buf, count); 273 mb(); 274 amdgpu_asic_flush_hdp(adev, NULL); 275 } else { 276 amdgpu_asic_invalidate_hdp(adev, NULL); 277 mb(); 278 memcpy_fromio(buf, addr, count); 279 } 280 281 if (count == size) 282 return; 283 284 pos += count; 285 buf += count / 4; 286 size -= count; 287 } 288 #endif 289 290 spin_lock_irqsave(&adev->mmio_idx_lock, flags); 291 for (last = pos + size; pos < last; pos += 4) { 292 uint32_t tmp = pos >> 31; 293 294 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000); 295 if (tmp != hi) { 296 WREG32_NO_KIQ(mmMM_INDEX_HI, tmp); 297 hi = tmp; 298 } 299 if (write) 300 WREG32_NO_KIQ(mmMM_DATA, *buf++); 301 else 302 *buf++ = RREG32_NO_KIQ(mmMM_DATA); 303 } 304 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags); 305 } 306 307 /* 308 * register access helper functions. 309 */ 310 /** 311 * amdgpu_device_rreg - read a memory mapped IO or indirect register 312 * 313 * @adev: amdgpu_device pointer 314 * @reg: dword aligned register offset 315 * @acc_flags: access flags which require special behavior 316 * 317 * Returns the 32 bit value from the offset specified. 318 */ 319 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev, 320 uint32_t reg, uint32_t acc_flags) 321 { 322 uint32_t ret; 323 324 if (adev->in_pci_err_recovery) 325 return 0; 326 327 if ((reg * 4) < adev->rmmio_size) { 328 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 329 amdgpu_sriov_runtime(adev) && 330 down_read_trylock(&adev->reset_sem)) { 331 ret = amdgpu_kiq_rreg(adev, reg); 332 up_read(&adev->reset_sem); 333 } else { 334 ret = readl(((void __iomem *)adev->rmmio) + (reg * 4)); 335 } 336 } else { 337 ret = adev->pcie_rreg(adev, reg * 4); 338 } 339 340 trace_amdgpu_device_rreg(adev->pdev->device, reg, ret); 341 342 return ret; 343 } 344 345 /* 346 * MMIO register read with bytes helper functions 347 * @offset:bytes offset from MMIO start 348 * 349 */ 350 351 /** 352 * amdgpu_mm_rreg8 - read a memory mapped IO register 353 * 354 * @adev: amdgpu_device pointer 355 * @offset: byte aligned register offset 356 * 357 * Returns the 8 bit value from the offset specified. 358 */ 359 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset) 360 { 361 if (adev->in_pci_err_recovery) 362 return 0; 363 364 if (offset < adev->rmmio_size) 365 return (readb(adev->rmmio + offset)); 366 BUG(); 367 } 368 369 /* 370 * MMIO register write with bytes helper functions 371 * @offset:bytes offset from MMIO start 372 * @value: the value want to be written to the register 373 * 374 */ 375 /** 376 * amdgpu_mm_wreg8 - read a memory mapped IO register 377 * 378 * @adev: amdgpu_device pointer 379 * @offset: byte aligned register offset 380 * @value: 8 bit value to write 381 * 382 * Writes the value specified to the offset specified. 383 */ 384 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value) 385 { 386 if (adev->in_pci_err_recovery) 387 return; 388 389 if (offset < adev->rmmio_size) 390 writeb(value, adev->rmmio + offset); 391 else 392 BUG(); 393 } 394 395 /** 396 * amdgpu_device_wreg - write to a memory mapped IO or indirect register 397 * 398 * @adev: amdgpu_device pointer 399 * @reg: dword aligned register offset 400 * @v: 32 bit value to write to the register 401 * @acc_flags: access flags which require special behavior 402 * 403 * Writes the value specified to the offset specified. 404 */ 405 void amdgpu_device_wreg(struct amdgpu_device *adev, 406 uint32_t reg, uint32_t v, 407 uint32_t acc_flags) 408 { 409 if (adev->in_pci_err_recovery) 410 return; 411 412 if ((reg * 4) < adev->rmmio_size) { 413 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) && 414 amdgpu_sriov_runtime(adev) && 415 down_read_trylock(&adev->reset_sem)) { 416 amdgpu_kiq_wreg(adev, reg, v); 417 up_read(&adev->reset_sem); 418 } else { 419 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 420 } 421 } else { 422 adev->pcie_wreg(adev, reg * 4, v); 423 } 424 425 trace_amdgpu_device_wreg(adev->pdev->device, reg, v); 426 } 427 428 /* 429 * amdgpu_mm_wreg_mmio_rlc - write register either with mmio or with RLC path if in range 430 * 431 * this function is invoked only the debugfs register access 432 * */ 433 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev, 434 uint32_t reg, uint32_t v) 435 { 436 if (adev->in_pci_err_recovery) 437 return; 438 439 if (amdgpu_sriov_fullaccess(adev) && 440 adev->gfx.rlc.funcs && 441 adev->gfx.rlc.funcs->is_rlcg_access_range) { 442 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg)) 443 return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v); 444 } else { 445 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4)); 446 } 447 } 448 449 /** 450 * amdgpu_io_rreg - read an IO register 451 * 452 * @adev: amdgpu_device pointer 453 * @reg: dword aligned register offset 454 * 455 * Returns the 32 bit value from the offset specified. 456 */ 457 u32 amdgpu_io_rreg(struct amdgpu_device *adev, u32 reg) 458 { 459 if (adev->in_pci_err_recovery) 460 return 0; 461 462 if ((reg * 4) < adev->rio_mem_size) 463 return ioread32(adev->rio_mem + (reg * 4)); 464 else { 465 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); 466 return ioread32(adev->rio_mem + (mmMM_DATA * 4)); 467 } 468 } 469 470 /** 471 * amdgpu_io_wreg - write to an IO register 472 * 473 * @adev: amdgpu_device pointer 474 * @reg: dword aligned register offset 475 * @v: 32 bit value to write to the register 476 * 477 * Writes the value specified to the offset specified. 478 */ 479 void amdgpu_io_wreg(struct amdgpu_device *adev, u32 reg, u32 v) 480 { 481 if (adev->in_pci_err_recovery) 482 return; 483 484 if ((reg * 4) < adev->rio_mem_size) 485 iowrite32(v, adev->rio_mem + (reg * 4)); 486 else { 487 iowrite32((reg * 4), adev->rio_mem + (mmMM_INDEX * 4)); 488 iowrite32(v, adev->rio_mem + (mmMM_DATA * 4)); 489 } 490 } 491 492 /** 493 * amdgpu_mm_rdoorbell - read a doorbell dword 494 * 495 * @adev: amdgpu_device pointer 496 * @index: doorbell index 497 * 498 * Returns the value in the doorbell aperture at the 499 * requested doorbell index (CIK). 500 */ 501 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index) 502 { 503 if (adev->in_pci_err_recovery) 504 return 0; 505 506 if (index < adev->doorbell.num_doorbells) { 507 return readl(adev->doorbell.ptr + index); 508 } else { 509 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 510 return 0; 511 } 512 } 513 514 /** 515 * amdgpu_mm_wdoorbell - write a doorbell dword 516 * 517 * @adev: amdgpu_device pointer 518 * @index: doorbell index 519 * @v: value to write 520 * 521 * Writes @v to the doorbell aperture at the 522 * requested doorbell index (CIK). 523 */ 524 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v) 525 { 526 if (adev->in_pci_err_recovery) 527 return; 528 529 if (index < adev->doorbell.num_doorbells) { 530 writel(v, adev->doorbell.ptr + index); 531 } else { 532 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 533 } 534 } 535 536 /** 537 * amdgpu_mm_rdoorbell64 - read a doorbell Qword 538 * 539 * @adev: amdgpu_device pointer 540 * @index: doorbell index 541 * 542 * Returns the value in the doorbell aperture at the 543 * requested doorbell index (VEGA10+). 544 */ 545 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index) 546 { 547 if (adev->in_pci_err_recovery) 548 return 0; 549 550 if (index < adev->doorbell.num_doorbells) { 551 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index)); 552 } else { 553 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index); 554 return 0; 555 } 556 } 557 558 /** 559 * amdgpu_mm_wdoorbell64 - write a doorbell Qword 560 * 561 * @adev: amdgpu_device pointer 562 * @index: doorbell index 563 * @v: value to write 564 * 565 * Writes @v to the doorbell aperture at the 566 * requested doorbell index (VEGA10+). 567 */ 568 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v) 569 { 570 if (adev->in_pci_err_recovery) 571 return; 572 573 if (index < adev->doorbell.num_doorbells) { 574 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v); 575 } else { 576 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index); 577 } 578 } 579 580 /** 581 * amdgpu_device_indirect_rreg - read an indirect register 582 * 583 * @adev: amdgpu_device pointer 584 * @pcie_index: mmio register offset 585 * @pcie_data: mmio register offset 586 * 587 * Returns the value of indirect register @reg_addr 588 */ 589 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev, 590 u32 pcie_index, u32 pcie_data, 591 u32 reg_addr) 592 { 593 unsigned long flags; 594 u32 r; 595 void __iomem *pcie_index_offset; 596 void __iomem *pcie_data_offset; 597 598 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 599 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 600 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 601 602 writel(reg_addr, pcie_index_offset); 603 readl(pcie_index_offset); 604 r = readl(pcie_data_offset); 605 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 606 607 return r; 608 } 609 610 /** 611 * amdgpu_device_indirect_rreg64 - read a 64bits indirect register 612 * 613 * @adev: amdgpu_device pointer 614 * @pcie_index: mmio register offset 615 * @pcie_data: mmio register offset 616 * 617 * Returns the value of indirect register @reg_addr 618 */ 619 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev, 620 u32 pcie_index, u32 pcie_data, 621 u32 reg_addr) 622 { 623 unsigned long flags; 624 u64 r; 625 void __iomem *pcie_index_offset; 626 void __iomem *pcie_data_offset; 627 628 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 629 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 630 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 631 632 /* read low 32 bits */ 633 writel(reg_addr, pcie_index_offset); 634 readl(pcie_index_offset); 635 r = readl(pcie_data_offset); 636 /* read high 32 bits */ 637 writel(reg_addr + 4, pcie_index_offset); 638 readl(pcie_index_offset); 639 r |= ((u64)readl(pcie_data_offset) << 32); 640 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 641 642 return r; 643 } 644 645 /** 646 * amdgpu_device_indirect_wreg - write an indirect register address 647 * 648 * @adev: amdgpu_device pointer 649 * @pcie_index: mmio register offset 650 * @pcie_data: mmio register offset 651 * @reg_addr: indirect register offset 652 * @reg_data: indirect register data 653 * 654 */ 655 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev, 656 u32 pcie_index, u32 pcie_data, 657 u32 reg_addr, u32 reg_data) 658 { 659 unsigned long flags; 660 void __iomem *pcie_index_offset; 661 void __iomem *pcie_data_offset; 662 663 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 664 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 665 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 666 667 writel(reg_addr, pcie_index_offset); 668 readl(pcie_index_offset); 669 writel(reg_data, pcie_data_offset); 670 readl(pcie_data_offset); 671 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 672 } 673 674 /** 675 * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address 676 * 677 * @adev: amdgpu_device pointer 678 * @pcie_index: mmio register offset 679 * @pcie_data: mmio register offset 680 * @reg_addr: indirect register offset 681 * @reg_data: indirect register data 682 * 683 */ 684 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev, 685 u32 pcie_index, u32 pcie_data, 686 u32 reg_addr, u64 reg_data) 687 { 688 unsigned long flags; 689 void __iomem *pcie_index_offset; 690 void __iomem *pcie_data_offset; 691 692 spin_lock_irqsave(&adev->pcie_idx_lock, flags); 693 pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4; 694 pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4; 695 696 /* write low 32 bits */ 697 writel(reg_addr, pcie_index_offset); 698 readl(pcie_index_offset); 699 writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset); 700 readl(pcie_data_offset); 701 /* write high 32 bits */ 702 writel(reg_addr + 4, pcie_index_offset); 703 readl(pcie_index_offset); 704 writel((u32)(reg_data >> 32), pcie_data_offset); 705 readl(pcie_data_offset); 706 spin_unlock_irqrestore(&adev->pcie_idx_lock, flags); 707 } 708 709 /** 710 * amdgpu_invalid_rreg - dummy reg read function 711 * 712 * @adev: amdgpu_device pointer 713 * @reg: offset of register 714 * 715 * Dummy register read function. Used for register blocks 716 * that certain asics don't have (all asics). 717 * Returns the value in the register. 718 */ 719 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg) 720 { 721 DRM_ERROR("Invalid callback to read register 0x%04X\n", reg); 722 BUG(); 723 return 0; 724 } 725 726 /** 727 * amdgpu_invalid_wreg - dummy reg write function 728 * 729 * @adev: amdgpu_device pointer 730 * @reg: offset of register 731 * @v: value to write to the register 732 * 733 * Dummy register read function. Used for register blocks 734 * that certain asics don't have (all asics). 735 */ 736 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v) 737 { 738 DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n", 739 reg, v); 740 BUG(); 741 } 742 743 /** 744 * amdgpu_invalid_rreg64 - dummy 64 bit reg read function 745 * 746 * @adev: amdgpu_device pointer 747 * @reg: offset of register 748 * 749 * Dummy register read function. Used for register blocks 750 * that certain asics don't have (all asics). 751 * Returns the value in the register. 752 */ 753 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg) 754 { 755 DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg); 756 BUG(); 757 return 0; 758 } 759 760 /** 761 * amdgpu_invalid_wreg64 - dummy reg write function 762 * 763 * @adev: amdgpu_device pointer 764 * @reg: offset of register 765 * @v: value to write to the register 766 * 767 * Dummy register read function. Used for register blocks 768 * that certain asics don't have (all asics). 769 */ 770 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v) 771 { 772 DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n", 773 reg, v); 774 BUG(); 775 } 776 777 /** 778 * amdgpu_block_invalid_rreg - dummy reg read function 779 * 780 * @adev: amdgpu_device pointer 781 * @block: offset of instance 782 * @reg: offset of register 783 * 784 * Dummy register read function. Used for register blocks 785 * that certain asics don't have (all asics). 786 * Returns the value in the register. 787 */ 788 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev, 789 uint32_t block, uint32_t reg) 790 { 791 DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n", 792 reg, block); 793 BUG(); 794 return 0; 795 } 796 797 /** 798 * amdgpu_block_invalid_wreg - dummy reg write function 799 * 800 * @adev: amdgpu_device pointer 801 * @block: offset of instance 802 * @reg: offset of register 803 * @v: value to write to the register 804 * 805 * Dummy register read function. Used for register blocks 806 * that certain asics don't have (all asics). 807 */ 808 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev, 809 uint32_t block, 810 uint32_t reg, uint32_t v) 811 { 812 DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n", 813 reg, block, v); 814 BUG(); 815 } 816 817 /** 818 * amdgpu_device_asic_init - Wrapper for atom asic_init 819 * 820 * @adev: amdgpu_device pointer 821 * 822 * Does any asic specific work and then calls atom asic init. 823 */ 824 static int amdgpu_device_asic_init(struct amdgpu_device *adev) 825 { 826 amdgpu_asic_pre_asic_init(adev); 827 828 return amdgpu_atom_asic_init(adev->mode_info.atom_context); 829 } 830 831 /** 832 * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page 833 * 834 * @adev: amdgpu_device pointer 835 * 836 * Allocates a scratch page of VRAM for use by various things in the 837 * driver. 838 */ 839 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev) 840 { 841 return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, 842 PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, 843 &adev->vram_scratch.robj, 844 &adev->vram_scratch.gpu_addr, 845 (void **)&adev->vram_scratch.ptr); 846 } 847 848 /** 849 * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page 850 * 851 * @adev: amdgpu_device pointer 852 * 853 * Frees the VRAM scratch page. 854 */ 855 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev) 856 { 857 amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL); 858 } 859 860 /** 861 * amdgpu_device_program_register_sequence - program an array of registers. 862 * 863 * @adev: amdgpu_device pointer 864 * @registers: pointer to the register array 865 * @array_size: size of the register array 866 * 867 * Programs an array or registers with and and or masks. 868 * This is a helper for setting golden registers. 869 */ 870 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev, 871 const u32 *registers, 872 const u32 array_size) 873 { 874 u32 tmp, reg, and_mask, or_mask; 875 int i; 876 877 if (array_size % 3) 878 return; 879 880 for (i = 0; i < array_size; i +=3) { 881 reg = registers[i + 0]; 882 and_mask = registers[i + 1]; 883 or_mask = registers[i + 2]; 884 885 if (and_mask == 0xffffffff) { 886 tmp = or_mask; 887 } else { 888 tmp = RREG32(reg); 889 tmp &= ~and_mask; 890 if (adev->family >= AMDGPU_FAMILY_AI) 891 tmp |= (or_mask & and_mask); 892 else 893 tmp |= or_mask; 894 } 895 WREG32(reg, tmp); 896 } 897 } 898 899 /** 900 * amdgpu_device_pci_config_reset - reset the GPU 901 * 902 * @adev: amdgpu_device pointer 903 * 904 * Resets the GPU using the pci config reset sequence. 905 * Only applicable to asics prior to vega10. 906 */ 907 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev) 908 { 909 pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA); 910 } 911 912 /* 913 * GPU doorbell aperture helpers function. 914 */ 915 /** 916 * amdgpu_device_doorbell_init - Init doorbell driver information. 917 * 918 * @adev: amdgpu_device pointer 919 * 920 * Init doorbell driver information (CIK) 921 * Returns 0 on success, error on failure. 922 */ 923 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev) 924 { 925 926 /* No doorbell on SI hardware generation */ 927 if (adev->asic_type < CHIP_BONAIRE) { 928 adev->doorbell.base = 0; 929 adev->doorbell.size = 0; 930 adev->doorbell.num_doorbells = 0; 931 adev->doorbell.ptr = NULL; 932 return 0; 933 } 934 935 if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET) 936 return -EINVAL; 937 938 amdgpu_asic_init_doorbell_index(adev); 939 940 /* doorbell bar mapping */ 941 adev->doorbell.base = pci_resource_start(adev->pdev, 2); 942 adev->doorbell.size = pci_resource_len(adev->pdev, 2); 943 944 adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32), 945 adev->doorbell_index.max_assignment+1); 946 if (adev->doorbell.num_doorbells == 0) 947 return -EINVAL; 948 949 /* For Vega, reserve and map two pages on doorbell BAR since SDMA 950 * paging queue doorbell use the second page. The 951 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the 952 * doorbells are in the first page. So with paging queue enabled, 953 * the max num_doorbells should + 1 page (0x400 in dword) 954 */ 955 if (adev->asic_type >= CHIP_VEGA10) 956 adev->doorbell.num_doorbells += 0x400; 957 958 adev->doorbell.ptr = ioremap(adev->doorbell.base, 959 adev->doorbell.num_doorbells * 960 sizeof(u32)); 961 if (adev->doorbell.ptr == NULL) 962 return -ENOMEM; 963 964 return 0; 965 } 966 967 /** 968 * amdgpu_device_doorbell_fini - Tear down doorbell driver information. 969 * 970 * @adev: amdgpu_device pointer 971 * 972 * Tear down doorbell driver information (CIK) 973 */ 974 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev) 975 { 976 iounmap(adev->doorbell.ptr); 977 adev->doorbell.ptr = NULL; 978 } 979 980 981 982 /* 983 * amdgpu_device_wb_*() 984 * Writeback is the method by which the GPU updates special pages in memory 985 * with the status of certain GPU events (fences, ring pointers,etc.). 986 */ 987 988 /** 989 * amdgpu_device_wb_fini - Disable Writeback and free memory 990 * 991 * @adev: amdgpu_device pointer 992 * 993 * Disables Writeback and frees the Writeback memory (all asics). 994 * Used at driver shutdown. 995 */ 996 static void amdgpu_device_wb_fini(struct amdgpu_device *adev) 997 { 998 if (adev->wb.wb_obj) { 999 amdgpu_bo_free_kernel(&adev->wb.wb_obj, 1000 &adev->wb.gpu_addr, 1001 (void **)&adev->wb.wb); 1002 adev->wb.wb_obj = NULL; 1003 } 1004 } 1005 1006 /** 1007 * amdgpu_device_wb_init- Init Writeback driver info and allocate memory 1008 * 1009 * @adev: amdgpu_device pointer 1010 * 1011 * Initializes writeback and allocates writeback memory (all asics). 1012 * Used at driver startup. 1013 * Returns 0 on success or an -error on failure. 1014 */ 1015 static int amdgpu_device_wb_init(struct amdgpu_device *adev) 1016 { 1017 int r; 1018 1019 if (adev->wb.wb_obj == NULL) { 1020 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */ 1021 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8, 1022 PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, 1023 &adev->wb.wb_obj, &adev->wb.gpu_addr, 1024 (void **)&adev->wb.wb); 1025 if (r) { 1026 dev_warn(adev->dev, "(%d) create WB bo failed\n", r); 1027 return r; 1028 } 1029 1030 adev->wb.num_wb = AMDGPU_MAX_WB; 1031 memset(&adev->wb.used, 0, sizeof(adev->wb.used)); 1032 1033 /* clear wb memory */ 1034 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8); 1035 } 1036 1037 return 0; 1038 } 1039 1040 /** 1041 * amdgpu_device_wb_get - Allocate a wb entry 1042 * 1043 * @adev: amdgpu_device pointer 1044 * @wb: wb index 1045 * 1046 * Allocate a wb slot for use by the driver (all asics). 1047 * Returns 0 on success or -EINVAL on failure. 1048 */ 1049 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb) 1050 { 1051 unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb); 1052 1053 if (offset < adev->wb.num_wb) { 1054 __set_bit(offset, adev->wb.used); 1055 *wb = offset << 3; /* convert to dw offset */ 1056 return 0; 1057 } else { 1058 return -EINVAL; 1059 } 1060 } 1061 1062 /** 1063 * amdgpu_device_wb_free - Free a wb entry 1064 * 1065 * @adev: amdgpu_device pointer 1066 * @wb: wb index 1067 * 1068 * Free a wb slot allocated for use by the driver (all asics) 1069 */ 1070 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb) 1071 { 1072 wb >>= 3; 1073 if (wb < adev->wb.num_wb) 1074 __clear_bit(wb, adev->wb.used); 1075 } 1076 1077 /** 1078 * amdgpu_device_resize_fb_bar - try to resize FB BAR 1079 * 1080 * @adev: amdgpu_device pointer 1081 * 1082 * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not 1083 * to fail, but if any of the BARs is not accessible after the size we abort 1084 * driver loading by returning -ENODEV. 1085 */ 1086 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev) 1087 { 1088 u64 space_needed = roundup_pow_of_two(adev->gmc.real_vram_size); 1089 u32 rbar_size = order_base_2(((space_needed >> 20) | 1)) - 1; 1090 struct pci_bus *root; 1091 struct resource *res; 1092 unsigned i; 1093 u16 cmd; 1094 int r; 1095 1096 /* Bypass for VF */ 1097 if (amdgpu_sriov_vf(adev)) 1098 return 0; 1099 1100 /* skip if the bios has already enabled large BAR */ 1101 if (adev->gmc.real_vram_size && 1102 (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size)) 1103 return 0; 1104 1105 /* Check if the root BUS has 64bit memory resources */ 1106 root = adev->pdev->bus; 1107 while (root->parent) 1108 root = root->parent; 1109 1110 pci_bus_for_each_resource(root, res, i) { 1111 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && 1112 res->start > 0x100000000ull) 1113 break; 1114 } 1115 1116 /* Trying to resize is pointless without a root hub window above 4GB */ 1117 if (!res) 1118 return 0; 1119 1120 /* Disable memory decoding while we change the BAR addresses and size */ 1121 pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd); 1122 pci_write_config_word(adev->pdev, PCI_COMMAND, 1123 cmd & ~PCI_COMMAND_MEMORY); 1124 1125 /* Free the VRAM and doorbell BAR, we most likely need to move both. */ 1126 amdgpu_device_doorbell_fini(adev); 1127 if (adev->asic_type >= CHIP_BONAIRE) 1128 pci_release_resource(adev->pdev, 2); 1129 1130 pci_release_resource(adev->pdev, 0); 1131 1132 r = pci_resize_resource(adev->pdev, 0, rbar_size); 1133 if (r == -ENOSPC) 1134 DRM_INFO("Not enough PCI address space for a large BAR."); 1135 else if (r && r != -ENOTSUPP) 1136 DRM_ERROR("Problem resizing BAR0 (%d).", r); 1137 1138 pci_assign_unassigned_bus_resources(adev->pdev->bus); 1139 1140 /* When the doorbell or fb BAR isn't available we have no chance of 1141 * using the device. 1142 */ 1143 r = amdgpu_device_doorbell_init(adev); 1144 if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET)) 1145 return -ENODEV; 1146 1147 pci_write_config_word(adev->pdev, PCI_COMMAND, cmd); 1148 1149 return 0; 1150 } 1151 1152 /* 1153 * GPU helpers function. 1154 */ 1155 /** 1156 * amdgpu_device_need_post - check if the hw need post or not 1157 * 1158 * @adev: amdgpu_device pointer 1159 * 1160 * Check if the asic has been initialized (all asics) at driver startup 1161 * or post is needed if hw reset is performed. 1162 * Returns true if need or false if not. 1163 */ 1164 bool amdgpu_device_need_post(struct amdgpu_device *adev) 1165 { 1166 uint32_t reg; 1167 1168 if (amdgpu_sriov_vf(adev)) 1169 return false; 1170 1171 if (amdgpu_passthrough(adev)) { 1172 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot 1173 * some old smc fw still need driver do vPost otherwise gpu hang, while 1174 * those smc fw version above 22.15 doesn't have this flaw, so we force 1175 * vpost executed for smc version below 22.15 1176 */ 1177 if (adev->asic_type == CHIP_FIJI) { 1178 int err; 1179 uint32_t fw_ver; 1180 err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev); 1181 /* force vPost if error occured */ 1182 if (err) 1183 return true; 1184 1185 fw_ver = *((uint32_t *)adev->pm.fw->data + 69); 1186 if (fw_ver < 0x00160e00) 1187 return true; 1188 } 1189 } 1190 1191 if (adev->has_hw_reset) { 1192 adev->has_hw_reset = false; 1193 return true; 1194 } 1195 1196 /* bios scratch used on CIK+ */ 1197 if (adev->asic_type >= CHIP_BONAIRE) 1198 return amdgpu_atombios_scratch_need_asic_init(adev); 1199 1200 /* check MEM_SIZE for older asics */ 1201 reg = amdgpu_asic_get_config_memsize(adev); 1202 1203 if ((reg != 0) && (reg != 0xffffffff)) 1204 return false; 1205 1206 return true; 1207 } 1208 1209 /* if we get transitioned to only one device, take VGA back */ 1210 /** 1211 * amdgpu_device_vga_set_decode - enable/disable vga decode 1212 * 1213 * @cookie: amdgpu_device pointer 1214 * @state: enable/disable vga decode 1215 * 1216 * Enable/disable vga decode (all asics). 1217 * Returns VGA resource flags. 1218 */ 1219 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state) 1220 { 1221 struct amdgpu_device *adev = cookie; 1222 amdgpu_asic_set_vga_state(adev, state); 1223 if (state) 1224 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM | 1225 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1226 else 1227 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 1228 } 1229 1230 /** 1231 * amdgpu_device_check_block_size - validate the vm block size 1232 * 1233 * @adev: amdgpu_device pointer 1234 * 1235 * Validates the vm block size specified via module parameter. 1236 * The vm block size defines number of bits in page table versus page directory, 1237 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1238 * page table and the remaining bits are in the page directory. 1239 */ 1240 static void amdgpu_device_check_block_size(struct amdgpu_device *adev) 1241 { 1242 /* defines number of bits in page table versus page directory, 1243 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the 1244 * page table and the remaining bits are in the page directory */ 1245 if (amdgpu_vm_block_size == -1) 1246 return; 1247 1248 if (amdgpu_vm_block_size < 9) { 1249 dev_warn(adev->dev, "VM page table size (%d) too small\n", 1250 amdgpu_vm_block_size); 1251 amdgpu_vm_block_size = -1; 1252 } 1253 } 1254 1255 /** 1256 * amdgpu_device_check_vm_size - validate the vm size 1257 * 1258 * @adev: amdgpu_device pointer 1259 * 1260 * Validates the vm size in GB specified via module parameter. 1261 * The VM size is the size of the GPU virtual memory space in GB. 1262 */ 1263 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev) 1264 { 1265 /* no need to check the default value */ 1266 if (amdgpu_vm_size == -1) 1267 return; 1268 1269 if (amdgpu_vm_size < 1) { 1270 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n", 1271 amdgpu_vm_size); 1272 amdgpu_vm_size = -1; 1273 } 1274 } 1275 1276 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev) 1277 { 1278 struct sysinfo si; 1279 bool is_os_64 = (sizeof(void *) == 8); 1280 uint64_t total_memory; 1281 uint64_t dram_size_seven_GB = 0x1B8000000; 1282 uint64_t dram_size_three_GB = 0xB8000000; 1283 1284 if (amdgpu_smu_memory_pool_size == 0) 1285 return; 1286 1287 if (!is_os_64) { 1288 DRM_WARN("Not 64-bit OS, feature not supported\n"); 1289 goto def_value; 1290 } 1291 si_meminfo(&si); 1292 total_memory = (uint64_t)si.totalram * si.mem_unit; 1293 1294 if ((amdgpu_smu_memory_pool_size == 1) || 1295 (amdgpu_smu_memory_pool_size == 2)) { 1296 if (total_memory < dram_size_three_GB) 1297 goto def_value1; 1298 } else if ((amdgpu_smu_memory_pool_size == 4) || 1299 (amdgpu_smu_memory_pool_size == 8)) { 1300 if (total_memory < dram_size_seven_GB) 1301 goto def_value1; 1302 } else { 1303 DRM_WARN("Smu memory pool size not supported\n"); 1304 goto def_value; 1305 } 1306 adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28; 1307 1308 return; 1309 1310 def_value1: 1311 DRM_WARN("No enough system memory\n"); 1312 def_value: 1313 adev->pm.smu_prv_buffer_size = 0; 1314 } 1315 1316 /** 1317 * amdgpu_device_check_arguments - validate module params 1318 * 1319 * @adev: amdgpu_device pointer 1320 * 1321 * Validates certain module parameters and updates 1322 * the associated values used by the driver (all asics). 1323 */ 1324 static int amdgpu_device_check_arguments(struct amdgpu_device *adev) 1325 { 1326 if (amdgpu_sched_jobs < 4) { 1327 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n", 1328 amdgpu_sched_jobs); 1329 amdgpu_sched_jobs = 4; 1330 } else if (!is_power_of_2(amdgpu_sched_jobs)){ 1331 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n", 1332 amdgpu_sched_jobs); 1333 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs); 1334 } 1335 1336 if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) { 1337 /* gart size must be greater or equal to 32M */ 1338 dev_warn(adev->dev, "gart size (%d) too small\n", 1339 amdgpu_gart_size); 1340 amdgpu_gart_size = -1; 1341 } 1342 1343 if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) { 1344 /* gtt size must be greater or equal to 32M */ 1345 dev_warn(adev->dev, "gtt size (%d) too small\n", 1346 amdgpu_gtt_size); 1347 amdgpu_gtt_size = -1; 1348 } 1349 1350 /* valid range is between 4 and 9 inclusive */ 1351 if (amdgpu_vm_fragment_size != -1 && 1352 (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) { 1353 dev_warn(adev->dev, "valid range is between 4 and 9\n"); 1354 amdgpu_vm_fragment_size = -1; 1355 } 1356 1357 if (amdgpu_sched_hw_submission < 2) { 1358 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n", 1359 amdgpu_sched_hw_submission); 1360 amdgpu_sched_hw_submission = 2; 1361 } else if (!is_power_of_2(amdgpu_sched_hw_submission)) { 1362 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n", 1363 amdgpu_sched_hw_submission); 1364 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission); 1365 } 1366 1367 amdgpu_device_check_smu_prv_buffer_size(adev); 1368 1369 amdgpu_device_check_vm_size(adev); 1370 1371 amdgpu_device_check_block_size(adev); 1372 1373 adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type); 1374 1375 amdgpu_gmc_tmz_set(adev); 1376 1377 amdgpu_gmc_noretry_set(adev); 1378 1379 return 0; 1380 } 1381 1382 /** 1383 * amdgpu_switcheroo_set_state - set switcheroo state 1384 * 1385 * @pdev: pci dev pointer 1386 * @state: vga_switcheroo state 1387 * 1388 * Callback for the switcheroo driver. Suspends or resumes the 1389 * the asics before or after it is powered up using ACPI methods. 1390 */ 1391 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev, 1392 enum vga_switcheroo_state state) 1393 { 1394 struct drm_device *dev = pci_get_drvdata(pdev); 1395 int r; 1396 1397 if (amdgpu_device_supports_boco(dev) && state == VGA_SWITCHEROO_OFF) 1398 return; 1399 1400 if (state == VGA_SWITCHEROO_ON) { 1401 pr_info("switched on\n"); 1402 /* don't suspend or resume card normally */ 1403 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1404 1405 pci_set_power_state(dev->pdev, PCI_D0); 1406 amdgpu_device_load_pci_state(dev->pdev); 1407 r = pci_enable_device(dev->pdev); 1408 if (r) 1409 DRM_WARN("pci_enable_device failed (%d)\n", r); 1410 amdgpu_device_resume(dev, true); 1411 1412 dev->switch_power_state = DRM_SWITCH_POWER_ON; 1413 drm_kms_helper_poll_enable(dev); 1414 } else { 1415 pr_info("switched off\n"); 1416 drm_kms_helper_poll_disable(dev); 1417 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING; 1418 amdgpu_device_suspend(dev, true); 1419 amdgpu_device_cache_pci_state(dev->pdev); 1420 /* Shut down the device */ 1421 pci_disable_device(dev->pdev); 1422 pci_set_power_state(dev->pdev, PCI_D3cold); 1423 dev->switch_power_state = DRM_SWITCH_POWER_OFF; 1424 } 1425 } 1426 1427 /** 1428 * amdgpu_switcheroo_can_switch - see if switcheroo state can change 1429 * 1430 * @pdev: pci dev pointer 1431 * 1432 * Callback for the switcheroo driver. Check of the switcheroo 1433 * state can be changed. 1434 * Returns true if the state can be changed, false if not. 1435 */ 1436 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev) 1437 { 1438 struct drm_device *dev = pci_get_drvdata(pdev); 1439 1440 /* 1441 * FIXME: open_count is protected by drm_global_mutex but that would lead to 1442 * locking inversion with the driver load path. And the access here is 1443 * completely racy anyway. So don't bother with locking for now. 1444 */ 1445 return atomic_read(&dev->open_count) == 0; 1446 } 1447 1448 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = { 1449 .set_gpu_state = amdgpu_switcheroo_set_state, 1450 .reprobe = NULL, 1451 .can_switch = amdgpu_switcheroo_can_switch, 1452 }; 1453 1454 /** 1455 * amdgpu_device_ip_set_clockgating_state - set the CG state 1456 * 1457 * @dev: amdgpu_device pointer 1458 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1459 * @state: clockgating state (gate or ungate) 1460 * 1461 * Sets the requested clockgating state for all instances of 1462 * the hardware IP specified. 1463 * Returns the error code from the last instance. 1464 */ 1465 int amdgpu_device_ip_set_clockgating_state(void *dev, 1466 enum amd_ip_block_type block_type, 1467 enum amd_clockgating_state state) 1468 { 1469 struct amdgpu_device *adev = dev; 1470 int i, r = 0; 1471 1472 for (i = 0; i < adev->num_ip_blocks; i++) { 1473 if (!adev->ip_blocks[i].status.valid) 1474 continue; 1475 if (adev->ip_blocks[i].version->type != block_type) 1476 continue; 1477 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state) 1478 continue; 1479 r = adev->ip_blocks[i].version->funcs->set_clockgating_state( 1480 (void *)adev, state); 1481 if (r) 1482 DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n", 1483 adev->ip_blocks[i].version->funcs->name, r); 1484 } 1485 return r; 1486 } 1487 1488 /** 1489 * amdgpu_device_ip_set_powergating_state - set the PG state 1490 * 1491 * @dev: amdgpu_device pointer 1492 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1493 * @state: powergating state (gate or ungate) 1494 * 1495 * Sets the requested powergating state for all instances of 1496 * the hardware IP specified. 1497 * Returns the error code from the last instance. 1498 */ 1499 int amdgpu_device_ip_set_powergating_state(void *dev, 1500 enum amd_ip_block_type block_type, 1501 enum amd_powergating_state state) 1502 { 1503 struct amdgpu_device *adev = dev; 1504 int i, r = 0; 1505 1506 for (i = 0; i < adev->num_ip_blocks; i++) { 1507 if (!adev->ip_blocks[i].status.valid) 1508 continue; 1509 if (adev->ip_blocks[i].version->type != block_type) 1510 continue; 1511 if (!adev->ip_blocks[i].version->funcs->set_powergating_state) 1512 continue; 1513 r = adev->ip_blocks[i].version->funcs->set_powergating_state( 1514 (void *)adev, state); 1515 if (r) 1516 DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n", 1517 adev->ip_blocks[i].version->funcs->name, r); 1518 } 1519 return r; 1520 } 1521 1522 /** 1523 * amdgpu_device_ip_get_clockgating_state - get the CG state 1524 * 1525 * @adev: amdgpu_device pointer 1526 * @flags: clockgating feature flags 1527 * 1528 * Walks the list of IPs on the device and updates the clockgating 1529 * flags for each IP. 1530 * Updates @flags with the feature flags for each hardware IP where 1531 * clockgating is enabled. 1532 */ 1533 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, 1534 u32 *flags) 1535 { 1536 int i; 1537 1538 for (i = 0; i < adev->num_ip_blocks; i++) { 1539 if (!adev->ip_blocks[i].status.valid) 1540 continue; 1541 if (adev->ip_blocks[i].version->funcs->get_clockgating_state) 1542 adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags); 1543 } 1544 } 1545 1546 /** 1547 * amdgpu_device_ip_wait_for_idle - wait for idle 1548 * 1549 * @adev: amdgpu_device pointer 1550 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1551 * 1552 * Waits for the request hardware IP to be idle. 1553 * Returns 0 for success or a negative error code on failure. 1554 */ 1555 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, 1556 enum amd_ip_block_type block_type) 1557 { 1558 int i, r; 1559 1560 for (i = 0; i < adev->num_ip_blocks; i++) { 1561 if (!adev->ip_blocks[i].status.valid) 1562 continue; 1563 if (adev->ip_blocks[i].version->type == block_type) { 1564 r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev); 1565 if (r) 1566 return r; 1567 break; 1568 } 1569 } 1570 return 0; 1571 1572 } 1573 1574 /** 1575 * amdgpu_device_ip_is_idle - is the hardware IP idle 1576 * 1577 * @adev: amdgpu_device pointer 1578 * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) 1579 * 1580 * Check if the hardware IP is idle or not. 1581 * Returns true if it the IP is idle, false if not. 1582 */ 1583 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev, 1584 enum amd_ip_block_type block_type) 1585 { 1586 int i; 1587 1588 for (i = 0; i < adev->num_ip_blocks; i++) { 1589 if (!adev->ip_blocks[i].status.valid) 1590 continue; 1591 if (adev->ip_blocks[i].version->type == block_type) 1592 return adev->ip_blocks[i].version->funcs->is_idle((void *)adev); 1593 } 1594 return true; 1595 1596 } 1597 1598 /** 1599 * amdgpu_device_ip_get_ip_block - get a hw IP pointer 1600 * 1601 * @adev: amdgpu_device pointer 1602 * @type: Type of hardware IP (SMU, GFX, UVD, etc.) 1603 * 1604 * Returns a pointer to the hardware IP block structure 1605 * if it exists for the asic, otherwise NULL. 1606 */ 1607 struct amdgpu_ip_block * 1608 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev, 1609 enum amd_ip_block_type type) 1610 { 1611 int i; 1612 1613 for (i = 0; i < adev->num_ip_blocks; i++) 1614 if (adev->ip_blocks[i].version->type == type) 1615 return &adev->ip_blocks[i]; 1616 1617 return NULL; 1618 } 1619 1620 /** 1621 * amdgpu_device_ip_block_version_cmp 1622 * 1623 * @adev: amdgpu_device pointer 1624 * @type: enum amd_ip_block_type 1625 * @major: major version 1626 * @minor: minor version 1627 * 1628 * return 0 if equal or greater 1629 * return 1 if smaller or the ip_block doesn't exist 1630 */ 1631 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev, 1632 enum amd_ip_block_type type, 1633 u32 major, u32 minor) 1634 { 1635 struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type); 1636 1637 if (ip_block && ((ip_block->version->major > major) || 1638 ((ip_block->version->major == major) && 1639 (ip_block->version->minor >= minor)))) 1640 return 0; 1641 1642 return 1; 1643 } 1644 1645 /** 1646 * amdgpu_device_ip_block_add 1647 * 1648 * @adev: amdgpu_device pointer 1649 * @ip_block_version: pointer to the IP to add 1650 * 1651 * Adds the IP block driver information to the collection of IPs 1652 * on the asic. 1653 */ 1654 int amdgpu_device_ip_block_add(struct amdgpu_device *adev, 1655 const struct amdgpu_ip_block_version *ip_block_version) 1656 { 1657 if (!ip_block_version) 1658 return -EINVAL; 1659 1660 DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks, 1661 ip_block_version->funcs->name); 1662 1663 adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version; 1664 1665 return 0; 1666 } 1667 1668 /** 1669 * amdgpu_device_enable_virtual_display - enable virtual display feature 1670 * 1671 * @adev: amdgpu_device pointer 1672 * 1673 * Enabled the virtual display feature if the user has enabled it via 1674 * the module parameter virtual_display. This feature provides a virtual 1675 * display hardware on headless boards or in virtualized environments. 1676 * This function parses and validates the configuration string specified by 1677 * the user and configues the virtual display configuration (number of 1678 * virtual connectors, crtcs, etc.) specified. 1679 */ 1680 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev) 1681 { 1682 adev->enable_virtual_display = false; 1683 1684 if (amdgpu_virtual_display) { 1685 struct drm_device *ddev = adev_to_drm(adev); 1686 const char *pci_address_name = pci_name(ddev->pdev); 1687 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname; 1688 1689 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL); 1690 pciaddstr_tmp = pciaddstr; 1691 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) { 1692 pciaddname = strsep(&pciaddname_tmp, ","); 1693 if (!strcmp("all", pciaddname) 1694 || !strcmp(pci_address_name, pciaddname)) { 1695 long num_crtc; 1696 int res = -1; 1697 1698 adev->enable_virtual_display = true; 1699 1700 if (pciaddname_tmp) 1701 res = kstrtol(pciaddname_tmp, 10, 1702 &num_crtc); 1703 1704 if (!res) { 1705 if (num_crtc < 1) 1706 num_crtc = 1; 1707 if (num_crtc > 6) 1708 num_crtc = 6; 1709 adev->mode_info.num_crtc = num_crtc; 1710 } else { 1711 adev->mode_info.num_crtc = 1; 1712 } 1713 break; 1714 } 1715 } 1716 1717 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n", 1718 amdgpu_virtual_display, pci_address_name, 1719 adev->enable_virtual_display, adev->mode_info.num_crtc); 1720 1721 kfree(pciaddstr); 1722 } 1723 } 1724 1725 /** 1726 * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware 1727 * 1728 * @adev: amdgpu_device pointer 1729 * 1730 * Parses the asic configuration parameters specified in the gpu info 1731 * firmware and makes them availale to the driver for use in configuring 1732 * the asic. 1733 * Returns 0 on success, -EINVAL on failure. 1734 */ 1735 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) 1736 { 1737 const char *chip_name; 1738 char fw_name[40]; 1739 int err; 1740 const struct gpu_info_firmware_header_v1_0 *hdr; 1741 1742 adev->firmware.gpu_info_fw = NULL; 1743 1744 if (adev->mman.discovery_bin) { 1745 amdgpu_discovery_get_gfx_info(adev); 1746 1747 /* 1748 * FIXME: The bounding box is still needed by Navi12, so 1749 * temporarily read it from gpu_info firmware. Should be droped 1750 * when DAL no longer needs it. 1751 */ 1752 if (adev->asic_type != CHIP_NAVI12) 1753 return 0; 1754 } 1755 1756 switch (adev->asic_type) { 1757 #ifdef CONFIG_DRM_AMDGPU_SI 1758 case CHIP_VERDE: 1759 case CHIP_TAHITI: 1760 case CHIP_PITCAIRN: 1761 case CHIP_OLAND: 1762 case CHIP_HAINAN: 1763 #endif 1764 #ifdef CONFIG_DRM_AMDGPU_CIK 1765 case CHIP_BONAIRE: 1766 case CHIP_HAWAII: 1767 case CHIP_KAVERI: 1768 case CHIP_KABINI: 1769 case CHIP_MULLINS: 1770 #endif 1771 case CHIP_TOPAZ: 1772 case CHIP_TONGA: 1773 case CHIP_FIJI: 1774 case CHIP_POLARIS10: 1775 case CHIP_POLARIS11: 1776 case CHIP_POLARIS12: 1777 case CHIP_VEGAM: 1778 case CHIP_CARRIZO: 1779 case CHIP_STONEY: 1780 case CHIP_VEGA20: 1781 case CHIP_SIENNA_CICHLID: 1782 case CHIP_NAVY_FLOUNDER: 1783 case CHIP_DIMGREY_CAVEFISH: 1784 default: 1785 return 0; 1786 case CHIP_VEGA10: 1787 chip_name = "vega10"; 1788 break; 1789 case CHIP_VEGA12: 1790 chip_name = "vega12"; 1791 break; 1792 case CHIP_RAVEN: 1793 if (adev->apu_flags & AMD_APU_IS_RAVEN2) 1794 chip_name = "raven2"; 1795 else if (adev->apu_flags & AMD_APU_IS_PICASSO) 1796 chip_name = "picasso"; 1797 else 1798 chip_name = "raven"; 1799 break; 1800 case CHIP_ARCTURUS: 1801 chip_name = "arcturus"; 1802 break; 1803 case CHIP_RENOIR: 1804 if (adev->apu_flags & AMD_APU_IS_RENOIR) 1805 chip_name = "renoir"; 1806 else 1807 chip_name = "green_sardine"; 1808 break; 1809 case CHIP_NAVI10: 1810 chip_name = "navi10"; 1811 break; 1812 case CHIP_NAVI14: 1813 chip_name = "navi14"; 1814 break; 1815 case CHIP_NAVI12: 1816 chip_name = "navi12"; 1817 break; 1818 case CHIP_VANGOGH: 1819 chip_name = "vangogh"; 1820 break; 1821 } 1822 1823 snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name); 1824 err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev); 1825 if (err) { 1826 dev_err(adev->dev, 1827 "Failed to load gpu_info firmware \"%s\"\n", 1828 fw_name); 1829 goto out; 1830 } 1831 err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw); 1832 if (err) { 1833 dev_err(adev->dev, 1834 "Failed to validate gpu_info firmware \"%s\"\n", 1835 fw_name); 1836 goto out; 1837 } 1838 1839 hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data; 1840 amdgpu_ucode_print_gpu_info_hdr(&hdr->header); 1841 1842 switch (hdr->version_major) { 1843 case 1: 1844 { 1845 const struct gpu_info_firmware_v1_0 *gpu_info_fw = 1846 (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data + 1847 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1848 1849 /* 1850 * Should be droped when DAL no longer needs it. 1851 */ 1852 if (adev->asic_type == CHIP_NAVI12) 1853 goto parse_soc_bounding_box; 1854 1855 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se); 1856 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh); 1857 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se); 1858 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se); 1859 adev->gfx.config.max_texture_channel_caches = 1860 le32_to_cpu(gpu_info_fw->gc_num_tccs); 1861 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs); 1862 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds); 1863 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth); 1864 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth); 1865 adev->gfx.config.double_offchip_lds_buf = 1866 le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer); 1867 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size); 1868 adev->gfx.cu_info.max_waves_per_simd = 1869 le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd); 1870 adev->gfx.cu_info.max_scratch_slots_per_cu = 1871 le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu); 1872 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size); 1873 if (hdr->version_minor >= 1) { 1874 const struct gpu_info_firmware_v1_1 *gpu_info_fw = 1875 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data + 1876 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1877 adev->gfx.config.num_sc_per_sh = 1878 le32_to_cpu(gpu_info_fw->num_sc_per_sh); 1879 adev->gfx.config.num_packer_per_sc = 1880 le32_to_cpu(gpu_info_fw->num_packer_per_sc); 1881 } 1882 1883 parse_soc_bounding_box: 1884 /* 1885 * soc bounding box info is not integrated in disocovery table, 1886 * we always need to parse it from gpu info firmware if needed. 1887 */ 1888 if (hdr->version_minor == 2) { 1889 const struct gpu_info_firmware_v1_2 *gpu_info_fw = 1890 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data + 1891 le32_to_cpu(hdr->header.ucode_array_offset_bytes)); 1892 adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box; 1893 } 1894 break; 1895 } 1896 default: 1897 dev_err(adev->dev, 1898 "Unsupported gpu_info table %d\n", hdr->header.ucode_version); 1899 err = -EINVAL; 1900 goto out; 1901 } 1902 out: 1903 return err; 1904 } 1905 1906 /** 1907 * amdgpu_device_ip_early_init - run early init for hardware IPs 1908 * 1909 * @adev: amdgpu_device pointer 1910 * 1911 * Early initialization pass for hardware IPs. The hardware IPs that make 1912 * up each asic are discovered each IP's early_init callback is run. This 1913 * is the first stage in initializing the asic. 1914 * Returns 0 on success, negative error code on failure. 1915 */ 1916 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) 1917 { 1918 int i, r; 1919 1920 amdgpu_device_enable_virtual_display(adev); 1921 1922 if (amdgpu_sriov_vf(adev)) { 1923 r = amdgpu_virt_request_full_gpu(adev, true); 1924 if (r) 1925 return r; 1926 } 1927 1928 switch (adev->asic_type) { 1929 #ifdef CONFIG_DRM_AMDGPU_SI 1930 case CHIP_VERDE: 1931 case CHIP_TAHITI: 1932 case CHIP_PITCAIRN: 1933 case CHIP_OLAND: 1934 case CHIP_HAINAN: 1935 adev->family = AMDGPU_FAMILY_SI; 1936 r = si_set_ip_blocks(adev); 1937 if (r) 1938 return r; 1939 break; 1940 #endif 1941 #ifdef CONFIG_DRM_AMDGPU_CIK 1942 case CHIP_BONAIRE: 1943 case CHIP_HAWAII: 1944 case CHIP_KAVERI: 1945 case CHIP_KABINI: 1946 case CHIP_MULLINS: 1947 if (adev->flags & AMD_IS_APU) 1948 adev->family = AMDGPU_FAMILY_KV; 1949 else 1950 adev->family = AMDGPU_FAMILY_CI; 1951 1952 r = cik_set_ip_blocks(adev); 1953 if (r) 1954 return r; 1955 break; 1956 #endif 1957 case CHIP_TOPAZ: 1958 case CHIP_TONGA: 1959 case CHIP_FIJI: 1960 case CHIP_POLARIS10: 1961 case CHIP_POLARIS11: 1962 case CHIP_POLARIS12: 1963 case CHIP_VEGAM: 1964 case CHIP_CARRIZO: 1965 case CHIP_STONEY: 1966 if (adev->flags & AMD_IS_APU) 1967 adev->family = AMDGPU_FAMILY_CZ; 1968 else 1969 adev->family = AMDGPU_FAMILY_VI; 1970 1971 r = vi_set_ip_blocks(adev); 1972 if (r) 1973 return r; 1974 break; 1975 case CHIP_VEGA10: 1976 case CHIP_VEGA12: 1977 case CHIP_VEGA20: 1978 case CHIP_RAVEN: 1979 case CHIP_ARCTURUS: 1980 case CHIP_RENOIR: 1981 if (adev->flags & AMD_IS_APU) 1982 adev->family = AMDGPU_FAMILY_RV; 1983 else 1984 adev->family = AMDGPU_FAMILY_AI; 1985 1986 r = soc15_set_ip_blocks(adev); 1987 if (r) 1988 return r; 1989 break; 1990 case CHIP_NAVI10: 1991 case CHIP_NAVI14: 1992 case CHIP_NAVI12: 1993 case CHIP_SIENNA_CICHLID: 1994 case CHIP_NAVY_FLOUNDER: 1995 case CHIP_DIMGREY_CAVEFISH: 1996 case CHIP_VANGOGH: 1997 if (adev->asic_type == CHIP_VANGOGH) 1998 adev->family = AMDGPU_FAMILY_VGH; 1999 else 2000 adev->family = AMDGPU_FAMILY_NV; 2001 2002 r = nv_set_ip_blocks(adev); 2003 if (r) 2004 return r; 2005 break; 2006 default: 2007 /* FIXME: not supported yet */ 2008 return -EINVAL; 2009 } 2010 2011 amdgpu_amdkfd_device_probe(adev); 2012 2013 adev->pm.pp_feature = amdgpu_pp_feature_mask; 2014 if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS) 2015 adev->pm.pp_feature &= ~PP_GFXOFF_MASK; 2016 2017 for (i = 0; i < adev->num_ip_blocks; i++) { 2018 if ((amdgpu_ip_block_mask & (1 << i)) == 0) { 2019 DRM_ERROR("disabled ip block: %d <%s>\n", 2020 i, adev->ip_blocks[i].version->funcs->name); 2021 adev->ip_blocks[i].status.valid = false; 2022 } else { 2023 if (adev->ip_blocks[i].version->funcs->early_init) { 2024 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev); 2025 if (r == -ENOENT) { 2026 adev->ip_blocks[i].status.valid = false; 2027 } else if (r) { 2028 DRM_ERROR("early_init of IP block <%s> failed %d\n", 2029 adev->ip_blocks[i].version->funcs->name, r); 2030 return r; 2031 } else { 2032 adev->ip_blocks[i].status.valid = true; 2033 } 2034 } else { 2035 adev->ip_blocks[i].status.valid = true; 2036 } 2037 } 2038 /* get the vbios after the asic_funcs are set up */ 2039 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) { 2040 r = amdgpu_device_parse_gpu_info_fw(adev); 2041 if (r) 2042 return r; 2043 2044 /* Read BIOS */ 2045 if (!amdgpu_get_bios(adev)) 2046 return -EINVAL; 2047 2048 r = amdgpu_atombios_init(adev); 2049 if (r) { 2050 dev_err(adev->dev, "amdgpu_atombios_init failed\n"); 2051 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0); 2052 return r; 2053 } 2054 } 2055 } 2056 2057 adev->cg_flags &= amdgpu_cg_mask; 2058 adev->pg_flags &= amdgpu_pg_mask; 2059 2060 return 0; 2061 } 2062 2063 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev) 2064 { 2065 int i, r; 2066 2067 for (i = 0; i < adev->num_ip_blocks; i++) { 2068 if (!adev->ip_blocks[i].status.sw) 2069 continue; 2070 if (adev->ip_blocks[i].status.hw) 2071 continue; 2072 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2073 (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) || 2074 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2075 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2076 if (r) { 2077 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2078 adev->ip_blocks[i].version->funcs->name, r); 2079 return r; 2080 } 2081 adev->ip_blocks[i].status.hw = true; 2082 } 2083 } 2084 2085 return 0; 2086 } 2087 2088 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev) 2089 { 2090 int i, r; 2091 2092 for (i = 0; i < adev->num_ip_blocks; i++) { 2093 if (!adev->ip_blocks[i].status.sw) 2094 continue; 2095 if (adev->ip_blocks[i].status.hw) 2096 continue; 2097 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2098 if (r) { 2099 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2100 adev->ip_blocks[i].version->funcs->name, r); 2101 return r; 2102 } 2103 adev->ip_blocks[i].status.hw = true; 2104 } 2105 2106 return 0; 2107 } 2108 2109 static int amdgpu_device_fw_loading(struct amdgpu_device *adev) 2110 { 2111 int r = 0; 2112 int i; 2113 uint32_t smu_version; 2114 2115 if (adev->asic_type >= CHIP_VEGA10) { 2116 for (i = 0; i < adev->num_ip_blocks; i++) { 2117 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP) 2118 continue; 2119 2120 /* no need to do the fw loading again if already done*/ 2121 if (adev->ip_blocks[i].status.hw == true) 2122 break; 2123 2124 if (amdgpu_in_reset(adev) || adev->in_suspend) { 2125 r = adev->ip_blocks[i].version->funcs->resume(adev); 2126 if (r) { 2127 DRM_ERROR("resume of IP block <%s> failed %d\n", 2128 adev->ip_blocks[i].version->funcs->name, r); 2129 return r; 2130 } 2131 } else { 2132 r = adev->ip_blocks[i].version->funcs->hw_init(adev); 2133 if (r) { 2134 DRM_ERROR("hw_init of IP block <%s> failed %d\n", 2135 adev->ip_blocks[i].version->funcs->name, r); 2136 return r; 2137 } 2138 } 2139 2140 adev->ip_blocks[i].status.hw = true; 2141 break; 2142 } 2143 } 2144 2145 if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA) 2146 r = amdgpu_pm_load_smu_firmware(adev, &smu_version); 2147 2148 return r; 2149 } 2150 2151 /** 2152 * amdgpu_device_ip_init - run init for hardware IPs 2153 * 2154 * @adev: amdgpu_device pointer 2155 * 2156 * Main initialization pass for hardware IPs. The list of all the hardware 2157 * IPs that make up the asic is walked and the sw_init and hw_init callbacks 2158 * are run. sw_init initializes the software state associated with each IP 2159 * and hw_init initializes the hardware associated with each IP. 2160 * Returns 0 on success, negative error code on failure. 2161 */ 2162 static int amdgpu_device_ip_init(struct amdgpu_device *adev) 2163 { 2164 int i, r; 2165 2166 r = amdgpu_ras_init(adev); 2167 if (r) 2168 return r; 2169 2170 for (i = 0; i < adev->num_ip_blocks; i++) { 2171 if (!adev->ip_blocks[i].status.valid) 2172 continue; 2173 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev); 2174 if (r) { 2175 DRM_ERROR("sw_init of IP block <%s> failed %d\n", 2176 adev->ip_blocks[i].version->funcs->name, r); 2177 goto init_failed; 2178 } 2179 adev->ip_blocks[i].status.sw = true; 2180 2181 /* need to do gmc hw init early so we can allocate gpu mem */ 2182 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2183 r = amdgpu_device_vram_scratch_init(adev); 2184 if (r) { 2185 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r); 2186 goto init_failed; 2187 } 2188 r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev); 2189 if (r) { 2190 DRM_ERROR("hw_init %d failed %d\n", i, r); 2191 goto init_failed; 2192 } 2193 r = amdgpu_device_wb_init(adev); 2194 if (r) { 2195 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r); 2196 goto init_failed; 2197 } 2198 adev->ip_blocks[i].status.hw = true; 2199 2200 /* right after GMC hw init, we create CSA */ 2201 if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) { 2202 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj, 2203 AMDGPU_GEM_DOMAIN_VRAM, 2204 AMDGPU_CSA_SIZE); 2205 if (r) { 2206 DRM_ERROR("allocate CSA failed %d\n", r); 2207 goto init_failed; 2208 } 2209 } 2210 } 2211 } 2212 2213 if (amdgpu_sriov_vf(adev)) 2214 amdgpu_virt_init_data_exchange(adev); 2215 2216 r = amdgpu_ib_pool_init(adev); 2217 if (r) { 2218 dev_err(adev->dev, "IB initialization failed (%d).\n", r); 2219 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r); 2220 goto init_failed; 2221 } 2222 2223 r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/ 2224 if (r) 2225 goto init_failed; 2226 2227 r = amdgpu_device_ip_hw_init_phase1(adev); 2228 if (r) 2229 goto init_failed; 2230 2231 r = amdgpu_device_fw_loading(adev); 2232 if (r) 2233 goto init_failed; 2234 2235 r = amdgpu_device_ip_hw_init_phase2(adev); 2236 if (r) 2237 goto init_failed; 2238 2239 /* 2240 * retired pages will be loaded from eeprom and reserved here, 2241 * it should be called after amdgpu_device_ip_hw_init_phase2 since 2242 * for some ASICs the RAS EEPROM code relies on SMU fully functioning 2243 * for I2C communication which only true at this point. 2244 * 2245 * amdgpu_ras_recovery_init may fail, but the upper only cares the 2246 * failure from bad gpu situation and stop amdgpu init process 2247 * accordingly. For other failed cases, it will still release all 2248 * the resource and print error message, rather than returning one 2249 * negative value to upper level. 2250 * 2251 * Note: theoretically, this should be called before all vram allocations 2252 * to protect retired page from abusing 2253 */ 2254 r = amdgpu_ras_recovery_init(adev); 2255 if (r) 2256 goto init_failed; 2257 2258 if (adev->gmc.xgmi.num_physical_nodes > 1) 2259 amdgpu_xgmi_add_device(adev); 2260 amdgpu_amdkfd_device_init(adev); 2261 2262 amdgpu_fru_get_product_info(adev); 2263 2264 init_failed: 2265 if (amdgpu_sriov_vf(adev)) 2266 amdgpu_virt_release_full_gpu(adev, true); 2267 2268 return r; 2269 } 2270 2271 /** 2272 * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer 2273 * 2274 * @adev: amdgpu_device pointer 2275 * 2276 * Writes a reset magic value to the gart pointer in VRAM. The driver calls 2277 * this function before a GPU reset. If the value is retained after a 2278 * GPU reset, VRAM has not been lost. Some GPU resets may destry VRAM contents. 2279 */ 2280 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev) 2281 { 2282 memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM); 2283 } 2284 2285 /** 2286 * amdgpu_device_check_vram_lost - check if vram is valid 2287 * 2288 * @adev: amdgpu_device pointer 2289 * 2290 * Checks the reset magic value written to the gart pointer in VRAM. 2291 * The driver calls this after a GPU reset to see if the contents of 2292 * VRAM is lost or now. 2293 * returns true if vram is lost, false if not. 2294 */ 2295 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) 2296 { 2297 if (memcmp(adev->gart.ptr, adev->reset_magic, 2298 AMDGPU_RESET_MAGIC_NUM)) 2299 return true; 2300 2301 if (!amdgpu_in_reset(adev)) 2302 return false; 2303 2304 /* 2305 * For all ASICs with baco/mode1 reset, the VRAM is 2306 * always assumed to be lost. 2307 */ 2308 switch (amdgpu_asic_reset_method(adev)) { 2309 case AMD_RESET_METHOD_BACO: 2310 case AMD_RESET_METHOD_MODE1: 2311 return true; 2312 default: 2313 return false; 2314 } 2315 } 2316 2317 /** 2318 * amdgpu_device_set_cg_state - set clockgating for amdgpu device 2319 * 2320 * @adev: amdgpu_device pointer 2321 * @state: clockgating state (gate or ungate) 2322 * 2323 * The list of all the hardware IPs that make up the asic is walked and the 2324 * set_clockgating_state callbacks are run. 2325 * Late initialization pass enabling clockgating for hardware IPs. 2326 * Fini or suspend, pass disabling clockgating for hardware IPs. 2327 * Returns 0 on success, negative error code on failure. 2328 */ 2329 2330 static int amdgpu_device_set_cg_state(struct amdgpu_device *adev, 2331 enum amd_clockgating_state state) 2332 { 2333 int i, j, r; 2334 2335 if (amdgpu_emu_mode == 1) 2336 return 0; 2337 2338 for (j = 0; j < adev->num_ip_blocks; j++) { 2339 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 2340 if (!adev->ip_blocks[i].status.late_initialized) 2341 continue; 2342 /* skip CG for VCE/UVD, it's handled specially */ 2343 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 2344 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 2345 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 2346 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 2347 adev->ip_blocks[i].version->funcs->set_clockgating_state) { 2348 /* enable clockgating to save power */ 2349 r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev, 2350 state); 2351 if (r) { 2352 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n", 2353 adev->ip_blocks[i].version->funcs->name, r); 2354 return r; 2355 } 2356 } 2357 } 2358 2359 return 0; 2360 } 2361 2362 static int amdgpu_device_set_pg_state(struct amdgpu_device *adev, enum amd_powergating_state state) 2363 { 2364 int i, j, r; 2365 2366 if (amdgpu_emu_mode == 1) 2367 return 0; 2368 2369 for (j = 0; j < adev->num_ip_blocks; j++) { 2370 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1; 2371 if (!adev->ip_blocks[i].status.late_initialized) 2372 continue; 2373 /* skip CG for VCE/UVD, it's handled specially */ 2374 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD && 2375 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE && 2376 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN && 2377 adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG && 2378 adev->ip_blocks[i].version->funcs->set_powergating_state) { 2379 /* enable powergating to save power */ 2380 r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev, 2381 state); 2382 if (r) { 2383 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n", 2384 adev->ip_blocks[i].version->funcs->name, r); 2385 return r; 2386 } 2387 } 2388 } 2389 return 0; 2390 } 2391 2392 static int amdgpu_device_enable_mgpu_fan_boost(void) 2393 { 2394 struct amdgpu_gpu_instance *gpu_ins; 2395 struct amdgpu_device *adev; 2396 int i, ret = 0; 2397 2398 mutex_lock(&mgpu_info.mutex); 2399 2400 /* 2401 * MGPU fan boost feature should be enabled 2402 * only when there are two or more dGPUs in 2403 * the system 2404 */ 2405 if (mgpu_info.num_dgpu < 2) 2406 goto out; 2407 2408 for (i = 0; i < mgpu_info.num_dgpu; i++) { 2409 gpu_ins = &(mgpu_info.gpu_ins[i]); 2410 adev = gpu_ins->adev; 2411 if (!(adev->flags & AMD_IS_APU) && 2412 !gpu_ins->mgpu_fan_enabled) { 2413 ret = amdgpu_dpm_enable_mgpu_fan_boost(adev); 2414 if (ret) 2415 break; 2416 2417 gpu_ins->mgpu_fan_enabled = 1; 2418 } 2419 } 2420 2421 out: 2422 mutex_unlock(&mgpu_info.mutex); 2423 2424 return ret; 2425 } 2426 2427 /** 2428 * amdgpu_device_ip_late_init - run late init for hardware IPs 2429 * 2430 * @adev: amdgpu_device pointer 2431 * 2432 * Late initialization pass for hardware IPs. The list of all the hardware 2433 * IPs that make up the asic is walked and the late_init callbacks are run. 2434 * late_init covers any special initialization that an IP requires 2435 * after all of the have been initialized or something that needs to happen 2436 * late in the init process. 2437 * Returns 0 on success, negative error code on failure. 2438 */ 2439 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev) 2440 { 2441 struct amdgpu_gpu_instance *gpu_instance; 2442 int i = 0, r; 2443 2444 for (i = 0; i < adev->num_ip_blocks; i++) { 2445 if (!adev->ip_blocks[i].status.hw) 2446 continue; 2447 if (adev->ip_blocks[i].version->funcs->late_init) { 2448 r = adev->ip_blocks[i].version->funcs->late_init((void *)adev); 2449 if (r) { 2450 DRM_ERROR("late_init of IP block <%s> failed %d\n", 2451 adev->ip_blocks[i].version->funcs->name, r); 2452 return r; 2453 } 2454 } 2455 adev->ip_blocks[i].status.late_initialized = true; 2456 } 2457 2458 amdgpu_ras_set_error_query_ready(adev, true); 2459 2460 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE); 2461 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE); 2462 2463 amdgpu_device_fill_reset_magic(adev); 2464 2465 r = amdgpu_device_enable_mgpu_fan_boost(); 2466 if (r) 2467 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r); 2468 2469 2470 if (adev->gmc.xgmi.num_physical_nodes > 1) { 2471 mutex_lock(&mgpu_info.mutex); 2472 2473 /* 2474 * Reset device p-state to low as this was booted with high. 2475 * 2476 * This should be performed only after all devices from the same 2477 * hive get initialized. 2478 * 2479 * However, it's unknown how many device in the hive in advance. 2480 * As this is counted one by one during devices initializations. 2481 * 2482 * So, we wait for all XGMI interlinked devices initialized. 2483 * This may bring some delays as those devices may come from 2484 * different hives. But that should be OK. 2485 */ 2486 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) { 2487 for (i = 0; i < mgpu_info.num_gpu; i++) { 2488 gpu_instance = &(mgpu_info.gpu_ins[i]); 2489 if (gpu_instance->adev->flags & AMD_IS_APU) 2490 continue; 2491 2492 r = amdgpu_xgmi_set_pstate(gpu_instance->adev, 2493 AMDGPU_XGMI_PSTATE_MIN); 2494 if (r) { 2495 DRM_ERROR("pstate setting failed (%d).\n", r); 2496 break; 2497 } 2498 } 2499 } 2500 2501 mutex_unlock(&mgpu_info.mutex); 2502 } 2503 2504 return 0; 2505 } 2506 2507 /** 2508 * amdgpu_device_ip_fini - run fini for hardware IPs 2509 * 2510 * @adev: amdgpu_device pointer 2511 * 2512 * Main teardown pass for hardware IPs. The list of all the hardware 2513 * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks 2514 * are run. hw_fini tears down the hardware associated with each IP 2515 * and sw_fini tears down any software state associated with each IP. 2516 * Returns 0 on success, negative error code on failure. 2517 */ 2518 static int amdgpu_device_ip_fini(struct amdgpu_device *adev) 2519 { 2520 int i, r; 2521 2522 if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done) 2523 amdgpu_virt_release_ras_err_handler_data(adev); 2524 2525 amdgpu_ras_pre_fini(adev); 2526 2527 if (adev->gmc.xgmi.num_physical_nodes > 1) 2528 amdgpu_xgmi_remove_device(adev); 2529 2530 amdgpu_amdkfd_device_fini(adev); 2531 2532 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 2533 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 2534 2535 /* need to disable SMC first */ 2536 for (i = 0; i < adev->num_ip_blocks; i++) { 2537 if (!adev->ip_blocks[i].status.hw) 2538 continue; 2539 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 2540 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); 2541 /* XXX handle errors */ 2542 if (r) { 2543 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 2544 adev->ip_blocks[i].version->funcs->name, r); 2545 } 2546 adev->ip_blocks[i].status.hw = false; 2547 break; 2548 } 2549 } 2550 2551 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2552 if (!adev->ip_blocks[i].status.hw) 2553 continue; 2554 2555 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev); 2556 /* XXX handle errors */ 2557 if (r) { 2558 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n", 2559 adev->ip_blocks[i].version->funcs->name, r); 2560 } 2561 2562 adev->ip_blocks[i].status.hw = false; 2563 } 2564 2565 2566 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2567 if (!adev->ip_blocks[i].status.sw) 2568 continue; 2569 2570 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) { 2571 amdgpu_ucode_free_bo(adev); 2572 amdgpu_free_static_csa(&adev->virt.csa_obj); 2573 amdgpu_device_wb_fini(adev); 2574 amdgpu_device_vram_scratch_fini(adev); 2575 amdgpu_ib_pool_fini(adev); 2576 } 2577 2578 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev); 2579 /* XXX handle errors */ 2580 if (r) { 2581 DRM_DEBUG("sw_fini of IP block <%s> failed %d\n", 2582 adev->ip_blocks[i].version->funcs->name, r); 2583 } 2584 adev->ip_blocks[i].status.sw = false; 2585 adev->ip_blocks[i].status.valid = false; 2586 } 2587 2588 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2589 if (!adev->ip_blocks[i].status.late_initialized) 2590 continue; 2591 if (adev->ip_blocks[i].version->funcs->late_fini) 2592 adev->ip_blocks[i].version->funcs->late_fini((void *)adev); 2593 adev->ip_blocks[i].status.late_initialized = false; 2594 } 2595 2596 amdgpu_ras_fini(adev); 2597 2598 if (amdgpu_sriov_vf(adev)) 2599 if (amdgpu_virt_release_full_gpu(adev, false)) 2600 DRM_ERROR("failed to release exclusive mode on fini\n"); 2601 2602 return 0; 2603 } 2604 2605 /** 2606 * amdgpu_device_delayed_init_work_handler - work handler for IB tests 2607 * 2608 * @work: work_struct. 2609 */ 2610 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work) 2611 { 2612 struct amdgpu_device *adev = 2613 container_of(work, struct amdgpu_device, delayed_init_work.work); 2614 int r; 2615 2616 r = amdgpu_ib_ring_tests(adev); 2617 if (r) 2618 DRM_ERROR("ib ring test failed (%d).\n", r); 2619 } 2620 2621 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work) 2622 { 2623 struct amdgpu_device *adev = 2624 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work); 2625 2626 mutex_lock(&adev->gfx.gfx_off_mutex); 2627 if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) { 2628 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true)) 2629 adev->gfx.gfx_off_state = true; 2630 } 2631 mutex_unlock(&adev->gfx.gfx_off_mutex); 2632 } 2633 2634 /** 2635 * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1) 2636 * 2637 * @adev: amdgpu_device pointer 2638 * 2639 * Main suspend function for hardware IPs. The list of all the hardware 2640 * IPs that make up the asic is walked, clockgating is disabled and the 2641 * suspend callbacks are run. suspend puts the hardware and software state 2642 * in each IP into a state suitable for suspend. 2643 * Returns 0 on success, negative error code on failure. 2644 */ 2645 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev) 2646 { 2647 int i, r; 2648 2649 amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE); 2650 amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE); 2651 2652 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2653 if (!adev->ip_blocks[i].status.valid) 2654 continue; 2655 2656 /* displays are handled separately */ 2657 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE) 2658 continue; 2659 2660 /* XXX handle errors */ 2661 r = adev->ip_blocks[i].version->funcs->suspend(adev); 2662 /* XXX handle errors */ 2663 if (r) { 2664 DRM_ERROR("suspend of IP block <%s> failed %d\n", 2665 adev->ip_blocks[i].version->funcs->name, r); 2666 return r; 2667 } 2668 2669 adev->ip_blocks[i].status.hw = false; 2670 } 2671 2672 return 0; 2673 } 2674 2675 /** 2676 * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2) 2677 * 2678 * @adev: amdgpu_device pointer 2679 * 2680 * Main suspend function for hardware IPs. The list of all the hardware 2681 * IPs that make up the asic is walked, clockgating is disabled and the 2682 * suspend callbacks are run. suspend puts the hardware and software state 2683 * in each IP into a state suitable for suspend. 2684 * Returns 0 on success, negative error code on failure. 2685 */ 2686 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev) 2687 { 2688 int i, r; 2689 2690 for (i = adev->num_ip_blocks - 1; i >= 0; i--) { 2691 if (!adev->ip_blocks[i].status.valid) 2692 continue; 2693 /* displays are handled in phase1 */ 2694 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) 2695 continue; 2696 /* PSP lost connection when err_event_athub occurs */ 2697 if (amdgpu_ras_intr_triggered() && 2698 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 2699 adev->ip_blocks[i].status.hw = false; 2700 continue; 2701 } 2702 /* XXX handle errors */ 2703 r = adev->ip_blocks[i].version->funcs->suspend(adev); 2704 /* XXX handle errors */ 2705 if (r) { 2706 DRM_ERROR("suspend of IP block <%s> failed %d\n", 2707 adev->ip_blocks[i].version->funcs->name, r); 2708 } 2709 adev->ip_blocks[i].status.hw = false; 2710 /* handle putting the SMC in the appropriate state */ 2711 if(!amdgpu_sriov_vf(adev)){ 2712 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) { 2713 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state); 2714 if (r) { 2715 DRM_ERROR("SMC failed to set mp1 state %d, %d\n", 2716 adev->mp1_state, r); 2717 return r; 2718 } 2719 } 2720 } 2721 adev->ip_blocks[i].status.hw = false; 2722 } 2723 2724 return 0; 2725 } 2726 2727 /** 2728 * amdgpu_device_ip_suspend - run suspend for hardware IPs 2729 * 2730 * @adev: amdgpu_device pointer 2731 * 2732 * Main suspend function for hardware IPs. The list of all the hardware 2733 * IPs that make up the asic is walked, clockgating is disabled and the 2734 * suspend callbacks are run. suspend puts the hardware and software state 2735 * in each IP into a state suitable for suspend. 2736 * Returns 0 on success, negative error code on failure. 2737 */ 2738 int amdgpu_device_ip_suspend(struct amdgpu_device *adev) 2739 { 2740 int r; 2741 2742 if (amdgpu_sriov_vf(adev)) 2743 amdgpu_virt_request_full_gpu(adev, false); 2744 2745 r = amdgpu_device_ip_suspend_phase1(adev); 2746 if (r) 2747 return r; 2748 r = amdgpu_device_ip_suspend_phase2(adev); 2749 2750 if (amdgpu_sriov_vf(adev)) 2751 amdgpu_virt_release_full_gpu(adev, false); 2752 2753 return r; 2754 } 2755 2756 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) 2757 { 2758 int i, r; 2759 2760 static enum amd_ip_block_type ip_order[] = { 2761 AMD_IP_BLOCK_TYPE_GMC, 2762 AMD_IP_BLOCK_TYPE_COMMON, 2763 AMD_IP_BLOCK_TYPE_PSP, 2764 AMD_IP_BLOCK_TYPE_IH, 2765 }; 2766 2767 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 2768 int j; 2769 struct amdgpu_ip_block *block; 2770 2771 block = &adev->ip_blocks[i]; 2772 block->status.hw = false; 2773 2774 for (j = 0; j < ARRAY_SIZE(ip_order); j++) { 2775 2776 if (block->version->type != ip_order[j] || 2777 !block->status.valid) 2778 continue; 2779 2780 r = block->version->funcs->hw_init(adev); 2781 DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); 2782 if (r) 2783 return r; 2784 block->status.hw = true; 2785 } 2786 } 2787 2788 return 0; 2789 } 2790 2791 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev) 2792 { 2793 int i, r; 2794 2795 static enum amd_ip_block_type ip_order[] = { 2796 AMD_IP_BLOCK_TYPE_SMC, 2797 AMD_IP_BLOCK_TYPE_DCE, 2798 AMD_IP_BLOCK_TYPE_GFX, 2799 AMD_IP_BLOCK_TYPE_SDMA, 2800 AMD_IP_BLOCK_TYPE_UVD, 2801 AMD_IP_BLOCK_TYPE_VCE, 2802 AMD_IP_BLOCK_TYPE_VCN 2803 }; 2804 2805 for (i = 0; i < ARRAY_SIZE(ip_order); i++) { 2806 int j; 2807 struct amdgpu_ip_block *block; 2808 2809 for (j = 0; j < adev->num_ip_blocks; j++) { 2810 block = &adev->ip_blocks[j]; 2811 2812 if (block->version->type != ip_order[i] || 2813 !block->status.valid || 2814 block->status.hw) 2815 continue; 2816 2817 if (block->version->type == AMD_IP_BLOCK_TYPE_SMC) 2818 r = block->version->funcs->resume(adev); 2819 else 2820 r = block->version->funcs->hw_init(adev); 2821 2822 DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded"); 2823 if (r) 2824 return r; 2825 block->status.hw = true; 2826 } 2827 } 2828 2829 return 0; 2830 } 2831 2832 /** 2833 * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs 2834 * 2835 * @adev: amdgpu_device pointer 2836 * 2837 * First resume function for hardware IPs. The list of all the hardware 2838 * IPs that make up the asic is walked and the resume callbacks are run for 2839 * COMMON, GMC, and IH. resume puts the hardware into a functional state 2840 * after a suspend and updates the software state as necessary. This 2841 * function is also used for restoring the GPU after a GPU reset. 2842 * Returns 0 on success, negative error code on failure. 2843 */ 2844 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev) 2845 { 2846 int i, r; 2847 2848 for (i = 0; i < adev->num_ip_blocks; i++) { 2849 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 2850 continue; 2851 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2852 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 2853 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) { 2854 2855 r = adev->ip_blocks[i].version->funcs->resume(adev); 2856 if (r) { 2857 DRM_ERROR("resume of IP block <%s> failed %d\n", 2858 adev->ip_blocks[i].version->funcs->name, r); 2859 return r; 2860 } 2861 adev->ip_blocks[i].status.hw = true; 2862 } 2863 } 2864 2865 return 0; 2866 } 2867 2868 /** 2869 * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs 2870 * 2871 * @adev: amdgpu_device pointer 2872 * 2873 * First resume function for hardware IPs. The list of all the hardware 2874 * IPs that make up the asic is walked and the resume callbacks are run for 2875 * all blocks except COMMON, GMC, and IH. resume puts the hardware into a 2876 * functional state after a suspend and updates the software state as 2877 * necessary. This function is also used for restoring the GPU after a GPU 2878 * reset. 2879 * Returns 0 on success, negative error code on failure. 2880 */ 2881 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev) 2882 { 2883 int i, r; 2884 2885 for (i = 0; i < adev->num_ip_blocks; i++) { 2886 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw) 2887 continue; 2888 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON || 2889 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC || 2890 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH || 2891 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) 2892 continue; 2893 r = adev->ip_blocks[i].version->funcs->resume(adev); 2894 if (r) { 2895 DRM_ERROR("resume of IP block <%s> failed %d\n", 2896 adev->ip_blocks[i].version->funcs->name, r); 2897 return r; 2898 } 2899 adev->ip_blocks[i].status.hw = true; 2900 } 2901 2902 return 0; 2903 } 2904 2905 /** 2906 * amdgpu_device_ip_resume - run resume for hardware IPs 2907 * 2908 * @adev: amdgpu_device pointer 2909 * 2910 * Main resume function for hardware IPs. The hardware IPs 2911 * are split into two resume functions because they are 2912 * are also used in in recovering from a GPU reset and some additional 2913 * steps need to be take between them. In this case (S3/S4) they are 2914 * run sequentially. 2915 * Returns 0 on success, negative error code on failure. 2916 */ 2917 static int amdgpu_device_ip_resume(struct amdgpu_device *adev) 2918 { 2919 int r; 2920 2921 r = amdgpu_device_ip_resume_phase1(adev); 2922 if (r) 2923 return r; 2924 2925 r = amdgpu_device_fw_loading(adev); 2926 if (r) 2927 return r; 2928 2929 r = amdgpu_device_ip_resume_phase2(adev); 2930 2931 return r; 2932 } 2933 2934 /** 2935 * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV 2936 * 2937 * @adev: amdgpu_device pointer 2938 * 2939 * Query the VBIOS data tables to determine if the board supports SR-IOV. 2940 */ 2941 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev) 2942 { 2943 if (amdgpu_sriov_vf(adev)) { 2944 if (adev->is_atom_fw) { 2945 if (amdgpu_atomfirmware_gpu_supports_virtualization(adev)) 2946 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 2947 } else { 2948 if (amdgpu_atombios_has_gpu_virtualization_table(adev)) 2949 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS; 2950 } 2951 2952 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS)) 2953 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0); 2954 } 2955 } 2956 2957 /** 2958 * amdgpu_device_asic_has_dc_support - determine if DC supports the asic 2959 * 2960 * @asic_type: AMD asic type 2961 * 2962 * Check if there is DC (new modesetting infrastructre) support for an asic. 2963 * returns true if DC has support, false if not. 2964 */ 2965 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type) 2966 { 2967 switch (asic_type) { 2968 #if defined(CONFIG_DRM_AMD_DC) 2969 #if defined(CONFIG_DRM_AMD_DC_SI) 2970 case CHIP_TAHITI: 2971 case CHIP_PITCAIRN: 2972 case CHIP_VERDE: 2973 case CHIP_OLAND: 2974 #endif 2975 case CHIP_BONAIRE: 2976 case CHIP_KAVERI: 2977 case CHIP_KABINI: 2978 case CHIP_MULLINS: 2979 /* 2980 * We have systems in the wild with these ASICs that require 2981 * LVDS and VGA support which is not supported with DC. 2982 * 2983 * Fallback to the non-DC driver here by default so as not to 2984 * cause regressions. 2985 */ 2986 return amdgpu_dc > 0; 2987 case CHIP_HAWAII: 2988 case CHIP_CARRIZO: 2989 case CHIP_STONEY: 2990 case CHIP_POLARIS10: 2991 case CHIP_POLARIS11: 2992 case CHIP_POLARIS12: 2993 case CHIP_VEGAM: 2994 case CHIP_TONGA: 2995 case CHIP_FIJI: 2996 case CHIP_VEGA10: 2997 case CHIP_VEGA12: 2998 case CHIP_VEGA20: 2999 #if defined(CONFIG_DRM_AMD_DC_DCN) 3000 case CHIP_RAVEN: 3001 case CHIP_NAVI10: 3002 case CHIP_NAVI14: 3003 case CHIP_NAVI12: 3004 case CHIP_RENOIR: 3005 #endif 3006 #if defined(CONFIG_DRM_AMD_DC_DCN3_0) 3007 case CHIP_SIENNA_CICHLID: 3008 case CHIP_NAVY_FLOUNDER: 3009 case CHIP_DIMGREY_CAVEFISH: 3010 case CHIP_VANGOGH: 3011 #endif 3012 return amdgpu_dc != 0; 3013 #endif 3014 default: 3015 if (amdgpu_dc > 0) 3016 DRM_INFO("Display Core has been requested via kernel parameter " 3017 "but isn't supported by ASIC, ignoring\n"); 3018 return false; 3019 } 3020 } 3021 3022 /** 3023 * amdgpu_device_has_dc_support - check if dc is supported 3024 * 3025 * @adev: amdgpu_device pointer 3026 * 3027 * Returns true for supported, false for not supported 3028 */ 3029 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev) 3030 { 3031 if (amdgpu_sriov_vf(adev) || adev->enable_virtual_display) 3032 return false; 3033 3034 return amdgpu_device_asic_has_dc_support(adev->asic_type); 3035 } 3036 3037 3038 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work) 3039 { 3040 struct amdgpu_device *adev = 3041 container_of(__work, struct amdgpu_device, xgmi_reset_work); 3042 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev); 3043 3044 /* It's a bug to not have a hive within this function */ 3045 if (WARN_ON(!hive)) 3046 return; 3047 3048 /* 3049 * Use task barrier to synchronize all xgmi reset works across the 3050 * hive. task_barrier_enter and task_barrier_exit will block 3051 * until all the threads running the xgmi reset works reach 3052 * those points. task_barrier_full will do both blocks. 3053 */ 3054 if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) { 3055 3056 task_barrier_enter(&hive->tb); 3057 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev)); 3058 3059 if (adev->asic_reset_res) 3060 goto fail; 3061 3062 task_barrier_exit(&hive->tb); 3063 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev)); 3064 3065 if (adev->asic_reset_res) 3066 goto fail; 3067 3068 if (adev->mmhub.funcs && adev->mmhub.funcs->reset_ras_error_count) 3069 adev->mmhub.funcs->reset_ras_error_count(adev); 3070 } else { 3071 3072 task_barrier_full(&hive->tb); 3073 adev->asic_reset_res = amdgpu_asic_reset(adev); 3074 } 3075 3076 fail: 3077 if (adev->asic_reset_res) 3078 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s", 3079 adev->asic_reset_res, adev_to_drm(adev)->unique); 3080 amdgpu_put_xgmi_hive(hive); 3081 } 3082 3083 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) 3084 { 3085 char *input = amdgpu_lockup_timeout; 3086 char *timeout_setting = NULL; 3087 int index = 0; 3088 long timeout; 3089 int ret = 0; 3090 3091 /* 3092 * By default timeout for non compute jobs is 10000. 3093 * And there is no timeout enforced on compute jobs. 3094 * In SR-IOV or passthrough mode, timeout for compute 3095 * jobs are 60000 by default. 3096 */ 3097 adev->gfx_timeout = msecs_to_jiffies(10000); 3098 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 3099 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) 3100 adev->compute_timeout = msecs_to_jiffies(60000); 3101 else 3102 adev->compute_timeout = MAX_SCHEDULE_TIMEOUT; 3103 3104 if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 3105 while ((timeout_setting = strsep(&input, ",")) && 3106 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) { 3107 ret = kstrtol(timeout_setting, 0, &timeout); 3108 if (ret) 3109 return ret; 3110 3111 if (timeout == 0) { 3112 index++; 3113 continue; 3114 } else if (timeout < 0) { 3115 timeout = MAX_SCHEDULE_TIMEOUT; 3116 } else { 3117 timeout = msecs_to_jiffies(timeout); 3118 } 3119 3120 switch (index++) { 3121 case 0: 3122 adev->gfx_timeout = timeout; 3123 break; 3124 case 1: 3125 adev->compute_timeout = timeout; 3126 break; 3127 case 2: 3128 adev->sdma_timeout = timeout; 3129 break; 3130 case 3: 3131 adev->video_timeout = timeout; 3132 break; 3133 default: 3134 break; 3135 } 3136 } 3137 /* 3138 * There is only one value specified and 3139 * it should apply to all non-compute jobs. 3140 */ 3141 if (index == 1) { 3142 adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout; 3143 if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev)) 3144 adev->compute_timeout = adev->gfx_timeout; 3145 } 3146 } 3147 3148 return ret; 3149 } 3150 3151 static const struct attribute *amdgpu_dev_attributes[] = { 3152 &dev_attr_product_name.attr, 3153 &dev_attr_product_number.attr, 3154 &dev_attr_serial_number.attr, 3155 &dev_attr_pcie_replay_count.attr, 3156 NULL 3157 }; 3158 3159 3160 /** 3161 * amdgpu_device_init - initialize the driver 3162 * 3163 * @adev: amdgpu_device pointer 3164 * @flags: driver flags 3165 * 3166 * Initializes the driver info and hw (all asics). 3167 * Returns 0 for success or an error on failure. 3168 * Called at driver startup. 3169 */ 3170 int amdgpu_device_init(struct amdgpu_device *adev, 3171 uint32_t flags) 3172 { 3173 struct drm_device *ddev = adev_to_drm(adev); 3174 struct pci_dev *pdev = adev->pdev; 3175 int r, i; 3176 bool boco = false; 3177 u32 max_MBps; 3178 3179 adev->shutdown = false; 3180 adev->flags = flags; 3181 3182 if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST) 3183 adev->asic_type = amdgpu_force_asic_type; 3184 else 3185 adev->asic_type = flags & AMD_ASIC_MASK; 3186 3187 adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT; 3188 if (amdgpu_emu_mode == 1) 3189 adev->usec_timeout *= 10; 3190 adev->gmc.gart_size = 512 * 1024 * 1024; 3191 adev->accel_working = false; 3192 adev->num_rings = 0; 3193 adev->mman.buffer_funcs = NULL; 3194 adev->mman.buffer_funcs_ring = NULL; 3195 adev->vm_manager.vm_pte_funcs = NULL; 3196 adev->vm_manager.vm_pte_num_scheds = 0; 3197 adev->gmc.gmc_funcs = NULL; 3198 adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS); 3199 bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES); 3200 3201 adev->smc_rreg = &amdgpu_invalid_rreg; 3202 adev->smc_wreg = &amdgpu_invalid_wreg; 3203 adev->pcie_rreg = &amdgpu_invalid_rreg; 3204 adev->pcie_wreg = &amdgpu_invalid_wreg; 3205 adev->pciep_rreg = &amdgpu_invalid_rreg; 3206 adev->pciep_wreg = &amdgpu_invalid_wreg; 3207 adev->pcie_rreg64 = &amdgpu_invalid_rreg64; 3208 adev->pcie_wreg64 = &amdgpu_invalid_wreg64; 3209 adev->uvd_ctx_rreg = &amdgpu_invalid_rreg; 3210 adev->uvd_ctx_wreg = &amdgpu_invalid_wreg; 3211 adev->didt_rreg = &amdgpu_invalid_rreg; 3212 adev->didt_wreg = &amdgpu_invalid_wreg; 3213 adev->gc_cac_rreg = &amdgpu_invalid_rreg; 3214 adev->gc_cac_wreg = &amdgpu_invalid_wreg; 3215 adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg; 3216 adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg; 3217 3218 DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n", 3219 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device, 3220 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision); 3221 3222 /* mutex initialization are all done here so we 3223 * can recall function without having locking issues */ 3224 atomic_set(&adev->irq.ih.lock, 0); 3225 mutex_init(&adev->firmware.mutex); 3226 mutex_init(&adev->pm.mutex); 3227 mutex_init(&adev->gfx.gpu_clock_mutex); 3228 mutex_init(&adev->srbm_mutex); 3229 mutex_init(&adev->gfx.pipe_reserve_mutex); 3230 mutex_init(&adev->gfx.gfx_off_mutex); 3231 mutex_init(&adev->grbm_idx_mutex); 3232 mutex_init(&adev->mn_lock); 3233 mutex_init(&adev->virt.vf_errors.lock); 3234 hash_init(adev->mn_hash); 3235 atomic_set(&adev->in_gpu_reset, 0); 3236 init_rwsem(&adev->reset_sem); 3237 mutex_init(&adev->psp.mutex); 3238 mutex_init(&adev->notifier_lock); 3239 3240 r = amdgpu_device_check_arguments(adev); 3241 if (r) 3242 return r; 3243 3244 spin_lock_init(&adev->mmio_idx_lock); 3245 spin_lock_init(&adev->smc_idx_lock); 3246 spin_lock_init(&adev->pcie_idx_lock); 3247 spin_lock_init(&adev->uvd_ctx_idx_lock); 3248 spin_lock_init(&adev->didt_idx_lock); 3249 spin_lock_init(&adev->gc_cac_idx_lock); 3250 spin_lock_init(&adev->se_cac_idx_lock); 3251 spin_lock_init(&adev->audio_endpt_idx_lock); 3252 spin_lock_init(&adev->mm_stats.lock); 3253 3254 INIT_LIST_HEAD(&adev->shadow_list); 3255 mutex_init(&adev->shadow_list_lock); 3256 3257 INIT_DELAYED_WORK(&adev->delayed_init_work, 3258 amdgpu_device_delayed_init_work_handler); 3259 INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work, 3260 amdgpu_device_delay_enable_gfx_off); 3261 3262 INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func); 3263 3264 adev->gfx.gfx_off_req_count = 1; 3265 adev->pm.ac_power = power_supply_is_system_supplied() > 0; 3266 3267 atomic_set(&adev->throttling_logging_enabled, 1); 3268 /* 3269 * If throttling continues, logging will be performed every minute 3270 * to avoid log flooding. "-1" is subtracted since the thermal 3271 * throttling interrupt comes every second. Thus, the total logging 3272 * interval is 59 seconds(retelimited printk interval) + 1(waiting 3273 * for throttling interrupt) = 60 seconds. 3274 */ 3275 ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1); 3276 ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE); 3277 3278 /* Registers mapping */ 3279 /* TODO: block userspace mapping of io register */ 3280 if (adev->asic_type >= CHIP_BONAIRE) { 3281 adev->rmmio_base = pci_resource_start(adev->pdev, 5); 3282 adev->rmmio_size = pci_resource_len(adev->pdev, 5); 3283 } else { 3284 adev->rmmio_base = pci_resource_start(adev->pdev, 2); 3285 adev->rmmio_size = pci_resource_len(adev->pdev, 2); 3286 } 3287 3288 adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size); 3289 if (adev->rmmio == NULL) { 3290 return -ENOMEM; 3291 } 3292 DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base); 3293 DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size); 3294 3295 /* io port mapping */ 3296 for (i = 0; i < DEVICE_COUNT_RESOURCE; i++) { 3297 if (pci_resource_flags(adev->pdev, i) & IORESOURCE_IO) { 3298 adev->rio_mem_size = pci_resource_len(adev->pdev, i); 3299 adev->rio_mem = pci_iomap(adev->pdev, i, adev->rio_mem_size); 3300 break; 3301 } 3302 } 3303 if (adev->rio_mem == NULL) 3304 DRM_INFO("PCI I/O BAR is not found.\n"); 3305 3306 /* enable PCIE atomic ops */ 3307 r = pci_enable_atomic_ops_to_root(adev->pdev, 3308 PCI_EXP_DEVCAP2_ATOMIC_COMP32 | 3309 PCI_EXP_DEVCAP2_ATOMIC_COMP64); 3310 if (r) { 3311 adev->have_atomics_support = false; 3312 DRM_INFO("PCIE atomic ops is not supported\n"); 3313 } else { 3314 adev->have_atomics_support = true; 3315 } 3316 3317 amdgpu_device_get_pcie_info(adev); 3318 3319 if (amdgpu_mcbp) 3320 DRM_INFO("MCBP is enabled\n"); 3321 3322 if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10) 3323 adev->enable_mes = true; 3324 3325 /* detect hw virtualization here */ 3326 amdgpu_detect_virtualization(adev); 3327 3328 r = amdgpu_device_get_job_timeout_settings(adev); 3329 if (r) { 3330 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n"); 3331 goto failed_unmap; 3332 } 3333 3334 /* early init functions */ 3335 r = amdgpu_device_ip_early_init(adev); 3336 if (r) 3337 goto failed_unmap; 3338 3339 /* doorbell bar mapping and doorbell index init*/ 3340 amdgpu_device_doorbell_init(adev); 3341 3342 /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */ 3343 /* this will fail for cards that aren't VGA class devices, just 3344 * ignore it */ 3345 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode); 3346 3347 if (amdgpu_device_supports_boco(ddev)) 3348 boco = true; 3349 if (amdgpu_has_atpx() && 3350 (amdgpu_is_atpx_hybrid() || 3351 amdgpu_has_atpx_dgpu_power_cntl()) && 3352 !pci_is_thunderbolt_attached(adev->pdev)) 3353 vga_switcheroo_register_client(adev->pdev, 3354 &amdgpu_switcheroo_ops, boco); 3355 if (boco) 3356 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain); 3357 3358 if (amdgpu_emu_mode == 1) { 3359 /* post the asic on emulation mode */ 3360 emu_soc_asic_init(adev); 3361 goto fence_driver_init; 3362 } 3363 3364 /* detect if we are with an SRIOV vbios */ 3365 amdgpu_device_detect_sriov_bios(adev); 3366 3367 /* check if we need to reset the asic 3368 * E.g., driver was not cleanly unloaded previously, etc. 3369 */ 3370 if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) { 3371 r = amdgpu_asic_reset(adev); 3372 if (r) { 3373 dev_err(adev->dev, "asic reset on init failed\n"); 3374 goto failed; 3375 } 3376 } 3377 3378 pci_enable_pcie_error_reporting(adev->ddev.pdev); 3379 3380 /* Post card if necessary */ 3381 if (amdgpu_device_need_post(adev)) { 3382 if (!adev->bios) { 3383 dev_err(adev->dev, "no vBIOS found\n"); 3384 r = -EINVAL; 3385 goto failed; 3386 } 3387 DRM_INFO("GPU posting now...\n"); 3388 r = amdgpu_device_asic_init(adev); 3389 if (r) { 3390 dev_err(adev->dev, "gpu post error!\n"); 3391 goto failed; 3392 } 3393 } 3394 3395 if (adev->is_atom_fw) { 3396 /* Initialize clocks */ 3397 r = amdgpu_atomfirmware_get_clock_info(adev); 3398 if (r) { 3399 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n"); 3400 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 3401 goto failed; 3402 } 3403 } else { 3404 /* Initialize clocks */ 3405 r = amdgpu_atombios_get_clock_info(adev); 3406 if (r) { 3407 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n"); 3408 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0); 3409 goto failed; 3410 } 3411 /* init i2c buses */ 3412 if (!amdgpu_device_has_dc_support(adev)) 3413 amdgpu_atombios_i2c_init(adev); 3414 } 3415 3416 fence_driver_init: 3417 /* Fence driver */ 3418 r = amdgpu_fence_driver_init(adev); 3419 if (r) { 3420 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n"); 3421 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0); 3422 goto failed; 3423 } 3424 3425 /* init the mode config */ 3426 drm_mode_config_init(adev_to_drm(adev)); 3427 3428 r = amdgpu_device_ip_init(adev); 3429 if (r) { 3430 /* failed in exclusive mode due to timeout */ 3431 if (amdgpu_sriov_vf(adev) && 3432 !amdgpu_sriov_runtime(adev) && 3433 amdgpu_virt_mmio_blocked(adev) && 3434 !amdgpu_virt_wait_reset(adev)) { 3435 dev_err(adev->dev, "VF exclusive mode timeout\n"); 3436 /* Don't send request since VF is inactive. */ 3437 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME; 3438 adev->virt.ops = NULL; 3439 r = -EAGAIN; 3440 goto failed; 3441 } 3442 dev_err(adev->dev, "amdgpu_device_ip_init failed\n"); 3443 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0); 3444 goto failed; 3445 } 3446 3447 dev_info(adev->dev, 3448 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n", 3449 adev->gfx.config.max_shader_engines, 3450 adev->gfx.config.max_sh_per_se, 3451 adev->gfx.config.max_cu_per_sh, 3452 adev->gfx.cu_info.number); 3453 3454 adev->accel_working = true; 3455 3456 amdgpu_vm_check_compute_bug(adev); 3457 3458 /* Initialize the buffer migration limit. */ 3459 if (amdgpu_moverate >= 0) 3460 max_MBps = amdgpu_moverate; 3461 else 3462 max_MBps = 8; /* Allow 8 MB/s. */ 3463 /* Get a log2 for easy divisions. */ 3464 adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps)); 3465 3466 amdgpu_fbdev_init(adev); 3467 3468 r = amdgpu_pm_sysfs_init(adev); 3469 if (r) { 3470 adev->pm_sysfs_en = false; 3471 DRM_ERROR("registering pm debugfs failed (%d).\n", r); 3472 } else 3473 adev->pm_sysfs_en = true; 3474 3475 r = amdgpu_ucode_sysfs_init(adev); 3476 if (r) { 3477 adev->ucode_sysfs_en = false; 3478 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r); 3479 } else 3480 adev->ucode_sysfs_en = true; 3481 3482 if ((amdgpu_testing & 1)) { 3483 if (adev->accel_working) 3484 amdgpu_test_moves(adev); 3485 else 3486 DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n"); 3487 } 3488 if (amdgpu_benchmarking) { 3489 if (adev->accel_working) 3490 amdgpu_benchmark(adev, amdgpu_benchmarking); 3491 else 3492 DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n"); 3493 } 3494 3495 /* 3496 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost. 3497 * Otherwise the mgpu fan boost feature will be skipped due to the 3498 * gpu instance is counted less. 3499 */ 3500 amdgpu_register_gpu_instance(adev); 3501 3502 /* enable clockgating, etc. after ib tests, etc. since some blocks require 3503 * explicit gating rather than handling it automatically. 3504 */ 3505 r = amdgpu_device_ip_late_init(adev); 3506 if (r) { 3507 dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n"); 3508 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r); 3509 goto failed; 3510 } 3511 3512 /* must succeed. */ 3513 amdgpu_ras_resume(adev); 3514 3515 queue_delayed_work(system_wq, &adev->delayed_init_work, 3516 msecs_to_jiffies(AMDGPU_RESUME_MS)); 3517 3518 if (amdgpu_sriov_vf(adev)) 3519 flush_delayed_work(&adev->delayed_init_work); 3520 3521 r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes); 3522 if (r) 3523 dev_err(adev->dev, "Could not create amdgpu device attr\n"); 3524 3525 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 3526 r = amdgpu_pmu_init(adev); 3527 if (r) 3528 dev_err(adev->dev, "amdgpu_pmu_init failed\n"); 3529 3530 /* Have stored pci confspace at hand for restore in sudden PCI error */ 3531 if (amdgpu_device_cache_pci_state(adev->pdev)) 3532 pci_restore_state(pdev); 3533 3534 return 0; 3535 3536 failed: 3537 amdgpu_vf_error_trans_all(adev); 3538 if (boco) 3539 vga_switcheroo_fini_domain_pm_ops(adev->dev); 3540 3541 failed_unmap: 3542 iounmap(adev->rmmio); 3543 adev->rmmio = NULL; 3544 3545 return r; 3546 } 3547 3548 /** 3549 * amdgpu_device_fini - tear down the driver 3550 * 3551 * @adev: amdgpu_device pointer 3552 * 3553 * Tear down the driver info (all asics). 3554 * Called at driver shutdown. 3555 */ 3556 void amdgpu_device_fini(struct amdgpu_device *adev) 3557 { 3558 dev_info(adev->dev, "amdgpu: finishing device.\n"); 3559 flush_delayed_work(&adev->delayed_init_work); 3560 adev->shutdown = true; 3561 3562 kfree(adev->pci_state); 3563 3564 /* make sure IB test finished before entering exclusive mode 3565 * to avoid preemption on IB test 3566 * */ 3567 if (amdgpu_sriov_vf(adev)) { 3568 amdgpu_virt_request_full_gpu(adev, false); 3569 amdgpu_virt_fini_data_exchange(adev); 3570 } 3571 3572 /* disable all interrupts */ 3573 amdgpu_irq_disable_all(adev); 3574 if (adev->mode_info.mode_config_initialized){ 3575 if (!amdgpu_device_has_dc_support(adev)) 3576 drm_helper_force_disable_all(adev_to_drm(adev)); 3577 else 3578 drm_atomic_helper_shutdown(adev_to_drm(adev)); 3579 } 3580 amdgpu_fence_driver_fini(adev); 3581 if (adev->pm_sysfs_en) 3582 amdgpu_pm_sysfs_fini(adev); 3583 amdgpu_fbdev_fini(adev); 3584 amdgpu_device_ip_fini(adev); 3585 release_firmware(adev->firmware.gpu_info_fw); 3586 adev->firmware.gpu_info_fw = NULL; 3587 adev->accel_working = false; 3588 /* free i2c buses */ 3589 if (!amdgpu_device_has_dc_support(adev)) 3590 amdgpu_i2c_fini(adev); 3591 3592 if (amdgpu_emu_mode != 1) 3593 amdgpu_atombios_fini(adev); 3594 3595 kfree(adev->bios); 3596 adev->bios = NULL; 3597 if (amdgpu_has_atpx() && 3598 (amdgpu_is_atpx_hybrid() || 3599 amdgpu_has_atpx_dgpu_power_cntl()) && 3600 !pci_is_thunderbolt_attached(adev->pdev)) 3601 vga_switcheroo_unregister_client(adev->pdev); 3602 if (amdgpu_device_supports_boco(adev_to_drm(adev))) 3603 vga_switcheroo_fini_domain_pm_ops(adev->dev); 3604 vga_client_register(adev->pdev, NULL, NULL, NULL); 3605 if (adev->rio_mem) 3606 pci_iounmap(adev->pdev, adev->rio_mem); 3607 adev->rio_mem = NULL; 3608 iounmap(adev->rmmio); 3609 adev->rmmio = NULL; 3610 amdgpu_device_doorbell_fini(adev); 3611 3612 if (adev->ucode_sysfs_en) 3613 amdgpu_ucode_sysfs_fini(adev); 3614 3615 sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes); 3616 if (IS_ENABLED(CONFIG_PERF_EVENTS)) 3617 amdgpu_pmu_fini(adev); 3618 if (adev->mman.discovery_bin) 3619 amdgpu_discovery_fini(adev); 3620 } 3621 3622 3623 /* 3624 * Suspend & resume. 3625 */ 3626 /** 3627 * amdgpu_device_suspend - initiate device suspend 3628 * 3629 * @dev: drm dev pointer 3630 * @fbcon : notify the fbdev of suspend 3631 * 3632 * Puts the hw in the suspend state (all asics). 3633 * Returns 0 for success or an error on failure. 3634 * Called at driver suspend. 3635 */ 3636 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) 3637 { 3638 struct amdgpu_device *adev; 3639 struct drm_crtc *crtc; 3640 struct drm_connector *connector; 3641 struct drm_connector_list_iter iter; 3642 int r; 3643 3644 adev = drm_to_adev(dev); 3645 3646 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 3647 return 0; 3648 3649 adev->in_suspend = true; 3650 drm_kms_helper_poll_disable(dev); 3651 3652 if (fbcon) 3653 amdgpu_fbdev_set_suspend(adev, 1); 3654 3655 cancel_delayed_work_sync(&adev->delayed_init_work); 3656 3657 if (!amdgpu_device_has_dc_support(adev)) { 3658 /* turn off display hw */ 3659 drm_modeset_lock_all(dev); 3660 drm_connector_list_iter_begin(dev, &iter); 3661 drm_for_each_connector_iter(connector, &iter) 3662 drm_helper_connector_dpms(connector, 3663 DRM_MODE_DPMS_OFF); 3664 drm_connector_list_iter_end(&iter); 3665 drm_modeset_unlock_all(dev); 3666 /* unpin the front buffers and cursors */ 3667 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { 3668 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); 3669 struct drm_framebuffer *fb = crtc->primary->fb; 3670 struct amdgpu_bo *robj; 3671 3672 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) { 3673 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 3674 r = amdgpu_bo_reserve(aobj, true); 3675 if (r == 0) { 3676 amdgpu_bo_unpin(aobj); 3677 amdgpu_bo_unreserve(aobj); 3678 } 3679 } 3680 3681 if (fb == NULL || fb->obj[0] == NULL) { 3682 continue; 3683 } 3684 robj = gem_to_amdgpu_bo(fb->obj[0]); 3685 /* don't unpin kernel fb objects */ 3686 if (!amdgpu_fbdev_robj_is_fb(adev, robj)) { 3687 r = amdgpu_bo_reserve(robj, true); 3688 if (r == 0) { 3689 amdgpu_bo_unpin(robj); 3690 amdgpu_bo_unreserve(robj); 3691 } 3692 } 3693 } 3694 } 3695 3696 amdgpu_ras_suspend(adev); 3697 3698 r = amdgpu_device_ip_suspend_phase1(adev); 3699 3700 amdgpu_amdkfd_suspend(adev, !fbcon); 3701 3702 /* evict vram memory */ 3703 amdgpu_bo_evict_vram(adev); 3704 3705 amdgpu_fence_driver_suspend(adev); 3706 3707 r = amdgpu_device_ip_suspend_phase2(adev); 3708 3709 /* evict remaining vram memory 3710 * This second call to evict vram is to evict the gart page table 3711 * using the CPU. 3712 */ 3713 amdgpu_bo_evict_vram(adev); 3714 3715 return 0; 3716 } 3717 3718 /** 3719 * amdgpu_device_resume - initiate device resume 3720 * 3721 * @dev: drm dev pointer 3722 * @fbcon : notify the fbdev of resume 3723 * 3724 * Bring the hw back to operating state (all asics). 3725 * Returns 0 for success or an error on failure. 3726 * Called at driver resume. 3727 */ 3728 int amdgpu_device_resume(struct drm_device *dev, bool fbcon) 3729 { 3730 struct drm_connector *connector; 3731 struct drm_connector_list_iter iter; 3732 struct amdgpu_device *adev = drm_to_adev(dev); 3733 struct drm_crtc *crtc; 3734 int r = 0; 3735 3736 if (dev->switch_power_state == DRM_SWITCH_POWER_OFF) 3737 return 0; 3738 3739 /* post card */ 3740 if (amdgpu_device_need_post(adev)) { 3741 r = amdgpu_device_asic_init(adev); 3742 if (r) 3743 dev_err(adev->dev, "amdgpu asic init failed\n"); 3744 } 3745 3746 r = amdgpu_device_ip_resume(adev); 3747 if (r) { 3748 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r); 3749 return r; 3750 } 3751 amdgpu_fence_driver_resume(adev); 3752 3753 3754 r = amdgpu_device_ip_late_init(adev); 3755 if (r) 3756 return r; 3757 3758 queue_delayed_work(system_wq, &adev->delayed_init_work, 3759 msecs_to_jiffies(AMDGPU_RESUME_MS)); 3760 3761 if (!amdgpu_device_has_dc_support(adev)) { 3762 /* pin cursors */ 3763 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { 3764 struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc); 3765 3766 if (amdgpu_crtc->cursor_bo && !adev->enable_virtual_display) { 3767 struct amdgpu_bo *aobj = gem_to_amdgpu_bo(amdgpu_crtc->cursor_bo); 3768 r = amdgpu_bo_reserve(aobj, true); 3769 if (r == 0) { 3770 r = amdgpu_bo_pin(aobj, AMDGPU_GEM_DOMAIN_VRAM); 3771 if (r != 0) 3772 dev_err(adev->dev, "Failed to pin cursor BO (%d)\n", r); 3773 amdgpu_crtc->cursor_addr = amdgpu_bo_gpu_offset(aobj); 3774 amdgpu_bo_unreserve(aobj); 3775 } 3776 } 3777 } 3778 } 3779 r = amdgpu_amdkfd_resume(adev, !fbcon); 3780 if (r) 3781 return r; 3782 3783 /* Make sure IB tests flushed */ 3784 flush_delayed_work(&adev->delayed_init_work); 3785 3786 /* blat the mode back in */ 3787 if (fbcon) { 3788 if (!amdgpu_device_has_dc_support(adev)) { 3789 /* pre DCE11 */ 3790 drm_helper_resume_force_mode(dev); 3791 3792 /* turn on display hw */ 3793 drm_modeset_lock_all(dev); 3794 3795 drm_connector_list_iter_begin(dev, &iter); 3796 drm_for_each_connector_iter(connector, &iter) 3797 drm_helper_connector_dpms(connector, 3798 DRM_MODE_DPMS_ON); 3799 drm_connector_list_iter_end(&iter); 3800 3801 drm_modeset_unlock_all(dev); 3802 } 3803 amdgpu_fbdev_set_suspend(adev, 0); 3804 } 3805 3806 drm_kms_helper_poll_enable(dev); 3807 3808 amdgpu_ras_resume(adev); 3809 3810 /* 3811 * Most of the connector probing functions try to acquire runtime pm 3812 * refs to ensure that the GPU is powered on when connector polling is 3813 * performed. Since we're calling this from a runtime PM callback, 3814 * trying to acquire rpm refs will cause us to deadlock. 3815 * 3816 * Since we're guaranteed to be holding the rpm lock, it's safe to 3817 * temporarily disable the rpm helpers so this doesn't deadlock us. 3818 */ 3819 #ifdef CONFIG_PM 3820 dev->dev->power.disable_depth++; 3821 #endif 3822 if (!amdgpu_device_has_dc_support(adev)) 3823 drm_helper_hpd_irq_event(dev); 3824 else 3825 drm_kms_helper_hotplug_event(dev); 3826 #ifdef CONFIG_PM 3827 dev->dev->power.disable_depth--; 3828 #endif 3829 adev->in_suspend = false; 3830 3831 return 0; 3832 } 3833 3834 /** 3835 * amdgpu_device_ip_check_soft_reset - did soft reset succeed 3836 * 3837 * @adev: amdgpu_device pointer 3838 * 3839 * The list of all the hardware IPs that make up the asic is walked and 3840 * the check_soft_reset callbacks are run. check_soft_reset determines 3841 * if the asic is still hung or not. 3842 * Returns true if any of the IPs are still in a hung state, false if not. 3843 */ 3844 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) 3845 { 3846 int i; 3847 bool asic_hang = false; 3848 3849 if (amdgpu_sriov_vf(adev)) 3850 return true; 3851 3852 if (amdgpu_asic_need_full_reset(adev)) 3853 return true; 3854 3855 for (i = 0; i < adev->num_ip_blocks; i++) { 3856 if (!adev->ip_blocks[i].status.valid) 3857 continue; 3858 if (adev->ip_blocks[i].version->funcs->check_soft_reset) 3859 adev->ip_blocks[i].status.hang = 3860 adev->ip_blocks[i].version->funcs->check_soft_reset(adev); 3861 if (adev->ip_blocks[i].status.hang) { 3862 dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); 3863 asic_hang = true; 3864 } 3865 } 3866 return asic_hang; 3867 } 3868 3869 /** 3870 * amdgpu_device_ip_pre_soft_reset - prepare for soft reset 3871 * 3872 * @adev: amdgpu_device pointer 3873 * 3874 * The list of all the hardware IPs that make up the asic is walked and the 3875 * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset 3876 * handles any IP specific hardware or software state changes that are 3877 * necessary for a soft reset to succeed. 3878 * Returns 0 on success, negative error code on failure. 3879 */ 3880 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) 3881 { 3882 int i, r = 0; 3883 3884 for (i = 0; i < adev->num_ip_blocks; i++) { 3885 if (!adev->ip_blocks[i].status.valid) 3886 continue; 3887 if (adev->ip_blocks[i].status.hang && 3888 adev->ip_blocks[i].version->funcs->pre_soft_reset) { 3889 r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev); 3890 if (r) 3891 return r; 3892 } 3893 } 3894 3895 return 0; 3896 } 3897 3898 /** 3899 * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed 3900 * 3901 * @adev: amdgpu_device pointer 3902 * 3903 * Some hardware IPs cannot be soft reset. If they are hung, a full gpu 3904 * reset is necessary to recover. 3905 * Returns true if a full asic reset is required, false if not. 3906 */ 3907 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) 3908 { 3909 int i; 3910 3911 if (amdgpu_asic_need_full_reset(adev)) 3912 return true; 3913 3914 for (i = 0; i < adev->num_ip_blocks; i++) { 3915 if (!adev->ip_blocks[i].status.valid) 3916 continue; 3917 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || 3918 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || 3919 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || 3920 (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || 3921 adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { 3922 if (adev->ip_blocks[i].status.hang) { 3923 dev_info(adev->dev, "Some block need full reset!\n"); 3924 return true; 3925 } 3926 } 3927 } 3928 return false; 3929 } 3930 3931 /** 3932 * amdgpu_device_ip_soft_reset - do a soft reset 3933 * 3934 * @adev: amdgpu_device pointer 3935 * 3936 * The list of all the hardware IPs that make up the asic is walked and the 3937 * soft_reset callbacks are run if the block is hung. soft_reset handles any 3938 * IP specific hardware or software state changes that are necessary to soft 3939 * reset the IP. 3940 * Returns 0 on success, negative error code on failure. 3941 */ 3942 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) 3943 { 3944 int i, r = 0; 3945 3946 for (i = 0; i < adev->num_ip_blocks; i++) { 3947 if (!adev->ip_blocks[i].status.valid) 3948 continue; 3949 if (adev->ip_blocks[i].status.hang && 3950 adev->ip_blocks[i].version->funcs->soft_reset) { 3951 r = adev->ip_blocks[i].version->funcs->soft_reset(adev); 3952 if (r) 3953 return r; 3954 } 3955 } 3956 3957 return 0; 3958 } 3959 3960 /** 3961 * amdgpu_device_ip_post_soft_reset - clean up from soft reset 3962 * 3963 * @adev: amdgpu_device pointer 3964 * 3965 * The list of all the hardware IPs that make up the asic is walked and the 3966 * post_soft_reset callbacks are run if the asic was hung. post_soft_reset 3967 * handles any IP specific hardware or software state changes that are 3968 * necessary after the IP has been soft reset. 3969 * Returns 0 on success, negative error code on failure. 3970 */ 3971 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) 3972 { 3973 int i, r = 0; 3974 3975 for (i = 0; i < adev->num_ip_blocks; i++) { 3976 if (!adev->ip_blocks[i].status.valid) 3977 continue; 3978 if (adev->ip_blocks[i].status.hang && 3979 adev->ip_blocks[i].version->funcs->post_soft_reset) 3980 r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev); 3981 if (r) 3982 return r; 3983 } 3984 3985 return 0; 3986 } 3987 3988 /** 3989 * amdgpu_device_recover_vram - Recover some VRAM contents 3990 * 3991 * @adev: amdgpu_device pointer 3992 * 3993 * Restores the contents of VRAM buffers from the shadows in GTT. Used to 3994 * restore things like GPUVM page tables after a GPU reset where 3995 * the contents of VRAM might be lost. 3996 * 3997 * Returns: 3998 * 0 on success, negative error code on failure. 3999 */ 4000 static int amdgpu_device_recover_vram(struct amdgpu_device *adev) 4001 { 4002 struct dma_fence *fence = NULL, *next = NULL; 4003 struct amdgpu_bo *shadow; 4004 long r = 1, tmo; 4005 4006 if (amdgpu_sriov_runtime(adev)) 4007 tmo = msecs_to_jiffies(8000); 4008 else 4009 tmo = msecs_to_jiffies(100); 4010 4011 dev_info(adev->dev, "recover vram bo from shadow start\n"); 4012 mutex_lock(&adev->shadow_list_lock); 4013 list_for_each_entry(shadow, &adev->shadow_list, shadow_list) { 4014 4015 /* No need to recover an evicted BO */ 4016 if (shadow->tbo.mem.mem_type != TTM_PL_TT || 4017 shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET || 4018 shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM) 4019 continue; 4020 4021 r = amdgpu_bo_restore_shadow(shadow, &next); 4022 if (r) 4023 break; 4024 4025 if (fence) { 4026 tmo = dma_fence_wait_timeout(fence, false, tmo); 4027 dma_fence_put(fence); 4028 fence = next; 4029 if (tmo == 0) { 4030 r = -ETIMEDOUT; 4031 break; 4032 } else if (tmo < 0) { 4033 r = tmo; 4034 break; 4035 } 4036 } else { 4037 fence = next; 4038 } 4039 } 4040 mutex_unlock(&adev->shadow_list_lock); 4041 4042 if (fence) 4043 tmo = dma_fence_wait_timeout(fence, false, tmo); 4044 dma_fence_put(fence); 4045 4046 if (r < 0 || tmo <= 0) { 4047 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo); 4048 return -EIO; 4049 } 4050 4051 dev_info(adev->dev, "recover vram bo from shadow done\n"); 4052 return 0; 4053 } 4054 4055 4056 /** 4057 * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf 4058 * 4059 * @adev: amdgpu_device pointer 4060 * @from_hypervisor: request from hypervisor 4061 * 4062 * do VF FLR and reinitialize Asic 4063 * return 0 means succeeded otherwise failed 4064 */ 4065 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev, 4066 bool from_hypervisor) 4067 { 4068 int r; 4069 4070 if (from_hypervisor) 4071 r = amdgpu_virt_request_full_gpu(adev, true); 4072 else 4073 r = amdgpu_virt_reset_gpu(adev); 4074 if (r) 4075 return r; 4076 4077 amdgpu_amdkfd_pre_reset(adev); 4078 4079 /* Resume IP prior to SMC */ 4080 r = amdgpu_device_ip_reinit_early_sriov(adev); 4081 if (r) 4082 goto error; 4083 4084 amdgpu_virt_init_data_exchange(adev); 4085 /* we need recover gart prior to run SMC/CP/SDMA resume */ 4086 amdgpu_gtt_mgr_recover(ttm_manager_type(&adev->mman.bdev, TTM_PL_TT)); 4087 4088 r = amdgpu_device_fw_loading(adev); 4089 if (r) 4090 return r; 4091 4092 /* now we are okay to resume SMC/CP/SDMA */ 4093 r = amdgpu_device_ip_reinit_late_sriov(adev); 4094 if (r) 4095 goto error; 4096 4097 amdgpu_irq_gpu_reset_resume_helper(adev); 4098 r = amdgpu_ib_ring_tests(adev); 4099 amdgpu_amdkfd_post_reset(adev); 4100 4101 error: 4102 amdgpu_virt_release_full_gpu(adev, true); 4103 if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) { 4104 amdgpu_inc_vram_lost(adev); 4105 r = amdgpu_device_recover_vram(adev); 4106 } 4107 4108 return r; 4109 } 4110 4111 /** 4112 * amdgpu_device_has_job_running - check if there is any job in mirror list 4113 * 4114 * @adev: amdgpu_device pointer 4115 * 4116 * check if there is any job in mirror list 4117 */ 4118 bool amdgpu_device_has_job_running(struct amdgpu_device *adev) 4119 { 4120 int i; 4121 struct drm_sched_job *job; 4122 4123 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4124 struct amdgpu_ring *ring = adev->rings[i]; 4125 4126 if (!ring || !ring->sched.thread) 4127 continue; 4128 4129 spin_lock(&ring->sched.job_list_lock); 4130 job = list_first_entry_or_null(&ring->sched.ring_mirror_list, 4131 struct drm_sched_job, node); 4132 spin_unlock(&ring->sched.job_list_lock); 4133 if (job) 4134 return true; 4135 } 4136 return false; 4137 } 4138 4139 /** 4140 * amdgpu_device_should_recover_gpu - check if we should try GPU recovery 4141 * 4142 * @adev: amdgpu_device pointer 4143 * 4144 * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover 4145 * a hung GPU. 4146 */ 4147 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev) 4148 { 4149 if (!amdgpu_device_ip_check_soft_reset(adev)) { 4150 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n"); 4151 return false; 4152 } 4153 4154 if (amdgpu_gpu_recovery == 0) 4155 goto disabled; 4156 4157 if (amdgpu_sriov_vf(adev)) 4158 return true; 4159 4160 if (amdgpu_gpu_recovery == -1) { 4161 switch (adev->asic_type) { 4162 case CHIP_BONAIRE: 4163 case CHIP_HAWAII: 4164 case CHIP_TOPAZ: 4165 case CHIP_TONGA: 4166 case CHIP_FIJI: 4167 case CHIP_POLARIS10: 4168 case CHIP_POLARIS11: 4169 case CHIP_POLARIS12: 4170 case CHIP_VEGAM: 4171 case CHIP_VEGA20: 4172 case CHIP_VEGA10: 4173 case CHIP_VEGA12: 4174 case CHIP_RAVEN: 4175 case CHIP_ARCTURUS: 4176 case CHIP_RENOIR: 4177 case CHIP_NAVI10: 4178 case CHIP_NAVI14: 4179 case CHIP_NAVI12: 4180 case CHIP_SIENNA_CICHLID: 4181 break; 4182 default: 4183 goto disabled; 4184 } 4185 } 4186 4187 return true; 4188 4189 disabled: 4190 dev_info(adev->dev, "GPU recovery disabled.\n"); 4191 return false; 4192 } 4193 4194 4195 static int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, 4196 struct amdgpu_job *job, 4197 bool *need_full_reset_arg) 4198 { 4199 int i, r = 0; 4200 bool need_full_reset = *need_full_reset_arg; 4201 4202 amdgpu_debugfs_wait_dump(adev); 4203 4204 if (amdgpu_sriov_vf(adev)) { 4205 /* stop the data exchange thread */ 4206 amdgpu_virt_fini_data_exchange(adev); 4207 } 4208 4209 /* block all schedulers and reset given job's ring */ 4210 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4211 struct amdgpu_ring *ring = adev->rings[i]; 4212 4213 if (!ring || !ring->sched.thread) 4214 continue; 4215 4216 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */ 4217 amdgpu_fence_driver_force_completion(ring); 4218 } 4219 4220 if(job) 4221 drm_sched_increase_karma(&job->base); 4222 4223 /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ 4224 if (!amdgpu_sriov_vf(adev)) { 4225 4226 if (!need_full_reset) 4227 need_full_reset = amdgpu_device_ip_need_full_reset(adev); 4228 4229 if (!need_full_reset) { 4230 amdgpu_device_ip_pre_soft_reset(adev); 4231 r = amdgpu_device_ip_soft_reset(adev); 4232 amdgpu_device_ip_post_soft_reset(adev); 4233 if (r || amdgpu_device_ip_check_soft_reset(adev)) { 4234 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); 4235 need_full_reset = true; 4236 } 4237 } 4238 4239 if (need_full_reset) 4240 r = amdgpu_device_ip_suspend(adev); 4241 4242 *need_full_reset_arg = need_full_reset; 4243 } 4244 4245 return r; 4246 } 4247 4248 static int amdgpu_do_asic_reset(struct amdgpu_hive_info *hive, 4249 struct list_head *device_list_handle, 4250 bool *need_full_reset_arg, 4251 bool skip_hw_reset) 4252 { 4253 struct amdgpu_device *tmp_adev = NULL; 4254 bool need_full_reset = *need_full_reset_arg, vram_lost = false; 4255 int r = 0; 4256 4257 /* 4258 * ASIC reset has to be done on all HGMI hive nodes ASAP 4259 * to allow proper links negotiation in FW (within 1 sec) 4260 */ 4261 if (!skip_hw_reset && need_full_reset) { 4262 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { 4263 /* For XGMI run all resets in parallel to speed up the process */ 4264 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 4265 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work)) 4266 r = -EALREADY; 4267 } else 4268 r = amdgpu_asic_reset(tmp_adev); 4269 4270 if (r) { 4271 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s", 4272 r, adev_to_drm(tmp_adev)->unique); 4273 break; 4274 } 4275 } 4276 4277 /* For XGMI wait for all resets to complete before proceed */ 4278 if (!r) { 4279 list_for_each_entry(tmp_adev, device_list_handle, 4280 gmc.xgmi.head) { 4281 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) { 4282 flush_work(&tmp_adev->xgmi_reset_work); 4283 r = tmp_adev->asic_reset_res; 4284 if (r) 4285 break; 4286 } 4287 } 4288 } 4289 } 4290 4291 if (!r && amdgpu_ras_intr_triggered()) { 4292 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { 4293 if (tmp_adev->mmhub.funcs && 4294 tmp_adev->mmhub.funcs->reset_ras_error_count) 4295 tmp_adev->mmhub.funcs->reset_ras_error_count(tmp_adev); 4296 } 4297 4298 amdgpu_ras_intr_cleared(); 4299 } 4300 4301 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { 4302 if (need_full_reset) { 4303 /* post card */ 4304 if (amdgpu_device_asic_init(tmp_adev)) 4305 dev_warn(tmp_adev->dev, "asic atom init failed!"); 4306 4307 if (!r) { 4308 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n"); 4309 r = amdgpu_device_ip_resume_phase1(tmp_adev); 4310 if (r) 4311 goto out; 4312 4313 vram_lost = amdgpu_device_check_vram_lost(tmp_adev); 4314 if (vram_lost) { 4315 DRM_INFO("VRAM is lost due to GPU reset!\n"); 4316 amdgpu_inc_vram_lost(tmp_adev); 4317 } 4318 4319 r = amdgpu_gtt_mgr_recover(ttm_manager_type(&tmp_adev->mman.bdev, TTM_PL_TT)); 4320 if (r) 4321 goto out; 4322 4323 r = amdgpu_device_fw_loading(tmp_adev); 4324 if (r) 4325 return r; 4326 4327 r = amdgpu_device_ip_resume_phase2(tmp_adev); 4328 if (r) 4329 goto out; 4330 4331 if (vram_lost) 4332 amdgpu_device_fill_reset_magic(tmp_adev); 4333 4334 /* 4335 * Add this ASIC as tracked as reset was already 4336 * complete successfully. 4337 */ 4338 amdgpu_register_gpu_instance(tmp_adev); 4339 4340 r = amdgpu_device_ip_late_init(tmp_adev); 4341 if (r) 4342 goto out; 4343 4344 amdgpu_fbdev_set_suspend(tmp_adev, 0); 4345 4346 /* 4347 * The GPU enters bad state once faulty pages 4348 * by ECC has reached the threshold, and ras 4349 * recovery is scheduled next. So add one check 4350 * here to break recovery if it indeed exceeds 4351 * bad page threshold, and remind user to 4352 * retire this GPU or setting one bigger 4353 * bad_page_threshold value to fix this once 4354 * probing driver again. 4355 */ 4356 if (!amdgpu_ras_check_err_threshold(tmp_adev)) { 4357 /* must succeed. */ 4358 amdgpu_ras_resume(tmp_adev); 4359 } else { 4360 r = -EINVAL; 4361 goto out; 4362 } 4363 4364 /* Update PSP FW topology after reset */ 4365 if (hive && tmp_adev->gmc.xgmi.num_physical_nodes > 1) 4366 r = amdgpu_xgmi_update_topology(hive, tmp_adev); 4367 } 4368 } 4369 4370 out: 4371 if (!r) { 4372 amdgpu_irq_gpu_reset_resume_helper(tmp_adev); 4373 r = amdgpu_ib_ring_tests(tmp_adev); 4374 if (r) { 4375 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r); 4376 r = amdgpu_device_ip_suspend(tmp_adev); 4377 need_full_reset = true; 4378 r = -EAGAIN; 4379 goto end; 4380 } 4381 } 4382 4383 if (!r) 4384 r = amdgpu_device_recover_vram(tmp_adev); 4385 else 4386 tmp_adev->asic_reset_res = r; 4387 } 4388 4389 end: 4390 *need_full_reset_arg = need_full_reset; 4391 return r; 4392 } 4393 4394 static bool amdgpu_device_lock_adev(struct amdgpu_device *adev, 4395 struct amdgpu_hive_info *hive) 4396 { 4397 if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0) 4398 return false; 4399 4400 if (hive) { 4401 down_write_nest_lock(&adev->reset_sem, &hive->hive_lock); 4402 } else { 4403 down_write(&adev->reset_sem); 4404 } 4405 4406 atomic_inc(&adev->gpu_reset_counter); 4407 switch (amdgpu_asic_reset_method(adev)) { 4408 case AMD_RESET_METHOD_MODE1: 4409 adev->mp1_state = PP_MP1_STATE_SHUTDOWN; 4410 break; 4411 case AMD_RESET_METHOD_MODE2: 4412 adev->mp1_state = PP_MP1_STATE_RESET; 4413 break; 4414 default: 4415 adev->mp1_state = PP_MP1_STATE_NONE; 4416 break; 4417 } 4418 4419 return true; 4420 } 4421 4422 static void amdgpu_device_unlock_adev(struct amdgpu_device *adev) 4423 { 4424 amdgpu_vf_error_trans_all(adev); 4425 adev->mp1_state = PP_MP1_STATE_NONE; 4426 atomic_set(&adev->in_gpu_reset, 0); 4427 up_write(&adev->reset_sem); 4428 } 4429 4430 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev) 4431 { 4432 struct pci_dev *p = NULL; 4433 4434 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 4435 adev->pdev->bus->number, 1); 4436 if (p) { 4437 pm_runtime_enable(&(p->dev)); 4438 pm_runtime_resume(&(p->dev)); 4439 } 4440 } 4441 4442 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) 4443 { 4444 enum amd_reset_method reset_method; 4445 struct pci_dev *p = NULL; 4446 u64 expires; 4447 4448 /* 4449 * For now, only BACO and mode1 reset are confirmed 4450 * to suffer the audio issue without proper suspended. 4451 */ 4452 reset_method = amdgpu_asic_reset_method(adev); 4453 if ((reset_method != AMD_RESET_METHOD_BACO) && 4454 (reset_method != AMD_RESET_METHOD_MODE1)) 4455 return -EINVAL; 4456 4457 p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), 4458 adev->pdev->bus->number, 1); 4459 if (!p) 4460 return -ENODEV; 4461 4462 expires = pm_runtime_autosuspend_expiration(&(p->dev)); 4463 if (!expires) 4464 /* 4465 * If we cannot get the audio device autosuspend delay, 4466 * a fixed 4S interval will be used. Considering 3S is 4467 * the audio controller default autosuspend delay setting. 4468 * 4S used here is guaranteed to cover that. 4469 */ 4470 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL; 4471 4472 while (!pm_runtime_status_suspended(&(p->dev))) { 4473 if (!pm_runtime_suspend(&(p->dev))) 4474 break; 4475 4476 if (expires < ktime_get_mono_fast_ns()) { 4477 dev_warn(adev->dev, "failed to suspend display audio\n"); 4478 /* TODO: abort the succeeding gpu reset? */ 4479 return -ETIMEDOUT; 4480 } 4481 } 4482 4483 pm_runtime_disable(&(p->dev)); 4484 4485 return 0; 4486 } 4487 4488 /** 4489 * amdgpu_device_gpu_recover - reset the asic and recover scheduler 4490 * 4491 * @adev: amdgpu_device pointer 4492 * @job: which job trigger hang 4493 * 4494 * Attempt to reset the GPU if it has hung (all asics). 4495 * Attempt to do soft-reset or full-reset and reinitialize Asic 4496 * Returns 0 for success or an error on failure. 4497 */ 4498 4499 int amdgpu_device_gpu_recover(struct amdgpu_device *adev, 4500 struct amdgpu_job *job) 4501 { 4502 struct list_head device_list, *device_list_handle = NULL; 4503 bool need_full_reset = false; 4504 bool job_signaled = false; 4505 struct amdgpu_hive_info *hive = NULL; 4506 struct amdgpu_device *tmp_adev = NULL; 4507 int i, r = 0; 4508 bool need_emergency_restart = false; 4509 bool audio_suspended = false; 4510 4511 /** 4512 * Special case: RAS triggered and full reset isn't supported 4513 */ 4514 need_emergency_restart = amdgpu_ras_need_emergency_restart(adev); 4515 4516 /* 4517 * Flush RAM to disk so that after reboot 4518 * the user can read log and see why the system rebooted. 4519 */ 4520 if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) { 4521 DRM_WARN("Emergency reboot."); 4522 4523 ksys_sync_helper(); 4524 emergency_restart(); 4525 } 4526 4527 dev_info(adev->dev, "GPU %s begin!\n", 4528 need_emergency_restart ? "jobs stop":"reset"); 4529 4530 /* 4531 * Here we trylock to avoid chain of resets executing from 4532 * either trigger by jobs on different adevs in XGMI hive or jobs on 4533 * different schedulers for same device while this TO handler is running. 4534 * We always reset all schedulers for device and all devices for XGMI 4535 * hive so that should take care of them too. 4536 */ 4537 hive = amdgpu_get_xgmi_hive(adev); 4538 if (hive) { 4539 if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) { 4540 DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress", 4541 job ? job->base.id : -1, hive->hive_id); 4542 amdgpu_put_xgmi_hive(hive); 4543 return 0; 4544 } 4545 mutex_lock(&hive->hive_lock); 4546 } 4547 4548 /* 4549 * Build list of devices to reset. 4550 * In case we are in XGMI hive mode, resort the device list 4551 * to put adev in the 1st position. 4552 */ 4553 INIT_LIST_HEAD(&device_list); 4554 if (adev->gmc.xgmi.num_physical_nodes > 1) { 4555 if (!hive) 4556 return -ENODEV; 4557 if (!list_is_first(&adev->gmc.xgmi.head, &hive->device_list)) 4558 list_rotate_to_front(&adev->gmc.xgmi.head, &hive->device_list); 4559 device_list_handle = &hive->device_list; 4560 } else { 4561 list_add_tail(&adev->gmc.xgmi.head, &device_list); 4562 device_list_handle = &device_list; 4563 } 4564 4565 /* block all schedulers and reset given job's ring */ 4566 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { 4567 if (!amdgpu_device_lock_adev(tmp_adev, hive)) { 4568 dev_info(tmp_adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress", 4569 job ? job->base.id : -1); 4570 r = 0; 4571 goto skip_recovery; 4572 } 4573 4574 /* 4575 * Try to put the audio codec into suspend state 4576 * before gpu reset started. 4577 * 4578 * Due to the power domain of the graphics device 4579 * is shared with AZ power domain. Without this, 4580 * we may change the audio hardware from behind 4581 * the audio driver's back. That will trigger 4582 * some audio codec errors. 4583 */ 4584 if (!amdgpu_device_suspend_display_audio(tmp_adev)) 4585 audio_suspended = true; 4586 4587 amdgpu_ras_set_error_query_ready(tmp_adev, false); 4588 4589 cancel_delayed_work_sync(&tmp_adev->delayed_init_work); 4590 4591 if (!amdgpu_sriov_vf(tmp_adev)) 4592 amdgpu_amdkfd_pre_reset(tmp_adev); 4593 4594 /* 4595 * Mark these ASICs to be reseted as untracked first 4596 * And add them back after reset completed 4597 */ 4598 amdgpu_unregister_gpu_instance(tmp_adev); 4599 4600 amdgpu_fbdev_set_suspend(tmp_adev, 1); 4601 4602 /* disable ras on ALL IPs */ 4603 if (!need_emergency_restart && 4604 amdgpu_device_ip_need_full_reset(tmp_adev)) 4605 amdgpu_ras_suspend(tmp_adev); 4606 4607 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4608 struct amdgpu_ring *ring = tmp_adev->rings[i]; 4609 4610 if (!ring || !ring->sched.thread) 4611 continue; 4612 4613 drm_sched_stop(&ring->sched, job ? &job->base : NULL); 4614 4615 if (need_emergency_restart) 4616 amdgpu_job_stop_all_jobs_on_sched(&ring->sched); 4617 } 4618 } 4619 4620 if (need_emergency_restart) 4621 goto skip_sched_resume; 4622 4623 /* 4624 * Must check guilty signal here since after this point all old 4625 * HW fences are force signaled. 4626 * 4627 * job->base holds a reference to parent fence 4628 */ 4629 if (job && job->base.s_fence->parent && 4630 dma_fence_is_signaled(job->base.s_fence->parent)) { 4631 job_signaled = true; 4632 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset"); 4633 goto skip_hw_reset; 4634 } 4635 4636 retry: /* Rest of adevs pre asic reset from XGMI hive. */ 4637 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { 4638 r = amdgpu_device_pre_asic_reset(tmp_adev, 4639 (tmp_adev == adev) ? job : NULL, 4640 &need_full_reset); 4641 /*TODO Should we stop ?*/ 4642 if (r) { 4643 dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ", 4644 r, adev_to_drm(tmp_adev)->unique); 4645 tmp_adev->asic_reset_res = r; 4646 } 4647 } 4648 4649 /* Actual ASIC resets if needed.*/ 4650 /* TODO Implement XGMI hive reset logic for SRIOV */ 4651 if (amdgpu_sriov_vf(adev)) { 4652 r = amdgpu_device_reset_sriov(adev, job ? false : true); 4653 if (r) 4654 adev->asic_reset_res = r; 4655 } else { 4656 r = amdgpu_do_asic_reset(hive, device_list_handle, &need_full_reset, false); 4657 if (r && r == -EAGAIN) 4658 goto retry; 4659 } 4660 4661 skip_hw_reset: 4662 4663 /* Post ASIC reset for all devs .*/ 4664 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { 4665 4666 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4667 struct amdgpu_ring *ring = tmp_adev->rings[i]; 4668 4669 if (!ring || !ring->sched.thread) 4670 continue; 4671 4672 /* No point to resubmit jobs if we didn't HW reset*/ 4673 if (!tmp_adev->asic_reset_res && !job_signaled) 4674 drm_sched_resubmit_jobs(&ring->sched); 4675 4676 drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res); 4677 } 4678 4679 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) { 4680 drm_helper_resume_force_mode(adev_to_drm(tmp_adev)); 4681 } 4682 4683 tmp_adev->asic_reset_res = 0; 4684 4685 if (r) { 4686 /* bad news, how to tell it to userspace ? */ 4687 dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter)); 4688 amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r); 4689 } else { 4690 dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter)); 4691 } 4692 } 4693 4694 skip_sched_resume: 4695 list_for_each_entry(tmp_adev, device_list_handle, gmc.xgmi.head) { 4696 /*unlock kfd: SRIOV would do it separately */ 4697 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev)) 4698 amdgpu_amdkfd_post_reset(tmp_adev); 4699 if (audio_suspended) 4700 amdgpu_device_resume_display_audio(tmp_adev); 4701 amdgpu_device_unlock_adev(tmp_adev); 4702 } 4703 4704 skip_recovery: 4705 if (hive) { 4706 atomic_set(&hive->in_reset, 0); 4707 mutex_unlock(&hive->hive_lock); 4708 amdgpu_put_xgmi_hive(hive); 4709 } 4710 4711 if (r) 4712 dev_info(adev->dev, "GPU reset end with ret = %d\n", r); 4713 return r; 4714 } 4715 4716 /** 4717 * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot 4718 * 4719 * @adev: amdgpu_device pointer 4720 * 4721 * Fetchs and stores in the driver the PCIE capabilities (gen speed 4722 * and lanes) of the slot the device is in. Handles APUs and 4723 * virtualized environments where PCIE config space may not be available. 4724 */ 4725 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev) 4726 { 4727 struct pci_dev *pdev; 4728 enum pci_bus_speed speed_cap, platform_speed_cap; 4729 enum pcie_link_width platform_link_width; 4730 4731 if (amdgpu_pcie_gen_cap) 4732 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap; 4733 4734 if (amdgpu_pcie_lane_cap) 4735 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap; 4736 4737 /* covers APUs as well */ 4738 if (pci_is_root_bus(adev->pdev->bus)) { 4739 if (adev->pm.pcie_gen_mask == 0) 4740 adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK; 4741 if (adev->pm.pcie_mlw_mask == 0) 4742 adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK; 4743 return; 4744 } 4745 4746 if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask) 4747 return; 4748 4749 pcie_bandwidth_available(adev->pdev, NULL, 4750 &platform_speed_cap, &platform_link_width); 4751 4752 if (adev->pm.pcie_gen_mask == 0) { 4753 /* asic caps */ 4754 pdev = adev->pdev; 4755 speed_cap = pcie_get_speed_cap(pdev); 4756 if (speed_cap == PCI_SPEED_UNKNOWN) { 4757 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 4758 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 4759 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 4760 } else { 4761 if (speed_cap == PCIE_SPEED_16_0GT) 4762 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 4763 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 4764 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 | 4765 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4); 4766 else if (speed_cap == PCIE_SPEED_8_0GT) 4767 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 4768 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 | 4769 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3); 4770 else if (speed_cap == PCIE_SPEED_5_0GT) 4771 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 | 4772 CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2); 4773 else 4774 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1; 4775 } 4776 /* platform caps */ 4777 if (platform_speed_cap == PCI_SPEED_UNKNOWN) { 4778 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 4779 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 4780 } else { 4781 if (platform_speed_cap == PCIE_SPEED_16_0GT) 4782 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 4783 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 4784 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 | 4785 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4); 4786 else if (platform_speed_cap == PCIE_SPEED_8_0GT) 4787 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 4788 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 | 4789 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3); 4790 else if (platform_speed_cap == PCIE_SPEED_5_0GT) 4791 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 | 4792 CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2); 4793 else 4794 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1; 4795 4796 } 4797 } 4798 if (adev->pm.pcie_mlw_mask == 0) { 4799 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) { 4800 adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK; 4801 } else { 4802 switch (platform_link_width) { 4803 case PCIE_LNK_X32: 4804 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 | 4805 CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 4806 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 4807 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 4808 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 4809 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 4810 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 4811 break; 4812 case PCIE_LNK_X16: 4813 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 | 4814 CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 4815 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 4816 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 4817 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 4818 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 4819 break; 4820 case PCIE_LNK_X12: 4821 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 | 4822 CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 4823 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 4824 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 4825 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 4826 break; 4827 case PCIE_LNK_X8: 4828 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 | 4829 CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 4830 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 4831 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 4832 break; 4833 case PCIE_LNK_X4: 4834 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 | 4835 CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 4836 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 4837 break; 4838 case PCIE_LNK_X2: 4839 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 | 4840 CAIL_PCIE_LINK_WIDTH_SUPPORT_X1); 4841 break; 4842 case PCIE_LNK_X1: 4843 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1; 4844 break; 4845 default: 4846 break; 4847 } 4848 } 4849 } 4850 } 4851 4852 int amdgpu_device_baco_enter(struct drm_device *dev) 4853 { 4854 struct amdgpu_device *adev = drm_to_adev(dev); 4855 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4856 4857 if (!amdgpu_device_supports_baco(adev_to_drm(adev))) 4858 return -ENOTSUPP; 4859 4860 if (ras && ras->supported) 4861 adev->nbio.funcs->enable_doorbell_interrupt(adev, false); 4862 4863 return amdgpu_dpm_baco_enter(adev); 4864 } 4865 4866 int amdgpu_device_baco_exit(struct drm_device *dev) 4867 { 4868 struct amdgpu_device *adev = drm_to_adev(dev); 4869 struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); 4870 int ret = 0; 4871 4872 if (!amdgpu_device_supports_baco(adev_to_drm(adev))) 4873 return -ENOTSUPP; 4874 4875 ret = amdgpu_dpm_baco_exit(adev); 4876 if (ret) 4877 return ret; 4878 4879 if (ras && ras->supported) 4880 adev->nbio.funcs->enable_doorbell_interrupt(adev, true); 4881 4882 return 0; 4883 } 4884 4885 static void amdgpu_cancel_all_tdr(struct amdgpu_device *adev) 4886 { 4887 int i; 4888 4889 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4890 struct amdgpu_ring *ring = adev->rings[i]; 4891 4892 if (!ring || !ring->sched.thread) 4893 continue; 4894 4895 cancel_delayed_work_sync(&ring->sched.work_tdr); 4896 } 4897 } 4898 4899 /** 4900 * amdgpu_pci_error_detected - Called when a PCI error is detected. 4901 * @pdev: PCI device struct 4902 * @state: PCI channel state 4903 * 4904 * Description: Called when a PCI error is detected. 4905 * 4906 * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT. 4907 */ 4908 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) 4909 { 4910 struct drm_device *dev = pci_get_drvdata(pdev); 4911 struct amdgpu_device *adev = drm_to_adev(dev); 4912 int i; 4913 4914 DRM_INFO("PCI error: detected callback, state(%d)!!\n", state); 4915 4916 if (adev->gmc.xgmi.num_physical_nodes > 1) { 4917 DRM_WARN("No support for XGMI hive yet..."); 4918 return PCI_ERS_RESULT_DISCONNECT; 4919 } 4920 4921 switch (state) { 4922 case pci_channel_io_normal: 4923 return PCI_ERS_RESULT_CAN_RECOVER; 4924 /* Fatal error, prepare for slot reset */ 4925 case pci_channel_io_frozen: 4926 /* 4927 * Cancel and wait for all TDRs in progress if failing to 4928 * set adev->in_gpu_reset in amdgpu_device_lock_adev 4929 * 4930 * Locking adev->reset_sem will prevent any external access 4931 * to GPU during PCI error recovery 4932 */ 4933 while (!amdgpu_device_lock_adev(adev, NULL)) 4934 amdgpu_cancel_all_tdr(adev); 4935 4936 /* 4937 * Block any work scheduling as we do for regular GPU reset 4938 * for the duration of the recovery 4939 */ 4940 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 4941 struct amdgpu_ring *ring = adev->rings[i]; 4942 4943 if (!ring || !ring->sched.thread) 4944 continue; 4945 4946 drm_sched_stop(&ring->sched, NULL); 4947 } 4948 return PCI_ERS_RESULT_NEED_RESET; 4949 case pci_channel_io_perm_failure: 4950 /* Permanent error, prepare for device removal */ 4951 return PCI_ERS_RESULT_DISCONNECT; 4952 } 4953 4954 return PCI_ERS_RESULT_NEED_RESET; 4955 } 4956 4957 /** 4958 * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers 4959 * @pdev: pointer to PCI device 4960 */ 4961 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev) 4962 { 4963 4964 DRM_INFO("PCI error: mmio enabled callback!!\n"); 4965 4966 /* TODO - dump whatever for debugging purposes */ 4967 4968 /* This called only if amdgpu_pci_error_detected returns 4969 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still 4970 * works, no need to reset slot. 4971 */ 4972 4973 return PCI_ERS_RESULT_RECOVERED; 4974 } 4975 4976 /** 4977 * amdgpu_pci_slot_reset - Called when PCI slot has been reset. 4978 * @pdev: PCI device struct 4979 * 4980 * Description: This routine is called by the pci error recovery 4981 * code after the PCI slot has been reset, just before we 4982 * should resume normal operations. 4983 */ 4984 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev) 4985 { 4986 struct drm_device *dev = pci_get_drvdata(pdev); 4987 struct amdgpu_device *adev = drm_to_adev(dev); 4988 int r, i; 4989 bool need_full_reset = true; 4990 u32 memsize; 4991 struct list_head device_list; 4992 4993 DRM_INFO("PCI error: slot reset callback!!\n"); 4994 4995 INIT_LIST_HEAD(&device_list); 4996 list_add_tail(&adev->gmc.xgmi.head, &device_list); 4997 4998 /* wait for asic to come out of reset */ 4999 msleep(500); 5000 5001 /* Restore PCI confspace */ 5002 amdgpu_device_load_pci_state(pdev); 5003 5004 /* confirm ASIC came out of reset */ 5005 for (i = 0; i < adev->usec_timeout; i++) { 5006 memsize = amdgpu_asic_get_config_memsize(adev); 5007 5008 if (memsize != 0xffffffff) 5009 break; 5010 udelay(1); 5011 } 5012 if (memsize == 0xffffffff) { 5013 r = -ETIME; 5014 goto out; 5015 } 5016 5017 adev->in_pci_err_recovery = true; 5018 r = amdgpu_device_pre_asic_reset(adev, NULL, &need_full_reset); 5019 adev->in_pci_err_recovery = false; 5020 if (r) 5021 goto out; 5022 5023 r = amdgpu_do_asic_reset(NULL, &device_list, &need_full_reset, true); 5024 5025 out: 5026 if (!r) { 5027 if (amdgpu_device_cache_pci_state(adev->pdev)) 5028 pci_restore_state(adev->pdev); 5029 5030 DRM_INFO("PCIe error recovery succeeded\n"); 5031 } else { 5032 DRM_ERROR("PCIe error recovery failed, err:%d", r); 5033 amdgpu_device_unlock_adev(adev); 5034 } 5035 5036 return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; 5037 } 5038 5039 /** 5040 * amdgpu_pci_resume() - resume normal ops after PCI reset 5041 * @pdev: pointer to PCI device 5042 * 5043 * Called when the error recovery driver tells us that its 5044 * OK to resume normal operation. Use completion to allow 5045 * halted scsi ops to resume. 5046 */ 5047 void amdgpu_pci_resume(struct pci_dev *pdev) 5048 { 5049 struct drm_device *dev = pci_get_drvdata(pdev); 5050 struct amdgpu_device *adev = drm_to_adev(dev); 5051 int i; 5052 5053 5054 DRM_INFO("PCI error: resume callback!!\n"); 5055 5056 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { 5057 struct amdgpu_ring *ring = adev->rings[i]; 5058 5059 if (!ring || !ring->sched.thread) 5060 continue; 5061 5062 5063 drm_sched_resubmit_jobs(&ring->sched); 5064 drm_sched_start(&ring->sched, true); 5065 } 5066 5067 amdgpu_device_unlock_adev(adev); 5068 } 5069 5070 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev) 5071 { 5072 struct drm_device *dev = pci_get_drvdata(pdev); 5073 struct amdgpu_device *adev = drm_to_adev(dev); 5074 int r; 5075 5076 r = pci_save_state(pdev); 5077 if (!r) { 5078 kfree(adev->pci_state); 5079 5080 adev->pci_state = pci_store_saved_state(pdev); 5081 5082 if (!adev->pci_state) { 5083 DRM_ERROR("Failed to store PCI saved state"); 5084 return false; 5085 } 5086 } else { 5087 DRM_WARN("Failed to save PCI state, err:%d\n", r); 5088 return false; 5089 } 5090 5091 return true; 5092 } 5093 5094 bool amdgpu_device_load_pci_state(struct pci_dev *pdev) 5095 { 5096 struct drm_device *dev = pci_get_drvdata(pdev); 5097 struct amdgpu_device *adev = drm_to_adev(dev); 5098 int r; 5099 5100 if (!adev->pci_state) 5101 return false; 5102 5103 r = pci_load_saved_state(pdev, adev->pci_state); 5104 5105 if (!r) { 5106 pci_restore_state(pdev); 5107 } else { 5108 DRM_WARN("Failed to load PCI state, err:%d\n", r); 5109 return false; 5110 } 5111 5112 return true; 5113 } 5114 5115 5116