1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/power_supply.h>
29 #include <linux/kthread.h>
30 #include <linux/module.h>
31 #include <linux/console.h>
32 #include <linux/slab.h>
33 #include <linux/iommu.h>
34 #include <linux/pci.h>
35 #include <linux/devcoredump.h>
36 #include <generated/utsrelease.h>
37 #include <linux/pci-p2pdma.h>
38 #include <linux/apple-gmux.h>
39 
40 #include <drm/drm_aperture.h>
41 #include <drm/drm_atomic_helper.h>
42 #include <drm/drm_crtc_helper.h>
43 #include <drm/drm_fb_helper.h>
44 #include <drm/drm_probe_helper.h>
45 #include <drm/amdgpu_drm.h>
46 #include <linux/vgaarb.h>
47 #include <linux/vga_switcheroo.h>
48 #include <linux/efi.h>
49 #include "amdgpu.h"
50 #include "amdgpu_trace.h"
51 #include "amdgpu_i2c.h"
52 #include "atom.h"
53 #include "amdgpu_atombios.h"
54 #include "amdgpu_atomfirmware.h"
55 #include "amd_pcie.h"
56 #ifdef CONFIG_DRM_AMDGPU_SI
57 #include "si.h"
58 #endif
59 #ifdef CONFIG_DRM_AMDGPU_CIK
60 #include "cik.h"
61 #endif
62 #include "vi.h"
63 #include "soc15.h"
64 #include "nv.h"
65 #include "bif/bif_4_1_d.h"
66 #include <linux/firmware.h>
67 #include "amdgpu_vf_error.h"
68 
69 #include "amdgpu_amdkfd.h"
70 #include "amdgpu_pm.h"
71 
72 #include "amdgpu_xgmi.h"
73 #include "amdgpu_ras.h"
74 #include "amdgpu_pmu.h"
75 #include "amdgpu_fru_eeprom.h"
76 #include "amdgpu_reset.h"
77 
78 #include <linux/suspend.h>
79 #include <drm/task_barrier.h>
80 #include <linux/pm_runtime.h>
81 
82 #include <drm/drm_drv.h>
83 
84 #if IS_ENABLED(CONFIG_X86)
85 #include <asm/intel-family.h>
86 #endif
87 
88 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
89 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
90 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
91 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
92 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
93 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
94 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
95 
96 #define AMDGPU_RESUME_MS		2000
97 #define AMDGPU_MAX_RETRY_LIMIT		2
98 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
99 
100 static const struct drm_driver amdgpu_kms_driver;
101 
102 const char *amdgpu_asic_name[] = {
103 	"TAHITI",
104 	"PITCAIRN",
105 	"VERDE",
106 	"OLAND",
107 	"HAINAN",
108 	"BONAIRE",
109 	"KAVERI",
110 	"KABINI",
111 	"HAWAII",
112 	"MULLINS",
113 	"TOPAZ",
114 	"TONGA",
115 	"FIJI",
116 	"CARRIZO",
117 	"STONEY",
118 	"POLARIS10",
119 	"POLARIS11",
120 	"POLARIS12",
121 	"VEGAM",
122 	"VEGA10",
123 	"VEGA12",
124 	"VEGA20",
125 	"RAVEN",
126 	"ARCTURUS",
127 	"RENOIR",
128 	"ALDEBARAN",
129 	"NAVI10",
130 	"CYAN_SKILLFISH",
131 	"NAVI14",
132 	"NAVI12",
133 	"SIENNA_CICHLID",
134 	"NAVY_FLOUNDER",
135 	"VANGOGH",
136 	"DIMGREY_CAVEFISH",
137 	"BEIGE_GOBY",
138 	"YELLOW_CARP",
139 	"IP DISCOVERY",
140 	"LAST",
141 };
142 
143 /**
144  * DOC: pcie_replay_count
145  *
146  * The amdgpu driver provides a sysfs API for reporting the total number
147  * of PCIe replays (NAKs)
148  * The file pcie_replay_count is used for this and returns the total
149  * number of replays as a sum of the NAKs generated and NAKs received
150  */
151 
152 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
153 		struct device_attribute *attr, char *buf)
154 {
155 	struct drm_device *ddev = dev_get_drvdata(dev);
156 	struct amdgpu_device *adev = drm_to_adev(ddev);
157 	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
158 
159 	return sysfs_emit(buf, "%llu\n", cnt);
160 }
161 
162 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
163 		amdgpu_device_get_pcie_replay_count, NULL);
164 
165 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
166 
167 /**
168  * DOC: product_name
169  *
170  * The amdgpu driver provides a sysfs API for reporting the product name
171  * for the device
172  * The file product_name is used for this and returns the product name
173  * as returned from the FRU.
174  * NOTE: This is only available for certain server cards
175  */
176 
177 static ssize_t amdgpu_device_get_product_name(struct device *dev,
178 		struct device_attribute *attr, char *buf)
179 {
180 	struct drm_device *ddev = dev_get_drvdata(dev);
181 	struct amdgpu_device *adev = drm_to_adev(ddev);
182 
183 	return sysfs_emit(buf, "%s\n", adev->product_name);
184 }
185 
186 static DEVICE_ATTR(product_name, S_IRUGO,
187 		amdgpu_device_get_product_name, NULL);
188 
189 /**
190  * DOC: product_number
191  *
192  * The amdgpu driver provides a sysfs API for reporting the part number
193  * for the device
194  * The file product_number is used for this and returns the part number
195  * as returned from the FRU.
196  * NOTE: This is only available for certain server cards
197  */
198 
199 static ssize_t amdgpu_device_get_product_number(struct device *dev,
200 		struct device_attribute *attr, char *buf)
201 {
202 	struct drm_device *ddev = dev_get_drvdata(dev);
203 	struct amdgpu_device *adev = drm_to_adev(ddev);
204 
205 	return sysfs_emit(buf, "%s\n", adev->product_number);
206 }
207 
208 static DEVICE_ATTR(product_number, S_IRUGO,
209 		amdgpu_device_get_product_number, NULL);
210 
211 /**
212  * DOC: serial_number
213  *
214  * The amdgpu driver provides a sysfs API for reporting the serial number
215  * for the device
216  * The file serial_number is used for this and returns the serial number
217  * as returned from the FRU.
218  * NOTE: This is only available for certain server cards
219  */
220 
221 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
222 		struct device_attribute *attr, char *buf)
223 {
224 	struct drm_device *ddev = dev_get_drvdata(dev);
225 	struct amdgpu_device *adev = drm_to_adev(ddev);
226 
227 	return sysfs_emit(buf, "%s\n", adev->serial);
228 }
229 
230 static DEVICE_ATTR(serial_number, S_IRUGO,
231 		amdgpu_device_get_serial_number, NULL);
232 
233 /**
234  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
235  *
236  * @dev: drm_device pointer
237  *
238  * Returns true if the device is a dGPU with ATPX power control,
239  * otherwise return false.
240  */
241 bool amdgpu_device_supports_px(struct drm_device *dev)
242 {
243 	struct amdgpu_device *adev = drm_to_adev(dev);
244 
245 	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
246 		return true;
247 	return false;
248 }
249 
250 /**
251  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
252  *
253  * @dev: drm_device pointer
254  *
255  * Returns true if the device is a dGPU with ACPI power control,
256  * otherwise return false.
257  */
258 bool amdgpu_device_supports_boco(struct drm_device *dev)
259 {
260 	struct amdgpu_device *adev = drm_to_adev(dev);
261 
262 	if (adev->has_pr3 ||
263 	    ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
264 		return true;
265 	return false;
266 }
267 
268 /**
269  * amdgpu_device_supports_baco - Does the device support BACO
270  *
271  * @dev: drm_device pointer
272  *
273  * Returns true if the device supporte BACO,
274  * otherwise return false.
275  */
276 bool amdgpu_device_supports_baco(struct drm_device *dev)
277 {
278 	struct amdgpu_device *adev = drm_to_adev(dev);
279 
280 	return amdgpu_asic_supports_baco(adev);
281 }
282 
283 /**
284  * amdgpu_device_supports_smart_shift - Is the device dGPU with
285  * smart shift support
286  *
287  * @dev: drm_device pointer
288  *
289  * Returns true if the device is a dGPU with Smart Shift support,
290  * otherwise returns false.
291  */
292 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
293 {
294 	return (amdgpu_device_supports_boco(dev) &&
295 		amdgpu_acpi_is_power_shift_control_supported());
296 }
297 
298 /*
299  * VRAM access helper functions
300  */
301 
302 /**
303  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
304  *
305  * @adev: amdgpu_device pointer
306  * @pos: offset of the buffer in vram
307  * @buf: virtual address of the buffer in system memory
308  * @size: read/write size, sizeof(@buf) must > @size
309  * @write: true - write to vram, otherwise - read from vram
310  */
311 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
312 			     void *buf, size_t size, bool write)
313 {
314 	unsigned long flags;
315 	uint32_t hi = ~0, tmp = 0;
316 	uint32_t *data = buf;
317 	uint64_t last;
318 	int idx;
319 
320 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
321 		return;
322 
323 	BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
324 
325 	spin_lock_irqsave(&adev->mmio_idx_lock, flags);
326 	for (last = pos + size; pos < last; pos += 4) {
327 		tmp = pos >> 31;
328 
329 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
330 		if (tmp != hi) {
331 			WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
332 			hi = tmp;
333 		}
334 		if (write)
335 			WREG32_NO_KIQ(mmMM_DATA, *data++);
336 		else
337 			*data++ = RREG32_NO_KIQ(mmMM_DATA);
338 	}
339 
340 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
341 	drm_dev_exit(idx);
342 }
343 
344 /**
345  * amdgpu_device_aper_access - access vram by vram aperature
346  *
347  * @adev: amdgpu_device pointer
348  * @pos: offset of the buffer in vram
349  * @buf: virtual address of the buffer in system memory
350  * @size: read/write size, sizeof(@buf) must > @size
351  * @write: true - write to vram, otherwise - read from vram
352  *
353  * The return value means how many bytes have been transferred.
354  */
355 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
356 				 void *buf, size_t size, bool write)
357 {
358 #ifdef CONFIG_64BIT
359 	void __iomem *addr;
360 	size_t count = 0;
361 	uint64_t last;
362 
363 	if (!adev->mman.aper_base_kaddr)
364 		return 0;
365 
366 	last = min(pos + size, adev->gmc.visible_vram_size);
367 	if (last > pos) {
368 		addr = adev->mman.aper_base_kaddr + pos;
369 		count = last - pos;
370 
371 		if (write) {
372 			memcpy_toio(addr, buf, count);
373 			mb();
374 			amdgpu_device_flush_hdp(adev, NULL);
375 		} else {
376 			amdgpu_device_invalidate_hdp(adev, NULL);
377 			mb();
378 			memcpy_fromio(buf, addr, count);
379 		}
380 
381 	}
382 
383 	return count;
384 #else
385 	return 0;
386 #endif
387 }
388 
389 /**
390  * amdgpu_device_vram_access - read/write a buffer in vram
391  *
392  * @adev: amdgpu_device pointer
393  * @pos: offset of the buffer in vram
394  * @buf: virtual address of the buffer in system memory
395  * @size: read/write size, sizeof(@buf) must > @size
396  * @write: true - write to vram, otherwise - read from vram
397  */
398 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
399 			       void *buf, size_t size, bool write)
400 {
401 	size_t count;
402 
403 	/* try to using vram apreature to access vram first */
404 	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
405 	size -= count;
406 	if (size) {
407 		/* using MM to access rest vram */
408 		pos += count;
409 		buf += count;
410 		amdgpu_device_mm_access(adev, pos, buf, size, write);
411 	}
412 }
413 
414 /*
415  * register access helper functions.
416  */
417 
418 /* Check if hw access should be skipped because of hotplug or device error */
419 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
420 {
421 	if (adev->no_hw_access)
422 		return true;
423 
424 #ifdef CONFIG_LOCKDEP
425 	/*
426 	 * This is a bit complicated to understand, so worth a comment. What we assert
427 	 * here is that the GPU reset is not running on another thread in parallel.
428 	 *
429 	 * For this we trylock the read side of the reset semaphore, if that succeeds
430 	 * we know that the reset is not running in paralell.
431 	 *
432 	 * If the trylock fails we assert that we are either already holding the read
433 	 * side of the lock or are the reset thread itself and hold the write side of
434 	 * the lock.
435 	 */
436 	if (in_task()) {
437 		if (down_read_trylock(&adev->reset_domain->sem))
438 			up_read(&adev->reset_domain->sem);
439 		else
440 			lockdep_assert_held(&adev->reset_domain->sem);
441 	}
442 #endif
443 	return false;
444 }
445 
446 /**
447  * amdgpu_device_rreg - read a memory mapped IO or indirect register
448  *
449  * @adev: amdgpu_device pointer
450  * @reg: dword aligned register offset
451  * @acc_flags: access flags which require special behavior
452  *
453  * Returns the 32 bit value from the offset specified.
454  */
455 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
456 			    uint32_t reg, uint32_t acc_flags)
457 {
458 	uint32_t ret;
459 
460 	if (amdgpu_device_skip_hw_access(adev))
461 		return 0;
462 
463 	if ((reg * 4) < adev->rmmio_size) {
464 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
465 		    amdgpu_sriov_runtime(adev) &&
466 		    down_read_trylock(&adev->reset_domain->sem)) {
467 			ret = amdgpu_kiq_rreg(adev, reg);
468 			up_read(&adev->reset_domain->sem);
469 		} else {
470 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
471 		}
472 	} else {
473 		ret = adev->pcie_rreg(adev, reg * 4);
474 	}
475 
476 	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
477 
478 	return ret;
479 }
480 
481 /*
482  * MMIO register read with bytes helper functions
483  * @offset:bytes offset from MMIO start
484  *
485 */
486 
487 /**
488  * amdgpu_mm_rreg8 - read a memory mapped IO register
489  *
490  * @adev: amdgpu_device pointer
491  * @offset: byte aligned register offset
492  *
493  * Returns the 8 bit value from the offset specified.
494  */
495 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
496 {
497 	if (amdgpu_device_skip_hw_access(adev))
498 		return 0;
499 
500 	if (offset < adev->rmmio_size)
501 		return (readb(adev->rmmio + offset));
502 	BUG();
503 }
504 
505 /*
506  * MMIO register write with bytes helper functions
507  * @offset:bytes offset from MMIO start
508  * @value: the value want to be written to the register
509  *
510 */
511 /**
512  * amdgpu_mm_wreg8 - read a memory mapped IO register
513  *
514  * @adev: amdgpu_device pointer
515  * @offset: byte aligned register offset
516  * @value: 8 bit value to write
517  *
518  * Writes the value specified to the offset specified.
519  */
520 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
521 {
522 	if (amdgpu_device_skip_hw_access(adev))
523 		return;
524 
525 	if (offset < adev->rmmio_size)
526 		writeb(value, adev->rmmio + offset);
527 	else
528 		BUG();
529 }
530 
531 /**
532  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
533  *
534  * @adev: amdgpu_device pointer
535  * @reg: dword aligned register offset
536  * @v: 32 bit value to write to the register
537  * @acc_flags: access flags which require special behavior
538  *
539  * Writes the value specified to the offset specified.
540  */
541 void amdgpu_device_wreg(struct amdgpu_device *adev,
542 			uint32_t reg, uint32_t v,
543 			uint32_t acc_flags)
544 {
545 	if (amdgpu_device_skip_hw_access(adev))
546 		return;
547 
548 	if ((reg * 4) < adev->rmmio_size) {
549 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
550 		    amdgpu_sriov_runtime(adev) &&
551 		    down_read_trylock(&adev->reset_domain->sem)) {
552 			amdgpu_kiq_wreg(adev, reg, v);
553 			up_read(&adev->reset_domain->sem);
554 		} else {
555 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
556 		}
557 	} else {
558 		adev->pcie_wreg(adev, reg * 4, v);
559 	}
560 
561 	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
562 }
563 
564 /**
565  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
566  *
567  * @adev: amdgpu_device pointer
568  * @reg: mmio/rlc register
569  * @v: value to write
570  *
571  * this function is invoked only for the debugfs register access
572  */
573 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
574 			     uint32_t reg, uint32_t v)
575 {
576 	if (amdgpu_device_skip_hw_access(adev))
577 		return;
578 
579 	if (amdgpu_sriov_fullaccess(adev) &&
580 	    adev->gfx.rlc.funcs &&
581 	    adev->gfx.rlc.funcs->is_rlcg_access_range) {
582 		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
583 			return amdgpu_sriov_wreg(adev, reg, v, 0, 0);
584 	} else if ((reg * 4) >= adev->rmmio_size) {
585 		adev->pcie_wreg(adev, reg * 4, v);
586 	} else {
587 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
588 	}
589 }
590 
591 /**
592  * amdgpu_mm_rdoorbell - read a doorbell dword
593  *
594  * @adev: amdgpu_device pointer
595  * @index: doorbell index
596  *
597  * Returns the value in the doorbell aperture at the
598  * requested doorbell index (CIK).
599  */
600 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
601 {
602 	if (amdgpu_device_skip_hw_access(adev))
603 		return 0;
604 
605 	if (index < adev->doorbell.num_kernel_doorbells) {
606 		return readl(adev->doorbell.ptr + index);
607 	} else {
608 		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
609 		return 0;
610 	}
611 }
612 
613 /**
614  * amdgpu_mm_wdoorbell - write a doorbell dword
615  *
616  * @adev: amdgpu_device pointer
617  * @index: doorbell index
618  * @v: value to write
619  *
620  * Writes @v to the doorbell aperture at the
621  * requested doorbell index (CIK).
622  */
623 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
624 {
625 	if (amdgpu_device_skip_hw_access(adev))
626 		return;
627 
628 	if (index < adev->doorbell.num_kernel_doorbells) {
629 		writel(v, adev->doorbell.ptr + index);
630 	} else {
631 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
632 	}
633 }
634 
635 /**
636  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
637  *
638  * @adev: amdgpu_device pointer
639  * @index: doorbell index
640  *
641  * Returns the value in the doorbell aperture at the
642  * requested doorbell index (VEGA10+).
643  */
644 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
645 {
646 	if (amdgpu_device_skip_hw_access(adev))
647 		return 0;
648 
649 	if (index < adev->doorbell.num_kernel_doorbells) {
650 		return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
651 	} else {
652 		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
653 		return 0;
654 	}
655 }
656 
657 /**
658  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
659  *
660  * @adev: amdgpu_device pointer
661  * @index: doorbell index
662  * @v: value to write
663  *
664  * Writes @v to the doorbell aperture at the
665  * requested doorbell index (VEGA10+).
666  */
667 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
668 {
669 	if (amdgpu_device_skip_hw_access(adev))
670 		return;
671 
672 	if (index < adev->doorbell.num_kernel_doorbells) {
673 		atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
674 	} else {
675 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
676 	}
677 }
678 
679 /**
680  * amdgpu_device_indirect_rreg - read an indirect register
681  *
682  * @adev: amdgpu_device pointer
683  * @reg_addr: indirect register address to read from
684  *
685  * Returns the value of indirect register @reg_addr
686  */
687 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
688 				u32 reg_addr)
689 {
690 	unsigned long flags, pcie_index, pcie_data;
691 	void __iomem *pcie_index_offset;
692 	void __iomem *pcie_data_offset;
693 	u32 r;
694 
695 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
696 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
697 
698 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
699 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
700 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
701 
702 	writel(reg_addr, pcie_index_offset);
703 	readl(pcie_index_offset);
704 	r = readl(pcie_data_offset);
705 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
706 
707 	return r;
708 }
709 
710 /**
711  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
712  *
713  * @adev: amdgpu_device pointer
714  * @reg_addr: indirect register address to read from
715  *
716  * Returns the value of indirect register @reg_addr
717  */
718 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
719 				  u32 reg_addr)
720 {
721 	unsigned long flags, pcie_index, pcie_data;
722 	void __iomem *pcie_index_offset;
723 	void __iomem *pcie_data_offset;
724 	u64 r;
725 
726 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
727 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
728 
729 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
730 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
731 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
732 
733 	/* read low 32 bits */
734 	writel(reg_addr, pcie_index_offset);
735 	readl(pcie_index_offset);
736 	r = readl(pcie_data_offset);
737 	/* read high 32 bits */
738 	writel(reg_addr + 4, pcie_index_offset);
739 	readl(pcie_index_offset);
740 	r |= ((u64)readl(pcie_data_offset) << 32);
741 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
742 
743 	return r;
744 }
745 
746 /**
747  * amdgpu_device_indirect_wreg - write an indirect register address
748  *
749  * @adev: amdgpu_device pointer
750  * @pcie_index: mmio register offset
751  * @pcie_data: mmio register offset
752  * @reg_addr: indirect register offset
753  * @reg_data: indirect register data
754  *
755  */
756 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
757 				 u32 reg_addr, u32 reg_data)
758 {
759 	unsigned long flags, pcie_index, pcie_data;
760 	void __iomem *pcie_index_offset;
761 	void __iomem *pcie_data_offset;
762 
763 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
764 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
765 
766 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
767 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
768 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
769 
770 	writel(reg_addr, pcie_index_offset);
771 	readl(pcie_index_offset);
772 	writel(reg_data, pcie_data_offset);
773 	readl(pcie_data_offset);
774 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
775 }
776 
777 /**
778  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
779  *
780  * @adev: amdgpu_device pointer
781  * @pcie_index: mmio register offset
782  * @pcie_data: mmio register offset
783  * @reg_addr: indirect register offset
784  * @reg_data: indirect register data
785  *
786  */
787 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
788 				   u32 reg_addr, u64 reg_data)
789 {
790 	unsigned long flags, pcie_index, pcie_data;
791 	void __iomem *pcie_index_offset;
792 	void __iomem *pcie_data_offset;
793 
794 	pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
795 	pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
796 
797 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
798 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
799 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
800 
801 	/* write low 32 bits */
802 	writel(reg_addr, pcie_index_offset);
803 	readl(pcie_index_offset);
804 	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
805 	readl(pcie_data_offset);
806 	/* write high 32 bits */
807 	writel(reg_addr + 4, pcie_index_offset);
808 	readl(pcie_index_offset);
809 	writel((u32)(reg_data >> 32), pcie_data_offset);
810 	readl(pcie_data_offset);
811 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
812 }
813 
814 /**
815  * amdgpu_device_get_rev_id - query device rev_id
816  *
817  * @adev: amdgpu_device pointer
818  *
819  * Return device rev_id
820  */
821 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
822 {
823 	return adev->nbio.funcs->get_rev_id(adev);
824 }
825 
826 /**
827  * amdgpu_invalid_rreg - dummy reg read function
828  *
829  * @adev: amdgpu_device pointer
830  * @reg: offset of register
831  *
832  * Dummy register read function.  Used for register blocks
833  * that certain asics don't have (all asics).
834  * Returns the value in the register.
835  */
836 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
837 {
838 	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
839 	BUG();
840 	return 0;
841 }
842 
843 /**
844  * amdgpu_invalid_wreg - dummy reg write function
845  *
846  * @adev: amdgpu_device pointer
847  * @reg: offset of register
848  * @v: value to write to the register
849  *
850  * Dummy register read function.  Used for register blocks
851  * that certain asics don't have (all asics).
852  */
853 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
854 {
855 	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
856 		  reg, v);
857 	BUG();
858 }
859 
860 /**
861  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
862  *
863  * @adev: amdgpu_device pointer
864  * @reg: offset of register
865  *
866  * Dummy register read function.  Used for register blocks
867  * that certain asics don't have (all asics).
868  * Returns the value in the register.
869  */
870 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
871 {
872 	DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
873 	BUG();
874 	return 0;
875 }
876 
877 /**
878  * amdgpu_invalid_wreg64 - dummy reg write function
879  *
880  * @adev: amdgpu_device pointer
881  * @reg: offset of register
882  * @v: value to write to the register
883  *
884  * Dummy register read function.  Used for register blocks
885  * that certain asics don't have (all asics).
886  */
887 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
888 {
889 	DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
890 		  reg, v);
891 	BUG();
892 }
893 
894 /**
895  * amdgpu_block_invalid_rreg - dummy reg read function
896  *
897  * @adev: amdgpu_device pointer
898  * @block: offset of instance
899  * @reg: offset of register
900  *
901  * Dummy register read function.  Used for register blocks
902  * that certain asics don't have (all asics).
903  * Returns the value in the register.
904  */
905 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
906 					  uint32_t block, uint32_t reg)
907 {
908 	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
909 		  reg, block);
910 	BUG();
911 	return 0;
912 }
913 
914 /**
915  * amdgpu_block_invalid_wreg - dummy reg write function
916  *
917  * @adev: amdgpu_device pointer
918  * @block: offset of instance
919  * @reg: offset of register
920  * @v: value to write to the register
921  *
922  * Dummy register read function.  Used for register blocks
923  * that certain asics don't have (all asics).
924  */
925 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
926 				      uint32_t block,
927 				      uint32_t reg, uint32_t v)
928 {
929 	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
930 		  reg, block, v);
931 	BUG();
932 }
933 
934 /**
935  * amdgpu_device_asic_init - Wrapper for atom asic_init
936  *
937  * @adev: amdgpu_device pointer
938  *
939  * Does any asic specific work and then calls atom asic init.
940  */
941 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
942 {
943 	amdgpu_asic_pre_asic_init(adev);
944 
945 	if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0))
946 		return amdgpu_atomfirmware_asic_init(adev, true);
947 	else
948 		return amdgpu_atom_asic_init(adev->mode_info.atom_context);
949 }
950 
951 /**
952  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
953  *
954  * @adev: amdgpu_device pointer
955  *
956  * Allocates a scratch page of VRAM for use by various things in the
957  * driver.
958  */
959 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
960 {
961 	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
962 				       AMDGPU_GEM_DOMAIN_VRAM |
963 				       AMDGPU_GEM_DOMAIN_GTT,
964 				       &adev->mem_scratch.robj,
965 				       &adev->mem_scratch.gpu_addr,
966 				       (void **)&adev->mem_scratch.ptr);
967 }
968 
969 /**
970  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
971  *
972  * @adev: amdgpu_device pointer
973  *
974  * Frees the VRAM scratch page.
975  */
976 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
977 {
978 	amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
979 }
980 
981 /**
982  * amdgpu_device_program_register_sequence - program an array of registers.
983  *
984  * @adev: amdgpu_device pointer
985  * @registers: pointer to the register array
986  * @array_size: size of the register array
987  *
988  * Programs an array or registers with and and or masks.
989  * This is a helper for setting golden registers.
990  */
991 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
992 					     const u32 *registers,
993 					     const u32 array_size)
994 {
995 	u32 tmp, reg, and_mask, or_mask;
996 	int i;
997 
998 	if (array_size % 3)
999 		return;
1000 
1001 	for (i = 0; i < array_size; i +=3) {
1002 		reg = registers[i + 0];
1003 		and_mask = registers[i + 1];
1004 		or_mask = registers[i + 2];
1005 
1006 		if (and_mask == 0xffffffff) {
1007 			tmp = or_mask;
1008 		} else {
1009 			tmp = RREG32(reg);
1010 			tmp &= ~and_mask;
1011 			if (adev->family >= AMDGPU_FAMILY_AI)
1012 				tmp |= (or_mask & and_mask);
1013 			else
1014 				tmp |= or_mask;
1015 		}
1016 		WREG32(reg, tmp);
1017 	}
1018 }
1019 
1020 /**
1021  * amdgpu_device_pci_config_reset - reset the GPU
1022  *
1023  * @adev: amdgpu_device pointer
1024  *
1025  * Resets the GPU using the pci config reset sequence.
1026  * Only applicable to asics prior to vega10.
1027  */
1028 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1029 {
1030 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1031 }
1032 
1033 /**
1034  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1035  *
1036  * @adev: amdgpu_device pointer
1037  *
1038  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1039  */
1040 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1041 {
1042 	return pci_reset_function(adev->pdev);
1043 }
1044 
1045 /*
1046  * GPU doorbell aperture helpers function.
1047  */
1048 /**
1049  * amdgpu_device_doorbell_init - Init doorbell driver information.
1050  *
1051  * @adev: amdgpu_device pointer
1052  *
1053  * Init doorbell driver information (CIK)
1054  * Returns 0 on success, error on failure.
1055  */
1056 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1057 {
1058 
1059 	/* No doorbell on SI hardware generation */
1060 	if (adev->asic_type < CHIP_BONAIRE) {
1061 		adev->doorbell.base = 0;
1062 		adev->doorbell.size = 0;
1063 		adev->doorbell.num_kernel_doorbells = 0;
1064 		adev->doorbell.ptr = NULL;
1065 		return 0;
1066 	}
1067 
1068 	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1069 		return -EINVAL;
1070 
1071 	amdgpu_asic_init_doorbell_index(adev);
1072 
1073 	/* doorbell bar mapping */
1074 	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1075 	adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1076 
1077 	if (adev->enable_mes) {
1078 		adev->doorbell.num_kernel_doorbells =
1079 			adev->doorbell.size / sizeof(u32);
1080 	} else {
1081 		adev->doorbell.num_kernel_doorbells =
1082 			min_t(u32, adev->doorbell.size / sizeof(u32),
1083 			      adev->doorbell_index.max_assignment+1);
1084 		if (adev->doorbell.num_kernel_doorbells == 0)
1085 			return -EINVAL;
1086 
1087 		/* For Vega, reserve and map two pages on doorbell BAR since SDMA
1088 		 * paging queue doorbell use the second page. The
1089 		 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1090 		 * doorbells are in the first page. So with paging queue enabled,
1091 		 * the max num_kernel_doorbells should + 1 page (0x400 in dword)
1092 		 */
1093 		if (adev->asic_type >= CHIP_VEGA10)
1094 			adev->doorbell.num_kernel_doorbells += 0x400;
1095 	}
1096 
1097 	adev->doorbell.ptr = ioremap(adev->doorbell.base,
1098 				     adev->doorbell.num_kernel_doorbells *
1099 				     sizeof(u32));
1100 	if (adev->doorbell.ptr == NULL)
1101 		return -ENOMEM;
1102 
1103 	return 0;
1104 }
1105 
1106 /**
1107  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1108  *
1109  * @adev: amdgpu_device pointer
1110  *
1111  * Tear down doorbell driver information (CIK)
1112  */
1113 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1114 {
1115 	iounmap(adev->doorbell.ptr);
1116 	adev->doorbell.ptr = NULL;
1117 }
1118 
1119 
1120 
1121 /*
1122  * amdgpu_device_wb_*()
1123  * Writeback is the method by which the GPU updates special pages in memory
1124  * with the status of certain GPU events (fences, ring pointers,etc.).
1125  */
1126 
1127 /**
1128  * amdgpu_device_wb_fini - Disable Writeback and free memory
1129  *
1130  * @adev: amdgpu_device pointer
1131  *
1132  * Disables Writeback and frees the Writeback memory (all asics).
1133  * Used at driver shutdown.
1134  */
1135 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1136 {
1137 	if (adev->wb.wb_obj) {
1138 		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1139 				      &adev->wb.gpu_addr,
1140 				      (void **)&adev->wb.wb);
1141 		adev->wb.wb_obj = NULL;
1142 	}
1143 }
1144 
1145 /**
1146  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1147  *
1148  * @adev: amdgpu_device pointer
1149  *
1150  * Initializes writeback and allocates writeback memory (all asics).
1151  * Used at driver startup.
1152  * Returns 0 on success or an -error on failure.
1153  */
1154 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1155 {
1156 	int r;
1157 
1158 	if (adev->wb.wb_obj == NULL) {
1159 		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1160 		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1161 					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1162 					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
1163 					    (void **)&adev->wb.wb);
1164 		if (r) {
1165 			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1166 			return r;
1167 		}
1168 
1169 		adev->wb.num_wb = AMDGPU_MAX_WB;
1170 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1171 
1172 		/* clear wb memory */
1173 		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1174 	}
1175 
1176 	return 0;
1177 }
1178 
1179 /**
1180  * amdgpu_device_wb_get - Allocate a wb entry
1181  *
1182  * @adev: amdgpu_device pointer
1183  * @wb: wb index
1184  *
1185  * Allocate a wb slot for use by the driver (all asics).
1186  * Returns 0 on success or -EINVAL on failure.
1187  */
1188 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1189 {
1190 	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1191 
1192 	if (offset < adev->wb.num_wb) {
1193 		__set_bit(offset, adev->wb.used);
1194 		*wb = offset << 3; /* convert to dw offset */
1195 		return 0;
1196 	} else {
1197 		return -EINVAL;
1198 	}
1199 }
1200 
1201 /**
1202  * amdgpu_device_wb_free - Free a wb entry
1203  *
1204  * @adev: amdgpu_device pointer
1205  * @wb: wb index
1206  *
1207  * Free a wb slot allocated for use by the driver (all asics)
1208  */
1209 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1210 {
1211 	wb >>= 3;
1212 	if (wb < adev->wb.num_wb)
1213 		__clear_bit(wb, adev->wb.used);
1214 }
1215 
1216 /**
1217  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1218  *
1219  * @adev: amdgpu_device pointer
1220  *
1221  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1222  * to fail, but if any of the BARs is not accessible after the size we abort
1223  * driver loading by returning -ENODEV.
1224  */
1225 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1226 {
1227 	int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1228 	struct pci_bus *root;
1229 	struct resource *res;
1230 	unsigned i;
1231 	u16 cmd;
1232 	int r;
1233 
1234 	/* Bypass for VF */
1235 	if (amdgpu_sriov_vf(adev))
1236 		return 0;
1237 
1238 	/* skip if the bios has already enabled large BAR */
1239 	if (adev->gmc.real_vram_size &&
1240 	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1241 		return 0;
1242 
1243 	/* Check if the root BUS has 64bit memory resources */
1244 	root = adev->pdev->bus;
1245 	while (root->parent)
1246 		root = root->parent;
1247 
1248 	pci_bus_for_each_resource(root, res, i) {
1249 		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1250 		    res->start > 0x100000000ull)
1251 			break;
1252 	}
1253 
1254 	/* Trying to resize is pointless without a root hub window above 4GB */
1255 	if (!res)
1256 		return 0;
1257 
1258 	/* Limit the BAR size to what is available */
1259 	rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1260 			rbar_size);
1261 
1262 	/* Disable memory decoding while we change the BAR addresses and size */
1263 	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1264 	pci_write_config_word(adev->pdev, PCI_COMMAND,
1265 			      cmd & ~PCI_COMMAND_MEMORY);
1266 
1267 	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
1268 	amdgpu_device_doorbell_fini(adev);
1269 	if (adev->asic_type >= CHIP_BONAIRE)
1270 		pci_release_resource(adev->pdev, 2);
1271 
1272 	pci_release_resource(adev->pdev, 0);
1273 
1274 	r = pci_resize_resource(adev->pdev, 0, rbar_size);
1275 	if (r == -ENOSPC)
1276 		DRM_INFO("Not enough PCI address space for a large BAR.");
1277 	else if (r && r != -ENOTSUPP)
1278 		DRM_ERROR("Problem resizing BAR0 (%d).", r);
1279 
1280 	pci_assign_unassigned_bus_resources(adev->pdev->bus);
1281 
1282 	/* When the doorbell or fb BAR isn't available we have no chance of
1283 	 * using the device.
1284 	 */
1285 	r = amdgpu_device_doorbell_init(adev);
1286 	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1287 		return -ENODEV;
1288 
1289 	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1290 
1291 	return 0;
1292 }
1293 
1294 /*
1295  * GPU helpers function.
1296  */
1297 /**
1298  * amdgpu_device_need_post - check if the hw need post or not
1299  *
1300  * @adev: amdgpu_device pointer
1301  *
1302  * Check if the asic has been initialized (all asics) at driver startup
1303  * or post is needed if  hw reset is performed.
1304  * Returns true if need or false if not.
1305  */
1306 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1307 {
1308 	uint32_t reg;
1309 
1310 	if (amdgpu_sriov_vf(adev))
1311 		return false;
1312 
1313 	if (amdgpu_passthrough(adev)) {
1314 		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1315 		 * some old smc fw still need driver do vPost otherwise gpu hang, while
1316 		 * those smc fw version above 22.15 doesn't have this flaw, so we force
1317 		 * vpost executed for smc version below 22.15
1318 		 */
1319 		if (adev->asic_type == CHIP_FIJI) {
1320 			int err;
1321 			uint32_t fw_ver;
1322 			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1323 			/* force vPost if error occured */
1324 			if (err)
1325 				return true;
1326 
1327 			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1328 			if (fw_ver < 0x00160e00)
1329 				return true;
1330 		}
1331 	}
1332 
1333 	/* Don't post if we need to reset whole hive on init */
1334 	if (adev->gmc.xgmi.pending_reset)
1335 		return false;
1336 
1337 	if (adev->has_hw_reset) {
1338 		adev->has_hw_reset = false;
1339 		return true;
1340 	}
1341 
1342 	/* bios scratch used on CIK+ */
1343 	if (adev->asic_type >= CHIP_BONAIRE)
1344 		return amdgpu_atombios_scratch_need_asic_init(adev);
1345 
1346 	/* check MEM_SIZE for older asics */
1347 	reg = amdgpu_asic_get_config_memsize(adev);
1348 
1349 	if ((reg != 0) && (reg != 0xffffffff))
1350 		return false;
1351 
1352 	return true;
1353 }
1354 
1355 /**
1356  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1357  *
1358  * @adev: amdgpu_device pointer
1359  *
1360  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1361  * be set for this device.
1362  *
1363  * Returns true if it should be used or false if not.
1364  */
1365 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1366 {
1367 	switch (amdgpu_aspm) {
1368 	case -1:
1369 		break;
1370 	case 0:
1371 		return false;
1372 	case 1:
1373 		return true;
1374 	default:
1375 		return false;
1376 	}
1377 	return pcie_aspm_enabled(adev->pdev);
1378 }
1379 
1380 bool amdgpu_device_aspm_support_quirk(void)
1381 {
1382 #if IS_ENABLED(CONFIG_X86)
1383 	struct cpuinfo_x86 *c = &cpu_data(0);
1384 
1385 	return !(c->x86 == 6 && c->x86_model == INTEL_FAM6_ALDERLAKE);
1386 #else
1387 	return true;
1388 #endif
1389 }
1390 
1391 /* if we get transitioned to only one device, take VGA back */
1392 /**
1393  * amdgpu_device_vga_set_decode - enable/disable vga decode
1394  *
1395  * @pdev: PCI device pointer
1396  * @state: enable/disable vga decode
1397  *
1398  * Enable/disable vga decode (all asics).
1399  * Returns VGA resource flags.
1400  */
1401 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1402 		bool state)
1403 {
1404 	struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1405 	amdgpu_asic_set_vga_state(adev, state);
1406 	if (state)
1407 		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1408 		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1409 	else
1410 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1411 }
1412 
1413 /**
1414  * amdgpu_device_check_block_size - validate the vm block size
1415  *
1416  * @adev: amdgpu_device pointer
1417  *
1418  * Validates the vm block size specified via module parameter.
1419  * The vm block size defines number of bits in page table versus page directory,
1420  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1421  * page table and the remaining bits are in the page directory.
1422  */
1423 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1424 {
1425 	/* defines number of bits in page table versus page directory,
1426 	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1427 	 * page table and the remaining bits are in the page directory */
1428 	if (amdgpu_vm_block_size == -1)
1429 		return;
1430 
1431 	if (amdgpu_vm_block_size < 9) {
1432 		dev_warn(adev->dev, "VM page table size (%d) too small\n",
1433 			 amdgpu_vm_block_size);
1434 		amdgpu_vm_block_size = -1;
1435 	}
1436 }
1437 
1438 /**
1439  * amdgpu_device_check_vm_size - validate the vm size
1440  *
1441  * @adev: amdgpu_device pointer
1442  *
1443  * Validates the vm size in GB specified via module parameter.
1444  * The VM size is the size of the GPU virtual memory space in GB.
1445  */
1446 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1447 {
1448 	/* no need to check the default value */
1449 	if (amdgpu_vm_size == -1)
1450 		return;
1451 
1452 	if (amdgpu_vm_size < 1) {
1453 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1454 			 amdgpu_vm_size);
1455 		amdgpu_vm_size = -1;
1456 	}
1457 }
1458 
1459 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1460 {
1461 	struct sysinfo si;
1462 	bool is_os_64 = (sizeof(void *) == 8);
1463 	uint64_t total_memory;
1464 	uint64_t dram_size_seven_GB = 0x1B8000000;
1465 	uint64_t dram_size_three_GB = 0xB8000000;
1466 
1467 	if (amdgpu_smu_memory_pool_size == 0)
1468 		return;
1469 
1470 	if (!is_os_64) {
1471 		DRM_WARN("Not 64-bit OS, feature not supported\n");
1472 		goto def_value;
1473 	}
1474 	si_meminfo(&si);
1475 	total_memory = (uint64_t)si.totalram * si.mem_unit;
1476 
1477 	if ((amdgpu_smu_memory_pool_size == 1) ||
1478 		(amdgpu_smu_memory_pool_size == 2)) {
1479 		if (total_memory < dram_size_three_GB)
1480 			goto def_value1;
1481 	} else if ((amdgpu_smu_memory_pool_size == 4) ||
1482 		(amdgpu_smu_memory_pool_size == 8)) {
1483 		if (total_memory < dram_size_seven_GB)
1484 			goto def_value1;
1485 	} else {
1486 		DRM_WARN("Smu memory pool size not supported\n");
1487 		goto def_value;
1488 	}
1489 	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1490 
1491 	return;
1492 
1493 def_value1:
1494 	DRM_WARN("No enough system memory\n");
1495 def_value:
1496 	adev->pm.smu_prv_buffer_size = 0;
1497 }
1498 
1499 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1500 {
1501 	if (!(adev->flags & AMD_IS_APU) ||
1502 	    adev->asic_type < CHIP_RAVEN)
1503 		return 0;
1504 
1505 	switch (adev->asic_type) {
1506 	case CHIP_RAVEN:
1507 		if (adev->pdev->device == 0x15dd)
1508 			adev->apu_flags |= AMD_APU_IS_RAVEN;
1509 		if (adev->pdev->device == 0x15d8)
1510 			adev->apu_flags |= AMD_APU_IS_PICASSO;
1511 		break;
1512 	case CHIP_RENOIR:
1513 		if ((adev->pdev->device == 0x1636) ||
1514 		    (adev->pdev->device == 0x164c))
1515 			adev->apu_flags |= AMD_APU_IS_RENOIR;
1516 		else
1517 			adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1518 		break;
1519 	case CHIP_VANGOGH:
1520 		adev->apu_flags |= AMD_APU_IS_VANGOGH;
1521 		break;
1522 	case CHIP_YELLOW_CARP:
1523 		break;
1524 	case CHIP_CYAN_SKILLFISH:
1525 		if ((adev->pdev->device == 0x13FE) ||
1526 		    (adev->pdev->device == 0x143F))
1527 			adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1528 		break;
1529 	default:
1530 		break;
1531 	}
1532 
1533 	return 0;
1534 }
1535 
1536 /**
1537  * amdgpu_device_check_arguments - validate module params
1538  *
1539  * @adev: amdgpu_device pointer
1540  *
1541  * Validates certain module parameters and updates
1542  * the associated values used by the driver (all asics).
1543  */
1544 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1545 {
1546 	if (amdgpu_sched_jobs < 4) {
1547 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1548 			 amdgpu_sched_jobs);
1549 		amdgpu_sched_jobs = 4;
1550 	} else if (!is_power_of_2(amdgpu_sched_jobs)){
1551 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1552 			 amdgpu_sched_jobs);
1553 		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1554 	}
1555 
1556 	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1557 		/* gart size must be greater or equal to 32M */
1558 		dev_warn(adev->dev, "gart size (%d) too small\n",
1559 			 amdgpu_gart_size);
1560 		amdgpu_gart_size = -1;
1561 	}
1562 
1563 	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1564 		/* gtt size must be greater or equal to 32M */
1565 		dev_warn(adev->dev, "gtt size (%d) too small\n",
1566 				 amdgpu_gtt_size);
1567 		amdgpu_gtt_size = -1;
1568 	}
1569 
1570 	/* valid range is between 4 and 9 inclusive */
1571 	if (amdgpu_vm_fragment_size != -1 &&
1572 	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1573 		dev_warn(adev->dev, "valid range is between 4 and 9\n");
1574 		amdgpu_vm_fragment_size = -1;
1575 	}
1576 
1577 	if (amdgpu_sched_hw_submission < 2) {
1578 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1579 			 amdgpu_sched_hw_submission);
1580 		amdgpu_sched_hw_submission = 2;
1581 	} else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1582 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1583 			 amdgpu_sched_hw_submission);
1584 		amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1585 	}
1586 
1587 	if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1588 		dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1589 		amdgpu_reset_method = -1;
1590 	}
1591 
1592 	amdgpu_device_check_smu_prv_buffer_size(adev);
1593 
1594 	amdgpu_device_check_vm_size(adev);
1595 
1596 	amdgpu_device_check_block_size(adev);
1597 
1598 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1599 
1600 	return 0;
1601 }
1602 
1603 /**
1604  * amdgpu_switcheroo_set_state - set switcheroo state
1605  *
1606  * @pdev: pci dev pointer
1607  * @state: vga_switcheroo state
1608  *
1609  * Callback for the switcheroo driver.  Suspends or resumes
1610  * the asics before or after it is powered up using ACPI methods.
1611  */
1612 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1613 					enum vga_switcheroo_state state)
1614 {
1615 	struct drm_device *dev = pci_get_drvdata(pdev);
1616 	int r;
1617 
1618 	if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1619 		return;
1620 
1621 	if (state == VGA_SWITCHEROO_ON) {
1622 		pr_info("switched on\n");
1623 		/* don't suspend or resume card normally */
1624 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1625 
1626 		pci_set_power_state(pdev, PCI_D0);
1627 		amdgpu_device_load_pci_state(pdev);
1628 		r = pci_enable_device(pdev);
1629 		if (r)
1630 			DRM_WARN("pci_enable_device failed (%d)\n", r);
1631 		amdgpu_device_resume(dev, true);
1632 
1633 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
1634 	} else {
1635 		pr_info("switched off\n");
1636 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1637 		amdgpu_device_suspend(dev, true);
1638 		amdgpu_device_cache_pci_state(pdev);
1639 		/* Shut down the device */
1640 		pci_disable_device(pdev);
1641 		pci_set_power_state(pdev, PCI_D3cold);
1642 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1643 	}
1644 }
1645 
1646 /**
1647  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1648  *
1649  * @pdev: pci dev pointer
1650  *
1651  * Callback for the switcheroo driver.  Check of the switcheroo
1652  * state can be changed.
1653  * Returns true if the state can be changed, false if not.
1654  */
1655 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1656 {
1657 	struct drm_device *dev = pci_get_drvdata(pdev);
1658 
1659 	/*
1660 	* FIXME: open_count is protected by drm_global_mutex but that would lead to
1661 	* locking inversion with the driver load path. And the access here is
1662 	* completely racy anyway. So don't bother with locking for now.
1663 	*/
1664 	return atomic_read(&dev->open_count) == 0;
1665 }
1666 
1667 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1668 	.set_gpu_state = amdgpu_switcheroo_set_state,
1669 	.reprobe = NULL,
1670 	.can_switch = amdgpu_switcheroo_can_switch,
1671 };
1672 
1673 /**
1674  * amdgpu_device_ip_set_clockgating_state - set the CG state
1675  *
1676  * @dev: amdgpu_device pointer
1677  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1678  * @state: clockgating state (gate or ungate)
1679  *
1680  * Sets the requested clockgating state for all instances of
1681  * the hardware IP specified.
1682  * Returns the error code from the last instance.
1683  */
1684 int amdgpu_device_ip_set_clockgating_state(void *dev,
1685 					   enum amd_ip_block_type block_type,
1686 					   enum amd_clockgating_state state)
1687 {
1688 	struct amdgpu_device *adev = dev;
1689 	int i, r = 0;
1690 
1691 	for (i = 0; i < adev->num_ip_blocks; i++) {
1692 		if (!adev->ip_blocks[i].status.valid)
1693 			continue;
1694 		if (adev->ip_blocks[i].version->type != block_type)
1695 			continue;
1696 		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1697 			continue;
1698 		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1699 			(void *)adev, state);
1700 		if (r)
1701 			DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1702 				  adev->ip_blocks[i].version->funcs->name, r);
1703 	}
1704 	return r;
1705 }
1706 
1707 /**
1708  * amdgpu_device_ip_set_powergating_state - set the PG state
1709  *
1710  * @dev: amdgpu_device pointer
1711  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1712  * @state: powergating state (gate or ungate)
1713  *
1714  * Sets the requested powergating state for all instances of
1715  * the hardware IP specified.
1716  * Returns the error code from the last instance.
1717  */
1718 int amdgpu_device_ip_set_powergating_state(void *dev,
1719 					   enum amd_ip_block_type block_type,
1720 					   enum amd_powergating_state state)
1721 {
1722 	struct amdgpu_device *adev = dev;
1723 	int i, r = 0;
1724 
1725 	for (i = 0; i < adev->num_ip_blocks; i++) {
1726 		if (!adev->ip_blocks[i].status.valid)
1727 			continue;
1728 		if (adev->ip_blocks[i].version->type != block_type)
1729 			continue;
1730 		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1731 			continue;
1732 		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1733 			(void *)adev, state);
1734 		if (r)
1735 			DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1736 				  adev->ip_blocks[i].version->funcs->name, r);
1737 	}
1738 	return r;
1739 }
1740 
1741 /**
1742  * amdgpu_device_ip_get_clockgating_state - get the CG state
1743  *
1744  * @adev: amdgpu_device pointer
1745  * @flags: clockgating feature flags
1746  *
1747  * Walks the list of IPs on the device and updates the clockgating
1748  * flags for each IP.
1749  * Updates @flags with the feature flags for each hardware IP where
1750  * clockgating is enabled.
1751  */
1752 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1753 					    u64 *flags)
1754 {
1755 	int i;
1756 
1757 	for (i = 0; i < adev->num_ip_blocks; i++) {
1758 		if (!adev->ip_blocks[i].status.valid)
1759 			continue;
1760 		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1761 			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1762 	}
1763 }
1764 
1765 /**
1766  * amdgpu_device_ip_wait_for_idle - wait for idle
1767  *
1768  * @adev: amdgpu_device pointer
1769  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1770  *
1771  * Waits for the request hardware IP to be idle.
1772  * Returns 0 for success or a negative error code on failure.
1773  */
1774 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1775 				   enum amd_ip_block_type block_type)
1776 {
1777 	int i, r;
1778 
1779 	for (i = 0; i < adev->num_ip_blocks; i++) {
1780 		if (!adev->ip_blocks[i].status.valid)
1781 			continue;
1782 		if (adev->ip_blocks[i].version->type == block_type) {
1783 			r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1784 			if (r)
1785 				return r;
1786 			break;
1787 		}
1788 	}
1789 	return 0;
1790 
1791 }
1792 
1793 /**
1794  * amdgpu_device_ip_is_idle - is the hardware IP idle
1795  *
1796  * @adev: amdgpu_device pointer
1797  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1798  *
1799  * Check if the hardware IP is idle or not.
1800  * Returns true if it the IP is idle, false if not.
1801  */
1802 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1803 			      enum amd_ip_block_type block_type)
1804 {
1805 	int i;
1806 
1807 	for (i = 0; i < adev->num_ip_blocks; i++) {
1808 		if (!adev->ip_blocks[i].status.valid)
1809 			continue;
1810 		if (adev->ip_blocks[i].version->type == block_type)
1811 			return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1812 	}
1813 	return true;
1814 
1815 }
1816 
1817 /**
1818  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1819  *
1820  * @adev: amdgpu_device pointer
1821  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1822  *
1823  * Returns a pointer to the hardware IP block structure
1824  * if it exists for the asic, otherwise NULL.
1825  */
1826 struct amdgpu_ip_block *
1827 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1828 			      enum amd_ip_block_type type)
1829 {
1830 	int i;
1831 
1832 	for (i = 0; i < adev->num_ip_blocks; i++)
1833 		if (adev->ip_blocks[i].version->type == type)
1834 			return &adev->ip_blocks[i];
1835 
1836 	return NULL;
1837 }
1838 
1839 /**
1840  * amdgpu_device_ip_block_version_cmp
1841  *
1842  * @adev: amdgpu_device pointer
1843  * @type: enum amd_ip_block_type
1844  * @major: major version
1845  * @minor: minor version
1846  *
1847  * return 0 if equal or greater
1848  * return 1 if smaller or the ip_block doesn't exist
1849  */
1850 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1851 				       enum amd_ip_block_type type,
1852 				       u32 major, u32 minor)
1853 {
1854 	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1855 
1856 	if (ip_block && ((ip_block->version->major > major) ||
1857 			((ip_block->version->major == major) &&
1858 			(ip_block->version->minor >= minor))))
1859 		return 0;
1860 
1861 	return 1;
1862 }
1863 
1864 /**
1865  * amdgpu_device_ip_block_add
1866  *
1867  * @adev: amdgpu_device pointer
1868  * @ip_block_version: pointer to the IP to add
1869  *
1870  * Adds the IP block driver information to the collection of IPs
1871  * on the asic.
1872  */
1873 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1874 			       const struct amdgpu_ip_block_version *ip_block_version)
1875 {
1876 	if (!ip_block_version)
1877 		return -EINVAL;
1878 
1879 	switch (ip_block_version->type) {
1880 	case AMD_IP_BLOCK_TYPE_VCN:
1881 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1882 			return 0;
1883 		break;
1884 	case AMD_IP_BLOCK_TYPE_JPEG:
1885 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1886 			return 0;
1887 		break;
1888 	default:
1889 		break;
1890 	}
1891 
1892 	DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1893 		  ip_block_version->funcs->name);
1894 
1895 	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1896 
1897 	return 0;
1898 }
1899 
1900 /**
1901  * amdgpu_device_enable_virtual_display - enable virtual display feature
1902  *
1903  * @adev: amdgpu_device pointer
1904  *
1905  * Enabled the virtual display feature if the user has enabled it via
1906  * the module parameter virtual_display.  This feature provides a virtual
1907  * display hardware on headless boards or in virtualized environments.
1908  * This function parses and validates the configuration string specified by
1909  * the user and configues the virtual display configuration (number of
1910  * virtual connectors, crtcs, etc.) specified.
1911  */
1912 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1913 {
1914 	adev->enable_virtual_display = false;
1915 
1916 	if (amdgpu_virtual_display) {
1917 		const char *pci_address_name = pci_name(adev->pdev);
1918 		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1919 
1920 		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1921 		pciaddstr_tmp = pciaddstr;
1922 		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1923 			pciaddname = strsep(&pciaddname_tmp, ",");
1924 			if (!strcmp("all", pciaddname)
1925 			    || !strcmp(pci_address_name, pciaddname)) {
1926 				long num_crtc;
1927 				int res = -1;
1928 
1929 				adev->enable_virtual_display = true;
1930 
1931 				if (pciaddname_tmp)
1932 					res = kstrtol(pciaddname_tmp, 10,
1933 						      &num_crtc);
1934 
1935 				if (!res) {
1936 					if (num_crtc < 1)
1937 						num_crtc = 1;
1938 					if (num_crtc > 6)
1939 						num_crtc = 6;
1940 					adev->mode_info.num_crtc = num_crtc;
1941 				} else {
1942 					adev->mode_info.num_crtc = 1;
1943 				}
1944 				break;
1945 			}
1946 		}
1947 
1948 		DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1949 			 amdgpu_virtual_display, pci_address_name,
1950 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
1951 
1952 		kfree(pciaddstr);
1953 	}
1954 }
1955 
1956 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
1957 {
1958 	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
1959 		adev->mode_info.num_crtc = 1;
1960 		adev->enable_virtual_display = true;
1961 		DRM_INFO("virtual_display:%d, num_crtc:%d\n",
1962 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
1963 	}
1964 }
1965 
1966 /**
1967  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1968  *
1969  * @adev: amdgpu_device pointer
1970  *
1971  * Parses the asic configuration parameters specified in the gpu info
1972  * firmware and makes them availale to the driver for use in configuring
1973  * the asic.
1974  * Returns 0 on success, -EINVAL on failure.
1975  */
1976 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1977 {
1978 	const char *chip_name;
1979 	char fw_name[40];
1980 	int err;
1981 	const struct gpu_info_firmware_header_v1_0 *hdr;
1982 
1983 	adev->firmware.gpu_info_fw = NULL;
1984 
1985 	if (adev->mman.discovery_bin) {
1986 		/*
1987 		 * FIXME: The bounding box is still needed by Navi12, so
1988 		 * temporarily read it from gpu_info firmware. Should be dropped
1989 		 * when DAL no longer needs it.
1990 		 */
1991 		if (adev->asic_type != CHIP_NAVI12)
1992 			return 0;
1993 	}
1994 
1995 	switch (adev->asic_type) {
1996 	default:
1997 		return 0;
1998 	case CHIP_VEGA10:
1999 		chip_name = "vega10";
2000 		break;
2001 	case CHIP_VEGA12:
2002 		chip_name = "vega12";
2003 		break;
2004 	case CHIP_RAVEN:
2005 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2006 			chip_name = "raven2";
2007 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2008 			chip_name = "picasso";
2009 		else
2010 			chip_name = "raven";
2011 		break;
2012 	case CHIP_ARCTURUS:
2013 		chip_name = "arcturus";
2014 		break;
2015 	case CHIP_NAVI12:
2016 		chip_name = "navi12";
2017 		break;
2018 	}
2019 
2020 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2021 	err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2022 	if (err) {
2023 		dev_err(adev->dev,
2024 			"Failed to get gpu_info firmware \"%s\"\n",
2025 			fw_name);
2026 		goto out;
2027 	}
2028 
2029 	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2030 	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2031 
2032 	switch (hdr->version_major) {
2033 	case 1:
2034 	{
2035 		const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2036 			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2037 								le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2038 
2039 		/*
2040 		 * Should be droped when DAL no longer needs it.
2041 		 */
2042 		if (adev->asic_type == CHIP_NAVI12)
2043 			goto parse_soc_bounding_box;
2044 
2045 		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2046 		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2047 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2048 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2049 		adev->gfx.config.max_texture_channel_caches =
2050 			le32_to_cpu(gpu_info_fw->gc_num_tccs);
2051 		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2052 		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2053 		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2054 		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2055 		adev->gfx.config.double_offchip_lds_buf =
2056 			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2057 		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2058 		adev->gfx.cu_info.max_waves_per_simd =
2059 			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2060 		adev->gfx.cu_info.max_scratch_slots_per_cu =
2061 			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2062 		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2063 		if (hdr->version_minor >= 1) {
2064 			const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2065 				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2066 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2067 			adev->gfx.config.num_sc_per_sh =
2068 				le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2069 			adev->gfx.config.num_packer_per_sc =
2070 				le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2071 		}
2072 
2073 parse_soc_bounding_box:
2074 		/*
2075 		 * soc bounding box info is not integrated in disocovery table,
2076 		 * we always need to parse it from gpu info firmware if needed.
2077 		 */
2078 		if (hdr->version_minor == 2) {
2079 			const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2080 				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2081 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2082 			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2083 		}
2084 		break;
2085 	}
2086 	default:
2087 		dev_err(adev->dev,
2088 			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2089 		err = -EINVAL;
2090 		goto out;
2091 	}
2092 out:
2093 	return err;
2094 }
2095 
2096 /**
2097  * amdgpu_device_ip_early_init - run early init for hardware IPs
2098  *
2099  * @adev: amdgpu_device pointer
2100  *
2101  * Early initialization pass for hardware IPs.  The hardware IPs that make
2102  * up each asic are discovered each IP's early_init callback is run.  This
2103  * is the first stage in initializing the asic.
2104  * Returns 0 on success, negative error code on failure.
2105  */
2106 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2107 {
2108 	struct drm_device *dev = adev_to_drm(adev);
2109 	struct pci_dev *parent;
2110 	int i, r;
2111 	bool total;
2112 
2113 	amdgpu_device_enable_virtual_display(adev);
2114 
2115 	if (amdgpu_sriov_vf(adev)) {
2116 		r = amdgpu_virt_request_full_gpu(adev, true);
2117 		if (r)
2118 			return r;
2119 	}
2120 
2121 	switch (adev->asic_type) {
2122 #ifdef CONFIG_DRM_AMDGPU_SI
2123 	case CHIP_VERDE:
2124 	case CHIP_TAHITI:
2125 	case CHIP_PITCAIRN:
2126 	case CHIP_OLAND:
2127 	case CHIP_HAINAN:
2128 		adev->family = AMDGPU_FAMILY_SI;
2129 		r = si_set_ip_blocks(adev);
2130 		if (r)
2131 			return r;
2132 		break;
2133 #endif
2134 #ifdef CONFIG_DRM_AMDGPU_CIK
2135 	case CHIP_BONAIRE:
2136 	case CHIP_HAWAII:
2137 	case CHIP_KAVERI:
2138 	case CHIP_KABINI:
2139 	case CHIP_MULLINS:
2140 		if (adev->flags & AMD_IS_APU)
2141 			adev->family = AMDGPU_FAMILY_KV;
2142 		else
2143 			adev->family = AMDGPU_FAMILY_CI;
2144 
2145 		r = cik_set_ip_blocks(adev);
2146 		if (r)
2147 			return r;
2148 		break;
2149 #endif
2150 	case CHIP_TOPAZ:
2151 	case CHIP_TONGA:
2152 	case CHIP_FIJI:
2153 	case CHIP_POLARIS10:
2154 	case CHIP_POLARIS11:
2155 	case CHIP_POLARIS12:
2156 	case CHIP_VEGAM:
2157 	case CHIP_CARRIZO:
2158 	case CHIP_STONEY:
2159 		if (adev->flags & AMD_IS_APU)
2160 			adev->family = AMDGPU_FAMILY_CZ;
2161 		else
2162 			adev->family = AMDGPU_FAMILY_VI;
2163 
2164 		r = vi_set_ip_blocks(adev);
2165 		if (r)
2166 			return r;
2167 		break;
2168 	default:
2169 		r = amdgpu_discovery_set_ip_blocks(adev);
2170 		if (r)
2171 			return r;
2172 		break;
2173 	}
2174 
2175 	if (amdgpu_has_atpx() &&
2176 	    (amdgpu_is_atpx_hybrid() ||
2177 	     amdgpu_has_atpx_dgpu_power_cntl()) &&
2178 	    ((adev->flags & AMD_IS_APU) == 0) &&
2179 	    !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
2180 		adev->flags |= AMD_IS_PX;
2181 
2182 	if (!(adev->flags & AMD_IS_APU)) {
2183 		parent = pci_upstream_bridge(adev->pdev);
2184 		adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2185 	}
2186 
2187 
2188 	adev->pm.pp_feature = amdgpu_pp_feature_mask;
2189 	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2190 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2191 	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2192 		adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2193 
2194 	total = true;
2195 	for (i = 0; i < adev->num_ip_blocks; i++) {
2196 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2197 			DRM_ERROR("disabled ip block: %d <%s>\n",
2198 				  i, adev->ip_blocks[i].version->funcs->name);
2199 			adev->ip_blocks[i].status.valid = false;
2200 		} else {
2201 			if (adev->ip_blocks[i].version->funcs->early_init) {
2202 				r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2203 				if (r == -ENOENT) {
2204 					adev->ip_blocks[i].status.valid = false;
2205 				} else if (r) {
2206 					DRM_ERROR("early_init of IP block <%s> failed %d\n",
2207 						  adev->ip_blocks[i].version->funcs->name, r);
2208 					total = false;
2209 				} else {
2210 					adev->ip_blocks[i].status.valid = true;
2211 				}
2212 			} else {
2213 				adev->ip_blocks[i].status.valid = true;
2214 			}
2215 		}
2216 		/* get the vbios after the asic_funcs are set up */
2217 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2218 			r = amdgpu_device_parse_gpu_info_fw(adev);
2219 			if (r)
2220 				return r;
2221 
2222 			/* Read BIOS */
2223 			if (!amdgpu_get_bios(adev))
2224 				return -EINVAL;
2225 
2226 			r = amdgpu_atombios_init(adev);
2227 			if (r) {
2228 				dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2229 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2230 				return r;
2231 			}
2232 
2233 			/*get pf2vf msg info at it's earliest time*/
2234 			if (amdgpu_sriov_vf(adev))
2235 				amdgpu_virt_init_data_exchange(adev);
2236 
2237 		}
2238 	}
2239 	if (!total)
2240 		return -ENODEV;
2241 
2242 	amdgpu_amdkfd_device_probe(adev);
2243 	adev->cg_flags &= amdgpu_cg_mask;
2244 	adev->pg_flags &= amdgpu_pg_mask;
2245 
2246 	return 0;
2247 }
2248 
2249 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2250 {
2251 	int i, r;
2252 
2253 	for (i = 0; i < adev->num_ip_blocks; i++) {
2254 		if (!adev->ip_blocks[i].status.sw)
2255 			continue;
2256 		if (adev->ip_blocks[i].status.hw)
2257 			continue;
2258 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2259 		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2260 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2261 			r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2262 			if (r) {
2263 				DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2264 					  adev->ip_blocks[i].version->funcs->name, r);
2265 				return r;
2266 			}
2267 			adev->ip_blocks[i].status.hw = true;
2268 		}
2269 	}
2270 
2271 	return 0;
2272 }
2273 
2274 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2275 {
2276 	int i, r;
2277 
2278 	for (i = 0; i < adev->num_ip_blocks; i++) {
2279 		if (!adev->ip_blocks[i].status.sw)
2280 			continue;
2281 		if (adev->ip_blocks[i].status.hw)
2282 			continue;
2283 		r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2284 		if (r) {
2285 			DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2286 				  adev->ip_blocks[i].version->funcs->name, r);
2287 			return r;
2288 		}
2289 		adev->ip_blocks[i].status.hw = true;
2290 	}
2291 
2292 	return 0;
2293 }
2294 
2295 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2296 {
2297 	int r = 0;
2298 	int i;
2299 	uint32_t smu_version;
2300 
2301 	if (adev->asic_type >= CHIP_VEGA10) {
2302 		for (i = 0; i < adev->num_ip_blocks; i++) {
2303 			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2304 				continue;
2305 
2306 			if (!adev->ip_blocks[i].status.sw)
2307 				continue;
2308 
2309 			/* no need to do the fw loading again if already done*/
2310 			if (adev->ip_blocks[i].status.hw == true)
2311 				break;
2312 
2313 			if (amdgpu_in_reset(adev) || adev->in_suspend) {
2314 				r = adev->ip_blocks[i].version->funcs->resume(adev);
2315 				if (r) {
2316 					DRM_ERROR("resume of IP block <%s> failed %d\n",
2317 							  adev->ip_blocks[i].version->funcs->name, r);
2318 					return r;
2319 				}
2320 			} else {
2321 				r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2322 				if (r) {
2323 					DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2324 							  adev->ip_blocks[i].version->funcs->name, r);
2325 					return r;
2326 				}
2327 			}
2328 
2329 			adev->ip_blocks[i].status.hw = true;
2330 			break;
2331 		}
2332 	}
2333 
2334 	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2335 		r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2336 
2337 	return r;
2338 }
2339 
2340 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2341 {
2342 	long timeout;
2343 	int r, i;
2344 
2345 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2346 		struct amdgpu_ring *ring = adev->rings[i];
2347 
2348 		/* No need to setup the GPU scheduler for rings that don't need it */
2349 		if (!ring || ring->no_scheduler)
2350 			continue;
2351 
2352 		switch (ring->funcs->type) {
2353 		case AMDGPU_RING_TYPE_GFX:
2354 			timeout = adev->gfx_timeout;
2355 			break;
2356 		case AMDGPU_RING_TYPE_COMPUTE:
2357 			timeout = adev->compute_timeout;
2358 			break;
2359 		case AMDGPU_RING_TYPE_SDMA:
2360 			timeout = adev->sdma_timeout;
2361 			break;
2362 		default:
2363 			timeout = adev->video_timeout;
2364 			break;
2365 		}
2366 
2367 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2368 				   ring->num_hw_submission, 0,
2369 				   timeout, adev->reset_domain->wq,
2370 				   ring->sched_score, ring->name,
2371 				   adev->dev);
2372 		if (r) {
2373 			DRM_ERROR("Failed to create scheduler on ring %s.\n",
2374 				  ring->name);
2375 			return r;
2376 		}
2377 	}
2378 
2379 	return 0;
2380 }
2381 
2382 
2383 /**
2384  * amdgpu_device_ip_init - run init for hardware IPs
2385  *
2386  * @adev: amdgpu_device pointer
2387  *
2388  * Main initialization pass for hardware IPs.  The list of all the hardware
2389  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2390  * are run.  sw_init initializes the software state associated with each IP
2391  * and hw_init initializes the hardware associated with each IP.
2392  * Returns 0 on success, negative error code on failure.
2393  */
2394 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2395 {
2396 	int i, r;
2397 
2398 	r = amdgpu_ras_init(adev);
2399 	if (r)
2400 		return r;
2401 
2402 	for (i = 0; i < adev->num_ip_blocks; i++) {
2403 		if (!adev->ip_blocks[i].status.valid)
2404 			continue;
2405 		r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2406 		if (r) {
2407 			DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2408 				  adev->ip_blocks[i].version->funcs->name, r);
2409 			goto init_failed;
2410 		}
2411 		adev->ip_blocks[i].status.sw = true;
2412 
2413 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2414 			/* need to do common hw init early so everything is set up for gmc */
2415 			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2416 			if (r) {
2417 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2418 				goto init_failed;
2419 			}
2420 			adev->ip_blocks[i].status.hw = true;
2421 		} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2422 			/* need to do gmc hw init early so we can allocate gpu mem */
2423 			/* Try to reserve bad pages early */
2424 			if (amdgpu_sriov_vf(adev))
2425 				amdgpu_virt_exchange_data(adev);
2426 
2427 			r = amdgpu_device_mem_scratch_init(adev);
2428 			if (r) {
2429 				DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2430 				goto init_failed;
2431 			}
2432 			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2433 			if (r) {
2434 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2435 				goto init_failed;
2436 			}
2437 			r = amdgpu_device_wb_init(adev);
2438 			if (r) {
2439 				DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2440 				goto init_failed;
2441 			}
2442 			adev->ip_blocks[i].status.hw = true;
2443 
2444 			/* right after GMC hw init, we create CSA */
2445 			if (amdgpu_mcbp) {
2446 				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2447 							       AMDGPU_GEM_DOMAIN_VRAM |
2448 							       AMDGPU_GEM_DOMAIN_GTT,
2449 							       AMDGPU_CSA_SIZE);
2450 				if (r) {
2451 					DRM_ERROR("allocate CSA failed %d\n", r);
2452 					goto init_failed;
2453 				}
2454 			}
2455 		}
2456 	}
2457 
2458 	if (amdgpu_sriov_vf(adev))
2459 		amdgpu_virt_init_data_exchange(adev);
2460 
2461 	r = amdgpu_ib_pool_init(adev);
2462 	if (r) {
2463 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2464 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2465 		goto init_failed;
2466 	}
2467 
2468 	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2469 	if (r)
2470 		goto init_failed;
2471 
2472 	r = amdgpu_device_ip_hw_init_phase1(adev);
2473 	if (r)
2474 		goto init_failed;
2475 
2476 	r = amdgpu_device_fw_loading(adev);
2477 	if (r)
2478 		goto init_failed;
2479 
2480 	r = amdgpu_device_ip_hw_init_phase2(adev);
2481 	if (r)
2482 		goto init_failed;
2483 
2484 	/*
2485 	 * retired pages will be loaded from eeprom and reserved here,
2486 	 * it should be called after amdgpu_device_ip_hw_init_phase2  since
2487 	 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2488 	 * for I2C communication which only true at this point.
2489 	 *
2490 	 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2491 	 * failure from bad gpu situation and stop amdgpu init process
2492 	 * accordingly. For other failed cases, it will still release all
2493 	 * the resource and print error message, rather than returning one
2494 	 * negative value to upper level.
2495 	 *
2496 	 * Note: theoretically, this should be called before all vram allocations
2497 	 * to protect retired page from abusing
2498 	 */
2499 	r = amdgpu_ras_recovery_init(adev);
2500 	if (r)
2501 		goto init_failed;
2502 
2503 	/**
2504 	 * In case of XGMI grab extra reference for reset domain for this device
2505 	 */
2506 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
2507 		if (amdgpu_xgmi_add_device(adev) == 0) {
2508 			if (!amdgpu_sriov_vf(adev)) {
2509 				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2510 
2511 				if (WARN_ON(!hive)) {
2512 					r = -ENOENT;
2513 					goto init_failed;
2514 				}
2515 
2516 				if (!hive->reset_domain ||
2517 				    !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2518 					r = -ENOENT;
2519 					amdgpu_put_xgmi_hive(hive);
2520 					goto init_failed;
2521 				}
2522 
2523 				/* Drop the early temporary reset domain we created for device */
2524 				amdgpu_reset_put_reset_domain(adev->reset_domain);
2525 				adev->reset_domain = hive->reset_domain;
2526 				amdgpu_put_xgmi_hive(hive);
2527 			}
2528 		}
2529 	}
2530 
2531 	r = amdgpu_device_init_schedulers(adev);
2532 	if (r)
2533 		goto init_failed;
2534 
2535 	/* Don't init kfd if whole hive need to be reset during init */
2536 	if (!adev->gmc.xgmi.pending_reset)
2537 		amdgpu_amdkfd_device_init(adev);
2538 
2539 	amdgpu_fru_get_product_info(adev);
2540 
2541 init_failed:
2542 
2543 	return r;
2544 }
2545 
2546 /**
2547  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2548  *
2549  * @adev: amdgpu_device pointer
2550  *
2551  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2552  * this function before a GPU reset.  If the value is retained after a
2553  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2554  */
2555 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2556 {
2557 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2558 }
2559 
2560 /**
2561  * amdgpu_device_check_vram_lost - check if vram is valid
2562  *
2563  * @adev: amdgpu_device pointer
2564  *
2565  * Checks the reset magic value written to the gart pointer in VRAM.
2566  * The driver calls this after a GPU reset to see if the contents of
2567  * VRAM is lost or now.
2568  * returns true if vram is lost, false if not.
2569  */
2570 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2571 {
2572 	if (memcmp(adev->gart.ptr, adev->reset_magic,
2573 			AMDGPU_RESET_MAGIC_NUM))
2574 		return true;
2575 
2576 	if (!amdgpu_in_reset(adev))
2577 		return false;
2578 
2579 	/*
2580 	 * For all ASICs with baco/mode1 reset, the VRAM is
2581 	 * always assumed to be lost.
2582 	 */
2583 	switch (amdgpu_asic_reset_method(adev)) {
2584 	case AMD_RESET_METHOD_BACO:
2585 	case AMD_RESET_METHOD_MODE1:
2586 		return true;
2587 	default:
2588 		return false;
2589 	}
2590 }
2591 
2592 /**
2593  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2594  *
2595  * @adev: amdgpu_device pointer
2596  * @state: clockgating state (gate or ungate)
2597  *
2598  * The list of all the hardware IPs that make up the asic is walked and the
2599  * set_clockgating_state callbacks are run.
2600  * Late initialization pass enabling clockgating for hardware IPs.
2601  * Fini or suspend, pass disabling clockgating for hardware IPs.
2602  * Returns 0 on success, negative error code on failure.
2603  */
2604 
2605 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2606 			       enum amd_clockgating_state state)
2607 {
2608 	int i, j, r;
2609 
2610 	if (amdgpu_emu_mode == 1)
2611 		return 0;
2612 
2613 	for (j = 0; j < adev->num_ip_blocks; j++) {
2614 		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2615 		if (!adev->ip_blocks[i].status.late_initialized)
2616 			continue;
2617 		/* skip CG for GFX, SDMA on S0ix */
2618 		if (adev->in_s0ix &&
2619 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2620 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2621 			continue;
2622 		/* skip CG for VCE/UVD, it's handled specially */
2623 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2624 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2625 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2626 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2627 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2628 			/* enable clockgating to save power */
2629 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2630 										     state);
2631 			if (r) {
2632 				DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2633 					  adev->ip_blocks[i].version->funcs->name, r);
2634 				return r;
2635 			}
2636 		}
2637 	}
2638 
2639 	return 0;
2640 }
2641 
2642 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2643 			       enum amd_powergating_state state)
2644 {
2645 	int i, j, r;
2646 
2647 	if (amdgpu_emu_mode == 1)
2648 		return 0;
2649 
2650 	for (j = 0; j < adev->num_ip_blocks; j++) {
2651 		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2652 		if (!adev->ip_blocks[i].status.late_initialized)
2653 			continue;
2654 		/* skip PG for GFX, SDMA on S0ix */
2655 		if (adev->in_s0ix &&
2656 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2657 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2658 			continue;
2659 		/* skip CG for VCE/UVD, it's handled specially */
2660 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2661 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2662 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2663 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2664 		    adev->ip_blocks[i].version->funcs->set_powergating_state) {
2665 			/* enable powergating to save power */
2666 			r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2667 											state);
2668 			if (r) {
2669 				DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2670 					  adev->ip_blocks[i].version->funcs->name, r);
2671 				return r;
2672 			}
2673 		}
2674 	}
2675 	return 0;
2676 }
2677 
2678 static int amdgpu_device_enable_mgpu_fan_boost(void)
2679 {
2680 	struct amdgpu_gpu_instance *gpu_ins;
2681 	struct amdgpu_device *adev;
2682 	int i, ret = 0;
2683 
2684 	mutex_lock(&mgpu_info.mutex);
2685 
2686 	/*
2687 	 * MGPU fan boost feature should be enabled
2688 	 * only when there are two or more dGPUs in
2689 	 * the system
2690 	 */
2691 	if (mgpu_info.num_dgpu < 2)
2692 		goto out;
2693 
2694 	for (i = 0; i < mgpu_info.num_dgpu; i++) {
2695 		gpu_ins = &(mgpu_info.gpu_ins[i]);
2696 		adev = gpu_ins->adev;
2697 		if (!(adev->flags & AMD_IS_APU) &&
2698 		    !gpu_ins->mgpu_fan_enabled) {
2699 			ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2700 			if (ret)
2701 				break;
2702 
2703 			gpu_ins->mgpu_fan_enabled = 1;
2704 		}
2705 	}
2706 
2707 out:
2708 	mutex_unlock(&mgpu_info.mutex);
2709 
2710 	return ret;
2711 }
2712 
2713 /**
2714  * amdgpu_device_ip_late_init - run late init for hardware IPs
2715  *
2716  * @adev: amdgpu_device pointer
2717  *
2718  * Late initialization pass for hardware IPs.  The list of all the hardware
2719  * IPs that make up the asic is walked and the late_init callbacks are run.
2720  * late_init covers any special initialization that an IP requires
2721  * after all of the have been initialized or something that needs to happen
2722  * late in the init process.
2723  * Returns 0 on success, negative error code on failure.
2724  */
2725 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2726 {
2727 	struct amdgpu_gpu_instance *gpu_instance;
2728 	int i = 0, r;
2729 
2730 	for (i = 0; i < adev->num_ip_blocks; i++) {
2731 		if (!adev->ip_blocks[i].status.hw)
2732 			continue;
2733 		if (adev->ip_blocks[i].version->funcs->late_init) {
2734 			r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2735 			if (r) {
2736 				DRM_ERROR("late_init of IP block <%s> failed %d\n",
2737 					  adev->ip_blocks[i].version->funcs->name, r);
2738 				return r;
2739 			}
2740 		}
2741 		adev->ip_blocks[i].status.late_initialized = true;
2742 	}
2743 
2744 	r = amdgpu_ras_late_init(adev);
2745 	if (r) {
2746 		DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2747 		return r;
2748 	}
2749 
2750 	amdgpu_ras_set_error_query_ready(adev, true);
2751 
2752 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2753 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2754 
2755 	amdgpu_device_fill_reset_magic(adev);
2756 
2757 	r = amdgpu_device_enable_mgpu_fan_boost();
2758 	if (r)
2759 		DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2760 
2761 	/* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2762 	if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)||
2763 			       adev->asic_type == CHIP_ALDEBARAN ))
2764 		amdgpu_dpm_handle_passthrough_sbr(adev, true);
2765 
2766 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
2767 		mutex_lock(&mgpu_info.mutex);
2768 
2769 		/*
2770 		 * Reset device p-state to low as this was booted with high.
2771 		 *
2772 		 * This should be performed only after all devices from the same
2773 		 * hive get initialized.
2774 		 *
2775 		 * However, it's unknown how many device in the hive in advance.
2776 		 * As this is counted one by one during devices initializations.
2777 		 *
2778 		 * So, we wait for all XGMI interlinked devices initialized.
2779 		 * This may bring some delays as those devices may come from
2780 		 * different hives. But that should be OK.
2781 		 */
2782 		if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2783 			for (i = 0; i < mgpu_info.num_gpu; i++) {
2784 				gpu_instance = &(mgpu_info.gpu_ins[i]);
2785 				if (gpu_instance->adev->flags & AMD_IS_APU)
2786 					continue;
2787 
2788 				r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2789 						AMDGPU_XGMI_PSTATE_MIN);
2790 				if (r) {
2791 					DRM_ERROR("pstate setting failed (%d).\n", r);
2792 					break;
2793 				}
2794 			}
2795 		}
2796 
2797 		mutex_unlock(&mgpu_info.mutex);
2798 	}
2799 
2800 	return 0;
2801 }
2802 
2803 /**
2804  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2805  *
2806  * @adev: amdgpu_device pointer
2807  *
2808  * For ASICs need to disable SMC first
2809  */
2810 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2811 {
2812 	int i, r;
2813 
2814 	if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2815 		return;
2816 
2817 	for (i = 0; i < adev->num_ip_blocks; i++) {
2818 		if (!adev->ip_blocks[i].status.hw)
2819 			continue;
2820 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2821 			r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2822 			/* XXX handle errors */
2823 			if (r) {
2824 				DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2825 					  adev->ip_blocks[i].version->funcs->name, r);
2826 			}
2827 			adev->ip_blocks[i].status.hw = false;
2828 			break;
2829 		}
2830 	}
2831 }
2832 
2833 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2834 {
2835 	int i, r;
2836 
2837 	for (i = 0; i < adev->num_ip_blocks; i++) {
2838 		if (!adev->ip_blocks[i].version->funcs->early_fini)
2839 			continue;
2840 
2841 		r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2842 		if (r) {
2843 			DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2844 				  adev->ip_blocks[i].version->funcs->name, r);
2845 		}
2846 	}
2847 
2848 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2849 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2850 
2851 	amdgpu_amdkfd_suspend(adev, false);
2852 
2853 	/* Workaroud for ASICs need to disable SMC first */
2854 	amdgpu_device_smu_fini_early(adev);
2855 
2856 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2857 		if (!adev->ip_blocks[i].status.hw)
2858 			continue;
2859 
2860 		r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2861 		/* XXX handle errors */
2862 		if (r) {
2863 			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2864 				  adev->ip_blocks[i].version->funcs->name, r);
2865 		}
2866 
2867 		adev->ip_blocks[i].status.hw = false;
2868 	}
2869 
2870 	if (amdgpu_sriov_vf(adev)) {
2871 		if (amdgpu_virt_release_full_gpu(adev, false))
2872 			DRM_ERROR("failed to release exclusive mode on fini\n");
2873 	}
2874 
2875 	return 0;
2876 }
2877 
2878 /**
2879  * amdgpu_device_ip_fini - run fini for hardware IPs
2880  *
2881  * @adev: amdgpu_device pointer
2882  *
2883  * Main teardown pass for hardware IPs.  The list of all the hardware
2884  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2885  * are run.  hw_fini tears down the hardware associated with each IP
2886  * and sw_fini tears down any software state associated with each IP.
2887  * Returns 0 on success, negative error code on failure.
2888  */
2889 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2890 {
2891 	int i, r;
2892 
2893 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2894 		amdgpu_virt_release_ras_err_handler_data(adev);
2895 
2896 	if (adev->gmc.xgmi.num_physical_nodes > 1)
2897 		amdgpu_xgmi_remove_device(adev);
2898 
2899 	amdgpu_amdkfd_device_fini_sw(adev);
2900 
2901 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2902 		if (!adev->ip_blocks[i].status.sw)
2903 			continue;
2904 
2905 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2906 			amdgpu_ucode_free_bo(adev);
2907 			amdgpu_free_static_csa(&adev->virt.csa_obj);
2908 			amdgpu_device_wb_fini(adev);
2909 			amdgpu_device_mem_scratch_fini(adev);
2910 			amdgpu_ib_pool_fini(adev);
2911 		}
2912 
2913 		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2914 		/* XXX handle errors */
2915 		if (r) {
2916 			DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2917 				  adev->ip_blocks[i].version->funcs->name, r);
2918 		}
2919 		adev->ip_blocks[i].status.sw = false;
2920 		adev->ip_blocks[i].status.valid = false;
2921 	}
2922 
2923 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2924 		if (!adev->ip_blocks[i].status.late_initialized)
2925 			continue;
2926 		if (adev->ip_blocks[i].version->funcs->late_fini)
2927 			adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2928 		adev->ip_blocks[i].status.late_initialized = false;
2929 	}
2930 
2931 	amdgpu_ras_fini(adev);
2932 
2933 	return 0;
2934 }
2935 
2936 /**
2937  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2938  *
2939  * @work: work_struct.
2940  */
2941 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2942 {
2943 	struct amdgpu_device *adev =
2944 		container_of(work, struct amdgpu_device, delayed_init_work.work);
2945 	int r;
2946 
2947 	r = amdgpu_ib_ring_tests(adev);
2948 	if (r)
2949 		DRM_ERROR("ib ring test failed (%d).\n", r);
2950 }
2951 
2952 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2953 {
2954 	struct amdgpu_device *adev =
2955 		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2956 
2957 	WARN_ON_ONCE(adev->gfx.gfx_off_state);
2958 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2959 
2960 	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2961 		adev->gfx.gfx_off_state = true;
2962 }
2963 
2964 /**
2965  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2966  *
2967  * @adev: amdgpu_device pointer
2968  *
2969  * Main suspend function for hardware IPs.  The list of all the hardware
2970  * IPs that make up the asic is walked, clockgating is disabled and the
2971  * suspend callbacks are run.  suspend puts the hardware and software state
2972  * in each IP into a state suitable for suspend.
2973  * Returns 0 on success, negative error code on failure.
2974  */
2975 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2976 {
2977 	int i, r;
2978 
2979 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2980 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2981 
2982 	/*
2983 	 * Per PMFW team's suggestion, driver needs to handle gfxoff
2984 	 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
2985 	 * scenario. Add the missing df cstate disablement here.
2986 	 */
2987 	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
2988 		dev_warn(adev->dev, "Failed to disallow df cstate");
2989 
2990 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2991 		if (!adev->ip_blocks[i].status.valid)
2992 			continue;
2993 
2994 		/* displays are handled separately */
2995 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2996 			continue;
2997 
2998 		/* XXX handle errors */
2999 		r = adev->ip_blocks[i].version->funcs->suspend(adev);
3000 		/* XXX handle errors */
3001 		if (r) {
3002 			DRM_ERROR("suspend of IP block <%s> failed %d\n",
3003 				  adev->ip_blocks[i].version->funcs->name, r);
3004 			return r;
3005 		}
3006 
3007 		adev->ip_blocks[i].status.hw = false;
3008 	}
3009 
3010 	return 0;
3011 }
3012 
3013 /**
3014  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3015  *
3016  * @adev: amdgpu_device pointer
3017  *
3018  * Main suspend function for hardware IPs.  The list of all the hardware
3019  * IPs that make up the asic is walked, clockgating is disabled and the
3020  * suspend callbacks are run.  suspend puts the hardware and software state
3021  * in each IP into a state suitable for suspend.
3022  * Returns 0 on success, negative error code on failure.
3023  */
3024 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3025 {
3026 	int i, r;
3027 
3028 	if (adev->in_s0ix)
3029 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3030 
3031 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3032 		if (!adev->ip_blocks[i].status.valid)
3033 			continue;
3034 		/* displays are handled in phase1 */
3035 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3036 			continue;
3037 		/* PSP lost connection when err_event_athub occurs */
3038 		if (amdgpu_ras_intr_triggered() &&
3039 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3040 			adev->ip_blocks[i].status.hw = false;
3041 			continue;
3042 		}
3043 
3044 		/* skip unnecessary suspend if we do not initialize them yet */
3045 		if (adev->gmc.xgmi.pending_reset &&
3046 		    !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3047 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3048 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3049 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3050 			adev->ip_blocks[i].status.hw = false;
3051 			continue;
3052 		}
3053 
3054 		/* skip suspend of gfx/mes and psp for S0ix
3055 		 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3056 		 * like at runtime. PSP is also part of the always on hardware
3057 		 * so no need to suspend it.
3058 		 */
3059 		if (adev->in_s0ix &&
3060 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3061 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3062 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3063 			continue;
3064 
3065 		/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3066 		if (adev->in_s0ix &&
3067 		    (adev->ip_versions[SDMA0_HWIP][0] >= IP_VERSION(5, 0, 0)) &&
3068 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
3069 			continue;
3070 
3071 		/* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3072 		 * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3073 		 * from this location and RLC Autoload automatically also gets loaded
3074 		 * from here based on PMFW -> PSP message during re-init sequence.
3075 		 * Therefore, the psp suspend & resume should be skipped to avoid destroy
3076 		 * the TMR and reload FWs again for IMU enabled APU ASICs.
3077 		 */
3078 		if (amdgpu_in_reset(adev) &&
3079 		    (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3080 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3081 			continue;
3082 
3083 		/* XXX handle errors */
3084 		r = adev->ip_blocks[i].version->funcs->suspend(adev);
3085 		/* XXX handle errors */
3086 		if (r) {
3087 			DRM_ERROR("suspend of IP block <%s> failed %d\n",
3088 				  adev->ip_blocks[i].version->funcs->name, r);
3089 		}
3090 		adev->ip_blocks[i].status.hw = false;
3091 		/* handle putting the SMC in the appropriate state */
3092 		if(!amdgpu_sriov_vf(adev)){
3093 			if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3094 				r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3095 				if (r) {
3096 					DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3097 							adev->mp1_state, r);
3098 					return r;
3099 				}
3100 			}
3101 		}
3102 	}
3103 
3104 	return 0;
3105 }
3106 
3107 /**
3108  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3109  *
3110  * @adev: amdgpu_device pointer
3111  *
3112  * Main suspend function for hardware IPs.  The list of all the hardware
3113  * IPs that make up the asic is walked, clockgating is disabled and the
3114  * suspend callbacks are run.  suspend puts the hardware and software state
3115  * in each IP into a state suitable for suspend.
3116  * Returns 0 on success, negative error code on failure.
3117  */
3118 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3119 {
3120 	int r;
3121 
3122 	if (amdgpu_sriov_vf(adev)) {
3123 		amdgpu_virt_fini_data_exchange(adev);
3124 		amdgpu_virt_request_full_gpu(adev, false);
3125 	}
3126 
3127 	r = amdgpu_device_ip_suspend_phase1(adev);
3128 	if (r)
3129 		return r;
3130 	r = amdgpu_device_ip_suspend_phase2(adev);
3131 
3132 	if (amdgpu_sriov_vf(adev))
3133 		amdgpu_virt_release_full_gpu(adev, false);
3134 
3135 	return r;
3136 }
3137 
3138 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3139 {
3140 	int i, r;
3141 
3142 	static enum amd_ip_block_type ip_order[] = {
3143 		AMD_IP_BLOCK_TYPE_COMMON,
3144 		AMD_IP_BLOCK_TYPE_GMC,
3145 		AMD_IP_BLOCK_TYPE_PSP,
3146 		AMD_IP_BLOCK_TYPE_IH,
3147 	};
3148 
3149 	for (i = 0; i < adev->num_ip_blocks; i++) {
3150 		int j;
3151 		struct amdgpu_ip_block *block;
3152 
3153 		block = &adev->ip_blocks[i];
3154 		block->status.hw = false;
3155 
3156 		for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3157 
3158 			if (block->version->type != ip_order[j] ||
3159 				!block->status.valid)
3160 				continue;
3161 
3162 			r = block->version->funcs->hw_init(adev);
3163 			DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3164 			if (r)
3165 				return r;
3166 			block->status.hw = true;
3167 		}
3168 	}
3169 
3170 	return 0;
3171 }
3172 
3173 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3174 {
3175 	int i, r;
3176 
3177 	static enum amd_ip_block_type ip_order[] = {
3178 		AMD_IP_BLOCK_TYPE_SMC,
3179 		AMD_IP_BLOCK_TYPE_DCE,
3180 		AMD_IP_BLOCK_TYPE_GFX,
3181 		AMD_IP_BLOCK_TYPE_SDMA,
3182 		AMD_IP_BLOCK_TYPE_MES,
3183 		AMD_IP_BLOCK_TYPE_UVD,
3184 		AMD_IP_BLOCK_TYPE_VCE,
3185 		AMD_IP_BLOCK_TYPE_VCN,
3186 		AMD_IP_BLOCK_TYPE_JPEG
3187 	};
3188 
3189 	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3190 		int j;
3191 		struct amdgpu_ip_block *block;
3192 
3193 		for (j = 0; j < adev->num_ip_blocks; j++) {
3194 			block = &adev->ip_blocks[j];
3195 
3196 			if (block->version->type != ip_order[i] ||
3197 				!block->status.valid ||
3198 				block->status.hw)
3199 				continue;
3200 
3201 			if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3202 				r = block->version->funcs->resume(adev);
3203 			else
3204 				r = block->version->funcs->hw_init(adev);
3205 
3206 			DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3207 			if (r)
3208 				return r;
3209 			block->status.hw = true;
3210 		}
3211 	}
3212 
3213 	return 0;
3214 }
3215 
3216 /**
3217  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3218  *
3219  * @adev: amdgpu_device pointer
3220  *
3221  * First resume function for hardware IPs.  The list of all the hardware
3222  * IPs that make up the asic is walked and the resume callbacks are run for
3223  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3224  * after a suspend and updates the software state as necessary.  This
3225  * function is also used for restoring the GPU after a GPU reset.
3226  * Returns 0 on success, negative error code on failure.
3227  */
3228 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3229 {
3230 	int i, r;
3231 
3232 	for (i = 0; i < adev->num_ip_blocks; i++) {
3233 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3234 			continue;
3235 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3236 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3237 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3238 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3239 
3240 			r = adev->ip_blocks[i].version->funcs->resume(adev);
3241 			if (r) {
3242 				DRM_ERROR("resume of IP block <%s> failed %d\n",
3243 					  adev->ip_blocks[i].version->funcs->name, r);
3244 				return r;
3245 			}
3246 			adev->ip_blocks[i].status.hw = true;
3247 		}
3248 	}
3249 
3250 	return 0;
3251 }
3252 
3253 /**
3254  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3255  *
3256  * @adev: amdgpu_device pointer
3257  *
3258  * First resume function for hardware IPs.  The list of all the hardware
3259  * IPs that make up the asic is walked and the resume callbacks are run for
3260  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3261  * functional state after a suspend and updates the software state as
3262  * necessary.  This function is also used for restoring the GPU after a GPU
3263  * reset.
3264  * Returns 0 on success, negative error code on failure.
3265  */
3266 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3267 {
3268 	int i, r;
3269 
3270 	for (i = 0; i < adev->num_ip_blocks; i++) {
3271 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3272 			continue;
3273 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3274 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3275 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3276 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3277 			continue;
3278 		r = adev->ip_blocks[i].version->funcs->resume(adev);
3279 		if (r) {
3280 			DRM_ERROR("resume of IP block <%s> failed %d\n",
3281 				  adev->ip_blocks[i].version->funcs->name, r);
3282 			return r;
3283 		}
3284 		adev->ip_blocks[i].status.hw = true;
3285 	}
3286 
3287 	return 0;
3288 }
3289 
3290 /**
3291  * amdgpu_device_ip_resume - run resume for hardware IPs
3292  *
3293  * @adev: amdgpu_device pointer
3294  *
3295  * Main resume function for hardware IPs.  The hardware IPs
3296  * are split into two resume functions because they are
3297  * are also used in in recovering from a GPU reset and some additional
3298  * steps need to be take between them.  In this case (S3/S4) they are
3299  * run sequentially.
3300  * Returns 0 on success, negative error code on failure.
3301  */
3302 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3303 {
3304 	int r;
3305 
3306 	if (!adev->in_s0ix) {
3307 		r = amdgpu_amdkfd_resume_iommu(adev);
3308 		if (r)
3309 			return r;
3310 	}
3311 
3312 	r = amdgpu_device_ip_resume_phase1(adev);
3313 	if (r)
3314 		return r;
3315 
3316 	r = amdgpu_device_fw_loading(adev);
3317 	if (r)
3318 		return r;
3319 
3320 	r = amdgpu_device_ip_resume_phase2(adev);
3321 
3322 	return r;
3323 }
3324 
3325 /**
3326  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3327  *
3328  * @adev: amdgpu_device pointer
3329  *
3330  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3331  */
3332 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3333 {
3334 	if (amdgpu_sriov_vf(adev)) {
3335 		if (adev->is_atom_fw) {
3336 			if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3337 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3338 		} else {
3339 			if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3340 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3341 		}
3342 
3343 		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3344 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3345 	}
3346 }
3347 
3348 /**
3349  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3350  *
3351  * @asic_type: AMD asic type
3352  *
3353  * Check if there is DC (new modesetting infrastructre) support for an asic.
3354  * returns true if DC has support, false if not.
3355  */
3356 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3357 {
3358 	switch (asic_type) {
3359 #ifdef CONFIG_DRM_AMDGPU_SI
3360 	case CHIP_HAINAN:
3361 #endif
3362 	case CHIP_TOPAZ:
3363 		/* chips with no display hardware */
3364 		return false;
3365 #if defined(CONFIG_DRM_AMD_DC)
3366 	case CHIP_TAHITI:
3367 	case CHIP_PITCAIRN:
3368 	case CHIP_VERDE:
3369 	case CHIP_OLAND:
3370 		/*
3371 		 * We have systems in the wild with these ASICs that require
3372 		 * LVDS and VGA support which is not supported with DC.
3373 		 *
3374 		 * Fallback to the non-DC driver here by default so as not to
3375 		 * cause regressions.
3376 		 */
3377 #if defined(CONFIG_DRM_AMD_DC_SI)
3378 		return amdgpu_dc > 0;
3379 #else
3380 		return false;
3381 #endif
3382 	case CHIP_BONAIRE:
3383 	case CHIP_KAVERI:
3384 	case CHIP_KABINI:
3385 	case CHIP_MULLINS:
3386 		/*
3387 		 * We have systems in the wild with these ASICs that require
3388 		 * VGA support which is not supported with DC.
3389 		 *
3390 		 * Fallback to the non-DC driver here by default so as not to
3391 		 * cause regressions.
3392 		 */
3393 		return amdgpu_dc > 0;
3394 	default:
3395 		return amdgpu_dc != 0;
3396 #else
3397 	default:
3398 		if (amdgpu_dc > 0)
3399 			DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3400 					 "but isn't supported by ASIC, ignoring\n");
3401 		return false;
3402 #endif
3403 	}
3404 }
3405 
3406 /**
3407  * amdgpu_device_has_dc_support - check if dc is supported
3408  *
3409  * @adev: amdgpu_device pointer
3410  *
3411  * Returns true for supported, false for not supported
3412  */
3413 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3414 {
3415 	if (adev->enable_virtual_display ||
3416 	    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3417 		return false;
3418 
3419 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
3420 }
3421 
3422 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3423 {
3424 	struct amdgpu_device *adev =
3425 		container_of(__work, struct amdgpu_device, xgmi_reset_work);
3426 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3427 
3428 	/* It's a bug to not have a hive within this function */
3429 	if (WARN_ON(!hive))
3430 		return;
3431 
3432 	/*
3433 	 * Use task barrier to synchronize all xgmi reset works across the
3434 	 * hive. task_barrier_enter and task_barrier_exit will block
3435 	 * until all the threads running the xgmi reset works reach
3436 	 * those points. task_barrier_full will do both blocks.
3437 	 */
3438 	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3439 
3440 		task_barrier_enter(&hive->tb);
3441 		adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3442 
3443 		if (adev->asic_reset_res)
3444 			goto fail;
3445 
3446 		task_barrier_exit(&hive->tb);
3447 		adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3448 
3449 		if (adev->asic_reset_res)
3450 			goto fail;
3451 
3452 		if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
3453 		    adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
3454 			adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
3455 	} else {
3456 
3457 		task_barrier_full(&hive->tb);
3458 		adev->asic_reset_res =  amdgpu_asic_reset(adev);
3459 	}
3460 
3461 fail:
3462 	if (adev->asic_reset_res)
3463 		DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3464 			 adev->asic_reset_res, adev_to_drm(adev)->unique);
3465 	amdgpu_put_xgmi_hive(hive);
3466 }
3467 
3468 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3469 {
3470 	char *input = amdgpu_lockup_timeout;
3471 	char *timeout_setting = NULL;
3472 	int index = 0;
3473 	long timeout;
3474 	int ret = 0;
3475 
3476 	/*
3477 	 * By default timeout for non compute jobs is 10000
3478 	 * and 60000 for compute jobs.
3479 	 * In SR-IOV or passthrough mode, timeout for compute
3480 	 * jobs are 60000 by default.
3481 	 */
3482 	adev->gfx_timeout = msecs_to_jiffies(10000);
3483 	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3484 	if (amdgpu_sriov_vf(adev))
3485 		adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3486 					msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3487 	else
3488 		adev->compute_timeout =  msecs_to_jiffies(60000);
3489 
3490 	if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3491 		while ((timeout_setting = strsep(&input, ",")) &&
3492 				strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3493 			ret = kstrtol(timeout_setting, 0, &timeout);
3494 			if (ret)
3495 				return ret;
3496 
3497 			if (timeout == 0) {
3498 				index++;
3499 				continue;
3500 			} else if (timeout < 0) {
3501 				timeout = MAX_SCHEDULE_TIMEOUT;
3502 				dev_warn(adev->dev, "lockup timeout disabled");
3503 				add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3504 			} else {
3505 				timeout = msecs_to_jiffies(timeout);
3506 			}
3507 
3508 			switch (index++) {
3509 			case 0:
3510 				adev->gfx_timeout = timeout;
3511 				break;
3512 			case 1:
3513 				adev->compute_timeout = timeout;
3514 				break;
3515 			case 2:
3516 				adev->sdma_timeout = timeout;
3517 				break;
3518 			case 3:
3519 				adev->video_timeout = timeout;
3520 				break;
3521 			default:
3522 				break;
3523 			}
3524 		}
3525 		/*
3526 		 * There is only one value specified and
3527 		 * it should apply to all non-compute jobs.
3528 		 */
3529 		if (index == 1) {
3530 			adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3531 			if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3532 				adev->compute_timeout = adev->gfx_timeout;
3533 		}
3534 	}
3535 
3536 	return ret;
3537 }
3538 
3539 /**
3540  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3541  *
3542  * @adev: amdgpu_device pointer
3543  *
3544  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3545  */
3546 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3547 {
3548 	struct iommu_domain *domain;
3549 
3550 	domain = iommu_get_domain_for_dev(adev->dev);
3551 	if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3552 		adev->ram_is_direct_mapped = true;
3553 }
3554 
3555 static const struct attribute *amdgpu_dev_attributes[] = {
3556 	&dev_attr_product_name.attr,
3557 	&dev_attr_product_number.attr,
3558 	&dev_attr_serial_number.attr,
3559 	&dev_attr_pcie_replay_count.attr,
3560 	NULL
3561 };
3562 
3563 /**
3564  * amdgpu_device_init - initialize the driver
3565  *
3566  * @adev: amdgpu_device pointer
3567  * @flags: driver flags
3568  *
3569  * Initializes the driver info and hw (all asics).
3570  * Returns 0 for success or an error on failure.
3571  * Called at driver startup.
3572  */
3573 int amdgpu_device_init(struct amdgpu_device *adev,
3574 		       uint32_t flags)
3575 {
3576 	struct drm_device *ddev = adev_to_drm(adev);
3577 	struct pci_dev *pdev = adev->pdev;
3578 	int r, i;
3579 	bool px = false;
3580 	u32 max_MBps;
3581 	int tmp;
3582 
3583 	adev->shutdown = false;
3584 	adev->flags = flags;
3585 
3586 	if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3587 		adev->asic_type = amdgpu_force_asic_type;
3588 	else
3589 		adev->asic_type = flags & AMD_ASIC_MASK;
3590 
3591 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3592 	if (amdgpu_emu_mode == 1)
3593 		adev->usec_timeout *= 10;
3594 	adev->gmc.gart_size = 512 * 1024 * 1024;
3595 	adev->accel_working = false;
3596 	adev->num_rings = 0;
3597 	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3598 	adev->mman.buffer_funcs = NULL;
3599 	adev->mman.buffer_funcs_ring = NULL;
3600 	adev->vm_manager.vm_pte_funcs = NULL;
3601 	adev->vm_manager.vm_pte_num_scheds = 0;
3602 	adev->gmc.gmc_funcs = NULL;
3603 	adev->harvest_ip_mask = 0x0;
3604 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3605 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3606 
3607 	adev->smc_rreg = &amdgpu_invalid_rreg;
3608 	adev->smc_wreg = &amdgpu_invalid_wreg;
3609 	adev->pcie_rreg = &amdgpu_invalid_rreg;
3610 	adev->pcie_wreg = &amdgpu_invalid_wreg;
3611 	adev->pciep_rreg = &amdgpu_invalid_rreg;
3612 	adev->pciep_wreg = &amdgpu_invalid_wreg;
3613 	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3614 	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3615 	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3616 	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3617 	adev->didt_rreg = &amdgpu_invalid_rreg;
3618 	adev->didt_wreg = &amdgpu_invalid_wreg;
3619 	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3620 	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3621 	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3622 	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3623 
3624 	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3625 		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3626 		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3627 
3628 	/* mutex initialization are all done here so we
3629 	 * can recall function without having locking issues */
3630 	mutex_init(&adev->firmware.mutex);
3631 	mutex_init(&adev->pm.mutex);
3632 	mutex_init(&adev->gfx.gpu_clock_mutex);
3633 	mutex_init(&adev->srbm_mutex);
3634 	mutex_init(&adev->gfx.pipe_reserve_mutex);
3635 	mutex_init(&adev->gfx.gfx_off_mutex);
3636 	mutex_init(&adev->grbm_idx_mutex);
3637 	mutex_init(&adev->mn_lock);
3638 	mutex_init(&adev->virt.vf_errors.lock);
3639 	hash_init(adev->mn_hash);
3640 	mutex_init(&adev->psp.mutex);
3641 	mutex_init(&adev->notifier_lock);
3642 	mutex_init(&adev->pm.stable_pstate_ctx_lock);
3643 	mutex_init(&adev->benchmark_mutex);
3644 
3645 	amdgpu_device_init_apu_flags(adev);
3646 
3647 	r = amdgpu_device_check_arguments(adev);
3648 	if (r)
3649 		return r;
3650 
3651 	spin_lock_init(&adev->mmio_idx_lock);
3652 	spin_lock_init(&adev->smc_idx_lock);
3653 	spin_lock_init(&adev->pcie_idx_lock);
3654 	spin_lock_init(&adev->uvd_ctx_idx_lock);
3655 	spin_lock_init(&adev->didt_idx_lock);
3656 	spin_lock_init(&adev->gc_cac_idx_lock);
3657 	spin_lock_init(&adev->se_cac_idx_lock);
3658 	spin_lock_init(&adev->audio_endpt_idx_lock);
3659 	spin_lock_init(&adev->mm_stats.lock);
3660 
3661 	INIT_LIST_HEAD(&adev->shadow_list);
3662 	mutex_init(&adev->shadow_list_lock);
3663 
3664 	INIT_LIST_HEAD(&adev->reset_list);
3665 
3666 	INIT_LIST_HEAD(&adev->ras_list);
3667 
3668 	INIT_DELAYED_WORK(&adev->delayed_init_work,
3669 			  amdgpu_device_delayed_init_work_handler);
3670 	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3671 			  amdgpu_device_delay_enable_gfx_off);
3672 
3673 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3674 
3675 	adev->gfx.gfx_off_req_count = 1;
3676 	adev->gfx.gfx_off_residency = 0;
3677 	adev->gfx.gfx_off_entrycount = 0;
3678 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3679 
3680 	atomic_set(&adev->throttling_logging_enabled, 1);
3681 	/*
3682 	 * If throttling continues, logging will be performed every minute
3683 	 * to avoid log flooding. "-1" is subtracted since the thermal
3684 	 * throttling interrupt comes every second. Thus, the total logging
3685 	 * interval is 59 seconds(retelimited printk interval) + 1(waiting
3686 	 * for throttling interrupt) = 60 seconds.
3687 	 */
3688 	ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3689 	ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3690 
3691 	/* Registers mapping */
3692 	/* TODO: block userspace mapping of io register */
3693 	if (adev->asic_type >= CHIP_BONAIRE) {
3694 		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3695 		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3696 	} else {
3697 		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3698 		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3699 	}
3700 
3701 	for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3702 		atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3703 
3704 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3705 	if (adev->rmmio == NULL) {
3706 		return -ENOMEM;
3707 	}
3708 	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3709 	DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3710 
3711 	amdgpu_device_get_pcie_info(adev);
3712 
3713 	if (amdgpu_mcbp)
3714 		DRM_INFO("MCBP is enabled\n");
3715 
3716 	/*
3717 	 * Reset domain needs to be present early, before XGMI hive discovered
3718 	 * (if any) and intitialized to use reset sem and in_gpu reset flag
3719 	 * early on during init and before calling to RREG32.
3720 	 */
3721 	adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3722 	if (!adev->reset_domain)
3723 		return -ENOMEM;
3724 
3725 	/* detect hw virtualization here */
3726 	amdgpu_detect_virtualization(adev);
3727 
3728 	r = amdgpu_device_get_job_timeout_settings(adev);
3729 	if (r) {
3730 		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3731 		return r;
3732 	}
3733 
3734 	/* early init functions */
3735 	r = amdgpu_device_ip_early_init(adev);
3736 	if (r)
3737 		return r;
3738 
3739 	/* Get rid of things like offb */
3740 	r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
3741 	if (r)
3742 		return r;
3743 
3744 	/* Enable TMZ based on IP_VERSION */
3745 	amdgpu_gmc_tmz_set(adev);
3746 
3747 	amdgpu_gmc_noretry_set(adev);
3748 	/* Need to get xgmi info early to decide the reset behavior*/
3749 	if (adev->gmc.xgmi.supported) {
3750 		r = adev->gfxhub.funcs->get_xgmi_info(adev);
3751 		if (r)
3752 			return r;
3753 	}
3754 
3755 	/* enable PCIE atomic ops */
3756 	if (amdgpu_sriov_vf(adev))
3757 		adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3758 			adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3759 			(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3760 	/* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
3761 	 * internal path natively support atomics, set have_atomics_support to true.
3762 	 */
3763 	else if ((adev->flags & AMD_IS_APU) &&
3764 		(adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0)))
3765 		adev->have_atomics_support = true;
3766 	else
3767 		adev->have_atomics_support =
3768 			!pci_enable_atomic_ops_to_root(adev->pdev,
3769 					  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3770 					  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3771 	if (!adev->have_atomics_support)
3772 		dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3773 
3774 	/* doorbell bar mapping and doorbell index init*/
3775 	amdgpu_device_doorbell_init(adev);
3776 
3777 	if (amdgpu_emu_mode == 1) {
3778 		/* post the asic on emulation mode */
3779 		emu_soc_asic_init(adev);
3780 		goto fence_driver_init;
3781 	}
3782 
3783 	amdgpu_reset_init(adev);
3784 
3785 	/* detect if we are with an SRIOV vbios */
3786 	amdgpu_device_detect_sriov_bios(adev);
3787 
3788 	/* check if we need to reset the asic
3789 	 *  E.g., driver was not cleanly unloaded previously, etc.
3790 	 */
3791 	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3792 		if (adev->gmc.xgmi.num_physical_nodes) {
3793 			dev_info(adev->dev, "Pending hive reset.\n");
3794 			adev->gmc.xgmi.pending_reset = true;
3795 			/* Only need to init necessary block for SMU to handle the reset */
3796 			for (i = 0; i < adev->num_ip_blocks; i++) {
3797 				if (!adev->ip_blocks[i].status.valid)
3798 					continue;
3799 				if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3800 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3801 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3802 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3803 					DRM_DEBUG("IP %s disabled for hw_init.\n",
3804 						adev->ip_blocks[i].version->funcs->name);
3805 					adev->ip_blocks[i].status.hw = true;
3806 				}
3807 			}
3808 		} else {
3809 			tmp = amdgpu_reset_method;
3810 			/* It should do a default reset when loading or reloading the driver,
3811 			 * regardless of the module parameter reset_method.
3812 			 */
3813 			amdgpu_reset_method = AMD_RESET_METHOD_NONE;
3814 			r = amdgpu_asic_reset(adev);
3815 			amdgpu_reset_method = tmp;
3816 			if (r) {
3817 				dev_err(adev->dev, "asic reset on init failed\n");
3818 				goto failed;
3819 			}
3820 		}
3821 	}
3822 
3823 	/* Post card if necessary */
3824 	if (amdgpu_device_need_post(adev)) {
3825 		if (!adev->bios) {
3826 			dev_err(adev->dev, "no vBIOS found\n");
3827 			r = -EINVAL;
3828 			goto failed;
3829 		}
3830 		DRM_INFO("GPU posting now...\n");
3831 		r = amdgpu_device_asic_init(adev);
3832 		if (r) {
3833 			dev_err(adev->dev, "gpu post error!\n");
3834 			goto failed;
3835 		}
3836 	}
3837 
3838 	if (adev->is_atom_fw) {
3839 		/* Initialize clocks */
3840 		r = amdgpu_atomfirmware_get_clock_info(adev);
3841 		if (r) {
3842 			dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3843 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3844 			goto failed;
3845 		}
3846 	} else {
3847 		/* Initialize clocks */
3848 		r = amdgpu_atombios_get_clock_info(adev);
3849 		if (r) {
3850 			dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3851 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3852 			goto failed;
3853 		}
3854 		/* init i2c buses */
3855 		if (!amdgpu_device_has_dc_support(adev))
3856 			amdgpu_atombios_i2c_init(adev);
3857 	}
3858 
3859 fence_driver_init:
3860 	/* Fence driver */
3861 	r = amdgpu_fence_driver_sw_init(adev);
3862 	if (r) {
3863 		dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3864 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3865 		goto failed;
3866 	}
3867 
3868 	/* init the mode config */
3869 	drm_mode_config_init(adev_to_drm(adev));
3870 
3871 	r = amdgpu_device_ip_init(adev);
3872 	if (r) {
3873 		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3874 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3875 		goto release_ras_con;
3876 	}
3877 
3878 	amdgpu_fence_driver_hw_init(adev);
3879 
3880 	dev_info(adev->dev,
3881 		"SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3882 			adev->gfx.config.max_shader_engines,
3883 			adev->gfx.config.max_sh_per_se,
3884 			adev->gfx.config.max_cu_per_sh,
3885 			adev->gfx.cu_info.number);
3886 
3887 	adev->accel_working = true;
3888 
3889 	amdgpu_vm_check_compute_bug(adev);
3890 
3891 	/* Initialize the buffer migration limit. */
3892 	if (amdgpu_moverate >= 0)
3893 		max_MBps = amdgpu_moverate;
3894 	else
3895 		max_MBps = 8; /* Allow 8 MB/s. */
3896 	/* Get a log2 for easy divisions. */
3897 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3898 
3899 	r = amdgpu_pm_sysfs_init(adev);
3900 	if (r)
3901 		DRM_ERROR("registering pm sysfs failed (%d).\n", r);
3902 
3903 	r = amdgpu_ucode_sysfs_init(adev);
3904 	if (r) {
3905 		adev->ucode_sysfs_en = false;
3906 		DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3907 	} else
3908 		adev->ucode_sysfs_en = true;
3909 
3910 	r = amdgpu_psp_sysfs_init(adev);
3911 	if (r) {
3912 		adev->psp_sysfs_en = false;
3913 		if (!amdgpu_sriov_vf(adev))
3914 			DRM_ERROR("Creating psp sysfs failed\n");
3915 	} else
3916 		adev->psp_sysfs_en = true;
3917 
3918 	/*
3919 	 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3920 	 * Otherwise the mgpu fan boost feature will be skipped due to the
3921 	 * gpu instance is counted less.
3922 	 */
3923 	amdgpu_register_gpu_instance(adev);
3924 
3925 	/* enable clockgating, etc. after ib tests, etc. since some blocks require
3926 	 * explicit gating rather than handling it automatically.
3927 	 */
3928 	if (!adev->gmc.xgmi.pending_reset) {
3929 		r = amdgpu_device_ip_late_init(adev);
3930 		if (r) {
3931 			dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3932 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3933 			goto release_ras_con;
3934 		}
3935 		/* must succeed. */
3936 		amdgpu_ras_resume(adev);
3937 		queue_delayed_work(system_wq, &adev->delayed_init_work,
3938 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
3939 	}
3940 
3941 	if (amdgpu_sriov_vf(adev)) {
3942 		amdgpu_virt_release_full_gpu(adev, true);
3943 		flush_delayed_work(&adev->delayed_init_work);
3944 	}
3945 
3946 	r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3947 	if (r)
3948 		dev_err(adev->dev, "Could not create amdgpu device attr\n");
3949 
3950 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
3951 		r = amdgpu_pmu_init(adev);
3952 	if (r)
3953 		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3954 
3955 	/* Have stored pci confspace at hand for restore in sudden PCI error */
3956 	if (amdgpu_device_cache_pci_state(adev->pdev))
3957 		pci_restore_state(pdev);
3958 
3959 	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3960 	/* this will fail for cards that aren't VGA class devices, just
3961 	 * ignore it */
3962 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3963 		vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
3964 
3965 	px = amdgpu_device_supports_px(ddev);
3966 
3967 	if (px || (!pci_is_thunderbolt_attached(adev->pdev) &&
3968 				apple_gmux_detect(NULL, NULL)))
3969 		vga_switcheroo_register_client(adev->pdev,
3970 					       &amdgpu_switcheroo_ops, px);
3971 
3972 	if (px)
3973 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3974 
3975 	if (adev->gmc.xgmi.pending_reset)
3976 		queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3977 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
3978 
3979 	amdgpu_device_check_iommu_direct_map(adev);
3980 
3981 	return 0;
3982 
3983 release_ras_con:
3984 	if (amdgpu_sriov_vf(adev))
3985 		amdgpu_virt_release_full_gpu(adev, true);
3986 
3987 	/* failed in exclusive mode due to timeout */
3988 	if (amdgpu_sriov_vf(adev) &&
3989 		!amdgpu_sriov_runtime(adev) &&
3990 		amdgpu_virt_mmio_blocked(adev) &&
3991 		!amdgpu_virt_wait_reset(adev)) {
3992 		dev_err(adev->dev, "VF exclusive mode timeout\n");
3993 		/* Don't send request since VF is inactive. */
3994 		adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3995 		adev->virt.ops = NULL;
3996 		r = -EAGAIN;
3997 	}
3998 	amdgpu_release_ras_context(adev);
3999 
4000 failed:
4001 	amdgpu_vf_error_trans_all(adev);
4002 
4003 	return r;
4004 }
4005 
4006 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4007 {
4008 
4009 	/* Clear all CPU mappings pointing to this device */
4010 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4011 
4012 	/* Unmap all mapped bars - Doorbell, registers and VRAM */
4013 	amdgpu_device_doorbell_fini(adev);
4014 
4015 	iounmap(adev->rmmio);
4016 	adev->rmmio = NULL;
4017 	if (adev->mman.aper_base_kaddr)
4018 		iounmap(adev->mman.aper_base_kaddr);
4019 	adev->mman.aper_base_kaddr = NULL;
4020 
4021 	/* Memory manager related */
4022 	if (!adev->gmc.xgmi.connected_to_cpu) {
4023 		arch_phys_wc_del(adev->gmc.vram_mtrr);
4024 		arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4025 	}
4026 }
4027 
4028 /**
4029  * amdgpu_device_fini_hw - tear down the driver
4030  *
4031  * @adev: amdgpu_device pointer
4032  *
4033  * Tear down the driver info (all asics).
4034  * Called at driver shutdown.
4035  */
4036 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4037 {
4038 	dev_info(adev->dev, "amdgpu: finishing device.\n");
4039 	flush_delayed_work(&adev->delayed_init_work);
4040 	adev->shutdown = true;
4041 
4042 	/* make sure IB test finished before entering exclusive mode
4043 	 * to avoid preemption on IB test
4044 	 * */
4045 	if (amdgpu_sriov_vf(adev)) {
4046 		amdgpu_virt_request_full_gpu(adev, false);
4047 		amdgpu_virt_fini_data_exchange(adev);
4048 	}
4049 
4050 	/* disable all interrupts */
4051 	amdgpu_irq_disable_all(adev);
4052 	if (adev->mode_info.mode_config_initialized){
4053 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4054 			drm_helper_force_disable_all(adev_to_drm(adev));
4055 		else
4056 			drm_atomic_helper_shutdown(adev_to_drm(adev));
4057 	}
4058 	amdgpu_fence_driver_hw_fini(adev);
4059 
4060 	if (adev->mman.initialized)
4061 		drain_workqueue(adev->mman.bdev.wq);
4062 
4063 	if (adev->pm.sysfs_initialized)
4064 		amdgpu_pm_sysfs_fini(adev);
4065 	if (adev->ucode_sysfs_en)
4066 		amdgpu_ucode_sysfs_fini(adev);
4067 	if (adev->psp_sysfs_en)
4068 		amdgpu_psp_sysfs_fini(adev);
4069 	sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4070 
4071 	/* disable ras feature must before hw fini */
4072 	amdgpu_ras_pre_fini(adev);
4073 
4074 	amdgpu_device_ip_fini_early(adev);
4075 
4076 	amdgpu_irq_fini_hw(adev);
4077 
4078 	if (adev->mman.initialized)
4079 		ttm_device_clear_dma_mappings(&adev->mman.bdev);
4080 
4081 	amdgpu_gart_dummy_page_fini(adev);
4082 
4083 	if (drm_dev_is_unplugged(adev_to_drm(adev)))
4084 		amdgpu_device_unmap_mmio(adev);
4085 
4086 }
4087 
4088 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4089 {
4090 	int idx;
4091 	bool px;
4092 
4093 	amdgpu_fence_driver_sw_fini(adev);
4094 	amdgpu_device_ip_fini(adev);
4095 	amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4096 	adev->accel_working = false;
4097 	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4098 
4099 	amdgpu_reset_fini(adev);
4100 
4101 	/* free i2c buses */
4102 	if (!amdgpu_device_has_dc_support(adev))
4103 		amdgpu_i2c_fini(adev);
4104 
4105 	if (amdgpu_emu_mode != 1)
4106 		amdgpu_atombios_fini(adev);
4107 
4108 	kfree(adev->bios);
4109 	adev->bios = NULL;
4110 
4111 	px = amdgpu_device_supports_px(adev_to_drm(adev));
4112 
4113 	if (px || (!pci_is_thunderbolt_attached(adev->pdev) &&
4114 				apple_gmux_detect(NULL, NULL)))
4115 		vga_switcheroo_unregister_client(adev->pdev);
4116 
4117 	if (px)
4118 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
4119 
4120 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4121 		vga_client_unregister(adev->pdev);
4122 
4123 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4124 
4125 		iounmap(adev->rmmio);
4126 		adev->rmmio = NULL;
4127 		amdgpu_device_doorbell_fini(adev);
4128 		drm_dev_exit(idx);
4129 	}
4130 
4131 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4132 		amdgpu_pmu_fini(adev);
4133 	if (adev->mman.discovery_bin)
4134 		amdgpu_discovery_fini(adev);
4135 
4136 	amdgpu_reset_put_reset_domain(adev->reset_domain);
4137 	adev->reset_domain = NULL;
4138 
4139 	kfree(adev->pci_state);
4140 
4141 }
4142 
4143 /**
4144  * amdgpu_device_evict_resources - evict device resources
4145  * @adev: amdgpu device object
4146  *
4147  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4148  * of the vram memory type. Mainly used for evicting device resources
4149  * at suspend time.
4150  *
4151  */
4152 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4153 {
4154 	int ret;
4155 
4156 	/* No need to evict vram on APUs for suspend to ram or s2idle */
4157 	if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4158 		return 0;
4159 
4160 	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4161 	if (ret)
4162 		DRM_WARN("evicting device resources failed\n");
4163 	return ret;
4164 }
4165 
4166 /*
4167  * Suspend & resume.
4168  */
4169 /**
4170  * amdgpu_device_suspend - initiate device suspend
4171  *
4172  * @dev: drm dev pointer
4173  * @fbcon : notify the fbdev of suspend
4174  *
4175  * Puts the hw in the suspend state (all asics).
4176  * Returns 0 for success or an error on failure.
4177  * Called at driver suspend.
4178  */
4179 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4180 {
4181 	struct amdgpu_device *adev = drm_to_adev(dev);
4182 	int r = 0;
4183 
4184 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4185 		return 0;
4186 
4187 	adev->in_suspend = true;
4188 
4189 	/* Evict the majority of BOs before grabbing the full access */
4190 	r = amdgpu_device_evict_resources(adev);
4191 	if (r)
4192 		return r;
4193 
4194 	if (amdgpu_sriov_vf(adev)) {
4195 		amdgpu_virt_fini_data_exchange(adev);
4196 		r = amdgpu_virt_request_full_gpu(adev, false);
4197 		if (r)
4198 			return r;
4199 	}
4200 
4201 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4202 		DRM_WARN("smart shift update failed\n");
4203 
4204 	if (fbcon)
4205 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4206 
4207 	cancel_delayed_work_sync(&adev->delayed_init_work);
4208 
4209 	amdgpu_ras_suspend(adev);
4210 
4211 	amdgpu_device_ip_suspend_phase1(adev);
4212 
4213 	if (!adev->in_s0ix)
4214 		amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4215 
4216 	r = amdgpu_device_evict_resources(adev);
4217 	if (r)
4218 		return r;
4219 
4220 	amdgpu_fence_driver_hw_fini(adev);
4221 
4222 	amdgpu_device_ip_suspend_phase2(adev);
4223 
4224 	if (amdgpu_sriov_vf(adev))
4225 		amdgpu_virt_release_full_gpu(adev, false);
4226 
4227 	return 0;
4228 }
4229 
4230 /**
4231  * amdgpu_device_resume - initiate device resume
4232  *
4233  * @dev: drm dev pointer
4234  * @fbcon : notify the fbdev of resume
4235  *
4236  * Bring the hw back to operating state (all asics).
4237  * Returns 0 for success or an error on failure.
4238  * Called at driver resume.
4239  */
4240 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4241 {
4242 	struct amdgpu_device *adev = drm_to_adev(dev);
4243 	int r = 0;
4244 
4245 	if (amdgpu_sriov_vf(adev)) {
4246 		r = amdgpu_virt_request_full_gpu(adev, true);
4247 		if (r)
4248 			return r;
4249 	}
4250 
4251 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4252 		return 0;
4253 
4254 	if (adev->in_s0ix)
4255 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4256 
4257 	/* post card */
4258 	if (amdgpu_device_need_post(adev)) {
4259 		r = amdgpu_device_asic_init(adev);
4260 		if (r)
4261 			dev_err(adev->dev, "amdgpu asic init failed\n");
4262 	}
4263 
4264 	r = amdgpu_device_ip_resume(adev);
4265 
4266 	if (r) {
4267 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4268 		goto exit;
4269 	}
4270 	amdgpu_fence_driver_hw_init(adev);
4271 
4272 	r = amdgpu_device_ip_late_init(adev);
4273 	if (r)
4274 		goto exit;
4275 
4276 	queue_delayed_work(system_wq, &adev->delayed_init_work,
4277 			   msecs_to_jiffies(AMDGPU_RESUME_MS));
4278 
4279 	if (!adev->in_s0ix) {
4280 		r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4281 		if (r)
4282 			goto exit;
4283 	}
4284 
4285 exit:
4286 	if (amdgpu_sriov_vf(adev)) {
4287 		amdgpu_virt_init_data_exchange(adev);
4288 		amdgpu_virt_release_full_gpu(adev, true);
4289 	}
4290 
4291 	if (r)
4292 		return r;
4293 
4294 	/* Make sure IB tests flushed */
4295 	flush_delayed_work(&adev->delayed_init_work);
4296 
4297 	if (fbcon)
4298 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4299 
4300 	amdgpu_ras_resume(adev);
4301 
4302 	if (adev->mode_info.num_crtc) {
4303 		/*
4304 		 * Most of the connector probing functions try to acquire runtime pm
4305 		 * refs to ensure that the GPU is powered on when connector polling is
4306 		 * performed. Since we're calling this from a runtime PM callback,
4307 		 * trying to acquire rpm refs will cause us to deadlock.
4308 		 *
4309 		 * Since we're guaranteed to be holding the rpm lock, it's safe to
4310 		 * temporarily disable the rpm helpers so this doesn't deadlock us.
4311 		 */
4312 #ifdef CONFIG_PM
4313 		dev->dev->power.disable_depth++;
4314 #endif
4315 		if (!adev->dc_enabled)
4316 			drm_helper_hpd_irq_event(dev);
4317 		else
4318 			drm_kms_helper_hotplug_event(dev);
4319 #ifdef CONFIG_PM
4320 		dev->dev->power.disable_depth--;
4321 #endif
4322 	}
4323 	adev->in_suspend = false;
4324 
4325 	if (adev->enable_mes)
4326 		amdgpu_mes_self_test(adev);
4327 
4328 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4329 		DRM_WARN("smart shift update failed\n");
4330 
4331 	return 0;
4332 }
4333 
4334 /**
4335  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4336  *
4337  * @adev: amdgpu_device pointer
4338  *
4339  * The list of all the hardware IPs that make up the asic is walked and
4340  * the check_soft_reset callbacks are run.  check_soft_reset determines
4341  * if the asic is still hung or not.
4342  * Returns true if any of the IPs are still in a hung state, false if not.
4343  */
4344 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4345 {
4346 	int i;
4347 	bool asic_hang = false;
4348 
4349 	if (amdgpu_sriov_vf(adev))
4350 		return true;
4351 
4352 	if (amdgpu_asic_need_full_reset(adev))
4353 		return true;
4354 
4355 	for (i = 0; i < adev->num_ip_blocks; i++) {
4356 		if (!adev->ip_blocks[i].status.valid)
4357 			continue;
4358 		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4359 			adev->ip_blocks[i].status.hang =
4360 				adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4361 		if (adev->ip_blocks[i].status.hang) {
4362 			dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4363 			asic_hang = true;
4364 		}
4365 	}
4366 	return asic_hang;
4367 }
4368 
4369 /**
4370  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4371  *
4372  * @adev: amdgpu_device pointer
4373  *
4374  * The list of all the hardware IPs that make up the asic is walked and the
4375  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4376  * handles any IP specific hardware or software state changes that are
4377  * necessary for a soft reset to succeed.
4378  * Returns 0 on success, negative error code on failure.
4379  */
4380 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4381 {
4382 	int i, r = 0;
4383 
4384 	for (i = 0; i < adev->num_ip_blocks; i++) {
4385 		if (!adev->ip_blocks[i].status.valid)
4386 			continue;
4387 		if (adev->ip_blocks[i].status.hang &&
4388 		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4389 			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4390 			if (r)
4391 				return r;
4392 		}
4393 	}
4394 
4395 	return 0;
4396 }
4397 
4398 /**
4399  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4400  *
4401  * @adev: amdgpu_device pointer
4402  *
4403  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4404  * reset is necessary to recover.
4405  * Returns true if a full asic reset is required, false if not.
4406  */
4407 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4408 {
4409 	int i;
4410 
4411 	if (amdgpu_asic_need_full_reset(adev))
4412 		return true;
4413 
4414 	for (i = 0; i < adev->num_ip_blocks; i++) {
4415 		if (!adev->ip_blocks[i].status.valid)
4416 			continue;
4417 		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4418 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4419 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4420 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4421 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4422 			if (adev->ip_blocks[i].status.hang) {
4423 				dev_info(adev->dev, "Some block need full reset!\n");
4424 				return true;
4425 			}
4426 		}
4427 	}
4428 	return false;
4429 }
4430 
4431 /**
4432  * amdgpu_device_ip_soft_reset - do a soft reset
4433  *
4434  * @adev: amdgpu_device pointer
4435  *
4436  * The list of all the hardware IPs that make up the asic is walked and the
4437  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4438  * IP specific hardware or software state changes that are necessary to soft
4439  * reset the IP.
4440  * Returns 0 on success, negative error code on failure.
4441  */
4442 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4443 {
4444 	int i, r = 0;
4445 
4446 	for (i = 0; i < adev->num_ip_blocks; i++) {
4447 		if (!adev->ip_blocks[i].status.valid)
4448 			continue;
4449 		if (adev->ip_blocks[i].status.hang &&
4450 		    adev->ip_blocks[i].version->funcs->soft_reset) {
4451 			r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4452 			if (r)
4453 				return r;
4454 		}
4455 	}
4456 
4457 	return 0;
4458 }
4459 
4460 /**
4461  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4462  *
4463  * @adev: amdgpu_device pointer
4464  *
4465  * The list of all the hardware IPs that make up the asic is walked and the
4466  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4467  * handles any IP specific hardware or software state changes that are
4468  * necessary after the IP has been soft reset.
4469  * Returns 0 on success, negative error code on failure.
4470  */
4471 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4472 {
4473 	int i, r = 0;
4474 
4475 	for (i = 0; i < adev->num_ip_blocks; i++) {
4476 		if (!adev->ip_blocks[i].status.valid)
4477 			continue;
4478 		if (adev->ip_blocks[i].status.hang &&
4479 		    adev->ip_blocks[i].version->funcs->post_soft_reset)
4480 			r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4481 		if (r)
4482 			return r;
4483 	}
4484 
4485 	return 0;
4486 }
4487 
4488 /**
4489  * amdgpu_device_recover_vram - Recover some VRAM contents
4490  *
4491  * @adev: amdgpu_device pointer
4492  *
4493  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4494  * restore things like GPUVM page tables after a GPU reset where
4495  * the contents of VRAM might be lost.
4496  *
4497  * Returns:
4498  * 0 on success, negative error code on failure.
4499  */
4500 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4501 {
4502 	struct dma_fence *fence = NULL, *next = NULL;
4503 	struct amdgpu_bo *shadow;
4504 	struct amdgpu_bo_vm *vmbo;
4505 	long r = 1, tmo;
4506 
4507 	if (amdgpu_sriov_runtime(adev))
4508 		tmo = msecs_to_jiffies(8000);
4509 	else
4510 		tmo = msecs_to_jiffies(100);
4511 
4512 	dev_info(adev->dev, "recover vram bo from shadow start\n");
4513 	mutex_lock(&adev->shadow_list_lock);
4514 	list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4515 		/* If vm is compute context or adev is APU, shadow will be NULL */
4516 		if (!vmbo->shadow)
4517 			continue;
4518 		shadow = vmbo->shadow;
4519 
4520 		/* No need to recover an evicted BO */
4521 		if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4522 		    shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4523 		    shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4524 			continue;
4525 
4526 		r = amdgpu_bo_restore_shadow(shadow, &next);
4527 		if (r)
4528 			break;
4529 
4530 		if (fence) {
4531 			tmo = dma_fence_wait_timeout(fence, false, tmo);
4532 			dma_fence_put(fence);
4533 			fence = next;
4534 			if (tmo == 0) {
4535 				r = -ETIMEDOUT;
4536 				break;
4537 			} else if (tmo < 0) {
4538 				r = tmo;
4539 				break;
4540 			}
4541 		} else {
4542 			fence = next;
4543 		}
4544 	}
4545 	mutex_unlock(&adev->shadow_list_lock);
4546 
4547 	if (fence)
4548 		tmo = dma_fence_wait_timeout(fence, false, tmo);
4549 	dma_fence_put(fence);
4550 
4551 	if (r < 0 || tmo <= 0) {
4552 		dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4553 		return -EIO;
4554 	}
4555 
4556 	dev_info(adev->dev, "recover vram bo from shadow done\n");
4557 	return 0;
4558 }
4559 
4560 
4561 /**
4562  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4563  *
4564  * @adev: amdgpu_device pointer
4565  * @from_hypervisor: request from hypervisor
4566  *
4567  * do VF FLR and reinitialize Asic
4568  * return 0 means succeeded otherwise failed
4569  */
4570 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4571 				     bool from_hypervisor)
4572 {
4573 	int r;
4574 	struct amdgpu_hive_info *hive = NULL;
4575 	int retry_limit = 0;
4576 
4577 retry:
4578 	amdgpu_amdkfd_pre_reset(adev);
4579 
4580 	if (from_hypervisor)
4581 		r = amdgpu_virt_request_full_gpu(adev, true);
4582 	else
4583 		r = amdgpu_virt_reset_gpu(adev);
4584 	if (r)
4585 		return r;
4586 
4587 	/* Resume IP prior to SMC */
4588 	r = amdgpu_device_ip_reinit_early_sriov(adev);
4589 	if (r)
4590 		goto error;
4591 
4592 	amdgpu_virt_init_data_exchange(adev);
4593 
4594 	r = amdgpu_device_fw_loading(adev);
4595 	if (r)
4596 		return r;
4597 
4598 	/* now we are okay to resume SMC/CP/SDMA */
4599 	r = amdgpu_device_ip_reinit_late_sriov(adev);
4600 	if (r)
4601 		goto error;
4602 
4603 	hive = amdgpu_get_xgmi_hive(adev);
4604 	/* Update PSP FW topology after reset */
4605 	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4606 		r = amdgpu_xgmi_update_topology(hive, adev);
4607 
4608 	if (hive)
4609 		amdgpu_put_xgmi_hive(hive);
4610 
4611 	if (!r) {
4612 		amdgpu_irq_gpu_reset_resume_helper(adev);
4613 		r = amdgpu_ib_ring_tests(adev);
4614 
4615 		amdgpu_amdkfd_post_reset(adev);
4616 	}
4617 
4618 error:
4619 	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4620 		amdgpu_inc_vram_lost(adev);
4621 		r = amdgpu_device_recover_vram(adev);
4622 	}
4623 	amdgpu_virt_release_full_gpu(adev, true);
4624 
4625 	if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4626 		if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4627 			retry_limit++;
4628 			goto retry;
4629 		} else
4630 			DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4631 	}
4632 
4633 	return r;
4634 }
4635 
4636 /**
4637  * amdgpu_device_has_job_running - check if there is any job in mirror list
4638  *
4639  * @adev: amdgpu_device pointer
4640  *
4641  * check if there is any job in mirror list
4642  */
4643 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4644 {
4645 	int i;
4646 	struct drm_sched_job *job;
4647 
4648 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4649 		struct amdgpu_ring *ring = adev->rings[i];
4650 
4651 		if (!ring || !ring->sched.thread)
4652 			continue;
4653 
4654 		spin_lock(&ring->sched.job_list_lock);
4655 		job = list_first_entry_or_null(&ring->sched.pending_list,
4656 					       struct drm_sched_job, list);
4657 		spin_unlock(&ring->sched.job_list_lock);
4658 		if (job)
4659 			return true;
4660 	}
4661 	return false;
4662 }
4663 
4664 /**
4665  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4666  *
4667  * @adev: amdgpu_device pointer
4668  *
4669  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4670  * a hung GPU.
4671  */
4672 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4673 {
4674 
4675 	if (amdgpu_gpu_recovery == 0)
4676 		goto disabled;
4677 
4678 	/* Skip soft reset check in fatal error mode */
4679 	if (!amdgpu_ras_is_poison_mode_supported(adev))
4680 		return true;
4681 
4682 	if (amdgpu_sriov_vf(adev))
4683 		return true;
4684 
4685 	if (amdgpu_gpu_recovery == -1) {
4686 		switch (adev->asic_type) {
4687 #ifdef CONFIG_DRM_AMDGPU_SI
4688 		case CHIP_VERDE:
4689 		case CHIP_TAHITI:
4690 		case CHIP_PITCAIRN:
4691 		case CHIP_OLAND:
4692 		case CHIP_HAINAN:
4693 #endif
4694 #ifdef CONFIG_DRM_AMDGPU_CIK
4695 		case CHIP_KAVERI:
4696 		case CHIP_KABINI:
4697 		case CHIP_MULLINS:
4698 #endif
4699 		case CHIP_CARRIZO:
4700 		case CHIP_STONEY:
4701 		case CHIP_CYAN_SKILLFISH:
4702 			goto disabled;
4703 		default:
4704 			break;
4705 		}
4706 	}
4707 
4708 	return true;
4709 
4710 disabled:
4711 		dev_info(adev->dev, "GPU recovery disabled.\n");
4712 		return false;
4713 }
4714 
4715 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4716 {
4717         u32 i;
4718         int ret = 0;
4719 
4720         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4721 
4722         dev_info(adev->dev, "GPU mode1 reset\n");
4723 
4724         /* disable BM */
4725         pci_clear_master(adev->pdev);
4726 
4727         amdgpu_device_cache_pci_state(adev->pdev);
4728 
4729         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4730                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4731                 ret = amdgpu_dpm_mode1_reset(adev);
4732         } else {
4733                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4734                 ret = psp_gpu_reset(adev);
4735         }
4736 
4737         if (ret)
4738                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4739 
4740         amdgpu_device_load_pci_state(adev->pdev);
4741 
4742         /* wait for asic to come out of reset */
4743         for (i = 0; i < adev->usec_timeout; i++) {
4744                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4745 
4746                 if (memsize != 0xffffffff)
4747                         break;
4748                 udelay(1);
4749         }
4750 
4751         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4752         return ret;
4753 }
4754 
4755 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4756 				 struct amdgpu_reset_context *reset_context)
4757 {
4758 	int i, r = 0;
4759 	struct amdgpu_job *job = NULL;
4760 	bool need_full_reset =
4761 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4762 
4763 	if (reset_context->reset_req_dev == adev)
4764 		job = reset_context->job;
4765 
4766 	if (amdgpu_sriov_vf(adev)) {
4767 		/* stop the data exchange thread */
4768 		amdgpu_virt_fini_data_exchange(adev);
4769 	}
4770 
4771 	amdgpu_fence_driver_isr_toggle(adev, true);
4772 
4773 	/* block all schedulers and reset given job's ring */
4774 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4775 		struct amdgpu_ring *ring = adev->rings[i];
4776 
4777 		if (!ring || !ring->sched.thread)
4778 			continue;
4779 
4780 		/*clear job fence from fence drv to avoid force_completion
4781 		 *leave NULL and vm flush fence in fence drv */
4782 		amdgpu_fence_driver_clear_job_fences(ring);
4783 
4784 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4785 		amdgpu_fence_driver_force_completion(ring);
4786 	}
4787 
4788 	amdgpu_fence_driver_isr_toggle(adev, false);
4789 
4790 	if (job && job->vm)
4791 		drm_sched_increase_karma(&job->base);
4792 
4793 	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4794 	/* If reset handler not implemented, continue; otherwise return */
4795 	if (r == -ENOSYS)
4796 		r = 0;
4797 	else
4798 		return r;
4799 
4800 	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4801 	if (!amdgpu_sriov_vf(adev)) {
4802 
4803 		if (!need_full_reset)
4804 			need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4805 
4806 		if (!need_full_reset && amdgpu_gpu_recovery &&
4807 		    amdgpu_device_ip_check_soft_reset(adev)) {
4808 			amdgpu_device_ip_pre_soft_reset(adev);
4809 			r = amdgpu_device_ip_soft_reset(adev);
4810 			amdgpu_device_ip_post_soft_reset(adev);
4811 			if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4812 				dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4813 				need_full_reset = true;
4814 			}
4815 		}
4816 
4817 		if (need_full_reset)
4818 			r = amdgpu_device_ip_suspend(adev);
4819 		if (need_full_reset)
4820 			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4821 		else
4822 			clear_bit(AMDGPU_NEED_FULL_RESET,
4823 				  &reset_context->flags);
4824 	}
4825 
4826 	return r;
4827 }
4828 
4829 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
4830 {
4831 	int i;
4832 
4833 	lockdep_assert_held(&adev->reset_domain->sem);
4834 
4835 	for (i = 0; i < adev->num_regs; i++) {
4836 		adev->reset_dump_reg_value[i] = RREG32(adev->reset_dump_reg_list[i]);
4837 		trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i],
4838 					     adev->reset_dump_reg_value[i]);
4839 	}
4840 
4841 	return 0;
4842 }
4843 
4844 #ifdef CONFIG_DEV_COREDUMP
4845 static ssize_t amdgpu_devcoredump_read(char *buffer, loff_t offset,
4846 		size_t count, void *data, size_t datalen)
4847 {
4848 	struct drm_printer p;
4849 	struct amdgpu_device *adev = data;
4850 	struct drm_print_iterator iter;
4851 	int i;
4852 
4853 	iter.data = buffer;
4854 	iter.offset = 0;
4855 	iter.start = offset;
4856 	iter.remain = count;
4857 
4858 	p = drm_coredump_printer(&iter);
4859 
4860 	drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
4861 	drm_printf(&p, "kernel: " UTS_RELEASE "\n");
4862 	drm_printf(&p, "module: " KBUILD_MODNAME "\n");
4863 	drm_printf(&p, "time: %lld.%09ld\n", adev->reset_time.tv_sec, adev->reset_time.tv_nsec);
4864 	if (adev->reset_task_info.pid)
4865 		drm_printf(&p, "process_name: %s PID: %d\n",
4866 			   adev->reset_task_info.process_name,
4867 			   adev->reset_task_info.pid);
4868 
4869 	if (adev->reset_vram_lost)
4870 		drm_printf(&p, "VRAM is lost due to GPU reset!\n");
4871 	if (adev->num_regs) {
4872 		drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
4873 
4874 		for (i = 0; i < adev->num_regs; i++)
4875 			drm_printf(&p, "0x%08x: 0x%08x\n",
4876 				   adev->reset_dump_reg_list[i],
4877 				   adev->reset_dump_reg_value[i]);
4878 	}
4879 
4880 	return count - iter.remain;
4881 }
4882 
4883 static void amdgpu_devcoredump_free(void *data)
4884 {
4885 }
4886 
4887 static void amdgpu_reset_capture_coredumpm(struct amdgpu_device *adev)
4888 {
4889 	struct drm_device *dev = adev_to_drm(adev);
4890 
4891 	ktime_get_ts64(&adev->reset_time);
4892 	dev_coredumpm(dev->dev, THIS_MODULE, adev, 0, GFP_KERNEL,
4893 		      amdgpu_devcoredump_read, amdgpu_devcoredump_free);
4894 }
4895 #endif
4896 
4897 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4898 			 struct amdgpu_reset_context *reset_context)
4899 {
4900 	struct amdgpu_device *tmp_adev = NULL;
4901 	bool need_full_reset, skip_hw_reset, vram_lost = false;
4902 	int r = 0;
4903 	bool gpu_reset_for_dev_remove = 0;
4904 
4905 	/* Try reset handler method first */
4906 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4907 				    reset_list);
4908 	amdgpu_reset_reg_dumps(tmp_adev);
4909 
4910 	reset_context->reset_device_list = device_list_handle;
4911 	r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4912 	/* If reset handler not implemented, continue; otherwise return */
4913 	if (r == -ENOSYS)
4914 		r = 0;
4915 	else
4916 		return r;
4917 
4918 	/* Reset handler not implemented, use the default method */
4919 	need_full_reset =
4920 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4921 	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4922 
4923 	gpu_reset_for_dev_remove =
4924 		test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
4925 			test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4926 
4927 	/*
4928 	 * ASIC reset has to be done on all XGMI hive nodes ASAP
4929 	 * to allow proper links negotiation in FW (within 1 sec)
4930 	 */
4931 	if (!skip_hw_reset && need_full_reset) {
4932 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4933 			/* For XGMI run all resets in parallel to speed up the process */
4934 			if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4935 				tmp_adev->gmc.xgmi.pending_reset = false;
4936 				if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4937 					r = -EALREADY;
4938 			} else
4939 				r = amdgpu_asic_reset(tmp_adev);
4940 
4941 			if (r) {
4942 				dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4943 					 r, adev_to_drm(tmp_adev)->unique);
4944 				break;
4945 			}
4946 		}
4947 
4948 		/* For XGMI wait for all resets to complete before proceed */
4949 		if (!r) {
4950 			list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4951 				if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4952 					flush_work(&tmp_adev->xgmi_reset_work);
4953 					r = tmp_adev->asic_reset_res;
4954 					if (r)
4955 						break;
4956 				}
4957 			}
4958 		}
4959 	}
4960 
4961 	if (!r && amdgpu_ras_intr_triggered()) {
4962 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4963 			if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops &&
4964 			    tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
4965 				tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev);
4966 		}
4967 
4968 		amdgpu_ras_intr_cleared();
4969 	}
4970 
4971 	/* Since the mode1 reset affects base ip blocks, the
4972 	 * phase1 ip blocks need to be resumed. Otherwise there
4973 	 * will be a BIOS signature error and the psp bootloader
4974 	 * can't load kdb on the next amdgpu install.
4975 	 */
4976 	if (gpu_reset_for_dev_remove) {
4977 		list_for_each_entry(tmp_adev, device_list_handle, reset_list)
4978 			amdgpu_device_ip_resume_phase1(tmp_adev);
4979 
4980 		goto end;
4981 	}
4982 
4983 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4984 		if (need_full_reset) {
4985 			/* post card */
4986 			r = amdgpu_device_asic_init(tmp_adev);
4987 			if (r) {
4988 				dev_warn(tmp_adev->dev, "asic atom init failed!");
4989 			} else {
4990 				dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4991 				r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4992 				if (r)
4993 					goto out;
4994 
4995 				r = amdgpu_device_ip_resume_phase1(tmp_adev);
4996 				if (r)
4997 					goto out;
4998 
4999 				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5000 #ifdef CONFIG_DEV_COREDUMP
5001 				tmp_adev->reset_vram_lost = vram_lost;
5002 				memset(&tmp_adev->reset_task_info, 0,
5003 						sizeof(tmp_adev->reset_task_info));
5004 				if (reset_context->job && reset_context->job->vm)
5005 					tmp_adev->reset_task_info =
5006 						reset_context->job->vm->task_info;
5007 				amdgpu_reset_capture_coredumpm(tmp_adev);
5008 #endif
5009 				if (vram_lost) {
5010 					DRM_INFO("VRAM is lost due to GPU reset!\n");
5011 					amdgpu_inc_vram_lost(tmp_adev);
5012 				}
5013 
5014 				r = amdgpu_device_fw_loading(tmp_adev);
5015 				if (r)
5016 					return r;
5017 
5018 				r = amdgpu_device_ip_resume_phase2(tmp_adev);
5019 				if (r)
5020 					goto out;
5021 
5022 				if (vram_lost)
5023 					amdgpu_device_fill_reset_magic(tmp_adev);
5024 
5025 				/*
5026 				 * Add this ASIC as tracked as reset was already
5027 				 * complete successfully.
5028 				 */
5029 				amdgpu_register_gpu_instance(tmp_adev);
5030 
5031 				if (!reset_context->hive &&
5032 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5033 					amdgpu_xgmi_add_device(tmp_adev);
5034 
5035 				r = amdgpu_device_ip_late_init(tmp_adev);
5036 				if (r)
5037 					goto out;
5038 
5039 				drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
5040 
5041 				/*
5042 				 * The GPU enters bad state once faulty pages
5043 				 * by ECC has reached the threshold, and ras
5044 				 * recovery is scheduled next. So add one check
5045 				 * here to break recovery if it indeed exceeds
5046 				 * bad page threshold, and remind user to
5047 				 * retire this GPU or setting one bigger
5048 				 * bad_page_threshold value to fix this once
5049 				 * probing driver again.
5050 				 */
5051 				if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5052 					/* must succeed. */
5053 					amdgpu_ras_resume(tmp_adev);
5054 				} else {
5055 					r = -EINVAL;
5056 					goto out;
5057 				}
5058 
5059 				/* Update PSP FW topology after reset */
5060 				if (reset_context->hive &&
5061 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5062 					r = amdgpu_xgmi_update_topology(
5063 						reset_context->hive, tmp_adev);
5064 			}
5065 		}
5066 
5067 out:
5068 		if (!r) {
5069 			amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5070 			r = amdgpu_ib_ring_tests(tmp_adev);
5071 			if (r) {
5072 				dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5073 				need_full_reset = true;
5074 				r = -EAGAIN;
5075 				goto end;
5076 			}
5077 		}
5078 
5079 		if (!r)
5080 			r = amdgpu_device_recover_vram(tmp_adev);
5081 		else
5082 			tmp_adev->asic_reset_res = r;
5083 	}
5084 
5085 end:
5086 	if (need_full_reset)
5087 		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5088 	else
5089 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5090 	return r;
5091 }
5092 
5093 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5094 {
5095 
5096 	switch (amdgpu_asic_reset_method(adev)) {
5097 	case AMD_RESET_METHOD_MODE1:
5098 		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5099 		break;
5100 	case AMD_RESET_METHOD_MODE2:
5101 		adev->mp1_state = PP_MP1_STATE_RESET;
5102 		break;
5103 	default:
5104 		adev->mp1_state = PP_MP1_STATE_NONE;
5105 		break;
5106 	}
5107 }
5108 
5109 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5110 {
5111 	amdgpu_vf_error_trans_all(adev);
5112 	adev->mp1_state = PP_MP1_STATE_NONE;
5113 }
5114 
5115 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5116 {
5117 	struct pci_dev *p = NULL;
5118 
5119 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5120 			adev->pdev->bus->number, 1);
5121 	if (p) {
5122 		pm_runtime_enable(&(p->dev));
5123 		pm_runtime_resume(&(p->dev));
5124 	}
5125 
5126 	pci_dev_put(p);
5127 }
5128 
5129 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5130 {
5131 	enum amd_reset_method reset_method;
5132 	struct pci_dev *p = NULL;
5133 	u64 expires;
5134 
5135 	/*
5136 	 * For now, only BACO and mode1 reset are confirmed
5137 	 * to suffer the audio issue without proper suspended.
5138 	 */
5139 	reset_method = amdgpu_asic_reset_method(adev);
5140 	if ((reset_method != AMD_RESET_METHOD_BACO) &&
5141 	     (reset_method != AMD_RESET_METHOD_MODE1))
5142 		return -EINVAL;
5143 
5144 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5145 			adev->pdev->bus->number, 1);
5146 	if (!p)
5147 		return -ENODEV;
5148 
5149 	expires = pm_runtime_autosuspend_expiration(&(p->dev));
5150 	if (!expires)
5151 		/*
5152 		 * If we cannot get the audio device autosuspend delay,
5153 		 * a fixed 4S interval will be used. Considering 3S is
5154 		 * the audio controller default autosuspend delay setting.
5155 		 * 4S used here is guaranteed to cover that.
5156 		 */
5157 		expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5158 
5159 	while (!pm_runtime_status_suspended(&(p->dev))) {
5160 		if (!pm_runtime_suspend(&(p->dev)))
5161 			break;
5162 
5163 		if (expires < ktime_get_mono_fast_ns()) {
5164 			dev_warn(adev->dev, "failed to suspend display audio\n");
5165 			pci_dev_put(p);
5166 			/* TODO: abort the succeeding gpu reset? */
5167 			return -ETIMEDOUT;
5168 		}
5169 	}
5170 
5171 	pm_runtime_disable(&(p->dev));
5172 
5173 	pci_dev_put(p);
5174 	return 0;
5175 }
5176 
5177 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5178 {
5179 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5180 
5181 #if defined(CONFIG_DEBUG_FS)
5182 	if (!amdgpu_sriov_vf(adev))
5183 		cancel_work(&adev->reset_work);
5184 #endif
5185 
5186 	if (adev->kfd.dev)
5187 		cancel_work(&adev->kfd.reset_work);
5188 
5189 	if (amdgpu_sriov_vf(adev))
5190 		cancel_work(&adev->virt.flr_work);
5191 
5192 	if (con && adev->ras_enabled)
5193 		cancel_work(&con->recovery_work);
5194 
5195 }
5196 
5197 /**
5198  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5199  *
5200  * @adev: amdgpu_device pointer
5201  * @job: which job trigger hang
5202  * @reset_context: amdgpu reset context pointer
5203  *
5204  * Attempt to reset the GPU if it has hung (all asics).
5205  * Attempt to do soft-reset or full-reset and reinitialize Asic
5206  * Returns 0 for success or an error on failure.
5207  */
5208 
5209 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5210 			      struct amdgpu_job *job,
5211 			      struct amdgpu_reset_context *reset_context)
5212 {
5213 	struct list_head device_list, *device_list_handle =  NULL;
5214 	bool job_signaled = false;
5215 	struct amdgpu_hive_info *hive = NULL;
5216 	struct amdgpu_device *tmp_adev = NULL;
5217 	int i, r = 0;
5218 	bool need_emergency_restart = false;
5219 	bool audio_suspended = false;
5220 	bool gpu_reset_for_dev_remove = false;
5221 
5222 	gpu_reset_for_dev_remove =
5223 			test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5224 				test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5225 
5226 	/*
5227 	 * Special case: RAS triggered and full reset isn't supported
5228 	 */
5229 	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5230 
5231 	/*
5232 	 * Flush RAM to disk so that after reboot
5233 	 * the user can read log and see why the system rebooted.
5234 	 */
5235 	if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
5236 		DRM_WARN("Emergency reboot.");
5237 
5238 		ksys_sync_helper();
5239 		emergency_restart();
5240 	}
5241 
5242 	dev_info(adev->dev, "GPU %s begin!\n",
5243 		need_emergency_restart ? "jobs stop":"reset");
5244 
5245 	if (!amdgpu_sriov_vf(adev))
5246 		hive = amdgpu_get_xgmi_hive(adev);
5247 	if (hive)
5248 		mutex_lock(&hive->hive_lock);
5249 
5250 	reset_context->job = job;
5251 	reset_context->hive = hive;
5252 	/*
5253 	 * Build list of devices to reset.
5254 	 * In case we are in XGMI hive mode, resort the device list
5255 	 * to put adev in the 1st position.
5256 	 */
5257 	INIT_LIST_HEAD(&device_list);
5258 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5259 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5260 			list_add_tail(&tmp_adev->reset_list, &device_list);
5261 			if (gpu_reset_for_dev_remove && adev->shutdown)
5262 				tmp_adev->shutdown = true;
5263 		}
5264 		if (!list_is_first(&adev->reset_list, &device_list))
5265 			list_rotate_to_front(&adev->reset_list, &device_list);
5266 		device_list_handle = &device_list;
5267 	} else {
5268 		list_add_tail(&adev->reset_list, &device_list);
5269 		device_list_handle = &device_list;
5270 	}
5271 
5272 	/* We need to lock reset domain only once both for XGMI and single device */
5273 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5274 				    reset_list);
5275 	amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5276 
5277 	/* block all schedulers and reset given job's ring */
5278 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5279 
5280 		amdgpu_device_set_mp1_state(tmp_adev);
5281 
5282 		/*
5283 		 * Try to put the audio codec into suspend state
5284 		 * before gpu reset started.
5285 		 *
5286 		 * Due to the power domain of the graphics device
5287 		 * is shared with AZ power domain. Without this,
5288 		 * we may change the audio hardware from behind
5289 		 * the audio driver's back. That will trigger
5290 		 * some audio codec errors.
5291 		 */
5292 		if (!amdgpu_device_suspend_display_audio(tmp_adev))
5293 			audio_suspended = true;
5294 
5295 		amdgpu_ras_set_error_query_ready(tmp_adev, false);
5296 
5297 		cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5298 
5299 		if (!amdgpu_sriov_vf(tmp_adev))
5300 			amdgpu_amdkfd_pre_reset(tmp_adev);
5301 
5302 		/*
5303 		 * Mark these ASICs to be reseted as untracked first
5304 		 * And add them back after reset completed
5305 		 */
5306 		amdgpu_unregister_gpu_instance(tmp_adev);
5307 
5308 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5309 
5310 		/* disable ras on ALL IPs */
5311 		if (!need_emergency_restart &&
5312 		      amdgpu_device_ip_need_full_reset(tmp_adev))
5313 			amdgpu_ras_suspend(tmp_adev);
5314 
5315 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5316 			struct amdgpu_ring *ring = tmp_adev->rings[i];
5317 
5318 			if (!ring || !ring->sched.thread)
5319 				continue;
5320 
5321 			drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5322 
5323 			if (need_emergency_restart)
5324 				amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5325 		}
5326 		atomic_inc(&tmp_adev->gpu_reset_counter);
5327 	}
5328 
5329 	if (need_emergency_restart)
5330 		goto skip_sched_resume;
5331 
5332 	/*
5333 	 * Must check guilty signal here since after this point all old
5334 	 * HW fences are force signaled.
5335 	 *
5336 	 * job->base holds a reference to parent fence
5337 	 */
5338 	if (job && dma_fence_is_signaled(&job->hw_fence)) {
5339 		job_signaled = true;
5340 		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5341 		goto skip_hw_reset;
5342 	}
5343 
5344 retry:	/* Rest of adevs pre asic reset from XGMI hive. */
5345 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5346 		if (gpu_reset_for_dev_remove) {
5347 			/* Workaroud for ASICs need to disable SMC first */
5348 			amdgpu_device_smu_fini_early(tmp_adev);
5349 		}
5350 		r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5351 		/*TODO Should we stop ?*/
5352 		if (r) {
5353 			dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5354 				  r, adev_to_drm(tmp_adev)->unique);
5355 			tmp_adev->asic_reset_res = r;
5356 		}
5357 
5358 		/*
5359 		 * Drop all pending non scheduler resets. Scheduler resets
5360 		 * were already dropped during drm_sched_stop
5361 		 */
5362 		amdgpu_device_stop_pending_resets(tmp_adev);
5363 	}
5364 
5365 	/* Actual ASIC resets if needed.*/
5366 	/* Host driver will handle XGMI hive reset for SRIOV */
5367 	if (amdgpu_sriov_vf(adev)) {
5368 		r = amdgpu_device_reset_sriov(adev, job ? false : true);
5369 		if (r)
5370 			adev->asic_reset_res = r;
5371 
5372 		/* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5373 		if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2) ||
5374 		    adev->ip_versions[GC_HWIP][0] == IP_VERSION(11, 0, 3))
5375 			amdgpu_ras_resume(adev);
5376 	} else {
5377 		r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5378 		if (r && r == -EAGAIN)
5379 			goto retry;
5380 
5381 		if (!r && gpu_reset_for_dev_remove)
5382 			goto recover_end;
5383 	}
5384 
5385 skip_hw_reset:
5386 
5387 	/* Post ASIC reset for all devs .*/
5388 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5389 
5390 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5391 			struct amdgpu_ring *ring = tmp_adev->rings[i];
5392 
5393 			if (!ring || !ring->sched.thread)
5394 				continue;
5395 
5396 			drm_sched_start(&ring->sched, true);
5397 		}
5398 
5399 		if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3))
5400 			amdgpu_mes_self_test(tmp_adev);
5401 
5402 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5403 			drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5404 		}
5405 
5406 		if (tmp_adev->asic_reset_res)
5407 			r = tmp_adev->asic_reset_res;
5408 
5409 		tmp_adev->asic_reset_res = 0;
5410 
5411 		if (r) {
5412 			/* bad news, how to tell it to userspace ? */
5413 			dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5414 			amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5415 		} else {
5416 			dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5417 			if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5418 				DRM_WARN("smart shift update failed\n");
5419 		}
5420 	}
5421 
5422 skip_sched_resume:
5423 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5424 		/* unlock kfd: SRIOV would do it separately */
5425 		if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5426 			amdgpu_amdkfd_post_reset(tmp_adev);
5427 
5428 		/* kfd_post_reset will do nothing if kfd device is not initialized,
5429 		 * need to bring up kfd here if it's not be initialized before
5430 		 */
5431 		if (!adev->kfd.init_complete)
5432 			amdgpu_amdkfd_device_init(adev);
5433 
5434 		if (audio_suspended)
5435 			amdgpu_device_resume_display_audio(tmp_adev);
5436 
5437 		amdgpu_device_unset_mp1_state(tmp_adev);
5438 
5439 		amdgpu_ras_set_error_query_ready(tmp_adev, true);
5440 	}
5441 
5442 recover_end:
5443 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5444 					    reset_list);
5445 	amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5446 
5447 	if (hive) {
5448 		mutex_unlock(&hive->hive_lock);
5449 		amdgpu_put_xgmi_hive(hive);
5450 	}
5451 
5452 	if (r)
5453 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5454 
5455 	atomic_set(&adev->reset_domain->reset_res, r);
5456 	return r;
5457 }
5458 
5459 /**
5460  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5461  *
5462  * @adev: amdgpu_device pointer
5463  *
5464  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5465  * and lanes) of the slot the device is in. Handles APUs and
5466  * virtualized environments where PCIE config space may not be available.
5467  */
5468 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5469 {
5470 	struct pci_dev *pdev;
5471 	enum pci_bus_speed speed_cap, platform_speed_cap;
5472 	enum pcie_link_width platform_link_width;
5473 
5474 	if (amdgpu_pcie_gen_cap)
5475 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5476 
5477 	if (amdgpu_pcie_lane_cap)
5478 		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5479 
5480 	/* covers APUs as well */
5481 	if (pci_is_root_bus(adev->pdev->bus)) {
5482 		if (adev->pm.pcie_gen_mask == 0)
5483 			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5484 		if (adev->pm.pcie_mlw_mask == 0)
5485 			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5486 		return;
5487 	}
5488 
5489 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5490 		return;
5491 
5492 	pcie_bandwidth_available(adev->pdev, NULL,
5493 				 &platform_speed_cap, &platform_link_width);
5494 
5495 	if (adev->pm.pcie_gen_mask == 0) {
5496 		/* asic caps */
5497 		pdev = adev->pdev;
5498 		speed_cap = pcie_get_speed_cap(pdev);
5499 		if (speed_cap == PCI_SPEED_UNKNOWN) {
5500 			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5501 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5502 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5503 		} else {
5504 			if (speed_cap == PCIE_SPEED_32_0GT)
5505 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5506 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5507 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5508 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5509 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5510 			else if (speed_cap == PCIE_SPEED_16_0GT)
5511 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5512 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5513 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5514 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5515 			else if (speed_cap == PCIE_SPEED_8_0GT)
5516 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5517 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5518 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5519 			else if (speed_cap == PCIE_SPEED_5_0GT)
5520 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5521 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5522 			else
5523 				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5524 		}
5525 		/* platform caps */
5526 		if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5527 			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5528 						   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5529 		} else {
5530 			if (platform_speed_cap == PCIE_SPEED_32_0GT)
5531 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5532 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5533 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5534 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5535 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5536 			else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5537 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5538 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5539 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5540 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5541 			else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5542 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5543 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5544 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5545 			else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5546 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5547 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5548 			else
5549 				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5550 
5551 		}
5552 	}
5553 	if (adev->pm.pcie_mlw_mask == 0) {
5554 		if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5555 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5556 		} else {
5557 			switch (platform_link_width) {
5558 			case PCIE_LNK_X32:
5559 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5560 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5561 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5562 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5563 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5564 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5565 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5566 				break;
5567 			case PCIE_LNK_X16:
5568 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5569 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5570 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5571 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5572 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5573 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5574 				break;
5575 			case PCIE_LNK_X12:
5576 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5577 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5578 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5579 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5580 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5581 				break;
5582 			case PCIE_LNK_X8:
5583 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5584 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5585 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5586 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5587 				break;
5588 			case PCIE_LNK_X4:
5589 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5590 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5591 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5592 				break;
5593 			case PCIE_LNK_X2:
5594 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5595 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5596 				break;
5597 			case PCIE_LNK_X1:
5598 				adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5599 				break;
5600 			default:
5601 				break;
5602 			}
5603 		}
5604 	}
5605 }
5606 
5607 /**
5608  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5609  *
5610  * @adev: amdgpu_device pointer
5611  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5612  *
5613  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5614  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5615  * @peer_adev.
5616  */
5617 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5618 				      struct amdgpu_device *peer_adev)
5619 {
5620 #ifdef CONFIG_HSA_AMD_P2P
5621 	uint64_t address_mask = peer_adev->dev->dma_mask ?
5622 		~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5623 	resource_size_t aper_limit =
5624 		adev->gmc.aper_base + adev->gmc.aper_size - 1;
5625 	bool p2p_access =
5626 		!adev->gmc.xgmi.connected_to_cpu &&
5627 		!(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5628 
5629 	return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5630 		adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5631 		!(adev->gmc.aper_base & address_mask ||
5632 		  aper_limit & address_mask));
5633 #else
5634 	return false;
5635 #endif
5636 }
5637 
5638 int amdgpu_device_baco_enter(struct drm_device *dev)
5639 {
5640 	struct amdgpu_device *adev = drm_to_adev(dev);
5641 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5642 
5643 	if (!amdgpu_device_supports_baco(dev))
5644 		return -ENOTSUPP;
5645 
5646 	if (ras && adev->ras_enabled &&
5647 	    adev->nbio.funcs->enable_doorbell_interrupt)
5648 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5649 
5650 	return amdgpu_dpm_baco_enter(adev);
5651 }
5652 
5653 int amdgpu_device_baco_exit(struct drm_device *dev)
5654 {
5655 	struct amdgpu_device *adev = drm_to_adev(dev);
5656 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5657 	int ret = 0;
5658 
5659 	if (!amdgpu_device_supports_baco(dev))
5660 		return -ENOTSUPP;
5661 
5662 	ret = amdgpu_dpm_baco_exit(adev);
5663 	if (ret)
5664 		return ret;
5665 
5666 	if (ras && adev->ras_enabled &&
5667 	    adev->nbio.funcs->enable_doorbell_interrupt)
5668 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5669 
5670 	if (amdgpu_passthrough(adev) &&
5671 	    adev->nbio.funcs->clear_doorbell_interrupt)
5672 		adev->nbio.funcs->clear_doorbell_interrupt(adev);
5673 
5674 	return 0;
5675 }
5676 
5677 /**
5678  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5679  * @pdev: PCI device struct
5680  * @state: PCI channel state
5681  *
5682  * Description: Called when a PCI error is detected.
5683  *
5684  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5685  */
5686 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5687 {
5688 	struct drm_device *dev = pci_get_drvdata(pdev);
5689 	struct amdgpu_device *adev = drm_to_adev(dev);
5690 	int i;
5691 
5692 	DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5693 
5694 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
5695 		DRM_WARN("No support for XGMI hive yet...");
5696 		return PCI_ERS_RESULT_DISCONNECT;
5697 	}
5698 
5699 	adev->pci_channel_state = state;
5700 
5701 	switch (state) {
5702 	case pci_channel_io_normal:
5703 		return PCI_ERS_RESULT_CAN_RECOVER;
5704 	/* Fatal error, prepare for slot reset */
5705 	case pci_channel_io_frozen:
5706 		/*
5707 		 * Locking adev->reset_domain->sem will prevent any external access
5708 		 * to GPU during PCI error recovery
5709 		 */
5710 		amdgpu_device_lock_reset_domain(adev->reset_domain);
5711 		amdgpu_device_set_mp1_state(adev);
5712 
5713 		/*
5714 		 * Block any work scheduling as we do for regular GPU reset
5715 		 * for the duration of the recovery
5716 		 */
5717 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5718 			struct amdgpu_ring *ring = adev->rings[i];
5719 
5720 			if (!ring || !ring->sched.thread)
5721 				continue;
5722 
5723 			drm_sched_stop(&ring->sched, NULL);
5724 		}
5725 		atomic_inc(&adev->gpu_reset_counter);
5726 		return PCI_ERS_RESULT_NEED_RESET;
5727 	case pci_channel_io_perm_failure:
5728 		/* Permanent error, prepare for device removal */
5729 		return PCI_ERS_RESULT_DISCONNECT;
5730 	}
5731 
5732 	return PCI_ERS_RESULT_NEED_RESET;
5733 }
5734 
5735 /**
5736  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5737  * @pdev: pointer to PCI device
5738  */
5739 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5740 {
5741 
5742 	DRM_INFO("PCI error: mmio enabled callback!!\n");
5743 
5744 	/* TODO - dump whatever for debugging purposes */
5745 
5746 	/* This called only if amdgpu_pci_error_detected returns
5747 	 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5748 	 * works, no need to reset slot.
5749 	 */
5750 
5751 	return PCI_ERS_RESULT_RECOVERED;
5752 }
5753 
5754 /**
5755  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5756  * @pdev: PCI device struct
5757  *
5758  * Description: This routine is called by the pci error recovery
5759  * code after the PCI slot has been reset, just before we
5760  * should resume normal operations.
5761  */
5762 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5763 {
5764 	struct drm_device *dev = pci_get_drvdata(pdev);
5765 	struct amdgpu_device *adev = drm_to_adev(dev);
5766 	int r, i;
5767 	struct amdgpu_reset_context reset_context;
5768 	u32 memsize;
5769 	struct list_head device_list;
5770 
5771 	DRM_INFO("PCI error: slot reset callback!!\n");
5772 
5773 	memset(&reset_context, 0, sizeof(reset_context));
5774 
5775 	INIT_LIST_HEAD(&device_list);
5776 	list_add_tail(&adev->reset_list, &device_list);
5777 
5778 	/* wait for asic to come out of reset */
5779 	msleep(500);
5780 
5781 	/* Restore PCI confspace */
5782 	amdgpu_device_load_pci_state(pdev);
5783 
5784 	/* confirm  ASIC came out of reset */
5785 	for (i = 0; i < adev->usec_timeout; i++) {
5786 		memsize = amdgpu_asic_get_config_memsize(adev);
5787 
5788 		if (memsize != 0xffffffff)
5789 			break;
5790 		udelay(1);
5791 	}
5792 	if (memsize == 0xffffffff) {
5793 		r = -ETIME;
5794 		goto out;
5795 	}
5796 
5797 	reset_context.method = AMD_RESET_METHOD_NONE;
5798 	reset_context.reset_req_dev = adev;
5799 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5800 	set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5801 
5802 	adev->no_hw_access = true;
5803 	r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5804 	adev->no_hw_access = false;
5805 	if (r)
5806 		goto out;
5807 
5808 	r = amdgpu_do_asic_reset(&device_list, &reset_context);
5809 
5810 out:
5811 	if (!r) {
5812 		if (amdgpu_device_cache_pci_state(adev->pdev))
5813 			pci_restore_state(adev->pdev);
5814 
5815 		DRM_INFO("PCIe error recovery succeeded\n");
5816 	} else {
5817 		DRM_ERROR("PCIe error recovery failed, err:%d", r);
5818 		amdgpu_device_unset_mp1_state(adev);
5819 		amdgpu_device_unlock_reset_domain(adev->reset_domain);
5820 	}
5821 
5822 	return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5823 }
5824 
5825 /**
5826  * amdgpu_pci_resume() - resume normal ops after PCI reset
5827  * @pdev: pointer to PCI device
5828  *
5829  * Called when the error recovery driver tells us that its
5830  * OK to resume normal operation.
5831  */
5832 void amdgpu_pci_resume(struct pci_dev *pdev)
5833 {
5834 	struct drm_device *dev = pci_get_drvdata(pdev);
5835 	struct amdgpu_device *adev = drm_to_adev(dev);
5836 	int i;
5837 
5838 
5839 	DRM_INFO("PCI error: resume callback!!\n");
5840 
5841 	/* Only continue execution for the case of pci_channel_io_frozen */
5842 	if (adev->pci_channel_state != pci_channel_io_frozen)
5843 		return;
5844 
5845 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5846 		struct amdgpu_ring *ring = adev->rings[i];
5847 
5848 		if (!ring || !ring->sched.thread)
5849 			continue;
5850 
5851 		drm_sched_start(&ring->sched, true);
5852 	}
5853 
5854 	amdgpu_device_unset_mp1_state(adev);
5855 	amdgpu_device_unlock_reset_domain(adev->reset_domain);
5856 }
5857 
5858 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5859 {
5860 	struct drm_device *dev = pci_get_drvdata(pdev);
5861 	struct amdgpu_device *adev = drm_to_adev(dev);
5862 	int r;
5863 
5864 	r = pci_save_state(pdev);
5865 	if (!r) {
5866 		kfree(adev->pci_state);
5867 
5868 		adev->pci_state = pci_store_saved_state(pdev);
5869 
5870 		if (!adev->pci_state) {
5871 			DRM_ERROR("Failed to store PCI saved state");
5872 			return false;
5873 		}
5874 	} else {
5875 		DRM_WARN("Failed to save PCI state, err:%d\n", r);
5876 		return false;
5877 	}
5878 
5879 	return true;
5880 }
5881 
5882 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5883 {
5884 	struct drm_device *dev = pci_get_drvdata(pdev);
5885 	struct amdgpu_device *adev = drm_to_adev(dev);
5886 	int r;
5887 
5888 	if (!adev->pci_state)
5889 		return false;
5890 
5891 	r = pci_load_saved_state(pdev, adev->pci_state);
5892 
5893 	if (!r) {
5894 		pci_restore_state(pdev);
5895 	} else {
5896 		DRM_WARN("Failed to load PCI state, err:%d\n", r);
5897 		return false;
5898 	}
5899 
5900 	return true;
5901 }
5902 
5903 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5904 		struct amdgpu_ring *ring)
5905 {
5906 #ifdef CONFIG_X86_64
5907 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5908 		return;
5909 #endif
5910 	if (adev->gmc.xgmi.connected_to_cpu)
5911 		return;
5912 
5913 	if (ring && ring->funcs->emit_hdp_flush)
5914 		amdgpu_ring_emit_hdp_flush(ring);
5915 	else
5916 		amdgpu_asic_flush_hdp(adev, ring);
5917 }
5918 
5919 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5920 		struct amdgpu_ring *ring)
5921 {
5922 #ifdef CONFIG_X86_64
5923 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5924 		return;
5925 #endif
5926 	if (adev->gmc.xgmi.connected_to_cpu)
5927 		return;
5928 
5929 	amdgpu_asic_invalidate_hdp(adev, ring);
5930 }
5931 
5932 int amdgpu_in_reset(struct amdgpu_device *adev)
5933 {
5934 	return atomic_read(&adev->reset_domain->in_gpu_reset);
5935 }
5936 
5937 /**
5938  * amdgpu_device_halt() - bring hardware to some kind of halt state
5939  *
5940  * @adev: amdgpu_device pointer
5941  *
5942  * Bring hardware to some kind of halt state so that no one can touch it
5943  * any more. It will help to maintain error context when error occurred.
5944  * Compare to a simple hang, the system will keep stable at least for SSH
5945  * access. Then it should be trivial to inspect the hardware state and
5946  * see what's going on. Implemented as following:
5947  *
5948  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
5949  *    clears all CPU mappings to device, disallows remappings through page faults
5950  * 2. amdgpu_irq_disable_all() disables all interrupts
5951  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
5952  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
5953  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
5954  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
5955  *    flush any in flight DMA operations
5956  */
5957 void amdgpu_device_halt(struct amdgpu_device *adev)
5958 {
5959 	struct pci_dev *pdev = adev->pdev;
5960 	struct drm_device *ddev = adev_to_drm(adev);
5961 
5962 	drm_dev_unplug(ddev);
5963 
5964 	amdgpu_irq_disable_all(adev);
5965 
5966 	amdgpu_fence_driver_hw_fini(adev);
5967 
5968 	adev->no_hw_access = true;
5969 
5970 	amdgpu_device_unmap_mmio(adev);
5971 
5972 	pci_disable_device(pdev);
5973 	pci_wait_for_pending_transaction(pdev);
5974 }
5975 
5976 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
5977 				u32 reg)
5978 {
5979 	unsigned long flags, address, data;
5980 	u32 r;
5981 
5982 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5983 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5984 
5985 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5986 	WREG32(address, reg * 4);
5987 	(void)RREG32(address);
5988 	r = RREG32(data);
5989 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5990 	return r;
5991 }
5992 
5993 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
5994 				u32 reg, u32 v)
5995 {
5996 	unsigned long flags, address, data;
5997 
5998 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5999 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6000 
6001 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6002 	WREG32(address, reg * 4);
6003 	(void)RREG32(address);
6004 	WREG32(data, v);
6005 	(void)RREG32(data);
6006 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6007 }
6008 
6009 /**
6010  * amdgpu_device_switch_gang - switch to a new gang
6011  * @adev: amdgpu_device pointer
6012  * @gang: the gang to switch to
6013  *
6014  * Try to switch to a new gang.
6015  * Returns: NULL if we switched to the new gang or a reference to the current
6016  * gang leader.
6017  */
6018 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6019 					    struct dma_fence *gang)
6020 {
6021 	struct dma_fence *old = NULL;
6022 
6023 	do {
6024 		dma_fence_put(old);
6025 		rcu_read_lock();
6026 		old = dma_fence_get_rcu_safe(&adev->gang_submit);
6027 		rcu_read_unlock();
6028 
6029 		if (old == gang)
6030 			break;
6031 
6032 		if (!dma_fence_is_signaled(old))
6033 			return old;
6034 
6035 	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6036 			 old, gang) != old);
6037 
6038 	dma_fence_put(old);
6039 	return NULL;
6040 }
6041 
6042 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6043 {
6044 	switch (adev->asic_type) {
6045 #ifdef CONFIG_DRM_AMDGPU_SI
6046 	case CHIP_HAINAN:
6047 #endif
6048 	case CHIP_TOPAZ:
6049 		/* chips with no display hardware */
6050 		return false;
6051 #ifdef CONFIG_DRM_AMDGPU_SI
6052 	case CHIP_TAHITI:
6053 	case CHIP_PITCAIRN:
6054 	case CHIP_VERDE:
6055 	case CHIP_OLAND:
6056 #endif
6057 #ifdef CONFIG_DRM_AMDGPU_CIK
6058 	case CHIP_BONAIRE:
6059 	case CHIP_HAWAII:
6060 	case CHIP_KAVERI:
6061 	case CHIP_KABINI:
6062 	case CHIP_MULLINS:
6063 #endif
6064 	case CHIP_TONGA:
6065 	case CHIP_FIJI:
6066 	case CHIP_POLARIS10:
6067 	case CHIP_POLARIS11:
6068 	case CHIP_POLARIS12:
6069 	case CHIP_VEGAM:
6070 	case CHIP_CARRIZO:
6071 	case CHIP_STONEY:
6072 		/* chips with display hardware */
6073 		return true;
6074 	default:
6075 		/* IP discovery */
6076 		if (!adev->ip_versions[DCE_HWIP][0] ||
6077 		    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6078 			return false;
6079 		return true;
6080 	}
6081 }
6082