xref: /openbmc/linux/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c (revision 69f685e0c125a9811b5489c2ece21708878fa6f9)
1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  * Authors: Dave Airlie
25  *          Alex Deucher
26  *          Jerome Glisse
27  */
28 #include <linux/power_supply.h>
29 #include <linux/kthread.h>
30 #include <linux/module.h>
31 #include <linux/console.h>
32 #include <linux/slab.h>
33 #include <linux/iommu.h>
34 #include <linux/pci.h>
35 #include <linux/devcoredump.h>
36 #include <generated/utsrelease.h>
37 #include <linux/pci-p2pdma.h>
38 
39 #include <drm/drm_atomic_helper.h>
40 #include <drm/drm_fb_helper.h>
41 #include <drm/drm_probe_helper.h>
42 #include <drm/amdgpu_drm.h>
43 #include <linux/vgaarb.h>
44 #include <linux/vga_switcheroo.h>
45 #include <linux/efi.h>
46 #include "amdgpu.h"
47 #include "amdgpu_trace.h"
48 #include "amdgpu_i2c.h"
49 #include "atom.h"
50 #include "amdgpu_atombios.h"
51 #include "amdgpu_atomfirmware.h"
52 #include "amd_pcie.h"
53 #ifdef CONFIG_DRM_AMDGPU_SI
54 #include "si.h"
55 #endif
56 #ifdef CONFIG_DRM_AMDGPU_CIK
57 #include "cik.h"
58 #endif
59 #include "vi.h"
60 #include "soc15.h"
61 #include "nv.h"
62 #include "bif/bif_4_1_d.h"
63 #include <linux/firmware.h>
64 #include "amdgpu_vf_error.h"
65 
66 #include "amdgpu_amdkfd.h"
67 #include "amdgpu_pm.h"
68 
69 #include "amdgpu_xgmi.h"
70 #include "amdgpu_ras.h"
71 #include "amdgpu_pmu.h"
72 #include "amdgpu_fru_eeprom.h"
73 #include "amdgpu_reset.h"
74 
75 #include <linux/suspend.h>
76 #include <drm/task_barrier.h>
77 #include <linux/pm_runtime.h>
78 
79 #include <drm/drm_drv.h>
80 
81 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
82 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
83 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
84 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
85 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
86 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
87 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
88 
89 #define AMDGPU_RESUME_MS		2000
90 #define AMDGPU_MAX_RETRY_LIMIT		2
91 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
92 
93 const char *amdgpu_asic_name[] = {
94 	"TAHITI",
95 	"PITCAIRN",
96 	"VERDE",
97 	"OLAND",
98 	"HAINAN",
99 	"BONAIRE",
100 	"KAVERI",
101 	"KABINI",
102 	"HAWAII",
103 	"MULLINS",
104 	"TOPAZ",
105 	"TONGA",
106 	"FIJI",
107 	"CARRIZO",
108 	"STONEY",
109 	"POLARIS10",
110 	"POLARIS11",
111 	"POLARIS12",
112 	"VEGAM",
113 	"VEGA10",
114 	"VEGA12",
115 	"VEGA20",
116 	"RAVEN",
117 	"ARCTURUS",
118 	"RENOIR",
119 	"ALDEBARAN",
120 	"NAVI10",
121 	"CYAN_SKILLFISH",
122 	"NAVI14",
123 	"NAVI12",
124 	"SIENNA_CICHLID",
125 	"NAVY_FLOUNDER",
126 	"VANGOGH",
127 	"DIMGREY_CAVEFISH",
128 	"BEIGE_GOBY",
129 	"YELLOW_CARP",
130 	"IP DISCOVERY",
131 	"LAST",
132 };
133 
134 /**
135  * DOC: pcie_replay_count
136  *
137  * The amdgpu driver provides a sysfs API for reporting the total number
138  * of PCIe replays (NAKs)
139  * The file pcie_replay_count is used for this and returns the total
140  * number of replays as a sum of the NAKs generated and NAKs received
141  */
142 
143 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
144 		struct device_attribute *attr, char *buf)
145 {
146 	struct drm_device *ddev = dev_get_drvdata(dev);
147 	struct amdgpu_device *adev = drm_to_adev(ddev);
148 	uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
149 
150 	return sysfs_emit(buf, "%llu\n", cnt);
151 }
152 
153 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
154 		amdgpu_device_get_pcie_replay_count, NULL);
155 
156 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
157 
158 /**
159  * DOC: product_name
160  *
161  * The amdgpu driver provides a sysfs API for reporting the product name
162  * for the device
163  * The file serial_number is used for this and returns the product name
164  * as returned from the FRU.
165  * NOTE: This is only available for certain server cards
166  */
167 
168 static ssize_t amdgpu_device_get_product_name(struct device *dev,
169 		struct device_attribute *attr, char *buf)
170 {
171 	struct drm_device *ddev = dev_get_drvdata(dev);
172 	struct amdgpu_device *adev = drm_to_adev(ddev);
173 
174 	return sysfs_emit(buf, "%s\n", adev->product_name);
175 }
176 
177 static DEVICE_ATTR(product_name, S_IRUGO,
178 		amdgpu_device_get_product_name, NULL);
179 
180 /**
181  * DOC: product_number
182  *
183  * The amdgpu driver provides a sysfs API for reporting the part number
184  * for the device
185  * The file serial_number is used for this and returns the part number
186  * as returned from the FRU.
187  * NOTE: This is only available for certain server cards
188  */
189 
190 static ssize_t amdgpu_device_get_product_number(struct device *dev,
191 		struct device_attribute *attr, char *buf)
192 {
193 	struct drm_device *ddev = dev_get_drvdata(dev);
194 	struct amdgpu_device *adev = drm_to_adev(ddev);
195 
196 	return sysfs_emit(buf, "%s\n", adev->product_number);
197 }
198 
199 static DEVICE_ATTR(product_number, S_IRUGO,
200 		amdgpu_device_get_product_number, NULL);
201 
202 /**
203  * DOC: serial_number
204  *
205  * The amdgpu driver provides a sysfs API for reporting the serial number
206  * for the device
207  * The file serial_number is used for this and returns the serial number
208  * as returned from the FRU.
209  * NOTE: This is only available for certain server cards
210  */
211 
212 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
213 		struct device_attribute *attr, char *buf)
214 {
215 	struct drm_device *ddev = dev_get_drvdata(dev);
216 	struct amdgpu_device *adev = drm_to_adev(ddev);
217 
218 	return sysfs_emit(buf, "%s\n", adev->serial);
219 }
220 
221 static DEVICE_ATTR(serial_number, S_IRUGO,
222 		amdgpu_device_get_serial_number, NULL);
223 
224 /**
225  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
226  *
227  * @dev: drm_device pointer
228  *
229  * Returns true if the device is a dGPU with ATPX power control,
230  * otherwise return false.
231  */
232 bool amdgpu_device_supports_px(struct drm_device *dev)
233 {
234 	struct amdgpu_device *adev = drm_to_adev(dev);
235 
236 	if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
237 		return true;
238 	return false;
239 }
240 
241 /**
242  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
243  *
244  * @dev: drm_device pointer
245  *
246  * Returns true if the device is a dGPU with ACPI power control,
247  * otherwise return false.
248  */
249 bool amdgpu_device_supports_boco(struct drm_device *dev)
250 {
251 	struct amdgpu_device *adev = drm_to_adev(dev);
252 
253 	if (adev->has_pr3 ||
254 	    ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
255 		return true;
256 	return false;
257 }
258 
259 /**
260  * amdgpu_device_supports_baco - Does the device support BACO
261  *
262  * @dev: drm_device pointer
263  *
264  * Returns true if the device supporte BACO,
265  * otherwise return false.
266  */
267 bool amdgpu_device_supports_baco(struct drm_device *dev)
268 {
269 	struct amdgpu_device *adev = drm_to_adev(dev);
270 
271 	return amdgpu_asic_supports_baco(adev);
272 }
273 
274 /**
275  * amdgpu_device_supports_smart_shift - Is the device dGPU with
276  * smart shift support
277  *
278  * @dev: drm_device pointer
279  *
280  * Returns true if the device is a dGPU with Smart Shift support,
281  * otherwise returns false.
282  */
283 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
284 {
285 	return (amdgpu_device_supports_boco(dev) &&
286 		amdgpu_acpi_is_power_shift_control_supported());
287 }
288 
289 /*
290  * VRAM access helper functions
291  */
292 
293 /**
294  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
295  *
296  * @adev: amdgpu_device pointer
297  * @pos: offset of the buffer in vram
298  * @buf: virtual address of the buffer in system memory
299  * @size: read/write size, sizeof(@buf) must > @size
300  * @write: true - write to vram, otherwise - read from vram
301  */
302 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
303 			     void *buf, size_t size, bool write)
304 {
305 	unsigned long flags;
306 	uint32_t hi = ~0, tmp = 0;
307 	uint32_t *data = buf;
308 	uint64_t last;
309 	int idx;
310 
311 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
312 		return;
313 
314 	BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
315 
316 	spin_lock_irqsave(&adev->mmio_idx_lock, flags);
317 	for (last = pos + size; pos < last; pos += 4) {
318 		tmp = pos >> 31;
319 
320 		WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
321 		if (tmp != hi) {
322 			WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
323 			hi = tmp;
324 		}
325 		if (write)
326 			WREG32_NO_KIQ(mmMM_DATA, *data++);
327 		else
328 			*data++ = RREG32_NO_KIQ(mmMM_DATA);
329 	}
330 
331 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
332 	drm_dev_exit(idx);
333 }
334 
335 /**
336  * amdgpu_device_aper_access - access vram by vram aperature
337  *
338  * @adev: amdgpu_device pointer
339  * @pos: offset of the buffer in vram
340  * @buf: virtual address of the buffer in system memory
341  * @size: read/write size, sizeof(@buf) must > @size
342  * @write: true - write to vram, otherwise - read from vram
343  *
344  * The return value means how many bytes have been transferred.
345  */
346 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
347 				 void *buf, size_t size, bool write)
348 {
349 #ifdef CONFIG_64BIT
350 	void __iomem *addr;
351 	size_t count = 0;
352 	uint64_t last;
353 
354 	if (!adev->mman.aper_base_kaddr)
355 		return 0;
356 
357 	last = min(pos + size, adev->gmc.visible_vram_size);
358 	if (last > pos) {
359 		addr = adev->mman.aper_base_kaddr + pos;
360 		count = last - pos;
361 
362 		if (write) {
363 			memcpy_toio(addr, buf, count);
364 			mb();
365 			amdgpu_device_flush_hdp(adev, NULL);
366 		} else {
367 			amdgpu_device_invalidate_hdp(adev, NULL);
368 			mb();
369 			memcpy_fromio(buf, addr, count);
370 		}
371 
372 	}
373 
374 	return count;
375 #else
376 	return 0;
377 #endif
378 }
379 
380 /**
381  * amdgpu_device_vram_access - read/write a buffer in vram
382  *
383  * @adev: amdgpu_device pointer
384  * @pos: offset of the buffer in vram
385  * @buf: virtual address of the buffer in system memory
386  * @size: read/write size, sizeof(@buf) must > @size
387  * @write: true - write to vram, otherwise - read from vram
388  */
389 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
390 			       void *buf, size_t size, bool write)
391 {
392 	size_t count;
393 
394 	/* try to using vram apreature to access vram first */
395 	count = amdgpu_device_aper_access(adev, pos, buf, size, write);
396 	size -= count;
397 	if (size) {
398 		/* using MM to access rest vram */
399 		pos += count;
400 		buf += count;
401 		amdgpu_device_mm_access(adev, pos, buf, size, write);
402 	}
403 }
404 
405 /*
406  * register access helper functions.
407  */
408 
409 /* Check if hw access should be skipped because of hotplug or device error */
410 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
411 {
412 	if (adev->no_hw_access)
413 		return true;
414 
415 #ifdef CONFIG_LOCKDEP
416 	/*
417 	 * This is a bit complicated to understand, so worth a comment. What we assert
418 	 * here is that the GPU reset is not running on another thread in parallel.
419 	 *
420 	 * For this we trylock the read side of the reset semaphore, if that succeeds
421 	 * we know that the reset is not running in paralell.
422 	 *
423 	 * If the trylock fails we assert that we are either already holding the read
424 	 * side of the lock or are the reset thread itself and hold the write side of
425 	 * the lock.
426 	 */
427 	if (in_task()) {
428 		if (down_read_trylock(&adev->reset_domain->sem))
429 			up_read(&adev->reset_domain->sem);
430 		else
431 			lockdep_assert_held(&adev->reset_domain->sem);
432 	}
433 #endif
434 	return false;
435 }
436 
437 /**
438  * amdgpu_device_rreg - read a memory mapped IO or indirect register
439  *
440  * @adev: amdgpu_device pointer
441  * @reg: dword aligned register offset
442  * @acc_flags: access flags which require special behavior
443  *
444  * Returns the 32 bit value from the offset specified.
445  */
446 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
447 			    uint32_t reg, uint32_t acc_flags)
448 {
449 	uint32_t ret;
450 
451 	if (amdgpu_device_skip_hw_access(adev))
452 		return 0;
453 
454 	if ((reg * 4) < adev->rmmio_size) {
455 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
456 		    amdgpu_sriov_runtime(adev) &&
457 		    down_read_trylock(&adev->reset_domain->sem)) {
458 			ret = amdgpu_kiq_rreg(adev, reg);
459 			up_read(&adev->reset_domain->sem);
460 		} else {
461 			ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
462 		}
463 	} else {
464 		ret = adev->pcie_rreg(adev, reg * 4);
465 	}
466 
467 	trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
468 
469 	return ret;
470 }
471 
472 /*
473  * MMIO register read with bytes helper functions
474  * @offset:bytes offset from MMIO start
475  *
476 */
477 
478 /**
479  * amdgpu_mm_rreg8 - read a memory mapped IO register
480  *
481  * @adev: amdgpu_device pointer
482  * @offset: byte aligned register offset
483  *
484  * Returns the 8 bit value from the offset specified.
485  */
486 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
487 {
488 	if (amdgpu_device_skip_hw_access(adev))
489 		return 0;
490 
491 	if (offset < adev->rmmio_size)
492 		return (readb(adev->rmmio + offset));
493 	BUG();
494 }
495 
496 /*
497  * MMIO register write with bytes helper functions
498  * @offset:bytes offset from MMIO start
499  * @value: the value want to be written to the register
500  *
501 */
502 /**
503  * amdgpu_mm_wreg8 - read a memory mapped IO register
504  *
505  * @adev: amdgpu_device pointer
506  * @offset: byte aligned register offset
507  * @value: 8 bit value to write
508  *
509  * Writes the value specified to the offset specified.
510  */
511 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
512 {
513 	if (amdgpu_device_skip_hw_access(adev))
514 		return;
515 
516 	if (offset < adev->rmmio_size)
517 		writeb(value, adev->rmmio + offset);
518 	else
519 		BUG();
520 }
521 
522 /**
523  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
524  *
525  * @adev: amdgpu_device pointer
526  * @reg: dword aligned register offset
527  * @v: 32 bit value to write to the register
528  * @acc_flags: access flags which require special behavior
529  *
530  * Writes the value specified to the offset specified.
531  */
532 void amdgpu_device_wreg(struct amdgpu_device *adev,
533 			uint32_t reg, uint32_t v,
534 			uint32_t acc_flags)
535 {
536 	if (amdgpu_device_skip_hw_access(adev))
537 		return;
538 
539 	if ((reg * 4) < adev->rmmio_size) {
540 		if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
541 		    amdgpu_sriov_runtime(adev) &&
542 		    down_read_trylock(&adev->reset_domain->sem)) {
543 			amdgpu_kiq_wreg(adev, reg, v);
544 			up_read(&adev->reset_domain->sem);
545 		} else {
546 			writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
547 		}
548 	} else {
549 		adev->pcie_wreg(adev, reg * 4, v);
550 	}
551 
552 	trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
553 }
554 
555 /**
556  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
557  *
558  * @adev: amdgpu_device pointer
559  * @reg: mmio/rlc register
560  * @v: value to write
561  *
562  * this function is invoked only for the debugfs register access
563  */
564 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
565 			     uint32_t reg, uint32_t v)
566 {
567 	if (amdgpu_device_skip_hw_access(adev))
568 		return;
569 
570 	if (amdgpu_sriov_fullaccess(adev) &&
571 	    adev->gfx.rlc.funcs &&
572 	    adev->gfx.rlc.funcs->is_rlcg_access_range) {
573 		if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
574 			return amdgpu_sriov_wreg(adev, reg, v, 0, 0);
575 	} else if ((reg * 4) >= adev->rmmio_size) {
576 		adev->pcie_wreg(adev, reg * 4, v);
577 	} else {
578 		writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
579 	}
580 }
581 
582 /**
583  * amdgpu_mm_rdoorbell - read a doorbell dword
584  *
585  * @adev: amdgpu_device pointer
586  * @index: doorbell index
587  *
588  * Returns the value in the doorbell aperture at the
589  * requested doorbell index (CIK).
590  */
591 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
592 {
593 	if (amdgpu_device_skip_hw_access(adev))
594 		return 0;
595 
596 	if (index < adev->doorbell.num_doorbells) {
597 		return readl(adev->doorbell.ptr + index);
598 	} else {
599 		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
600 		return 0;
601 	}
602 }
603 
604 /**
605  * amdgpu_mm_wdoorbell - write a doorbell dword
606  *
607  * @adev: amdgpu_device pointer
608  * @index: doorbell index
609  * @v: value to write
610  *
611  * Writes @v to the doorbell aperture at the
612  * requested doorbell index (CIK).
613  */
614 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
615 {
616 	if (amdgpu_device_skip_hw_access(adev))
617 		return;
618 
619 	if (index < adev->doorbell.num_doorbells) {
620 		writel(v, adev->doorbell.ptr + index);
621 	} else {
622 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
623 	}
624 }
625 
626 /**
627  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
628  *
629  * @adev: amdgpu_device pointer
630  * @index: doorbell index
631  *
632  * Returns the value in the doorbell aperture at the
633  * requested doorbell index (VEGA10+).
634  */
635 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
636 {
637 	if (amdgpu_device_skip_hw_access(adev))
638 		return 0;
639 
640 	if (index < adev->doorbell.num_doorbells) {
641 		return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
642 	} else {
643 		DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
644 		return 0;
645 	}
646 }
647 
648 /**
649  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
650  *
651  * @adev: amdgpu_device pointer
652  * @index: doorbell index
653  * @v: value to write
654  *
655  * Writes @v to the doorbell aperture at the
656  * requested doorbell index (VEGA10+).
657  */
658 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
659 {
660 	if (amdgpu_device_skip_hw_access(adev))
661 		return;
662 
663 	if (index < adev->doorbell.num_doorbells) {
664 		atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
665 	} else {
666 		DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
667 	}
668 }
669 
670 /**
671  * amdgpu_device_indirect_rreg - read an indirect register
672  *
673  * @adev: amdgpu_device pointer
674  * @pcie_index: mmio register offset
675  * @pcie_data: mmio register offset
676  * @reg_addr: indirect register address to read from
677  *
678  * Returns the value of indirect register @reg_addr
679  */
680 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
681 				u32 pcie_index, u32 pcie_data,
682 				u32 reg_addr)
683 {
684 	unsigned long flags;
685 	u32 r;
686 	void __iomem *pcie_index_offset;
687 	void __iomem *pcie_data_offset;
688 
689 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
690 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
691 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
692 
693 	writel(reg_addr, pcie_index_offset);
694 	readl(pcie_index_offset);
695 	r = readl(pcie_data_offset);
696 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
697 
698 	return r;
699 }
700 
701 /**
702  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
703  *
704  * @adev: amdgpu_device pointer
705  * @pcie_index: mmio register offset
706  * @pcie_data: mmio register offset
707  * @reg_addr: indirect register address to read from
708  *
709  * Returns the value of indirect register @reg_addr
710  */
711 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
712 				  u32 pcie_index, u32 pcie_data,
713 				  u32 reg_addr)
714 {
715 	unsigned long flags;
716 	u64 r;
717 	void __iomem *pcie_index_offset;
718 	void __iomem *pcie_data_offset;
719 
720 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
721 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
722 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
723 
724 	/* read low 32 bits */
725 	writel(reg_addr, pcie_index_offset);
726 	readl(pcie_index_offset);
727 	r = readl(pcie_data_offset);
728 	/* read high 32 bits */
729 	writel(reg_addr + 4, pcie_index_offset);
730 	readl(pcie_index_offset);
731 	r |= ((u64)readl(pcie_data_offset) << 32);
732 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
733 
734 	return r;
735 }
736 
737 /**
738  * amdgpu_device_indirect_wreg - write an indirect register address
739  *
740  * @adev: amdgpu_device pointer
741  * @pcie_index: mmio register offset
742  * @pcie_data: mmio register offset
743  * @reg_addr: indirect register offset
744  * @reg_data: indirect register data
745  *
746  */
747 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
748 				 u32 pcie_index, u32 pcie_data,
749 				 u32 reg_addr, u32 reg_data)
750 {
751 	unsigned long flags;
752 	void __iomem *pcie_index_offset;
753 	void __iomem *pcie_data_offset;
754 
755 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
756 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
757 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
758 
759 	writel(reg_addr, pcie_index_offset);
760 	readl(pcie_index_offset);
761 	writel(reg_data, pcie_data_offset);
762 	readl(pcie_data_offset);
763 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
764 }
765 
766 /**
767  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
768  *
769  * @adev: amdgpu_device pointer
770  * @pcie_index: mmio register offset
771  * @pcie_data: mmio register offset
772  * @reg_addr: indirect register offset
773  * @reg_data: indirect register data
774  *
775  */
776 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
777 				   u32 pcie_index, u32 pcie_data,
778 				   u32 reg_addr, u64 reg_data)
779 {
780 	unsigned long flags;
781 	void __iomem *pcie_index_offset;
782 	void __iomem *pcie_data_offset;
783 
784 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
785 	pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
786 	pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
787 
788 	/* write low 32 bits */
789 	writel(reg_addr, pcie_index_offset);
790 	readl(pcie_index_offset);
791 	writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
792 	readl(pcie_data_offset);
793 	/* write high 32 bits */
794 	writel(reg_addr + 4, pcie_index_offset);
795 	readl(pcie_index_offset);
796 	writel((u32)(reg_data >> 32), pcie_data_offset);
797 	readl(pcie_data_offset);
798 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
799 }
800 
801 /**
802  * amdgpu_invalid_rreg - dummy reg read function
803  *
804  * @adev: amdgpu_device pointer
805  * @reg: offset of register
806  *
807  * Dummy register read function.  Used for register blocks
808  * that certain asics don't have (all asics).
809  * Returns the value in the register.
810  */
811 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
812 {
813 	DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
814 	BUG();
815 	return 0;
816 }
817 
818 /**
819  * amdgpu_invalid_wreg - dummy reg write function
820  *
821  * @adev: amdgpu_device pointer
822  * @reg: offset of register
823  * @v: value to write to the register
824  *
825  * Dummy register read function.  Used for register blocks
826  * that certain asics don't have (all asics).
827  */
828 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
829 {
830 	DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
831 		  reg, v);
832 	BUG();
833 }
834 
835 /**
836  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
837  *
838  * @adev: amdgpu_device pointer
839  * @reg: offset of register
840  *
841  * Dummy register read function.  Used for register blocks
842  * that certain asics don't have (all asics).
843  * Returns the value in the register.
844  */
845 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
846 {
847 	DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
848 	BUG();
849 	return 0;
850 }
851 
852 /**
853  * amdgpu_invalid_wreg64 - dummy reg write function
854  *
855  * @adev: amdgpu_device pointer
856  * @reg: offset of register
857  * @v: value to write to the register
858  *
859  * Dummy register read function.  Used for register blocks
860  * that certain asics don't have (all asics).
861  */
862 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
863 {
864 	DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
865 		  reg, v);
866 	BUG();
867 }
868 
869 /**
870  * amdgpu_block_invalid_rreg - dummy reg read function
871  *
872  * @adev: amdgpu_device pointer
873  * @block: offset of instance
874  * @reg: offset of register
875  *
876  * Dummy register read function.  Used for register blocks
877  * that certain asics don't have (all asics).
878  * Returns the value in the register.
879  */
880 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
881 					  uint32_t block, uint32_t reg)
882 {
883 	DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
884 		  reg, block);
885 	BUG();
886 	return 0;
887 }
888 
889 /**
890  * amdgpu_block_invalid_wreg - dummy reg write function
891  *
892  * @adev: amdgpu_device pointer
893  * @block: offset of instance
894  * @reg: offset of register
895  * @v: value to write to the register
896  *
897  * Dummy register read function.  Used for register blocks
898  * that certain asics don't have (all asics).
899  */
900 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
901 				      uint32_t block,
902 				      uint32_t reg, uint32_t v)
903 {
904 	DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
905 		  reg, block, v);
906 	BUG();
907 }
908 
909 /**
910  * amdgpu_device_asic_init - Wrapper for atom asic_init
911  *
912  * @adev: amdgpu_device pointer
913  *
914  * Does any asic specific work and then calls atom asic init.
915  */
916 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
917 {
918 	amdgpu_asic_pre_asic_init(adev);
919 
920 	if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0))
921 		return amdgpu_atomfirmware_asic_init(adev, true);
922 	else
923 		return amdgpu_atom_asic_init(adev->mode_info.atom_context);
924 }
925 
926 /**
927  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
928  *
929  * @adev: amdgpu_device pointer
930  *
931  * Allocates a scratch page of VRAM for use by various things in the
932  * driver.
933  */
934 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
935 {
936 	return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
937 				       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
938 				       &adev->vram_scratch.robj,
939 				       &adev->vram_scratch.gpu_addr,
940 				       (void **)&adev->vram_scratch.ptr);
941 }
942 
943 /**
944  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
945  *
946  * @adev: amdgpu_device pointer
947  *
948  * Frees the VRAM scratch page.
949  */
950 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
951 {
952 	amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
953 }
954 
955 /**
956  * amdgpu_device_program_register_sequence - program an array of registers.
957  *
958  * @adev: amdgpu_device pointer
959  * @registers: pointer to the register array
960  * @array_size: size of the register array
961  *
962  * Programs an array or registers with and and or masks.
963  * This is a helper for setting golden registers.
964  */
965 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
966 					     const u32 *registers,
967 					     const u32 array_size)
968 {
969 	u32 tmp, reg, and_mask, or_mask;
970 	int i;
971 
972 	if (array_size % 3)
973 		return;
974 
975 	for (i = 0; i < array_size; i +=3) {
976 		reg = registers[i + 0];
977 		and_mask = registers[i + 1];
978 		or_mask = registers[i + 2];
979 
980 		if (and_mask == 0xffffffff) {
981 			tmp = or_mask;
982 		} else {
983 			tmp = RREG32(reg);
984 			tmp &= ~and_mask;
985 			if (adev->family >= AMDGPU_FAMILY_AI)
986 				tmp |= (or_mask & and_mask);
987 			else
988 				tmp |= or_mask;
989 		}
990 		WREG32(reg, tmp);
991 	}
992 }
993 
994 /**
995  * amdgpu_device_pci_config_reset - reset the GPU
996  *
997  * @adev: amdgpu_device pointer
998  *
999  * Resets the GPU using the pci config reset sequence.
1000  * Only applicable to asics prior to vega10.
1001  */
1002 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1003 {
1004 	pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1005 }
1006 
1007 /**
1008  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1009  *
1010  * @adev: amdgpu_device pointer
1011  *
1012  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1013  */
1014 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1015 {
1016 	return pci_reset_function(adev->pdev);
1017 }
1018 
1019 /*
1020  * GPU doorbell aperture helpers function.
1021  */
1022 /**
1023  * amdgpu_device_doorbell_init - Init doorbell driver information.
1024  *
1025  * @adev: amdgpu_device pointer
1026  *
1027  * Init doorbell driver information (CIK)
1028  * Returns 0 on success, error on failure.
1029  */
1030 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1031 {
1032 
1033 	/* No doorbell on SI hardware generation */
1034 	if (adev->asic_type < CHIP_BONAIRE) {
1035 		adev->doorbell.base = 0;
1036 		adev->doorbell.size = 0;
1037 		adev->doorbell.num_doorbells = 0;
1038 		adev->doorbell.ptr = NULL;
1039 		return 0;
1040 	}
1041 
1042 	if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1043 		return -EINVAL;
1044 
1045 	amdgpu_asic_init_doorbell_index(adev);
1046 
1047 	/* doorbell bar mapping */
1048 	adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1049 	adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1050 
1051 	if (adev->enable_mes) {
1052 		adev->doorbell.num_doorbells =
1053 			adev->doorbell.size / sizeof(u32);
1054 	} else {
1055 		adev->doorbell.num_doorbells =
1056 			min_t(u32, adev->doorbell.size / sizeof(u32),
1057 			      adev->doorbell_index.max_assignment+1);
1058 		if (adev->doorbell.num_doorbells == 0)
1059 			return -EINVAL;
1060 
1061 		/* For Vega, reserve and map two pages on doorbell BAR since SDMA
1062 		 * paging queue doorbell use the second page. The
1063 		 * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1064 		 * doorbells are in the first page. So with paging queue enabled,
1065 		 * the max num_doorbells should + 1 page (0x400 in dword)
1066 		 */
1067 		if (adev->asic_type >= CHIP_VEGA10)
1068 			adev->doorbell.num_doorbells += 0x400;
1069 	}
1070 
1071 	adev->doorbell.ptr = ioremap(adev->doorbell.base,
1072 				     adev->doorbell.num_doorbells *
1073 				     sizeof(u32));
1074 	if (adev->doorbell.ptr == NULL)
1075 		return -ENOMEM;
1076 
1077 	return 0;
1078 }
1079 
1080 /**
1081  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1082  *
1083  * @adev: amdgpu_device pointer
1084  *
1085  * Tear down doorbell driver information (CIK)
1086  */
1087 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1088 {
1089 	iounmap(adev->doorbell.ptr);
1090 	adev->doorbell.ptr = NULL;
1091 }
1092 
1093 
1094 
1095 /*
1096  * amdgpu_device_wb_*()
1097  * Writeback is the method by which the GPU updates special pages in memory
1098  * with the status of certain GPU events (fences, ring pointers,etc.).
1099  */
1100 
1101 /**
1102  * amdgpu_device_wb_fini - Disable Writeback and free memory
1103  *
1104  * @adev: amdgpu_device pointer
1105  *
1106  * Disables Writeback and frees the Writeback memory (all asics).
1107  * Used at driver shutdown.
1108  */
1109 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1110 {
1111 	if (adev->wb.wb_obj) {
1112 		amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1113 				      &adev->wb.gpu_addr,
1114 				      (void **)&adev->wb.wb);
1115 		adev->wb.wb_obj = NULL;
1116 	}
1117 }
1118 
1119 /**
1120  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1121  *
1122  * @adev: amdgpu_device pointer
1123  *
1124  * Initializes writeback and allocates writeback memory (all asics).
1125  * Used at driver startup.
1126  * Returns 0 on success or an -error on failure.
1127  */
1128 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1129 {
1130 	int r;
1131 
1132 	if (adev->wb.wb_obj == NULL) {
1133 		/* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1134 		r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1135 					    PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1136 					    &adev->wb.wb_obj, &adev->wb.gpu_addr,
1137 					    (void **)&adev->wb.wb);
1138 		if (r) {
1139 			dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1140 			return r;
1141 		}
1142 
1143 		adev->wb.num_wb = AMDGPU_MAX_WB;
1144 		memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1145 
1146 		/* clear wb memory */
1147 		memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1148 	}
1149 
1150 	return 0;
1151 }
1152 
1153 /**
1154  * amdgpu_device_wb_get - Allocate a wb entry
1155  *
1156  * @adev: amdgpu_device pointer
1157  * @wb: wb index
1158  *
1159  * Allocate a wb slot for use by the driver (all asics).
1160  * Returns 0 on success or -EINVAL on failure.
1161  */
1162 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1163 {
1164 	unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1165 
1166 	if (offset < adev->wb.num_wb) {
1167 		__set_bit(offset, adev->wb.used);
1168 		*wb = offset << 3; /* convert to dw offset */
1169 		return 0;
1170 	} else {
1171 		return -EINVAL;
1172 	}
1173 }
1174 
1175 /**
1176  * amdgpu_device_wb_free - Free a wb entry
1177  *
1178  * @adev: amdgpu_device pointer
1179  * @wb: wb index
1180  *
1181  * Free a wb slot allocated for use by the driver (all asics)
1182  */
1183 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1184 {
1185 	wb >>= 3;
1186 	if (wb < adev->wb.num_wb)
1187 		__clear_bit(wb, adev->wb.used);
1188 }
1189 
1190 /**
1191  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1192  *
1193  * @adev: amdgpu_device pointer
1194  *
1195  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1196  * to fail, but if any of the BARs is not accessible after the size we abort
1197  * driver loading by returning -ENODEV.
1198  */
1199 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1200 {
1201 	int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1202 	struct pci_bus *root;
1203 	struct resource *res;
1204 	unsigned i;
1205 	u16 cmd;
1206 	int r;
1207 
1208 	/* Bypass for VF */
1209 	if (amdgpu_sriov_vf(adev))
1210 		return 0;
1211 
1212 	/* skip if the bios has already enabled large BAR */
1213 	if (adev->gmc.real_vram_size &&
1214 	    (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1215 		return 0;
1216 
1217 	/* Check if the root BUS has 64bit memory resources */
1218 	root = adev->pdev->bus;
1219 	while (root->parent)
1220 		root = root->parent;
1221 
1222 	pci_bus_for_each_resource(root, res, i) {
1223 		if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1224 		    res->start > 0x100000000ull)
1225 			break;
1226 	}
1227 
1228 	/* Trying to resize is pointless without a root hub window above 4GB */
1229 	if (!res)
1230 		return 0;
1231 
1232 	/* Limit the BAR size to what is available */
1233 	rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1234 			rbar_size);
1235 
1236 	/* Disable memory decoding while we change the BAR addresses and size */
1237 	pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1238 	pci_write_config_word(adev->pdev, PCI_COMMAND,
1239 			      cmd & ~PCI_COMMAND_MEMORY);
1240 
1241 	/* Free the VRAM and doorbell BAR, we most likely need to move both. */
1242 	amdgpu_device_doorbell_fini(adev);
1243 	if (adev->asic_type >= CHIP_BONAIRE)
1244 		pci_release_resource(adev->pdev, 2);
1245 
1246 	pci_release_resource(adev->pdev, 0);
1247 
1248 	r = pci_resize_resource(adev->pdev, 0, rbar_size);
1249 	if (r == -ENOSPC)
1250 		DRM_INFO("Not enough PCI address space for a large BAR.");
1251 	else if (r && r != -ENOTSUPP)
1252 		DRM_ERROR("Problem resizing BAR0 (%d).", r);
1253 
1254 	pci_assign_unassigned_bus_resources(adev->pdev->bus);
1255 
1256 	/* When the doorbell or fb BAR isn't available we have no chance of
1257 	 * using the device.
1258 	 */
1259 	r = amdgpu_device_doorbell_init(adev);
1260 	if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1261 		return -ENODEV;
1262 
1263 	pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1264 
1265 	return 0;
1266 }
1267 
1268 /*
1269  * GPU helpers function.
1270  */
1271 /**
1272  * amdgpu_device_need_post - check if the hw need post or not
1273  *
1274  * @adev: amdgpu_device pointer
1275  *
1276  * Check if the asic has been initialized (all asics) at driver startup
1277  * or post is needed if  hw reset is performed.
1278  * Returns true if need or false if not.
1279  */
1280 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1281 {
1282 	uint32_t reg;
1283 
1284 	if (amdgpu_sriov_vf(adev))
1285 		return false;
1286 
1287 	if (amdgpu_passthrough(adev)) {
1288 		/* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1289 		 * some old smc fw still need driver do vPost otherwise gpu hang, while
1290 		 * those smc fw version above 22.15 doesn't have this flaw, so we force
1291 		 * vpost executed for smc version below 22.15
1292 		 */
1293 		if (adev->asic_type == CHIP_FIJI) {
1294 			int err;
1295 			uint32_t fw_ver;
1296 			err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1297 			/* force vPost if error occured */
1298 			if (err)
1299 				return true;
1300 
1301 			fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1302 			if (fw_ver < 0x00160e00)
1303 				return true;
1304 		}
1305 	}
1306 
1307 	/* Don't post if we need to reset whole hive on init */
1308 	if (adev->gmc.xgmi.pending_reset)
1309 		return false;
1310 
1311 	if (adev->has_hw_reset) {
1312 		adev->has_hw_reset = false;
1313 		return true;
1314 	}
1315 
1316 	/* bios scratch used on CIK+ */
1317 	if (adev->asic_type >= CHIP_BONAIRE)
1318 		return amdgpu_atombios_scratch_need_asic_init(adev);
1319 
1320 	/* check MEM_SIZE for older asics */
1321 	reg = amdgpu_asic_get_config_memsize(adev);
1322 
1323 	if ((reg != 0) && (reg != 0xffffffff))
1324 		return false;
1325 
1326 	return true;
1327 }
1328 
1329 /**
1330  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1331  *
1332  * @adev: amdgpu_device pointer
1333  *
1334  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1335  * be set for this device.
1336  *
1337  * Returns true if it should be used or false if not.
1338  */
1339 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1340 {
1341 	switch (amdgpu_aspm) {
1342 	case -1:
1343 		break;
1344 	case 0:
1345 		return false;
1346 	case 1:
1347 		return true;
1348 	default:
1349 		return false;
1350 	}
1351 	return pcie_aspm_enabled(adev->pdev);
1352 }
1353 
1354 /* if we get transitioned to only one device, take VGA back */
1355 /**
1356  * amdgpu_device_vga_set_decode - enable/disable vga decode
1357  *
1358  * @pdev: PCI device pointer
1359  * @state: enable/disable vga decode
1360  *
1361  * Enable/disable vga decode (all asics).
1362  * Returns VGA resource flags.
1363  */
1364 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1365 		bool state)
1366 {
1367 	struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1368 	amdgpu_asic_set_vga_state(adev, state);
1369 	if (state)
1370 		return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1371 		       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1372 	else
1373 		return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1374 }
1375 
1376 /**
1377  * amdgpu_device_check_block_size - validate the vm block size
1378  *
1379  * @adev: amdgpu_device pointer
1380  *
1381  * Validates the vm block size specified via module parameter.
1382  * The vm block size defines number of bits in page table versus page directory,
1383  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1384  * page table and the remaining bits are in the page directory.
1385  */
1386 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1387 {
1388 	/* defines number of bits in page table versus page directory,
1389 	 * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1390 	 * page table and the remaining bits are in the page directory */
1391 	if (amdgpu_vm_block_size == -1)
1392 		return;
1393 
1394 	if (amdgpu_vm_block_size < 9) {
1395 		dev_warn(adev->dev, "VM page table size (%d) too small\n",
1396 			 amdgpu_vm_block_size);
1397 		amdgpu_vm_block_size = -1;
1398 	}
1399 }
1400 
1401 /**
1402  * amdgpu_device_check_vm_size - validate the vm size
1403  *
1404  * @adev: amdgpu_device pointer
1405  *
1406  * Validates the vm size in GB specified via module parameter.
1407  * The VM size is the size of the GPU virtual memory space in GB.
1408  */
1409 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1410 {
1411 	/* no need to check the default value */
1412 	if (amdgpu_vm_size == -1)
1413 		return;
1414 
1415 	if (amdgpu_vm_size < 1) {
1416 		dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1417 			 amdgpu_vm_size);
1418 		amdgpu_vm_size = -1;
1419 	}
1420 }
1421 
1422 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1423 {
1424 	struct sysinfo si;
1425 	bool is_os_64 = (sizeof(void *) == 8);
1426 	uint64_t total_memory;
1427 	uint64_t dram_size_seven_GB = 0x1B8000000;
1428 	uint64_t dram_size_three_GB = 0xB8000000;
1429 
1430 	if (amdgpu_smu_memory_pool_size == 0)
1431 		return;
1432 
1433 	if (!is_os_64) {
1434 		DRM_WARN("Not 64-bit OS, feature not supported\n");
1435 		goto def_value;
1436 	}
1437 	si_meminfo(&si);
1438 	total_memory = (uint64_t)si.totalram * si.mem_unit;
1439 
1440 	if ((amdgpu_smu_memory_pool_size == 1) ||
1441 		(amdgpu_smu_memory_pool_size == 2)) {
1442 		if (total_memory < dram_size_three_GB)
1443 			goto def_value1;
1444 	} else if ((amdgpu_smu_memory_pool_size == 4) ||
1445 		(amdgpu_smu_memory_pool_size == 8)) {
1446 		if (total_memory < dram_size_seven_GB)
1447 			goto def_value1;
1448 	} else {
1449 		DRM_WARN("Smu memory pool size not supported\n");
1450 		goto def_value;
1451 	}
1452 	adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1453 
1454 	return;
1455 
1456 def_value1:
1457 	DRM_WARN("No enough system memory\n");
1458 def_value:
1459 	adev->pm.smu_prv_buffer_size = 0;
1460 }
1461 
1462 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1463 {
1464 	if (!(adev->flags & AMD_IS_APU) ||
1465 	    adev->asic_type < CHIP_RAVEN)
1466 		return 0;
1467 
1468 	switch (adev->asic_type) {
1469 	case CHIP_RAVEN:
1470 		if (adev->pdev->device == 0x15dd)
1471 			adev->apu_flags |= AMD_APU_IS_RAVEN;
1472 		if (adev->pdev->device == 0x15d8)
1473 			adev->apu_flags |= AMD_APU_IS_PICASSO;
1474 		break;
1475 	case CHIP_RENOIR:
1476 		if ((adev->pdev->device == 0x1636) ||
1477 		    (adev->pdev->device == 0x164c))
1478 			adev->apu_flags |= AMD_APU_IS_RENOIR;
1479 		else
1480 			adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1481 		break;
1482 	case CHIP_VANGOGH:
1483 		adev->apu_flags |= AMD_APU_IS_VANGOGH;
1484 		break;
1485 	case CHIP_YELLOW_CARP:
1486 		break;
1487 	case CHIP_CYAN_SKILLFISH:
1488 		if ((adev->pdev->device == 0x13FE) ||
1489 		    (adev->pdev->device == 0x143F))
1490 			adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1491 		break;
1492 	default:
1493 		break;
1494 	}
1495 
1496 	return 0;
1497 }
1498 
1499 /**
1500  * amdgpu_device_check_arguments - validate module params
1501  *
1502  * @adev: amdgpu_device pointer
1503  *
1504  * Validates certain module parameters and updates
1505  * the associated values used by the driver (all asics).
1506  */
1507 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1508 {
1509 	if (amdgpu_sched_jobs < 4) {
1510 		dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1511 			 amdgpu_sched_jobs);
1512 		amdgpu_sched_jobs = 4;
1513 	} else if (!is_power_of_2(amdgpu_sched_jobs)){
1514 		dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1515 			 amdgpu_sched_jobs);
1516 		amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1517 	}
1518 
1519 	if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1520 		/* gart size must be greater or equal to 32M */
1521 		dev_warn(adev->dev, "gart size (%d) too small\n",
1522 			 amdgpu_gart_size);
1523 		amdgpu_gart_size = -1;
1524 	}
1525 
1526 	if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1527 		/* gtt size must be greater or equal to 32M */
1528 		dev_warn(adev->dev, "gtt size (%d) too small\n",
1529 				 amdgpu_gtt_size);
1530 		amdgpu_gtt_size = -1;
1531 	}
1532 
1533 	/* valid range is between 4 and 9 inclusive */
1534 	if (amdgpu_vm_fragment_size != -1 &&
1535 	    (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1536 		dev_warn(adev->dev, "valid range is between 4 and 9\n");
1537 		amdgpu_vm_fragment_size = -1;
1538 	}
1539 
1540 	if (amdgpu_sched_hw_submission < 2) {
1541 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1542 			 amdgpu_sched_hw_submission);
1543 		amdgpu_sched_hw_submission = 2;
1544 	} else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1545 		dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1546 			 amdgpu_sched_hw_submission);
1547 		amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1548 	}
1549 
1550 	if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1551 		dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1552 		amdgpu_reset_method = -1;
1553 	}
1554 
1555 	amdgpu_device_check_smu_prv_buffer_size(adev);
1556 
1557 	amdgpu_device_check_vm_size(adev);
1558 
1559 	amdgpu_device_check_block_size(adev);
1560 
1561 	adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1562 
1563 	return 0;
1564 }
1565 
1566 /**
1567  * amdgpu_switcheroo_set_state - set switcheroo state
1568  *
1569  * @pdev: pci dev pointer
1570  * @state: vga_switcheroo state
1571  *
1572  * Callback for the switcheroo driver.  Suspends or resumes
1573  * the asics before or after it is powered up using ACPI methods.
1574  */
1575 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1576 					enum vga_switcheroo_state state)
1577 {
1578 	struct drm_device *dev = pci_get_drvdata(pdev);
1579 	int r;
1580 
1581 	if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1582 		return;
1583 
1584 	if (state == VGA_SWITCHEROO_ON) {
1585 		pr_info("switched on\n");
1586 		/* don't suspend or resume card normally */
1587 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1588 
1589 		pci_set_power_state(pdev, PCI_D0);
1590 		amdgpu_device_load_pci_state(pdev);
1591 		r = pci_enable_device(pdev);
1592 		if (r)
1593 			DRM_WARN("pci_enable_device failed (%d)\n", r);
1594 		amdgpu_device_resume(dev, true);
1595 
1596 		dev->switch_power_state = DRM_SWITCH_POWER_ON;
1597 	} else {
1598 		pr_info("switched off\n");
1599 		dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1600 		amdgpu_device_suspend(dev, true);
1601 		amdgpu_device_cache_pci_state(pdev);
1602 		/* Shut down the device */
1603 		pci_disable_device(pdev);
1604 		pci_set_power_state(pdev, PCI_D3cold);
1605 		dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1606 	}
1607 }
1608 
1609 /**
1610  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1611  *
1612  * @pdev: pci dev pointer
1613  *
1614  * Callback for the switcheroo driver.  Check of the switcheroo
1615  * state can be changed.
1616  * Returns true if the state can be changed, false if not.
1617  */
1618 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1619 {
1620 	struct drm_device *dev = pci_get_drvdata(pdev);
1621 
1622 	/*
1623 	* FIXME: open_count is protected by drm_global_mutex but that would lead to
1624 	* locking inversion with the driver load path. And the access here is
1625 	* completely racy anyway. So don't bother with locking for now.
1626 	*/
1627 	return atomic_read(&dev->open_count) == 0;
1628 }
1629 
1630 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1631 	.set_gpu_state = amdgpu_switcheroo_set_state,
1632 	.reprobe = NULL,
1633 	.can_switch = amdgpu_switcheroo_can_switch,
1634 };
1635 
1636 /**
1637  * amdgpu_device_ip_set_clockgating_state - set the CG state
1638  *
1639  * @dev: amdgpu_device pointer
1640  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1641  * @state: clockgating state (gate or ungate)
1642  *
1643  * Sets the requested clockgating state for all instances of
1644  * the hardware IP specified.
1645  * Returns the error code from the last instance.
1646  */
1647 int amdgpu_device_ip_set_clockgating_state(void *dev,
1648 					   enum amd_ip_block_type block_type,
1649 					   enum amd_clockgating_state state)
1650 {
1651 	struct amdgpu_device *adev = dev;
1652 	int i, r = 0;
1653 
1654 	for (i = 0; i < adev->num_ip_blocks; i++) {
1655 		if (!adev->ip_blocks[i].status.valid)
1656 			continue;
1657 		if (adev->ip_blocks[i].version->type != block_type)
1658 			continue;
1659 		if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1660 			continue;
1661 		r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1662 			(void *)adev, state);
1663 		if (r)
1664 			DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1665 				  adev->ip_blocks[i].version->funcs->name, r);
1666 	}
1667 	return r;
1668 }
1669 
1670 /**
1671  * amdgpu_device_ip_set_powergating_state - set the PG state
1672  *
1673  * @dev: amdgpu_device pointer
1674  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1675  * @state: powergating state (gate or ungate)
1676  *
1677  * Sets the requested powergating state for all instances of
1678  * the hardware IP specified.
1679  * Returns the error code from the last instance.
1680  */
1681 int amdgpu_device_ip_set_powergating_state(void *dev,
1682 					   enum amd_ip_block_type block_type,
1683 					   enum amd_powergating_state state)
1684 {
1685 	struct amdgpu_device *adev = dev;
1686 	int i, r = 0;
1687 
1688 	for (i = 0; i < adev->num_ip_blocks; i++) {
1689 		if (!adev->ip_blocks[i].status.valid)
1690 			continue;
1691 		if (adev->ip_blocks[i].version->type != block_type)
1692 			continue;
1693 		if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1694 			continue;
1695 		r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1696 			(void *)adev, state);
1697 		if (r)
1698 			DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1699 				  adev->ip_blocks[i].version->funcs->name, r);
1700 	}
1701 	return r;
1702 }
1703 
1704 /**
1705  * amdgpu_device_ip_get_clockgating_state - get the CG state
1706  *
1707  * @adev: amdgpu_device pointer
1708  * @flags: clockgating feature flags
1709  *
1710  * Walks the list of IPs on the device and updates the clockgating
1711  * flags for each IP.
1712  * Updates @flags with the feature flags for each hardware IP where
1713  * clockgating is enabled.
1714  */
1715 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1716 					    u64 *flags)
1717 {
1718 	int i;
1719 
1720 	for (i = 0; i < adev->num_ip_blocks; i++) {
1721 		if (!adev->ip_blocks[i].status.valid)
1722 			continue;
1723 		if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1724 			adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1725 	}
1726 }
1727 
1728 /**
1729  * amdgpu_device_ip_wait_for_idle - wait for idle
1730  *
1731  * @adev: amdgpu_device pointer
1732  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1733  *
1734  * Waits for the request hardware IP to be idle.
1735  * Returns 0 for success or a negative error code on failure.
1736  */
1737 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1738 				   enum amd_ip_block_type block_type)
1739 {
1740 	int i, r;
1741 
1742 	for (i = 0; i < adev->num_ip_blocks; i++) {
1743 		if (!adev->ip_blocks[i].status.valid)
1744 			continue;
1745 		if (adev->ip_blocks[i].version->type == block_type) {
1746 			r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1747 			if (r)
1748 				return r;
1749 			break;
1750 		}
1751 	}
1752 	return 0;
1753 
1754 }
1755 
1756 /**
1757  * amdgpu_device_ip_is_idle - is the hardware IP idle
1758  *
1759  * @adev: amdgpu_device pointer
1760  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1761  *
1762  * Check if the hardware IP is idle or not.
1763  * Returns true if it the IP is idle, false if not.
1764  */
1765 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1766 			      enum amd_ip_block_type block_type)
1767 {
1768 	int i;
1769 
1770 	for (i = 0; i < adev->num_ip_blocks; i++) {
1771 		if (!adev->ip_blocks[i].status.valid)
1772 			continue;
1773 		if (adev->ip_blocks[i].version->type == block_type)
1774 			return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1775 	}
1776 	return true;
1777 
1778 }
1779 
1780 /**
1781  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1782  *
1783  * @adev: amdgpu_device pointer
1784  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1785  *
1786  * Returns a pointer to the hardware IP block structure
1787  * if it exists for the asic, otherwise NULL.
1788  */
1789 struct amdgpu_ip_block *
1790 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1791 			      enum amd_ip_block_type type)
1792 {
1793 	int i;
1794 
1795 	for (i = 0; i < adev->num_ip_blocks; i++)
1796 		if (adev->ip_blocks[i].version->type == type)
1797 			return &adev->ip_blocks[i];
1798 
1799 	return NULL;
1800 }
1801 
1802 /**
1803  * amdgpu_device_ip_block_version_cmp
1804  *
1805  * @adev: amdgpu_device pointer
1806  * @type: enum amd_ip_block_type
1807  * @major: major version
1808  * @minor: minor version
1809  *
1810  * return 0 if equal or greater
1811  * return 1 if smaller or the ip_block doesn't exist
1812  */
1813 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1814 				       enum amd_ip_block_type type,
1815 				       u32 major, u32 minor)
1816 {
1817 	struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1818 
1819 	if (ip_block && ((ip_block->version->major > major) ||
1820 			((ip_block->version->major == major) &&
1821 			(ip_block->version->minor >= minor))))
1822 		return 0;
1823 
1824 	return 1;
1825 }
1826 
1827 /**
1828  * amdgpu_device_ip_block_add
1829  *
1830  * @adev: amdgpu_device pointer
1831  * @ip_block_version: pointer to the IP to add
1832  *
1833  * Adds the IP block driver information to the collection of IPs
1834  * on the asic.
1835  */
1836 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1837 			       const struct amdgpu_ip_block_version *ip_block_version)
1838 {
1839 	if (!ip_block_version)
1840 		return -EINVAL;
1841 
1842 	switch (ip_block_version->type) {
1843 	case AMD_IP_BLOCK_TYPE_VCN:
1844 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1845 			return 0;
1846 		break;
1847 	case AMD_IP_BLOCK_TYPE_JPEG:
1848 		if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1849 			return 0;
1850 		break;
1851 	default:
1852 		break;
1853 	}
1854 
1855 	DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1856 		  ip_block_version->funcs->name);
1857 
1858 	adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1859 
1860 	return 0;
1861 }
1862 
1863 /**
1864  * amdgpu_device_enable_virtual_display - enable virtual display feature
1865  *
1866  * @adev: amdgpu_device pointer
1867  *
1868  * Enabled the virtual display feature if the user has enabled it via
1869  * the module parameter virtual_display.  This feature provides a virtual
1870  * display hardware on headless boards or in virtualized environments.
1871  * This function parses and validates the configuration string specified by
1872  * the user and configues the virtual display configuration (number of
1873  * virtual connectors, crtcs, etc.) specified.
1874  */
1875 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1876 {
1877 	adev->enable_virtual_display = false;
1878 
1879 	if (amdgpu_virtual_display) {
1880 		const char *pci_address_name = pci_name(adev->pdev);
1881 		char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1882 
1883 		pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1884 		pciaddstr_tmp = pciaddstr;
1885 		while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1886 			pciaddname = strsep(&pciaddname_tmp, ",");
1887 			if (!strcmp("all", pciaddname)
1888 			    || !strcmp(pci_address_name, pciaddname)) {
1889 				long num_crtc;
1890 				int res = -1;
1891 
1892 				adev->enable_virtual_display = true;
1893 
1894 				if (pciaddname_tmp)
1895 					res = kstrtol(pciaddname_tmp, 10,
1896 						      &num_crtc);
1897 
1898 				if (!res) {
1899 					if (num_crtc < 1)
1900 						num_crtc = 1;
1901 					if (num_crtc > 6)
1902 						num_crtc = 6;
1903 					adev->mode_info.num_crtc = num_crtc;
1904 				} else {
1905 					adev->mode_info.num_crtc = 1;
1906 				}
1907 				break;
1908 			}
1909 		}
1910 
1911 		DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1912 			 amdgpu_virtual_display, pci_address_name,
1913 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
1914 
1915 		kfree(pciaddstr);
1916 	}
1917 }
1918 
1919 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
1920 {
1921 	if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
1922 		adev->mode_info.num_crtc = 1;
1923 		adev->enable_virtual_display = true;
1924 		DRM_INFO("virtual_display:%d, num_crtc:%d\n",
1925 			 adev->enable_virtual_display, adev->mode_info.num_crtc);
1926 	}
1927 }
1928 
1929 /**
1930  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1931  *
1932  * @adev: amdgpu_device pointer
1933  *
1934  * Parses the asic configuration parameters specified in the gpu info
1935  * firmware and makes them availale to the driver for use in configuring
1936  * the asic.
1937  * Returns 0 on success, -EINVAL on failure.
1938  */
1939 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1940 {
1941 	const char *chip_name;
1942 	char fw_name[40];
1943 	int err;
1944 	const struct gpu_info_firmware_header_v1_0 *hdr;
1945 
1946 	adev->firmware.gpu_info_fw = NULL;
1947 
1948 	if (adev->mman.discovery_bin) {
1949 		/*
1950 		 * FIXME: The bounding box is still needed by Navi12, so
1951 		 * temporarily read it from gpu_info firmware. Should be dropped
1952 		 * when DAL no longer needs it.
1953 		 */
1954 		if (adev->asic_type != CHIP_NAVI12)
1955 			return 0;
1956 	}
1957 
1958 	switch (adev->asic_type) {
1959 	default:
1960 		return 0;
1961 	case CHIP_VEGA10:
1962 		chip_name = "vega10";
1963 		break;
1964 	case CHIP_VEGA12:
1965 		chip_name = "vega12";
1966 		break;
1967 	case CHIP_RAVEN:
1968 		if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1969 			chip_name = "raven2";
1970 		else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1971 			chip_name = "picasso";
1972 		else
1973 			chip_name = "raven";
1974 		break;
1975 	case CHIP_ARCTURUS:
1976 		chip_name = "arcturus";
1977 		break;
1978 	case CHIP_NAVI12:
1979 		chip_name = "navi12";
1980 		break;
1981 	}
1982 
1983 	snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1984 	err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1985 	if (err) {
1986 		dev_err(adev->dev,
1987 			"Failed to load gpu_info firmware \"%s\"\n",
1988 			fw_name);
1989 		goto out;
1990 	}
1991 	err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1992 	if (err) {
1993 		dev_err(adev->dev,
1994 			"Failed to validate gpu_info firmware \"%s\"\n",
1995 			fw_name);
1996 		goto out;
1997 	}
1998 
1999 	hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2000 	amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2001 
2002 	switch (hdr->version_major) {
2003 	case 1:
2004 	{
2005 		const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2006 			(const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2007 								le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2008 
2009 		/*
2010 		 * Should be droped when DAL no longer needs it.
2011 		 */
2012 		if (adev->asic_type == CHIP_NAVI12)
2013 			goto parse_soc_bounding_box;
2014 
2015 		adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2016 		adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2017 		adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2018 		adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2019 		adev->gfx.config.max_texture_channel_caches =
2020 			le32_to_cpu(gpu_info_fw->gc_num_tccs);
2021 		adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2022 		adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2023 		adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2024 		adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2025 		adev->gfx.config.double_offchip_lds_buf =
2026 			le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2027 		adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2028 		adev->gfx.cu_info.max_waves_per_simd =
2029 			le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2030 		adev->gfx.cu_info.max_scratch_slots_per_cu =
2031 			le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2032 		adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2033 		if (hdr->version_minor >= 1) {
2034 			const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2035 				(const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2036 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2037 			adev->gfx.config.num_sc_per_sh =
2038 				le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2039 			adev->gfx.config.num_packer_per_sc =
2040 				le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2041 		}
2042 
2043 parse_soc_bounding_box:
2044 		/*
2045 		 * soc bounding box info is not integrated in disocovery table,
2046 		 * we always need to parse it from gpu info firmware if needed.
2047 		 */
2048 		if (hdr->version_minor == 2) {
2049 			const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2050 				(const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2051 									le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2052 			adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2053 		}
2054 		break;
2055 	}
2056 	default:
2057 		dev_err(adev->dev,
2058 			"Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2059 		err = -EINVAL;
2060 		goto out;
2061 	}
2062 out:
2063 	return err;
2064 }
2065 
2066 /**
2067  * amdgpu_device_ip_early_init - run early init for hardware IPs
2068  *
2069  * @adev: amdgpu_device pointer
2070  *
2071  * Early initialization pass for hardware IPs.  The hardware IPs that make
2072  * up each asic are discovered each IP's early_init callback is run.  This
2073  * is the first stage in initializing the asic.
2074  * Returns 0 on success, negative error code on failure.
2075  */
2076 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2077 {
2078 	struct drm_device *dev = adev_to_drm(adev);
2079 	struct pci_dev *parent;
2080 	int i, r;
2081 
2082 	amdgpu_device_enable_virtual_display(adev);
2083 
2084 	if (amdgpu_sriov_vf(adev)) {
2085 		r = amdgpu_virt_request_full_gpu(adev, true);
2086 		if (r)
2087 			return r;
2088 	}
2089 
2090 	switch (adev->asic_type) {
2091 #ifdef CONFIG_DRM_AMDGPU_SI
2092 	case CHIP_VERDE:
2093 	case CHIP_TAHITI:
2094 	case CHIP_PITCAIRN:
2095 	case CHIP_OLAND:
2096 	case CHIP_HAINAN:
2097 		adev->family = AMDGPU_FAMILY_SI;
2098 		r = si_set_ip_blocks(adev);
2099 		if (r)
2100 			return r;
2101 		break;
2102 #endif
2103 #ifdef CONFIG_DRM_AMDGPU_CIK
2104 	case CHIP_BONAIRE:
2105 	case CHIP_HAWAII:
2106 	case CHIP_KAVERI:
2107 	case CHIP_KABINI:
2108 	case CHIP_MULLINS:
2109 		if (adev->flags & AMD_IS_APU)
2110 			adev->family = AMDGPU_FAMILY_KV;
2111 		else
2112 			adev->family = AMDGPU_FAMILY_CI;
2113 
2114 		r = cik_set_ip_blocks(adev);
2115 		if (r)
2116 			return r;
2117 		break;
2118 #endif
2119 	case CHIP_TOPAZ:
2120 	case CHIP_TONGA:
2121 	case CHIP_FIJI:
2122 	case CHIP_POLARIS10:
2123 	case CHIP_POLARIS11:
2124 	case CHIP_POLARIS12:
2125 	case CHIP_VEGAM:
2126 	case CHIP_CARRIZO:
2127 	case CHIP_STONEY:
2128 		if (adev->flags & AMD_IS_APU)
2129 			adev->family = AMDGPU_FAMILY_CZ;
2130 		else
2131 			adev->family = AMDGPU_FAMILY_VI;
2132 
2133 		r = vi_set_ip_blocks(adev);
2134 		if (r)
2135 			return r;
2136 		break;
2137 	default:
2138 		r = amdgpu_discovery_set_ip_blocks(adev);
2139 		if (r)
2140 			return r;
2141 		break;
2142 	}
2143 
2144 	if (amdgpu_has_atpx() &&
2145 	    (amdgpu_is_atpx_hybrid() ||
2146 	     amdgpu_has_atpx_dgpu_power_cntl()) &&
2147 	    ((adev->flags & AMD_IS_APU) == 0) &&
2148 	    !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
2149 		adev->flags |= AMD_IS_PX;
2150 
2151 	if (!(adev->flags & AMD_IS_APU)) {
2152 		parent = pci_upstream_bridge(adev->pdev);
2153 		adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2154 	}
2155 
2156 	amdgpu_amdkfd_device_probe(adev);
2157 
2158 	adev->pm.pp_feature = amdgpu_pp_feature_mask;
2159 	if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2160 		adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2161 	if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2162 		adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2163 
2164 	for (i = 0; i < adev->num_ip_blocks; i++) {
2165 		if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2166 			DRM_ERROR("disabled ip block: %d <%s>\n",
2167 				  i, adev->ip_blocks[i].version->funcs->name);
2168 			adev->ip_blocks[i].status.valid = false;
2169 		} else {
2170 			if (adev->ip_blocks[i].version->funcs->early_init) {
2171 				r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2172 				if (r == -ENOENT) {
2173 					adev->ip_blocks[i].status.valid = false;
2174 				} else if (r) {
2175 					DRM_ERROR("early_init of IP block <%s> failed %d\n",
2176 						  adev->ip_blocks[i].version->funcs->name, r);
2177 					return r;
2178 				} else {
2179 					adev->ip_blocks[i].status.valid = true;
2180 				}
2181 			} else {
2182 				adev->ip_blocks[i].status.valid = true;
2183 			}
2184 		}
2185 		/* get the vbios after the asic_funcs are set up */
2186 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2187 			r = amdgpu_device_parse_gpu_info_fw(adev);
2188 			if (r)
2189 				return r;
2190 
2191 			/* Read BIOS */
2192 			if (!amdgpu_get_bios(adev))
2193 				return -EINVAL;
2194 
2195 			r = amdgpu_atombios_init(adev);
2196 			if (r) {
2197 				dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2198 				amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2199 				return r;
2200 			}
2201 
2202 			/*get pf2vf msg info at it's earliest time*/
2203 			if (amdgpu_sriov_vf(adev))
2204 				amdgpu_virt_init_data_exchange(adev);
2205 
2206 		}
2207 	}
2208 
2209 	adev->cg_flags &= amdgpu_cg_mask;
2210 	adev->pg_flags &= amdgpu_pg_mask;
2211 
2212 	return 0;
2213 }
2214 
2215 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2216 {
2217 	int i, r;
2218 
2219 	for (i = 0; i < adev->num_ip_blocks; i++) {
2220 		if (!adev->ip_blocks[i].status.sw)
2221 			continue;
2222 		if (adev->ip_blocks[i].status.hw)
2223 			continue;
2224 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2225 		    (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2226 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2227 			r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2228 			if (r) {
2229 				DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2230 					  adev->ip_blocks[i].version->funcs->name, r);
2231 				return r;
2232 			}
2233 			adev->ip_blocks[i].status.hw = true;
2234 		}
2235 	}
2236 
2237 	return 0;
2238 }
2239 
2240 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2241 {
2242 	int i, r;
2243 
2244 	for (i = 0; i < adev->num_ip_blocks; i++) {
2245 		if (!adev->ip_blocks[i].status.sw)
2246 			continue;
2247 		if (adev->ip_blocks[i].status.hw)
2248 			continue;
2249 		r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2250 		if (r) {
2251 			DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2252 				  adev->ip_blocks[i].version->funcs->name, r);
2253 			return r;
2254 		}
2255 		adev->ip_blocks[i].status.hw = true;
2256 	}
2257 
2258 	return 0;
2259 }
2260 
2261 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2262 {
2263 	int r = 0;
2264 	int i;
2265 	uint32_t smu_version;
2266 
2267 	if (adev->asic_type >= CHIP_VEGA10) {
2268 		for (i = 0; i < adev->num_ip_blocks; i++) {
2269 			if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2270 				continue;
2271 
2272 			if (!adev->ip_blocks[i].status.sw)
2273 				continue;
2274 
2275 			/* no need to do the fw loading again if already done*/
2276 			if (adev->ip_blocks[i].status.hw == true)
2277 				break;
2278 
2279 			if (amdgpu_in_reset(adev) || adev->in_suspend) {
2280 				r = adev->ip_blocks[i].version->funcs->resume(adev);
2281 				if (r) {
2282 					DRM_ERROR("resume of IP block <%s> failed %d\n",
2283 							  adev->ip_blocks[i].version->funcs->name, r);
2284 					return r;
2285 				}
2286 			} else {
2287 				r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2288 				if (r) {
2289 					DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2290 							  adev->ip_blocks[i].version->funcs->name, r);
2291 					return r;
2292 				}
2293 			}
2294 
2295 			adev->ip_blocks[i].status.hw = true;
2296 			break;
2297 		}
2298 	}
2299 
2300 	if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2301 		r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2302 
2303 	return r;
2304 }
2305 
2306 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2307 {
2308 	long timeout;
2309 	int r, i;
2310 
2311 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2312 		struct amdgpu_ring *ring = adev->rings[i];
2313 
2314 		/* No need to setup the GPU scheduler for rings that don't need it */
2315 		if (!ring || ring->no_scheduler)
2316 			continue;
2317 
2318 		switch (ring->funcs->type) {
2319 		case AMDGPU_RING_TYPE_GFX:
2320 			timeout = adev->gfx_timeout;
2321 			break;
2322 		case AMDGPU_RING_TYPE_COMPUTE:
2323 			timeout = adev->compute_timeout;
2324 			break;
2325 		case AMDGPU_RING_TYPE_SDMA:
2326 			timeout = adev->sdma_timeout;
2327 			break;
2328 		default:
2329 			timeout = adev->video_timeout;
2330 			break;
2331 		}
2332 
2333 		r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2334 				   ring->num_hw_submission, amdgpu_job_hang_limit,
2335 				   timeout, adev->reset_domain->wq,
2336 				   ring->sched_score, ring->name,
2337 				   adev->dev);
2338 		if (r) {
2339 			DRM_ERROR("Failed to create scheduler on ring %s.\n",
2340 				  ring->name);
2341 			return r;
2342 		}
2343 	}
2344 
2345 	return 0;
2346 }
2347 
2348 
2349 /**
2350  * amdgpu_device_ip_init - run init for hardware IPs
2351  *
2352  * @adev: amdgpu_device pointer
2353  *
2354  * Main initialization pass for hardware IPs.  The list of all the hardware
2355  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2356  * are run.  sw_init initializes the software state associated with each IP
2357  * and hw_init initializes the hardware associated with each IP.
2358  * Returns 0 on success, negative error code on failure.
2359  */
2360 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2361 {
2362 	int i, r;
2363 
2364 	r = amdgpu_ras_init(adev);
2365 	if (r)
2366 		return r;
2367 
2368 	for (i = 0; i < adev->num_ip_blocks; i++) {
2369 		if (!adev->ip_blocks[i].status.valid)
2370 			continue;
2371 		r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2372 		if (r) {
2373 			DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2374 				  adev->ip_blocks[i].version->funcs->name, r);
2375 			goto init_failed;
2376 		}
2377 		adev->ip_blocks[i].status.sw = true;
2378 
2379 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2380 			/* need to do common hw init early so everything is set up for gmc */
2381 			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2382 			if (r) {
2383 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2384 				goto init_failed;
2385 			}
2386 			adev->ip_blocks[i].status.hw = true;
2387 		} else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2388 			/* need to do gmc hw init early so we can allocate gpu mem */
2389 			/* Try to reserve bad pages early */
2390 			if (amdgpu_sriov_vf(adev))
2391 				amdgpu_virt_exchange_data(adev);
2392 
2393 			r = amdgpu_device_vram_scratch_init(adev);
2394 			if (r) {
2395 				DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2396 				goto init_failed;
2397 			}
2398 			r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2399 			if (r) {
2400 				DRM_ERROR("hw_init %d failed %d\n", i, r);
2401 				goto init_failed;
2402 			}
2403 			r = amdgpu_device_wb_init(adev);
2404 			if (r) {
2405 				DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2406 				goto init_failed;
2407 			}
2408 			adev->ip_blocks[i].status.hw = true;
2409 
2410 			/* right after GMC hw init, we create CSA */
2411 			if (amdgpu_mcbp) {
2412 				r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2413 								AMDGPU_GEM_DOMAIN_VRAM,
2414 								AMDGPU_CSA_SIZE);
2415 				if (r) {
2416 					DRM_ERROR("allocate CSA failed %d\n", r);
2417 					goto init_failed;
2418 				}
2419 			}
2420 		}
2421 	}
2422 
2423 	if (amdgpu_sriov_vf(adev))
2424 		amdgpu_virt_init_data_exchange(adev);
2425 
2426 	r = amdgpu_ib_pool_init(adev);
2427 	if (r) {
2428 		dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2429 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2430 		goto init_failed;
2431 	}
2432 
2433 	r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2434 	if (r)
2435 		goto init_failed;
2436 
2437 	r = amdgpu_device_ip_hw_init_phase1(adev);
2438 	if (r)
2439 		goto init_failed;
2440 
2441 	r = amdgpu_device_fw_loading(adev);
2442 	if (r)
2443 		goto init_failed;
2444 
2445 	r = amdgpu_device_ip_hw_init_phase2(adev);
2446 	if (r)
2447 		goto init_failed;
2448 
2449 	/*
2450 	 * retired pages will be loaded from eeprom and reserved here,
2451 	 * it should be called after amdgpu_device_ip_hw_init_phase2  since
2452 	 * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2453 	 * for I2C communication which only true at this point.
2454 	 *
2455 	 * amdgpu_ras_recovery_init may fail, but the upper only cares the
2456 	 * failure from bad gpu situation and stop amdgpu init process
2457 	 * accordingly. For other failed cases, it will still release all
2458 	 * the resource and print error message, rather than returning one
2459 	 * negative value to upper level.
2460 	 *
2461 	 * Note: theoretically, this should be called before all vram allocations
2462 	 * to protect retired page from abusing
2463 	 */
2464 	r = amdgpu_ras_recovery_init(adev);
2465 	if (r)
2466 		goto init_failed;
2467 
2468 	/**
2469 	 * In case of XGMI grab extra reference for reset domain for this device
2470 	 */
2471 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
2472 		if (amdgpu_xgmi_add_device(adev) == 0) {
2473 			if (!amdgpu_sriov_vf(adev)) {
2474 				struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2475 
2476 				if (WARN_ON(!hive)) {
2477 					r = -ENOENT;
2478 					goto init_failed;
2479 				}
2480 
2481 				if (!hive->reset_domain ||
2482 				    !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2483 					r = -ENOENT;
2484 					amdgpu_put_xgmi_hive(hive);
2485 					goto init_failed;
2486 				}
2487 
2488 				/* Drop the early temporary reset domain we created for device */
2489 				amdgpu_reset_put_reset_domain(adev->reset_domain);
2490 				adev->reset_domain = hive->reset_domain;
2491 				amdgpu_put_xgmi_hive(hive);
2492 			}
2493 		}
2494 	}
2495 
2496 	r = amdgpu_device_init_schedulers(adev);
2497 	if (r)
2498 		goto init_failed;
2499 
2500 	/* Don't init kfd if whole hive need to be reset during init */
2501 	if (!adev->gmc.xgmi.pending_reset)
2502 		amdgpu_amdkfd_device_init(adev);
2503 
2504 	amdgpu_fru_get_product_info(adev);
2505 
2506 init_failed:
2507 	if (amdgpu_sriov_vf(adev))
2508 		amdgpu_virt_release_full_gpu(adev, true);
2509 
2510 	return r;
2511 }
2512 
2513 /**
2514  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2515  *
2516  * @adev: amdgpu_device pointer
2517  *
2518  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2519  * this function before a GPU reset.  If the value is retained after a
2520  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2521  */
2522 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2523 {
2524 	memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2525 }
2526 
2527 /**
2528  * amdgpu_device_check_vram_lost - check if vram is valid
2529  *
2530  * @adev: amdgpu_device pointer
2531  *
2532  * Checks the reset magic value written to the gart pointer in VRAM.
2533  * The driver calls this after a GPU reset to see if the contents of
2534  * VRAM is lost or now.
2535  * returns true if vram is lost, false if not.
2536  */
2537 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2538 {
2539 	if (memcmp(adev->gart.ptr, adev->reset_magic,
2540 			AMDGPU_RESET_MAGIC_NUM))
2541 		return true;
2542 
2543 	if (!amdgpu_in_reset(adev))
2544 		return false;
2545 
2546 	/*
2547 	 * For all ASICs with baco/mode1 reset, the VRAM is
2548 	 * always assumed to be lost.
2549 	 */
2550 	switch (amdgpu_asic_reset_method(adev)) {
2551 	case AMD_RESET_METHOD_BACO:
2552 	case AMD_RESET_METHOD_MODE1:
2553 		return true;
2554 	default:
2555 		return false;
2556 	}
2557 }
2558 
2559 /**
2560  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2561  *
2562  * @adev: amdgpu_device pointer
2563  * @state: clockgating state (gate or ungate)
2564  *
2565  * The list of all the hardware IPs that make up the asic is walked and the
2566  * set_clockgating_state callbacks are run.
2567  * Late initialization pass enabling clockgating for hardware IPs.
2568  * Fini or suspend, pass disabling clockgating for hardware IPs.
2569  * Returns 0 on success, negative error code on failure.
2570  */
2571 
2572 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2573 			       enum amd_clockgating_state state)
2574 {
2575 	int i, j, r;
2576 
2577 	if (amdgpu_emu_mode == 1)
2578 		return 0;
2579 
2580 	for (j = 0; j < adev->num_ip_blocks; j++) {
2581 		i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2582 		if (!adev->ip_blocks[i].status.late_initialized)
2583 			continue;
2584 		/* skip CG for GFX on S0ix */
2585 		if (adev->in_s0ix &&
2586 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2587 			continue;
2588 		/* skip CG for VCE/UVD, it's handled specially */
2589 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2590 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2591 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2592 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2593 		    adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2594 			/* enable clockgating to save power */
2595 			r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2596 										     state);
2597 			if (r) {
2598 				DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2599 					  adev->ip_blocks[i].version->funcs->name, r);
2600 				return r;
2601 			}
2602 		}
2603 	}
2604 
2605 	return 0;
2606 }
2607 
2608 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2609 			       enum amd_powergating_state state)
2610 {
2611 	int i, j, r;
2612 
2613 	if (amdgpu_emu_mode == 1)
2614 		return 0;
2615 
2616 	for (j = 0; j < adev->num_ip_blocks; j++) {
2617 		i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2618 		if (!adev->ip_blocks[i].status.late_initialized)
2619 			continue;
2620 		/* skip PG for GFX on S0ix */
2621 		if (adev->in_s0ix &&
2622 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2623 			continue;
2624 		/* skip CG for VCE/UVD, it's handled specially */
2625 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2626 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2627 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2628 		    adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2629 		    adev->ip_blocks[i].version->funcs->set_powergating_state) {
2630 			/* enable powergating to save power */
2631 			r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2632 											state);
2633 			if (r) {
2634 				DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2635 					  adev->ip_blocks[i].version->funcs->name, r);
2636 				return r;
2637 			}
2638 		}
2639 	}
2640 	return 0;
2641 }
2642 
2643 static int amdgpu_device_enable_mgpu_fan_boost(void)
2644 {
2645 	struct amdgpu_gpu_instance *gpu_ins;
2646 	struct amdgpu_device *adev;
2647 	int i, ret = 0;
2648 
2649 	mutex_lock(&mgpu_info.mutex);
2650 
2651 	/*
2652 	 * MGPU fan boost feature should be enabled
2653 	 * only when there are two or more dGPUs in
2654 	 * the system
2655 	 */
2656 	if (mgpu_info.num_dgpu < 2)
2657 		goto out;
2658 
2659 	for (i = 0; i < mgpu_info.num_dgpu; i++) {
2660 		gpu_ins = &(mgpu_info.gpu_ins[i]);
2661 		adev = gpu_ins->adev;
2662 		if (!(adev->flags & AMD_IS_APU) &&
2663 		    !gpu_ins->mgpu_fan_enabled) {
2664 			ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2665 			if (ret)
2666 				break;
2667 
2668 			gpu_ins->mgpu_fan_enabled = 1;
2669 		}
2670 	}
2671 
2672 out:
2673 	mutex_unlock(&mgpu_info.mutex);
2674 
2675 	return ret;
2676 }
2677 
2678 /**
2679  * amdgpu_device_ip_late_init - run late init for hardware IPs
2680  *
2681  * @adev: amdgpu_device pointer
2682  *
2683  * Late initialization pass for hardware IPs.  The list of all the hardware
2684  * IPs that make up the asic is walked and the late_init callbacks are run.
2685  * late_init covers any special initialization that an IP requires
2686  * after all of the have been initialized or something that needs to happen
2687  * late in the init process.
2688  * Returns 0 on success, negative error code on failure.
2689  */
2690 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2691 {
2692 	struct amdgpu_gpu_instance *gpu_instance;
2693 	int i = 0, r;
2694 
2695 	for (i = 0; i < adev->num_ip_blocks; i++) {
2696 		if (!adev->ip_blocks[i].status.hw)
2697 			continue;
2698 		if (adev->ip_blocks[i].version->funcs->late_init) {
2699 			r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2700 			if (r) {
2701 				DRM_ERROR("late_init of IP block <%s> failed %d\n",
2702 					  adev->ip_blocks[i].version->funcs->name, r);
2703 				return r;
2704 			}
2705 		}
2706 		adev->ip_blocks[i].status.late_initialized = true;
2707 	}
2708 
2709 	r = amdgpu_ras_late_init(adev);
2710 	if (r) {
2711 		DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2712 		return r;
2713 	}
2714 
2715 	amdgpu_ras_set_error_query_ready(adev, true);
2716 
2717 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2718 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2719 
2720 	amdgpu_device_fill_reset_magic(adev);
2721 
2722 	r = amdgpu_device_enable_mgpu_fan_boost();
2723 	if (r)
2724 		DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2725 
2726 	/* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2727 	if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)||
2728 			       adev->asic_type == CHIP_ALDEBARAN ))
2729 		amdgpu_dpm_handle_passthrough_sbr(adev, true);
2730 
2731 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
2732 		mutex_lock(&mgpu_info.mutex);
2733 
2734 		/*
2735 		 * Reset device p-state to low as this was booted with high.
2736 		 *
2737 		 * This should be performed only after all devices from the same
2738 		 * hive get initialized.
2739 		 *
2740 		 * However, it's unknown how many device in the hive in advance.
2741 		 * As this is counted one by one during devices initializations.
2742 		 *
2743 		 * So, we wait for all XGMI interlinked devices initialized.
2744 		 * This may bring some delays as those devices may come from
2745 		 * different hives. But that should be OK.
2746 		 */
2747 		if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2748 			for (i = 0; i < mgpu_info.num_gpu; i++) {
2749 				gpu_instance = &(mgpu_info.gpu_ins[i]);
2750 				if (gpu_instance->adev->flags & AMD_IS_APU)
2751 					continue;
2752 
2753 				r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2754 						AMDGPU_XGMI_PSTATE_MIN);
2755 				if (r) {
2756 					DRM_ERROR("pstate setting failed (%d).\n", r);
2757 					break;
2758 				}
2759 			}
2760 		}
2761 
2762 		mutex_unlock(&mgpu_info.mutex);
2763 	}
2764 
2765 	return 0;
2766 }
2767 
2768 /**
2769  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2770  *
2771  * @adev: amdgpu_device pointer
2772  *
2773  * For ASICs need to disable SMC first
2774  */
2775 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2776 {
2777 	int i, r;
2778 
2779 	if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2780 		return;
2781 
2782 	for (i = 0; i < adev->num_ip_blocks; i++) {
2783 		if (!adev->ip_blocks[i].status.hw)
2784 			continue;
2785 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2786 			r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2787 			/* XXX handle errors */
2788 			if (r) {
2789 				DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2790 					  adev->ip_blocks[i].version->funcs->name, r);
2791 			}
2792 			adev->ip_blocks[i].status.hw = false;
2793 			break;
2794 		}
2795 	}
2796 }
2797 
2798 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2799 {
2800 	int i, r;
2801 
2802 	for (i = 0; i < adev->num_ip_blocks; i++) {
2803 		if (!adev->ip_blocks[i].version->funcs->early_fini)
2804 			continue;
2805 
2806 		r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2807 		if (r) {
2808 			DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2809 				  adev->ip_blocks[i].version->funcs->name, r);
2810 		}
2811 	}
2812 
2813 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2814 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2815 
2816 	amdgpu_amdkfd_suspend(adev, false);
2817 
2818 	/* Workaroud for ASICs need to disable SMC first */
2819 	amdgpu_device_smu_fini_early(adev);
2820 
2821 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2822 		if (!adev->ip_blocks[i].status.hw)
2823 			continue;
2824 
2825 		r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2826 		/* XXX handle errors */
2827 		if (r) {
2828 			DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2829 				  adev->ip_blocks[i].version->funcs->name, r);
2830 		}
2831 
2832 		adev->ip_blocks[i].status.hw = false;
2833 	}
2834 
2835 	if (amdgpu_sriov_vf(adev)) {
2836 		if (amdgpu_virt_release_full_gpu(adev, false))
2837 			DRM_ERROR("failed to release exclusive mode on fini\n");
2838 	}
2839 
2840 	return 0;
2841 }
2842 
2843 /**
2844  * amdgpu_device_ip_fini - run fini for hardware IPs
2845  *
2846  * @adev: amdgpu_device pointer
2847  *
2848  * Main teardown pass for hardware IPs.  The list of all the hardware
2849  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2850  * are run.  hw_fini tears down the hardware associated with each IP
2851  * and sw_fini tears down any software state associated with each IP.
2852  * Returns 0 on success, negative error code on failure.
2853  */
2854 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2855 {
2856 	int i, r;
2857 
2858 	if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2859 		amdgpu_virt_release_ras_err_handler_data(adev);
2860 
2861 	if (adev->gmc.xgmi.num_physical_nodes > 1)
2862 		amdgpu_xgmi_remove_device(adev);
2863 
2864 	amdgpu_amdkfd_device_fini_sw(adev);
2865 
2866 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2867 		if (!adev->ip_blocks[i].status.sw)
2868 			continue;
2869 
2870 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2871 			amdgpu_ucode_free_bo(adev);
2872 			amdgpu_free_static_csa(&adev->virt.csa_obj);
2873 			amdgpu_device_wb_fini(adev);
2874 			amdgpu_device_vram_scratch_fini(adev);
2875 			amdgpu_ib_pool_fini(adev);
2876 		}
2877 
2878 		r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2879 		/* XXX handle errors */
2880 		if (r) {
2881 			DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2882 				  adev->ip_blocks[i].version->funcs->name, r);
2883 		}
2884 		adev->ip_blocks[i].status.sw = false;
2885 		adev->ip_blocks[i].status.valid = false;
2886 	}
2887 
2888 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2889 		if (!adev->ip_blocks[i].status.late_initialized)
2890 			continue;
2891 		if (adev->ip_blocks[i].version->funcs->late_fini)
2892 			adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2893 		adev->ip_blocks[i].status.late_initialized = false;
2894 	}
2895 
2896 	amdgpu_ras_fini(adev);
2897 
2898 	return 0;
2899 }
2900 
2901 /**
2902  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2903  *
2904  * @work: work_struct.
2905  */
2906 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2907 {
2908 	struct amdgpu_device *adev =
2909 		container_of(work, struct amdgpu_device, delayed_init_work.work);
2910 	int r;
2911 
2912 	r = amdgpu_ib_ring_tests(adev);
2913 	if (r)
2914 		DRM_ERROR("ib ring test failed (%d).\n", r);
2915 }
2916 
2917 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2918 {
2919 	struct amdgpu_device *adev =
2920 		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2921 
2922 	WARN_ON_ONCE(adev->gfx.gfx_off_state);
2923 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2924 
2925 	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2926 		adev->gfx.gfx_off_state = true;
2927 }
2928 
2929 /**
2930  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2931  *
2932  * @adev: amdgpu_device pointer
2933  *
2934  * Main suspend function for hardware IPs.  The list of all the hardware
2935  * IPs that make up the asic is walked, clockgating is disabled and the
2936  * suspend callbacks are run.  suspend puts the hardware and software state
2937  * in each IP into a state suitable for suspend.
2938  * Returns 0 on success, negative error code on failure.
2939  */
2940 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2941 {
2942 	int i, r;
2943 
2944 	amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2945 	amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2946 
2947 	/*
2948 	 * Per PMFW team's suggestion, driver needs to handle gfxoff
2949 	 * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
2950 	 * scenario. Add the missing df cstate disablement here.
2951 	 */
2952 	if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
2953 		dev_warn(adev->dev, "Failed to disallow df cstate");
2954 
2955 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2956 		if (!adev->ip_blocks[i].status.valid)
2957 			continue;
2958 
2959 		/* displays are handled separately */
2960 		if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2961 			continue;
2962 
2963 		/* XXX handle errors */
2964 		r = adev->ip_blocks[i].version->funcs->suspend(adev);
2965 		/* XXX handle errors */
2966 		if (r) {
2967 			DRM_ERROR("suspend of IP block <%s> failed %d\n",
2968 				  adev->ip_blocks[i].version->funcs->name, r);
2969 			return r;
2970 		}
2971 
2972 		adev->ip_blocks[i].status.hw = false;
2973 	}
2974 
2975 	return 0;
2976 }
2977 
2978 /**
2979  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2980  *
2981  * @adev: amdgpu_device pointer
2982  *
2983  * Main suspend function for hardware IPs.  The list of all the hardware
2984  * IPs that make up the asic is walked, clockgating is disabled and the
2985  * suspend callbacks are run.  suspend puts the hardware and software state
2986  * in each IP into a state suitable for suspend.
2987  * Returns 0 on success, negative error code on failure.
2988  */
2989 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2990 {
2991 	int i, r;
2992 
2993 	if (adev->in_s0ix)
2994 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
2995 
2996 	for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2997 		if (!adev->ip_blocks[i].status.valid)
2998 			continue;
2999 		/* displays are handled in phase1 */
3000 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3001 			continue;
3002 		/* PSP lost connection when err_event_athub occurs */
3003 		if (amdgpu_ras_intr_triggered() &&
3004 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3005 			adev->ip_blocks[i].status.hw = false;
3006 			continue;
3007 		}
3008 
3009 		/* skip unnecessary suspend if we do not initialize them yet */
3010 		if (adev->gmc.xgmi.pending_reset &&
3011 		    !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3012 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3013 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3014 		      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3015 			adev->ip_blocks[i].status.hw = false;
3016 			continue;
3017 		}
3018 
3019 		/* skip suspend of gfx and psp for S0ix
3020 		 * gfx is in gfxoff state, so on resume it will exit gfxoff just
3021 		 * like at runtime. PSP is also part of the always on hardware
3022 		 * so no need to suspend it.
3023 		 */
3024 		if (adev->in_s0ix &&
3025 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3026 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX))
3027 			continue;
3028 
3029 		/* XXX handle errors */
3030 		r = adev->ip_blocks[i].version->funcs->suspend(adev);
3031 		/* XXX handle errors */
3032 		if (r) {
3033 			DRM_ERROR("suspend of IP block <%s> failed %d\n",
3034 				  adev->ip_blocks[i].version->funcs->name, r);
3035 		}
3036 		adev->ip_blocks[i].status.hw = false;
3037 		/* handle putting the SMC in the appropriate state */
3038 		if(!amdgpu_sriov_vf(adev)){
3039 			if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3040 				r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3041 				if (r) {
3042 					DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3043 							adev->mp1_state, r);
3044 					return r;
3045 				}
3046 			}
3047 		}
3048 	}
3049 
3050 	return 0;
3051 }
3052 
3053 /**
3054  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3055  *
3056  * @adev: amdgpu_device pointer
3057  *
3058  * Main suspend function for hardware IPs.  The list of all the hardware
3059  * IPs that make up the asic is walked, clockgating is disabled and the
3060  * suspend callbacks are run.  suspend puts the hardware and software state
3061  * in each IP into a state suitable for suspend.
3062  * Returns 0 on success, negative error code on failure.
3063  */
3064 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3065 {
3066 	int r;
3067 
3068 	if (amdgpu_sriov_vf(adev)) {
3069 		amdgpu_virt_fini_data_exchange(adev);
3070 		amdgpu_virt_request_full_gpu(adev, false);
3071 	}
3072 
3073 	r = amdgpu_device_ip_suspend_phase1(adev);
3074 	if (r)
3075 		return r;
3076 	r = amdgpu_device_ip_suspend_phase2(adev);
3077 
3078 	if (amdgpu_sriov_vf(adev))
3079 		amdgpu_virt_release_full_gpu(adev, false);
3080 
3081 	return r;
3082 }
3083 
3084 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3085 {
3086 	int i, r;
3087 
3088 	static enum amd_ip_block_type ip_order[] = {
3089 		AMD_IP_BLOCK_TYPE_COMMON,
3090 		AMD_IP_BLOCK_TYPE_GMC,
3091 		AMD_IP_BLOCK_TYPE_PSP,
3092 		AMD_IP_BLOCK_TYPE_IH,
3093 	};
3094 
3095 	for (i = 0; i < adev->num_ip_blocks; i++) {
3096 		int j;
3097 		struct amdgpu_ip_block *block;
3098 
3099 		block = &adev->ip_blocks[i];
3100 		block->status.hw = false;
3101 
3102 		for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3103 
3104 			if (block->version->type != ip_order[j] ||
3105 				!block->status.valid)
3106 				continue;
3107 
3108 			r = block->version->funcs->hw_init(adev);
3109 			DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3110 			if (r)
3111 				return r;
3112 			block->status.hw = true;
3113 		}
3114 	}
3115 
3116 	return 0;
3117 }
3118 
3119 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3120 {
3121 	int i, r;
3122 
3123 	static enum amd_ip_block_type ip_order[] = {
3124 		AMD_IP_BLOCK_TYPE_SMC,
3125 		AMD_IP_BLOCK_TYPE_DCE,
3126 		AMD_IP_BLOCK_TYPE_GFX,
3127 		AMD_IP_BLOCK_TYPE_SDMA,
3128 		AMD_IP_BLOCK_TYPE_UVD,
3129 		AMD_IP_BLOCK_TYPE_VCE,
3130 		AMD_IP_BLOCK_TYPE_VCN
3131 	};
3132 
3133 	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3134 		int j;
3135 		struct amdgpu_ip_block *block;
3136 
3137 		for (j = 0; j < adev->num_ip_blocks; j++) {
3138 			block = &adev->ip_blocks[j];
3139 
3140 			if (block->version->type != ip_order[i] ||
3141 				!block->status.valid ||
3142 				block->status.hw)
3143 				continue;
3144 
3145 			if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3146 				r = block->version->funcs->resume(adev);
3147 			else
3148 				r = block->version->funcs->hw_init(adev);
3149 
3150 			DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3151 			if (r)
3152 				return r;
3153 			block->status.hw = true;
3154 		}
3155 	}
3156 
3157 	return 0;
3158 }
3159 
3160 /**
3161  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3162  *
3163  * @adev: amdgpu_device pointer
3164  *
3165  * First resume function for hardware IPs.  The list of all the hardware
3166  * IPs that make up the asic is walked and the resume callbacks are run for
3167  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3168  * after a suspend and updates the software state as necessary.  This
3169  * function is also used for restoring the GPU after a GPU reset.
3170  * Returns 0 on success, negative error code on failure.
3171  */
3172 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3173 {
3174 	int i, r;
3175 
3176 	for (i = 0; i < adev->num_ip_blocks; i++) {
3177 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3178 			continue;
3179 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3180 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3181 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3182 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3183 
3184 			r = adev->ip_blocks[i].version->funcs->resume(adev);
3185 			if (r) {
3186 				DRM_ERROR("resume of IP block <%s> failed %d\n",
3187 					  adev->ip_blocks[i].version->funcs->name, r);
3188 				return r;
3189 			}
3190 			adev->ip_blocks[i].status.hw = true;
3191 		}
3192 	}
3193 
3194 	return 0;
3195 }
3196 
3197 /**
3198  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3199  *
3200  * @adev: amdgpu_device pointer
3201  *
3202  * First resume function for hardware IPs.  The list of all the hardware
3203  * IPs that make up the asic is walked and the resume callbacks are run for
3204  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3205  * functional state after a suspend and updates the software state as
3206  * necessary.  This function is also used for restoring the GPU after a GPU
3207  * reset.
3208  * Returns 0 on success, negative error code on failure.
3209  */
3210 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3211 {
3212 	int i, r;
3213 
3214 	for (i = 0; i < adev->num_ip_blocks; i++) {
3215 		if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3216 			continue;
3217 		if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3218 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3219 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3220 		    adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3221 			continue;
3222 		r = adev->ip_blocks[i].version->funcs->resume(adev);
3223 		if (r) {
3224 			DRM_ERROR("resume of IP block <%s> failed %d\n",
3225 				  adev->ip_blocks[i].version->funcs->name, r);
3226 			return r;
3227 		}
3228 		adev->ip_blocks[i].status.hw = true;
3229 
3230 		if (adev->in_s0ix && adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3231 			/* disable gfxoff for IP resume. The gfxoff will be re-enabled in
3232 			 * amdgpu_device_resume() after IP resume.
3233 			 */
3234 			amdgpu_gfx_off_ctrl(adev, false);
3235 			DRM_DEBUG("will disable gfxoff for re-initializing other blocks\n");
3236 		}
3237 
3238 	}
3239 
3240 	return 0;
3241 }
3242 
3243 /**
3244  * amdgpu_device_ip_resume - run resume for hardware IPs
3245  *
3246  * @adev: amdgpu_device pointer
3247  *
3248  * Main resume function for hardware IPs.  The hardware IPs
3249  * are split into two resume functions because they are
3250  * are also used in in recovering from a GPU reset and some additional
3251  * steps need to be take between them.  In this case (S3/S4) they are
3252  * run sequentially.
3253  * Returns 0 on success, negative error code on failure.
3254  */
3255 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3256 {
3257 	int r;
3258 
3259 	r = amdgpu_amdkfd_resume_iommu(adev);
3260 	if (r)
3261 		return r;
3262 
3263 	r = amdgpu_device_ip_resume_phase1(adev);
3264 	if (r)
3265 		return r;
3266 
3267 	r = amdgpu_device_fw_loading(adev);
3268 	if (r)
3269 		return r;
3270 
3271 	r = amdgpu_device_ip_resume_phase2(adev);
3272 
3273 	return r;
3274 }
3275 
3276 /**
3277  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3278  *
3279  * @adev: amdgpu_device pointer
3280  *
3281  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3282  */
3283 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3284 {
3285 	if (amdgpu_sriov_vf(adev)) {
3286 		if (adev->is_atom_fw) {
3287 			if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3288 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3289 		} else {
3290 			if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3291 				adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3292 		}
3293 
3294 		if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3295 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3296 	}
3297 }
3298 
3299 /**
3300  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3301  *
3302  * @asic_type: AMD asic type
3303  *
3304  * Check if there is DC (new modesetting infrastructre) support for an asic.
3305  * returns true if DC has support, false if not.
3306  */
3307 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3308 {
3309 	switch (asic_type) {
3310 #ifdef CONFIG_DRM_AMDGPU_SI
3311 	case CHIP_HAINAN:
3312 #endif
3313 	case CHIP_TOPAZ:
3314 		/* chips with no display hardware */
3315 		return false;
3316 #if defined(CONFIG_DRM_AMD_DC)
3317 	case CHIP_TAHITI:
3318 	case CHIP_PITCAIRN:
3319 	case CHIP_VERDE:
3320 	case CHIP_OLAND:
3321 		/*
3322 		 * We have systems in the wild with these ASICs that require
3323 		 * LVDS and VGA support which is not supported with DC.
3324 		 *
3325 		 * Fallback to the non-DC driver here by default so as not to
3326 		 * cause regressions.
3327 		 */
3328 #if defined(CONFIG_DRM_AMD_DC_SI)
3329 		return amdgpu_dc > 0;
3330 #else
3331 		return false;
3332 #endif
3333 	case CHIP_BONAIRE:
3334 	case CHIP_KAVERI:
3335 	case CHIP_KABINI:
3336 	case CHIP_MULLINS:
3337 		/*
3338 		 * We have systems in the wild with these ASICs that require
3339 		 * VGA support which is not supported with DC.
3340 		 *
3341 		 * Fallback to the non-DC driver here by default so as not to
3342 		 * cause regressions.
3343 		 */
3344 		return amdgpu_dc > 0;
3345 	default:
3346 		return amdgpu_dc != 0;
3347 #else
3348 	default:
3349 		if (amdgpu_dc > 0)
3350 			DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3351 					 "but isn't supported by ASIC, ignoring\n");
3352 		return false;
3353 #endif
3354 	}
3355 }
3356 
3357 /**
3358  * amdgpu_device_has_dc_support - check if dc is supported
3359  *
3360  * @adev: amdgpu_device pointer
3361  *
3362  * Returns true for supported, false for not supported
3363  */
3364 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3365 {
3366 	if (adev->enable_virtual_display ||
3367 	    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3368 		return false;
3369 
3370 	return amdgpu_device_asic_has_dc_support(adev->asic_type);
3371 }
3372 
3373 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3374 {
3375 	struct amdgpu_device *adev =
3376 		container_of(__work, struct amdgpu_device, xgmi_reset_work);
3377 	struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3378 
3379 	/* It's a bug to not have a hive within this function */
3380 	if (WARN_ON(!hive))
3381 		return;
3382 
3383 	/*
3384 	 * Use task barrier to synchronize all xgmi reset works across the
3385 	 * hive. task_barrier_enter and task_barrier_exit will block
3386 	 * until all the threads running the xgmi reset works reach
3387 	 * those points. task_barrier_full will do both blocks.
3388 	 */
3389 	if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3390 
3391 		task_barrier_enter(&hive->tb);
3392 		adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3393 
3394 		if (adev->asic_reset_res)
3395 			goto fail;
3396 
3397 		task_barrier_exit(&hive->tb);
3398 		adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3399 
3400 		if (adev->asic_reset_res)
3401 			goto fail;
3402 
3403 		if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
3404 		    adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
3405 			adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
3406 	} else {
3407 
3408 		task_barrier_full(&hive->tb);
3409 		adev->asic_reset_res =  amdgpu_asic_reset(adev);
3410 	}
3411 
3412 fail:
3413 	if (adev->asic_reset_res)
3414 		DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3415 			 adev->asic_reset_res, adev_to_drm(adev)->unique);
3416 	amdgpu_put_xgmi_hive(hive);
3417 }
3418 
3419 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3420 {
3421 	char *input = amdgpu_lockup_timeout;
3422 	char *timeout_setting = NULL;
3423 	int index = 0;
3424 	long timeout;
3425 	int ret = 0;
3426 
3427 	/*
3428 	 * By default timeout for non compute jobs is 10000
3429 	 * and 60000 for compute jobs.
3430 	 * In SR-IOV or passthrough mode, timeout for compute
3431 	 * jobs are 60000 by default.
3432 	 */
3433 	adev->gfx_timeout = msecs_to_jiffies(10000);
3434 	adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3435 	if (amdgpu_sriov_vf(adev))
3436 		adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3437 					msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3438 	else
3439 		adev->compute_timeout =  msecs_to_jiffies(60000);
3440 
3441 	if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3442 		while ((timeout_setting = strsep(&input, ",")) &&
3443 				strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3444 			ret = kstrtol(timeout_setting, 0, &timeout);
3445 			if (ret)
3446 				return ret;
3447 
3448 			if (timeout == 0) {
3449 				index++;
3450 				continue;
3451 			} else if (timeout < 0) {
3452 				timeout = MAX_SCHEDULE_TIMEOUT;
3453 				dev_warn(adev->dev, "lockup timeout disabled");
3454 				add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3455 			} else {
3456 				timeout = msecs_to_jiffies(timeout);
3457 			}
3458 
3459 			switch (index++) {
3460 			case 0:
3461 				adev->gfx_timeout = timeout;
3462 				break;
3463 			case 1:
3464 				adev->compute_timeout = timeout;
3465 				break;
3466 			case 2:
3467 				adev->sdma_timeout = timeout;
3468 				break;
3469 			case 3:
3470 				adev->video_timeout = timeout;
3471 				break;
3472 			default:
3473 				break;
3474 			}
3475 		}
3476 		/*
3477 		 * There is only one value specified and
3478 		 * it should apply to all non-compute jobs.
3479 		 */
3480 		if (index == 1) {
3481 			adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3482 			if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3483 				adev->compute_timeout = adev->gfx_timeout;
3484 		}
3485 	}
3486 
3487 	return ret;
3488 }
3489 
3490 /**
3491  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3492  *
3493  * @adev: amdgpu_device pointer
3494  *
3495  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3496  */
3497 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3498 {
3499 	struct iommu_domain *domain;
3500 
3501 	domain = iommu_get_domain_for_dev(adev->dev);
3502 	if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3503 		adev->ram_is_direct_mapped = true;
3504 }
3505 
3506 static const struct attribute *amdgpu_dev_attributes[] = {
3507 	&dev_attr_product_name.attr,
3508 	&dev_attr_product_number.attr,
3509 	&dev_attr_serial_number.attr,
3510 	&dev_attr_pcie_replay_count.attr,
3511 	NULL
3512 };
3513 
3514 /**
3515  * amdgpu_device_init - initialize the driver
3516  *
3517  * @adev: amdgpu_device pointer
3518  * @flags: driver flags
3519  *
3520  * Initializes the driver info and hw (all asics).
3521  * Returns 0 for success or an error on failure.
3522  * Called at driver startup.
3523  */
3524 int amdgpu_device_init(struct amdgpu_device *adev,
3525 		       uint32_t flags)
3526 {
3527 	struct drm_device *ddev = adev_to_drm(adev);
3528 	struct pci_dev *pdev = adev->pdev;
3529 	int r, i;
3530 	bool px = false;
3531 	u32 max_MBps;
3532 
3533 	adev->shutdown = false;
3534 	adev->flags = flags;
3535 
3536 	if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3537 		adev->asic_type = amdgpu_force_asic_type;
3538 	else
3539 		adev->asic_type = flags & AMD_ASIC_MASK;
3540 
3541 	adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3542 	if (amdgpu_emu_mode == 1)
3543 		adev->usec_timeout *= 10;
3544 	adev->gmc.gart_size = 512 * 1024 * 1024;
3545 	adev->accel_working = false;
3546 	adev->num_rings = 0;
3547 	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3548 	adev->mman.buffer_funcs = NULL;
3549 	adev->mman.buffer_funcs_ring = NULL;
3550 	adev->vm_manager.vm_pte_funcs = NULL;
3551 	adev->vm_manager.vm_pte_num_scheds = 0;
3552 	adev->gmc.gmc_funcs = NULL;
3553 	adev->harvest_ip_mask = 0x0;
3554 	adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3555 	bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3556 
3557 	adev->smc_rreg = &amdgpu_invalid_rreg;
3558 	adev->smc_wreg = &amdgpu_invalid_wreg;
3559 	adev->pcie_rreg = &amdgpu_invalid_rreg;
3560 	adev->pcie_wreg = &amdgpu_invalid_wreg;
3561 	adev->pciep_rreg = &amdgpu_invalid_rreg;
3562 	adev->pciep_wreg = &amdgpu_invalid_wreg;
3563 	adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3564 	adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3565 	adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3566 	adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3567 	adev->didt_rreg = &amdgpu_invalid_rreg;
3568 	adev->didt_wreg = &amdgpu_invalid_wreg;
3569 	adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3570 	adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3571 	adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3572 	adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3573 
3574 	DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3575 		 amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3576 		 pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3577 
3578 	/* mutex initialization are all done here so we
3579 	 * can recall function without having locking issues */
3580 	mutex_init(&adev->firmware.mutex);
3581 	mutex_init(&adev->pm.mutex);
3582 	mutex_init(&adev->gfx.gpu_clock_mutex);
3583 	mutex_init(&adev->srbm_mutex);
3584 	mutex_init(&adev->gfx.pipe_reserve_mutex);
3585 	mutex_init(&adev->gfx.gfx_off_mutex);
3586 	mutex_init(&adev->grbm_idx_mutex);
3587 	mutex_init(&adev->mn_lock);
3588 	mutex_init(&adev->virt.vf_errors.lock);
3589 	hash_init(adev->mn_hash);
3590 	mutex_init(&adev->psp.mutex);
3591 	mutex_init(&adev->notifier_lock);
3592 	mutex_init(&adev->pm.stable_pstate_ctx_lock);
3593 	mutex_init(&adev->benchmark_mutex);
3594 
3595 	amdgpu_device_init_apu_flags(adev);
3596 
3597 	r = amdgpu_device_check_arguments(adev);
3598 	if (r)
3599 		return r;
3600 
3601 	spin_lock_init(&adev->mmio_idx_lock);
3602 	spin_lock_init(&adev->smc_idx_lock);
3603 	spin_lock_init(&adev->pcie_idx_lock);
3604 	spin_lock_init(&adev->uvd_ctx_idx_lock);
3605 	spin_lock_init(&adev->didt_idx_lock);
3606 	spin_lock_init(&adev->gc_cac_idx_lock);
3607 	spin_lock_init(&adev->se_cac_idx_lock);
3608 	spin_lock_init(&adev->audio_endpt_idx_lock);
3609 	spin_lock_init(&adev->mm_stats.lock);
3610 
3611 	INIT_LIST_HEAD(&adev->shadow_list);
3612 	mutex_init(&adev->shadow_list_lock);
3613 
3614 	INIT_LIST_HEAD(&adev->reset_list);
3615 
3616 	INIT_LIST_HEAD(&adev->ras_list);
3617 
3618 	INIT_DELAYED_WORK(&adev->delayed_init_work,
3619 			  amdgpu_device_delayed_init_work_handler);
3620 	INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3621 			  amdgpu_device_delay_enable_gfx_off);
3622 
3623 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3624 
3625 	adev->gfx.gfx_off_req_count = 1;
3626 	adev->gfx.gfx_off_residency = 0;
3627 	adev->gfx.gfx_off_entrycount = 0;
3628 	adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3629 
3630 	atomic_set(&adev->throttling_logging_enabled, 1);
3631 	/*
3632 	 * If throttling continues, logging will be performed every minute
3633 	 * to avoid log flooding. "-1" is subtracted since the thermal
3634 	 * throttling interrupt comes every second. Thus, the total logging
3635 	 * interval is 59 seconds(retelimited printk interval) + 1(waiting
3636 	 * for throttling interrupt) = 60 seconds.
3637 	 */
3638 	ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3639 	ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3640 
3641 	/* Registers mapping */
3642 	/* TODO: block userspace mapping of io register */
3643 	if (adev->asic_type >= CHIP_BONAIRE) {
3644 		adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3645 		adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3646 	} else {
3647 		adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3648 		adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3649 	}
3650 
3651 	for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3652 		atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3653 
3654 	adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3655 	if (adev->rmmio == NULL) {
3656 		return -ENOMEM;
3657 	}
3658 	DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3659 	DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3660 
3661 	amdgpu_device_get_pcie_info(adev);
3662 
3663 	if (amdgpu_mcbp)
3664 		DRM_INFO("MCBP is enabled\n");
3665 
3666 	/*
3667 	 * Reset domain needs to be present early, before XGMI hive discovered
3668 	 * (if any) and intitialized to use reset sem and in_gpu reset flag
3669 	 * early on during init and before calling to RREG32.
3670 	 */
3671 	adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3672 	if (!adev->reset_domain)
3673 		return -ENOMEM;
3674 
3675 	/* detect hw virtualization here */
3676 	amdgpu_detect_virtualization(adev);
3677 
3678 	r = amdgpu_device_get_job_timeout_settings(adev);
3679 	if (r) {
3680 		dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3681 		return r;
3682 	}
3683 
3684 	/* early init functions */
3685 	r = amdgpu_device_ip_early_init(adev);
3686 	if (r)
3687 		return r;
3688 
3689 	/* Enable TMZ based on IP_VERSION */
3690 	amdgpu_gmc_tmz_set(adev);
3691 
3692 	amdgpu_gmc_noretry_set(adev);
3693 	/* Need to get xgmi info early to decide the reset behavior*/
3694 	if (adev->gmc.xgmi.supported) {
3695 		r = adev->gfxhub.funcs->get_xgmi_info(adev);
3696 		if (r)
3697 			return r;
3698 	}
3699 
3700 	/* enable PCIE atomic ops */
3701 	if (amdgpu_sriov_vf(adev))
3702 		adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3703 			adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3704 			(PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3705 	else
3706 		adev->have_atomics_support =
3707 			!pci_enable_atomic_ops_to_root(adev->pdev,
3708 					  PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3709 					  PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3710 	if (!adev->have_atomics_support)
3711 		dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3712 
3713 	/* doorbell bar mapping and doorbell index init*/
3714 	amdgpu_device_doorbell_init(adev);
3715 
3716 	if (amdgpu_emu_mode == 1) {
3717 		/* post the asic on emulation mode */
3718 		emu_soc_asic_init(adev);
3719 		goto fence_driver_init;
3720 	}
3721 
3722 	amdgpu_reset_init(adev);
3723 
3724 	/* detect if we are with an SRIOV vbios */
3725 	amdgpu_device_detect_sriov_bios(adev);
3726 
3727 	/* check if we need to reset the asic
3728 	 *  E.g., driver was not cleanly unloaded previously, etc.
3729 	 */
3730 	if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3731 		if (adev->gmc.xgmi.num_physical_nodes) {
3732 			dev_info(adev->dev, "Pending hive reset.\n");
3733 			adev->gmc.xgmi.pending_reset = true;
3734 			/* Only need to init necessary block for SMU to handle the reset */
3735 			for (i = 0; i < adev->num_ip_blocks; i++) {
3736 				if (!adev->ip_blocks[i].status.valid)
3737 					continue;
3738 				if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3739 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3740 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3741 				      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3742 					DRM_DEBUG("IP %s disabled for hw_init.\n",
3743 						adev->ip_blocks[i].version->funcs->name);
3744 					adev->ip_blocks[i].status.hw = true;
3745 				}
3746 			}
3747 		} else {
3748 			r = amdgpu_asic_reset(adev);
3749 			if (r) {
3750 				dev_err(adev->dev, "asic reset on init failed\n");
3751 				goto failed;
3752 			}
3753 		}
3754 	}
3755 
3756 	pci_enable_pcie_error_reporting(adev->pdev);
3757 
3758 	/* Post card if necessary */
3759 	if (amdgpu_device_need_post(adev)) {
3760 		if (!adev->bios) {
3761 			dev_err(adev->dev, "no vBIOS found\n");
3762 			r = -EINVAL;
3763 			goto failed;
3764 		}
3765 		DRM_INFO("GPU posting now...\n");
3766 		r = amdgpu_device_asic_init(adev);
3767 		if (r) {
3768 			dev_err(adev->dev, "gpu post error!\n");
3769 			goto failed;
3770 		}
3771 	}
3772 
3773 	if (adev->is_atom_fw) {
3774 		/* Initialize clocks */
3775 		r = amdgpu_atomfirmware_get_clock_info(adev);
3776 		if (r) {
3777 			dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3778 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3779 			goto failed;
3780 		}
3781 	} else {
3782 		/* Initialize clocks */
3783 		r = amdgpu_atombios_get_clock_info(adev);
3784 		if (r) {
3785 			dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3786 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3787 			goto failed;
3788 		}
3789 		/* init i2c buses */
3790 		if (!amdgpu_device_has_dc_support(adev))
3791 			amdgpu_atombios_i2c_init(adev);
3792 	}
3793 
3794 fence_driver_init:
3795 	/* Fence driver */
3796 	r = amdgpu_fence_driver_sw_init(adev);
3797 	if (r) {
3798 		dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3799 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3800 		goto failed;
3801 	}
3802 
3803 	/* init the mode config */
3804 	drm_mode_config_init(adev_to_drm(adev));
3805 
3806 	r = amdgpu_device_ip_init(adev);
3807 	if (r) {
3808 		/* failed in exclusive mode due to timeout */
3809 		if (amdgpu_sriov_vf(adev) &&
3810 		    !amdgpu_sriov_runtime(adev) &&
3811 		    amdgpu_virt_mmio_blocked(adev) &&
3812 		    !amdgpu_virt_wait_reset(adev)) {
3813 			dev_err(adev->dev, "VF exclusive mode timeout\n");
3814 			/* Don't send request since VF is inactive. */
3815 			adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3816 			adev->virt.ops = NULL;
3817 			r = -EAGAIN;
3818 			goto release_ras_con;
3819 		}
3820 		dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3821 		amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3822 		goto release_ras_con;
3823 	}
3824 
3825 	amdgpu_fence_driver_hw_init(adev);
3826 
3827 	dev_info(adev->dev,
3828 		"SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3829 			adev->gfx.config.max_shader_engines,
3830 			adev->gfx.config.max_sh_per_se,
3831 			adev->gfx.config.max_cu_per_sh,
3832 			adev->gfx.cu_info.number);
3833 
3834 	adev->accel_working = true;
3835 
3836 	amdgpu_vm_check_compute_bug(adev);
3837 
3838 	/* Initialize the buffer migration limit. */
3839 	if (amdgpu_moverate >= 0)
3840 		max_MBps = amdgpu_moverate;
3841 	else
3842 		max_MBps = 8; /* Allow 8 MB/s. */
3843 	/* Get a log2 for easy divisions. */
3844 	adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3845 
3846 	r = amdgpu_pm_sysfs_init(adev);
3847 	if (r) {
3848 		adev->pm_sysfs_en = false;
3849 		DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3850 	} else
3851 		adev->pm_sysfs_en = true;
3852 
3853 	r = amdgpu_ucode_sysfs_init(adev);
3854 	if (r) {
3855 		adev->ucode_sysfs_en = false;
3856 		DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3857 	} else
3858 		adev->ucode_sysfs_en = true;
3859 
3860 	r = amdgpu_psp_sysfs_init(adev);
3861 	if (r) {
3862 		adev->psp_sysfs_en = false;
3863 		if (!amdgpu_sriov_vf(adev))
3864 			DRM_ERROR("Creating psp sysfs failed\n");
3865 	} else
3866 		adev->psp_sysfs_en = true;
3867 
3868 	/*
3869 	 * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3870 	 * Otherwise the mgpu fan boost feature will be skipped due to the
3871 	 * gpu instance is counted less.
3872 	 */
3873 	amdgpu_register_gpu_instance(adev);
3874 
3875 	/* enable clockgating, etc. after ib tests, etc. since some blocks require
3876 	 * explicit gating rather than handling it automatically.
3877 	 */
3878 	if (!adev->gmc.xgmi.pending_reset) {
3879 		r = amdgpu_device_ip_late_init(adev);
3880 		if (r) {
3881 			dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3882 			amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3883 			goto release_ras_con;
3884 		}
3885 		/* must succeed. */
3886 		amdgpu_ras_resume(adev);
3887 		queue_delayed_work(system_wq, &adev->delayed_init_work,
3888 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
3889 	}
3890 
3891 	if (amdgpu_sriov_vf(adev))
3892 		flush_delayed_work(&adev->delayed_init_work);
3893 
3894 	r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3895 	if (r)
3896 		dev_err(adev->dev, "Could not create amdgpu device attr\n");
3897 
3898 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
3899 		r = amdgpu_pmu_init(adev);
3900 	if (r)
3901 		dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3902 
3903 	/* Have stored pci confspace at hand for restore in sudden PCI error */
3904 	if (amdgpu_device_cache_pci_state(adev->pdev))
3905 		pci_restore_state(pdev);
3906 
3907 	/* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3908 	/* this will fail for cards that aren't VGA class devices, just
3909 	 * ignore it */
3910 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3911 		vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
3912 
3913 	if (amdgpu_device_supports_px(ddev)) {
3914 		px = true;
3915 		vga_switcheroo_register_client(adev->pdev,
3916 					       &amdgpu_switcheroo_ops, px);
3917 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3918 	}
3919 
3920 	if (adev->gmc.xgmi.pending_reset)
3921 		queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3922 				   msecs_to_jiffies(AMDGPU_RESUME_MS));
3923 
3924 	amdgpu_device_check_iommu_direct_map(adev);
3925 
3926 	return 0;
3927 
3928 release_ras_con:
3929 	amdgpu_release_ras_context(adev);
3930 
3931 failed:
3932 	amdgpu_vf_error_trans_all(adev);
3933 
3934 	return r;
3935 }
3936 
3937 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3938 {
3939 
3940 	/* Clear all CPU mappings pointing to this device */
3941 	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3942 
3943 	/* Unmap all mapped bars - Doorbell, registers and VRAM */
3944 	amdgpu_device_doorbell_fini(adev);
3945 
3946 	iounmap(adev->rmmio);
3947 	adev->rmmio = NULL;
3948 	if (adev->mman.aper_base_kaddr)
3949 		iounmap(adev->mman.aper_base_kaddr);
3950 	adev->mman.aper_base_kaddr = NULL;
3951 
3952 	/* Memory manager related */
3953 	if (!adev->gmc.xgmi.connected_to_cpu) {
3954 		arch_phys_wc_del(adev->gmc.vram_mtrr);
3955 		arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
3956 	}
3957 }
3958 
3959 /**
3960  * amdgpu_device_fini_hw - tear down the driver
3961  *
3962  * @adev: amdgpu_device pointer
3963  *
3964  * Tear down the driver info (all asics).
3965  * Called at driver shutdown.
3966  */
3967 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
3968 {
3969 	dev_info(adev->dev, "amdgpu: finishing device.\n");
3970 	flush_delayed_work(&adev->delayed_init_work);
3971 	adev->shutdown = true;
3972 
3973 	/* make sure IB test finished before entering exclusive mode
3974 	 * to avoid preemption on IB test
3975 	 * */
3976 	if (amdgpu_sriov_vf(adev)) {
3977 		amdgpu_virt_request_full_gpu(adev, false);
3978 		amdgpu_virt_fini_data_exchange(adev);
3979 	}
3980 
3981 	/* disable all interrupts */
3982 	amdgpu_irq_disable_all(adev);
3983 	if (adev->mode_info.mode_config_initialized){
3984 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
3985 			drm_helper_force_disable_all(adev_to_drm(adev));
3986 		else
3987 			drm_atomic_helper_shutdown(adev_to_drm(adev));
3988 	}
3989 	amdgpu_fence_driver_hw_fini(adev);
3990 
3991 	if (adev->mman.initialized) {
3992 		flush_delayed_work(&adev->mman.bdev.wq);
3993 		ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3994 	}
3995 
3996 	if (adev->pm_sysfs_en)
3997 		amdgpu_pm_sysfs_fini(adev);
3998 	if (adev->ucode_sysfs_en)
3999 		amdgpu_ucode_sysfs_fini(adev);
4000 	if (adev->psp_sysfs_en)
4001 		amdgpu_psp_sysfs_fini(adev);
4002 	sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4003 
4004 	/* disable ras feature must before hw fini */
4005 	amdgpu_ras_pre_fini(adev);
4006 
4007 	amdgpu_device_ip_fini_early(adev);
4008 
4009 	amdgpu_irq_fini_hw(adev);
4010 
4011 	if (adev->mman.initialized)
4012 		ttm_device_clear_dma_mappings(&adev->mman.bdev);
4013 
4014 	amdgpu_gart_dummy_page_fini(adev);
4015 
4016 	amdgpu_device_unmap_mmio(adev);
4017 
4018 }
4019 
4020 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4021 {
4022 	int idx;
4023 
4024 	amdgpu_fence_driver_sw_fini(adev);
4025 	amdgpu_device_ip_fini(adev);
4026 	release_firmware(adev->firmware.gpu_info_fw);
4027 	adev->firmware.gpu_info_fw = NULL;
4028 	adev->accel_working = false;
4029 	dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4030 
4031 	amdgpu_reset_fini(adev);
4032 
4033 	/* free i2c buses */
4034 	if (!amdgpu_device_has_dc_support(adev))
4035 		amdgpu_i2c_fini(adev);
4036 
4037 	if (amdgpu_emu_mode != 1)
4038 		amdgpu_atombios_fini(adev);
4039 
4040 	kfree(adev->bios);
4041 	adev->bios = NULL;
4042 	if (amdgpu_device_supports_px(adev_to_drm(adev))) {
4043 		vga_switcheroo_unregister_client(adev->pdev);
4044 		vga_switcheroo_fini_domain_pm_ops(adev->dev);
4045 	}
4046 	if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4047 		vga_client_unregister(adev->pdev);
4048 
4049 	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4050 
4051 		iounmap(adev->rmmio);
4052 		adev->rmmio = NULL;
4053 		amdgpu_device_doorbell_fini(adev);
4054 		drm_dev_exit(idx);
4055 	}
4056 
4057 	if (IS_ENABLED(CONFIG_PERF_EVENTS))
4058 		amdgpu_pmu_fini(adev);
4059 	if (adev->mman.discovery_bin)
4060 		amdgpu_discovery_fini(adev);
4061 
4062 	amdgpu_reset_put_reset_domain(adev->reset_domain);
4063 	adev->reset_domain = NULL;
4064 
4065 	kfree(adev->pci_state);
4066 
4067 }
4068 
4069 /**
4070  * amdgpu_device_evict_resources - evict device resources
4071  * @adev: amdgpu device object
4072  *
4073  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4074  * of the vram memory type. Mainly used for evicting device resources
4075  * at suspend time.
4076  *
4077  */
4078 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4079 {
4080 	int ret;
4081 
4082 	/* No need to evict vram on APUs for suspend to ram or s2idle */
4083 	if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4084 		return 0;
4085 
4086 	ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4087 	if (ret)
4088 		DRM_WARN("evicting device resources failed\n");
4089 	return ret;
4090 }
4091 
4092 /*
4093  * Suspend & resume.
4094  */
4095 /**
4096  * amdgpu_device_suspend - initiate device suspend
4097  *
4098  * @dev: drm dev pointer
4099  * @fbcon : notify the fbdev of suspend
4100  *
4101  * Puts the hw in the suspend state (all asics).
4102  * Returns 0 for success or an error on failure.
4103  * Called at driver suspend.
4104  */
4105 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4106 {
4107 	struct amdgpu_device *adev = drm_to_adev(dev);
4108 	int r = 0;
4109 
4110 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4111 		return 0;
4112 
4113 	adev->in_suspend = true;
4114 
4115 	if (amdgpu_sriov_vf(adev)) {
4116 		amdgpu_virt_fini_data_exchange(adev);
4117 		r = amdgpu_virt_request_full_gpu(adev, false);
4118 		if (r)
4119 			return r;
4120 	}
4121 
4122 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4123 		DRM_WARN("smart shift update failed\n");
4124 
4125 	drm_kms_helper_poll_disable(dev);
4126 
4127 	if (fbcon)
4128 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4129 
4130 	cancel_delayed_work_sync(&adev->delayed_init_work);
4131 
4132 	amdgpu_ras_suspend(adev);
4133 
4134 	amdgpu_device_ip_suspend_phase1(adev);
4135 
4136 	if (!adev->in_s0ix)
4137 		amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4138 
4139 	r = amdgpu_device_evict_resources(adev);
4140 	if (r)
4141 		return r;
4142 
4143 	amdgpu_fence_driver_hw_fini(adev);
4144 
4145 	amdgpu_device_ip_suspend_phase2(adev);
4146 
4147 	if (amdgpu_sriov_vf(adev))
4148 		amdgpu_virt_release_full_gpu(adev, false);
4149 
4150 	return 0;
4151 }
4152 
4153 /**
4154  * amdgpu_device_resume - initiate device resume
4155  *
4156  * @dev: drm dev pointer
4157  * @fbcon : notify the fbdev of resume
4158  *
4159  * Bring the hw back to operating state (all asics).
4160  * Returns 0 for success or an error on failure.
4161  * Called at driver resume.
4162  */
4163 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4164 {
4165 	struct amdgpu_device *adev = drm_to_adev(dev);
4166 	int r = 0;
4167 
4168 	if (amdgpu_sriov_vf(adev)) {
4169 		r = amdgpu_virt_request_full_gpu(adev, true);
4170 		if (r)
4171 			return r;
4172 	}
4173 
4174 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4175 		return 0;
4176 
4177 	if (adev->in_s0ix)
4178 		amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4179 
4180 	/* post card */
4181 	if (amdgpu_device_need_post(adev)) {
4182 		r = amdgpu_device_asic_init(adev);
4183 		if (r)
4184 			dev_err(adev->dev, "amdgpu asic init failed\n");
4185 	}
4186 
4187 	r = amdgpu_device_ip_resume(adev);
4188 
4189 	if (r) {
4190 		dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4191 		goto exit;
4192 	}
4193 	amdgpu_fence_driver_hw_init(adev);
4194 
4195 	r = amdgpu_device_ip_late_init(adev);
4196 	if (r)
4197 		goto exit;
4198 
4199 	queue_delayed_work(system_wq, &adev->delayed_init_work,
4200 			   msecs_to_jiffies(AMDGPU_RESUME_MS));
4201 
4202 	if (!adev->in_s0ix) {
4203 		r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4204 		if (r)
4205 			goto exit;
4206 	}
4207 
4208 exit:
4209 	if (amdgpu_sriov_vf(adev)) {
4210 		amdgpu_virt_init_data_exchange(adev);
4211 		amdgpu_virt_release_full_gpu(adev, true);
4212 	}
4213 
4214 	if (r)
4215 		return r;
4216 
4217 	/* Make sure IB tests flushed */
4218 	flush_delayed_work(&adev->delayed_init_work);
4219 
4220 	if (adev->in_s0ix) {
4221 		/* re-enable gfxoff after IP resume. This re-enables gfxoff after
4222 		 * it was disabled for IP resume in amdgpu_device_ip_resume_phase2().
4223 		 */
4224 		amdgpu_gfx_off_ctrl(adev, true);
4225 		DRM_DEBUG("will enable gfxoff for the mission mode\n");
4226 	}
4227 	if (fbcon)
4228 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4229 
4230 	drm_kms_helper_poll_enable(dev);
4231 
4232 	amdgpu_ras_resume(adev);
4233 
4234 	if (adev->mode_info.num_crtc) {
4235 		/*
4236 		 * Most of the connector probing functions try to acquire runtime pm
4237 		 * refs to ensure that the GPU is powered on when connector polling is
4238 		 * performed. Since we're calling this from a runtime PM callback,
4239 		 * trying to acquire rpm refs will cause us to deadlock.
4240 		 *
4241 		 * Since we're guaranteed to be holding the rpm lock, it's safe to
4242 		 * temporarily disable the rpm helpers so this doesn't deadlock us.
4243 		 */
4244 #ifdef CONFIG_PM
4245 		dev->dev->power.disable_depth++;
4246 #endif
4247 		if (!adev->dc_enabled)
4248 			drm_helper_hpd_irq_event(dev);
4249 		else
4250 			drm_kms_helper_hotplug_event(dev);
4251 #ifdef CONFIG_PM
4252 		dev->dev->power.disable_depth--;
4253 #endif
4254 	}
4255 	adev->in_suspend = false;
4256 
4257 	if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4258 		DRM_WARN("smart shift update failed\n");
4259 
4260 	return 0;
4261 }
4262 
4263 /**
4264  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4265  *
4266  * @adev: amdgpu_device pointer
4267  *
4268  * The list of all the hardware IPs that make up the asic is walked and
4269  * the check_soft_reset callbacks are run.  check_soft_reset determines
4270  * if the asic is still hung or not.
4271  * Returns true if any of the IPs are still in a hung state, false if not.
4272  */
4273 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4274 {
4275 	int i;
4276 	bool asic_hang = false;
4277 
4278 	if (amdgpu_sriov_vf(adev))
4279 		return true;
4280 
4281 	if (amdgpu_asic_need_full_reset(adev))
4282 		return true;
4283 
4284 	for (i = 0; i < adev->num_ip_blocks; i++) {
4285 		if (!adev->ip_blocks[i].status.valid)
4286 			continue;
4287 		if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4288 			adev->ip_blocks[i].status.hang =
4289 				adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4290 		if (adev->ip_blocks[i].status.hang) {
4291 			dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4292 			asic_hang = true;
4293 		}
4294 	}
4295 	return asic_hang;
4296 }
4297 
4298 /**
4299  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4300  *
4301  * @adev: amdgpu_device pointer
4302  *
4303  * The list of all the hardware IPs that make up the asic is walked and the
4304  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4305  * handles any IP specific hardware or software state changes that are
4306  * necessary for a soft reset to succeed.
4307  * Returns 0 on success, negative error code on failure.
4308  */
4309 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4310 {
4311 	int i, r = 0;
4312 
4313 	for (i = 0; i < adev->num_ip_blocks; i++) {
4314 		if (!adev->ip_blocks[i].status.valid)
4315 			continue;
4316 		if (adev->ip_blocks[i].status.hang &&
4317 		    adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4318 			r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4319 			if (r)
4320 				return r;
4321 		}
4322 	}
4323 
4324 	return 0;
4325 }
4326 
4327 /**
4328  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4329  *
4330  * @adev: amdgpu_device pointer
4331  *
4332  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4333  * reset is necessary to recover.
4334  * Returns true if a full asic reset is required, false if not.
4335  */
4336 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4337 {
4338 	int i;
4339 
4340 	if (amdgpu_asic_need_full_reset(adev))
4341 		return true;
4342 
4343 	for (i = 0; i < adev->num_ip_blocks; i++) {
4344 		if (!adev->ip_blocks[i].status.valid)
4345 			continue;
4346 		if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4347 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4348 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4349 		    (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4350 		     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4351 			if (adev->ip_blocks[i].status.hang) {
4352 				dev_info(adev->dev, "Some block need full reset!\n");
4353 				return true;
4354 			}
4355 		}
4356 	}
4357 	return false;
4358 }
4359 
4360 /**
4361  * amdgpu_device_ip_soft_reset - do a soft reset
4362  *
4363  * @adev: amdgpu_device pointer
4364  *
4365  * The list of all the hardware IPs that make up the asic is walked and the
4366  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4367  * IP specific hardware or software state changes that are necessary to soft
4368  * reset the IP.
4369  * Returns 0 on success, negative error code on failure.
4370  */
4371 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4372 {
4373 	int i, r = 0;
4374 
4375 	for (i = 0; i < adev->num_ip_blocks; i++) {
4376 		if (!adev->ip_blocks[i].status.valid)
4377 			continue;
4378 		if (adev->ip_blocks[i].status.hang &&
4379 		    adev->ip_blocks[i].version->funcs->soft_reset) {
4380 			r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4381 			if (r)
4382 				return r;
4383 		}
4384 	}
4385 
4386 	return 0;
4387 }
4388 
4389 /**
4390  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4391  *
4392  * @adev: amdgpu_device pointer
4393  *
4394  * The list of all the hardware IPs that make up the asic is walked and the
4395  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4396  * handles any IP specific hardware or software state changes that are
4397  * necessary after the IP has been soft reset.
4398  * Returns 0 on success, negative error code on failure.
4399  */
4400 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4401 {
4402 	int i, r = 0;
4403 
4404 	for (i = 0; i < adev->num_ip_blocks; i++) {
4405 		if (!adev->ip_blocks[i].status.valid)
4406 			continue;
4407 		if (adev->ip_blocks[i].status.hang &&
4408 		    adev->ip_blocks[i].version->funcs->post_soft_reset)
4409 			r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4410 		if (r)
4411 			return r;
4412 	}
4413 
4414 	return 0;
4415 }
4416 
4417 /**
4418  * amdgpu_device_recover_vram - Recover some VRAM contents
4419  *
4420  * @adev: amdgpu_device pointer
4421  *
4422  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4423  * restore things like GPUVM page tables after a GPU reset where
4424  * the contents of VRAM might be lost.
4425  *
4426  * Returns:
4427  * 0 on success, negative error code on failure.
4428  */
4429 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4430 {
4431 	struct dma_fence *fence = NULL, *next = NULL;
4432 	struct amdgpu_bo *shadow;
4433 	struct amdgpu_bo_vm *vmbo;
4434 	long r = 1, tmo;
4435 
4436 	if (amdgpu_sriov_runtime(adev))
4437 		tmo = msecs_to_jiffies(8000);
4438 	else
4439 		tmo = msecs_to_jiffies(100);
4440 
4441 	dev_info(adev->dev, "recover vram bo from shadow start\n");
4442 	mutex_lock(&adev->shadow_list_lock);
4443 	list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4444 		shadow = &vmbo->bo;
4445 		/* No need to recover an evicted BO */
4446 		if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4447 		    shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4448 		    shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4449 			continue;
4450 
4451 		r = amdgpu_bo_restore_shadow(shadow, &next);
4452 		if (r)
4453 			break;
4454 
4455 		if (fence) {
4456 			tmo = dma_fence_wait_timeout(fence, false, tmo);
4457 			dma_fence_put(fence);
4458 			fence = next;
4459 			if (tmo == 0) {
4460 				r = -ETIMEDOUT;
4461 				break;
4462 			} else if (tmo < 0) {
4463 				r = tmo;
4464 				break;
4465 			}
4466 		} else {
4467 			fence = next;
4468 		}
4469 	}
4470 	mutex_unlock(&adev->shadow_list_lock);
4471 
4472 	if (fence)
4473 		tmo = dma_fence_wait_timeout(fence, false, tmo);
4474 	dma_fence_put(fence);
4475 
4476 	if (r < 0 || tmo <= 0) {
4477 		dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4478 		return -EIO;
4479 	}
4480 
4481 	dev_info(adev->dev, "recover vram bo from shadow done\n");
4482 	return 0;
4483 }
4484 
4485 
4486 /**
4487  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4488  *
4489  * @adev: amdgpu_device pointer
4490  * @from_hypervisor: request from hypervisor
4491  *
4492  * do VF FLR and reinitialize Asic
4493  * return 0 means succeeded otherwise failed
4494  */
4495 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4496 				     bool from_hypervisor)
4497 {
4498 	int r;
4499 	struct amdgpu_hive_info *hive = NULL;
4500 	int retry_limit = 0;
4501 
4502 retry:
4503 	amdgpu_amdkfd_pre_reset(adev);
4504 
4505 	if (from_hypervisor)
4506 		r = amdgpu_virt_request_full_gpu(adev, true);
4507 	else
4508 		r = amdgpu_virt_reset_gpu(adev);
4509 	if (r)
4510 		return r;
4511 
4512 	/* Resume IP prior to SMC */
4513 	r = amdgpu_device_ip_reinit_early_sriov(adev);
4514 	if (r)
4515 		goto error;
4516 
4517 	amdgpu_virt_init_data_exchange(adev);
4518 
4519 	r = amdgpu_device_fw_loading(adev);
4520 	if (r)
4521 		return r;
4522 
4523 	/* now we are okay to resume SMC/CP/SDMA */
4524 	r = amdgpu_device_ip_reinit_late_sriov(adev);
4525 	if (r)
4526 		goto error;
4527 
4528 	hive = amdgpu_get_xgmi_hive(adev);
4529 	/* Update PSP FW topology after reset */
4530 	if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4531 		r = amdgpu_xgmi_update_topology(hive, adev);
4532 
4533 	if (hive)
4534 		amdgpu_put_xgmi_hive(hive);
4535 
4536 	if (!r) {
4537 		amdgpu_irq_gpu_reset_resume_helper(adev);
4538 		r = amdgpu_ib_ring_tests(adev);
4539 
4540 		amdgpu_amdkfd_post_reset(adev);
4541 	}
4542 
4543 error:
4544 	if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4545 		amdgpu_inc_vram_lost(adev);
4546 		r = amdgpu_device_recover_vram(adev);
4547 	}
4548 	amdgpu_virt_release_full_gpu(adev, true);
4549 
4550 	if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4551 		if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4552 			retry_limit++;
4553 			goto retry;
4554 		} else
4555 			DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4556 	}
4557 
4558 	return r;
4559 }
4560 
4561 /**
4562  * amdgpu_device_has_job_running - check if there is any job in mirror list
4563  *
4564  * @adev: amdgpu_device pointer
4565  *
4566  * check if there is any job in mirror list
4567  */
4568 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4569 {
4570 	int i;
4571 	struct drm_sched_job *job;
4572 
4573 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4574 		struct amdgpu_ring *ring = adev->rings[i];
4575 
4576 		if (!ring || !ring->sched.thread)
4577 			continue;
4578 
4579 		spin_lock(&ring->sched.job_list_lock);
4580 		job = list_first_entry_or_null(&ring->sched.pending_list,
4581 					       struct drm_sched_job, list);
4582 		spin_unlock(&ring->sched.job_list_lock);
4583 		if (job)
4584 			return true;
4585 	}
4586 	return false;
4587 }
4588 
4589 /**
4590  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4591  *
4592  * @adev: amdgpu_device pointer
4593  *
4594  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4595  * a hung GPU.
4596  */
4597 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4598 {
4599 
4600 	if (amdgpu_gpu_recovery == 0)
4601 		goto disabled;
4602 
4603 	/* Skip soft reset check in fatal error mode */
4604 	if (!amdgpu_ras_is_poison_mode_supported(adev))
4605 		return true;
4606 
4607 	if (!amdgpu_device_ip_check_soft_reset(adev)) {
4608 		dev_info(adev->dev,"Timeout, but no hardware hang detected.\n");
4609 		return false;
4610 	}
4611 
4612 	if (amdgpu_sriov_vf(adev))
4613 		return true;
4614 
4615 	if (amdgpu_gpu_recovery == -1) {
4616 		switch (adev->asic_type) {
4617 #ifdef CONFIG_DRM_AMDGPU_SI
4618 		case CHIP_VERDE:
4619 		case CHIP_TAHITI:
4620 		case CHIP_PITCAIRN:
4621 		case CHIP_OLAND:
4622 		case CHIP_HAINAN:
4623 #endif
4624 #ifdef CONFIG_DRM_AMDGPU_CIK
4625 		case CHIP_KAVERI:
4626 		case CHIP_KABINI:
4627 		case CHIP_MULLINS:
4628 #endif
4629 		case CHIP_CARRIZO:
4630 		case CHIP_STONEY:
4631 		case CHIP_CYAN_SKILLFISH:
4632 			goto disabled;
4633 		default:
4634 			break;
4635 		}
4636 	}
4637 
4638 	return true;
4639 
4640 disabled:
4641 		dev_info(adev->dev, "GPU recovery disabled.\n");
4642 		return false;
4643 }
4644 
4645 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4646 {
4647         u32 i;
4648         int ret = 0;
4649 
4650         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4651 
4652         dev_info(adev->dev, "GPU mode1 reset\n");
4653 
4654         /* disable BM */
4655         pci_clear_master(adev->pdev);
4656 
4657         amdgpu_device_cache_pci_state(adev->pdev);
4658 
4659         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4660                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4661                 ret = amdgpu_dpm_mode1_reset(adev);
4662         } else {
4663                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4664                 ret = psp_gpu_reset(adev);
4665         }
4666 
4667         if (ret)
4668                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4669 
4670         amdgpu_device_load_pci_state(adev->pdev);
4671 
4672         /* wait for asic to come out of reset */
4673         for (i = 0; i < adev->usec_timeout; i++) {
4674                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4675 
4676                 if (memsize != 0xffffffff)
4677                         break;
4678                 udelay(1);
4679         }
4680 
4681         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4682         return ret;
4683 }
4684 
4685 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4686 				 struct amdgpu_reset_context *reset_context)
4687 {
4688 	int i, r = 0;
4689 	struct amdgpu_job *job = NULL;
4690 	bool need_full_reset =
4691 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4692 
4693 	if (reset_context->reset_req_dev == adev)
4694 		job = reset_context->job;
4695 
4696 	if (amdgpu_sriov_vf(adev)) {
4697 		/* stop the data exchange thread */
4698 		amdgpu_virt_fini_data_exchange(adev);
4699 	}
4700 
4701 	amdgpu_fence_driver_isr_toggle(adev, true);
4702 
4703 	/* block all schedulers and reset given job's ring */
4704 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4705 		struct amdgpu_ring *ring = adev->rings[i];
4706 
4707 		if (!ring || !ring->sched.thread)
4708 			continue;
4709 
4710 		/*clear job fence from fence drv to avoid force_completion
4711 		 *leave NULL and vm flush fence in fence drv */
4712 		amdgpu_fence_driver_clear_job_fences(ring);
4713 
4714 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4715 		amdgpu_fence_driver_force_completion(ring);
4716 	}
4717 
4718 	amdgpu_fence_driver_isr_toggle(adev, false);
4719 
4720 	if (job && job->vm)
4721 		drm_sched_increase_karma(&job->base);
4722 
4723 	r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4724 	/* If reset handler not implemented, continue; otherwise return */
4725 	if (r == -ENOSYS)
4726 		r = 0;
4727 	else
4728 		return r;
4729 
4730 	/* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4731 	if (!amdgpu_sriov_vf(adev)) {
4732 
4733 		if (!need_full_reset)
4734 			need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4735 
4736 		if (!need_full_reset && amdgpu_gpu_recovery) {
4737 			amdgpu_device_ip_pre_soft_reset(adev);
4738 			r = amdgpu_device_ip_soft_reset(adev);
4739 			amdgpu_device_ip_post_soft_reset(adev);
4740 			if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4741 				dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4742 				need_full_reset = true;
4743 			}
4744 		}
4745 
4746 		if (need_full_reset)
4747 			r = amdgpu_device_ip_suspend(adev);
4748 		if (need_full_reset)
4749 			set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4750 		else
4751 			clear_bit(AMDGPU_NEED_FULL_RESET,
4752 				  &reset_context->flags);
4753 	}
4754 
4755 	return r;
4756 }
4757 
4758 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
4759 {
4760 	int i;
4761 
4762 	lockdep_assert_held(&adev->reset_domain->sem);
4763 
4764 	for (i = 0; i < adev->num_regs; i++) {
4765 		adev->reset_dump_reg_value[i] = RREG32(adev->reset_dump_reg_list[i]);
4766 		trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i],
4767 					     adev->reset_dump_reg_value[i]);
4768 	}
4769 
4770 	return 0;
4771 }
4772 
4773 #ifdef CONFIG_DEV_COREDUMP
4774 static ssize_t amdgpu_devcoredump_read(char *buffer, loff_t offset,
4775 		size_t count, void *data, size_t datalen)
4776 {
4777 	struct drm_printer p;
4778 	struct amdgpu_device *adev = data;
4779 	struct drm_print_iterator iter;
4780 	int i;
4781 
4782 	iter.data = buffer;
4783 	iter.offset = 0;
4784 	iter.start = offset;
4785 	iter.remain = count;
4786 
4787 	p = drm_coredump_printer(&iter);
4788 
4789 	drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
4790 	drm_printf(&p, "kernel: " UTS_RELEASE "\n");
4791 	drm_printf(&p, "module: " KBUILD_MODNAME "\n");
4792 	drm_printf(&p, "time: %lld.%09ld\n", adev->reset_time.tv_sec, adev->reset_time.tv_nsec);
4793 	if (adev->reset_task_info.pid)
4794 		drm_printf(&p, "process_name: %s PID: %d\n",
4795 			   adev->reset_task_info.process_name,
4796 			   adev->reset_task_info.pid);
4797 
4798 	if (adev->reset_vram_lost)
4799 		drm_printf(&p, "VRAM is lost due to GPU reset!\n");
4800 	if (adev->num_regs) {
4801 		drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
4802 
4803 		for (i = 0; i < adev->num_regs; i++)
4804 			drm_printf(&p, "0x%08x: 0x%08x\n",
4805 				   adev->reset_dump_reg_list[i],
4806 				   adev->reset_dump_reg_value[i]);
4807 	}
4808 
4809 	return count - iter.remain;
4810 }
4811 
4812 static void amdgpu_devcoredump_free(void *data)
4813 {
4814 }
4815 
4816 static void amdgpu_reset_capture_coredumpm(struct amdgpu_device *adev)
4817 {
4818 	struct drm_device *dev = adev_to_drm(adev);
4819 
4820 	ktime_get_ts64(&adev->reset_time);
4821 	dev_coredumpm(dev->dev, THIS_MODULE, adev, 0, GFP_KERNEL,
4822 		      amdgpu_devcoredump_read, amdgpu_devcoredump_free);
4823 }
4824 #endif
4825 
4826 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4827 			 struct amdgpu_reset_context *reset_context)
4828 {
4829 	struct amdgpu_device *tmp_adev = NULL;
4830 	bool need_full_reset, skip_hw_reset, vram_lost = false;
4831 	int r = 0;
4832 	bool gpu_reset_for_dev_remove = 0;
4833 
4834 	/* Try reset handler method first */
4835 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4836 				    reset_list);
4837 	amdgpu_reset_reg_dumps(tmp_adev);
4838 
4839 	reset_context->reset_device_list = device_list_handle;
4840 	r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4841 	/* If reset handler not implemented, continue; otherwise return */
4842 	if (r == -ENOSYS)
4843 		r = 0;
4844 	else
4845 		return r;
4846 
4847 	/* Reset handler not implemented, use the default method */
4848 	need_full_reset =
4849 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4850 	skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4851 
4852 	gpu_reset_for_dev_remove =
4853 		test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
4854 			test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4855 
4856 	/*
4857 	 * ASIC reset has to be done on all XGMI hive nodes ASAP
4858 	 * to allow proper links negotiation in FW (within 1 sec)
4859 	 */
4860 	if (!skip_hw_reset && need_full_reset) {
4861 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4862 			/* For XGMI run all resets in parallel to speed up the process */
4863 			if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4864 				tmp_adev->gmc.xgmi.pending_reset = false;
4865 				if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4866 					r = -EALREADY;
4867 			} else
4868 				r = amdgpu_asic_reset(tmp_adev);
4869 
4870 			if (r) {
4871 				dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4872 					 r, adev_to_drm(tmp_adev)->unique);
4873 				break;
4874 			}
4875 		}
4876 
4877 		/* For XGMI wait for all resets to complete before proceed */
4878 		if (!r) {
4879 			list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4880 				if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4881 					flush_work(&tmp_adev->xgmi_reset_work);
4882 					r = tmp_adev->asic_reset_res;
4883 					if (r)
4884 						break;
4885 				}
4886 			}
4887 		}
4888 	}
4889 
4890 	if (!r && amdgpu_ras_intr_triggered()) {
4891 		list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4892 			if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops &&
4893 			    tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
4894 				tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev);
4895 		}
4896 
4897 		amdgpu_ras_intr_cleared();
4898 	}
4899 
4900 	/* Since the mode1 reset affects base ip blocks, the
4901 	 * phase1 ip blocks need to be resumed. Otherwise there
4902 	 * will be a BIOS signature error and the psp bootloader
4903 	 * can't load kdb on the next amdgpu install.
4904 	 */
4905 	if (gpu_reset_for_dev_remove) {
4906 		list_for_each_entry(tmp_adev, device_list_handle, reset_list)
4907 			amdgpu_device_ip_resume_phase1(tmp_adev);
4908 
4909 		goto end;
4910 	}
4911 
4912 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4913 		if (need_full_reset) {
4914 			/* post card */
4915 			r = amdgpu_device_asic_init(tmp_adev);
4916 			if (r) {
4917 				dev_warn(tmp_adev->dev, "asic atom init failed!");
4918 			} else {
4919 				dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4920 				r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4921 				if (r)
4922 					goto out;
4923 
4924 				r = amdgpu_device_ip_resume_phase1(tmp_adev);
4925 				if (r)
4926 					goto out;
4927 
4928 				vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4929 #ifdef CONFIG_DEV_COREDUMP
4930 				tmp_adev->reset_vram_lost = vram_lost;
4931 				memset(&tmp_adev->reset_task_info, 0,
4932 						sizeof(tmp_adev->reset_task_info));
4933 				if (reset_context->job && reset_context->job->vm)
4934 					tmp_adev->reset_task_info =
4935 						reset_context->job->vm->task_info;
4936 				amdgpu_reset_capture_coredumpm(tmp_adev);
4937 #endif
4938 				if (vram_lost) {
4939 					DRM_INFO("VRAM is lost due to GPU reset!\n");
4940 					amdgpu_inc_vram_lost(tmp_adev);
4941 				}
4942 
4943 				r = amdgpu_device_fw_loading(tmp_adev);
4944 				if (r)
4945 					return r;
4946 
4947 				r = amdgpu_device_ip_resume_phase2(tmp_adev);
4948 				if (r)
4949 					goto out;
4950 
4951 				if (vram_lost)
4952 					amdgpu_device_fill_reset_magic(tmp_adev);
4953 
4954 				/*
4955 				 * Add this ASIC as tracked as reset was already
4956 				 * complete successfully.
4957 				 */
4958 				amdgpu_register_gpu_instance(tmp_adev);
4959 
4960 				if (!reset_context->hive &&
4961 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4962 					amdgpu_xgmi_add_device(tmp_adev);
4963 
4964 				r = amdgpu_device_ip_late_init(tmp_adev);
4965 				if (r)
4966 					goto out;
4967 
4968 				drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
4969 
4970 				/*
4971 				 * The GPU enters bad state once faulty pages
4972 				 * by ECC has reached the threshold, and ras
4973 				 * recovery is scheduled next. So add one check
4974 				 * here to break recovery if it indeed exceeds
4975 				 * bad page threshold, and remind user to
4976 				 * retire this GPU or setting one bigger
4977 				 * bad_page_threshold value to fix this once
4978 				 * probing driver again.
4979 				 */
4980 				if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4981 					/* must succeed. */
4982 					amdgpu_ras_resume(tmp_adev);
4983 				} else {
4984 					r = -EINVAL;
4985 					goto out;
4986 				}
4987 
4988 				/* Update PSP FW topology after reset */
4989 				if (reset_context->hive &&
4990 				    tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4991 					r = amdgpu_xgmi_update_topology(
4992 						reset_context->hive, tmp_adev);
4993 			}
4994 		}
4995 
4996 out:
4997 		if (!r) {
4998 			amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4999 			r = amdgpu_ib_ring_tests(tmp_adev);
5000 			if (r) {
5001 				dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5002 				need_full_reset = true;
5003 				r = -EAGAIN;
5004 				goto end;
5005 			}
5006 		}
5007 
5008 		if (!r)
5009 			r = amdgpu_device_recover_vram(tmp_adev);
5010 		else
5011 			tmp_adev->asic_reset_res = r;
5012 	}
5013 
5014 end:
5015 	if (need_full_reset)
5016 		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5017 	else
5018 		clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5019 	return r;
5020 }
5021 
5022 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5023 {
5024 
5025 	switch (amdgpu_asic_reset_method(adev)) {
5026 	case AMD_RESET_METHOD_MODE1:
5027 		adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5028 		break;
5029 	case AMD_RESET_METHOD_MODE2:
5030 		adev->mp1_state = PP_MP1_STATE_RESET;
5031 		break;
5032 	default:
5033 		adev->mp1_state = PP_MP1_STATE_NONE;
5034 		break;
5035 	}
5036 }
5037 
5038 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5039 {
5040 	amdgpu_vf_error_trans_all(adev);
5041 	adev->mp1_state = PP_MP1_STATE_NONE;
5042 }
5043 
5044 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5045 {
5046 	struct pci_dev *p = NULL;
5047 
5048 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5049 			adev->pdev->bus->number, 1);
5050 	if (p) {
5051 		pm_runtime_enable(&(p->dev));
5052 		pm_runtime_resume(&(p->dev));
5053 	}
5054 
5055 	pci_dev_put(p);
5056 }
5057 
5058 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5059 {
5060 	enum amd_reset_method reset_method;
5061 	struct pci_dev *p = NULL;
5062 	u64 expires;
5063 
5064 	/*
5065 	 * For now, only BACO and mode1 reset are confirmed
5066 	 * to suffer the audio issue without proper suspended.
5067 	 */
5068 	reset_method = amdgpu_asic_reset_method(adev);
5069 	if ((reset_method != AMD_RESET_METHOD_BACO) &&
5070 	     (reset_method != AMD_RESET_METHOD_MODE1))
5071 		return -EINVAL;
5072 
5073 	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5074 			adev->pdev->bus->number, 1);
5075 	if (!p)
5076 		return -ENODEV;
5077 
5078 	expires = pm_runtime_autosuspend_expiration(&(p->dev));
5079 	if (!expires)
5080 		/*
5081 		 * If we cannot get the audio device autosuspend delay,
5082 		 * a fixed 4S interval will be used. Considering 3S is
5083 		 * the audio controller default autosuspend delay setting.
5084 		 * 4S used here is guaranteed to cover that.
5085 		 */
5086 		expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5087 
5088 	while (!pm_runtime_status_suspended(&(p->dev))) {
5089 		if (!pm_runtime_suspend(&(p->dev)))
5090 			break;
5091 
5092 		if (expires < ktime_get_mono_fast_ns()) {
5093 			dev_warn(adev->dev, "failed to suspend display audio\n");
5094 			pci_dev_put(p);
5095 			/* TODO: abort the succeeding gpu reset? */
5096 			return -ETIMEDOUT;
5097 		}
5098 	}
5099 
5100 	pm_runtime_disable(&(p->dev));
5101 
5102 	pci_dev_put(p);
5103 	return 0;
5104 }
5105 
5106 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5107 {
5108 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5109 
5110 #if defined(CONFIG_DEBUG_FS)
5111 	if (!amdgpu_sriov_vf(adev))
5112 		cancel_work(&adev->reset_work);
5113 #endif
5114 
5115 	if (adev->kfd.dev)
5116 		cancel_work(&adev->kfd.reset_work);
5117 
5118 	if (amdgpu_sriov_vf(adev))
5119 		cancel_work(&adev->virt.flr_work);
5120 
5121 	if (con && adev->ras_enabled)
5122 		cancel_work(&con->recovery_work);
5123 
5124 }
5125 
5126 /**
5127  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5128  *
5129  * @adev: amdgpu_device pointer
5130  * @job: which job trigger hang
5131  *
5132  * Attempt to reset the GPU if it has hung (all asics).
5133  * Attempt to do soft-reset or full-reset and reinitialize Asic
5134  * Returns 0 for success or an error on failure.
5135  */
5136 
5137 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5138 			      struct amdgpu_job *job,
5139 			      struct amdgpu_reset_context *reset_context)
5140 {
5141 	struct list_head device_list, *device_list_handle =  NULL;
5142 	bool job_signaled = false;
5143 	struct amdgpu_hive_info *hive = NULL;
5144 	struct amdgpu_device *tmp_adev = NULL;
5145 	int i, r = 0;
5146 	bool need_emergency_restart = false;
5147 	bool audio_suspended = false;
5148 	bool gpu_reset_for_dev_remove = false;
5149 
5150 	gpu_reset_for_dev_remove =
5151 			test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5152 				test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5153 
5154 	/*
5155 	 * Special case: RAS triggered and full reset isn't supported
5156 	 */
5157 	need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5158 
5159 	/*
5160 	 * Flush RAM to disk so that after reboot
5161 	 * the user can read log and see why the system rebooted.
5162 	 */
5163 	if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
5164 		DRM_WARN("Emergency reboot.");
5165 
5166 		ksys_sync_helper();
5167 		emergency_restart();
5168 	}
5169 
5170 	dev_info(adev->dev, "GPU %s begin!\n",
5171 		need_emergency_restart ? "jobs stop":"reset");
5172 
5173 	if (!amdgpu_sriov_vf(adev))
5174 		hive = amdgpu_get_xgmi_hive(adev);
5175 	if (hive)
5176 		mutex_lock(&hive->hive_lock);
5177 
5178 	reset_context->job = job;
5179 	reset_context->hive = hive;
5180 	/*
5181 	 * Build list of devices to reset.
5182 	 * In case we are in XGMI hive mode, resort the device list
5183 	 * to put adev in the 1st position.
5184 	 */
5185 	INIT_LIST_HEAD(&device_list);
5186 	if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5187 		list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5188 			list_add_tail(&tmp_adev->reset_list, &device_list);
5189 			if (gpu_reset_for_dev_remove && adev->shutdown)
5190 				tmp_adev->shutdown = true;
5191 		}
5192 		if (!list_is_first(&adev->reset_list, &device_list))
5193 			list_rotate_to_front(&adev->reset_list, &device_list);
5194 		device_list_handle = &device_list;
5195 	} else {
5196 		list_add_tail(&adev->reset_list, &device_list);
5197 		device_list_handle = &device_list;
5198 	}
5199 
5200 	/* We need to lock reset domain only once both for XGMI and single device */
5201 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5202 				    reset_list);
5203 	amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5204 
5205 	/* block all schedulers and reset given job's ring */
5206 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5207 
5208 		amdgpu_device_set_mp1_state(tmp_adev);
5209 
5210 		/*
5211 		 * Try to put the audio codec into suspend state
5212 		 * before gpu reset started.
5213 		 *
5214 		 * Due to the power domain of the graphics device
5215 		 * is shared with AZ power domain. Without this,
5216 		 * we may change the audio hardware from behind
5217 		 * the audio driver's back. That will trigger
5218 		 * some audio codec errors.
5219 		 */
5220 		if (!amdgpu_device_suspend_display_audio(tmp_adev))
5221 			audio_suspended = true;
5222 
5223 		amdgpu_ras_set_error_query_ready(tmp_adev, false);
5224 
5225 		cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5226 
5227 		if (!amdgpu_sriov_vf(tmp_adev))
5228 			amdgpu_amdkfd_pre_reset(tmp_adev);
5229 
5230 		/*
5231 		 * Mark these ASICs to be reseted as untracked first
5232 		 * And add them back after reset completed
5233 		 */
5234 		amdgpu_unregister_gpu_instance(tmp_adev);
5235 
5236 		drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5237 
5238 		/* disable ras on ALL IPs */
5239 		if (!need_emergency_restart &&
5240 		      amdgpu_device_ip_need_full_reset(tmp_adev))
5241 			amdgpu_ras_suspend(tmp_adev);
5242 
5243 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5244 			struct amdgpu_ring *ring = tmp_adev->rings[i];
5245 
5246 			if (!ring || !ring->sched.thread)
5247 				continue;
5248 
5249 			drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5250 
5251 			if (need_emergency_restart)
5252 				amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5253 		}
5254 		atomic_inc(&tmp_adev->gpu_reset_counter);
5255 	}
5256 
5257 	if (need_emergency_restart)
5258 		goto skip_sched_resume;
5259 
5260 	/*
5261 	 * Must check guilty signal here since after this point all old
5262 	 * HW fences are force signaled.
5263 	 *
5264 	 * job->base holds a reference to parent fence
5265 	 */
5266 	if (job && dma_fence_is_signaled(&job->hw_fence)) {
5267 		job_signaled = true;
5268 		dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5269 		goto skip_hw_reset;
5270 	}
5271 
5272 retry:	/* Rest of adevs pre asic reset from XGMI hive. */
5273 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5274 		if (gpu_reset_for_dev_remove) {
5275 			/* Workaroud for ASICs need to disable SMC first */
5276 			amdgpu_device_smu_fini_early(tmp_adev);
5277 		}
5278 		r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5279 		/*TODO Should we stop ?*/
5280 		if (r) {
5281 			dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5282 				  r, adev_to_drm(tmp_adev)->unique);
5283 			tmp_adev->asic_reset_res = r;
5284 		}
5285 
5286 		/*
5287 		 * Drop all pending non scheduler resets. Scheduler resets
5288 		 * were already dropped during drm_sched_stop
5289 		 */
5290 		amdgpu_device_stop_pending_resets(tmp_adev);
5291 	}
5292 
5293 	/* Actual ASIC resets if needed.*/
5294 	/* Host driver will handle XGMI hive reset for SRIOV */
5295 	if (amdgpu_sriov_vf(adev)) {
5296 		r = amdgpu_device_reset_sriov(adev, job ? false : true);
5297 		if (r)
5298 			adev->asic_reset_res = r;
5299 
5300 		/* Aldebaran supports ras in SRIOV, so need resume ras during reset */
5301 		if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
5302 			amdgpu_ras_resume(adev);
5303 	} else {
5304 		r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5305 		if (r && r == -EAGAIN)
5306 			goto retry;
5307 
5308 		if (!r && gpu_reset_for_dev_remove)
5309 			goto recover_end;
5310 	}
5311 
5312 skip_hw_reset:
5313 
5314 	/* Post ASIC reset for all devs .*/
5315 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5316 
5317 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5318 			struct amdgpu_ring *ring = tmp_adev->rings[i];
5319 
5320 			if (!ring || !ring->sched.thread)
5321 				continue;
5322 
5323 			drm_sched_start(&ring->sched, true);
5324 		}
5325 
5326 		if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3))
5327 			amdgpu_mes_self_test(tmp_adev);
5328 
5329 		if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5330 			drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5331 		}
5332 
5333 		if (tmp_adev->asic_reset_res)
5334 			r = tmp_adev->asic_reset_res;
5335 
5336 		tmp_adev->asic_reset_res = 0;
5337 
5338 		if (r) {
5339 			/* bad news, how to tell it to userspace ? */
5340 			dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5341 			amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5342 		} else {
5343 			dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5344 			if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5345 				DRM_WARN("smart shift update failed\n");
5346 		}
5347 	}
5348 
5349 skip_sched_resume:
5350 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5351 		/* unlock kfd: SRIOV would do it separately */
5352 		if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5353 			amdgpu_amdkfd_post_reset(tmp_adev);
5354 
5355 		/* kfd_post_reset will do nothing if kfd device is not initialized,
5356 		 * need to bring up kfd here if it's not be initialized before
5357 		 */
5358 		if (!adev->kfd.init_complete)
5359 			amdgpu_amdkfd_device_init(adev);
5360 
5361 		if (audio_suspended)
5362 			amdgpu_device_resume_display_audio(tmp_adev);
5363 
5364 		amdgpu_device_unset_mp1_state(tmp_adev);
5365 
5366 		amdgpu_ras_set_error_query_ready(tmp_adev, true);
5367 	}
5368 
5369 recover_end:
5370 	tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5371 					    reset_list);
5372 	amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5373 
5374 	if (hive) {
5375 		mutex_unlock(&hive->hive_lock);
5376 		amdgpu_put_xgmi_hive(hive);
5377 	}
5378 
5379 	if (r)
5380 		dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5381 
5382 	atomic_set(&adev->reset_domain->reset_res, r);
5383 	return r;
5384 }
5385 
5386 /**
5387  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5388  *
5389  * @adev: amdgpu_device pointer
5390  *
5391  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5392  * and lanes) of the slot the device is in. Handles APUs and
5393  * virtualized environments where PCIE config space may not be available.
5394  */
5395 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5396 {
5397 	struct pci_dev *pdev;
5398 	enum pci_bus_speed speed_cap, platform_speed_cap;
5399 	enum pcie_link_width platform_link_width;
5400 
5401 	if (amdgpu_pcie_gen_cap)
5402 		adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5403 
5404 	if (amdgpu_pcie_lane_cap)
5405 		adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5406 
5407 	/* covers APUs as well */
5408 	if (pci_is_root_bus(adev->pdev->bus)) {
5409 		if (adev->pm.pcie_gen_mask == 0)
5410 			adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5411 		if (adev->pm.pcie_mlw_mask == 0)
5412 			adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5413 		return;
5414 	}
5415 
5416 	if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5417 		return;
5418 
5419 	pcie_bandwidth_available(adev->pdev, NULL,
5420 				 &platform_speed_cap, &platform_link_width);
5421 
5422 	if (adev->pm.pcie_gen_mask == 0) {
5423 		/* asic caps */
5424 		pdev = adev->pdev;
5425 		speed_cap = pcie_get_speed_cap(pdev);
5426 		if (speed_cap == PCI_SPEED_UNKNOWN) {
5427 			adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5428 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5429 						  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5430 		} else {
5431 			if (speed_cap == PCIE_SPEED_32_0GT)
5432 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5433 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5434 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5435 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5436 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5437 			else if (speed_cap == PCIE_SPEED_16_0GT)
5438 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5439 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5440 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5441 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5442 			else if (speed_cap == PCIE_SPEED_8_0GT)
5443 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5444 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5445 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5446 			else if (speed_cap == PCIE_SPEED_5_0GT)
5447 				adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5448 							  CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5449 			else
5450 				adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5451 		}
5452 		/* platform caps */
5453 		if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5454 			adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5455 						   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5456 		} else {
5457 			if (platform_speed_cap == PCIE_SPEED_32_0GT)
5458 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5459 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5460 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5461 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5462 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5463 			else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5464 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5465 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5466 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5467 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5468 			else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5469 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5470 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5471 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5472 			else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5473 				adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5474 							   CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5475 			else
5476 				adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5477 
5478 		}
5479 	}
5480 	if (adev->pm.pcie_mlw_mask == 0) {
5481 		if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5482 			adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5483 		} else {
5484 			switch (platform_link_width) {
5485 			case PCIE_LNK_X32:
5486 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5487 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5488 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5489 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5490 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5491 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5492 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5493 				break;
5494 			case PCIE_LNK_X16:
5495 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5496 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5497 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5498 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5499 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5500 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5501 				break;
5502 			case PCIE_LNK_X12:
5503 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5504 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5505 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5506 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5507 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5508 				break;
5509 			case PCIE_LNK_X8:
5510 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5511 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5512 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5513 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5514 				break;
5515 			case PCIE_LNK_X4:
5516 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5517 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5518 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5519 				break;
5520 			case PCIE_LNK_X2:
5521 				adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5522 							  CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5523 				break;
5524 			case PCIE_LNK_X1:
5525 				adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5526 				break;
5527 			default:
5528 				break;
5529 			}
5530 		}
5531 	}
5532 }
5533 
5534 /**
5535  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5536  *
5537  * @adev: amdgpu_device pointer
5538  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5539  *
5540  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5541  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5542  * @peer_adev.
5543  */
5544 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5545 				      struct amdgpu_device *peer_adev)
5546 {
5547 #ifdef CONFIG_HSA_AMD_P2P
5548 	uint64_t address_mask = peer_adev->dev->dma_mask ?
5549 		~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5550 	resource_size_t aper_limit =
5551 		adev->gmc.aper_base + adev->gmc.aper_size - 1;
5552 	bool p2p_access =
5553 		!adev->gmc.xgmi.connected_to_cpu &&
5554 		!(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5555 
5556 	return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5557 		adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5558 		!(adev->gmc.aper_base & address_mask ||
5559 		  aper_limit & address_mask));
5560 #else
5561 	return false;
5562 #endif
5563 }
5564 
5565 int amdgpu_device_baco_enter(struct drm_device *dev)
5566 {
5567 	struct amdgpu_device *adev = drm_to_adev(dev);
5568 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5569 
5570 	if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5571 		return -ENOTSUPP;
5572 
5573 	if (ras && adev->ras_enabled &&
5574 	    adev->nbio.funcs->enable_doorbell_interrupt)
5575 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5576 
5577 	return amdgpu_dpm_baco_enter(adev);
5578 }
5579 
5580 int amdgpu_device_baco_exit(struct drm_device *dev)
5581 {
5582 	struct amdgpu_device *adev = drm_to_adev(dev);
5583 	struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5584 	int ret = 0;
5585 
5586 	if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5587 		return -ENOTSUPP;
5588 
5589 	ret = amdgpu_dpm_baco_exit(adev);
5590 	if (ret)
5591 		return ret;
5592 
5593 	if (ras && adev->ras_enabled &&
5594 	    adev->nbio.funcs->enable_doorbell_interrupt)
5595 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5596 
5597 	if (amdgpu_passthrough(adev) &&
5598 	    adev->nbio.funcs->clear_doorbell_interrupt)
5599 		adev->nbio.funcs->clear_doorbell_interrupt(adev);
5600 
5601 	return 0;
5602 }
5603 
5604 /**
5605  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5606  * @pdev: PCI device struct
5607  * @state: PCI channel state
5608  *
5609  * Description: Called when a PCI error is detected.
5610  *
5611  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5612  */
5613 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5614 {
5615 	struct drm_device *dev = pci_get_drvdata(pdev);
5616 	struct amdgpu_device *adev = drm_to_adev(dev);
5617 	int i;
5618 
5619 	DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5620 
5621 	if (adev->gmc.xgmi.num_physical_nodes > 1) {
5622 		DRM_WARN("No support for XGMI hive yet...");
5623 		return PCI_ERS_RESULT_DISCONNECT;
5624 	}
5625 
5626 	adev->pci_channel_state = state;
5627 
5628 	switch (state) {
5629 	case pci_channel_io_normal:
5630 		return PCI_ERS_RESULT_CAN_RECOVER;
5631 	/* Fatal error, prepare for slot reset */
5632 	case pci_channel_io_frozen:
5633 		/*
5634 		 * Locking adev->reset_domain->sem will prevent any external access
5635 		 * to GPU during PCI error recovery
5636 		 */
5637 		amdgpu_device_lock_reset_domain(adev->reset_domain);
5638 		amdgpu_device_set_mp1_state(adev);
5639 
5640 		/*
5641 		 * Block any work scheduling as we do for regular GPU reset
5642 		 * for the duration of the recovery
5643 		 */
5644 		for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5645 			struct amdgpu_ring *ring = adev->rings[i];
5646 
5647 			if (!ring || !ring->sched.thread)
5648 				continue;
5649 
5650 			drm_sched_stop(&ring->sched, NULL);
5651 		}
5652 		atomic_inc(&adev->gpu_reset_counter);
5653 		return PCI_ERS_RESULT_NEED_RESET;
5654 	case pci_channel_io_perm_failure:
5655 		/* Permanent error, prepare for device removal */
5656 		return PCI_ERS_RESULT_DISCONNECT;
5657 	}
5658 
5659 	return PCI_ERS_RESULT_NEED_RESET;
5660 }
5661 
5662 /**
5663  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5664  * @pdev: pointer to PCI device
5665  */
5666 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5667 {
5668 
5669 	DRM_INFO("PCI error: mmio enabled callback!!\n");
5670 
5671 	/* TODO - dump whatever for debugging purposes */
5672 
5673 	/* This called only if amdgpu_pci_error_detected returns
5674 	 * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5675 	 * works, no need to reset slot.
5676 	 */
5677 
5678 	return PCI_ERS_RESULT_RECOVERED;
5679 }
5680 
5681 /**
5682  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5683  * @pdev: PCI device struct
5684  *
5685  * Description: This routine is called by the pci error recovery
5686  * code after the PCI slot has been reset, just before we
5687  * should resume normal operations.
5688  */
5689 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5690 {
5691 	struct drm_device *dev = pci_get_drvdata(pdev);
5692 	struct amdgpu_device *adev = drm_to_adev(dev);
5693 	int r, i;
5694 	struct amdgpu_reset_context reset_context;
5695 	u32 memsize;
5696 	struct list_head device_list;
5697 
5698 	DRM_INFO("PCI error: slot reset callback!!\n");
5699 
5700 	memset(&reset_context, 0, sizeof(reset_context));
5701 
5702 	INIT_LIST_HEAD(&device_list);
5703 	list_add_tail(&adev->reset_list, &device_list);
5704 
5705 	/* wait for asic to come out of reset */
5706 	msleep(500);
5707 
5708 	/* Restore PCI confspace */
5709 	amdgpu_device_load_pci_state(pdev);
5710 
5711 	/* confirm  ASIC came out of reset */
5712 	for (i = 0; i < adev->usec_timeout; i++) {
5713 		memsize = amdgpu_asic_get_config_memsize(adev);
5714 
5715 		if (memsize != 0xffffffff)
5716 			break;
5717 		udelay(1);
5718 	}
5719 	if (memsize == 0xffffffff) {
5720 		r = -ETIME;
5721 		goto out;
5722 	}
5723 
5724 	reset_context.method = AMD_RESET_METHOD_NONE;
5725 	reset_context.reset_req_dev = adev;
5726 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5727 	set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5728 
5729 	adev->no_hw_access = true;
5730 	r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5731 	adev->no_hw_access = false;
5732 	if (r)
5733 		goto out;
5734 
5735 	r = amdgpu_do_asic_reset(&device_list, &reset_context);
5736 
5737 out:
5738 	if (!r) {
5739 		if (amdgpu_device_cache_pci_state(adev->pdev))
5740 			pci_restore_state(adev->pdev);
5741 
5742 		DRM_INFO("PCIe error recovery succeeded\n");
5743 	} else {
5744 		DRM_ERROR("PCIe error recovery failed, err:%d", r);
5745 		amdgpu_device_unset_mp1_state(adev);
5746 		amdgpu_device_unlock_reset_domain(adev->reset_domain);
5747 	}
5748 
5749 	return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5750 }
5751 
5752 /**
5753  * amdgpu_pci_resume() - resume normal ops after PCI reset
5754  * @pdev: pointer to PCI device
5755  *
5756  * Called when the error recovery driver tells us that its
5757  * OK to resume normal operation.
5758  */
5759 void amdgpu_pci_resume(struct pci_dev *pdev)
5760 {
5761 	struct drm_device *dev = pci_get_drvdata(pdev);
5762 	struct amdgpu_device *adev = drm_to_adev(dev);
5763 	int i;
5764 
5765 
5766 	DRM_INFO("PCI error: resume callback!!\n");
5767 
5768 	/* Only continue execution for the case of pci_channel_io_frozen */
5769 	if (adev->pci_channel_state != pci_channel_io_frozen)
5770 		return;
5771 
5772 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5773 		struct amdgpu_ring *ring = adev->rings[i];
5774 
5775 		if (!ring || !ring->sched.thread)
5776 			continue;
5777 
5778 		drm_sched_start(&ring->sched, true);
5779 	}
5780 
5781 	amdgpu_device_unset_mp1_state(adev);
5782 	amdgpu_device_unlock_reset_domain(adev->reset_domain);
5783 }
5784 
5785 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5786 {
5787 	struct drm_device *dev = pci_get_drvdata(pdev);
5788 	struct amdgpu_device *adev = drm_to_adev(dev);
5789 	int r;
5790 
5791 	r = pci_save_state(pdev);
5792 	if (!r) {
5793 		kfree(adev->pci_state);
5794 
5795 		adev->pci_state = pci_store_saved_state(pdev);
5796 
5797 		if (!adev->pci_state) {
5798 			DRM_ERROR("Failed to store PCI saved state");
5799 			return false;
5800 		}
5801 	} else {
5802 		DRM_WARN("Failed to save PCI state, err:%d\n", r);
5803 		return false;
5804 	}
5805 
5806 	return true;
5807 }
5808 
5809 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5810 {
5811 	struct drm_device *dev = pci_get_drvdata(pdev);
5812 	struct amdgpu_device *adev = drm_to_adev(dev);
5813 	int r;
5814 
5815 	if (!adev->pci_state)
5816 		return false;
5817 
5818 	r = pci_load_saved_state(pdev, adev->pci_state);
5819 
5820 	if (!r) {
5821 		pci_restore_state(pdev);
5822 	} else {
5823 		DRM_WARN("Failed to load PCI state, err:%d\n", r);
5824 		return false;
5825 	}
5826 
5827 	return true;
5828 }
5829 
5830 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5831 		struct amdgpu_ring *ring)
5832 {
5833 #ifdef CONFIG_X86_64
5834 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5835 		return;
5836 #endif
5837 	if (adev->gmc.xgmi.connected_to_cpu)
5838 		return;
5839 
5840 	if (ring && ring->funcs->emit_hdp_flush)
5841 		amdgpu_ring_emit_hdp_flush(ring);
5842 	else
5843 		amdgpu_asic_flush_hdp(adev, ring);
5844 }
5845 
5846 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5847 		struct amdgpu_ring *ring)
5848 {
5849 #ifdef CONFIG_X86_64
5850 	if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5851 		return;
5852 #endif
5853 	if (adev->gmc.xgmi.connected_to_cpu)
5854 		return;
5855 
5856 	amdgpu_asic_invalidate_hdp(adev, ring);
5857 }
5858 
5859 int amdgpu_in_reset(struct amdgpu_device *adev)
5860 {
5861 	return atomic_read(&adev->reset_domain->in_gpu_reset);
5862 	}
5863 
5864 /**
5865  * amdgpu_device_halt() - bring hardware to some kind of halt state
5866  *
5867  * @adev: amdgpu_device pointer
5868  *
5869  * Bring hardware to some kind of halt state so that no one can touch it
5870  * any more. It will help to maintain error context when error occurred.
5871  * Compare to a simple hang, the system will keep stable at least for SSH
5872  * access. Then it should be trivial to inspect the hardware state and
5873  * see what's going on. Implemented as following:
5874  *
5875  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
5876  *    clears all CPU mappings to device, disallows remappings through page faults
5877  * 2. amdgpu_irq_disable_all() disables all interrupts
5878  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
5879  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
5880  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
5881  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
5882  *    flush any in flight DMA operations
5883  */
5884 void amdgpu_device_halt(struct amdgpu_device *adev)
5885 {
5886 	struct pci_dev *pdev = adev->pdev;
5887 	struct drm_device *ddev = adev_to_drm(adev);
5888 
5889 	drm_dev_unplug(ddev);
5890 
5891 	amdgpu_irq_disable_all(adev);
5892 
5893 	amdgpu_fence_driver_hw_fini(adev);
5894 
5895 	adev->no_hw_access = true;
5896 
5897 	amdgpu_device_unmap_mmio(adev);
5898 
5899 	pci_disable_device(pdev);
5900 	pci_wait_for_pending_transaction(pdev);
5901 }
5902 
5903 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
5904 				u32 reg)
5905 {
5906 	unsigned long flags, address, data;
5907 	u32 r;
5908 
5909 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5910 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5911 
5912 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5913 	WREG32(address, reg * 4);
5914 	(void)RREG32(address);
5915 	r = RREG32(data);
5916 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5917 	return r;
5918 }
5919 
5920 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
5921 				u32 reg, u32 v)
5922 {
5923 	unsigned long flags, address, data;
5924 
5925 	address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5926 	data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5927 
5928 	spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5929 	WREG32(address, reg * 4);
5930 	(void)RREG32(address);
5931 	WREG32(data, v);
5932 	(void)RREG32(data);
5933 	spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5934 }
5935 
5936 /**
5937  * amdgpu_device_switch_gang - switch to a new gang
5938  * @adev: amdgpu_device pointer
5939  * @gang: the gang to switch to
5940  *
5941  * Try to switch to a new gang.
5942  * Returns: NULL if we switched to the new gang or a reference to the current
5943  * gang leader.
5944  */
5945 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
5946 					    struct dma_fence *gang)
5947 {
5948 	struct dma_fence *old = NULL;
5949 
5950 	do {
5951 		dma_fence_put(old);
5952 		rcu_read_lock();
5953 		old = dma_fence_get_rcu_safe(&adev->gang_submit);
5954 		rcu_read_unlock();
5955 
5956 		if (old == gang)
5957 			break;
5958 
5959 		if (!dma_fence_is_signaled(old))
5960 			return old;
5961 
5962 	} while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
5963 			 old, gang) != old);
5964 
5965 	dma_fence_put(old);
5966 	return NULL;
5967 }
5968 
5969 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
5970 {
5971 	switch (adev->asic_type) {
5972 #ifdef CONFIG_DRM_AMDGPU_SI
5973 	case CHIP_HAINAN:
5974 #endif
5975 	case CHIP_TOPAZ:
5976 		/* chips with no display hardware */
5977 		return false;
5978 #ifdef CONFIG_DRM_AMDGPU_SI
5979 	case CHIP_TAHITI:
5980 	case CHIP_PITCAIRN:
5981 	case CHIP_VERDE:
5982 	case CHIP_OLAND:
5983 #endif
5984 #ifdef CONFIG_DRM_AMDGPU_CIK
5985 	case CHIP_BONAIRE:
5986 	case CHIP_HAWAII:
5987 	case CHIP_KAVERI:
5988 	case CHIP_KABINI:
5989 	case CHIP_MULLINS:
5990 #endif
5991 	case CHIP_TONGA:
5992 	case CHIP_FIJI:
5993 	case CHIP_POLARIS10:
5994 	case CHIP_POLARIS11:
5995 	case CHIP_POLARIS12:
5996 	case CHIP_VEGAM:
5997 	case CHIP_CARRIZO:
5998 	case CHIP_STONEY:
5999 		/* chips with display hardware */
6000 		return true;
6001 	default:
6002 		/* IP discovery */
6003 		if (!adev->ip_versions[DCE_HWIP][0] ||
6004 		    (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6005 			return false;
6006 		return true;
6007 	}
6008 }
6009