1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  */
25 
26 #include <linux/kthread.h>
27 #include <linux/pci.h>
28 #include <linux/uaccess.h>
29 #include <linux/pm_runtime.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_pm.h"
33 #include "amdgpu_dm_debugfs.h"
34 #include "amdgpu_ras.h"
35 #include "amdgpu_rap.h"
36 #include "amdgpu_securedisplay.h"
37 #include "amdgpu_fw_attestation.h"
38 #include "amdgpu_umr.h"
39 
40 #include "amdgpu_reset.h"
41 #include "amdgpu_psp_ta.h"
42 
43 #if defined(CONFIG_DEBUG_FS)
44 
45 /**
46  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
47  *
48  * @read: True if reading
49  * @f: open file handle
50  * @buf: User buffer to write/read to
51  * @size: Number of bytes to write/read
52  * @pos:  Offset to seek to
53  *
54  * This debugfs entry has special meaning on the offset being sought.
55  * Various bits have different meanings:
56  *
57  * Bit 62:  Indicates a GRBM bank switch is needed
58  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
59  * 	    zero)
60  * Bits 24..33: The SE or ME selector if needed
61  * Bits 34..43: The SH (or SA) or PIPE selector if needed
62  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
63  *
64  * Bit 23:  Indicates that the PM power gating lock should be held
65  * 	    This is necessary to read registers that might be
66  * 	    unreliable during a power gating transistion.
67  *
68  * The lower bits are the BYTE offset of the register to read.  This
69  * allows reading multiple registers in a single call and having
70  * the returned size reflect that.
71  */
72 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
73 		char __user *buf, size_t size, loff_t *pos)
74 {
75 	struct amdgpu_device *adev = file_inode(f)->i_private;
76 	ssize_t result = 0;
77 	int r;
78 	bool pm_pg_lock, use_bank, use_ring;
79 	unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
80 
81 	pm_pg_lock = use_bank = use_ring = false;
82 	instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
83 
84 	if (size & 0x3 || *pos & 0x3 ||
85 			((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
86 		return -EINVAL;
87 
88 	/* are we reading registers for which a PG lock is necessary? */
89 	pm_pg_lock = (*pos >> 23) & 1;
90 
91 	if (*pos & (1ULL << 62)) {
92 		se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
93 		sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
94 		instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
95 
96 		if (se_bank == 0x3FF)
97 			se_bank = 0xFFFFFFFF;
98 		if (sh_bank == 0x3FF)
99 			sh_bank = 0xFFFFFFFF;
100 		if (instance_bank == 0x3FF)
101 			instance_bank = 0xFFFFFFFF;
102 		use_bank = true;
103 	} else if (*pos & (1ULL << 61)) {
104 
105 		me = (*pos & GENMASK_ULL(33, 24)) >> 24;
106 		pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
107 		queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
108 		vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
109 
110 		use_ring = true;
111 	} else {
112 		use_bank = use_ring = false;
113 	}
114 
115 	*pos &= (1UL << 22) - 1;
116 
117 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
118 	if (r < 0) {
119 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
120 		return r;
121 	}
122 
123 	r = amdgpu_virt_enable_access_debugfs(adev);
124 	if (r < 0) {
125 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
126 		return r;
127 	}
128 
129 	if (use_bank) {
130 		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
131 		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
132 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
133 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
134 			amdgpu_virt_disable_access_debugfs(adev);
135 			return -EINVAL;
136 		}
137 		mutex_lock(&adev->grbm_idx_mutex);
138 		amdgpu_gfx_select_se_sh(adev, se_bank,
139 					sh_bank, instance_bank);
140 	} else if (use_ring) {
141 		mutex_lock(&adev->srbm_mutex);
142 		amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
143 	}
144 
145 	if (pm_pg_lock)
146 		mutex_lock(&adev->pm.mutex);
147 
148 	while (size) {
149 		uint32_t value;
150 
151 		if (read) {
152 			value = RREG32(*pos >> 2);
153 			r = put_user(value, (uint32_t *)buf);
154 		} else {
155 			r = get_user(value, (uint32_t *)buf);
156 			if (!r)
157 				amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
158 		}
159 		if (r) {
160 			result = r;
161 			goto end;
162 		}
163 
164 		result += 4;
165 		buf += 4;
166 		*pos += 4;
167 		size -= 4;
168 	}
169 
170 end:
171 	if (use_bank) {
172 		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
173 		mutex_unlock(&adev->grbm_idx_mutex);
174 	} else if (use_ring) {
175 		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
176 		mutex_unlock(&adev->srbm_mutex);
177 	}
178 
179 	if (pm_pg_lock)
180 		mutex_unlock(&adev->pm.mutex);
181 
182 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
183 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
184 
185 	amdgpu_virt_disable_access_debugfs(adev);
186 	return result;
187 }
188 
189 /*
190  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
191  */
192 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
193 					size_t size, loff_t *pos)
194 {
195 	return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
196 }
197 
198 /*
199  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
200  */
201 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
202 					 size_t size, loff_t *pos)
203 {
204 	return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
205 }
206 
207 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
208 {
209 	struct amdgpu_debugfs_regs2_data *rd;
210 
211 	rd = kzalloc(sizeof *rd, GFP_KERNEL);
212 	if (!rd)
213 		return -ENOMEM;
214 	rd->adev = file_inode(file)->i_private;
215 	file->private_data = rd;
216 	mutex_init(&rd->lock);
217 
218 	return 0;
219 }
220 
221 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
222 {
223 	struct amdgpu_debugfs_regs2_data *rd = file->private_data;
224 	mutex_destroy(&rd->lock);
225 	kfree(file->private_data);
226 	return 0;
227 }
228 
229 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
230 {
231 	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
232 	struct amdgpu_device *adev = rd->adev;
233 	ssize_t result = 0;
234 	int r;
235 	uint32_t value;
236 
237 	if (size & 0x3 || offset & 0x3)
238 		return -EINVAL;
239 
240 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
241 	if (r < 0) {
242 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
243 		return r;
244 	}
245 
246 	r = amdgpu_virt_enable_access_debugfs(adev);
247 	if (r < 0) {
248 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
249 		return r;
250 	}
251 
252 	mutex_lock(&rd->lock);
253 
254 	if (rd->id.use_grbm) {
255 		if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
256 		    (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
257 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
258 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
259 			amdgpu_virt_disable_access_debugfs(adev);
260 			mutex_unlock(&rd->lock);
261 			return -EINVAL;
262 		}
263 		mutex_lock(&adev->grbm_idx_mutex);
264 		amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
265 								rd->id.grbm.sh,
266 								rd->id.grbm.instance);
267 	}
268 
269 	if (rd->id.use_srbm) {
270 		mutex_lock(&adev->srbm_mutex);
271 		amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
272 									rd->id.srbm.queue, rd->id.srbm.vmid);
273 	}
274 
275 	if (rd->id.pg_lock)
276 		mutex_lock(&adev->pm.mutex);
277 
278 	while (size) {
279 		if (!write_en) {
280 			value = RREG32(offset >> 2);
281 			r = put_user(value, (uint32_t *)buf);
282 		} else {
283 			r = get_user(value, (uint32_t *)buf);
284 			if (!r)
285 				amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value);
286 		}
287 		if (r) {
288 			result = r;
289 			goto end;
290 		}
291 		offset += 4;
292 		size -= 4;
293 		result += 4;
294 		buf += 4;
295 	}
296 end:
297 	if (rd->id.use_grbm) {
298 		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
299 		mutex_unlock(&adev->grbm_idx_mutex);
300 	}
301 
302 	if (rd->id.use_srbm) {
303 		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
304 		mutex_unlock(&adev->srbm_mutex);
305 	}
306 
307 	if (rd->id.pg_lock)
308 		mutex_unlock(&adev->pm.mutex);
309 
310 	mutex_unlock(&rd->lock);
311 
312 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
313 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
314 
315 	amdgpu_virt_disable_access_debugfs(adev);
316 	return result;
317 }
318 
319 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
320 {
321 	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
322 	int r;
323 
324 	switch (cmd) {
325 	case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
326 		mutex_lock(&rd->lock);
327 		r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata *)data, sizeof rd->id);
328 		mutex_unlock(&rd->lock);
329 		return r ? -EINVAL : 0;
330 	default:
331 		return -EINVAL;
332 	}
333 	return 0;
334 }
335 
336 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
337 {
338 	return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
339 }
340 
341 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
342 {
343 	return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
344 }
345 
346 
347 /**
348  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
349  *
350  * @f: open file handle
351  * @buf: User buffer to store read data in
352  * @size: Number of bytes to read
353  * @pos:  Offset to seek to
354  *
355  * The lower bits are the BYTE offset of the register to read.  This
356  * allows reading multiple registers in a single call and having
357  * the returned size reflect that.
358  */
359 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
360 					size_t size, loff_t *pos)
361 {
362 	struct amdgpu_device *adev = file_inode(f)->i_private;
363 	ssize_t result = 0;
364 	int r;
365 
366 	if (size & 0x3 || *pos & 0x3)
367 		return -EINVAL;
368 
369 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
370 	if (r < 0) {
371 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
372 		return r;
373 	}
374 
375 	r = amdgpu_virt_enable_access_debugfs(adev);
376 	if (r < 0) {
377 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
378 		return r;
379 	}
380 
381 	while (size) {
382 		uint32_t value;
383 
384 		value = RREG32_PCIE(*pos);
385 		r = put_user(value, (uint32_t *)buf);
386 		if (r)
387 			goto out;
388 
389 		result += 4;
390 		buf += 4;
391 		*pos += 4;
392 		size -= 4;
393 	}
394 
395 	r = result;
396 out:
397 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
398 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
399 	amdgpu_virt_disable_access_debugfs(adev);
400 	return r;
401 }
402 
403 /**
404  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
405  *
406  * @f: open file handle
407  * @buf: User buffer to write data from
408  * @size: Number of bytes to write
409  * @pos:  Offset to seek to
410  *
411  * The lower bits are the BYTE offset of the register to write.  This
412  * allows writing multiple registers in a single call and having
413  * the returned size reflect that.
414  */
415 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
416 					 size_t size, loff_t *pos)
417 {
418 	struct amdgpu_device *adev = file_inode(f)->i_private;
419 	ssize_t result = 0;
420 	int r;
421 
422 	if (size & 0x3 || *pos & 0x3)
423 		return -EINVAL;
424 
425 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
426 	if (r < 0) {
427 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
428 		return r;
429 	}
430 
431 	r = amdgpu_virt_enable_access_debugfs(adev);
432 	if (r < 0) {
433 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
434 		return r;
435 	}
436 
437 	while (size) {
438 		uint32_t value;
439 
440 		r = get_user(value, (uint32_t *)buf);
441 		if (r)
442 			goto out;
443 
444 		WREG32_PCIE(*pos, value);
445 
446 		result += 4;
447 		buf += 4;
448 		*pos += 4;
449 		size -= 4;
450 	}
451 
452 	r = result;
453 out:
454 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
455 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
456 	amdgpu_virt_disable_access_debugfs(adev);
457 	return r;
458 }
459 
460 /**
461  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
462  *
463  * @f: open file handle
464  * @buf: User buffer to store read data in
465  * @size: Number of bytes to read
466  * @pos:  Offset to seek to
467  *
468  * The lower bits are the BYTE offset of the register to read.  This
469  * allows reading multiple registers in a single call and having
470  * the returned size reflect that.
471  */
472 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
473 					size_t size, loff_t *pos)
474 {
475 	struct amdgpu_device *adev = file_inode(f)->i_private;
476 	ssize_t result = 0;
477 	int r;
478 
479 	if (size & 0x3 || *pos & 0x3)
480 		return -EINVAL;
481 
482 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
483 	if (r < 0) {
484 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
485 		return r;
486 	}
487 
488 	r = amdgpu_virt_enable_access_debugfs(adev);
489 	if (r < 0) {
490 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
491 		return r;
492 	}
493 
494 	while (size) {
495 		uint32_t value;
496 
497 		value = RREG32_DIDT(*pos >> 2);
498 		r = put_user(value, (uint32_t *)buf);
499 		if (r)
500 			goto out;
501 
502 		result += 4;
503 		buf += 4;
504 		*pos += 4;
505 		size -= 4;
506 	}
507 
508 	r = result;
509 out:
510 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
511 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
512 	amdgpu_virt_disable_access_debugfs(adev);
513 	return r;
514 }
515 
516 /**
517  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
518  *
519  * @f: open file handle
520  * @buf: User buffer to write data from
521  * @size: Number of bytes to write
522  * @pos:  Offset to seek to
523  *
524  * The lower bits are the BYTE offset of the register to write.  This
525  * allows writing multiple registers in a single call and having
526  * the returned size reflect that.
527  */
528 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
529 					 size_t size, loff_t *pos)
530 {
531 	struct amdgpu_device *adev = file_inode(f)->i_private;
532 	ssize_t result = 0;
533 	int r;
534 
535 	if (size & 0x3 || *pos & 0x3)
536 		return -EINVAL;
537 
538 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
539 	if (r < 0) {
540 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
541 		return r;
542 	}
543 
544 	r = amdgpu_virt_enable_access_debugfs(adev);
545 	if (r < 0) {
546 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
547 		return r;
548 	}
549 
550 	while (size) {
551 		uint32_t value;
552 
553 		r = get_user(value, (uint32_t *)buf);
554 		if (r)
555 			goto out;
556 
557 		WREG32_DIDT(*pos >> 2, value);
558 
559 		result += 4;
560 		buf += 4;
561 		*pos += 4;
562 		size -= 4;
563 	}
564 
565 	r = result;
566 out:
567 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
568 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
569 	amdgpu_virt_disable_access_debugfs(adev);
570 	return r;
571 }
572 
573 /**
574  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
575  *
576  * @f: open file handle
577  * @buf: User buffer to store read data in
578  * @size: Number of bytes to read
579  * @pos:  Offset to seek to
580  *
581  * The lower bits are the BYTE offset of the register to read.  This
582  * allows reading multiple registers in a single call and having
583  * the returned size reflect that.
584  */
585 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
586 					size_t size, loff_t *pos)
587 {
588 	struct amdgpu_device *adev = file_inode(f)->i_private;
589 	ssize_t result = 0;
590 	int r;
591 
592 	if (size & 0x3 || *pos & 0x3)
593 		return -EINVAL;
594 
595 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
596 	if (r < 0) {
597 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
598 		return r;
599 	}
600 
601 	r = amdgpu_virt_enable_access_debugfs(adev);
602 	if (r < 0) {
603 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
604 		return r;
605 	}
606 
607 	while (size) {
608 		uint32_t value;
609 
610 		value = RREG32_SMC(*pos);
611 		r = put_user(value, (uint32_t *)buf);
612 		if (r)
613 			goto out;
614 
615 		result += 4;
616 		buf += 4;
617 		*pos += 4;
618 		size -= 4;
619 	}
620 
621 	r = result;
622 out:
623 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
624 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
625 	amdgpu_virt_disable_access_debugfs(adev);
626 	return r;
627 }
628 
629 /**
630  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
631  *
632  * @f: open file handle
633  * @buf: User buffer to write data from
634  * @size: Number of bytes to write
635  * @pos:  Offset to seek to
636  *
637  * The lower bits are the BYTE offset of the register to write.  This
638  * allows writing multiple registers in a single call and having
639  * the returned size reflect that.
640  */
641 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
642 					 size_t size, loff_t *pos)
643 {
644 	struct amdgpu_device *adev = file_inode(f)->i_private;
645 	ssize_t result = 0;
646 	int r;
647 
648 	if (size & 0x3 || *pos & 0x3)
649 		return -EINVAL;
650 
651 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
652 	if (r < 0) {
653 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
654 		return r;
655 	}
656 
657 	r = amdgpu_virt_enable_access_debugfs(adev);
658 	if (r < 0) {
659 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
660 		return r;
661 	}
662 
663 	while (size) {
664 		uint32_t value;
665 
666 		r = get_user(value, (uint32_t *)buf);
667 		if (r)
668 			goto out;
669 
670 		WREG32_SMC(*pos, value);
671 
672 		result += 4;
673 		buf += 4;
674 		*pos += 4;
675 		size -= 4;
676 	}
677 
678 	r = result;
679 out:
680 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
681 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
682 	amdgpu_virt_disable_access_debugfs(adev);
683 	return r;
684 }
685 
686 /**
687  * amdgpu_debugfs_gca_config_read - Read from gfx config data
688  *
689  * @f: open file handle
690  * @buf: User buffer to store read data in
691  * @size: Number of bytes to read
692  * @pos:  Offset to seek to
693  *
694  * This file is used to access configuration data in a somewhat
695  * stable fashion.  The format is a series of DWORDs with the first
696  * indicating which revision it is.  New content is appended to the
697  * end so that older software can still read the data.
698  */
699 
700 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
701 					size_t size, loff_t *pos)
702 {
703 	struct amdgpu_device *adev = file_inode(f)->i_private;
704 	ssize_t result = 0;
705 	int r;
706 	uint32_t *config, no_regs = 0;
707 
708 	if (size & 0x3 || *pos & 0x3)
709 		return -EINVAL;
710 
711 	config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
712 	if (!config)
713 		return -ENOMEM;
714 
715 	/* version, increment each time something is added */
716 	config[no_regs++] = 5;
717 	config[no_regs++] = adev->gfx.config.max_shader_engines;
718 	config[no_regs++] = adev->gfx.config.max_tile_pipes;
719 	config[no_regs++] = adev->gfx.config.max_cu_per_sh;
720 	config[no_regs++] = adev->gfx.config.max_sh_per_se;
721 	config[no_regs++] = adev->gfx.config.max_backends_per_se;
722 	config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
723 	config[no_regs++] = adev->gfx.config.max_gprs;
724 	config[no_regs++] = adev->gfx.config.max_gs_threads;
725 	config[no_regs++] = adev->gfx.config.max_hw_contexts;
726 	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
727 	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
728 	config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
729 	config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
730 	config[no_regs++] = adev->gfx.config.num_tile_pipes;
731 	config[no_regs++] = adev->gfx.config.backend_enable_mask;
732 	config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
733 	config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
734 	config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
735 	config[no_regs++] = adev->gfx.config.num_gpus;
736 	config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
737 	config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
738 	config[no_regs++] = adev->gfx.config.gb_addr_config;
739 	config[no_regs++] = adev->gfx.config.num_rbs;
740 
741 	/* rev==1 */
742 	config[no_regs++] = adev->rev_id;
743 	config[no_regs++] = lower_32_bits(adev->pg_flags);
744 	config[no_regs++] = lower_32_bits(adev->cg_flags);
745 
746 	/* rev==2 */
747 	config[no_regs++] = adev->family;
748 	config[no_regs++] = adev->external_rev_id;
749 
750 	/* rev==3 */
751 	config[no_regs++] = adev->pdev->device;
752 	config[no_regs++] = adev->pdev->revision;
753 	config[no_regs++] = adev->pdev->subsystem_device;
754 	config[no_regs++] = adev->pdev->subsystem_vendor;
755 
756 	/* rev==4 APU flag */
757 	config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
758 
759 	/* rev==5 PG/CG flag upper 32bit */
760 	config[no_regs++] = upper_32_bits(adev->pg_flags);
761 	config[no_regs++] = upper_32_bits(adev->cg_flags);
762 
763 	while (size && (*pos < no_regs * 4)) {
764 		uint32_t value;
765 
766 		value = config[*pos >> 2];
767 		r = put_user(value, (uint32_t *)buf);
768 		if (r) {
769 			kfree(config);
770 			return r;
771 		}
772 
773 		result += 4;
774 		buf += 4;
775 		*pos += 4;
776 		size -= 4;
777 	}
778 
779 	kfree(config);
780 	return result;
781 }
782 
783 /**
784  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
785  *
786  * @f: open file handle
787  * @buf: User buffer to store read data in
788  * @size: Number of bytes to read
789  * @pos:  Offset to seek to
790  *
791  * The offset is treated as the BYTE address of one of the sensors
792  * enumerated in amd/include/kgd_pp_interface.h under the
793  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
794  * you would use the offset 3 * 4 = 12.
795  */
796 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
797 					size_t size, loff_t *pos)
798 {
799 	struct amdgpu_device *adev = file_inode(f)->i_private;
800 	int idx, x, outsize, r, valuesize;
801 	uint32_t values[16];
802 
803 	if (size & 3 || *pos & 0x3)
804 		return -EINVAL;
805 
806 	if (!adev->pm.dpm_enabled)
807 		return -EINVAL;
808 
809 	/* convert offset to sensor number */
810 	idx = *pos >> 2;
811 
812 	valuesize = sizeof(values);
813 
814 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
815 	if (r < 0) {
816 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
817 		return r;
818 	}
819 
820 	r = amdgpu_virt_enable_access_debugfs(adev);
821 	if (r < 0) {
822 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
823 		return r;
824 	}
825 
826 	r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
827 
828 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
829 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
830 
831 	if (r) {
832 		amdgpu_virt_disable_access_debugfs(adev);
833 		return r;
834 	}
835 
836 	if (size > valuesize) {
837 		amdgpu_virt_disable_access_debugfs(adev);
838 		return -EINVAL;
839 	}
840 
841 	outsize = 0;
842 	x = 0;
843 	if (!r) {
844 		while (size) {
845 			r = put_user(values[x++], (int32_t *)buf);
846 			buf += 4;
847 			size -= 4;
848 			outsize += 4;
849 		}
850 	}
851 
852 	amdgpu_virt_disable_access_debugfs(adev);
853 	return !r ? outsize : r;
854 }
855 
856 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
857  *
858  * @f: open file handle
859  * @buf: User buffer to store read data in
860  * @size: Number of bytes to read
861  * @pos:  Offset to seek to
862  *
863  * The offset being sought changes which wave that the status data
864  * will be returned for.  The bits are used as follows:
865  *
866  * Bits 0..6: 	Byte offset into data
867  * Bits 7..14:	SE selector
868  * Bits 15..22:	SH/SA selector
869  * Bits 23..30: CU/{WGP+SIMD} selector
870  * Bits 31..36: WAVE ID selector
871  * Bits 37..44: SIMD ID selector
872  *
873  * The returned data begins with one DWORD of version information
874  * Followed by WAVE STATUS registers relevant to the GFX IP version
875  * being used.  See gfx_v8_0_read_wave_data() for an example output.
876  */
877 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
878 					size_t size, loff_t *pos)
879 {
880 	struct amdgpu_device *adev = f->f_inode->i_private;
881 	int r, x;
882 	ssize_t result = 0;
883 	uint32_t offset, se, sh, cu, wave, simd, data[32];
884 
885 	if (size & 3 || *pos & 3)
886 		return -EINVAL;
887 
888 	/* decode offset */
889 	offset = (*pos & GENMASK_ULL(6, 0));
890 	se = (*pos & GENMASK_ULL(14, 7)) >> 7;
891 	sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
892 	cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
893 	wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
894 	simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
895 
896 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
897 	if (r < 0) {
898 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
899 		return r;
900 	}
901 
902 	r = amdgpu_virt_enable_access_debugfs(adev);
903 	if (r < 0) {
904 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
905 		return r;
906 	}
907 
908 	/* switch to the specific se/sh/cu */
909 	mutex_lock(&adev->grbm_idx_mutex);
910 	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
911 
912 	x = 0;
913 	if (adev->gfx.funcs->read_wave_data)
914 		adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
915 
916 	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
917 	mutex_unlock(&adev->grbm_idx_mutex);
918 
919 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
920 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
921 
922 	if (!x) {
923 		amdgpu_virt_disable_access_debugfs(adev);
924 		return -EINVAL;
925 	}
926 
927 	while (size && (offset < x * 4)) {
928 		uint32_t value;
929 
930 		value = data[offset >> 2];
931 		r = put_user(value, (uint32_t *)buf);
932 		if (r) {
933 			amdgpu_virt_disable_access_debugfs(adev);
934 			return r;
935 		}
936 
937 		result += 4;
938 		buf += 4;
939 		offset += 4;
940 		size -= 4;
941 	}
942 
943 	amdgpu_virt_disable_access_debugfs(adev);
944 	return result;
945 }
946 
947 /** amdgpu_debugfs_gpr_read - Read wave gprs
948  *
949  * @f: open file handle
950  * @buf: User buffer to store read data in
951  * @size: Number of bytes to read
952  * @pos:  Offset to seek to
953  *
954  * The offset being sought changes which wave that the status data
955  * will be returned for.  The bits are used as follows:
956  *
957  * Bits 0..11:	Byte offset into data
958  * Bits 12..19:	SE selector
959  * Bits 20..27:	SH/SA selector
960  * Bits 28..35: CU/{WGP+SIMD} selector
961  * Bits 36..43: WAVE ID selector
962  * Bits 37..44: SIMD ID selector
963  * Bits 52..59: Thread selector
964  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
965  *
966  * The return data comes from the SGPR or VGPR register bank for
967  * the selected operational unit.
968  */
969 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
970 					size_t size, loff_t *pos)
971 {
972 	struct amdgpu_device *adev = f->f_inode->i_private;
973 	int r;
974 	ssize_t result = 0;
975 	uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
976 
977 	if (size > 4096 || size & 3 || *pos & 3)
978 		return -EINVAL;
979 
980 	/* decode offset */
981 	offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
982 	se = (*pos & GENMASK_ULL(19, 12)) >> 12;
983 	sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
984 	cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
985 	wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
986 	simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
987 	thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
988 	bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
989 
990 	data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
991 	if (!data)
992 		return -ENOMEM;
993 
994 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
995 	if (r < 0)
996 		goto err;
997 
998 	r = amdgpu_virt_enable_access_debugfs(adev);
999 	if (r < 0)
1000 		goto err;
1001 
1002 	/* switch to the specific se/sh/cu */
1003 	mutex_lock(&adev->grbm_idx_mutex);
1004 	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
1005 
1006 	if (bank == 0) {
1007 		if (adev->gfx.funcs->read_wave_vgprs)
1008 			adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
1009 	} else {
1010 		if (adev->gfx.funcs->read_wave_sgprs)
1011 			adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
1012 	}
1013 
1014 	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
1015 	mutex_unlock(&adev->grbm_idx_mutex);
1016 
1017 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1018 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1019 
1020 	while (size) {
1021 		uint32_t value;
1022 
1023 		value = data[result >> 2];
1024 		r = put_user(value, (uint32_t *)buf);
1025 		if (r) {
1026 			amdgpu_virt_disable_access_debugfs(adev);
1027 			goto err;
1028 		}
1029 
1030 		result += 4;
1031 		buf += 4;
1032 		size -= 4;
1033 	}
1034 
1035 	kfree(data);
1036 	amdgpu_virt_disable_access_debugfs(adev);
1037 	return result;
1038 
1039 err:
1040 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1041 	kfree(data);
1042 	return r;
1043 }
1044 
1045 /**
1046  * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1047  *
1048  * @f: open file handle
1049  * @buf: User buffer to store read data in
1050  * @size: Number of bytes to read
1051  * @pos:  Offset to seek to
1052  *
1053  * Read the last residency value logged. It doesn't auto update, one needs to
1054  * stop logging before getting the current value.
1055  */
1056 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1057 						    size_t size, loff_t *pos)
1058 {
1059 	struct amdgpu_device *adev = file_inode(f)->i_private;
1060 	ssize_t result = 0;
1061 	int r;
1062 
1063 	if (size & 0x3 || *pos & 0x3)
1064 		return -EINVAL;
1065 
1066 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1067 	if (r < 0) {
1068 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1069 		return r;
1070 	}
1071 
1072 	while (size) {
1073 		uint32_t value;
1074 
1075 		r = amdgpu_get_gfx_off_residency(adev, &value);
1076 		if (r)
1077 			goto out;
1078 
1079 		r = put_user(value, (uint32_t *)buf);
1080 		if (r)
1081 			goto out;
1082 
1083 		result += 4;
1084 		buf += 4;
1085 		*pos += 4;
1086 		size -= 4;
1087 	}
1088 
1089 	r = result;
1090 out:
1091 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1092 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1093 
1094 	return r;
1095 }
1096 
1097 /**
1098  * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1099  *
1100  * @f: open file handle
1101  * @buf: User buffer to write data from
1102  * @size: Number of bytes to write
1103  * @pos:  Offset to seek to
1104  *
1105  * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1106  */
1107 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1108 						     size_t size, loff_t *pos)
1109 {
1110 	struct amdgpu_device *adev = file_inode(f)->i_private;
1111 	ssize_t result = 0;
1112 	int r;
1113 
1114 	if (size & 0x3 || *pos & 0x3)
1115 		return -EINVAL;
1116 
1117 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1118 	if (r < 0) {
1119 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1120 		return r;
1121 	}
1122 
1123 	while (size) {
1124 		u32 value;
1125 
1126 		r = get_user(value, (uint32_t *)buf);
1127 		if (r)
1128 			goto out;
1129 
1130 		amdgpu_set_gfx_off_residency(adev, value ? true : false);
1131 
1132 		result += 4;
1133 		buf += 4;
1134 		*pos += 4;
1135 		size -= 4;
1136 	}
1137 
1138 	r = result;
1139 out:
1140 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1141 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1142 
1143 	return r;
1144 }
1145 
1146 
1147 /**
1148  * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1149  *
1150  * @f: open file handle
1151  * @buf: User buffer to store read data in
1152  * @size: Number of bytes to read
1153  * @pos:  Offset to seek to
1154  */
1155 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1156 						size_t size, loff_t *pos)
1157 {
1158 	struct amdgpu_device *adev = file_inode(f)->i_private;
1159 	ssize_t result = 0;
1160 	int r;
1161 
1162 	if (size & 0x3 || *pos & 0x3)
1163 		return -EINVAL;
1164 
1165 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1166 	if (r < 0) {
1167 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1168 		return r;
1169 	}
1170 
1171 	while (size) {
1172 		u64 value = 0;
1173 
1174 		r = amdgpu_get_gfx_off_entrycount(adev, &value);
1175 		if (r)
1176 			goto out;
1177 
1178 		r = put_user(value, (u64 *)buf);
1179 		if (r)
1180 			goto out;
1181 
1182 		result += 4;
1183 		buf += 4;
1184 		*pos += 4;
1185 		size -= 4;
1186 	}
1187 
1188 	r = result;
1189 out:
1190 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1191 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1192 
1193 	return r;
1194 }
1195 
1196 /**
1197  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1198  *
1199  * @f: open file handle
1200  * @buf: User buffer to write data from
1201  * @size: Number of bytes to write
1202  * @pos:  Offset to seek to
1203  *
1204  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1205  */
1206 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1207 					 size_t size, loff_t *pos)
1208 {
1209 	struct amdgpu_device *adev = file_inode(f)->i_private;
1210 	ssize_t result = 0;
1211 	int r;
1212 
1213 	if (size & 0x3 || *pos & 0x3)
1214 		return -EINVAL;
1215 
1216 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1217 	if (r < 0) {
1218 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1219 		return r;
1220 	}
1221 
1222 	while (size) {
1223 		uint32_t value;
1224 
1225 		r = get_user(value, (uint32_t *)buf);
1226 		if (r)
1227 			goto out;
1228 
1229 		amdgpu_gfx_off_ctrl(adev, value ? true : false);
1230 
1231 		result += 4;
1232 		buf += 4;
1233 		*pos += 4;
1234 		size -= 4;
1235 	}
1236 
1237 	r = result;
1238 out:
1239 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1240 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1241 
1242 	return r;
1243 }
1244 
1245 
1246 /**
1247  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1248  *
1249  * @f: open file handle
1250  * @buf: User buffer to store read data in
1251  * @size: Number of bytes to read
1252  * @pos:  Offset to seek to
1253  */
1254 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1255 					 size_t size, loff_t *pos)
1256 {
1257 	struct amdgpu_device *adev = file_inode(f)->i_private;
1258 	ssize_t result = 0;
1259 	int r;
1260 
1261 	if (size & 0x3 || *pos & 0x3)
1262 		return -EINVAL;
1263 
1264 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1265 	if (r < 0) {
1266 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1267 		return r;
1268 	}
1269 
1270 	while (size) {
1271 		u32 value = adev->gfx.gfx_off_state;
1272 
1273 		r = put_user(value, (u32 *)buf);
1274 		if (r)
1275 			goto out;
1276 
1277 		result += 4;
1278 		buf += 4;
1279 		*pos += 4;
1280 		size -= 4;
1281 	}
1282 
1283 	r = result;
1284 out:
1285 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1286 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1287 
1288 	return r;
1289 }
1290 
1291 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1292 						 size_t size, loff_t *pos)
1293 {
1294 	struct amdgpu_device *adev = file_inode(f)->i_private;
1295 	ssize_t result = 0;
1296 	int r;
1297 
1298 	if (size & 0x3 || *pos & 0x3)
1299 		return -EINVAL;
1300 
1301 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1302 	if (r < 0) {
1303 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1304 		return r;
1305 	}
1306 
1307 	while (size) {
1308 		u32 value;
1309 
1310 		r = amdgpu_get_gfx_off_status(adev, &value);
1311 		if (r)
1312 			goto out;
1313 
1314 		r = put_user(value, (u32 *)buf);
1315 		if (r)
1316 			goto out;
1317 
1318 		result += 4;
1319 		buf += 4;
1320 		*pos += 4;
1321 		size -= 4;
1322 	}
1323 
1324 	r = result;
1325 out:
1326 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1327 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1328 
1329 	return r;
1330 }
1331 
1332 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1333 	.owner = THIS_MODULE,
1334 	.unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1335 	.read = amdgpu_debugfs_regs2_read,
1336 	.write = amdgpu_debugfs_regs2_write,
1337 	.open = amdgpu_debugfs_regs2_open,
1338 	.release = amdgpu_debugfs_regs2_release,
1339 	.llseek = default_llseek
1340 };
1341 
1342 static const struct file_operations amdgpu_debugfs_regs_fops = {
1343 	.owner = THIS_MODULE,
1344 	.read = amdgpu_debugfs_regs_read,
1345 	.write = amdgpu_debugfs_regs_write,
1346 	.llseek = default_llseek
1347 };
1348 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1349 	.owner = THIS_MODULE,
1350 	.read = amdgpu_debugfs_regs_didt_read,
1351 	.write = amdgpu_debugfs_regs_didt_write,
1352 	.llseek = default_llseek
1353 };
1354 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1355 	.owner = THIS_MODULE,
1356 	.read = amdgpu_debugfs_regs_pcie_read,
1357 	.write = amdgpu_debugfs_regs_pcie_write,
1358 	.llseek = default_llseek
1359 };
1360 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1361 	.owner = THIS_MODULE,
1362 	.read = amdgpu_debugfs_regs_smc_read,
1363 	.write = amdgpu_debugfs_regs_smc_write,
1364 	.llseek = default_llseek
1365 };
1366 
1367 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1368 	.owner = THIS_MODULE,
1369 	.read = amdgpu_debugfs_gca_config_read,
1370 	.llseek = default_llseek
1371 };
1372 
1373 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1374 	.owner = THIS_MODULE,
1375 	.read = amdgpu_debugfs_sensor_read,
1376 	.llseek = default_llseek
1377 };
1378 
1379 static const struct file_operations amdgpu_debugfs_wave_fops = {
1380 	.owner = THIS_MODULE,
1381 	.read = amdgpu_debugfs_wave_read,
1382 	.llseek = default_llseek
1383 };
1384 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1385 	.owner = THIS_MODULE,
1386 	.read = amdgpu_debugfs_gpr_read,
1387 	.llseek = default_llseek
1388 };
1389 
1390 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1391 	.owner = THIS_MODULE,
1392 	.read = amdgpu_debugfs_gfxoff_read,
1393 	.write = amdgpu_debugfs_gfxoff_write,
1394 	.llseek = default_llseek
1395 };
1396 
1397 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1398 	.owner = THIS_MODULE,
1399 	.read = amdgpu_debugfs_gfxoff_status_read,
1400 	.llseek = default_llseek
1401 };
1402 
1403 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1404 	.owner = THIS_MODULE,
1405 	.read = amdgpu_debugfs_gfxoff_count_read,
1406 	.llseek = default_llseek
1407 };
1408 
1409 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1410 	.owner = THIS_MODULE,
1411 	.read = amdgpu_debugfs_gfxoff_residency_read,
1412 	.write = amdgpu_debugfs_gfxoff_residency_write,
1413 	.llseek = default_llseek
1414 };
1415 
1416 static const struct file_operations *debugfs_regs[] = {
1417 	&amdgpu_debugfs_regs_fops,
1418 	&amdgpu_debugfs_regs2_fops,
1419 	&amdgpu_debugfs_regs_didt_fops,
1420 	&amdgpu_debugfs_regs_pcie_fops,
1421 	&amdgpu_debugfs_regs_smc_fops,
1422 	&amdgpu_debugfs_gca_config_fops,
1423 	&amdgpu_debugfs_sensors_fops,
1424 	&amdgpu_debugfs_wave_fops,
1425 	&amdgpu_debugfs_gpr_fops,
1426 	&amdgpu_debugfs_gfxoff_fops,
1427 	&amdgpu_debugfs_gfxoff_status_fops,
1428 	&amdgpu_debugfs_gfxoff_count_fops,
1429 	&amdgpu_debugfs_gfxoff_residency_fops,
1430 };
1431 
1432 static const char *debugfs_regs_names[] = {
1433 	"amdgpu_regs",
1434 	"amdgpu_regs2",
1435 	"amdgpu_regs_didt",
1436 	"amdgpu_regs_pcie",
1437 	"amdgpu_regs_smc",
1438 	"amdgpu_gca_config",
1439 	"amdgpu_sensors",
1440 	"amdgpu_wave",
1441 	"amdgpu_gpr",
1442 	"amdgpu_gfxoff",
1443 	"amdgpu_gfxoff_status",
1444 	"amdgpu_gfxoff_count",
1445 	"amdgpu_gfxoff_residency",
1446 };
1447 
1448 /**
1449  * amdgpu_debugfs_regs_init -	Initialize debugfs entries that provide
1450  * 				register access.
1451  *
1452  * @adev: The device to attach the debugfs entries to
1453  */
1454 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1455 {
1456 	struct drm_minor *minor = adev_to_drm(adev)->primary;
1457 	struct dentry *ent, *root = minor->debugfs_root;
1458 	unsigned int i;
1459 
1460 	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1461 		ent = debugfs_create_file(debugfs_regs_names[i],
1462 					  S_IFREG | S_IRUGO, root,
1463 					  adev, debugfs_regs[i]);
1464 		if (!i && !IS_ERR_OR_NULL(ent))
1465 			i_size_write(ent->d_inode, adev->rmmio_size);
1466 	}
1467 
1468 	return 0;
1469 }
1470 
1471 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1472 {
1473 	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1474 	struct drm_device *dev = adev_to_drm(adev);
1475 	int r = 0, i;
1476 
1477 	r = pm_runtime_get_sync(dev->dev);
1478 	if (r < 0) {
1479 		pm_runtime_put_autosuspend(dev->dev);
1480 		return r;
1481 	}
1482 
1483 	/* Avoid accidently unparking the sched thread during GPU reset */
1484 	r = down_write_killable(&adev->reset_domain->sem);
1485 	if (r)
1486 		return r;
1487 
1488 	/* hold on the scheduler */
1489 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1490 		struct amdgpu_ring *ring = adev->rings[i];
1491 
1492 		if (!ring || !ring->sched.thread)
1493 			continue;
1494 		kthread_park(ring->sched.thread);
1495 	}
1496 
1497 	seq_printf(m, "run ib test:\n");
1498 	r = amdgpu_ib_ring_tests(adev);
1499 	if (r)
1500 		seq_printf(m, "ib ring tests failed (%d).\n", r);
1501 	else
1502 		seq_printf(m, "ib ring tests passed.\n");
1503 
1504 	/* go on the scheduler */
1505 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1506 		struct amdgpu_ring *ring = adev->rings[i];
1507 
1508 		if (!ring || !ring->sched.thread)
1509 			continue;
1510 		kthread_unpark(ring->sched.thread);
1511 	}
1512 
1513 	up_write(&adev->reset_domain->sem);
1514 
1515 	pm_runtime_mark_last_busy(dev->dev);
1516 	pm_runtime_put_autosuspend(dev->dev);
1517 
1518 	return 0;
1519 }
1520 
1521 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1522 {
1523 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1524 	struct drm_device *dev = adev_to_drm(adev);
1525 	int r;
1526 
1527 	r = pm_runtime_get_sync(dev->dev);
1528 	if (r < 0) {
1529 		pm_runtime_put_autosuspend(dev->dev);
1530 		return r;
1531 	}
1532 
1533 	*val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1534 
1535 	pm_runtime_mark_last_busy(dev->dev);
1536 	pm_runtime_put_autosuspend(dev->dev);
1537 
1538 	return 0;
1539 }
1540 
1541 
1542 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1543 {
1544 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1545 	struct drm_device *dev = adev_to_drm(adev);
1546 	int r;
1547 
1548 	r = pm_runtime_get_sync(dev->dev);
1549 	if (r < 0) {
1550 		pm_runtime_put_autosuspend(dev->dev);
1551 		return r;
1552 	}
1553 
1554 	*val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1555 
1556 	pm_runtime_mark_last_busy(dev->dev);
1557 	pm_runtime_put_autosuspend(dev->dev);
1558 
1559 	return 0;
1560 }
1561 
1562 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1563 {
1564 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1565 	struct drm_device *dev = adev_to_drm(adev);
1566 	int r;
1567 
1568 	r = pm_runtime_get_sync(dev->dev);
1569 	if (r < 0) {
1570 		pm_runtime_put_autosuspend(dev->dev);
1571 		return r;
1572 	}
1573 
1574 	r = amdgpu_benchmark(adev, val);
1575 
1576 	pm_runtime_mark_last_busy(dev->dev);
1577 	pm_runtime_put_autosuspend(dev->dev);
1578 
1579 	return r;
1580 }
1581 
1582 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1583 {
1584 	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1585 	struct drm_device *dev = adev_to_drm(adev);
1586 	struct drm_file *file;
1587 	int r;
1588 
1589 	r = mutex_lock_interruptible(&dev->filelist_mutex);
1590 	if (r)
1591 		return r;
1592 
1593 	list_for_each_entry(file, &dev->filelist, lhead) {
1594 		struct amdgpu_fpriv *fpriv = file->driver_priv;
1595 		struct amdgpu_vm *vm = &fpriv->vm;
1596 
1597 		seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1598 				vm->task_info.pid, vm->task_info.process_name);
1599 		r = amdgpu_bo_reserve(vm->root.bo, true);
1600 		if (r)
1601 			break;
1602 		amdgpu_debugfs_vm_bo_info(vm, m);
1603 		amdgpu_bo_unreserve(vm->root.bo);
1604 	}
1605 
1606 	mutex_unlock(&dev->filelist_mutex);
1607 
1608 	return r;
1609 }
1610 
1611 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1612 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1613 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1614 			 NULL, "%lld\n");
1615 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1616 			 NULL, "%lld\n");
1617 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1618 			 "%lld\n");
1619 
1620 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1621 					  struct dma_fence **fences)
1622 {
1623 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
1624 	uint32_t sync_seq, last_seq;
1625 
1626 	last_seq = atomic_read(&ring->fence_drv.last_seq);
1627 	sync_seq = ring->fence_drv.sync_seq;
1628 
1629 	last_seq &= drv->num_fences_mask;
1630 	sync_seq &= drv->num_fences_mask;
1631 
1632 	do {
1633 		struct dma_fence *fence, **ptr;
1634 
1635 		++last_seq;
1636 		last_seq &= drv->num_fences_mask;
1637 		ptr = &drv->fences[last_seq];
1638 
1639 		fence = rcu_dereference_protected(*ptr, 1);
1640 		RCU_INIT_POINTER(*ptr, NULL);
1641 
1642 		if (!fence)
1643 			continue;
1644 
1645 		fences[last_seq] = fence;
1646 
1647 	} while (last_seq != sync_seq);
1648 }
1649 
1650 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1651 					    int length)
1652 {
1653 	int i;
1654 	struct dma_fence *fence;
1655 
1656 	for (i = 0; i < length; i++) {
1657 		fence = fences[i];
1658 		if (!fence)
1659 			continue;
1660 		dma_fence_signal(fence);
1661 		dma_fence_put(fence);
1662 	}
1663 }
1664 
1665 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1666 {
1667 	struct drm_sched_job *s_job;
1668 	struct dma_fence *fence;
1669 
1670 	spin_lock(&sched->job_list_lock);
1671 	list_for_each_entry(s_job, &sched->pending_list, list) {
1672 		fence = sched->ops->run_job(s_job);
1673 		dma_fence_put(fence);
1674 	}
1675 	spin_unlock(&sched->job_list_lock);
1676 }
1677 
1678 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1679 {
1680 	struct amdgpu_job *job;
1681 	struct drm_sched_job *s_job, *tmp;
1682 	uint32_t preempt_seq;
1683 	struct dma_fence *fence, **ptr;
1684 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
1685 	struct drm_gpu_scheduler *sched = &ring->sched;
1686 	bool preempted = true;
1687 
1688 	if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1689 		return;
1690 
1691 	preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1692 	if (preempt_seq <= atomic_read(&drv->last_seq)) {
1693 		preempted = false;
1694 		goto no_preempt;
1695 	}
1696 
1697 	preempt_seq &= drv->num_fences_mask;
1698 	ptr = &drv->fences[preempt_seq];
1699 	fence = rcu_dereference_protected(*ptr, 1);
1700 
1701 no_preempt:
1702 	spin_lock(&sched->job_list_lock);
1703 	list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1704 		if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1705 			/* remove job from ring_mirror_list */
1706 			list_del_init(&s_job->list);
1707 			sched->ops->free_job(s_job);
1708 			continue;
1709 		}
1710 		job = to_amdgpu_job(s_job);
1711 		if (preempted && (&job->hw_fence) == fence)
1712 			/* mark the job as preempted */
1713 			job->preemption_status |= AMDGPU_IB_PREEMPTED;
1714 	}
1715 	spin_unlock(&sched->job_list_lock);
1716 }
1717 
1718 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1719 {
1720 	int r, length;
1721 	struct amdgpu_ring *ring;
1722 	struct dma_fence **fences = NULL;
1723 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1724 
1725 	if (val >= AMDGPU_MAX_RINGS)
1726 		return -EINVAL;
1727 
1728 	ring = adev->rings[val];
1729 
1730 	if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1731 		return -EINVAL;
1732 
1733 	/* the last preemption failed */
1734 	if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1735 		return -EBUSY;
1736 
1737 	length = ring->fence_drv.num_fences_mask + 1;
1738 	fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1739 	if (!fences)
1740 		return -ENOMEM;
1741 
1742 	/* Avoid accidently unparking the sched thread during GPU reset */
1743 	r = down_read_killable(&adev->reset_domain->sem);
1744 	if (r)
1745 		goto pro_end;
1746 
1747 	/* stop the scheduler */
1748 	kthread_park(ring->sched.thread);
1749 
1750 	/* preempt the IB */
1751 	r = amdgpu_ring_preempt_ib(ring);
1752 	if (r) {
1753 		DRM_WARN("failed to preempt ring %d\n", ring->idx);
1754 		goto failure;
1755 	}
1756 
1757 	amdgpu_fence_process(ring);
1758 
1759 	if (atomic_read(&ring->fence_drv.last_seq) !=
1760 	    ring->fence_drv.sync_seq) {
1761 		DRM_INFO("ring %d was preempted\n", ring->idx);
1762 
1763 		amdgpu_ib_preempt_mark_partial_job(ring);
1764 
1765 		/* swap out the old fences */
1766 		amdgpu_ib_preempt_fences_swap(ring, fences);
1767 
1768 		amdgpu_fence_driver_force_completion(ring);
1769 
1770 		/* resubmit unfinished jobs */
1771 		amdgpu_ib_preempt_job_recovery(&ring->sched);
1772 
1773 		/* wait for jobs finished */
1774 		amdgpu_fence_wait_empty(ring);
1775 
1776 		/* signal the old fences */
1777 		amdgpu_ib_preempt_signal_fences(fences, length);
1778 	}
1779 
1780 failure:
1781 	/* restart the scheduler */
1782 	kthread_unpark(ring->sched.thread);
1783 
1784 	up_read(&adev->reset_domain->sem);
1785 
1786 pro_end:
1787 	kfree(fences);
1788 
1789 	return r;
1790 }
1791 
1792 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1793 {
1794 	int ret = 0;
1795 	uint32_t max_freq, min_freq;
1796 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1797 
1798 	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1799 		return -EINVAL;
1800 
1801 	ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1802 	if (ret < 0) {
1803 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1804 		return ret;
1805 	}
1806 
1807 	ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
1808 	if (ret == -EOPNOTSUPP) {
1809 		ret = 0;
1810 		goto out;
1811 	}
1812 	if (ret || val > max_freq || val < min_freq) {
1813 		ret = -EINVAL;
1814 		goto out;
1815 	}
1816 
1817 	ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
1818 	if (ret)
1819 		ret = -EINVAL;
1820 
1821 out:
1822 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1823 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1824 
1825 	return ret;
1826 }
1827 
1828 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1829 			amdgpu_debugfs_ib_preempt, "%llu\n");
1830 
1831 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
1832 			amdgpu_debugfs_sclk_set, "%llu\n");
1833 
1834 static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
1835 				char __user *buf, size_t size, loff_t *pos)
1836 {
1837 	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
1838 	char reg_offset[12];
1839 	int i, ret, len = 0;
1840 
1841 	if (*pos)
1842 		return 0;
1843 
1844 	memset(reg_offset, 0, 12);
1845 	ret = down_read_killable(&adev->reset_domain->sem);
1846 	if (ret)
1847 		return ret;
1848 
1849 	for (i = 0; i < adev->num_regs; i++) {
1850 		sprintf(reg_offset, "0x%x\n", adev->reset_dump_reg_list[i]);
1851 		up_read(&adev->reset_domain->sem);
1852 		if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
1853 			return -EFAULT;
1854 
1855 		len += strlen(reg_offset);
1856 		ret = down_read_killable(&adev->reset_domain->sem);
1857 		if (ret)
1858 			return ret;
1859 	}
1860 
1861 	up_read(&adev->reset_domain->sem);
1862 	*pos += len;
1863 
1864 	return len;
1865 }
1866 
1867 static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
1868 			const char __user *buf, size_t size, loff_t *pos)
1869 {
1870 	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
1871 	char reg_offset[11];
1872 	uint32_t *new = NULL, *tmp = NULL;
1873 	int ret, i = 0, len = 0;
1874 
1875 	do {
1876 		memset(reg_offset, 0, 11);
1877 		if (copy_from_user(reg_offset, buf + len,
1878 					min(10, ((int)size-len)))) {
1879 			ret = -EFAULT;
1880 			goto error_free;
1881 		}
1882 
1883 		new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
1884 		if (!new) {
1885 			ret = -ENOMEM;
1886 			goto error_free;
1887 		}
1888 		tmp = new;
1889 		if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
1890 			ret = -EINVAL;
1891 			goto error_free;
1892 		}
1893 
1894 		len += ret;
1895 		i++;
1896 	} while (len < size);
1897 
1898 	new = kmalloc_array(i, sizeof(uint32_t), GFP_KERNEL);
1899 	if (!new) {
1900 		ret = -ENOMEM;
1901 		goto error_free;
1902 	}
1903 	ret = down_write_killable(&adev->reset_domain->sem);
1904 	if (ret)
1905 		goto error_free;
1906 
1907 	swap(adev->reset_dump_reg_list, tmp);
1908 	swap(adev->reset_dump_reg_value, new);
1909 	adev->num_regs = i;
1910 	up_write(&adev->reset_domain->sem);
1911 	ret = size;
1912 
1913 error_free:
1914 	if (tmp != new)
1915 		kfree(tmp);
1916 	kfree(new);
1917 	return ret;
1918 }
1919 
1920 static const struct file_operations amdgpu_reset_dump_register_list = {
1921 	.owner = THIS_MODULE,
1922 	.read = amdgpu_reset_dump_register_list_read,
1923 	.write = amdgpu_reset_dump_register_list_write,
1924 	.llseek = default_llseek
1925 };
1926 
1927 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1928 {
1929 	struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
1930 	struct dentry *ent;
1931 	int r, i;
1932 
1933 	if (!debugfs_initialized())
1934 		return 0;
1935 
1936 	debugfs_create_x32("amdgpu_smu_debug", 0600, root,
1937 			   &adev->pm.smu_debug_mask);
1938 
1939 	ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
1940 				  &fops_ib_preempt);
1941 	if (IS_ERR(ent)) {
1942 		DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1943 		return PTR_ERR(ent);
1944 	}
1945 
1946 	ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
1947 				  &fops_sclk_set);
1948 	if (IS_ERR(ent)) {
1949 		DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1950 		return PTR_ERR(ent);
1951 	}
1952 
1953 	/* Register debugfs entries for amdgpu_ttm */
1954 	amdgpu_ttm_debugfs_init(adev);
1955 	amdgpu_debugfs_pm_init(adev);
1956 	amdgpu_debugfs_sa_init(adev);
1957 	amdgpu_debugfs_fence_init(adev);
1958 	amdgpu_debugfs_gem_init(adev);
1959 
1960 	r = amdgpu_debugfs_regs_init(adev);
1961 	if (r)
1962 		DRM_ERROR("registering register debugfs failed (%d).\n", r);
1963 
1964 	amdgpu_debugfs_firmware_init(adev);
1965 	amdgpu_ta_if_debugfs_init(adev);
1966 
1967 #if defined(CONFIG_DRM_AMD_DC)
1968 	if (adev->dc_enabled)
1969 		dtn_debugfs_init(adev);
1970 #endif
1971 
1972 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1973 		struct amdgpu_ring *ring = adev->rings[i];
1974 
1975 		if (!ring)
1976 			continue;
1977 
1978 		amdgpu_debugfs_ring_init(adev, ring);
1979 	}
1980 
1981 	for ( i = 0; i < adev->vcn.num_vcn_inst; i++) {
1982 		if (!amdgpu_vcnfw_log)
1983 			break;
1984 
1985 		if (adev->vcn.harvest_config & (1 << i))
1986 			continue;
1987 
1988 		amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
1989 	}
1990 
1991 	amdgpu_ras_debugfs_create_all(adev);
1992 	amdgpu_rap_debugfs_init(adev);
1993 	amdgpu_securedisplay_debugfs_init(adev);
1994 	amdgpu_fw_attestation_debugfs_init(adev);
1995 
1996 	debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
1997 			    &amdgpu_evict_vram_fops);
1998 	debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
1999 			    &amdgpu_evict_gtt_fops);
2000 	debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2001 			    &amdgpu_debugfs_test_ib_fops);
2002 	debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2003 			    &amdgpu_debugfs_vm_info_fops);
2004 	debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2005 			    &amdgpu_benchmark_fops);
2006 	debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
2007 			    &amdgpu_reset_dump_register_list);
2008 
2009 	adev->debugfs_vbios_blob.data = adev->bios;
2010 	adev->debugfs_vbios_blob.size = adev->bios_size;
2011 	debugfs_create_blob("amdgpu_vbios", 0444, root,
2012 			    &adev->debugfs_vbios_blob);
2013 
2014 	adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2015 	adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2016 	debugfs_create_blob("amdgpu_discovery", 0444, root,
2017 			    &adev->debugfs_discovery_blob);
2018 
2019 	return 0;
2020 }
2021 
2022 #else
2023 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2024 {
2025 	return 0;
2026 }
2027 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2028 {
2029 	return 0;
2030 }
2031 #endif
2032