1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  */
25 
26 #include <linux/kthread.h>
27 #include <linux/pci.h>
28 #include <linux/uaccess.h>
29 #include <linux/pm_runtime.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_pm.h"
33 #include "amdgpu_dm_debugfs.h"
34 #include "amdgpu_ras.h"
35 #include "amdgpu_rap.h"
36 #include "amdgpu_securedisplay.h"
37 #include "amdgpu_fw_attestation.h"
38 #include "amdgpu_umr.h"
39 
40 #if defined(CONFIG_DEBUG_FS)
41 
42 /**
43  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
44  *
45  * @read: True if reading
46  * @f: open file handle
47  * @buf: User buffer to write/read to
48  * @size: Number of bytes to write/read
49  * @pos:  Offset to seek to
50  *
51  * This debugfs entry has special meaning on the offset being sought.
52  * Various bits have different meanings:
53  *
54  * Bit 62:  Indicates a GRBM bank switch is needed
55  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
56  * 	    zero)
57  * Bits 24..33: The SE or ME selector if needed
58  * Bits 34..43: The SH (or SA) or PIPE selector if needed
59  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
60  *
61  * Bit 23:  Indicates that the PM power gating lock should be held
62  * 	    This is necessary to read registers that might be
63  * 	    unreliable during a power gating transistion.
64  *
65  * The lower bits are the BYTE offset of the register to read.  This
66  * allows reading multiple registers in a single call and having
67  * the returned size reflect that.
68  */
69 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
70 		char __user *buf, size_t size, loff_t *pos)
71 {
72 	struct amdgpu_device *adev = file_inode(f)->i_private;
73 	ssize_t result = 0;
74 	int r;
75 	bool pm_pg_lock, use_bank, use_ring;
76 	unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
77 
78 	pm_pg_lock = use_bank = use_ring = false;
79 	instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
80 
81 	if (size & 0x3 || *pos & 0x3 ||
82 			((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
83 		return -EINVAL;
84 
85 	/* are we reading registers for which a PG lock is necessary? */
86 	pm_pg_lock = (*pos >> 23) & 1;
87 
88 	if (*pos & (1ULL << 62)) {
89 		se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
90 		sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
91 		instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
92 
93 		if (se_bank == 0x3FF)
94 			se_bank = 0xFFFFFFFF;
95 		if (sh_bank == 0x3FF)
96 			sh_bank = 0xFFFFFFFF;
97 		if (instance_bank == 0x3FF)
98 			instance_bank = 0xFFFFFFFF;
99 		use_bank = true;
100 	} else if (*pos & (1ULL << 61)) {
101 
102 		me = (*pos & GENMASK_ULL(33, 24)) >> 24;
103 		pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
104 		queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
105 		vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
106 
107 		use_ring = true;
108 	} else {
109 		use_bank = use_ring = false;
110 	}
111 
112 	*pos &= (1UL << 22) - 1;
113 
114 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
115 	if (r < 0) {
116 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
117 		return r;
118 	}
119 
120 	r = amdgpu_virt_enable_access_debugfs(adev);
121 	if (r < 0) {
122 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
123 		return r;
124 	}
125 
126 	if (use_bank) {
127 		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
128 		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
129 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
130 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
131 			amdgpu_virt_disable_access_debugfs(adev);
132 			return -EINVAL;
133 		}
134 		mutex_lock(&adev->grbm_idx_mutex);
135 		amdgpu_gfx_select_se_sh(adev, se_bank,
136 					sh_bank, instance_bank);
137 	} else if (use_ring) {
138 		mutex_lock(&adev->srbm_mutex);
139 		amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
140 	}
141 
142 	if (pm_pg_lock)
143 		mutex_lock(&adev->pm.mutex);
144 
145 	while (size) {
146 		uint32_t value;
147 
148 		if (read) {
149 			value = RREG32(*pos >> 2);
150 			r = put_user(value, (uint32_t *)buf);
151 		} else {
152 			r = get_user(value, (uint32_t *)buf);
153 			if (!r)
154 				amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
155 		}
156 		if (r) {
157 			result = r;
158 			goto end;
159 		}
160 
161 		result += 4;
162 		buf += 4;
163 		*pos += 4;
164 		size -= 4;
165 	}
166 
167 end:
168 	if (use_bank) {
169 		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
170 		mutex_unlock(&adev->grbm_idx_mutex);
171 	} else if (use_ring) {
172 		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
173 		mutex_unlock(&adev->srbm_mutex);
174 	}
175 
176 	if (pm_pg_lock)
177 		mutex_unlock(&adev->pm.mutex);
178 
179 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
180 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
181 
182 	amdgpu_virt_disable_access_debugfs(adev);
183 	return result;
184 }
185 
186 /*
187  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
188  */
189 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
190 					size_t size, loff_t *pos)
191 {
192 	return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
193 }
194 
195 /*
196  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
197  */
198 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
199 					 size_t size, loff_t *pos)
200 {
201 	return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
202 }
203 
204 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
205 {
206 	struct amdgpu_debugfs_regs2_data *rd;
207 
208 	rd = kzalloc(sizeof *rd, GFP_KERNEL);
209 	if (!rd)
210 		return -ENOMEM;
211 	rd->adev = file_inode(file)->i_private;
212 	file->private_data = rd;
213 	mutex_init(&rd->lock);
214 
215 	return 0;
216 }
217 
218 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
219 {
220 	struct amdgpu_debugfs_regs2_data *rd = file->private_data;
221 	mutex_destroy(&rd->lock);
222 	kfree(file->private_data);
223 	return 0;
224 }
225 
226 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
227 {
228 	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
229 	struct amdgpu_device *adev = rd->adev;
230 	ssize_t result = 0;
231 	int r;
232 	uint32_t value;
233 
234 	if (size & 0x3 || offset & 0x3)
235 		return -EINVAL;
236 
237 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
238 	if (r < 0) {
239 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
240 		return r;
241 	}
242 
243 	r = amdgpu_virt_enable_access_debugfs(adev);
244 	if (r < 0) {
245 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
246 		return r;
247 	}
248 
249 	mutex_lock(&rd->lock);
250 
251 	if (rd->id.use_grbm) {
252 		if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
253 		    (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
254 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
255 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
256 			amdgpu_virt_disable_access_debugfs(adev);
257 			mutex_unlock(&rd->lock);
258 			return -EINVAL;
259 		}
260 		mutex_lock(&adev->grbm_idx_mutex);
261 		amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
262 								rd->id.grbm.sh,
263 								rd->id.grbm.instance);
264 	}
265 
266 	if (rd->id.use_srbm) {
267 		mutex_lock(&adev->srbm_mutex);
268 		amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
269 									rd->id.srbm.queue, rd->id.srbm.vmid);
270 	}
271 
272 	if (rd->id.pg_lock)
273 		mutex_lock(&adev->pm.mutex);
274 
275 	while (size) {
276 		if (!write_en) {
277 			value = RREG32(offset >> 2);
278 			r = put_user(value, (uint32_t *)buf);
279 		} else {
280 			r = get_user(value, (uint32_t *)buf);
281 			if (!r)
282 				amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value);
283 		}
284 		if (r) {
285 			result = r;
286 			goto end;
287 		}
288 		offset += 4;
289 		size -= 4;
290 		result += 4;
291 		buf += 4;
292 	}
293 end:
294 	if (rd->id.use_grbm) {
295 		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
296 		mutex_unlock(&adev->grbm_idx_mutex);
297 	}
298 
299 	if (rd->id.use_srbm) {
300 		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
301 		mutex_unlock(&adev->srbm_mutex);
302 	}
303 
304 	if (rd->id.pg_lock)
305 		mutex_unlock(&adev->pm.mutex);
306 
307 	mutex_unlock(&rd->lock);
308 
309 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
310 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
311 
312 	amdgpu_virt_disable_access_debugfs(adev);
313 	return result;
314 }
315 
316 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
317 {
318 	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
319 	int r;
320 
321 	switch (cmd) {
322 	case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
323 		mutex_lock(&rd->lock);
324 		r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata *)data, sizeof rd->id);
325 		mutex_unlock(&rd->lock);
326 		return r ? -EINVAL : 0;
327 	default:
328 		return -EINVAL;
329 	}
330 	return 0;
331 }
332 
333 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
334 {
335 	return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
336 }
337 
338 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
339 {
340 	return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
341 }
342 
343 
344 /**
345  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
346  *
347  * @f: open file handle
348  * @buf: User buffer to store read data in
349  * @size: Number of bytes to read
350  * @pos:  Offset to seek to
351  *
352  * The lower bits are the BYTE offset of the register to read.  This
353  * allows reading multiple registers in a single call and having
354  * the returned size reflect that.
355  */
356 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
357 					size_t size, loff_t *pos)
358 {
359 	struct amdgpu_device *adev = file_inode(f)->i_private;
360 	ssize_t result = 0;
361 	int r;
362 
363 	if (size & 0x3 || *pos & 0x3)
364 		return -EINVAL;
365 
366 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
367 	if (r < 0) {
368 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
369 		return r;
370 	}
371 
372 	r = amdgpu_virt_enable_access_debugfs(adev);
373 	if (r < 0) {
374 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
375 		return r;
376 	}
377 
378 	while (size) {
379 		uint32_t value;
380 
381 		value = RREG32_PCIE(*pos);
382 		r = put_user(value, (uint32_t *)buf);
383 		if (r) {
384 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
385 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
386 			amdgpu_virt_disable_access_debugfs(adev);
387 			return r;
388 		}
389 
390 		result += 4;
391 		buf += 4;
392 		*pos += 4;
393 		size -= 4;
394 	}
395 
396 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
397 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
398 
399 	amdgpu_virt_disable_access_debugfs(adev);
400 	return result;
401 }
402 
403 /**
404  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
405  *
406  * @f: open file handle
407  * @buf: User buffer to write data from
408  * @size: Number of bytes to write
409  * @pos:  Offset to seek to
410  *
411  * The lower bits are the BYTE offset of the register to write.  This
412  * allows writing multiple registers in a single call and having
413  * the returned size reflect that.
414  */
415 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
416 					 size_t size, loff_t *pos)
417 {
418 	struct amdgpu_device *adev = file_inode(f)->i_private;
419 	ssize_t result = 0;
420 	int r;
421 
422 	if (size & 0x3 || *pos & 0x3)
423 		return -EINVAL;
424 
425 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
426 	if (r < 0) {
427 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
428 		return r;
429 	}
430 
431 	r = amdgpu_virt_enable_access_debugfs(adev);
432 	if (r < 0) {
433 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
434 		return r;
435 	}
436 
437 	while (size) {
438 		uint32_t value;
439 
440 		r = get_user(value, (uint32_t *)buf);
441 		if (r) {
442 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
443 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
444 			amdgpu_virt_disable_access_debugfs(adev);
445 			return r;
446 		}
447 
448 		WREG32_PCIE(*pos, value);
449 
450 		result += 4;
451 		buf += 4;
452 		*pos += 4;
453 		size -= 4;
454 	}
455 
456 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
457 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
458 
459 	amdgpu_virt_disable_access_debugfs(adev);
460 	return result;
461 }
462 
463 /**
464  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
465  *
466  * @f: open file handle
467  * @buf: User buffer to store read data in
468  * @size: Number of bytes to read
469  * @pos:  Offset to seek to
470  *
471  * The lower bits are the BYTE offset of the register to read.  This
472  * allows reading multiple registers in a single call and having
473  * the returned size reflect that.
474  */
475 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
476 					size_t size, loff_t *pos)
477 {
478 	struct amdgpu_device *adev = file_inode(f)->i_private;
479 	ssize_t result = 0;
480 	int r;
481 
482 	if (size & 0x3 || *pos & 0x3)
483 		return -EINVAL;
484 
485 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
486 	if (r < 0) {
487 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
488 		return r;
489 	}
490 
491 	r = amdgpu_virt_enable_access_debugfs(adev);
492 	if (r < 0) {
493 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
494 		return r;
495 	}
496 
497 	while (size) {
498 		uint32_t value;
499 
500 		value = RREG32_DIDT(*pos >> 2);
501 		r = put_user(value, (uint32_t *)buf);
502 		if (r) {
503 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
504 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
505 			amdgpu_virt_disable_access_debugfs(adev);
506 			return r;
507 		}
508 
509 		result += 4;
510 		buf += 4;
511 		*pos += 4;
512 		size -= 4;
513 	}
514 
515 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
516 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
517 
518 	amdgpu_virt_disable_access_debugfs(adev);
519 	return result;
520 }
521 
522 /**
523  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
524  *
525  * @f: open file handle
526  * @buf: User buffer to write data from
527  * @size: Number of bytes to write
528  * @pos:  Offset to seek to
529  *
530  * The lower bits are the BYTE offset of the register to write.  This
531  * allows writing multiple registers in a single call and having
532  * the returned size reflect that.
533  */
534 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
535 					 size_t size, loff_t *pos)
536 {
537 	struct amdgpu_device *adev = file_inode(f)->i_private;
538 	ssize_t result = 0;
539 	int r;
540 
541 	if (size & 0x3 || *pos & 0x3)
542 		return -EINVAL;
543 
544 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
545 	if (r < 0) {
546 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
547 		return r;
548 	}
549 
550 	r = amdgpu_virt_enable_access_debugfs(adev);
551 	if (r < 0) {
552 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
553 		return r;
554 	}
555 
556 	while (size) {
557 		uint32_t value;
558 
559 		r = get_user(value, (uint32_t *)buf);
560 		if (r) {
561 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
562 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
563 			amdgpu_virt_disable_access_debugfs(adev);
564 			return r;
565 		}
566 
567 		WREG32_DIDT(*pos >> 2, value);
568 
569 		result += 4;
570 		buf += 4;
571 		*pos += 4;
572 		size -= 4;
573 	}
574 
575 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
576 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
577 
578 	amdgpu_virt_disable_access_debugfs(adev);
579 	return result;
580 }
581 
582 /**
583  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
584  *
585  * @f: open file handle
586  * @buf: User buffer to store read data in
587  * @size: Number of bytes to read
588  * @pos:  Offset to seek to
589  *
590  * The lower bits are the BYTE offset of the register to read.  This
591  * allows reading multiple registers in a single call and having
592  * the returned size reflect that.
593  */
594 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
595 					size_t size, loff_t *pos)
596 {
597 	struct amdgpu_device *adev = file_inode(f)->i_private;
598 	ssize_t result = 0;
599 	int r;
600 
601 	if (size & 0x3 || *pos & 0x3)
602 		return -EINVAL;
603 
604 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
605 	if (r < 0) {
606 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
607 		return r;
608 	}
609 
610 	r = amdgpu_virt_enable_access_debugfs(adev);
611 	if (r < 0) {
612 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
613 		return r;
614 	}
615 
616 	while (size) {
617 		uint32_t value;
618 
619 		value = RREG32_SMC(*pos);
620 		r = put_user(value, (uint32_t *)buf);
621 		if (r) {
622 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
623 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
624 			amdgpu_virt_disable_access_debugfs(adev);
625 			return r;
626 		}
627 
628 		result += 4;
629 		buf += 4;
630 		*pos += 4;
631 		size -= 4;
632 	}
633 
634 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
635 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
636 
637 	amdgpu_virt_disable_access_debugfs(adev);
638 	return result;
639 }
640 
641 /**
642  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
643  *
644  * @f: open file handle
645  * @buf: User buffer to write data from
646  * @size: Number of bytes to write
647  * @pos:  Offset to seek to
648  *
649  * The lower bits are the BYTE offset of the register to write.  This
650  * allows writing multiple registers in a single call and having
651  * the returned size reflect that.
652  */
653 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
654 					 size_t size, loff_t *pos)
655 {
656 	struct amdgpu_device *adev = file_inode(f)->i_private;
657 	ssize_t result = 0;
658 	int r;
659 
660 	if (size & 0x3 || *pos & 0x3)
661 		return -EINVAL;
662 
663 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
664 	if (r < 0) {
665 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
666 		return r;
667 	}
668 
669 	r = amdgpu_virt_enable_access_debugfs(adev);
670 	if (r < 0) {
671 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
672 		return r;
673 	}
674 
675 	while (size) {
676 		uint32_t value;
677 
678 		r = get_user(value, (uint32_t *)buf);
679 		if (r) {
680 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
681 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
682 			amdgpu_virt_disable_access_debugfs(adev);
683 			return r;
684 		}
685 
686 		WREG32_SMC(*pos, value);
687 
688 		result += 4;
689 		buf += 4;
690 		*pos += 4;
691 		size -= 4;
692 	}
693 
694 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
695 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
696 
697 	amdgpu_virt_disable_access_debugfs(adev);
698 	return result;
699 }
700 
701 /**
702  * amdgpu_debugfs_gca_config_read - Read from gfx config data
703  *
704  * @f: open file handle
705  * @buf: User buffer to store read data in
706  * @size: Number of bytes to read
707  * @pos:  Offset to seek to
708  *
709  * This file is used to access configuration data in a somewhat
710  * stable fashion.  The format is a series of DWORDs with the first
711  * indicating which revision it is.  New content is appended to the
712  * end so that older software can still read the data.
713  */
714 
715 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
716 					size_t size, loff_t *pos)
717 {
718 	struct amdgpu_device *adev = file_inode(f)->i_private;
719 	ssize_t result = 0;
720 	int r;
721 	uint32_t *config, no_regs = 0;
722 
723 	if (size & 0x3 || *pos & 0x3)
724 		return -EINVAL;
725 
726 	config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
727 	if (!config)
728 		return -ENOMEM;
729 
730 	/* version, increment each time something is added */
731 	config[no_regs++] = 3;
732 	config[no_regs++] = adev->gfx.config.max_shader_engines;
733 	config[no_regs++] = adev->gfx.config.max_tile_pipes;
734 	config[no_regs++] = adev->gfx.config.max_cu_per_sh;
735 	config[no_regs++] = adev->gfx.config.max_sh_per_se;
736 	config[no_regs++] = adev->gfx.config.max_backends_per_se;
737 	config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
738 	config[no_regs++] = adev->gfx.config.max_gprs;
739 	config[no_regs++] = adev->gfx.config.max_gs_threads;
740 	config[no_regs++] = adev->gfx.config.max_hw_contexts;
741 	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
742 	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
743 	config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
744 	config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
745 	config[no_regs++] = adev->gfx.config.num_tile_pipes;
746 	config[no_regs++] = adev->gfx.config.backend_enable_mask;
747 	config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
748 	config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
749 	config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
750 	config[no_regs++] = adev->gfx.config.num_gpus;
751 	config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
752 	config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
753 	config[no_regs++] = adev->gfx.config.gb_addr_config;
754 	config[no_regs++] = adev->gfx.config.num_rbs;
755 
756 	/* rev==1 */
757 	config[no_regs++] = adev->rev_id;
758 	config[no_regs++] = adev->pg_flags;
759 	config[no_regs++] = adev->cg_flags;
760 
761 	/* rev==2 */
762 	config[no_regs++] = adev->family;
763 	config[no_regs++] = adev->external_rev_id;
764 
765 	/* rev==3 */
766 	config[no_regs++] = adev->pdev->device;
767 	config[no_regs++] = adev->pdev->revision;
768 	config[no_regs++] = adev->pdev->subsystem_device;
769 	config[no_regs++] = adev->pdev->subsystem_vendor;
770 
771 	while (size && (*pos < no_regs * 4)) {
772 		uint32_t value;
773 
774 		value = config[*pos >> 2];
775 		r = put_user(value, (uint32_t *)buf);
776 		if (r) {
777 			kfree(config);
778 			return r;
779 		}
780 
781 		result += 4;
782 		buf += 4;
783 		*pos += 4;
784 		size -= 4;
785 	}
786 
787 	kfree(config);
788 	return result;
789 }
790 
791 /**
792  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
793  *
794  * @f: open file handle
795  * @buf: User buffer to store read data in
796  * @size: Number of bytes to read
797  * @pos:  Offset to seek to
798  *
799  * The offset is treated as the BYTE address of one of the sensors
800  * enumerated in amd/include/kgd_pp_interface.h under the
801  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
802  * you would use the offset 3 * 4 = 12.
803  */
804 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
805 					size_t size, loff_t *pos)
806 {
807 	struct amdgpu_device *adev = file_inode(f)->i_private;
808 	int idx, x, outsize, r, valuesize;
809 	uint32_t values[16];
810 
811 	if (size & 3 || *pos & 0x3)
812 		return -EINVAL;
813 
814 	if (!adev->pm.dpm_enabled)
815 		return -EINVAL;
816 
817 	/* convert offset to sensor number */
818 	idx = *pos >> 2;
819 
820 	valuesize = sizeof(values);
821 
822 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
823 	if (r < 0) {
824 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
825 		return r;
826 	}
827 
828 	r = amdgpu_virt_enable_access_debugfs(adev);
829 	if (r < 0) {
830 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
831 		return r;
832 	}
833 
834 	r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
835 
836 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
837 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
838 
839 	if (r) {
840 		amdgpu_virt_disable_access_debugfs(adev);
841 		return r;
842 	}
843 
844 	if (size > valuesize) {
845 		amdgpu_virt_disable_access_debugfs(adev);
846 		return -EINVAL;
847 	}
848 
849 	outsize = 0;
850 	x = 0;
851 	if (!r) {
852 		while (size) {
853 			r = put_user(values[x++], (int32_t *)buf);
854 			buf += 4;
855 			size -= 4;
856 			outsize += 4;
857 		}
858 	}
859 
860 	amdgpu_virt_disable_access_debugfs(adev);
861 	return !r ? outsize : r;
862 }
863 
864 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
865  *
866  * @f: open file handle
867  * @buf: User buffer to store read data in
868  * @size: Number of bytes to read
869  * @pos:  Offset to seek to
870  *
871  * The offset being sought changes which wave that the status data
872  * will be returned for.  The bits are used as follows:
873  *
874  * Bits 0..6: 	Byte offset into data
875  * Bits 7..14:	SE selector
876  * Bits 15..22:	SH/SA selector
877  * Bits 23..30: CU/{WGP+SIMD} selector
878  * Bits 31..36: WAVE ID selector
879  * Bits 37..44: SIMD ID selector
880  *
881  * The returned data begins with one DWORD of version information
882  * Followed by WAVE STATUS registers relevant to the GFX IP version
883  * being used.  See gfx_v8_0_read_wave_data() for an example output.
884  */
885 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
886 					size_t size, loff_t *pos)
887 {
888 	struct amdgpu_device *adev = f->f_inode->i_private;
889 	int r, x;
890 	ssize_t result = 0;
891 	uint32_t offset, se, sh, cu, wave, simd, data[32];
892 
893 	if (size & 3 || *pos & 3)
894 		return -EINVAL;
895 
896 	/* decode offset */
897 	offset = (*pos & GENMASK_ULL(6, 0));
898 	se = (*pos & GENMASK_ULL(14, 7)) >> 7;
899 	sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
900 	cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
901 	wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
902 	simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
903 
904 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
905 	if (r < 0) {
906 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
907 		return r;
908 	}
909 
910 	r = amdgpu_virt_enable_access_debugfs(adev);
911 	if (r < 0) {
912 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
913 		return r;
914 	}
915 
916 	/* switch to the specific se/sh/cu */
917 	mutex_lock(&adev->grbm_idx_mutex);
918 	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
919 
920 	x = 0;
921 	if (adev->gfx.funcs->read_wave_data)
922 		adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
923 
924 	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
925 	mutex_unlock(&adev->grbm_idx_mutex);
926 
927 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
928 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
929 
930 	if (!x) {
931 		amdgpu_virt_disable_access_debugfs(adev);
932 		return -EINVAL;
933 	}
934 
935 	while (size && (offset < x * 4)) {
936 		uint32_t value;
937 
938 		value = data[offset >> 2];
939 		r = put_user(value, (uint32_t *)buf);
940 		if (r) {
941 			amdgpu_virt_disable_access_debugfs(adev);
942 			return r;
943 		}
944 
945 		result += 4;
946 		buf += 4;
947 		offset += 4;
948 		size -= 4;
949 	}
950 
951 	amdgpu_virt_disable_access_debugfs(adev);
952 	return result;
953 }
954 
955 /** amdgpu_debugfs_gpr_read - Read wave gprs
956  *
957  * @f: open file handle
958  * @buf: User buffer to store read data in
959  * @size: Number of bytes to read
960  * @pos:  Offset to seek to
961  *
962  * The offset being sought changes which wave that the status data
963  * will be returned for.  The bits are used as follows:
964  *
965  * Bits 0..11:	Byte offset into data
966  * Bits 12..19:	SE selector
967  * Bits 20..27:	SH/SA selector
968  * Bits 28..35: CU/{WGP+SIMD} selector
969  * Bits 36..43: WAVE ID selector
970  * Bits 37..44: SIMD ID selector
971  * Bits 52..59: Thread selector
972  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
973  *
974  * The return data comes from the SGPR or VGPR register bank for
975  * the selected operational unit.
976  */
977 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
978 					size_t size, loff_t *pos)
979 {
980 	struct amdgpu_device *adev = f->f_inode->i_private;
981 	int r;
982 	ssize_t result = 0;
983 	uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
984 
985 	if (size > 4096 || size & 3 || *pos & 3)
986 		return -EINVAL;
987 
988 	/* decode offset */
989 	offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
990 	se = (*pos & GENMASK_ULL(19, 12)) >> 12;
991 	sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
992 	cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
993 	wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
994 	simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
995 	thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
996 	bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
997 
998 	data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
999 	if (!data)
1000 		return -ENOMEM;
1001 
1002 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1003 	if (r < 0)
1004 		goto err;
1005 
1006 	r = amdgpu_virt_enable_access_debugfs(adev);
1007 	if (r < 0)
1008 		goto err;
1009 
1010 	/* switch to the specific se/sh/cu */
1011 	mutex_lock(&adev->grbm_idx_mutex);
1012 	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
1013 
1014 	if (bank == 0) {
1015 		if (adev->gfx.funcs->read_wave_vgprs)
1016 			adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
1017 	} else {
1018 		if (adev->gfx.funcs->read_wave_sgprs)
1019 			adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
1020 	}
1021 
1022 	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
1023 	mutex_unlock(&adev->grbm_idx_mutex);
1024 
1025 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1026 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1027 
1028 	while (size) {
1029 		uint32_t value;
1030 
1031 		value = data[result >> 2];
1032 		r = put_user(value, (uint32_t *)buf);
1033 		if (r) {
1034 			amdgpu_virt_disable_access_debugfs(adev);
1035 			goto err;
1036 		}
1037 
1038 		result += 4;
1039 		buf += 4;
1040 		size -= 4;
1041 	}
1042 
1043 	kfree(data);
1044 	amdgpu_virt_disable_access_debugfs(adev);
1045 	return result;
1046 
1047 err:
1048 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1049 	kfree(data);
1050 	return r;
1051 }
1052 
1053 /**
1054  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1055  *
1056  * @f: open file handle
1057  * @buf: User buffer to write data from
1058  * @size: Number of bytes to write
1059  * @pos:  Offset to seek to
1060  *
1061  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1062  */
1063 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1064 					 size_t size, loff_t *pos)
1065 {
1066 	struct amdgpu_device *adev = file_inode(f)->i_private;
1067 	ssize_t result = 0;
1068 	int r;
1069 
1070 	if (size & 0x3 || *pos & 0x3)
1071 		return -EINVAL;
1072 
1073 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1074 	if (r < 0) {
1075 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1076 		return r;
1077 	}
1078 
1079 	while (size) {
1080 		uint32_t value;
1081 
1082 		r = get_user(value, (uint32_t *)buf);
1083 		if (r) {
1084 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1085 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1086 			return r;
1087 		}
1088 
1089 		amdgpu_gfx_off_ctrl(adev, value ? true : false);
1090 
1091 		result += 4;
1092 		buf += 4;
1093 		*pos += 4;
1094 		size -= 4;
1095 	}
1096 
1097 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1098 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1099 
1100 	return result;
1101 }
1102 
1103 
1104 /**
1105  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1106  *
1107  * @f: open file handle
1108  * @buf: User buffer to store read data in
1109  * @size: Number of bytes to read
1110  * @pos:  Offset to seek to
1111  */
1112 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1113 					 size_t size, loff_t *pos)
1114 {
1115 	struct amdgpu_device *adev = file_inode(f)->i_private;
1116 	ssize_t result = 0;
1117 	int r;
1118 
1119 	if (size & 0x3 || *pos & 0x3)
1120 		return -EINVAL;
1121 
1122 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1123 	if (r < 0)
1124 		return r;
1125 
1126 	while (size) {
1127 		uint32_t value;
1128 
1129 		r = amdgpu_get_gfx_off_status(adev, &value);
1130 		if (r) {
1131 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1132 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1133 			return r;
1134 		}
1135 
1136 		r = put_user(value, (uint32_t *)buf);
1137 		if (r) {
1138 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1139 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1140 			return r;
1141 		}
1142 
1143 		result += 4;
1144 		buf += 4;
1145 		*pos += 4;
1146 		size -= 4;
1147 	}
1148 
1149 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1150 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1151 
1152 	return result;
1153 }
1154 
1155 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1156 	.owner = THIS_MODULE,
1157 	.unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1158 	.read = amdgpu_debugfs_regs2_read,
1159 	.write = amdgpu_debugfs_regs2_write,
1160 	.open = amdgpu_debugfs_regs2_open,
1161 	.release = amdgpu_debugfs_regs2_release,
1162 	.llseek = default_llseek
1163 };
1164 
1165 static const struct file_operations amdgpu_debugfs_regs_fops = {
1166 	.owner = THIS_MODULE,
1167 	.read = amdgpu_debugfs_regs_read,
1168 	.write = amdgpu_debugfs_regs_write,
1169 	.llseek = default_llseek
1170 };
1171 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1172 	.owner = THIS_MODULE,
1173 	.read = amdgpu_debugfs_regs_didt_read,
1174 	.write = amdgpu_debugfs_regs_didt_write,
1175 	.llseek = default_llseek
1176 };
1177 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1178 	.owner = THIS_MODULE,
1179 	.read = amdgpu_debugfs_regs_pcie_read,
1180 	.write = amdgpu_debugfs_regs_pcie_write,
1181 	.llseek = default_llseek
1182 };
1183 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1184 	.owner = THIS_MODULE,
1185 	.read = amdgpu_debugfs_regs_smc_read,
1186 	.write = amdgpu_debugfs_regs_smc_write,
1187 	.llseek = default_llseek
1188 };
1189 
1190 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1191 	.owner = THIS_MODULE,
1192 	.read = amdgpu_debugfs_gca_config_read,
1193 	.llseek = default_llseek
1194 };
1195 
1196 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1197 	.owner = THIS_MODULE,
1198 	.read = amdgpu_debugfs_sensor_read,
1199 	.llseek = default_llseek
1200 };
1201 
1202 static const struct file_operations amdgpu_debugfs_wave_fops = {
1203 	.owner = THIS_MODULE,
1204 	.read = amdgpu_debugfs_wave_read,
1205 	.llseek = default_llseek
1206 };
1207 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1208 	.owner = THIS_MODULE,
1209 	.read = amdgpu_debugfs_gpr_read,
1210 	.llseek = default_llseek
1211 };
1212 
1213 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1214 	.owner = THIS_MODULE,
1215 	.read = amdgpu_debugfs_gfxoff_read,
1216 	.write = amdgpu_debugfs_gfxoff_write,
1217 	.llseek = default_llseek
1218 };
1219 
1220 static const struct file_operations *debugfs_regs[] = {
1221 	&amdgpu_debugfs_regs_fops,
1222 	&amdgpu_debugfs_regs2_fops,
1223 	&amdgpu_debugfs_regs_didt_fops,
1224 	&amdgpu_debugfs_regs_pcie_fops,
1225 	&amdgpu_debugfs_regs_smc_fops,
1226 	&amdgpu_debugfs_gca_config_fops,
1227 	&amdgpu_debugfs_sensors_fops,
1228 	&amdgpu_debugfs_wave_fops,
1229 	&amdgpu_debugfs_gpr_fops,
1230 	&amdgpu_debugfs_gfxoff_fops,
1231 };
1232 
1233 static const char *debugfs_regs_names[] = {
1234 	"amdgpu_regs",
1235 	"amdgpu_regs2",
1236 	"amdgpu_regs_didt",
1237 	"amdgpu_regs_pcie",
1238 	"amdgpu_regs_smc",
1239 	"amdgpu_gca_config",
1240 	"amdgpu_sensors",
1241 	"amdgpu_wave",
1242 	"amdgpu_gpr",
1243 	"amdgpu_gfxoff",
1244 };
1245 
1246 /**
1247  * amdgpu_debugfs_regs_init -	Initialize debugfs entries that provide
1248  * 				register access.
1249  *
1250  * @adev: The device to attach the debugfs entries to
1251  */
1252 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1253 {
1254 	struct drm_minor *minor = adev_to_drm(adev)->primary;
1255 	struct dentry *ent, *root = minor->debugfs_root;
1256 	unsigned int i;
1257 
1258 	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1259 		ent = debugfs_create_file(debugfs_regs_names[i],
1260 					  S_IFREG | S_IRUGO, root,
1261 					  adev, debugfs_regs[i]);
1262 		if (!i && !IS_ERR_OR_NULL(ent))
1263 			i_size_write(ent->d_inode, adev->rmmio_size);
1264 	}
1265 
1266 	return 0;
1267 }
1268 
1269 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1270 {
1271 	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1272 	struct drm_device *dev = adev_to_drm(adev);
1273 	int r = 0, i;
1274 
1275 	r = pm_runtime_get_sync(dev->dev);
1276 	if (r < 0) {
1277 		pm_runtime_put_autosuspend(dev->dev);
1278 		return r;
1279 	}
1280 
1281 	/* Avoid accidently unparking the sched thread during GPU reset */
1282 	r = down_write_killable(&adev->reset_sem);
1283 	if (r)
1284 		return r;
1285 
1286 	/* hold on the scheduler */
1287 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1288 		struct amdgpu_ring *ring = adev->rings[i];
1289 
1290 		if (!ring || !ring->sched.thread)
1291 			continue;
1292 		kthread_park(ring->sched.thread);
1293 	}
1294 
1295 	seq_printf(m, "run ib test:\n");
1296 	r = amdgpu_ib_ring_tests(adev);
1297 	if (r)
1298 		seq_printf(m, "ib ring tests failed (%d).\n", r);
1299 	else
1300 		seq_printf(m, "ib ring tests passed.\n");
1301 
1302 	/* go on the scheduler */
1303 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1304 		struct amdgpu_ring *ring = adev->rings[i];
1305 
1306 		if (!ring || !ring->sched.thread)
1307 			continue;
1308 		kthread_unpark(ring->sched.thread);
1309 	}
1310 
1311 	up_write(&adev->reset_sem);
1312 
1313 	pm_runtime_mark_last_busy(dev->dev);
1314 	pm_runtime_put_autosuspend(dev->dev);
1315 
1316 	return 0;
1317 }
1318 
1319 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1320 {
1321 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1322 	struct drm_device *dev = adev_to_drm(adev);
1323 	int r;
1324 
1325 	r = pm_runtime_get_sync(dev->dev);
1326 	if (r < 0) {
1327 		pm_runtime_put_autosuspend(dev->dev);
1328 		return r;
1329 	}
1330 
1331 	*val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1332 
1333 	pm_runtime_mark_last_busy(dev->dev);
1334 	pm_runtime_put_autosuspend(dev->dev);
1335 
1336 	return 0;
1337 }
1338 
1339 
1340 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1341 {
1342 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1343 	struct drm_device *dev = adev_to_drm(adev);
1344 	int r;
1345 
1346 	r = pm_runtime_get_sync(dev->dev);
1347 	if (r < 0) {
1348 		pm_runtime_put_autosuspend(dev->dev);
1349 		return r;
1350 	}
1351 
1352 	*val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1353 
1354 	pm_runtime_mark_last_busy(dev->dev);
1355 	pm_runtime_put_autosuspend(dev->dev);
1356 
1357 	return 0;
1358 }
1359 
1360 
1361 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1362 {
1363 	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1364 	struct drm_device *dev = adev_to_drm(adev);
1365 	struct drm_file *file;
1366 	int r;
1367 
1368 	r = mutex_lock_interruptible(&dev->filelist_mutex);
1369 	if (r)
1370 		return r;
1371 
1372 	list_for_each_entry(file, &dev->filelist, lhead) {
1373 		struct amdgpu_fpriv *fpriv = file->driver_priv;
1374 		struct amdgpu_vm *vm = &fpriv->vm;
1375 
1376 		seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1377 				vm->task_info.pid, vm->task_info.process_name);
1378 		r = amdgpu_bo_reserve(vm->root.bo, true);
1379 		if (r)
1380 			break;
1381 		amdgpu_debugfs_vm_bo_info(vm, m);
1382 		amdgpu_bo_unreserve(vm->root.bo);
1383 	}
1384 
1385 	mutex_unlock(&dev->filelist_mutex);
1386 
1387 	return r;
1388 }
1389 
1390 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1391 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1392 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1393 			 NULL, "%lld\n");
1394 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1395 			 NULL, "%lld\n");
1396 
1397 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1398 					  struct dma_fence **fences)
1399 {
1400 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
1401 	uint32_t sync_seq, last_seq;
1402 
1403 	last_seq = atomic_read(&ring->fence_drv.last_seq);
1404 	sync_seq = ring->fence_drv.sync_seq;
1405 
1406 	last_seq &= drv->num_fences_mask;
1407 	sync_seq &= drv->num_fences_mask;
1408 
1409 	do {
1410 		struct dma_fence *fence, **ptr;
1411 
1412 		++last_seq;
1413 		last_seq &= drv->num_fences_mask;
1414 		ptr = &drv->fences[last_seq];
1415 
1416 		fence = rcu_dereference_protected(*ptr, 1);
1417 		RCU_INIT_POINTER(*ptr, NULL);
1418 
1419 		if (!fence)
1420 			continue;
1421 
1422 		fences[last_seq] = fence;
1423 
1424 	} while (last_seq != sync_seq);
1425 }
1426 
1427 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1428 					    int length)
1429 {
1430 	int i;
1431 	struct dma_fence *fence;
1432 
1433 	for (i = 0; i < length; i++) {
1434 		fence = fences[i];
1435 		if (!fence)
1436 			continue;
1437 		dma_fence_signal(fence);
1438 		dma_fence_put(fence);
1439 	}
1440 }
1441 
1442 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1443 {
1444 	struct drm_sched_job *s_job;
1445 	struct dma_fence *fence;
1446 
1447 	spin_lock(&sched->job_list_lock);
1448 	list_for_each_entry(s_job, &sched->pending_list, list) {
1449 		fence = sched->ops->run_job(s_job);
1450 		dma_fence_put(fence);
1451 	}
1452 	spin_unlock(&sched->job_list_lock);
1453 }
1454 
1455 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1456 {
1457 	struct amdgpu_job *job;
1458 	struct drm_sched_job *s_job, *tmp;
1459 	uint32_t preempt_seq;
1460 	struct dma_fence *fence, **ptr;
1461 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
1462 	struct drm_gpu_scheduler *sched = &ring->sched;
1463 	bool preempted = true;
1464 
1465 	if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1466 		return;
1467 
1468 	preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1469 	if (preempt_seq <= atomic_read(&drv->last_seq)) {
1470 		preempted = false;
1471 		goto no_preempt;
1472 	}
1473 
1474 	preempt_seq &= drv->num_fences_mask;
1475 	ptr = &drv->fences[preempt_seq];
1476 	fence = rcu_dereference_protected(*ptr, 1);
1477 
1478 no_preempt:
1479 	spin_lock(&sched->job_list_lock);
1480 	list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1481 		if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1482 			/* remove job from ring_mirror_list */
1483 			list_del_init(&s_job->list);
1484 			sched->ops->free_job(s_job);
1485 			continue;
1486 		}
1487 		job = to_amdgpu_job(s_job);
1488 		if (preempted && (&job->hw_fence) == fence)
1489 			/* mark the job as preempted */
1490 			job->preemption_status |= AMDGPU_IB_PREEMPTED;
1491 	}
1492 	spin_unlock(&sched->job_list_lock);
1493 }
1494 
1495 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1496 {
1497 	int r, resched, length;
1498 	struct amdgpu_ring *ring;
1499 	struct dma_fence **fences = NULL;
1500 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1501 
1502 	if (val >= AMDGPU_MAX_RINGS)
1503 		return -EINVAL;
1504 
1505 	ring = adev->rings[val];
1506 
1507 	if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1508 		return -EINVAL;
1509 
1510 	/* the last preemption failed */
1511 	if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1512 		return -EBUSY;
1513 
1514 	length = ring->fence_drv.num_fences_mask + 1;
1515 	fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1516 	if (!fences)
1517 		return -ENOMEM;
1518 
1519 	/* Avoid accidently unparking the sched thread during GPU reset */
1520 	r = down_read_killable(&adev->reset_sem);
1521 	if (r)
1522 		goto pro_end;
1523 
1524 	/* stop the scheduler */
1525 	kthread_park(ring->sched.thread);
1526 
1527 	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1528 
1529 	/* preempt the IB */
1530 	r = amdgpu_ring_preempt_ib(ring);
1531 	if (r) {
1532 		DRM_WARN("failed to preempt ring %d\n", ring->idx);
1533 		goto failure;
1534 	}
1535 
1536 	amdgpu_fence_process(ring);
1537 
1538 	if (atomic_read(&ring->fence_drv.last_seq) !=
1539 	    ring->fence_drv.sync_seq) {
1540 		DRM_INFO("ring %d was preempted\n", ring->idx);
1541 
1542 		amdgpu_ib_preempt_mark_partial_job(ring);
1543 
1544 		/* swap out the old fences */
1545 		amdgpu_ib_preempt_fences_swap(ring, fences);
1546 
1547 		amdgpu_fence_driver_force_completion(ring);
1548 
1549 		/* resubmit unfinished jobs */
1550 		amdgpu_ib_preempt_job_recovery(&ring->sched);
1551 
1552 		/* wait for jobs finished */
1553 		amdgpu_fence_wait_empty(ring);
1554 
1555 		/* signal the old fences */
1556 		amdgpu_ib_preempt_signal_fences(fences, length);
1557 	}
1558 
1559 failure:
1560 	/* restart the scheduler */
1561 	kthread_unpark(ring->sched.thread);
1562 
1563 	up_read(&adev->reset_sem);
1564 
1565 	ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1566 
1567 pro_end:
1568 	kfree(fences);
1569 
1570 	return r;
1571 }
1572 
1573 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1574 {
1575 	int ret = 0;
1576 	uint32_t max_freq, min_freq;
1577 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1578 
1579 	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1580 		return -EINVAL;
1581 
1582 	ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1583 	if (ret < 0) {
1584 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1585 		return ret;
1586 	}
1587 
1588 	if (is_support_sw_smu(adev)) {
1589 		ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
1590 		if (ret || val > max_freq || val < min_freq)
1591 			return -EINVAL;
1592 		ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
1593 	} else {
1594 		return 0;
1595 	}
1596 
1597 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1598 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1599 
1600 	if (ret)
1601 		return -EINVAL;
1602 
1603 	return 0;
1604 }
1605 
1606 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1607 			amdgpu_debugfs_ib_preempt, "%llu\n");
1608 
1609 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
1610 			amdgpu_debugfs_sclk_set, "%llu\n");
1611 
1612 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1613 {
1614 	struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
1615 	struct dentry *ent;
1616 	int r, i;
1617 
1618 	if (!debugfs_initialized())
1619 		return 0;
1620 
1621 	ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
1622 				  &fops_ib_preempt);
1623 	if (IS_ERR(ent)) {
1624 		DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1625 		return PTR_ERR(ent);
1626 	}
1627 
1628 	ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
1629 				  &fops_sclk_set);
1630 	if (IS_ERR(ent)) {
1631 		DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1632 		return PTR_ERR(ent);
1633 	}
1634 
1635 	/* Register debugfs entries for amdgpu_ttm */
1636 	amdgpu_ttm_debugfs_init(adev);
1637 	amdgpu_debugfs_pm_init(adev);
1638 	amdgpu_debugfs_sa_init(adev);
1639 	amdgpu_debugfs_fence_init(adev);
1640 	amdgpu_debugfs_gem_init(adev);
1641 
1642 	r = amdgpu_debugfs_regs_init(adev);
1643 	if (r)
1644 		DRM_ERROR("registering register debugfs failed (%d).\n", r);
1645 
1646 	amdgpu_debugfs_firmware_init(adev);
1647 
1648 #if defined(CONFIG_DRM_AMD_DC)
1649 	if (amdgpu_device_has_dc_support(adev))
1650 		dtn_debugfs_init(adev);
1651 #endif
1652 
1653 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1654 		struct amdgpu_ring *ring = adev->rings[i];
1655 
1656 		if (!ring)
1657 			continue;
1658 
1659 		amdgpu_debugfs_ring_init(adev, ring);
1660 	}
1661 
1662 	amdgpu_ras_debugfs_create_all(adev);
1663 	amdgpu_rap_debugfs_init(adev);
1664 	amdgpu_securedisplay_debugfs_init(adev);
1665 	amdgpu_fw_attestation_debugfs_init(adev);
1666 
1667 	debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
1668 			    &amdgpu_evict_vram_fops);
1669 	debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
1670 			    &amdgpu_evict_gtt_fops);
1671 	debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
1672 			    &amdgpu_debugfs_test_ib_fops);
1673 	debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
1674 			    &amdgpu_debugfs_vm_info_fops);
1675 
1676 	adev->debugfs_vbios_blob.data = adev->bios;
1677 	adev->debugfs_vbios_blob.size = adev->bios_size;
1678 	debugfs_create_blob("amdgpu_vbios", 0444, root,
1679 			    &adev->debugfs_vbios_blob);
1680 
1681 	adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
1682 	adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
1683 	debugfs_create_blob("amdgpu_discovery", 0444, root,
1684 			    &adev->debugfs_discovery_blob);
1685 
1686 	return 0;
1687 }
1688 
1689 #else
1690 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1691 {
1692 	return 0;
1693 }
1694 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1695 {
1696 	return 0;
1697 }
1698 #endif
1699