1 /*
2  * Copyright 2008 Advanced Micro Devices, Inc.
3  * Copyright 2008 Red Hat Inc.
4  * Copyright 2009 Jerome Glisse.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22  * OTHER DEALINGS IN THE SOFTWARE.
23  *
24  */
25 
26 #include <linux/kthread.h>
27 #include <linux/pci.h>
28 #include <linux/uaccess.h>
29 #include <linux/pm_runtime.h>
30 
31 #include "amdgpu.h"
32 #include "amdgpu_pm.h"
33 #include "amdgpu_dm_debugfs.h"
34 #include "amdgpu_ras.h"
35 #include "amdgpu_rap.h"
36 #include "amdgpu_securedisplay.h"
37 #include "amdgpu_fw_attestation.h"
38 #include "amdgpu_umr.h"
39 
40 #include "amdgpu_reset.h"
41 #include "amdgpu_psp_ta.h"
42 
43 #if defined(CONFIG_DEBUG_FS)
44 
45 /**
46  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
47  *
48  * @read: True if reading
49  * @f: open file handle
50  * @buf: User buffer to write/read to
51  * @size: Number of bytes to write/read
52  * @pos:  Offset to seek to
53  *
54  * This debugfs entry has special meaning on the offset being sought.
55  * Various bits have different meanings:
56  *
57  * Bit 62:  Indicates a GRBM bank switch is needed
58  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
59  *	    zero)
60  * Bits 24..33: The SE or ME selector if needed
61  * Bits 34..43: The SH (or SA) or PIPE selector if needed
62  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
63  *
64  * Bit 23:  Indicates that the PM power gating lock should be held
65  *	    This is necessary to read registers that might be
66  *	    unreliable during a power gating transistion.
67  *
68  * The lower bits are the BYTE offset of the register to read.  This
69  * allows reading multiple registers in a single call and having
70  * the returned size reflect that.
71  */
72 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
73 		char __user *buf, size_t size, loff_t *pos)
74 {
75 	struct amdgpu_device *adev = file_inode(f)->i_private;
76 	ssize_t result = 0;
77 	int r;
78 	bool pm_pg_lock, use_bank, use_ring;
79 	unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
80 
81 	pm_pg_lock = use_bank = use_ring = false;
82 	instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
83 
84 	if (size & 0x3 || *pos & 0x3 ||
85 			((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
86 		return -EINVAL;
87 
88 	/* are we reading registers for which a PG lock is necessary? */
89 	pm_pg_lock = (*pos >> 23) & 1;
90 
91 	if (*pos & (1ULL << 62)) {
92 		se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
93 		sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
94 		instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
95 
96 		if (se_bank == 0x3FF)
97 			se_bank = 0xFFFFFFFF;
98 		if (sh_bank == 0x3FF)
99 			sh_bank = 0xFFFFFFFF;
100 		if (instance_bank == 0x3FF)
101 			instance_bank = 0xFFFFFFFF;
102 		use_bank = true;
103 	} else if (*pos & (1ULL << 61)) {
104 
105 		me = (*pos & GENMASK_ULL(33, 24)) >> 24;
106 		pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
107 		queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
108 		vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
109 
110 		use_ring = true;
111 	} else {
112 		use_bank = use_ring = false;
113 	}
114 
115 	*pos &= (1UL << 22) - 1;
116 
117 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
118 	if (r < 0) {
119 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
120 		return r;
121 	}
122 
123 	r = amdgpu_virt_enable_access_debugfs(adev);
124 	if (r < 0) {
125 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
126 		return r;
127 	}
128 
129 	if (use_bank) {
130 		if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
131 		    (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
132 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
133 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
134 			amdgpu_virt_disable_access_debugfs(adev);
135 			return -EINVAL;
136 		}
137 		mutex_lock(&adev->grbm_idx_mutex);
138 		amdgpu_gfx_select_se_sh(adev, se_bank,
139 					sh_bank, instance_bank, 0);
140 	} else if (use_ring) {
141 		mutex_lock(&adev->srbm_mutex);
142 		amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
143 	}
144 
145 	if (pm_pg_lock)
146 		mutex_lock(&adev->pm.mutex);
147 
148 	while (size) {
149 		uint32_t value;
150 
151 		if (read) {
152 			value = RREG32(*pos >> 2);
153 			r = put_user(value, (uint32_t *)buf);
154 		} else {
155 			r = get_user(value, (uint32_t *)buf);
156 			if (!r)
157 				amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
158 		}
159 		if (r) {
160 			result = r;
161 			goto end;
162 		}
163 
164 		result += 4;
165 		buf += 4;
166 		*pos += 4;
167 		size -= 4;
168 	}
169 
170 end:
171 	if (use_bank) {
172 		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
173 		mutex_unlock(&adev->grbm_idx_mutex);
174 	} else if (use_ring) {
175 		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
176 		mutex_unlock(&adev->srbm_mutex);
177 	}
178 
179 	if (pm_pg_lock)
180 		mutex_unlock(&adev->pm.mutex);
181 
182 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
183 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
184 
185 	amdgpu_virt_disable_access_debugfs(adev);
186 	return result;
187 }
188 
189 /*
190  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
191  */
192 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
193 					size_t size, loff_t *pos)
194 {
195 	return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
196 }
197 
198 /*
199  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
200  */
201 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
202 					 size_t size, loff_t *pos)
203 {
204 	return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
205 }
206 
207 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
208 {
209 	struct amdgpu_debugfs_regs2_data *rd;
210 
211 	rd = kzalloc(sizeof(*rd), GFP_KERNEL);
212 	if (!rd)
213 		return -ENOMEM;
214 	rd->adev = file_inode(file)->i_private;
215 	file->private_data = rd;
216 	mutex_init(&rd->lock);
217 
218 	return 0;
219 }
220 
221 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
222 {
223 	struct amdgpu_debugfs_regs2_data *rd = file->private_data;
224 
225 	mutex_destroy(&rd->lock);
226 	kfree(file->private_data);
227 	return 0;
228 }
229 
230 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
231 {
232 	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
233 	struct amdgpu_device *adev = rd->adev;
234 	ssize_t result = 0;
235 	int r;
236 	uint32_t value;
237 
238 	if (size & 0x3 || offset & 0x3)
239 		return -EINVAL;
240 
241 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
242 	if (r < 0) {
243 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
244 		return r;
245 	}
246 
247 	r = amdgpu_virt_enable_access_debugfs(adev);
248 	if (r < 0) {
249 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
250 		return r;
251 	}
252 
253 	mutex_lock(&rd->lock);
254 
255 	if (rd->id.use_grbm) {
256 		if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
257 		    (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
258 			pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
259 			pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
260 			amdgpu_virt_disable_access_debugfs(adev);
261 			mutex_unlock(&rd->lock);
262 			return -EINVAL;
263 		}
264 		mutex_lock(&adev->grbm_idx_mutex);
265 		amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
266 						  rd->id.grbm.sh,
267 						  rd->id.grbm.instance, rd->id.xcc_id);
268 	}
269 
270 	if (rd->id.use_srbm) {
271 		mutex_lock(&adev->srbm_mutex);
272 		amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
273 					    rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
274 	}
275 
276 	if (rd->id.pg_lock)
277 		mutex_lock(&adev->pm.mutex);
278 
279 	while (size) {
280 		if (!write_en) {
281 			value = RREG32(offset >> 2);
282 			r = put_user(value, (uint32_t *)buf);
283 		} else {
284 			r = get_user(value, (uint32_t *)buf);
285 			if (!r)
286 				amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id);
287 		}
288 		if (r) {
289 			result = r;
290 			goto end;
291 		}
292 		offset += 4;
293 		size -= 4;
294 		result += 4;
295 		buf += 4;
296 	}
297 end:
298 	if (rd->id.use_grbm) {
299 		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
300 		mutex_unlock(&adev->grbm_idx_mutex);
301 	}
302 
303 	if (rd->id.use_srbm) {
304 		amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
305 		mutex_unlock(&adev->srbm_mutex);
306 	}
307 
308 	if (rd->id.pg_lock)
309 		mutex_unlock(&adev->pm.mutex);
310 
311 	mutex_unlock(&rd->lock);
312 
313 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
314 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
315 
316 	amdgpu_virt_disable_access_debugfs(adev);
317 	return result;
318 }
319 
320 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
321 {
322 	struct amdgpu_debugfs_regs2_data *rd = f->private_data;
323 	struct amdgpu_debugfs_regs2_iocdata v1_data;
324 	int r;
325 
326 	mutex_lock(&rd->lock);
327 
328 	switch (cmd) {
329 	case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
330 		r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
331 				   sizeof(rd->id));
332 		if (r)
333 			r = -EINVAL;
334 		goto done;
335 	case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
336 		r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
337 				   sizeof(v1_data));
338 		if (r) {
339 			r = -EINVAL;
340 			goto done;
341 		}
342 		goto v1_copy;
343 	default:
344 		r = -EINVAL;
345 		goto done;
346 	}
347 
348 v1_copy:
349 	rd->id.use_srbm = v1_data.use_srbm;
350 	rd->id.use_grbm = v1_data.use_grbm;
351 	rd->id.pg_lock = v1_data.pg_lock;
352 	rd->id.grbm.se = v1_data.grbm.se;
353 	rd->id.grbm.sh = v1_data.grbm.sh;
354 	rd->id.grbm.instance = v1_data.grbm.instance;
355 	rd->id.srbm.me = v1_data.srbm.me;
356 	rd->id.srbm.pipe = v1_data.srbm.pipe;
357 	rd->id.srbm.queue = v1_data.srbm.queue;
358 	rd->id.xcc_id = 0;
359 done:
360 	mutex_unlock(&rd->lock);
361 	return r;
362 }
363 
364 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
365 {
366 	return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
367 }
368 
369 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
370 {
371 	return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
372 }
373 
374 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
375 {
376 	struct amdgpu_debugfs_gprwave_data *rd;
377 
378 	rd = kzalloc(sizeof(*rd), GFP_KERNEL);
379 	if (!rd)
380 		return -ENOMEM;
381 	rd->adev = file_inode(file)->i_private;
382 	file->private_data = rd;
383 	mutex_init(&rd->lock);
384 
385 	return 0;
386 }
387 
388 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
389 {
390 	struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
391 
392 	mutex_destroy(&rd->lock);
393 	kfree(file->private_data);
394 	return 0;
395 }
396 
397 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
398 {
399 	struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
400 	struct amdgpu_device *adev = rd->adev;
401 	ssize_t result = 0;
402 	int r;
403 	uint32_t *data, x;
404 
405 	if (size & 0x3 || *pos & 0x3)
406 		return -EINVAL;
407 
408 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
409 	if (r < 0) {
410 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
411 		return r;
412 	}
413 
414 	r = amdgpu_virt_enable_access_debugfs(adev);
415 	if (r < 0) {
416 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
417 		return r;
418 	}
419 
420 	data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
421 	if (!data) {
422 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
423 		amdgpu_virt_disable_access_debugfs(adev);
424 		return -ENOMEM;
425 	}
426 
427 	/* switch to the specific se/sh/cu */
428 	mutex_lock(&adev->grbm_idx_mutex);
429 	amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
430 
431 	if (!rd->id.gpr_or_wave) {
432 		x = 0;
433 		if (adev->gfx.funcs->read_wave_data)
434 			adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
435 	} else {
436 		x = size >> 2;
437 		if (rd->id.gpr.vpgr_or_sgpr) {
438 			if (adev->gfx.funcs->read_wave_vgprs)
439 				adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
440 		} else {
441 			if (adev->gfx.funcs->read_wave_sgprs)
442 				adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
443 		}
444 	}
445 
446 	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
447 	mutex_unlock(&adev->grbm_idx_mutex);
448 
449 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
450 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
451 
452 	if (!x) {
453 		result = -EINVAL;
454 		goto done;
455 	}
456 
457 	while (size && (*pos < x * 4)) {
458 		uint32_t value;
459 
460 		value = data[*pos >> 2];
461 		r = put_user(value, (uint32_t *)buf);
462 		if (r) {
463 			result = r;
464 			goto done;
465 		}
466 
467 		result += 4;
468 		buf += 4;
469 		*pos += 4;
470 		size -= 4;
471 	}
472 
473 done:
474 	amdgpu_virt_disable_access_debugfs(adev);
475 	kfree(data);
476 	return result;
477 }
478 
479 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
480 {
481 	struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
482 	int r = 0;
483 
484 	mutex_lock(&rd->lock);
485 
486 	switch (cmd) {
487 	case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
488 		if (copy_from_user(&rd->id,
489 				   (struct amdgpu_debugfs_gprwave_iocdata *)data,
490 				   sizeof(rd->id)))
491 			r = -EFAULT;
492 		goto done;
493 	default:
494 		r = -EINVAL;
495 		goto done;
496 	}
497 
498 done:
499 	mutex_unlock(&rd->lock);
500 	return r;
501 }
502 
503 
504 
505 
506 /**
507  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
508  *
509  * @f: open file handle
510  * @buf: User buffer to store read data in
511  * @size: Number of bytes to read
512  * @pos:  Offset to seek to
513  *
514  * The lower bits are the BYTE offset of the register to read.  This
515  * allows reading multiple registers in a single call and having
516  * the returned size reflect that.
517  */
518 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
519 					size_t size, loff_t *pos)
520 {
521 	struct amdgpu_device *adev = file_inode(f)->i_private;
522 	ssize_t result = 0;
523 	int r;
524 
525 	if (size & 0x3 || *pos & 0x3)
526 		return -EINVAL;
527 
528 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
529 	if (r < 0) {
530 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
531 		return r;
532 	}
533 
534 	r = amdgpu_virt_enable_access_debugfs(adev);
535 	if (r < 0) {
536 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
537 		return r;
538 	}
539 
540 	while (size) {
541 		uint32_t value;
542 
543 		value = RREG32_PCIE(*pos);
544 		r = put_user(value, (uint32_t *)buf);
545 		if (r)
546 			goto out;
547 
548 		result += 4;
549 		buf += 4;
550 		*pos += 4;
551 		size -= 4;
552 	}
553 
554 	r = result;
555 out:
556 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
557 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
558 	amdgpu_virt_disable_access_debugfs(adev);
559 	return r;
560 }
561 
562 /**
563  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
564  *
565  * @f: open file handle
566  * @buf: User buffer to write data from
567  * @size: Number of bytes to write
568  * @pos:  Offset to seek to
569  *
570  * The lower bits are the BYTE offset of the register to write.  This
571  * allows writing multiple registers in a single call and having
572  * the returned size reflect that.
573  */
574 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
575 					 size_t size, loff_t *pos)
576 {
577 	struct amdgpu_device *adev = file_inode(f)->i_private;
578 	ssize_t result = 0;
579 	int r;
580 
581 	if (size & 0x3 || *pos & 0x3)
582 		return -EINVAL;
583 
584 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
585 	if (r < 0) {
586 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
587 		return r;
588 	}
589 
590 	r = amdgpu_virt_enable_access_debugfs(adev);
591 	if (r < 0) {
592 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
593 		return r;
594 	}
595 
596 	while (size) {
597 		uint32_t value;
598 
599 		r = get_user(value, (uint32_t *)buf);
600 		if (r)
601 			goto out;
602 
603 		WREG32_PCIE(*pos, value);
604 
605 		result += 4;
606 		buf += 4;
607 		*pos += 4;
608 		size -= 4;
609 	}
610 
611 	r = result;
612 out:
613 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
614 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
615 	amdgpu_virt_disable_access_debugfs(adev);
616 	return r;
617 }
618 
619 /**
620  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
621  *
622  * @f: open file handle
623  * @buf: User buffer to store read data in
624  * @size: Number of bytes to read
625  * @pos:  Offset to seek to
626  *
627  * The lower bits are the BYTE offset of the register to read.  This
628  * allows reading multiple registers in a single call and having
629  * the returned size reflect that.
630  */
631 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
632 					size_t size, loff_t *pos)
633 {
634 	struct amdgpu_device *adev = file_inode(f)->i_private;
635 	ssize_t result = 0;
636 	int r;
637 
638 	if (size & 0x3 || *pos & 0x3)
639 		return -EINVAL;
640 
641 	if (!adev->didt_rreg)
642 		return -EOPNOTSUPP;
643 
644 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
645 	if (r < 0) {
646 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
647 		return r;
648 	}
649 
650 	r = amdgpu_virt_enable_access_debugfs(adev);
651 	if (r < 0) {
652 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
653 		return r;
654 	}
655 
656 	while (size) {
657 		uint32_t value;
658 
659 		value = RREG32_DIDT(*pos >> 2);
660 		r = put_user(value, (uint32_t *)buf);
661 		if (r)
662 			goto out;
663 
664 		result += 4;
665 		buf += 4;
666 		*pos += 4;
667 		size -= 4;
668 	}
669 
670 	r = result;
671 out:
672 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
673 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
674 	amdgpu_virt_disable_access_debugfs(adev);
675 	return r;
676 }
677 
678 /**
679  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
680  *
681  * @f: open file handle
682  * @buf: User buffer to write data from
683  * @size: Number of bytes to write
684  * @pos:  Offset to seek to
685  *
686  * The lower bits are the BYTE offset of the register to write.  This
687  * allows writing multiple registers in a single call and having
688  * the returned size reflect that.
689  */
690 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
691 					 size_t size, loff_t *pos)
692 {
693 	struct amdgpu_device *adev = file_inode(f)->i_private;
694 	ssize_t result = 0;
695 	int r;
696 
697 	if (size & 0x3 || *pos & 0x3)
698 		return -EINVAL;
699 
700 	if (!adev->didt_wreg)
701 		return -EOPNOTSUPP;
702 
703 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
704 	if (r < 0) {
705 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
706 		return r;
707 	}
708 
709 	r = amdgpu_virt_enable_access_debugfs(adev);
710 	if (r < 0) {
711 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
712 		return r;
713 	}
714 
715 	while (size) {
716 		uint32_t value;
717 
718 		r = get_user(value, (uint32_t *)buf);
719 		if (r)
720 			goto out;
721 
722 		WREG32_DIDT(*pos >> 2, value);
723 
724 		result += 4;
725 		buf += 4;
726 		*pos += 4;
727 		size -= 4;
728 	}
729 
730 	r = result;
731 out:
732 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
733 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
734 	amdgpu_virt_disable_access_debugfs(adev);
735 	return r;
736 }
737 
738 /**
739  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
740  *
741  * @f: open file handle
742  * @buf: User buffer to store read data in
743  * @size: Number of bytes to read
744  * @pos:  Offset to seek to
745  *
746  * The lower bits are the BYTE offset of the register to read.  This
747  * allows reading multiple registers in a single call and having
748  * the returned size reflect that.
749  */
750 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
751 					size_t size, loff_t *pos)
752 {
753 	struct amdgpu_device *adev = file_inode(f)->i_private;
754 	ssize_t result = 0;
755 	int r;
756 
757 	if (!adev->smc_rreg)
758 		return -EOPNOTSUPP;
759 
760 	if (size & 0x3 || *pos & 0x3)
761 		return -EINVAL;
762 
763 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
764 	if (r < 0) {
765 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
766 		return r;
767 	}
768 
769 	r = amdgpu_virt_enable_access_debugfs(adev);
770 	if (r < 0) {
771 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
772 		return r;
773 	}
774 
775 	while (size) {
776 		uint32_t value;
777 
778 		value = RREG32_SMC(*pos);
779 		r = put_user(value, (uint32_t *)buf);
780 		if (r)
781 			goto out;
782 
783 		result += 4;
784 		buf += 4;
785 		*pos += 4;
786 		size -= 4;
787 	}
788 
789 	r = result;
790 out:
791 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
792 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
793 	amdgpu_virt_disable_access_debugfs(adev);
794 	return r;
795 }
796 
797 /**
798  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
799  *
800  * @f: open file handle
801  * @buf: User buffer to write data from
802  * @size: Number of bytes to write
803  * @pos:  Offset to seek to
804  *
805  * The lower bits are the BYTE offset of the register to write.  This
806  * allows writing multiple registers in a single call and having
807  * the returned size reflect that.
808  */
809 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
810 					 size_t size, loff_t *pos)
811 {
812 	struct amdgpu_device *adev = file_inode(f)->i_private;
813 	ssize_t result = 0;
814 	int r;
815 
816 	if (!adev->smc_wreg)
817 		return -EOPNOTSUPP;
818 
819 	if (size & 0x3 || *pos & 0x3)
820 		return -EINVAL;
821 
822 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
823 	if (r < 0) {
824 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
825 		return r;
826 	}
827 
828 	r = amdgpu_virt_enable_access_debugfs(adev);
829 	if (r < 0) {
830 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
831 		return r;
832 	}
833 
834 	while (size) {
835 		uint32_t value;
836 
837 		r = get_user(value, (uint32_t *)buf);
838 		if (r)
839 			goto out;
840 
841 		WREG32_SMC(*pos, value);
842 
843 		result += 4;
844 		buf += 4;
845 		*pos += 4;
846 		size -= 4;
847 	}
848 
849 	r = result;
850 out:
851 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
852 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
853 	amdgpu_virt_disable_access_debugfs(adev);
854 	return r;
855 }
856 
857 /**
858  * amdgpu_debugfs_gca_config_read - Read from gfx config data
859  *
860  * @f: open file handle
861  * @buf: User buffer to store read data in
862  * @size: Number of bytes to read
863  * @pos:  Offset to seek to
864  *
865  * This file is used to access configuration data in a somewhat
866  * stable fashion.  The format is a series of DWORDs with the first
867  * indicating which revision it is.  New content is appended to the
868  * end so that older software can still read the data.
869  */
870 
871 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
872 					size_t size, loff_t *pos)
873 {
874 	struct amdgpu_device *adev = file_inode(f)->i_private;
875 	ssize_t result = 0;
876 	int r;
877 	uint32_t *config, no_regs = 0;
878 
879 	if (size & 0x3 || *pos & 0x3)
880 		return -EINVAL;
881 
882 	config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
883 	if (!config)
884 		return -ENOMEM;
885 
886 	/* version, increment each time something is added */
887 	config[no_regs++] = 5;
888 	config[no_regs++] = adev->gfx.config.max_shader_engines;
889 	config[no_regs++] = adev->gfx.config.max_tile_pipes;
890 	config[no_regs++] = adev->gfx.config.max_cu_per_sh;
891 	config[no_regs++] = adev->gfx.config.max_sh_per_se;
892 	config[no_regs++] = adev->gfx.config.max_backends_per_se;
893 	config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
894 	config[no_regs++] = adev->gfx.config.max_gprs;
895 	config[no_regs++] = adev->gfx.config.max_gs_threads;
896 	config[no_regs++] = adev->gfx.config.max_hw_contexts;
897 	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
898 	config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
899 	config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
900 	config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
901 	config[no_regs++] = adev->gfx.config.num_tile_pipes;
902 	config[no_regs++] = adev->gfx.config.backend_enable_mask;
903 	config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
904 	config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
905 	config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
906 	config[no_regs++] = adev->gfx.config.num_gpus;
907 	config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
908 	config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
909 	config[no_regs++] = adev->gfx.config.gb_addr_config;
910 	config[no_regs++] = adev->gfx.config.num_rbs;
911 
912 	/* rev==1 */
913 	config[no_regs++] = adev->rev_id;
914 	config[no_regs++] = lower_32_bits(adev->pg_flags);
915 	config[no_regs++] = lower_32_bits(adev->cg_flags);
916 
917 	/* rev==2 */
918 	config[no_regs++] = adev->family;
919 	config[no_regs++] = adev->external_rev_id;
920 
921 	/* rev==3 */
922 	config[no_regs++] = adev->pdev->device;
923 	config[no_regs++] = adev->pdev->revision;
924 	config[no_regs++] = adev->pdev->subsystem_device;
925 	config[no_regs++] = adev->pdev->subsystem_vendor;
926 
927 	/* rev==4 APU flag */
928 	config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
929 
930 	/* rev==5 PG/CG flag upper 32bit */
931 	config[no_regs++] = upper_32_bits(adev->pg_flags);
932 	config[no_regs++] = upper_32_bits(adev->cg_flags);
933 
934 	while (size && (*pos < no_regs * 4)) {
935 		uint32_t value;
936 
937 		value = config[*pos >> 2];
938 		r = put_user(value, (uint32_t *)buf);
939 		if (r) {
940 			kfree(config);
941 			return r;
942 		}
943 
944 		result += 4;
945 		buf += 4;
946 		*pos += 4;
947 		size -= 4;
948 	}
949 
950 	kfree(config);
951 	return result;
952 }
953 
954 /**
955  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
956  *
957  * @f: open file handle
958  * @buf: User buffer to store read data in
959  * @size: Number of bytes to read
960  * @pos:  Offset to seek to
961  *
962  * The offset is treated as the BYTE address of one of the sensors
963  * enumerated in amd/include/kgd_pp_interface.h under the
964  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
965  * you would use the offset 3 * 4 = 12.
966  */
967 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
968 					size_t size, loff_t *pos)
969 {
970 	struct amdgpu_device *adev = file_inode(f)->i_private;
971 	int idx, x, outsize, r, valuesize;
972 	uint32_t values[16];
973 
974 	if (size & 3 || *pos & 0x3)
975 		return -EINVAL;
976 
977 	if (!adev->pm.dpm_enabled)
978 		return -EINVAL;
979 
980 	/* convert offset to sensor number */
981 	idx = *pos >> 2;
982 
983 	valuesize = sizeof(values);
984 
985 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
986 	if (r < 0) {
987 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
988 		return r;
989 	}
990 
991 	r = amdgpu_virt_enable_access_debugfs(adev);
992 	if (r < 0) {
993 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
994 		return r;
995 	}
996 
997 	r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
998 
999 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1000 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1001 
1002 	if (r) {
1003 		amdgpu_virt_disable_access_debugfs(adev);
1004 		return r;
1005 	}
1006 
1007 	if (size > valuesize) {
1008 		amdgpu_virt_disable_access_debugfs(adev);
1009 		return -EINVAL;
1010 	}
1011 
1012 	outsize = 0;
1013 	x = 0;
1014 	if (!r) {
1015 		while (size) {
1016 			r = put_user(values[x++], (int32_t *)buf);
1017 			buf += 4;
1018 			size -= 4;
1019 			outsize += 4;
1020 		}
1021 	}
1022 
1023 	amdgpu_virt_disable_access_debugfs(adev);
1024 	return !r ? outsize : r;
1025 }
1026 
1027 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1028  *
1029  * @f: open file handle
1030  * @buf: User buffer to store read data in
1031  * @size: Number of bytes to read
1032  * @pos:  Offset to seek to
1033  *
1034  * The offset being sought changes which wave that the status data
1035  * will be returned for.  The bits are used as follows:
1036  *
1037  * Bits 0..6:	Byte offset into data
1038  * Bits 7..14:	SE selector
1039  * Bits 15..22:	SH/SA selector
1040  * Bits 23..30: CU/{WGP+SIMD} selector
1041  * Bits 31..36: WAVE ID selector
1042  * Bits 37..44: SIMD ID selector
1043  *
1044  * The returned data begins with one DWORD of version information
1045  * Followed by WAVE STATUS registers relevant to the GFX IP version
1046  * being used.  See gfx_v8_0_read_wave_data() for an example output.
1047  */
1048 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1049 					size_t size, loff_t *pos)
1050 {
1051 	struct amdgpu_device *adev = f->f_inode->i_private;
1052 	int r, x;
1053 	ssize_t result = 0;
1054 	uint32_t offset, se, sh, cu, wave, simd, data[32];
1055 
1056 	if (size & 3 || *pos & 3)
1057 		return -EINVAL;
1058 
1059 	/* decode offset */
1060 	offset = (*pos & GENMASK_ULL(6, 0));
1061 	se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1062 	sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1063 	cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1064 	wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1065 	simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1066 
1067 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1068 	if (r < 0) {
1069 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1070 		return r;
1071 	}
1072 
1073 	r = amdgpu_virt_enable_access_debugfs(adev);
1074 	if (r < 0) {
1075 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1076 		return r;
1077 	}
1078 
1079 	/* switch to the specific se/sh/cu */
1080 	mutex_lock(&adev->grbm_idx_mutex);
1081 	amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1082 
1083 	x = 0;
1084 	if (adev->gfx.funcs->read_wave_data)
1085 		adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1086 
1087 	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1088 	mutex_unlock(&adev->grbm_idx_mutex);
1089 
1090 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1091 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1092 
1093 	if (!x) {
1094 		amdgpu_virt_disable_access_debugfs(adev);
1095 		return -EINVAL;
1096 	}
1097 
1098 	while (size && (offset < x * 4)) {
1099 		uint32_t value;
1100 
1101 		value = data[offset >> 2];
1102 		r = put_user(value, (uint32_t *)buf);
1103 		if (r) {
1104 			amdgpu_virt_disable_access_debugfs(adev);
1105 			return r;
1106 		}
1107 
1108 		result += 4;
1109 		buf += 4;
1110 		offset += 4;
1111 		size -= 4;
1112 	}
1113 
1114 	amdgpu_virt_disable_access_debugfs(adev);
1115 	return result;
1116 }
1117 
1118 /** amdgpu_debugfs_gpr_read - Read wave gprs
1119  *
1120  * @f: open file handle
1121  * @buf: User buffer to store read data in
1122  * @size: Number of bytes to read
1123  * @pos:  Offset to seek to
1124  *
1125  * The offset being sought changes which wave that the status data
1126  * will be returned for.  The bits are used as follows:
1127  *
1128  * Bits 0..11:	Byte offset into data
1129  * Bits 12..19:	SE selector
1130  * Bits 20..27:	SH/SA selector
1131  * Bits 28..35: CU/{WGP+SIMD} selector
1132  * Bits 36..43: WAVE ID selector
1133  * Bits 37..44: SIMD ID selector
1134  * Bits 52..59: Thread selector
1135  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1136  *
1137  * The return data comes from the SGPR or VGPR register bank for
1138  * the selected operational unit.
1139  */
1140 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1141 					size_t size, loff_t *pos)
1142 {
1143 	struct amdgpu_device *adev = f->f_inode->i_private;
1144 	int r;
1145 	ssize_t result = 0;
1146 	uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1147 
1148 	if (size > 4096 || size & 3 || *pos & 3)
1149 		return -EINVAL;
1150 
1151 	/* decode offset */
1152 	offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1153 	se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1154 	sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1155 	cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1156 	wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1157 	simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1158 	thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1159 	bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1160 
1161 	data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1162 	if (!data)
1163 		return -ENOMEM;
1164 
1165 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1166 	if (r < 0)
1167 		goto err;
1168 
1169 	r = amdgpu_virt_enable_access_debugfs(adev);
1170 	if (r < 0)
1171 		goto err;
1172 
1173 	/* switch to the specific se/sh/cu */
1174 	mutex_lock(&adev->grbm_idx_mutex);
1175 	amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1176 
1177 	if (bank == 0) {
1178 		if (adev->gfx.funcs->read_wave_vgprs)
1179 			adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1180 	} else {
1181 		if (adev->gfx.funcs->read_wave_sgprs)
1182 			adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1183 	}
1184 
1185 	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1186 	mutex_unlock(&adev->grbm_idx_mutex);
1187 
1188 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1189 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1190 
1191 	while (size) {
1192 		uint32_t value;
1193 
1194 		value = data[result >> 2];
1195 		r = put_user(value, (uint32_t *)buf);
1196 		if (r) {
1197 			amdgpu_virt_disable_access_debugfs(adev);
1198 			goto err;
1199 		}
1200 
1201 		result += 4;
1202 		buf += 4;
1203 		size -= 4;
1204 	}
1205 
1206 	kfree(data);
1207 	amdgpu_virt_disable_access_debugfs(adev);
1208 	return result;
1209 
1210 err:
1211 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1212 	kfree(data);
1213 	return r;
1214 }
1215 
1216 /**
1217  * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1218  *
1219  * @f: open file handle
1220  * @buf: User buffer to store read data in
1221  * @size: Number of bytes to read
1222  * @pos:  Offset to seek to
1223  *
1224  * Read the last residency value logged. It doesn't auto update, one needs to
1225  * stop logging before getting the current value.
1226  */
1227 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1228 						    size_t size, loff_t *pos)
1229 {
1230 	struct amdgpu_device *adev = file_inode(f)->i_private;
1231 	ssize_t result = 0;
1232 	int r;
1233 
1234 	if (size & 0x3 || *pos & 0x3)
1235 		return -EINVAL;
1236 
1237 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1238 	if (r < 0) {
1239 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1240 		return r;
1241 	}
1242 
1243 	while (size) {
1244 		uint32_t value;
1245 
1246 		r = amdgpu_get_gfx_off_residency(adev, &value);
1247 		if (r)
1248 			goto out;
1249 
1250 		r = put_user(value, (uint32_t *)buf);
1251 		if (r)
1252 			goto out;
1253 
1254 		result += 4;
1255 		buf += 4;
1256 		*pos += 4;
1257 		size -= 4;
1258 	}
1259 
1260 	r = result;
1261 out:
1262 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1263 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1264 
1265 	return r;
1266 }
1267 
1268 /**
1269  * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1270  *
1271  * @f: open file handle
1272  * @buf: User buffer to write data from
1273  * @size: Number of bytes to write
1274  * @pos:  Offset to seek to
1275  *
1276  * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1277  */
1278 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1279 						     size_t size, loff_t *pos)
1280 {
1281 	struct amdgpu_device *adev = file_inode(f)->i_private;
1282 	ssize_t result = 0;
1283 	int r;
1284 
1285 	if (size & 0x3 || *pos & 0x3)
1286 		return -EINVAL;
1287 
1288 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1289 	if (r < 0) {
1290 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1291 		return r;
1292 	}
1293 
1294 	while (size) {
1295 		u32 value;
1296 
1297 		r = get_user(value, (uint32_t *)buf);
1298 		if (r)
1299 			goto out;
1300 
1301 		amdgpu_set_gfx_off_residency(adev, value ? true : false);
1302 
1303 		result += 4;
1304 		buf += 4;
1305 		*pos += 4;
1306 		size -= 4;
1307 	}
1308 
1309 	r = result;
1310 out:
1311 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1312 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1313 
1314 	return r;
1315 }
1316 
1317 
1318 /**
1319  * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1320  *
1321  * @f: open file handle
1322  * @buf: User buffer to store read data in
1323  * @size: Number of bytes to read
1324  * @pos:  Offset to seek to
1325  */
1326 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1327 						size_t size, loff_t *pos)
1328 {
1329 	struct amdgpu_device *adev = file_inode(f)->i_private;
1330 	ssize_t result = 0;
1331 	int r;
1332 
1333 	if (size & 0x3 || *pos & 0x3)
1334 		return -EINVAL;
1335 
1336 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1337 	if (r < 0) {
1338 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1339 		return r;
1340 	}
1341 
1342 	while (size) {
1343 		u64 value = 0;
1344 
1345 		r = amdgpu_get_gfx_off_entrycount(adev, &value);
1346 		if (r)
1347 			goto out;
1348 
1349 		r = put_user(value, (u64 *)buf);
1350 		if (r)
1351 			goto out;
1352 
1353 		result += 4;
1354 		buf += 4;
1355 		*pos += 4;
1356 		size -= 4;
1357 	}
1358 
1359 	r = result;
1360 out:
1361 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1362 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1363 
1364 	return r;
1365 }
1366 
1367 /**
1368  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1369  *
1370  * @f: open file handle
1371  * @buf: User buffer to write data from
1372  * @size: Number of bytes to write
1373  * @pos:  Offset to seek to
1374  *
1375  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1376  */
1377 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1378 					 size_t size, loff_t *pos)
1379 {
1380 	struct amdgpu_device *adev = file_inode(f)->i_private;
1381 	ssize_t result = 0;
1382 	int r;
1383 
1384 	if (size & 0x3 || *pos & 0x3)
1385 		return -EINVAL;
1386 
1387 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1388 	if (r < 0) {
1389 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1390 		return r;
1391 	}
1392 
1393 	while (size) {
1394 		uint32_t value;
1395 
1396 		r = get_user(value, (uint32_t *)buf);
1397 		if (r)
1398 			goto out;
1399 
1400 		amdgpu_gfx_off_ctrl(adev, value ? true : false);
1401 
1402 		result += 4;
1403 		buf += 4;
1404 		*pos += 4;
1405 		size -= 4;
1406 	}
1407 
1408 	r = result;
1409 out:
1410 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1411 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1412 
1413 	return r;
1414 }
1415 
1416 
1417 /**
1418  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1419  *
1420  * @f: open file handle
1421  * @buf: User buffer to store read data in
1422  * @size: Number of bytes to read
1423  * @pos:  Offset to seek to
1424  */
1425 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1426 					 size_t size, loff_t *pos)
1427 {
1428 	struct amdgpu_device *adev = file_inode(f)->i_private;
1429 	ssize_t result = 0;
1430 	int r;
1431 
1432 	if (size & 0x3 || *pos & 0x3)
1433 		return -EINVAL;
1434 
1435 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1436 	if (r < 0) {
1437 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1438 		return r;
1439 	}
1440 
1441 	while (size) {
1442 		u32 value = adev->gfx.gfx_off_state;
1443 
1444 		r = put_user(value, (u32 *)buf);
1445 		if (r)
1446 			goto out;
1447 
1448 		result += 4;
1449 		buf += 4;
1450 		*pos += 4;
1451 		size -= 4;
1452 	}
1453 
1454 	r = result;
1455 out:
1456 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1457 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1458 
1459 	return r;
1460 }
1461 
1462 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1463 						 size_t size, loff_t *pos)
1464 {
1465 	struct amdgpu_device *adev = file_inode(f)->i_private;
1466 	ssize_t result = 0;
1467 	int r;
1468 
1469 	if (size & 0x3 || *pos & 0x3)
1470 		return -EINVAL;
1471 
1472 	r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1473 	if (r < 0) {
1474 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1475 		return r;
1476 	}
1477 
1478 	while (size) {
1479 		u32 value;
1480 
1481 		r = amdgpu_get_gfx_off_status(adev, &value);
1482 		if (r)
1483 			goto out;
1484 
1485 		r = put_user(value, (u32 *)buf);
1486 		if (r)
1487 			goto out;
1488 
1489 		result += 4;
1490 		buf += 4;
1491 		*pos += 4;
1492 		size -= 4;
1493 	}
1494 
1495 	r = result;
1496 out:
1497 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1498 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1499 
1500 	return r;
1501 }
1502 
1503 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1504 	.owner = THIS_MODULE,
1505 	.unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1506 	.read = amdgpu_debugfs_regs2_read,
1507 	.write = amdgpu_debugfs_regs2_write,
1508 	.open = amdgpu_debugfs_regs2_open,
1509 	.release = amdgpu_debugfs_regs2_release,
1510 	.llseek = default_llseek
1511 };
1512 
1513 static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1514 	.owner = THIS_MODULE,
1515 	.unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1516 	.read = amdgpu_debugfs_gprwave_read,
1517 	.open = amdgpu_debugfs_gprwave_open,
1518 	.release = amdgpu_debugfs_gprwave_release,
1519 	.llseek = default_llseek
1520 };
1521 
1522 static const struct file_operations amdgpu_debugfs_regs_fops = {
1523 	.owner = THIS_MODULE,
1524 	.read = amdgpu_debugfs_regs_read,
1525 	.write = amdgpu_debugfs_regs_write,
1526 	.llseek = default_llseek
1527 };
1528 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1529 	.owner = THIS_MODULE,
1530 	.read = amdgpu_debugfs_regs_didt_read,
1531 	.write = amdgpu_debugfs_regs_didt_write,
1532 	.llseek = default_llseek
1533 };
1534 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1535 	.owner = THIS_MODULE,
1536 	.read = amdgpu_debugfs_regs_pcie_read,
1537 	.write = amdgpu_debugfs_regs_pcie_write,
1538 	.llseek = default_llseek
1539 };
1540 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1541 	.owner = THIS_MODULE,
1542 	.read = amdgpu_debugfs_regs_smc_read,
1543 	.write = amdgpu_debugfs_regs_smc_write,
1544 	.llseek = default_llseek
1545 };
1546 
1547 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1548 	.owner = THIS_MODULE,
1549 	.read = amdgpu_debugfs_gca_config_read,
1550 	.llseek = default_llseek
1551 };
1552 
1553 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1554 	.owner = THIS_MODULE,
1555 	.read = amdgpu_debugfs_sensor_read,
1556 	.llseek = default_llseek
1557 };
1558 
1559 static const struct file_operations amdgpu_debugfs_wave_fops = {
1560 	.owner = THIS_MODULE,
1561 	.read = amdgpu_debugfs_wave_read,
1562 	.llseek = default_llseek
1563 };
1564 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1565 	.owner = THIS_MODULE,
1566 	.read = amdgpu_debugfs_gpr_read,
1567 	.llseek = default_llseek
1568 };
1569 
1570 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1571 	.owner = THIS_MODULE,
1572 	.read = amdgpu_debugfs_gfxoff_read,
1573 	.write = amdgpu_debugfs_gfxoff_write,
1574 	.llseek = default_llseek
1575 };
1576 
1577 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1578 	.owner = THIS_MODULE,
1579 	.read = amdgpu_debugfs_gfxoff_status_read,
1580 	.llseek = default_llseek
1581 };
1582 
1583 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1584 	.owner = THIS_MODULE,
1585 	.read = amdgpu_debugfs_gfxoff_count_read,
1586 	.llseek = default_llseek
1587 };
1588 
1589 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1590 	.owner = THIS_MODULE,
1591 	.read = amdgpu_debugfs_gfxoff_residency_read,
1592 	.write = amdgpu_debugfs_gfxoff_residency_write,
1593 	.llseek = default_llseek
1594 };
1595 
1596 static const struct file_operations *debugfs_regs[] = {
1597 	&amdgpu_debugfs_regs_fops,
1598 	&amdgpu_debugfs_regs2_fops,
1599 	&amdgpu_debugfs_gprwave_fops,
1600 	&amdgpu_debugfs_regs_didt_fops,
1601 	&amdgpu_debugfs_regs_pcie_fops,
1602 	&amdgpu_debugfs_regs_smc_fops,
1603 	&amdgpu_debugfs_gca_config_fops,
1604 	&amdgpu_debugfs_sensors_fops,
1605 	&amdgpu_debugfs_wave_fops,
1606 	&amdgpu_debugfs_gpr_fops,
1607 	&amdgpu_debugfs_gfxoff_fops,
1608 	&amdgpu_debugfs_gfxoff_status_fops,
1609 	&amdgpu_debugfs_gfxoff_count_fops,
1610 	&amdgpu_debugfs_gfxoff_residency_fops,
1611 };
1612 
1613 static const char * const debugfs_regs_names[] = {
1614 	"amdgpu_regs",
1615 	"amdgpu_regs2",
1616 	"amdgpu_gprwave",
1617 	"amdgpu_regs_didt",
1618 	"amdgpu_regs_pcie",
1619 	"amdgpu_regs_smc",
1620 	"amdgpu_gca_config",
1621 	"amdgpu_sensors",
1622 	"amdgpu_wave",
1623 	"amdgpu_gpr",
1624 	"amdgpu_gfxoff",
1625 	"amdgpu_gfxoff_status",
1626 	"amdgpu_gfxoff_count",
1627 	"amdgpu_gfxoff_residency",
1628 };
1629 
1630 /**
1631  * amdgpu_debugfs_regs_init -	Initialize debugfs entries that provide
1632  *				register access.
1633  *
1634  * @adev: The device to attach the debugfs entries to
1635  */
1636 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1637 {
1638 	struct drm_minor *minor = adev_to_drm(adev)->primary;
1639 	struct dentry *ent, *root = minor->debugfs_root;
1640 	unsigned int i;
1641 
1642 	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1643 		ent = debugfs_create_file(debugfs_regs_names[i],
1644 					  S_IFREG | 0444, root,
1645 					  adev, debugfs_regs[i]);
1646 		if (!i && !IS_ERR_OR_NULL(ent))
1647 			i_size_write(ent->d_inode, adev->rmmio_size);
1648 	}
1649 
1650 	return 0;
1651 }
1652 
1653 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1654 {
1655 	struct amdgpu_device *adev = m->private;
1656 	struct drm_device *dev = adev_to_drm(adev);
1657 	int r = 0, i;
1658 
1659 	r = pm_runtime_get_sync(dev->dev);
1660 	if (r < 0) {
1661 		pm_runtime_put_autosuspend(dev->dev);
1662 		return r;
1663 	}
1664 
1665 	/* Avoid accidently unparking the sched thread during GPU reset */
1666 	r = down_write_killable(&adev->reset_domain->sem);
1667 	if (r)
1668 		return r;
1669 
1670 	/* hold on the scheduler */
1671 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1672 		struct amdgpu_ring *ring = adev->rings[i];
1673 
1674 		if (!ring || !ring->sched.thread)
1675 			continue;
1676 		kthread_park(ring->sched.thread);
1677 	}
1678 
1679 	seq_puts(m, "run ib test:\n");
1680 	r = amdgpu_ib_ring_tests(adev);
1681 	if (r)
1682 		seq_printf(m, "ib ring tests failed (%d).\n", r);
1683 	else
1684 		seq_puts(m, "ib ring tests passed.\n");
1685 
1686 	/* go on the scheduler */
1687 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1688 		struct amdgpu_ring *ring = adev->rings[i];
1689 
1690 		if (!ring || !ring->sched.thread)
1691 			continue;
1692 		kthread_unpark(ring->sched.thread);
1693 	}
1694 
1695 	up_write(&adev->reset_domain->sem);
1696 
1697 	pm_runtime_mark_last_busy(dev->dev);
1698 	pm_runtime_put_autosuspend(dev->dev);
1699 
1700 	return 0;
1701 }
1702 
1703 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1704 {
1705 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1706 	struct drm_device *dev = adev_to_drm(adev);
1707 	int r;
1708 
1709 	r = pm_runtime_get_sync(dev->dev);
1710 	if (r < 0) {
1711 		pm_runtime_put_autosuspend(dev->dev);
1712 		return r;
1713 	}
1714 
1715 	*val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1716 
1717 	pm_runtime_mark_last_busy(dev->dev);
1718 	pm_runtime_put_autosuspend(dev->dev);
1719 
1720 	return 0;
1721 }
1722 
1723 
1724 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1725 {
1726 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1727 	struct drm_device *dev = adev_to_drm(adev);
1728 	int r;
1729 
1730 	r = pm_runtime_get_sync(dev->dev);
1731 	if (r < 0) {
1732 		pm_runtime_put_autosuspend(dev->dev);
1733 		return r;
1734 	}
1735 
1736 	*val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1737 
1738 	pm_runtime_mark_last_busy(dev->dev);
1739 	pm_runtime_put_autosuspend(dev->dev);
1740 
1741 	return 0;
1742 }
1743 
1744 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1745 {
1746 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1747 	struct drm_device *dev = adev_to_drm(adev);
1748 	int r;
1749 
1750 	r = pm_runtime_get_sync(dev->dev);
1751 	if (r < 0) {
1752 		pm_runtime_put_autosuspend(dev->dev);
1753 		return r;
1754 	}
1755 
1756 	r = amdgpu_benchmark(adev, val);
1757 
1758 	pm_runtime_mark_last_busy(dev->dev);
1759 	pm_runtime_put_autosuspend(dev->dev);
1760 
1761 	return r;
1762 }
1763 
1764 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1765 {
1766 	struct amdgpu_device *adev = m->private;
1767 	struct drm_device *dev = adev_to_drm(adev);
1768 	struct drm_file *file;
1769 	int r;
1770 
1771 	r = mutex_lock_interruptible(&dev->filelist_mutex);
1772 	if (r)
1773 		return r;
1774 
1775 	list_for_each_entry(file, &dev->filelist, lhead) {
1776 		struct amdgpu_fpriv *fpriv = file->driver_priv;
1777 		struct amdgpu_vm *vm = &fpriv->vm;
1778 
1779 		seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1780 				vm->task_info.pid, vm->task_info.process_name);
1781 		r = amdgpu_bo_reserve(vm->root.bo, true);
1782 		if (r)
1783 			break;
1784 		amdgpu_debugfs_vm_bo_info(vm, m);
1785 		amdgpu_bo_unreserve(vm->root.bo);
1786 	}
1787 
1788 	mutex_unlock(&dev->filelist_mutex);
1789 
1790 	return r;
1791 }
1792 
1793 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1794 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1795 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1796 			 NULL, "%lld\n");
1797 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1798 			 NULL, "%lld\n");
1799 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1800 			 "%lld\n");
1801 
1802 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1803 					  struct dma_fence **fences)
1804 {
1805 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
1806 	uint32_t sync_seq, last_seq;
1807 
1808 	last_seq = atomic_read(&ring->fence_drv.last_seq);
1809 	sync_seq = ring->fence_drv.sync_seq;
1810 
1811 	last_seq &= drv->num_fences_mask;
1812 	sync_seq &= drv->num_fences_mask;
1813 
1814 	do {
1815 		struct dma_fence *fence, **ptr;
1816 
1817 		++last_seq;
1818 		last_seq &= drv->num_fences_mask;
1819 		ptr = &drv->fences[last_seq];
1820 
1821 		fence = rcu_dereference_protected(*ptr, 1);
1822 		RCU_INIT_POINTER(*ptr, NULL);
1823 
1824 		if (!fence)
1825 			continue;
1826 
1827 		fences[last_seq] = fence;
1828 
1829 	} while (last_seq != sync_seq);
1830 }
1831 
1832 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1833 					    int length)
1834 {
1835 	int i;
1836 	struct dma_fence *fence;
1837 
1838 	for (i = 0; i < length; i++) {
1839 		fence = fences[i];
1840 		if (!fence)
1841 			continue;
1842 		dma_fence_signal(fence);
1843 		dma_fence_put(fence);
1844 	}
1845 }
1846 
1847 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1848 {
1849 	struct drm_sched_job *s_job;
1850 	struct dma_fence *fence;
1851 
1852 	spin_lock(&sched->job_list_lock);
1853 	list_for_each_entry(s_job, &sched->pending_list, list) {
1854 		fence = sched->ops->run_job(s_job);
1855 		dma_fence_put(fence);
1856 	}
1857 	spin_unlock(&sched->job_list_lock);
1858 }
1859 
1860 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1861 {
1862 	struct amdgpu_job *job;
1863 	struct drm_sched_job *s_job, *tmp;
1864 	uint32_t preempt_seq;
1865 	struct dma_fence *fence, **ptr;
1866 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
1867 	struct drm_gpu_scheduler *sched = &ring->sched;
1868 	bool preempted = true;
1869 
1870 	if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1871 		return;
1872 
1873 	preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1874 	if (preempt_seq <= atomic_read(&drv->last_seq)) {
1875 		preempted = false;
1876 		goto no_preempt;
1877 	}
1878 
1879 	preempt_seq &= drv->num_fences_mask;
1880 	ptr = &drv->fences[preempt_seq];
1881 	fence = rcu_dereference_protected(*ptr, 1);
1882 
1883 no_preempt:
1884 	spin_lock(&sched->job_list_lock);
1885 	list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1886 		if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1887 			/* remove job from ring_mirror_list */
1888 			list_del_init(&s_job->list);
1889 			sched->ops->free_job(s_job);
1890 			continue;
1891 		}
1892 		job = to_amdgpu_job(s_job);
1893 		if (preempted && (&job->hw_fence) == fence)
1894 			/* mark the job as preempted */
1895 			job->preemption_status |= AMDGPU_IB_PREEMPTED;
1896 	}
1897 	spin_unlock(&sched->job_list_lock);
1898 }
1899 
1900 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1901 {
1902 	int r, length;
1903 	struct amdgpu_ring *ring;
1904 	struct dma_fence **fences = NULL;
1905 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1906 
1907 	if (val >= AMDGPU_MAX_RINGS)
1908 		return -EINVAL;
1909 
1910 	ring = adev->rings[val];
1911 
1912 	if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1913 		return -EINVAL;
1914 
1915 	/* the last preemption failed */
1916 	if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1917 		return -EBUSY;
1918 
1919 	length = ring->fence_drv.num_fences_mask + 1;
1920 	fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1921 	if (!fences)
1922 		return -ENOMEM;
1923 
1924 	/* Avoid accidently unparking the sched thread during GPU reset */
1925 	r = down_read_killable(&adev->reset_domain->sem);
1926 	if (r)
1927 		goto pro_end;
1928 
1929 	/* stop the scheduler */
1930 	kthread_park(ring->sched.thread);
1931 
1932 	/* preempt the IB */
1933 	r = amdgpu_ring_preempt_ib(ring);
1934 	if (r) {
1935 		DRM_WARN("failed to preempt ring %d\n", ring->idx);
1936 		goto failure;
1937 	}
1938 
1939 	amdgpu_fence_process(ring);
1940 
1941 	if (atomic_read(&ring->fence_drv.last_seq) !=
1942 	    ring->fence_drv.sync_seq) {
1943 		DRM_INFO("ring %d was preempted\n", ring->idx);
1944 
1945 		amdgpu_ib_preempt_mark_partial_job(ring);
1946 
1947 		/* swap out the old fences */
1948 		amdgpu_ib_preempt_fences_swap(ring, fences);
1949 
1950 		amdgpu_fence_driver_force_completion(ring);
1951 
1952 		/* resubmit unfinished jobs */
1953 		amdgpu_ib_preempt_job_recovery(&ring->sched);
1954 
1955 		/* wait for jobs finished */
1956 		amdgpu_fence_wait_empty(ring);
1957 
1958 		/* signal the old fences */
1959 		amdgpu_ib_preempt_signal_fences(fences, length);
1960 	}
1961 
1962 failure:
1963 	/* restart the scheduler */
1964 	kthread_unpark(ring->sched.thread);
1965 
1966 	up_read(&adev->reset_domain->sem);
1967 
1968 pro_end:
1969 	kfree(fences);
1970 
1971 	return r;
1972 }
1973 
1974 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1975 {
1976 	int ret = 0;
1977 	uint32_t max_freq, min_freq;
1978 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
1979 
1980 	if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1981 		return -EINVAL;
1982 
1983 	ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1984 	if (ret < 0) {
1985 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1986 		return ret;
1987 	}
1988 
1989 	ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
1990 	if (ret == -EOPNOTSUPP) {
1991 		ret = 0;
1992 		goto out;
1993 	}
1994 	if (ret || val > max_freq || val < min_freq) {
1995 		ret = -EINVAL;
1996 		goto out;
1997 	}
1998 
1999 	ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
2000 	if (ret)
2001 		ret = -EINVAL;
2002 
2003 out:
2004 	pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
2005 	pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2006 
2007 	return ret;
2008 }
2009 
2010 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
2011 			amdgpu_debugfs_ib_preempt, "%llu\n");
2012 
2013 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2014 			amdgpu_debugfs_sclk_set, "%llu\n");
2015 
2016 static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
2017 				char __user *buf, size_t size, loff_t *pos)
2018 {
2019 	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2020 	char reg_offset[12];
2021 	int i, ret, len = 0;
2022 
2023 	if (*pos)
2024 		return 0;
2025 
2026 	memset(reg_offset, 0, 12);
2027 	ret = down_read_killable(&adev->reset_domain->sem);
2028 	if (ret)
2029 		return ret;
2030 
2031 	for (i = 0; i < adev->num_regs; i++) {
2032 		sprintf(reg_offset, "0x%x\n", adev->reset_dump_reg_list[i]);
2033 		up_read(&adev->reset_domain->sem);
2034 		if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
2035 			return -EFAULT;
2036 
2037 		len += strlen(reg_offset);
2038 		ret = down_read_killable(&adev->reset_domain->sem);
2039 		if (ret)
2040 			return ret;
2041 	}
2042 
2043 	up_read(&adev->reset_domain->sem);
2044 	*pos += len;
2045 
2046 	return len;
2047 }
2048 
2049 static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
2050 			const char __user *buf, size_t size, loff_t *pos)
2051 {
2052 	struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2053 	char reg_offset[11];
2054 	uint32_t *new = NULL, *tmp = NULL;
2055 	int ret, i = 0, len = 0;
2056 
2057 	do {
2058 		memset(reg_offset, 0, 11);
2059 		if (copy_from_user(reg_offset, buf + len,
2060 					min(10, ((int)size-len)))) {
2061 			ret = -EFAULT;
2062 			goto error_free;
2063 		}
2064 
2065 		new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
2066 		if (!new) {
2067 			ret = -ENOMEM;
2068 			goto error_free;
2069 		}
2070 		tmp = new;
2071 		if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
2072 			ret = -EINVAL;
2073 			goto error_free;
2074 		}
2075 
2076 		len += ret;
2077 		i++;
2078 	} while (len < size);
2079 
2080 	new = kmalloc_array(i, sizeof(uint32_t), GFP_KERNEL);
2081 	if (!new) {
2082 		ret = -ENOMEM;
2083 		goto error_free;
2084 	}
2085 	ret = down_write_killable(&adev->reset_domain->sem);
2086 	if (ret)
2087 		goto error_free;
2088 
2089 	swap(adev->reset_dump_reg_list, tmp);
2090 	swap(adev->reset_dump_reg_value, new);
2091 	adev->num_regs = i;
2092 	up_write(&adev->reset_domain->sem);
2093 	ret = size;
2094 
2095 error_free:
2096 	if (tmp != new)
2097 		kfree(tmp);
2098 	kfree(new);
2099 	return ret;
2100 }
2101 
2102 static const struct file_operations amdgpu_reset_dump_register_list = {
2103 	.owner = THIS_MODULE,
2104 	.read = amdgpu_reset_dump_register_list_read,
2105 	.write = amdgpu_reset_dump_register_list_write,
2106 	.llseek = default_llseek
2107 };
2108 
2109 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2110 {
2111 	struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2112 	struct dentry *ent;
2113 	int r, i;
2114 
2115 	if (!debugfs_initialized())
2116 		return 0;
2117 
2118 	debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2119 			   &adev->pm.smu_debug_mask);
2120 
2121 	ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2122 				  &fops_ib_preempt);
2123 	if (IS_ERR(ent)) {
2124 		DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
2125 		return PTR_ERR(ent);
2126 	}
2127 
2128 	ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2129 				  &fops_sclk_set);
2130 	if (IS_ERR(ent)) {
2131 		DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
2132 		return PTR_ERR(ent);
2133 	}
2134 
2135 	/* Register debugfs entries for amdgpu_ttm */
2136 	amdgpu_ttm_debugfs_init(adev);
2137 	amdgpu_debugfs_pm_init(adev);
2138 	amdgpu_debugfs_sa_init(adev);
2139 	amdgpu_debugfs_fence_init(adev);
2140 	amdgpu_debugfs_gem_init(adev);
2141 
2142 	r = amdgpu_debugfs_regs_init(adev);
2143 	if (r)
2144 		DRM_ERROR("registering register debugfs failed (%d).\n", r);
2145 
2146 	amdgpu_debugfs_firmware_init(adev);
2147 	amdgpu_ta_if_debugfs_init(adev);
2148 
2149 #if defined(CONFIG_DRM_AMD_DC)
2150 	if (adev->dc_enabled)
2151 		dtn_debugfs_init(adev);
2152 #endif
2153 
2154 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2155 		struct amdgpu_ring *ring = adev->rings[i];
2156 
2157 		if (!ring)
2158 			continue;
2159 
2160 		amdgpu_debugfs_ring_init(adev, ring);
2161 	}
2162 
2163 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2164 		if (!amdgpu_vcnfw_log)
2165 			break;
2166 
2167 		if (adev->vcn.harvest_config & (1 << i))
2168 			continue;
2169 
2170 		amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2171 	}
2172 
2173 	amdgpu_ras_debugfs_create_all(adev);
2174 	amdgpu_rap_debugfs_init(adev);
2175 	amdgpu_securedisplay_debugfs_init(adev);
2176 	amdgpu_fw_attestation_debugfs_init(adev);
2177 
2178 	debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
2179 			    &amdgpu_evict_vram_fops);
2180 	debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
2181 			    &amdgpu_evict_gtt_fops);
2182 	debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2183 			    &amdgpu_debugfs_test_ib_fops);
2184 	debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2185 			    &amdgpu_debugfs_vm_info_fops);
2186 	debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2187 			    &amdgpu_benchmark_fops);
2188 	debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
2189 			    &amdgpu_reset_dump_register_list);
2190 
2191 	adev->debugfs_vbios_blob.data = adev->bios;
2192 	adev->debugfs_vbios_blob.size = adev->bios_size;
2193 	debugfs_create_blob("amdgpu_vbios", 0444, root,
2194 			    &adev->debugfs_vbios_blob);
2195 
2196 	adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2197 	adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2198 	debugfs_create_blob("amdgpu_discovery", 0444, root,
2199 			    &adev->debugfs_discovery_blob);
2200 
2201 	return 0;
2202 }
2203 
2204 #else
2205 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2206 {
2207 	return 0;
2208 }
2209 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2210 {
2211 	return 0;
2212 }
2213 #endif
2214