1 /*
2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
23 *
24 */
25
26 #include <linux/kthread.h>
27 #include <linux/pci.h>
28 #include <linux/uaccess.h>
29 #include <linux/pm_runtime.h>
30
31 #include "amdgpu.h"
32 #include "amdgpu_pm.h"
33 #include "amdgpu_dm_debugfs.h"
34 #include "amdgpu_ras.h"
35 #include "amdgpu_rap.h"
36 #include "amdgpu_securedisplay.h"
37 #include "amdgpu_fw_attestation.h"
38 #include "amdgpu_umr.h"
39
40 #include "amdgpu_reset.h"
41 #include "amdgpu_psp_ta.h"
42
43 #if defined(CONFIG_DEBUG_FS)
44
45 /**
46 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
47 *
48 * @read: True if reading
49 * @f: open file handle
50 * @buf: User buffer to write/read to
51 * @size: Number of bytes to write/read
52 * @pos: Offset to seek to
53 *
54 * This debugfs entry has special meaning on the offset being sought.
55 * Various bits have different meanings:
56 *
57 * Bit 62: Indicates a GRBM bank switch is needed
58 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
59 * zero)
60 * Bits 24..33: The SE or ME selector if needed
61 * Bits 34..43: The SH (or SA) or PIPE selector if needed
62 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
63 *
64 * Bit 23: Indicates that the PM power gating lock should be held
65 * This is necessary to read registers that might be
66 * unreliable during a power gating transistion.
67 *
68 * The lower bits are the BYTE offset of the register to read. This
69 * allows reading multiple registers in a single call and having
70 * the returned size reflect that.
71 */
amdgpu_debugfs_process_reg_op(bool read,struct file * f,char __user * buf,size_t size,loff_t * pos)72 static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
73 char __user *buf, size_t size, loff_t *pos)
74 {
75 struct amdgpu_device *adev = file_inode(f)->i_private;
76 ssize_t result = 0;
77 int r;
78 bool pm_pg_lock, use_bank, use_ring;
79 unsigned int instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
80
81 pm_pg_lock = use_bank = use_ring = false;
82 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
83
84 if (size & 0x3 || *pos & 0x3 ||
85 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
86 return -EINVAL;
87
88 /* are we reading registers for which a PG lock is necessary? */
89 pm_pg_lock = (*pos >> 23) & 1;
90
91 if (*pos & (1ULL << 62)) {
92 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
93 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
94 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
95
96 if (se_bank == 0x3FF)
97 se_bank = 0xFFFFFFFF;
98 if (sh_bank == 0x3FF)
99 sh_bank = 0xFFFFFFFF;
100 if (instance_bank == 0x3FF)
101 instance_bank = 0xFFFFFFFF;
102 use_bank = true;
103 } else if (*pos & (1ULL << 61)) {
104
105 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
106 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
107 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
108 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
109
110 use_ring = true;
111 } else {
112 use_bank = use_ring = false;
113 }
114
115 *pos &= (1UL << 22) - 1;
116
117 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
118 if (r < 0) {
119 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
120 return r;
121 }
122
123 r = amdgpu_virt_enable_access_debugfs(adev);
124 if (r < 0) {
125 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
126 return r;
127 }
128
129 if (use_bank) {
130 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
131 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
132 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
133 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
134 amdgpu_virt_disable_access_debugfs(adev);
135 return -EINVAL;
136 }
137 mutex_lock(&adev->grbm_idx_mutex);
138 amdgpu_gfx_select_se_sh(adev, se_bank,
139 sh_bank, instance_bank, 0);
140 } else if (use_ring) {
141 mutex_lock(&adev->srbm_mutex);
142 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid, 0);
143 }
144
145 if (pm_pg_lock)
146 mutex_lock(&adev->pm.mutex);
147
148 while (size) {
149 uint32_t value;
150
151 if (read) {
152 value = RREG32(*pos >> 2);
153 r = put_user(value, (uint32_t *)buf);
154 } else {
155 r = get_user(value, (uint32_t *)buf);
156 if (!r)
157 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
158 }
159 if (r) {
160 result = r;
161 goto end;
162 }
163
164 result += 4;
165 buf += 4;
166 *pos += 4;
167 size -= 4;
168 }
169
170 end:
171 if (use_bank) {
172 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
173 mutex_unlock(&adev->grbm_idx_mutex);
174 } else if (use_ring) {
175 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, 0);
176 mutex_unlock(&adev->srbm_mutex);
177 }
178
179 if (pm_pg_lock)
180 mutex_unlock(&adev->pm.mutex);
181
182 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
183 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
184
185 amdgpu_virt_disable_access_debugfs(adev);
186 return result;
187 }
188
189 /*
190 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
191 */
amdgpu_debugfs_regs_read(struct file * f,char __user * buf,size_t size,loff_t * pos)192 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
193 size_t size, loff_t *pos)
194 {
195 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
196 }
197
198 /*
199 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
200 */
amdgpu_debugfs_regs_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)201 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
202 size_t size, loff_t *pos)
203 {
204 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
205 }
206
amdgpu_debugfs_regs2_open(struct inode * inode,struct file * file)207 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
208 {
209 struct amdgpu_debugfs_regs2_data *rd;
210
211 rd = kzalloc(sizeof(*rd), GFP_KERNEL);
212 if (!rd)
213 return -ENOMEM;
214 rd->adev = file_inode(file)->i_private;
215 file->private_data = rd;
216 mutex_init(&rd->lock);
217
218 return 0;
219 }
220
amdgpu_debugfs_regs2_release(struct inode * inode,struct file * file)221 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
222 {
223 struct amdgpu_debugfs_regs2_data *rd = file->private_data;
224
225 mutex_destroy(&rd->lock);
226 kfree(file->private_data);
227 return 0;
228 }
229
amdgpu_debugfs_regs2_op(struct file * f,char __user * buf,u32 offset,size_t size,int write_en)230 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
231 {
232 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
233 struct amdgpu_device *adev = rd->adev;
234 ssize_t result = 0;
235 int r;
236 uint32_t value;
237
238 if (size & 0x3 || offset & 0x3)
239 return -EINVAL;
240
241 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
242 if (r < 0) {
243 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
244 return r;
245 }
246
247 r = amdgpu_virt_enable_access_debugfs(adev);
248 if (r < 0) {
249 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
250 return r;
251 }
252
253 mutex_lock(&rd->lock);
254
255 if (rd->id.use_grbm) {
256 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
257 (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
258 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
259 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
260 amdgpu_virt_disable_access_debugfs(adev);
261 mutex_unlock(&rd->lock);
262 return -EINVAL;
263 }
264 mutex_lock(&adev->grbm_idx_mutex);
265 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
266 rd->id.grbm.sh,
267 rd->id.grbm.instance, rd->id.xcc_id);
268 }
269
270 if (rd->id.use_srbm) {
271 mutex_lock(&adev->srbm_mutex);
272 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
273 rd->id.srbm.queue, rd->id.srbm.vmid, rd->id.xcc_id);
274 }
275
276 if (rd->id.pg_lock)
277 mutex_lock(&adev->pm.mutex);
278
279 while (size) {
280 if (!write_en) {
281 value = RREG32(offset >> 2);
282 r = put_user(value, (uint32_t *)buf);
283 } else {
284 r = get_user(value, (uint32_t *)buf);
285 if (!r)
286 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value, rd->id.xcc_id);
287 }
288 if (r) {
289 result = r;
290 goto end;
291 }
292 offset += 4;
293 size -= 4;
294 result += 4;
295 buf += 4;
296 }
297 end:
298 if (rd->id.use_grbm) {
299 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, rd->id.xcc_id);
300 mutex_unlock(&adev->grbm_idx_mutex);
301 }
302
303 if (rd->id.use_srbm) {
304 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0, rd->id.xcc_id);
305 mutex_unlock(&adev->srbm_mutex);
306 }
307
308 if (rd->id.pg_lock)
309 mutex_unlock(&adev->pm.mutex);
310
311 mutex_unlock(&rd->lock);
312
313 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
314 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
315
316 amdgpu_virt_disable_access_debugfs(adev);
317 return result;
318 }
319
amdgpu_debugfs_regs2_ioctl(struct file * f,unsigned int cmd,unsigned long data)320 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
321 {
322 struct amdgpu_debugfs_regs2_data *rd = f->private_data;
323 struct amdgpu_debugfs_regs2_iocdata v1_data;
324 int r;
325
326 mutex_lock(&rd->lock);
327
328 switch (cmd) {
329 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE_V2:
330 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata_v2 *)data,
331 sizeof(rd->id));
332 if (r)
333 r = -EINVAL;
334 goto done;
335 case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
336 r = copy_from_user(&v1_data, (struct amdgpu_debugfs_regs2_iocdata *)data,
337 sizeof(v1_data));
338 if (r) {
339 r = -EINVAL;
340 goto done;
341 }
342 goto v1_copy;
343 default:
344 r = -EINVAL;
345 goto done;
346 }
347
348 v1_copy:
349 rd->id.use_srbm = v1_data.use_srbm;
350 rd->id.use_grbm = v1_data.use_grbm;
351 rd->id.pg_lock = v1_data.pg_lock;
352 rd->id.grbm.se = v1_data.grbm.se;
353 rd->id.grbm.sh = v1_data.grbm.sh;
354 rd->id.grbm.instance = v1_data.grbm.instance;
355 rd->id.srbm.me = v1_data.srbm.me;
356 rd->id.srbm.pipe = v1_data.srbm.pipe;
357 rd->id.srbm.queue = v1_data.srbm.queue;
358 rd->id.xcc_id = 0;
359 done:
360 mutex_unlock(&rd->lock);
361 return r;
362 }
363
amdgpu_debugfs_regs2_read(struct file * f,char __user * buf,size_t size,loff_t * pos)364 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
365 {
366 return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
367 }
368
amdgpu_debugfs_regs2_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)369 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
370 {
371 return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
372 }
373
amdgpu_debugfs_gprwave_open(struct inode * inode,struct file * file)374 static int amdgpu_debugfs_gprwave_open(struct inode *inode, struct file *file)
375 {
376 struct amdgpu_debugfs_gprwave_data *rd;
377
378 rd = kzalloc(sizeof(*rd), GFP_KERNEL);
379 if (!rd)
380 return -ENOMEM;
381 rd->adev = file_inode(file)->i_private;
382 file->private_data = rd;
383 mutex_init(&rd->lock);
384
385 return 0;
386 }
387
amdgpu_debugfs_gprwave_release(struct inode * inode,struct file * file)388 static int amdgpu_debugfs_gprwave_release(struct inode *inode, struct file *file)
389 {
390 struct amdgpu_debugfs_gprwave_data *rd = file->private_data;
391
392 mutex_destroy(&rd->lock);
393 kfree(file->private_data);
394 return 0;
395 }
396
amdgpu_debugfs_gprwave_read(struct file * f,char __user * buf,size_t size,loff_t * pos)397 static ssize_t amdgpu_debugfs_gprwave_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
398 {
399 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
400 struct amdgpu_device *adev = rd->adev;
401 ssize_t result = 0;
402 int r;
403 uint32_t *data, x;
404
405 if (size & 0x3 || *pos & 0x3)
406 return -EINVAL;
407
408 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
409 if (r < 0) {
410 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
411 return r;
412 }
413
414 r = amdgpu_virt_enable_access_debugfs(adev);
415 if (r < 0) {
416 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
417 return r;
418 }
419
420 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
421 if (!data) {
422 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
423 amdgpu_virt_disable_access_debugfs(adev);
424 return -ENOMEM;
425 }
426
427 /* switch to the specific se/sh/cu */
428 mutex_lock(&adev->grbm_idx_mutex);
429 amdgpu_gfx_select_se_sh(adev, rd->id.se, rd->id.sh, rd->id.cu, rd->id.xcc_id);
430
431 if (!rd->id.gpr_or_wave) {
432 x = 0;
433 if (adev->gfx.funcs->read_wave_data)
434 adev->gfx.funcs->read_wave_data(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, data, &x);
435 } else {
436 x = size >> 2;
437 if (rd->id.gpr.vpgr_or_sgpr) {
438 if (adev->gfx.funcs->read_wave_vgprs)
439 adev->gfx.funcs->read_wave_vgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, rd->id.gpr.thread, *pos, size>>2, data);
440 } else {
441 if (adev->gfx.funcs->read_wave_sgprs)
442 adev->gfx.funcs->read_wave_sgprs(adev, rd->id.xcc_id, rd->id.simd, rd->id.wave, *pos, size>>2, data);
443 }
444 }
445
446 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, rd->id.xcc_id);
447 mutex_unlock(&adev->grbm_idx_mutex);
448
449 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
450 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
451
452 if (!x) {
453 result = -EINVAL;
454 goto done;
455 }
456
457 while (size && (*pos < x * 4)) {
458 uint32_t value;
459
460 value = data[*pos >> 2];
461 r = put_user(value, (uint32_t *)buf);
462 if (r) {
463 result = r;
464 goto done;
465 }
466
467 result += 4;
468 buf += 4;
469 *pos += 4;
470 size -= 4;
471 }
472
473 done:
474 amdgpu_virt_disable_access_debugfs(adev);
475 kfree(data);
476 return result;
477 }
478
amdgpu_debugfs_gprwave_ioctl(struct file * f,unsigned int cmd,unsigned long data)479 static long amdgpu_debugfs_gprwave_ioctl(struct file *f, unsigned int cmd, unsigned long data)
480 {
481 struct amdgpu_debugfs_gprwave_data *rd = f->private_data;
482 int r = 0;
483
484 mutex_lock(&rd->lock);
485
486 switch (cmd) {
487 case AMDGPU_DEBUGFS_GPRWAVE_IOC_SET_STATE:
488 if (copy_from_user(&rd->id,
489 (struct amdgpu_debugfs_gprwave_iocdata *)data,
490 sizeof(rd->id)))
491 r = -EFAULT;
492 goto done;
493 default:
494 r = -EINVAL;
495 goto done;
496 }
497
498 done:
499 mutex_unlock(&rd->lock);
500 return r;
501 }
502
503
504
505
506 /**
507 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
508 *
509 * @f: open file handle
510 * @buf: User buffer to store read data in
511 * @size: Number of bytes to read
512 * @pos: Offset to seek to
513 *
514 * The lower bits are the BYTE offset of the register to read. This
515 * allows reading multiple registers in a single call and having
516 * the returned size reflect that.
517 */
amdgpu_debugfs_regs_pcie_read(struct file * f,char __user * buf,size_t size,loff_t * pos)518 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
519 size_t size, loff_t *pos)
520 {
521 struct amdgpu_device *adev = file_inode(f)->i_private;
522 ssize_t result = 0;
523 int r;
524
525 if (size & 0x3 || *pos & 0x3)
526 return -EINVAL;
527
528 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
529 if (r < 0) {
530 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
531 return r;
532 }
533
534 r = amdgpu_virt_enable_access_debugfs(adev);
535 if (r < 0) {
536 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
537 return r;
538 }
539
540 while (size) {
541 uint32_t value;
542
543 value = RREG32_PCIE(*pos);
544 r = put_user(value, (uint32_t *)buf);
545 if (r)
546 goto out;
547
548 result += 4;
549 buf += 4;
550 *pos += 4;
551 size -= 4;
552 }
553
554 r = result;
555 out:
556 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
557 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
558 amdgpu_virt_disable_access_debugfs(adev);
559 return r;
560 }
561
562 /**
563 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
564 *
565 * @f: open file handle
566 * @buf: User buffer to write data from
567 * @size: Number of bytes to write
568 * @pos: Offset to seek to
569 *
570 * The lower bits are the BYTE offset of the register to write. This
571 * allows writing multiple registers in a single call and having
572 * the returned size reflect that.
573 */
amdgpu_debugfs_regs_pcie_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)574 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
575 size_t size, loff_t *pos)
576 {
577 struct amdgpu_device *adev = file_inode(f)->i_private;
578 ssize_t result = 0;
579 int r;
580
581 if (size & 0x3 || *pos & 0x3)
582 return -EINVAL;
583
584 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
585 if (r < 0) {
586 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
587 return r;
588 }
589
590 r = amdgpu_virt_enable_access_debugfs(adev);
591 if (r < 0) {
592 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
593 return r;
594 }
595
596 while (size) {
597 uint32_t value;
598
599 r = get_user(value, (uint32_t *)buf);
600 if (r)
601 goto out;
602
603 WREG32_PCIE(*pos, value);
604
605 result += 4;
606 buf += 4;
607 *pos += 4;
608 size -= 4;
609 }
610
611 r = result;
612 out:
613 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
614 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
615 amdgpu_virt_disable_access_debugfs(adev);
616 return r;
617 }
618
619 /**
620 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
621 *
622 * @f: open file handle
623 * @buf: User buffer to store read data in
624 * @size: Number of bytes to read
625 * @pos: Offset to seek to
626 *
627 * The lower bits are the BYTE offset of the register to read. This
628 * allows reading multiple registers in a single call and having
629 * the returned size reflect that.
630 */
amdgpu_debugfs_regs_didt_read(struct file * f,char __user * buf,size_t size,loff_t * pos)631 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
632 size_t size, loff_t *pos)
633 {
634 struct amdgpu_device *adev = file_inode(f)->i_private;
635 ssize_t result = 0;
636 int r;
637
638 if (size & 0x3 || *pos & 0x3)
639 return -EINVAL;
640
641 if (!adev->didt_rreg)
642 return -EOPNOTSUPP;
643
644 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
645 if (r < 0) {
646 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
647 return r;
648 }
649
650 r = amdgpu_virt_enable_access_debugfs(adev);
651 if (r < 0) {
652 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
653 return r;
654 }
655
656 while (size) {
657 uint32_t value;
658
659 value = RREG32_DIDT(*pos >> 2);
660 r = put_user(value, (uint32_t *)buf);
661 if (r)
662 goto out;
663
664 result += 4;
665 buf += 4;
666 *pos += 4;
667 size -= 4;
668 }
669
670 r = result;
671 out:
672 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
673 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
674 amdgpu_virt_disable_access_debugfs(adev);
675 return r;
676 }
677
678 /**
679 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
680 *
681 * @f: open file handle
682 * @buf: User buffer to write data from
683 * @size: Number of bytes to write
684 * @pos: Offset to seek to
685 *
686 * The lower bits are the BYTE offset of the register to write. This
687 * allows writing multiple registers in a single call and having
688 * the returned size reflect that.
689 */
amdgpu_debugfs_regs_didt_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)690 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
691 size_t size, loff_t *pos)
692 {
693 struct amdgpu_device *adev = file_inode(f)->i_private;
694 ssize_t result = 0;
695 int r;
696
697 if (size & 0x3 || *pos & 0x3)
698 return -EINVAL;
699
700 if (!adev->didt_wreg)
701 return -EOPNOTSUPP;
702
703 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
704 if (r < 0) {
705 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
706 return r;
707 }
708
709 r = amdgpu_virt_enable_access_debugfs(adev);
710 if (r < 0) {
711 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
712 return r;
713 }
714
715 while (size) {
716 uint32_t value;
717
718 r = get_user(value, (uint32_t *)buf);
719 if (r)
720 goto out;
721
722 WREG32_DIDT(*pos >> 2, value);
723
724 result += 4;
725 buf += 4;
726 *pos += 4;
727 size -= 4;
728 }
729
730 r = result;
731 out:
732 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
733 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
734 amdgpu_virt_disable_access_debugfs(adev);
735 return r;
736 }
737
738 /**
739 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
740 *
741 * @f: open file handle
742 * @buf: User buffer to store read data in
743 * @size: Number of bytes to read
744 * @pos: Offset to seek to
745 *
746 * The lower bits are the BYTE offset of the register to read. This
747 * allows reading multiple registers in a single call and having
748 * the returned size reflect that.
749 */
amdgpu_debugfs_regs_smc_read(struct file * f,char __user * buf,size_t size,loff_t * pos)750 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
751 size_t size, loff_t *pos)
752 {
753 struct amdgpu_device *adev = file_inode(f)->i_private;
754 ssize_t result = 0;
755 int r;
756
757 if (!adev->smc_rreg)
758 return -EOPNOTSUPP;
759
760 if (size & 0x3 || *pos & 0x3)
761 return -EINVAL;
762
763 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
764 if (r < 0) {
765 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
766 return r;
767 }
768
769 r = amdgpu_virt_enable_access_debugfs(adev);
770 if (r < 0) {
771 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
772 return r;
773 }
774
775 while (size) {
776 uint32_t value;
777
778 value = RREG32_SMC(*pos);
779 r = put_user(value, (uint32_t *)buf);
780 if (r)
781 goto out;
782
783 result += 4;
784 buf += 4;
785 *pos += 4;
786 size -= 4;
787 }
788
789 r = result;
790 out:
791 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
792 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
793 amdgpu_virt_disable_access_debugfs(adev);
794 return r;
795 }
796
797 /**
798 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
799 *
800 * @f: open file handle
801 * @buf: User buffer to write data from
802 * @size: Number of bytes to write
803 * @pos: Offset to seek to
804 *
805 * The lower bits are the BYTE offset of the register to write. This
806 * allows writing multiple registers in a single call and having
807 * the returned size reflect that.
808 */
amdgpu_debugfs_regs_smc_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)809 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
810 size_t size, loff_t *pos)
811 {
812 struct amdgpu_device *adev = file_inode(f)->i_private;
813 ssize_t result = 0;
814 int r;
815
816 if (!adev->smc_wreg)
817 return -EOPNOTSUPP;
818
819 if (size & 0x3 || *pos & 0x3)
820 return -EINVAL;
821
822 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
823 if (r < 0) {
824 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
825 return r;
826 }
827
828 r = amdgpu_virt_enable_access_debugfs(adev);
829 if (r < 0) {
830 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
831 return r;
832 }
833
834 while (size) {
835 uint32_t value;
836
837 r = get_user(value, (uint32_t *)buf);
838 if (r)
839 goto out;
840
841 WREG32_SMC(*pos, value);
842
843 result += 4;
844 buf += 4;
845 *pos += 4;
846 size -= 4;
847 }
848
849 r = result;
850 out:
851 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
852 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
853 amdgpu_virt_disable_access_debugfs(adev);
854 return r;
855 }
856
857 /**
858 * amdgpu_debugfs_gca_config_read - Read from gfx config data
859 *
860 * @f: open file handle
861 * @buf: User buffer to store read data in
862 * @size: Number of bytes to read
863 * @pos: Offset to seek to
864 *
865 * This file is used to access configuration data in a somewhat
866 * stable fashion. The format is a series of DWORDs with the first
867 * indicating which revision it is. New content is appended to the
868 * end so that older software can still read the data.
869 */
870
amdgpu_debugfs_gca_config_read(struct file * f,char __user * buf,size_t size,loff_t * pos)871 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
872 size_t size, loff_t *pos)
873 {
874 struct amdgpu_device *adev = file_inode(f)->i_private;
875 ssize_t result = 0;
876 int r;
877 uint32_t *config, no_regs = 0;
878
879 if (size & 0x3 || *pos & 0x3)
880 return -EINVAL;
881
882 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
883 if (!config)
884 return -ENOMEM;
885
886 /* version, increment each time something is added */
887 config[no_regs++] = 5;
888 config[no_regs++] = adev->gfx.config.max_shader_engines;
889 config[no_regs++] = adev->gfx.config.max_tile_pipes;
890 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
891 config[no_regs++] = adev->gfx.config.max_sh_per_se;
892 config[no_regs++] = adev->gfx.config.max_backends_per_se;
893 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
894 config[no_regs++] = adev->gfx.config.max_gprs;
895 config[no_regs++] = adev->gfx.config.max_gs_threads;
896 config[no_regs++] = adev->gfx.config.max_hw_contexts;
897 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
898 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
899 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
900 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
901 config[no_regs++] = adev->gfx.config.num_tile_pipes;
902 config[no_regs++] = adev->gfx.config.backend_enable_mask;
903 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
904 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
905 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
906 config[no_regs++] = adev->gfx.config.num_gpus;
907 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
908 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
909 config[no_regs++] = adev->gfx.config.gb_addr_config;
910 config[no_regs++] = adev->gfx.config.num_rbs;
911
912 /* rev==1 */
913 config[no_regs++] = adev->rev_id;
914 config[no_regs++] = lower_32_bits(adev->pg_flags);
915 config[no_regs++] = lower_32_bits(adev->cg_flags);
916
917 /* rev==2 */
918 config[no_regs++] = adev->family;
919 config[no_regs++] = adev->external_rev_id;
920
921 /* rev==3 */
922 config[no_regs++] = adev->pdev->device;
923 config[no_regs++] = adev->pdev->revision;
924 config[no_regs++] = adev->pdev->subsystem_device;
925 config[no_regs++] = adev->pdev->subsystem_vendor;
926
927 /* rev==4 APU flag */
928 config[no_regs++] = adev->flags & AMD_IS_APU ? 1 : 0;
929
930 /* rev==5 PG/CG flag upper 32bit */
931 config[no_regs++] = upper_32_bits(adev->pg_flags);
932 config[no_regs++] = upper_32_bits(adev->cg_flags);
933
934 while (size && (*pos < no_regs * 4)) {
935 uint32_t value;
936
937 value = config[*pos >> 2];
938 r = put_user(value, (uint32_t *)buf);
939 if (r) {
940 kfree(config);
941 return r;
942 }
943
944 result += 4;
945 buf += 4;
946 *pos += 4;
947 size -= 4;
948 }
949
950 kfree(config);
951 return result;
952 }
953
954 /**
955 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
956 *
957 * @f: open file handle
958 * @buf: User buffer to store read data in
959 * @size: Number of bytes to read
960 * @pos: Offset to seek to
961 *
962 * The offset is treated as the BYTE address of one of the sensors
963 * enumerated in amd/include/kgd_pp_interface.h under the
964 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
965 * you would use the offset 3 * 4 = 12.
966 */
amdgpu_debugfs_sensor_read(struct file * f,char __user * buf,size_t size,loff_t * pos)967 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
968 size_t size, loff_t *pos)
969 {
970 struct amdgpu_device *adev = file_inode(f)->i_private;
971 int idx, x, outsize, r, valuesize;
972 uint32_t values[16];
973
974 if (size & 3 || *pos & 0x3)
975 return -EINVAL;
976
977 if (!adev->pm.dpm_enabled)
978 return -EINVAL;
979
980 /* convert offset to sensor number */
981 idx = *pos >> 2;
982
983 valuesize = sizeof(values);
984
985 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
986 if (r < 0) {
987 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
988 return r;
989 }
990
991 r = amdgpu_virt_enable_access_debugfs(adev);
992 if (r < 0) {
993 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
994 return r;
995 }
996
997 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
998
999 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1000 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1001
1002 if (r) {
1003 amdgpu_virt_disable_access_debugfs(adev);
1004 return r;
1005 }
1006
1007 if (size > valuesize) {
1008 amdgpu_virt_disable_access_debugfs(adev);
1009 return -EINVAL;
1010 }
1011
1012 outsize = 0;
1013 x = 0;
1014 if (!r) {
1015 while (size) {
1016 r = put_user(values[x++], (int32_t *)buf);
1017 buf += 4;
1018 size -= 4;
1019 outsize += 4;
1020 }
1021 }
1022
1023 amdgpu_virt_disable_access_debugfs(adev);
1024 return !r ? outsize : r;
1025 }
1026
1027 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
1028 *
1029 * @f: open file handle
1030 * @buf: User buffer to store read data in
1031 * @size: Number of bytes to read
1032 * @pos: Offset to seek to
1033 *
1034 * The offset being sought changes which wave that the status data
1035 * will be returned for. The bits are used as follows:
1036 *
1037 * Bits 0..6: Byte offset into data
1038 * Bits 7..14: SE selector
1039 * Bits 15..22: SH/SA selector
1040 * Bits 23..30: CU/{WGP+SIMD} selector
1041 * Bits 31..36: WAVE ID selector
1042 * Bits 37..44: SIMD ID selector
1043 *
1044 * The returned data begins with one DWORD of version information
1045 * Followed by WAVE STATUS registers relevant to the GFX IP version
1046 * being used. See gfx_v8_0_read_wave_data() for an example output.
1047 */
amdgpu_debugfs_wave_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1048 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
1049 size_t size, loff_t *pos)
1050 {
1051 struct amdgpu_device *adev = f->f_inode->i_private;
1052 int r, x;
1053 ssize_t result = 0;
1054 uint32_t offset, se, sh, cu, wave, simd, data[32];
1055
1056 if (size & 3 || *pos & 3)
1057 return -EINVAL;
1058
1059 /* decode offset */
1060 offset = (*pos & GENMASK_ULL(6, 0));
1061 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
1062 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
1063 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
1064 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
1065 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
1066
1067 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1068 if (r < 0) {
1069 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1070 return r;
1071 }
1072
1073 r = amdgpu_virt_enable_access_debugfs(adev);
1074 if (r < 0) {
1075 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1076 return r;
1077 }
1078
1079 /* switch to the specific se/sh/cu */
1080 mutex_lock(&adev->grbm_idx_mutex);
1081 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1082
1083 x = 0;
1084 if (adev->gfx.funcs->read_wave_data)
1085 adev->gfx.funcs->read_wave_data(adev, 0, simd, wave, data, &x);
1086
1087 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1088 mutex_unlock(&adev->grbm_idx_mutex);
1089
1090 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1091 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1092
1093 if (!x) {
1094 amdgpu_virt_disable_access_debugfs(adev);
1095 return -EINVAL;
1096 }
1097
1098 while (size && (offset < x * 4)) {
1099 uint32_t value;
1100
1101 value = data[offset >> 2];
1102 r = put_user(value, (uint32_t *)buf);
1103 if (r) {
1104 amdgpu_virt_disable_access_debugfs(adev);
1105 return r;
1106 }
1107
1108 result += 4;
1109 buf += 4;
1110 offset += 4;
1111 size -= 4;
1112 }
1113
1114 amdgpu_virt_disable_access_debugfs(adev);
1115 return result;
1116 }
1117
1118 /** amdgpu_debugfs_gpr_read - Read wave gprs
1119 *
1120 * @f: open file handle
1121 * @buf: User buffer to store read data in
1122 * @size: Number of bytes to read
1123 * @pos: Offset to seek to
1124 *
1125 * The offset being sought changes which wave that the status data
1126 * will be returned for. The bits are used as follows:
1127 *
1128 * Bits 0..11: Byte offset into data
1129 * Bits 12..19: SE selector
1130 * Bits 20..27: SH/SA selector
1131 * Bits 28..35: CU/{WGP+SIMD} selector
1132 * Bits 36..43: WAVE ID selector
1133 * Bits 37..44: SIMD ID selector
1134 * Bits 52..59: Thread selector
1135 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
1136 *
1137 * The return data comes from the SGPR or VGPR register bank for
1138 * the selected operational unit.
1139 */
amdgpu_debugfs_gpr_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1140 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
1141 size_t size, loff_t *pos)
1142 {
1143 struct amdgpu_device *adev = f->f_inode->i_private;
1144 int r;
1145 ssize_t result = 0;
1146 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
1147
1148 if (size > 4096 || size & 3 || *pos & 3)
1149 return -EINVAL;
1150
1151 /* decode offset */
1152 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
1153 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
1154 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
1155 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
1156 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
1157 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
1158 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
1159 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
1160
1161 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
1162 if (!data)
1163 return -ENOMEM;
1164
1165 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1166 if (r < 0)
1167 goto err;
1168
1169 r = amdgpu_virt_enable_access_debugfs(adev);
1170 if (r < 0)
1171 goto err;
1172
1173 /* switch to the specific se/sh/cu */
1174 mutex_lock(&adev->grbm_idx_mutex);
1175 amdgpu_gfx_select_se_sh(adev, se, sh, cu, 0);
1176
1177 if (bank == 0) {
1178 if (adev->gfx.funcs->read_wave_vgprs)
1179 adev->gfx.funcs->read_wave_vgprs(adev, 0, simd, wave, thread, offset, size>>2, data);
1180 } else {
1181 if (adev->gfx.funcs->read_wave_sgprs)
1182 adev->gfx.funcs->read_wave_sgprs(adev, 0, simd, wave, offset, size>>2, data);
1183 }
1184
1185 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0);
1186 mutex_unlock(&adev->grbm_idx_mutex);
1187
1188 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1189 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1190
1191 while (size) {
1192 uint32_t value;
1193
1194 value = data[result >> 2];
1195 r = put_user(value, (uint32_t *)buf);
1196 if (r) {
1197 amdgpu_virt_disable_access_debugfs(adev);
1198 goto err;
1199 }
1200
1201 result += 4;
1202 buf += 4;
1203 size -= 4;
1204 }
1205
1206 kfree(data);
1207 amdgpu_virt_disable_access_debugfs(adev);
1208 return result;
1209
1210 err:
1211 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1212 kfree(data);
1213 return r;
1214 }
1215
1216 /**
1217 * amdgpu_debugfs_gfxoff_residency_read - Read GFXOFF residency
1218 *
1219 * @f: open file handle
1220 * @buf: User buffer to store read data in
1221 * @size: Number of bytes to read
1222 * @pos: Offset to seek to
1223 *
1224 * Read the last residency value logged. It doesn't auto update, one needs to
1225 * stop logging before getting the current value.
1226 */
amdgpu_debugfs_gfxoff_residency_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1227 static ssize_t amdgpu_debugfs_gfxoff_residency_read(struct file *f, char __user *buf,
1228 size_t size, loff_t *pos)
1229 {
1230 struct amdgpu_device *adev = file_inode(f)->i_private;
1231 ssize_t result = 0;
1232 int r;
1233
1234 if (size & 0x3 || *pos & 0x3)
1235 return -EINVAL;
1236
1237 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1238 if (r < 0) {
1239 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1240 return r;
1241 }
1242
1243 while (size) {
1244 uint32_t value;
1245
1246 r = amdgpu_get_gfx_off_residency(adev, &value);
1247 if (r)
1248 goto out;
1249
1250 r = put_user(value, (uint32_t *)buf);
1251 if (r)
1252 goto out;
1253
1254 result += 4;
1255 buf += 4;
1256 *pos += 4;
1257 size -= 4;
1258 }
1259
1260 r = result;
1261 out:
1262 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1263 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1264
1265 return r;
1266 }
1267
1268 /**
1269 * amdgpu_debugfs_gfxoff_residency_write - Log GFXOFF Residency
1270 *
1271 * @f: open file handle
1272 * @buf: User buffer to write data from
1273 * @size: Number of bytes to write
1274 * @pos: Offset to seek to
1275 *
1276 * Write a 32-bit non-zero to start logging; write a 32-bit zero to stop
1277 */
amdgpu_debugfs_gfxoff_residency_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)1278 static ssize_t amdgpu_debugfs_gfxoff_residency_write(struct file *f, const char __user *buf,
1279 size_t size, loff_t *pos)
1280 {
1281 struct amdgpu_device *adev = file_inode(f)->i_private;
1282 ssize_t result = 0;
1283 int r;
1284
1285 if (size & 0x3 || *pos & 0x3)
1286 return -EINVAL;
1287
1288 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1289 if (r < 0) {
1290 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1291 return r;
1292 }
1293
1294 while (size) {
1295 u32 value;
1296
1297 r = get_user(value, (uint32_t *)buf);
1298 if (r)
1299 goto out;
1300
1301 amdgpu_set_gfx_off_residency(adev, value ? true : false);
1302
1303 result += 4;
1304 buf += 4;
1305 *pos += 4;
1306 size -= 4;
1307 }
1308
1309 r = result;
1310 out:
1311 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1312 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1313
1314 return r;
1315 }
1316
1317
1318 /**
1319 * amdgpu_debugfs_gfxoff_count_read - Read GFXOFF entry count
1320 *
1321 * @f: open file handle
1322 * @buf: User buffer to store read data in
1323 * @size: Number of bytes to read
1324 * @pos: Offset to seek to
1325 */
amdgpu_debugfs_gfxoff_count_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1326 static ssize_t amdgpu_debugfs_gfxoff_count_read(struct file *f, char __user *buf,
1327 size_t size, loff_t *pos)
1328 {
1329 struct amdgpu_device *adev = file_inode(f)->i_private;
1330 ssize_t result = 0;
1331 int r;
1332
1333 if (size & 0x3 || *pos & 0x3)
1334 return -EINVAL;
1335
1336 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1337 if (r < 0) {
1338 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1339 return r;
1340 }
1341
1342 while (size) {
1343 u64 value = 0;
1344
1345 r = amdgpu_get_gfx_off_entrycount(adev, &value);
1346 if (r)
1347 goto out;
1348
1349 r = put_user(value, (u64 *)buf);
1350 if (r)
1351 goto out;
1352
1353 result += 4;
1354 buf += 4;
1355 *pos += 4;
1356 size -= 4;
1357 }
1358
1359 r = result;
1360 out:
1361 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1362 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1363
1364 return r;
1365 }
1366
1367 /**
1368 * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1369 *
1370 * @f: open file handle
1371 * @buf: User buffer to write data from
1372 * @size: Number of bytes to write
1373 * @pos: Offset to seek to
1374 *
1375 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1376 */
amdgpu_debugfs_gfxoff_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)1377 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1378 size_t size, loff_t *pos)
1379 {
1380 struct amdgpu_device *adev = file_inode(f)->i_private;
1381 ssize_t result = 0;
1382 int r;
1383
1384 if (size & 0x3 || *pos & 0x3)
1385 return -EINVAL;
1386
1387 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1388 if (r < 0) {
1389 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1390 return r;
1391 }
1392
1393 while (size) {
1394 uint32_t value;
1395
1396 r = get_user(value, (uint32_t *)buf);
1397 if (r)
1398 goto out;
1399
1400 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1401
1402 result += 4;
1403 buf += 4;
1404 *pos += 4;
1405 size -= 4;
1406 }
1407
1408 r = result;
1409 out:
1410 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1411 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1412
1413 return r;
1414 }
1415
1416
1417 /**
1418 * amdgpu_debugfs_gfxoff_read - read gfxoff status
1419 *
1420 * @f: open file handle
1421 * @buf: User buffer to store read data in
1422 * @size: Number of bytes to read
1423 * @pos: Offset to seek to
1424 */
amdgpu_debugfs_gfxoff_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1425 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1426 size_t size, loff_t *pos)
1427 {
1428 struct amdgpu_device *adev = file_inode(f)->i_private;
1429 ssize_t result = 0;
1430 int r;
1431
1432 if (size & 0x3 || *pos & 0x3)
1433 return -EINVAL;
1434
1435 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1436 if (r < 0) {
1437 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1438 return r;
1439 }
1440
1441 while (size) {
1442 u32 value = adev->gfx.gfx_off_state;
1443
1444 r = put_user(value, (u32 *)buf);
1445 if (r)
1446 goto out;
1447
1448 result += 4;
1449 buf += 4;
1450 *pos += 4;
1451 size -= 4;
1452 }
1453
1454 r = result;
1455 out:
1456 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1457 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1458
1459 return r;
1460 }
1461
amdgpu_debugfs_gfxoff_status_read(struct file * f,char __user * buf,size_t size,loff_t * pos)1462 static ssize_t amdgpu_debugfs_gfxoff_status_read(struct file *f, char __user *buf,
1463 size_t size, loff_t *pos)
1464 {
1465 struct amdgpu_device *adev = file_inode(f)->i_private;
1466 ssize_t result = 0;
1467 int r;
1468
1469 if (size & 0x3 || *pos & 0x3)
1470 return -EINVAL;
1471
1472 r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1473 if (r < 0) {
1474 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1475 return r;
1476 }
1477
1478 while (size) {
1479 u32 value;
1480
1481 r = amdgpu_get_gfx_off_status(adev, &value);
1482 if (r)
1483 goto out;
1484
1485 r = put_user(value, (u32 *)buf);
1486 if (r)
1487 goto out;
1488
1489 result += 4;
1490 buf += 4;
1491 *pos += 4;
1492 size -= 4;
1493 }
1494
1495 r = result;
1496 out:
1497 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1498 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1499
1500 return r;
1501 }
1502
1503 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1504 .owner = THIS_MODULE,
1505 .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1506 .read = amdgpu_debugfs_regs2_read,
1507 .write = amdgpu_debugfs_regs2_write,
1508 .open = amdgpu_debugfs_regs2_open,
1509 .release = amdgpu_debugfs_regs2_release,
1510 .llseek = default_llseek
1511 };
1512
1513 static const struct file_operations amdgpu_debugfs_gprwave_fops = {
1514 .owner = THIS_MODULE,
1515 .unlocked_ioctl = amdgpu_debugfs_gprwave_ioctl,
1516 .read = amdgpu_debugfs_gprwave_read,
1517 .open = amdgpu_debugfs_gprwave_open,
1518 .release = amdgpu_debugfs_gprwave_release,
1519 .llseek = default_llseek
1520 };
1521
1522 static const struct file_operations amdgpu_debugfs_regs_fops = {
1523 .owner = THIS_MODULE,
1524 .read = amdgpu_debugfs_regs_read,
1525 .write = amdgpu_debugfs_regs_write,
1526 .llseek = default_llseek
1527 };
1528 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1529 .owner = THIS_MODULE,
1530 .read = amdgpu_debugfs_regs_didt_read,
1531 .write = amdgpu_debugfs_regs_didt_write,
1532 .llseek = default_llseek
1533 };
1534 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1535 .owner = THIS_MODULE,
1536 .read = amdgpu_debugfs_regs_pcie_read,
1537 .write = amdgpu_debugfs_regs_pcie_write,
1538 .llseek = default_llseek
1539 };
1540 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1541 .owner = THIS_MODULE,
1542 .read = amdgpu_debugfs_regs_smc_read,
1543 .write = amdgpu_debugfs_regs_smc_write,
1544 .llseek = default_llseek
1545 };
1546
1547 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1548 .owner = THIS_MODULE,
1549 .read = amdgpu_debugfs_gca_config_read,
1550 .llseek = default_llseek
1551 };
1552
1553 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1554 .owner = THIS_MODULE,
1555 .read = amdgpu_debugfs_sensor_read,
1556 .llseek = default_llseek
1557 };
1558
1559 static const struct file_operations amdgpu_debugfs_wave_fops = {
1560 .owner = THIS_MODULE,
1561 .read = amdgpu_debugfs_wave_read,
1562 .llseek = default_llseek
1563 };
1564 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1565 .owner = THIS_MODULE,
1566 .read = amdgpu_debugfs_gpr_read,
1567 .llseek = default_llseek
1568 };
1569
1570 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1571 .owner = THIS_MODULE,
1572 .read = amdgpu_debugfs_gfxoff_read,
1573 .write = amdgpu_debugfs_gfxoff_write,
1574 .llseek = default_llseek
1575 };
1576
1577 static const struct file_operations amdgpu_debugfs_gfxoff_status_fops = {
1578 .owner = THIS_MODULE,
1579 .read = amdgpu_debugfs_gfxoff_status_read,
1580 .llseek = default_llseek
1581 };
1582
1583 static const struct file_operations amdgpu_debugfs_gfxoff_count_fops = {
1584 .owner = THIS_MODULE,
1585 .read = amdgpu_debugfs_gfxoff_count_read,
1586 .llseek = default_llseek
1587 };
1588
1589 static const struct file_operations amdgpu_debugfs_gfxoff_residency_fops = {
1590 .owner = THIS_MODULE,
1591 .read = amdgpu_debugfs_gfxoff_residency_read,
1592 .write = amdgpu_debugfs_gfxoff_residency_write,
1593 .llseek = default_llseek
1594 };
1595
1596 static const struct file_operations *debugfs_regs[] = {
1597 &amdgpu_debugfs_regs_fops,
1598 &amdgpu_debugfs_regs2_fops,
1599 &amdgpu_debugfs_gprwave_fops,
1600 &amdgpu_debugfs_regs_didt_fops,
1601 &amdgpu_debugfs_regs_pcie_fops,
1602 &amdgpu_debugfs_regs_smc_fops,
1603 &amdgpu_debugfs_gca_config_fops,
1604 &amdgpu_debugfs_sensors_fops,
1605 &amdgpu_debugfs_wave_fops,
1606 &amdgpu_debugfs_gpr_fops,
1607 &amdgpu_debugfs_gfxoff_fops,
1608 &amdgpu_debugfs_gfxoff_status_fops,
1609 &amdgpu_debugfs_gfxoff_count_fops,
1610 &amdgpu_debugfs_gfxoff_residency_fops,
1611 };
1612
1613 static const char * const debugfs_regs_names[] = {
1614 "amdgpu_regs",
1615 "amdgpu_regs2",
1616 "amdgpu_gprwave",
1617 "amdgpu_regs_didt",
1618 "amdgpu_regs_pcie",
1619 "amdgpu_regs_smc",
1620 "amdgpu_gca_config",
1621 "amdgpu_sensors",
1622 "amdgpu_wave",
1623 "amdgpu_gpr",
1624 "amdgpu_gfxoff",
1625 "amdgpu_gfxoff_status",
1626 "amdgpu_gfxoff_count",
1627 "amdgpu_gfxoff_residency",
1628 };
1629
1630 /**
1631 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1632 * register access.
1633 *
1634 * @adev: The device to attach the debugfs entries to
1635 */
amdgpu_debugfs_regs_init(struct amdgpu_device * adev)1636 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1637 {
1638 struct drm_minor *minor = adev_to_drm(adev)->primary;
1639 struct dentry *ent, *root = minor->debugfs_root;
1640 unsigned int i;
1641
1642 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1643 ent = debugfs_create_file(debugfs_regs_names[i],
1644 S_IFREG | 0444, root,
1645 adev, debugfs_regs[i]);
1646 if (!i && !IS_ERR_OR_NULL(ent))
1647 i_size_write(ent->d_inode, adev->rmmio_size);
1648 }
1649
1650 return 0;
1651 }
1652
amdgpu_debugfs_test_ib_show(struct seq_file * m,void * unused)1653 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1654 {
1655 struct amdgpu_device *adev = m->private;
1656 struct drm_device *dev = adev_to_drm(adev);
1657 int r = 0, i;
1658
1659 r = pm_runtime_get_sync(dev->dev);
1660 if (r < 0) {
1661 pm_runtime_put_autosuspend(dev->dev);
1662 return r;
1663 }
1664
1665 /* Avoid accidently unparking the sched thread during GPU reset */
1666 r = down_write_killable(&adev->reset_domain->sem);
1667 if (r)
1668 return r;
1669
1670 /* hold on the scheduler */
1671 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1672 struct amdgpu_ring *ring = adev->rings[i];
1673
1674 if (!ring || !ring->sched.thread)
1675 continue;
1676 kthread_park(ring->sched.thread);
1677 }
1678
1679 seq_puts(m, "run ib test:\n");
1680 r = amdgpu_ib_ring_tests(adev);
1681 if (r)
1682 seq_printf(m, "ib ring tests failed (%d).\n", r);
1683 else
1684 seq_puts(m, "ib ring tests passed.\n");
1685
1686 /* go on the scheduler */
1687 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1688 struct amdgpu_ring *ring = adev->rings[i];
1689
1690 if (!ring || !ring->sched.thread)
1691 continue;
1692 kthread_unpark(ring->sched.thread);
1693 }
1694
1695 up_write(&adev->reset_domain->sem);
1696
1697 pm_runtime_mark_last_busy(dev->dev);
1698 pm_runtime_put_autosuspend(dev->dev);
1699
1700 return 0;
1701 }
1702
amdgpu_debugfs_evict_vram(void * data,u64 * val)1703 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1704 {
1705 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1706 struct drm_device *dev = adev_to_drm(adev);
1707 int r;
1708
1709 r = pm_runtime_get_sync(dev->dev);
1710 if (r < 0) {
1711 pm_runtime_put_autosuspend(dev->dev);
1712 return r;
1713 }
1714
1715 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1716
1717 pm_runtime_mark_last_busy(dev->dev);
1718 pm_runtime_put_autosuspend(dev->dev);
1719
1720 return 0;
1721 }
1722
1723
amdgpu_debugfs_evict_gtt(void * data,u64 * val)1724 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1725 {
1726 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1727 struct drm_device *dev = adev_to_drm(adev);
1728 int r;
1729
1730 r = pm_runtime_get_sync(dev->dev);
1731 if (r < 0) {
1732 pm_runtime_put_autosuspend(dev->dev);
1733 return r;
1734 }
1735
1736 *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1737
1738 pm_runtime_mark_last_busy(dev->dev);
1739 pm_runtime_put_autosuspend(dev->dev);
1740
1741 return 0;
1742 }
1743
amdgpu_debugfs_benchmark(void * data,u64 val)1744 static int amdgpu_debugfs_benchmark(void *data, u64 val)
1745 {
1746 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1747 struct drm_device *dev = adev_to_drm(adev);
1748 int r;
1749
1750 r = pm_runtime_get_sync(dev->dev);
1751 if (r < 0) {
1752 pm_runtime_put_autosuspend(dev->dev);
1753 return r;
1754 }
1755
1756 r = amdgpu_benchmark(adev, val);
1757
1758 pm_runtime_mark_last_busy(dev->dev);
1759 pm_runtime_put_autosuspend(dev->dev);
1760
1761 return r;
1762 }
1763
amdgpu_debugfs_vm_info_show(struct seq_file * m,void * unused)1764 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1765 {
1766 struct amdgpu_device *adev = m->private;
1767 struct drm_device *dev = adev_to_drm(adev);
1768 struct drm_file *file;
1769 int r;
1770
1771 r = mutex_lock_interruptible(&dev->filelist_mutex);
1772 if (r)
1773 return r;
1774
1775 list_for_each_entry(file, &dev->filelist, lhead) {
1776 struct amdgpu_fpriv *fpriv = file->driver_priv;
1777 struct amdgpu_vm *vm = &fpriv->vm;
1778
1779 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1780 vm->task_info.pid, vm->task_info.process_name);
1781 r = amdgpu_bo_reserve(vm->root.bo, true);
1782 if (r)
1783 break;
1784 amdgpu_debugfs_vm_bo_info(vm, m);
1785 amdgpu_bo_unreserve(vm->root.bo);
1786 }
1787
1788 mutex_unlock(&dev->filelist_mutex);
1789
1790 return r;
1791 }
1792
1793 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1794 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1795 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1796 NULL, "%lld\n");
1797 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1798 NULL, "%lld\n");
1799 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_benchmark_fops, NULL, amdgpu_debugfs_benchmark,
1800 "%lld\n");
1801
amdgpu_ib_preempt_fences_swap(struct amdgpu_ring * ring,struct dma_fence ** fences)1802 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1803 struct dma_fence **fences)
1804 {
1805 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1806 uint32_t sync_seq, last_seq;
1807
1808 last_seq = atomic_read(&ring->fence_drv.last_seq);
1809 sync_seq = ring->fence_drv.sync_seq;
1810
1811 last_seq &= drv->num_fences_mask;
1812 sync_seq &= drv->num_fences_mask;
1813
1814 do {
1815 struct dma_fence *fence, **ptr;
1816
1817 ++last_seq;
1818 last_seq &= drv->num_fences_mask;
1819 ptr = &drv->fences[last_seq];
1820
1821 fence = rcu_dereference_protected(*ptr, 1);
1822 RCU_INIT_POINTER(*ptr, NULL);
1823
1824 if (!fence)
1825 continue;
1826
1827 fences[last_seq] = fence;
1828
1829 } while (last_seq != sync_seq);
1830 }
1831
amdgpu_ib_preempt_signal_fences(struct dma_fence ** fences,int length)1832 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1833 int length)
1834 {
1835 int i;
1836 struct dma_fence *fence;
1837
1838 for (i = 0; i < length; i++) {
1839 fence = fences[i];
1840 if (!fence)
1841 continue;
1842 dma_fence_signal(fence);
1843 dma_fence_put(fence);
1844 }
1845 }
1846
amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler * sched)1847 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1848 {
1849 struct drm_sched_job *s_job;
1850 struct dma_fence *fence;
1851
1852 spin_lock(&sched->job_list_lock);
1853 list_for_each_entry(s_job, &sched->pending_list, list) {
1854 fence = sched->ops->run_job(s_job);
1855 dma_fence_put(fence);
1856 }
1857 spin_unlock(&sched->job_list_lock);
1858 }
1859
amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring * ring)1860 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1861 {
1862 struct amdgpu_job *job;
1863 struct drm_sched_job *s_job, *tmp;
1864 uint32_t preempt_seq;
1865 struct dma_fence *fence, **ptr;
1866 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1867 struct drm_gpu_scheduler *sched = &ring->sched;
1868 bool preempted = true;
1869
1870 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1871 return;
1872
1873 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1874 if (preempt_seq <= atomic_read(&drv->last_seq)) {
1875 preempted = false;
1876 goto no_preempt;
1877 }
1878
1879 preempt_seq &= drv->num_fences_mask;
1880 ptr = &drv->fences[preempt_seq];
1881 fence = rcu_dereference_protected(*ptr, 1);
1882
1883 no_preempt:
1884 spin_lock(&sched->job_list_lock);
1885 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1886 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1887 /* remove job from ring_mirror_list */
1888 list_del_init(&s_job->list);
1889 sched->ops->free_job(s_job);
1890 continue;
1891 }
1892 job = to_amdgpu_job(s_job);
1893 if (preempted && (&job->hw_fence) == fence)
1894 /* mark the job as preempted */
1895 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1896 }
1897 spin_unlock(&sched->job_list_lock);
1898 }
1899
amdgpu_debugfs_ib_preempt(void * data,u64 val)1900 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1901 {
1902 int r, length;
1903 struct amdgpu_ring *ring;
1904 struct dma_fence **fences = NULL;
1905 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1906
1907 if (val >= AMDGPU_MAX_RINGS)
1908 return -EINVAL;
1909
1910 ring = adev->rings[val];
1911
1912 if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1913 return -EINVAL;
1914
1915 /* the last preemption failed */
1916 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1917 return -EBUSY;
1918
1919 length = ring->fence_drv.num_fences_mask + 1;
1920 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1921 if (!fences)
1922 return -ENOMEM;
1923
1924 /* Avoid accidently unparking the sched thread during GPU reset */
1925 r = down_read_killable(&adev->reset_domain->sem);
1926 if (r)
1927 goto pro_end;
1928
1929 /* stop the scheduler */
1930 kthread_park(ring->sched.thread);
1931
1932 /* preempt the IB */
1933 r = amdgpu_ring_preempt_ib(ring);
1934 if (r) {
1935 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1936 goto failure;
1937 }
1938
1939 amdgpu_fence_process(ring);
1940
1941 if (atomic_read(&ring->fence_drv.last_seq) !=
1942 ring->fence_drv.sync_seq) {
1943 DRM_INFO("ring %d was preempted\n", ring->idx);
1944
1945 amdgpu_ib_preempt_mark_partial_job(ring);
1946
1947 /* swap out the old fences */
1948 amdgpu_ib_preempt_fences_swap(ring, fences);
1949
1950 amdgpu_fence_driver_force_completion(ring);
1951
1952 /* resubmit unfinished jobs */
1953 amdgpu_ib_preempt_job_recovery(&ring->sched);
1954
1955 /* wait for jobs finished */
1956 amdgpu_fence_wait_empty(ring);
1957
1958 /* signal the old fences */
1959 amdgpu_ib_preempt_signal_fences(fences, length);
1960 }
1961
1962 failure:
1963 /* restart the scheduler */
1964 kthread_unpark(ring->sched.thread);
1965
1966 up_read(&adev->reset_domain->sem);
1967
1968 pro_end:
1969 kfree(fences);
1970
1971 return r;
1972 }
1973
amdgpu_debugfs_sclk_set(void * data,u64 val)1974 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1975 {
1976 int ret = 0;
1977 uint32_t max_freq, min_freq;
1978 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1979
1980 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1981 return -EINVAL;
1982
1983 ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1984 if (ret < 0) {
1985 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1986 return ret;
1987 }
1988
1989 ret = amdgpu_dpm_get_dpm_freq_range(adev, PP_SCLK, &min_freq, &max_freq);
1990 if (ret == -EOPNOTSUPP) {
1991 ret = 0;
1992 goto out;
1993 }
1994 if (ret || val > max_freq || val < min_freq) {
1995 ret = -EINVAL;
1996 goto out;
1997 }
1998
1999 ret = amdgpu_dpm_set_soft_freq_range(adev, PP_SCLK, (uint32_t)val, (uint32_t)val);
2000 if (ret)
2001 ret = -EINVAL;
2002
2003 out:
2004 pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
2005 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
2006
2007 return ret;
2008 }
2009
2010 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
2011 amdgpu_debugfs_ib_preempt, "%llu\n");
2012
2013 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
2014 amdgpu_debugfs_sclk_set, "%llu\n");
2015
amdgpu_reset_dump_register_list_read(struct file * f,char __user * buf,size_t size,loff_t * pos)2016 static ssize_t amdgpu_reset_dump_register_list_read(struct file *f,
2017 char __user *buf, size_t size, loff_t *pos)
2018 {
2019 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2020 char reg_offset[12];
2021 int i, ret, len = 0;
2022
2023 if (*pos)
2024 return 0;
2025
2026 memset(reg_offset, 0, 12);
2027 ret = down_read_killable(&adev->reset_domain->sem);
2028 if (ret)
2029 return ret;
2030
2031 for (i = 0; i < adev->num_regs; i++) {
2032 sprintf(reg_offset, "0x%x\n", adev->reset_dump_reg_list[i]);
2033 up_read(&adev->reset_domain->sem);
2034 if (copy_to_user(buf + len, reg_offset, strlen(reg_offset)))
2035 return -EFAULT;
2036
2037 len += strlen(reg_offset);
2038 ret = down_read_killable(&adev->reset_domain->sem);
2039 if (ret)
2040 return ret;
2041 }
2042
2043 up_read(&adev->reset_domain->sem);
2044 *pos += len;
2045
2046 return len;
2047 }
2048
amdgpu_reset_dump_register_list_write(struct file * f,const char __user * buf,size_t size,loff_t * pos)2049 static ssize_t amdgpu_reset_dump_register_list_write(struct file *f,
2050 const char __user *buf, size_t size, loff_t *pos)
2051 {
2052 struct amdgpu_device *adev = (struct amdgpu_device *)file_inode(f)->i_private;
2053 char reg_offset[11];
2054 uint32_t *new = NULL, *tmp = NULL;
2055 unsigned int len = 0;
2056 int ret, i = 0;
2057
2058 do {
2059 memset(reg_offset, 0, 11);
2060 if (copy_from_user(reg_offset, buf + len,
2061 min(10, (size-len)))) {
2062 ret = -EFAULT;
2063 goto error_free;
2064 }
2065
2066 new = krealloc_array(tmp, i + 1, sizeof(uint32_t), GFP_KERNEL);
2067 if (!new) {
2068 ret = -ENOMEM;
2069 goto error_free;
2070 }
2071 tmp = new;
2072 if (sscanf(reg_offset, "%X %n", &tmp[i], &ret) != 1) {
2073 ret = -EINVAL;
2074 goto error_free;
2075 }
2076
2077 len += ret;
2078 i++;
2079 } while (len < size);
2080
2081 new = kmalloc_array(i, sizeof(uint32_t), GFP_KERNEL);
2082 if (!new) {
2083 ret = -ENOMEM;
2084 goto error_free;
2085 }
2086 ret = down_write_killable(&adev->reset_domain->sem);
2087 if (ret)
2088 goto error_free;
2089
2090 swap(adev->reset_dump_reg_list, tmp);
2091 swap(adev->reset_dump_reg_value, new);
2092 adev->num_regs = i;
2093 up_write(&adev->reset_domain->sem);
2094 ret = size;
2095
2096 error_free:
2097 if (tmp != new)
2098 kfree(tmp);
2099 kfree(new);
2100 return ret;
2101 }
2102
2103 static const struct file_operations amdgpu_reset_dump_register_list = {
2104 .owner = THIS_MODULE,
2105 .read = amdgpu_reset_dump_register_list_read,
2106 .write = amdgpu_reset_dump_register_list_write,
2107 .llseek = default_llseek
2108 };
2109
amdgpu_debugfs_init(struct amdgpu_device * adev)2110 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2111 {
2112 struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
2113 struct dentry *ent;
2114 int r, i;
2115
2116 if (!debugfs_initialized())
2117 return 0;
2118
2119 debugfs_create_x32("amdgpu_smu_debug", 0600, root,
2120 &adev->pm.smu_debug_mask);
2121
2122 ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
2123 &fops_ib_preempt);
2124 if (IS_ERR(ent)) {
2125 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
2126 return PTR_ERR(ent);
2127 }
2128
2129 ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
2130 &fops_sclk_set);
2131 if (IS_ERR(ent)) {
2132 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
2133 return PTR_ERR(ent);
2134 }
2135
2136 /* Register debugfs entries for amdgpu_ttm */
2137 amdgpu_ttm_debugfs_init(adev);
2138 amdgpu_debugfs_pm_init(adev);
2139 amdgpu_debugfs_sa_init(adev);
2140 amdgpu_debugfs_fence_init(adev);
2141 amdgpu_debugfs_gem_init(adev);
2142
2143 r = amdgpu_debugfs_regs_init(adev);
2144 if (r)
2145 DRM_ERROR("registering register debugfs failed (%d).\n", r);
2146
2147 amdgpu_debugfs_firmware_init(adev);
2148 amdgpu_ta_if_debugfs_init(adev);
2149
2150 #if defined(CONFIG_DRM_AMD_DC)
2151 if (adev->dc_enabled)
2152 dtn_debugfs_init(adev);
2153 #endif
2154
2155 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2156 struct amdgpu_ring *ring = adev->rings[i];
2157
2158 if (!ring)
2159 continue;
2160
2161 amdgpu_debugfs_ring_init(adev, ring);
2162 }
2163
2164 for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
2165 if (!amdgpu_vcnfw_log)
2166 break;
2167
2168 if (adev->vcn.harvest_config & (1 << i))
2169 continue;
2170
2171 amdgpu_debugfs_vcn_fwlog_init(adev, i, &adev->vcn.inst[i]);
2172 }
2173
2174 amdgpu_ras_debugfs_create_all(adev);
2175 amdgpu_rap_debugfs_init(adev);
2176 amdgpu_securedisplay_debugfs_init(adev);
2177 amdgpu_fw_attestation_debugfs_init(adev);
2178
2179 debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
2180 &amdgpu_evict_vram_fops);
2181 debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
2182 &amdgpu_evict_gtt_fops);
2183 debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
2184 &amdgpu_debugfs_test_ib_fops);
2185 debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
2186 &amdgpu_debugfs_vm_info_fops);
2187 debugfs_create_file("amdgpu_benchmark", 0200, root, adev,
2188 &amdgpu_benchmark_fops);
2189 debugfs_create_file("amdgpu_reset_dump_register_list", 0644, root, adev,
2190 &amdgpu_reset_dump_register_list);
2191
2192 adev->debugfs_vbios_blob.data = adev->bios;
2193 adev->debugfs_vbios_blob.size = adev->bios_size;
2194 debugfs_create_blob("amdgpu_vbios", 0444, root,
2195 &adev->debugfs_vbios_blob);
2196
2197 adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
2198 adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
2199 debugfs_create_blob("amdgpu_discovery", 0444, root,
2200 &adev->debugfs_discovery_blob);
2201
2202 return 0;
2203 }
2204
2205 #else
amdgpu_debugfs_init(struct amdgpu_device * adev)2206 int amdgpu_debugfs_init(struct amdgpu_device *adev)
2207 {
2208 return 0;
2209 }
amdgpu_debugfs_regs_init(struct amdgpu_device * adev)2210 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
2211 {
2212 return 0;
2213 }
2214 #endif
2215