1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3 * Copyright 2014-2022 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <linux/mutex.h>
25 #include <linux/log2.h>
26 #include <linux/sched.h>
27 #include <linux/sched/mm.h>
28 #include <linux/sched/task.h>
29 #include <linux/mmu_context.h>
30 #include <linux/slab.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu.h"
38 #include "amdgpu_reset.h"
39
40 struct mm_struct;
41
42 #include "kfd_priv.h"
43 #include "kfd_device_queue_manager.h"
44 #include "kfd_svm.h"
45 #include "kfd_smi_events.h"
46 #include "kfd_debug.h"
47
48 /*
49 * List of struct kfd_process (field kfd_process).
50 * Unique/indexed by mm_struct*
51 */
52 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
53 DEFINE_MUTEX(kfd_processes_mutex);
54
55 DEFINE_SRCU(kfd_processes_srcu);
56
57 /* For process termination handling */
58 static struct workqueue_struct *kfd_process_wq;
59
60 /* Ordered, single-threaded workqueue for restoring evicted
61 * processes. Restoring multiple processes concurrently under memory
62 * pressure can lead to processes blocking each other from validating
63 * their BOs and result in a live-lock situation where processes
64 * remain evicted indefinitely.
65 */
66 static struct workqueue_struct *kfd_restore_wq;
67
68 static struct kfd_process *find_process(const struct task_struct *thread,
69 bool ref);
70 static void kfd_process_ref_release(struct kref *ref);
71 static struct kfd_process *create_process(const struct task_struct *thread);
72
73 static void evict_process_worker(struct work_struct *work);
74 static void restore_process_worker(struct work_struct *work);
75
76 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd);
77
78 struct kfd_procfs_tree {
79 struct kobject *kobj;
80 };
81
82 static struct kfd_procfs_tree procfs;
83
84 /*
85 * Structure for SDMA activity tracking
86 */
87 struct kfd_sdma_activity_handler_workarea {
88 struct work_struct sdma_activity_work;
89 struct kfd_process_device *pdd;
90 uint64_t sdma_activity_counter;
91 };
92
93 struct temp_sdma_queue_list {
94 uint64_t __user *rptr;
95 uint64_t sdma_val;
96 unsigned int queue_id;
97 struct list_head list;
98 };
99
kfd_sdma_activity_worker(struct work_struct * work)100 static void kfd_sdma_activity_worker(struct work_struct *work)
101 {
102 struct kfd_sdma_activity_handler_workarea *workarea;
103 struct kfd_process_device *pdd;
104 uint64_t val;
105 struct mm_struct *mm;
106 struct queue *q;
107 struct qcm_process_device *qpd;
108 struct device_queue_manager *dqm;
109 int ret = 0;
110 struct temp_sdma_queue_list sdma_q_list;
111 struct temp_sdma_queue_list *sdma_q, *next;
112
113 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
114 sdma_activity_work);
115
116 pdd = workarea->pdd;
117 if (!pdd)
118 return;
119 dqm = pdd->dev->dqm;
120 qpd = &pdd->qpd;
121 if (!dqm || !qpd)
122 return;
123 /*
124 * Total SDMA activity is current SDMA activity + past SDMA activity
125 * Past SDMA count is stored in pdd.
126 * To get the current activity counters for all active SDMA queues,
127 * we loop over all SDMA queues and get their counts from user-space.
128 *
129 * We cannot call get_user() with dqm_lock held as it can cause
130 * a circular lock dependency situation. To read the SDMA stats,
131 * we need to do the following:
132 *
133 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
134 * with dqm_lock/dqm_unlock().
135 * 2. Call get_user() for each node in temporary list without dqm_lock.
136 * Save the SDMA count for each node and also add the count to the total
137 * SDMA count counter.
138 * Its possible, during this step, a few SDMA queue nodes got deleted
139 * from the qpd->queues_list.
140 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
141 * If any node got deleted, its SDMA count would be captured in the sdma
142 * past activity counter. So subtract the SDMA counter stored in step 2
143 * for this node from the total SDMA count.
144 */
145 INIT_LIST_HEAD(&sdma_q_list.list);
146
147 /*
148 * Create the temp list of all SDMA queues
149 */
150 dqm_lock(dqm);
151
152 list_for_each_entry(q, &qpd->queues_list, list) {
153 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
154 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
155 continue;
156
157 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
158 if (!sdma_q) {
159 dqm_unlock(dqm);
160 goto cleanup;
161 }
162
163 INIT_LIST_HEAD(&sdma_q->list);
164 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
165 sdma_q->queue_id = q->properties.queue_id;
166 list_add_tail(&sdma_q->list, &sdma_q_list.list);
167 }
168
169 /*
170 * If the temp list is empty, then no SDMA queues nodes were found in
171 * qpd->queues_list. Return the past activity count as the total sdma
172 * count
173 */
174 if (list_empty(&sdma_q_list.list)) {
175 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
176 dqm_unlock(dqm);
177 return;
178 }
179
180 dqm_unlock(dqm);
181
182 /*
183 * Get the usage count for each SDMA queue in temp_list.
184 */
185 mm = get_task_mm(pdd->process->lead_thread);
186 if (!mm)
187 goto cleanup;
188
189 kthread_use_mm(mm);
190
191 list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
192 val = 0;
193 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
194 if (ret) {
195 pr_debug("Failed to read SDMA queue active counter for queue id: %d",
196 sdma_q->queue_id);
197 } else {
198 sdma_q->sdma_val = val;
199 workarea->sdma_activity_counter += val;
200 }
201 }
202
203 kthread_unuse_mm(mm);
204 mmput(mm);
205
206 /*
207 * Do a second iteration over qpd_queues_list to check if any SDMA
208 * nodes got deleted while fetching SDMA counter.
209 */
210 dqm_lock(dqm);
211
212 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
213
214 list_for_each_entry(q, &qpd->queues_list, list) {
215 if (list_empty(&sdma_q_list.list))
216 break;
217
218 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
219 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
220 continue;
221
222 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
223 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
224 (sdma_q->queue_id == q->properties.queue_id)) {
225 list_del(&sdma_q->list);
226 kfree(sdma_q);
227 break;
228 }
229 }
230 }
231
232 dqm_unlock(dqm);
233
234 /*
235 * If temp list is not empty, it implies some queues got deleted
236 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
237 * count for each node from the total SDMA count.
238 */
239 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
240 workarea->sdma_activity_counter -= sdma_q->sdma_val;
241 list_del(&sdma_q->list);
242 kfree(sdma_q);
243 }
244
245 return;
246
247 cleanup:
248 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
249 list_del(&sdma_q->list);
250 kfree(sdma_q);
251 }
252 }
253
254 /**
255 * kfd_get_cu_occupancy - Collect number of waves in-flight on this device
256 * by current process. Translates acquired wave count into number of compute units
257 * that are occupied.
258 *
259 * @attr: Handle of attribute that allows reporting of wave count. The attribute
260 * handle encapsulates GPU device it is associated with, thereby allowing collection
261 * of waves in flight, etc
262 * @buffer: Handle of user provided buffer updated with wave count
263 *
264 * Return: Number of bytes written to user buffer or an error value
265 */
kfd_get_cu_occupancy(struct attribute * attr,char * buffer)266 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
267 {
268 int cu_cnt;
269 int wave_cnt;
270 int max_waves_per_cu;
271 struct kfd_node *dev = NULL;
272 struct kfd_process *proc = NULL;
273 struct kfd_process_device *pdd = NULL;
274
275 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
276 dev = pdd->dev;
277 if (dev->kfd2kgd->get_cu_occupancy == NULL)
278 return -EINVAL;
279
280 cu_cnt = 0;
281 proc = pdd->process;
282 if (pdd->qpd.queue_count == 0) {
283 pr_debug("Gpu-Id: %d has no active queues for process %d\n",
284 dev->id, proc->pasid);
285 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
286 }
287
288 /* Collect wave count from device if it supports */
289 wave_cnt = 0;
290 max_waves_per_cu = 0;
291 dev->kfd2kgd->get_cu_occupancy(dev->adev, proc->pasid, &wave_cnt,
292 &max_waves_per_cu, 0);
293
294 /* Translate wave count to number of compute units */
295 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
296 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
297 }
298
kfd_procfs_show(struct kobject * kobj,struct attribute * attr,char * buffer)299 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
300 char *buffer)
301 {
302 if (strcmp(attr->name, "pasid") == 0) {
303 struct kfd_process *p = container_of(attr, struct kfd_process,
304 attr_pasid);
305
306 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
307 } else if (strncmp(attr->name, "vram_", 5) == 0) {
308 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
309 attr_vram);
310 return snprintf(buffer, PAGE_SIZE, "%llu\n", atomic64_read(&pdd->vram_usage));
311 } else if (strncmp(attr->name, "sdma_", 5) == 0) {
312 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
313 attr_sdma);
314 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
315
316 INIT_WORK_ONSTACK(&sdma_activity_work_handler.sdma_activity_work,
317 kfd_sdma_activity_worker);
318
319 sdma_activity_work_handler.pdd = pdd;
320 sdma_activity_work_handler.sdma_activity_counter = 0;
321
322 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
323
324 flush_work(&sdma_activity_work_handler.sdma_activity_work);
325 destroy_work_on_stack(&sdma_activity_work_handler.sdma_activity_work);
326
327 return snprintf(buffer, PAGE_SIZE, "%llu\n",
328 (sdma_activity_work_handler.sdma_activity_counter)/
329 SDMA_ACTIVITY_DIVISOR);
330 } else {
331 pr_err("Invalid attribute");
332 return -EINVAL;
333 }
334
335 return 0;
336 }
337
kfd_procfs_kobj_release(struct kobject * kobj)338 static void kfd_procfs_kobj_release(struct kobject *kobj)
339 {
340 kfree(kobj);
341 }
342
343 static const struct sysfs_ops kfd_procfs_ops = {
344 .show = kfd_procfs_show,
345 };
346
347 static const struct kobj_type procfs_type = {
348 .release = kfd_procfs_kobj_release,
349 .sysfs_ops = &kfd_procfs_ops,
350 };
351
kfd_procfs_init(void)352 void kfd_procfs_init(void)
353 {
354 int ret = 0;
355
356 procfs.kobj = kfd_alloc_struct(procfs.kobj);
357 if (!procfs.kobj)
358 return;
359
360 ret = kobject_init_and_add(procfs.kobj, &procfs_type,
361 &kfd_device->kobj, "proc");
362 if (ret) {
363 pr_warn("Could not create procfs proc folder");
364 /* If we fail to create the procfs, clean up */
365 kfd_procfs_shutdown();
366 }
367 }
368
kfd_procfs_shutdown(void)369 void kfd_procfs_shutdown(void)
370 {
371 if (procfs.kobj) {
372 kobject_del(procfs.kobj);
373 kobject_put(procfs.kobj);
374 procfs.kobj = NULL;
375 }
376 }
377
kfd_procfs_queue_show(struct kobject * kobj,struct attribute * attr,char * buffer)378 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
379 struct attribute *attr, char *buffer)
380 {
381 struct queue *q = container_of(kobj, struct queue, kobj);
382
383 if (!strcmp(attr->name, "size"))
384 return snprintf(buffer, PAGE_SIZE, "%llu",
385 q->properties.queue_size);
386 else if (!strcmp(attr->name, "type"))
387 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
388 else if (!strcmp(attr->name, "gpuid"))
389 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
390 else
391 pr_err("Invalid attribute");
392
393 return 0;
394 }
395
kfd_procfs_stats_show(struct kobject * kobj,struct attribute * attr,char * buffer)396 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
397 struct attribute *attr, char *buffer)
398 {
399 if (strcmp(attr->name, "evicted_ms") == 0) {
400 struct kfd_process_device *pdd = container_of(attr,
401 struct kfd_process_device,
402 attr_evict);
403 uint64_t evict_jiffies;
404
405 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
406
407 return snprintf(buffer,
408 PAGE_SIZE,
409 "%llu\n",
410 jiffies64_to_msecs(evict_jiffies));
411
412 /* Sysfs handle that gets CU occupancy is per device */
413 } else if (strcmp(attr->name, "cu_occupancy") == 0) {
414 return kfd_get_cu_occupancy(attr, buffer);
415 } else {
416 pr_err("Invalid attribute");
417 }
418
419 return 0;
420 }
421
kfd_sysfs_counters_show(struct kobject * kobj,struct attribute * attr,char * buf)422 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
423 struct attribute *attr, char *buf)
424 {
425 struct kfd_process_device *pdd;
426
427 if (!strcmp(attr->name, "faults")) {
428 pdd = container_of(attr, struct kfd_process_device,
429 attr_faults);
430 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
431 }
432 if (!strcmp(attr->name, "page_in")) {
433 pdd = container_of(attr, struct kfd_process_device,
434 attr_page_in);
435 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
436 }
437 if (!strcmp(attr->name, "page_out")) {
438 pdd = container_of(attr, struct kfd_process_device,
439 attr_page_out);
440 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
441 }
442 return 0;
443 }
444
445 static struct attribute attr_queue_size = {
446 .name = "size",
447 .mode = KFD_SYSFS_FILE_MODE
448 };
449
450 static struct attribute attr_queue_type = {
451 .name = "type",
452 .mode = KFD_SYSFS_FILE_MODE
453 };
454
455 static struct attribute attr_queue_gpuid = {
456 .name = "gpuid",
457 .mode = KFD_SYSFS_FILE_MODE
458 };
459
460 static struct attribute *procfs_queue_attrs[] = {
461 &attr_queue_size,
462 &attr_queue_type,
463 &attr_queue_gpuid,
464 NULL
465 };
466 ATTRIBUTE_GROUPS(procfs_queue);
467
468 static const struct sysfs_ops procfs_queue_ops = {
469 .show = kfd_procfs_queue_show,
470 };
471
472 static const struct kobj_type procfs_queue_type = {
473 .sysfs_ops = &procfs_queue_ops,
474 .default_groups = procfs_queue_groups,
475 };
476
477 static const struct sysfs_ops procfs_stats_ops = {
478 .show = kfd_procfs_stats_show,
479 };
480
481 static const struct kobj_type procfs_stats_type = {
482 .sysfs_ops = &procfs_stats_ops,
483 .release = kfd_procfs_kobj_release,
484 };
485
486 static const struct sysfs_ops sysfs_counters_ops = {
487 .show = kfd_sysfs_counters_show,
488 };
489
490 static const struct kobj_type sysfs_counters_type = {
491 .sysfs_ops = &sysfs_counters_ops,
492 .release = kfd_procfs_kobj_release,
493 };
494
kfd_procfs_add_queue(struct queue * q)495 int kfd_procfs_add_queue(struct queue *q)
496 {
497 struct kfd_process *proc;
498 int ret;
499
500 if (!q || !q->process)
501 return -EINVAL;
502 proc = q->process;
503
504 /* Create proc/<pid>/queues/<queue id> folder */
505 if (!proc->kobj_queues)
506 return -EFAULT;
507 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
508 proc->kobj_queues, "%u", q->properties.queue_id);
509 if (ret < 0) {
510 pr_warn("Creating proc/<pid>/queues/%u failed",
511 q->properties.queue_id);
512 kobject_put(&q->kobj);
513 return ret;
514 }
515
516 return 0;
517 }
518
kfd_sysfs_create_file(struct kobject * kobj,struct attribute * attr,char * name)519 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
520 char *name)
521 {
522 int ret;
523
524 if (!kobj || !attr || !name)
525 return;
526
527 attr->name = name;
528 attr->mode = KFD_SYSFS_FILE_MODE;
529 sysfs_attr_init(attr);
530
531 ret = sysfs_create_file(kobj, attr);
532 if (ret)
533 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
534 }
535
kfd_procfs_add_sysfs_stats(struct kfd_process * p)536 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
537 {
538 int ret;
539 int i;
540 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
541
542 if (!p || !p->kobj)
543 return;
544
545 /*
546 * Create sysfs files for each GPU:
547 * - proc/<pid>/stats_<gpuid>/
548 * - proc/<pid>/stats_<gpuid>/evicted_ms
549 * - proc/<pid>/stats_<gpuid>/cu_occupancy
550 */
551 for (i = 0; i < p->n_pdds; i++) {
552 struct kfd_process_device *pdd = p->pdds[i];
553
554 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
555 "stats_%u", pdd->dev->id);
556 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
557 if (!pdd->kobj_stats)
558 return;
559
560 ret = kobject_init_and_add(pdd->kobj_stats,
561 &procfs_stats_type,
562 p->kobj,
563 stats_dir_filename);
564
565 if (ret) {
566 pr_warn("Creating KFD proc/stats_%s folder failed",
567 stats_dir_filename);
568 kobject_put(pdd->kobj_stats);
569 pdd->kobj_stats = NULL;
570 return;
571 }
572
573 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
574 "evicted_ms");
575 /* Add sysfs file to report compute unit occupancy */
576 if (pdd->dev->kfd2kgd->get_cu_occupancy)
577 kfd_sysfs_create_file(pdd->kobj_stats,
578 &pdd->attr_cu_occupancy,
579 "cu_occupancy");
580 }
581 }
582
kfd_procfs_add_sysfs_counters(struct kfd_process * p)583 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
584 {
585 int ret = 0;
586 int i;
587 char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
588
589 if (!p || !p->kobj)
590 return;
591
592 /*
593 * Create sysfs files for each GPU which supports SVM
594 * - proc/<pid>/counters_<gpuid>/
595 * - proc/<pid>/counters_<gpuid>/faults
596 * - proc/<pid>/counters_<gpuid>/page_in
597 * - proc/<pid>/counters_<gpuid>/page_out
598 */
599 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
600 struct kfd_process_device *pdd = p->pdds[i];
601 struct kobject *kobj_counters;
602
603 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
604 "counters_%u", pdd->dev->id);
605 kobj_counters = kfd_alloc_struct(kobj_counters);
606 if (!kobj_counters)
607 return;
608
609 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
610 p->kobj, counters_dir_filename);
611 if (ret) {
612 pr_warn("Creating KFD proc/%s folder failed",
613 counters_dir_filename);
614 kobject_put(kobj_counters);
615 return;
616 }
617
618 pdd->kobj_counters = kobj_counters;
619 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
620 "faults");
621 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
622 "page_in");
623 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
624 "page_out");
625 }
626 }
627
kfd_procfs_add_sysfs_files(struct kfd_process * p)628 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
629 {
630 int i;
631
632 if (!p || !p->kobj)
633 return;
634
635 /*
636 * Create sysfs files for each GPU:
637 * - proc/<pid>/vram_<gpuid>
638 * - proc/<pid>/sdma_<gpuid>
639 */
640 for (i = 0; i < p->n_pdds; i++) {
641 struct kfd_process_device *pdd = p->pdds[i];
642
643 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
644 pdd->dev->id);
645 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
646 pdd->vram_filename);
647
648 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
649 pdd->dev->id);
650 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
651 pdd->sdma_filename);
652 }
653 }
654
kfd_procfs_del_queue(struct queue * q)655 void kfd_procfs_del_queue(struct queue *q)
656 {
657 if (!q)
658 return;
659
660 kobject_del(&q->kobj);
661 kobject_put(&q->kobj);
662 }
663
kfd_process_create_wq(void)664 int kfd_process_create_wq(void)
665 {
666 if (!kfd_process_wq)
667 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
668 if (!kfd_restore_wq)
669 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
670
671 if (!kfd_process_wq || !kfd_restore_wq) {
672 kfd_process_destroy_wq();
673 return -ENOMEM;
674 }
675
676 return 0;
677 }
678
kfd_process_destroy_wq(void)679 void kfd_process_destroy_wq(void)
680 {
681 if (kfd_process_wq) {
682 destroy_workqueue(kfd_process_wq);
683 kfd_process_wq = NULL;
684 }
685 if (kfd_restore_wq) {
686 destroy_workqueue(kfd_restore_wq);
687 kfd_restore_wq = NULL;
688 }
689 }
690
kfd_process_free_gpuvm(struct kgd_mem * mem,struct kfd_process_device * pdd,void ** kptr)691 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
692 struct kfd_process_device *pdd, void **kptr)
693 {
694 struct kfd_node *dev = pdd->dev;
695
696 if (kptr && *kptr) {
697 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
698 *kptr = NULL;
699 }
700
701 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->adev, mem, pdd->drm_priv);
702 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, mem, pdd->drm_priv,
703 NULL);
704 }
705
706 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
707 * This function should be only called right after the process
708 * is created and when kfd_processes_mutex is still being held
709 * to avoid concurrency. Because of that exclusiveness, we do
710 * not need to take p->mutex.
711 */
kfd_process_alloc_gpuvm(struct kfd_process_device * pdd,uint64_t gpu_va,uint32_t size,uint32_t flags,struct kgd_mem ** mem,void ** kptr)712 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
713 uint64_t gpu_va, uint32_t size,
714 uint32_t flags, struct kgd_mem **mem, void **kptr)
715 {
716 struct kfd_node *kdev = pdd->dev;
717 int err;
718
719 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->adev, gpu_va, size,
720 pdd->drm_priv, mem, NULL,
721 flags, false);
722 if (err)
723 goto err_alloc_mem;
724
725 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->adev, *mem,
726 pdd->drm_priv);
727 if (err)
728 goto err_map_mem;
729
730 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->adev, *mem, true);
731 if (err) {
732 pr_debug("Sync memory failed, wait interrupted by user signal\n");
733 goto sync_memory_failed;
734 }
735
736 if (kptr) {
737 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(
738 (struct kgd_mem *)*mem, kptr, NULL);
739 if (err) {
740 pr_debug("Map GTT BO to kernel failed\n");
741 goto sync_memory_failed;
742 }
743 }
744
745 return err;
746
747 sync_memory_failed:
748 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(kdev->adev, *mem, pdd->drm_priv);
749
750 err_map_mem:
751 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->adev, *mem, pdd->drm_priv,
752 NULL);
753 err_alloc_mem:
754 *mem = NULL;
755 *kptr = NULL;
756 return err;
757 }
758
759 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
760 * process for IB usage The memory reserved is for KFD to submit
761 * IB to AMDGPU from kernel. If the memory is reserved
762 * successfully, ib_kaddr will have the CPU/kernel
763 * address. Check ib_kaddr before accessing the memory.
764 */
kfd_process_device_reserve_ib_mem(struct kfd_process_device * pdd)765 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
766 {
767 struct qcm_process_device *qpd = &pdd->qpd;
768 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
769 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
770 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
771 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
772 struct kgd_mem *mem;
773 void *kaddr;
774 int ret;
775
776 if (qpd->ib_kaddr || !qpd->ib_base)
777 return 0;
778
779 /* ib_base is only set for dGPU */
780 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
781 &mem, &kaddr);
782 if (ret)
783 return ret;
784
785 qpd->ib_mem = mem;
786 qpd->ib_kaddr = kaddr;
787
788 return 0;
789 }
790
kfd_process_device_destroy_ib_mem(struct kfd_process_device * pdd)791 static void kfd_process_device_destroy_ib_mem(struct kfd_process_device *pdd)
792 {
793 struct qcm_process_device *qpd = &pdd->qpd;
794
795 if (!qpd->ib_kaddr || !qpd->ib_base)
796 return;
797
798 kfd_process_free_gpuvm(qpd->ib_mem, pdd, &qpd->ib_kaddr);
799 }
800
kfd_create_process(struct task_struct * thread)801 struct kfd_process *kfd_create_process(struct task_struct *thread)
802 {
803 struct kfd_process *process;
804 int ret;
805
806 if (!(thread->mm && mmget_not_zero(thread->mm)))
807 return ERR_PTR(-EINVAL);
808
809 /* Only the pthreads threading model is supported. */
810 if (thread->group_leader->mm != thread->mm) {
811 mmput(thread->mm);
812 return ERR_PTR(-EINVAL);
813 }
814
815 /* If the process just called exec(3), it is possible that the
816 * cleanup of the kfd_process (following the release of the mm
817 * of the old process image) is still in the cleanup work queue.
818 * Make sure to drain any job before trying to recreate any
819 * resource for this process.
820 */
821 flush_workqueue(kfd_process_wq);
822
823 /*
824 * take kfd processes mutex before starting of process creation
825 * so there won't be a case where two threads of the same process
826 * create two kfd_process structures
827 */
828 mutex_lock(&kfd_processes_mutex);
829
830 if (kfd_is_locked()) {
831 pr_debug("KFD is locked! Cannot create process");
832 process = ERR_PTR(-EINVAL);
833 goto out;
834 }
835
836 /* A prior open of /dev/kfd could have already created the process. */
837 process = find_process(thread, false);
838 if (process) {
839 pr_debug("Process already found\n");
840 } else {
841 process = create_process(thread);
842 if (IS_ERR(process))
843 goto out;
844
845 if (!procfs.kobj)
846 goto out;
847
848 process->kobj = kfd_alloc_struct(process->kobj);
849 if (!process->kobj) {
850 pr_warn("Creating procfs kobject failed");
851 goto out;
852 }
853 ret = kobject_init_and_add(process->kobj, &procfs_type,
854 procfs.kobj, "%d",
855 (int)process->lead_thread->pid);
856 if (ret) {
857 pr_warn("Creating procfs pid directory failed");
858 kobject_put(process->kobj);
859 goto out;
860 }
861
862 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
863 "pasid");
864
865 process->kobj_queues = kobject_create_and_add("queues",
866 process->kobj);
867 if (!process->kobj_queues)
868 pr_warn("Creating KFD proc/queues folder failed");
869
870 kfd_procfs_add_sysfs_stats(process);
871 kfd_procfs_add_sysfs_files(process);
872 kfd_procfs_add_sysfs_counters(process);
873
874 init_waitqueue_head(&process->wait_irq_drain);
875 }
876 out:
877 if (!IS_ERR(process))
878 kref_get(&process->ref);
879 mutex_unlock(&kfd_processes_mutex);
880 mmput(thread->mm);
881
882 return process;
883 }
884
kfd_get_process(const struct task_struct * thread)885 struct kfd_process *kfd_get_process(const struct task_struct *thread)
886 {
887 struct kfd_process *process;
888
889 if (!thread->mm)
890 return ERR_PTR(-EINVAL);
891
892 /* Only the pthreads threading model is supported. */
893 if (thread->group_leader->mm != thread->mm)
894 return ERR_PTR(-EINVAL);
895
896 process = find_process(thread, false);
897 if (!process)
898 return ERR_PTR(-EINVAL);
899
900 return process;
901 }
902
find_process_by_mm(const struct mm_struct * mm)903 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
904 {
905 struct kfd_process *process;
906
907 hash_for_each_possible_rcu(kfd_processes_table, process,
908 kfd_processes, (uintptr_t)mm)
909 if (process->mm == mm)
910 return process;
911
912 return NULL;
913 }
914
find_process(const struct task_struct * thread,bool ref)915 static struct kfd_process *find_process(const struct task_struct *thread,
916 bool ref)
917 {
918 struct kfd_process *p;
919 int idx;
920
921 idx = srcu_read_lock(&kfd_processes_srcu);
922 p = find_process_by_mm(thread->mm);
923 if (p && ref)
924 kref_get(&p->ref);
925 srcu_read_unlock(&kfd_processes_srcu, idx);
926
927 return p;
928 }
929
kfd_unref_process(struct kfd_process * p)930 void kfd_unref_process(struct kfd_process *p)
931 {
932 kref_put(&p->ref, kfd_process_ref_release);
933 }
934
935 /* This increments the process->ref counter. */
kfd_lookup_process_by_pid(struct pid * pid)936 struct kfd_process *kfd_lookup_process_by_pid(struct pid *pid)
937 {
938 struct task_struct *task = NULL;
939 struct kfd_process *p = NULL;
940
941 if (!pid) {
942 task = current;
943 get_task_struct(task);
944 } else {
945 task = get_pid_task(pid, PIDTYPE_PID);
946 }
947
948 if (task) {
949 p = find_process(task, true);
950 put_task_struct(task);
951 }
952
953 return p;
954 }
955
kfd_process_device_free_bos(struct kfd_process_device * pdd)956 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
957 {
958 struct kfd_process *p = pdd->process;
959 void *mem;
960 int id;
961 int i;
962
963 /*
964 * Remove all handles from idr and release appropriate
965 * local memory object
966 */
967 idr_for_each_entry(&pdd->alloc_idr, mem, id) {
968
969 for (i = 0; i < p->n_pdds; i++) {
970 struct kfd_process_device *peer_pdd = p->pdds[i];
971
972 if (!peer_pdd->drm_priv)
973 continue;
974 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
975 peer_pdd->dev->adev, mem, peer_pdd->drm_priv);
976 }
977
978 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, mem,
979 pdd->drm_priv, NULL);
980 kfd_process_device_remove_obj_handle(pdd, id);
981 }
982 }
983
984 /*
985 * Just kunmap and unpin signal BO here. It will be freed in
986 * kfd_process_free_outstanding_kfd_bos()
987 */
kfd_process_kunmap_signal_bo(struct kfd_process * p)988 static void kfd_process_kunmap_signal_bo(struct kfd_process *p)
989 {
990 struct kfd_process_device *pdd;
991 struct kfd_node *kdev;
992 void *mem;
993
994 kdev = kfd_device_by_id(GET_GPU_ID(p->signal_handle));
995 if (!kdev)
996 return;
997
998 mutex_lock(&p->mutex);
999
1000 pdd = kfd_get_process_device_data(kdev, p);
1001 if (!pdd)
1002 goto out;
1003
1004 mem = kfd_process_device_translate_handle(
1005 pdd, GET_IDR_HANDLE(p->signal_handle));
1006 if (!mem)
1007 goto out;
1008
1009 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
1010
1011 out:
1012 mutex_unlock(&p->mutex);
1013 }
1014
kfd_process_free_outstanding_kfd_bos(struct kfd_process * p)1015 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
1016 {
1017 int i;
1018
1019 for (i = 0; i < p->n_pdds; i++)
1020 kfd_process_device_free_bos(p->pdds[i]);
1021 }
1022
kfd_process_destroy_pdds(struct kfd_process * p)1023 static void kfd_process_destroy_pdds(struct kfd_process *p)
1024 {
1025 int i;
1026
1027 for (i = 0; i < p->n_pdds; i++) {
1028 struct kfd_process_device *pdd = p->pdds[i];
1029
1030 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
1031 pdd->dev->id, p->pasid);
1032
1033 kfd_process_device_destroy_cwsr_dgpu(pdd);
1034 kfd_process_device_destroy_ib_mem(pdd);
1035
1036 if (pdd->drm_file) {
1037 amdgpu_amdkfd_gpuvm_release_process_vm(
1038 pdd->dev->adev, pdd->drm_priv);
1039 fput(pdd->drm_file);
1040 }
1041
1042 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
1043 free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
1044 get_order(KFD_CWSR_TBA_TMA_SIZE));
1045
1046 idr_destroy(&pdd->alloc_idr);
1047
1048 kfd_free_process_doorbells(pdd->dev->kfd, pdd);
1049
1050 if (pdd->dev->kfd->shared_resources.enable_mes &&
1051 pdd->proc_ctx_cpu_ptr)
1052 amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev,
1053 &pdd->proc_ctx_bo);
1054 /*
1055 * before destroying pdd, make sure to report availability
1056 * for auto suspend
1057 */
1058 if (pdd->runtime_inuse) {
1059 pm_runtime_mark_last_busy(adev_to_drm(pdd->dev->adev)->dev);
1060 pm_runtime_put_autosuspend(adev_to_drm(pdd->dev->adev)->dev);
1061 pdd->runtime_inuse = false;
1062 }
1063
1064 kfree(pdd);
1065 p->pdds[i] = NULL;
1066 }
1067 p->n_pdds = 0;
1068 }
1069
kfd_process_remove_sysfs(struct kfd_process * p)1070 static void kfd_process_remove_sysfs(struct kfd_process *p)
1071 {
1072 struct kfd_process_device *pdd;
1073 int i;
1074
1075 if (!p->kobj)
1076 return;
1077
1078 sysfs_remove_file(p->kobj, &p->attr_pasid);
1079 kobject_del(p->kobj_queues);
1080 kobject_put(p->kobj_queues);
1081 p->kobj_queues = NULL;
1082
1083 for (i = 0; i < p->n_pdds; i++) {
1084 pdd = p->pdds[i];
1085
1086 sysfs_remove_file(p->kobj, &pdd->attr_vram);
1087 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1088
1089 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1090 if (pdd->dev->kfd2kgd->get_cu_occupancy)
1091 sysfs_remove_file(pdd->kobj_stats,
1092 &pdd->attr_cu_occupancy);
1093 kobject_del(pdd->kobj_stats);
1094 kobject_put(pdd->kobj_stats);
1095 pdd->kobj_stats = NULL;
1096 }
1097
1098 for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1099 pdd = p->pdds[i];
1100
1101 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1102 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1103 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1104 kobject_del(pdd->kobj_counters);
1105 kobject_put(pdd->kobj_counters);
1106 pdd->kobj_counters = NULL;
1107 }
1108
1109 kobject_del(p->kobj);
1110 kobject_put(p->kobj);
1111 p->kobj = NULL;
1112 }
1113
1114 /*
1115 * If any GPU is ongoing reset, wait for reset complete.
1116 */
kfd_process_wait_gpu_reset_complete(struct kfd_process * p)1117 static void kfd_process_wait_gpu_reset_complete(struct kfd_process *p)
1118 {
1119 int i;
1120
1121 for (i = 0; i < p->n_pdds; i++)
1122 flush_workqueue(p->pdds[i]->dev->adev->reset_domain->wq);
1123 }
1124
1125 /* No process locking is needed in this function, because the process
1126 * is not findable any more. We must assume that no other thread is
1127 * using it any more, otherwise we couldn't safely free the process
1128 * structure in the end.
1129 */
kfd_process_wq_release(struct work_struct * work)1130 static void kfd_process_wq_release(struct work_struct *work)
1131 {
1132 struct kfd_process *p = container_of(work, struct kfd_process,
1133 release_work);
1134
1135 kfd_process_dequeue_from_all_devices(p);
1136 pqm_uninit(&p->pqm);
1137
1138 /*
1139 * If GPU in reset, user queues may still running, wait for reset complete.
1140 */
1141 kfd_process_wait_gpu_reset_complete(p);
1142
1143 /* Signal the eviction fence after user mode queues are
1144 * destroyed. This allows any BOs to be freed without
1145 * triggering pointless evictions or waiting for fences.
1146 */
1147 dma_fence_signal(p->ef);
1148
1149 kfd_process_remove_sysfs(p);
1150
1151 kfd_process_kunmap_signal_bo(p);
1152 kfd_process_free_outstanding_kfd_bos(p);
1153 svm_range_list_fini(p);
1154
1155 kfd_process_destroy_pdds(p);
1156 dma_fence_put(p->ef);
1157
1158 kfd_event_free_process(p);
1159
1160 kfd_pasid_free(p->pasid);
1161 mutex_destroy(&p->mutex);
1162
1163 put_task_struct(p->lead_thread);
1164
1165 kfree(p);
1166 }
1167
kfd_process_ref_release(struct kref * ref)1168 static void kfd_process_ref_release(struct kref *ref)
1169 {
1170 struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1171
1172 INIT_WORK(&p->release_work, kfd_process_wq_release);
1173 queue_work(kfd_process_wq, &p->release_work);
1174 }
1175
kfd_process_alloc_notifier(struct mm_struct * mm)1176 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1177 {
1178 int idx = srcu_read_lock(&kfd_processes_srcu);
1179 struct kfd_process *p = find_process_by_mm(mm);
1180
1181 srcu_read_unlock(&kfd_processes_srcu, idx);
1182
1183 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1184 }
1185
kfd_process_free_notifier(struct mmu_notifier * mn)1186 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1187 {
1188 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1189 }
1190
kfd_process_notifier_release_internal(struct kfd_process * p)1191 static void kfd_process_notifier_release_internal(struct kfd_process *p)
1192 {
1193 int i;
1194
1195 cancel_delayed_work_sync(&p->eviction_work);
1196 cancel_delayed_work_sync(&p->restore_work);
1197
1198 for (i = 0; i < p->n_pdds; i++) {
1199 struct kfd_process_device *pdd = p->pdds[i];
1200
1201 /* re-enable GFX OFF since runtime enable with ttmp setup disabled it. */
1202 if (!kfd_dbg_is_rlc_restore_supported(pdd->dev) && p->runtime_info.ttmp_setup)
1203 amdgpu_gfx_off_ctrl(pdd->dev->adev, true);
1204 }
1205
1206 /* Indicate to other users that MM is no longer valid */
1207 p->mm = NULL;
1208 kfd_dbg_trap_disable(p);
1209
1210 if (atomic_read(&p->debugged_process_count) > 0) {
1211 struct kfd_process *target;
1212 unsigned int temp;
1213 int idx = srcu_read_lock(&kfd_processes_srcu);
1214
1215 hash_for_each_rcu(kfd_processes_table, temp, target, kfd_processes) {
1216 if (target->debugger_process && target->debugger_process == p) {
1217 mutex_lock_nested(&target->mutex, 1);
1218 kfd_dbg_trap_disable(target);
1219 mutex_unlock(&target->mutex);
1220 if (atomic_read(&p->debugged_process_count) == 0)
1221 break;
1222 }
1223 }
1224
1225 srcu_read_unlock(&kfd_processes_srcu, idx);
1226 }
1227
1228 mmu_notifier_put(&p->mmu_notifier);
1229 }
1230
kfd_process_notifier_release(struct mmu_notifier * mn,struct mm_struct * mm)1231 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1232 struct mm_struct *mm)
1233 {
1234 struct kfd_process *p;
1235
1236 /*
1237 * The kfd_process structure can not be free because the
1238 * mmu_notifier srcu is read locked
1239 */
1240 p = container_of(mn, struct kfd_process, mmu_notifier);
1241 if (WARN_ON(p->mm != mm))
1242 return;
1243
1244 mutex_lock(&kfd_processes_mutex);
1245 /*
1246 * Do early return if table is empty.
1247 *
1248 * This could potentially happen if this function is called concurrently
1249 * by mmu_notifier and by kfd_cleanup_pocesses.
1250 *
1251 */
1252 if (hash_empty(kfd_processes_table)) {
1253 mutex_unlock(&kfd_processes_mutex);
1254 return;
1255 }
1256 hash_del_rcu(&p->kfd_processes);
1257 mutex_unlock(&kfd_processes_mutex);
1258 synchronize_srcu(&kfd_processes_srcu);
1259
1260 kfd_process_notifier_release_internal(p);
1261 }
1262
1263 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1264 .release = kfd_process_notifier_release,
1265 .alloc_notifier = kfd_process_alloc_notifier,
1266 .free_notifier = kfd_process_free_notifier,
1267 };
1268
1269 /*
1270 * This code handles the case when driver is being unloaded before all
1271 * mm_struct are released. We need to safely free the kfd_process and
1272 * avoid race conditions with mmu_notifier that might try to free them.
1273 *
1274 */
kfd_cleanup_processes(void)1275 void kfd_cleanup_processes(void)
1276 {
1277 struct kfd_process *p;
1278 struct hlist_node *p_temp;
1279 unsigned int temp;
1280 HLIST_HEAD(cleanup_list);
1281
1282 /*
1283 * Move all remaining kfd_process from the process table to a
1284 * temp list for processing. Once done, callback from mmu_notifier
1285 * release will not see the kfd_process in the table and do early return,
1286 * avoiding double free issues.
1287 */
1288 mutex_lock(&kfd_processes_mutex);
1289 hash_for_each_safe(kfd_processes_table, temp, p_temp, p, kfd_processes) {
1290 hash_del_rcu(&p->kfd_processes);
1291 synchronize_srcu(&kfd_processes_srcu);
1292 hlist_add_head(&p->kfd_processes, &cleanup_list);
1293 }
1294 mutex_unlock(&kfd_processes_mutex);
1295
1296 hlist_for_each_entry_safe(p, p_temp, &cleanup_list, kfd_processes)
1297 kfd_process_notifier_release_internal(p);
1298
1299 /*
1300 * Ensures that all outstanding free_notifier get called, triggering
1301 * the release of the kfd_process struct.
1302 */
1303 mmu_notifier_synchronize();
1304 }
1305
kfd_process_init_cwsr_apu(struct kfd_process * p,struct file * filep)1306 int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1307 {
1308 unsigned long offset;
1309 int i;
1310
1311 if (p->has_cwsr)
1312 return 0;
1313
1314 for (i = 0; i < p->n_pdds; i++) {
1315 struct kfd_node *dev = p->pdds[i]->dev;
1316 struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1317
1318 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1319 continue;
1320
1321 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1322 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1323 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1324 MAP_SHARED, offset);
1325
1326 if (IS_ERR_VALUE(qpd->tba_addr)) {
1327 int err = qpd->tba_addr;
1328
1329 dev_err(dev->adev->dev,
1330 "Failure to set tba address. error %d.\n", err);
1331 qpd->tba_addr = 0;
1332 qpd->cwsr_kaddr = NULL;
1333 return err;
1334 }
1335
1336 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1337
1338 kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
1339
1340 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1341 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1342 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1343 }
1344
1345 p->has_cwsr = true;
1346
1347 return 0;
1348 }
1349
kfd_process_device_init_cwsr_dgpu(struct kfd_process_device * pdd)1350 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1351 {
1352 struct kfd_node *dev = pdd->dev;
1353 struct qcm_process_device *qpd = &pdd->qpd;
1354 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1355 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1356 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1357 struct kgd_mem *mem;
1358 void *kaddr;
1359 int ret;
1360
1361 if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1362 return 0;
1363
1364 /* cwsr_base is only set for dGPU */
1365 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1366 KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr);
1367 if (ret)
1368 return ret;
1369
1370 qpd->cwsr_mem = mem;
1371 qpd->cwsr_kaddr = kaddr;
1372 qpd->tba_addr = qpd->cwsr_base;
1373
1374 memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
1375
1376 kfd_process_set_trap_debug_flag(&pdd->qpd,
1377 pdd->process->debug_trap_enabled);
1378
1379 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1380 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1381 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1382
1383 return 0;
1384 }
1385
kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device * pdd)1386 static void kfd_process_device_destroy_cwsr_dgpu(struct kfd_process_device *pdd)
1387 {
1388 struct kfd_node *dev = pdd->dev;
1389 struct qcm_process_device *qpd = &pdd->qpd;
1390
1391 if (!dev->kfd->cwsr_enabled || !qpd->cwsr_kaddr || !qpd->cwsr_base)
1392 return;
1393
1394 kfd_process_free_gpuvm(qpd->cwsr_mem, pdd, &qpd->cwsr_kaddr);
1395 }
1396
kfd_process_set_trap_handler(struct qcm_process_device * qpd,uint64_t tba_addr,uint64_t tma_addr)1397 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1398 uint64_t tba_addr,
1399 uint64_t tma_addr)
1400 {
1401 if (qpd->cwsr_kaddr) {
1402 /* KFD trap handler is bound, record as second-level TBA/TMA
1403 * in first-level TMA. First-level trap will jump to second.
1404 */
1405 uint64_t *tma =
1406 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1407 tma[0] = tba_addr;
1408 tma[1] = tma_addr;
1409 } else {
1410 /* No trap handler bound, bind as first-level TBA/TMA. */
1411 qpd->tba_addr = tba_addr;
1412 qpd->tma_addr = tma_addr;
1413 }
1414 }
1415
kfd_process_xnack_mode(struct kfd_process * p,bool supported)1416 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1417 {
1418 int i;
1419
1420 /* On most GFXv9 GPUs, the retry mode in the SQ must match the
1421 * boot time retry setting. Mixing processes with different
1422 * XNACK/retry settings can hang the GPU.
1423 *
1424 * Different GPUs can have different noretry settings depending
1425 * on HW bugs or limitations. We need to find at least one
1426 * XNACK mode for this process that's compatible with all GPUs.
1427 * Fortunately GPUs with retry enabled (noretry=0) can run code
1428 * built for XNACK-off. On GFXv9 it may perform slower.
1429 *
1430 * Therefore applications built for XNACK-off can always be
1431 * supported and will be our fallback if any GPU does not
1432 * support retry.
1433 */
1434 for (i = 0; i < p->n_pdds; i++) {
1435 struct kfd_node *dev = p->pdds[i]->dev;
1436
1437 /* Only consider GFXv9 and higher GPUs. Older GPUs don't
1438 * support the SVM APIs and don't need to be considered
1439 * for the XNACK mode selection.
1440 */
1441 if (!KFD_IS_SOC15(dev))
1442 continue;
1443 /* Aldebaran can always support XNACK because it can support
1444 * per-process XNACK mode selection. But let the dev->noretry
1445 * setting still influence the default XNACK mode.
1446 */
1447 if (supported && KFD_SUPPORT_XNACK_PER_PROCESS(dev))
1448 continue;
1449
1450 /* GFXv10 and later GPUs do not support shader preemption
1451 * during page faults. This can lead to poor QoS for queue
1452 * management and memory-manager-related preemptions or
1453 * even deadlocks.
1454 */
1455 if (KFD_GC_VERSION(dev) >= IP_VERSION(10, 1, 1))
1456 return false;
1457
1458 if (dev->kfd->noretry)
1459 return false;
1460 }
1461
1462 return true;
1463 }
1464
kfd_process_set_trap_debug_flag(struct qcm_process_device * qpd,bool enabled)1465 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
1466 bool enabled)
1467 {
1468 if (qpd->cwsr_kaddr) {
1469 uint64_t *tma =
1470 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1471 tma[2] = enabled;
1472 }
1473 }
1474
1475 /*
1476 * On return the kfd_process is fully operational and will be freed when the
1477 * mm is released
1478 */
create_process(const struct task_struct * thread)1479 static struct kfd_process *create_process(const struct task_struct *thread)
1480 {
1481 struct kfd_process *process;
1482 struct mmu_notifier *mn;
1483 int err = -ENOMEM;
1484
1485 process = kzalloc(sizeof(*process), GFP_KERNEL);
1486 if (!process)
1487 goto err_alloc_process;
1488
1489 kref_init(&process->ref);
1490 mutex_init(&process->mutex);
1491 process->mm = thread->mm;
1492 process->lead_thread = thread->group_leader;
1493 process->n_pdds = 0;
1494 process->queues_paused = false;
1495 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1496 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1497 process->last_restore_timestamp = get_jiffies_64();
1498 err = kfd_event_init_process(process);
1499 if (err)
1500 goto err_event_init;
1501 process->is_32bit_user_mode = in_compat_syscall();
1502 process->debug_trap_enabled = false;
1503 process->debugger_process = NULL;
1504 process->exception_enable_mask = 0;
1505 atomic_set(&process->debugged_process_count, 0);
1506 sema_init(&process->runtime_enable_sema, 0);
1507
1508 process->pasid = kfd_pasid_alloc();
1509 if (process->pasid == 0) {
1510 err = -ENOSPC;
1511 goto err_alloc_pasid;
1512 }
1513
1514 err = pqm_init(&process->pqm, process);
1515 if (err != 0)
1516 goto err_process_pqm_init;
1517
1518 /* init process apertures*/
1519 err = kfd_init_apertures(process);
1520 if (err != 0)
1521 goto err_init_apertures;
1522
1523 /* Check XNACK support after PDDs are created in kfd_init_apertures */
1524 process->xnack_enabled = kfd_process_xnack_mode(process, false);
1525
1526 err = svm_range_list_init(process);
1527 if (err)
1528 goto err_init_svm_range_list;
1529
1530 /* alloc_notifier needs to find the process in the hash table */
1531 hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1532 (uintptr_t)process->mm);
1533
1534 /* Avoid free_notifier to start kfd_process_wq_release if
1535 * mmu_notifier_get failed because of pending signal.
1536 */
1537 kref_get(&process->ref);
1538
1539 /* MMU notifier registration must be the last call that can fail
1540 * because after this point we cannot unwind the process creation.
1541 * After this point, mmu_notifier_put will trigger the cleanup by
1542 * dropping the last process reference in the free_notifier.
1543 */
1544 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1545 if (IS_ERR(mn)) {
1546 err = PTR_ERR(mn);
1547 goto err_register_notifier;
1548 }
1549 BUG_ON(mn != &process->mmu_notifier);
1550
1551 kfd_unref_process(process);
1552 get_task_struct(process->lead_thread);
1553
1554 INIT_WORK(&process->debug_event_workarea, debug_event_write_work_handler);
1555
1556 return process;
1557
1558 err_register_notifier:
1559 hash_del_rcu(&process->kfd_processes);
1560 svm_range_list_fini(process);
1561 err_init_svm_range_list:
1562 kfd_process_free_outstanding_kfd_bos(process);
1563 kfd_process_destroy_pdds(process);
1564 err_init_apertures:
1565 pqm_uninit(&process->pqm);
1566 err_process_pqm_init:
1567 kfd_pasid_free(process->pasid);
1568 err_alloc_pasid:
1569 kfd_event_free_process(process);
1570 err_event_init:
1571 mutex_destroy(&process->mutex);
1572 kfree(process);
1573 err_alloc_process:
1574 return ERR_PTR(err);
1575 }
1576
kfd_get_process_device_data(struct kfd_node * dev,struct kfd_process * p)1577 struct kfd_process_device *kfd_get_process_device_data(struct kfd_node *dev,
1578 struct kfd_process *p)
1579 {
1580 int i;
1581
1582 for (i = 0; i < p->n_pdds; i++)
1583 if (p->pdds[i]->dev == dev)
1584 return p->pdds[i];
1585
1586 return NULL;
1587 }
1588
kfd_create_process_device_data(struct kfd_node * dev,struct kfd_process * p)1589 struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
1590 struct kfd_process *p)
1591 {
1592 struct kfd_process_device *pdd = NULL;
1593
1594 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1595 return NULL;
1596 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1597 if (!pdd)
1598 return NULL;
1599
1600 pdd->dev = dev;
1601 INIT_LIST_HEAD(&pdd->qpd.queues_list);
1602 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1603 pdd->qpd.dqm = dev->dqm;
1604 pdd->qpd.pqm = &p->pqm;
1605 pdd->qpd.evicted = 0;
1606 pdd->qpd.mapped_gws_queue = false;
1607 pdd->process = p;
1608 pdd->bound = PDD_UNBOUND;
1609 pdd->already_dequeued = false;
1610 pdd->runtime_inuse = false;
1611 atomic64_set(&pdd->vram_usage, 0);
1612 pdd->sdma_past_activity_counter = 0;
1613 pdd->user_gpu_id = dev->id;
1614 atomic64_set(&pdd->evict_duration_counter, 0);
1615
1616 p->pdds[p->n_pdds++] = pdd;
1617 if (kfd_dbg_is_per_vmid_supported(pdd->dev))
1618 pdd->spi_dbg_override = pdd->dev->kfd2kgd->disable_debug_trap(
1619 pdd->dev->adev,
1620 false,
1621 0);
1622
1623 /* Init idr used for memory handle translation */
1624 idr_init(&pdd->alloc_idr);
1625
1626 return pdd;
1627 }
1628
1629 /**
1630 * kfd_process_device_init_vm - Initialize a VM for a process-device
1631 *
1632 * @pdd: The process-device
1633 * @drm_file: Optional pointer to a DRM file descriptor
1634 *
1635 * If @drm_file is specified, it will be used to acquire the VM from
1636 * that file descriptor. If successful, the @pdd takes ownership of
1637 * the file descriptor.
1638 *
1639 * If @drm_file is NULL, a new VM is created.
1640 *
1641 * Returns 0 on success, -errno on failure.
1642 */
kfd_process_device_init_vm(struct kfd_process_device * pdd,struct file * drm_file)1643 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1644 struct file *drm_file)
1645 {
1646 struct amdgpu_fpriv *drv_priv;
1647 struct amdgpu_vm *avm;
1648 struct kfd_process *p;
1649 struct kfd_node *dev;
1650 int ret;
1651
1652 if (!drm_file)
1653 return -EINVAL;
1654
1655 if (pdd->drm_priv)
1656 return -EBUSY;
1657
1658 ret = amdgpu_file_to_fpriv(drm_file, &drv_priv);
1659 if (ret)
1660 return ret;
1661 avm = &drv_priv->vm;
1662
1663 p = pdd->process;
1664 dev = pdd->dev;
1665
1666 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(dev->adev, avm,
1667 &p->kgd_process_info,
1668 &p->ef);
1669 if (ret) {
1670 dev_err(dev->adev->dev, "Failed to create process VM object\n");
1671 return ret;
1672 }
1673 pdd->drm_priv = drm_file->private_data;
1674 atomic64_set(&pdd->tlb_seq, 0);
1675
1676 ret = kfd_process_device_reserve_ib_mem(pdd);
1677 if (ret)
1678 goto err_reserve_ib_mem;
1679 ret = kfd_process_device_init_cwsr_dgpu(pdd);
1680 if (ret)
1681 goto err_init_cwsr;
1682
1683 ret = amdgpu_amdkfd_gpuvm_set_vm_pasid(dev->adev, avm, p->pasid);
1684 if (ret)
1685 goto err_set_pasid;
1686
1687 pdd->drm_file = drm_file;
1688
1689 return 0;
1690
1691 err_set_pasid:
1692 kfd_process_device_destroy_cwsr_dgpu(pdd);
1693 err_init_cwsr:
1694 kfd_process_device_destroy_ib_mem(pdd);
1695 err_reserve_ib_mem:
1696 pdd->drm_priv = NULL;
1697 amdgpu_amdkfd_gpuvm_destroy_cb(dev->adev, avm);
1698
1699 return ret;
1700 }
1701
1702 /*
1703 * Direct the IOMMU to bind the process (specifically the pasid->mm)
1704 * to the device.
1705 * Unbinding occurs when the process dies or the device is removed.
1706 *
1707 * Assumes that the process lock is held.
1708 */
kfd_bind_process_to_device(struct kfd_node * dev,struct kfd_process * p)1709 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_node *dev,
1710 struct kfd_process *p)
1711 {
1712 struct kfd_process_device *pdd;
1713 int err;
1714
1715 pdd = kfd_get_process_device_data(dev, p);
1716 if (!pdd) {
1717 dev_err(dev->adev->dev, "Process device data doesn't exist\n");
1718 return ERR_PTR(-ENOMEM);
1719 }
1720
1721 if (!pdd->drm_priv)
1722 return ERR_PTR(-ENODEV);
1723
1724 /*
1725 * signal runtime-pm system to auto resume and prevent
1726 * further runtime suspend once device pdd is created until
1727 * pdd is destroyed.
1728 */
1729 if (!pdd->runtime_inuse) {
1730 err = pm_runtime_get_sync(adev_to_drm(dev->adev)->dev);
1731 if (err < 0) {
1732 pm_runtime_put_autosuspend(adev_to_drm(dev->adev)->dev);
1733 return ERR_PTR(err);
1734 }
1735 }
1736
1737 /*
1738 * make sure that runtime_usage counter is incremented just once
1739 * per pdd
1740 */
1741 pdd->runtime_inuse = true;
1742
1743 return pdd;
1744 }
1745
1746 /* Create specific handle mapped to mem from process local memory idr
1747 * Assumes that the process lock is held.
1748 */
kfd_process_device_create_obj_handle(struct kfd_process_device * pdd,void * mem)1749 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1750 void *mem)
1751 {
1752 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1753 }
1754
1755 /* Translate specific handle from process local memory idr
1756 * Assumes that the process lock is held.
1757 */
kfd_process_device_translate_handle(struct kfd_process_device * pdd,int handle)1758 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1759 int handle)
1760 {
1761 if (handle < 0)
1762 return NULL;
1763
1764 return idr_find(&pdd->alloc_idr, handle);
1765 }
1766
1767 /* Remove specific handle from process local memory idr
1768 * Assumes that the process lock is held.
1769 */
kfd_process_device_remove_obj_handle(struct kfd_process_device * pdd,int handle)1770 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1771 int handle)
1772 {
1773 if (handle >= 0)
1774 idr_remove(&pdd->alloc_idr, handle);
1775 }
1776
1777 /* This increments the process->ref counter. */
kfd_lookup_process_by_pasid(u32 pasid)1778 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1779 {
1780 struct kfd_process *p, *ret_p = NULL;
1781 unsigned int temp;
1782
1783 int idx = srcu_read_lock(&kfd_processes_srcu);
1784
1785 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1786 if (p->pasid == pasid) {
1787 kref_get(&p->ref);
1788 ret_p = p;
1789 break;
1790 }
1791 }
1792
1793 srcu_read_unlock(&kfd_processes_srcu, idx);
1794
1795 return ret_p;
1796 }
1797
1798 /* This increments the process->ref counter. */
kfd_lookup_process_by_mm(const struct mm_struct * mm)1799 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1800 {
1801 struct kfd_process *p;
1802
1803 int idx = srcu_read_lock(&kfd_processes_srcu);
1804
1805 p = find_process_by_mm(mm);
1806 if (p)
1807 kref_get(&p->ref);
1808
1809 srcu_read_unlock(&kfd_processes_srcu, idx);
1810
1811 return p;
1812 }
1813
1814 /* kfd_process_evict_queues - Evict all user queues of a process
1815 *
1816 * Eviction is reference-counted per process-device. This means multiple
1817 * evictions from different sources can be nested safely.
1818 */
kfd_process_evict_queues(struct kfd_process * p,uint32_t trigger)1819 int kfd_process_evict_queues(struct kfd_process *p, uint32_t trigger)
1820 {
1821 int r = 0;
1822 int i;
1823 unsigned int n_evicted = 0;
1824
1825 for (i = 0; i < p->n_pdds; i++) {
1826 struct kfd_process_device *pdd = p->pdds[i];
1827 struct device *dev = pdd->dev->adev->dev;
1828
1829 kfd_smi_event_queue_eviction(pdd->dev, p->lead_thread->pid,
1830 trigger);
1831
1832 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1833 &pdd->qpd);
1834 /* evict return -EIO if HWS is hang or asic is resetting, in this case
1835 * we would like to set all the queues to be in evicted state to prevent
1836 * them been add back since they actually not be saved right now.
1837 */
1838 if (r && r != -EIO) {
1839 dev_err(dev, "Failed to evict process queues\n");
1840 goto fail;
1841 }
1842 n_evicted++;
1843 }
1844
1845 return r;
1846
1847 fail:
1848 /* To keep state consistent, roll back partial eviction by
1849 * restoring queues
1850 */
1851 for (i = 0; i < p->n_pdds; i++) {
1852 struct kfd_process_device *pdd = p->pdds[i];
1853
1854 if (n_evicted == 0)
1855 break;
1856
1857 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1858
1859 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1860 &pdd->qpd))
1861 dev_err(pdd->dev->adev->dev,
1862 "Failed to restore queues\n");
1863
1864 n_evicted--;
1865 }
1866
1867 return r;
1868 }
1869
1870 /* kfd_process_restore_queues - Restore all user queues of a process */
kfd_process_restore_queues(struct kfd_process * p)1871 int kfd_process_restore_queues(struct kfd_process *p)
1872 {
1873 int r, ret = 0;
1874 int i;
1875
1876 for (i = 0; i < p->n_pdds; i++) {
1877 struct kfd_process_device *pdd = p->pdds[i];
1878 struct device *dev = pdd->dev->adev->dev;
1879
1880 kfd_smi_event_queue_restore(pdd->dev, p->lead_thread->pid);
1881
1882 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1883 &pdd->qpd);
1884 if (r) {
1885 dev_err(dev, "Failed to restore process queues\n");
1886 if (!ret)
1887 ret = r;
1888 }
1889 }
1890
1891 return ret;
1892 }
1893
kfd_process_gpuidx_from_gpuid(struct kfd_process * p,uint32_t gpu_id)1894 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1895 {
1896 int i;
1897
1898 for (i = 0; i < p->n_pdds; i++)
1899 if (p->pdds[i] && gpu_id == p->pdds[i]->user_gpu_id)
1900 return i;
1901 return -EINVAL;
1902 }
1903
1904 int
kfd_process_gpuid_from_node(struct kfd_process * p,struct kfd_node * node,uint32_t * gpuid,uint32_t * gpuidx)1905 kfd_process_gpuid_from_node(struct kfd_process *p, struct kfd_node *node,
1906 uint32_t *gpuid, uint32_t *gpuidx)
1907 {
1908 int i;
1909
1910 for (i = 0; i < p->n_pdds; i++)
1911 if (p->pdds[i] && p->pdds[i]->dev == node) {
1912 *gpuid = p->pdds[i]->user_gpu_id;
1913 *gpuidx = i;
1914 return 0;
1915 }
1916 return -EINVAL;
1917 }
1918
evict_process_worker(struct work_struct * work)1919 static void evict_process_worker(struct work_struct *work)
1920 {
1921 int ret;
1922 struct kfd_process *p;
1923 struct delayed_work *dwork;
1924
1925 dwork = to_delayed_work(work);
1926
1927 /* Process termination destroys this worker thread. So during the
1928 * lifetime of this thread, kfd_process p will be valid
1929 */
1930 p = container_of(dwork, struct kfd_process, eviction_work);
1931 WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1932 "Eviction fence mismatch\n");
1933
1934 /* Narrow window of overlap between restore and evict work
1935 * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
1936 * unreserves KFD BOs, it is possible to evicted again. But
1937 * restore has few more steps of finish. So lets wait for any
1938 * previous restore work to complete
1939 */
1940 flush_delayed_work(&p->restore_work);
1941
1942 pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1943 ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_TTM);
1944 if (!ret) {
1945 dma_fence_signal(p->ef);
1946 dma_fence_put(p->ef);
1947 p->ef = NULL;
1948 queue_delayed_work(kfd_restore_wq, &p->restore_work,
1949 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1950
1951 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1952 } else
1953 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1954 }
1955
restore_process_worker(struct work_struct * work)1956 static void restore_process_worker(struct work_struct *work)
1957 {
1958 struct delayed_work *dwork;
1959 struct kfd_process *p;
1960 int ret = 0;
1961
1962 dwork = to_delayed_work(work);
1963
1964 /* Process termination destroys this worker thread. So during the
1965 * lifetime of this thread, kfd_process p will be valid
1966 */
1967 p = container_of(dwork, struct kfd_process, restore_work);
1968 pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1969
1970 /* Setting last_restore_timestamp before successful restoration.
1971 * Otherwise this would have to be set by KGD (restore_process_bos)
1972 * before KFD BOs are unreserved. If not, the process can be evicted
1973 * again before the timestamp is set.
1974 * If restore fails, the timestamp will be set again in the next
1975 * attempt. This would mean that the minimum GPU quanta would be
1976 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
1977 * functions)
1978 */
1979
1980 p->last_restore_timestamp = get_jiffies_64();
1981 /* VMs may not have been acquired yet during debugging. */
1982 if (p->kgd_process_info)
1983 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1984 &p->ef);
1985 if (ret) {
1986 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1987 p->pasid, PROCESS_BACK_OFF_TIME_MS);
1988 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1989 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1990 WARN(!ret, "reschedule restore work failed\n");
1991 return;
1992 }
1993
1994 ret = kfd_process_restore_queues(p);
1995 if (!ret)
1996 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1997 else
1998 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1999 }
2000
kfd_suspend_all_processes(void)2001 void kfd_suspend_all_processes(void)
2002 {
2003 struct kfd_process *p;
2004 unsigned int temp;
2005 int idx = srcu_read_lock(&kfd_processes_srcu);
2006
2007 WARN(debug_evictions, "Evicting all processes");
2008 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2009 cancel_delayed_work_sync(&p->eviction_work);
2010 flush_delayed_work(&p->restore_work);
2011
2012 if (kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_TRIGGER_SUSPEND))
2013 pr_err("Failed to suspend process 0x%x\n", p->pasid);
2014 dma_fence_signal(p->ef);
2015 dma_fence_put(p->ef);
2016 p->ef = NULL;
2017 }
2018 srcu_read_unlock(&kfd_processes_srcu, idx);
2019 }
2020
kfd_resume_all_processes(void)2021 int kfd_resume_all_processes(void)
2022 {
2023 struct kfd_process *p;
2024 unsigned int temp;
2025 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
2026
2027 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2028 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
2029 pr_err("Restore process %d failed during resume\n",
2030 p->pasid);
2031 ret = -EFAULT;
2032 }
2033 }
2034 srcu_read_unlock(&kfd_processes_srcu, idx);
2035 return ret;
2036 }
2037
kfd_reserved_mem_mmap(struct kfd_node * dev,struct kfd_process * process,struct vm_area_struct * vma)2038 int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
2039 struct vm_area_struct *vma)
2040 {
2041 struct kfd_process_device *pdd;
2042 struct qcm_process_device *qpd;
2043
2044 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
2045 dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");
2046 return -EINVAL;
2047 }
2048
2049 pdd = kfd_get_process_device_data(dev, process);
2050 if (!pdd)
2051 return -EINVAL;
2052 qpd = &pdd->qpd;
2053
2054 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
2055 get_order(KFD_CWSR_TBA_TMA_SIZE));
2056 if (!qpd->cwsr_kaddr) {
2057 dev_err(dev->adev->dev,
2058 "Error allocating per process CWSR buffer.\n");
2059 return -ENOMEM;
2060 }
2061
2062 vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
2063 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
2064 /* Mapping pages to user process */
2065 return remap_pfn_range(vma, vma->vm_start,
2066 PFN_DOWN(__pa(qpd->cwsr_kaddr)),
2067 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
2068 }
2069
kfd_flush_tlb(struct kfd_process_device * pdd,enum TLB_FLUSH_TYPE type)2070 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
2071 {
2072 struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv);
2073 uint64_t tlb_seq = amdgpu_vm_tlb_seq(vm);
2074 struct kfd_node *dev = pdd->dev;
2075 uint32_t xcc_mask = dev->xcc_mask;
2076 int xcc = 0;
2077
2078 /*
2079 * It can be that we race and lose here, but that is extremely unlikely
2080 * and the worst thing which could happen is that we flush the changes
2081 * into the TLB once more which is harmless.
2082 */
2083 if (atomic64_xchg(&pdd->tlb_seq, tlb_seq) == tlb_seq)
2084 return;
2085
2086 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
2087 /* Nothing to flush until a VMID is assigned, which
2088 * only happens when the first queue is created.
2089 */
2090 if (pdd->qpd.vmid)
2091 amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->adev,
2092 pdd->qpd.vmid);
2093 } else {
2094 for_each_inst(xcc, xcc_mask)
2095 amdgpu_amdkfd_flush_gpu_tlb_pasid(
2096 dev->adev, pdd->process->pasid, type, xcc);
2097 }
2098 }
2099
2100 /* assumes caller holds process lock. */
kfd_process_drain_interrupts(struct kfd_process_device * pdd)2101 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
2102 {
2103 uint32_t irq_drain_fence[8];
2104 uint8_t node_id = 0;
2105 int r = 0;
2106
2107 if (!KFD_IS_SOC15(pdd->dev))
2108 return 0;
2109
2110 pdd->process->irq_drain_is_open = true;
2111
2112 memset(irq_drain_fence, 0, sizeof(irq_drain_fence));
2113 irq_drain_fence[0] = (KFD_IRQ_FENCE_SOURCEID << 8) |
2114 KFD_IRQ_FENCE_CLIENTID;
2115 irq_drain_fence[3] = pdd->process->pasid;
2116
2117 /*
2118 * For GFX 9.4.3, send the NodeId also in IH cookie DW[3]
2119 */
2120 if (KFD_GC_VERSION(pdd->dev->kfd) == IP_VERSION(9, 4, 3)) {
2121 node_id = ffs(pdd->dev->interrupt_bitmap) - 1;
2122 irq_drain_fence[3] |= node_id << 16;
2123 }
2124
2125 /* ensure stale irqs scheduled KFD interrupts and send drain fence. */
2126 if (amdgpu_amdkfd_send_close_event_drain_irq(pdd->dev->adev,
2127 irq_drain_fence)) {
2128 pdd->process->irq_drain_is_open = false;
2129 return 0;
2130 }
2131
2132 r = wait_event_interruptible(pdd->process->wait_irq_drain,
2133 !READ_ONCE(pdd->process->irq_drain_is_open));
2134 if (r)
2135 pdd->process->irq_drain_is_open = false;
2136
2137 return r;
2138 }
2139
kfd_process_close_interrupt_drain(unsigned int pasid)2140 void kfd_process_close_interrupt_drain(unsigned int pasid)
2141 {
2142 struct kfd_process *p;
2143
2144 p = kfd_lookup_process_by_pasid(pasid);
2145
2146 if (!p)
2147 return;
2148
2149 WRITE_ONCE(p->irq_drain_is_open, false);
2150 wake_up_all(&p->wait_irq_drain);
2151 kfd_unref_process(p);
2152 }
2153
2154 struct send_exception_work_handler_workarea {
2155 struct work_struct work;
2156 struct kfd_process *p;
2157 unsigned int queue_id;
2158 uint64_t error_reason;
2159 };
2160
send_exception_work_handler(struct work_struct * work)2161 static void send_exception_work_handler(struct work_struct *work)
2162 {
2163 struct send_exception_work_handler_workarea *workarea;
2164 struct kfd_process *p;
2165 struct queue *q;
2166 struct mm_struct *mm;
2167 struct kfd_context_save_area_header __user *csa_header;
2168 uint64_t __user *err_payload_ptr;
2169 uint64_t cur_err;
2170 uint32_t ev_id;
2171
2172 workarea = container_of(work,
2173 struct send_exception_work_handler_workarea,
2174 work);
2175 p = workarea->p;
2176
2177 mm = get_task_mm(p->lead_thread);
2178
2179 if (!mm)
2180 return;
2181
2182 kthread_use_mm(mm);
2183
2184 q = pqm_get_user_queue(&p->pqm, workarea->queue_id);
2185
2186 if (!q)
2187 goto out;
2188
2189 csa_header = (void __user *)q->properties.ctx_save_restore_area_address;
2190
2191 get_user(err_payload_ptr, (uint64_t __user **)&csa_header->err_payload_addr);
2192 get_user(cur_err, err_payload_ptr);
2193 cur_err |= workarea->error_reason;
2194 put_user(cur_err, err_payload_ptr);
2195 get_user(ev_id, &csa_header->err_event_id);
2196
2197 kfd_set_event(p, ev_id);
2198
2199 out:
2200 kthread_unuse_mm(mm);
2201 mmput(mm);
2202 }
2203
kfd_send_exception_to_runtime(struct kfd_process * p,unsigned int queue_id,uint64_t error_reason)2204 int kfd_send_exception_to_runtime(struct kfd_process *p,
2205 unsigned int queue_id,
2206 uint64_t error_reason)
2207 {
2208 struct send_exception_work_handler_workarea worker;
2209
2210 INIT_WORK_ONSTACK(&worker.work, send_exception_work_handler);
2211
2212 worker.p = p;
2213 worker.queue_id = queue_id;
2214 worker.error_reason = error_reason;
2215
2216 schedule_work(&worker.work);
2217 flush_work(&worker.work);
2218 destroy_work_on_stack(&worker.work);
2219
2220 return 0;
2221 }
2222
kfd_process_device_data_by_id(struct kfd_process * p,uint32_t gpu_id)2223 struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p, uint32_t gpu_id)
2224 {
2225 int i;
2226
2227 if (gpu_id) {
2228 for (i = 0; i < p->n_pdds; i++) {
2229 struct kfd_process_device *pdd = p->pdds[i];
2230
2231 if (pdd->user_gpu_id == gpu_id)
2232 return pdd;
2233 }
2234 }
2235 return NULL;
2236 }
2237
kfd_process_get_user_gpu_id(struct kfd_process * p,uint32_t actual_gpu_id)2238 int kfd_process_get_user_gpu_id(struct kfd_process *p, uint32_t actual_gpu_id)
2239 {
2240 int i;
2241
2242 if (!actual_gpu_id)
2243 return 0;
2244
2245 for (i = 0; i < p->n_pdds; i++) {
2246 struct kfd_process_device *pdd = p->pdds[i];
2247
2248 if (pdd->dev->id == actual_gpu_id)
2249 return pdd->user_gpu_id;
2250 }
2251 return -EINVAL;
2252 }
2253
2254 #if defined(CONFIG_DEBUG_FS)
2255
kfd_debugfs_mqds_by_process(struct seq_file * m,void * data)2256 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
2257 {
2258 struct kfd_process *p;
2259 unsigned int temp;
2260 int r = 0;
2261
2262 int idx = srcu_read_lock(&kfd_processes_srcu);
2263
2264 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
2265 seq_printf(m, "Process %d PASID 0x%x:\n",
2266 p->lead_thread->tgid, p->pasid);
2267
2268 mutex_lock(&p->mutex);
2269 r = pqm_debugfs_mqds(m, &p->pqm);
2270 mutex_unlock(&p->mutex);
2271
2272 if (r)
2273 break;
2274 }
2275
2276 srcu_read_unlock(&kfd_processes_srcu, idx);
2277
2278 return r;
2279 }
2280
2281 #endif
2282