1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/mutex.h>
24 #include <linux/log2.h>
25 #include <linux/sched.h>
26 #include <linux/sched/mm.h>
27 #include <linux/sched/task.h>
28 #include <linux/mmu_context.h>
29 #include <linux/slab.h>
30 #include <linux/amd-iommu.h>
31 #include <linux/notifier.h>
32 #include <linux/compat.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/pm_runtime.h>
36 #include "amdgpu_amdkfd.h"
37 #include "amdgpu.h"
38 
39 struct mm_struct;
40 
41 #include "kfd_priv.h"
42 #include "kfd_device_queue_manager.h"
43 #include "kfd_dbgmgr.h"
44 #include "kfd_iommu.h"
45 
46 /*
47  * List of struct kfd_process (field kfd_process).
48  * Unique/indexed by mm_struct*
49  */
50 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
51 static DEFINE_MUTEX(kfd_processes_mutex);
52 
53 DEFINE_SRCU(kfd_processes_srcu);
54 
55 /* For process termination handling */
56 static struct workqueue_struct *kfd_process_wq;
57 
58 /* Ordered, single-threaded workqueue for restoring evicted
59  * processes. Restoring multiple processes concurrently under memory
60  * pressure can lead to processes blocking each other from validating
61  * their BOs and result in a live-lock situation where processes
62  * remain evicted indefinitely.
63  */
64 static struct workqueue_struct *kfd_restore_wq;
65 
66 static struct kfd_process *find_process(const struct task_struct *thread);
67 static void kfd_process_ref_release(struct kref *ref);
68 static struct kfd_process *create_process(const struct task_struct *thread);
69 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep);
70 
71 static void evict_process_worker(struct work_struct *work);
72 static void restore_process_worker(struct work_struct *work);
73 
74 struct kfd_procfs_tree {
75 	struct kobject *kobj;
76 };
77 
78 static struct kfd_procfs_tree procfs;
79 
80 /*
81  * Structure for SDMA activity tracking
82  */
83 struct kfd_sdma_activity_handler_workarea {
84 	struct work_struct sdma_activity_work;
85 	struct kfd_process_device *pdd;
86 	uint64_t sdma_activity_counter;
87 };
88 
89 struct temp_sdma_queue_list {
90 	uint64_t __user *rptr;
91 	uint64_t sdma_val;
92 	unsigned int queue_id;
93 	struct list_head list;
94 };
95 
96 static void kfd_sdma_activity_worker(struct work_struct *work)
97 {
98 	struct kfd_sdma_activity_handler_workarea *workarea;
99 	struct kfd_process_device *pdd;
100 	uint64_t val;
101 	struct mm_struct *mm;
102 	struct queue *q;
103 	struct qcm_process_device *qpd;
104 	struct device_queue_manager *dqm;
105 	int ret = 0;
106 	struct temp_sdma_queue_list sdma_q_list;
107 	struct temp_sdma_queue_list *sdma_q, *next;
108 
109 	workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
110 				sdma_activity_work);
111 	if (!workarea)
112 		return;
113 
114 	pdd = workarea->pdd;
115 	if (!pdd)
116 		return;
117 	dqm = pdd->dev->dqm;
118 	qpd = &pdd->qpd;
119 	if (!dqm || !qpd)
120 		return;
121 	/*
122 	 * Total SDMA activity is current SDMA activity + past SDMA activity
123 	 * Past SDMA count is stored in pdd.
124 	 * To get the current activity counters for all active SDMA queues,
125 	 * we loop over all SDMA queues and get their counts from user-space.
126 	 *
127 	 * We cannot call get_user() with dqm_lock held as it can cause
128 	 * a circular lock dependency situation. To read the SDMA stats,
129 	 * we need to do the following:
130 	 *
131 	 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
132 	 *    with dqm_lock/dqm_unlock().
133 	 * 2. Call get_user() for each node in temporary list without dqm_lock.
134 	 *    Save the SDMA count for each node and also add the count to the total
135 	 *    SDMA count counter.
136 	 *    Its possible, during this step, a few SDMA queue nodes got deleted
137 	 *    from the qpd->queues_list.
138 	 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
139 	 *    If any node got deleted, its SDMA count would be captured in the sdma
140 	 *    past activity counter. So subtract the SDMA counter stored in step 2
141 	 *    for this node from the total SDMA count.
142 	 */
143 	INIT_LIST_HEAD(&sdma_q_list.list);
144 
145 	/*
146 	 * Create the temp list of all SDMA queues
147 	 */
148 	dqm_lock(dqm);
149 
150 	list_for_each_entry(q, &qpd->queues_list, list) {
151 		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
152 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
153 			continue;
154 
155 		sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
156 		if (!sdma_q) {
157 			dqm_unlock(dqm);
158 			goto cleanup;
159 		}
160 
161 		INIT_LIST_HEAD(&sdma_q->list);
162 		sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
163 		sdma_q->queue_id = q->properties.queue_id;
164 		list_add_tail(&sdma_q->list, &sdma_q_list.list);
165 	}
166 
167 	/*
168 	 * If the temp list is empty, then no SDMA queues nodes were found in
169 	 * qpd->queues_list. Return the past activity count as the total sdma
170 	 * count
171 	 */
172 	if (list_empty(&sdma_q_list.list)) {
173 		workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
174 		dqm_unlock(dqm);
175 		return;
176 	}
177 
178 	dqm_unlock(dqm);
179 
180 	/*
181 	 * Get the usage count for each SDMA queue in temp_list.
182 	 */
183 	mm = get_task_mm(pdd->process->lead_thread);
184 	if (!mm)
185 		goto cleanup;
186 
187 	kthread_use_mm(mm);
188 
189 	list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
190 		val = 0;
191 		ret = read_sdma_queue_counter(sdma_q->rptr, &val);
192 		if (ret) {
193 			pr_debug("Failed to read SDMA queue active counter for queue id: %d",
194 				 sdma_q->queue_id);
195 		} else {
196 			sdma_q->sdma_val = val;
197 			workarea->sdma_activity_counter += val;
198 		}
199 	}
200 
201 	kthread_unuse_mm(mm);
202 	mmput(mm);
203 
204 	/*
205 	 * Do a second iteration over qpd_queues_list to check if any SDMA
206 	 * nodes got deleted while fetching SDMA counter.
207 	 */
208 	dqm_lock(dqm);
209 
210 	workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
211 
212 	list_for_each_entry(q, &qpd->queues_list, list) {
213 		if (list_empty(&sdma_q_list.list))
214 			break;
215 
216 		if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
217 		    (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
218 			continue;
219 
220 		list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
221 			if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
222 			     (sdma_q->queue_id == q->properties.queue_id)) {
223 				list_del(&sdma_q->list);
224 				kfree(sdma_q);
225 				break;
226 			}
227 		}
228 	}
229 
230 	dqm_unlock(dqm);
231 
232 	/*
233 	 * If temp list is not empty, it implies some queues got deleted
234 	 * from qpd->queues_list during SDMA usage read. Subtract the SDMA
235 	 * count for each node from the total SDMA count.
236 	 */
237 	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
238 		workarea->sdma_activity_counter -= sdma_q->sdma_val;
239 		list_del(&sdma_q->list);
240 		kfree(sdma_q);
241 	}
242 
243 	return;
244 
245 cleanup:
246 	list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
247 		list_del(&sdma_q->list);
248 		kfree(sdma_q);
249 	}
250 }
251 
252 /**
253  * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
254  * by current process. Translates acquired wave count into number of compute units
255  * that are occupied.
256  *
257  * @atr: Handle of attribute that allows reporting of wave count. The attribute
258  * handle encapsulates GPU device it is associated with, thereby allowing collection
259  * of waves in flight, etc
260  *
261  * @buffer: Handle of user provided buffer updated with wave count
262  *
263  * Return: Number of bytes written to user buffer or an error value
264  */
265 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
266 {
267 	int cu_cnt;
268 	int wave_cnt;
269 	int max_waves_per_cu;
270 	struct kfd_dev *dev = NULL;
271 	struct kfd_process *proc = NULL;
272 	struct kfd_process_device *pdd = NULL;
273 
274 	pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
275 	dev = pdd->dev;
276 	if (dev->kfd2kgd->get_cu_occupancy == NULL)
277 		return -EINVAL;
278 
279 	cu_cnt = 0;
280 	proc = pdd->process;
281 	if (pdd->qpd.queue_count == 0) {
282 		pr_debug("Gpu-Id: %d has no active queues for process %d\n",
283 			 dev->id, proc->pasid);
284 		return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
285 	}
286 
287 	/* Collect wave count from device if it supports */
288 	wave_cnt = 0;
289 	max_waves_per_cu = 0;
290 	dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
291 			&max_waves_per_cu);
292 
293 	/* Translate wave count to number of compute units */
294 	cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
295 	return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
296 }
297 
298 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
299 			       char *buffer)
300 {
301 	if (strcmp(attr->name, "pasid") == 0) {
302 		struct kfd_process *p = container_of(attr, struct kfd_process,
303 						     attr_pasid);
304 
305 		return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
306 	} else if (strncmp(attr->name, "vram_", 5) == 0) {
307 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
308 							      attr_vram);
309 		return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
310 	} else if (strncmp(attr->name, "sdma_", 5) == 0) {
311 		struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
312 							      attr_sdma);
313 		struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
314 
315 		INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
316 					kfd_sdma_activity_worker);
317 
318 		sdma_activity_work_handler.pdd = pdd;
319 		sdma_activity_work_handler.sdma_activity_counter = 0;
320 
321 		schedule_work(&sdma_activity_work_handler.sdma_activity_work);
322 
323 		flush_work(&sdma_activity_work_handler.sdma_activity_work);
324 
325 		return snprintf(buffer, PAGE_SIZE, "%llu\n",
326 				(sdma_activity_work_handler.sdma_activity_counter)/
327 				 SDMA_ACTIVITY_DIVISOR);
328 	} else {
329 		pr_err("Invalid attribute");
330 		return -EINVAL;
331 	}
332 
333 	return 0;
334 }
335 
336 static void kfd_procfs_kobj_release(struct kobject *kobj)
337 {
338 	kfree(kobj);
339 }
340 
341 static const struct sysfs_ops kfd_procfs_ops = {
342 	.show = kfd_procfs_show,
343 };
344 
345 static struct kobj_type procfs_type = {
346 	.release = kfd_procfs_kobj_release,
347 	.sysfs_ops = &kfd_procfs_ops,
348 };
349 
350 void kfd_procfs_init(void)
351 {
352 	int ret = 0;
353 
354 	procfs.kobj = kfd_alloc_struct(procfs.kobj);
355 	if (!procfs.kobj)
356 		return;
357 
358 	ret = kobject_init_and_add(procfs.kobj, &procfs_type,
359 				   &kfd_device->kobj, "proc");
360 	if (ret) {
361 		pr_warn("Could not create procfs proc folder");
362 		/* If we fail to create the procfs, clean up */
363 		kfd_procfs_shutdown();
364 	}
365 }
366 
367 void kfd_procfs_shutdown(void)
368 {
369 	if (procfs.kobj) {
370 		kobject_del(procfs.kobj);
371 		kobject_put(procfs.kobj);
372 		procfs.kobj = NULL;
373 	}
374 }
375 
376 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
377 				     struct attribute *attr, char *buffer)
378 {
379 	struct queue *q = container_of(kobj, struct queue, kobj);
380 
381 	if (!strcmp(attr->name, "size"))
382 		return snprintf(buffer, PAGE_SIZE, "%llu",
383 				q->properties.queue_size);
384 	else if (!strcmp(attr->name, "type"))
385 		return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
386 	else if (!strcmp(attr->name, "gpuid"))
387 		return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
388 	else
389 		pr_err("Invalid attribute");
390 
391 	return 0;
392 }
393 
394 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
395 				     struct attribute *attr, char *buffer)
396 {
397 	if (strcmp(attr->name, "evicted_ms") == 0) {
398 		struct kfd_process_device *pdd = container_of(attr,
399 				struct kfd_process_device,
400 				attr_evict);
401 		uint64_t evict_jiffies;
402 
403 		evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
404 
405 		return snprintf(buffer,
406 				PAGE_SIZE,
407 				"%llu\n",
408 				jiffies64_to_msecs(evict_jiffies));
409 
410 	/* Sysfs handle that gets CU occupancy is per device */
411 	} else if (strcmp(attr->name, "cu_occupancy") == 0) {
412 		return kfd_get_cu_occupancy(attr, buffer);
413 	} else {
414 		pr_err("Invalid attribute");
415 	}
416 
417 	return 0;
418 }
419 
420 static struct attribute attr_queue_size = {
421 	.name = "size",
422 	.mode = KFD_SYSFS_FILE_MODE
423 };
424 
425 static struct attribute attr_queue_type = {
426 	.name = "type",
427 	.mode = KFD_SYSFS_FILE_MODE
428 };
429 
430 static struct attribute attr_queue_gpuid = {
431 	.name = "gpuid",
432 	.mode = KFD_SYSFS_FILE_MODE
433 };
434 
435 static struct attribute *procfs_queue_attrs[] = {
436 	&attr_queue_size,
437 	&attr_queue_type,
438 	&attr_queue_gpuid,
439 	NULL
440 };
441 
442 static const struct sysfs_ops procfs_queue_ops = {
443 	.show = kfd_procfs_queue_show,
444 };
445 
446 static struct kobj_type procfs_queue_type = {
447 	.sysfs_ops = &procfs_queue_ops,
448 	.default_attrs = procfs_queue_attrs,
449 };
450 
451 static const struct sysfs_ops procfs_stats_ops = {
452 	.show = kfd_procfs_stats_show,
453 };
454 
455 static struct attribute *procfs_stats_attrs[] = {
456 	NULL
457 };
458 
459 static struct kobj_type procfs_stats_type = {
460 	.sysfs_ops = &procfs_stats_ops,
461 	.default_attrs = procfs_stats_attrs,
462 };
463 
464 int kfd_procfs_add_queue(struct queue *q)
465 {
466 	struct kfd_process *proc;
467 	int ret;
468 
469 	if (!q || !q->process)
470 		return -EINVAL;
471 	proc = q->process;
472 
473 	/* Create proc/<pid>/queues/<queue id> folder */
474 	if (!proc->kobj_queues)
475 		return -EFAULT;
476 	ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
477 			proc->kobj_queues, "%u", q->properties.queue_id);
478 	if (ret < 0) {
479 		pr_warn("Creating proc/<pid>/queues/%u failed",
480 			q->properties.queue_id);
481 		kobject_put(&q->kobj);
482 		return ret;
483 	}
484 
485 	return 0;
486 }
487 
488 static int kfd_sysfs_create_file(struct kfd_process *p, struct attribute *attr,
489 				 char *name)
490 {
491 	int ret = 0;
492 
493 	if (!p || !attr || !name)
494 		return -EINVAL;
495 
496 	attr->name = name;
497 	attr->mode = KFD_SYSFS_FILE_MODE;
498 	sysfs_attr_init(attr);
499 
500 	ret = sysfs_create_file(p->kobj, attr);
501 
502 	return ret;
503 }
504 
505 static int kfd_procfs_add_sysfs_stats(struct kfd_process *p)
506 {
507 	int ret = 0;
508 	struct kfd_process_device *pdd;
509 	char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
510 
511 	if (!p)
512 		return -EINVAL;
513 
514 	if (!p->kobj)
515 		return -EFAULT;
516 
517 	/*
518 	 * Create sysfs files for each GPU:
519 	 * - proc/<pid>/stats_<gpuid>/
520 	 * - proc/<pid>/stats_<gpuid>/evicted_ms
521 	 * - proc/<pid>/stats_<gpuid>/cu_occupancy
522 	 */
523 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
524 		struct kobject *kobj_stats;
525 
526 		snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
527 				"stats_%u", pdd->dev->id);
528 		kobj_stats = kfd_alloc_struct(kobj_stats);
529 		if (!kobj_stats)
530 			return -ENOMEM;
531 
532 		ret = kobject_init_and_add(kobj_stats,
533 						&procfs_stats_type,
534 						p->kobj,
535 						stats_dir_filename);
536 
537 		if (ret) {
538 			pr_warn("Creating KFD proc/stats_%s folder failed",
539 					stats_dir_filename);
540 			kobject_put(kobj_stats);
541 			goto err;
542 		}
543 
544 		pdd->kobj_stats = kobj_stats;
545 		pdd->attr_evict.name = "evicted_ms";
546 		pdd->attr_evict.mode = KFD_SYSFS_FILE_MODE;
547 		sysfs_attr_init(&pdd->attr_evict);
548 		ret = sysfs_create_file(kobj_stats, &pdd->attr_evict);
549 		if (ret)
550 			pr_warn("Creating eviction stats for gpuid %d failed",
551 					(int)pdd->dev->id);
552 
553 		/* Add sysfs file to report compute unit occupancy */
554 		if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) {
555 			pdd->attr_cu_occupancy.name = "cu_occupancy";
556 			pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE;
557 			sysfs_attr_init(&pdd->attr_cu_occupancy);
558 			ret = sysfs_create_file(kobj_stats,
559 						&pdd->attr_cu_occupancy);
560 			if (ret)
561 				pr_warn("Creating %s failed for gpuid: %d",
562 					pdd->attr_cu_occupancy.name,
563 					(int)pdd->dev->id);
564 		}
565 	}
566 err:
567 	return ret;
568 }
569 
570 
571 static int kfd_procfs_add_sysfs_files(struct kfd_process *p)
572 {
573 	int ret = 0;
574 	struct kfd_process_device *pdd;
575 
576 	if (!p)
577 		return -EINVAL;
578 
579 	if (!p->kobj)
580 		return -EFAULT;
581 
582 	/*
583 	 * Create sysfs files for each GPU:
584 	 * - proc/<pid>/vram_<gpuid>
585 	 * - proc/<pid>/sdma_<gpuid>
586 	 */
587 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
588 		snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
589 			 pdd->dev->id);
590 		ret = kfd_sysfs_create_file(p, &pdd->attr_vram, pdd->vram_filename);
591 		if (ret)
592 			pr_warn("Creating vram usage for gpu id %d failed",
593 				(int)pdd->dev->id);
594 
595 		snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
596 			 pdd->dev->id);
597 		ret = kfd_sysfs_create_file(p, &pdd->attr_sdma, pdd->sdma_filename);
598 		if (ret)
599 			pr_warn("Creating sdma usage for gpu id %d failed",
600 				(int)pdd->dev->id);
601 	}
602 
603 	return ret;
604 }
605 
606 void kfd_procfs_del_queue(struct queue *q)
607 {
608 	if (!q)
609 		return;
610 
611 	kobject_del(&q->kobj);
612 	kobject_put(&q->kobj);
613 }
614 
615 int kfd_process_create_wq(void)
616 {
617 	if (!kfd_process_wq)
618 		kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
619 	if (!kfd_restore_wq)
620 		kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
621 
622 	if (!kfd_process_wq || !kfd_restore_wq) {
623 		kfd_process_destroy_wq();
624 		return -ENOMEM;
625 	}
626 
627 	return 0;
628 }
629 
630 void kfd_process_destroy_wq(void)
631 {
632 	if (kfd_process_wq) {
633 		destroy_workqueue(kfd_process_wq);
634 		kfd_process_wq = NULL;
635 	}
636 	if (kfd_restore_wq) {
637 		destroy_workqueue(kfd_restore_wq);
638 		kfd_restore_wq = NULL;
639 	}
640 }
641 
642 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
643 			struct kfd_process_device *pdd)
644 {
645 	struct kfd_dev *dev = pdd->dev;
646 
647 	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->vm);
648 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, NULL);
649 }
650 
651 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
652  *	This function should be only called right after the process
653  *	is created and when kfd_processes_mutex is still being held
654  *	to avoid concurrency. Because of that exclusiveness, we do
655  *	not need to take p->mutex.
656  */
657 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
658 				   uint64_t gpu_va, uint32_t size,
659 				   uint32_t flags, void **kptr)
660 {
661 	struct kfd_dev *kdev = pdd->dev;
662 	struct kgd_mem *mem = NULL;
663 	int handle;
664 	int err;
665 
666 	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
667 						 pdd->vm, &mem, NULL, flags);
668 	if (err)
669 		goto err_alloc_mem;
670 
671 	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->vm);
672 	if (err)
673 		goto err_map_mem;
674 
675 	err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true);
676 	if (err) {
677 		pr_debug("Sync memory failed, wait interrupted by user signal\n");
678 		goto sync_memory_failed;
679 	}
680 
681 	/* Create an obj handle so kfd_process_device_remove_obj_handle
682 	 * will take care of the bo removal when the process finishes.
683 	 * We do not need to take p->mutex, because the process is just
684 	 * created and the ioctls have not had the chance to run.
685 	 */
686 	handle = kfd_process_device_create_obj_handle(pdd, mem);
687 
688 	if (handle < 0) {
689 		err = handle;
690 		goto free_gpuvm;
691 	}
692 
693 	if (kptr) {
694 		err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd,
695 				(struct kgd_mem *)mem, kptr, NULL);
696 		if (err) {
697 			pr_debug("Map GTT BO to kernel failed\n");
698 			goto free_obj_handle;
699 		}
700 	}
701 
702 	return err;
703 
704 free_obj_handle:
705 	kfd_process_device_remove_obj_handle(pdd, handle);
706 free_gpuvm:
707 sync_memory_failed:
708 	kfd_process_free_gpuvm(mem, pdd);
709 	return err;
710 
711 err_map_mem:
712 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, NULL);
713 err_alloc_mem:
714 	*kptr = NULL;
715 	return err;
716 }
717 
718 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
719  *	process for IB usage The memory reserved is for KFD to submit
720  *	IB to AMDGPU from kernel.  If the memory is reserved
721  *	successfully, ib_kaddr will have the CPU/kernel
722  *	address. Check ib_kaddr before accessing the memory.
723  */
724 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
725 {
726 	struct qcm_process_device *qpd = &pdd->qpd;
727 	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
728 			KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
729 			KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
730 			KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
731 	void *kaddr;
732 	int ret;
733 
734 	if (qpd->ib_kaddr || !qpd->ib_base)
735 		return 0;
736 
737 	/* ib_base is only set for dGPU */
738 	ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
739 				      &kaddr);
740 	if (ret)
741 		return ret;
742 
743 	qpd->ib_kaddr = kaddr;
744 
745 	return 0;
746 }
747 
748 struct kfd_process *kfd_create_process(struct file *filep)
749 {
750 	struct kfd_process *process;
751 	struct task_struct *thread = current;
752 	int ret;
753 
754 	if (!thread->mm)
755 		return ERR_PTR(-EINVAL);
756 
757 	/* Only the pthreads threading model is supported. */
758 	if (thread->group_leader->mm != thread->mm)
759 		return ERR_PTR(-EINVAL);
760 
761 	/*
762 	 * take kfd processes mutex before starting of process creation
763 	 * so there won't be a case where two threads of the same process
764 	 * create two kfd_process structures
765 	 */
766 	mutex_lock(&kfd_processes_mutex);
767 
768 	/* A prior open of /dev/kfd could have already created the process. */
769 	process = find_process(thread);
770 	if (process) {
771 		pr_debug("Process already found\n");
772 	} else {
773 		process = create_process(thread);
774 		if (IS_ERR(process))
775 			goto out;
776 
777 		ret = kfd_process_init_cwsr_apu(process, filep);
778 		if (ret) {
779 			process = ERR_PTR(ret);
780 			goto out;
781 		}
782 
783 		if (!procfs.kobj)
784 			goto out;
785 
786 		process->kobj = kfd_alloc_struct(process->kobj);
787 		if (!process->kobj) {
788 			pr_warn("Creating procfs kobject failed");
789 			goto out;
790 		}
791 		ret = kobject_init_and_add(process->kobj, &procfs_type,
792 					   procfs.kobj, "%d",
793 					   (int)process->lead_thread->pid);
794 		if (ret) {
795 			pr_warn("Creating procfs pid directory failed");
796 			kobject_put(process->kobj);
797 			goto out;
798 		}
799 
800 		process->attr_pasid.name = "pasid";
801 		process->attr_pasid.mode = KFD_SYSFS_FILE_MODE;
802 		sysfs_attr_init(&process->attr_pasid);
803 		ret = sysfs_create_file(process->kobj, &process->attr_pasid);
804 		if (ret)
805 			pr_warn("Creating pasid for pid %d failed",
806 					(int)process->lead_thread->pid);
807 
808 		process->kobj_queues = kobject_create_and_add("queues",
809 							process->kobj);
810 		if (!process->kobj_queues)
811 			pr_warn("Creating KFD proc/queues folder failed");
812 
813 		ret = kfd_procfs_add_sysfs_stats(process);
814 		if (ret)
815 			pr_warn("Creating sysfs stats dir for pid %d failed",
816 				(int)process->lead_thread->pid);
817 
818 		ret = kfd_procfs_add_sysfs_files(process);
819 		if (ret)
820 			pr_warn("Creating sysfs usage file for pid %d failed",
821 				(int)process->lead_thread->pid);
822 	}
823 out:
824 	if (!IS_ERR(process))
825 		kref_get(&process->ref);
826 	mutex_unlock(&kfd_processes_mutex);
827 
828 	return process;
829 }
830 
831 struct kfd_process *kfd_get_process(const struct task_struct *thread)
832 {
833 	struct kfd_process *process;
834 
835 	if (!thread->mm)
836 		return ERR_PTR(-EINVAL);
837 
838 	/* Only the pthreads threading model is supported. */
839 	if (thread->group_leader->mm != thread->mm)
840 		return ERR_PTR(-EINVAL);
841 
842 	process = find_process(thread);
843 	if (!process)
844 		return ERR_PTR(-EINVAL);
845 
846 	return process;
847 }
848 
849 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
850 {
851 	struct kfd_process *process;
852 
853 	hash_for_each_possible_rcu(kfd_processes_table, process,
854 					kfd_processes, (uintptr_t)mm)
855 		if (process->mm == mm)
856 			return process;
857 
858 	return NULL;
859 }
860 
861 static struct kfd_process *find_process(const struct task_struct *thread)
862 {
863 	struct kfd_process *p;
864 	int idx;
865 
866 	idx = srcu_read_lock(&kfd_processes_srcu);
867 	p = find_process_by_mm(thread->mm);
868 	srcu_read_unlock(&kfd_processes_srcu, idx);
869 
870 	return p;
871 }
872 
873 void kfd_unref_process(struct kfd_process *p)
874 {
875 	kref_put(&p->ref, kfd_process_ref_release);
876 }
877 
878 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
879 {
880 	struct kfd_process *p = pdd->process;
881 	void *mem;
882 	int id;
883 
884 	/*
885 	 * Remove all handles from idr and release appropriate
886 	 * local memory object
887 	 */
888 	idr_for_each_entry(&pdd->alloc_idr, mem, id) {
889 		struct kfd_process_device *peer_pdd;
890 
891 		list_for_each_entry(peer_pdd, &p->per_device_data,
892 				    per_device_list) {
893 			if (!peer_pdd->vm)
894 				continue;
895 			amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
896 				peer_pdd->dev->kgd, mem, peer_pdd->vm);
897 		}
898 
899 		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, NULL);
900 		kfd_process_device_remove_obj_handle(pdd, id);
901 	}
902 }
903 
904 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
905 {
906 	struct kfd_process_device *pdd;
907 
908 	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
909 		kfd_process_device_free_bos(pdd);
910 }
911 
912 static void kfd_process_destroy_pdds(struct kfd_process *p)
913 {
914 	struct kfd_process_device *pdd, *temp;
915 
916 	list_for_each_entry_safe(pdd, temp, &p->per_device_data,
917 				 per_device_list) {
918 		pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
919 				pdd->dev->id, p->pasid);
920 
921 		if (pdd->drm_file) {
922 			amdgpu_amdkfd_gpuvm_release_process_vm(
923 					pdd->dev->kgd, pdd->vm);
924 			fput(pdd->drm_file);
925 		}
926 		else if (pdd->vm)
927 			amdgpu_amdkfd_gpuvm_destroy_process_vm(
928 				pdd->dev->kgd, pdd->vm);
929 
930 		list_del(&pdd->per_device_list);
931 
932 		if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
933 			free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
934 				get_order(KFD_CWSR_TBA_TMA_SIZE));
935 
936 		kfree(pdd->qpd.doorbell_bitmap);
937 		idr_destroy(&pdd->alloc_idr);
938 
939 		kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
940 
941 		/*
942 		 * before destroying pdd, make sure to report availability
943 		 * for auto suspend
944 		 */
945 		if (pdd->runtime_inuse) {
946 			pm_runtime_mark_last_busy(pdd->dev->ddev->dev);
947 			pm_runtime_put_autosuspend(pdd->dev->ddev->dev);
948 			pdd->runtime_inuse = false;
949 		}
950 
951 		kfree(pdd);
952 	}
953 }
954 
955 /* No process locking is needed in this function, because the process
956  * is not findable any more. We must assume that no other thread is
957  * using it any more, otherwise we couldn't safely free the process
958  * structure in the end.
959  */
960 static void kfd_process_wq_release(struct work_struct *work)
961 {
962 	struct kfd_process *p = container_of(work, struct kfd_process,
963 					     release_work);
964 	struct kfd_process_device *pdd;
965 
966 	/* Remove the procfs files */
967 	if (p->kobj) {
968 		sysfs_remove_file(p->kobj, &p->attr_pasid);
969 		kobject_del(p->kobj_queues);
970 		kobject_put(p->kobj_queues);
971 		p->kobj_queues = NULL;
972 
973 		list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
974 			sysfs_remove_file(p->kobj, &pdd->attr_vram);
975 			sysfs_remove_file(p->kobj, &pdd->attr_sdma);
976 			sysfs_remove_file(p->kobj, &pdd->attr_evict);
977 			if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL)
978 				sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy);
979 			kobject_del(pdd->kobj_stats);
980 			kobject_put(pdd->kobj_stats);
981 			pdd->kobj_stats = NULL;
982 		}
983 
984 		kobject_del(p->kobj);
985 		kobject_put(p->kobj);
986 		p->kobj = NULL;
987 	}
988 
989 	kfd_iommu_unbind_process(p);
990 
991 	kfd_process_free_outstanding_kfd_bos(p);
992 
993 	kfd_process_destroy_pdds(p);
994 	dma_fence_put(p->ef);
995 
996 	kfd_event_free_process(p);
997 
998 	kfd_pasid_free(p->pasid);
999 	mutex_destroy(&p->mutex);
1000 
1001 	put_task_struct(p->lead_thread);
1002 
1003 	kfree(p);
1004 }
1005 
1006 static void kfd_process_ref_release(struct kref *ref)
1007 {
1008 	struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1009 
1010 	INIT_WORK(&p->release_work, kfd_process_wq_release);
1011 	queue_work(kfd_process_wq, &p->release_work);
1012 }
1013 
1014 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1015 {
1016 	kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1017 }
1018 
1019 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1020 					struct mm_struct *mm)
1021 {
1022 	struct kfd_process *p;
1023 	struct kfd_process_device *pdd = NULL;
1024 
1025 	/*
1026 	 * The kfd_process structure can not be free because the
1027 	 * mmu_notifier srcu is read locked
1028 	 */
1029 	p = container_of(mn, struct kfd_process, mmu_notifier);
1030 	if (WARN_ON(p->mm != mm))
1031 		return;
1032 
1033 	mutex_lock(&kfd_processes_mutex);
1034 	hash_del_rcu(&p->kfd_processes);
1035 	mutex_unlock(&kfd_processes_mutex);
1036 	synchronize_srcu(&kfd_processes_srcu);
1037 
1038 	cancel_delayed_work_sync(&p->eviction_work);
1039 	cancel_delayed_work_sync(&p->restore_work);
1040 
1041 	mutex_lock(&p->mutex);
1042 
1043 	/* Iterate over all process device data structures and if the
1044 	 * pdd is in debug mode, we should first force unregistration,
1045 	 * then we will be able to destroy the queues
1046 	 */
1047 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1048 		struct kfd_dev *dev = pdd->dev;
1049 
1050 		mutex_lock(kfd_get_dbgmgr_mutex());
1051 		if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
1052 			if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
1053 				kfd_dbgmgr_destroy(dev->dbgmgr);
1054 				dev->dbgmgr = NULL;
1055 			}
1056 		}
1057 		mutex_unlock(kfd_get_dbgmgr_mutex());
1058 	}
1059 
1060 	kfd_process_dequeue_from_all_devices(p);
1061 	pqm_uninit(&p->pqm);
1062 
1063 	/* Indicate to other users that MM is no longer valid */
1064 	p->mm = NULL;
1065 	/* Signal the eviction fence after user mode queues are
1066 	 * destroyed. This allows any BOs to be freed without
1067 	 * triggering pointless evictions or waiting for fences.
1068 	 */
1069 	dma_fence_signal(p->ef);
1070 
1071 	mutex_unlock(&p->mutex);
1072 
1073 	mmu_notifier_put(&p->mmu_notifier);
1074 }
1075 
1076 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1077 	.release = kfd_process_notifier_release,
1078 	.free_notifier = kfd_process_free_notifier,
1079 };
1080 
1081 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1082 {
1083 	unsigned long  offset;
1084 	struct kfd_process_device *pdd;
1085 
1086 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1087 		struct kfd_dev *dev = pdd->dev;
1088 		struct qcm_process_device *qpd = &pdd->qpd;
1089 
1090 		if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1091 			continue;
1092 
1093 		offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1094 		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1095 			KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1096 			MAP_SHARED, offset);
1097 
1098 		if (IS_ERR_VALUE(qpd->tba_addr)) {
1099 			int err = qpd->tba_addr;
1100 
1101 			pr_err("Failure to set tba address. error %d.\n", err);
1102 			qpd->tba_addr = 0;
1103 			qpd->cwsr_kaddr = NULL;
1104 			return err;
1105 		}
1106 
1107 		memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1108 
1109 		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1110 		pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1111 			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1112 	}
1113 
1114 	return 0;
1115 }
1116 
1117 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1118 {
1119 	struct kfd_dev *dev = pdd->dev;
1120 	struct qcm_process_device *qpd = &pdd->qpd;
1121 	uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1122 			| KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1123 			| KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1124 	void *kaddr;
1125 	int ret;
1126 
1127 	if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1128 		return 0;
1129 
1130 	/* cwsr_base is only set for dGPU */
1131 	ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1132 				      KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr);
1133 	if (ret)
1134 		return ret;
1135 
1136 	qpd->cwsr_kaddr = kaddr;
1137 	qpd->tba_addr = qpd->cwsr_base;
1138 
1139 	memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1140 
1141 	qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1142 	pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1143 		 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1144 
1145 	return 0;
1146 }
1147 
1148 /*
1149  * On return the kfd_process is fully operational and will be freed when the
1150  * mm is released
1151  */
1152 static struct kfd_process *create_process(const struct task_struct *thread)
1153 {
1154 	struct kfd_process *process;
1155 	int err = -ENOMEM;
1156 
1157 	process = kzalloc(sizeof(*process), GFP_KERNEL);
1158 	if (!process)
1159 		goto err_alloc_process;
1160 
1161 	kref_init(&process->ref);
1162 	mutex_init(&process->mutex);
1163 	process->mm = thread->mm;
1164 	process->lead_thread = thread->group_leader;
1165 	INIT_LIST_HEAD(&process->per_device_data);
1166 	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1167 	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1168 	process->last_restore_timestamp = get_jiffies_64();
1169 	kfd_event_init_process(process);
1170 	process->is_32bit_user_mode = in_compat_syscall();
1171 
1172 	process->pasid = kfd_pasid_alloc();
1173 	if (process->pasid == 0)
1174 		goto err_alloc_pasid;
1175 
1176 	err = pqm_init(&process->pqm, process);
1177 	if (err != 0)
1178 		goto err_process_pqm_init;
1179 
1180 	/* init process apertures*/
1181 	err = kfd_init_apertures(process);
1182 	if (err != 0)
1183 		goto err_init_apertures;
1184 
1185 	/* Must be last, have to use release destruction after this */
1186 	process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops;
1187 	err = mmu_notifier_register(&process->mmu_notifier, process->mm);
1188 	if (err)
1189 		goto err_register_notifier;
1190 
1191 	get_task_struct(process->lead_thread);
1192 	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1193 			(uintptr_t)process->mm);
1194 
1195 	return process;
1196 
1197 err_register_notifier:
1198 	kfd_process_free_outstanding_kfd_bos(process);
1199 	kfd_process_destroy_pdds(process);
1200 err_init_apertures:
1201 	pqm_uninit(&process->pqm);
1202 err_process_pqm_init:
1203 	kfd_pasid_free(process->pasid);
1204 err_alloc_pasid:
1205 	mutex_destroy(&process->mutex);
1206 	kfree(process);
1207 err_alloc_process:
1208 	return ERR_PTR(err);
1209 }
1210 
1211 static int init_doorbell_bitmap(struct qcm_process_device *qpd,
1212 			struct kfd_dev *dev)
1213 {
1214 	unsigned int i;
1215 	int range_start = dev->shared_resources.non_cp_doorbells_start;
1216 	int range_end = dev->shared_resources.non_cp_doorbells_end;
1217 
1218 	if (!KFD_IS_SOC15(dev->device_info->asic_family))
1219 		return 0;
1220 
1221 	qpd->doorbell_bitmap =
1222 		kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
1223 				     BITS_PER_BYTE), GFP_KERNEL);
1224 	if (!qpd->doorbell_bitmap)
1225 		return -ENOMEM;
1226 
1227 	/* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */
1228 	pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end);
1229 	pr_debug("reserved doorbell 0x%03x - 0x%03x\n",
1230 			range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1231 			range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET);
1232 
1233 	for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) {
1234 		if (i >= range_start && i <= range_end) {
1235 			set_bit(i, qpd->doorbell_bitmap);
1236 			set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1237 				qpd->doorbell_bitmap);
1238 		}
1239 	}
1240 
1241 	return 0;
1242 }
1243 
1244 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
1245 							struct kfd_process *p)
1246 {
1247 	struct kfd_process_device *pdd = NULL;
1248 
1249 	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
1250 		if (pdd->dev == dev)
1251 			return pdd;
1252 
1253 	return NULL;
1254 }
1255 
1256 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
1257 							struct kfd_process *p)
1258 {
1259 	struct kfd_process_device *pdd = NULL;
1260 
1261 	pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1262 	if (!pdd)
1263 		return NULL;
1264 
1265 	if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
1266 		pr_err("Failed to alloc doorbell for pdd\n");
1267 		goto err_free_pdd;
1268 	}
1269 
1270 	if (init_doorbell_bitmap(&pdd->qpd, dev)) {
1271 		pr_err("Failed to init doorbell for process\n");
1272 		goto err_free_pdd;
1273 	}
1274 
1275 	pdd->dev = dev;
1276 	INIT_LIST_HEAD(&pdd->qpd.queues_list);
1277 	INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1278 	pdd->qpd.dqm = dev->dqm;
1279 	pdd->qpd.pqm = &p->pqm;
1280 	pdd->qpd.evicted = 0;
1281 	pdd->qpd.mapped_gws_queue = false;
1282 	pdd->process = p;
1283 	pdd->bound = PDD_UNBOUND;
1284 	pdd->already_dequeued = false;
1285 	pdd->runtime_inuse = false;
1286 	pdd->vram_usage = 0;
1287 	pdd->sdma_past_activity_counter = 0;
1288 	atomic64_set(&pdd->evict_duration_counter, 0);
1289 	list_add(&pdd->per_device_list, &p->per_device_data);
1290 
1291 	/* Init idr used for memory handle translation */
1292 	idr_init(&pdd->alloc_idr);
1293 
1294 	return pdd;
1295 
1296 err_free_pdd:
1297 	kfree(pdd);
1298 	return NULL;
1299 }
1300 
1301 /**
1302  * kfd_process_device_init_vm - Initialize a VM for a process-device
1303  *
1304  * @pdd: The process-device
1305  * @drm_file: Optional pointer to a DRM file descriptor
1306  *
1307  * If @drm_file is specified, it will be used to acquire the VM from
1308  * that file descriptor. If successful, the @pdd takes ownership of
1309  * the file descriptor.
1310  *
1311  * If @drm_file is NULL, a new VM is created.
1312  *
1313  * Returns 0 on success, -errno on failure.
1314  */
1315 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1316 			       struct file *drm_file)
1317 {
1318 	struct kfd_process *p;
1319 	struct kfd_dev *dev;
1320 	int ret;
1321 
1322 	if (pdd->vm)
1323 		return drm_file ? -EBUSY : 0;
1324 
1325 	p = pdd->process;
1326 	dev = pdd->dev;
1327 
1328 	if (drm_file)
1329 		ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1330 			dev->kgd, drm_file, p->pasid,
1331 			&pdd->vm, &p->kgd_process_info, &p->ef);
1332 	else
1333 		ret = amdgpu_amdkfd_gpuvm_create_process_vm(dev->kgd, p->pasid,
1334 			&pdd->vm, &p->kgd_process_info, &p->ef);
1335 	if (ret) {
1336 		pr_err("Failed to create process VM object\n");
1337 		return ret;
1338 	}
1339 
1340 	amdgpu_vm_set_task_info(pdd->vm);
1341 
1342 	ret = kfd_process_device_reserve_ib_mem(pdd);
1343 	if (ret)
1344 		goto err_reserve_ib_mem;
1345 	ret = kfd_process_device_init_cwsr_dgpu(pdd);
1346 	if (ret)
1347 		goto err_init_cwsr;
1348 
1349 	pdd->drm_file = drm_file;
1350 
1351 	return 0;
1352 
1353 err_init_cwsr:
1354 err_reserve_ib_mem:
1355 	kfd_process_device_free_bos(pdd);
1356 	if (!drm_file)
1357 		amdgpu_amdkfd_gpuvm_destroy_process_vm(dev->kgd, pdd->vm);
1358 	pdd->vm = NULL;
1359 
1360 	return ret;
1361 }
1362 
1363 /*
1364  * Direct the IOMMU to bind the process (specifically the pasid->mm)
1365  * to the device.
1366  * Unbinding occurs when the process dies or the device is removed.
1367  *
1368  * Assumes that the process lock is held.
1369  */
1370 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
1371 							struct kfd_process *p)
1372 {
1373 	struct kfd_process_device *pdd;
1374 	int err;
1375 
1376 	pdd = kfd_get_process_device_data(dev, p);
1377 	if (!pdd) {
1378 		pr_err("Process device data doesn't exist\n");
1379 		return ERR_PTR(-ENOMEM);
1380 	}
1381 
1382 	/*
1383 	 * signal runtime-pm system to auto resume and prevent
1384 	 * further runtime suspend once device pdd is created until
1385 	 * pdd is destroyed.
1386 	 */
1387 	if (!pdd->runtime_inuse) {
1388 		err = pm_runtime_get_sync(dev->ddev->dev);
1389 		if (err < 0) {
1390 			pm_runtime_put_autosuspend(dev->ddev->dev);
1391 			return ERR_PTR(err);
1392 		}
1393 	}
1394 
1395 	err = kfd_iommu_bind_process_to_device(pdd);
1396 	if (err)
1397 		goto out;
1398 
1399 	err = kfd_process_device_init_vm(pdd, NULL);
1400 	if (err)
1401 		goto out;
1402 
1403 	/*
1404 	 * make sure that runtime_usage counter is incremented just once
1405 	 * per pdd
1406 	 */
1407 	pdd->runtime_inuse = true;
1408 
1409 	return pdd;
1410 
1411 out:
1412 	/* balance runpm reference count and exit with error */
1413 	if (!pdd->runtime_inuse) {
1414 		pm_runtime_mark_last_busy(dev->ddev->dev);
1415 		pm_runtime_put_autosuspend(dev->ddev->dev);
1416 	}
1417 
1418 	return ERR_PTR(err);
1419 }
1420 
1421 struct kfd_process_device *kfd_get_first_process_device_data(
1422 						struct kfd_process *p)
1423 {
1424 	return list_first_entry(&p->per_device_data,
1425 				struct kfd_process_device,
1426 				per_device_list);
1427 }
1428 
1429 struct kfd_process_device *kfd_get_next_process_device_data(
1430 						struct kfd_process *p,
1431 						struct kfd_process_device *pdd)
1432 {
1433 	if (list_is_last(&pdd->per_device_list, &p->per_device_data))
1434 		return NULL;
1435 	return list_next_entry(pdd, per_device_list);
1436 }
1437 
1438 bool kfd_has_process_device_data(struct kfd_process *p)
1439 {
1440 	return !(list_empty(&p->per_device_data));
1441 }
1442 
1443 /* Create specific handle mapped to mem from process local memory idr
1444  * Assumes that the process lock is held.
1445  */
1446 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1447 					void *mem)
1448 {
1449 	return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1450 }
1451 
1452 /* Translate specific handle from process local memory idr
1453  * Assumes that the process lock is held.
1454  */
1455 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1456 					int handle)
1457 {
1458 	if (handle < 0)
1459 		return NULL;
1460 
1461 	return idr_find(&pdd->alloc_idr, handle);
1462 }
1463 
1464 /* Remove specific handle from process local memory idr
1465  * Assumes that the process lock is held.
1466  */
1467 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1468 					int handle)
1469 {
1470 	if (handle >= 0)
1471 		idr_remove(&pdd->alloc_idr, handle);
1472 }
1473 
1474 /* This increments the process->ref counter. */
1475 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1476 {
1477 	struct kfd_process *p, *ret_p = NULL;
1478 	unsigned int temp;
1479 
1480 	int idx = srcu_read_lock(&kfd_processes_srcu);
1481 
1482 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1483 		if (p->pasid == pasid) {
1484 			kref_get(&p->ref);
1485 			ret_p = p;
1486 			break;
1487 		}
1488 	}
1489 
1490 	srcu_read_unlock(&kfd_processes_srcu, idx);
1491 
1492 	return ret_p;
1493 }
1494 
1495 /* This increments the process->ref counter. */
1496 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1497 {
1498 	struct kfd_process *p;
1499 
1500 	int idx = srcu_read_lock(&kfd_processes_srcu);
1501 
1502 	p = find_process_by_mm(mm);
1503 	if (p)
1504 		kref_get(&p->ref);
1505 
1506 	srcu_read_unlock(&kfd_processes_srcu, idx);
1507 
1508 	return p;
1509 }
1510 
1511 /* kfd_process_evict_queues - Evict all user queues of a process
1512  *
1513  * Eviction is reference-counted per process-device. This means multiple
1514  * evictions from different sources can be nested safely.
1515  */
1516 int kfd_process_evict_queues(struct kfd_process *p)
1517 {
1518 	struct kfd_process_device *pdd;
1519 	int r = 0;
1520 	unsigned int n_evicted = 0;
1521 
1522 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1523 		r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1524 							    &pdd->qpd);
1525 		if (r) {
1526 			pr_err("Failed to evict process queues\n");
1527 			goto fail;
1528 		}
1529 		n_evicted++;
1530 	}
1531 
1532 	return r;
1533 
1534 fail:
1535 	/* To keep state consistent, roll back partial eviction by
1536 	 * restoring queues
1537 	 */
1538 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1539 		if (n_evicted == 0)
1540 			break;
1541 		if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1542 							      &pdd->qpd))
1543 			pr_err("Failed to restore queues\n");
1544 
1545 		n_evicted--;
1546 	}
1547 
1548 	return r;
1549 }
1550 
1551 /* kfd_process_restore_queues - Restore all user queues of a process */
1552 int kfd_process_restore_queues(struct kfd_process *p)
1553 {
1554 	struct kfd_process_device *pdd;
1555 	int r, ret = 0;
1556 
1557 	list_for_each_entry(pdd, &p->per_device_data, per_device_list) {
1558 		r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1559 							      &pdd->qpd);
1560 		if (r) {
1561 			pr_err("Failed to restore process queues\n");
1562 			if (!ret)
1563 				ret = r;
1564 		}
1565 	}
1566 
1567 	return ret;
1568 }
1569 
1570 static void evict_process_worker(struct work_struct *work)
1571 {
1572 	int ret;
1573 	struct kfd_process *p;
1574 	struct delayed_work *dwork;
1575 
1576 	dwork = to_delayed_work(work);
1577 
1578 	/* Process termination destroys this worker thread. So during the
1579 	 * lifetime of this thread, kfd_process p will be valid
1580 	 */
1581 	p = container_of(dwork, struct kfd_process, eviction_work);
1582 	WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1583 		  "Eviction fence mismatch\n");
1584 
1585 	/* Narrow window of overlap between restore and evict work
1586 	 * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
1587 	 * unreserves KFD BOs, it is possible to evicted again. But
1588 	 * restore has few more steps of finish. So lets wait for any
1589 	 * previous restore work to complete
1590 	 */
1591 	flush_delayed_work(&p->restore_work);
1592 
1593 	pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1594 	ret = kfd_process_evict_queues(p);
1595 	if (!ret) {
1596 		dma_fence_signal(p->ef);
1597 		dma_fence_put(p->ef);
1598 		p->ef = NULL;
1599 		queue_delayed_work(kfd_restore_wq, &p->restore_work,
1600 				msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1601 
1602 		pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1603 	} else
1604 		pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1605 }
1606 
1607 static void restore_process_worker(struct work_struct *work)
1608 {
1609 	struct delayed_work *dwork;
1610 	struct kfd_process *p;
1611 	int ret = 0;
1612 
1613 	dwork = to_delayed_work(work);
1614 
1615 	/* Process termination destroys this worker thread. So during the
1616 	 * lifetime of this thread, kfd_process p will be valid
1617 	 */
1618 	p = container_of(dwork, struct kfd_process, restore_work);
1619 	pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1620 
1621 	/* Setting last_restore_timestamp before successful restoration.
1622 	 * Otherwise this would have to be set by KGD (restore_process_bos)
1623 	 * before KFD BOs are unreserved. If not, the process can be evicted
1624 	 * again before the timestamp is set.
1625 	 * If restore fails, the timestamp will be set again in the next
1626 	 * attempt. This would mean that the minimum GPU quanta would be
1627 	 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
1628 	 * functions)
1629 	 */
1630 
1631 	p->last_restore_timestamp = get_jiffies_64();
1632 	ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1633 						     &p->ef);
1634 	if (ret) {
1635 		pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1636 			 p->pasid, PROCESS_BACK_OFF_TIME_MS);
1637 		ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1638 				msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1639 		WARN(!ret, "reschedule restore work failed\n");
1640 		return;
1641 	}
1642 
1643 	ret = kfd_process_restore_queues(p);
1644 	if (!ret)
1645 		pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1646 	else
1647 		pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1648 }
1649 
1650 void kfd_suspend_all_processes(void)
1651 {
1652 	struct kfd_process *p;
1653 	unsigned int temp;
1654 	int idx = srcu_read_lock(&kfd_processes_srcu);
1655 
1656 	WARN(debug_evictions, "Evicting all processes");
1657 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1658 		cancel_delayed_work_sync(&p->eviction_work);
1659 		cancel_delayed_work_sync(&p->restore_work);
1660 
1661 		if (kfd_process_evict_queues(p))
1662 			pr_err("Failed to suspend process 0x%x\n", p->pasid);
1663 		dma_fence_signal(p->ef);
1664 		dma_fence_put(p->ef);
1665 		p->ef = NULL;
1666 	}
1667 	srcu_read_unlock(&kfd_processes_srcu, idx);
1668 }
1669 
1670 int kfd_resume_all_processes(void)
1671 {
1672 	struct kfd_process *p;
1673 	unsigned int temp;
1674 	int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
1675 
1676 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1677 		if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
1678 			pr_err("Restore process %d failed during resume\n",
1679 			       p->pasid);
1680 			ret = -EFAULT;
1681 		}
1682 	}
1683 	srcu_read_unlock(&kfd_processes_srcu, idx);
1684 	return ret;
1685 }
1686 
1687 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
1688 			  struct vm_area_struct *vma)
1689 {
1690 	struct kfd_process_device *pdd;
1691 	struct qcm_process_device *qpd;
1692 
1693 	if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
1694 		pr_err("Incorrect CWSR mapping size.\n");
1695 		return -EINVAL;
1696 	}
1697 
1698 	pdd = kfd_get_process_device_data(dev, process);
1699 	if (!pdd)
1700 		return -EINVAL;
1701 	qpd = &pdd->qpd;
1702 
1703 	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1704 					get_order(KFD_CWSR_TBA_TMA_SIZE));
1705 	if (!qpd->cwsr_kaddr) {
1706 		pr_err("Error allocating per process CWSR buffer.\n");
1707 		return -ENOMEM;
1708 	}
1709 
1710 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
1711 		| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
1712 	/* Mapping pages to user process */
1713 	return remap_pfn_range(vma, vma->vm_start,
1714 			       PFN_DOWN(__pa(qpd->cwsr_kaddr)),
1715 			       KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
1716 }
1717 
1718 void kfd_flush_tlb(struct kfd_process_device *pdd)
1719 {
1720 	struct kfd_dev *dev = pdd->dev;
1721 
1722 	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1723 		/* Nothing to flush until a VMID is assigned, which
1724 		 * only happens when the first queue is created.
1725 		 */
1726 		if (pdd->qpd.vmid)
1727 			amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd,
1728 							pdd->qpd.vmid);
1729 	} else {
1730 		amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
1731 						pdd->process->pasid);
1732 	}
1733 }
1734 
1735 #if defined(CONFIG_DEBUG_FS)
1736 
1737 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
1738 {
1739 	struct kfd_process *p;
1740 	unsigned int temp;
1741 	int r = 0;
1742 
1743 	int idx = srcu_read_lock(&kfd_processes_srcu);
1744 
1745 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1746 		seq_printf(m, "Process %d PASID 0x%x:\n",
1747 			   p->lead_thread->tgid, p->pasid);
1748 
1749 		mutex_lock(&p->mutex);
1750 		r = pqm_debugfs_mqds(m, &p->pqm);
1751 		mutex_unlock(&p->mutex);
1752 
1753 		if (r)
1754 			break;
1755 	}
1756 
1757 	srcu_read_unlock(&kfd_processes_srcu, idx);
1758 
1759 	return r;
1760 }
1761 
1762 #endif
1763 
1764