1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <linux/device.h>
25 #include <linux/export.h>
26 #include <linux/err.h>
27 #include <linux/fs.h>
28 #include <linux/file.h>
29 #include <linux/sched.h>
30 #include <linux/slab.h>
31 #include <linux/uaccess.h>
32 #include <linux/compat.h>
33 #include <uapi/linux/kfd_ioctl.h>
34 #include <linux/time.h>
35 #include <linux/mm.h>
36 #include <linux/mman.h>
37 #include <linux/ptrace.h>
38 #include <linux/dma-buf.h>
39 #include <linux/fdtable.h>
40 #include <linux/processor.h>
41 #include "kfd_priv.h"
42 #include "kfd_device_queue_manager.h"
43 #include "kfd_svm.h"
44 #include "amdgpu_amdkfd.h"
45 #include "kfd_smi_events.h"
46 #include "amdgpu_dma_buf.h"
47 
48 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
49 static int kfd_open(struct inode *, struct file *);
50 static int kfd_release(struct inode *, struct file *);
51 static int kfd_mmap(struct file *, struct vm_area_struct *);
52 
53 static const char kfd_dev_name[] = "kfd";
54 
55 static const struct file_operations kfd_fops = {
56 	.owner = THIS_MODULE,
57 	.unlocked_ioctl = kfd_ioctl,
58 	.compat_ioctl = compat_ptr_ioctl,
59 	.open = kfd_open,
60 	.release = kfd_release,
61 	.mmap = kfd_mmap,
62 };
63 
64 static int kfd_char_dev_major = -1;
65 static struct class *kfd_class;
66 struct device *kfd_device;
67 
68 static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
69 {
70 	struct kfd_process_device *pdd;
71 
72 	mutex_lock(&p->mutex);
73 	pdd = kfd_process_device_data_by_id(p, gpu_id);
74 
75 	if (pdd)
76 		return pdd;
77 
78 	mutex_unlock(&p->mutex);
79 	return NULL;
80 }
81 
82 static inline void kfd_unlock_pdd(struct kfd_process_device *pdd)
83 {
84 	mutex_unlock(&pdd->process->mutex);
85 }
86 
87 int kfd_chardev_init(void)
88 {
89 	int err = 0;
90 
91 	kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
92 	err = kfd_char_dev_major;
93 	if (err < 0)
94 		goto err_register_chrdev;
95 
96 	kfd_class = class_create(THIS_MODULE, kfd_dev_name);
97 	err = PTR_ERR(kfd_class);
98 	if (IS_ERR(kfd_class))
99 		goto err_class_create;
100 
101 	kfd_device = device_create(kfd_class, NULL,
102 					MKDEV(kfd_char_dev_major, 0),
103 					NULL, kfd_dev_name);
104 	err = PTR_ERR(kfd_device);
105 	if (IS_ERR(kfd_device))
106 		goto err_device_create;
107 
108 	return 0;
109 
110 err_device_create:
111 	class_destroy(kfd_class);
112 err_class_create:
113 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
114 err_register_chrdev:
115 	return err;
116 }
117 
118 void kfd_chardev_exit(void)
119 {
120 	device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
121 	class_destroy(kfd_class);
122 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
123 	kfd_device = NULL;
124 }
125 
126 
127 static int kfd_open(struct inode *inode, struct file *filep)
128 {
129 	struct kfd_process *process;
130 	bool is_32bit_user_mode;
131 
132 	if (iminor(inode) != 0)
133 		return -ENODEV;
134 
135 	is_32bit_user_mode = in_compat_syscall();
136 
137 	if (is_32bit_user_mode) {
138 		dev_warn(kfd_device,
139 			"Process %d (32-bit) failed to open /dev/kfd\n"
140 			"32-bit processes are not supported by amdkfd\n",
141 			current->pid);
142 		return -EPERM;
143 	}
144 
145 	process = kfd_create_process(filep);
146 	if (IS_ERR(process))
147 		return PTR_ERR(process);
148 
149 	if (kfd_is_locked()) {
150 		dev_dbg(kfd_device, "kfd is locked!\n"
151 				"process %d unreferenced", process->pasid);
152 		kfd_unref_process(process);
153 		return -EAGAIN;
154 	}
155 
156 	/* filep now owns the reference returned by kfd_create_process */
157 	filep->private_data = process;
158 
159 	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
160 		process->pasid, process->is_32bit_user_mode);
161 
162 	return 0;
163 }
164 
165 static int kfd_release(struct inode *inode, struct file *filep)
166 {
167 	struct kfd_process *process = filep->private_data;
168 
169 	if (process)
170 		kfd_unref_process(process);
171 
172 	return 0;
173 }
174 
175 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
176 					void *data)
177 {
178 	struct kfd_ioctl_get_version_args *args = data;
179 
180 	args->major_version = KFD_IOCTL_MAJOR_VERSION;
181 	args->minor_version = KFD_IOCTL_MINOR_VERSION;
182 
183 	return 0;
184 }
185 
186 static int set_queue_properties_from_user(struct queue_properties *q_properties,
187 				struct kfd_ioctl_create_queue_args *args)
188 {
189 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
190 		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
191 		return -EINVAL;
192 	}
193 
194 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
195 		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
196 		return -EINVAL;
197 	}
198 
199 	if ((args->ring_base_address) &&
200 		(!access_ok((const void __user *) args->ring_base_address,
201 			sizeof(uint64_t)))) {
202 		pr_err("Can't access ring base address\n");
203 		return -EFAULT;
204 	}
205 
206 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
207 		pr_err("Ring size must be a power of 2 or 0\n");
208 		return -EINVAL;
209 	}
210 
211 	if (!access_ok((const void __user *) args->read_pointer_address,
212 			sizeof(uint32_t))) {
213 		pr_err("Can't access read pointer\n");
214 		return -EFAULT;
215 	}
216 
217 	if (!access_ok((const void __user *) args->write_pointer_address,
218 			sizeof(uint32_t))) {
219 		pr_err("Can't access write pointer\n");
220 		return -EFAULT;
221 	}
222 
223 	if (args->eop_buffer_address &&
224 		!access_ok((const void __user *) args->eop_buffer_address,
225 			sizeof(uint32_t))) {
226 		pr_debug("Can't access eop buffer");
227 		return -EFAULT;
228 	}
229 
230 	if (args->ctx_save_restore_address &&
231 		!access_ok((const void __user *) args->ctx_save_restore_address,
232 			sizeof(uint32_t))) {
233 		pr_debug("Can't access ctx save restore buffer");
234 		return -EFAULT;
235 	}
236 
237 	q_properties->is_interop = false;
238 	q_properties->is_gws = false;
239 	q_properties->queue_percent = args->queue_percentage;
240 	q_properties->priority = args->queue_priority;
241 	q_properties->queue_address = args->ring_base_address;
242 	q_properties->queue_size = args->ring_size;
243 	q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
244 	q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
245 	q_properties->eop_ring_buffer_address = args->eop_buffer_address;
246 	q_properties->eop_ring_buffer_size = args->eop_buffer_size;
247 	q_properties->ctx_save_restore_area_address =
248 			args->ctx_save_restore_address;
249 	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
250 	q_properties->ctl_stack_size = args->ctl_stack_size;
251 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
252 		args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
253 		q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
254 	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
255 		q_properties->type = KFD_QUEUE_TYPE_SDMA;
256 	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
257 		q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
258 	else
259 		return -ENOTSUPP;
260 
261 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
262 		q_properties->format = KFD_QUEUE_FORMAT_AQL;
263 	else
264 		q_properties->format = KFD_QUEUE_FORMAT_PM4;
265 
266 	pr_debug("Queue Percentage: %d, %d\n",
267 			q_properties->queue_percent, args->queue_percentage);
268 
269 	pr_debug("Queue Priority: %d, %d\n",
270 			q_properties->priority, args->queue_priority);
271 
272 	pr_debug("Queue Address: 0x%llX, 0x%llX\n",
273 			q_properties->queue_address, args->ring_base_address);
274 
275 	pr_debug("Queue Size: 0x%llX, %u\n",
276 			q_properties->queue_size, args->ring_size);
277 
278 	pr_debug("Queue r/w Pointers: %px, %px\n",
279 			q_properties->read_ptr,
280 			q_properties->write_ptr);
281 
282 	pr_debug("Queue Format: %d\n", q_properties->format);
283 
284 	pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
285 
286 	pr_debug("Queue CTX save area: 0x%llX\n",
287 			q_properties->ctx_save_restore_area_address);
288 
289 	return 0;
290 }
291 
292 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
293 					void *data)
294 {
295 	struct kfd_ioctl_create_queue_args *args = data;
296 	struct kfd_dev *dev;
297 	int err = 0;
298 	unsigned int queue_id;
299 	struct kfd_process_device *pdd;
300 	struct queue_properties q_properties;
301 	uint32_t doorbell_offset_in_process = 0;
302 	struct amdgpu_bo *wptr_bo = NULL;
303 
304 	memset(&q_properties, 0, sizeof(struct queue_properties));
305 
306 	pr_debug("Creating queue ioctl\n");
307 
308 	err = set_queue_properties_from_user(&q_properties, args);
309 	if (err)
310 		return err;
311 
312 	pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
313 
314 	mutex_lock(&p->mutex);
315 
316 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
317 	if (!pdd) {
318 		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
319 		err = -EINVAL;
320 		goto err_pdd;
321 	}
322 	dev = pdd->dev;
323 
324 	pdd = kfd_bind_process_to_device(dev, p);
325 	if (IS_ERR(pdd)) {
326 		err = -ESRCH;
327 		goto err_bind_process;
328 	}
329 
330 	if (!pdd->doorbell_index &&
331 	    kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
332 		err = -ENOMEM;
333 		goto err_alloc_doorbells;
334 	}
335 
336 	/* Starting with GFX11, wptr BOs must be mapped to GART for MES to determine work
337 	 * on unmapped queues for usermode queue oversubscription (no aggregated doorbell)
338 	 */
339 	if (dev->shared_resources.enable_mes &&
340 			((dev->adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK)
341 			>> AMDGPU_MES_API_VERSION_SHIFT) >= 2) {
342 		struct amdgpu_bo_va_mapping *wptr_mapping;
343 		struct amdgpu_vm *wptr_vm;
344 
345 		wptr_vm = drm_priv_to_vm(pdd->drm_priv);
346 		err = amdgpu_bo_reserve(wptr_vm->root.bo, false);
347 		if (err)
348 			goto err_wptr_map_gart;
349 
350 		wptr_mapping = amdgpu_vm_bo_lookup_mapping(
351 				wptr_vm, args->write_pointer_address >> PAGE_SHIFT);
352 		amdgpu_bo_unreserve(wptr_vm->root.bo);
353 		if (!wptr_mapping) {
354 			pr_err("Failed to lookup wptr bo\n");
355 			err = -EINVAL;
356 			goto err_wptr_map_gart;
357 		}
358 
359 		wptr_bo = wptr_mapping->bo_va->base.bo;
360 		if (wptr_bo->tbo.base.size > PAGE_SIZE) {
361 			pr_err("Requested GART mapping for wptr bo larger than one page\n");
362 			err = -EINVAL;
363 			goto err_wptr_map_gart;
364 		}
365 
366 		err = amdgpu_amdkfd_map_gtt_bo_to_gart(dev->adev, wptr_bo);
367 		if (err) {
368 			pr_err("Failed to map wptr bo to GART\n");
369 			goto err_wptr_map_gart;
370 		}
371 	}
372 
373 	pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
374 			p->pasid,
375 			dev->id);
376 
377 	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, wptr_bo,
378 			NULL, NULL, NULL, &doorbell_offset_in_process);
379 	if (err != 0)
380 		goto err_create_queue;
381 
382 	args->queue_id = queue_id;
383 
384 
385 	/* Return gpu_id as doorbell offset for mmap usage */
386 	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
387 	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
388 	if (KFD_IS_SOC15(dev))
389 		/* On SOC15 ASICs, include the doorbell offset within the
390 		 * process doorbell frame, which is 2 pages.
391 		 */
392 		args->doorbell_offset |= doorbell_offset_in_process;
393 
394 	mutex_unlock(&p->mutex);
395 
396 	pr_debug("Queue id %d was created successfully\n", args->queue_id);
397 
398 	pr_debug("Ring buffer address == 0x%016llX\n",
399 			args->ring_base_address);
400 
401 	pr_debug("Read ptr address    == 0x%016llX\n",
402 			args->read_pointer_address);
403 
404 	pr_debug("Write ptr address   == 0x%016llX\n",
405 			args->write_pointer_address);
406 
407 	return 0;
408 
409 err_create_queue:
410 	if (wptr_bo)
411 		amdgpu_amdkfd_free_gtt_mem(dev->adev, wptr_bo);
412 err_wptr_map_gart:
413 err_alloc_doorbells:
414 err_bind_process:
415 err_pdd:
416 	mutex_unlock(&p->mutex);
417 	return err;
418 }
419 
420 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
421 					void *data)
422 {
423 	int retval;
424 	struct kfd_ioctl_destroy_queue_args *args = data;
425 
426 	pr_debug("Destroying queue id %d for pasid 0x%x\n",
427 				args->queue_id,
428 				p->pasid);
429 
430 	mutex_lock(&p->mutex);
431 
432 	retval = pqm_destroy_queue(&p->pqm, args->queue_id);
433 
434 	mutex_unlock(&p->mutex);
435 	return retval;
436 }
437 
438 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
439 					void *data)
440 {
441 	int retval;
442 	struct kfd_ioctl_update_queue_args *args = data;
443 	struct queue_properties properties;
444 
445 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
446 		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
447 		return -EINVAL;
448 	}
449 
450 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
451 		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
452 		return -EINVAL;
453 	}
454 
455 	if ((args->ring_base_address) &&
456 		(!access_ok((const void __user *) args->ring_base_address,
457 			sizeof(uint64_t)))) {
458 		pr_err("Can't access ring base address\n");
459 		return -EFAULT;
460 	}
461 
462 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
463 		pr_err("Ring size must be a power of 2 or 0\n");
464 		return -EINVAL;
465 	}
466 
467 	properties.queue_address = args->ring_base_address;
468 	properties.queue_size = args->ring_size;
469 	properties.queue_percent = args->queue_percentage;
470 	properties.priority = args->queue_priority;
471 
472 	pr_debug("Updating queue id %d for pasid 0x%x\n",
473 			args->queue_id, p->pasid);
474 
475 	mutex_lock(&p->mutex);
476 
477 	retval = pqm_update_queue_properties(&p->pqm, args->queue_id, &properties);
478 
479 	mutex_unlock(&p->mutex);
480 
481 	return retval;
482 }
483 
484 static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
485 					void *data)
486 {
487 	int retval;
488 	const int max_num_cus = 1024;
489 	struct kfd_ioctl_set_cu_mask_args *args = data;
490 	struct mqd_update_info minfo = {0};
491 	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
492 	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
493 
494 	if ((args->num_cu_mask % 32) != 0) {
495 		pr_debug("num_cu_mask 0x%x must be a multiple of 32",
496 				args->num_cu_mask);
497 		return -EINVAL;
498 	}
499 
500 	minfo.cu_mask.count = args->num_cu_mask;
501 	if (minfo.cu_mask.count == 0) {
502 		pr_debug("CU mask cannot be 0");
503 		return -EINVAL;
504 	}
505 
506 	/* To prevent an unreasonably large CU mask size, set an arbitrary
507 	 * limit of max_num_cus bits.  We can then just drop any CU mask bits
508 	 * past max_num_cus bits and just use the first max_num_cus bits.
509 	 */
510 	if (minfo.cu_mask.count > max_num_cus) {
511 		pr_debug("CU mask cannot be greater than 1024 bits");
512 		minfo.cu_mask.count = max_num_cus;
513 		cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
514 	}
515 
516 	minfo.cu_mask.ptr = kzalloc(cu_mask_size, GFP_KERNEL);
517 	if (!minfo.cu_mask.ptr)
518 		return -ENOMEM;
519 
520 	retval = copy_from_user(minfo.cu_mask.ptr, cu_mask_ptr, cu_mask_size);
521 	if (retval) {
522 		pr_debug("Could not copy CU mask from userspace");
523 		retval = -EFAULT;
524 		goto out;
525 	}
526 
527 	minfo.update_flag = UPDATE_FLAG_CU_MASK;
528 
529 	mutex_lock(&p->mutex);
530 
531 	retval = pqm_update_mqd(&p->pqm, args->queue_id, &minfo);
532 
533 	mutex_unlock(&p->mutex);
534 
535 out:
536 	kfree(minfo.cu_mask.ptr);
537 	return retval;
538 }
539 
540 static int kfd_ioctl_get_queue_wave_state(struct file *filep,
541 					  struct kfd_process *p, void *data)
542 {
543 	struct kfd_ioctl_get_queue_wave_state_args *args = data;
544 	int r;
545 
546 	mutex_lock(&p->mutex);
547 
548 	r = pqm_get_wave_state(&p->pqm, args->queue_id,
549 			       (void __user *)args->ctl_stack_address,
550 			       &args->ctl_stack_used_size,
551 			       &args->save_area_used_size);
552 
553 	mutex_unlock(&p->mutex);
554 
555 	return r;
556 }
557 
558 static int kfd_ioctl_set_memory_policy(struct file *filep,
559 					struct kfd_process *p, void *data)
560 {
561 	struct kfd_ioctl_set_memory_policy_args *args = data;
562 	int err = 0;
563 	struct kfd_process_device *pdd;
564 	enum cache_policy default_policy, alternate_policy;
565 
566 	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
567 	    && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
568 		return -EINVAL;
569 	}
570 
571 	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
572 	    && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
573 		return -EINVAL;
574 	}
575 
576 	mutex_lock(&p->mutex);
577 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
578 	if (!pdd) {
579 		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
580 		err = -EINVAL;
581 		goto err_pdd;
582 	}
583 
584 	pdd = kfd_bind_process_to_device(pdd->dev, p);
585 	if (IS_ERR(pdd)) {
586 		err = -ESRCH;
587 		goto out;
588 	}
589 
590 	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
591 			 ? cache_policy_coherent : cache_policy_noncoherent;
592 
593 	alternate_policy =
594 		(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
595 		   ? cache_policy_coherent : cache_policy_noncoherent;
596 
597 	if (!pdd->dev->dqm->ops.set_cache_memory_policy(pdd->dev->dqm,
598 				&pdd->qpd,
599 				default_policy,
600 				alternate_policy,
601 				(void __user *)args->alternate_aperture_base,
602 				args->alternate_aperture_size))
603 		err = -EINVAL;
604 
605 out:
606 err_pdd:
607 	mutex_unlock(&p->mutex);
608 
609 	return err;
610 }
611 
612 static int kfd_ioctl_set_trap_handler(struct file *filep,
613 					struct kfd_process *p, void *data)
614 {
615 	struct kfd_ioctl_set_trap_handler_args *args = data;
616 	int err = 0;
617 	struct kfd_process_device *pdd;
618 
619 	mutex_lock(&p->mutex);
620 
621 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
622 	if (!pdd) {
623 		err = -EINVAL;
624 		goto err_pdd;
625 	}
626 
627 	pdd = kfd_bind_process_to_device(pdd->dev, p);
628 	if (IS_ERR(pdd)) {
629 		err = -ESRCH;
630 		goto out;
631 	}
632 
633 	kfd_process_set_trap_handler(&pdd->qpd, args->tba_addr, args->tma_addr);
634 
635 out:
636 err_pdd:
637 	mutex_unlock(&p->mutex);
638 
639 	return err;
640 }
641 
642 static int kfd_ioctl_dbg_register(struct file *filep,
643 				struct kfd_process *p, void *data)
644 {
645 	return -EPERM;
646 }
647 
648 static int kfd_ioctl_dbg_unregister(struct file *filep,
649 				struct kfd_process *p, void *data)
650 {
651 	return -EPERM;
652 }
653 
654 static int kfd_ioctl_dbg_address_watch(struct file *filep,
655 					struct kfd_process *p, void *data)
656 {
657 	return -EPERM;
658 }
659 
660 /* Parse and generate fixed size data structure for wave control */
661 static int kfd_ioctl_dbg_wave_control(struct file *filep,
662 					struct kfd_process *p, void *data)
663 {
664 	return -EPERM;
665 }
666 
667 static int kfd_ioctl_get_clock_counters(struct file *filep,
668 				struct kfd_process *p, void *data)
669 {
670 	struct kfd_ioctl_get_clock_counters_args *args = data;
671 	struct kfd_process_device *pdd;
672 
673 	mutex_lock(&p->mutex);
674 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
675 	mutex_unlock(&p->mutex);
676 	if (pdd)
677 		/* Reading GPU clock counter from KGD */
678 		args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(pdd->dev->adev);
679 	else
680 		/* Node without GPU resource */
681 		args->gpu_clock_counter = 0;
682 
683 	/* No access to rdtsc. Using raw monotonic time */
684 	args->cpu_clock_counter = ktime_get_raw_ns();
685 	args->system_clock_counter = ktime_get_boottime_ns();
686 
687 	/* Since the counter is in nano-seconds we use 1GHz frequency */
688 	args->system_clock_freq = 1000000000;
689 
690 	return 0;
691 }
692 
693 
694 static int kfd_ioctl_get_process_apertures(struct file *filp,
695 				struct kfd_process *p, void *data)
696 {
697 	struct kfd_ioctl_get_process_apertures_args *args = data;
698 	struct kfd_process_device_apertures *pAperture;
699 	int i;
700 
701 	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
702 
703 	args->num_of_nodes = 0;
704 
705 	mutex_lock(&p->mutex);
706 	/* Run over all pdd of the process */
707 	for (i = 0; i < p->n_pdds; i++) {
708 		struct kfd_process_device *pdd = p->pdds[i];
709 
710 		pAperture =
711 			&args->process_apertures[args->num_of_nodes];
712 		pAperture->gpu_id = pdd->dev->id;
713 		pAperture->lds_base = pdd->lds_base;
714 		pAperture->lds_limit = pdd->lds_limit;
715 		pAperture->gpuvm_base = pdd->gpuvm_base;
716 		pAperture->gpuvm_limit = pdd->gpuvm_limit;
717 		pAperture->scratch_base = pdd->scratch_base;
718 		pAperture->scratch_limit = pdd->scratch_limit;
719 
720 		dev_dbg(kfd_device,
721 			"node id %u\n", args->num_of_nodes);
722 		dev_dbg(kfd_device,
723 			"gpu id %u\n", pdd->dev->id);
724 		dev_dbg(kfd_device,
725 			"lds_base %llX\n", pdd->lds_base);
726 		dev_dbg(kfd_device,
727 			"lds_limit %llX\n", pdd->lds_limit);
728 		dev_dbg(kfd_device,
729 			"gpuvm_base %llX\n", pdd->gpuvm_base);
730 		dev_dbg(kfd_device,
731 			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
732 		dev_dbg(kfd_device,
733 			"scratch_base %llX\n", pdd->scratch_base);
734 		dev_dbg(kfd_device,
735 			"scratch_limit %llX\n", pdd->scratch_limit);
736 
737 		if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
738 			break;
739 	}
740 	mutex_unlock(&p->mutex);
741 
742 	return 0;
743 }
744 
745 static int kfd_ioctl_get_process_apertures_new(struct file *filp,
746 				struct kfd_process *p, void *data)
747 {
748 	struct kfd_ioctl_get_process_apertures_new_args *args = data;
749 	struct kfd_process_device_apertures *pa;
750 	int ret;
751 	int i;
752 
753 	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
754 
755 	if (args->num_of_nodes == 0) {
756 		/* Return number of nodes, so that user space can alloacate
757 		 * sufficient memory
758 		 */
759 		mutex_lock(&p->mutex);
760 		args->num_of_nodes = p->n_pdds;
761 		goto out_unlock;
762 	}
763 
764 	/* Fill in process-aperture information for all available
765 	 * nodes, but not more than args->num_of_nodes as that is
766 	 * the amount of memory allocated by user
767 	 */
768 	pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
769 				args->num_of_nodes), GFP_KERNEL);
770 	if (!pa)
771 		return -ENOMEM;
772 
773 	mutex_lock(&p->mutex);
774 
775 	if (!p->n_pdds) {
776 		args->num_of_nodes = 0;
777 		kfree(pa);
778 		goto out_unlock;
779 	}
780 
781 	/* Run over all pdd of the process */
782 	for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
783 		struct kfd_process_device *pdd = p->pdds[i];
784 
785 		pa[i].gpu_id = pdd->dev->id;
786 		pa[i].lds_base = pdd->lds_base;
787 		pa[i].lds_limit = pdd->lds_limit;
788 		pa[i].gpuvm_base = pdd->gpuvm_base;
789 		pa[i].gpuvm_limit = pdd->gpuvm_limit;
790 		pa[i].scratch_base = pdd->scratch_base;
791 		pa[i].scratch_limit = pdd->scratch_limit;
792 
793 		dev_dbg(kfd_device,
794 			"gpu id %u\n", pdd->dev->id);
795 		dev_dbg(kfd_device,
796 			"lds_base %llX\n", pdd->lds_base);
797 		dev_dbg(kfd_device,
798 			"lds_limit %llX\n", pdd->lds_limit);
799 		dev_dbg(kfd_device,
800 			"gpuvm_base %llX\n", pdd->gpuvm_base);
801 		dev_dbg(kfd_device,
802 			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
803 		dev_dbg(kfd_device,
804 			"scratch_base %llX\n", pdd->scratch_base);
805 		dev_dbg(kfd_device,
806 			"scratch_limit %llX\n", pdd->scratch_limit);
807 	}
808 	mutex_unlock(&p->mutex);
809 
810 	args->num_of_nodes = i;
811 	ret = copy_to_user(
812 			(void __user *)args->kfd_process_device_apertures_ptr,
813 			pa,
814 			(i * sizeof(struct kfd_process_device_apertures)));
815 	kfree(pa);
816 	return ret ? -EFAULT : 0;
817 
818 out_unlock:
819 	mutex_unlock(&p->mutex);
820 	return 0;
821 }
822 
823 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
824 					void *data)
825 {
826 	struct kfd_ioctl_create_event_args *args = data;
827 	int err;
828 
829 	/* For dGPUs the event page is allocated in user mode. The
830 	 * handle is passed to KFD with the first call to this IOCTL
831 	 * through the event_page_offset field.
832 	 */
833 	if (args->event_page_offset) {
834 		mutex_lock(&p->mutex);
835 		err = kfd_kmap_event_page(p, args->event_page_offset);
836 		mutex_unlock(&p->mutex);
837 		if (err)
838 			return err;
839 	}
840 
841 	err = kfd_event_create(filp, p, args->event_type,
842 				args->auto_reset != 0, args->node_id,
843 				&args->event_id, &args->event_trigger_data,
844 				&args->event_page_offset,
845 				&args->event_slot_index);
846 
847 	pr_debug("Created event (id:0x%08x) (%s)\n", args->event_id, __func__);
848 	return err;
849 }
850 
851 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
852 					void *data)
853 {
854 	struct kfd_ioctl_destroy_event_args *args = data;
855 
856 	return kfd_event_destroy(p, args->event_id);
857 }
858 
859 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
860 				void *data)
861 {
862 	struct kfd_ioctl_set_event_args *args = data;
863 
864 	return kfd_set_event(p, args->event_id);
865 }
866 
867 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
868 				void *data)
869 {
870 	struct kfd_ioctl_reset_event_args *args = data;
871 
872 	return kfd_reset_event(p, args->event_id);
873 }
874 
875 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
876 				void *data)
877 {
878 	struct kfd_ioctl_wait_events_args *args = data;
879 
880 	return kfd_wait_on_events(p, args->num_events,
881 			(void __user *)args->events_ptr,
882 			(args->wait_for_all != 0),
883 			&args->timeout, &args->wait_result);
884 }
885 static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
886 					struct kfd_process *p, void *data)
887 {
888 	struct kfd_ioctl_set_scratch_backing_va_args *args = data;
889 	struct kfd_process_device *pdd;
890 	struct kfd_dev *dev;
891 	long err;
892 
893 	mutex_lock(&p->mutex);
894 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
895 	if (!pdd) {
896 		err = -EINVAL;
897 		goto err_pdd;
898 	}
899 	dev = pdd->dev;
900 
901 	pdd = kfd_bind_process_to_device(dev, p);
902 	if (IS_ERR(pdd)) {
903 		err = PTR_ERR(pdd);
904 		goto bind_process_to_device_fail;
905 	}
906 
907 	pdd->qpd.sh_hidden_private_base = args->va_addr;
908 
909 	mutex_unlock(&p->mutex);
910 
911 	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
912 	    pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
913 		dev->kfd2kgd->set_scratch_backing_va(
914 			dev->adev, args->va_addr, pdd->qpd.vmid);
915 
916 	return 0;
917 
918 bind_process_to_device_fail:
919 err_pdd:
920 	mutex_unlock(&p->mutex);
921 	return err;
922 }
923 
924 static int kfd_ioctl_get_tile_config(struct file *filep,
925 		struct kfd_process *p, void *data)
926 {
927 	struct kfd_ioctl_get_tile_config_args *args = data;
928 	struct kfd_process_device *pdd;
929 	struct tile_config config;
930 	int err = 0;
931 
932 	mutex_lock(&p->mutex);
933 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
934 	mutex_unlock(&p->mutex);
935 	if (!pdd)
936 		return -EINVAL;
937 
938 	amdgpu_amdkfd_get_tile_config(pdd->dev->adev, &config);
939 
940 	args->gb_addr_config = config.gb_addr_config;
941 	args->num_banks = config.num_banks;
942 	args->num_ranks = config.num_ranks;
943 
944 	if (args->num_tile_configs > config.num_tile_configs)
945 		args->num_tile_configs = config.num_tile_configs;
946 	err = copy_to_user((void __user *)args->tile_config_ptr,
947 			config.tile_config_ptr,
948 			args->num_tile_configs * sizeof(uint32_t));
949 	if (err) {
950 		args->num_tile_configs = 0;
951 		return -EFAULT;
952 	}
953 
954 	if (args->num_macro_tile_configs > config.num_macro_tile_configs)
955 		args->num_macro_tile_configs =
956 				config.num_macro_tile_configs;
957 	err = copy_to_user((void __user *)args->macro_tile_config_ptr,
958 			config.macro_tile_config_ptr,
959 			args->num_macro_tile_configs * sizeof(uint32_t));
960 	if (err) {
961 		args->num_macro_tile_configs = 0;
962 		return -EFAULT;
963 	}
964 
965 	return 0;
966 }
967 
968 static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
969 				void *data)
970 {
971 	struct kfd_ioctl_acquire_vm_args *args = data;
972 	struct kfd_process_device *pdd;
973 	struct file *drm_file;
974 	int ret;
975 
976 	drm_file = fget(args->drm_fd);
977 	if (!drm_file)
978 		return -EINVAL;
979 
980 	mutex_lock(&p->mutex);
981 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
982 	if (!pdd) {
983 		ret = -EINVAL;
984 		goto err_pdd;
985 	}
986 
987 	if (pdd->drm_file) {
988 		ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
989 		goto err_drm_file;
990 	}
991 
992 	ret = kfd_process_device_init_vm(pdd, drm_file);
993 	if (ret)
994 		goto err_unlock;
995 
996 	/* On success, the PDD keeps the drm_file reference */
997 	mutex_unlock(&p->mutex);
998 
999 	return 0;
1000 
1001 err_unlock:
1002 err_pdd:
1003 err_drm_file:
1004 	mutex_unlock(&p->mutex);
1005 	fput(drm_file);
1006 	return ret;
1007 }
1008 
1009 bool kfd_dev_is_large_bar(struct kfd_dev *dev)
1010 {
1011 	if (debug_largebar) {
1012 		pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1013 		return true;
1014 	}
1015 
1016 	if (dev->use_iommu_v2)
1017 		return false;
1018 
1019 	if (dev->local_mem_info.local_mem_size_private == 0 &&
1020 			dev->local_mem_info.local_mem_size_public > 0)
1021 		return true;
1022 	return false;
1023 }
1024 
1025 static int kfd_ioctl_get_available_memory(struct file *filep,
1026 					  struct kfd_process *p, void *data)
1027 {
1028 	struct kfd_ioctl_get_available_memory_args *args = data;
1029 	struct kfd_process_device *pdd = kfd_lock_pdd_by_id(p, args->gpu_id);
1030 
1031 	if (!pdd)
1032 		return -EINVAL;
1033 	args->available = amdgpu_amdkfd_get_available_memory(pdd->dev->adev);
1034 	kfd_unlock_pdd(pdd);
1035 	return 0;
1036 }
1037 
1038 static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1039 					struct kfd_process *p, void *data)
1040 {
1041 	struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1042 	struct kfd_process_device *pdd;
1043 	void *mem;
1044 	struct kfd_dev *dev;
1045 	int idr_handle;
1046 	long err;
1047 	uint64_t offset = args->mmap_offset;
1048 	uint32_t flags = args->flags;
1049 
1050 	if (args->size == 0)
1051 		return -EINVAL;
1052 
1053 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1054 	/* Flush pending deferred work to avoid racing with deferred actions
1055 	 * from previous memory map changes (e.g. munmap).
1056 	 */
1057 	svm_range_list_lock_and_flush_work(&p->svms, current->mm);
1058 	mutex_lock(&p->svms.lock);
1059 	mmap_write_unlock(current->mm);
1060 	if (interval_tree_iter_first(&p->svms.objects,
1061 				     args->va_addr >> PAGE_SHIFT,
1062 				     (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1063 		pr_err("Address: 0x%llx already allocated by SVM\n",
1064 			args->va_addr);
1065 		mutex_unlock(&p->svms.lock);
1066 		return -EADDRINUSE;
1067 	}
1068 	mutex_unlock(&p->svms.lock);
1069 #endif
1070 	mutex_lock(&p->mutex);
1071 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1072 	if (!pdd) {
1073 		err = -EINVAL;
1074 		goto err_pdd;
1075 	}
1076 
1077 	dev = pdd->dev;
1078 
1079 	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1080 		(flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1081 		!kfd_dev_is_large_bar(dev)) {
1082 		pr_err("Alloc host visible vram on small bar is not allowed\n");
1083 		err = -EINVAL;
1084 		goto err_large_bar;
1085 	}
1086 
1087 	pdd = kfd_bind_process_to_device(dev, p);
1088 	if (IS_ERR(pdd)) {
1089 		err = PTR_ERR(pdd);
1090 		goto err_unlock;
1091 	}
1092 
1093 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1094 		if (args->size != kfd_doorbell_process_slice(dev)) {
1095 			err = -EINVAL;
1096 			goto err_unlock;
1097 		}
1098 		offset = kfd_get_process_doorbells(pdd);
1099 		if (!offset) {
1100 			err = -ENOMEM;
1101 			goto err_unlock;
1102 		}
1103 	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1104 		if (args->size != PAGE_SIZE) {
1105 			err = -EINVAL;
1106 			goto err_unlock;
1107 		}
1108 		offset = dev->adev->rmmio_remap.bus_addr;
1109 		if (!offset) {
1110 			err = -ENOMEM;
1111 			goto err_unlock;
1112 		}
1113 	}
1114 
1115 	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1116 		dev->adev, args->va_addr, args->size,
1117 		pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
1118 		flags, false);
1119 
1120 	if (err)
1121 		goto err_unlock;
1122 
1123 	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1124 	if (idr_handle < 0) {
1125 		err = -EFAULT;
1126 		goto err_free;
1127 	}
1128 
1129 	/* Update the VRAM usage count */
1130 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
1131 		WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + args->size);
1132 
1133 	mutex_unlock(&p->mutex);
1134 
1135 	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1136 	args->mmap_offset = offset;
1137 
1138 	/* MMIO is mapped through kfd device
1139 	 * Generate a kfd mmap offset
1140 	 */
1141 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1142 		args->mmap_offset = KFD_MMAP_TYPE_MMIO
1143 					| KFD_MMAP_GPU_ID(args->gpu_id);
1144 
1145 	return 0;
1146 
1147 err_free:
1148 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->adev, (struct kgd_mem *)mem,
1149 					       pdd->drm_priv, NULL);
1150 err_unlock:
1151 err_pdd:
1152 err_large_bar:
1153 	mutex_unlock(&p->mutex);
1154 	return err;
1155 }
1156 
1157 static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1158 					struct kfd_process *p, void *data)
1159 {
1160 	struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1161 	struct kfd_process_device *pdd;
1162 	void *mem;
1163 	int ret;
1164 	uint64_t size = 0;
1165 
1166 	mutex_lock(&p->mutex);
1167 	/*
1168 	 * Safeguard to prevent user space from freeing signal BO.
1169 	 * It will be freed at process termination.
1170 	 */
1171 	if (p->signal_handle && (p->signal_handle == args->handle)) {
1172 		pr_err("Free signal BO is not allowed\n");
1173 		ret = -EPERM;
1174 		goto err_unlock;
1175 	}
1176 
1177 	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1178 	if (!pdd) {
1179 		pr_err("Process device data doesn't exist\n");
1180 		ret = -EINVAL;
1181 		goto err_pdd;
1182 	}
1183 
1184 	mem = kfd_process_device_translate_handle(
1185 		pdd, GET_IDR_HANDLE(args->handle));
1186 	if (!mem) {
1187 		ret = -EINVAL;
1188 		goto err_unlock;
1189 	}
1190 
1191 	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev,
1192 				(struct kgd_mem *)mem, pdd->drm_priv, &size);
1193 
1194 	/* If freeing the buffer failed, leave the handle in place for
1195 	 * clean-up during process tear-down.
1196 	 */
1197 	if (!ret)
1198 		kfd_process_device_remove_obj_handle(
1199 			pdd, GET_IDR_HANDLE(args->handle));
1200 
1201 	WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1202 
1203 err_unlock:
1204 err_pdd:
1205 	mutex_unlock(&p->mutex);
1206 	return ret;
1207 }
1208 
1209 static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1210 					struct kfd_process *p, void *data)
1211 {
1212 	struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1213 	struct kfd_process_device *pdd, *peer_pdd;
1214 	void *mem;
1215 	struct kfd_dev *dev;
1216 	long err = 0;
1217 	int i;
1218 	uint32_t *devices_arr = NULL;
1219 
1220 	if (!args->n_devices) {
1221 		pr_debug("Device IDs array empty\n");
1222 		return -EINVAL;
1223 	}
1224 	if (args->n_success > args->n_devices) {
1225 		pr_debug("n_success exceeds n_devices\n");
1226 		return -EINVAL;
1227 	}
1228 
1229 	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1230 				    GFP_KERNEL);
1231 	if (!devices_arr)
1232 		return -ENOMEM;
1233 
1234 	err = copy_from_user(devices_arr,
1235 			     (void __user *)args->device_ids_array_ptr,
1236 			     args->n_devices * sizeof(*devices_arr));
1237 	if (err != 0) {
1238 		err = -EFAULT;
1239 		goto copy_from_user_failed;
1240 	}
1241 
1242 	mutex_lock(&p->mutex);
1243 	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1244 	if (!pdd) {
1245 		err = -EINVAL;
1246 		goto get_process_device_data_failed;
1247 	}
1248 	dev = pdd->dev;
1249 
1250 	pdd = kfd_bind_process_to_device(dev, p);
1251 	if (IS_ERR(pdd)) {
1252 		err = PTR_ERR(pdd);
1253 		goto bind_process_to_device_failed;
1254 	}
1255 
1256 	mem = kfd_process_device_translate_handle(pdd,
1257 						GET_IDR_HANDLE(args->handle));
1258 	if (!mem) {
1259 		err = -ENOMEM;
1260 		goto get_mem_obj_from_handle_failed;
1261 	}
1262 
1263 	for (i = args->n_success; i < args->n_devices; i++) {
1264 		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1265 		if (!peer_pdd) {
1266 			pr_debug("Getting device by id failed for 0x%x\n",
1267 				 devices_arr[i]);
1268 			err = -EINVAL;
1269 			goto get_mem_obj_from_handle_failed;
1270 		}
1271 
1272 		peer_pdd = kfd_bind_process_to_device(peer_pdd->dev, p);
1273 		if (IS_ERR(peer_pdd)) {
1274 			err = PTR_ERR(peer_pdd);
1275 			goto get_mem_obj_from_handle_failed;
1276 		}
1277 
1278 		err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1279 			peer_pdd->dev->adev, (struct kgd_mem *)mem,
1280 			peer_pdd->drm_priv);
1281 		if (err) {
1282 			struct pci_dev *pdev = peer_pdd->dev->adev->pdev;
1283 
1284 			dev_err(dev->adev->dev,
1285 			       "Failed to map peer:%04x:%02x:%02x.%d mem_domain:%d\n",
1286 			       pci_domain_nr(pdev->bus),
1287 			       pdev->bus->number,
1288 			       PCI_SLOT(pdev->devfn),
1289 			       PCI_FUNC(pdev->devfn),
1290 			       ((struct kgd_mem *)mem)->domain);
1291 			goto map_memory_to_gpu_failed;
1292 		}
1293 		args->n_success = i+1;
1294 	}
1295 
1296 	mutex_unlock(&p->mutex);
1297 
1298 	err = amdgpu_amdkfd_gpuvm_sync_memory(dev->adev, (struct kgd_mem *) mem, true);
1299 	if (err) {
1300 		pr_debug("Sync memory failed, wait interrupted by user signal\n");
1301 		goto sync_memory_failed;
1302 	}
1303 
1304 	/* Flush TLBs after waiting for the page table updates to complete */
1305 	for (i = 0; i < args->n_devices; i++) {
1306 		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1307 		if (WARN_ON_ONCE(!peer_pdd))
1308 			continue;
1309 		kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
1310 	}
1311 	kfree(devices_arr);
1312 
1313 	return err;
1314 
1315 get_process_device_data_failed:
1316 bind_process_to_device_failed:
1317 get_mem_obj_from_handle_failed:
1318 map_memory_to_gpu_failed:
1319 	mutex_unlock(&p->mutex);
1320 copy_from_user_failed:
1321 sync_memory_failed:
1322 	kfree(devices_arr);
1323 
1324 	return err;
1325 }
1326 
1327 static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1328 					struct kfd_process *p, void *data)
1329 {
1330 	struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1331 	struct kfd_process_device *pdd, *peer_pdd;
1332 	void *mem;
1333 	long err = 0;
1334 	uint32_t *devices_arr = NULL, i;
1335 
1336 	if (!args->n_devices) {
1337 		pr_debug("Device IDs array empty\n");
1338 		return -EINVAL;
1339 	}
1340 	if (args->n_success > args->n_devices) {
1341 		pr_debug("n_success exceeds n_devices\n");
1342 		return -EINVAL;
1343 	}
1344 
1345 	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1346 				    GFP_KERNEL);
1347 	if (!devices_arr)
1348 		return -ENOMEM;
1349 
1350 	err = copy_from_user(devices_arr,
1351 			     (void __user *)args->device_ids_array_ptr,
1352 			     args->n_devices * sizeof(*devices_arr));
1353 	if (err != 0) {
1354 		err = -EFAULT;
1355 		goto copy_from_user_failed;
1356 	}
1357 
1358 	mutex_lock(&p->mutex);
1359 	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(args->handle));
1360 	if (!pdd) {
1361 		err = -EINVAL;
1362 		goto bind_process_to_device_failed;
1363 	}
1364 
1365 	mem = kfd_process_device_translate_handle(pdd,
1366 						GET_IDR_HANDLE(args->handle));
1367 	if (!mem) {
1368 		err = -ENOMEM;
1369 		goto get_mem_obj_from_handle_failed;
1370 	}
1371 
1372 	for (i = args->n_success; i < args->n_devices; i++) {
1373 		peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1374 		if (!peer_pdd) {
1375 			err = -EINVAL;
1376 			goto get_mem_obj_from_handle_failed;
1377 		}
1378 		err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1379 			peer_pdd->dev->adev, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1380 		if (err) {
1381 			pr_err("Failed to unmap from gpu %d/%d\n",
1382 			       i, args->n_devices);
1383 			goto unmap_memory_from_gpu_failed;
1384 		}
1385 		args->n_success = i+1;
1386 	}
1387 	mutex_unlock(&p->mutex);
1388 
1389 	if (kfd_flush_tlb_after_unmap(pdd->dev)) {
1390 		err = amdgpu_amdkfd_gpuvm_sync_memory(pdd->dev->adev,
1391 				(struct kgd_mem *) mem, true);
1392 		if (err) {
1393 			pr_debug("Sync memory failed, wait interrupted by user signal\n");
1394 			goto sync_memory_failed;
1395 		}
1396 
1397 		/* Flush TLBs after waiting for the page table updates to complete */
1398 		for (i = 0; i < args->n_devices; i++) {
1399 			peer_pdd = kfd_process_device_data_by_id(p, devices_arr[i]);
1400 			if (WARN_ON_ONCE(!peer_pdd))
1401 				continue;
1402 			kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
1403 		}
1404 	}
1405 	kfree(devices_arr);
1406 
1407 	return 0;
1408 
1409 bind_process_to_device_failed:
1410 get_mem_obj_from_handle_failed:
1411 unmap_memory_from_gpu_failed:
1412 	mutex_unlock(&p->mutex);
1413 copy_from_user_failed:
1414 sync_memory_failed:
1415 	kfree(devices_arr);
1416 	return err;
1417 }
1418 
1419 static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1420 		struct kfd_process *p, void *data)
1421 {
1422 	int retval;
1423 	struct kfd_ioctl_alloc_queue_gws_args *args = data;
1424 	struct queue *q;
1425 	struct kfd_dev *dev;
1426 
1427 	mutex_lock(&p->mutex);
1428 	q = pqm_get_user_queue(&p->pqm, args->queue_id);
1429 
1430 	if (q) {
1431 		dev = q->device;
1432 	} else {
1433 		retval = -EINVAL;
1434 		goto out_unlock;
1435 	}
1436 
1437 	if (!dev->gws) {
1438 		retval = -ENODEV;
1439 		goto out_unlock;
1440 	}
1441 
1442 	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1443 		retval = -ENODEV;
1444 		goto out_unlock;
1445 	}
1446 
1447 	retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1448 	mutex_unlock(&p->mutex);
1449 
1450 	args->first_gws = 0;
1451 	return retval;
1452 
1453 out_unlock:
1454 	mutex_unlock(&p->mutex);
1455 	return retval;
1456 }
1457 
1458 static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1459 		struct kfd_process *p, void *data)
1460 {
1461 	struct kfd_ioctl_get_dmabuf_info_args *args = data;
1462 	struct kfd_dev *dev = NULL;
1463 	struct amdgpu_device *dmabuf_adev;
1464 	void *metadata_buffer = NULL;
1465 	uint32_t flags;
1466 	unsigned int i;
1467 	int r;
1468 
1469 	/* Find a KFD GPU device that supports the get_dmabuf_info query */
1470 	for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1471 		if (dev)
1472 			break;
1473 	if (!dev)
1474 		return -EINVAL;
1475 
1476 	if (args->metadata_ptr) {
1477 		metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1478 		if (!metadata_buffer)
1479 			return -ENOMEM;
1480 	}
1481 
1482 	/* Get dmabuf info from KGD */
1483 	r = amdgpu_amdkfd_get_dmabuf_info(dev->adev, args->dmabuf_fd,
1484 					  &dmabuf_adev, &args->size,
1485 					  metadata_buffer, args->metadata_size,
1486 					  &args->metadata_size, &flags);
1487 	if (r)
1488 		goto exit;
1489 
1490 	/* Reverse-lookup gpu_id from kgd pointer */
1491 	dev = kfd_device_by_adev(dmabuf_adev);
1492 	if (!dev) {
1493 		r = -EINVAL;
1494 		goto exit;
1495 	}
1496 	args->gpu_id = dev->id;
1497 	args->flags = flags;
1498 
1499 	/* Copy metadata buffer to user mode */
1500 	if (metadata_buffer) {
1501 		r = copy_to_user((void __user *)args->metadata_ptr,
1502 				 metadata_buffer, args->metadata_size);
1503 		if (r != 0)
1504 			r = -EFAULT;
1505 	}
1506 
1507 exit:
1508 	kfree(metadata_buffer);
1509 
1510 	return r;
1511 }
1512 
1513 static int kfd_ioctl_import_dmabuf(struct file *filep,
1514 				   struct kfd_process *p, void *data)
1515 {
1516 	struct kfd_ioctl_import_dmabuf_args *args = data;
1517 	struct kfd_process_device *pdd;
1518 	struct dma_buf *dmabuf;
1519 	int idr_handle;
1520 	uint64_t size;
1521 	void *mem;
1522 	int r;
1523 
1524 	dmabuf = dma_buf_get(args->dmabuf_fd);
1525 	if (IS_ERR(dmabuf))
1526 		return PTR_ERR(dmabuf);
1527 
1528 	mutex_lock(&p->mutex);
1529 	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
1530 	if (!pdd) {
1531 		r = -EINVAL;
1532 		goto err_unlock;
1533 	}
1534 
1535 	pdd = kfd_bind_process_to_device(pdd->dev, p);
1536 	if (IS_ERR(pdd)) {
1537 		r = PTR_ERR(pdd);
1538 		goto err_unlock;
1539 	}
1540 
1541 	r = amdgpu_amdkfd_gpuvm_import_dmabuf(pdd->dev->adev, dmabuf,
1542 					      args->va_addr, pdd->drm_priv,
1543 					      (struct kgd_mem **)&mem, &size,
1544 					      NULL);
1545 	if (r)
1546 		goto err_unlock;
1547 
1548 	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1549 	if (idr_handle < 0) {
1550 		r = -EFAULT;
1551 		goto err_free;
1552 	}
1553 
1554 	mutex_unlock(&p->mutex);
1555 	dma_buf_put(dmabuf);
1556 
1557 	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1558 
1559 	return 0;
1560 
1561 err_free:
1562 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, (struct kgd_mem *)mem,
1563 					       pdd->drm_priv, NULL);
1564 err_unlock:
1565 	mutex_unlock(&p->mutex);
1566 	dma_buf_put(dmabuf);
1567 	return r;
1568 }
1569 
1570 /* Handle requests for watching SMI events */
1571 static int kfd_ioctl_smi_events(struct file *filep,
1572 				struct kfd_process *p, void *data)
1573 {
1574 	struct kfd_ioctl_smi_events_args *args = data;
1575 	struct kfd_process_device *pdd;
1576 
1577 	mutex_lock(&p->mutex);
1578 
1579 	pdd = kfd_process_device_data_by_id(p, args->gpuid);
1580 	mutex_unlock(&p->mutex);
1581 	if (!pdd)
1582 		return -EINVAL;
1583 
1584 	return kfd_smi_event_open(pdd->dev, &args->anon_fd);
1585 }
1586 
1587 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1588 
1589 static int kfd_ioctl_set_xnack_mode(struct file *filep,
1590 				    struct kfd_process *p, void *data)
1591 {
1592 	struct kfd_ioctl_set_xnack_mode_args *args = data;
1593 	int r = 0;
1594 
1595 	mutex_lock(&p->mutex);
1596 	if (args->xnack_enabled >= 0) {
1597 		if (!list_empty(&p->pqm.queues)) {
1598 			pr_debug("Process has user queues running\n");
1599 			r = -EBUSY;
1600 			goto out_unlock;
1601 		}
1602 
1603 		if (p->xnack_enabled == args->xnack_enabled)
1604 			goto out_unlock;
1605 
1606 		if (args->xnack_enabled && !kfd_process_xnack_mode(p, true)) {
1607 			r = -EPERM;
1608 			goto out_unlock;
1609 		}
1610 
1611 		r = svm_range_switch_xnack_reserve_mem(p, args->xnack_enabled);
1612 	} else {
1613 		args->xnack_enabled = p->xnack_enabled;
1614 	}
1615 
1616 out_unlock:
1617 	mutex_unlock(&p->mutex);
1618 
1619 	return r;
1620 }
1621 
1622 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1623 {
1624 	struct kfd_ioctl_svm_args *args = data;
1625 	int r = 0;
1626 
1627 	pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1628 		 args->start_addr, args->size, args->op, args->nattr);
1629 
1630 	if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1631 		return -EINVAL;
1632 	if (!args->start_addr || !args->size)
1633 		return -EINVAL;
1634 
1635 	r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
1636 		      args->attrs);
1637 
1638 	return r;
1639 }
1640 #else
1641 static int kfd_ioctl_set_xnack_mode(struct file *filep,
1642 				    struct kfd_process *p, void *data)
1643 {
1644 	return -EPERM;
1645 }
1646 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1647 {
1648 	return -EPERM;
1649 }
1650 #endif
1651 
1652 static int criu_checkpoint_process(struct kfd_process *p,
1653 			     uint8_t __user *user_priv_data,
1654 			     uint64_t *priv_offset)
1655 {
1656 	struct kfd_criu_process_priv_data process_priv;
1657 	int ret;
1658 
1659 	memset(&process_priv, 0, sizeof(process_priv));
1660 
1661 	process_priv.version = KFD_CRIU_PRIV_VERSION;
1662 	/* For CR, we don't consider negative xnack mode which is used for
1663 	 * querying without changing it, here 0 simply means disabled and 1
1664 	 * means enabled so retry for finding a valid PTE.
1665 	 */
1666 	process_priv.xnack_mode = p->xnack_enabled ? 1 : 0;
1667 
1668 	ret = copy_to_user(user_priv_data + *priv_offset,
1669 				&process_priv, sizeof(process_priv));
1670 
1671 	if (ret) {
1672 		pr_err("Failed to copy process information to user\n");
1673 		ret = -EFAULT;
1674 	}
1675 
1676 	*priv_offset += sizeof(process_priv);
1677 	return ret;
1678 }
1679 
1680 static int criu_checkpoint_devices(struct kfd_process *p,
1681 			     uint32_t num_devices,
1682 			     uint8_t __user *user_addr,
1683 			     uint8_t __user *user_priv_data,
1684 			     uint64_t *priv_offset)
1685 {
1686 	struct kfd_criu_device_priv_data *device_priv = NULL;
1687 	struct kfd_criu_device_bucket *device_buckets = NULL;
1688 	int ret = 0, i;
1689 
1690 	device_buckets = kvzalloc(num_devices * sizeof(*device_buckets), GFP_KERNEL);
1691 	if (!device_buckets) {
1692 		ret = -ENOMEM;
1693 		goto exit;
1694 	}
1695 
1696 	device_priv = kvzalloc(num_devices * sizeof(*device_priv), GFP_KERNEL);
1697 	if (!device_priv) {
1698 		ret = -ENOMEM;
1699 		goto exit;
1700 	}
1701 
1702 	for (i = 0; i < num_devices; i++) {
1703 		struct kfd_process_device *pdd = p->pdds[i];
1704 
1705 		device_buckets[i].user_gpu_id = pdd->user_gpu_id;
1706 		device_buckets[i].actual_gpu_id = pdd->dev->id;
1707 
1708 		/*
1709 		 * priv_data does not contain useful information for now and is reserved for
1710 		 * future use, so we do not set its contents.
1711 		 */
1712 	}
1713 
1714 	ret = copy_to_user(user_addr, device_buckets, num_devices * sizeof(*device_buckets));
1715 	if (ret) {
1716 		pr_err("Failed to copy device information to user\n");
1717 		ret = -EFAULT;
1718 		goto exit;
1719 	}
1720 
1721 	ret = copy_to_user(user_priv_data + *priv_offset,
1722 			   device_priv,
1723 			   num_devices * sizeof(*device_priv));
1724 	if (ret) {
1725 		pr_err("Failed to copy device information to user\n");
1726 		ret = -EFAULT;
1727 	}
1728 	*priv_offset += num_devices * sizeof(*device_priv);
1729 
1730 exit:
1731 	kvfree(device_buckets);
1732 	kvfree(device_priv);
1733 	return ret;
1734 }
1735 
1736 static uint32_t get_process_num_bos(struct kfd_process *p)
1737 {
1738 	uint32_t num_of_bos = 0;
1739 	int i;
1740 
1741 	/* Run over all PDDs of the process */
1742 	for (i = 0; i < p->n_pdds; i++) {
1743 		struct kfd_process_device *pdd = p->pdds[i];
1744 		void *mem;
1745 		int id;
1746 
1747 		idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1748 			struct kgd_mem *kgd_mem = (struct kgd_mem *)mem;
1749 
1750 			if ((uint64_t)kgd_mem->va > pdd->gpuvm_base)
1751 				num_of_bos++;
1752 		}
1753 	}
1754 	return num_of_bos;
1755 }
1756 
1757 static int criu_get_prime_handle(struct drm_gem_object *gobj, int flags,
1758 				      u32 *shared_fd)
1759 {
1760 	struct dma_buf *dmabuf;
1761 	int ret;
1762 
1763 	dmabuf = amdgpu_gem_prime_export(gobj, flags);
1764 	if (IS_ERR(dmabuf)) {
1765 		ret = PTR_ERR(dmabuf);
1766 		pr_err("dmabuf export failed for the BO\n");
1767 		return ret;
1768 	}
1769 
1770 	ret = dma_buf_fd(dmabuf, flags);
1771 	if (ret < 0) {
1772 		pr_err("dmabuf create fd failed, ret:%d\n", ret);
1773 		goto out_free_dmabuf;
1774 	}
1775 
1776 	*shared_fd = ret;
1777 	return 0;
1778 
1779 out_free_dmabuf:
1780 	dma_buf_put(dmabuf);
1781 	return ret;
1782 }
1783 
1784 static int criu_checkpoint_bos(struct kfd_process *p,
1785 			       uint32_t num_bos,
1786 			       uint8_t __user *user_bos,
1787 			       uint8_t __user *user_priv_data,
1788 			       uint64_t *priv_offset)
1789 {
1790 	struct kfd_criu_bo_bucket *bo_buckets;
1791 	struct kfd_criu_bo_priv_data *bo_privs;
1792 	int ret = 0, pdd_index, bo_index = 0, id;
1793 	void *mem;
1794 
1795 	bo_buckets = kvzalloc(num_bos * sizeof(*bo_buckets), GFP_KERNEL);
1796 	if (!bo_buckets)
1797 		return -ENOMEM;
1798 
1799 	bo_privs = kvzalloc(num_bos * sizeof(*bo_privs), GFP_KERNEL);
1800 	if (!bo_privs) {
1801 		ret = -ENOMEM;
1802 		goto exit;
1803 	}
1804 
1805 	for (pdd_index = 0; pdd_index < p->n_pdds; pdd_index++) {
1806 		struct kfd_process_device *pdd = p->pdds[pdd_index];
1807 		struct amdgpu_bo *dumper_bo;
1808 		struct kgd_mem *kgd_mem;
1809 
1810 		idr_for_each_entry(&pdd->alloc_idr, mem, id) {
1811 			struct kfd_criu_bo_bucket *bo_bucket;
1812 			struct kfd_criu_bo_priv_data *bo_priv;
1813 			int i, dev_idx = 0;
1814 
1815 			if (!mem) {
1816 				ret = -ENOMEM;
1817 				goto exit;
1818 			}
1819 
1820 			kgd_mem = (struct kgd_mem *)mem;
1821 			dumper_bo = kgd_mem->bo;
1822 
1823 			if ((uint64_t)kgd_mem->va <= pdd->gpuvm_base)
1824 				continue;
1825 
1826 			bo_bucket = &bo_buckets[bo_index];
1827 			bo_priv = &bo_privs[bo_index];
1828 
1829 			bo_bucket->gpu_id = pdd->user_gpu_id;
1830 			bo_bucket->addr = (uint64_t)kgd_mem->va;
1831 			bo_bucket->size = amdgpu_bo_size(dumper_bo);
1832 			bo_bucket->alloc_flags = (uint32_t)kgd_mem->alloc_flags;
1833 			bo_priv->idr_handle = id;
1834 
1835 			if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
1836 				ret = amdgpu_ttm_tt_get_userptr(&dumper_bo->tbo,
1837 								&bo_priv->user_addr);
1838 				if (ret) {
1839 					pr_err("Failed to obtain user address for user-pointer bo\n");
1840 					goto exit;
1841 				}
1842 			}
1843 			if (bo_bucket->alloc_flags
1844 			    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
1845 				ret = criu_get_prime_handle(&dumper_bo->tbo.base,
1846 						bo_bucket->alloc_flags &
1847 						KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DRM_RDWR : 0,
1848 						&bo_bucket->dmabuf_fd);
1849 				if (ret)
1850 					goto exit;
1851 			} else {
1852 				bo_bucket->dmabuf_fd = KFD_INVALID_FD;
1853 			}
1854 
1855 			if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
1856 				bo_bucket->offset = KFD_MMAP_TYPE_DOORBELL |
1857 					KFD_MMAP_GPU_ID(pdd->dev->id);
1858 			else if (bo_bucket->alloc_flags &
1859 				KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1860 				bo_bucket->offset = KFD_MMAP_TYPE_MMIO |
1861 					KFD_MMAP_GPU_ID(pdd->dev->id);
1862 			else
1863 				bo_bucket->offset = amdgpu_bo_mmap_offset(dumper_bo);
1864 
1865 			for (i = 0; i < p->n_pdds; i++) {
1866 				if (amdgpu_amdkfd_bo_mapped_to_dev(p->pdds[i]->dev->adev, kgd_mem))
1867 					bo_priv->mapped_gpuids[dev_idx++] = p->pdds[i]->user_gpu_id;
1868 			}
1869 
1870 			pr_debug("bo_size = 0x%llx, bo_addr = 0x%llx bo_offset = 0x%llx\n"
1871 					"gpu_id = 0x%x alloc_flags = 0x%x idr_handle = 0x%x",
1872 					bo_bucket->size,
1873 					bo_bucket->addr,
1874 					bo_bucket->offset,
1875 					bo_bucket->gpu_id,
1876 					bo_bucket->alloc_flags,
1877 					bo_priv->idr_handle);
1878 			bo_index++;
1879 		}
1880 	}
1881 
1882 	ret = copy_to_user(user_bos, bo_buckets, num_bos * sizeof(*bo_buckets));
1883 	if (ret) {
1884 		pr_err("Failed to copy BO information to user\n");
1885 		ret = -EFAULT;
1886 		goto exit;
1887 	}
1888 
1889 	ret = copy_to_user(user_priv_data + *priv_offset, bo_privs, num_bos * sizeof(*bo_privs));
1890 	if (ret) {
1891 		pr_err("Failed to copy BO priv information to user\n");
1892 		ret = -EFAULT;
1893 		goto exit;
1894 	}
1895 
1896 	*priv_offset += num_bos * sizeof(*bo_privs);
1897 
1898 exit:
1899 	while (ret && bo_index--) {
1900 		if (bo_buckets[bo_index].alloc_flags
1901 		    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
1902 			close_fd(bo_buckets[bo_index].dmabuf_fd);
1903 	}
1904 
1905 	kvfree(bo_buckets);
1906 	kvfree(bo_privs);
1907 	return ret;
1908 }
1909 
1910 static int criu_get_process_object_info(struct kfd_process *p,
1911 					uint32_t *num_devices,
1912 					uint32_t *num_bos,
1913 					uint32_t *num_objects,
1914 					uint64_t *objs_priv_size)
1915 {
1916 	uint64_t queues_priv_data_size, svm_priv_data_size, priv_size;
1917 	uint32_t num_queues, num_events, num_svm_ranges;
1918 	int ret;
1919 
1920 	*num_devices = p->n_pdds;
1921 	*num_bos = get_process_num_bos(p);
1922 
1923 	ret = kfd_process_get_queue_info(p, &num_queues, &queues_priv_data_size);
1924 	if (ret)
1925 		return ret;
1926 
1927 	num_events = kfd_get_num_events(p);
1928 
1929 	ret = svm_range_get_info(p, &num_svm_ranges, &svm_priv_data_size);
1930 	if (ret)
1931 		return ret;
1932 
1933 	*num_objects = num_queues + num_events + num_svm_ranges;
1934 
1935 	if (objs_priv_size) {
1936 		priv_size = sizeof(struct kfd_criu_process_priv_data);
1937 		priv_size += *num_devices * sizeof(struct kfd_criu_device_priv_data);
1938 		priv_size += *num_bos * sizeof(struct kfd_criu_bo_priv_data);
1939 		priv_size += queues_priv_data_size;
1940 		priv_size += num_events * sizeof(struct kfd_criu_event_priv_data);
1941 		priv_size += svm_priv_data_size;
1942 		*objs_priv_size = priv_size;
1943 	}
1944 	return 0;
1945 }
1946 
1947 static int criu_checkpoint(struct file *filep,
1948 			   struct kfd_process *p,
1949 			   struct kfd_ioctl_criu_args *args)
1950 {
1951 	int ret;
1952 	uint32_t num_devices, num_bos, num_objects;
1953 	uint64_t priv_size, priv_offset = 0, bo_priv_offset;
1954 
1955 	if (!args->devices || !args->bos || !args->priv_data)
1956 		return -EINVAL;
1957 
1958 	mutex_lock(&p->mutex);
1959 
1960 	if (!p->n_pdds) {
1961 		pr_err("No pdd for given process\n");
1962 		ret = -ENODEV;
1963 		goto exit_unlock;
1964 	}
1965 
1966 	/* Confirm all process queues are evicted */
1967 	if (!p->queues_paused) {
1968 		pr_err("Cannot dump process when queues are not in evicted state\n");
1969 		/* CRIU plugin did not call op PROCESS_INFO before checkpointing */
1970 		ret = -EINVAL;
1971 		goto exit_unlock;
1972 	}
1973 
1974 	ret = criu_get_process_object_info(p, &num_devices, &num_bos, &num_objects, &priv_size);
1975 	if (ret)
1976 		goto exit_unlock;
1977 
1978 	if (num_devices != args->num_devices ||
1979 	    num_bos != args->num_bos ||
1980 	    num_objects != args->num_objects ||
1981 	    priv_size != args->priv_data_size) {
1982 
1983 		ret = -EINVAL;
1984 		goto exit_unlock;
1985 	}
1986 
1987 	/* each function will store private data inside priv_data and adjust priv_offset */
1988 	ret = criu_checkpoint_process(p, (uint8_t __user *)args->priv_data, &priv_offset);
1989 	if (ret)
1990 		goto exit_unlock;
1991 
1992 	ret = criu_checkpoint_devices(p, num_devices, (uint8_t __user *)args->devices,
1993 				(uint8_t __user *)args->priv_data, &priv_offset);
1994 	if (ret)
1995 		goto exit_unlock;
1996 
1997 	/* Leave room for BOs in the private data. They need to be restored
1998 	 * before events, but we checkpoint them last to simplify the error
1999 	 * handling.
2000 	 */
2001 	bo_priv_offset = priv_offset;
2002 	priv_offset += num_bos * sizeof(struct kfd_criu_bo_priv_data);
2003 
2004 	if (num_objects) {
2005 		ret = kfd_criu_checkpoint_queues(p, (uint8_t __user *)args->priv_data,
2006 						 &priv_offset);
2007 		if (ret)
2008 			goto exit_unlock;
2009 
2010 		ret = kfd_criu_checkpoint_events(p, (uint8_t __user *)args->priv_data,
2011 						 &priv_offset);
2012 		if (ret)
2013 			goto exit_unlock;
2014 
2015 		ret = kfd_criu_checkpoint_svm(p, (uint8_t __user *)args->priv_data, &priv_offset);
2016 		if (ret)
2017 			goto exit_unlock;
2018 	}
2019 
2020 	/* This must be the last thing in this function that can fail.
2021 	 * Otherwise we leak dmabuf file descriptors.
2022 	 */
2023 	ret = criu_checkpoint_bos(p, num_bos, (uint8_t __user *)args->bos,
2024 			   (uint8_t __user *)args->priv_data, &bo_priv_offset);
2025 
2026 exit_unlock:
2027 	mutex_unlock(&p->mutex);
2028 	if (ret)
2029 		pr_err("Failed to dump CRIU ret:%d\n", ret);
2030 	else
2031 		pr_debug("CRIU dump ret:%d\n", ret);
2032 
2033 	return ret;
2034 }
2035 
2036 static int criu_restore_process(struct kfd_process *p,
2037 				struct kfd_ioctl_criu_args *args,
2038 				uint64_t *priv_offset,
2039 				uint64_t max_priv_data_size)
2040 {
2041 	int ret = 0;
2042 	struct kfd_criu_process_priv_data process_priv;
2043 
2044 	if (*priv_offset + sizeof(process_priv) > max_priv_data_size)
2045 		return -EINVAL;
2046 
2047 	ret = copy_from_user(&process_priv,
2048 				(void __user *)(args->priv_data + *priv_offset),
2049 				sizeof(process_priv));
2050 	if (ret) {
2051 		pr_err("Failed to copy process private information from user\n");
2052 		ret = -EFAULT;
2053 		goto exit;
2054 	}
2055 	*priv_offset += sizeof(process_priv);
2056 
2057 	if (process_priv.version != KFD_CRIU_PRIV_VERSION) {
2058 		pr_err("Invalid CRIU API version (checkpointed:%d current:%d)\n",
2059 			process_priv.version, KFD_CRIU_PRIV_VERSION);
2060 		return -EINVAL;
2061 	}
2062 
2063 	pr_debug("Setting XNACK mode\n");
2064 	if (process_priv.xnack_mode && !kfd_process_xnack_mode(p, true)) {
2065 		pr_err("xnack mode cannot be set\n");
2066 		ret = -EPERM;
2067 		goto exit;
2068 	} else {
2069 		pr_debug("set xnack mode: %d\n", process_priv.xnack_mode);
2070 		p->xnack_enabled = process_priv.xnack_mode;
2071 	}
2072 
2073 exit:
2074 	return ret;
2075 }
2076 
2077 static int criu_restore_devices(struct kfd_process *p,
2078 				struct kfd_ioctl_criu_args *args,
2079 				uint64_t *priv_offset,
2080 				uint64_t max_priv_data_size)
2081 {
2082 	struct kfd_criu_device_bucket *device_buckets;
2083 	struct kfd_criu_device_priv_data *device_privs;
2084 	int ret = 0;
2085 	uint32_t i;
2086 
2087 	if (args->num_devices != p->n_pdds)
2088 		return -EINVAL;
2089 
2090 	if (*priv_offset + (args->num_devices * sizeof(*device_privs)) > max_priv_data_size)
2091 		return -EINVAL;
2092 
2093 	device_buckets = kmalloc_array(args->num_devices, sizeof(*device_buckets), GFP_KERNEL);
2094 	if (!device_buckets)
2095 		return -ENOMEM;
2096 
2097 	ret = copy_from_user(device_buckets, (void __user *)args->devices,
2098 				args->num_devices * sizeof(*device_buckets));
2099 	if (ret) {
2100 		pr_err("Failed to copy devices buckets from user\n");
2101 		ret = -EFAULT;
2102 		goto exit;
2103 	}
2104 
2105 	for (i = 0; i < args->num_devices; i++) {
2106 		struct kfd_dev *dev;
2107 		struct kfd_process_device *pdd;
2108 		struct file *drm_file;
2109 
2110 		/* device private data is not currently used */
2111 
2112 		if (!device_buckets[i].user_gpu_id) {
2113 			pr_err("Invalid user gpu_id\n");
2114 			ret = -EINVAL;
2115 			goto exit;
2116 		}
2117 
2118 		dev = kfd_device_by_id(device_buckets[i].actual_gpu_id);
2119 		if (!dev) {
2120 			pr_err("Failed to find device with gpu_id = %x\n",
2121 				device_buckets[i].actual_gpu_id);
2122 			ret = -EINVAL;
2123 			goto exit;
2124 		}
2125 
2126 		pdd = kfd_get_process_device_data(dev, p);
2127 		if (!pdd) {
2128 			pr_err("Failed to get pdd for gpu_id = %x\n",
2129 					device_buckets[i].actual_gpu_id);
2130 			ret = -EINVAL;
2131 			goto exit;
2132 		}
2133 		pdd->user_gpu_id = device_buckets[i].user_gpu_id;
2134 
2135 		drm_file = fget(device_buckets[i].drm_fd);
2136 		if (!drm_file) {
2137 			pr_err("Invalid render node file descriptor sent from plugin (%d)\n",
2138 				device_buckets[i].drm_fd);
2139 			ret = -EINVAL;
2140 			goto exit;
2141 		}
2142 
2143 		if (pdd->drm_file) {
2144 			ret = -EINVAL;
2145 			goto exit;
2146 		}
2147 
2148 		/* create the vm using render nodes for kfd pdd */
2149 		if (kfd_process_device_init_vm(pdd, drm_file)) {
2150 			pr_err("could not init vm for given pdd\n");
2151 			/* On success, the PDD keeps the drm_file reference */
2152 			fput(drm_file);
2153 			ret = -EINVAL;
2154 			goto exit;
2155 		}
2156 		/*
2157 		 * pdd now already has the vm bound to render node so below api won't create a new
2158 		 * exclusive kfd mapping but use existing one with renderDXXX but is still needed
2159 		 * for iommu v2 binding  and runtime pm.
2160 		 */
2161 		pdd = kfd_bind_process_to_device(dev, p);
2162 		if (IS_ERR(pdd)) {
2163 			ret = PTR_ERR(pdd);
2164 			goto exit;
2165 		}
2166 
2167 		if (!pdd->doorbell_index &&
2168 		    kfd_alloc_process_doorbells(pdd->dev, &pdd->doorbell_index) < 0) {
2169 			ret = -ENOMEM;
2170 			goto exit;
2171 		}
2172 	}
2173 
2174 	/*
2175 	 * We are not copying device private data from user as we are not using the data for now,
2176 	 * but we still adjust for its private data.
2177 	 */
2178 	*priv_offset += args->num_devices * sizeof(*device_privs);
2179 
2180 exit:
2181 	kfree(device_buckets);
2182 	return ret;
2183 }
2184 
2185 static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
2186 				      struct kfd_criu_bo_bucket *bo_bucket,
2187 				      struct kfd_criu_bo_priv_data *bo_priv,
2188 				      struct kgd_mem **kgd_mem)
2189 {
2190 	int idr_handle;
2191 	int ret;
2192 	const bool criu_resume = true;
2193 	u64 offset;
2194 
2195 	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
2196 		if (bo_bucket->size != kfd_doorbell_process_slice(pdd->dev))
2197 			return -EINVAL;
2198 
2199 		offset = kfd_get_process_doorbells(pdd);
2200 		if (!offset)
2201 			return -ENOMEM;
2202 	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2203 		/* MMIO BOs need remapped bus address */
2204 		if (bo_bucket->size != PAGE_SIZE) {
2205 			pr_err("Invalid page size\n");
2206 			return -EINVAL;
2207 		}
2208 		offset = pdd->dev->adev->rmmio_remap.bus_addr;
2209 		if (!offset) {
2210 			pr_err("amdgpu_amdkfd_get_mmio_remap_phys_addr failed\n");
2211 			return -ENOMEM;
2212 		}
2213 	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) {
2214 		offset = bo_priv->user_addr;
2215 	}
2216 	/* Create the BO */
2217 	ret = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(pdd->dev->adev, bo_bucket->addr,
2218 						      bo_bucket->size, pdd->drm_priv, kgd_mem,
2219 						      &offset, bo_bucket->alloc_flags, criu_resume);
2220 	if (ret) {
2221 		pr_err("Could not create the BO\n");
2222 		return ret;
2223 	}
2224 	pr_debug("New BO created: size:0x%llx addr:0x%llx offset:0x%llx\n",
2225 		 bo_bucket->size, bo_bucket->addr, offset);
2226 
2227 	/* Restore previous IDR handle */
2228 	pr_debug("Restoring old IDR handle for the BO");
2229 	idr_handle = idr_alloc(&pdd->alloc_idr, *kgd_mem, bo_priv->idr_handle,
2230 			       bo_priv->idr_handle + 1, GFP_KERNEL);
2231 
2232 	if (idr_handle < 0) {
2233 		pr_err("Could not allocate idr\n");
2234 		amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->adev, *kgd_mem, pdd->drm_priv,
2235 						       NULL);
2236 		return -ENOMEM;
2237 	}
2238 
2239 	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL)
2240 		bo_bucket->restored_offset = KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(pdd->dev->id);
2241 	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
2242 		bo_bucket->restored_offset = KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(pdd->dev->id);
2243 	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_GTT) {
2244 		bo_bucket->restored_offset = offset;
2245 	} else if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) {
2246 		bo_bucket->restored_offset = offset;
2247 		/* Update the VRAM usage count */
2248 		WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + bo_bucket->size);
2249 	}
2250 	return 0;
2251 }
2252 
2253 static int criu_restore_bo(struct kfd_process *p,
2254 			   struct kfd_criu_bo_bucket *bo_bucket,
2255 			   struct kfd_criu_bo_priv_data *bo_priv)
2256 {
2257 	struct kfd_process_device *pdd;
2258 	struct kgd_mem *kgd_mem;
2259 	int ret;
2260 	int j;
2261 
2262 	pr_debug("Restoring BO size:0x%llx addr:0x%llx gpu_id:0x%x flags:0x%x idr_handle:0x%x\n",
2263 		 bo_bucket->size, bo_bucket->addr, bo_bucket->gpu_id, bo_bucket->alloc_flags,
2264 		 bo_priv->idr_handle);
2265 
2266 	pdd = kfd_process_device_data_by_id(p, bo_bucket->gpu_id);
2267 	if (!pdd) {
2268 		pr_err("Failed to get pdd\n");
2269 		return -ENODEV;
2270 	}
2271 
2272 	ret = criu_restore_memory_of_gpu(pdd, bo_bucket, bo_priv, &kgd_mem);
2273 	if (ret)
2274 		return ret;
2275 
2276 	/* now map these BOs to GPU/s */
2277 	for (j = 0; j < p->n_pdds; j++) {
2278 		struct kfd_dev *peer;
2279 		struct kfd_process_device *peer_pdd;
2280 
2281 		if (!bo_priv->mapped_gpuids[j])
2282 			break;
2283 
2284 		peer_pdd = kfd_process_device_data_by_id(p, bo_priv->mapped_gpuids[j]);
2285 		if (!peer_pdd)
2286 			return -EINVAL;
2287 
2288 		peer = peer_pdd->dev;
2289 
2290 		peer_pdd = kfd_bind_process_to_device(peer, p);
2291 		if (IS_ERR(peer_pdd))
2292 			return PTR_ERR(peer_pdd);
2293 
2294 		ret = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(peer->adev, kgd_mem,
2295 							    peer_pdd->drm_priv);
2296 		if (ret) {
2297 			pr_err("Failed to map to gpu %d/%d\n", j, p->n_pdds);
2298 			return ret;
2299 		}
2300 	}
2301 
2302 	pr_debug("map memory was successful for the BO\n");
2303 	/* create the dmabuf object and export the bo */
2304 	if (bo_bucket->alloc_flags
2305 	    & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT)) {
2306 		ret = criu_get_prime_handle(&kgd_mem->bo->tbo.base, DRM_RDWR,
2307 					    &bo_bucket->dmabuf_fd);
2308 		if (ret)
2309 			return ret;
2310 	} else {
2311 		bo_bucket->dmabuf_fd = KFD_INVALID_FD;
2312 	}
2313 
2314 	return 0;
2315 }
2316 
2317 static int criu_restore_bos(struct kfd_process *p,
2318 			    struct kfd_ioctl_criu_args *args,
2319 			    uint64_t *priv_offset,
2320 			    uint64_t max_priv_data_size)
2321 {
2322 	struct kfd_criu_bo_bucket *bo_buckets = NULL;
2323 	struct kfd_criu_bo_priv_data *bo_privs = NULL;
2324 	int ret = 0;
2325 	uint32_t i = 0;
2326 
2327 	if (*priv_offset + (args->num_bos * sizeof(*bo_privs)) > max_priv_data_size)
2328 		return -EINVAL;
2329 
2330 	/* Prevent MMU notifications until stage-4 IOCTL (CRIU_RESUME) is received */
2331 	amdgpu_amdkfd_block_mmu_notifications(p->kgd_process_info);
2332 
2333 	bo_buckets = kvmalloc_array(args->num_bos, sizeof(*bo_buckets), GFP_KERNEL);
2334 	if (!bo_buckets)
2335 		return -ENOMEM;
2336 
2337 	ret = copy_from_user(bo_buckets, (void __user *)args->bos,
2338 			     args->num_bos * sizeof(*bo_buckets));
2339 	if (ret) {
2340 		pr_err("Failed to copy BOs information from user\n");
2341 		ret = -EFAULT;
2342 		goto exit;
2343 	}
2344 
2345 	bo_privs = kvmalloc_array(args->num_bos, sizeof(*bo_privs), GFP_KERNEL);
2346 	if (!bo_privs) {
2347 		ret = -ENOMEM;
2348 		goto exit;
2349 	}
2350 
2351 	ret = copy_from_user(bo_privs, (void __user *)args->priv_data + *priv_offset,
2352 			     args->num_bos * sizeof(*bo_privs));
2353 	if (ret) {
2354 		pr_err("Failed to copy BOs information from user\n");
2355 		ret = -EFAULT;
2356 		goto exit;
2357 	}
2358 	*priv_offset += args->num_bos * sizeof(*bo_privs);
2359 
2360 	/* Create and map new BOs */
2361 	for (; i < args->num_bos; i++) {
2362 		ret = criu_restore_bo(p, &bo_buckets[i], &bo_privs[i]);
2363 		if (ret) {
2364 			pr_debug("Failed to restore BO[%d] ret%d\n", i, ret);
2365 			goto exit;
2366 		}
2367 	} /* done */
2368 
2369 	/* Copy only the buckets back so user can read bo_buckets[N].restored_offset */
2370 	ret = copy_to_user((void __user *)args->bos,
2371 				bo_buckets,
2372 				(args->num_bos * sizeof(*bo_buckets)));
2373 	if (ret)
2374 		ret = -EFAULT;
2375 
2376 exit:
2377 	while (ret && i--) {
2378 		if (bo_buckets[i].alloc_flags
2379 		   & (KFD_IOC_ALLOC_MEM_FLAGS_VRAM | KFD_IOC_ALLOC_MEM_FLAGS_GTT))
2380 			close_fd(bo_buckets[i].dmabuf_fd);
2381 	}
2382 	kvfree(bo_buckets);
2383 	kvfree(bo_privs);
2384 	return ret;
2385 }
2386 
2387 static int criu_restore_objects(struct file *filep,
2388 				struct kfd_process *p,
2389 				struct kfd_ioctl_criu_args *args,
2390 				uint64_t *priv_offset,
2391 				uint64_t max_priv_data_size)
2392 {
2393 	int ret = 0;
2394 	uint32_t i;
2395 
2396 	BUILD_BUG_ON(offsetof(struct kfd_criu_queue_priv_data, object_type));
2397 	BUILD_BUG_ON(offsetof(struct kfd_criu_event_priv_data, object_type));
2398 	BUILD_BUG_ON(offsetof(struct kfd_criu_svm_range_priv_data, object_type));
2399 
2400 	for (i = 0; i < args->num_objects; i++) {
2401 		uint32_t object_type;
2402 
2403 		if (*priv_offset + sizeof(object_type) > max_priv_data_size) {
2404 			pr_err("Invalid private data size\n");
2405 			return -EINVAL;
2406 		}
2407 
2408 		ret = get_user(object_type, (uint32_t __user *)(args->priv_data + *priv_offset));
2409 		if (ret) {
2410 			pr_err("Failed to copy private information from user\n");
2411 			goto exit;
2412 		}
2413 
2414 		switch (object_type) {
2415 		case KFD_CRIU_OBJECT_TYPE_QUEUE:
2416 			ret = kfd_criu_restore_queue(p, (uint8_t __user *)args->priv_data,
2417 						     priv_offset, max_priv_data_size);
2418 			if (ret)
2419 				goto exit;
2420 			break;
2421 		case KFD_CRIU_OBJECT_TYPE_EVENT:
2422 			ret = kfd_criu_restore_event(filep, p, (uint8_t __user *)args->priv_data,
2423 						     priv_offset, max_priv_data_size);
2424 			if (ret)
2425 				goto exit;
2426 			break;
2427 		case KFD_CRIU_OBJECT_TYPE_SVM_RANGE:
2428 			ret = kfd_criu_restore_svm(p, (uint8_t __user *)args->priv_data,
2429 						     priv_offset, max_priv_data_size);
2430 			if (ret)
2431 				goto exit;
2432 			break;
2433 		default:
2434 			pr_err("Invalid object type:%u at index:%d\n", object_type, i);
2435 			ret = -EINVAL;
2436 			goto exit;
2437 		}
2438 	}
2439 exit:
2440 	return ret;
2441 }
2442 
2443 static int criu_restore(struct file *filep,
2444 			struct kfd_process *p,
2445 			struct kfd_ioctl_criu_args *args)
2446 {
2447 	uint64_t priv_offset = 0;
2448 	int ret = 0;
2449 
2450 	pr_debug("CRIU restore (num_devices:%u num_bos:%u num_objects:%u priv_data_size:%llu)\n",
2451 		 args->num_devices, args->num_bos, args->num_objects, args->priv_data_size);
2452 
2453 	if (!args->bos || !args->devices || !args->priv_data || !args->priv_data_size ||
2454 	    !args->num_devices || !args->num_bos)
2455 		return -EINVAL;
2456 
2457 	mutex_lock(&p->mutex);
2458 
2459 	/*
2460 	 * Set the process to evicted state to avoid running any new queues before all the memory
2461 	 * mappings are ready.
2462 	 */
2463 	ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_RESTORE);
2464 	if (ret)
2465 		goto exit_unlock;
2466 
2467 	/* Each function will adjust priv_offset based on how many bytes they consumed */
2468 	ret = criu_restore_process(p, args, &priv_offset, args->priv_data_size);
2469 	if (ret)
2470 		goto exit_unlock;
2471 
2472 	ret = criu_restore_devices(p, args, &priv_offset, args->priv_data_size);
2473 	if (ret)
2474 		goto exit_unlock;
2475 
2476 	ret = criu_restore_bos(p, args, &priv_offset, args->priv_data_size);
2477 	if (ret)
2478 		goto exit_unlock;
2479 
2480 	ret = criu_restore_objects(filep, p, args, &priv_offset, args->priv_data_size);
2481 	if (ret)
2482 		goto exit_unlock;
2483 
2484 	if (priv_offset != args->priv_data_size) {
2485 		pr_err("Invalid private data size\n");
2486 		ret = -EINVAL;
2487 	}
2488 
2489 exit_unlock:
2490 	mutex_unlock(&p->mutex);
2491 	if (ret)
2492 		pr_err("Failed to restore CRIU ret:%d\n", ret);
2493 	else
2494 		pr_debug("CRIU restore successful\n");
2495 
2496 	return ret;
2497 }
2498 
2499 static int criu_unpause(struct file *filep,
2500 			struct kfd_process *p,
2501 			struct kfd_ioctl_criu_args *args)
2502 {
2503 	int ret;
2504 
2505 	mutex_lock(&p->mutex);
2506 
2507 	if (!p->queues_paused) {
2508 		mutex_unlock(&p->mutex);
2509 		return -EINVAL;
2510 	}
2511 
2512 	ret = kfd_process_restore_queues(p);
2513 	if (ret)
2514 		pr_err("Failed to unpause queues ret:%d\n", ret);
2515 	else
2516 		p->queues_paused = false;
2517 
2518 	mutex_unlock(&p->mutex);
2519 
2520 	return ret;
2521 }
2522 
2523 static int criu_resume(struct file *filep,
2524 			struct kfd_process *p,
2525 			struct kfd_ioctl_criu_args *args)
2526 {
2527 	struct kfd_process *target = NULL;
2528 	struct pid *pid = NULL;
2529 	int ret = 0;
2530 
2531 	pr_debug("Inside %s, target pid for criu restore: %d\n", __func__,
2532 		 args->pid);
2533 
2534 	pid = find_get_pid(args->pid);
2535 	if (!pid) {
2536 		pr_err("Cannot find pid info for %i\n", args->pid);
2537 		return -ESRCH;
2538 	}
2539 
2540 	pr_debug("calling kfd_lookup_process_by_pid\n");
2541 	target = kfd_lookup_process_by_pid(pid);
2542 
2543 	put_pid(pid);
2544 
2545 	if (!target) {
2546 		pr_debug("Cannot find process info for %i\n", args->pid);
2547 		return -ESRCH;
2548 	}
2549 
2550 	mutex_lock(&target->mutex);
2551 	ret = kfd_criu_resume_svm(target);
2552 	if (ret) {
2553 		pr_err("kfd_criu_resume_svm failed for %i\n", args->pid);
2554 		goto exit;
2555 	}
2556 
2557 	ret =  amdgpu_amdkfd_criu_resume(target->kgd_process_info);
2558 	if (ret)
2559 		pr_err("amdgpu_amdkfd_criu_resume failed for %i\n", args->pid);
2560 
2561 exit:
2562 	mutex_unlock(&target->mutex);
2563 
2564 	kfd_unref_process(target);
2565 	return ret;
2566 }
2567 
2568 static int criu_process_info(struct file *filep,
2569 				struct kfd_process *p,
2570 				struct kfd_ioctl_criu_args *args)
2571 {
2572 	int ret = 0;
2573 
2574 	mutex_lock(&p->mutex);
2575 
2576 	if (!p->n_pdds) {
2577 		pr_err("No pdd for given process\n");
2578 		ret = -ENODEV;
2579 		goto err_unlock;
2580 	}
2581 
2582 	ret = kfd_process_evict_queues(p, KFD_QUEUE_EVICTION_CRIU_CHECKPOINT);
2583 	if (ret)
2584 		goto err_unlock;
2585 
2586 	p->queues_paused = true;
2587 
2588 	args->pid = task_pid_nr_ns(p->lead_thread,
2589 					task_active_pid_ns(p->lead_thread));
2590 
2591 	ret = criu_get_process_object_info(p, &args->num_devices, &args->num_bos,
2592 					   &args->num_objects, &args->priv_data_size);
2593 	if (ret)
2594 		goto err_unlock;
2595 
2596 	dev_dbg(kfd_device, "Num of devices:%u bos:%u objects:%u priv_data_size:%lld\n",
2597 				args->num_devices, args->num_bos, args->num_objects,
2598 				args->priv_data_size);
2599 
2600 err_unlock:
2601 	if (ret) {
2602 		kfd_process_restore_queues(p);
2603 		p->queues_paused = false;
2604 	}
2605 	mutex_unlock(&p->mutex);
2606 	return ret;
2607 }
2608 
2609 static int kfd_ioctl_criu(struct file *filep, struct kfd_process *p, void *data)
2610 {
2611 	struct kfd_ioctl_criu_args *args = data;
2612 	int ret;
2613 
2614 	dev_dbg(kfd_device, "CRIU operation: %d\n", args->op);
2615 	switch (args->op) {
2616 	case KFD_CRIU_OP_PROCESS_INFO:
2617 		ret = criu_process_info(filep, p, args);
2618 		break;
2619 	case KFD_CRIU_OP_CHECKPOINT:
2620 		ret = criu_checkpoint(filep, p, args);
2621 		break;
2622 	case KFD_CRIU_OP_UNPAUSE:
2623 		ret = criu_unpause(filep, p, args);
2624 		break;
2625 	case KFD_CRIU_OP_RESTORE:
2626 		ret = criu_restore(filep, p, args);
2627 		break;
2628 	case KFD_CRIU_OP_RESUME:
2629 		ret = criu_resume(filep, p, args);
2630 		break;
2631 	default:
2632 		dev_dbg(kfd_device, "Unsupported CRIU operation:%d\n", args->op);
2633 		ret = -EINVAL;
2634 		break;
2635 	}
2636 
2637 	if (ret)
2638 		dev_dbg(kfd_device, "CRIU operation:%d err:%d\n", args->op, ret);
2639 
2640 	return ret;
2641 }
2642 
2643 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
2644 	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
2645 			    .cmd_drv = 0, .name = #ioctl}
2646 
2647 /** Ioctl table */
2648 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
2649 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
2650 			kfd_ioctl_get_version, 0),
2651 
2652 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
2653 			kfd_ioctl_create_queue, 0),
2654 
2655 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
2656 			kfd_ioctl_destroy_queue, 0),
2657 
2658 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
2659 			kfd_ioctl_set_memory_policy, 0),
2660 
2661 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
2662 			kfd_ioctl_get_clock_counters, 0),
2663 
2664 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
2665 			kfd_ioctl_get_process_apertures, 0),
2666 
2667 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
2668 			kfd_ioctl_update_queue, 0),
2669 
2670 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
2671 			kfd_ioctl_create_event, 0),
2672 
2673 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
2674 			kfd_ioctl_destroy_event, 0),
2675 
2676 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
2677 			kfd_ioctl_set_event, 0),
2678 
2679 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
2680 			kfd_ioctl_reset_event, 0),
2681 
2682 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
2683 			kfd_ioctl_wait_events, 0),
2684 
2685 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER_DEPRECATED,
2686 			kfd_ioctl_dbg_register, 0),
2687 
2688 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER_DEPRECATED,
2689 			kfd_ioctl_dbg_unregister, 0),
2690 
2691 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH_DEPRECATED,
2692 			kfd_ioctl_dbg_address_watch, 0),
2693 
2694 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL_DEPRECATED,
2695 			kfd_ioctl_dbg_wave_control, 0),
2696 
2697 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
2698 			kfd_ioctl_set_scratch_backing_va, 0),
2699 
2700 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
2701 			kfd_ioctl_get_tile_config, 0),
2702 
2703 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
2704 			kfd_ioctl_set_trap_handler, 0),
2705 
2706 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
2707 			kfd_ioctl_get_process_apertures_new, 0),
2708 
2709 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
2710 			kfd_ioctl_acquire_vm, 0),
2711 
2712 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
2713 			kfd_ioctl_alloc_memory_of_gpu, 0),
2714 
2715 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
2716 			kfd_ioctl_free_memory_of_gpu, 0),
2717 
2718 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
2719 			kfd_ioctl_map_memory_to_gpu, 0),
2720 
2721 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
2722 			kfd_ioctl_unmap_memory_from_gpu, 0),
2723 
2724 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
2725 			kfd_ioctl_set_cu_mask, 0),
2726 
2727 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
2728 			kfd_ioctl_get_queue_wave_state, 0),
2729 
2730 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
2731 				kfd_ioctl_get_dmabuf_info, 0),
2732 
2733 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
2734 				kfd_ioctl_import_dmabuf, 0),
2735 
2736 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
2737 			kfd_ioctl_alloc_queue_gws, 0),
2738 
2739 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
2740 			kfd_ioctl_smi_events, 0),
2741 
2742 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
2743 
2744 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
2745 			kfd_ioctl_set_xnack_mode, 0),
2746 
2747 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CRIU_OP,
2748 			kfd_ioctl_criu, KFD_IOC_FLAG_CHECKPOINT_RESTORE),
2749 
2750 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_AVAILABLE_MEMORY,
2751 			kfd_ioctl_get_available_memory, 0),
2752 };
2753 
2754 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
2755 
2756 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
2757 {
2758 	struct kfd_process *process;
2759 	amdkfd_ioctl_t *func;
2760 	const struct amdkfd_ioctl_desc *ioctl = NULL;
2761 	unsigned int nr = _IOC_NR(cmd);
2762 	char stack_kdata[128];
2763 	char *kdata = NULL;
2764 	unsigned int usize, asize;
2765 	int retcode = -EINVAL;
2766 	bool ptrace_attached = false;
2767 
2768 	if (nr >= AMDKFD_CORE_IOCTL_COUNT)
2769 		goto err_i1;
2770 
2771 	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
2772 		u32 amdkfd_size;
2773 
2774 		ioctl = &amdkfd_ioctls[nr];
2775 
2776 		amdkfd_size = _IOC_SIZE(ioctl->cmd);
2777 		usize = asize = _IOC_SIZE(cmd);
2778 		if (amdkfd_size > asize)
2779 			asize = amdkfd_size;
2780 
2781 		cmd = ioctl->cmd;
2782 	} else
2783 		goto err_i1;
2784 
2785 	dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
2786 
2787 	/* Get the process struct from the filep. Only the process
2788 	 * that opened /dev/kfd can use the file descriptor. Child
2789 	 * processes need to create their own KFD device context.
2790 	 */
2791 	process = filep->private_data;
2792 
2793 	rcu_read_lock();
2794 	if ((ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE) &&
2795 	    ptrace_parent(process->lead_thread) == current)
2796 		ptrace_attached = true;
2797 	rcu_read_unlock();
2798 
2799 	if (process->lead_thread != current->group_leader
2800 	    && !ptrace_attached) {
2801 		dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
2802 		retcode = -EBADF;
2803 		goto err_i1;
2804 	}
2805 
2806 	/* Do not trust userspace, use our own definition */
2807 	func = ioctl->func;
2808 
2809 	if (unlikely(!func)) {
2810 		dev_dbg(kfd_device, "no function\n");
2811 		retcode = -EINVAL;
2812 		goto err_i1;
2813 	}
2814 
2815 	/*
2816 	 * Versions of docker shipped in Ubuntu 18.xx and 20.xx do not support
2817 	 * CAP_CHECKPOINT_RESTORE, so we also allow access if CAP_SYS_ADMIN as CAP_SYS_ADMIN is a
2818 	 * more priviledged access.
2819 	 */
2820 	if (unlikely(ioctl->flags & KFD_IOC_FLAG_CHECKPOINT_RESTORE)) {
2821 		if (!capable(CAP_CHECKPOINT_RESTORE) &&
2822 						!capable(CAP_SYS_ADMIN)) {
2823 			retcode = -EACCES;
2824 			goto err_i1;
2825 		}
2826 	}
2827 
2828 	if (cmd & (IOC_IN | IOC_OUT)) {
2829 		if (asize <= sizeof(stack_kdata)) {
2830 			kdata = stack_kdata;
2831 		} else {
2832 			kdata = kmalloc(asize, GFP_KERNEL);
2833 			if (!kdata) {
2834 				retcode = -ENOMEM;
2835 				goto err_i1;
2836 			}
2837 		}
2838 		if (asize > usize)
2839 			memset(kdata + usize, 0, asize - usize);
2840 	}
2841 
2842 	if (cmd & IOC_IN) {
2843 		if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
2844 			retcode = -EFAULT;
2845 			goto err_i1;
2846 		}
2847 	} else if (cmd & IOC_OUT) {
2848 		memset(kdata, 0, usize);
2849 	}
2850 
2851 	retcode = func(filep, process, kdata);
2852 
2853 	if (cmd & IOC_OUT)
2854 		if (copy_to_user((void __user *)arg, kdata, usize) != 0)
2855 			retcode = -EFAULT;
2856 
2857 err_i1:
2858 	if (!ioctl)
2859 		dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
2860 			  task_pid_nr(current), cmd, nr);
2861 
2862 	if (kdata != stack_kdata)
2863 		kfree(kdata);
2864 
2865 	if (retcode)
2866 		dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
2867 				nr, arg, retcode);
2868 
2869 	return retcode;
2870 }
2871 
2872 static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
2873 		      struct vm_area_struct *vma)
2874 {
2875 	phys_addr_t address;
2876 
2877 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2878 		return -EINVAL;
2879 
2880 	address = dev->adev->rmmio_remap.bus_addr;
2881 
2882 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
2883 				VM_DONTDUMP | VM_PFNMAP;
2884 
2885 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2886 
2887 	pr_debug("pasid 0x%x mapping mmio page\n"
2888 		 "     target user address == 0x%08llX\n"
2889 		 "     physical address    == 0x%08llX\n"
2890 		 "     vm_flags            == 0x%04lX\n"
2891 		 "     size                == 0x%04lX\n",
2892 		 process->pasid, (unsigned long long) vma->vm_start,
2893 		 address, vma->vm_flags, PAGE_SIZE);
2894 
2895 	return io_remap_pfn_range(vma,
2896 				vma->vm_start,
2897 				address >> PAGE_SHIFT,
2898 				PAGE_SIZE,
2899 				vma->vm_page_prot);
2900 }
2901 
2902 
2903 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
2904 {
2905 	struct kfd_process *process;
2906 	struct kfd_dev *dev = NULL;
2907 	unsigned long mmap_offset;
2908 	unsigned int gpu_id;
2909 
2910 	process = kfd_get_process(current);
2911 	if (IS_ERR(process))
2912 		return PTR_ERR(process);
2913 
2914 	mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
2915 	gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
2916 	if (gpu_id)
2917 		dev = kfd_device_by_id(gpu_id);
2918 
2919 	switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
2920 	case KFD_MMAP_TYPE_DOORBELL:
2921 		if (!dev)
2922 			return -ENODEV;
2923 		return kfd_doorbell_mmap(dev, process, vma);
2924 
2925 	case KFD_MMAP_TYPE_EVENTS:
2926 		return kfd_event_mmap(process, vma);
2927 
2928 	case KFD_MMAP_TYPE_RESERVED_MEM:
2929 		if (!dev)
2930 			return -ENODEV;
2931 		return kfd_reserved_mem_mmap(dev, process, vma);
2932 	case KFD_MMAP_TYPE_MMIO:
2933 		if (!dev)
2934 			return -ENODEV;
2935 		return kfd_mmio_mmap(dev, process, vma);
2936 	}
2937 
2938 	return -EFAULT;
2939 }
2940