1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
26 #include <linux/fs.h>
27 #include <linux/file.h>
28 #include <linux/sched.h>
29 #include <linux/slab.h>
30 #include <linux/uaccess.h>
31 #include <linux/compat.h>
32 #include <uapi/linux/kfd_ioctl.h>
33 #include <linux/time.h>
34 #include <linux/mm.h>
35 #include <linux/mman.h>
36 #include <linux/dma-buf.h>
37 #include <asm/processor.h>
38 #include "kfd_priv.h"
39 #include "kfd_device_queue_manager.h"
40 #include "kfd_dbgmgr.h"
41 #include "amdgpu_amdkfd.h"
42 #include "kfd_smi_events.h"
43 
44 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
45 static int kfd_open(struct inode *, struct file *);
46 static int kfd_release(struct inode *, struct file *);
47 static int kfd_mmap(struct file *, struct vm_area_struct *);
48 
49 static const char kfd_dev_name[] = "kfd";
50 
51 static const struct file_operations kfd_fops = {
52 	.owner = THIS_MODULE,
53 	.unlocked_ioctl = kfd_ioctl,
54 	.compat_ioctl = compat_ptr_ioctl,
55 	.open = kfd_open,
56 	.release = kfd_release,
57 	.mmap = kfd_mmap,
58 };
59 
60 static int kfd_char_dev_major = -1;
61 static struct class *kfd_class;
62 struct device *kfd_device;
63 
64 int kfd_chardev_init(void)
65 {
66 	int err = 0;
67 
68 	kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
69 	err = kfd_char_dev_major;
70 	if (err < 0)
71 		goto err_register_chrdev;
72 
73 	kfd_class = class_create(THIS_MODULE, kfd_dev_name);
74 	err = PTR_ERR(kfd_class);
75 	if (IS_ERR(kfd_class))
76 		goto err_class_create;
77 
78 	kfd_device = device_create(kfd_class, NULL,
79 					MKDEV(kfd_char_dev_major, 0),
80 					NULL, kfd_dev_name);
81 	err = PTR_ERR(kfd_device);
82 	if (IS_ERR(kfd_device))
83 		goto err_device_create;
84 
85 	return 0;
86 
87 err_device_create:
88 	class_destroy(kfd_class);
89 err_class_create:
90 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
91 err_register_chrdev:
92 	return err;
93 }
94 
95 void kfd_chardev_exit(void)
96 {
97 	device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
98 	class_destroy(kfd_class);
99 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
100 	kfd_device = NULL;
101 }
102 
103 struct device *kfd_chardev(void)
104 {
105 	return kfd_device;
106 }
107 
108 
109 static int kfd_open(struct inode *inode, struct file *filep)
110 {
111 	struct kfd_process *process;
112 	bool is_32bit_user_mode;
113 
114 	if (iminor(inode) != 0)
115 		return -ENODEV;
116 
117 	is_32bit_user_mode = in_compat_syscall();
118 
119 	if (is_32bit_user_mode) {
120 		dev_warn(kfd_device,
121 			"Process %d (32-bit) failed to open /dev/kfd\n"
122 			"32-bit processes are not supported by amdkfd\n",
123 			current->pid);
124 		return -EPERM;
125 	}
126 
127 	process = kfd_create_process(filep);
128 	if (IS_ERR(process))
129 		return PTR_ERR(process);
130 
131 	if (kfd_is_locked()) {
132 		dev_dbg(kfd_device, "kfd is locked!\n"
133 				"process %d unreferenced", process->pasid);
134 		kfd_unref_process(process);
135 		return -EAGAIN;
136 	}
137 
138 	/* filep now owns the reference returned by kfd_create_process */
139 	filep->private_data = process;
140 
141 	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
142 		process->pasid, process->is_32bit_user_mode);
143 
144 	return 0;
145 }
146 
147 static int kfd_release(struct inode *inode, struct file *filep)
148 {
149 	struct kfd_process *process = filep->private_data;
150 
151 	if (process)
152 		kfd_unref_process(process);
153 
154 	return 0;
155 }
156 
157 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
158 					void *data)
159 {
160 	struct kfd_ioctl_get_version_args *args = data;
161 
162 	args->major_version = KFD_IOCTL_MAJOR_VERSION;
163 	args->minor_version = KFD_IOCTL_MINOR_VERSION;
164 
165 	return 0;
166 }
167 
168 static int set_queue_properties_from_user(struct queue_properties *q_properties,
169 				struct kfd_ioctl_create_queue_args *args)
170 {
171 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
172 		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
173 		return -EINVAL;
174 	}
175 
176 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
177 		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
178 		return -EINVAL;
179 	}
180 
181 	if ((args->ring_base_address) &&
182 		(!access_ok((const void __user *) args->ring_base_address,
183 			sizeof(uint64_t)))) {
184 		pr_err("Can't access ring base address\n");
185 		return -EFAULT;
186 	}
187 
188 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
189 		pr_err("Ring size must be a power of 2 or 0\n");
190 		return -EINVAL;
191 	}
192 
193 	if (!access_ok((const void __user *) args->read_pointer_address,
194 			sizeof(uint32_t))) {
195 		pr_err("Can't access read pointer\n");
196 		return -EFAULT;
197 	}
198 
199 	if (!access_ok((const void __user *) args->write_pointer_address,
200 			sizeof(uint32_t))) {
201 		pr_err("Can't access write pointer\n");
202 		return -EFAULT;
203 	}
204 
205 	if (args->eop_buffer_address &&
206 		!access_ok((const void __user *) args->eop_buffer_address,
207 			sizeof(uint32_t))) {
208 		pr_debug("Can't access eop buffer");
209 		return -EFAULT;
210 	}
211 
212 	if (args->ctx_save_restore_address &&
213 		!access_ok((const void __user *) args->ctx_save_restore_address,
214 			sizeof(uint32_t))) {
215 		pr_debug("Can't access ctx save restore buffer");
216 		return -EFAULT;
217 	}
218 
219 	q_properties->is_interop = false;
220 	q_properties->is_gws = false;
221 	q_properties->queue_percent = args->queue_percentage;
222 	q_properties->priority = args->queue_priority;
223 	q_properties->queue_address = args->ring_base_address;
224 	q_properties->queue_size = args->ring_size;
225 	q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
226 	q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
227 	q_properties->eop_ring_buffer_address = args->eop_buffer_address;
228 	q_properties->eop_ring_buffer_size = args->eop_buffer_size;
229 	q_properties->ctx_save_restore_area_address =
230 			args->ctx_save_restore_address;
231 	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
232 	q_properties->ctl_stack_size = args->ctl_stack_size;
233 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
234 		args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
235 		q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
236 	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
237 		q_properties->type = KFD_QUEUE_TYPE_SDMA;
238 	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
239 		q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
240 	else
241 		return -ENOTSUPP;
242 
243 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
244 		q_properties->format = KFD_QUEUE_FORMAT_AQL;
245 	else
246 		q_properties->format = KFD_QUEUE_FORMAT_PM4;
247 
248 	pr_debug("Queue Percentage: %d, %d\n",
249 			q_properties->queue_percent, args->queue_percentage);
250 
251 	pr_debug("Queue Priority: %d, %d\n",
252 			q_properties->priority, args->queue_priority);
253 
254 	pr_debug("Queue Address: 0x%llX, 0x%llX\n",
255 			q_properties->queue_address, args->ring_base_address);
256 
257 	pr_debug("Queue Size: 0x%llX, %u\n",
258 			q_properties->queue_size, args->ring_size);
259 
260 	pr_debug("Queue r/w Pointers: %px, %px\n",
261 			q_properties->read_ptr,
262 			q_properties->write_ptr);
263 
264 	pr_debug("Queue Format: %d\n", q_properties->format);
265 
266 	pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
267 
268 	pr_debug("Queue CTX save area: 0x%llX\n",
269 			q_properties->ctx_save_restore_area_address);
270 
271 	return 0;
272 }
273 
274 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
275 					void *data)
276 {
277 	struct kfd_ioctl_create_queue_args *args = data;
278 	struct kfd_dev *dev;
279 	int err = 0;
280 	unsigned int queue_id;
281 	struct kfd_process_device *pdd;
282 	struct queue_properties q_properties;
283 	uint32_t doorbell_offset_in_process = 0;
284 
285 	memset(&q_properties, 0, sizeof(struct queue_properties));
286 
287 	pr_debug("Creating queue ioctl\n");
288 
289 	err = set_queue_properties_from_user(&q_properties, args);
290 	if (err)
291 		return err;
292 
293 	pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
294 	dev = kfd_device_by_id(args->gpu_id);
295 	if (!dev) {
296 		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
297 		return -EINVAL;
298 	}
299 
300 	mutex_lock(&p->mutex);
301 
302 	pdd = kfd_bind_process_to_device(dev, p);
303 	if (IS_ERR(pdd)) {
304 		err = -ESRCH;
305 		goto err_bind_process;
306 	}
307 
308 	pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
309 			p->pasid,
310 			dev->id);
311 
312 	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id,
313 			&doorbell_offset_in_process);
314 	if (err != 0)
315 		goto err_create_queue;
316 
317 	args->queue_id = queue_id;
318 
319 
320 	/* Return gpu_id as doorbell offset for mmap usage */
321 	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
322 	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
323 	if (KFD_IS_SOC15(dev->device_info->asic_family))
324 		/* On SOC15 ASICs, include the doorbell offset within the
325 		 * process doorbell frame, which is 2 pages.
326 		 */
327 		args->doorbell_offset |= doorbell_offset_in_process;
328 
329 	mutex_unlock(&p->mutex);
330 
331 	pr_debug("Queue id %d was created successfully\n", args->queue_id);
332 
333 	pr_debug("Ring buffer address == 0x%016llX\n",
334 			args->ring_base_address);
335 
336 	pr_debug("Read ptr address    == 0x%016llX\n",
337 			args->read_pointer_address);
338 
339 	pr_debug("Write ptr address   == 0x%016llX\n",
340 			args->write_pointer_address);
341 
342 	return 0;
343 
344 err_create_queue:
345 err_bind_process:
346 	mutex_unlock(&p->mutex);
347 	return err;
348 }
349 
350 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
351 					void *data)
352 {
353 	int retval;
354 	struct kfd_ioctl_destroy_queue_args *args = data;
355 
356 	pr_debug("Destroying queue id %d for pasid 0x%x\n",
357 				args->queue_id,
358 				p->pasid);
359 
360 	mutex_lock(&p->mutex);
361 
362 	retval = pqm_destroy_queue(&p->pqm, args->queue_id);
363 
364 	mutex_unlock(&p->mutex);
365 	return retval;
366 }
367 
368 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
369 					void *data)
370 {
371 	int retval;
372 	struct kfd_ioctl_update_queue_args *args = data;
373 	struct queue_properties properties;
374 
375 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
376 		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
377 		return -EINVAL;
378 	}
379 
380 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
381 		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
382 		return -EINVAL;
383 	}
384 
385 	if ((args->ring_base_address) &&
386 		(!access_ok((const void __user *) args->ring_base_address,
387 			sizeof(uint64_t)))) {
388 		pr_err("Can't access ring base address\n");
389 		return -EFAULT;
390 	}
391 
392 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
393 		pr_err("Ring size must be a power of 2 or 0\n");
394 		return -EINVAL;
395 	}
396 
397 	properties.queue_address = args->ring_base_address;
398 	properties.queue_size = args->ring_size;
399 	properties.queue_percent = args->queue_percentage;
400 	properties.priority = args->queue_priority;
401 
402 	pr_debug("Updating queue id %d for pasid 0x%x\n",
403 			args->queue_id, p->pasid);
404 
405 	mutex_lock(&p->mutex);
406 
407 	retval = pqm_update_queue(&p->pqm, args->queue_id, &properties);
408 
409 	mutex_unlock(&p->mutex);
410 
411 	return retval;
412 }
413 
414 static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
415 					void *data)
416 {
417 	int retval;
418 	const int max_num_cus = 1024;
419 	struct kfd_ioctl_set_cu_mask_args *args = data;
420 	struct queue_properties properties;
421 	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
422 	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
423 
424 	if ((args->num_cu_mask % 32) != 0) {
425 		pr_debug("num_cu_mask 0x%x must be a multiple of 32",
426 				args->num_cu_mask);
427 		return -EINVAL;
428 	}
429 
430 	properties.cu_mask_count = args->num_cu_mask;
431 	if (properties.cu_mask_count == 0) {
432 		pr_debug("CU mask cannot be 0");
433 		return -EINVAL;
434 	}
435 
436 	/* To prevent an unreasonably large CU mask size, set an arbitrary
437 	 * limit of max_num_cus bits.  We can then just drop any CU mask bits
438 	 * past max_num_cus bits and just use the first max_num_cus bits.
439 	 */
440 	if (properties.cu_mask_count > max_num_cus) {
441 		pr_debug("CU mask cannot be greater than 1024 bits");
442 		properties.cu_mask_count = max_num_cus;
443 		cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
444 	}
445 
446 	properties.cu_mask = kzalloc(cu_mask_size, GFP_KERNEL);
447 	if (!properties.cu_mask)
448 		return -ENOMEM;
449 
450 	retval = copy_from_user(properties.cu_mask, cu_mask_ptr, cu_mask_size);
451 	if (retval) {
452 		pr_debug("Could not copy CU mask from userspace");
453 		kfree(properties.cu_mask);
454 		return -EFAULT;
455 	}
456 
457 	mutex_lock(&p->mutex);
458 
459 	retval = pqm_set_cu_mask(&p->pqm, args->queue_id, &properties);
460 
461 	mutex_unlock(&p->mutex);
462 
463 	if (retval)
464 		kfree(properties.cu_mask);
465 
466 	return retval;
467 }
468 
469 static int kfd_ioctl_get_queue_wave_state(struct file *filep,
470 					  struct kfd_process *p, void *data)
471 {
472 	struct kfd_ioctl_get_queue_wave_state_args *args = data;
473 	int r;
474 
475 	mutex_lock(&p->mutex);
476 
477 	r = pqm_get_wave_state(&p->pqm, args->queue_id,
478 			       (void __user *)args->ctl_stack_address,
479 			       &args->ctl_stack_used_size,
480 			       &args->save_area_used_size);
481 
482 	mutex_unlock(&p->mutex);
483 
484 	return r;
485 }
486 
487 static int kfd_ioctl_set_memory_policy(struct file *filep,
488 					struct kfd_process *p, void *data)
489 {
490 	struct kfd_ioctl_set_memory_policy_args *args = data;
491 	struct kfd_dev *dev;
492 	int err = 0;
493 	struct kfd_process_device *pdd;
494 	enum cache_policy default_policy, alternate_policy;
495 
496 	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
497 	    && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
498 		return -EINVAL;
499 	}
500 
501 	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
502 	    && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
503 		return -EINVAL;
504 	}
505 
506 	dev = kfd_device_by_id(args->gpu_id);
507 	if (!dev)
508 		return -EINVAL;
509 
510 	mutex_lock(&p->mutex);
511 
512 	pdd = kfd_bind_process_to_device(dev, p);
513 	if (IS_ERR(pdd)) {
514 		err = -ESRCH;
515 		goto out;
516 	}
517 
518 	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
519 			 ? cache_policy_coherent : cache_policy_noncoherent;
520 
521 	alternate_policy =
522 		(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
523 		   ? cache_policy_coherent : cache_policy_noncoherent;
524 
525 	if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm,
526 				&pdd->qpd,
527 				default_policy,
528 				alternate_policy,
529 				(void __user *)args->alternate_aperture_base,
530 				args->alternate_aperture_size))
531 		err = -EINVAL;
532 
533 out:
534 	mutex_unlock(&p->mutex);
535 
536 	return err;
537 }
538 
539 static int kfd_ioctl_set_trap_handler(struct file *filep,
540 					struct kfd_process *p, void *data)
541 {
542 	struct kfd_ioctl_set_trap_handler_args *args = data;
543 	struct kfd_dev *dev;
544 	int err = 0;
545 	struct kfd_process_device *pdd;
546 
547 	dev = kfd_device_by_id(args->gpu_id);
548 	if (!dev)
549 		return -EINVAL;
550 
551 	mutex_lock(&p->mutex);
552 
553 	pdd = kfd_bind_process_to_device(dev, p);
554 	if (IS_ERR(pdd)) {
555 		err = -ESRCH;
556 		goto out;
557 	}
558 
559 	if (dev->dqm->ops.set_trap_handler(dev->dqm,
560 					&pdd->qpd,
561 					args->tba_addr,
562 					args->tma_addr))
563 		err = -EINVAL;
564 
565 out:
566 	mutex_unlock(&p->mutex);
567 
568 	return err;
569 }
570 
571 static int kfd_ioctl_dbg_register(struct file *filep,
572 				struct kfd_process *p, void *data)
573 {
574 	struct kfd_ioctl_dbg_register_args *args = data;
575 	struct kfd_dev *dev;
576 	struct kfd_dbgmgr *dbgmgr_ptr;
577 	struct kfd_process_device *pdd;
578 	bool create_ok;
579 	long status = 0;
580 
581 	dev = kfd_device_by_id(args->gpu_id);
582 	if (!dev)
583 		return -EINVAL;
584 
585 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
586 		pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
587 		return -EINVAL;
588 	}
589 
590 	mutex_lock(&p->mutex);
591 	mutex_lock(kfd_get_dbgmgr_mutex());
592 
593 	/*
594 	 * make sure that we have pdd, if this the first queue created for
595 	 * this process
596 	 */
597 	pdd = kfd_bind_process_to_device(dev, p);
598 	if (IS_ERR(pdd)) {
599 		status = PTR_ERR(pdd);
600 		goto out;
601 	}
602 
603 	if (!dev->dbgmgr) {
604 		/* In case of a legal call, we have no dbgmgr yet */
605 		create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev);
606 		if (create_ok) {
607 			status = kfd_dbgmgr_register(dbgmgr_ptr, p);
608 			if (status != 0)
609 				kfd_dbgmgr_destroy(dbgmgr_ptr);
610 			else
611 				dev->dbgmgr = dbgmgr_ptr;
612 		}
613 	} else {
614 		pr_debug("debugger already registered\n");
615 		status = -EINVAL;
616 	}
617 
618 out:
619 	mutex_unlock(kfd_get_dbgmgr_mutex());
620 	mutex_unlock(&p->mutex);
621 
622 	return status;
623 }
624 
625 static int kfd_ioctl_dbg_unregister(struct file *filep,
626 				struct kfd_process *p, void *data)
627 {
628 	struct kfd_ioctl_dbg_unregister_args *args = data;
629 	struct kfd_dev *dev;
630 	long status;
631 
632 	dev = kfd_device_by_id(args->gpu_id);
633 	if (!dev || !dev->dbgmgr)
634 		return -EINVAL;
635 
636 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
637 		pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n");
638 		return -EINVAL;
639 	}
640 
641 	mutex_lock(kfd_get_dbgmgr_mutex());
642 
643 	status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
644 	if (!status) {
645 		kfd_dbgmgr_destroy(dev->dbgmgr);
646 		dev->dbgmgr = NULL;
647 	}
648 
649 	mutex_unlock(kfd_get_dbgmgr_mutex());
650 
651 	return status;
652 }
653 
654 /*
655  * Parse and generate variable size data structure for address watch.
656  * Total size of the buffer and # watch points is limited in order
657  * to prevent kernel abuse. (no bearing to the much smaller HW limitation
658  * which is enforced by dbgdev module)
659  * please also note that the watch address itself are not "copied from user",
660  * since it be set into the HW in user mode values.
661  *
662  */
663 static int kfd_ioctl_dbg_address_watch(struct file *filep,
664 					struct kfd_process *p, void *data)
665 {
666 	struct kfd_ioctl_dbg_address_watch_args *args = data;
667 	struct kfd_dev *dev;
668 	struct dbg_address_watch_info aw_info;
669 	unsigned char *args_buff;
670 	long status;
671 	void __user *cmd_from_user;
672 	uint64_t watch_mask_value = 0;
673 	unsigned int args_idx = 0;
674 
675 	memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info));
676 
677 	dev = kfd_device_by_id(args->gpu_id);
678 	if (!dev)
679 		return -EINVAL;
680 
681 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
682 		pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
683 		return -EINVAL;
684 	}
685 
686 	cmd_from_user = (void __user *) args->content_ptr;
687 
688 	/* Validate arguments */
689 
690 	if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) ||
691 		(args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) ||
692 		(cmd_from_user == NULL))
693 		return -EINVAL;
694 
695 	/* this is the actual buffer to work with */
696 	args_buff = memdup_user(cmd_from_user,
697 				args->buf_size_in_bytes - sizeof(*args));
698 	if (IS_ERR(args_buff))
699 		return PTR_ERR(args_buff);
700 
701 	aw_info.process = p;
702 
703 	aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx]));
704 	args_idx += sizeof(aw_info.num_watch_points);
705 
706 	aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx];
707 	args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points;
708 
709 	/*
710 	 * set watch address base pointer to point on the array base
711 	 * within args_buff
712 	 */
713 	aw_info.watch_address = (uint64_t *) &args_buff[args_idx];
714 
715 	/* skip over the addresses buffer */
716 	args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
717 
718 	if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) {
719 		status = -EINVAL;
720 		goto out;
721 	}
722 
723 	watch_mask_value = (uint64_t) args_buff[args_idx];
724 
725 	if (watch_mask_value > 0) {
726 		/*
727 		 * There is an array of masks.
728 		 * set watch mask base pointer to point on the array base
729 		 * within args_buff
730 		 */
731 		aw_info.watch_mask = (uint64_t *) &args_buff[args_idx];
732 
733 		/* skip over the masks buffer */
734 		args_idx += sizeof(aw_info.watch_mask) *
735 				aw_info.num_watch_points;
736 	} else {
737 		/* just the NULL mask, set to NULL and skip over it */
738 		aw_info.watch_mask = NULL;
739 		args_idx += sizeof(aw_info.watch_mask);
740 	}
741 
742 	if (args_idx >= args->buf_size_in_bytes - sizeof(args)) {
743 		status = -EINVAL;
744 		goto out;
745 	}
746 
747 	/* Currently HSA Event is not supported for DBG */
748 	aw_info.watch_event = NULL;
749 
750 	mutex_lock(kfd_get_dbgmgr_mutex());
751 
752 	status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info);
753 
754 	mutex_unlock(kfd_get_dbgmgr_mutex());
755 
756 out:
757 	kfree(args_buff);
758 
759 	return status;
760 }
761 
762 /* Parse and generate fixed size data structure for wave control */
763 static int kfd_ioctl_dbg_wave_control(struct file *filep,
764 					struct kfd_process *p, void *data)
765 {
766 	struct kfd_ioctl_dbg_wave_control_args *args = data;
767 	struct kfd_dev *dev;
768 	struct dbg_wave_control_info wac_info;
769 	unsigned char *args_buff;
770 	uint32_t computed_buff_size;
771 	long status;
772 	void __user *cmd_from_user;
773 	unsigned int args_idx = 0;
774 
775 	memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info));
776 
777 	/* we use compact form, independent of the packing attribute value */
778 	computed_buff_size = sizeof(*args) +
779 				sizeof(wac_info.mode) +
780 				sizeof(wac_info.operand) +
781 				sizeof(wac_info.dbgWave_msg.DbgWaveMsg) +
782 				sizeof(wac_info.dbgWave_msg.MemoryVA) +
783 				sizeof(wac_info.trapId);
784 
785 	dev = kfd_device_by_id(args->gpu_id);
786 	if (!dev)
787 		return -EINVAL;
788 
789 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
790 		pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
791 		return -EINVAL;
792 	}
793 
794 	/* input size must match the computed "compact" size */
795 	if (args->buf_size_in_bytes != computed_buff_size) {
796 		pr_debug("size mismatch, computed : actual %u : %u\n",
797 				args->buf_size_in_bytes, computed_buff_size);
798 		return -EINVAL;
799 	}
800 
801 	cmd_from_user = (void __user *) args->content_ptr;
802 
803 	if (cmd_from_user == NULL)
804 		return -EINVAL;
805 
806 	/* copy the entire buffer from user */
807 
808 	args_buff = memdup_user(cmd_from_user,
809 				args->buf_size_in_bytes - sizeof(*args));
810 	if (IS_ERR(args_buff))
811 		return PTR_ERR(args_buff);
812 
813 	/* move ptr to the start of the "pay-load" area */
814 	wac_info.process = p;
815 
816 	wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx]));
817 	args_idx += sizeof(wac_info.operand);
818 
819 	wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx]));
820 	args_idx += sizeof(wac_info.mode);
821 
822 	wac_info.trapId = *((uint32_t *)(&args_buff[args_idx]));
823 	args_idx += sizeof(wac_info.trapId);
824 
825 	wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value =
826 					*((uint32_t *)(&args_buff[args_idx]));
827 	wac_info.dbgWave_msg.MemoryVA = NULL;
828 
829 	mutex_lock(kfd_get_dbgmgr_mutex());
830 
831 	pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
832 			wac_info.process, wac_info.operand,
833 			wac_info.mode, wac_info.trapId,
834 			wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
835 
836 	status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info);
837 
838 	pr_debug("Returned status of dbg manager is %ld\n", status);
839 
840 	mutex_unlock(kfd_get_dbgmgr_mutex());
841 
842 	kfree(args_buff);
843 
844 	return status;
845 }
846 
847 static int kfd_ioctl_get_clock_counters(struct file *filep,
848 				struct kfd_process *p, void *data)
849 {
850 	struct kfd_ioctl_get_clock_counters_args *args = data;
851 	struct kfd_dev *dev;
852 
853 	dev = kfd_device_by_id(args->gpu_id);
854 	if (dev)
855 		/* Reading GPU clock counter from KGD */
856 		args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(dev->kgd);
857 	else
858 		/* Node without GPU resource */
859 		args->gpu_clock_counter = 0;
860 
861 	/* No access to rdtsc. Using raw monotonic time */
862 	args->cpu_clock_counter = ktime_get_raw_ns();
863 	args->system_clock_counter = ktime_get_boottime_ns();
864 
865 	/* Since the counter is in nano-seconds we use 1GHz frequency */
866 	args->system_clock_freq = 1000000000;
867 
868 	return 0;
869 }
870 
871 
872 static int kfd_ioctl_get_process_apertures(struct file *filp,
873 				struct kfd_process *p, void *data)
874 {
875 	struct kfd_ioctl_get_process_apertures_args *args = data;
876 	struct kfd_process_device_apertures *pAperture;
877 	struct kfd_process_device *pdd;
878 
879 	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
880 
881 	args->num_of_nodes = 0;
882 
883 	mutex_lock(&p->mutex);
884 
885 	/*if the process-device list isn't empty*/
886 	if (kfd_has_process_device_data(p)) {
887 		/* Run over all pdd of the process */
888 		pdd = kfd_get_first_process_device_data(p);
889 		do {
890 			pAperture =
891 				&args->process_apertures[args->num_of_nodes];
892 			pAperture->gpu_id = pdd->dev->id;
893 			pAperture->lds_base = pdd->lds_base;
894 			pAperture->lds_limit = pdd->lds_limit;
895 			pAperture->gpuvm_base = pdd->gpuvm_base;
896 			pAperture->gpuvm_limit = pdd->gpuvm_limit;
897 			pAperture->scratch_base = pdd->scratch_base;
898 			pAperture->scratch_limit = pdd->scratch_limit;
899 
900 			dev_dbg(kfd_device,
901 				"node id %u\n", args->num_of_nodes);
902 			dev_dbg(kfd_device,
903 				"gpu id %u\n", pdd->dev->id);
904 			dev_dbg(kfd_device,
905 				"lds_base %llX\n", pdd->lds_base);
906 			dev_dbg(kfd_device,
907 				"lds_limit %llX\n", pdd->lds_limit);
908 			dev_dbg(kfd_device,
909 				"gpuvm_base %llX\n", pdd->gpuvm_base);
910 			dev_dbg(kfd_device,
911 				"gpuvm_limit %llX\n", pdd->gpuvm_limit);
912 			dev_dbg(kfd_device,
913 				"scratch_base %llX\n", pdd->scratch_base);
914 			dev_dbg(kfd_device,
915 				"scratch_limit %llX\n", pdd->scratch_limit);
916 
917 			args->num_of_nodes++;
918 
919 			pdd = kfd_get_next_process_device_data(p, pdd);
920 		} while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
921 	}
922 
923 	mutex_unlock(&p->mutex);
924 
925 	return 0;
926 }
927 
928 static int kfd_ioctl_get_process_apertures_new(struct file *filp,
929 				struct kfd_process *p, void *data)
930 {
931 	struct kfd_ioctl_get_process_apertures_new_args *args = data;
932 	struct kfd_process_device_apertures *pa;
933 	struct kfd_process_device *pdd;
934 	uint32_t nodes = 0;
935 	int ret;
936 
937 	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
938 
939 	if (args->num_of_nodes == 0) {
940 		/* Return number of nodes, so that user space can alloacate
941 		 * sufficient memory
942 		 */
943 		mutex_lock(&p->mutex);
944 
945 		if (!kfd_has_process_device_data(p))
946 			goto out_unlock;
947 
948 		/* Run over all pdd of the process */
949 		pdd = kfd_get_first_process_device_data(p);
950 		do {
951 			args->num_of_nodes++;
952 			pdd = kfd_get_next_process_device_data(p, pdd);
953 		} while (pdd);
954 
955 		goto out_unlock;
956 	}
957 
958 	/* Fill in process-aperture information for all available
959 	 * nodes, but not more than args->num_of_nodes as that is
960 	 * the amount of memory allocated by user
961 	 */
962 	pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
963 				args->num_of_nodes), GFP_KERNEL);
964 	if (!pa)
965 		return -ENOMEM;
966 
967 	mutex_lock(&p->mutex);
968 
969 	if (!kfd_has_process_device_data(p)) {
970 		args->num_of_nodes = 0;
971 		kfree(pa);
972 		goto out_unlock;
973 	}
974 
975 	/* Run over all pdd of the process */
976 	pdd = kfd_get_first_process_device_data(p);
977 	do {
978 		pa[nodes].gpu_id = pdd->dev->id;
979 		pa[nodes].lds_base = pdd->lds_base;
980 		pa[nodes].lds_limit = pdd->lds_limit;
981 		pa[nodes].gpuvm_base = pdd->gpuvm_base;
982 		pa[nodes].gpuvm_limit = pdd->gpuvm_limit;
983 		pa[nodes].scratch_base = pdd->scratch_base;
984 		pa[nodes].scratch_limit = pdd->scratch_limit;
985 
986 		dev_dbg(kfd_device,
987 			"gpu id %u\n", pdd->dev->id);
988 		dev_dbg(kfd_device,
989 			"lds_base %llX\n", pdd->lds_base);
990 		dev_dbg(kfd_device,
991 			"lds_limit %llX\n", pdd->lds_limit);
992 		dev_dbg(kfd_device,
993 			"gpuvm_base %llX\n", pdd->gpuvm_base);
994 		dev_dbg(kfd_device,
995 			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
996 		dev_dbg(kfd_device,
997 			"scratch_base %llX\n", pdd->scratch_base);
998 		dev_dbg(kfd_device,
999 			"scratch_limit %llX\n", pdd->scratch_limit);
1000 		nodes++;
1001 
1002 		pdd = kfd_get_next_process_device_data(p, pdd);
1003 	} while (pdd && (nodes < args->num_of_nodes));
1004 	mutex_unlock(&p->mutex);
1005 
1006 	args->num_of_nodes = nodes;
1007 	ret = copy_to_user(
1008 			(void __user *)args->kfd_process_device_apertures_ptr,
1009 			pa,
1010 			(nodes * sizeof(struct kfd_process_device_apertures)));
1011 	kfree(pa);
1012 	return ret ? -EFAULT : 0;
1013 
1014 out_unlock:
1015 	mutex_unlock(&p->mutex);
1016 	return 0;
1017 }
1018 
1019 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
1020 					void *data)
1021 {
1022 	struct kfd_ioctl_create_event_args *args = data;
1023 	int err;
1024 
1025 	/* For dGPUs the event page is allocated in user mode. The
1026 	 * handle is passed to KFD with the first call to this IOCTL
1027 	 * through the event_page_offset field.
1028 	 */
1029 	if (args->event_page_offset) {
1030 		struct kfd_dev *kfd;
1031 		struct kfd_process_device *pdd;
1032 		void *mem, *kern_addr;
1033 		uint64_t size;
1034 
1035 		if (p->signal_page) {
1036 			pr_err("Event page is already set\n");
1037 			return -EINVAL;
1038 		}
1039 
1040 		kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset));
1041 		if (!kfd) {
1042 			pr_err("Getting device by id failed in %s\n", __func__);
1043 			return -EINVAL;
1044 		}
1045 
1046 		mutex_lock(&p->mutex);
1047 		pdd = kfd_bind_process_to_device(kfd, p);
1048 		if (IS_ERR(pdd)) {
1049 			err = PTR_ERR(pdd);
1050 			goto out_unlock;
1051 		}
1052 
1053 		mem = kfd_process_device_translate_handle(pdd,
1054 				GET_IDR_HANDLE(args->event_page_offset));
1055 		if (!mem) {
1056 			pr_err("Can't find BO, offset is 0x%llx\n",
1057 			       args->event_page_offset);
1058 			err = -EINVAL;
1059 			goto out_unlock;
1060 		}
1061 		mutex_unlock(&p->mutex);
1062 
1063 		err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd,
1064 						mem, &kern_addr, &size);
1065 		if (err) {
1066 			pr_err("Failed to map event page to kernel\n");
1067 			return err;
1068 		}
1069 
1070 		err = kfd_event_page_set(p, kern_addr, size);
1071 		if (err) {
1072 			pr_err("Failed to set event page\n");
1073 			return err;
1074 		}
1075 	}
1076 
1077 	err = kfd_event_create(filp, p, args->event_type,
1078 				args->auto_reset != 0, args->node_id,
1079 				&args->event_id, &args->event_trigger_data,
1080 				&args->event_page_offset,
1081 				&args->event_slot_index);
1082 
1083 	return err;
1084 
1085 out_unlock:
1086 	mutex_unlock(&p->mutex);
1087 	return err;
1088 }
1089 
1090 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
1091 					void *data)
1092 {
1093 	struct kfd_ioctl_destroy_event_args *args = data;
1094 
1095 	return kfd_event_destroy(p, args->event_id);
1096 }
1097 
1098 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
1099 				void *data)
1100 {
1101 	struct kfd_ioctl_set_event_args *args = data;
1102 
1103 	return kfd_set_event(p, args->event_id);
1104 }
1105 
1106 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
1107 				void *data)
1108 {
1109 	struct kfd_ioctl_reset_event_args *args = data;
1110 
1111 	return kfd_reset_event(p, args->event_id);
1112 }
1113 
1114 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
1115 				void *data)
1116 {
1117 	struct kfd_ioctl_wait_events_args *args = data;
1118 	int err;
1119 
1120 	err = kfd_wait_on_events(p, args->num_events,
1121 			(void __user *)args->events_ptr,
1122 			(args->wait_for_all != 0),
1123 			args->timeout, &args->wait_result);
1124 
1125 	return err;
1126 }
1127 static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
1128 					struct kfd_process *p, void *data)
1129 {
1130 	struct kfd_ioctl_set_scratch_backing_va_args *args = data;
1131 	struct kfd_process_device *pdd;
1132 	struct kfd_dev *dev;
1133 	long err;
1134 
1135 	dev = kfd_device_by_id(args->gpu_id);
1136 	if (!dev)
1137 		return -EINVAL;
1138 
1139 	mutex_lock(&p->mutex);
1140 
1141 	pdd = kfd_bind_process_to_device(dev, p);
1142 	if (IS_ERR(pdd)) {
1143 		err = PTR_ERR(pdd);
1144 		goto bind_process_to_device_fail;
1145 	}
1146 
1147 	pdd->qpd.sh_hidden_private_base = args->va_addr;
1148 
1149 	mutex_unlock(&p->mutex);
1150 
1151 	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
1152 	    pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
1153 		dev->kfd2kgd->set_scratch_backing_va(
1154 			dev->kgd, args->va_addr, pdd->qpd.vmid);
1155 
1156 	return 0;
1157 
1158 bind_process_to_device_fail:
1159 	mutex_unlock(&p->mutex);
1160 	return err;
1161 }
1162 
1163 static int kfd_ioctl_get_tile_config(struct file *filep,
1164 		struct kfd_process *p, void *data)
1165 {
1166 	struct kfd_ioctl_get_tile_config_args *args = data;
1167 	struct kfd_dev *dev;
1168 	struct tile_config config;
1169 	int err = 0;
1170 
1171 	dev = kfd_device_by_id(args->gpu_id);
1172 	if (!dev)
1173 		return -EINVAL;
1174 
1175 	amdgpu_amdkfd_get_tile_config(dev->kgd, &config);
1176 
1177 	args->gb_addr_config = config.gb_addr_config;
1178 	args->num_banks = config.num_banks;
1179 	args->num_ranks = config.num_ranks;
1180 
1181 	if (args->num_tile_configs > config.num_tile_configs)
1182 		args->num_tile_configs = config.num_tile_configs;
1183 	err = copy_to_user((void __user *)args->tile_config_ptr,
1184 			config.tile_config_ptr,
1185 			args->num_tile_configs * sizeof(uint32_t));
1186 	if (err) {
1187 		args->num_tile_configs = 0;
1188 		return -EFAULT;
1189 	}
1190 
1191 	if (args->num_macro_tile_configs > config.num_macro_tile_configs)
1192 		args->num_macro_tile_configs =
1193 				config.num_macro_tile_configs;
1194 	err = copy_to_user((void __user *)args->macro_tile_config_ptr,
1195 			config.macro_tile_config_ptr,
1196 			args->num_macro_tile_configs * sizeof(uint32_t));
1197 	if (err) {
1198 		args->num_macro_tile_configs = 0;
1199 		return -EFAULT;
1200 	}
1201 
1202 	return 0;
1203 }
1204 
1205 static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
1206 				void *data)
1207 {
1208 	struct kfd_ioctl_acquire_vm_args *args = data;
1209 	struct kfd_process_device *pdd;
1210 	struct kfd_dev *dev;
1211 	struct file *drm_file;
1212 	int ret;
1213 
1214 	dev = kfd_device_by_id(args->gpu_id);
1215 	if (!dev)
1216 		return -EINVAL;
1217 
1218 	drm_file = fget(args->drm_fd);
1219 	if (!drm_file)
1220 		return -EINVAL;
1221 
1222 	mutex_lock(&p->mutex);
1223 
1224 	pdd = kfd_get_process_device_data(dev, p);
1225 	if (!pdd) {
1226 		ret = -EINVAL;
1227 		goto err_unlock;
1228 	}
1229 
1230 	if (pdd->drm_file) {
1231 		ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
1232 		goto err_unlock;
1233 	}
1234 
1235 	ret = kfd_process_device_init_vm(pdd, drm_file);
1236 	if (ret)
1237 		goto err_unlock;
1238 	/* On success, the PDD keeps the drm_file reference */
1239 	mutex_unlock(&p->mutex);
1240 
1241 	return 0;
1242 
1243 err_unlock:
1244 	mutex_unlock(&p->mutex);
1245 	fput(drm_file);
1246 	return ret;
1247 }
1248 
1249 bool kfd_dev_is_large_bar(struct kfd_dev *dev)
1250 {
1251 	struct kfd_local_mem_info mem_info;
1252 
1253 	if (debug_largebar) {
1254 		pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1255 		return true;
1256 	}
1257 
1258 	if (dev->use_iommu_v2)
1259 		return false;
1260 
1261 	amdgpu_amdkfd_get_local_mem_info(dev->kgd, &mem_info);
1262 	if (mem_info.local_mem_size_private == 0 &&
1263 			mem_info.local_mem_size_public > 0)
1264 		return true;
1265 	return false;
1266 }
1267 
1268 static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1269 					struct kfd_process *p, void *data)
1270 {
1271 	struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1272 	struct kfd_process_device *pdd;
1273 	void *mem;
1274 	struct kfd_dev *dev;
1275 	int idr_handle;
1276 	long err;
1277 	uint64_t offset = args->mmap_offset;
1278 	uint32_t flags = args->flags;
1279 
1280 	if (args->size == 0)
1281 		return -EINVAL;
1282 
1283 	dev = kfd_device_by_id(args->gpu_id);
1284 	if (!dev)
1285 		return -EINVAL;
1286 
1287 	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1288 		(flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1289 		!kfd_dev_is_large_bar(dev)) {
1290 		pr_err("Alloc host visible vram on small bar is not allowed\n");
1291 		return -EINVAL;
1292 	}
1293 
1294 	mutex_lock(&p->mutex);
1295 
1296 	pdd = kfd_bind_process_to_device(dev, p);
1297 	if (IS_ERR(pdd)) {
1298 		err = PTR_ERR(pdd);
1299 		goto err_unlock;
1300 	}
1301 
1302 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1303 		if (args->size != kfd_doorbell_process_slice(dev)) {
1304 			err = -EINVAL;
1305 			goto err_unlock;
1306 		}
1307 		offset = kfd_get_process_doorbells(pdd);
1308 	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1309 		if (args->size != PAGE_SIZE) {
1310 			err = -EINVAL;
1311 			goto err_unlock;
1312 		}
1313 		offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
1314 		if (!offset) {
1315 			err = -ENOMEM;
1316 			goto err_unlock;
1317 		}
1318 	}
1319 
1320 	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1321 		dev->kgd, args->va_addr, args->size,
1322 		pdd->vm, (struct kgd_mem **) &mem, &offset,
1323 		flags);
1324 
1325 	if (err)
1326 		goto err_unlock;
1327 
1328 	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1329 	if (idr_handle < 0) {
1330 		err = -EFAULT;
1331 		goto err_free;
1332 	}
1333 
1334 	/* Update the VRAM usage count */
1335 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
1336 		WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + args->size);
1337 
1338 	mutex_unlock(&p->mutex);
1339 
1340 	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1341 	args->mmap_offset = offset;
1342 
1343 	/* MMIO is mapped through kfd device
1344 	 * Generate a kfd mmap offset
1345 	 */
1346 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1347 		args->mmap_offset = KFD_MMAP_TYPE_MMIO
1348 					| KFD_MMAP_GPU_ID(args->gpu_id);
1349 
1350 	return 0;
1351 
1352 err_free:
1353 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
1354 err_unlock:
1355 	mutex_unlock(&p->mutex);
1356 	return err;
1357 }
1358 
1359 static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1360 					struct kfd_process *p, void *data)
1361 {
1362 	struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1363 	struct kfd_process_device *pdd;
1364 	void *mem;
1365 	struct kfd_dev *dev;
1366 	int ret;
1367 	uint64_t size = 0;
1368 
1369 	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1370 	if (!dev)
1371 		return -EINVAL;
1372 
1373 	mutex_lock(&p->mutex);
1374 
1375 	pdd = kfd_get_process_device_data(dev, p);
1376 	if (!pdd) {
1377 		pr_err("Process device data doesn't exist\n");
1378 		ret = -EINVAL;
1379 		goto err_unlock;
1380 	}
1381 
1382 	mem = kfd_process_device_translate_handle(
1383 		pdd, GET_IDR_HANDLE(args->handle));
1384 	if (!mem) {
1385 		ret = -EINVAL;
1386 		goto err_unlock;
1387 	}
1388 
1389 	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd,
1390 						(struct kgd_mem *)mem, &size);
1391 
1392 	/* If freeing the buffer failed, leave the handle in place for
1393 	 * clean-up during process tear-down.
1394 	 */
1395 	if (!ret)
1396 		kfd_process_device_remove_obj_handle(
1397 			pdd, GET_IDR_HANDLE(args->handle));
1398 
1399 	WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1400 
1401 err_unlock:
1402 	mutex_unlock(&p->mutex);
1403 	return ret;
1404 }
1405 
1406 static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1407 					struct kfd_process *p, void *data)
1408 {
1409 	struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1410 	struct kfd_process_device *pdd, *peer_pdd;
1411 	void *mem;
1412 	struct kfd_dev *dev, *peer;
1413 	long err = 0;
1414 	int i;
1415 	uint32_t *devices_arr = NULL;
1416 
1417 	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1418 	if (!dev)
1419 		return -EINVAL;
1420 
1421 	if (!args->n_devices) {
1422 		pr_debug("Device IDs array empty\n");
1423 		return -EINVAL;
1424 	}
1425 	if (args->n_success > args->n_devices) {
1426 		pr_debug("n_success exceeds n_devices\n");
1427 		return -EINVAL;
1428 	}
1429 
1430 	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1431 				    GFP_KERNEL);
1432 	if (!devices_arr)
1433 		return -ENOMEM;
1434 
1435 	err = copy_from_user(devices_arr,
1436 			     (void __user *)args->device_ids_array_ptr,
1437 			     args->n_devices * sizeof(*devices_arr));
1438 	if (err != 0) {
1439 		err = -EFAULT;
1440 		goto copy_from_user_failed;
1441 	}
1442 
1443 	mutex_lock(&p->mutex);
1444 
1445 	pdd = kfd_bind_process_to_device(dev, p);
1446 	if (IS_ERR(pdd)) {
1447 		err = PTR_ERR(pdd);
1448 		goto bind_process_to_device_failed;
1449 	}
1450 
1451 	mem = kfd_process_device_translate_handle(pdd,
1452 						GET_IDR_HANDLE(args->handle));
1453 	if (!mem) {
1454 		err = -ENOMEM;
1455 		goto get_mem_obj_from_handle_failed;
1456 	}
1457 
1458 	for (i = args->n_success; i < args->n_devices; i++) {
1459 		peer = kfd_device_by_id(devices_arr[i]);
1460 		if (!peer) {
1461 			pr_debug("Getting device by id failed for 0x%x\n",
1462 				 devices_arr[i]);
1463 			err = -EINVAL;
1464 			goto get_mem_obj_from_handle_failed;
1465 		}
1466 
1467 		peer_pdd = kfd_bind_process_to_device(peer, p);
1468 		if (IS_ERR(peer_pdd)) {
1469 			err = PTR_ERR(peer_pdd);
1470 			goto get_mem_obj_from_handle_failed;
1471 		}
1472 		err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1473 			peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
1474 		if (err) {
1475 			pr_err("Failed to map to gpu %d/%d\n",
1476 			       i, args->n_devices);
1477 			goto map_memory_to_gpu_failed;
1478 		}
1479 		args->n_success = i+1;
1480 	}
1481 
1482 	mutex_unlock(&p->mutex);
1483 
1484 	err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd, (struct kgd_mem *) mem, true);
1485 	if (err) {
1486 		pr_debug("Sync memory failed, wait interrupted by user signal\n");
1487 		goto sync_memory_failed;
1488 	}
1489 
1490 	/* Flush TLBs after waiting for the page table updates to complete */
1491 	for (i = 0; i < args->n_devices; i++) {
1492 		peer = kfd_device_by_id(devices_arr[i]);
1493 		if (WARN_ON_ONCE(!peer))
1494 			continue;
1495 		peer_pdd = kfd_get_process_device_data(peer, p);
1496 		if (WARN_ON_ONCE(!peer_pdd))
1497 			continue;
1498 		kfd_flush_tlb(peer_pdd);
1499 	}
1500 
1501 	kfree(devices_arr);
1502 
1503 	return err;
1504 
1505 bind_process_to_device_failed:
1506 get_mem_obj_from_handle_failed:
1507 map_memory_to_gpu_failed:
1508 	mutex_unlock(&p->mutex);
1509 copy_from_user_failed:
1510 sync_memory_failed:
1511 	kfree(devices_arr);
1512 
1513 	return err;
1514 }
1515 
1516 static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1517 					struct kfd_process *p, void *data)
1518 {
1519 	struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1520 	struct kfd_process_device *pdd, *peer_pdd;
1521 	void *mem;
1522 	struct kfd_dev *dev, *peer;
1523 	long err = 0;
1524 	uint32_t *devices_arr = NULL, i;
1525 
1526 	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1527 	if (!dev)
1528 		return -EINVAL;
1529 
1530 	if (!args->n_devices) {
1531 		pr_debug("Device IDs array empty\n");
1532 		return -EINVAL;
1533 	}
1534 	if (args->n_success > args->n_devices) {
1535 		pr_debug("n_success exceeds n_devices\n");
1536 		return -EINVAL;
1537 	}
1538 
1539 	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1540 				    GFP_KERNEL);
1541 	if (!devices_arr)
1542 		return -ENOMEM;
1543 
1544 	err = copy_from_user(devices_arr,
1545 			     (void __user *)args->device_ids_array_ptr,
1546 			     args->n_devices * sizeof(*devices_arr));
1547 	if (err != 0) {
1548 		err = -EFAULT;
1549 		goto copy_from_user_failed;
1550 	}
1551 
1552 	mutex_lock(&p->mutex);
1553 
1554 	pdd = kfd_get_process_device_data(dev, p);
1555 	if (!pdd) {
1556 		err = -EINVAL;
1557 		goto bind_process_to_device_failed;
1558 	}
1559 
1560 	mem = kfd_process_device_translate_handle(pdd,
1561 						GET_IDR_HANDLE(args->handle));
1562 	if (!mem) {
1563 		err = -ENOMEM;
1564 		goto get_mem_obj_from_handle_failed;
1565 	}
1566 
1567 	for (i = args->n_success; i < args->n_devices; i++) {
1568 		peer = kfd_device_by_id(devices_arr[i]);
1569 		if (!peer) {
1570 			err = -EINVAL;
1571 			goto get_mem_obj_from_handle_failed;
1572 		}
1573 
1574 		peer_pdd = kfd_get_process_device_data(peer, p);
1575 		if (!peer_pdd) {
1576 			err = -ENODEV;
1577 			goto get_mem_obj_from_handle_failed;
1578 		}
1579 		err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1580 			peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
1581 		if (err) {
1582 			pr_err("Failed to unmap from gpu %d/%d\n",
1583 			       i, args->n_devices);
1584 			goto unmap_memory_from_gpu_failed;
1585 		}
1586 		args->n_success = i+1;
1587 	}
1588 	kfree(devices_arr);
1589 
1590 	mutex_unlock(&p->mutex);
1591 
1592 	return 0;
1593 
1594 bind_process_to_device_failed:
1595 get_mem_obj_from_handle_failed:
1596 unmap_memory_from_gpu_failed:
1597 	mutex_unlock(&p->mutex);
1598 copy_from_user_failed:
1599 	kfree(devices_arr);
1600 	return err;
1601 }
1602 
1603 static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1604 		struct kfd_process *p, void *data)
1605 {
1606 	int retval;
1607 	struct kfd_ioctl_alloc_queue_gws_args *args = data;
1608 	struct queue *q;
1609 	struct kfd_dev *dev;
1610 
1611 	mutex_lock(&p->mutex);
1612 	q = pqm_get_user_queue(&p->pqm, args->queue_id);
1613 
1614 	if (q) {
1615 		dev = q->device;
1616 	} else {
1617 		retval = -EINVAL;
1618 		goto out_unlock;
1619 	}
1620 
1621 	if (!dev->gws) {
1622 		retval = -ENODEV;
1623 		goto out_unlock;
1624 	}
1625 
1626 	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1627 		retval = -ENODEV;
1628 		goto out_unlock;
1629 	}
1630 
1631 	retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1632 	mutex_unlock(&p->mutex);
1633 
1634 	args->first_gws = 0;
1635 	return retval;
1636 
1637 out_unlock:
1638 	mutex_unlock(&p->mutex);
1639 	return retval;
1640 }
1641 
1642 static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1643 		struct kfd_process *p, void *data)
1644 {
1645 	struct kfd_ioctl_get_dmabuf_info_args *args = data;
1646 	struct kfd_dev *dev = NULL;
1647 	struct kgd_dev *dma_buf_kgd;
1648 	void *metadata_buffer = NULL;
1649 	uint32_t flags;
1650 	unsigned int i;
1651 	int r;
1652 
1653 	/* Find a KFD GPU device that supports the get_dmabuf_info query */
1654 	for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1655 		if (dev)
1656 			break;
1657 	if (!dev)
1658 		return -EINVAL;
1659 
1660 	if (args->metadata_ptr) {
1661 		metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1662 		if (!metadata_buffer)
1663 			return -ENOMEM;
1664 	}
1665 
1666 	/* Get dmabuf info from KGD */
1667 	r = amdgpu_amdkfd_get_dmabuf_info(dev->kgd, args->dmabuf_fd,
1668 					  &dma_buf_kgd, &args->size,
1669 					  metadata_buffer, args->metadata_size,
1670 					  &args->metadata_size, &flags);
1671 	if (r)
1672 		goto exit;
1673 
1674 	/* Reverse-lookup gpu_id from kgd pointer */
1675 	dev = kfd_device_by_kgd(dma_buf_kgd);
1676 	if (!dev) {
1677 		r = -EINVAL;
1678 		goto exit;
1679 	}
1680 	args->gpu_id = dev->id;
1681 	args->flags = flags;
1682 
1683 	/* Copy metadata buffer to user mode */
1684 	if (metadata_buffer) {
1685 		r = copy_to_user((void __user *)args->metadata_ptr,
1686 				 metadata_buffer, args->metadata_size);
1687 		if (r != 0)
1688 			r = -EFAULT;
1689 	}
1690 
1691 exit:
1692 	kfree(metadata_buffer);
1693 
1694 	return r;
1695 }
1696 
1697 static int kfd_ioctl_import_dmabuf(struct file *filep,
1698 				   struct kfd_process *p, void *data)
1699 {
1700 	struct kfd_ioctl_import_dmabuf_args *args = data;
1701 	struct kfd_process_device *pdd;
1702 	struct dma_buf *dmabuf;
1703 	struct kfd_dev *dev;
1704 	int idr_handle;
1705 	uint64_t size;
1706 	void *mem;
1707 	int r;
1708 
1709 	dev = kfd_device_by_id(args->gpu_id);
1710 	if (!dev)
1711 		return -EINVAL;
1712 
1713 	dmabuf = dma_buf_get(args->dmabuf_fd);
1714 	if (IS_ERR(dmabuf))
1715 		return PTR_ERR(dmabuf);
1716 
1717 	mutex_lock(&p->mutex);
1718 
1719 	pdd = kfd_bind_process_to_device(dev, p);
1720 	if (IS_ERR(pdd)) {
1721 		r = PTR_ERR(pdd);
1722 		goto err_unlock;
1723 	}
1724 
1725 	r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf,
1726 					      args->va_addr, pdd->vm,
1727 					      (struct kgd_mem **)&mem, &size,
1728 					      NULL);
1729 	if (r)
1730 		goto err_unlock;
1731 
1732 	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1733 	if (idr_handle < 0) {
1734 		r = -EFAULT;
1735 		goto err_free;
1736 	}
1737 
1738 	mutex_unlock(&p->mutex);
1739 	dma_buf_put(dmabuf);
1740 
1741 	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1742 
1743 	return 0;
1744 
1745 err_free:
1746 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
1747 err_unlock:
1748 	mutex_unlock(&p->mutex);
1749 	dma_buf_put(dmabuf);
1750 	return r;
1751 }
1752 
1753 /* Handle requests for watching SMI events */
1754 static int kfd_ioctl_smi_events(struct file *filep,
1755 				struct kfd_process *p, void *data)
1756 {
1757 	struct kfd_ioctl_smi_events_args *args = data;
1758 	struct kfd_dev *dev;
1759 
1760 	dev = kfd_device_by_id(args->gpuid);
1761 	if (!dev)
1762 		return -EINVAL;
1763 
1764 	return kfd_smi_event_open(dev, &args->anon_fd);
1765 }
1766 
1767 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
1768 	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
1769 			    .cmd_drv = 0, .name = #ioctl}
1770 
1771 /** Ioctl table */
1772 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
1773 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
1774 			kfd_ioctl_get_version, 0),
1775 
1776 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
1777 			kfd_ioctl_create_queue, 0),
1778 
1779 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
1780 			kfd_ioctl_destroy_queue, 0),
1781 
1782 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
1783 			kfd_ioctl_set_memory_policy, 0),
1784 
1785 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
1786 			kfd_ioctl_get_clock_counters, 0),
1787 
1788 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
1789 			kfd_ioctl_get_process_apertures, 0),
1790 
1791 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
1792 			kfd_ioctl_update_queue, 0),
1793 
1794 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
1795 			kfd_ioctl_create_event, 0),
1796 
1797 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
1798 			kfd_ioctl_destroy_event, 0),
1799 
1800 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
1801 			kfd_ioctl_set_event, 0),
1802 
1803 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
1804 			kfd_ioctl_reset_event, 0),
1805 
1806 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
1807 			kfd_ioctl_wait_events, 0),
1808 
1809 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER,
1810 			kfd_ioctl_dbg_register, 0),
1811 
1812 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER,
1813 			kfd_ioctl_dbg_unregister, 0),
1814 
1815 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH,
1816 			kfd_ioctl_dbg_address_watch, 0),
1817 
1818 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL,
1819 			kfd_ioctl_dbg_wave_control, 0),
1820 
1821 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
1822 			kfd_ioctl_set_scratch_backing_va, 0),
1823 
1824 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
1825 			kfd_ioctl_get_tile_config, 0),
1826 
1827 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
1828 			kfd_ioctl_set_trap_handler, 0),
1829 
1830 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
1831 			kfd_ioctl_get_process_apertures_new, 0),
1832 
1833 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
1834 			kfd_ioctl_acquire_vm, 0),
1835 
1836 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
1837 			kfd_ioctl_alloc_memory_of_gpu, 0),
1838 
1839 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
1840 			kfd_ioctl_free_memory_of_gpu, 0),
1841 
1842 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
1843 			kfd_ioctl_map_memory_to_gpu, 0),
1844 
1845 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
1846 			kfd_ioctl_unmap_memory_from_gpu, 0),
1847 
1848 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
1849 			kfd_ioctl_set_cu_mask, 0),
1850 
1851 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
1852 			kfd_ioctl_get_queue_wave_state, 0),
1853 
1854 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
1855 				kfd_ioctl_get_dmabuf_info, 0),
1856 
1857 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
1858 				kfd_ioctl_import_dmabuf, 0),
1859 
1860 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
1861 			kfd_ioctl_alloc_queue_gws, 0),
1862 
1863 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
1864 			kfd_ioctl_smi_events, 0),
1865 };
1866 
1867 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
1868 
1869 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
1870 {
1871 	struct kfd_process *process;
1872 	amdkfd_ioctl_t *func;
1873 	const struct amdkfd_ioctl_desc *ioctl = NULL;
1874 	unsigned int nr = _IOC_NR(cmd);
1875 	char stack_kdata[128];
1876 	char *kdata = NULL;
1877 	unsigned int usize, asize;
1878 	int retcode = -EINVAL;
1879 
1880 	if (nr >= AMDKFD_CORE_IOCTL_COUNT)
1881 		goto err_i1;
1882 
1883 	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
1884 		u32 amdkfd_size;
1885 
1886 		ioctl = &amdkfd_ioctls[nr];
1887 
1888 		amdkfd_size = _IOC_SIZE(ioctl->cmd);
1889 		usize = asize = _IOC_SIZE(cmd);
1890 		if (amdkfd_size > asize)
1891 			asize = amdkfd_size;
1892 
1893 		cmd = ioctl->cmd;
1894 	} else
1895 		goto err_i1;
1896 
1897 	dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
1898 
1899 	/* Get the process struct from the filep. Only the process
1900 	 * that opened /dev/kfd can use the file descriptor. Child
1901 	 * processes need to create their own KFD device context.
1902 	 */
1903 	process = filep->private_data;
1904 	if (process->lead_thread != current->group_leader) {
1905 		dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
1906 		retcode = -EBADF;
1907 		goto err_i1;
1908 	}
1909 
1910 	/* Do not trust userspace, use our own definition */
1911 	func = ioctl->func;
1912 
1913 	if (unlikely(!func)) {
1914 		dev_dbg(kfd_device, "no function\n");
1915 		retcode = -EINVAL;
1916 		goto err_i1;
1917 	}
1918 
1919 	if (cmd & (IOC_IN | IOC_OUT)) {
1920 		if (asize <= sizeof(stack_kdata)) {
1921 			kdata = stack_kdata;
1922 		} else {
1923 			kdata = kmalloc(asize, GFP_KERNEL);
1924 			if (!kdata) {
1925 				retcode = -ENOMEM;
1926 				goto err_i1;
1927 			}
1928 		}
1929 		if (asize > usize)
1930 			memset(kdata + usize, 0, asize - usize);
1931 	}
1932 
1933 	if (cmd & IOC_IN) {
1934 		if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
1935 			retcode = -EFAULT;
1936 			goto err_i1;
1937 		}
1938 	} else if (cmd & IOC_OUT) {
1939 		memset(kdata, 0, usize);
1940 	}
1941 
1942 	retcode = func(filep, process, kdata);
1943 
1944 	if (cmd & IOC_OUT)
1945 		if (copy_to_user((void __user *)arg, kdata, usize) != 0)
1946 			retcode = -EFAULT;
1947 
1948 err_i1:
1949 	if (!ioctl)
1950 		dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
1951 			  task_pid_nr(current), cmd, nr);
1952 
1953 	if (kdata != stack_kdata)
1954 		kfree(kdata);
1955 
1956 	if (retcode)
1957 		dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
1958 				nr, arg, retcode);
1959 
1960 	return retcode;
1961 }
1962 
1963 static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
1964 		      struct vm_area_struct *vma)
1965 {
1966 	phys_addr_t address;
1967 	int ret;
1968 
1969 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1970 		return -EINVAL;
1971 
1972 	address = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
1973 
1974 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
1975 				VM_DONTDUMP | VM_PFNMAP;
1976 
1977 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1978 
1979 	pr_debug("pasid 0x%x mapping mmio page\n"
1980 		 "     target user address == 0x%08llX\n"
1981 		 "     physical address    == 0x%08llX\n"
1982 		 "     vm_flags            == 0x%04lX\n"
1983 		 "     size                == 0x%04lX\n",
1984 		 process->pasid, (unsigned long long) vma->vm_start,
1985 		 address, vma->vm_flags, PAGE_SIZE);
1986 
1987 	ret = io_remap_pfn_range(vma,
1988 				vma->vm_start,
1989 				address >> PAGE_SHIFT,
1990 				PAGE_SIZE,
1991 				vma->vm_page_prot);
1992 	return ret;
1993 }
1994 
1995 
1996 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
1997 {
1998 	struct kfd_process *process;
1999 	struct kfd_dev *dev = NULL;
2000 	unsigned long mmap_offset;
2001 	unsigned int gpu_id;
2002 
2003 	process = kfd_get_process(current);
2004 	if (IS_ERR(process))
2005 		return PTR_ERR(process);
2006 
2007 	mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
2008 	gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
2009 	if (gpu_id)
2010 		dev = kfd_device_by_id(gpu_id);
2011 
2012 	switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
2013 	case KFD_MMAP_TYPE_DOORBELL:
2014 		if (!dev)
2015 			return -ENODEV;
2016 		return kfd_doorbell_mmap(dev, process, vma);
2017 
2018 	case KFD_MMAP_TYPE_EVENTS:
2019 		return kfd_event_mmap(process, vma);
2020 
2021 	case KFD_MMAP_TYPE_RESERVED_MEM:
2022 		if (!dev)
2023 			return -ENODEV;
2024 		return kfd_reserved_mem_mmap(dev, process, vma);
2025 	case KFD_MMAP_TYPE_MMIO:
2026 		if (!dev)
2027 			return -ENODEV;
2028 		return kfd_mmio_mmap(dev, process, vma);
2029 	}
2030 
2031 	return -EFAULT;
2032 }
2033