1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
26 #include <linux/fs.h>
27 #include <linux/file.h>
28 #include <linux/sched.h>
29 #include <linux/slab.h>
30 #include <linux/uaccess.h>
31 #include <linux/compat.h>
32 #include <uapi/linux/kfd_ioctl.h>
33 #include <linux/time.h>
34 #include <linux/mm.h>
35 #include <linux/mman.h>
36 #include <linux/dma-buf.h>
37 #include <asm/processor.h>
38 #include "kfd_priv.h"
39 #include "kfd_device_queue_manager.h"
40 #include "kfd_dbgmgr.h"
41 #include "kfd_svm.h"
42 #include "amdgpu_amdkfd.h"
43 #include "kfd_smi_events.h"
44 
45 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
46 static int kfd_open(struct inode *, struct file *);
47 static int kfd_release(struct inode *, struct file *);
48 static int kfd_mmap(struct file *, struct vm_area_struct *);
49 
50 static const char kfd_dev_name[] = "kfd";
51 
52 static const struct file_operations kfd_fops = {
53 	.owner = THIS_MODULE,
54 	.unlocked_ioctl = kfd_ioctl,
55 	.compat_ioctl = compat_ptr_ioctl,
56 	.open = kfd_open,
57 	.release = kfd_release,
58 	.mmap = kfd_mmap,
59 };
60 
61 static int kfd_char_dev_major = -1;
62 static struct class *kfd_class;
63 struct device *kfd_device;
64 
65 int kfd_chardev_init(void)
66 {
67 	int err = 0;
68 
69 	kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
70 	err = kfd_char_dev_major;
71 	if (err < 0)
72 		goto err_register_chrdev;
73 
74 	kfd_class = class_create(THIS_MODULE, kfd_dev_name);
75 	err = PTR_ERR(kfd_class);
76 	if (IS_ERR(kfd_class))
77 		goto err_class_create;
78 
79 	kfd_device = device_create(kfd_class, NULL,
80 					MKDEV(kfd_char_dev_major, 0),
81 					NULL, kfd_dev_name);
82 	err = PTR_ERR(kfd_device);
83 	if (IS_ERR(kfd_device))
84 		goto err_device_create;
85 
86 	return 0;
87 
88 err_device_create:
89 	class_destroy(kfd_class);
90 err_class_create:
91 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
92 err_register_chrdev:
93 	return err;
94 }
95 
96 void kfd_chardev_exit(void)
97 {
98 	device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
99 	class_destroy(kfd_class);
100 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
101 	kfd_device = NULL;
102 }
103 
104 struct device *kfd_chardev(void)
105 {
106 	return kfd_device;
107 }
108 
109 
110 static int kfd_open(struct inode *inode, struct file *filep)
111 {
112 	struct kfd_process *process;
113 	bool is_32bit_user_mode;
114 
115 	if (iminor(inode) != 0)
116 		return -ENODEV;
117 
118 	is_32bit_user_mode = in_compat_syscall();
119 
120 	if (is_32bit_user_mode) {
121 		dev_warn(kfd_device,
122 			"Process %d (32-bit) failed to open /dev/kfd\n"
123 			"32-bit processes are not supported by amdkfd\n",
124 			current->pid);
125 		return -EPERM;
126 	}
127 
128 	process = kfd_create_process(filep);
129 	if (IS_ERR(process))
130 		return PTR_ERR(process);
131 
132 	if (kfd_is_locked()) {
133 		dev_dbg(kfd_device, "kfd is locked!\n"
134 				"process %d unreferenced", process->pasid);
135 		kfd_unref_process(process);
136 		return -EAGAIN;
137 	}
138 
139 	/* filep now owns the reference returned by kfd_create_process */
140 	filep->private_data = process;
141 
142 	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
143 		process->pasid, process->is_32bit_user_mode);
144 
145 	return 0;
146 }
147 
148 static int kfd_release(struct inode *inode, struct file *filep)
149 {
150 	struct kfd_process *process = filep->private_data;
151 
152 	if (process)
153 		kfd_unref_process(process);
154 
155 	return 0;
156 }
157 
158 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
159 					void *data)
160 {
161 	struct kfd_ioctl_get_version_args *args = data;
162 
163 	args->major_version = KFD_IOCTL_MAJOR_VERSION;
164 	args->minor_version = KFD_IOCTL_MINOR_VERSION;
165 
166 	return 0;
167 }
168 
169 static int set_queue_properties_from_user(struct queue_properties *q_properties,
170 				struct kfd_ioctl_create_queue_args *args)
171 {
172 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
173 		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
174 		return -EINVAL;
175 	}
176 
177 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
178 		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
179 		return -EINVAL;
180 	}
181 
182 	if ((args->ring_base_address) &&
183 		(!access_ok((const void __user *) args->ring_base_address,
184 			sizeof(uint64_t)))) {
185 		pr_err("Can't access ring base address\n");
186 		return -EFAULT;
187 	}
188 
189 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
190 		pr_err("Ring size must be a power of 2 or 0\n");
191 		return -EINVAL;
192 	}
193 
194 	if (!access_ok((const void __user *) args->read_pointer_address,
195 			sizeof(uint32_t))) {
196 		pr_err("Can't access read pointer\n");
197 		return -EFAULT;
198 	}
199 
200 	if (!access_ok((const void __user *) args->write_pointer_address,
201 			sizeof(uint32_t))) {
202 		pr_err("Can't access write pointer\n");
203 		return -EFAULT;
204 	}
205 
206 	if (args->eop_buffer_address &&
207 		!access_ok((const void __user *) args->eop_buffer_address,
208 			sizeof(uint32_t))) {
209 		pr_debug("Can't access eop buffer");
210 		return -EFAULT;
211 	}
212 
213 	if (args->ctx_save_restore_address &&
214 		!access_ok((const void __user *) args->ctx_save_restore_address,
215 			sizeof(uint32_t))) {
216 		pr_debug("Can't access ctx save restore buffer");
217 		return -EFAULT;
218 	}
219 
220 	q_properties->is_interop = false;
221 	q_properties->is_gws = false;
222 	q_properties->queue_percent = args->queue_percentage;
223 	q_properties->priority = args->queue_priority;
224 	q_properties->queue_address = args->ring_base_address;
225 	q_properties->queue_size = args->ring_size;
226 	q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
227 	q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
228 	q_properties->eop_ring_buffer_address = args->eop_buffer_address;
229 	q_properties->eop_ring_buffer_size = args->eop_buffer_size;
230 	q_properties->ctx_save_restore_area_address =
231 			args->ctx_save_restore_address;
232 	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
233 	q_properties->ctl_stack_size = args->ctl_stack_size;
234 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
235 		args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
236 		q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
237 	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
238 		q_properties->type = KFD_QUEUE_TYPE_SDMA;
239 	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI)
240 		q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI;
241 	else
242 		return -ENOTSUPP;
243 
244 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
245 		q_properties->format = KFD_QUEUE_FORMAT_AQL;
246 	else
247 		q_properties->format = KFD_QUEUE_FORMAT_PM4;
248 
249 	pr_debug("Queue Percentage: %d, %d\n",
250 			q_properties->queue_percent, args->queue_percentage);
251 
252 	pr_debug("Queue Priority: %d, %d\n",
253 			q_properties->priority, args->queue_priority);
254 
255 	pr_debug("Queue Address: 0x%llX, 0x%llX\n",
256 			q_properties->queue_address, args->ring_base_address);
257 
258 	pr_debug("Queue Size: 0x%llX, %u\n",
259 			q_properties->queue_size, args->ring_size);
260 
261 	pr_debug("Queue r/w Pointers: %px, %px\n",
262 			q_properties->read_ptr,
263 			q_properties->write_ptr);
264 
265 	pr_debug("Queue Format: %d\n", q_properties->format);
266 
267 	pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address);
268 
269 	pr_debug("Queue CTX save area: 0x%llX\n",
270 			q_properties->ctx_save_restore_area_address);
271 
272 	return 0;
273 }
274 
275 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
276 					void *data)
277 {
278 	struct kfd_ioctl_create_queue_args *args = data;
279 	struct kfd_dev *dev;
280 	int err = 0;
281 	unsigned int queue_id;
282 	struct kfd_process_device *pdd;
283 	struct queue_properties q_properties;
284 	uint32_t doorbell_offset_in_process = 0;
285 
286 	memset(&q_properties, 0, sizeof(struct queue_properties));
287 
288 	pr_debug("Creating queue ioctl\n");
289 
290 	err = set_queue_properties_from_user(&q_properties, args);
291 	if (err)
292 		return err;
293 
294 	pr_debug("Looking for gpu id 0x%x\n", args->gpu_id);
295 	dev = kfd_device_by_id(args->gpu_id);
296 	if (!dev) {
297 		pr_debug("Could not find gpu id 0x%x\n", args->gpu_id);
298 		return -EINVAL;
299 	}
300 
301 	mutex_lock(&p->mutex);
302 
303 	pdd = kfd_bind_process_to_device(dev, p);
304 	if (IS_ERR(pdd)) {
305 		err = -ESRCH;
306 		goto err_bind_process;
307 	}
308 
309 	pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n",
310 			p->pasid,
311 			dev->id);
312 
313 	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id,
314 			&doorbell_offset_in_process);
315 	if (err != 0)
316 		goto err_create_queue;
317 
318 	args->queue_id = queue_id;
319 
320 
321 	/* Return gpu_id as doorbell offset for mmap usage */
322 	args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL;
323 	args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id);
324 	if (KFD_IS_SOC15(dev->device_info->asic_family))
325 		/* On SOC15 ASICs, include the doorbell offset within the
326 		 * process doorbell frame, which is 2 pages.
327 		 */
328 		args->doorbell_offset |= doorbell_offset_in_process;
329 
330 	mutex_unlock(&p->mutex);
331 
332 	pr_debug("Queue id %d was created successfully\n", args->queue_id);
333 
334 	pr_debug("Ring buffer address == 0x%016llX\n",
335 			args->ring_base_address);
336 
337 	pr_debug("Read ptr address    == 0x%016llX\n",
338 			args->read_pointer_address);
339 
340 	pr_debug("Write ptr address   == 0x%016llX\n",
341 			args->write_pointer_address);
342 
343 	return 0;
344 
345 err_create_queue:
346 err_bind_process:
347 	mutex_unlock(&p->mutex);
348 	return err;
349 }
350 
351 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
352 					void *data)
353 {
354 	int retval;
355 	struct kfd_ioctl_destroy_queue_args *args = data;
356 
357 	pr_debug("Destroying queue id %d for pasid 0x%x\n",
358 				args->queue_id,
359 				p->pasid);
360 
361 	mutex_lock(&p->mutex);
362 
363 	retval = pqm_destroy_queue(&p->pqm, args->queue_id);
364 
365 	mutex_unlock(&p->mutex);
366 	return retval;
367 }
368 
369 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
370 					void *data)
371 {
372 	int retval;
373 	struct kfd_ioctl_update_queue_args *args = data;
374 	struct queue_properties properties;
375 
376 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
377 		pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
378 		return -EINVAL;
379 	}
380 
381 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
382 		pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
383 		return -EINVAL;
384 	}
385 
386 	if ((args->ring_base_address) &&
387 		(!access_ok((const void __user *) args->ring_base_address,
388 			sizeof(uint64_t)))) {
389 		pr_err("Can't access ring base address\n");
390 		return -EFAULT;
391 	}
392 
393 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
394 		pr_err("Ring size must be a power of 2 or 0\n");
395 		return -EINVAL;
396 	}
397 
398 	properties.queue_address = args->ring_base_address;
399 	properties.queue_size = args->ring_size;
400 	properties.queue_percent = args->queue_percentage;
401 	properties.priority = args->queue_priority;
402 
403 	pr_debug("Updating queue id %d for pasid 0x%x\n",
404 			args->queue_id, p->pasid);
405 
406 	mutex_lock(&p->mutex);
407 
408 	retval = pqm_update_queue_properties(&p->pqm, args->queue_id, &properties);
409 
410 	mutex_unlock(&p->mutex);
411 
412 	return retval;
413 }
414 
415 static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
416 					void *data)
417 {
418 	int retval;
419 	const int max_num_cus = 1024;
420 	struct kfd_ioctl_set_cu_mask_args *args = data;
421 	struct mqd_update_info minfo = {0};
422 	uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
423 	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
424 
425 	if ((args->num_cu_mask % 32) != 0) {
426 		pr_debug("num_cu_mask 0x%x must be a multiple of 32",
427 				args->num_cu_mask);
428 		return -EINVAL;
429 	}
430 
431 	minfo.cu_mask.count = args->num_cu_mask;
432 	if (minfo.cu_mask.count == 0) {
433 		pr_debug("CU mask cannot be 0");
434 		return -EINVAL;
435 	}
436 
437 	/* To prevent an unreasonably large CU mask size, set an arbitrary
438 	 * limit of max_num_cus bits.  We can then just drop any CU mask bits
439 	 * past max_num_cus bits and just use the first max_num_cus bits.
440 	 */
441 	if (minfo.cu_mask.count > max_num_cus) {
442 		pr_debug("CU mask cannot be greater than 1024 bits");
443 		minfo.cu_mask.count = max_num_cus;
444 		cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
445 	}
446 
447 	minfo.cu_mask.ptr = kzalloc(cu_mask_size, GFP_KERNEL);
448 	if (!minfo.cu_mask.ptr)
449 		return -ENOMEM;
450 
451 	retval = copy_from_user(minfo.cu_mask.ptr, cu_mask_ptr, cu_mask_size);
452 	if (retval) {
453 		pr_debug("Could not copy CU mask from userspace");
454 		retval = -EFAULT;
455 		goto out;
456 	}
457 
458 	minfo.update_flag = UPDATE_FLAG_CU_MASK;
459 
460 	mutex_lock(&p->mutex);
461 
462 	retval = pqm_update_mqd(&p->pqm, args->queue_id, &minfo);
463 
464 	mutex_unlock(&p->mutex);
465 
466 out:
467 	kfree(minfo.cu_mask.ptr);
468 	return retval;
469 }
470 
471 static int kfd_ioctl_get_queue_wave_state(struct file *filep,
472 					  struct kfd_process *p, void *data)
473 {
474 	struct kfd_ioctl_get_queue_wave_state_args *args = data;
475 	int r;
476 
477 	mutex_lock(&p->mutex);
478 
479 	r = pqm_get_wave_state(&p->pqm, args->queue_id,
480 			       (void __user *)args->ctl_stack_address,
481 			       &args->ctl_stack_used_size,
482 			       &args->save_area_used_size);
483 
484 	mutex_unlock(&p->mutex);
485 
486 	return r;
487 }
488 
489 static int kfd_ioctl_set_memory_policy(struct file *filep,
490 					struct kfd_process *p, void *data)
491 {
492 	struct kfd_ioctl_set_memory_policy_args *args = data;
493 	struct kfd_dev *dev;
494 	int err = 0;
495 	struct kfd_process_device *pdd;
496 	enum cache_policy default_policy, alternate_policy;
497 
498 	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
499 	    && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
500 		return -EINVAL;
501 	}
502 
503 	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
504 	    && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
505 		return -EINVAL;
506 	}
507 
508 	dev = kfd_device_by_id(args->gpu_id);
509 	if (!dev)
510 		return -EINVAL;
511 
512 	mutex_lock(&p->mutex);
513 
514 	pdd = kfd_bind_process_to_device(dev, p);
515 	if (IS_ERR(pdd)) {
516 		err = -ESRCH;
517 		goto out;
518 	}
519 
520 	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
521 			 ? cache_policy_coherent : cache_policy_noncoherent;
522 
523 	alternate_policy =
524 		(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
525 		   ? cache_policy_coherent : cache_policy_noncoherent;
526 
527 	if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm,
528 				&pdd->qpd,
529 				default_policy,
530 				alternate_policy,
531 				(void __user *)args->alternate_aperture_base,
532 				args->alternate_aperture_size))
533 		err = -EINVAL;
534 
535 out:
536 	mutex_unlock(&p->mutex);
537 
538 	return err;
539 }
540 
541 static int kfd_ioctl_set_trap_handler(struct file *filep,
542 					struct kfd_process *p, void *data)
543 {
544 	struct kfd_ioctl_set_trap_handler_args *args = data;
545 	struct kfd_dev *dev;
546 	int err = 0;
547 	struct kfd_process_device *pdd;
548 
549 	dev = kfd_device_by_id(args->gpu_id);
550 	if (!dev)
551 		return -EINVAL;
552 
553 	mutex_lock(&p->mutex);
554 
555 	pdd = kfd_bind_process_to_device(dev, p);
556 	if (IS_ERR(pdd)) {
557 		err = -ESRCH;
558 		goto out;
559 	}
560 
561 	kfd_process_set_trap_handler(&pdd->qpd, args->tba_addr, args->tma_addr);
562 
563 out:
564 	mutex_unlock(&p->mutex);
565 
566 	return err;
567 }
568 
569 static int kfd_ioctl_dbg_register(struct file *filep,
570 				struct kfd_process *p, void *data)
571 {
572 	struct kfd_ioctl_dbg_register_args *args = data;
573 	struct kfd_dev *dev;
574 	struct kfd_dbgmgr *dbgmgr_ptr;
575 	struct kfd_process_device *pdd;
576 	bool create_ok;
577 	long status = 0;
578 
579 	dev = kfd_device_by_id(args->gpu_id);
580 	if (!dev)
581 		return -EINVAL;
582 
583 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
584 		pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
585 		return -EINVAL;
586 	}
587 
588 	mutex_lock(&p->mutex);
589 	mutex_lock(kfd_get_dbgmgr_mutex());
590 
591 	/*
592 	 * make sure that we have pdd, if this the first queue created for
593 	 * this process
594 	 */
595 	pdd = kfd_bind_process_to_device(dev, p);
596 	if (IS_ERR(pdd)) {
597 		status = PTR_ERR(pdd);
598 		goto out;
599 	}
600 
601 	if (!dev->dbgmgr) {
602 		/* In case of a legal call, we have no dbgmgr yet */
603 		create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev);
604 		if (create_ok) {
605 			status = kfd_dbgmgr_register(dbgmgr_ptr, p);
606 			if (status != 0)
607 				kfd_dbgmgr_destroy(dbgmgr_ptr);
608 			else
609 				dev->dbgmgr = dbgmgr_ptr;
610 		}
611 	} else {
612 		pr_debug("debugger already registered\n");
613 		status = -EINVAL;
614 	}
615 
616 out:
617 	mutex_unlock(kfd_get_dbgmgr_mutex());
618 	mutex_unlock(&p->mutex);
619 
620 	return status;
621 }
622 
623 static int kfd_ioctl_dbg_unregister(struct file *filep,
624 				struct kfd_process *p, void *data)
625 {
626 	struct kfd_ioctl_dbg_unregister_args *args = data;
627 	struct kfd_dev *dev;
628 	long status;
629 
630 	dev = kfd_device_by_id(args->gpu_id);
631 	if (!dev || !dev->dbgmgr)
632 		return -EINVAL;
633 
634 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
635 		pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n");
636 		return -EINVAL;
637 	}
638 
639 	mutex_lock(kfd_get_dbgmgr_mutex());
640 
641 	status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
642 	if (!status) {
643 		kfd_dbgmgr_destroy(dev->dbgmgr);
644 		dev->dbgmgr = NULL;
645 	}
646 
647 	mutex_unlock(kfd_get_dbgmgr_mutex());
648 
649 	return status;
650 }
651 
652 /*
653  * Parse and generate variable size data structure for address watch.
654  * Total size of the buffer and # watch points is limited in order
655  * to prevent kernel abuse. (no bearing to the much smaller HW limitation
656  * which is enforced by dbgdev module)
657  * please also note that the watch address itself are not "copied from user",
658  * since it be set into the HW in user mode values.
659  *
660  */
661 static int kfd_ioctl_dbg_address_watch(struct file *filep,
662 					struct kfd_process *p, void *data)
663 {
664 	struct kfd_ioctl_dbg_address_watch_args *args = data;
665 	struct kfd_dev *dev;
666 	struct dbg_address_watch_info aw_info;
667 	unsigned char *args_buff;
668 	long status;
669 	void __user *cmd_from_user;
670 	uint64_t watch_mask_value = 0;
671 	unsigned int args_idx = 0;
672 
673 	memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info));
674 
675 	dev = kfd_device_by_id(args->gpu_id);
676 	if (!dev)
677 		return -EINVAL;
678 
679 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
680 		pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
681 		return -EINVAL;
682 	}
683 
684 	cmd_from_user = (void __user *) args->content_ptr;
685 
686 	/* Validate arguments */
687 
688 	if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) ||
689 		(args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) ||
690 		(cmd_from_user == NULL))
691 		return -EINVAL;
692 
693 	/* this is the actual buffer to work with */
694 	args_buff = memdup_user(cmd_from_user,
695 				args->buf_size_in_bytes - sizeof(*args));
696 	if (IS_ERR(args_buff))
697 		return PTR_ERR(args_buff);
698 
699 	aw_info.process = p;
700 
701 	aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx]));
702 	args_idx += sizeof(aw_info.num_watch_points);
703 
704 	aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx];
705 	args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points;
706 
707 	/*
708 	 * set watch address base pointer to point on the array base
709 	 * within args_buff
710 	 */
711 	aw_info.watch_address = (uint64_t *) &args_buff[args_idx];
712 
713 	/* skip over the addresses buffer */
714 	args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
715 
716 	if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) {
717 		status = -EINVAL;
718 		goto out;
719 	}
720 
721 	watch_mask_value = (uint64_t) args_buff[args_idx];
722 
723 	if (watch_mask_value > 0) {
724 		/*
725 		 * There is an array of masks.
726 		 * set watch mask base pointer to point on the array base
727 		 * within args_buff
728 		 */
729 		aw_info.watch_mask = (uint64_t *) &args_buff[args_idx];
730 
731 		/* skip over the masks buffer */
732 		args_idx += sizeof(aw_info.watch_mask) *
733 				aw_info.num_watch_points;
734 	} else {
735 		/* just the NULL mask, set to NULL and skip over it */
736 		aw_info.watch_mask = NULL;
737 		args_idx += sizeof(aw_info.watch_mask);
738 	}
739 
740 	if (args_idx >= args->buf_size_in_bytes - sizeof(args)) {
741 		status = -EINVAL;
742 		goto out;
743 	}
744 
745 	/* Currently HSA Event is not supported for DBG */
746 	aw_info.watch_event = NULL;
747 
748 	mutex_lock(kfd_get_dbgmgr_mutex());
749 
750 	status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info);
751 
752 	mutex_unlock(kfd_get_dbgmgr_mutex());
753 
754 out:
755 	kfree(args_buff);
756 
757 	return status;
758 }
759 
760 /* Parse and generate fixed size data structure for wave control */
761 static int kfd_ioctl_dbg_wave_control(struct file *filep,
762 					struct kfd_process *p, void *data)
763 {
764 	struct kfd_ioctl_dbg_wave_control_args *args = data;
765 	struct kfd_dev *dev;
766 	struct dbg_wave_control_info wac_info;
767 	unsigned char *args_buff;
768 	uint32_t computed_buff_size;
769 	long status;
770 	void __user *cmd_from_user;
771 	unsigned int args_idx = 0;
772 
773 	memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info));
774 
775 	/* we use compact form, independent of the packing attribute value */
776 	computed_buff_size = sizeof(*args) +
777 				sizeof(wac_info.mode) +
778 				sizeof(wac_info.operand) +
779 				sizeof(wac_info.dbgWave_msg.DbgWaveMsg) +
780 				sizeof(wac_info.dbgWave_msg.MemoryVA) +
781 				sizeof(wac_info.trapId);
782 
783 	dev = kfd_device_by_id(args->gpu_id);
784 	if (!dev)
785 		return -EINVAL;
786 
787 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
788 		pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
789 		return -EINVAL;
790 	}
791 
792 	/* input size must match the computed "compact" size */
793 	if (args->buf_size_in_bytes != computed_buff_size) {
794 		pr_debug("size mismatch, computed : actual %u : %u\n",
795 				args->buf_size_in_bytes, computed_buff_size);
796 		return -EINVAL;
797 	}
798 
799 	cmd_from_user = (void __user *) args->content_ptr;
800 
801 	if (cmd_from_user == NULL)
802 		return -EINVAL;
803 
804 	/* copy the entire buffer from user */
805 
806 	args_buff = memdup_user(cmd_from_user,
807 				args->buf_size_in_bytes - sizeof(*args));
808 	if (IS_ERR(args_buff))
809 		return PTR_ERR(args_buff);
810 
811 	/* move ptr to the start of the "pay-load" area */
812 	wac_info.process = p;
813 
814 	wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx]));
815 	args_idx += sizeof(wac_info.operand);
816 
817 	wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx]));
818 	args_idx += sizeof(wac_info.mode);
819 
820 	wac_info.trapId = *((uint32_t *)(&args_buff[args_idx]));
821 	args_idx += sizeof(wac_info.trapId);
822 
823 	wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value =
824 					*((uint32_t *)(&args_buff[args_idx]));
825 	wac_info.dbgWave_msg.MemoryVA = NULL;
826 
827 	mutex_lock(kfd_get_dbgmgr_mutex());
828 
829 	pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
830 			wac_info.process, wac_info.operand,
831 			wac_info.mode, wac_info.trapId,
832 			wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
833 
834 	status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info);
835 
836 	pr_debug("Returned status of dbg manager is %ld\n", status);
837 
838 	mutex_unlock(kfd_get_dbgmgr_mutex());
839 
840 	kfree(args_buff);
841 
842 	return status;
843 }
844 
845 static int kfd_ioctl_get_clock_counters(struct file *filep,
846 				struct kfd_process *p, void *data)
847 {
848 	struct kfd_ioctl_get_clock_counters_args *args = data;
849 	struct kfd_dev *dev;
850 
851 	dev = kfd_device_by_id(args->gpu_id);
852 	if (dev)
853 		/* Reading GPU clock counter from KGD */
854 		args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(dev->kgd);
855 	else
856 		/* Node without GPU resource */
857 		args->gpu_clock_counter = 0;
858 
859 	/* No access to rdtsc. Using raw monotonic time */
860 	args->cpu_clock_counter = ktime_get_raw_ns();
861 	args->system_clock_counter = ktime_get_boottime_ns();
862 
863 	/* Since the counter is in nano-seconds we use 1GHz frequency */
864 	args->system_clock_freq = 1000000000;
865 
866 	return 0;
867 }
868 
869 
870 static int kfd_ioctl_get_process_apertures(struct file *filp,
871 				struct kfd_process *p, void *data)
872 {
873 	struct kfd_ioctl_get_process_apertures_args *args = data;
874 	struct kfd_process_device_apertures *pAperture;
875 	int i;
876 
877 	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
878 
879 	args->num_of_nodes = 0;
880 
881 	mutex_lock(&p->mutex);
882 	/* Run over all pdd of the process */
883 	for (i = 0; i < p->n_pdds; i++) {
884 		struct kfd_process_device *pdd = p->pdds[i];
885 
886 		pAperture =
887 			&args->process_apertures[args->num_of_nodes];
888 		pAperture->gpu_id = pdd->dev->id;
889 		pAperture->lds_base = pdd->lds_base;
890 		pAperture->lds_limit = pdd->lds_limit;
891 		pAperture->gpuvm_base = pdd->gpuvm_base;
892 		pAperture->gpuvm_limit = pdd->gpuvm_limit;
893 		pAperture->scratch_base = pdd->scratch_base;
894 		pAperture->scratch_limit = pdd->scratch_limit;
895 
896 		dev_dbg(kfd_device,
897 			"node id %u\n", args->num_of_nodes);
898 		dev_dbg(kfd_device,
899 			"gpu id %u\n", pdd->dev->id);
900 		dev_dbg(kfd_device,
901 			"lds_base %llX\n", pdd->lds_base);
902 		dev_dbg(kfd_device,
903 			"lds_limit %llX\n", pdd->lds_limit);
904 		dev_dbg(kfd_device,
905 			"gpuvm_base %llX\n", pdd->gpuvm_base);
906 		dev_dbg(kfd_device,
907 			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
908 		dev_dbg(kfd_device,
909 			"scratch_base %llX\n", pdd->scratch_base);
910 		dev_dbg(kfd_device,
911 			"scratch_limit %llX\n", pdd->scratch_limit);
912 
913 		if (++args->num_of_nodes >= NUM_OF_SUPPORTED_GPUS)
914 			break;
915 	}
916 	mutex_unlock(&p->mutex);
917 
918 	return 0;
919 }
920 
921 static int kfd_ioctl_get_process_apertures_new(struct file *filp,
922 				struct kfd_process *p, void *data)
923 {
924 	struct kfd_ioctl_get_process_apertures_new_args *args = data;
925 	struct kfd_process_device_apertures *pa;
926 	int ret;
927 	int i;
928 
929 	dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid);
930 
931 	if (args->num_of_nodes == 0) {
932 		/* Return number of nodes, so that user space can alloacate
933 		 * sufficient memory
934 		 */
935 		mutex_lock(&p->mutex);
936 		args->num_of_nodes = p->n_pdds;
937 		goto out_unlock;
938 	}
939 
940 	/* Fill in process-aperture information for all available
941 	 * nodes, but not more than args->num_of_nodes as that is
942 	 * the amount of memory allocated by user
943 	 */
944 	pa = kzalloc((sizeof(struct kfd_process_device_apertures) *
945 				args->num_of_nodes), GFP_KERNEL);
946 	if (!pa)
947 		return -ENOMEM;
948 
949 	mutex_lock(&p->mutex);
950 
951 	if (!p->n_pdds) {
952 		args->num_of_nodes = 0;
953 		kfree(pa);
954 		goto out_unlock;
955 	}
956 
957 	/* Run over all pdd of the process */
958 	for (i = 0; i < min(p->n_pdds, args->num_of_nodes); i++) {
959 		struct kfd_process_device *pdd = p->pdds[i];
960 
961 		pa[i].gpu_id = pdd->dev->id;
962 		pa[i].lds_base = pdd->lds_base;
963 		pa[i].lds_limit = pdd->lds_limit;
964 		pa[i].gpuvm_base = pdd->gpuvm_base;
965 		pa[i].gpuvm_limit = pdd->gpuvm_limit;
966 		pa[i].scratch_base = pdd->scratch_base;
967 		pa[i].scratch_limit = pdd->scratch_limit;
968 
969 		dev_dbg(kfd_device,
970 			"gpu id %u\n", pdd->dev->id);
971 		dev_dbg(kfd_device,
972 			"lds_base %llX\n", pdd->lds_base);
973 		dev_dbg(kfd_device,
974 			"lds_limit %llX\n", pdd->lds_limit);
975 		dev_dbg(kfd_device,
976 			"gpuvm_base %llX\n", pdd->gpuvm_base);
977 		dev_dbg(kfd_device,
978 			"gpuvm_limit %llX\n", pdd->gpuvm_limit);
979 		dev_dbg(kfd_device,
980 			"scratch_base %llX\n", pdd->scratch_base);
981 		dev_dbg(kfd_device,
982 			"scratch_limit %llX\n", pdd->scratch_limit);
983 	}
984 	mutex_unlock(&p->mutex);
985 
986 	args->num_of_nodes = i;
987 	ret = copy_to_user(
988 			(void __user *)args->kfd_process_device_apertures_ptr,
989 			pa,
990 			(i * sizeof(struct kfd_process_device_apertures)));
991 	kfree(pa);
992 	return ret ? -EFAULT : 0;
993 
994 out_unlock:
995 	mutex_unlock(&p->mutex);
996 	return 0;
997 }
998 
999 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
1000 					void *data)
1001 {
1002 	struct kfd_ioctl_create_event_args *args = data;
1003 	int err;
1004 
1005 	/* For dGPUs the event page is allocated in user mode. The
1006 	 * handle is passed to KFD with the first call to this IOCTL
1007 	 * through the event_page_offset field.
1008 	 */
1009 	if (args->event_page_offset) {
1010 		struct kfd_dev *kfd;
1011 		struct kfd_process_device *pdd;
1012 		void *mem, *kern_addr;
1013 		uint64_t size;
1014 
1015 		kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset));
1016 		if (!kfd) {
1017 			pr_err("Getting device by id failed in %s\n", __func__);
1018 			return -EINVAL;
1019 		}
1020 
1021 		mutex_lock(&p->mutex);
1022 
1023 		if (p->signal_page) {
1024 			pr_err("Event page is already set\n");
1025 			err = -EINVAL;
1026 			goto out_unlock;
1027 		}
1028 
1029 		pdd = kfd_bind_process_to_device(kfd, p);
1030 		if (IS_ERR(pdd)) {
1031 			err = PTR_ERR(pdd);
1032 			goto out_unlock;
1033 		}
1034 
1035 		mem = kfd_process_device_translate_handle(pdd,
1036 				GET_IDR_HANDLE(args->event_page_offset));
1037 		if (!mem) {
1038 			pr_err("Can't find BO, offset is 0x%llx\n",
1039 			       args->event_page_offset);
1040 			err = -EINVAL;
1041 			goto out_unlock;
1042 		}
1043 
1044 		err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd,
1045 						mem, &kern_addr, &size);
1046 		if (err) {
1047 			pr_err("Failed to map event page to kernel\n");
1048 			goto out_unlock;
1049 		}
1050 
1051 		err = kfd_event_page_set(p, kern_addr, size);
1052 		if (err) {
1053 			pr_err("Failed to set event page\n");
1054 			amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kfd->kgd, mem);
1055 			goto out_unlock;
1056 		}
1057 
1058 		p->signal_handle = args->event_page_offset;
1059 
1060 		mutex_unlock(&p->mutex);
1061 	}
1062 
1063 	err = kfd_event_create(filp, p, args->event_type,
1064 				args->auto_reset != 0, args->node_id,
1065 				&args->event_id, &args->event_trigger_data,
1066 				&args->event_page_offset,
1067 				&args->event_slot_index);
1068 
1069 	return err;
1070 
1071 out_unlock:
1072 	mutex_unlock(&p->mutex);
1073 	return err;
1074 }
1075 
1076 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
1077 					void *data)
1078 {
1079 	struct kfd_ioctl_destroy_event_args *args = data;
1080 
1081 	return kfd_event_destroy(p, args->event_id);
1082 }
1083 
1084 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
1085 				void *data)
1086 {
1087 	struct kfd_ioctl_set_event_args *args = data;
1088 
1089 	return kfd_set_event(p, args->event_id);
1090 }
1091 
1092 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
1093 				void *data)
1094 {
1095 	struct kfd_ioctl_reset_event_args *args = data;
1096 
1097 	return kfd_reset_event(p, args->event_id);
1098 }
1099 
1100 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
1101 				void *data)
1102 {
1103 	struct kfd_ioctl_wait_events_args *args = data;
1104 	int err;
1105 
1106 	err = kfd_wait_on_events(p, args->num_events,
1107 			(void __user *)args->events_ptr,
1108 			(args->wait_for_all != 0),
1109 			args->timeout, &args->wait_result);
1110 
1111 	return err;
1112 }
1113 static int kfd_ioctl_set_scratch_backing_va(struct file *filep,
1114 					struct kfd_process *p, void *data)
1115 {
1116 	struct kfd_ioctl_set_scratch_backing_va_args *args = data;
1117 	struct kfd_process_device *pdd;
1118 	struct kfd_dev *dev;
1119 	long err;
1120 
1121 	dev = kfd_device_by_id(args->gpu_id);
1122 	if (!dev)
1123 		return -EINVAL;
1124 
1125 	mutex_lock(&p->mutex);
1126 
1127 	pdd = kfd_bind_process_to_device(dev, p);
1128 	if (IS_ERR(pdd)) {
1129 		err = PTR_ERR(pdd);
1130 		goto bind_process_to_device_fail;
1131 	}
1132 
1133 	pdd->qpd.sh_hidden_private_base = args->va_addr;
1134 
1135 	mutex_unlock(&p->mutex);
1136 
1137 	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS &&
1138 	    pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va)
1139 		dev->kfd2kgd->set_scratch_backing_va(
1140 			dev->kgd, args->va_addr, pdd->qpd.vmid);
1141 
1142 	return 0;
1143 
1144 bind_process_to_device_fail:
1145 	mutex_unlock(&p->mutex);
1146 	return err;
1147 }
1148 
1149 static int kfd_ioctl_get_tile_config(struct file *filep,
1150 		struct kfd_process *p, void *data)
1151 {
1152 	struct kfd_ioctl_get_tile_config_args *args = data;
1153 	struct kfd_dev *dev;
1154 	struct tile_config config;
1155 	int err = 0;
1156 
1157 	dev = kfd_device_by_id(args->gpu_id);
1158 	if (!dev)
1159 		return -EINVAL;
1160 
1161 	amdgpu_amdkfd_get_tile_config(dev->kgd, &config);
1162 
1163 	args->gb_addr_config = config.gb_addr_config;
1164 	args->num_banks = config.num_banks;
1165 	args->num_ranks = config.num_ranks;
1166 
1167 	if (args->num_tile_configs > config.num_tile_configs)
1168 		args->num_tile_configs = config.num_tile_configs;
1169 	err = copy_to_user((void __user *)args->tile_config_ptr,
1170 			config.tile_config_ptr,
1171 			args->num_tile_configs * sizeof(uint32_t));
1172 	if (err) {
1173 		args->num_tile_configs = 0;
1174 		return -EFAULT;
1175 	}
1176 
1177 	if (args->num_macro_tile_configs > config.num_macro_tile_configs)
1178 		args->num_macro_tile_configs =
1179 				config.num_macro_tile_configs;
1180 	err = copy_to_user((void __user *)args->macro_tile_config_ptr,
1181 			config.macro_tile_config_ptr,
1182 			args->num_macro_tile_configs * sizeof(uint32_t));
1183 	if (err) {
1184 		args->num_macro_tile_configs = 0;
1185 		return -EFAULT;
1186 	}
1187 
1188 	return 0;
1189 }
1190 
1191 static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p,
1192 				void *data)
1193 {
1194 	struct kfd_ioctl_acquire_vm_args *args = data;
1195 	struct kfd_process_device *pdd;
1196 	struct kfd_dev *dev;
1197 	struct file *drm_file;
1198 	int ret;
1199 
1200 	dev = kfd_device_by_id(args->gpu_id);
1201 	if (!dev)
1202 		return -EINVAL;
1203 
1204 	drm_file = fget(args->drm_fd);
1205 	if (!drm_file)
1206 		return -EINVAL;
1207 
1208 	mutex_lock(&p->mutex);
1209 
1210 	pdd = kfd_get_process_device_data(dev, p);
1211 	if (!pdd) {
1212 		ret = -EINVAL;
1213 		goto err_unlock;
1214 	}
1215 
1216 	if (pdd->drm_file) {
1217 		ret = pdd->drm_file == drm_file ? 0 : -EBUSY;
1218 		goto err_unlock;
1219 	}
1220 
1221 	ret = kfd_process_device_init_vm(pdd, drm_file);
1222 	if (ret)
1223 		goto err_unlock;
1224 	/* On success, the PDD keeps the drm_file reference */
1225 	mutex_unlock(&p->mutex);
1226 
1227 	return 0;
1228 
1229 err_unlock:
1230 	mutex_unlock(&p->mutex);
1231 	fput(drm_file);
1232 	return ret;
1233 }
1234 
1235 bool kfd_dev_is_large_bar(struct kfd_dev *dev)
1236 {
1237 	struct kfd_local_mem_info mem_info;
1238 
1239 	if (debug_largebar) {
1240 		pr_debug("Simulate large-bar allocation on non large-bar machine\n");
1241 		return true;
1242 	}
1243 
1244 	if (dev->use_iommu_v2)
1245 		return false;
1246 
1247 	amdgpu_amdkfd_get_local_mem_info(dev->kgd, &mem_info);
1248 	if (mem_info.local_mem_size_private == 0 &&
1249 			mem_info.local_mem_size_public > 0)
1250 		return true;
1251 	return false;
1252 }
1253 
1254 static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep,
1255 					struct kfd_process *p, void *data)
1256 {
1257 	struct kfd_ioctl_alloc_memory_of_gpu_args *args = data;
1258 	struct kfd_process_device *pdd;
1259 	void *mem;
1260 	struct kfd_dev *dev;
1261 	int idr_handle;
1262 	long err;
1263 	uint64_t offset = args->mmap_offset;
1264 	uint32_t flags = args->flags;
1265 
1266 	if (args->size == 0)
1267 		return -EINVAL;
1268 
1269 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1270 	/* Flush pending deferred work to avoid racing with deferred actions
1271 	 * from previous memory map changes (e.g. munmap).
1272 	 */
1273 	svm_range_list_lock_and_flush_work(&p->svms, current->mm);
1274 	mutex_lock(&p->svms.lock);
1275 	mmap_write_unlock(current->mm);
1276 	if (interval_tree_iter_first(&p->svms.objects,
1277 				     args->va_addr >> PAGE_SHIFT,
1278 				     (args->va_addr + args->size - 1) >> PAGE_SHIFT)) {
1279 		pr_err("Address: 0x%llx already allocated by SVM\n",
1280 			args->va_addr);
1281 		mutex_unlock(&p->svms.lock);
1282 		return -EADDRINUSE;
1283 	}
1284 	mutex_unlock(&p->svms.lock);
1285 #endif
1286 	dev = kfd_device_by_id(args->gpu_id);
1287 	if (!dev)
1288 		return -EINVAL;
1289 
1290 	if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) &&
1291 		(flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) &&
1292 		!kfd_dev_is_large_bar(dev)) {
1293 		pr_err("Alloc host visible vram on small bar is not allowed\n");
1294 		return -EINVAL;
1295 	}
1296 
1297 	mutex_lock(&p->mutex);
1298 
1299 	pdd = kfd_bind_process_to_device(dev, p);
1300 	if (IS_ERR(pdd)) {
1301 		err = PTR_ERR(pdd);
1302 		goto err_unlock;
1303 	}
1304 
1305 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
1306 		if (args->size != kfd_doorbell_process_slice(dev)) {
1307 			err = -EINVAL;
1308 			goto err_unlock;
1309 		}
1310 		offset = kfd_get_process_doorbells(pdd);
1311 	} else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) {
1312 		if (args->size != PAGE_SIZE) {
1313 			err = -EINVAL;
1314 			goto err_unlock;
1315 		}
1316 		offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
1317 		if (!offset) {
1318 			err = -ENOMEM;
1319 			goto err_unlock;
1320 		}
1321 	}
1322 
1323 	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1324 		dev->kgd, args->va_addr, args->size,
1325 		pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
1326 		flags);
1327 
1328 	if (err)
1329 		goto err_unlock;
1330 
1331 	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1332 	if (idr_handle < 0) {
1333 		err = -EFAULT;
1334 		goto err_free;
1335 	}
1336 
1337 	/* Update the VRAM usage count */
1338 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)
1339 		WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + args->size);
1340 
1341 	mutex_unlock(&p->mutex);
1342 
1343 	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1344 	args->mmap_offset = offset;
1345 
1346 	/* MMIO is mapped through kfd device
1347 	 * Generate a kfd mmap offset
1348 	 */
1349 	if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP)
1350 		args->mmap_offset = KFD_MMAP_TYPE_MMIO
1351 					| KFD_MMAP_GPU_ID(args->gpu_id);
1352 
1353 	return 0;
1354 
1355 err_free:
1356 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
1357 					       pdd->drm_priv, NULL);
1358 err_unlock:
1359 	mutex_unlock(&p->mutex);
1360 	return err;
1361 }
1362 
1363 static int kfd_ioctl_free_memory_of_gpu(struct file *filep,
1364 					struct kfd_process *p, void *data)
1365 {
1366 	struct kfd_ioctl_free_memory_of_gpu_args *args = data;
1367 	struct kfd_process_device *pdd;
1368 	void *mem;
1369 	struct kfd_dev *dev;
1370 	int ret;
1371 	uint64_t size = 0;
1372 
1373 	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1374 	if (!dev)
1375 		return -EINVAL;
1376 
1377 	mutex_lock(&p->mutex);
1378 	/*
1379 	 * Safeguard to prevent user space from freeing signal BO.
1380 	 * It will be freed at process termination.
1381 	 */
1382 	if (p->signal_handle && (p->signal_handle == args->handle)) {
1383 		pr_err("Free signal BO is not allowed\n");
1384 		ret = -EPERM;
1385 		goto err_unlock;
1386 	}
1387 
1388 	pdd = kfd_get_process_device_data(dev, p);
1389 	if (!pdd) {
1390 		pr_err("Process device data doesn't exist\n");
1391 		ret = -EINVAL;
1392 		goto err_unlock;
1393 	}
1394 
1395 	mem = kfd_process_device_translate_handle(
1396 		pdd, GET_IDR_HANDLE(args->handle));
1397 	if (!mem) {
1398 		ret = -EINVAL;
1399 		goto err_unlock;
1400 	}
1401 
1402 	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd,
1403 				(struct kgd_mem *)mem, pdd->drm_priv, &size);
1404 
1405 	/* If freeing the buffer failed, leave the handle in place for
1406 	 * clean-up during process tear-down.
1407 	 */
1408 	if (!ret)
1409 		kfd_process_device_remove_obj_handle(
1410 			pdd, GET_IDR_HANDLE(args->handle));
1411 
1412 	WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size);
1413 
1414 err_unlock:
1415 	mutex_unlock(&p->mutex);
1416 	return ret;
1417 }
1418 
1419 static int kfd_ioctl_map_memory_to_gpu(struct file *filep,
1420 					struct kfd_process *p, void *data)
1421 {
1422 	struct kfd_ioctl_map_memory_to_gpu_args *args = data;
1423 	struct kfd_process_device *pdd, *peer_pdd;
1424 	void *mem;
1425 	struct kfd_dev *dev, *peer;
1426 	long err = 0;
1427 	int i;
1428 	uint32_t *devices_arr = NULL;
1429 	bool table_freed = false;
1430 
1431 	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1432 	if (!dev)
1433 		return -EINVAL;
1434 
1435 	if (!args->n_devices) {
1436 		pr_debug("Device IDs array empty\n");
1437 		return -EINVAL;
1438 	}
1439 	if (args->n_success > args->n_devices) {
1440 		pr_debug("n_success exceeds n_devices\n");
1441 		return -EINVAL;
1442 	}
1443 
1444 	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1445 				    GFP_KERNEL);
1446 	if (!devices_arr)
1447 		return -ENOMEM;
1448 
1449 	err = copy_from_user(devices_arr,
1450 			     (void __user *)args->device_ids_array_ptr,
1451 			     args->n_devices * sizeof(*devices_arr));
1452 	if (err != 0) {
1453 		err = -EFAULT;
1454 		goto copy_from_user_failed;
1455 	}
1456 
1457 	mutex_lock(&p->mutex);
1458 
1459 	pdd = kfd_bind_process_to_device(dev, p);
1460 	if (IS_ERR(pdd)) {
1461 		err = PTR_ERR(pdd);
1462 		goto bind_process_to_device_failed;
1463 	}
1464 
1465 	mem = kfd_process_device_translate_handle(pdd,
1466 						GET_IDR_HANDLE(args->handle));
1467 	if (!mem) {
1468 		err = -ENOMEM;
1469 		goto get_mem_obj_from_handle_failed;
1470 	}
1471 
1472 	for (i = args->n_success; i < args->n_devices; i++) {
1473 		peer = kfd_device_by_id(devices_arr[i]);
1474 		if (!peer) {
1475 			pr_debug("Getting device by id failed for 0x%x\n",
1476 				 devices_arr[i]);
1477 			err = -EINVAL;
1478 			goto get_mem_obj_from_handle_failed;
1479 		}
1480 
1481 		peer_pdd = kfd_bind_process_to_device(peer, p);
1482 		if (IS_ERR(peer_pdd)) {
1483 			err = PTR_ERR(peer_pdd);
1484 			goto get_mem_obj_from_handle_failed;
1485 		}
1486 		err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1487 			peer->kgd, (struct kgd_mem *)mem,
1488 			peer_pdd->drm_priv, &table_freed);
1489 		if (err) {
1490 			pr_err("Failed to map to gpu %d/%d\n",
1491 			       i, args->n_devices);
1492 			goto map_memory_to_gpu_failed;
1493 		}
1494 		args->n_success = i+1;
1495 	}
1496 
1497 	mutex_unlock(&p->mutex);
1498 
1499 	err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd, (struct kgd_mem *) mem, true);
1500 	if (err) {
1501 		pr_debug("Sync memory failed, wait interrupted by user signal\n");
1502 		goto sync_memory_failed;
1503 	}
1504 
1505 	/* Flush TLBs after waiting for the page table updates to complete */
1506 	if (table_freed) {
1507 		for (i = 0; i < args->n_devices; i++) {
1508 			peer = kfd_device_by_id(devices_arr[i]);
1509 			if (WARN_ON_ONCE(!peer))
1510 				continue;
1511 			peer_pdd = kfd_get_process_device_data(peer, p);
1512 			if (WARN_ON_ONCE(!peer_pdd))
1513 				continue;
1514 			kfd_flush_tlb(peer_pdd, TLB_FLUSH_LEGACY);
1515 		}
1516 	}
1517 	kfree(devices_arr);
1518 
1519 	return err;
1520 
1521 bind_process_to_device_failed:
1522 get_mem_obj_from_handle_failed:
1523 map_memory_to_gpu_failed:
1524 	mutex_unlock(&p->mutex);
1525 copy_from_user_failed:
1526 sync_memory_failed:
1527 	kfree(devices_arr);
1528 
1529 	return err;
1530 }
1531 
1532 static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep,
1533 					struct kfd_process *p, void *data)
1534 {
1535 	struct kfd_ioctl_unmap_memory_from_gpu_args *args = data;
1536 	struct kfd_process_device *pdd, *peer_pdd;
1537 	void *mem;
1538 	struct kfd_dev *dev, *peer;
1539 	long err = 0;
1540 	uint32_t *devices_arr = NULL, i;
1541 
1542 	dev = kfd_device_by_id(GET_GPU_ID(args->handle));
1543 	if (!dev)
1544 		return -EINVAL;
1545 
1546 	if (!args->n_devices) {
1547 		pr_debug("Device IDs array empty\n");
1548 		return -EINVAL;
1549 	}
1550 	if (args->n_success > args->n_devices) {
1551 		pr_debug("n_success exceeds n_devices\n");
1552 		return -EINVAL;
1553 	}
1554 
1555 	devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr),
1556 				    GFP_KERNEL);
1557 	if (!devices_arr)
1558 		return -ENOMEM;
1559 
1560 	err = copy_from_user(devices_arr,
1561 			     (void __user *)args->device_ids_array_ptr,
1562 			     args->n_devices * sizeof(*devices_arr));
1563 	if (err != 0) {
1564 		err = -EFAULT;
1565 		goto copy_from_user_failed;
1566 	}
1567 
1568 	mutex_lock(&p->mutex);
1569 
1570 	pdd = kfd_get_process_device_data(dev, p);
1571 	if (!pdd) {
1572 		err = -EINVAL;
1573 		goto bind_process_to_device_failed;
1574 	}
1575 
1576 	mem = kfd_process_device_translate_handle(pdd,
1577 						GET_IDR_HANDLE(args->handle));
1578 	if (!mem) {
1579 		err = -ENOMEM;
1580 		goto get_mem_obj_from_handle_failed;
1581 	}
1582 
1583 	for (i = args->n_success; i < args->n_devices; i++) {
1584 		peer = kfd_device_by_id(devices_arr[i]);
1585 		if (!peer) {
1586 			err = -EINVAL;
1587 			goto get_mem_obj_from_handle_failed;
1588 		}
1589 
1590 		peer_pdd = kfd_get_process_device_data(peer, p);
1591 		if (!peer_pdd) {
1592 			err = -ENODEV;
1593 			goto get_mem_obj_from_handle_failed;
1594 		}
1595 		err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1596 			peer->kgd, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1597 		if (err) {
1598 			pr_err("Failed to unmap from gpu %d/%d\n",
1599 			       i, args->n_devices);
1600 			goto unmap_memory_from_gpu_failed;
1601 		}
1602 		args->n_success = i+1;
1603 	}
1604 	mutex_unlock(&p->mutex);
1605 
1606 	if (dev->device_info->asic_family == CHIP_ALDEBARAN) {
1607 		err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd,
1608 				(struct kgd_mem *) mem, true);
1609 		if (err) {
1610 			pr_debug("Sync memory failed, wait interrupted by user signal\n");
1611 			goto sync_memory_failed;
1612 		}
1613 
1614 		/* Flush TLBs after waiting for the page table updates to complete */
1615 		for (i = 0; i < args->n_devices; i++) {
1616 			peer = kfd_device_by_id(devices_arr[i]);
1617 			if (WARN_ON_ONCE(!peer))
1618 				continue;
1619 			peer_pdd = kfd_get_process_device_data(peer, p);
1620 			if (WARN_ON_ONCE(!peer_pdd))
1621 				continue;
1622 			kfd_flush_tlb(peer_pdd, TLB_FLUSH_HEAVYWEIGHT);
1623 		}
1624 	}
1625 	kfree(devices_arr);
1626 
1627 	return 0;
1628 
1629 bind_process_to_device_failed:
1630 get_mem_obj_from_handle_failed:
1631 unmap_memory_from_gpu_failed:
1632 	mutex_unlock(&p->mutex);
1633 copy_from_user_failed:
1634 sync_memory_failed:
1635 	kfree(devices_arr);
1636 	return err;
1637 }
1638 
1639 static int kfd_ioctl_alloc_queue_gws(struct file *filep,
1640 		struct kfd_process *p, void *data)
1641 {
1642 	int retval;
1643 	struct kfd_ioctl_alloc_queue_gws_args *args = data;
1644 	struct queue *q;
1645 	struct kfd_dev *dev;
1646 
1647 	mutex_lock(&p->mutex);
1648 	q = pqm_get_user_queue(&p->pqm, args->queue_id);
1649 
1650 	if (q) {
1651 		dev = q->device;
1652 	} else {
1653 		retval = -EINVAL;
1654 		goto out_unlock;
1655 	}
1656 
1657 	if (!dev->gws) {
1658 		retval = -ENODEV;
1659 		goto out_unlock;
1660 	}
1661 
1662 	if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1663 		retval = -ENODEV;
1664 		goto out_unlock;
1665 	}
1666 
1667 	retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL);
1668 	mutex_unlock(&p->mutex);
1669 
1670 	args->first_gws = 0;
1671 	return retval;
1672 
1673 out_unlock:
1674 	mutex_unlock(&p->mutex);
1675 	return retval;
1676 }
1677 
1678 static int kfd_ioctl_get_dmabuf_info(struct file *filep,
1679 		struct kfd_process *p, void *data)
1680 {
1681 	struct kfd_ioctl_get_dmabuf_info_args *args = data;
1682 	struct kfd_dev *dev = NULL;
1683 	struct kgd_dev *dma_buf_kgd;
1684 	void *metadata_buffer = NULL;
1685 	uint32_t flags;
1686 	unsigned int i;
1687 	int r;
1688 
1689 	/* Find a KFD GPU device that supports the get_dmabuf_info query */
1690 	for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++)
1691 		if (dev)
1692 			break;
1693 	if (!dev)
1694 		return -EINVAL;
1695 
1696 	if (args->metadata_ptr) {
1697 		metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL);
1698 		if (!metadata_buffer)
1699 			return -ENOMEM;
1700 	}
1701 
1702 	/* Get dmabuf info from KGD */
1703 	r = amdgpu_amdkfd_get_dmabuf_info(dev->kgd, args->dmabuf_fd,
1704 					  &dma_buf_kgd, &args->size,
1705 					  metadata_buffer, args->metadata_size,
1706 					  &args->metadata_size, &flags);
1707 	if (r)
1708 		goto exit;
1709 
1710 	/* Reverse-lookup gpu_id from kgd pointer */
1711 	dev = kfd_device_by_kgd(dma_buf_kgd);
1712 	if (!dev) {
1713 		r = -EINVAL;
1714 		goto exit;
1715 	}
1716 	args->gpu_id = dev->id;
1717 	args->flags = flags;
1718 
1719 	/* Copy metadata buffer to user mode */
1720 	if (metadata_buffer) {
1721 		r = copy_to_user((void __user *)args->metadata_ptr,
1722 				 metadata_buffer, args->metadata_size);
1723 		if (r != 0)
1724 			r = -EFAULT;
1725 	}
1726 
1727 exit:
1728 	kfree(metadata_buffer);
1729 
1730 	return r;
1731 }
1732 
1733 static int kfd_ioctl_import_dmabuf(struct file *filep,
1734 				   struct kfd_process *p, void *data)
1735 {
1736 	struct kfd_ioctl_import_dmabuf_args *args = data;
1737 	struct kfd_process_device *pdd;
1738 	struct dma_buf *dmabuf;
1739 	struct kfd_dev *dev;
1740 	int idr_handle;
1741 	uint64_t size;
1742 	void *mem;
1743 	int r;
1744 
1745 	dev = kfd_device_by_id(args->gpu_id);
1746 	if (!dev)
1747 		return -EINVAL;
1748 
1749 	dmabuf = dma_buf_get(args->dmabuf_fd);
1750 	if (IS_ERR(dmabuf))
1751 		return PTR_ERR(dmabuf);
1752 
1753 	mutex_lock(&p->mutex);
1754 
1755 	pdd = kfd_bind_process_to_device(dev, p);
1756 	if (IS_ERR(pdd)) {
1757 		r = PTR_ERR(pdd);
1758 		goto err_unlock;
1759 	}
1760 
1761 	r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf,
1762 					      args->va_addr, pdd->drm_priv,
1763 					      (struct kgd_mem **)&mem, &size,
1764 					      NULL);
1765 	if (r)
1766 		goto err_unlock;
1767 
1768 	idr_handle = kfd_process_device_create_obj_handle(pdd, mem);
1769 	if (idr_handle < 0) {
1770 		r = -EFAULT;
1771 		goto err_free;
1772 	}
1773 
1774 	mutex_unlock(&p->mutex);
1775 	dma_buf_put(dmabuf);
1776 
1777 	args->handle = MAKE_HANDLE(args->gpu_id, idr_handle);
1778 
1779 	return 0;
1780 
1781 err_free:
1782 	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
1783 					       pdd->drm_priv, NULL);
1784 err_unlock:
1785 	mutex_unlock(&p->mutex);
1786 	dma_buf_put(dmabuf);
1787 	return r;
1788 }
1789 
1790 /* Handle requests for watching SMI events */
1791 static int kfd_ioctl_smi_events(struct file *filep,
1792 				struct kfd_process *p, void *data)
1793 {
1794 	struct kfd_ioctl_smi_events_args *args = data;
1795 	struct kfd_dev *dev;
1796 
1797 	dev = kfd_device_by_id(args->gpuid);
1798 	if (!dev)
1799 		return -EINVAL;
1800 
1801 	return kfd_smi_event_open(dev, &args->anon_fd);
1802 }
1803 
1804 static int kfd_ioctl_set_xnack_mode(struct file *filep,
1805 				    struct kfd_process *p, void *data)
1806 {
1807 	struct kfd_ioctl_set_xnack_mode_args *args = data;
1808 	int r = 0;
1809 
1810 	mutex_lock(&p->mutex);
1811 	if (args->xnack_enabled >= 0) {
1812 		if (!list_empty(&p->pqm.queues)) {
1813 			pr_debug("Process has user queues running\n");
1814 			mutex_unlock(&p->mutex);
1815 			return -EBUSY;
1816 		}
1817 		if (args->xnack_enabled && !kfd_process_xnack_mode(p, true))
1818 			r = -EPERM;
1819 		else
1820 			p->xnack_enabled = args->xnack_enabled;
1821 	} else {
1822 		args->xnack_enabled = p->xnack_enabled;
1823 	}
1824 	mutex_unlock(&p->mutex);
1825 
1826 	return r;
1827 }
1828 
1829 #if IS_ENABLED(CONFIG_HSA_AMD_SVM)
1830 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1831 {
1832 	struct kfd_ioctl_svm_args *args = data;
1833 	int r = 0;
1834 
1835 	pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
1836 		 args->start_addr, args->size, args->op, args->nattr);
1837 
1838 	if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
1839 		return -EINVAL;
1840 	if (!args->start_addr || !args->size)
1841 		return -EINVAL;
1842 
1843 	mutex_lock(&p->mutex);
1844 
1845 	r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
1846 		      args->attrs);
1847 
1848 	mutex_unlock(&p->mutex);
1849 
1850 	return r;
1851 }
1852 #else
1853 static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
1854 {
1855 	return -EPERM;
1856 }
1857 #endif
1858 
1859 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
1860 	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
1861 			    .cmd_drv = 0, .name = #ioctl}
1862 
1863 /** Ioctl table */
1864 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
1865 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
1866 			kfd_ioctl_get_version, 0),
1867 
1868 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
1869 			kfd_ioctl_create_queue, 0),
1870 
1871 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
1872 			kfd_ioctl_destroy_queue, 0),
1873 
1874 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
1875 			kfd_ioctl_set_memory_policy, 0),
1876 
1877 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
1878 			kfd_ioctl_get_clock_counters, 0),
1879 
1880 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
1881 			kfd_ioctl_get_process_apertures, 0),
1882 
1883 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
1884 			kfd_ioctl_update_queue, 0),
1885 
1886 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
1887 			kfd_ioctl_create_event, 0),
1888 
1889 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
1890 			kfd_ioctl_destroy_event, 0),
1891 
1892 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
1893 			kfd_ioctl_set_event, 0),
1894 
1895 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
1896 			kfd_ioctl_reset_event, 0),
1897 
1898 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
1899 			kfd_ioctl_wait_events, 0),
1900 
1901 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER,
1902 			kfd_ioctl_dbg_register, 0),
1903 
1904 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER,
1905 			kfd_ioctl_dbg_unregister, 0),
1906 
1907 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH,
1908 			kfd_ioctl_dbg_address_watch, 0),
1909 
1910 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL,
1911 			kfd_ioctl_dbg_wave_control, 0),
1912 
1913 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA,
1914 			kfd_ioctl_set_scratch_backing_va, 0),
1915 
1916 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG,
1917 			kfd_ioctl_get_tile_config, 0),
1918 
1919 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER,
1920 			kfd_ioctl_set_trap_handler, 0),
1921 
1922 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW,
1923 			kfd_ioctl_get_process_apertures_new, 0),
1924 
1925 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM,
1926 			kfd_ioctl_acquire_vm, 0),
1927 
1928 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU,
1929 			kfd_ioctl_alloc_memory_of_gpu, 0),
1930 
1931 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU,
1932 			kfd_ioctl_free_memory_of_gpu, 0),
1933 
1934 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU,
1935 			kfd_ioctl_map_memory_to_gpu, 0),
1936 
1937 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
1938 			kfd_ioctl_unmap_memory_from_gpu, 0),
1939 
1940 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
1941 			kfd_ioctl_set_cu_mask, 0),
1942 
1943 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE,
1944 			kfd_ioctl_get_queue_wave_state, 0),
1945 
1946 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO,
1947 				kfd_ioctl_get_dmabuf_info, 0),
1948 
1949 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF,
1950 				kfd_ioctl_import_dmabuf, 0),
1951 
1952 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS,
1953 			kfd_ioctl_alloc_queue_gws, 0),
1954 
1955 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
1956 			kfd_ioctl_smi_events, 0),
1957 
1958 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
1959 
1960 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
1961 			kfd_ioctl_set_xnack_mode, 0),
1962 };
1963 
1964 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
1965 
1966 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
1967 {
1968 	struct kfd_process *process;
1969 	amdkfd_ioctl_t *func;
1970 	const struct amdkfd_ioctl_desc *ioctl = NULL;
1971 	unsigned int nr = _IOC_NR(cmd);
1972 	char stack_kdata[128];
1973 	char *kdata = NULL;
1974 	unsigned int usize, asize;
1975 	int retcode = -EINVAL;
1976 
1977 	if (nr >= AMDKFD_CORE_IOCTL_COUNT)
1978 		goto err_i1;
1979 
1980 	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
1981 		u32 amdkfd_size;
1982 
1983 		ioctl = &amdkfd_ioctls[nr];
1984 
1985 		amdkfd_size = _IOC_SIZE(ioctl->cmd);
1986 		usize = asize = _IOC_SIZE(cmd);
1987 		if (amdkfd_size > asize)
1988 			asize = amdkfd_size;
1989 
1990 		cmd = ioctl->cmd;
1991 	} else
1992 		goto err_i1;
1993 
1994 	dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg);
1995 
1996 	/* Get the process struct from the filep. Only the process
1997 	 * that opened /dev/kfd can use the file descriptor. Child
1998 	 * processes need to create their own KFD device context.
1999 	 */
2000 	process = filep->private_data;
2001 	if (process->lead_thread != current->group_leader) {
2002 		dev_dbg(kfd_device, "Using KFD FD in wrong process\n");
2003 		retcode = -EBADF;
2004 		goto err_i1;
2005 	}
2006 
2007 	/* Do not trust userspace, use our own definition */
2008 	func = ioctl->func;
2009 
2010 	if (unlikely(!func)) {
2011 		dev_dbg(kfd_device, "no function\n");
2012 		retcode = -EINVAL;
2013 		goto err_i1;
2014 	}
2015 
2016 	if (cmd & (IOC_IN | IOC_OUT)) {
2017 		if (asize <= sizeof(stack_kdata)) {
2018 			kdata = stack_kdata;
2019 		} else {
2020 			kdata = kmalloc(asize, GFP_KERNEL);
2021 			if (!kdata) {
2022 				retcode = -ENOMEM;
2023 				goto err_i1;
2024 			}
2025 		}
2026 		if (asize > usize)
2027 			memset(kdata + usize, 0, asize - usize);
2028 	}
2029 
2030 	if (cmd & IOC_IN) {
2031 		if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
2032 			retcode = -EFAULT;
2033 			goto err_i1;
2034 		}
2035 	} else if (cmd & IOC_OUT) {
2036 		memset(kdata, 0, usize);
2037 	}
2038 
2039 	retcode = func(filep, process, kdata);
2040 
2041 	if (cmd & IOC_OUT)
2042 		if (copy_to_user((void __user *)arg, kdata, usize) != 0)
2043 			retcode = -EFAULT;
2044 
2045 err_i1:
2046 	if (!ioctl)
2047 		dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
2048 			  task_pid_nr(current), cmd, nr);
2049 
2050 	if (kdata != stack_kdata)
2051 		kfree(kdata);
2052 
2053 	if (retcode)
2054 		dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n",
2055 				nr, arg, retcode);
2056 
2057 	return retcode;
2058 }
2059 
2060 static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process,
2061 		      struct vm_area_struct *vma)
2062 {
2063 	phys_addr_t address;
2064 	int ret;
2065 
2066 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2067 		return -EINVAL;
2068 
2069 	address = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd);
2070 
2071 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE |
2072 				VM_DONTDUMP | VM_PFNMAP;
2073 
2074 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2075 
2076 	pr_debug("pasid 0x%x mapping mmio page\n"
2077 		 "     target user address == 0x%08llX\n"
2078 		 "     physical address    == 0x%08llX\n"
2079 		 "     vm_flags            == 0x%04lX\n"
2080 		 "     size                == 0x%04lX\n",
2081 		 process->pasid, (unsigned long long) vma->vm_start,
2082 		 address, vma->vm_flags, PAGE_SIZE);
2083 
2084 	ret = io_remap_pfn_range(vma,
2085 				vma->vm_start,
2086 				address >> PAGE_SHIFT,
2087 				PAGE_SIZE,
2088 				vma->vm_page_prot);
2089 	return ret;
2090 }
2091 
2092 
2093 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
2094 {
2095 	struct kfd_process *process;
2096 	struct kfd_dev *dev = NULL;
2097 	unsigned long mmap_offset;
2098 	unsigned int gpu_id;
2099 
2100 	process = kfd_get_process(current);
2101 	if (IS_ERR(process))
2102 		return PTR_ERR(process);
2103 
2104 	mmap_offset = vma->vm_pgoff << PAGE_SHIFT;
2105 	gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset);
2106 	if (gpu_id)
2107 		dev = kfd_device_by_id(gpu_id);
2108 
2109 	switch (mmap_offset & KFD_MMAP_TYPE_MASK) {
2110 	case KFD_MMAP_TYPE_DOORBELL:
2111 		if (!dev)
2112 			return -ENODEV;
2113 		return kfd_doorbell_mmap(dev, process, vma);
2114 
2115 	case KFD_MMAP_TYPE_EVENTS:
2116 		return kfd_event_mmap(process, vma);
2117 
2118 	case KFD_MMAP_TYPE_RESERVED_MEM:
2119 		if (!dev)
2120 			return -ENODEV;
2121 		return kfd_reserved_mem_mmap(dev, process, vma);
2122 	case KFD_MMAP_TYPE_MMIO:
2123 		if (!dev)
2124 			return -ENODEV;
2125 		return kfd_mmio_mmap(dev, process, vma);
2126 	}
2127 
2128 	return -EFAULT;
2129 }
2130