1 /*
2  * Copyright 2014 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 
23 #include <linux/device.h>
24 #include <linux/export.h>
25 #include <linux/err.h>
26 #include <linux/fs.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
29 #include <linux/uaccess.h>
30 #include <linux/compat.h>
31 #include <uapi/linux/kfd_ioctl.h>
32 #include <linux/time.h>
33 #include <linux/mm.h>
34 #include <linux/mman.h>
35 #include <asm/processor.h>
36 #include "kfd_priv.h"
37 #include "kfd_device_queue_manager.h"
38 #include "kfd_dbgmgr.h"
39 
40 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
41 static int kfd_open(struct inode *, struct file *);
42 static int kfd_mmap(struct file *, struct vm_area_struct *);
43 
44 static const char kfd_dev_name[] = "kfd";
45 
46 static const struct file_operations kfd_fops = {
47 	.owner = THIS_MODULE,
48 	.unlocked_ioctl = kfd_ioctl,
49 	.compat_ioctl = kfd_ioctl,
50 	.open = kfd_open,
51 	.mmap = kfd_mmap,
52 };
53 
54 static int kfd_char_dev_major = -1;
55 static struct class *kfd_class;
56 struct device *kfd_device;
57 
58 int kfd_chardev_init(void)
59 {
60 	int err = 0;
61 
62 	kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops);
63 	err = kfd_char_dev_major;
64 	if (err < 0)
65 		goto err_register_chrdev;
66 
67 	kfd_class = class_create(THIS_MODULE, kfd_dev_name);
68 	err = PTR_ERR(kfd_class);
69 	if (IS_ERR(kfd_class))
70 		goto err_class_create;
71 
72 	kfd_device = device_create(kfd_class, NULL,
73 					MKDEV(kfd_char_dev_major, 0),
74 					NULL, kfd_dev_name);
75 	err = PTR_ERR(kfd_device);
76 	if (IS_ERR(kfd_device))
77 		goto err_device_create;
78 
79 	return 0;
80 
81 err_device_create:
82 	class_destroy(kfd_class);
83 err_class_create:
84 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
85 err_register_chrdev:
86 	return err;
87 }
88 
89 void kfd_chardev_exit(void)
90 {
91 	device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0));
92 	class_destroy(kfd_class);
93 	unregister_chrdev(kfd_char_dev_major, kfd_dev_name);
94 }
95 
96 struct device *kfd_chardev(void)
97 {
98 	return kfd_device;
99 }
100 
101 
102 static int kfd_open(struct inode *inode, struct file *filep)
103 {
104 	struct kfd_process *process;
105 	bool is_32bit_user_mode;
106 
107 	if (iminor(inode) != 0)
108 		return -ENODEV;
109 
110 	is_32bit_user_mode = in_compat_syscall();
111 
112 	if (is_32bit_user_mode) {
113 		dev_warn(kfd_device,
114 			"Process %d (32-bit) failed to open /dev/kfd\n"
115 			"32-bit processes are not supported by amdkfd\n",
116 			current->pid);
117 		return -EPERM;
118 	}
119 
120 	process = kfd_create_process(current);
121 	if (IS_ERR(process))
122 		return PTR_ERR(process);
123 
124 	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
125 		process->pasid, process->is_32bit_user_mode);
126 
127 	return 0;
128 }
129 
130 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p,
131 					void *data)
132 {
133 	struct kfd_ioctl_get_version_args *args = data;
134 
135 	args->major_version = KFD_IOCTL_MAJOR_VERSION;
136 	args->minor_version = KFD_IOCTL_MINOR_VERSION;
137 
138 	return 0;
139 }
140 
141 static int set_queue_properties_from_user(struct queue_properties *q_properties,
142 				struct kfd_ioctl_create_queue_args *args)
143 {
144 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
145 		pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
146 		return -EINVAL;
147 	}
148 
149 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
150 		pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
151 		return -EINVAL;
152 	}
153 
154 	if ((args->ring_base_address) &&
155 		(!access_ok(VERIFY_WRITE,
156 			(const void __user *) args->ring_base_address,
157 			sizeof(uint64_t)))) {
158 		pr_err("kfd: can't access ring base address\n");
159 		return -EFAULT;
160 	}
161 
162 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
163 		pr_err("kfd: ring size must be a power of 2 or 0\n");
164 		return -EINVAL;
165 	}
166 
167 	if (!access_ok(VERIFY_WRITE,
168 			(const void __user *) args->read_pointer_address,
169 			sizeof(uint32_t))) {
170 		pr_err("kfd: can't access read pointer\n");
171 		return -EFAULT;
172 	}
173 
174 	if (!access_ok(VERIFY_WRITE,
175 			(const void __user *) args->write_pointer_address,
176 			sizeof(uint32_t))) {
177 		pr_err("kfd: can't access write pointer\n");
178 		return -EFAULT;
179 	}
180 
181 	if (args->eop_buffer_address &&
182 		!access_ok(VERIFY_WRITE,
183 			(const void __user *) args->eop_buffer_address,
184 			sizeof(uint32_t))) {
185 		pr_debug("kfd: can't access eop buffer");
186 		return -EFAULT;
187 	}
188 
189 	if (args->ctx_save_restore_address &&
190 		!access_ok(VERIFY_WRITE,
191 			(const void __user *) args->ctx_save_restore_address,
192 			sizeof(uint32_t))) {
193 		pr_debug("kfd: can't access ctx save restore buffer");
194 		return -EFAULT;
195 	}
196 
197 	q_properties->is_interop = false;
198 	q_properties->queue_percent = args->queue_percentage;
199 	q_properties->priority = args->queue_priority;
200 	q_properties->queue_address = args->ring_base_address;
201 	q_properties->queue_size = args->ring_size;
202 	q_properties->read_ptr = (uint32_t *) args->read_pointer_address;
203 	q_properties->write_ptr = (uint32_t *) args->write_pointer_address;
204 	q_properties->eop_ring_buffer_address = args->eop_buffer_address;
205 	q_properties->eop_ring_buffer_size = args->eop_buffer_size;
206 	q_properties->ctx_save_restore_area_address =
207 			args->ctx_save_restore_address;
208 	q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size;
209 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE ||
210 		args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
211 		q_properties->type = KFD_QUEUE_TYPE_COMPUTE;
212 	else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA)
213 		q_properties->type = KFD_QUEUE_TYPE_SDMA;
214 	else
215 		return -ENOTSUPP;
216 
217 	if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL)
218 		q_properties->format = KFD_QUEUE_FORMAT_AQL;
219 	else
220 		q_properties->format = KFD_QUEUE_FORMAT_PM4;
221 
222 	pr_debug("Queue Percentage (%d, %d)\n",
223 			q_properties->queue_percent, args->queue_percentage);
224 
225 	pr_debug("Queue Priority (%d, %d)\n",
226 			q_properties->priority, args->queue_priority);
227 
228 	pr_debug("Queue Address (0x%llX, 0x%llX)\n",
229 			q_properties->queue_address, args->ring_base_address);
230 
231 	pr_debug("Queue Size (0x%llX, %u)\n",
232 			q_properties->queue_size, args->ring_size);
233 
234 	pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n",
235 			(uint64_t) q_properties->read_ptr,
236 			(uint64_t) q_properties->write_ptr);
237 
238 	pr_debug("Queue Format (%d)\n", q_properties->format);
239 
240 	pr_debug("Queue EOP (0x%llX)\n", q_properties->eop_ring_buffer_address);
241 
242 	pr_debug("Queue CTX save arex (0x%llX)\n",
243 			q_properties->ctx_save_restore_area_address);
244 
245 	return 0;
246 }
247 
248 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p,
249 					void *data)
250 {
251 	struct kfd_ioctl_create_queue_args *args = data;
252 	struct kfd_dev *dev;
253 	int err = 0;
254 	unsigned int queue_id;
255 	struct kfd_process_device *pdd;
256 	struct queue_properties q_properties;
257 
258 	memset(&q_properties, 0, sizeof(struct queue_properties));
259 
260 	pr_debug("kfd: creating queue ioctl\n");
261 
262 	err = set_queue_properties_from_user(&q_properties, args);
263 	if (err)
264 		return err;
265 
266 	pr_debug("kfd: looking for gpu id 0x%x\n", args->gpu_id);
267 	dev = kfd_device_by_id(args->gpu_id);
268 	if (dev == NULL) {
269 		pr_debug("kfd: gpu id 0x%x was not found\n", args->gpu_id);
270 		return -EINVAL;
271 	}
272 
273 	mutex_lock(&p->mutex);
274 
275 	pdd = kfd_bind_process_to_device(dev, p);
276 	if (IS_ERR(pdd)) {
277 		err = -ESRCH;
278 		goto err_bind_process;
279 	}
280 
281 	pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n",
282 			p->pasid,
283 			dev->id);
284 
285 	err = pqm_create_queue(&p->pqm, dev, filep, &q_properties,
286 				0, q_properties.type, &queue_id);
287 	if (err != 0)
288 		goto err_create_queue;
289 
290 	args->queue_id = queue_id;
291 
292 
293 	/* Return gpu_id as doorbell offset for mmap usage */
294 	args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id);
295 	args->doorbell_offset <<= PAGE_SHIFT;
296 
297 	mutex_unlock(&p->mutex);
298 
299 	pr_debug("kfd: queue id %d was created successfully\n", args->queue_id);
300 
301 	pr_debug("ring buffer address == 0x%016llX\n",
302 			args->ring_base_address);
303 
304 	pr_debug("read ptr address    == 0x%016llX\n",
305 			args->read_pointer_address);
306 
307 	pr_debug("write ptr address   == 0x%016llX\n",
308 			args->write_pointer_address);
309 
310 	return 0;
311 
312 err_create_queue:
313 err_bind_process:
314 	mutex_unlock(&p->mutex);
315 	return err;
316 }
317 
318 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p,
319 					void *data)
320 {
321 	int retval;
322 	struct kfd_ioctl_destroy_queue_args *args = data;
323 
324 	pr_debug("kfd: destroying queue id %d for PASID %d\n",
325 				args->queue_id,
326 				p->pasid);
327 
328 	mutex_lock(&p->mutex);
329 
330 	retval = pqm_destroy_queue(&p->pqm, args->queue_id);
331 
332 	mutex_unlock(&p->mutex);
333 	return retval;
334 }
335 
336 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p,
337 					void *data)
338 {
339 	int retval;
340 	struct kfd_ioctl_update_queue_args *args = data;
341 	struct queue_properties properties;
342 
343 	if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) {
344 		pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n");
345 		return -EINVAL;
346 	}
347 
348 	if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) {
349 		pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n");
350 		return -EINVAL;
351 	}
352 
353 	if ((args->ring_base_address) &&
354 		(!access_ok(VERIFY_WRITE,
355 			(const void __user *) args->ring_base_address,
356 			sizeof(uint64_t)))) {
357 		pr_err("kfd: can't access ring base address\n");
358 		return -EFAULT;
359 	}
360 
361 	if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) {
362 		pr_err("kfd: ring size must be a power of 2 or 0\n");
363 		return -EINVAL;
364 	}
365 
366 	properties.queue_address = args->ring_base_address;
367 	properties.queue_size = args->ring_size;
368 	properties.queue_percent = args->queue_percentage;
369 	properties.priority = args->queue_priority;
370 
371 	pr_debug("kfd: updating queue id %d for PASID %d\n",
372 			args->queue_id, p->pasid);
373 
374 	mutex_lock(&p->mutex);
375 
376 	retval = pqm_update_queue(&p->pqm, args->queue_id, &properties);
377 
378 	mutex_unlock(&p->mutex);
379 
380 	return retval;
381 }
382 
383 static int kfd_ioctl_set_memory_policy(struct file *filep,
384 					struct kfd_process *p, void *data)
385 {
386 	struct kfd_ioctl_set_memory_policy_args *args = data;
387 	struct kfd_dev *dev;
388 	int err = 0;
389 	struct kfd_process_device *pdd;
390 	enum cache_policy default_policy, alternate_policy;
391 
392 	if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT
393 	    && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
394 		return -EINVAL;
395 	}
396 
397 	if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT
398 	    && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) {
399 		return -EINVAL;
400 	}
401 
402 	dev = kfd_device_by_id(args->gpu_id);
403 	if (dev == NULL)
404 		return -EINVAL;
405 
406 	mutex_lock(&p->mutex);
407 
408 	pdd = kfd_bind_process_to_device(dev, p);
409 	if (IS_ERR(pdd)) {
410 		err = -ESRCH;
411 		goto out;
412 	}
413 
414 	default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT)
415 			 ? cache_policy_coherent : cache_policy_noncoherent;
416 
417 	alternate_policy =
418 		(args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT)
419 		   ? cache_policy_coherent : cache_policy_noncoherent;
420 
421 	if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm,
422 				&pdd->qpd,
423 				default_policy,
424 				alternate_policy,
425 				(void __user *)args->alternate_aperture_base,
426 				args->alternate_aperture_size))
427 		err = -EINVAL;
428 
429 out:
430 	mutex_unlock(&p->mutex);
431 
432 	return err;
433 }
434 
435 static int kfd_ioctl_dbg_register(struct file *filep,
436 				struct kfd_process *p, void *data)
437 {
438 	struct kfd_ioctl_dbg_register_args *args = data;
439 	struct kfd_dev *dev;
440 	struct kfd_dbgmgr *dbgmgr_ptr;
441 	struct kfd_process_device *pdd;
442 	bool create_ok;
443 	long status = 0;
444 
445 	dev = kfd_device_by_id(args->gpu_id);
446 	if (dev == NULL)
447 		return -EINVAL;
448 
449 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
450 		pr_debug("kfd_ioctl_dbg_register not supported on CZ\n");
451 		return -EINVAL;
452 	}
453 
454 	mutex_lock(kfd_get_dbgmgr_mutex());
455 	mutex_lock(&p->mutex);
456 
457 	/*
458 	 * make sure that we have pdd, if this the first queue created for
459 	 * this process
460 	 */
461 	pdd = kfd_bind_process_to_device(dev, p);
462 	if (IS_ERR(pdd)) {
463 		mutex_unlock(&p->mutex);
464 		mutex_unlock(kfd_get_dbgmgr_mutex());
465 		return PTR_ERR(pdd);
466 	}
467 
468 	if (dev->dbgmgr == NULL) {
469 		/* In case of a legal call, we have no dbgmgr yet */
470 		create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev);
471 		if (create_ok) {
472 			status = kfd_dbgmgr_register(dbgmgr_ptr, p);
473 			if (status != 0)
474 				kfd_dbgmgr_destroy(dbgmgr_ptr);
475 			else
476 				dev->dbgmgr = dbgmgr_ptr;
477 		}
478 	} else {
479 		pr_debug("debugger already registered\n");
480 		status = -EINVAL;
481 	}
482 
483 	mutex_unlock(&p->mutex);
484 	mutex_unlock(kfd_get_dbgmgr_mutex());
485 
486 	return status;
487 }
488 
489 static int kfd_ioctl_dbg_unregister(struct file *filep,
490 				struct kfd_process *p, void *data)
491 {
492 	struct kfd_ioctl_dbg_unregister_args *args = data;
493 	struct kfd_dev *dev;
494 	long status;
495 
496 	dev = kfd_device_by_id(args->gpu_id);
497 	if (dev == NULL)
498 		return -EINVAL;
499 
500 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
501 		pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n");
502 		return -EINVAL;
503 	}
504 
505 	mutex_lock(kfd_get_dbgmgr_mutex());
506 
507 	status = kfd_dbgmgr_unregister(dev->dbgmgr, p);
508 	if (status == 0) {
509 		kfd_dbgmgr_destroy(dev->dbgmgr);
510 		dev->dbgmgr = NULL;
511 	}
512 
513 	mutex_unlock(kfd_get_dbgmgr_mutex());
514 
515 	return status;
516 }
517 
518 /*
519  * Parse and generate variable size data structure for address watch.
520  * Total size of the buffer and # watch points is limited in order
521  * to prevent kernel abuse. (no bearing to the much smaller HW limitation
522  * which is enforced by dbgdev module)
523  * please also note that the watch address itself are not "copied from user",
524  * since it be set into the HW in user mode values.
525  *
526  */
527 static int kfd_ioctl_dbg_address_watch(struct file *filep,
528 					struct kfd_process *p, void *data)
529 {
530 	struct kfd_ioctl_dbg_address_watch_args *args = data;
531 	struct kfd_dev *dev;
532 	struct dbg_address_watch_info aw_info;
533 	unsigned char *args_buff;
534 	long status;
535 	void __user *cmd_from_user;
536 	uint64_t watch_mask_value = 0;
537 	unsigned int args_idx = 0;
538 
539 	memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info));
540 
541 	dev = kfd_device_by_id(args->gpu_id);
542 	if (dev == NULL)
543 		return -EINVAL;
544 
545 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
546 		pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
547 		return -EINVAL;
548 	}
549 
550 	cmd_from_user = (void __user *) args->content_ptr;
551 
552 	/* Validate arguments */
553 
554 	if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) ||
555 		(args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) ||
556 		(cmd_from_user == NULL))
557 		return -EINVAL;
558 
559 	/* this is the actual buffer to work with */
560 	args_buff = memdup_user(cmd_from_user,
561 				args->buf_size_in_bytes - sizeof(*args));
562 	if (IS_ERR(args_buff))
563 		return PTR_ERR(args_buff);
564 
565 	aw_info.process = p;
566 
567 	aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx]));
568 	args_idx += sizeof(aw_info.num_watch_points);
569 
570 	aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx];
571 	args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points;
572 
573 	/*
574 	 * set watch address base pointer to point on the array base
575 	 * within args_buff
576 	 */
577 	aw_info.watch_address = (uint64_t *) &args_buff[args_idx];
578 
579 	/* skip over the addresses buffer */
580 	args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points;
581 
582 	if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) {
583 		kfree(args_buff);
584 		return -EINVAL;
585 	}
586 
587 	watch_mask_value = (uint64_t) args_buff[args_idx];
588 
589 	if (watch_mask_value > 0) {
590 		/*
591 		 * There is an array of masks.
592 		 * set watch mask base pointer to point on the array base
593 		 * within args_buff
594 		 */
595 		aw_info.watch_mask = (uint64_t *) &args_buff[args_idx];
596 
597 		/* skip over the masks buffer */
598 		args_idx += sizeof(aw_info.watch_mask) *
599 				aw_info.num_watch_points;
600 	} else {
601 		/* just the NULL mask, set to NULL and skip over it */
602 		aw_info.watch_mask = NULL;
603 		args_idx += sizeof(aw_info.watch_mask);
604 	}
605 
606 	if (args_idx >= args->buf_size_in_bytes - sizeof(args)) {
607 		kfree(args_buff);
608 		return -EINVAL;
609 	}
610 
611 	/* Currently HSA Event is not supported for DBG */
612 	aw_info.watch_event = NULL;
613 
614 	mutex_lock(kfd_get_dbgmgr_mutex());
615 
616 	status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info);
617 
618 	mutex_unlock(kfd_get_dbgmgr_mutex());
619 
620 	kfree(args_buff);
621 
622 	return status;
623 }
624 
625 /* Parse and generate fixed size data structure for wave control */
626 static int kfd_ioctl_dbg_wave_control(struct file *filep,
627 					struct kfd_process *p, void *data)
628 {
629 	struct kfd_ioctl_dbg_wave_control_args *args = data;
630 	struct kfd_dev *dev;
631 	struct dbg_wave_control_info wac_info;
632 	unsigned char *args_buff;
633 	uint32_t computed_buff_size;
634 	long status;
635 	void __user *cmd_from_user;
636 	unsigned int args_idx = 0;
637 
638 	memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info));
639 
640 	/* we use compact form, independent of the packing attribute value */
641 	computed_buff_size = sizeof(*args) +
642 				sizeof(wac_info.mode) +
643 				sizeof(wac_info.operand) +
644 				sizeof(wac_info.dbgWave_msg.DbgWaveMsg) +
645 				sizeof(wac_info.dbgWave_msg.MemoryVA) +
646 				sizeof(wac_info.trapId);
647 
648 	dev = kfd_device_by_id(args->gpu_id);
649 	if (dev == NULL)
650 		return -EINVAL;
651 
652 	if (dev->device_info->asic_family == CHIP_CARRIZO) {
653 		pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n");
654 		return -EINVAL;
655 	}
656 
657 	/* input size must match the computed "compact" size */
658 	if (args->buf_size_in_bytes != computed_buff_size) {
659 		pr_debug("size mismatch, computed : actual %u : %u\n",
660 				args->buf_size_in_bytes, computed_buff_size);
661 		return -EINVAL;
662 	}
663 
664 	cmd_from_user = (void __user *) args->content_ptr;
665 
666 	if (cmd_from_user == NULL)
667 		return -EINVAL;
668 
669 	/* copy the entire buffer from user */
670 
671 	args_buff = memdup_user(cmd_from_user,
672 				args->buf_size_in_bytes - sizeof(*args));
673 	if (IS_ERR(args_buff))
674 		return PTR_ERR(args_buff);
675 
676 	/* move ptr to the start of the "pay-load" area */
677 	wac_info.process = p;
678 
679 	wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx]));
680 	args_idx += sizeof(wac_info.operand);
681 
682 	wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx]));
683 	args_idx += sizeof(wac_info.mode);
684 
685 	wac_info.trapId = *((uint32_t *)(&args_buff[args_idx]));
686 	args_idx += sizeof(wac_info.trapId);
687 
688 	wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value =
689 					*((uint32_t *)(&args_buff[args_idx]));
690 	wac_info.dbgWave_msg.MemoryVA = NULL;
691 
692 	mutex_lock(kfd_get_dbgmgr_mutex());
693 
694 	pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n",
695 			wac_info.process, wac_info.operand,
696 			wac_info.mode, wac_info.trapId,
697 			wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value);
698 
699 	status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info);
700 
701 	pr_debug("Returned status of dbg manager is %ld\n", status);
702 
703 	mutex_unlock(kfd_get_dbgmgr_mutex());
704 
705 	kfree(args_buff);
706 
707 	return status;
708 }
709 
710 static int kfd_ioctl_get_clock_counters(struct file *filep,
711 				struct kfd_process *p, void *data)
712 {
713 	struct kfd_ioctl_get_clock_counters_args *args = data;
714 	struct kfd_dev *dev;
715 	struct timespec64 time;
716 
717 	dev = kfd_device_by_id(args->gpu_id);
718 	if (dev == NULL)
719 		return -EINVAL;
720 
721 	/* Reading GPU clock counter from KGD */
722 	args->gpu_clock_counter =
723 		dev->kfd2kgd->get_gpu_clock_counter(dev->kgd);
724 
725 	/* No access to rdtsc. Using raw monotonic time */
726 	getrawmonotonic64(&time);
727 	args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time);
728 
729 	get_monotonic_boottime64(&time);
730 	args->system_clock_counter = (uint64_t)timespec64_to_ns(&time);
731 
732 	/* Since the counter is in nano-seconds we use 1GHz frequency */
733 	args->system_clock_freq = 1000000000;
734 
735 	return 0;
736 }
737 
738 
739 static int kfd_ioctl_get_process_apertures(struct file *filp,
740 				struct kfd_process *p, void *data)
741 {
742 	struct kfd_ioctl_get_process_apertures_args *args = data;
743 	struct kfd_process_device_apertures *pAperture;
744 	struct kfd_process_device *pdd;
745 
746 	dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid);
747 
748 	args->num_of_nodes = 0;
749 
750 	mutex_lock(&p->mutex);
751 
752 	/*if the process-device list isn't empty*/
753 	if (kfd_has_process_device_data(p)) {
754 		/* Run over all pdd of the process */
755 		pdd = kfd_get_first_process_device_data(p);
756 		do {
757 			pAperture =
758 				&args->process_apertures[args->num_of_nodes];
759 			pAperture->gpu_id = pdd->dev->id;
760 			pAperture->lds_base = pdd->lds_base;
761 			pAperture->lds_limit = pdd->lds_limit;
762 			pAperture->gpuvm_base = pdd->gpuvm_base;
763 			pAperture->gpuvm_limit = pdd->gpuvm_limit;
764 			pAperture->scratch_base = pdd->scratch_base;
765 			pAperture->scratch_limit = pdd->scratch_limit;
766 
767 			dev_dbg(kfd_device,
768 				"node id %u\n", args->num_of_nodes);
769 			dev_dbg(kfd_device,
770 				"gpu id %u\n", pdd->dev->id);
771 			dev_dbg(kfd_device,
772 				"lds_base %llX\n", pdd->lds_base);
773 			dev_dbg(kfd_device,
774 				"lds_limit %llX\n", pdd->lds_limit);
775 			dev_dbg(kfd_device,
776 				"gpuvm_base %llX\n", pdd->gpuvm_base);
777 			dev_dbg(kfd_device,
778 				"gpuvm_limit %llX\n", pdd->gpuvm_limit);
779 			dev_dbg(kfd_device,
780 				"scratch_base %llX\n", pdd->scratch_base);
781 			dev_dbg(kfd_device,
782 				"scratch_limit %llX\n", pdd->scratch_limit);
783 
784 			args->num_of_nodes++;
785 		} while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL &&
786 				(args->num_of_nodes < NUM_OF_SUPPORTED_GPUS));
787 	}
788 
789 	mutex_unlock(&p->mutex);
790 
791 	return 0;
792 }
793 
794 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p,
795 					void *data)
796 {
797 	struct kfd_ioctl_create_event_args *args = data;
798 	int err;
799 
800 	err = kfd_event_create(filp, p, args->event_type,
801 				args->auto_reset != 0, args->node_id,
802 				&args->event_id, &args->event_trigger_data,
803 				&args->event_page_offset,
804 				&args->event_slot_index);
805 
806 	return err;
807 }
808 
809 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p,
810 					void *data)
811 {
812 	struct kfd_ioctl_destroy_event_args *args = data;
813 
814 	return kfd_event_destroy(p, args->event_id);
815 }
816 
817 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p,
818 				void *data)
819 {
820 	struct kfd_ioctl_set_event_args *args = data;
821 
822 	return kfd_set_event(p, args->event_id);
823 }
824 
825 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p,
826 				void *data)
827 {
828 	struct kfd_ioctl_reset_event_args *args = data;
829 
830 	return kfd_reset_event(p, args->event_id);
831 }
832 
833 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p,
834 				void *data)
835 {
836 	struct kfd_ioctl_wait_events_args *args = data;
837 	enum kfd_event_wait_result wait_result;
838 	int err;
839 
840 	err = kfd_wait_on_events(p, args->num_events,
841 			(void __user *)args->events_ptr,
842 			(args->wait_for_all != 0),
843 			args->timeout, &wait_result);
844 
845 	args->wait_result = wait_result;
846 
847 	return err;
848 }
849 
850 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
851 	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl}
852 
853 /** Ioctl table */
854 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
855 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION,
856 			kfd_ioctl_get_version, 0),
857 
858 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE,
859 			kfd_ioctl_create_queue, 0),
860 
861 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE,
862 			kfd_ioctl_destroy_queue, 0),
863 
864 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY,
865 			kfd_ioctl_set_memory_policy, 0),
866 
867 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS,
868 			kfd_ioctl_get_clock_counters, 0),
869 
870 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES,
871 			kfd_ioctl_get_process_apertures, 0),
872 
873 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE,
874 			kfd_ioctl_update_queue, 0),
875 
876 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT,
877 			kfd_ioctl_create_event, 0),
878 
879 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT,
880 			kfd_ioctl_destroy_event, 0),
881 
882 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT,
883 			kfd_ioctl_set_event, 0),
884 
885 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT,
886 			kfd_ioctl_reset_event, 0),
887 
888 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS,
889 			kfd_ioctl_wait_events, 0),
890 
891 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER,
892 			kfd_ioctl_dbg_register, 0),
893 
894 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER,
895 			kfd_ioctl_dbg_unregister, 0),
896 
897 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH,
898 			kfd_ioctl_dbg_address_watch, 0),
899 
900 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL,
901 			kfd_ioctl_dbg_wave_control, 0),
902 };
903 
904 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
905 
906 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
907 {
908 	struct kfd_process *process;
909 	amdkfd_ioctl_t *func;
910 	const struct amdkfd_ioctl_desc *ioctl = NULL;
911 	unsigned int nr = _IOC_NR(cmd);
912 	char stack_kdata[128];
913 	char *kdata = NULL;
914 	unsigned int usize, asize;
915 	int retcode = -EINVAL;
916 
917 	if (nr >= AMDKFD_CORE_IOCTL_COUNT)
918 		goto err_i1;
919 
920 	if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) {
921 		u32 amdkfd_size;
922 
923 		ioctl = &amdkfd_ioctls[nr];
924 
925 		amdkfd_size = _IOC_SIZE(ioctl->cmd);
926 		usize = asize = _IOC_SIZE(cmd);
927 		if (amdkfd_size > asize)
928 			asize = amdkfd_size;
929 
930 		cmd = ioctl->cmd;
931 	} else
932 		goto err_i1;
933 
934 	dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg);
935 
936 	process = kfd_get_process(current);
937 	if (IS_ERR(process)) {
938 		dev_dbg(kfd_device, "no process\n");
939 		goto err_i1;
940 	}
941 
942 	/* Do not trust userspace, use our own definition */
943 	func = ioctl->func;
944 
945 	if (unlikely(!func)) {
946 		dev_dbg(kfd_device, "no function\n");
947 		retcode = -EINVAL;
948 		goto err_i1;
949 	}
950 
951 	if (cmd & (IOC_IN | IOC_OUT)) {
952 		if (asize <= sizeof(stack_kdata)) {
953 			kdata = stack_kdata;
954 		} else {
955 			kdata = kmalloc(asize, GFP_KERNEL);
956 			if (!kdata) {
957 				retcode = -ENOMEM;
958 				goto err_i1;
959 			}
960 		}
961 		if (asize > usize)
962 			memset(kdata + usize, 0, asize - usize);
963 	}
964 
965 	if (cmd & IOC_IN) {
966 		if (copy_from_user(kdata, (void __user *)arg, usize) != 0) {
967 			retcode = -EFAULT;
968 			goto err_i1;
969 		}
970 	} else if (cmd & IOC_OUT) {
971 		memset(kdata, 0, usize);
972 	}
973 
974 	retcode = func(filep, process, kdata);
975 
976 	if (cmd & IOC_OUT)
977 		if (copy_to_user((void __user *)arg, kdata, usize) != 0)
978 			retcode = -EFAULT;
979 
980 err_i1:
981 	if (!ioctl)
982 		dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n",
983 			  task_pid_nr(current), cmd, nr);
984 
985 	if (kdata != stack_kdata)
986 		kfree(kdata);
987 
988 	if (retcode)
989 		dev_dbg(kfd_device, "ret = %d\n", retcode);
990 
991 	return retcode;
992 }
993 
994 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma)
995 {
996 	struct kfd_process *process;
997 
998 	process = kfd_get_process(current);
999 	if (IS_ERR(process))
1000 		return PTR_ERR(process);
1001 
1002 	if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) ==
1003 			KFD_MMAP_DOORBELL_MASK) {
1004 		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK;
1005 		return kfd_doorbell_mmap(process, vma);
1006 	} else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) ==
1007 			KFD_MMAP_EVENTS_MASK) {
1008 		vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK;
1009 		return kfd_event_mmap(process, vma);
1010 	}
1011 
1012 	return -EFAULT;
1013 }
1014