1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/device.h> 24 #include <linux/export.h> 25 #include <linux/err.h> 26 #include <linux/fs.h> 27 #include <linux/sched.h> 28 #include <linux/slab.h> 29 #include <linux/uaccess.h> 30 #include <linux/compat.h> 31 #include <uapi/linux/kfd_ioctl.h> 32 #include <linux/time.h> 33 #include <linux/mm.h> 34 #include <linux/mman.h> 35 #include <asm/processor.h> 36 #include "kfd_priv.h" 37 #include "kfd_device_queue_manager.h" 38 #include "kfd_dbgmgr.h" 39 40 static long kfd_ioctl(struct file *, unsigned int, unsigned long); 41 static int kfd_open(struct inode *, struct file *); 42 static int kfd_mmap(struct file *, struct vm_area_struct *); 43 44 static const char kfd_dev_name[] = "kfd"; 45 46 static const struct file_operations kfd_fops = { 47 .owner = THIS_MODULE, 48 .unlocked_ioctl = kfd_ioctl, 49 .compat_ioctl = kfd_ioctl, 50 .open = kfd_open, 51 .mmap = kfd_mmap, 52 }; 53 54 static int kfd_char_dev_major = -1; 55 static struct class *kfd_class; 56 struct device *kfd_device; 57 58 int kfd_chardev_init(void) 59 { 60 int err = 0; 61 62 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); 63 err = kfd_char_dev_major; 64 if (err < 0) 65 goto err_register_chrdev; 66 67 kfd_class = class_create(THIS_MODULE, kfd_dev_name); 68 err = PTR_ERR(kfd_class); 69 if (IS_ERR(kfd_class)) 70 goto err_class_create; 71 72 kfd_device = device_create(kfd_class, NULL, 73 MKDEV(kfd_char_dev_major, 0), 74 NULL, kfd_dev_name); 75 err = PTR_ERR(kfd_device); 76 if (IS_ERR(kfd_device)) 77 goto err_device_create; 78 79 return 0; 80 81 err_device_create: 82 class_destroy(kfd_class); 83 err_class_create: 84 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 85 err_register_chrdev: 86 return err; 87 } 88 89 void kfd_chardev_exit(void) 90 { 91 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); 92 class_destroy(kfd_class); 93 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 94 } 95 96 struct device *kfd_chardev(void) 97 { 98 return kfd_device; 99 } 100 101 102 static int kfd_open(struct inode *inode, struct file *filep) 103 { 104 struct kfd_process *process; 105 bool is_32bit_user_mode; 106 107 if (iminor(inode) != 0) 108 return -ENODEV; 109 110 is_32bit_user_mode = in_compat_syscall(); 111 112 if (is_32bit_user_mode) { 113 dev_warn(kfd_device, 114 "Process %d (32-bit) failed to open /dev/kfd\n" 115 "32-bit processes are not supported by amdkfd\n", 116 current->pid); 117 return -EPERM; 118 } 119 120 process = kfd_create_process(current); 121 if (IS_ERR(process)) 122 return PTR_ERR(process); 123 124 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", 125 process->pasid, process->is_32bit_user_mode); 126 127 return 0; 128 } 129 130 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, 131 void *data) 132 { 133 struct kfd_ioctl_get_version_args *args = data; 134 135 args->major_version = KFD_IOCTL_MAJOR_VERSION; 136 args->minor_version = KFD_IOCTL_MINOR_VERSION; 137 138 return 0; 139 } 140 141 static int set_queue_properties_from_user(struct queue_properties *q_properties, 142 struct kfd_ioctl_create_queue_args *args) 143 { 144 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 145 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 146 return -EINVAL; 147 } 148 149 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 150 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 151 return -EINVAL; 152 } 153 154 if ((args->ring_base_address) && 155 (!access_ok(VERIFY_WRITE, 156 (const void __user *) args->ring_base_address, 157 sizeof(uint64_t)))) { 158 pr_err("kfd: can't access ring base address\n"); 159 return -EFAULT; 160 } 161 162 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 163 pr_err("kfd: ring size must be a power of 2 or 0\n"); 164 return -EINVAL; 165 } 166 167 if (!access_ok(VERIFY_WRITE, 168 (const void __user *) args->read_pointer_address, 169 sizeof(uint32_t))) { 170 pr_err("kfd: can't access read pointer\n"); 171 return -EFAULT; 172 } 173 174 if (!access_ok(VERIFY_WRITE, 175 (const void __user *) args->write_pointer_address, 176 sizeof(uint32_t))) { 177 pr_err("kfd: can't access write pointer\n"); 178 return -EFAULT; 179 } 180 181 if (args->eop_buffer_address && 182 !access_ok(VERIFY_WRITE, 183 (const void __user *) args->eop_buffer_address, 184 sizeof(uint32_t))) { 185 pr_debug("kfd: can't access eop buffer"); 186 return -EFAULT; 187 } 188 189 if (args->ctx_save_restore_address && 190 !access_ok(VERIFY_WRITE, 191 (const void __user *) args->ctx_save_restore_address, 192 sizeof(uint32_t))) { 193 pr_debug("kfd: can't access ctx save restore buffer"); 194 return -EFAULT; 195 } 196 197 q_properties->is_interop = false; 198 q_properties->queue_percent = args->queue_percentage; 199 q_properties->priority = args->queue_priority; 200 q_properties->queue_address = args->ring_base_address; 201 q_properties->queue_size = args->ring_size; 202 q_properties->read_ptr = (uint32_t *) args->read_pointer_address; 203 q_properties->write_ptr = (uint32_t *) args->write_pointer_address; 204 q_properties->eop_ring_buffer_address = args->eop_buffer_address; 205 q_properties->eop_ring_buffer_size = args->eop_buffer_size; 206 q_properties->ctx_save_restore_area_address = 207 args->ctx_save_restore_address; 208 q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size; 209 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || 210 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 211 q_properties->type = KFD_QUEUE_TYPE_COMPUTE; 212 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) 213 q_properties->type = KFD_QUEUE_TYPE_SDMA; 214 else 215 return -ENOTSUPP; 216 217 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 218 q_properties->format = KFD_QUEUE_FORMAT_AQL; 219 else 220 q_properties->format = KFD_QUEUE_FORMAT_PM4; 221 222 pr_debug("Queue Percentage (%d, %d)\n", 223 q_properties->queue_percent, args->queue_percentage); 224 225 pr_debug("Queue Priority (%d, %d)\n", 226 q_properties->priority, args->queue_priority); 227 228 pr_debug("Queue Address (0x%llX, 0x%llX)\n", 229 q_properties->queue_address, args->ring_base_address); 230 231 pr_debug("Queue Size (0x%llX, %u)\n", 232 q_properties->queue_size, args->ring_size); 233 234 pr_debug("Queue r/w Pointers (0x%llX, 0x%llX)\n", 235 (uint64_t) q_properties->read_ptr, 236 (uint64_t) q_properties->write_ptr); 237 238 pr_debug("Queue Format (%d)\n", q_properties->format); 239 240 pr_debug("Queue EOP (0x%llX)\n", q_properties->eop_ring_buffer_address); 241 242 pr_debug("Queue CTX save arex (0x%llX)\n", 243 q_properties->ctx_save_restore_area_address); 244 245 return 0; 246 } 247 248 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 249 void *data) 250 { 251 struct kfd_ioctl_create_queue_args *args = data; 252 struct kfd_dev *dev; 253 int err = 0; 254 unsigned int queue_id; 255 struct kfd_process_device *pdd; 256 struct queue_properties q_properties; 257 258 memset(&q_properties, 0, sizeof(struct queue_properties)); 259 260 pr_debug("kfd: creating queue ioctl\n"); 261 262 err = set_queue_properties_from_user(&q_properties, args); 263 if (err) 264 return err; 265 266 pr_debug("kfd: looking for gpu id 0x%x\n", args->gpu_id); 267 dev = kfd_device_by_id(args->gpu_id); 268 if (dev == NULL) { 269 pr_debug("kfd: gpu id 0x%x was not found\n", args->gpu_id); 270 return -EINVAL; 271 } 272 273 mutex_lock(&p->mutex); 274 275 pdd = kfd_bind_process_to_device(dev, p); 276 if (IS_ERR(pdd)) { 277 err = -ESRCH; 278 goto err_bind_process; 279 } 280 281 pr_debug("kfd: creating queue for PASID %d on GPU 0x%x\n", 282 p->pasid, 283 dev->id); 284 285 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 286 0, q_properties.type, &queue_id); 287 if (err != 0) 288 goto err_create_queue; 289 290 args->queue_id = queue_id; 291 292 293 /* Return gpu_id as doorbell offset for mmap usage */ 294 args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id); 295 args->doorbell_offset <<= PAGE_SHIFT; 296 297 mutex_unlock(&p->mutex); 298 299 pr_debug("kfd: queue id %d was created successfully\n", args->queue_id); 300 301 pr_debug("ring buffer address == 0x%016llX\n", 302 args->ring_base_address); 303 304 pr_debug("read ptr address == 0x%016llX\n", 305 args->read_pointer_address); 306 307 pr_debug("write ptr address == 0x%016llX\n", 308 args->write_pointer_address); 309 310 return 0; 311 312 err_create_queue: 313 err_bind_process: 314 mutex_unlock(&p->mutex); 315 return err; 316 } 317 318 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, 319 void *data) 320 { 321 int retval; 322 struct kfd_ioctl_destroy_queue_args *args = data; 323 324 pr_debug("kfd: destroying queue id %d for PASID %d\n", 325 args->queue_id, 326 p->pasid); 327 328 mutex_lock(&p->mutex); 329 330 retval = pqm_destroy_queue(&p->pqm, args->queue_id); 331 332 mutex_unlock(&p->mutex); 333 return retval; 334 } 335 336 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, 337 void *data) 338 { 339 int retval; 340 struct kfd_ioctl_update_queue_args *args = data; 341 struct queue_properties properties; 342 343 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 344 pr_err("kfd: queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 345 return -EINVAL; 346 } 347 348 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 349 pr_err("kfd: queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 350 return -EINVAL; 351 } 352 353 if ((args->ring_base_address) && 354 (!access_ok(VERIFY_WRITE, 355 (const void __user *) args->ring_base_address, 356 sizeof(uint64_t)))) { 357 pr_err("kfd: can't access ring base address\n"); 358 return -EFAULT; 359 } 360 361 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 362 pr_err("kfd: ring size must be a power of 2 or 0\n"); 363 return -EINVAL; 364 } 365 366 properties.queue_address = args->ring_base_address; 367 properties.queue_size = args->ring_size; 368 properties.queue_percent = args->queue_percentage; 369 properties.priority = args->queue_priority; 370 371 pr_debug("kfd: updating queue id %d for PASID %d\n", 372 args->queue_id, p->pasid); 373 374 mutex_lock(&p->mutex); 375 376 retval = pqm_update_queue(&p->pqm, args->queue_id, &properties); 377 378 mutex_unlock(&p->mutex); 379 380 return retval; 381 } 382 383 static int kfd_ioctl_set_memory_policy(struct file *filep, 384 struct kfd_process *p, void *data) 385 { 386 struct kfd_ioctl_set_memory_policy_args *args = data; 387 struct kfd_dev *dev; 388 int err = 0; 389 struct kfd_process_device *pdd; 390 enum cache_policy default_policy, alternate_policy; 391 392 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT 393 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 394 return -EINVAL; 395 } 396 397 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT 398 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 399 return -EINVAL; 400 } 401 402 dev = kfd_device_by_id(args->gpu_id); 403 if (dev == NULL) 404 return -EINVAL; 405 406 mutex_lock(&p->mutex); 407 408 pdd = kfd_bind_process_to_device(dev, p); 409 if (IS_ERR(pdd)) { 410 err = -ESRCH; 411 goto out; 412 } 413 414 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT) 415 ? cache_policy_coherent : cache_policy_noncoherent; 416 417 alternate_policy = 418 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) 419 ? cache_policy_coherent : cache_policy_noncoherent; 420 421 if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm, 422 &pdd->qpd, 423 default_policy, 424 alternate_policy, 425 (void __user *)args->alternate_aperture_base, 426 args->alternate_aperture_size)) 427 err = -EINVAL; 428 429 out: 430 mutex_unlock(&p->mutex); 431 432 return err; 433 } 434 435 static int kfd_ioctl_dbg_register(struct file *filep, 436 struct kfd_process *p, void *data) 437 { 438 struct kfd_ioctl_dbg_register_args *args = data; 439 struct kfd_dev *dev; 440 struct kfd_dbgmgr *dbgmgr_ptr; 441 struct kfd_process_device *pdd; 442 bool create_ok; 443 long status = 0; 444 445 dev = kfd_device_by_id(args->gpu_id); 446 if (dev == NULL) 447 return -EINVAL; 448 449 if (dev->device_info->asic_family == CHIP_CARRIZO) { 450 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n"); 451 return -EINVAL; 452 } 453 454 mutex_lock(kfd_get_dbgmgr_mutex()); 455 mutex_lock(&p->mutex); 456 457 /* 458 * make sure that we have pdd, if this the first queue created for 459 * this process 460 */ 461 pdd = kfd_bind_process_to_device(dev, p); 462 if (IS_ERR(pdd)) { 463 mutex_unlock(&p->mutex); 464 mutex_unlock(kfd_get_dbgmgr_mutex()); 465 return PTR_ERR(pdd); 466 } 467 468 if (dev->dbgmgr == NULL) { 469 /* In case of a legal call, we have no dbgmgr yet */ 470 create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev); 471 if (create_ok) { 472 status = kfd_dbgmgr_register(dbgmgr_ptr, p); 473 if (status != 0) 474 kfd_dbgmgr_destroy(dbgmgr_ptr); 475 else 476 dev->dbgmgr = dbgmgr_ptr; 477 } 478 } else { 479 pr_debug("debugger already registered\n"); 480 status = -EINVAL; 481 } 482 483 mutex_unlock(&p->mutex); 484 mutex_unlock(kfd_get_dbgmgr_mutex()); 485 486 return status; 487 } 488 489 static int kfd_ioctl_dbg_unregister(struct file *filep, 490 struct kfd_process *p, void *data) 491 { 492 struct kfd_ioctl_dbg_unregister_args *args = data; 493 struct kfd_dev *dev; 494 long status; 495 496 dev = kfd_device_by_id(args->gpu_id); 497 if (dev == NULL) 498 return -EINVAL; 499 500 if (dev->device_info->asic_family == CHIP_CARRIZO) { 501 pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n"); 502 return -EINVAL; 503 } 504 505 mutex_lock(kfd_get_dbgmgr_mutex()); 506 507 status = kfd_dbgmgr_unregister(dev->dbgmgr, p); 508 if (status == 0) { 509 kfd_dbgmgr_destroy(dev->dbgmgr); 510 dev->dbgmgr = NULL; 511 } 512 513 mutex_unlock(kfd_get_dbgmgr_mutex()); 514 515 return status; 516 } 517 518 /* 519 * Parse and generate variable size data structure for address watch. 520 * Total size of the buffer and # watch points is limited in order 521 * to prevent kernel abuse. (no bearing to the much smaller HW limitation 522 * which is enforced by dbgdev module) 523 * please also note that the watch address itself are not "copied from user", 524 * since it be set into the HW in user mode values. 525 * 526 */ 527 static int kfd_ioctl_dbg_address_watch(struct file *filep, 528 struct kfd_process *p, void *data) 529 { 530 struct kfd_ioctl_dbg_address_watch_args *args = data; 531 struct kfd_dev *dev; 532 struct dbg_address_watch_info aw_info; 533 unsigned char *args_buff; 534 long status; 535 void __user *cmd_from_user; 536 uint64_t watch_mask_value = 0; 537 unsigned int args_idx = 0; 538 539 memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info)); 540 541 dev = kfd_device_by_id(args->gpu_id); 542 if (dev == NULL) 543 return -EINVAL; 544 545 if (dev->device_info->asic_family == CHIP_CARRIZO) { 546 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 547 return -EINVAL; 548 } 549 550 cmd_from_user = (void __user *) args->content_ptr; 551 552 /* Validate arguments */ 553 554 if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) || 555 (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) || 556 (cmd_from_user == NULL)) 557 return -EINVAL; 558 559 /* this is the actual buffer to work with */ 560 args_buff = memdup_user(cmd_from_user, 561 args->buf_size_in_bytes - sizeof(*args)); 562 if (IS_ERR(args_buff)) 563 return PTR_ERR(args_buff); 564 565 aw_info.process = p; 566 567 aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx])); 568 args_idx += sizeof(aw_info.num_watch_points); 569 570 aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx]; 571 args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points; 572 573 /* 574 * set watch address base pointer to point on the array base 575 * within args_buff 576 */ 577 aw_info.watch_address = (uint64_t *) &args_buff[args_idx]; 578 579 /* skip over the addresses buffer */ 580 args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points; 581 582 if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) { 583 kfree(args_buff); 584 return -EINVAL; 585 } 586 587 watch_mask_value = (uint64_t) args_buff[args_idx]; 588 589 if (watch_mask_value > 0) { 590 /* 591 * There is an array of masks. 592 * set watch mask base pointer to point on the array base 593 * within args_buff 594 */ 595 aw_info.watch_mask = (uint64_t *) &args_buff[args_idx]; 596 597 /* skip over the masks buffer */ 598 args_idx += sizeof(aw_info.watch_mask) * 599 aw_info.num_watch_points; 600 } else { 601 /* just the NULL mask, set to NULL and skip over it */ 602 aw_info.watch_mask = NULL; 603 args_idx += sizeof(aw_info.watch_mask); 604 } 605 606 if (args_idx >= args->buf_size_in_bytes - sizeof(args)) { 607 kfree(args_buff); 608 return -EINVAL; 609 } 610 611 /* Currently HSA Event is not supported for DBG */ 612 aw_info.watch_event = NULL; 613 614 mutex_lock(kfd_get_dbgmgr_mutex()); 615 616 status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info); 617 618 mutex_unlock(kfd_get_dbgmgr_mutex()); 619 620 kfree(args_buff); 621 622 return status; 623 } 624 625 /* Parse and generate fixed size data structure for wave control */ 626 static int kfd_ioctl_dbg_wave_control(struct file *filep, 627 struct kfd_process *p, void *data) 628 { 629 struct kfd_ioctl_dbg_wave_control_args *args = data; 630 struct kfd_dev *dev; 631 struct dbg_wave_control_info wac_info; 632 unsigned char *args_buff; 633 uint32_t computed_buff_size; 634 long status; 635 void __user *cmd_from_user; 636 unsigned int args_idx = 0; 637 638 memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info)); 639 640 /* we use compact form, independent of the packing attribute value */ 641 computed_buff_size = sizeof(*args) + 642 sizeof(wac_info.mode) + 643 sizeof(wac_info.operand) + 644 sizeof(wac_info.dbgWave_msg.DbgWaveMsg) + 645 sizeof(wac_info.dbgWave_msg.MemoryVA) + 646 sizeof(wac_info.trapId); 647 648 dev = kfd_device_by_id(args->gpu_id); 649 if (dev == NULL) 650 return -EINVAL; 651 652 if (dev->device_info->asic_family == CHIP_CARRIZO) { 653 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 654 return -EINVAL; 655 } 656 657 /* input size must match the computed "compact" size */ 658 if (args->buf_size_in_bytes != computed_buff_size) { 659 pr_debug("size mismatch, computed : actual %u : %u\n", 660 args->buf_size_in_bytes, computed_buff_size); 661 return -EINVAL; 662 } 663 664 cmd_from_user = (void __user *) args->content_ptr; 665 666 if (cmd_from_user == NULL) 667 return -EINVAL; 668 669 /* copy the entire buffer from user */ 670 671 args_buff = memdup_user(cmd_from_user, 672 args->buf_size_in_bytes - sizeof(*args)); 673 if (IS_ERR(args_buff)) 674 return PTR_ERR(args_buff); 675 676 /* move ptr to the start of the "pay-load" area */ 677 wac_info.process = p; 678 679 wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx])); 680 args_idx += sizeof(wac_info.operand); 681 682 wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx])); 683 args_idx += sizeof(wac_info.mode); 684 685 wac_info.trapId = *((uint32_t *)(&args_buff[args_idx])); 686 args_idx += sizeof(wac_info.trapId); 687 688 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = 689 *((uint32_t *)(&args_buff[args_idx])); 690 wac_info.dbgWave_msg.MemoryVA = NULL; 691 692 mutex_lock(kfd_get_dbgmgr_mutex()); 693 694 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n", 695 wac_info.process, wac_info.operand, 696 wac_info.mode, wac_info.trapId, 697 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value); 698 699 status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info); 700 701 pr_debug("Returned status of dbg manager is %ld\n", status); 702 703 mutex_unlock(kfd_get_dbgmgr_mutex()); 704 705 kfree(args_buff); 706 707 return status; 708 } 709 710 static int kfd_ioctl_get_clock_counters(struct file *filep, 711 struct kfd_process *p, void *data) 712 { 713 struct kfd_ioctl_get_clock_counters_args *args = data; 714 struct kfd_dev *dev; 715 struct timespec64 time; 716 717 dev = kfd_device_by_id(args->gpu_id); 718 if (dev == NULL) 719 return -EINVAL; 720 721 /* Reading GPU clock counter from KGD */ 722 args->gpu_clock_counter = 723 dev->kfd2kgd->get_gpu_clock_counter(dev->kgd); 724 725 /* No access to rdtsc. Using raw monotonic time */ 726 getrawmonotonic64(&time); 727 args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time); 728 729 get_monotonic_boottime64(&time); 730 args->system_clock_counter = (uint64_t)timespec64_to_ns(&time); 731 732 /* Since the counter is in nano-seconds we use 1GHz frequency */ 733 args->system_clock_freq = 1000000000; 734 735 return 0; 736 } 737 738 739 static int kfd_ioctl_get_process_apertures(struct file *filp, 740 struct kfd_process *p, void *data) 741 { 742 struct kfd_ioctl_get_process_apertures_args *args = data; 743 struct kfd_process_device_apertures *pAperture; 744 struct kfd_process_device *pdd; 745 746 dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); 747 748 args->num_of_nodes = 0; 749 750 mutex_lock(&p->mutex); 751 752 /*if the process-device list isn't empty*/ 753 if (kfd_has_process_device_data(p)) { 754 /* Run over all pdd of the process */ 755 pdd = kfd_get_first_process_device_data(p); 756 do { 757 pAperture = 758 &args->process_apertures[args->num_of_nodes]; 759 pAperture->gpu_id = pdd->dev->id; 760 pAperture->lds_base = pdd->lds_base; 761 pAperture->lds_limit = pdd->lds_limit; 762 pAperture->gpuvm_base = pdd->gpuvm_base; 763 pAperture->gpuvm_limit = pdd->gpuvm_limit; 764 pAperture->scratch_base = pdd->scratch_base; 765 pAperture->scratch_limit = pdd->scratch_limit; 766 767 dev_dbg(kfd_device, 768 "node id %u\n", args->num_of_nodes); 769 dev_dbg(kfd_device, 770 "gpu id %u\n", pdd->dev->id); 771 dev_dbg(kfd_device, 772 "lds_base %llX\n", pdd->lds_base); 773 dev_dbg(kfd_device, 774 "lds_limit %llX\n", pdd->lds_limit); 775 dev_dbg(kfd_device, 776 "gpuvm_base %llX\n", pdd->gpuvm_base); 777 dev_dbg(kfd_device, 778 "gpuvm_limit %llX\n", pdd->gpuvm_limit); 779 dev_dbg(kfd_device, 780 "scratch_base %llX\n", pdd->scratch_base); 781 dev_dbg(kfd_device, 782 "scratch_limit %llX\n", pdd->scratch_limit); 783 784 args->num_of_nodes++; 785 } while ((pdd = kfd_get_next_process_device_data(p, pdd)) != NULL && 786 (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); 787 } 788 789 mutex_unlock(&p->mutex); 790 791 return 0; 792 } 793 794 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p, 795 void *data) 796 { 797 struct kfd_ioctl_create_event_args *args = data; 798 int err; 799 800 err = kfd_event_create(filp, p, args->event_type, 801 args->auto_reset != 0, args->node_id, 802 &args->event_id, &args->event_trigger_data, 803 &args->event_page_offset, 804 &args->event_slot_index); 805 806 return err; 807 } 808 809 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p, 810 void *data) 811 { 812 struct kfd_ioctl_destroy_event_args *args = data; 813 814 return kfd_event_destroy(p, args->event_id); 815 } 816 817 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p, 818 void *data) 819 { 820 struct kfd_ioctl_set_event_args *args = data; 821 822 return kfd_set_event(p, args->event_id); 823 } 824 825 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p, 826 void *data) 827 { 828 struct kfd_ioctl_reset_event_args *args = data; 829 830 return kfd_reset_event(p, args->event_id); 831 } 832 833 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, 834 void *data) 835 { 836 struct kfd_ioctl_wait_events_args *args = data; 837 enum kfd_event_wait_result wait_result; 838 int err; 839 840 err = kfd_wait_on_events(p, args->num_events, 841 (void __user *)args->events_ptr, 842 (args->wait_for_all != 0), 843 args->timeout, &wait_result); 844 845 args->wait_result = wait_result; 846 847 return err; 848 } 849 850 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ 851 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, .cmd_drv = 0, .name = #ioctl} 852 853 /** Ioctl table */ 854 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { 855 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION, 856 kfd_ioctl_get_version, 0), 857 858 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE, 859 kfd_ioctl_create_queue, 0), 860 861 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE, 862 kfd_ioctl_destroy_queue, 0), 863 864 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY, 865 kfd_ioctl_set_memory_policy, 0), 866 867 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS, 868 kfd_ioctl_get_clock_counters, 0), 869 870 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES, 871 kfd_ioctl_get_process_apertures, 0), 872 873 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE, 874 kfd_ioctl_update_queue, 0), 875 876 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT, 877 kfd_ioctl_create_event, 0), 878 879 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT, 880 kfd_ioctl_destroy_event, 0), 881 882 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT, 883 kfd_ioctl_set_event, 0), 884 885 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT, 886 kfd_ioctl_reset_event, 0), 887 888 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS, 889 kfd_ioctl_wait_events, 0), 890 891 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER, 892 kfd_ioctl_dbg_register, 0), 893 894 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER, 895 kfd_ioctl_dbg_unregister, 0), 896 897 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH, 898 kfd_ioctl_dbg_address_watch, 0), 899 900 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL, 901 kfd_ioctl_dbg_wave_control, 0), 902 }; 903 904 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) 905 906 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 907 { 908 struct kfd_process *process; 909 amdkfd_ioctl_t *func; 910 const struct amdkfd_ioctl_desc *ioctl = NULL; 911 unsigned int nr = _IOC_NR(cmd); 912 char stack_kdata[128]; 913 char *kdata = NULL; 914 unsigned int usize, asize; 915 int retcode = -EINVAL; 916 917 if (nr >= AMDKFD_CORE_IOCTL_COUNT) 918 goto err_i1; 919 920 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { 921 u32 amdkfd_size; 922 923 ioctl = &amdkfd_ioctls[nr]; 924 925 amdkfd_size = _IOC_SIZE(ioctl->cmd); 926 usize = asize = _IOC_SIZE(cmd); 927 if (amdkfd_size > asize) 928 asize = amdkfd_size; 929 930 cmd = ioctl->cmd; 931 } else 932 goto err_i1; 933 934 dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg); 935 936 process = kfd_get_process(current); 937 if (IS_ERR(process)) { 938 dev_dbg(kfd_device, "no process\n"); 939 goto err_i1; 940 } 941 942 /* Do not trust userspace, use our own definition */ 943 func = ioctl->func; 944 945 if (unlikely(!func)) { 946 dev_dbg(kfd_device, "no function\n"); 947 retcode = -EINVAL; 948 goto err_i1; 949 } 950 951 if (cmd & (IOC_IN | IOC_OUT)) { 952 if (asize <= sizeof(stack_kdata)) { 953 kdata = stack_kdata; 954 } else { 955 kdata = kmalloc(asize, GFP_KERNEL); 956 if (!kdata) { 957 retcode = -ENOMEM; 958 goto err_i1; 959 } 960 } 961 if (asize > usize) 962 memset(kdata + usize, 0, asize - usize); 963 } 964 965 if (cmd & IOC_IN) { 966 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { 967 retcode = -EFAULT; 968 goto err_i1; 969 } 970 } else if (cmd & IOC_OUT) { 971 memset(kdata, 0, usize); 972 } 973 974 retcode = func(filep, process, kdata); 975 976 if (cmd & IOC_OUT) 977 if (copy_to_user((void __user *)arg, kdata, usize) != 0) 978 retcode = -EFAULT; 979 980 err_i1: 981 if (!ioctl) 982 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", 983 task_pid_nr(current), cmd, nr); 984 985 if (kdata != stack_kdata) 986 kfree(kdata); 987 988 if (retcode) 989 dev_dbg(kfd_device, "ret = %d\n", retcode); 990 991 return retcode; 992 } 993 994 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) 995 { 996 struct kfd_process *process; 997 998 process = kfd_get_process(current); 999 if (IS_ERR(process)) 1000 return PTR_ERR(process); 1001 1002 if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) == 1003 KFD_MMAP_DOORBELL_MASK) { 1004 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK; 1005 return kfd_doorbell_mmap(process, vma); 1006 } else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) == 1007 KFD_MMAP_EVENTS_MASK) { 1008 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK; 1009 return kfd_event_mmap(process, vma); 1010 } 1011 1012 return -EFAULT; 1013 } 1014