1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/device.h> 24 #include <linux/export.h> 25 #include <linux/err.h> 26 #include <linux/fs.h> 27 #include <linux/sched.h> 28 #include <linux/slab.h> 29 #include <linux/uaccess.h> 30 #include <linux/compat.h> 31 #include <uapi/linux/kfd_ioctl.h> 32 #include <linux/time.h> 33 #include <linux/mm.h> 34 #include <linux/mman.h> 35 #include <asm/processor.h> 36 #include "kfd_priv.h" 37 #include "kfd_device_queue_manager.h" 38 #include "kfd_dbgmgr.h" 39 40 static long kfd_ioctl(struct file *, unsigned int, unsigned long); 41 static int kfd_open(struct inode *, struct file *); 42 static int kfd_mmap(struct file *, struct vm_area_struct *); 43 44 static const char kfd_dev_name[] = "kfd"; 45 46 static const struct file_operations kfd_fops = { 47 .owner = THIS_MODULE, 48 .unlocked_ioctl = kfd_ioctl, 49 .compat_ioctl = kfd_ioctl, 50 .open = kfd_open, 51 .mmap = kfd_mmap, 52 }; 53 54 static int kfd_char_dev_major = -1; 55 static struct class *kfd_class; 56 struct device *kfd_device; 57 58 int kfd_chardev_init(void) 59 { 60 int err = 0; 61 62 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); 63 err = kfd_char_dev_major; 64 if (err < 0) 65 goto err_register_chrdev; 66 67 kfd_class = class_create(THIS_MODULE, kfd_dev_name); 68 err = PTR_ERR(kfd_class); 69 if (IS_ERR(kfd_class)) 70 goto err_class_create; 71 72 kfd_device = device_create(kfd_class, NULL, 73 MKDEV(kfd_char_dev_major, 0), 74 NULL, kfd_dev_name); 75 err = PTR_ERR(kfd_device); 76 if (IS_ERR(kfd_device)) 77 goto err_device_create; 78 79 return 0; 80 81 err_device_create: 82 class_destroy(kfd_class); 83 err_class_create: 84 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 85 err_register_chrdev: 86 return err; 87 } 88 89 void kfd_chardev_exit(void) 90 { 91 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); 92 class_destroy(kfd_class); 93 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 94 } 95 96 struct device *kfd_chardev(void) 97 { 98 return kfd_device; 99 } 100 101 102 static int kfd_open(struct inode *inode, struct file *filep) 103 { 104 struct kfd_process *process; 105 bool is_32bit_user_mode; 106 107 if (iminor(inode) != 0) 108 return -ENODEV; 109 110 is_32bit_user_mode = in_compat_syscall(); 111 112 if (is_32bit_user_mode) { 113 dev_warn(kfd_device, 114 "Process %d (32-bit) failed to open /dev/kfd\n" 115 "32-bit processes are not supported by amdkfd\n", 116 current->pid); 117 return -EPERM; 118 } 119 120 process = kfd_create_process(current); 121 if (IS_ERR(process)) 122 return PTR_ERR(process); 123 124 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", 125 process->pasid, process->is_32bit_user_mode); 126 127 return 0; 128 } 129 130 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, 131 void *data) 132 { 133 struct kfd_ioctl_get_version_args *args = data; 134 135 args->major_version = KFD_IOCTL_MAJOR_VERSION; 136 args->minor_version = KFD_IOCTL_MINOR_VERSION; 137 138 return 0; 139 } 140 141 static int set_queue_properties_from_user(struct queue_properties *q_properties, 142 struct kfd_ioctl_create_queue_args *args) 143 { 144 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 145 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 146 return -EINVAL; 147 } 148 149 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 150 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 151 return -EINVAL; 152 } 153 154 if ((args->ring_base_address) && 155 (!access_ok(VERIFY_WRITE, 156 (const void __user *) args->ring_base_address, 157 sizeof(uint64_t)))) { 158 pr_err("Can't access ring base address\n"); 159 return -EFAULT; 160 } 161 162 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 163 pr_err("Ring size must be a power of 2 or 0\n"); 164 return -EINVAL; 165 } 166 167 if (!access_ok(VERIFY_WRITE, 168 (const void __user *) args->read_pointer_address, 169 sizeof(uint32_t))) { 170 pr_err("Can't access read pointer\n"); 171 return -EFAULT; 172 } 173 174 if (!access_ok(VERIFY_WRITE, 175 (const void __user *) args->write_pointer_address, 176 sizeof(uint32_t))) { 177 pr_err("Can't access write pointer\n"); 178 return -EFAULT; 179 } 180 181 if (args->eop_buffer_address && 182 !access_ok(VERIFY_WRITE, 183 (const void __user *) args->eop_buffer_address, 184 sizeof(uint32_t))) { 185 pr_debug("Can't access eop buffer"); 186 return -EFAULT; 187 } 188 189 if (args->ctx_save_restore_address && 190 !access_ok(VERIFY_WRITE, 191 (const void __user *) args->ctx_save_restore_address, 192 sizeof(uint32_t))) { 193 pr_debug("Can't access ctx save restore buffer"); 194 return -EFAULT; 195 } 196 197 q_properties->is_interop = false; 198 q_properties->queue_percent = args->queue_percentage; 199 q_properties->priority = args->queue_priority; 200 q_properties->queue_address = args->ring_base_address; 201 q_properties->queue_size = args->ring_size; 202 q_properties->read_ptr = (uint32_t *) args->read_pointer_address; 203 q_properties->write_ptr = (uint32_t *) args->write_pointer_address; 204 q_properties->eop_ring_buffer_address = args->eop_buffer_address; 205 q_properties->eop_ring_buffer_size = args->eop_buffer_size; 206 q_properties->ctx_save_restore_area_address = 207 args->ctx_save_restore_address; 208 q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size; 209 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || 210 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 211 q_properties->type = KFD_QUEUE_TYPE_COMPUTE; 212 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) 213 q_properties->type = KFD_QUEUE_TYPE_SDMA; 214 else 215 return -ENOTSUPP; 216 217 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 218 q_properties->format = KFD_QUEUE_FORMAT_AQL; 219 else 220 q_properties->format = KFD_QUEUE_FORMAT_PM4; 221 222 pr_debug("Queue Percentage: %d, %d\n", 223 q_properties->queue_percent, args->queue_percentage); 224 225 pr_debug("Queue Priority: %d, %d\n", 226 q_properties->priority, args->queue_priority); 227 228 pr_debug("Queue Address: 0x%llX, 0x%llX\n", 229 q_properties->queue_address, args->ring_base_address); 230 231 pr_debug("Queue Size: 0x%llX, %u\n", 232 q_properties->queue_size, args->ring_size); 233 234 pr_debug("Queue r/w Pointers: %p, %p\n", 235 q_properties->read_ptr, 236 q_properties->write_ptr); 237 238 pr_debug("Queue Format: %d\n", q_properties->format); 239 240 pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address); 241 242 pr_debug("Queue CTX save area: 0x%llX\n", 243 q_properties->ctx_save_restore_area_address); 244 245 return 0; 246 } 247 248 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 249 void *data) 250 { 251 struct kfd_ioctl_create_queue_args *args = data; 252 struct kfd_dev *dev; 253 int err = 0; 254 unsigned int queue_id; 255 struct kfd_process_device *pdd; 256 struct queue_properties q_properties; 257 258 memset(&q_properties, 0, sizeof(struct queue_properties)); 259 260 pr_debug("Creating queue ioctl\n"); 261 262 err = set_queue_properties_from_user(&q_properties, args); 263 if (err) 264 return err; 265 266 pr_debug("Looking for gpu id 0x%x\n", args->gpu_id); 267 dev = kfd_device_by_id(args->gpu_id); 268 if (!dev) { 269 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id); 270 return -EINVAL; 271 } 272 273 mutex_lock(&p->mutex); 274 275 pdd = kfd_bind_process_to_device(dev, p); 276 if (IS_ERR(pdd)) { 277 err = -ESRCH; 278 goto err_bind_process; 279 } 280 281 pr_debug("Creating queue for PASID %d on gpu 0x%x\n", 282 p->pasid, 283 dev->id); 284 285 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, 286 0, q_properties.type, &queue_id); 287 if (err != 0) 288 goto err_create_queue; 289 290 args->queue_id = queue_id; 291 292 293 /* Return gpu_id as doorbell offset for mmap usage */ 294 args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id); 295 args->doorbell_offset <<= PAGE_SHIFT; 296 297 mutex_unlock(&p->mutex); 298 299 pr_debug("Queue id %d was created successfully\n", args->queue_id); 300 301 pr_debug("Ring buffer address == 0x%016llX\n", 302 args->ring_base_address); 303 304 pr_debug("Read ptr address == 0x%016llX\n", 305 args->read_pointer_address); 306 307 pr_debug("Write ptr address == 0x%016llX\n", 308 args->write_pointer_address); 309 310 return 0; 311 312 err_create_queue: 313 err_bind_process: 314 mutex_unlock(&p->mutex); 315 return err; 316 } 317 318 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, 319 void *data) 320 { 321 int retval; 322 struct kfd_ioctl_destroy_queue_args *args = data; 323 324 pr_debug("Destroying queue id %d for pasid %d\n", 325 args->queue_id, 326 p->pasid); 327 328 mutex_lock(&p->mutex); 329 330 retval = pqm_destroy_queue(&p->pqm, args->queue_id); 331 332 mutex_unlock(&p->mutex); 333 return retval; 334 } 335 336 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, 337 void *data) 338 { 339 int retval; 340 struct kfd_ioctl_update_queue_args *args = data; 341 struct queue_properties properties; 342 343 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 344 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 345 return -EINVAL; 346 } 347 348 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 349 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 350 return -EINVAL; 351 } 352 353 if ((args->ring_base_address) && 354 (!access_ok(VERIFY_WRITE, 355 (const void __user *) args->ring_base_address, 356 sizeof(uint64_t)))) { 357 pr_err("Can't access ring base address\n"); 358 return -EFAULT; 359 } 360 361 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 362 pr_err("Ring size must be a power of 2 or 0\n"); 363 return -EINVAL; 364 } 365 366 properties.queue_address = args->ring_base_address; 367 properties.queue_size = args->ring_size; 368 properties.queue_percent = args->queue_percentage; 369 properties.priority = args->queue_priority; 370 371 pr_debug("Updating queue id %d for pasid %d\n", 372 args->queue_id, p->pasid); 373 374 mutex_lock(&p->mutex); 375 376 retval = pqm_update_queue(&p->pqm, args->queue_id, &properties); 377 378 mutex_unlock(&p->mutex); 379 380 return retval; 381 } 382 383 static int kfd_ioctl_set_memory_policy(struct file *filep, 384 struct kfd_process *p, void *data) 385 { 386 struct kfd_ioctl_set_memory_policy_args *args = data; 387 struct kfd_dev *dev; 388 int err = 0; 389 struct kfd_process_device *pdd; 390 enum cache_policy default_policy, alternate_policy; 391 392 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT 393 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 394 return -EINVAL; 395 } 396 397 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT 398 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 399 return -EINVAL; 400 } 401 402 dev = kfd_device_by_id(args->gpu_id); 403 if (!dev) 404 return -EINVAL; 405 406 mutex_lock(&p->mutex); 407 408 pdd = kfd_bind_process_to_device(dev, p); 409 if (IS_ERR(pdd)) { 410 err = -ESRCH; 411 goto out; 412 } 413 414 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT) 415 ? cache_policy_coherent : cache_policy_noncoherent; 416 417 alternate_policy = 418 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) 419 ? cache_policy_coherent : cache_policy_noncoherent; 420 421 if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm, 422 &pdd->qpd, 423 default_policy, 424 alternate_policy, 425 (void __user *)args->alternate_aperture_base, 426 args->alternate_aperture_size)) 427 err = -EINVAL; 428 429 out: 430 mutex_unlock(&p->mutex); 431 432 return err; 433 } 434 435 static int kfd_ioctl_dbg_register(struct file *filep, 436 struct kfd_process *p, void *data) 437 { 438 struct kfd_ioctl_dbg_register_args *args = data; 439 struct kfd_dev *dev; 440 struct kfd_dbgmgr *dbgmgr_ptr; 441 struct kfd_process_device *pdd; 442 bool create_ok; 443 long status = 0; 444 445 dev = kfd_device_by_id(args->gpu_id); 446 if (!dev) 447 return -EINVAL; 448 449 if (dev->device_info->asic_family == CHIP_CARRIZO) { 450 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n"); 451 return -EINVAL; 452 } 453 454 mutex_lock(kfd_get_dbgmgr_mutex()); 455 mutex_lock(&p->mutex); 456 457 /* 458 * make sure that we have pdd, if this the first queue created for 459 * this process 460 */ 461 pdd = kfd_bind_process_to_device(dev, p); 462 if (IS_ERR(pdd)) { 463 status = PTR_ERR(pdd); 464 goto out; 465 } 466 467 if (!dev->dbgmgr) { 468 /* In case of a legal call, we have no dbgmgr yet */ 469 create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev); 470 if (create_ok) { 471 status = kfd_dbgmgr_register(dbgmgr_ptr, p); 472 if (status != 0) 473 kfd_dbgmgr_destroy(dbgmgr_ptr); 474 else 475 dev->dbgmgr = dbgmgr_ptr; 476 } 477 } else { 478 pr_debug("debugger already registered\n"); 479 status = -EINVAL; 480 } 481 482 out: 483 mutex_unlock(&p->mutex); 484 mutex_unlock(kfd_get_dbgmgr_mutex()); 485 486 return status; 487 } 488 489 static int kfd_ioctl_dbg_unregister(struct file *filep, 490 struct kfd_process *p, void *data) 491 { 492 struct kfd_ioctl_dbg_unregister_args *args = data; 493 struct kfd_dev *dev; 494 long status; 495 496 dev = kfd_device_by_id(args->gpu_id); 497 if (!dev) 498 return -EINVAL; 499 500 if (dev->device_info->asic_family == CHIP_CARRIZO) { 501 pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n"); 502 return -EINVAL; 503 } 504 505 mutex_lock(kfd_get_dbgmgr_mutex()); 506 507 status = kfd_dbgmgr_unregister(dev->dbgmgr, p); 508 if (!status) { 509 kfd_dbgmgr_destroy(dev->dbgmgr); 510 dev->dbgmgr = NULL; 511 } 512 513 mutex_unlock(kfd_get_dbgmgr_mutex()); 514 515 return status; 516 } 517 518 /* 519 * Parse and generate variable size data structure for address watch. 520 * Total size of the buffer and # watch points is limited in order 521 * to prevent kernel abuse. (no bearing to the much smaller HW limitation 522 * which is enforced by dbgdev module) 523 * please also note that the watch address itself are not "copied from user", 524 * since it be set into the HW in user mode values. 525 * 526 */ 527 static int kfd_ioctl_dbg_address_watch(struct file *filep, 528 struct kfd_process *p, void *data) 529 { 530 struct kfd_ioctl_dbg_address_watch_args *args = data; 531 struct kfd_dev *dev; 532 struct dbg_address_watch_info aw_info; 533 unsigned char *args_buff; 534 long status; 535 void __user *cmd_from_user; 536 uint64_t watch_mask_value = 0; 537 unsigned int args_idx = 0; 538 539 memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info)); 540 541 dev = kfd_device_by_id(args->gpu_id); 542 if (!dev) 543 return -EINVAL; 544 545 if (dev->device_info->asic_family == CHIP_CARRIZO) { 546 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 547 return -EINVAL; 548 } 549 550 cmd_from_user = (void __user *) args->content_ptr; 551 552 /* Validate arguments */ 553 554 if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) || 555 (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) || 556 (cmd_from_user == NULL)) 557 return -EINVAL; 558 559 /* this is the actual buffer to work with */ 560 args_buff = memdup_user(cmd_from_user, 561 args->buf_size_in_bytes - sizeof(*args)); 562 if (IS_ERR(args_buff)) 563 return PTR_ERR(args_buff); 564 565 aw_info.process = p; 566 567 aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx])); 568 args_idx += sizeof(aw_info.num_watch_points); 569 570 aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx]; 571 args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points; 572 573 /* 574 * set watch address base pointer to point on the array base 575 * within args_buff 576 */ 577 aw_info.watch_address = (uint64_t *) &args_buff[args_idx]; 578 579 /* skip over the addresses buffer */ 580 args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points; 581 582 if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) { 583 status = -EINVAL; 584 goto out; 585 } 586 587 watch_mask_value = (uint64_t) args_buff[args_idx]; 588 589 if (watch_mask_value > 0) { 590 /* 591 * There is an array of masks. 592 * set watch mask base pointer to point on the array base 593 * within args_buff 594 */ 595 aw_info.watch_mask = (uint64_t *) &args_buff[args_idx]; 596 597 /* skip over the masks buffer */ 598 args_idx += sizeof(aw_info.watch_mask) * 599 aw_info.num_watch_points; 600 } else { 601 /* just the NULL mask, set to NULL and skip over it */ 602 aw_info.watch_mask = NULL; 603 args_idx += sizeof(aw_info.watch_mask); 604 } 605 606 if (args_idx >= args->buf_size_in_bytes - sizeof(args)) { 607 status = -EINVAL; 608 goto out; 609 } 610 611 /* Currently HSA Event is not supported for DBG */ 612 aw_info.watch_event = NULL; 613 614 mutex_lock(kfd_get_dbgmgr_mutex()); 615 616 status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info); 617 618 mutex_unlock(kfd_get_dbgmgr_mutex()); 619 620 out: 621 kfree(args_buff); 622 623 return status; 624 } 625 626 /* Parse and generate fixed size data structure for wave control */ 627 static int kfd_ioctl_dbg_wave_control(struct file *filep, 628 struct kfd_process *p, void *data) 629 { 630 struct kfd_ioctl_dbg_wave_control_args *args = data; 631 struct kfd_dev *dev; 632 struct dbg_wave_control_info wac_info; 633 unsigned char *args_buff; 634 uint32_t computed_buff_size; 635 long status; 636 void __user *cmd_from_user; 637 unsigned int args_idx = 0; 638 639 memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info)); 640 641 /* we use compact form, independent of the packing attribute value */ 642 computed_buff_size = sizeof(*args) + 643 sizeof(wac_info.mode) + 644 sizeof(wac_info.operand) + 645 sizeof(wac_info.dbgWave_msg.DbgWaveMsg) + 646 sizeof(wac_info.dbgWave_msg.MemoryVA) + 647 sizeof(wac_info.trapId); 648 649 dev = kfd_device_by_id(args->gpu_id); 650 if (!dev) 651 return -EINVAL; 652 653 if (dev->device_info->asic_family == CHIP_CARRIZO) { 654 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 655 return -EINVAL; 656 } 657 658 /* input size must match the computed "compact" size */ 659 if (args->buf_size_in_bytes != computed_buff_size) { 660 pr_debug("size mismatch, computed : actual %u : %u\n", 661 args->buf_size_in_bytes, computed_buff_size); 662 return -EINVAL; 663 } 664 665 cmd_from_user = (void __user *) args->content_ptr; 666 667 if (cmd_from_user == NULL) 668 return -EINVAL; 669 670 /* copy the entire buffer from user */ 671 672 args_buff = memdup_user(cmd_from_user, 673 args->buf_size_in_bytes - sizeof(*args)); 674 if (IS_ERR(args_buff)) 675 return PTR_ERR(args_buff); 676 677 /* move ptr to the start of the "pay-load" area */ 678 wac_info.process = p; 679 680 wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx])); 681 args_idx += sizeof(wac_info.operand); 682 683 wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx])); 684 args_idx += sizeof(wac_info.mode); 685 686 wac_info.trapId = *((uint32_t *)(&args_buff[args_idx])); 687 args_idx += sizeof(wac_info.trapId); 688 689 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = 690 *((uint32_t *)(&args_buff[args_idx])); 691 wac_info.dbgWave_msg.MemoryVA = NULL; 692 693 mutex_lock(kfd_get_dbgmgr_mutex()); 694 695 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n", 696 wac_info.process, wac_info.operand, 697 wac_info.mode, wac_info.trapId, 698 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value); 699 700 status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info); 701 702 pr_debug("Returned status of dbg manager is %ld\n", status); 703 704 mutex_unlock(kfd_get_dbgmgr_mutex()); 705 706 kfree(args_buff); 707 708 return status; 709 } 710 711 static int kfd_ioctl_get_clock_counters(struct file *filep, 712 struct kfd_process *p, void *data) 713 { 714 struct kfd_ioctl_get_clock_counters_args *args = data; 715 struct kfd_dev *dev; 716 struct timespec64 time; 717 718 dev = kfd_device_by_id(args->gpu_id); 719 if (dev == NULL) 720 return -EINVAL; 721 722 /* Reading GPU clock counter from KGD */ 723 args->gpu_clock_counter = 724 dev->kfd2kgd->get_gpu_clock_counter(dev->kgd); 725 726 /* No access to rdtsc. Using raw monotonic time */ 727 getrawmonotonic64(&time); 728 args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time); 729 730 get_monotonic_boottime64(&time); 731 args->system_clock_counter = (uint64_t)timespec64_to_ns(&time); 732 733 /* Since the counter is in nano-seconds we use 1GHz frequency */ 734 args->system_clock_freq = 1000000000; 735 736 return 0; 737 } 738 739 740 static int kfd_ioctl_get_process_apertures(struct file *filp, 741 struct kfd_process *p, void *data) 742 { 743 struct kfd_ioctl_get_process_apertures_args *args = data; 744 struct kfd_process_device_apertures *pAperture; 745 struct kfd_process_device *pdd; 746 747 dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); 748 749 args->num_of_nodes = 0; 750 751 mutex_lock(&p->mutex); 752 753 /*if the process-device list isn't empty*/ 754 if (kfd_has_process_device_data(p)) { 755 /* Run over all pdd of the process */ 756 pdd = kfd_get_first_process_device_data(p); 757 do { 758 pAperture = 759 &args->process_apertures[args->num_of_nodes]; 760 pAperture->gpu_id = pdd->dev->id; 761 pAperture->lds_base = pdd->lds_base; 762 pAperture->lds_limit = pdd->lds_limit; 763 pAperture->gpuvm_base = pdd->gpuvm_base; 764 pAperture->gpuvm_limit = pdd->gpuvm_limit; 765 pAperture->scratch_base = pdd->scratch_base; 766 pAperture->scratch_limit = pdd->scratch_limit; 767 768 dev_dbg(kfd_device, 769 "node id %u\n", args->num_of_nodes); 770 dev_dbg(kfd_device, 771 "gpu id %u\n", pdd->dev->id); 772 dev_dbg(kfd_device, 773 "lds_base %llX\n", pdd->lds_base); 774 dev_dbg(kfd_device, 775 "lds_limit %llX\n", pdd->lds_limit); 776 dev_dbg(kfd_device, 777 "gpuvm_base %llX\n", pdd->gpuvm_base); 778 dev_dbg(kfd_device, 779 "gpuvm_limit %llX\n", pdd->gpuvm_limit); 780 dev_dbg(kfd_device, 781 "scratch_base %llX\n", pdd->scratch_base); 782 dev_dbg(kfd_device, 783 "scratch_limit %llX\n", pdd->scratch_limit); 784 785 args->num_of_nodes++; 786 787 pdd = kfd_get_next_process_device_data(p, pdd); 788 } while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); 789 } 790 791 mutex_unlock(&p->mutex); 792 793 return 0; 794 } 795 796 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p, 797 void *data) 798 { 799 struct kfd_ioctl_create_event_args *args = data; 800 int err; 801 802 err = kfd_event_create(filp, p, args->event_type, 803 args->auto_reset != 0, args->node_id, 804 &args->event_id, &args->event_trigger_data, 805 &args->event_page_offset, 806 &args->event_slot_index); 807 808 return err; 809 } 810 811 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p, 812 void *data) 813 { 814 struct kfd_ioctl_destroy_event_args *args = data; 815 816 return kfd_event_destroy(p, args->event_id); 817 } 818 819 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p, 820 void *data) 821 { 822 struct kfd_ioctl_set_event_args *args = data; 823 824 return kfd_set_event(p, args->event_id); 825 } 826 827 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p, 828 void *data) 829 { 830 struct kfd_ioctl_reset_event_args *args = data; 831 832 return kfd_reset_event(p, args->event_id); 833 } 834 835 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, 836 void *data) 837 { 838 struct kfd_ioctl_wait_events_args *args = data; 839 enum kfd_event_wait_result wait_result; 840 int err; 841 842 err = kfd_wait_on_events(p, args->num_events, 843 (void __user *)args->events_ptr, 844 (args->wait_for_all != 0), 845 args->timeout, &wait_result); 846 847 args->wait_result = wait_result; 848 849 return err; 850 } 851 static int kfd_ioctl_set_scratch_backing_va(struct file *filep, 852 struct kfd_process *p, void *data) 853 { 854 struct kfd_ioctl_set_scratch_backing_va_args *args = data; 855 struct kfd_process_device *pdd; 856 struct kfd_dev *dev; 857 long err; 858 859 dev = kfd_device_by_id(args->gpu_id); 860 if (!dev) 861 return -EINVAL; 862 863 mutex_lock(&p->mutex); 864 865 pdd = kfd_bind_process_to_device(dev, p); 866 if (IS_ERR(pdd)) { 867 err = PTR_ERR(pdd); 868 goto bind_process_to_device_fail; 869 } 870 871 pdd->qpd.sh_hidden_private_base = args->va_addr; 872 873 mutex_unlock(&p->mutex); 874 875 if (sched_policy == KFD_SCHED_POLICY_NO_HWS && pdd->qpd.vmid != 0) 876 dev->kfd2kgd->set_scratch_backing_va( 877 dev->kgd, args->va_addr, pdd->qpd.vmid); 878 879 return 0; 880 881 bind_process_to_device_fail: 882 mutex_unlock(&p->mutex); 883 return err; 884 } 885 886 static int kfd_ioctl_get_tile_config(struct file *filep, 887 struct kfd_process *p, void *data) 888 { 889 struct kfd_ioctl_get_tile_config_args *args = data; 890 struct kfd_dev *dev; 891 struct tile_config config; 892 int err = 0; 893 894 dev = kfd_device_by_id(args->gpu_id); 895 if (!dev) 896 return -EINVAL; 897 898 dev->kfd2kgd->get_tile_config(dev->kgd, &config); 899 900 args->gb_addr_config = config.gb_addr_config; 901 args->num_banks = config.num_banks; 902 args->num_ranks = config.num_ranks; 903 904 if (args->num_tile_configs > config.num_tile_configs) 905 args->num_tile_configs = config.num_tile_configs; 906 err = copy_to_user((void __user *)args->tile_config_ptr, 907 config.tile_config_ptr, 908 args->num_tile_configs * sizeof(uint32_t)); 909 if (err) { 910 args->num_tile_configs = 0; 911 return -EFAULT; 912 } 913 914 if (args->num_macro_tile_configs > config.num_macro_tile_configs) 915 args->num_macro_tile_configs = 916 config.num_macro_tile_configs; 917 err = copy_to_user((void __user *)args->macro_tile_config_ptr, 918 config.macro_tile_config_ptr, 919 args->num_macro_tile_configs * sizeof(uint32_t)); 920 if (err) { 921 args->num_macro_tile_configs = 0; 922 return -EFAULT; 923 } 924 925 return 0; 926 } 927 928 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ 929 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \ 930 .cmd_drv = 0, .name = #ioctl} 931 932 /** Ioctl table */ 933 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { 934 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION, 935 kfd_ioctl_get_version, 0), 936 937 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE, 938 kfd_ioctl_create_queue, 0), 939 940 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE, 941 kfd_ioctl_destroy_queue, 0), 942 943 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY, 944 kfd_ioctl_set_memory_policy, 0), 945 946 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS, 947 kfd_ioctl_get_clock_counters, 0), 948 949 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES, 950 kfd_ioctl_get_process_apertures, 0), 951 952 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE, 953 kfd_ioctl_update_queue, 0), 954 955 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT, 956 kfd_ioctl_create_event, 0), 957 958 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT, 959 kfd_ioctl_destroy_event, 0), 960 961 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT, 962 kfd_ioctl_set_event, 0), 963 964 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT, 965 kfd_ioctl_reset_event, 0), 966 967 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS, 968 kfd_ioctl_wait_events, 0), 969 970 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER, 971 kfd_ioctl_dbg_register, 0), 972 973 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER, 974 kfd_ioctl_dbg_unregister, 0), 975 976 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH, 977 kfd_ioctl_dbg_address_watch, 0), 978 979 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL, 980 kfd_ioctl_dbg_wave_control, 0), 981 982 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA, 983 kfd_ioctl_set_scratch_backing_va, 0), 984 985 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG, 986 kfd_ioctl_get_tile_config, 0) 987 }; 988 989 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) 990 991 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 992 { 993 struct kfd_process *process; 994 amdkfd_ioctl_t *func; 995 const struct amdkfd_ioctl_desc *ioctl = NULL; 996 unsigned int nr = _IOC_NR(cmd); 997 char stack_kdata[128]; 998 char *kdata = NULL; 999 unsigned int usize, asize; 1000 int retcode = -EINVAL; 1001 1002 if (nr >= AMDKFD_CORE_IOCTL_COUNT) 1003 goto err_i1; 1004 1005 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { 1006 u32 amdkfd_size; 1007 1008 ioctl = &amdkfd_ioctls[nr]; 1009 1010 amdkfd_size = _IOC_SIZE(ioctl->cmd); 1011 usize = asize = _IOC_SIZE(cmd); 1012 if (amdkfd_size > asize) 1013 asize = amdkfd_size; 1014 1015 cmd = ioctl->cmd; 1016 } else 1017 goto err_i1; 1018 1019 dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg); 1020 1021 process = kfd_get_process(current); 1022 if (IS_ERR(process)) { 1023 dev_dbg(kfd_device, "no process\n"); 1024 goto err_i1; 1025 } 1026 1027 /* Do not trust userspace, use our own definition */ 1028 func = ioctl->func; 1029 1030 if (unlikely(!func)) { 1031 dev_dbg(kfd_device, "no function\n"); 1032 retcode = -EINVAL; 1033 goto err_i1; 1034 } 1035 1036 if (cmd & (IOC_IN | IOC_OUT)) { 1037 if (asize <= sizeof(stack_kdata)) { 1038 kdata = stack_kdata; 1039 } else { 1040 kdata = kmalloc(asize, GFP_KERNEL); 1041 if (!kdata) { 1042 retcode = -ENOMEM; 1043 goto err_i1; 1044 } 1045 } 1046 if (asize > usize) 1047 memset(kdata + usize, 0, asize - usize); 1048 } 1049 1050 if (cmd & IOC_IN) { 1051 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { 1052 retcode = -EFAULT; 1053 goto err_i1; 1054 } 1055 } else if (cmd & IOC_OUT) { 1056 memset(kdata, 0, usize); 1057 } 1058 1059 retcode = func(filep, process, kdata); 1060 1061 if (cmd & IOC_OUT) 1062 if (copy_to_user((void __user *)arg, kdata, usize) != 0) 1063 retcode = -EFAULT; 1064 1065 err_i1: 1066 if (!ioctl) 1067 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", 1068 task_pid_nr(current), cmd, nr); 1069 1070 if (kdata != stack_kdata) 1071 kfree(kdata); 1072 1073 if (retcode) 1074 dev_dbg(kfd_device, "ret = %d\n", retcode); 1075 1076 return retcode; 1077 } 1078 1079 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) 1080 { 1081 struct kfd_process *process; 1082 1083 process = kfd_get_process(current); 1084 if (IS_ERR(process)) 1085 return PTR_ERR(process); 1086 1087 if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) == 1088 KFD_MMAP_DOORBELL_MASK) { 1089 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK; 1090 return kfd_doorbell_mmap(process, vma); 1091 } else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) == 1092 KFD_MMAP_EVENTS_MASK) { 1093 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK; 1094 return kfd_event_mmap(process, vma); 1095 } 1096 1097 return -EFAULT; 1098 } 1099