1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/device.h> 24 #include <linux/export.h> 25 #include <linux/err.h> 26 #include <linux/fs.h> 27 #include <linux/sched.h> 28 #include <linux/slab.h> 29 #include <linux/uaccess.h> 30 #include <linux/compat.h> 31 #include <uapi/linux/kfd_ioctl.h> 32 #include <linux/time.h> 33 #include <linux/mm.h> 34 #include <linux/mman.h> 35 #include <asm/processor.h> 36 #include "kfd_priv.h" 37 #include "kfd_device_queue_manager.h" 38 #include "kfd_dbgmgr.h" 39 40 static long kfd_ioctl(struct file *, unsigned int, unsigned long); 41 static int kfd_open(struct inode *, struct file *); 42 static int kfd_mmap(struct file *, struct vm_area_struct *); 43 44 static const char kfd_dev_name[] = "kfd"; 45 46 static const struct file_operations kfd_fops = { 47 .owner = THIS_MODULE, 48 .unlocked_ioctl = kfd_ioctl, 49 .compat_ioctl = kfd_ioctl, 50 .open = kfd_open, 51 .mmap = kfd_mmap, 52 }; 53 54 static int kfd_char_dev_major = -1; 55 static struct class *kfd_class; 56 struct device *kfd_device; 57 58 int kfd_chardev_init(void) 59 { 60 int err = 0; 61 62 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); 63 err = kfd_char_dev_major; 64 if (err < 0) 65 goto err_register_chrdev; 66 67 kfd_class = class_create(THIS_MODULE, kfd_dev_name); 68 err = PTR_ERR(kfd_class); 69 if (IS_ERR(kfd_class)) 70 goto err_class_create; 71 72 kfd_device = device_create(kfd_class, NULL, 73 MKDEV(kfd_char_dev_major, 0), 74 NULL, kfd_dev_name); 75 err = PTR_ERR(kfd_device); 76 if (IS_ERR(kfd_device)) 77 goto err_device_create; 78 79 return 0; 80 81 err_device_create: 82 class_destroy(kfd_class); 83 err_class_create: 84 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 85 err_register_chrdev: 86 return err; 87 } 88 89 void kfd_chardev_exit(void) 90 { 91 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); 92 class_destroy(kfd_class); 93 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 94 } 95 96 struct device *kfd_chardev(void) 97 { 98 return kfd_device; 99 } 100 101 102 static int kfd_open(struct inode *inode, struct file *filep) 103 { 104 struct kfd_process *process; 105 bool is_32bit_user_mode; 106 107 if (iminor(inode) != 0) 108 return -ENODEV; 109 110 is_32bit_user_mode = in_compat_syscall(); 111 112 if (is_32bit_user_mode) { 113 dev_warn(kfd_device, 114 "Process %d (32-bit) failed to open /dev/kfd\n" 115 "32-bit processes are not supported by amdkfd\n", 116 current->pid); 117 return -EPERM; 118 } 119 120 process = kfd_create_process(current); 121 if (IS_ERR(process)) 122 return PTR_ERR(process); 123 124 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", 125 process->pasid, process->is_32bit_user_mode); 126 127 return 0; 128 } 129 130 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, 131 void *data) 132 { 133 struct kfd_ioctl_get_version_args *args = data; 134 135 args->major_version = KFD_IOCTL_MAJOR_VERSION; 136 args->minor_version = KFD_IOCTL_MINOR_VERSION; 137 138 return 0; 139 } 140 141 static int set_queue_properties_from_user(struct queue_properties *q_properties, 142 struct kfd_ioctl_create_queue_args *args) 143 { 144 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 145 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 146 return -EINVAL; 147 } 148 149 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 150 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 151 return -EINVAL; 152 } 153 154 if ((args->ring_base_address) && 155 (!access_ok(VERIFY_WRITE, 156 (const void __user *) args->ring_base_address, 157 sizeof(uint64_t)))) { 158 pr_err("Can't access ring base address\n"); 159 return -EFAULT; 160 } 161 162 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 163 pr_err("Ring size must be a power of 2 or 0\n"); 164 return -EINVAL; 165 } 166 167 if (!access_ok(VERIFY_WRITE, 168 (const void __user *) args->read_pointer_address, 169 sizeof(uint32_t))) { 170 pr_err("Can't access read pointer\n"); 171 return -EFAULT; 172 } 173 174 if (!access_ok(VERIFY_WRITE, 175 (const void __user *) args->write_pointer_address, 176 sizeof(uint32_t))) { 177 pr_err("Can't access write pointer\n"); 178 return -EFAULT; 179 } 180 181 if (args->eop_buffer_address && 182 !access_ok(VERIFY_WRITE, 183 (const void __user *) args->eop_buffer_address, 184 sizeof(uint32_t))) { 185 pr_debug("Can't access eop buffer"); 186 return -EFAULT; 187 } 188 189 if (args->ctx_save_restore_address && 190 !access_ok(VERIFY_WRITE, 191 (const void __user *) args->ctx_save_restore_address, 192 sizeof(uint32_t))) { 193 pr_debug("Can't access ctx save restore buffer"); 194 return -EFAULT; 195 } 196 197 q_properties->is_interop = false; 198 q_properties->queue_percent = args->queue_percentage; 199 q_properties->priority = args->queue_priority; 200 q_properties->queue_address = args->ring_base_address; 201 q_properties->queue_size = args->ring_size; 202 q_properties->read_ptr = (uint32_t *) args->read_pointer_address; 203 q_properties->write_ptr = (uint32_t *) args->write_pointer_address; 204 q_properties->eop_ring_buffer_address = args->eop_buffer_address; 205 q_properties->eop_ring_buffer_size = args->eop_buffer_size; 206 q_properties->ctx_save_restore_area_address = 207 args->ctx_save_restore_address; 208 q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size; 209 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || 210 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 211 q_properties->type = KFD_QUEUE_TYPE_COMPUTE; 212 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) 213 q_properties->type = KFD_QUEUE_TYPE_SDMA; 214 else 215 return -ENOTSUPP; 216 217 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 218 q_properties->format = KFD_QUEUE_FORMAT_AQL; 219 else 220 q_properties->format = KFD_QUEUE_FORMAT_PM4; 221 222 pr_debug("Queue Percentage: %d, %d\n", 223 q_properties->queue_percent, args->queue_percentage); 224 225 pr_debug("Queue Priority: %d, %d\n", 226 q_properties->priority, args->queue_priority); 227 228 pr_debug("Queue Address: 0x%llX, 0x%llX\n", 229 q_properties->queue_address, args->ring_base_address); 230 231 pr_debug("Queue Size: 0x%llX, %u\n", 232 q_properties->queue_size, args->ring_size); 233 234 pr_debug("Queue r/w Pointers: %p, %p\n", 235 q_properties->read_ptr, 236 q_properties->write_ptr); 237 238 pr_debug("Queue Format: %d\n", q_properties->format); 239 240 pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address); 241 242 pr_debug("Queue CTX save area: 0x%llX\n", 243 q_properties->ctx_save_restore_area_address); 244 245 return 0; 246 } 247 248 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 249 void *data) 250 { 251 struct kfd_ioctl_create_queue_args *args = data; 252 struct kfd_dev *dev; 253 int err = 0; 254 unsigned int queue_id; 255 struct kfd_process_device *pdd; 256 struct queue_properties q_properties; 257 258 memset(&q_properties, 0, sizeof(struct queue_properties)); 259 260 pr_debug("Creating queue ioctl\n"); 261 262 err = set_queue_properties_from_user(&q_properties, args); 263 if (err) 264 return err; 265 266 pr_debug("Looking for gpu id 0x%x\n", args->gpu_id); 267 dev = kfd_device_by_id(args->gpu_id); 268 if (!dev) { 269 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id); 270 return -EINVAL; 271 } 272 273 mutex_lock(&p->mutex); 274 275 pdd = kfd_bind_process_to_device(dev, p); 276 if (IS_ERR(pdd)) { 277 err = -ESRCH; 278 goto err_bind_process; 279 } 280 281 pr_debug("Creating queue for PASID %d on gpu 0x%x\n", 282 p->pasid, 283 dev->id); 284 285 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id); 286 if (err != 0) 287 goto err_create_queue; 288 289 args->queue_id = queue_id; 290 291 292 /* Return gpu_id as doorbell offset for mmap usage */ 293 args->doorbell_offset = (KFD_MMAP_DOORBELL_MASK | args->gpu_id); 294 args->doorbell_offset <<= PAGE_SHIFT; 295 296 mutex_unlock(&p->mutex); 297 298 pr_debug("Queue id %d was created successfully\n", args->queue_id); 299 300 pr_debug("Ring buffer address == 0x%016llX\n", 301 args->ring_base_address); 302 303 pr_debug("Read ptr address == 0x%016llX\n", 304 args->read_pointer_address); 305 306 pr_debug("Write ptr address == 0x%016llX\n", 307 args->write_pointer_address); 308 309 return 0; 310 311 err_create_queue: 312 err_bind_process: 313 mutex_unlock(&p->mutex); 314 return err; 315 } 316 317 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, 318 void *data) 319 { 320 int retval; 321 struct kfd_ioctl_destroy_queue_args *args = data; 322 323 pr_debug("Destroying queue id %d for pasid %d\n", 324 args->queue_id, 325 p->pasid); 326 327 mutex_lock(&p->mutex); 328 329 retval = pqm_destroy_queue(&p->pqm, args->queue_id); 330 331 mutex_unlock(&p->mutex); 332 return retval; 333 } 334 335 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, 336 void *data) 337 { 338 int retval; 339 struct kfd_ioctl_update_queue_args *args = data; 340 struct queue_properties properties; 341 342 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 343 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 344 return -EINVAL; 345 } 346 347 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 348 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 349 return -EINVAL; 350 } 351 352 if ((args->ring_base_address) && 353 (!access_ok(VERIFY_WRITE, 354 (const void __user *) args->ring_base_address, 355 sizeof(uint64_t)))) { 356 pr_err("Can't access ring base address\n"); 357 return -EFAULT; 358 } 359 360 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 361 pr_err("Ring size must be a power of 2 or 0\n"); 362 return -EINVAL; 363 } 364 365 properties.queue_address = args->ring_base_address; 366 properties.queue_size = args->ring_size; 367 properties.queue_percent = args->queue_percentage; 368 properties.priority = args->queue_priority; 369 370 pr_debug("Updating queue id %d for pasid %d\n", 371 args->queue_id, p->pasid); 372 373 mutex_lock(&p->mutex); 374 375 retval = pqm_update_queue(&p->pqm, args->queue_id, &properties); 376 377 mutex_unlock(&p->mutex); 378 379 return retval; 380 } 381 382 static int kfd_ioctl_set_memory_policy(struct file *filep, 383 struct kfd_process *p, void *data) 384 { 385 struct kfd_ioctl_set_memory_policy_args *args = data; 386 struct kfd_dev *dev; 387 int err = 0; 388 struct kfd_process_device *pdd; 389 enum cache_policy default_policy, alternate_policy; 390 391 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT 392 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 393 return -EINVAL; 394 } 395 396 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT 397 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 398 return -EINVAL; 399 } 400 401 dev = kfd_device_by_id(args->gpu_id); 402 if (!dev) 403 return -EINVAL; 404 405 mutex_lock(&p->mutex); 406 407 pdd = kfd_bind_process_to_device(dev, p); 408 if (IS_ERR(pdd)) { 409 err = -ESRCH; 410 goto out; 411 } 412 413 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT) 414 ? cache_policy_coherent : cache_policy_noncoherent; 415 416 alternate_policy = 417 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) 418 ? cache_policy_coherent : cache_policy_noncoherent; 419 420 if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm, 421 &pdd->qpd, 422 default_policy, 423 alternate_policy, 424 (void __user *)args->alternate_aperture_base, 425 args->alternate_aperture_size)) 426 err = -EINVAL; 427 428 out: 429 mutex_unlock(&p->mutex); 430 431 return err; 432 } 433 434 static int kfd_ioctl_dbg_register(struct file *filep, 435 struct kfd_process *p, void *data) 436 { 437 struct kfd_ioctl_dbg_register_args *args = data; 438 struct kfd_dev *dev; 439 struct kfd_dbgmgr *dbgmgr_ptr; 440 struct kfd_process_device *pdd; 441 bool create_ok; 442 long status = 0; 443 444 dev = kfd_device_by_id(args->gpu_id); 445 if (!dev) 446 return -EINVAL; 447 448 if (dev->device_info->asic_family == CHIP_CARRIZO) { 449 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n"); 450 return -EINVAL; 451 } 452 453 mutex_lock(&p->mutex); 454 mutex_lock(kfd_get_dbgmgr_mutex()); 455 456 /* 457 * make sure that we have pdd, if this the first queue created for 458 * this process 459 */ 460 pdd = kfd_bind_process_to_device(dev, p); 461 if (IS_ERR(pdd)) { 462 status = PTR_ERR(pdd); 463 goto out; 464 } 465 466 if (!dev->dbgmgr) { 467 /* In case of a legal call, we have no dbgmgr yet */ 468 create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev); 469 if (create_ok) { 470 status = kfd_dbgmgr_register(dbgmgr_ptr, p); 471 if (status != 0) 472 kfd_dbgmgr_destroy(dbgmgr_ptr); 473 else 474 dev->dbgmgr = dbgmgr_ptr; 475 } 476 } else { 477 pr_debug("debugger already registered\n"); 478 status = -EINVAL; 479 } 480 481 out: 482 mutex_unlock(kfd_get_dbgmgr_mutex()); 483 mutex_unlock(&p->mutex); 484 485 return status; 486 } 487 488 static int kfd_ioctl_dbg_unregister(struct file *filep, 489 struct kfd_process *p, void *data) 490 { 491 struct kfd_ioctl_dbg_unregister_args *args = data; 492 struct kfd_dev *dev; 493 long status; 494 495 dev = kfd_device_by_id(args->gpu_id); 496 if (!dev) 497 return -EINVAL; 498 499 if (dev->device_info->asic_family == CHIP_CARRIZO) { 500 pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n"); 501 return -EINVAL; 502 } 503 504 mutex_lock(kfd_get_dbgmgr_mutex()); 505 506 status = kfd_dbgmgr_unregister(dev->dbgmgr, p); 507 if (!status) { 508 kfd_dbgmgr_destroy(dev->dbgmgr); 509 dev->dbgmgr = NULL; 510 } 511 512 mutex_unlock(kfd_get_dbgmgr_mutex()); 513 514 return status; 515 } 516 517 /* 518 * Parse and generate variable size data structure for address watch. 519 * Total size of the buffer and # watch points is limited in order 520 * to prevent kernel abuse. (no bearing to the much smaller HW limitation 521 * which is enforced by dbgdev module) 522 * please also note that the watch address itself are not "copied from user", 523 * since it be set into the HW in user mode values. 524 * 525 */ 526 static int kfd_ioctl_dbg_address_watch(struct file *filep, 527 struct kfd_process *p, void *data) 528 { 529 struct kfd_ioctl_dbg_address_watch_args *args = data; 530 struct kfd_dev *dev; 531 struct dbg_address_watch_info aw_info; 532 unsigned char *args_buff; 533 long status; 534 void __user *cmd_from_user; 535 uint64_t watch_mask_value = 0; 536 unsigned int args_idx = 0; 537 538 memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info)); 539 540 dev = kfd_device_by_id(args->gpu_id); 541 if (!dev) 542 return -EINVAL; 543 544 if (dev->device_info->asic_family == CHIP_CARRIZO) { 545 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 546 return -EINVAL; 547 } 548 549 cmd_from_user = (void __user *) args->content_ptr; 550 551 /* Validate arguments */ 552 553 if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) || 554 (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) || 555 (cmd_from_user == NULL)) 556 return -EINVAL; 557 558 /* this is the actual buffer to work with */ 559 args_buff = memdup_user(cmd_from_user, 560 args->buf_size_in_bytes - sizeof(*args)); 561 if (IS_ERR(args_buff)) 562 return PTR_ERR(args_buff); 563 564 aw_info.process = p; 565 566 aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx])); 567 args_idx += sizeof(aw_info.num_watch_points); 568 569 aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx]; 570 args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points; 571 572 /* 573 * set watch address base pointer to point on the array base 574 * within args_buff 575 */ 576 aw_info.watch_address = (uint64_t *) &args_buff[args_idx]; 577 578 /* skip over the addresses buffer */ 579 args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points; 580 581 if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) { 582 status = -EINVAL; 583 goto out; 584 } 585 586 watch_mask_value = (uint64_t) args_buff[args_idx]; 587 588 if (watch_mask_value > 0) { 589 /* 590 * There is an array of masks. 591 * set watch mask base pointer to point on the array base 592 * within args_buff 593 */ 594 aw_info.watch_mask = (uint64_t *) &args_buff[args_idx]; 595 596 /* skip over the masks buffer */ 597 args_idx += sizeof(aw_info.watch_mask) * 598 aw_info.num_watch_points; 599 } else { 600 /* just the NULL mask, set to NULL and skip over it */ 601 aw_info.watch_mask = NULL; 602 args_idx += sizeof(aw_info.watch_mask); 603 } 604 605 if (args_idx >= args->buf_size_in_bytes - sizeof(args)) { 606 status = -EINVAL; 607 goto out; 608 } 609 610 /* Currently HSA Event is not supported for DBG */ 611 aw_info.watch_event = NULL; 612 613 mutex_lock(kfd_get_dbgmgr_mutex()); 614 615 status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info); 616 617 mutex_unlock(kfd_get_dbgmgr_mutex()); 618 619 out: 620 kfree(args_buff); 621 622 return status; 623 } 624 625 /* Parse and generate fixed size data structure for wave control */ 626 static int kfd_ioctl_dbg_wave_control(struct file *filep, 627 struct kfd_process *p, void *data) 628 { 629 struct kfd_ioctl_dbg_wave_control_args *args = data; 630 struct kfd_dev *dev; 631 struct dbg_wave_control_info wac_info; 632 unsigned char *args_buff; 633 uint32_t computed_buff_size; 634 long status; 635 void __user *cmd_from_user; 636 unsigned int args_idx = 0; 637 638 memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info)); 639 640 /* we use compact form, independent of the packing attribute value */ 641 computed_buff_size = sizeof(*args) + 642 sizeof(wac_info.mode) + 643 sizeof(wac_info.operand) + 644 sizeof(wac_info.dbgWave_msg.DbgWaveMsg) + 645 sizeof(wac_info.dbgWave_msg.MemoryVA) + 646 sizeof(wac_info.trapId); 647 648 dev = kfd_device_by_id(args->gpu_id); 649 if (!dev) 650 return -EINVAL; 651 652 if (dev->device_info->asic_family == CHIP_CARRIZO) { 653 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 654 return -EINVAL; 655 } 656 657 /* input size must match the computed "compact" size */ 658 if (args->buf_size_in_bytes != computed_buff_size) { 659 pr_debug("size mismatch, computed : actual %u : %u\n", 660 args->buf_size_in_bytes, computed_buff_size); 661 return -EINVAL; 662 } 663 664 cmd_from_user = (void __user *) args->content_ptr; 665 666 if (cmd_from_user == NULL) 667 return -EINVAL; 668 669 /* copy the entire buffer from user */ 670 671 args_buff = memdup_user(cmd_from_user, 672 args->buf_size_in_bytes - sizeof(*args)); 673 if (IS_ERR(args_buff)) 674 return PTR_ERR(args_buff); 675 676 /* move ptr to the start of the "pay-load" area */ 677 wac_info.process = p; 678 679 wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx])); 680 args_idx += sizeof(wac_info.operand); 681 682 wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx])); 683 args_idx += sizeof(wac_info.mode); 684 685 wac_info.trapId = *((uint32_t *)(&args_buff[args_idx])); 686 args_idx += sizeof(wac_info.trapId); 687 688 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = 689 *((uint32_t *)(&args_buff[args_idx])); 690 wac_info.dbgWave_msg.MemoryVA = NULL; 691 692 mutex_lock(kfd_get_dbgmgr_mutex()); 693 694 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n", 695 wac_info.process, wac_info.operand, 696 wac_info.mode, wac_info.trapId, 697 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value); 698 699 status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info); 700 701 pr_debug("Returned status of dbg manager is %ld\n", status); 702 703 mutex_unlock(kfd_get_dbgmgr_mutex()); 704 705 kfree(args_buff); 706 707 return status; 708 } 709 710 static int kfd_ioctl_get_clock_counters(struct file *filep, 711 struct kfd_process *p, void *data) 712 { 713 struct kfd_ioctl_get_clock_counters_args *args = data; 714 struct kfd_dev *dev; 715 struct timespec64 time; 716 717 dev = kfd_device_by_id(args->gpu_id); 718 if (dev == NULL) 719 return -EINVAL; 720 721 /* Reading GPU clock counter from KGD */ 722 args->gpu_clock_counter = 723 dev->kfd2kgd->get_gpu_clock_counter(dev->kgd); 724 725 /* No access to rdtsc. Using raw monotonic time */ 726 getrawmonotonic64(&time); 727 args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time); 728 729 get_monotonic_boottime64(&time); 730 args->system_clock_counter = (uint64_t)timespec64_to_ns(&time); 731 732 /* Since the counter is in nano-seconds we use 1GHz frequency */ 733 args->system_clock_freq = 1000000000; 734 735 return 0; 736 } 737 738 739 static int kfd_ioctl_get_process_apertures(struct file *filp, 740 struct kfd_process *p, void *data) 741 { 742 struct kfd_ioctl_get_process_apertures_args *args = data; 743 struct kfd_process_device_apertures *pAperture; 744 struct kfd_process_device *pdd; 745 746 dev_dbg(kfd_device, "get apertures for PASID %d", p->pasid); 747 748 args->num_of_nodes = 0; 749 750 mutex_lock(&p->mutex); 751 752 /*if the process-device list isn't empty*/ 753 if (kfd_has_process_device_data(p)) { 754 /* Run over all pdd of the process */ 755 pdd = kfd_get_first_process_device_data(p); 756 do { 757 pAperture = 758 &args->process_apertures[args->num_of_nodes]; 759 pAperture->gpu_id = pdd->dev->id; 760 pAperture->lds_base = pdd->lds_base; 761 pAperture->lds_limit = pdd->lds_limit; 762 pAperture->gpuvm_base = pdd->gpuvm_base; 763 pAperture->gpuvm_limit = pdd->gpuvm_limit; 764 pAperture->scratch_base = pdd->scratch_base; 765 pAperture->scratch_limit = pdd->scratch_limit; 766 767 dev_dbg(kfd_device, 768 "node id %u\n", args->num_of_nodes); 769 dev_dbg(kfd_device, 770 "gpu id %u\n", pdd->dev->id); 771 dev_dbg(kfd_device, 772 "lds_base %llX\n", pdd->lds_base); 773 dev_dbg(kfd_device, 774 "lds_limit %llX\n", pdd->lds_limit); 775 dev_dbg(kfd_device, 776 "gpuvm_base %llX\n", pdd->gpuvm_base); 777 dev_dbg(kfd_device, 778 "gpuvm_limit %llX\n", pdd->gpuvm_limit); 779 dev_dbg(kfd_device, 780 "scratch_base %llX\n", pdd->scratch_base); 781 dev_dbg(kfd_device, 782 "scratch_limit %llX\n", pdd->scratch_limit); 783 784 args->num_of_nodes++; 785 786 pdd = kfd_get_next_process_device_data(p, pdd); 787 } while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); 788 } 789 790 mutex_unlock(&p->mutex); 791 792 return 0; 793 } 794 795 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p, 796 void *data) 797 { 798 struct kfd_ioctl_create_event_args *args = data; 799 int err; 800 801 err = kfd_event_create(filp, p, args->event_type, 802 args->auto_reset != 0, args->node_id, 803 &args->event_id, &args->event_trigger_data, 804 &args->event_page_offset, 805 &args->event_slot_index); 806 807 return err; 808 } 809 810 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p, 811 void *data) 812 { 813 struct kfd_ioctl_destroy_event_args *args = data; 814 815 return kfd_event_destroy(p, args->event_id); 816 } 817 818 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p, 819 void *data) 820 { 821 struct kfd_ioctl_set_event_args *args = data; 822 823 return kfd_set_event(p, args->event_id); 824 } 825 826 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p, 827 void *data) 828 { 829 struct kfd_ioctl_reset_event_args *args = data; 830 831 return kfd_reset_event(p, args->event_id); 832 } 833 834 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, 835 void *data) 836 { 837 struct kfd_ioctl_wait_events_args *args = data; 838 int err; 839 840 err = kfd_wait_on_events(p, args->num_events, 841 (void __user *)args->events_ptr, 842 (args->wait_for_all != 0), 843 args->timeout, &args->wait_result); 844 845 return err; 846 } 847 static int kfd_ioctl_set_scratch_backing_va(struct file *filep, 848 struct kfd_process *p, void *data) 849 { 850 struct kfd_ioctl_set_scratch_backing_va_args *args = data; 851 struct kfd_process_device *pdd; 852 struct kfd_dev *dev; 853 long err; 854 855 dev = kfd_device_by_id(args->gpu_id); 856 if (!dev) 857 return -EINVAL; 858 859 mutex_lock(&p->mutex); 860 861 pdd = kfd_bind_process_to_device(dev, p); 862 if (IS_ERR(pdd)) { 863 err = PTR_ERR(pdd); 864 goto bind_process_to_device_fail; 865 } 866 867 pdd->qpd.sh_hidden_private_base = args->va_addr; 868 869 mutex_unlock(&p->mutex); 870 871 if (sched_policy == KFD_SCHED_POLICY_NO_HWS && pdd->qpd.vmid != 0) 872 dev->kfd2kgd->set_scratch_backing_va( 873 dev->kgd, args->va_addr, pdd->qpd.vmid); 874 875 return 0; 876 877 bind_process_to_device_fail: 878 mutex_unlock(&p->mutex); 879 return err; 880 } 881 882 static int kfd_ioctl_get_tile_config(struct file *filep, 883 struct kfd_process *p, void *data) 884 { 885 struct kfd_ioctl_get_tile_config_args *args = data; 886 struct kfd_dev *dev; 887 struct tile_config config; 888 int err = 0; 889 890 dev = kfd_device_by_id(args->gpu_id); 891 if (!dev) 892 return -EINVAL; 893 894 dev->kfd2kgd->get_tile_config(dev->kgd, &config); 895 896 args->gb_addr_config = config.gb_addr_config; 897 args->num_banks = config.num_banks; 898 args->num_ranks = config.num_ranks; 899 900 if (args->num_tile_configs > config.num_tile_configs) 901 args->num_tile_configs = config.num_tile_configs; 902 err = copy_to_user((void __user *)args->tile_config_ptr, 903 config.tile_config_ptr, 904 args->num_tile_configs * sizeof(uint32_t)); 905 if (err) { 906 args->num_tile_configs = 0; 907 return -EFAULT; 908 } 909 910 if (args->num_macro_tile_configs > config.num_macro_tile_configs) 911 args->num_macro_tile_configs = 912 config.num_macro_tile_configs; 913 err = copy_to_user((void __user *)args->macro_tile_config_ptr, 914 config.macro_tile_config_ptr, 915 args->num_macro_tile_configs * sizeof(uint32_t)); 916 if (err) { 917 args->num_macro_tile_configs = 0; 918 return -EFAULT; 919 } 920 921 return 0; 922 } 923 924 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ 925 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \ 926 .cmd_drv = 0, .name = #ioctl} 927 928 /** Ioctl table */ 929 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { 930 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION, 931 kfd_ioctl_get_version, 0), 932 933 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE, 934 kfd_ioctl_create_queue, 0), 935 936 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE, 937 kfd_ioctl_destroy_queue, 0), 938 939 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY, 940 kfd_ioctl_set_memory_policy, 0), 941 942 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS, 943 kfd_ioctl_get_clock_counters, 0), 944 945 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES, 946 kfd_ioctl_get_process_apertures, 0), 947 948 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE, 949 kfd_ioctl_update_queue, 0), 950 951 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT, 952 kfd_ioctl_create_event, 0), 953 954 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT, 955 kfd_ioctl_destroy_event, 0), 956 957 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT, 958 kfd_ioctl_set_event, 0), 959 960 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT, 961 kfd_ioctl_reset_event, 0), 962 963 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS, 964 kfd_ioctl_wait_events, 0), 965 966 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER, 967 kfd_ioctl_dbg_register, 0), 968 969 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER, 970 kfd_ioctl_dbg_unregister, 0), 971 972 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH, 973 kfd_ioctl_dbg_address_watch, 0), 974 975 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL, 976 kfd_ioctl_dbg_wave_control, 0), 977 978 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA, 979 kfd_ioctl_set_scratch_backing_va, 0), 980 981 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG, 982 kfd_ioctl_get_tile_config, 0) 983 }; 984 985 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) 986 987 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 988 { 989 struct kfd_process *process; 990 amdkfd_ioctl_t *func; 991 const struct amdkfd_ioctl_desc *ioctl = NULL; 992 unsigned int nr = _IOC_NR(cmd); 993 char stack_kdata[128]; 994 char *kdata = NULL; 995 unsigned int usize, asize; 996 int retcode = -EINVAL; 997 998 if (nr >= AMDKFD_CORE_IOCTL_COUNT) 999 goto err_i1; 1000 1001 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { 1002 u32 amdkfd_size; 1003 1004 ioctl = &amdkfd_ioctls[nr]; 1005 1006 amdkfd_size = _IOC_SIZE(ioctl->cmd); 1007 usize = asize = _IOC_SIZE(cmd); 1008 if (amdkfd_size > asize) 1009 asize = amdkfd_size; 1010 1011 cmd = ioctl->cmd; 1012 } else 1013 goto err_i1; 1014 1015 dev_dbg(kfd_device, "ioctl cmd 0x%x (#%d), arg 0x%lx\n", cmd, nr, arg); 1016 1017 process = kfd_get_process(current); 1018 if (IS_ERR(process)) { 1019 dev_dbg(kfd_device, "no process\n"); 1020 goto err_i1; 1021 } 1022 1023 /* Do not trust userspace, use our own definition */ 1024 func = ioctl->func; 1025 1026 if (unlikely(!func)) { 1027 dev_dbg(kfd_device, "no function\n"); 1028 retcode = -EINVAL; 1029 goto err_i1; 1030 } 1031 1032 if (cmd & (IOC_IN | IOC_OUT)) { 1033 if (asize <= sizeof(stack_kdata)) { 1034 kdata = stack_kdata; 1035 } else { 1036 kdata = kmalloc(asize, GFP_KERNEL); 1037 if (!kdata) { 1038 retcode = -ENOMEM; 1039 goto err_i1; 1040 } 1041 } 1042 if (asize > usize) 1043 memset(kdata + usize, 0, asize - usize); 1044 } 1045 1046 if (cmd & IOC_IN) { 1047 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { 1048 retcode = -EFAULT; 1049 goto err_i1; 1050 } 1051 } else if (cmd & IOC_OUT) { 1052 memset(kdata, 0, usize); 1053 } 1054 1055 retcode = func(filep, process, kdata); 1056 1057 if (cmd & IOC_OUT) 1058 if (copy_to_user((void __user *)arg, kdata, usize) != 0) 1059 retcode = -EFAULT; 1060 1061 err_i1: 1062 if (!ioctl) 1063 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", 1064 task_pid_nr(current), cmd, nr); 1065 1066 if (kdata != stack_kdata) 1067 kfree(kdata); 1068 1069 if (retcode) 1070 dev_dbg(kfd_device, "ret = %d\n", retcode); 1071 1072 return retcode; 1073 } 1074 1075 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) 1076 { 1077 struct kfd_process *process; 1078 1079 process = kfd_get_process(current); 1080 if (IS_ERR(process)) 1081 return PTR_ERR(process); 1082 1083 if ((vma->vm_pgoff & KFD_MMAP_DOORBELL_MASK) == 1084 KFD_MMAP_DOORBELL_MASK) { 1085 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_DOORBELL_MASK; 1086 return kfd_doorbell_mmap(process, vma); 1087 } else if ((vma->vm_pgoff & KFD_MMAP_EVENTS_MASK) == 1088 KFD_MMAP_EVENTS_MASK) { 1089 vma->vm_pgoff = vma->vm_pgoff ^ KFD_MMAP_EVENTS_MASK; 1090 return kfd_event_mmap(process, vma); 1091 } 1092 1093 return -EFAULT; 1094 } 1095