1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/device.h> 24 #include <linux/export.h> 25 #include <linux/err.h> 26 #include <linux/fs.h> 27 #include <linux/file.h> 28 #include <linux/sched.h> 29 #include <linux/slab.h> 30 #include <linux/uaccess.h> 31 #include <linux/compat.h> 32 #include <uapi/linux/kfd_ioctl.h> 33 #include <linux/time.h> 34 #include <linux/mm.h> 35 #include <linux/mman.h> 36 #include <linux/dma-buf.h> 37 #include <asm/processor.h> 38 #include "kfd_priv.h" 39 #include "kfd_device_queue_manager.h" 40 #include "kfd_dbgmgr.h" 41 #include "amdgpu_amdkfd.h" 42 #include "kfd_smi_events.h" 43 44 static long kfd_ioctl(struct file *, unsigned int, unsigned long); 45 static int kfd_open(struct inode *, struct file *); 46 static int kfd_release(struct inode *, struct file *); 47 static int kfd_mmap(struct file *, struct vm_area_struct *); 48 49 static const char kfd_dev_name[] = "kfd"; 50 51 static const struct file_operations kfd_fops = { 52 .owner = THIS_MODULE, 53 .unlocked_ioctl = kfd_ioctl, 54 .compat_ioctl = compat_ptr_ioctl, 55 .open = kfd_open, 56 .release = kfd_release, 57 .mmap = kfd_mmap, 58 }; 59 60 static int kfd_char_dev_major = -1; 61 static struct class *kfd_class; 62 struct device *kfd_device; 63 64 int kfd_chardev_init(void) 65 { 66 int err = 0; 67 68 kfd_char_dev_major = register_chrdev(0, kfd_dev_name, &kfd_fops); 69 err = kfd_char_dev_major; 70 if (err < 0) 71 goto err_register_chrdev; 72 73 kfd_class = class_create(THIS_MODULE, kfd_dev_name); 74 err = PTR_ERR(kfd_class); 75 if (IS_ERR(kfd_class)) 76 goto err_class_create; 77 78 kfd_device = device_create(kfd_class, NULL, 79 MKDEV(kfd_char_dev_major, 0), 80 NULL, kfd_dev_name); 81 err = PTR_ERR(kfd_device); 82 if (IS_ERR(kfd_device)) 83 goto err_device_create; 84 85 return 0; 86 87 err_device_create: 88 class_destroy(kfd_class); 89 err_class_create: 90 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 91 err_register_chrdev: 92 return err; 93 } 94 95 void kfd_chardev_exit(void) 96 { 97 device_destroy(kfd_class, MKDEV(kfd_char_dev_major, 0)); 98 class_destroy(kfd_class); 99 unregister_chrdev(kfd_char_dev_major, kfd_dev_name); 100 } 101 102 struct device *kfd_chardev(void) 103 { 104 return kfd_device; 105 } 106 107 108 static int kfd_open(struct inode *inode, struct file *filep) 109 { 110 struct kfd_process *process; 111 bool is_32bit_user_mode; 112 113 if (iminor(inode) != 0) 114 return -ENODEV; 115 116 is_32bit_user_mode = in_compat_syscall(); 117 118 if (is_32bit_user_mode) { 119 dev_warn(kfd_device, 120 "Process %d (32-bit) failed to open /dev/kfd\n" 121 "32-bit processes are not supported by amdkfd\n", 122 current->pid); 123 return -EPERM; 124 } 125 126 process = kfd_create_process(filep); 127 if (IS_ERR(process)) 128 return PTR_ERR(process); 129 130 if (kfd_is_locked()) { 131 dev_dbg(kfd_device, "kfd is locked!\n" 132 "process %d unreferenced", process->pasid); 133 kfd_unref_process(process); 134 return -EAGAIN; 135 } 136 137 /* filep now owns the reference returned by kfd_create_process */ 138 filep->private_data = process; 139 140 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n", 141 process->pasid, process->is_32bit_user_mode); 142 143 return 0; 144 } 145 146 static int kfd_release(struct inode *inode, struct file *filep) 147 { 148 struct kfd_process *process = filep->private_data; 149 150 if (process) 151 kfd_unref_process(process); 152 153 return 0; 154 } 155 156 static int kfd_ioctl_get_version(struct file *filep, struct kfd_process *p, 157 void *data) 158 { 159 struct kfd_ioctl_get_version_args *args = data; 160 161 args->major_version = KFD_IOCTL_MAJOR_VERSION; 162 args->minor_version = KFD_IOCTL_MINOR_VERSION; 163 164 return 0; 165 } 166 167 static int set_queue_properties_from_user(struct queue_properties *q_properties, 168 struct kfd_ioctl_create_queue_args *args) 169 { 170 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 171 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 172 return -EINVAL; 173 } 174 175 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 176 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 177 return -EINVAL; 178 } 179 180 if ((args->ring_base_address) && 181 (!access_ok((const void __user *) args->ring_base_address, 182 sizeof(uint64_t)))) { 183 pr_err("Can't access ring base address\n"); 184 return -EFAULT; 185 } 186 187 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 188 pr_err("Ring size must be a power of 2 or 0\n"); 189 return -EINVAL; 190 } 191 192 if (!access_ok((const void __user *) args->read_pointer_address, 193 sizeof(uint32_t))) { 194 pr_err("Can't access read pointer\n"); 195 return -EFAULT; 196 } 197 198 if (!access_ok((const void __user *) args->write_pointer_address, 199 sizeof(uint32_t))) { 200 pr_err("Can't access write pointer\n"); 201 return -EFAULT; 202 } 203 204 if (args->eop_buffer_address && 205 !access_ok((const void __user *) args->eop_buffer_address, 206 sizeof(uint32_t))) { 207 pr_debug("Can't access eop buffer"); 208 return -EFAULT; 209 } 210 211 if (args->ctx_save_restore_address && 212 !access_ok((const void __user *) args->ctx_save_restore_address, 213 sizeof(uint32_t))) { 214 pr_debug("Can't access ctx save restore buffer"); 215 return -EFAULT; 216 } 217 218 q_properties->is_interop = false; 219 q_properties->is_gws = false; 220 q_properties->queue_percent = args->queue_percentage; 221 q_properties->priority = args->queue_priority; 222 q_properties->queue_address = args->ring_base_address; 223 q_properties->queue_size = args->ring_size; 224 q_properties->read_ptr = (uint32_t *) args->read_pointer_address; 225 q_properties->write_ptr = (uint32_t *) args->write_pointer_address; 226 q_properties->eop_ring_buffer_address = args->eop_buffer_address; 227 q_properties->eop_ring_buffer_size = args->eop_buffer_size; 228 q_properties->ctx_save_restore_area_address = 229 args->ctx_save_restore_address; 230 q_properties->ctx_save_restore_area_size = args->ctx_save_restore_size; 231 q_properties->ctl_stack_size = args->ctl_stack_size; 232 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE || 233 args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 234 q_properties->type = KFD_QUEUE_TYPE_COMPUTE; 235 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA) 236 q_properties->type = KFD_QUEUE_TYPE_SDMA; 237 else if (args->queue_type == KFD_IOC_QUEUE_TYPE_SDMA_XGMI) 238 q_properties->type = KFD_QUEUE_TYPE_SDMA_XGMI; 239 else 240 return -ENOTSUPP; 241 242 if (args->queue_type == KFD_IOC_QUEUE_TYPE_COMPUTE_AQL) 243 q_properties->format = KFD_QUEUE_FORMAT_AQL; 244 else 245 q_properties->format = KFD_QUEUE_FORMAT_PM4; 246 247 pr_debug("Queue Percentage: %d, %d\n", 248 q_properties->queue_percent, args->queue_percentage); 249 250 pr_debug("Queue Priority: %d, %d\n", 251 q_properties->priority, args->queue_priority); 252 253 pr_debug("Queue Address: 0x%llX, 0x%llX\n", 254 q_properties->queue_address, args->ring_base_address); 255 256 pr_debug("Queue Size: 0x%llX, %u\n", 257 q_properties->queue_size, args->ring_size); 258 259 pr_debug("Queue r/w Pointers: %px, %px\n", 260 q_properties->read_ptr, 261 q_properties->write_ptr); 262 263 pr_debug("Queue Format: %d\n", q_properties->format); 264 265 pr_debug("Queue EOP: 0x%llX\n", q_properties->eop_ring_buffer_address); 266 267 pr_debug("Queue CTX save area: 0x%llX\n", 268 q_properties->ctx_save_restore_area_address); 269 270 return 0; 271 } 272 273 static int kfd_ioctl_create_queue(struct file *filep, struct kfd_process *p, 274 void *data) 275 { 276 struct kfd_ioctl_create_queue_args *args = data; 277 struct kfd_dev *dev; 278 int err = 0; 279 unsigned int queue_id; 280 struct kfd_process_device *pdd; 281 struct queue_properties q_properties; 282 uint32_t doorbell_offset_in_process = 0; 283 284 memset(&q_properties, 0, sizeof(struct queue_properties)); 285 286 pr_debug("Creating queue ioctl\n"); 287 288 err = set_queue_properties_from_user(&q_properties, args); 289 if (err) 290 return err; 291 292 pr_debug("Looking for gpu id 0x%x\n", args->gpu_id); 293 dev = kfd_device_by_id(args->gpu_id); 294 if (!dev) { 295 pr_debug("Could not find gpu id 0x%x\n", args->gpu_id); 296 return -EINVAL; 297 } 298 299 mutex_lock(&p->mutex); 300 301 pdd = kfd_bind_process_to_device(dev, p); 302 if (IS_ERR(pdd)) { 303 err = -ESRCH; 304 goto err_bind_process; 305 } 306 307 pr_debug("Creating queue for PASID 0x%x on gpu 0x%x\n", 308 p->pasid, 309 dev->id); 310 311 err = pqm_create_queue(&p->pqm, dev, filep, &q_properties, &queue_id, 312 &doorbell_offset_in_process); 313 if (err != 0) 314 goto err_create_queue; 315 316 args->queue_id = queue_id; 317 318 319 /* Return gpu_id as doorbell offset for mmap usage */ 320 args->doorbell_offset = KFD_MMAP_TYPE_DOORBELL; 321 args->doorbell_offset |= KFD_MMAP_GPU_ID(args->gpu_id); 322 if (KFD_IS_SOC15(dev->device_info->asic_family)) 323 /* On SOC15 ASICs, include the doorbell offset within the 324 * process doorbell frame, which is 2 pages. 325 */ 326 args->doorbell_offset |= doorbell_offset_in_process; 327 328 mutex_unlock(&p->mutex); 329 330 pr_debug("Queue id %d was created successfully\n", args->queue_id); 331 332 pr_debug("Ring buffer address == 0x%016llX\n", 333 args->ring_base_address); 334 335 pr_debug("Read ptr address == 0x%016llX\n", 336 args->read_pointer_address); 337 338 pr_debug("Write ptr address == 0x%016llX\n", 339 args->write_pointer_address); 340 341 return 0; 342 343 err_create_queue: 344 err_bind_process: 345 mutex_unlock(&p->mutex); 346 return err; 347 } 348 349 static int kfd_ioctl_destroy_queue(struct file *filp, struct kfd_process *p, 350 void *data) 351 { 352 int retval; 353 struct kfd_ioctl_destroy_queue_args *args = data; 354 355 pr_debug("Destroying queue id %d for pasid 0x%x\n", 356 args->queue_id, 357 p->pasid); 358 359 mutex_lock(&p->mutex); 360 361 retval = pqm_destroy_queue(&p->pqm, args->queue_id); 362 363 mutex_unlock(&p->mutex); 364 return retval; 365 } 366 367 static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, 368 void *data) 369 { 370 int retval; 371 struct kfd_ioctl_update_queue_args *args = data; 372 struct queue_properties properties; 373 374 if (args->queue_percentage > KFD_MAX_QUEUE_PERCENTAGE) { 375 pr_err("Queue percentage must be between 0 to KFD_MAX_QUEUE_PERCENTAGE\n"); 376 return -EINVAL; 377 } 378 379 if (args->queue_priority > KFD_MAX_QUEUE_PRIORITY) { 380 pr_err("Queue priority must be between 0 to KFD_MAX_QUEUE_PRIORITY\n"); 381 return -EINVAL; 382 } 383 384 if ((args->ring_base_address) && 385 (!access_ok((const void __user *) args->ring_base_address, 386 sizeof(uint64_t)))) { 387 pr_err("Can't access ring base address\n"); 388 return -EFAULT; 389 } 390 391 if (!is_power_of_2(args->ring_size) && (args->ring_size != 0)) { 392 pr_err("Ring size must be a power of 2 or 0\n"); 393 return -EINVAL; 394 } 395 396 properties.queue_address = args->ring_base_address; 397 properties.queue_size = args->ring_size; 398 properties.queue_percent = args->queue_percentage; 399 properties.priority = args->queue_priority; 400 401 pr_debug("Updating queue id %d for pasid 0x%x\n", 402 args->queue_id, p->pasid); 403 404 mutex_lock(&p->mutex); 405 406 retval = pqm_update_queue(&p->pqm, args->queue_id, &properties); 407 408 mutex_unlock(&p->mutex); 409 410 return retval; 411 } 412 413 static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p, 414 void *data) 415 { 416 int retval; 417 const int max_num_cus = 1024; 418 struct kfd_ioctl_set_cu_mask_args *args = data; 419 struct queue_properties properties; 420 uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr; 421 size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32); 422 423 if ((args->num_cu_mask % 32) != 0) { 424 pr_debug("num_cu_mask 0x%x must be a multiple of 32", 425 args->num_cu_mask); 426 return -EINVAL; 427 } 428 429 properties.cu_mask_count = args->num_cu_mask; 430 if (properties.cu_mask_count == 0) { 431 pr_debug("CU mask cannot be 0"); 432 return -EINVAL; 433 } 434 435 /* To prevent an unreasonably large CU mask size, set an arbitrary 436 * limit of max_num_cus bits. We can then just drop any CU mask bits 437 * past max_num_cus bits and just use the first max_num_cus bits. 438 */ 439 if (properties.cu_mask_count > max_num_cus) { 440 pr_debug("CU mask cannot be greater than 1024 bits"); 441 properties.cu_mask_count = max_num_cus; 442 cu_mask_size = sizeof(uint32_t) * (max_num_cus/32); 443 } 444 445 properties.cu_mask = kzalloc(cu_mask_size, GFP_KERNEL); 446 if (!properties.cu_mask) 447 return -ENOMEM; 448 449 retval = copy_from_user(properties.cu_mask, cu_mask_ptr, cu_mask_size); 450 if (retval) { 451 pr_debug("Could not copy CU mask from userspace"); 452 kfree(properties.cu_mask); 453 return -EFAULT; 454 } 455 456 mutex_lock(&p->mutex); 457 458 retval = pqm_set_cu_mask(&p->pqm, args->queue_id, &properties); 459 460 mutex_unlock(&p->mutex); 461 462 if (retval) 463 kfree(properties.cu_mask); 464 465 return retval; 466 } 467 468 static int kfd_ioctl_get_queue_wave_state(struct file *filep, 469 struct kfd_process *p, void *data) 470 { 471 struct kfd_ioctl_get_queue_wave_state_args *args = data; 472 int r; 473 474 mutex_lock(&p->mutex); 475 476 r = pqm_get_wave_state(&p->pqm, args->queue_id, 477 (void __user *)args->ctl_stack_address, 478 &args->ctl_stack_used_size, 479 &args->save_area_used_size); 480 481 mutex_unlock(&p->mutex); 482 483 return r; 484 } 485 486 static int kfd_ioctl_set_memory_policy(struct file *filep, 487 struct kfd_process *p, void *data) 488 { 489 struct kfd_ioctl_set_memory_policy_args *args = data; 490 struct kfd_dev *dev; 491 int err = 0; 492 struct kfd_process_device *pdd; 493 enum cache_policy default_policy, alternate_policy; 494 495 if (args->default_policy != KFD_IOC_CACHE_POLICY_COHERENT 496 && args->default_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 497 return -EINVAL; 498 } 499 500 if (args->alternate_policy != KFD_IOC_CACHE_POLICY_COHERENT 501 && args->alternate_policy != KFD_IOC_CACHE_POLICY_NONCOHERENT) { 502 return -EINVAL; 503 } 504 505 dev = kfd_device_by_id(args->gpu_id); 506 if (!dev) 507 return -EINVAL; 508 509 mutex_lock(&p->mutex); 510 511 pdd = kfd_bind_process_to_device(dev, p); 512 if (IS_ERR(pdd)) { 513 err = -ESRCH; 514 goto out; 515 } 516 517 default_policy = (args->default_policy == KFD_IOC_CACHE_POLICY_COHERENT) 518 ? cache_policy_coherent : cache_policy_noncoherent; 519 520 alternate_policy = 521 (args->alternate_policy == KFD_IOC_CACHE_POLICY_COHERENT) 522 ? cache_policy_coherent : cache_policy_noncoherent; 523 524 if (!dev->dqm->ops.set_cache_memory_policy(dev->dqm, 525 &pdd->qpd, 526 default_policy, 527 alternate_policy, 528 (void __user *)args->alternate_aperture_base, 529 args->alternate_aperture_size)) 530 err = -EINVAL; 531 532 out: 533 mutex_unlock(&p->mutex); 534 535 return err; 536 } 537 538 static int kfd_ioctl_set_trap_handler(struct file *filep, 539 struct kfd_process *p, void *data) 540 { 541 struct kfd_ioctl_set_trap_handler_args *args = data; 542 struct kfd_dev *dev; 543 int err = 0; 544 struct kfd_process_device *pdd; 545 546 dev = kfd_device_by_id(args->gpu_id); 547 if (!dev) 548 return -EINVAL; 549 550 mutex_lock(&p->mutex); 551 552 pdd = kfd_bind_process_to_device(dev, p); 553 if (IS_ERR(pdd)) { 554 err = -ESRCH; 555 goto out; 556 } 557 558 if (dev->dqm->ops.set_trap_handler(dev->dqm, 559 &pdd->qpd, 560 args->tba_addr, 561 args->tma_addr)) 562 err = -EINVAL; 563 564 out: 565 mutex_unlock(&p->mutex); 566 567 return err; 568 } 569 570 static int kfd_ioctl_dbg_register(struct file *filep, 571 struct kfd_process *p, void *data) 572 { 573 struct kfd_ioctl_dbg_register_args *args = data; 574 struct kfd_dev *dev; 575 struct kfd_dbgmgr *dbgmgr_ptr; 576 struct kfd_process_device *pdd; 577 bool create_ok; 578 long status = 0; 579 580 dev = kfd_device_by_id(args->gpu_id); 581 if (!dev) 582 return -EINVAL; 583 584 if (dev->device_info->asic_family == CHIP_CARRIZO) { 585 pr_debug("kfd_ioctl_dbg_register not supported on CZ\n"); 586 return -EINVAL; 587 } 588 589 mutex_lock(&p->mutex); 590 mutex_lock(kfd_get_dbgmgr_mutex()); 591 592 /* 593 * make sure that we have pdd, if this the first queue created for 594 * this process 595 */ 596 pdd = kfd_bind_process_to_device(dev, p); 597 if (IS_ERR(pdd)) { 598 status = PTR_ERR(pdd); 599 goto out; 600 } 601 602 if (!dev->dbgmgr) { 603 /* In case of a legal call, we have no dbgmgr yet */ 604 create_ok = kfd_dbgmgr_create(&dbgmgr_ptr, dev); 605 if (create_ok) { 606 status = kfd_dbgmgr_register(dbgmgr_ptr, p); 607 if (status != 0) 608 kfd_dbgmgr_destroy(dbgmgr_ptr); 609 else 610 dev->dbgmgr = dbgmgr_ptr; 611 } 612 } else { 613 pr_debug("debugger already registered\n"); 614 status = -EINVAL; 615 } 616 617 out: 618 mutex_unlock(kfd_get_dbgmgr_mutex()); 619 mutex_unlock(&p->mutex); 620 621 return status; 622 } 623 624 static int kfd_ioctl_dbg_unregister(struct file *filep, 625 struct kfd_process *p, void *data) 626 { 627 struct kfd_ioctl_dbg_unregister_args *args = data; 628 struct kfd_dev *dev; 629 long status; 630 631 dev = kfd_device_by_id(args->gpu_id); 632 if (!dev || !dev->dbgmgr) 633 return -EINVAL; 634 635 if (dev->device_info->asic_family == CHIP_CARRIZO) { 636 pr_debug("kfd_ioctl_dbg_unregister not supported on CZ\n"); 637 return -EINVAL; 638 } 639 640 mutex_lock(kfd_get_dbgmgr_mutex()); 641 642 status = kfd_dbgmgr_unregister(dev->dbgmgr, p); 643 if (!status) { 644 kfd_dbgmgr_destroy(dev->dbgmgr); 645 dev->dbgmgr = NULL; 646 } 647 648 mutex_unlock(kfd_get_dbgmgr_mutex()); 649 650 return status; 651 } 652 653 /* 654 * Parse and generate variable size data structure for address watch. 655 * Total size of the buffer and # watch points is limited in order 656 * to prevent kernel abuse. (no bearing to the much smaller HW limitation 657 * which is enforced by dbgdev module) 658 * please also note that the watch address itself are not "copied from user", 659 * since it be set into the HW in user mode values. 660 * 661 */ 662 static int kfd_ioctl_dbg_address_watch(struct file *filep, 663 struct kfd_process *p, void *data) 664 { 665 struct kfd_ioctl_dbg_address_watch_args *args = data; 666 struct kfd_dev *dev; 667 struct dbg_address_watch_info aw_info; 668 unsigned char *args_buff; 669 long status; 670 void __user *cmd_from_user; 671 uint64_t watch_mask_value = 0; 672 unsigned int args_idx = 0; 673 674 memset((void *) &aw_info, 0, sizeof(struct dbg_address_watch_info)); 675 676 dev = kfd_device_by_id(args->gpu_id); 677 if (!dev) 678 return -EINVAL; 679 680 if (dev->device_info->asic_family == CHIP_CARRIZO) { 681 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 682 return -EINVAL; 683 } 684 685 cmd_from_user = (void __user *) args->content_ptr; 686 687 /* Validate arguments */ 688 689 if ((args->buf_size_in_bytes > MAX_ALLOWED_AW_BUFF_SIZE) || 690 (args->buf_size_in_bytes <= sizeof(*args) + sizeof(int) * 2) || 691 (cmd_from_user == NULL)) 692 return -EINVAL; 693 694 /* this is the actual buffer to work with */ 695 args_buff = memdup_user(cmd_from_user, 696 args->buf_size_in_bytes - sizeof(*args)); 697 if (IS_ERR(args_buff)) 698 return PTR_ERR(args_buff); 699 700 aw_info.process = p; 701 702 aw_info.num_watch_points = *((uint32_t *)(&args_buff[args_idx])); 703 args_idx += sizeof(aw_info.num_watch_points); 704 705 aw_info.watch_mode = (enum HSA_DBG_WATCH_MODE *) &args_buff[args_idx]; 706 args_idx += sizeof(enum HSA_DBG_WATCH_MODE) * aw_info.num_watch_points; 707 708 /* 709 * set watch address base pointer to point on the array base 710 * within args_buff 711 */ 712 aw_info.watch_address = (uint64_t *) &args_buff[args_idx]; 713 714 /* skip over the addresses buffer */ 715 args_idx += sizeof(aw_info.watch_address) * aw_info.num_watch_points; 716 717 if (args_idx >= args->buf_size_in_bytes - sizeof(*args)) { 718 status = -EINVAL; 719 goto out; 720 } 721 722 watch_mask_value = (uint64_t) args_buff[args_idx]; 723 724 if (watch_mask_value > 0) { 725 /* 726 * There is an array of masks. 727 * set watch mask base pointer to point on the array base 728 * within args_buff 729 */ 730 aw_info.watch_mask = (uint64_t *) &args_buff[args_idx]; 731 732 /* skip over the masks buffer */ 733 args_idx += sizeof(aw_info.watch_mask) * 734 aw_info.num_watch_points; 735 } else { 736 /* just the NULL mask, set to NULL and skip over it */ 737 aw_info.watch_mask = NULL; 738 args_idx += sizeof(aw_info.watch_mask); 739 } 740 741 if (args_idx >= args->buf_size_in_bytes - sizeof(args)) { 742 status = -EINVAL; 743 goto out; 744 } 745 746 /* Currently HSA Event is not supported for DBG */ 747 aw_info.watch_event = NULL; 748 749 mutex_lock(kfd_get_dbgmgr_mutex()); 750 751 status = kfd_dbgmgr_address_watch(dev->dbgmgr, &aw_info); 752 753 mutex_unlock(kfd_get_dbgmgr_mutex()); 754 755 out: 756 kfree(args_buff); 757 758 return status; 759 } 760 761 /* Parse and generate fixed size data structure for wave control */ 762 static int kfd_ioctl_dbg_wave_control(struct file *filep, 763 struct kfd_process *p, void *data) 764 { 765 struct kfd_ioctl_dbg_wave_control_args *args = data; 766 struct kfd_dev *dev; 767 struct dbg_wave_control_info wac_info; 768 unsigned char *args_buff; 769 uint32_t computed_buff_size; 770 long status; 771 void __user *cmd_from_user; 772 unsigned int args_idx = 0; 773 774 memset((void *) &wac_info, 0, sizeof(struct dbg_wave_control_info)); 775 776 /* we use compact form, independent of the packing attribute value */ 777 computed_buff_size = sizeof(*args) + 778 sizeof(wac_info.mode) + 779 sizeof(wac_info.operand) + 780 sizeof(wac_info.dbgWave_msg.DbgWaveMsg) + 781 sizeof(wac_info.dbgWave_msg.MemoryVA) + 782 sizeof(wac_info.trapId); 783 784 dev = kfd_device_by_id(args->gpu_id); 785 if (!dev) 786 return -EINVAL; 787 788 if (dev->device_info->asic_family == CHIP_CARRIZO) { 789 pr_debug("kfd_ioctl_dbg_wave_control not supported on CZ\n"); 790 return -EINVAL; 791 } 792 793 /* input size must match the computed "compact" size */ 794 if (args->buf_size_in_bytes != computed_buff_size) { 795 pr_debug("size mismatch, computed : actual %u : %u\n", 796 args->buf_size_in_bytes, computed_buff_size); 797 return -EINVAL; 798 } 799 800 cmd_from_user = (void __user *) args->content_ptr; 801 802 if (cmd_from_user == NULL) 803 return -EINVAL; 804 805 /* copy the entire buffer from user */ 806 807 args_buff = memdup_user(cmd_from_user, 808 args->buf_size_in_bytes - sizeof(*args)); 809 if (IS_ERR(args_buff)) 810 return PTR_ERR(args_buff); 811 812 /* move ptr to the start of the "pay-load" area */ 813 wac_info.process = p; 814 815 wac_info.operand = *((enum HSA_DBG_WAVEOP *)(&args_buff[args_idx])); 816 args_idx += sizeof(wac_info.operand); 817 818 wac_info.mode = *((enum HSA_DBG_WAVEMODE *)(&args_buff[args_idx])); 819 args_idx += sizeof(wac_info.mode); 820 821 wac_info.trapId = *((uint32_t *)(&args_buff[args_idx])); 822 args_idx += sizeof(wac_info.trapId); 823 824 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value = 825 *((uint32_t *)(&args_buff[args_idx])); 826 wac_info.dbgWave_msg.MemoryVA = NULL; 827 828 mutex_lock(kfd_get_dbgmgr_mutex()); 829 830 pr_debug("Calling dbg manager process %p, operand %u, mode %u, trapId %u, message %u\n", 831 wac_info.process, wac_info.operand, 832 wac_info.mode, wac_info.trapId, 833 wac_info.dbgWave_msg.DbgWaveMsg.WaveMsgInfoGen2.Value); 834 835 status = kfd_dbgmgr_wave_control(dev->dbgmgr, &wac_info); 836 837 pr_debug("Returned status of dbg manager is %ld\n", status); 838 839 mutex_unlock(kfd_get_dbgmgr_mutex()); 840 841 kfree(args_buff); 842 843 return status; 844 } 845 846 static int kfd_ioctl_get_clock_counters(struct file *filep, 847 struct kfd_process *p, void *data) 848 { 849 struct kfd_ioctl_get_clock_counters_args *args = data; 850 struct kfd_dev *dev; 851 852 dev = kfd_device_by_id(args->gpu_id); 853 if (dev) 854 /* Reading GPU clock counter from KGD */ 855 args->gpu_clock_counter = amdgpu_amdkfd_get_gpu_clock_counter(dev->kgd); 856 else 857 /* Node without GPU resource */ 858 args->gpu_clock_counter = 0; 859 860 /* No access to rdtsc. Using raw monotonic time */ 861 args->cpu_clock_counter = ktime_get_raw_ns(); 862 args->system_clock_counter = ktime_get_boottime_ns(); 863 864 /* Since the counter is in nano-seconds we use 1GHz frequency */ 865 args->system_clock_freq = 1000000000; 866 867 return 0; 868 } 869 870 871 static int kfd_ioctl_get_process_apertures(struct file *filp, 872 struct kfd_process *p, void *data) 873 { 874 struct kfd_ioctl_get_process_apertures_args *args = data; 875 struct kfd_process_device_apertures *pAperture; 876 struct kfd_process_device *pdd; 877 878 dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid); 879 880 args->num_of_nodes = 0; 881 882 mutex_lock(&p->mutex); 883 884 /*if the process-device list isn't empty*/ 885 if (kfd_has_process_device_data(p)) { 886 /* Run over all pdd of the process */ 887 pdd = kfd_get_first_process_device_data(p); 888 do { 889 pAperture = 890 &args->process_apertures[args->num_of_nodes]; 891 pAperture->gpu_id = pdd->dev->id; 892 pAperture->lds_base = pdd->lds_base; 893 pAperture->lds_limit = pdd->lds_limit; 894 pAperture->gpuvm_base = pdd->gpuvm_base; 895 pAperture->gpuvm_limit = pdd->gpuvm_limit; 896 pAperture->scratch_base = pdd->scratch_base; 897 pAperture->scratch_limit = pdd->scratch_limit; 898 899 dev_dbg(kfd_device, 900 "node id %u\n", args->num_of_nodes); 901 dev_dbg(kfd_device, 902 "gpu id %u\n", pdd->dev->id); 903 dev_dbg(kfd_device, 904 "lds_base %llX\n", pdd->lds_base); 905 dev_dbg(kfd_device, 906 "lds_limit %llX\n", pdd->lds_limit); 907 dev_dbg(kfd_device, 908 "gpuvm_base %llX\n", pdd->gpuvm_base); 909 dev_dbg(kfd_device, 910 "gpuvm_limit %llX\n", pdd->gpuvm_limit); 911 dev_dbg(kfd_device, 912 "scratch_base %llX\n", pdd->scratch_base); 913 dev_dbg(kfd_device, 914 "scratch_limit %llX\n", pdd->scratch_limit); 915 916 args->num_of_nodes++; 917 918 pdd = kfd_get_next_process_device_data(p, pdd); 919 } while (pdd && (args->num_of_nodes < NUM_OF_SUPPORTED_GPUS)); 920 } 921 922 mutex_unlock(&p->mutex); 923 924 return 0; 925 } 926 927 static int kfd_ioctl_get_process_apertures_new(struct file *filp, 928 struct kfd_process *p, void *data) 929 { 930 struct kfd_ioctl_get_process_apertures_new_args *args = data; 931 struct kfd_process_device_apertures *pa; 932 struct kfd_process_device *pdd; 933 uint32_t nodes = 0; 934 int ret; 935 936 dev_dbg(kfd_device, "get apertures for PASID 0x%x", p->pasid); 937 938 if (args->num_of_nodes == 0) { 939 /* Return number of nodes, so that user space can alloacate 940 * sufficient memory 941 */ 942 mutex_lock(&p->mutex); 943 944 if (!kfd_has_process_device_data(p)) 945 goto out_unlock; 946 947 /* Run over all pdd of the process */ 948 pdd = kfd_get_first_process_device_data(p); 949 do { 950 args->num_of_nodes++; 951 pdd = kfd_get_next_process_device_data(p, pdd); 952 } while (pdd); 953 954 goto out_unlock; 955 } 956 957 /* Fill in process-aperture information for all available 958 * nodes, but not more than args->num_of_nodes as that is 959 * the amount of memory allocated by user 960 */ 961 pa = kzalloc((sizeof(struct kfd_process_device_apertures) * 962 args->num_of_nodes), GFP_KERNEL); 963 if (!pa) 964 return -ENOMEM; 965 966 mutex_lock(&p->mutex); 967 968 if (!kfd_has_process_device_data(p)) { 969 args->num_of_nodes = 0; 970 kfree(pa); 971 goto out_unlock; 972 } 973 974 /* Run over all pdd of the process */ 975 pdd = kfd_get_first_process_device_data(p); 976 do { 977 pa[nodes].gpu_id = pdd->dev->id; 978 pa[nodes].lds_base = pdd->lds_base; 979 pa[nodes].lds_limit = pdd->lds_limit; 980 pa[nodes].gpuvm_base = pdd->gpuvm_base; 981 pa[nodes].gpuvm_limit = pdd->gpuvm_limit; 982 pa[nodes].scratch_base = pdd->scratch_base; 983 pa[nodes].scratch_limit = pdd->scratch_limit; 984 985 dev_dbg(kfd_device, 986 "gpu id %u\n", pdd->dev->id); 987 dev_dbg(kfd_device, 988 "lds_base %llX\n", pdd->lds_base); 989 dev_dbg(kfd_device, 990 "lds_limit %llX\n", pdd->lds_limit); 991 dev_dbg(kfd_device, 992 "gpuvm_base %llX\n", pdd->gpuvm_base); 993 dev_dbg(kfd_device, 994 "gpuvm_limit %llX\n", pdd->gpuvm_limit); 995 dev_dbg(kfd_device, 996 "scratch_base %llX\n", pdd->scratch_base); 997 dev_dbg(kfd_device, 998 "scratch_limit %llX\n", pdd->scratch_limit); 999 nodes++; 1000 1001 pdd = kfd_get_next_process_device_data(p, pdd); 1002 } while (pdd && (nodes < args->num_of_nodes)); 1003 mutex_unlock(&p->mutex); 1004 1005 args->num_of_nodes = nodes; 1006 ret = copy_to_user( 1007 (void __user *)args->kfd_process_device_apertures_ptr, 1008 pa, 1009 (nodes * sizeof(struct kfd_process_device_apertures))); 1010 kfree(pa); 1011 return ret ? -EFAULT : 0; 1012 1013 out_unlock: 1014 mutex_unlock(&p->mutex); 1015 return 0; 1016 } 1017 1018 static int kfd_ioctl_create_event(struct file *filp, struct kfd_process *p, 1019 void *data) 1020 { 1021 struct kfd_ioctl_create_event_args *args = data; 1022 int err; 1023 1024 /* For dGPUs the event page is allocated in user mode. The 1025 * handle is passed to KFD with the first call to this IOCTL 1026 * through the event_page_offset field. 1027 */ 1028 if (args->event_page_offset) { 1029 struct kfd_dev *kfd; 1030 struct kfd_process_device *pdd; 1031 void *mem, *kern_addr; 1032 uint64_t size; 1033 1034 if (p->signal_page) { 1035 pr_err("Event page is already set\n"); 1036 return -EINVAL; 1037 } 1038 1039 kfd = kfd_device_by_id(GET_GPU_ID(args->event_page_offset)); 1040 if (!kfd) { 1041 pr_err("Getting device by id failed in %s\n", __func__); 1042 return -EINVAL; 1043 } 1044 1045 mutex_lock(&p->mutex); 1046 pdd = kfd_bind_process_to_device(kfd, p); 1047 if (IS_ERR(pdd)) { 1048 err = PTR_ERR(pdd); 1049 goto out_unlock; 1050 } 1051 1052 mem = kfd_process_device_translate_handle(pdd, 1053 GET_IDR_HANDLE(args->event_page_offset)); 1054 if (!mem) { 1055 pr_err("Can't find BO, offset is 0x%llx\n", 1056 args->event_page_offset); 1057 err = -EINVAL; 1058 goto out_unlock; 1059 } 1060 mutex_unlock(&p->mutex); 1061 1062 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->kgd, 1063 mem, &kern_addr, &size); 1064 if (err) { 1065 pr_err("Failed to map event page to kernel\n"); 1066 return err; 1067 } 1068 1069 err = kfd_event_page_set(p, kern_addr, size); 1070 if (err) { 1071 pr_err("Failed to set event page\n"); 1072 return err; 1073 } 1074 } 1075 1076 err = kfd_event_create(filp, p, args->event_type, 1077 args->auto_reset != 0, args->node_id, 1078 &args->event_id, &args->event_trigger_data, 1079 &args->event_page_offset, 1080 &args->event_slot_index); 1081 1082 return err; 1083 1084 out_unlock: 1085 mutex_unlock(&p->mutex); 1086 return err; 1087 } 1088 1089 static int kfd_ioctl_destroy_event(struct file *filp, struct kfd_process *p, 1090 void *data) 1091 { 1092 struct kfd_ioctl_destroy_event_args *args = data; 1093 1094 return kfd_event_destroy(p, args->event_id); 1095 } 1096 1097 static int kfd_ioctl_set_event(struct file *filp, struct kfd_process *p, 1098 void *data) 1099 { 1100 struct kfd_ioctl_set_event_args *args = data; 1101 1102 return kfd_set_event(p, args->event_id); 1103 } 1104 1105 static int kfd_ioctl_reset_event(struct file *filp, struct kfd_process *p, 1106 void *data) 1107 { 1108 struct kfd_ioctl_reset_event_args *args = data; 1109 1110 return kfd_reset_event(p, args->event_id); 1111 } 1112 1113 static int kfd_ioctl_wait_events(struct file *filp, struct kfd_process *p, 1114 void *data) 1115 { 1116 struct kfd_ioctl_wait_events_args *args = data; 1117 int err; 1118 1119 err = kfd_wait_on_events(p, args->num_events, 1120 (void __user *)args->events_ptr, 1121 (args->wait_for_all != 0), 1122 args->timeout, &args->wait_result); 1123 1124 return err; 1125 } 1126 static int kfd_ioctl_set_scratch_backing_va(struct file *filep, 1127 struct kfd_process *p, void *data) 1128 { 1129 struct kfd_ioctl_set_scratch_backing_va_args *args = data; 1130 struct kfd_process_device *pdd; 1131 struct kfd_dev *dev; 1132 long err; 1133 1134 dev = kfd_device_by_id(args->gpu_id); 1135 if (!dev) 1136 return -EINVAL; 1137 1138 mutex_lock(&p->mutex); 1139 1140 pdd = kfd_bind_process_to_device(dev, p); 1141 if (IS_ERR(pdd)) { 1142 err = PTR_ERR(pdd); 1143 goto bind_process_to_device_fail; 1144 } 1145 1146 pdd->qpd.sh_hidden_private_base = args->va_addr; 1147 1148 mutex_unlock(&p->mutex); 1149 1150 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS && 1151 pdd->qpd.vmid != 0 && dev->kfd2kgd->set_scratch_backing_va) 1152 dev->kfd2kgd->set_scratch_backing_va( 1153 dev->kgd, args->va_addr, pdd->qpd.vmid); 1154 1155 return 0; 1156 1157 bind_process_to_device_fail: 1158 mutex_unlock(&p->mutex); 1159 return err; 1160 } 1161 1162 static int kfd_ioctl_get_tile_config(struct file *filep, 1163 struct kfd_process *p, void *data) 1164 { 1165 struct kfd_ioctl_get_tile_config_args *args = data; 1166 struct kfd_dev *dev; 1167 struct tile_config config; 1168 int err = 0; 1169 1170 dev = kfd_device_by_id(args->gpu_id); 1171 if (!dev) 1172 return -EINVAL; 1173 1174 amdgpu_amdkfd_get_tile_config(dev->kgd, &config); 1175 1176 args->gb_addr_config = config.gb_addr_config; 1177 args->num_banks = config.num_banks; 1178 args->num_ranks = config.num_ranks; 1179 1180 if (args->num_tile_configs > config.num_tile_configs) 1181 args->num_tile_configs = config.num_tile_configs; 1182 err = copy_to_user((void __user *)args->tile_config_ptr, 1183 config.tile_config_ptr, 1184 args->num_tile_configs * sizeof(uint32_t)); 1185 if (err) { 1186 args->num_tile_configs = 0; 1187 return -EFAULT; 1188 } 1189 1190 if (args->num_macro_tile_configs > config.num_macro_tile_configs) 1191 args->num_macro_tile_configs = 1192 config.num_macro_tile_configs; 1193 err = copy_to_user((void __user *)args->macro_tile_config_ptr, 1194 config.macro_tile_config_ptr, 1195 args->num_macro_tile_configs * sizeof(uint32_t)); 1196 if (err) { 1197 args->num_macro_tile_configs = 0; 1198 return -EFAULT; 1199 } 1200 1201 return 0; 1202 } 1203 1204 static int kfd_ioctl_acquire_vm(struct file *filep, struct kfd_process *p, 1205 void *data) 1206 { 1207 struct kfd_ioctl_acquire_vm_args *args = data; 1208 struct kfd_process_device *pdd; 1209 struct kfd_dev *dev; 1210 struct file *drm_file; 1211 int ret; 1212 1213 dev = kfd_device_by_id(args->gpu_id); 1214 if (!dev) 1215 return -EINVAL; 1216 1217 drm_file = fget(args->drm_fd); 1218 if (!drm_file) 1219 return -EINVAL; 1220 1221 mutex_lock(&p->mutex); 1222 1223 pdd = kfd_get_process_device_data(dev, p); 1224 if (!pdd) { 1225 ret = -EINVAL; 1226 goto err_unlock; 1227 } 1228 1229 if (pdd->drm_file) { 1230 ret = pdd->drm_file == drm_file ? 0 : -EBUSY; 1231 goto err_unlock; 1232 } 1233 1234 ret = kfd_process_device_init_vm(pdd, drm_file); 1235 if (ret) 1236 goto err_unlock; 1237 /* On success, the PDD keeps the drm_file reference */ 1238 mutex_unlock(&p->mutex); 1239 1240 return 0; 1241 1242 err_unlock: 1243 mutex_unlock(&p->mutex); 1244 fput(drm_file); 1245 return ret; 1246 } 1247 1248 bool kfd_dev_is_large_bar(struct kfd_dev *dev) 1249 { 1250 struct kfd_local_mem_info mem_info; 1251 1252 if (debug_largebar) { 1253 pr_debug("Simulate large-bar allocation on non large-bar machine\n"); 1254 return true; 1255 } 1256 1257 if (dev->device_info->needs_iommu_device) 1258 return false; 1259 1260 amdgpu_amdkfd_get_local_mem_info(dev->kgd, &mem_info); 1261 if (mem_info.local_mem_size_private == 0 && 1262 mem_info.local_mem_size_public > 0) 1263 return true; 1264 return false; 1265 } 1266 1267 static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, 1268 struct kfd_process *p, void *data) 1269 { 1270 struct kfd_ioctl_alloc_memory_of_gpu_args *args = data; 1271 struct kfd_process_device *pdd; 1272 void *mem; 1273 struct kfd_dev *dev; 1274 int idr_handle; 1275 long err; 1276 uint64_t offset = args->mmap_offset; 1277 uint32_t flags = args->flags; 1278 1279 if (args->size == 0) 1280 return -EINVAL; 1281 1282 dev = kfd_device_by_id(args->gpu_id); 1283 if (!dev) 1284 return -EINVAL; 1285 1286 if ((flags & KFD_IOC_ALLOC_MEM_FLAGS_PUBLIC) && 1287 (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) && 1288 !kfd_dev_is_large_bar(dev)) { 1289 pr_err("Alloc host visible vram on small bar is not allowed\n"); 1290 return -EINVAL; 1291 } 1292 1293 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) { 1294 if (args->size != kfd_doorbell_process_slice(dev)) 1295 return -EINVAL; 1296 offset = kfd_get_process_doorbells(dev, p); 1297 } else if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) { 1298 if (args->size != PAGE_SIZE) 1299 return -EINVAL; 1300 offset = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd); 1301 if (!offset) 1302 return -ENOMEM; 1303 } 1304 1305 mutex_lock(&p->mutex); 1306 1307 pdd = kfd_bind_process_to_device(dev, p); 1308 if (IS_ERR(pdd)) { 1309 err = PTR_ERR(pdd); 1310 goto err_unlock; 1311 } 1312 1313 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( 1314 dev->kgd, args->va_addr, args->size, 1315 pdd->vm, (struct kgd_mem **) &mem, &offset, 1316 flags); 1317 1318 if (err) 1319 goto err_unlock; 1320 1321 idr_handle = kfd_process_device_create_obj_handle(pdd, mem); 1322 if (idr_handle < 0) { 1323 err = -EFAULT; 1324 goto err_free; 1325 } 1326 1327 /* Update the VRAM usage count */ 1328 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) 1329 WRITE_ONCE(pdd->vram_usage, pdd->vram_usage + args->size); 1330 1331 mutex_unlock(&p->mutex); 1332 1333 args->handle = MAKE_HANDLE(args->gpu_id, idr_handle); 1334 args->mmap_offset = offset; 1335 1336 /* MMIO is mapped through kfd device 1337 * Generate a kfd mmap offset 1338 */ 1339 if (flags & KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP) 1340 args->mmap_offset = KFD_MMAP_TYPE_MMIO 1341 | KFD_MMAP_GPU_ID(args->gpu_id); 1342 1343 return 0; 1344 1345 err_free: 1346 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL); 1347 err_unlock: 1348 mutex_unlock(&p->mutex); 1349 return err; 1350 } 1351 1352 static int kfd_ioctl_free_memory_of_gpu(struct file *filep, 1353 struct kfd_process *p, void *data) 1354 { 1355 struct kfd_ioctl_free_memory_of_gpu_args *args = data; 1356 struct kfd_process_device *pdd; 1357 void *mem; 1358 struct kfd_dev *dev; 1359 int ret; 1360 uint64_t size = 0; 1361 1362 dev = kfd_device_by_id(GET_GPU_ID(args->handle)); 1363 if (!dev) 1364 return -EINVAL; 1365 1366 mutex_lock(&p->mutex); 1367 1368 pdd = kfd_get_process_device_data(dev, p); 1369 if (!pdd) { 1370 pr_err("Process device data doesn't exist\n"); 1371 ret = -EINVAL; 1372 goto err_unlock; 1373 } 1374 1375 mem = kfd_process_device_translate_handle( 1376 pdd, GET_IDR_HANDLE(args->handle)); 1377 if (!mem) { 1378 ret = -EINVAL; 1379 goto err_unlock; 1380 } 1381 1382 ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, 1383 (struct kgd_mem *)mem, &size); 1384 1385 /* If freeing the buffer failed, leave the handle in place for 1386 * clean-up during process tear-down. 1387 */ 1388 if (!ret) 1389 kfd_process_device_remove_obj_handle( 1390 pdd, GET_IDR_HANDLE(args->handle)); 1391 1392 WRITE_ONCE(pdd->vram_usage, pdd->vram_usage - size); 1393 1394 err_unlock: 1395 mutex_unlock(&p->mutex); 1396 return ret; 1397 } 1398 1399 static int kfd_ioctl_map_memory_to_gpu(struct file *filep, 1400 struct kfd_process *p, void *data) 1401 { 1402 struct kfd_ioctl_map_memory_to_gpu_args *args = data; 1403 struct kfd_process_device *pdd, *peer_pdd; 1404 void *mem; 1405 struct kfd_dev *dev, *peer; 1406 long err = 0; 1407 int i; 1408 uint32_t *devices_arr = NULL; 1409 1410 dev = kfd_device_by_id(GET_GPU_ID(args->handle)); 1411 if (!dev) 1412 return -EINVAL; 1413 1414 if (!args->n_devices) { 1415 pr_debug("Device IDs array empty\n"); 1416 return -EINVAL; 1417 } 1418 if (args->n_success > args->n_devices) { 1419 pr_debug("n_success exceeds n_devices\n"); 1420 return -EINVAL; 1421 } 1422 1423 devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr), 1424 GFP_KERNEL); 1425 if (!devices_arr) 1426 return -ENOMEM; 1427 1428 err = copy_from_user(devices_arr, 1429 (void __user *)args->device_ids_array_ptr, 1430 args->n_devices * sizeof(*devices_arr)); 1431 if (err != 0) { 1432 err = -EFAULT; 1433 goto copy_from_user_failed; 1434 } 1435 1436 mutex_lock(&p->mutex); 1437 1438 pdd = kfd_bind_process_to_device(dev, p); 1439 if (IS_ERR(pdd)) { 1440 err = PTR_ERR(pdd); 1441 goto bind_process_to_device_failed; 1442 } 1443 1444 mem = kfd_process_device_translate_handle(pdd, 1445 GET_IDR_HANDLE(args->handle)); 1446 if (!mem) { 1447 err = -ENOMEM; 1448 goto get_mem_obj_from_handle_failed; 1449 } 1450 1451 for (i = args->n_success; i < args->n_devices; i++) { 1452 peer = kfd_device_by_id(devices_arr[i]); 1453 if (!peer) { 1454 pr_debug("Getting device by id failed for 0x%x\n", 1455 devices_arr[i]); 1456 err = -EINVAL; 1457 goto get_mem_obj_from_handle_failed; 1458 } 1459 1460 peer_pdd = kfd_bind_process_to_device(peer, p); 1461 if (IS_ERR(peer_pdd)) { 1462 err = PTR_ERR(peer_pdd); 1463 goto get_mem_obj_from_handle_failed; 1464 } 1465 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu( 1466 peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm); 1467 if (err) { 1468 pr_err("Failed to map to gpu %d/%d\n", 1469 i, args->n_devices); 1470 goto map_memory_to_gpu_failed; 1471 } 1472 args->n_success = i+1; 1473 } 1474 1475 mutex_unlock(&p->mutex); 1476 1477 err = amdgpu_amdkfd_gpuvm_sync_memory(dev->kgd, (struct kgd_mem *) mem, true); 1478 if (err) { 1479 pr_debug("Sync memory failed, wait interrupted by user signal\n"); 1480 goto sync_memory_failed; 1481 } 1482 1483 /* Flush TLBs after waiting for the page table updates to complete */ 1484 for (i = 0; i < args->n_devices; i++) { 1485 peer = kfd_device_by_id(devices_arr[i]); 1486 if (WARN_ON_ONCE(!peer)) 1487 continue; 1488 peer_pdd = kfd_get_process_device_data(peer, p); 1489 if (WARN_ON_ONCE(!peer_pdd)) 1490 continue; 1491 kfd_flush_tlb(peer_pdd); 1492 } 1493 1494 kfree(devices_arr); 1495 1496 return err; 1497 1498 bind_process_to_device_failed: 1499 get_mem_obj_from_handle_failed: 1500 map_memory_to_gpu_failed: 1501 mutex_unlock(&p->mutex); 1502 copy_from_user_failed: 1503 sync_memory_failed: 1504 kfree(devices_arr); 1505 1506 return err; 1507 } 1508 1509 static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep, 1510 struct kfd_process *p, void *data) 1511 { 1512 struct kfd_ioctl_unmap_memory_from_gpu_args *args = data; 1513 struct kfd_process_device *pdd, *peer_pdd; 1514 void *mem; 1515 struct kfd_dev *dev, *peer; 1516 long err = 0; 1517 uint32_t *devices_arr = NULL, i; 1518 1519 dev = kfd_device_by_id(GET_GPU_ID(args->handle)); 1520 if (!dev) 1521 return -EINVAL; 1522 1523 if (!args->n_devices) { 1524 pr_debug("Device IDs array empty\n"); 1525 return -EINVAL; 1526 } 1527 if (args->n_success > args->n_devices) { 1528 pr_debug("n_success exceeds n_devices\n"); 1529 return -EINVAL; 1530 } 1531 1532 devices_arr = kmalloc_array(args->n_devices, sizeof(*devices_arr), 1533 GFP_KERNEL); 1534 if (!devices_arr) 1535 return -ENOMEM; 1536 1537 err = copy_from_user(devices_arr, 1538 (void __user *)args->device_ids_array_ptr, 1539 args->n_devices * sizeof(*devices_arr)); 1540 if (err != 0) { 1541 err = -EFAULT; 1542 goto copy_from_user_failed; 1543 } 1544 1545 mutex_lock(&p->mutex); 1546 1547 pdd = kfd_get_process_device_data(dev, p); 1548 if (!pdd) { 1549 err = -EINVAL; 1550 goto bind_process_to_device_failed; 1551 } 1552 1553 mem = kfd_process_device_translate_handle(pdd, 1554 GET_IDR_HANDLE(args->handle)); 1555 if (!mem) { 1556 err = -ENOMEM; 1557 goto get_mem_obj_from_handle_failed; 1558 } 1559 1560 for (i = args->n_success; i < args->n_devices; i++) { 1561 peer = kfd_device_by_id(devices_arr[i]); 1562 if (!peer) { 1563 err = -EINVAL; 1564 goto get_mem_obj_from_handle_failed; 1565 } 1566 1567 peer_pdd = kfd_get_process_device_data(peer, p); 1568 if (!peer_pdd) { 1569 err = -ENODEV; 1570 goto get_mem_obj_from_handle_failed; 1571 } 1572 err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( 1573 peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm); 1574 if (err) { 1575 pr_err("Failed to unmap from gpu %d/%d\n", 1576 i, args->n_devices); 1577 goto unmap_memory_from_gpu_failed; 1578 } 1579 args->n_success = i+1; 1580 } 1581 kfree(devices_arr); 1582 1583 mutex_unlock(&p->mutex); 1584 1585 return 0; 1586 1587 bind_process_to_device_failed: 1588 get_mem_obj_from_handle_failed: 1589 unmap_memory_from_gpu_failed: 1590 mutex_unlock(&p->mutex); 1591 copy_from_user_failed: 1592 kfree(devices_arr); 1593 return err; 1594 } 1595 1596 static int kfd_ioctl_alloc_queue_gws(struct file *filep, 1597 struct kfd_process *p, void *data) 1598 { 1599 int retval; 1600 struct kfd_ioctl_alloc_queue_gws_args *args = data; 1601 struct queue *q; 1602 struct kfd_dev *dev; 1603 1604 mutex_lock(&p->mutex); 1605 q = pqm_get_user_queue(&p->pqm, args->queue_id); 1606 1607 if (q) { 1608 dev = q->device; 1609 } else { 1610 retval = -EINVAL; 1611 goto out_unlock; 1612 } 1613 1614 if (!dev->gws) { 1615 retval = -ENODEV; 1616 goto out_unlock; 1617 } 1618 1619 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 1620 retval = -ENODEV; 1621 goto out_unlock; 1622 } 1623 1624 retval = pqm_set_gws(&p->pqm, args->queue_id, args->num_gws ? dev->gws : NULL); 1625 mutex_unlock(&p->mutex); 1626 1627 args->first_gws = 0; 1628 return retval; 1629 1630 out_unlock: 1631 mutex_unlock(&p->mutex); 1632 return retval; 1633 } 1634 1635 static int kfd_ioctl_get_dmabuf_info(struct file *filep, 1636 struct kfd_process *p, void *data) 1637 { 1638 struct kfd_ioctl_get_dmabuf_info_args *args = data; 1639 struct kfd_dev *dev = NULL; 1640 struct kgd_dev *dma_buf_kgd; 1641 void *metadata_buffer = NULL; 1642 uint32_t flags; 1643 unsigned int i; 1644 int r; 1645 1646 /* Find a KFD GPU device that supports the get_dmabuf_info query */ 1647 for (i = 0; kfd_topology_enum_kfd_devices(i, &dev) == 0; i++) 1648 if (dev) 1649 break; 1650 if (!dev) 1651 return -EINVAL; 1652 1653 if (args->metadata_ptr) { 1654 metadata_buffer = kzalloc(args->metadata_size, GFP_KERNEL); 1655 if (!metadata_buffer) 1656 return -ENOMEM; 1657 } 1658 1659 /* Get dmabuf info from KGD */ 1660 r = amdgpu_amdkfd_get_dmabuf_info(dev->kgd, args->dmabuf_fd, 1661 &dma_buf_kgd, &args->size, 1662 metadata_buffer, args->metadata_size, 1663 &args->metadata_size, &flags); 1664 if (r) 1665 goto exit; 1666 1667 /* Reverse-lookup gpu_id from kgd pointer */ 1668 dev = kfd_device_by_kgd(dma_buf_kgd); 1669 if (!dev) { 1670 r = -EINVAL; 1671 goto exit; 1672 } 1673 args->gpu_id = dev->id; 1674 args->flags = flags; 1675 1676 /* Copy metadata buffer to user mode */ 1677 if (metadata_buffer) { 1678 r = copy_to_user((void __user *)args->metadata_ptr, 1679 metadata_buffer, args->metadata_size); 1680 if (r != 0) 1681 r = -EFAULT; 1682 } 1683 1684 exit: 1685 kfree(metadata_buffer); 1686 1687 return r; 1688 } 1689 1690 static int kfd_ioctl_import_dmabuf(struct file *filep, 1691 struct kfd_process *p, void *data) 1692 { 1693 struct kfd_ioctl_import_dmabuf_args *args = data; 1694 struct kfd_process_device *pdd; 1695 struct dma_buf *dmabuf; 1696 struct kfd_dev *dev; 1697 int idr_handle; 1698 uint64_t size; 1699 void *mem; 1700 int r; 1701 1702 dev = kfd_device_by_id(args->gpu_id); 1703 if (!dev) 1704 return -EINVAL; 1705 1706 dmabuf = dma_buf_get(args->dmabuf_fd); 1707 if (IS_ERR(dmabuf)) 1708 return PTR_ERR(dmabuf); 1709 1710 mutex_lock(&p->mutex); 1711 1712 pdd = kfd_bind_process_to_device(dev, p); 1713 if (IS_ERR(pdd)) { 1714 r = PTR_ERR(pdd); 1715 goto err_unlock; 1716 } 1717 1718 r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf, 1719 args->va_addr, pdd->vm, 1720 (struct kgd_mem **)&mem, &size, 1721 NULL); 1722 if (r) 1723 goto err_unlock; 1724 1725 idr_handle = kfd_process_device_create_obj_handle(pdd, mem); 1726 if (idr_handle < 0) { 1727 r = -EFAULT; 1728 goto err_free; 1729 } 1730 1731 mutex_unlock(&p->mutex); 1732 1733 args->handle = MAKE_HANDLE(args->gpu_id, idr_handle); 1734 1735 return 0; 1736 1737 err_free: 1738 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL); 1739 err_unlock: 1740 mutex_unlock(&p->mutex); 1741 return r; 1742 } 1743 1744 /* Handle requests for watching SMI events */ 1745 static int kfd_ioctl_smi_events(struct file *filep, 1746 struct kfd_process *p, void *data) 1747 { 1748 struct kfd_ioctl_smi_events_args *args = data; 1749 struct kfd_dev *dev; 1750 1751 dev = kfd_device_by_id(args->gpuid); 1752 if (!dev) 1753 return -EINVAL; 1754 1755 return kfd_smi_event_open(dev, &args->anon_fd); 1756 } 1757 1758 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \ 1759 [_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \ 1760 .cmd_drv = 0, .name = #ioctl} 1761 1762 /** Ioctl table */ 1763 static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { 1764 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_VERSION, 1765 kfd_ioctl_get_version, 0), 1766 1767 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_QUEUE, 1768 kfd_ioctl_create_queue, 0), 1769 1770 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_QUEUE, 1771 kfd_ioctl_destroy_queue, 0), 1772 1773 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_MEMORY_POLICY, 1774 kfd_ioctl_set_memory_policy, 0), 1775 1776 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_CLOCK_COUNTERS, 1777 kfd_ioctl_get_clock_counters, 0), 1778 1779 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES, 1780 kfd_ioctl_get_process_apertures, 0), 1781 1782 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UPDATE_QUEUE, 1783 kfd_ioctl_update_queue, 0), 1784 1785 AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_EVENT, 1786 kfd_ioctl_create_event, 0), 1787 1788 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DESTROY_EVENT, 1789 kfd_ioctl_destroy_event, 0), 1790 1791 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_EVENT, 1792 kfd_ioctl_set_event, 0), 1793 1794 AMDKFD_IOCTL_DEF(AMDKFD_IOC_RESET_EVENT, 1795 kfd_ioctl_reset_event, 0), 1796 1797 AMDKFD_IOCTL_DEF(AMDKFD_IOC_WAIT_EVENTS, 1798 kfd_ioctl_wait_events, 0), 1799 1800 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_REGISTER, 1801 kfd_ioctl_dbg_register, 0), 1802 1803 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_UNREGISTER, 1804 kfd_ioctl_dbg_unregister, 0), 1805 1806 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_ADDRESS_WATCH, 1807 kfd_ioctl_dbg_address_watch, 0), 1808 1809 AMDKFD_IOCTL_DEF(AMDKFD_IOC_DBG_WAVE_CONTROL, 1810 kfd_ioctl_dbg_wave_control, 0), 1811 1812 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_SCRATCH_BACKING_VA, 1813 kfd_ioctl_set_scratch_backing_va, 0), 1814 1815 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_TILE_CONFIG, 1816 kfd_ioctl_get_tile_config, 0), 1817 1818 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_TRAP_HANDLER, 1819 kfd_ioctl_set_trap_handler, 0), 1820 1821 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_PROCESS_APERTURES_NEW, 1822 kfd_ioctl_get_process_apertures_new, 0), 1823 1824 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ACQUIRE_VM, 1825 kfd_ioctl_acquire_vm, 0), 1826 1827 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_MEMORY_OF_GPU, 1828 kfd_ioctl_alloc_memory_of_gpu, 0), 1829 1830 AMDKFD_IOCTL_DEF(AMDKFD_IOC_FREE_MEMORY_OF_GPU, 1831 kfd_ioctl_free_memory_of_gpu, 0), 1832 1833 AMDKFD_IOCTL_DEF(AMDKFD_IOC_MAP_MEMORY_TO_GPU, 1834 kfd_ioctl_map_memory_to_gpu, 0), 1835 1836 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU, 1837 kfd_ioctl_unmap_memory_from_gpu, 0), 1838 1839 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK, 1840 kfd_ioctl_set_cu_mask, 0), 1841 1842 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_QUEUE_WAVE_STATE, 1843 kfd_ioctl_get_queue_wave_state, 0), 1844 1845 AMDKFD_IOCTL_DEF(AMDKFD_IOC_GET_DMABUF_INFO, 1846 kfd_ioctl_get_dmabuf_info, 0), 1847 1848 AMDKFD_IOCTL_DEF(AMDKFD_IOC_IMPORT_DMABUF, 1849 kfd_ioctl_import_dmabuf, 0), 1850 1851 AMDKFD_IOCTL_DEF(AMDKFD_IOC_ALLOC_QUEUE_GWS, 1852 kfd_ioctl_alloc_queue_gws, 0), 1853 1854 AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS, 1855 kfd_ioctl_smi_events, 0), 1856 }; 1857 1858 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls) 1859 1860 static long kfd_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) 1861 { 1862 struct kfd_process *process; 1863 amdkfd_ioctl_t *func; 1864 const struct amdkfd_ioctl_desc *ioctl = NULL; 1865 unsigned int nr = _IOC_NR(cmd); 1866 char stack_kdata[128]; 1867 char *kdata = NULL; 1868 unsigned int usize, asize; 1869 int retcode = -EINVAL; 1870 1871 if (nr >= AMDKFD_CORE_IOCTL_COUNT) 1872 goto err_i1; 1873 1874 if ((nr >= AMDKFD_COMMAND_START) && (nr < AMDKFD_COMMAND_END)) { 1875 u32 amdkfd_size; 1876 1877 ioctl = &amdkfd_ioctls[nr]; 1878 1879 amdkfd_size = _IOC_SIZE(ioctl->cmd); 1880 usize = asize = _IOC_SIZE(cmd); 1881 if (amdkfd_size > asize) 1882 asize = amdkfd_size; 1883 1884 cmd = ioctl->cmd; 1885 } else 1886 goto err_i1; 1887 1888 dev_dbg(kfd_device, "ioctl cmd 0x%x (#0x%x), arg 0x%lx\n", cmd, nr, arg); 1889 1890 /* Get the process struct from the filep. Only the process 1891 * that opened /dev/kfd can use the file descriptor. Child 1892 * processes need to create their own KFD device context. 1893 */ 1894 process = filep->private_data; 1895 if (process->lead_thread != current->group_leader) { 1896 dev_dbg(kfd_device, "Using KFD FD in wrong process\n"); 1897 retcode = -EBADF; 1898 goto err_i1; 1899 } 1900 1901 /* Do not trust userspace, use our own definition */ 1902 func = ioctl->func; 1903 1904 if (unlikely(!func)) { 1905 dev_dbg(kfd_device, "no function\n"); 1906 retcode = -EINVAL; 1907 goto err_i1; 1908 } 1909 1910 if (cmd & (IOC_IN | IOC_OUT)) { 1911 if (asize <= sizeof(stack_kdata)) { 1912 kdata = stack_kdata; 1913 } else { 1914 kdata = kmalloc(asize, GFP_KERNEL); 1915 if (!kdata) { 1916 retcode = -ENOMEM; 1917 goto err_i1; 1918 } 1919 } 1920 if (asize > usize) 1921 memset(kdata + usize, 0, asize - usize); 1922 } 1923 1924 if (cmd & IOC_IN) { 1925 if (copy_from_user(kdata, (void __user *)arg, usize) != 0) { 1926 retcode = -EFAULT; 1927 goto err_i1; 1928 } 1929 } else if (cmd & IOC_OUT) { 1930 memset(kdata, 0, usize); 1931 } 1932 1933 retcode = func(filep, process, kdata); 1934 1935 if (cmd & IOC_OUT) 1936 if (copy_to_user((void __user *)arg, kdata, usize) != 0) 1937 retcode = -EFAULT; 1938 1939 err_i1: 1940 if (!ioctl) 1941 dev_dbg(kfd_device, "invalid ioctl: pid=%d, cmd=0x%02x, nr=0x%02x\n", 1942 task_pid_nr(current), cmd, nr); 1943 1944 if (kdata != stack_kdata) 1945 kfree(kdata); 1946 1947 if (retcode) 1948 dev_dbg(kfd_device, "ioctl cmd (#0x%x), arg 0x%lx, ret = %d\n", 1949 nr, arg, retcode); 1950 1951 return retcode; 1952 } 1953 1954 static int kfd_mmio_mmap(struct kfd_dev *dev, struct kfd_process *process, 1955 struct vm_area_struct *vma) 1956 { 1957 phys_addr_t address; 1958 int ret; 1959 1960 if (vma->vm_end - vma->vm_start != PAGE_SIZE) 1961 return -EINVAL; 1962 1963 address = amdgpu_amdkfd_get_mmio_remap_phys_addr(dev->kgd); 1964 1965 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | 1966 VM_DONTDUMP | VM_PFNMAP; 1967 1968 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 1969 1970 pr_debug("pasid 0x%x mapping mmio page\n" 1971 " target user address == 0x%08llX\n" 1972 " physical address == 0x%08llX\n" 1973 " vm_flags == 0x%04lX\n" 1974 " size == 0x%04lX\n", 1975 process->pasid, (unsigned long long) vma->vm_start, 1976 address, vma->vm_flags, PAGE_SIZE); 1977 1978 ret = io_remap_pfn_range(vma, 1979 vma->vm_start, 1980 address >> PAGE_SHIFT, 1981 PAGE_SIZE, 1982 vma->vm_page_prot); 1983 return ret; 1984 } 1985 1986 1987 static int kfd_mmap(struct file *filp, struct vm_area_struct *vma) 1988 { 1989 struct kfd_process *process; 1990 struct kfd_dev *dev = NULL; 1991 unsigned long mmap_offset; 1992 unsigned int gpu_id; 1993 1994 process = kfd_get_process(current); 1995 if (IS_ERR(process)) 1996 return PTR_ERR(process); 1997 1998 mmap_offset = vma->vm_pgoff << PAGE_SHIFT; 1999 gpu_id = KFD_MMAP_GET_GPU_ID(mmap_offset); 2000 if (gpu_id) 2001 dev = kfd_device_by_id(gpu_id); 2002 2003 switch (mmap_offset & KFD_MMAP_TYPE_MASK) { 2004 case KFD_MMAP_TYPE_DOORBELL: 2005 if (!dev) 2006 return -ENODEV; 2007 return kfd_doorbell_mmap(dev, process, vma); 2008 2009 case KFD_MMAP_TYPE_EVENTS: 2010 return kfd_event_mmap(process, vma); 2011 2012 case KFD_MMAP_TYPE_RESERVED_MEM: 2013 if (!dev) 2014 return -ENODEV; 2015 return kfd_reserved_mem_mmap(dev, process, vma); 2016 case KFD_MMAP_TYPE_MMIO: 2017 if (!dev) 2018 return -ENODEV; 2019 return kfd_mmio_mmap(dev, process, vma); 2020 } 2021 2022 return -EFAULT; 2023 } 2024