1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/mutex.h> 24 #include <linux/log2.h> 25 #include <linux/sched.h> 26 #include <linux/sched/mm.h> 27 #include <linux/sched/task.h> 28 #include <linux/slab.h> 29 #include <linux/amd-iommu.h> 30 #include <linux/notifier.h> 31 #include <linux/compat.h> 32 #include <linux/mman.h> 33 #include <linux/file.h> 34 #include <linux/pm_runtime.h> 35 #include "amdgpu_amdkfd.h" 36 #include "amdgpu.h" 37 38 struct mm_struct; 39 40 #include "kfd_priv.h" 41 #include "kfd_device_queue_manager.h" 42 #include "kfd_dbgmgr.h" 43 #include "kfd_iommu.h" 44 45 /* 46 * List of struct kfd_process (field kfd_process). 47 * Unique/indexed by mm_struct* 48 */ 49 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); 50 static DEFINE_MUTEX(kfd_processes_mutex); 51 52 DEFINE_SRCU(kfd_processes_srcu); 53 54 /* For process termination handling */ 55 static struct workqueue_struct *kfd_process_wq; 56 57 /* Ordered, single-threaded workqueue for restoring evicted 58 * processes. Restoring multiple processes concurrently under memory 59 * pressure can lead to processes blocking each other from validating 60 * their BOs and result in a live-lock situation where processes 61 * remain evicted indefinitely. 62 */ 63 static struct workqueue_struct *kfd_restore_wq; 64 65 static struct kfd_process *find_process(const struct task_struct *thread); 66 static void kfd_process_ref_release(struct kref *ref); 67 static struct kfd_process *create_process(const struct task_struct *thread); 68 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep); 69 70 static void evict_process_worker(struct work_struct *work); 71 static void restore_process_worker(struct work_struct *work); 72 73 struct kfd_procfs_tree { 74 struct kobject *kobj; 75 }; 76 77 static struct kfd_procfs_tree procfs; 78 79 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr, 80 char *buffer) 81 { 82 int val = 0; 83 84 if (strcmp(attr->name, "pasid") == 0) { 85 struct kfd_process *p = container_of(attr, struct kfd_process, 86 attr_pasid); 87 val = p->pasid; 88 } else { 89 pr_err("Invalid attribute"); 90 return -EINVAL; 91 } 92 93 return snprintf(buffer, PAGE_SIZE, "%d\n", val); 94 } 95 96 static void kfd_procfs_kobj_release(struct kobject *kobj) 97 { 98 kfree(kobj); 99 } 100 101 static const struct sysfs_ops kfd_procfs_ops = { 102 .show = kfd_procfs_show, 103 }; 104 105 static struct kobj_type procfs_type = { 106 .release = kfd_procfs_kobj_release, 107 .sysfs_ops = &kfd_procfs_ops, 108 }; 109 110 void kfd_procfs_init(void) 111 { 112 int ret = 0; 113 114 procfs.kobj = kfd_alloc_struct(procfs.kobj); 115 if (!procfs.kobj) 116 return; 117 118 ret = kobject_init_and_add(procfs.kobj, &procfs_type, 119 &kfd_device->kobj, "proc"); 120 if (ret) { 121 pr_warn("Could not create procfs proc folder"); 122 /* If we fail to create the procfs, clean up */ 123 kfd_procfs_shutdown(); 124 } 125 } 126 127 void kfd_procfs_shutdown(void) 128 { 129 if (procfs.kobj) { 130 kobject_del(procfs.kobj); 131 kobject_put(procfs.kobj); 132 procfs.kobj = NULL; 133 } 134 } 135 136 static ssize_t kfd_procfs_queue_show(struct kobject *kobj, 137 struct attribute *attr, char *buffer) 138 { 139 struct queue *q = container_of(kobj, struct queue, kobj); 140 141 if (!strcmp(attr->name, "size")) 142 return snprintf(buffer, PAGE_SIZE, "%llu", 143 q->properties.queue_size); 144 else if (!strcmp(attr->name, "type")) 145 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type); 146 else if (!strcmp(attr->name, "gpuid")) 147 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id); 148 else 149 pr_err("Invalid attribute"); 150 151 return 0; 152 } 153 154 static struct attribute attr_queue_size = { 155 .name = "size", 156 .mode = KFD_SYSFS_FILE_MODE 157 }; 158 159 static struct attribute attr_queue_type = { 160 .name = "type", 161 .mode = KFD_SYSFS_FILE_MODE 162 }; 163 164 static struct attribute attr_queue_gpuid = { 165 .name = "gpuid", 166 .mode = KFD_SYSFS_FILE_MODE 167 }; 168 169 static struct attribute *procfs_queue_attrs[] = { 170 &attr_queue_size, 171 &attr_queue_type, 172 &attr_queue_gpuid, 173 NULL 174 }; 175 176 static const struct sysfs_ops procfs_queue_ops = { 177 .show = kfd_procfs_queue_show, 178 }; 179 180 static struct kobj_type procfs_queue_type = { 181 .sysfs_ops = &procfs_queue_ops, 182 .default_attrs = procfs_queue_attrs, 183 }; 184 185 int kfd_procfs_add_queue(struct queue *q) 186 { 187 struct kfd_process *proc; 188 int ret; 189 190 if (!q || !q->process) 191 return -EINVAL; 192 proc = q->process; 193 194 /* Create proc/<pid>/queues/<queue id> folder */ 195 if (!proc->kobj_queues) 196 return -EFAULT; 197 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type, 198 proc->kobj_queues, "%u", q->properties.queue_id); 199 if (ret < 0) { 200 pr_warn("Creating proc/<pid>/queues/%u failed", 201 q->properties.queue_id); 202 kobject_put(&q->kobj); 203 return ret; 204 } 205 206 return 0; 207 } 208 209 void kfd_procfs_del_queue(struct queue *q) 210 { 211 if (!q) 212 return; 213 214 kobject_del(&q->kobj); 215 kobject_put(&q->kobj); 216 } 217 218 int kfd_process_create_wq(void) 219 { 220 if (!kfd_process_wq) 221 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0); 222 if (!kfd_restore_wq) 223 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0); 224 225 if (!kfd_process_wq || !kfd_restore_wq) { 226 kfd_process_destroy_wq(); 227 return -ENOMEM; 228 } 229 230 return 0; 231 } 232 233 void kfd_process_destroy_wq(void) 234 { 235 if (kfd_process_wq) { 236 destroy_workqueue(kfd_process_wq); 237 kfd_process_wq = NULL; 238 } 239 if (kfd_restore_wq) { 240 destroy_workqueue(kfd_restore_wq); 241 kfd_restore_wq = NULL; 242 } 243 } 244 245 static void kfd_process_free_gpuvm(struct kgd_mem *mem, 246 struct kfd_process_device *pdd) 247 { 248 struct kfd_dev *dev = pdd->dev; 249 250 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->vm); 251 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem); 252 } 253 254 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process 255 * This function should be only called right after the process 256 * is created and when kfd_processes_mutex is still being held 257 * to avoid concurrency. Because of that exclusiveness, we do 258 * not need to take p->mutex. 259 */ 260 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, 261 uint64_t gpu_va, uint32_t size, 262 uint32_t flags, void **kptr) 263 { 264 struct kfd_dev *kdev = pdd->dev; 265 struct kgd_mem *mem = NULL; 266 int handle; 267 int err; 268 269 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size, 270 pdd->vm, &mem, NULL, flags); 271 if (err) 272 goto err_alloc_mem; 273 274 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->vm); 275 if (err) 276 goto err_map_mem; 277 278 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true); 279 if (err) { 280 pr_debug("Sync memory failed, wait interrupted by user signal\n"); 281 goto sync_memory_failed; 282 } 283 284 /* Create an obj handle so kfd_process_device_remove_obj_handle 285 * will take care of the bo removal when the process finishes. 286 * We do not need to take p->mutex, because the process is just 287 * created and the ioctls have not had the chance to run. 288 */ 289 handle = kfd_process_device_create_obj_handle(pdd, mem); 290 291 if (handle < 0) { 292 err = handle; 293 goto free_gpuvm; 294 } 295 296 if (kptr) { 297 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd, 298 (struct kgd_mem *)mem, kptr, NULL); 299 if (err) { 300 pr_debug("Map GTT BO to kernel failed\n"); 301 goto free_obj_handle; 302 } 303 } 304 305 return err; 306 307 free_obj_handle: 308 kfd_process_device_remove_obj_handle(pdd, handle); 309 free_gpuvm: 310 sync_memory_failed: 311 kfd_process_free_gpuvm(mem, pdd); 312 return err; 313 314 err_map_mem: 315 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem); 316 err_alloc_mem: 317 *kptr = NULL; 318 return err; 319 } 320 321 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the 322 * process for IB usage The memory reserved is for KFD to submit 323 * IB to AMDGPU from kernel. If the memory is reserved 324 * successfully, ib_kaddr will have the CPU/kernel 325 * address. Check ib_kaddr before accessing the memory. 326 */ 327 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd) 328 { 329 struct qcm_process_device *qpd = &pdd->qpd; 330 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT | 331 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE | 332 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | 333 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; 334 void *kaddr; 335 int ret; 336 337 if (qpd->ib_kaddr || !qpd->ib_base) 338 return 0; 339 340 /* ib_base is only set for dGPU */ 341 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, 342 &kaddr); 343 if (ret) 344 return ret; 345 346 qpd->ib_kaddr = kaddr; 347 348 return 0; 349 } 350 351 struct kfd_process *kfd_create_process(struct file *filep) 352 { 353 struct kfd_process *process; 354 struct task_struct *thread = current; 355 int ret; 356 357 if (!thread->mm) 358 return ERR_PTR(-EINVAL); 359 360 /* Only the pthreads threading model is supported. */ 361 if (thread->group_leader->mm != thread->mm) 362 return ERR_PTR(-EINVAL); 363 364 /* 365 * take kfd processes mutex before starting of process creation 366 * so there won't be a case where two threads of the same process 367 * create two kfd_process structures 368 */ 369 mutex_lock(&kfd_processes_mutex); 370 371 /* A prior open of /dev/kfd could have already created the process. */ 372 process = find_process(thread); 373 if (process) { 374 pr_debug("Process already found\n"); 375 } else { 376 process = create_process(thread); 377 if (IS_ERR(process)) 378 goto out; 379 380 ret = kfd_process_init_cwsr_apu(process, filep); 381 if (ret) { 382 process = ERR_PTR(ret); 383 goto out; 384 } 385 386 if (!procfs.kobj) 387 goto out; 388 389 process->kobj = kfd_alloc_struct(process->kobj); 390 if (!process->kobj) { 391 pr_warn("Creating procfs kobject failed"); 392 goto out; 393 } 394 ret = kobject_init_and_add(process->kobj, &procfs_type, 395 procfs.kobj, "%d", 396 (int)process->lead_thread->pid); 397 if (ret) { 398 pr_warn("Creating procfs pid directory failed"); 399 goto out; 400 } 401 402 process->attr_pasid.name = "pasid"; 403 process->attr_pasid.mode = KFD_SYSFS_FILE_MODE; 404 sysfs_attr_init(&process->attr_pasid); 405 ret = sysfs_create_file(process->kobj, &process->attr_pasid); 406 if (ret) 407 pr_warn("Creating pasid for pid %d failed", 408 (int)process->lead_thread->pid); 409 410 process->kobj_queues = kobject_create_and_add("queues", 411 process->kobj); 412 if (!process->kobj_queues) 413 pr_warn("Creating KFD proc/queues folder failed"); 414 } 415 out: 416 if (!IS_ERR(process)) 417 kref_get(&process->ref); 418 mutex_unlock(&kfd_processes_mutex); 419 420 return process; 421 } 422 423 struct kfd_process *kfd_get_process(const struct task_struct *thread) 424 { 425 struct kfd_process *process; 426 427 if (!thread->mm) 428 return ERR_PTR(-EINVAL); 429 430 /* Only the pthreads threading model is supported. */ 431 if (thread->group_leader->mm != thread->mm) 432 return ERR_PTR(-EINVAL); 433 434 process = find_process(thread); 435 if (!process) 436 return ERR_PTR(-EINVAL); 437 438 return process; 439 } 440 441 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) 442 { 443 struct kfd_process *process; 444 445 hash_for_each_possible_rcu(kfd_processes_table, process, 446 kfd_processes, (uintptr_t)mm) 447 if (process->mm == mm) 448 return process; 449 450 return NULL; 451 } 452 453 static struct kfd_process *find_process(const struct task_struct *thread) 454 { 455 struct kfd_process *p; 456 int idx; 457 458 idx = srcu_read_lock(&kfd_processes_srcu); 459 p = find_process_by_mm(thread->mm); 460 srcu_read_unlock(&kfd_processes_srcu, idx); 461 462 return p; 463 } 464 465 void kfd_unref_process(struct kfd_process *p) 466 { 467 kref_put(&p->ref, kfd_process_ref_release); 468 } 469 470 static void kfd_process_device_free_bos(struct kfd_process_device *pdd) 471 { 472 struct kfd_process *p = pdd->process; 473 void *mem; 474 int id; 475 476 /* 477 * Remove all handles from idr and release appropriate 478 * local memory object 479 */ 480 idr_for_each_entry(&pdd->alloc_idr, mem, id) { 481 struct kfd_process_device *peer_pdd; 482 483 list_for_each_entry(peer_pdd, &p->per_device_data, 484 per_device_list) { 485 if (!peer_pdd->vm) 486 continue; 487 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( 488 peer_pdd->dev->kgd, mem, peer_pdd->vm); 489 } 490 491 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem); 492 kfd_process_device_remove_obj_handle(pdd, id); 493 } 494 } 495 496 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p) 497 { 498 struct kfd_process_device *pdd; 499 500 list_for_each_entry(pdd, &p->per_device_data, per_device_list) 501 kfd_process_device_free_bos(pdd); 502 } 503 504 static void kfd_process_destroy_pdds(struct kfd_process *p) 505 { 506 struct kfd_process_device *pdd, *temp; 507 508 list_for_each_entry_safe(pdd, temp, &p->per_device_data, 509 per_device_list) { 510 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n", 511 pdd->dev->id, p->pasid); 512 513 if (pdd->drm_file) { 514 amdgpu_amdkfd_gpuvm_release_process_vm( 515 pdd->dev->kgd, pdd->vm); 516 fput(pdd->drm_file); 517 } 518 else if (pdd->vm) 519 amdgpu_amdkfd_gpuvm_destroy_process_vm( 520 pdd->dev->kgd, pdd->vm); 521 522 list_del(&pdd->per_device_list); 523 524 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) 525 free_pages((unsigned long)pdd->qpd.cwsr_kaddr, 526 get_order(KFD_CWSR_TBA_TMA_SIZE)); 527 528 kfree(pdd->qpd.doorbell_bitmap); 529 idr_destroy(&pdd->alloc_idr); 530 531 /* 532 * before destroying pdd, make sure to report availability 533 * for auto suspend 534 */ 535 if (pdd->runtime_inuse) { 536 pm_runtime_mark_last_busy(pdd->dev->ddev->dev); 537 pm_runtime_put_autosuspend(pdd->dev->ddev->dev); 538 pdd->runtime_inuse = false; 539 } 540 541 kfree(pdd); 542 } 543 } 544 545 /* No process locking is needed in this function, because the process 546 * is not findable any more. We must assume that no other thread is 547 * using it any more, otherwise we couldn't safely free the process 548 * structure in the end. 549 */ 550 static void kfd_process_wq_release(struct work_struct *work) 551 { 552 struct kfd_process *p = container_of(work, struct kfd_process, 553 release_work); 554 555 /* Remove the procfs files */ 556 if (p->kobj) { 557 sysfs_remove_file(p->kobj, &p->attr_pasid); 558 kobject_del(p->kobj_queues); 559 kobject_put(p->kobj_queues); 560 p->kobj_queues = NULL; 561 kobject_del(p->kobj); 562 kobject_put(p->kobj); 563 p->kobj = NULL; 564 } 565 566 kfd_iommu_unbind_process(p); 567 568 kfd_process_free_outstanding_kfd_bos(p); 569 570 kfd_process_destroy_pdds(p); 571 dma_fence_put(p->ef); 572 573 kfd_event_free_process(p); 574 575 kfd_pasid_free(p->pasid); 576 kfd_free_process_doorbells(p); 577 578 mutex_destroy(&p->mutex); 579 580 put_task_struct(p->lead_thread); 581 582 kfree(p); 583 } 584 585 static void kfd_process_ref_release(struct kref *ref) 586 { 587 struct kfd_process *p = container_of(ref, struct kfd_process, ref); 588 589 INIT_WORK(&p->release_work, kfd_process_wq_release); 590 queue_work(kfd_process_wq, &p->release_work); 591 } 592 593 static void kfd_process_free_notifier(struct mmu_notifier *mn) 594 { 595 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier)); 596 } 597 598 static void kfd_process_notifier_release(struct mmu_notifier *mn, 599 struct mm_struct *mm) 600 { 601 struct kfd_process *p; 602 struct kfd_process_device *pdd = NULL; 603 604 /* 605 * The kfd_process structure can not be free because the 606 * mmu_notifier srcu is read locked 607 */ 608 p = container_of(mn, struct kfd_process, mmu_notifier); 609 if (WARN_ON(p->mm != mm)) 610 return; 611 612 mutex_lock(&kfd_processes_mutex); 613 hash_del_rcu(&p->kfd_processes); 614 mutex_unlock(&kfd_processes_mutex); 615 synchronize_srcu(&kfd_processes_srcu); 616 617 cancel_delayed_work_sync(&p->eviction_work); 618 cancel_delayed_work_sync(&p->restore_work); 619 620 mutex_lock(&p->mutex); 621 622 /* Iterate over all process device data structures and if the 623 * pdd is in debug mode, we should first force unregistration, 624 * then we will be able to destroy the queues 625 */ 626 list_for_each_entry(pdd, &p->per_device_data, per_device_list) { 627 struct kfd_dev *dev = pdd->dev; 628 629 mutex_lock(kfd_get_dbgmgr_mutex()); 630 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) { 631 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) { 632 kfd_dbgmgr_destroy(dev->dbgmgr); 633 dev->dbgmgr = NULL; 634 } 635 } 636 mutex_unlock(kfd_get_dbgmgr_mutex()); 637 } 638 639 kfd_process_dequeue_from_all_devices(p); 640 pqm_uninit(&p->pqm); 641 642 /* Indicate to other users that MM is no longer valid */ 643 p->mm = NULL; 644 /* Signal the eviction fence after user mode queues are 645 * destroyed. This allows any BOs to be freed without 646 * triggering pointless evictions or waiting for fences. 647 */ 648 dma_fence_signal(p->ef); 649 650 mutex_unlock(&p->mutex); 651 652 mmu_notifier_put(&p->mmu_notifier); 653 } 654 655 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = { 656 .release = kfd_process_notifier_release, 657 .free_notifier = kfd_process_free_notifier, 658 }; 659 660 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep) 661 { 662 unsigned long offset; 663 struct kfd_process_device *pdd; 664 665 list_for_each_entry(pdd, &p->per_device_data, per_device_list) { 666 struct kfd_dev *dev = pdd->dev; 667 struct qcm_process_device *qpd = &pdd->qpd; 668 669 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base) 670 continue; 671 672 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id); 673 qpd->tba_addr = (int64_t)vm_mmap(filep, 0, 674 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC, 675 MAP_SHARED, offset); 676 677 if (IS_ERR_VALUE(qpd->tba_addr)) { 678 int err = qpd->tba_addr; 679 680 pr_err("Failure to set tba address. error %d.\n", err); 681 qpd->tba_addr = 0; 682 qpd->cwsr_kaddr = NULL; 683 return err; 684 } 685 686 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size); 687 688 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; 689 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n", 690 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); 691 } 692 693 return 0; 694 } 695 696 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) 697 { 698 struct kfd_dev *dev = pdd->dev; 699 struct qcm_process_device *qpd = &pdd->qpd; 700 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT 701 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE 702 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; 703 void *kaddr; 704 int ret; 705 706 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base) 707 return 0; 708 709 /* cwsr_base is only set for dGPU */ 710 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, 711 KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr); 712 if (ret) 713 return ret; 714 715 qpd->cwsr_kaddr = kaddr; 716 qpd->tba_addr = qpd->cwsr_base; 717 718 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size); 719 720 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; 721 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n", 722 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); 723 724 return 0; 725 } 726 727 /* 728 * On return the kfd_process is fully operational and will be freed when the 729 * mm is released 730 */ 731 static struct kfd_process *create_process(const struct task_struct *thread) 732 { 733 struct kfd_process *process; 734 int err = -ENOMEM; 735 736 process = kzalloc(sizeof(*process), GFP_KERNEL); 737 if (!process) 738 goto err_alloc_process; 739 740 kref_init(&process->ref); 741 mutex_init(&process->mutex); 742 process->mm = thread->mm; 743 process->lead_thread = thread->group_leader; 744 INIT_LIST_HEAD(&process->per_device_data); 745 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker); 746 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); 747 process->last_restore_timestamp = get_jiffies_64(); 748 kfd_event_init_process(process); 749 process->is_32bit_user_mode = in_compat_syscall(); 750 751 process->pasid = kfd_pasid_alloc(); 752 if (process->pasid == 0) 753 goto err_alloc_pasid; 754 755 if (kfd_alloc_process_doorbells(process) < 0) 756 goto err_alloc_doorbells; 757 758 err = pqm_init(&process->pqm, process); 759 if (err != 0) 760 goto err_process_pqm_init; 761 762 /* init process apertures*/ 763 err = kfd_init_apertures(process); 764 if (err != 0) 765 goto err_init_apertures; 766 767 /* Must be last, have to use release destruction after this */ 768 process->mmu_notifier.ops = &kfd_process_mmu_notifier_ops; 769 err = mmu_notifier_register(&process->mmu_notifier, process->mm); 770 if (err) 771 goto err_register_notifier; 772 773 get_task_struct(process->lead_thread); 774 hash_add_rcu(kfd_processes_table, &process->kfd_processes, 775 (uintptr_t)process->mm); 776 777 return process; 778 779 err_register_notifier: 780 kfd_process_free_outstanding_kfd_bos(process); 781 kfd_process_destroy_pdds(process); 782 err_init_apertures: 783 pqm_uninit(&process->pqm); 784 err_process_pqm_init: 785 kfd_free_process_doorbells(process); 786 err_alloc_doorbells: 787 kfd_pasid_free(process->pasid); 788 err_alloc_pasid: 789 mutex_destroy(&process->mutex); 790 kfree(process); 791 err_alloc_process: 792 return ERR_PTR(err); 793 } 794 795 static int init_doorbell_bitmap(struct qcm_process_device *qpd, 796 struct kfd_dev *dev) 797 { 798 unsigned int i; 799 int range_start = dev->shared_resources.non_cp_doorbells_start; 800 int range_end = dev->shared_resources.non_cp_doorbells_end; 801 802 if (!KFD_IS_SOC15(dev->device_info->asic_family)) 803 return 0; 804 805 qpd->doorbell_bitmap = 806 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, 807 BITS_PER_BYTE), GFP_KERNEL); 808 if (!qpd->doorbell_bitmap) 809 return -ENOMEM; 810 811 /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */ 812 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end); 813 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", 814 range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, 815 range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET); 816 817 for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) { 818 if (i >= range_start && i <= range_end) { 819 set_bit(i, qpd->doorbell_bitmap); 820 set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, 821 qpd->doorbell_bitmap); 822 } 823 } 824 825 return 0; 826 } 827 828 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, 829 struct kfd_process *p) 830 { 831 struct kfd_process_device *pdd = NULL; 832 833 list_for_each_entry(pdd, &p->per_device_data, per_device_list) 834 if (pdd->dev == dev) 835 return pdd; 836 837 return NULL; 838 } 839 840 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, 841 struct kfd_process *p) 842 { 843 struct kfd_process_device *pdd = NULL; 844 845 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); 846 if (!pdd) 847 return NULL; 848 849 if (init_doorbell_bitmap(&pdd->qpd, dev)) { 850 pr_err("Failed to init doorbell for process\n"); 851 kfree(pdd); 852 return NULL; 853 } 854 855 pdd->dev = dev; 856 INIT_LIST_HEAD(&pdd->qpd.queues_list); 857 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); 858 pdd->qpd.dqm = dev->dqm; 859 pdd->qpd.pqm = &p->pqm; 860 pdd->qpd.evicted = 0; 861 pdd->process = p; 862 pdd->bound = PDD_UNBOUND; 863 pdd->already_dequeued = false; 864 pdd->runtime_inuse = false; 865 list_add(&pdd->per_device_list, &p->per_device_data); 866 867 /* Init idr used for memory handle translation */ 868 idr_init(&pdd->alloc_idr); 869 870 return pdd; 871 } 872 873 /** 874 * kfd_process_device_init_vm - Initialize a VM for a process-device 875 * 876 * @pdd: The process-device 877 * @drm_file: Optional pointer to a DRM file descriptor 878 * 879 * If @drm_file is specified, it will be used to acquire the VM from 880 * that file descriptor. If successful, the @pdd takes ownership of 881 * the file descriptor. 882 * 883 * If @drm_file is NULL, a new VM is created. 884 * 885 * Returns 0 on success, -errno on failure. 886 */ 887 int kfd_process_device_init_vm(struct kfd_process_device *pdd, 888 struct file *drm_file) 889 { 890 struct kfd_process *p; 891 struct kfd_dev *dev; 892 int ret; 893 894 if (pdd->vm) 895 return drm_file ? -EBUSY : 0; 896 897 p = pdd->process; 898 dev = pdd->dev; 899 900 if (drm_file) 901 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm( 902 dev->kgd, drm_file, p->pasid, 903 &pdd->vm, &p->kgd_process_info, &p->ef); 904 else 905 ret = amdgpu_amdkfd_gpuvm_create_process_vm(dev->kgd, p->pasid, 906 &pdd->vm, &p->kgd_process_info, &p->ef); 907 if (ret) { 908 pr_err("Failed to create process VM object\n"); 909 return ret; 910 } 911 912 amdgpu_vm_set_task_info(pdd->vm); 913 914 ret = kfd_process_device_reserve_ib_mem(pdd); 915 if (ret) 916 goto err_reserve_ib_mem; 917 ret = kfd_process_device_init_cwsr_dgpu(pdd); 918 if (ret) 919 goto err_init_cwsr; 920 921 pdd->drm_file = drm_file; 922 923 return 0; 924 925 err_init_cwsr: 926 err_reserve_ib_mem: 927 kfd_process_device_free_bos(pdd); 928 if (!drm_file) 929 amdgpu_amdkfd_gpuvm_destroy_process_vm(dev->kgd, pdd->vm); 930 pdd->vm = NULL; 931 932 return ret; 933 } 934 935 /* 936 * Direct the IOMMU to bind the process (specifically the pasid->mm) 937 * to the device. 938 * Unbinding occurs when the process dies or the device is removed. 939 * 940 * Assumes that the process lock is held. 941 */ 942 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, 943 struct kfd_process *p) 944 { 945 struct kfd_process_device *pdd; 946 int err; 947 948 pdd = kfd_get_process_device_data(dev, p); 949 if (!pdd) { 950 pr_err("Process device data doesn't exist\n"); 951 return ERR_PTR(-ENOMEM); 952 } 953 954 /* 955 * signal runtime-pm system to auto resume and prevent 956 * further runtime suspend once device pdd is created until 957 * pdd is destroyed. 958 */ 959 if (!pdd->runtime_inuse) { 960 err = pm_runtime_get_sync(dev->ddev->dev); 961 if (err < 0) 962 return ERR_PTR(err); 963 } 964 965 err = kfd_iommu_bind_process_to_device(pdd); 966 if (err) 967 goto out; 968 969 err = kfd_process_device_init_vm(pdd, NULL); 970 if (err) 971 goto out; 972 973 /* 974 * make sure that runtime_usage counter is incremented just once 975 * per pdd 976 */ 977 pdd->runtime_inuse = true; 978 979 return pdd; 980 981 out: 982 /* balance runpm reference count and exit with error */ 983 if (!pdd->runtime_inuse) { 984 pm_runtime_mark_last_busy(dev->ddev->dev); 985 pm_runtime_put_autosuspend(dev->ddev->dev); 986 } 987 988 return ERR_PTR(err); 989 } 990 991 struct kfd_process_device *kfd_get_first_process_device_data( 992 struct kfd_process *p) 993 { 994 return list_first_entry(&p->per_device_data, 995 struct kfd_process_device, 996 per_device_list); 997 } 998 999 struct kfd_process_device *kfd_get_next_process_device_data( 1000 struct kfd_process *p, 1001 struct kfd_process_device *pdd) 1002 { 1003 if (list_is_last(&pdd->per_device_list, &p->per_device_data)) 1004 return NULL; 1005 return list_next_entry(pdd, per_device_list); 1006 } 1007 1008 bool kfd_has_process_device_data(struct kfd_process *p) 1009 { 1010 return !(list_empty(&p->per_device_data)); 1011 } 1012 1013 /* Create specific handle mapped to mem from process local memory idr 1014 * Assumes that the process lock is held. 1015 */ 1016 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, 1017 void *mem) 1018 { 1019 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); 1020 } 1021 1022 /* Translate specific handle from process local memory idr 1023 * Assumes that the process lock is held. 1024 */ 1025 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd, 1026 int handle) 1027 { 1028 if (handle < 0) 1029 return NULL; 1030 1031 return idr_find(&pdd->alloc_idr, handle); 1032 } 1033 1034 /* Remove specific handle from process local memory idr 1035 * Assumes that the process lock is held. 1036 */ 1037 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, 1038 int handle) 1039 { 1040 if (handle >= 0) 1041 idr_remove(&pdd->alloc_idr, handle); 1042 } 1043 1044 /* This increments the process->ref counter. */ 1045 struct kfd_process *kfd_lookup_process_by_pasid(unsigned int pasid) 1046 { 1047 struct kfd_process *p, *ret_p = NULL; 1048 unsigned int temp; 1049 1050 int idx = srcu_read_lock(&kfd_processes_srcu); 1051 1052 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1053 if (p->pasid == pasid) { 1054 kref_get(&p->ref); 1055 ret_p = p; 1056 break; 1057 } 1058 } 1059 1060 srcu_read_unlock(&kfd_processes_srcu, idx); 1061 1062 return ret_p; 1063 } 1064 1065 /* This increments the process->ref counter. */ 1066 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm) 1067 { 1068 struct kfd_process *p; 1069 1070 int idx = srcu_read_lock(&kfd_processes_srcu); 1071 1072 p = find_process_by_mm(mm); 1073 if (p) 1074 kref_get(&p->ref); 1075 1076 srcu_read_unlock(&kfd_processes_srcu, idx); 1077 1078 return p; 1079 } 1080 1081 /* process_evict_queues - Evict all user queues of a process 1082 * 1083 * Eviction is reference-counted per process-device. This means multiple 1084 * evictions from different sources can be nested safely. 1085 */ 1086 int kfd_process_evict_queues(struct kfd_process *p) 1087 { 1088 struct kfd_process_device *pdd; 1089 int r = 0; 1090 unsigned int n_evicted = 0; 1091 1092 list_for_each_entry(pdd, &p->per_device_data, per_device_list) { 1093 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, 1094 &pdd->qpd); 1095 if (r) { 1096 pr_err("Failed to evict process queues\n"); 1097 goto fail; 1098 } 1099 n_evicted++; 1100 } 1101 1102 return r; 1103 1104 fail: 1105 /* To keep state consistent, roll back partial eviction by 1106 * restoring queues 1107 */ 1108 list_for_each_entry(pdd, &p->per_device_data, per_device_list) { 1109 if (n_evicted == 0) 1110 break; 1111 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, 1112 &pdd->qpd)) 1113 pr_err("Failed to restore queues\n"); 1114 1115 n_evicted--; 1116 } 1117 1118 return r; 1119 } 1120 1121 /* process_restore_queues - Restore all user queues of a process */ 1122 int kfd_process_restore_queues(struct kfd_process *p) 1123 { 1124 struct kfd_process_device *pdd; 1125 int r, ret = 0; 1126 1127 list_for_each_entry(pdd, &p->per_device_data, per_device_list) { 1128 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, 1129 &pdd->qpd); 1130 if (r) { 1131 pr_err("Failed to restore process queues\n"); 1132 if (!ret) 1133 ret = r; 1134 } 1135 } 1136 1137 return ret; 1138 } 1139 1140 static void evict_process_worker(struct work_struct *work) 1141 { 1142 int ret; 1143 struct kfd_process *p; 1144 struct delayed_work *dwork; 1145 1146 dwork = to_delayed_work(work); 1147 1148 /* Process termination destroys this worker thread. So during the 1149 * lifetime of this thread, kfd_process p will be valid 1150 */ 1151 p = container_of(dwork, struct kfd_process, eviction_work); 1152 WARN_ONCE(p->last_eviction_seqno != p->ef->seqno, 1153 "Eviction fence mismatch\n"); 1154 1155 /* Narrow window of overlap between restore and evict work 1156 * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos 1157 * unreserves KFD BOs, it is possible to evicted again. But 1158 * restore has few more steps of finish. So lets wait for any 1159 * previous restore work to complete 1160 */ 1161 flush_delayed_work(&p->restore_work); 1162 1163 pr_debug("Started evicting pasid 0x%x\n", p->pasid); 1164 ret = kfd_process_evict_queues(p); 1165 if (!ret) { 1166 dma_fence_signal(p->ef); 1167 dma_fence_put(p->ef); 1168 p->ef = NULL; 1169 queue_delayed_work(kfd_restore_wq, &p->restore_work, 1170 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)); 1171 1172 pr_debug("Finished evicting pasid 0x%x\n", p->pasid); 1173 } else 1174 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid); 1175 } 1176 1177 static void restore_process_worker(struct work_struct *work) 1178 { 1179 struct delayed_work *dwork; 1180 struct kfd_process *p; 1181 int ret = 0; 1182 1183 dwork = to_delayed_work(work); 1184 1185 /* Process termination destroys this worker thread. So during the 1186 * lifetime of this thread, kfd_process p will be valid 1187 */ 1188 p = container_of(dwork, struct kfd_process, restore_work); 1189 pr_debug("Started restoring pasid 0x%x\n", p->pasid); 1190 1191 /* Setting last_restore_timestamp before successful restoration. 1192 * Otherwise this would have to be set by KGD (restore_process_bos) 1193 * before KFD BOs are unreserved. If not, the process can be evicted 1194 * again before the timestamp is set. 1195 * If restore fails, the timestamp will be set again in the next 1196 * attempt. This would mean that the minimum GPU quanta would be 1197 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two 1198 * functions) 1199 */ 1200 1201 p->last_restore_timestamp = get_jiffies_64(); 1202 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info, 1203 &p->ef); 1204 if (ret) { 1205 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n", 1206 p->pasid, PROCESS_BACK_OFF_TIME_MS); 1207 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work, 1208 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS)); 1209 WARN(!ret, "reschedule restore work failed\n"); 1210 return; 1211 } 1212 1213 ret = kfd_process_restore_queues(p); 1214 if (!ret) 1215 pr_debug("Finished restoring pasid 0x%x\n", p->pasid); 1216 else 1217 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid); 1218 } 1219 1220 void kfd_suspend_all_processes(void) 1221 { 1222 struct kfd_process *p; 1223 unsigned int temp; 1224 int idx = srcu_read_lock(&kfd_processes_srcu); 1225 1226 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1227 cancel_delayed_work_sync(&p->eviction_work); 1228 cancel_delayed_work_sync(&p->restore_work); 1229 1230 if (kfd_process_evict_queues(p)) 1231 pr_err("Failed to suspend process 0x%x\n", p->pasid); 1232 dma_fence_signal(p->ef); 1233 dma_fence_put(p->ef); 1234 p->ef = NULL; 1235 } 1236 srcu_read_unlock(&kfd_processes_srcu, idx); 1237 } 1238 1239 int kfd_resume_all_processes(void) 1240 { 1241 struct kfd_process *p; 1242 unsigned int temp; 1243 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu); 1244 1245 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1246 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) { 1247 pr_err("Restore process %d failed during resume\n", 1248 p->pasid); 1249 ret = -EFAULT; 1250 } 1251 } 1252 srcu_read_unlock(&kfd_processes_srcu, idx); 1253 return ret; 1254 } 1255 1256 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process, 1257 struct vm_area_struct *vma) 1258 { 1259 struct kfd_process_device *pdd; 1260 struct qcm_process_device *qpd; 1261 1262 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) { 1263 pr_err("Incorrect CWSR mapping size.\n"); 1264 return -EINVAL; 1265 } 1266 1267 pdd = kfd_get_process_device_data(dev, process); 1268 if (!pdd) 1269 return -EINVAL; 1270 qpd = &pdd->qpd; 1271 1272 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1273 get_order(KFD_CWSR_TBA_TMA_SIZE)); 1274 if (!qpd->cwsr_kaddr) { 1275 pr_err("Error allocating per process CWSR buffer.\n"); 1276 return -ENOMEM; 1277 } 1278 1279 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND 1280 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP; 1281 /* Mapping pages to user process */ 1282 return remap_pfn_range(vma, vma->vm_start, 1283 PFN_DOWN(__pa(qpd->cwsr_kaddr)), 1284 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot); 1285 } 1286 1287 void kfd_flush_tlb(struct kfd_process_device *pdd) 1288 { 1289 struct kfd_dev *dev = pdd->dev; 1290 1291 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 1292 /* Nothing to flush until a VMID is assigned, which 1293 * only happens when the first queue is created. 1294 */ 1295 if (pdd->qpd.vmid) 1296 amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd, 1297 pdd->qpd.vmid); 1298 } else { 1299 amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd, 1300 pdd->process->pasid); 1301 } 1302 } 1303 1304 #if defined(CONFIG_DEBUG_FS) 1305 1306 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data) 1307 { 1308 struct kfd_process *p; 1309 unsigned int temp; 1310 int r = 0; 1311 1312 int idx = srcu_read_lock(&kfd_processes_srcu); 1313 1314 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1315 seq_printf(m, "Process %d PASID 0x%x:\n", 1316 p->lead_thread->tgid, p->pasid); 1317 1318 mutex_lock(&p->mutex); 1319 r = pqm_debugfs_mqds(m, &p->pqm); 1320 mutex_unlock(&p->mutex); 1321 1322 if (r) 1323 break; 1324 } 1325 1326 srcu_read_unlock(&kfd_processes_srcu, idx); 1327 1328 return r; 1329 } 1330 1331 #endif 1332 1333