1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/mutex.h> 24 #include <linux/log2.h> 25 #include <linux/sched.h> 26 #include <linux/sched/mm.h> 27 #include <linux/sched/task.h> 28 #include <linux/mmu_context.h> 29 #include <linux/slab.h> 30 #include <linux/amd-iommu.h> 31 #include <linux/notifier.h> 32 #include <linux/compat.h> 33 #include <linux/mman.h> 34 #include <linux/file.h> 35 #include <linux/pm_runtime.h> 36 #include "amdgpu_amdkfd.h" 37 #include "amdgpu.h" 38 39 struct mm_struct; 40 41 #include "kfd_priv.h" 42 #include "kfd_device_queue_manager.h" 43 #include "kfd_dbgmgr.h" 44 #include "kfd_iommu.h" 45 #include "kfd_svm.h" 46 47 /* 48 * List of struct kfd_process (field kfd_process). 49 * Unique/indexed by mm_struct* 50 */ 51 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE); 52 static DEFINE_MUTEX(kfd_processes_mutex); 53 54 DEFINE_SRCU(kfd_processes_srcu); 55 56 /* For process termination handling */ 57 static struct workqueue_struct *kfd_process_wq; 58 59 /* Ordered, single-threaded workqueue for restoring evicted 60 * processes. Restoring multiple processes concurrently under memory 61 * pressure can lead to processes blocking each other from validating 62 * their BOs and result in a live-lock situation where processes 63 * remain evicted indefinitely. 64 */ 65 static struct workqueue_struct *kfd_restore_wq; 66 67 static struct kfd_process *find_process(const struct task_struct *thread); 68 static void kfd_process_ref_release(struct kref *ref); 69 static struct kfd_process *create_process(const struct task_struct *thread); 70 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep); 71 72 static void evict_process_worker(struct work_struct *work); 73 static void restore_process_worker(struct work_struct *work); 74 75 struct kfd_procfs_tree { 76 struct kobject *kobj; 77 }; 78 79 static struct kfd_procfs_tree procfs; 80 81 /* 82 * Structure for SDMA activity tracking 83 */ 84 struct kfd_sdma_activity_handler_workarea { 85 struct work_struct sdma_activity_work; 86 struct kfd_process_device *pdd; 87 uint64_t sdma_activity_counter; 88 }; 89 90 struct temp_sdma_queue_list { 91 uint64_t __user *rptr; 92 uint64_t sdma_val; 93 unsigned int queue_id; 94 struct list_head list; 95 }; 96 97 static void kfd_sdma_activity_worker(struct work_struct *work) 98 { 99 struct kfd_sdma_activity_handler_workarea *workarea; 100 struct kfd_process_device *pdd; 101 uint64_t val; 102 struct mm_struct *mm; 103 struct queue *q; 104 struct qcm_process_device *qpd; 105 struct device_queue_manager *dqm; 106 int ret = 0; 107 struct temp_sdma_queue_list sdma_q_list; 108 struct temp_sdma_queue_list *sdma_q, *next; 109 110 workarea = container_of(work, struct kfd_sdma_activity_handler_workarea, 111 sdma_activity_work); 112 113 pdd = workarea->pdd; 114 if (!pdd) 115 return; 116 dqm = pdd->dev->dqm; 117 qpd = &pdd->qpd; 118 if (!dqm || !qpd) 119 return; 120 /* 121 * Total SDMA activity is current SDMA activity + past SDMA activity 122 * Past SDMA count is stored in pdd. 123 * To get the current activity counters for all active SDMA queues, 124 * we loop over all SDMA queues and get their counts from user-space. 125 * 126 * We cannot call get_user() with dqm_lock held as it can cause 127 * a circular lock dependency situation. To read the SDMA stats, 128 * we need to do the following: 129 * 130 * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list, 131 * with dqm_lock/dqm_unlock(). 132 * 2. Call get_user() for each node in temporary list without dqm_lock. 133 * Save the SDMA count for each node and also add the count to the total 134 * SDMA count counter. 135 * Its possible, during this step, a few SDMA queue nodes got deleted 136 * from the qpd->queues_list. 137 * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted. 138 * If any node got deleted, its SDMA count would be captured in the sdma 139 * past activity counter. So subtract the SDMA counter stored in step 2 140 * for this node from the total SDMA count. 141 */ 142 INIT_LIST_HEAD(&sdma_q_list.list); 143 144 /* 145 * Create the temp list of all SDMA queues 146 */ 147 dqm_lock(dqm); 148 149 list_for_each_entry(q, &qpd->queues_list, list) { 150 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && 151 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) 152 continue; 153 154 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL); 155 if (!sdma_q) { 156 dqm_unlock(dqm); 157 goto cleanup; 158 } 159 160 INIT_LIST_HEAD(&sdma_q->list); 161 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr; 162 sdma_q->queue_id = q->properties.queue_id; 163 list_add_tail(&sdma_q->list, &sdma_q_list.list); 164 } 165 166 /* 167 * If the temp list is empty, then no SDMA queues nodes were found in 168 * qpd->queues_list. Return the past activity count as the total sdma 169 * count 170 */ 171 if (list_empty(&sdma_q_list.list)) { 172 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; 173 dqm_unlock(dqm); 174 return; 175 } 176 177 dqm_unlock(dqm); 178 179 /* 180 * Get the usage count for each SDMA queue in temp_list. 181 */ 182 mm = get_task_mm(pdd->process->lead_thread); 183 if (!mm) 184 goto cleanup; 185 186 kthread_use_mm(mm); 187 188 list_for_each_entry(sdma_q, &sdma_q_list.list, list) { 189 val = 0; 190 ret = read_sdma_queue_counter(sdma_q->rptr, &val); 191 if (ret) { 192 pr_debug("Failed to read SDMA queue active counter for queue id: %d", 193 sdma_q->queue_id); 194 } else { 195 sdma_q->sdma_val = val; 196 workarea->sdma_activity_counter += val; 197 } 198 } 199 200 kthread_unuse_mm(mm); 201 mmput(mm); 202 203 /* 204 * Do a second iteration over qpd_queues_list to check if any SDMA 205 * nodes got deleted while fetching SDMA counter. 206 */ 207 dqm_lock(dqm); 208 209 workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; 210 211 list_for_each_entry(q, &qpd->queues_list, list) { 212 if (list_empty(&sdma_q_list.list)) 213 break; 214 215 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) && 216 (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) 217 continue; 218 219 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { 220 if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) && 221 (sdma_q->queue_id == q->properties.queue_id)) { 222 list_del(&sdma_q->list); 223 kfree(sdma_q); 224 break; 225 } 226 } 227 } 228 229 dqm_unlock(dqm); 230 231 /* 232 * If temp list is not empty, it implies some queues got deleted 233 * from qpd->queues_list during SDMA usage read. Subtract the SDMA 234 * count for each node from the total SDMA count. 235 */ 236 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { 237 workarea->sdma_activity_counter -= sdma_q->sdma_val; 238 list_del(&sdma_q->list); 239 kfree(sdma_q); 240 } 241 242 return; 243 244 cleanup: 245 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) { 246 list_del(&sdma_q->list); 247 kfree(sdma_q); 248 } 249 } 250 251 /** 252 * @kfd_get_cu_occupancy - Collect number of waves in-flight on this device 253 * by current process. Translates acquired wave count into number of compute units 254 * that are occupied. 255 * 256 * @atr: Handle of attribute that allows reporting of wave count. The attribute 257 * handle encapsulates GPU device it is associated with, thereby allowing collection 258 * of waves in flight, etc 259 * 260 * @buffer: Handle of user provided buffer updated with wave count 261 * 262 * Return: Number of bytes written to user buffer or an error value 263 */ 264 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer) 265 { 266 int cu_cnt; 267 int wave_cnt; 268 int max_waves_per_cu; 269 struct kfd_dev *dev = NULL; 270 struct kfd_process *proc = NULL; 271 struct kfd_process_device *pdd = NULL; 272 273 pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy); 274 dev = pdd->dev; 275 if (dev->kfd2kgd->get_cu_occupancy == NULL) 276 return -EINVAL; 277 278 cu_cnt = 0; 279 proc = pdd->process; 280 if (pdd->qpd.queue_count == 0) { 281 pr_debug("Gpu-Id: %d has no active queues for process %d\n", 282 dev->id, proc->pasid); 283 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); 284 } 285 286 /* Collect wave count from device if it supports */ 287 wave_cnt = 0; 288 max_waves_per_cu = 0; 289 dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt, 290 &max_waves_per_cu); 291 292 /* Translate wave count to number of compute units */ 293 cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu; 294 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt); 295 } 296 297 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr, 298 char *buffer) 299 { 300 if (strcmp(attr->name, "pasid") == 0) { 301 struct kfd_process *p = container_of(attr, struct kfd_process, 302 attr_pasid); 303 304 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid); 305 } else if (strncmp(attr->name, "vram_", 5) == 0) { 306 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, 307 attr_vram); 308 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage)); 309 } else if (strncmp(attr->name, "sdma_", 5) == 0) { 310 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device, 311 attr_sdma); 312 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler; 313 314 INIT_WORK(&sdma_activity_work_handler.sdma_activity_work, 315 kfd_sdma_activity_worker); 316 317 sdma_activity_work_handler.pdd = pdd; 318 sdma_activity_work_handler.sdma_activity_counter = 0; 319 320 schedule_work(&sdma_activity_work_handler.sdma_activity_work); 321 322 flush_work(&sdma_activity_work_handler.sdma_activity_work); 323 324 return snprintf(buffer, PAGE_SIZE, "%llu\n", 325 (sdma_activity_work_handler.sdma_activity_counter)/ 326 SDMA_ACTIVITY_DIVISOR); 327 } else { 328 pr_err("Invalid attribute"); 329 return -EINVAL; 330 } 331 332 return 0; 333 } 334 335 static void kfd_procfs_kobj_release(struct kobject *kobj) 336 { 337 kfree(kobj); 338 } 339 340 static const struct sysfs_ops kfd_procfs_ops = { 341 .show = kfd_procfs_show, 342 }; 343 344 static struct kobj_type procfs_type = { 345 .release = kfd_procfs_kobj_release, 346 .sysfs_ops = &kfd_procfs_ops, 347 }; 348 349 void kfd_procfs_init(void) 350 { 351 int ret = 0; 352 353 procfs.kobj = kfd_alloc_struct(procfs.kobj); 354 if (!procfs.kobj) 355 return; 356 357 ret = kobject_init_and_add(procfs.kobj, &procfs_type, 358 &kfd_device->kobj, "proc"); 359 if (ret) { 360 pr_warn("Could not create procfs proc folder"); 361 /* If we fail to create the procfs, clean up */ 362 kfd_procfs_shutdown(); 363 } 364 } 365 366 void kfd_procfs_shutdown(void) 367 { 368 if (procfs.kobj) { 369 kobject_del(procfs.kobj); 370 kobject_put(procfs.kobj); 371 procfs.kobj = NULL; 372 } 373 } 374 375 static ssize_t kfd_procfs_queue_show(struct kobject *kobj, 376 struct attribute *attr, char *buffer) 377 { 378 struct queue *q = container_of(kobj, struct queue, kobj); 379 380 if (!strcmp(attr->name, "size")) 381 return snprintf(buffer, PAGE_SIZE, "%llu", 382 q->properties.queue_size); 383 else if (!strcmp(attr->name, "type")) 384 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type); 385 else if (!strcmp(attr->name, "gpuid")) 386 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id); 387 else 388 pr_err("Invalid attribute"); 389 390 return 0; 391 } 392 393 static ssize_t kfd_procfs_stats_show(struct kobject *kobj, 394 struct attribute *attr, char *buffer) 395 { 396 if (strcmp(attr->name, "evicted_ms") == 0) { 397 struct kfd_process_device *pdd = container_of(attr, 398 struct kfd_process_device, 399 attr_evict); 400 uint64_t evict_jiffies; 401 402 evict_jiffies = atomic64_read(&pdd->evict_duration_counter); 403 404 return snprintf(buffer, 405 PAGE_SIZE, 406 "%llu\n", 407 jiffies64_to_msecs(evict_jiffies)); 408 409 /* Sysfs handle that gets CU occupancy is per device */ 410 } else if (strcmp(attr->name, "cu_occupancy") == 0) { 411 return kfd_get_cu_occupancy(attr, buffer); 412 } else { 413 pr_err("Invalid attribute"); 414 } 415 416 return 0; 417 } 418 419 static struct attribute attr_queue_size = { 420 .name = "size", 421 .mode = KFD_SYSFS_FILE_MODE 422 }; 423 424 static struct attribute attr_queue_type = { 425 .name = "type", 426 .mode = KFD_SYSFS_FILE_MODE 427 }; 428 429 static struct attribute attr_queue_gpuid = { 430 .name = "gpuid", 431 .mode = KFD_SYSFS_FILE_MODE 432 }; 433 434 static struct attribute *procfs_queue_attrs[] = { 435 &attr_queue_size, 436 &attr_queue_type, 437 &attr_queue_gpuid, 438 NULL 439 }; 440 441 static const struct sysfs_ops procfs_queue_ops = { 442 .show = kfd_procfs_queue_show, 443 }; 444 445 static struct kobj_type procfs_queue_type = { 446 .sysfs_ops = &procfs_queue_ops, 447 .default_attrs = procfs_queue_attrs, 448 }; 449 450 static const struct sysfs_ops procfs_stats_ops = { 451 .show = kfd_procfs_stats_show, 452 }; 453 454 static struct attribute *procfs_stats_attrs[] = { 455 NULL 456 }; 457 458 static struct kobj_type procfs_stats_type = { 459 .sysfs_ops = &procfs_stats_ops, 460 .default_attrs = procfs_stats_attrs, 461 }; 462 463 int kfd_procfs_add_queue(struct queue *q) 464 { 465 struct kfd_process *proc; 466 int ret; 467 468 if (!q || !q->process) 469 return -EINVAL; 470 proc = q->process; 471 472 /* Create proc/<pid>/queues/<queue id> folder */ 473 if (!proc->kobj_queues) 474 return -EFAULT; 475 ret = kobject_init_and_add(&q->kobj, &procfs_queue_type, 476 proc->kobj_queues, "%u", q->properties.queue_id); 477 if (ret < 0) { 478 pr_warn("Creating proc/<pid>/queues/%u failed", 479 q->properties.queue_id); 480 kobject_put(&q->kobj); 481 return ret; 482 } 483 484 return 0; 485 } 486 487 static int kfd_sysfs_create_file(struct kfd_process *p, struct attribute *attr, 488 char *name) 489 { 490 int ret = 0; 491 492 if (!p || !attr || !name) 493 return -EINVAL; 494 495 attr->name = name; 496 attr->mode = KFD_SYSFS_FILE_MODE; 497 sysfs_attr_init(attr); 498 499 ret = sysfs_create_file(p->kobj, attr); 500 501 return ret; 502 } 503 504 static int kfd_procfs_add_sysfs_stats(struct kfd_process *p) 505 { 506 int ret = 0; 507 int i; 508 char stats_dir_filename[MAX_SYSFS_FILENAME_LEN]; 509 510 if (!p) 511 return -EINVAL; 512 513 if (!p->kobj) 514 return -EFAULT; 515 516 /* 517 * Create sysfs files for each GPU: 518 * - proc/<pid>/stats_<gpuid>/ 519 * - proc/<pid>/stats_<gpuid>/evicted_ms 520 * - proc/<pid>/stats_<gpuid>/cu_occupancy 521 */ 522 for (i = 0; i < p->n_pdds; i++) { 523 struct kfd_process_device *pdd = p->pdds[i]; 524 struct kobject *kobj_stats; 525 526 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN, 527 "stats_%u", pdd->dev->id); 528 kobj_stats = kfd_alloc_struct(kobj_stats); 529 if (!kobj_stats) 530 return -ENOMEM; 531 532 ret = kobject_init_and_add(kobj_stats, 533 &procfs_stats_type, 534 p->kobj, 535 stats_dir_filename); 536 537 if (ret) { 538 pr_warn("Creating KFD proc/stats_%s folder failed", 539 stats_dir_filename); 540 kobject_put(kobj_stats); 541 goto err; 542 } 543 544 pdd->kobj_stats = kobj_stats; 545 pdd->attr_evict.name = "evicted_ms"; 546 pdd->attr_evict.mode = KFD_SYSFS_FILE_MODE; 547 sysfs_attr_init(&pdd->attr_evict); 548 ret = sysfs_create_file(kobj_stats, &pdd->attr_evict); 549 if (ret) 550 pr_warn("Creating eviction stats for gpuid %d failed", 551 (int)pdd->dev->id); 552 553 /* Add sysfs file to report compute unit occupancy */ 554 if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) { 555 pdd->attr_cu_occupancy.name = "cu_occupancy"; 556 pdd->attr_cu_occupancy.mode = KFD_SYSFS_FILE_MODE; 557 sysfs_attr_init(&pdd->attr_cu_occupancy); 558 ret = sysfs_create_file(kobj_stats, 559 &pdd->attr_cu_occupancy); 560 if (ret) 561 pr_warn("Creating %s failed for gpuid: %d", 562 pdd->attr_cu_occupancy.name, 563 (int)pdd->dev->id); 564 } 565 } 566 err: 567 return ret; 568 } 569 570 571 static int kfd_procfs_add_sysfs_files(struct kfd_process *p) 572 { 573 int ret = 0; 574 int i; 575 576 if (!p) 577 return -EINVAL; 578 579 if (!p->kobj) 580 return -EFAULT; 581 582 /* 583 * Create sysfs files for each GPU: 584 * - proc/<pid>/vram_<gpuid> 585 * - proc/<pid>/sdma_<gpuid> 586 */ 587 for (i = 0; i < p->n_pdds; i++) { 588 struct kfd_process_device *pdd = p->pdds[i]; 589 590 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u", 591 pdd->dev->id); 592 ret = kfd_sysfs_create_file(p, &pdd->attr_vram, pdd->vram_filename); 593 if (ret) 594 pr_warn("Creating vram usage for gpu id %d failed", 595 (int)pdd->dev->id); 596 597 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u", 598 pdd->dev->id); 599 ret = kfd_sysfs_create_file(p, &pdd->attr_sdma, pdd->sdma_filename); 600 if (ret) 601 pr_warn("Creating sdma usage for gpu id %d failed", 602 (int)pdd->dev->id); 603 } 604 605 return ret; 606 } 607 608 void kfd_procfs_del_queue(struct queue *q) 609 { 610 if (!q) 611 return; 612 613 kobject_del(&q->kobj); 614 kobject_put(&q->kobj); 615 } 616 617 int kfd_process_create_wq(void) 618 { 619 if (!kfd_process_wq) 620 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0); 621 if (!kfd_restore_wq) 622 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0); 623 624 if (!kfd_process_wq || !kfd_restore_wq) { 625 kfd_process_destroy_wq(); 626 return -ENOMEM; 627 } 628 629 return 0; 630 } 631 632 void kfd_process_destroy_wq(void) 633 { 634 if (kfd_process_wq) { 635 destroy_workqueue(kfd_process_wq); 636 kfd_process_wq = NULL; 637 } 638 if (kfd_restore_wq) { 639 destroy_workqueue(kfd_restore_wq); 640 kfd_restore_wq = NULL; 641 } 642 } 643 644 static void kfd_process_free_gpuvm(struct kgd_mem *mem, 645 struct kfd_process_device *pdd) 646 { 647 struct kfd_dev *dev = pdd->dev; 648 649 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv); 650 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv, 651 NULL); 652 } 653 654 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process 655 * This function should be only called right after the process 656 * is created and when kfd_processes_mutex is still being held 657 * to avoid concurrency. Because of that exclusiveness, we do 658 * not need to take p->mutex. 659 */ 660 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, 661 uint64_t gpu_va, uint32_t size, 662 uint32_t flags, void **kptr) 663 { 664 struct kfd_dev *kdev = pdd->dev; 665 struct kgd_mem *mem = NULL; 666 int handle; 667 int err; 668 669 err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size, 670 pdd->drm_priv, &mem, NULL, flags); 671 if (err) 672 goto err_alloc_mem; 673 674 err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, 675 pdd->drm_priv, NULL); 676 if (err) 677 goto err_map_mem; 678 679 err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true); 680 if (err) { 681 pr_debug("Sync memory failed, wait interrupted by user signal\n"); 682 goto sync_memory_failed; 683 } 684 685 /* Create an obj handle so kfd_process_device_remove_obj_handle 686 * will take care of the bo removal when the process finishes. 687 * We do not need to take p->mutex, because the process is just 688 * created and the ioctls have not had the chance to run. 689 */ 690 handle = kfd_process_device_create_obj_handle(pdd, mem); 691 692 if (handle < 0) { 693 err = handle; 694 goto free_gpuvm; 695 } 696 697 if (kptr) { 698 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd, 699 (struct kgd_mem *)mem, kptr, NULL); 700 if (err) { 701 pr_debug("Map GTT BO to kernel failed\n"); 702 goto free_obj_handle; 703 } 704 } 705 706 return err; 707 708 free_obj_handle: 709 kfd_process_device_remove_obj_handle(pdd, handle); 710 free_gpuvm: 711 sync_memory_failed: 712 kfd_process_free_gpuvm(mem, pdd); 713 return err; 714 715 err_map_mem: 716 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, pdd->drm_priv, 717 NULL); 718 err_alloc_mem: 719 *kptr = NULL; 720 return err; 721 } 722 723 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the 724 * process for IB usage The memory reserved is for KFD to submit 725 * IB to AMDGPU from kernel. If the memory is reserved 726 * successfully, ib_kaddr will have the CPU/kernel 727 * address. Check ib_kaddr before accessing the memory. 728 */ 729 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd) 730 { 731 struct qcm_process_device *qpd = &pdd->qpd; 732 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT | 733 KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE | 734 KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE | 735 KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; 736 void *kaddr; 737 int ret; 738 739 if (qpd->ib_kaddr || !qpd->ib_base) 740 return 0; 741 742 /* ib_base is only set for dGPU */ 743 ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags, 744 &kaddr); 745 if (ret) 746 return ret; 747 748 qpd->ib_kaddr = kaddr; 749 750 return 0; 751 } 752 753 struct kfd_process *kfd_create_process(struct file *filep) 754 { 755 struct kfd_process *process; 756 struct task_struct *thread = current; 757 int ret; 758 759 if (!thread->mm) 760 return ERR_PTR(-EINVAL); 761 762 /* Only the pthreads threading model is supported. */ 763 if (thread->group_leader->mm != thread->mm) 764 return ERR_PTR(-EINVAL); 765 766 /* 767 * take kfd processes mutex before starting of process creation 768 * so there won't be a case where two threads of the same process 769 * create two kfd_process structures 770 */ 771 mutex_lock(&kfd_processes_mutex); 772 773 /* A prior open of /dev/kfd could have already created the process. */ 774 process = find_process(thread); 775 if (process) { 776 pr_debug("Process already found\n"); 777 } else { 778 process = create_process(thread); 779 if (IS_ERR(process)) 780 goto out; 781 782 ret = kfd_process_init_cwsr_apu(process, filep); 783 if (ret) 784 goto out_destroy; 785 786 if (!procfs.kobj) 787 goto out; 788 789 process->kobj = kfd_alloc_struct(process->kobj); 790 if (!process->kobj) { 791 pr_warn("Creating procfs kobject failed"); 792 goto out; 793 } 794 ret = kobject_init_and_add(process->kobj, &procfs_type, 795 procfs.kobj, "%d", 796 (int)process->lead_thread->pid); 797 if (ret) { 798 pr_warn("Creating procfs pid directory failed"); 799 kobject_put(process->kobj); 800 goto out; 801 } 802 803 process->attr_pasid.name = "pasid"; 804 process->attr_pasid.mode = KFD_SYSFS_FILE_MODE; 805 sysfs_attr_init(&process->attr_pasid); 806 ret = sysfs_create_file(process->kobj, &process->attr_pasid); 807 if (ret) 808 pr_warn("Creating pasid for pid %d failed", 809 (int)process->lead_thread->pid); 810 811 process->kobj_queues = kobject_create_and_add("queues", 812 process->kobj); 813 if (!process->kobj_queues) 814 pr_warn("Creating KFD proc/queues folder failed"); 815 816 ret = kfd_procfs_add_sysfs_stats(process); 817 if (ret) 818 pr_warn("Creating sysfs stats dir for pid %d failed", 819 (int)process->lead_thread->pid); 820 821 ret = kfd_procfs_add_sysfs_files(process); 822 if (ret) 823 pr_warn("Creating sysfs usage file for pid %d failed", 824 (int)process->lead_thread->pid); 825 } 826 out: 827 if (!IS_ERR(process)) 828 kref_get(&process->ref); 829 mutex_unlock(&kfd_processes_mutex); 830 831 return process; 832 833 out_destroy: 834 hash_del_rcu(&process->kfd_processes); 835 mutex_unlock(&kfd_processes_mutex); 836 synchronize_srcu(&kfd_processes_srcu); 837 /* kfd_process_free_notifier will trigger the cleanup */ 838 mmu_notifier_put(&process->mmu_notifier); 839 return ERR_PTR(ret); 840 } 841 842 struct kfd_process *kfd_get_process(const struct task_struct *thread) 843 { 844 struct kfd_process *process; 845 846 if (!thread->mm) 847 return ERR_PTR(-EINVAL); 848 849 /* Only the pthreads threading model is supported. */ 850 if (thread->group_leader->mm != thread->mm) 851 return ERR_PTR(-EINVAL); 852 853 process = find_process(thread); 854 if (!process) 855 return ERR_PTR(-EINVAL); 856 857 return process; 858 } 859 860 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) 861 { 862 struct kfd_process *process; 863 864 hash_for_each_possible_rcu(kfd_processes_table, process, 865 kfd_processes, (uintptr_t)mm) 866 if (process->mm == mm) 867 return process; 868 869 return NULL; 870 } 871 872 static struct kfd_process *find_process(const struct task_struct *thread) 873 { 874 struct kfd_process *p; 875 int idx; 876 877 idx = srcu_read_lock(&kfd_processes_srcu); 878 p = find_process_by_mm(thread->mm); 879 srcu_read_unlock(&kfd_processes_srcu, idx); 880 881 return p; 882 } 883 884 void kfd_unref_process(struct kfd_process *p) 885 { 886 kref_put(&p->ref, kfd_process_ref_release); 887 } 888 889 890 static void kfd_process_device_free_bos(struct kfd_process_device *pdd) 891 { 892 struct kfd_process *p = pdd->process; 893 void *mem; 894 int id; 895 int i; 896 897 /* 898 * Remove all handles from idr and release appropriate 899 * local memory object 900 */ 901 idr_for_each_entry(&pdd->alloc_idr, mem, id) { 902 903 for (i = 0; i < p->n_pdds; i++) { 904 struct kfd_process_device *peer_pdd = p->pdds[i]; 905 906 if (!peer_pdd->drm_priv) 907 continue; 908 amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( 909 peer_pdd->dev->kgd, mem, peer_pdd->drm_priv); 910 } 911 912 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, 913 pdd->drm_priv, NULL); 914 kfd_process_device_remove_obj_handle(pdd, id); 915 } 916 } 917 918 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p) 919 { 920 int i; 921 922 for (i = 0; i < p->n_pdds; i++) 923 kfd_process_device_free_bos(p->pdds[i]); 924 } 925 926 static void kfd_process_destroy_pdds(struct kfd_process *p) 927 { 928 int i; 929 930 for (i = 0; i < p->n_pdds; i++) { 931 struct kfd_process_device *pdd = p->pdds[i]; 932 933 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n", 934 pdd->dev->id, p->pasid); 935 936 if (pdd->drm_file) { 937 amdgpu_amdkfd_gpuvm_release_process_vm( 938 pdd->dev->kgd, pdd->drm_priv); 939 fput(pdd->drm_file); 940 } 941 942 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base) 943 free_pages((unsigned long)pdd->qpd.cwsr_kaddr, 944 get_order(KFD_CWSR_TBA_TMA_SIZE)); 945 946 kfree(pdd->qpd.doorbell_bitmap); 947 idr_destroy(&pdd->alloc_idr); 948 949 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index); 950 951 /* 952 * before destroying pdd, make sure to report availability 953 * for auto suspend 954 */ 955 if (pdd->runtime_inuse) { 956 pm_runtime_mark_last_busy(pdd->dev->ddev->dev); 957 pm_runtime_put_autosuspend(pdd->dev->ddev->dev); 958 pdd->runtime_inuse = false; 959 } 960 961 kfree(pdd); 962 p->pdds[i] = NULL; 963 } 964 p->n_pdds = 0; 965 } 966 967 /* No process locking is needed in this function, because the process 968 * is not findable any more. We must assume that no other thread is 969 * using it any more, otherwise we couldn't safely free the process 970 * structure in the end. 971 */ 972 static void kfd_process_wq_release(struct work_struct *work) 973 { 974 struct kfd_process *p = container_of(work, struct kfd_process, 975 release_work); 976 int i; 977 978 /* Remove the procfs files */ 979 if (p->kobj) { 980 sysfs_remove_file(p->kobj, &p->attr_pasid); 981 kobject_del(p->kobj_queues); 982 kobject_put(p->kobj_queues); 983 p->kobj_queues = NULL; 984 985 for (i = 0; i < p->n_pdds; i++) { 986 struct kfd_process_device *pdd = p->pdds[i]; 987 988 sysfs_remove_file(p->kobj, &pdd->attr_vram); 989 sysfs_remove_file(p->kobj, &pdd->attr_sdma); 990 sysfs_remove_file(p->kobj, &pdd->attr_evict); 991 if (pdd->dev->kfd2kgd->get_cu_occupancy != NULL) 992 sysfs_remove_file(p->kobj, &pdd->attr_cu_occupancy); 993 kobject_del(pdd->kobj_stats); 994 kobject_put(pdd->kobj_stats); 995 pdd->kobj_stats = NULL; 996 } 997 998 kobject_del(p->kobj); 999 kobject_put(p->kobj); 1000 p->kobj = NULL; 1001 } 1002 1003 kfd_iommu_unbind_process(p); 1004 1005 kfd_process_free_outstanding_kfd_bos(p); 1006 svm_range_list_fini(p); 1007 1008 kfd_process_destroy_pdds(p); 1009 dma_fence_put(p->ef); 1010 1011 kfd_event_free_process(p); 1012 1013 kfd_pasid_free(p->pasid); 1014 mutex_destroy(&p->mutex); 1015 1016 put_task_struct(p->lead_thread); 1017 1018 kfree(p); 1019 } 1020 1021 static void kfd_process_ref_release(struct kref *ref) 1022 { 1023 struct kfd_process *p = container_of(ref, struct kfd_process, ref); 1024 1025 INIT_WORK(&p->release_work, kfd_process_wq_release); 1026 queue_work(kfd_process_wq, &p->release_work); 1027 } 1028 1029 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm) 1030 { 1031 int idx = srcu_read_lock(&kfd_processes_srcu); 1032 struct kfd_process *p = find_process_by_mm(mm); 1033 1034 srcu_read_unlock(&kfd_processes_srcu, idx); 1035 1036 return p ? &p->mmu_notifier : ERR_PTR(-ESRCH); 1037 } 1038 1039 static void kfd_process_free_notifier(struct mmu_notifier *mn) 1040 { 1041 kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier)); 1042 } 1043 1044 static void kfd_process_notifier_release(struct mmu_notifier *mn, 1045 struct mm_struct *mm) 1046 { 1047 struct kfd_process *p; 1048 int i; 1049 1050 /* 1051 * The kfd_process structure can not be free because the 1052 * mmu_notifier srcu is read locked 1053 */ 1054 p = container_of(mn, struct kfd_process, mmu_notifier); 1055 if (WARN_ON(p->mm != mm)) 1056 return; 1057 1058 mutex_lock(&kfd_processes_mutex); 1059 hash_del_rcu(&p->kfd_processes); 1060 mutex_unlock(&kfd_processes_mutex); 1061 synchronize_srcu(&kfd_processes_srcu); 1062 1063 cancel_delayed_work_sync(&p->eviction_work); 1064 cancel_delayed_work_sync(&p->restore_work); 1065 cancel_delayed_work_sync(&p->svms.restore_work); 1066 1067 mutex_lock(&p->mutex); 1068 1069 /* Iterate over all process device data structures and if the 1070 * pdd is in debug mode, we should first force unregistration, 1071 * then we will be able to destroy the queues 1072 */ 1073 for (i = 0; i < p->n_pdds; i++) { 1074 struct kfd_dev *dev = p->pdds[i]->dev; 1075 1076 mutex_lock(kfd_get_dbgmgr_mutex()); 1077 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) { 1078 if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) { 1079 kfd_dbgmgr_destroy(dev->dbgmgr); 1080 dev->dbgmgr = NULL; 1081 } 1082 } 1083 mutex_unlock(kfd_get_dbgmgr_mutex()); 1084 } 1085 1086 kfd_process_dequeue_from_all_devices(p); 1087 pqm_uninit(&p->pqm); 1088 1089 /* Indicate to other users that MM is no longer valid */ 1090 p->mm = NULL; 1091 /* Signal the eviction fence after user mode queues are 1092 * destroyed. This allows any BOs to be freed without 1093 * triggering pointless evictions or waiting for fences. 1094 */ 1095 dma_fence_signal(p->ef); 1096 1097 mutex_unlock(&p->mutex); 1098 1099 mmu_notifier_put(&p->mmu_notifier); 1100 } 1101 1102 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = { 1103 .release = kfd_process_notifier_release, 1104 .alloc_notifier = kfd_process_alloc_notifier, 1105 .free_notifier = kfd_process_free_notifier, 1106 }; 1107 1108 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep) 1109 { 1110 unsigned long offset; 1111 int i; 1112 1113 for (i = 0; i < p->n_pdds; i++) { 1114 struct kfd_dev *dev = p->pdds[i]->dev; 1115 struct qcm_process_device *qpd = &p->pdds[i]->qpd; 1116 1117 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base) 1118 continue; 1119 1120 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id); 1121 qpd->tba_addr = (int64_t)vm_mmap(filep, 0, 1122 KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC, 1123 MAP_SHARED, offset); 1124 1125 if (IS_ERR_VALUE(qpd->tba_addr)) { 1126 int err = qpd->tba_addr; 1127 1128 pr_err("Failure to set tba address. error %d.\n", err); 1129 qpd->tba_addr = 0; 1130 qpd->cwsr_kaddr = NULL; 1131 return err; 1132 } 1133 1134 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size); 1135 1136 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; 1137 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n", 1138 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); 1139 } 1140 1141 return 0; 1142 } 1143 1144 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) 1145 { 1146 struct kfd_dev *dev = pdd->dev; 1147 struct qcm_process_device *qpd = &pdd->qpd; 1148 uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT 1149 | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE 1150 | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; 1151 void *kaddr; 1152 int ret; 1153 1154 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base) 1155 return 0; 1156 1157 /* cwsr_base is only set for dGPU */ 1158 ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, 1159 KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr); 1160 if (ret) 1161 return ret; 1162 1163 qpd->cwsr_kaddr = kaddr; 1164 qpd->tba_addr = qpd->cwsr_base; 1165 1166 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size); 1167 1168 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET; 1169 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n", 1170 qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr); 1171 1172 return 0; 1173 } 1174 1175 void kfd_process_set_trap_handler(struct qcm_process_device *qpd, 1176 uint64_t tba_addr, 1177 uint64_t tma_addr) 1178 { 1179 if (qpd->cwsr_kaddr) { 1180 /* KFD trap handler is bound, record as second-level TBA/TMA 1181 * in first-level TMA. First-level trap will jump to second. 1182 */ 1183 uint64_t *tma = 1184 (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET); 1185 tma[0] = tba_addr; 1186 tma[1] = tma_addr; 1187 } else { 1188 /* No trap handler bound, bind as first-level TBA/TMA. */ 1189 qpd->tba_addr = tba_addr; 1190 qpd->tma_addr = tma_addr; 1191 } 1192 } 1193 1194 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported) 1195 { 1196 int i; 1197 1198 /* On most GFXv9 GPUs, the retry mode in the SQ must match the 1199 * boot time retry setting. Mixing processes with different 1200 * XNACK/retry settings can hang the GPU. 1201 * 1202 * Different GPUs can have different noretry settings depending 1203 * on HW bugs or limitations. We need to find at least one 1204 * XNACK mode for this process that's compatible with all GPUs. 1205 * Fortunately GPUs with retry enabled (noretry=0) can run code 1206 * built for XNACK-off. On GFXv9 it may perform slower. 1207 * 1208 * Therefore applications built for XNACK-off can always be 1209 * supported and will be our fallback if any GPU does not 1210 * support retry. 1211 */ 1212 for (i = 0; i < p->n_pdds; i++) { 1213 struct kfd_dev *dev = p->pdds[i]->dev; 1214 1215 /* Only consider GFXv9 and higher GPUs. Older GPUs don't 1216 * support the SVM APIs and don't need to be considered 1217 * for the XNACK mode selection. 1218 */ 1219 if (dev->device_info->asic_family < CHIP_VEGA10) 1220 continue; 1221 /* Aldebaran can always support XNACK because it can support 1222 * per-process XNACK mode selection. But let the dev->noretry 1223 * setting still influence the default XNACK mode. 1224 */ 1225 if (supported && 1226 dev->device_info->asic_family == CHIP_ALDEBARAN) 1227 continue; 1228 1229 /* GFXv10 and later GPUs do not support shader preemption 1230 * during page faults. This can lead to poor QoS for queue 1231 * management and memory-manager-related preemptions or 1232 * even deadlocks. 1233 */ 1234 if (dev->device_info->asic_family >= CHIP_NAVI10) 1235 return false; 1236 1237 if (dev->noretry) 1238 return false; 1239 } 1240 1241 return true; 1242 } 1243 1244 /* 1245 * On return the kfd_process is fully operational and will be freed when the 1246 * mm is released 1247 */ 1248 static struct kfd_process *create_process(const struct task_struct *thread) 1249 { 1250 struct kfd_process *process; 1251 struct mmu_notifier *mn; 1252 int err = -ENOMEM; 1253 1254 process = kzalloc(sizeof(*process), GFP_KERNEL); 1255 if (!process) 1256 goto err_alloc_process; 1257 1258 kref_init(&process->ref); 1259 mutex_init(&process->mutex); 1260 process->mm = thread->mm; 1261 process->lead_thread = thread->group_leader; 1262 process->n_pdds = 0; 1263 process->svm_disabled = false; 1264 INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker); 1265 INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); 1266 process->last_restore_timestamp = get_jiffies_64(); 1267 kfd_event_init_process(process); 1268 process->is_32bit_user_mode = in_compat_syscall(); 1269 1270 process->pasid = kfd_pasid_alloc(); 1271 if (process->pasid == 0) 1272 goto err_alloc_pasid; 1273 1274 err = pqm_init(&process->pqm, process); 1275 if (err != 0) 1276 goto err_process_pqm_init; 1277 1278 /* init process apertures*/ 1279 err = kfd_init_apertures(process); 1280 if (err != 0) 1281 goto err_init_apertures; 1282 1283 /* Check XNACK support after PDDs are created in kfd_init_apertures */ 1284 process->xnack_enabled = kfd_process_xnack_mode(process, false); 1285 1286 err = svm_range_list_init(process); 1287 if (err) 1288 goto err_init_svm_range_list; 1289 1290 /* alloc_notifier needs to find the process in the hash table */ 1291 hash_add_rcu(kfd_processes_table, &process->kfd_processes, 1292 (uintptr_t)process->mm); 1293 1294 /* MMU notifier registration must be the last call that can fail 1295 * because after this point we cannot unwind the process creation. 1296 * After this point, mmu_notifier_put will trigger the cleanup by 1297 * dropping the last process reference in the free_notifier. 1298 */ 1299 mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm); 1300 if (IS_ERR(mn)) { 1301 err = PTR_ERR(mn); 1302 goto err_register_notifier; 1303 } 1304 BUG_ON(mn != &process->mmu_notifier); 1305 1306 get_task_struct(process->lead_thread); 1307 1308 return process; 1309 1310 err_register_notifier: 1311 hash_del_rcu(&process->kfd_processes); 1312 svm_range_list_fini(process); 1313 err_init_svm_range_list: 1314 kfd_process_free_outstanding_kfd_bos(process); 1315 kfd_process_destroy_pdds(process); 1316 err_init_apertures: 1317 pqm_uninit(&process->pqm); 1318 err_process_pqm_init: 1319 kfd_pasid_free(process->pasid); 1320 err_alloc_pasid: 1321 mutex_destroy(&process->mutex); 1322 kfree(process); 1323 err_alloc_process: 1324 return ERR_PTR(err); 1325 } 1326 1327 static int init_doorbell_bitmap(struct qcm_process_device *qpd, 1328 struct kfd_dev *dev) 1329 { 1330 unsigned int i; 1331 int range_start = dev->shared_resources.non_cp_doorbells_start; 1332 int range_end = dev->shared_resources.non_cp_doorbells_end; 1333 1334 if (!KFD_IS_SOC15(dev->device_info->asic_family)) 1335 return 0; 1336 1337 qpd->doorbell_bitmap = 1338 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS, 1339 BITS_PER_BYTE), GFP_KERNEL); 1340 if (!qpd->doorbell_bitmap) 1341 return -ENOMEM; 1342 1343 /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */ 1344 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end); 1345 pr_debug("reserved doorbell 0x%03x - 0x%03x\n", 1346 range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, 1347 range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET); 1348 1349 for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) { 1350 if (i >= range_start && i <= range_end) { 1351 set_bit(i, qpd->doorbell_bitmap); 1352 set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET, 1353 qpd->doorbell_bitmap); 1354 } 1355 } 1356 1357 return 0; 1358 } 1359 1360 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev, 1361 struct kfd_process *p) 1362 { 1363 int i; 1364 1365 for (i = 0; i < p->n_pdds; i++) 1366 if (p->pdds[i]->dev == dev) 1367 return p->pdds[i]; 1368 1369 return NULL; 1370 } 1371 1372 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev, 1373 struct kfd_process *p) 1374 { 1375 struct kfd_process_device *pdd = NULL; 1376 1377 if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE)) 1378 return NULL; 1379 pdd = kzalloc(sizeof(*pdd), GFP_KERNEL); 1380 if (!pdd) 1381 return NULL; 1382 1383 if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) { 1384 pr_err("Failed to alloc doorbell for pdd\n"); 1385 goto err_free_pdd; 1386 } 1387 1388 if (init_doorbell_bitmap(&pdd->qpd, dev)) { 1389 pr_err("Failed to init doorbell for process\n"); 1390 goto err_free_pdd; 1391 } 1392 1393 pdd->dev = dev; 1394 INIT_LIST_HEAD(&pdd->qpd.queues_list); 1395 INIT_LIST_HEAD(&pdd->qpd.priv_queue_list); 1396 pdd->qpd.dqm = dev->dqm; 1397 pdd->qpd.pqm = &p->pqm; 1398 pdd->qpd.evicted = 0; 1399 pdd->qpd.mapped_gws_queue = false; 1400 pdd->process = p; 1401 pdd->bound = PDD_UNBOUND; 1402 pdd->already_dequeued = false; 1403 pdd->runtime_inuse = false; 1404 pdd->vram_usage = 0; 1405 pdd->sdma_past_activity_counter = 0; 1406 atomic64_set(&pdd->evict_duration_counter, 0); 1407 p->pdds[p->n_pdds++] = pdd; 1408 1409 /* Init idr used for memory handle translation */ 1410 idr_init(&pdd->alloc_idr); 1411 1412 return pdd; 1413 1414 err_free_pdd: 1415 kfree(pdd); 1416 return NULL; 1417 } 1418 1419 /** 1420 * kfd_process_device_init_vm - Initialize a VM for a process-device 1421 * 1422 * @pdd: The process-device 1423 * @drm_file: Optional pointer to a DRM file descriptor 1424 * 1425 * If @drm_file is specified, it will be used to acquire the VM from 1426 * that file descriptor. If successful, the @pdd takes ownership of 1427 * the file descriptor. 1428 * 1429 * If @drm_file is NULL, a new VM is created. 1430 * 1431 * Returns 0 on success, -errno on failure. 1432 */ 1433 int kfd_process_device_init_vm(struct kfd_process_device *pdd, 1434 struct file *drm_file) 1435 { 1436 struct kfd_process *p; 1437 struct kfd_dev *dev; 1438 int ret; 1439 1440 if (!drm_file) 1441 return -EINVAL; 1442 1443 if (pdd->drm_priv) 1444 return -EBUSY; 1445 1446 p = pdd->process; 1447 dev = pdd->dev; 1448 1449 ret = amdgpu_amdkfd_gpuvm_acquire_process_vm( 1450 dev->kgd, drm_file, p->pasid, 1451 &p->kgd_process_info, &p->ef); 1452 if (ret) { 1453 pr_err("Failed to create process VM object\n"); 1454 return ret; 1455 } 1456 pdd->drm_priv = drm_file->private_data; 1457 1458 ret = kfd_process_device_reserve_ib_mem(pdd); 1459 if (ret) 1460 goto err_reserve_ib_mem; 1461 ret = kfd_process_device_init_cwsr_dgpu(pdd); 1462 if (ret) 1463 goto err_init_cwsr; 1464 1465 pdd->drm_file = drm_file; 1466 1467 return 0; 1468 1469 err_init_cwsr: 1470 err_reserve_ib_mem: 1471 kfd_process_device_free_bos(pdd); 1472 pdd->drm_priv = NULL; 1473 1474 return ret; 1475 } 1476 1477 /* 1478 * Direct the IOMMU to bind the process (specifically the pasid->mm) 1479 * to the device. 1480 * Unbinding occurs when the process dies or the device is removed. 1481 * 1482 * Assumes that the process lock is held. 1483 */ 1484 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev, 1485 struct kfd_process *p) 1486 { 1487 struct kfd_process_device *pdd; 1488 int err; 1489 1490 pdd = kfd_get_process_device_data(dev, p); 1491 if (!pdd) { 1492 pr_err("Process device data doesn't exist\n"); 1493 return ERR_PTR(-ENOMEM); 1494 } 1495 1496 if (!pdd->drm_priv) 1497 return ERR_PTR(-ENODEV); 1498 1499 /* 1500 * signal runtime-pm system to auto resume and prevent 1501 * further runtime suspend once device pdd is created until 1502 * pdd is destroyed. 1503 */ 1504 if (!pdd->runtime_inuse) { 1505 err = pm_runtime_get_sync(dev->ddev->dev); 1506 if (err < 0) { 1507 pm_runtime_put_autosuspend(dev->ddev->dev); 1508 return ERR_PTR(err); 1509 } 1510 } 1511 1512 err = kfd_iommu_bind_process_to_device(pdd); 1513 if (err) 1514 goto out; 1515 1516 /* 1517 * make sure that runtime_usage counter is incremented just once 1518 * per pdd 1519 */ 1520 pdd->runtime_inuse = true; 1521 1522 return pdd; 1523 1524 out: 1525 /* balance runpm reference count and exit with error */ 1526 if (!pdd->runtime_inuse) { 1527 pm_runtime_mark_last_busy(dev->ddev->dev); 1528 pm_runtime_put_autosuspend(dev->ddev->dev); 1529 } 1530 1531 return ERR_PTR(err); 1532 } 1533 1534 /* Create specific handle mapped to mem from process local memory idr 1535 * Assumes that the process lock is held. 1536 */ 1537 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd, 1538 void *mem) 1539 { 1540 return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL); 1541 } 1542 1543 /* Translate specific handle from process local memory idr 1544 * Assumes that the process lock is held. 1545 */ 1546 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd, 1547 int handle) 1548 { 1549 if (handle < 0) 1550 return NULL; 1551 1552 return idr_find(&pdd->alloc_idr, handle); 1553 } 1554 1555 /* Remove specific handle from process local memory idr 1556 * Assumes that the process lock is held. 1557 */ 1558 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd, 1559 int handle) 1560 { 1561 if (handle >= 0) 1562 idr_remove(&pdd->alloc_idr, handle); 1563 } 1564 1565 /* This increments the process->ref counter. */ 1566 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid) 1567 { 1568 struct kfd_process *p, *ret_p = NULL; 1569 unsigned int temp; 1570 1571 int idx = srcu_read_lock(&kfd_processes_srcu); 1572 1573 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1574 if (p->pasid == pasid) { 1575 kref_get(&p->ref); 1576 ret_p = p; 1577 break; 1578 } 1579 } 1580 1581 srcu_read_unlock(&kfd_processes_srcu, idx); 1582 1583 return ret_p; 1584 } 1585 1586 /* This increments the process->ref counter. */ 1587 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm) 1588 { 1589 struct kfd_process *p; 1590 1591 int idx = srcu_read_lock(&kfd_processes_srcu); 1592 1593 p = find_process_by_mm(mm); 1594 if (p) 1595 kref_get(&p->ref); 1596 1597 srcu_read_unlock(&kfd_processes_srcu, idx); 1598 1599 return p; 1600 } 1601 1602 /* kfd_process_evict_queues - Evict all user queues of a process 1603 * 1604 * Eviction is reference-counted per process-device. This means multiple 1605 * evictions from different sources can be nested safely. 1606 */ 1607 int kfd_process_evict_queues(struct kfd_process *p) 1608 { 1609 int r = 0; 1610 int i; 1611 unsigned int n_evicted = 0; 1612 1613 for (i = 0; i < p->n_pdds; i++) { 1614 struct kfd_process_device *pdd = p->pdds[i]; 1615 1616 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm, 1617 &pdd->qpd); 1618 if (r) { 1619 pr_err("Failed to evict process queues\n"); 1620 goto fail; 1621 } 1622 n_evicted++; 1623 } 1624 1625 return r; 1626 1627 fail: 1628 /* To keep state consistent, roll back partial eviction by 1629 * restoring queues 1630 */ 1631 for (i = 0; i < p->n_pdds; i++) { 1632 struct kfd_process_device *pdd = p->pdds[i]; 1633 1634 if (n_evicted == 0) 1635 break; 1636 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, 1637 &pdd->qpd)) 1638 pr_err("Failed to restore queues\n"); 1639 1640 n_evicted--; 1641 } 1642 1643 return r; 1644 } 1645 1646 /* kfd_process_restore_queues - Restore all user queues of a process */ 1647 int kfd_process_restore_queues(struct kfd_process *p) 1648 { 1649 int r, ret = 0; 1650 int i; 1651 1652 for (i = 0; i < p->n_pdds; i++) { 1653 struct kfd_process_device *pdd = p->pdds[i]; 1654 1655 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm, 1656 &pdd->qpd); 1657 if (r) { 1658 pr_err("Failed to restore process queues\n"); 1659 if (!ret) 1660 ret = r; 1661 } 1662 } 1663 1664 return ret; 1665 } 1666 1667 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id) 1668 { 1669 int i; 1670 1671 for (i = 0; i < p->n_pdds; i++) 1672 if (p->pdds[i] && gpu_id == p->pdds[i]->dev->id) 1673 return i; 1674 return -EINVAL; 1675 } 1676 1677 int 1678 kfd_process_gpuid_from_kgd(struct kfd_process *p, struct amdgpu_device *adev, 1679 uint32_t *gpuid, uint32_t *gpuidx) 1680 { 1681 struct kgd_dev *kgd = (struct kgd_dev *)adev; 1682 int i; 1683 1684 for (i = 0; i < p->n_pdds; i++) 1685 if (p->pdds[i] && p->pdds[i]->dev->kgd == kgd) { 1686 *gpuid = p->pdds[i]->dev->id; 1687 *gpuidx = i; 1688 return 0; 1689 } 1690 return -EINVAL; 1691 } 1692 1693 static void evict_process_worker(struct work_struct *work) 1694 { 1695 int ret; 1696 struct kfd_process *p; 1697 struct delayed_work *dwork; 1698 1699 dwork = to_delayed_work(work); 1700 1701 /* Process termination destroys this worker thread. So during the 1702 * lifetime of this thread, kfd_process p will be valid 1703 */ 1704 p = container_of(dwork, struct kfd_process, eviction_work); 1705 WARN_ONCE(p->last_eviction_seqno != p->ef->seqno, 1706 "Eviction fence mismatch\n"); 1707 1708 /* Narrow window of overlap between restore and evict work 1709 * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos 1710 * unreserves KFD BOs, it is possible to evicted again. But 1711 * restore has few more steps of finish. So lets wait for any 1712 * previous restore work to complete 1713 */ 1714 flush_delayed_work(&p->restore_work); 1715 1716 pr_debug("Started evicting pasid 0x%x\n", p->pasid); 1717 ret = kfd_process_evict_queues(p); 1718 if (!ret) { 1719 dma_fence_signal(p->ef); 1720 dma_fence_put(p->ef); 1721 p->ef = NULL; 1722 queue_delayed_work(kfd_restore_wq, &p->restore_work, 1723 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS)); 1724 1725 pr_debug("Finished evicting pasid 0x%x\n", p->pasid); 1726 } else 1727 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid); 1728 } 1729 1730 static void restore_process_worker(struct work_struct *work) 1731 { 1732 struct delayed_work *dwork; 1733 struct kfd_process *p; 1734 int ret = 0; 1735 1736 dwork = to_delayed_work(work); 1737 1738 /* Process termination destroys this worker thread. So during the 1739 * lifetime of this thread, kfd_process p will be valid 1740 */ 1741 p = container_of(dwork, struct kfd_process, restore_work); 1742 pr_debug("Started restoring pasid 0x%x\n", p->pasid); 1743 1744 /* Setting last_restore_timestamp before successful restoration. 1745 * Otherwise this would have to be set by KGD (restore_process_bos) 1746 * before KFD BOs are unreserved. If not, the process can be evicted 1747 * again before the timestamp is set. 1748 * If restore fails, the timestamp will be set again in the next 1749 * attempt. This would mean that the minimum GPU quanta would be 1750 * PROCESS_ACTIVE_TIME_MS - (time to execute the following two 1751 * functions) 1752 */ 1753 1754 p->last_restore_timestamp = get_jiffies_64(); 1755 ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info, 1756 &p->ef); 1757 if (ret) { 1758 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n", 1759 p->pasid, PROCESS_BACK_OFF_TIME_MS); 1760 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work, 1761 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS)); 1762 WARN(!ret, "reschedule restore work failed\n"); 1763 return; 1764 } 1765 1766 ret = kfd_process_restore_queues(p); 1767 if (!ret) 1768 pr_debug("Finished restoring pasid 0x%x\n", p->pasid); 1769 else 1770 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid); 1771 } 1772 1773 void kfd_suspend_all_processes(void) 1774 { 1775 struct kfd_process *p; 1776 unsigned int temp; 1777 int idx = srcu_read_lock(&kfd_processes_srcu); 1778 1779 WARN(debug_evictions, "Evicting all processes"); 1780 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1781 cancel_delayed_work_sync(&p->eviction_work); 1782 cancel_delayed_work_sync(&p->restore_work); 1783 1784 if (kfd_process_evict_queues(p)) 1785 pr_err("Failed to suspend process 0x%x\n", p->pasid); 1786 dma_fence_signal(p->ef); 1787 dma_fence_put(p->ef); 1788 p->ef = NULL; 1789 } 1790 srcu_read_unlock(&kfd_processes_srcu, idx); 1791 } 1792 1793 int kfd_resume_all_processes(void) 1794 { 1795 struct kfd_process *p; 1796 unsigned int temp; 1797 int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu); 1798 1799 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1800 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) { 1801 pr_err("Restore process %d failed during resume\n", 1802 p->pasid); 1803 ret = -EFAULT; 1804 } 1805 } 1806 srcu_read_unlock(&kfd_processes_srcu, idx); 1807 return ret; 1808 } 1809 1810 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process, 1811 struct vm_area_struct *vma) 1812 { 1813 struct kfd_process_device *pdd; 1814 struct qcm_process_device *qpd; 1815 1816 if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) { 1817 pr_err("Incorrect CWSR mapping size.\n"); 1818 return -EINVAL; 1819 } 1820 1821 pdd = kfd_get_process_device_data(dev, process); 1822 if (!pdd) 1823 return -EINVAL; 1824 qpd = &pdd->qpd; 1825 1826 qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1827 get_order(KFD_CWSR_TBA_TMA_SIZE)); 1828 if (!qpd->cwsr_kaddr) { 1829 pr_err("Error allocating per process CWSR buffer.\n"); 1830 return -ENOMEM; 1831 } 1832 1833 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND 1834 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP; 1835 /* Mapping pages to user process */ 1836 return remap_pfn_range(vma, vma->vm_start, 1837 PFN_DOWN(__pa(qpd->cwsr_kaddr)), 1838 KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot); 1839 } 1840 1841 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type) 1842 { 1843 struct kfd_dev *dev = pdd->dev; 1844 1845 if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) { 1846 /* Nothing to flush until a VMID is assigned, which 1847 * only happens when the first queue is created. 1848 */ 1849 if (pdd->qpd.vmid) 1850 amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd, 1851 pdd->qpd.vmid); 1852 } else { 1853 amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd, 1854 pdd->process->pasid, type); 1855 } 1856 } 1857 1858 #if defined(CONFIG_DEBUG_FS) 1859 1860 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data) 1861 { 1862 struct kfd_process *p; 1863 unsigned int temp; 1864 int r = 0; 1865 1866 int idx = srcu_read_lock(&kfd_processes_srcu); 1867 1868 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1869 seq_printf(m, "Process %d PASID 0x%x:\n", 1870 p->lead_thread->tgid, p->pasid); 1871 1872 mutex_lock(&p->mutex); 1873 r = pqm_debugfs_mqds(m, &p->pqm); 1874 mutex_unlock(&p->mutex); 1875 1876 if (r) 1877 break; 1878 } 1879 1880 srcu_read_unlock(&kfd_processes_srcu, idx); 1881 1882 return r; 1883 } 1884 1885 #endif 1886 1887