1 // SPDX-License-Identifier: GPL-2.0 OR MIT 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <linux/mm_types.h> 25 #include <linux/slab.h> 26 #include <linux/types.h> 27 #include <linux/sched/signal.h> 28 #include <linux/sched/mm.h> 29 #include <linux/uaccess.h> 30 #include <linux/mman.h> 31 #include <linux/memory.h> 32 #include "kfd_priv.h" 33 #include "kfd_events.h" 34 #include "kfd_iommu.h" 35 #include <linux/device.h> 36 37 /* 38 * Wrapper around wait_queue_entry_t 39 */ 40 struct kfd_event_waiter { 41 wait_queue_entry_t wait; 42 struct kfd_event *event; /* Event to wait for */ 43 bool activated; /* Becomes true when event is signaled */ 44 }; 45 46 /* 47 * Each signal event needs a 64-bit signal slot where the signaler will write 48 * a 1 before sending an interrupt. (This is needed because some interrupts 49 * do not contain enough spare data bits to identify an event.) 50 * We get whole pages and map them to the process VA. 51 * Individual signal events use their event_id as slot index. 52 */ 53 struct kfd_signal_page { 54 uint64_t *kernel_address; 55 uint64_t __user *user_address; 56 bool need_to_free_pages; 57 }; 58 59 static uint64_t *page_slots(struct kfd_signal_page *page) 60 { 61 return page->kernel_address; 62 } 63 64 static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p) 65 { 66 void *backing_store; 67 struct kfd_signal_page *page; 68 69 page = kzalloc(sizeof(*page), GFP_KERNEL); 70 if (!page) 71 return NULL; 72 73 backing_store = (void *) __get_free_pages(GFP_KERNEL, 74 get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); 75 if (!backing_store) 76 goto fail_alloc_signal_store; 77 78 /* Initialize all events to unsignaled */ 79 memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT, 80 KFD_SIGNAL_EVENT_LIMIT * 8); 81 82 page->kernel_address = backing_store; 83 page->need_to_free_pages = true; 84 pr_debug("Allocated new event signal page at %p, for process %p\n", 85 page, p); 86 87 return page; 88 89 fail_alloc_signal_store: 90 kfree(page); 91 return NULL; 92 } 93 94 static int allocate_event_notification_slot(struct kfd_process *p, 95 struct kfd_event *ev, 96 const int *restore_id) 97 { 98 int id; 99 100 if (!p->signal_page) { 101 p->signal_page = allocate_signal_page(p); 102 if (!p->signal_page) 103 return -ENOMEM; 104 /* Oldest user mode expects 256 event slots */ 105 p->signal_mapped_size = 256*8; 106 } 107 108 if (restore_id) { 109 id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1, 110 GFP_KERNEL); 111 } else { 112 /* 113 * Compatibility with old user mode: Only use signal slots 114 * user mode has mapped, may be less than 115 * KFD_SIGNAL_EVENT_LIMIT. This also allows future increase 116 * of the event limit without breaking user mode. 117 */ 118 id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8, 119 GFP_KERNEL); 120 } 121 if (id < 0) 122 return id; 123 124 ev->event_id = id; 125 page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT; 126 127 return 0; 128 } 129 130 /* 131 * Assumes that p->event_mutex or rcu_readlock is held and of course that p is 132 * not going away. 133 */ 134 static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id) 135 { 136 return idr_find(&p->event_idr, id); 137 } 138 139 /** 140 * lookup_signaled_event_by_partial_id - Lookup signaled event from partial ID 141 * @p: Pointer to struct kfd_process 142 * @id: ID to look up 143 * @bits: Number of valid bits in @id 144 * 145 * Finds the first signaled event with a matching partial ID. If no 146 * matching signaled event is found, returns NULL. In that case the 147 * caller should assume that the partial ID is invalid and do an 148 * exhaustive search of all siglaned events. 149 * 150 * If multiple events with the same partial ID signal at the same 151 * time, they will be found one interrupt at a time, not necessarily 152 * in the same order the interrupts occurred. As long as the number of 153 * interrupts is correct, all signaled events will be seen by the 154 * driver. 155 */ 156 static struct kfd_event *lookup_signaled_event_by_partial_id( 157 struct kfd_process *p, uint32_t id, uint32_t bits) 158 { 159 struct kfd_event *ev; 160 161 if (!p->signal_page || id >= KFD_SIGNAL_EVENT_LIMIT) 162 return NULL; 163 164 /* Fast path for the common case that @id is not a partial ID 165 * and we only need a single lookup. 166 */ 167 if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) { 168 if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) 169 return NULL; 170 171 return idr_find(&p->event_idr, id); 172 } 173 174 /* General case for partial IDs: Iterate over all matching IDs 175 * and find the first one that has signaled. 176 */ 177 for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) { 178 if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) 179 continue; 180 181 ev = idr_find(&p->event_idr, id); 182 } 183 184 return ev; 185 } 186 187 static int create_signal_event(struct file *devkfd, struct kfd_process *p, 188 struct kfd_event *ev, const int *restore_id) 189 { 190 int ret; 191 192 if (p->signal_mapped_size && 193 p->signal_event_count == p->signal_mapped_size / 8) { 194 if (!p->signal_event_limit_reached) { 195 pr_debug("Signal event wasn't created because limit was reached\n"); 196 p->signal_event_limit_reached = true; 197 } 198 return -ENOSPC; 199 } 200 201 ret = allocate_event_notification_slot(p, ev, restore_id); 202 if (ret) { 203 pr_warn("Signal event wasn't created because out of kernel memory\n"); 204 return ret; 205 } 206 207 p->signal_event_count++; 208 209 ev->user_signal_address = &p->signal_page->user_address[ev->event_id]; 210 pr_debug("Signal event number %zu created with id %d, address %p\n", 211 p->signal_event_count, ev->event_id, 212 ev->user_signal_address); 213 214 return 0; 215 } 216 217 static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id) 218 { 219 int id; 220 221 if (restore_id) 222 id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1, 223 GFP_KERNEL); 224 else 225 /* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an 226 * intentional integer overflow to -1 without a compiler 227 * warning. idr_alloc treats a negative value as "maximum 228 * signed integer". 229 */ 230 id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID, 231 (uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1, 232 GFP_KERNEL); 233 234 if (id < 0) 235 return id; 236 ev->event_id = id; 237 238 return 0; 239 } 240 241 void kfd_event_init_process(struct kfd_process *p) 242 { 243 mutex_init(&p->event_mutex); 244 idr_init(&p->event_idr); 245 p->signal_page = NULL; 246 p->signal_event_count = 0; 247 } 248 249 static void destroy_event(struct kfd_process *p, struct kfd_event *ev) 250 { 251 struct kfd_event_waiter *waiter; 252 253 /* Wake up pending waiters. They will return failure */ 254 spin_lock(&ev->lock); 255 list_for_each_entry(waiter, &ev->wq.head, wait.entry) 256 WRITE_ONCE(waiter->event, NULL); 257 wake_up_all(&ev->wq); 258 spin_unlock(&ev->lock); 259 260 if (ev->type == KFD_EVENT_TYPE_SIGNAL || 261 ev->type == KFD_EVENT_TYPE_DEBUG) 262 p->signal_event_count--; 263 264 idr_remove(&p->event_idr, ev->event_id); 265 synchronize_rcu(); 266 kfree(ev); 267 } 268 269 static void destroy_events(struct kfd_process *p) 270 { 271 struct kfd_event *ev; 272 uint32_t id; 273 274 idr_for_each_entry(&p->event_idr, ev, id) 275 destroy_event(p, ev); 276 idr_destroy(&p->event_idr); 277 } 278 279 /* 280 * We assume that the process is being destroyed and there is no need to 281 * unmap the pages or keep bookkeeping data in order. 282 */ 283 static void shutdown_signal_page(struct kfd_process *p) 284 { 285 struct kfd_signal_page *page = p->signal_page; 286 287 if (page) { 288 if (page->need_to_free_pages) 289 free_pages((unsigned long)page->kernel_address, 290 get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); 291 kfree(page); 292 } 293 } 294 295 void kfd_event_free_process(struct kfd_process *p) 296 { 297 destroy_events(p); 298 shutdown_signal_page(p); 299 } 300 301 static bool event_can_be_gpu_signaled(const struct kfd_event *ev) 302 { 303 return ev->type == KFD_EVENT_TYPE_SIGNAL || 304 ev->type == KFD_EVENT_TYPE_DEBUG; 305 } 306 307 static bool event_can_be_cpu_signaled(const struct kfd_event *ev) 308 { 309 return ev->type == KFD_EVENT_TYPE_SIGNAL; 310 } 311 312 static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, 313 uint64_t size, uint64_t user_handle) 314 { 315 struct kfd_signal_page *page; 316 317 if (p->signal_page) 318 return -EBUSY; 319 320 page = kzalloc(sizeof(*page), GFP_KERNEL); 321 if (!page) 322 return -ENOMEM; 323 324 /* Initialize all events to unsignaled */ 325 memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT, 326 KFD_SIGNAL_EVENT_LIMIT * 8); 327 328 page->kernel_address = kernel_address; 329 330 p->signal_page = page; 331 p->signal_mapped_size = size; 332 p->signal_handle = user_handle; 333 return 0; 334 } 335 336 int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset) 337 { 338 struct kfd_dev *kfd; 339 struct kfd_process_device *pdd; 340 void *mem, *kern_addr; 341 uint64_t size; 342 int err = 0; 343 344 if (p->signal_page) { 345 pr_err("Event page is already set\n"); 346 return -EINVAL; 347 } 348 349 pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(event_page_offset)); 350 if (!pdd) { 351 pr_err("Getting device by id failed in %s\n", __func__); 352 return -EINVAL; 353 } 354 kfd = pdd->dev; 355 356 pdd = kfd_bind_process_to_device(kfd, p); 357 if (IS_ERR(pdd)) 358 return PTR_ERR(pdd); 359 360 mem = kfd_process_device_translate_handle(pdd, 361 GET_IDR_HANDLE(event_page_offset)); 362 if (!mem) { 363 pr_err("Can't find BO, offset is 0x%llx\n", event_page_offset); 364 return -EINVAL; 365 } 366 367 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kfd->adev, 368 mem, &kern_addr, &size); 369 if (err) { 370 pr_err("Failed to map event page to kernel\n"); 371 return err; 372 } 373 374 err = kfd_event_page_set(p, kern_addr, size, event_page_offset); 375 if (err) { 376 pr_err("Failed to set event page\n"); 377 amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(kfd->adev, mem); 378 return err; 379 } 380 return err; 381 } 382 383 int kfd_event_create(struct file *devkfd, struct kfd_process *p, 384 uint32_t event_type, bool auto_reset, uint32_t node_id, 385 uint32_t *event_id, uint32_t *event_trigger_data, 386 uint64_t *event_page_offset, uint32_t *event_slot_index) 387 { 388 int ret = 0; 389 struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL); 390 391 if (!ev) 392 return -ENOMEM; 393 394 ev->type = event_type; 395 ev->auto_reset = auto_reset; 396 ev->signaled = false; 397 398 spin_lock_init(&ev->lock); 399 init_waitqueue_head(&ev->wq); 400 401 *event_page_offset = 0; 402 403 mutex_lock(&p->event_mutex); 404 405 switch (event_type) { 406 case KFD_EVENT_TYPE_SIGNAL: 407 case KFD_EVENT_TYPE_DEBUG: 408 ret = create_signal_event(devkfd, p, ev, NULL); 409 if (!ret) { 410 *event_page_offset = KFD_MMAP_TYPE_EVENTS; 411 *event_slot_index = ev->event_id; 412 } 413 break; 414 default: 415 ret = create_other_event(p, ev, NULL); 416 break; 417 } 418 419 if (!ret) { 420 *event_id = ev->event_id; 421 *event_trigger_data = ev->event_id; 422 } else { 423 kfree(ev); 424 } 425 426 mutex_unlock(&p->event_mutex); 427 428 return ret; 429 } 430 431 int kfd_criu_restore_event(struct file *devkfd, 432 struct kfd_process *p, 433 uint8_t __user *user_priv_ptr, 434 uint64_t *priv_data_offset, 435 uint64_t max_priv_data_size) 436 { 437 struct kfd_criu_event_priv_data *ev_priv; 438 struct kfd_event *ev = NULL; 439 int ret = 0; 440 441 ev_priv = kmalloc(sizeof(*ev_priv), GFP_KERNEL); 442 if (!ev_priv) 443 return -ENOMEM; 444 445 ev = kzalloc(sizeof(*ev), GFP_KERNEL); 446 if (!ev) { 447 ret = -ENOMEM; 448 goto exit; 449 } 450 451 if (*priv_data_offset + sizeof(*ev_priv) > max_priv_data_size) { 452 ret = -EINVAL; 453 goto exit; 454 } 455 456 ret = copy_from_user(ev_priv, user_priv_ptr + *priv_data_offset, sizeof(*ev_priv)); 457 if (ret) { 458 ret = -EFAULT; 459 goto exit; 460 } 461 *priv_data_offset += sizeof(*ev_priv); 462 463 if (ev_priv->user_handle) { 464 ret = kfd_kmap_event_page(p, ev_priv->user_handle); 465 if (ret) 466 goto exit; 467 } 468 469 ev->type = ev_priv->type; 470 ev->auto_reset = ev_priv->auto_reset; 471 ev->signaled = ev_priv->signaled; 472 473 spin_lock_init(&ev->lock); 474 init_waitqueue_head(&ev->wq); 475 476 mutex_lock(&p->event_mutex); 477 switch (ev->type) { 478 case KFD_EVENT_TYPE_SIGNAL: 479 case KFD_EVENT_TYPE_DEBUG: 480 ret = create_signal_event(devkfd, p, ev, &ev_priv->event_id); 481 break; 482 case KFD_EVENT_TYPE_MEMORY: 483 memcpy(&ev->memory_exception_data, 484 &ev_priv->memory_exception_data, 485 sizeof(struct kfd_hsa_memory_exception_data)); 486 487 ret = create_other_event(p, ev, &ev_priv->event_id); 488 break; 489 case KFD_EVENT_TYPE_HW_EXCEPTION: 490 memcpy(&ev->hw_exception_data, 491 &ev_priv->hw_exception_data, 492 sizeof(struct kfd_hsa_hw_exception_data)); 493 494 ret = create_other_event(p, ev, &ev_priv->event_id); 495 break; 496 } 497 498 exit: 499 if (ret) 500 kfree(ev); 501 502 kfree(ev_priv); 503 504 mutex_unlock(&p->event_mutex); 505 506 return ret; 507 } 508 509 int kfd_criu_checkpoint_events(struct kfd_process *p, 510 uint8_t __user *user_priv_data, 511 uint64_t *priv_data_offset) 512 { 513 struct kfd_criu_event_priv_data *ev_privs; 514 int i = 0; 515 int ret = 0; 516 struct kfd_event *ev; 517 uint32_t ev_id; 518 519 uint32_t num_events = kfd_get_num_events(p); 520 521 if (!num_events) 522 return 0; 523 524 ev_privs = kvzalloc(num_events * sizeof(*ev_privs), GFP_KERNEL); 525 if (!ev_privs) 526 return -ENOMEM; 527 528 529 idr_for_each_entry(&p->event_idr, ev, ev_id) { 530 struct kfd_criu_event_priv_data *ev_priv; 531 532 /* 533 * Currently, all events have same size of private_data, but the current ioctl's 534 * and CRIU plugin supports private_data of variable sizes 535 */ 536 ev_priv = &ev_privs[i]; 537 538 ev_priv->object_type = KFD_CRIU_OBJECT_TYPE_EVENT; 539 540 /* We store the user_handle with the first event */ 541 if (i == 0 && p->signal_page) 542 ev_priv->user_handle = p->signal_handle; 543 544 ev_priv->event_id = ev->event_id; 545 ev_priv->auto_reset = ev->auto_reset; 546 ev_priv->type = ev->type; 547 ev_priv->signaled = ev->signaled; 548 549 if (ev_priv->type == KFD_EVENT_TYPE_MEMORY) 550 memcpy(&ev_priv->memory_exception_data, 551 &ev->memory_exception_data, 552 sizeof(struct kfd_hsa_memory_exception_data)); 553 else if (ev_priv->type == KFD_EVENT_TYPE_HW_EXCEPTION) 554 memcpy(&ev_priv->hw_exception_data, 555 &ev->hw_exception_data, 556 sizeof(struct kfd_hsa_hw_exception_data)); 557 558 pr_debug("Checkpointed event[%d] id = 0x%08x auto_reset = %x type = %x signaled = %x\n", 559 i, 560 ev_priv->event_id, 561 ev_priv->auto_reset, 562 ev_priv->type, 563 ev_priv->signaled); 564 i++; 565 } 566 567 ret = copy_to_user(user_priv_data + *priv_data_offset, 568 ev_privs, num_events * sizeof(*ev_privs)); 569 if (ret) { 570 pr_err("Failed to copy events priv to user\n"); 571 ret = -EFAULT; 572 } 573 574 *priv_data_offset += num_events * sizeof(*ev_privs); 575 576 kvfree(ev_privs); 577 return ret; 578 } 579 580 int kfd_get_num_events(struct kfd_process *p) 581 { 582 struct kfd_event *ev; 583 uint32_t id; 584 u32 num_events = 0; 585 586 idr_for_each_entry(&p->event_idr, ev, id) 587 num_events++; 588 589 return num_events; 590 } 591 592 /* Assumes that p is current. */ 593 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id) 594 { 595 struct kfd_event *ev; 596 int ret = 0; 597 598 mutex_lock(&p->event_mutex); 599 600 ev = lookup_event_by_id(p, event_id); 601 602 if (ev) 603 destroy_event(p, ev); 604 else 605 ret = -EINVAL; 606 607 mutex_unlock(&p->event_mutex); 608 return ret; 609 } 610 611 static void set_event(struct kfd_event *ev) 612 { 613 struct kfd_event_waiter *waiter; 614 615 /* Auto reset if the list is non-empty and we're waking 616 * someone. waitqueue_active is safe here because we're 617 * protected by the ev->lock, which is also held when 618 * updating the wait queues in kfd_wait_on_events. 619 */ 620 ev->signaled = !ev->auto_reset || !waitqueue_active(&ev->wq); 621 622 list_for_each_entry(waiter, &ev->wq.head, wait.entry) 623 WRITE_ONCE(waiter->activated, true); 624 625 wake_up_all(&ev->wq); 626 } 627 628 /* Assumes that p is current. */ 629 int kfd_set_event(struct kfd_process *p, uint32_t event_id) 630 { 631 int ret = 0; 632 struct kfd_event *ev; 633 634 rcu_read_lock(); 635 636 ev = lookup_event_by_id(p, event_id); 637 spin_lock(&ev->lock); 638 639 if (ev && event_can_be_cpu_signaled(ev)) 640 set_event(ev); 641 else 642 ret = -EINVAL; 643 644 spin_unlock(&ev->lock); 645 rcu_read_unlock(); 646 return ret; 647 } 648 649 static void reset_event(struct kfd_event *ev) 650 { 651 ev->signaled = false; 652 } 653 654 /* Assumes that p is current. */ 655 int kfd_reset_event(struct kfd_process *p, uint32_t event_id) 656 { 657 int ret = 0; 658 struct kfd_event *ev; 659 660 rcu_read_lock(); 661 662 ev = lookup_event_by_id(p, event_id); 663 spin_lock(&ev->lock); 664 665 if (ev && event_can_be_cpu_signaled(ev)) 666 reset_event(ev); 667 else 668 ret = -EINVAL; 669 670 spin_unlock(&ev->lock); 671 rcu_read_unlock(); 672 return ret; 673 674 } 675 676 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev) 677 { 678 WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT); 679 } 680 681 static void set_event_from_interrupt(struct kfd_process *p, 682 struct kfd_event *ev) 683 { 684 if (ev && event_can_be_gpu_signaled(ev)) { 685 acknowledge_signal(p, ev); 686 spin_lock(&ev->lock); 687 set_event(ev); 688 spin_unlock(&ev->lock); 689 } 690 } 691 692 void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, 693 uint32_t valid_id_bits) 694 { 695 struct kfd_event *ev = NULL; 696 697 /* 698 * Because we are called from arbitrary context (workqueue) as opposed 699 * to process context, kfd_process could attempt to exit while we are 700 * running so the lookup function increments the process ref count. 701 */ 702 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 703 704 if (!p) 705 return; /* Presumably process exited. */ 706 707 rcu_read_lock(); 708 709 if (valid_id_bits) 710 ev = lookup_signaled_event_by_partial_id(p, partial_id, 711 valid_id_bits); 712 if (ev) { 713 set_event_from_interrupt(p, ev); 714 } else if (p->signal_page) { 715 /* 716 * Partial ID lookup failed. Assume that the event ID 717 * in the interrupt payload was invalid and do an 718 * exhaustive search of signaled events. 719 */ 720 uint64_t *slots = page_slots(p->signal_page); 721 uint32_t id; 722 723 if (valid_id_bits) 724 pr_debug_ratelimited("Partial ID invalid: %u (%u valid bits)\n", 725 partial_id, valid_id_bits); 726 727 if (p->signal_event_count < KFD_SIGNAL_EVENT_LIMIT / 64) { 728 /* With relatively few events, it's faster to 729 * iterate over the event IDR 730 */ 731 idr_for_each_entry(&p->event_idr, ev, id) { 732 if (id >= KFD_SIGNAL_EVENT_LIMIT) 733 break; 734 735 if (READ_ONCE(slots[id]) != UNSIGNALED_EVENT_SLOT) 736 set_event_from_interrupt(p, ev); 737 } 738 } else { 739 /* With relatively many events, it's faster to 740 * iterate over the signal slots and lookup 741 * only signaled events from the IDR. 742 */ 743 for (id = 0; id < KFD_SIGNAL_EVENT_LIMIT; id++) 744 if (READ_ONCE(slots[id]) != UNSIGNALED_EVENT_SLOT) { 745 ev = lookup_event_by_id(p, id); 746 set_event_from_interrupt(p, ev); 747 } 748 } 749 } 750 751 rcu_read_unlock(); 752 kfd_unref_process(p); 753 } 754 755 static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events) 756 { 757 struct kfd_event_waiter *event_waiters; 758 uint32_t i; 759 760 event_waiters = kmalloc_array(num_events, 761 sizeof(struct kfd_event_waiter), 762 GFP_KERNEL); 763 if (!event_waiters) 764 return NULL; 765 766 for (i = 0; (event_waiters) && (i < num_events) ; i++) { 767 init_wait(&event_waiters[i].wait); 768 event_waiters[i].activated = false; 769 } 770 771 return event_waiters; 772 } 773 774 static int init_event_waiter_get_status(struct kfd_process *p, 775 struct kfd_event_waiter *waiter, 776 uint32_t event_id) 777 { 778 struct kfd_event *ev = lookup_event_by_id(p, event_id); 779 780 if (!ev) 781 return -EINVAL; 782 783 spin_lock(&ev->lock); 784 waiter->event = ev; 785 waiter->activated = ev->signaled; 786 ev->signaled = ev->signaled && !ev->auto_reset; 787 spin_unlock(&ev->lock); 788 789 return 0; 790 } 791 792 static void init_event_waiter_add_to_waitlist(struct kfd_event_waiter *waiter) 793 { 794 struct kfd_event *ev = waiter->event; 795 796 /* Only add to the wait list if we actually need to 797 * wait on this event. 798 */ 799 if (!waiter->activated) { 800 spin_lock(&ev->lock); 801 add_wait_queue(&ev->wq, &waiter->wait); 802 spin_unlock(&ev->lock); 803 } 804 } 805 806 /* test_event_condition - Test condition of events being waited for 807 * @all: Return completion only if all events have signaled 808 * @num_events: Number of events to wait for 809 * @event_waiters: Array of event waiters, one per event 810 * 811 * Returns KFD_IOC_WAIT_RESULT_COMPLETE if all (or one) event(s) have 812 * signaled. Returns KFD_IOC_WAIT_RESULT_TIMEOUT if no (or not all) 813 * events have signaled. Returns KFD_IOC_WAIT_RESULT_FAIL if any of 814 * the events have been destroyed. 815 */ 816 static uint32_t test_event_condition(bool all, uint32_t num_events, 817 struct kfd_event_waiter *event_waiters) 818 { 819 uint32_t i; 820 uint32_t activated_count = 0; 821 822 for (i = 0; i < num_events; i++) { 823 if (!READ_ONCE(event_waiters[i].event)) 824 return KFD_IOC_WAIT_RESULT_FAIL; 825 826 if (READ_ONCE(event_waiters[i].activated)) { 827 if (!all) 828 return KFD_IOC_WAIT_RESULT_COMPLETE; 829 830 activated_count++; 831 } 832 } 833 834 return activated_count == num_events ? 835 KFD_IOC_WAIT_RESULT_COMPLETE : KFD_IOC_WAIT_RESULT_TIMEOUT; 836 } 837 838 /* 839 * Copy event specific data, if defined. 840 * Currently only memory exception events have additional data to copy to user 841 */ 842 static int copy_signaled_event_data(uint32_t num_events, 843 struct kfd_event_waiter *event_waiters, 844 struct kfd_event_data __user *data) 845 { 846 struct kfd_hsa_memory_exception_data *src; 847 struct kfd_hsa_memory_exception_data __user *dst; 848 struct kfd_event_waiter *waiter; 849 struct kfd_event *event; 850 uint32_t i; 851 852 for (i = 0; i < num_events; i++) { 853 waiter = &event_waiters[i]; 854 event = waiter->event; 855 if (!event) 856 return -EINVAL; /* event was destroyed */ 857 if (waiter->activated && event->type == KFD_EVENT_TYPE_MEMORY) { 858 dst = &data[i].memory_exception_data; 859 src = &event->memory_exception_data; 860 if (copy_to_user(dst, src, 861 sizeof(struct kfd_hsa_memory_exception_data))) 862 return -EFAULT; 863 } 864 } 865 866 return 0; 867 } 868 869 static long user_timeout_to_jiffies(uint32_t user_timeout_ms) 870 { 871 if (user_timeout_ms == KFD_EVENT_TIMEOUT_IMMEDIATE) 872 return 0; 873 874 if (user_timeout_ms == KFD_EVENT_TIMEOUT_INFINITE) 875 return MAX_SCHEDULE_TIMEOUT; 876 877 /* 878 * msecs_to_jiffies interprets all values above 2^31-1 as infinite, 879 * but we consider them finite. 880 * This hack is wrong, but nobody is likely to notice. 881 */ 882 user_timeout_ms = min_t(uint32_t, user_timeout_ms, 0x7FFFFFFF); 883 884 return msecs_to_jiffies(user_timeout_ms) + 1; 885 } 886 887 static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters) 888 { 889 uint32_t i; 890 891 for (i = 0; i < num_events; i++) 892 if (waiters[i].event) { 893 spin_lock(&waiters[i].event->lock); 894 remove_wait_queue(&waiters[i].event->wq, 895 &waiters[i].wait); 896 spin_unlock(&waiters[i].event->lock); 897 } 898 899 kfree(waiters); 900 } 901 902 int kfd_wait_on_events(struct kfd_process *p, 903 uint32_t num_events, void __user *data, 904 bool all, uint32_t user_timeout_ms, 905 uint32_t *wait_result) 906 { 907 struct kfd_event_data __user *events = 908 (struct kfd_event_data __user *) data; 909 uint32_t i; 910 int ret = 0; 911 912 struct kfd_event_waiter *event_waiters = NULL; 913 long timeout = user_timeout_to_jiffies(user_timeout_ms); 914 915 event_waiters = alloc_event_waiters(num_events); 916 if (!event_waiters) { 917 ret = -ENOMEM; 918 goto out; 919 } 920 921 /* Use p->event_mutex here to protect against concurrent creation and 922 * destruction of events while we initialize event_waiters. 923 */ 924 mutex_lock(&p->event_mutex); 925 926 for (i = 0; i < num_events; i++) { 927 struct kfd_event_data event_data; 928 929 if (copy_from_user(&event_data, &events[i], 930 sizeof(struct kfd_event_data))) { 931 ret = -EFAULT; 932 goto out_unlock; 933 } 934 935 ret = init_event_waiter_get_status(p, &event_waiters[i], 936 event_data.event_id); 937 if (ret) 938 goto out_unlock; 939 } 940 941 /* Check condition once. */ 942 *wait_result = test_event_condition(all, num_events, event_waiters); 943 if (*wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) { 944 ret = copy_signaled_event_data(num_events, 945 event_waiters, events); 946 goto out_unlock; 947 } else if (WARN_ON(*wait_result == KFD_IOC_WAIT_RESULT_FAIL)) { 948 /* This should not happen. Events shouldn't be 949 * destroyed while we're holding the event_mutex 950 */ 951 goto out_unlock; 952 } 953 954 /* Add to wait lists if we need to wait. */ 955 for (i = 0; i < num_events; i++) 956 init_event_waiter_add_to_waitlist(&event_waiters[i]); 957 958 mutex_unlock(&p->event_mutex); 959 960 while (true) { 961 if (fatal_signal_pending(current)) { 962 ret = -EINTR; 963 break; 964 } 965 966 if (signal_pending(current)) { 967 /* 968 * This is wrong when a nonzero, non-infinite timeout 969 * is specified. We need to use 970 * ERESTARTSYS_RESTARTBLOCK, but struct restart_block 971 * contains a union with data for each user and it's 972 * in generic kernel code that I don't want to 973 * touch yet. 974 */ 975 ret = -ERESTARTSYS; 976 break; 977 } 978 979 /* Set task state to interruptible sleep before 980 * checking wake-up conditions. A concurrent wake-up 981 * will put the task back into runnable state. In that 982 * case schedule_timeout will not put the task to 983 * sleep and we'll get a chance to re-check the 984 * updated conditions almost immediately. Otherwise, 985 * this race condition would lead to a soft hang or a 986 * very long sleep. 987 */ 988 set_current_state(TASK_INTERRUPTIBLE); 989 990 *wait_result = test_event_condition(all, num_events, 991 event_waiters); 992 if (*wait_result != KFD_IOC_WAIT_RESULT_TIMEOUT) 993 break; 994 995 if (timeout <= 0) 996 break; 997 998 timeout = schedule_timeout(timeout); 999 } 1000 __set_current_state(TASK_RUNNING); 1001 1002 mutex_lock(&p->event_mutex); 1003 /* copy_signaled_event_data may sleep. So this has to happen 1004 * after the task state is set back to RUNNING. 1005 * 1006 * The event may also have been destroyed after signaling. So 1007 * copy_signaled_event_data also must confirm that the event 1008 * still exists. Therefore this must be under the p->event_mutex 1009 * which is also held when events are destroyed. 1010 */ 1011 if (!ret && *wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) 1012 ret = copy_signaled_event_data(num_events, 1013 event_waiters, events); 1014 1015 out_unlock: 1016 free_waiters(num_events, event_waiters); 1017 mutex_unlock(&p->event_mutex); 1018 out: 1019 if (ret) 1020 *wait_result = KFD_IOC_WAIT_RESULT_FAIL; 1021 else if (*wait_result == KFD_IOC_WAIT_RESULT_FAIL) 1022 ret = -EIO; 1023 1024 return ret; 1025 } 1026 1027 int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma) 1028 { 1029 unsigned long pfn; 1030 struct kfd_signal_page *page; 1031 int ret; 1032 1033 /* check required size doesn't exceed the allocated size */ 1034 if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) < 1035 get_order(vma->vm_end - vma->vm_start)) { 1036 pr_err("Event page mmap requested illegal size\n"); 1037 return -EINVAL; 1038 } 1039 1040 page = p->signal_page; 1041 if (!page) { 1042 /* Probably KFD bug, but mmap is user-accessible. */ 1043 pr_debug("Signal page could not be found\n"); 1044 return -EINVAL; 1045 } 1046 1047 pfn = __pa(page->kernel_address); 1048 pfn >>= PAGE_SHIFT; 1049 1050 vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE 1051 | VM_DONTDUMP | VM_PFNMAP; 1052 1053 pr_debug("Mapping signal page\n"); 1054 pr_debug(" start user address == 0x%08lx\n", vma->vm_start); 1055 pr_debug(" end user address == 0x%08lx\n", vma->vm_end); 1056 pr_debug(" pfn == 0x%016lX\n", pfn); 1057 pr_debug(" vm_flags == 0x%08lX\n", vma->vm_flags); 1058 pr_debug(" size == 0x%08lX\n", 1059 vma->vm_end - vma->vm_start); 1060 1061 page->user_address = (uint64_t __user *)vma->vm_start; 1062 1063 /* mapping the page to user process */ 1064 ret = remap_pfn_range(vma, vma->vm_start, pfn, 1065 vma->vm_end - vma->vm_start, vma->vm_page_prot); 1066 if (!ret) 1067 p->signal_mapped_size = vma->vm_end - vma->vm_start; 1068 1069 return ret; 1070 } 1071 1072 /* 1073 * Assumes that p is not going away. 1074 */ 1075 static void lookup_events_by_type_and_signal(struct kfd_process *p, 1076 int type, void *event_data) 1077 { 1078 struct kfd_hsa_memory_exception_data *ev_data; 1079 struct kfd_event *ev; 1080 uint32_t id; 1081 bool send_signal = true; 1082 1083 ev_data = (struct kfd_hsa_memory_exception_data *) event_data; 1084 1085 rcu_read_lock(); 1086 1087 id = KFD_FIRST_NONSIGNAL_EVENT_ID; 1088 idr_for_each_entry_continue(&p->event_idr, ev, id) 1089 if (ev->type == type) { 1090 send_signal = false; 1091 dev_dbg(kfd_device, 1092 "Event found: id %X type %d", 1093 ev->event_id, ev->type); 1094 spin_lock(&ev->lock); 1095 set_event(ev); 1096 if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data) 1097 ev->memory_exception_data = *ev_data; 1098 spin_unlock(&ev->lock); 1099 } 1100 1101 if (type == KFD_EVENT_TYPE_MEMORY) { 1102 dev_warn(kfd_device, 1103 "Sending SIGSEGV to process %d (pasid 0x%x)", 1104 p->lead_thread->pid, p->pasid); 1105 send_sig(SIGSEGV, p->lead_thread, 0); 1106 } 1107 1108 /* Send SIGTERM no event of type "type" has been found*/ 1109 if (send_signal) { 1110 if (send_sigterm) { 1111 dev_warn(kfd_device, 1112 "Sending SIGTERM to process %d (pasid 0x%x)", 1113 p->lead_thread->pid, p->pasid); 1114 send_sig(SIGTERM, p->lead_thread, 0); 1115 } else { 1116 dev_err(kfd_device, 1117 "Process %d (pasid 0x%x) got unhandled exception", 1118 p->lead_thread->pid, p->pasid); 1119 } 1120 } 1121 1122 rcu_read_unlock(); 1123 } 1124 1125 #ifdef KFD_SUPPORT_IOMMU_V2 1126 void kfd_signal_iommu_event(struct kfd_dev *dev, u32 pasid, 1127 unsigned long address, bool is_write_requested, 1128 bool is_execute_requested) 1129 { 1130 struct kfd_hsa_memory_exception_data memory_exception_data; 1131 struct vm_area_struct *vma; 1132 int user_gpu_id; 1133 1134 /* 1135 * Because we are called from arbitrary context (workqueue) as opposed 1136 * to process context, kfd_process could attempt to exit while we are 1137 * running so the lookup function increments the process ref count. 1138 */ 1139 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 1140 struct mm_struct *mm; 1141 1142 if (!p) 1143 return; /* Presumably process exited. */ 1144 1145 /* Take a safe reference to the mm_struct, which may otherwise 1146 * disappear even while the kfd_process is still referenced. 1147 */ 1148 mm = get_task_mm(p->lead_thread); 1149 if (!mm) { 1150 kfd_unref_process(p); 1151 return; /* Process is exiting */ 1152 } 1153 1154 user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id); 1155 if (unlikely(user_gpu_id == -EINVAL)) { 1156 WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id); 1157 return; 1158 } 1159 memset(&memory_exception_data, 0, sizeof(memory_exception_data)); 1160 1161 mmap_read_lock(mm); 1162 vma = find_vma(mm, address); 1163 1164 memory_exception_data.gpu_id = user_gpu_id; 1165 memory_exception_data.va = address; 1166 /* Set failure reason */ 1167 memory_exception_data.failure.NotPresent = 1; 1168 memory_exception_data.failure.NoExecute = 0; 1169 memory_exception_data.failure.ReadOnly = 0; 1170 if (vma && address >= vma->vm_start) { 1171 memory_exception_data.failure.NotPresent = 0; 1172 1173 if (is_write_requested && !(vma->vm_flags & VM_WRITE)) 1174 memory_exception_data.failure.ReadOnly = 1; 1175 else 1176 memory_exception_data.failure.ReadOnly = 0; 1177 1178 if (is_execute_requested && !(vma->vm_flags & VM_EXEC)) 1179 memory_exception_data.failure.NoExecute = 1; 1180 else 1181 memory_exception_data.failure.NoExecute = 0; 1182 } 1183 1184 mmap_read_unlock(mm); 1185 mmput(mm); 1186 1187 pr_debug("notpresent %d, noexecute %d, readonly %d\n", 1188 memory_exception_data.failure.NotPresent, 1189 memory_exception_data.failure.NoExecute, 1190 memory_exception_data.failure.ReadOnly); 1191 1192 /* Workaround on Raven to not kill the process when memory is freed 1193 * before IOMMU is able to finish processing all the excessive PPRs 1194 */ 1195 1196 if (KFD_GC_VERSION(dev) != IP_VERSION(9, 1, 0) && 1197 KFD_GC_VERSION(dev) != IP_VERSION(9, 2, 2) && 1198 KFD_GC_VERSION(dev) != IP_VERSION(9, 3, 0)) 1199 lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY, 1200 &memory_exception_data); 1201 1202 kfd_unref_process(p); 1203 } 1204 #endif /* KFD_SUPPORT_IOMMU_V2 */ 1205 1206 void kfd_signal_hw_exception_event(u32 pasid) 1207 { 1208 /* 1209 * Because we are called from arbitrary context (workqueue) as opposed 1210 * to process context, kfd_process could attempt to exit while we are 1211 * running so the lookup function increments the process ref count. 1212 */ 1213 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 1214 1215 if (!p) 1216 return; /* Presumably process exited. */ 1217 1218 lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL); 1219 kfd_unref_process(p); 1220 } 1221 1222 void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid, 1223 struct kfd_vm_fault_info *info) 1224 { 1225 struct kfd_event *ev; 1226 uint32_t id; 1227 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 1228 struct kfd_hsa_memory_exception_data memory_exception_data; 1229 int user_gpu_id; 1230 1231 if (!p) 1232 return; /* Presumably process exited. */ 1233 1234 user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id); 1235 if (unlikely(user_gpu_id == -EINVAL)) { 1236 WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id); 1237 return; 1238 } 1239 1240 memset(&memory_exception_data, 0, sizeof(memory_exception_data)); 1241 memory_exception_data.gpu_id = user_gpu_id; 1242 memory_exception_data.failure.imprecise = true; 1243 /* Set failure reason */ 1244 if (info) { 1245 memory_exception_data.va = (info->page_addr) << PAGE_SHIFT; 1246 memory_exception_data.failure.NotPresent = 1247 info->prot_valid ? 1 : 0; 1248 memory_exception_data.failure.NoExecute = 1249 info->prot_exec ? 1 : 0; 1250 memory_exception_data.failure.ReadOnly = 1251 info->prot_write ? 1 : 0; 1252 memory_exception_data.failure.imprecise = 0; 1253 } 1254 1255 rcu_read_lock(); 1256 1257 id = KFD_FIRST_NONSIGNAL_EVENT_ID; 1258 idr_for_each_entry_continue(&p->event_idr, ev, id) 1259 if (ev->type == KFD_EVENT_TYPE_MEMORY) { 1260 spin_lock(&ev->lock); 1261 ev->memory_exception_data = memory_exception_data; 1262 set_event(ev); 1263 spin_unlock(&ev->lock); 1264 } 1265 1266 rcu_read_unlock(); 1267 kfd_unref_process(p); 1268 } 1269 1270 void kfd_signal_reset_event(struct kfd_dev *dev) 1271 { 1272 struct kfd_hsa_hw_exception_data hw_exception_data; 1273 struct kfd_hsa_memory_exception_data memory_exception_data; 1274 struct kfd_process *p; 1275 struct kfd_event *ev; 1276 unsigned int temp; 1277 uint32_t id, idx; 1278 int reset_cause = atomic_read(&dev->sram_ecc_flag) ? 1279 KFD_HW_EXCEPTION_ECC : 1280 KFD_HW_EXCEPTION_GPU_HANG; 1281 1282 /* Whole gpu reset caused by GPU hang and memory is lost */ 1283 memset(&hw_exception_data, 0, sizeof(hw_exception_data)); 1284 hw_exception_data.memory_lost = 1; 1285 hw_exception_data.reset_cause = reset_cause; 1286 1287 memset(&memory_exception_data, 0, sizeof(memory_exception_data)); 1288 memory_exception_data.ErrorType = KFD_MEM_ERR_SRAM_ECC; 1289 memory_exception_data.failure.imprecise = true; 1290 1291 idx = srcu_read_lock(&kfd_processes_srcu); 1292 hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) { 1293 int user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id); 1294 1295 if (unlikely(user_gpu_id == -EINVAL)) { 1296 WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id); 1297 continue; 1298 } 1299 1300 rcu_read_lock(); 1301 1302 id = KFD_FIRST_NONSIGNAL_EVENT_ID; 1303 idr_for_each_entry_continue(&p->event_idr, ev, id) { 1304 if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) { 1305 spin_lock(&ev->lock); 1306 ev->hw_exception_data = hw_exception_data; 1307 ev->hw_exception_data.gpu_id = user_gpu_id; 1308 set_event(ev); 1309 spin_unlock(&ev->lock); 1310 } 1311 if (ev->type == KFD_EVENT_TYPE_MEMORY && 1312 reset_cause == KFD_HW_EXCEPTION_ECC) { 1313 spin_lock(&ev->lock); 1314 ev->memory_exception_data = memory_exception_data; 1315 ev->memory_exception_data.gpu_id = user_gpu_id; 1316 set_event(ev); 1317 spin_unlock(&ev->lock); 1318 } 1319 } 1320 1321 rcu_read_unlock(); 1322 } 1323 srcu_read_unlock(&kfd_processes_srcu, idx); 1324 } 1325 1326 void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid) 1327 { 1328 struct kfd_process *p = kfd_lookup_process_by_pasid(pasid); 1329 struct kfd_hsa_memory_exception_data memory_exception_data; 1330 struct kfd_hsa_hw_exception_data hw_exception_data; 1331 struct kfd_event *ev; 1332 uint32_t id = KFD_FIRST_NONSIGNAL_EVENT_ID; 1333 int user_gpu_id; 1334 1335 if (!p) 1336 return; /* Presumably process exited. */ 1337 1338 user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id); 1339 if (unlikely(user_gpu_id == -EINVAL)) { 1340 WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id); 1341 return; 1342 } 1343 1344 memset(&hw_exception_data, 0, sizeof(hw_exception_data)); 1345 hw_exception_data.gpu_id = user_gpu_id; 1346 hw_exception_data.memory_lost = 1; 1347 hw_exception_data.reset_cause = KFD_HW_EXCEPTION_ECC; 1348 1349 memset(&memory_exception_data, 0, sizeof(memory_exception_data)); 1350 memory_exception_data.ErrorType = KFD_MEM_ERR_POISON_CONSUMED; 1351 memory_exception_data.gpu_id = user_gpu_id; 1352 memory_exception_data.failure.imprecise = true; 1353 1354 rcu_read_lock(); 1355 1356 idr_for_each_entry_continue(&p->event_idr, ev, id) { 1357 if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) { 1358 spin_lock(&ev->lock); 1359 ev->hw_exception_data = hw_exception_data; 1360 set_event(ev); 1361 spin_unlock(&ev->lock); 1362 } 1363 1364 if (ev->type == KFD_EVENT_TYPE_MEMORY) { 1365 spin_lock(&ev->lock); 1366 ev->memory_exception_data = memory_exception_data; 1367 set_event(ev); 1368 spin_unlock(&ev->lock); 1369 } 1370 } 1371 1372 rcu_read_unlock(); 1373 1374 /* user application will handle SIGBUS signal */ 1375 send_sig(SIGBUS, p->lead_thread, 0); 1376 1377 kfd_unref_process(p); 1378 } 1379