1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /*
3  * Copyright 2014-2022 Advanced Micro Devices, Inc.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21  * OTHER DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include <linux/mm_types.h>
25 #include <linux/slab.h>
26 #include <linux/types.h>
27 #include <linux/sched/signal.h>
28 #include <linux/sched/mm.h>
29 #include <linux/uaccess.h>
30 #include <linux/mman.h>
31 #include <linux/memory.h>
32 #include "kfd_priv.h"
33 #include "kfd_events.h"
34 #include "kfd_iommu.h"
35 #include <linux/device.h>
36 
37 /*
38  * Wrapper around wait_queue_entry_t
39  */
40 struct kfd_event_waiter {
41 	wait_queue_entry_t wait;
42 	struct kfd_event *event; /* Event to wait for */
43 	bool activated;		 /* Becomes true when event is signaled */
44 };
45 
46 /*
47  * Each signal event needs a 64-bit signal slot where the signaler will write
48  * a 1 before sending an interrupt. (This is needed because some interrupts
49  * do not contain enough spare data bits to identify an event.)
50  * We get whole pages and map them to the process VA.
51  * Individual signal events use their event_id as slot index.
52  */
53 struct kfd_signal_page {
54 	uint64_t *kernel_address;
55 	uint64_t __user *user_address;
56 	bool need_to_free_pages;
57 };
58 
59 static uint64_t *page_slots(struct kfd_signal_page *page)
60 {
61 	return page->kernel_address;
62 }
63 
64 static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p)
65 {
66 	void *backing_store;
67 	struct kfd_signal_page *page;
68 
69 	page = kzalloc(sizeof(*page), GFP_KERNEL);
70 	if (!page)
71 		return NULL;
72 
73 	backing_store = (void *) __get_free_pages(GFP_KERNEL,
74 					get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
75 	if (!backing_store)
76 		goto fail_alloc_signal_store;
77 
78 	/* Initialize all events to unsignaled */
79 	memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT,
80 	       KFD_SIGNAL_EVENT_LIMIT * 8);
81 
82 	page->kernel_address = backing_store;
83 	page->need_to_free_pages = true;
84 	pr_debug("Allocated new event signal page at %p, for process %p\n",
85 			page, p);
86 
87 	return page;
88 
89 fail_alloc_signal_store:
90 	kfree(page);
91 	return NULL;
92 }
93 
94 static int allocate_event_notification_slot(struct kfd_process *p,
95 					    struct kfd_event *ev,
96 					    const int *restore_id)
97 {
98 	int id;
99 
100 	if (!p->signal_page) {
101 		p->signal_page = allocate_signal_page(p);
102 		if (!p->signal_page)
103 			return -ENOMEM;
104 		/* Oldest user mode expects 256 event slots */
105 		p->signal_mapped_size = 256*8;
106 	}
107 
108 	if (restore_id) {
109 		id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,
110 				GFP_KERNEL);
111 	} else {
112 		/*
113 		 * Compatibility with old user mode: Only use signal slots
114 		 * user mode has mapped, may be less than
115 		 * KFD_SIGNAL_EVENT_LIMIT. This also allows future increase
116 		 * of the event limit without breaking user mode.
117 		 */
118 		id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,
119 				GFP_KERNEL);
120 	}
121 	if (id < 0)
122 		return id;
123 
124 	ev->event_id = id;
125 	page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT;
126 
127 	return 0;
128 }
129 
130 /*
131  * Assumes that p->event_mutex or rcu_readlock is held and of course that p is
132  * not going away.
133  */
134 static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)
135 {
136 	return idr_find(&p->event_idr, id);
137 }
138 
139 /**
140  * lookup_signaled_event_by_partial_id - Lookup signaled event from partial ID
141  * @p:     Pointer to struct kfd_process
142  * @id:    ID to look up
143  * @bits:  Number of valid bits in @id
144  *
145  * Finds the first signaled event with a matching partial ID. If no
146  * matching signaled event is found, returns NULL. In that case the
147  * caller should assume that the partial ID is invalid and do an
148  * exhaustive search of all siglaned events.
149  *
150  * If multiple events with the same partial ID signal at the same
151  * time, they will be found one interrupt at a time, not necessarily
152  * in the same order the interrupts occurred. As long as the number of
153  * interrupts is correct, all signaled events will be seen by the
154  * driver.
155  */
156 static struct kfd_event *lookup_signaled_event_by_partial_id(
157 	struct kfd_process *p, uint32_t id, uint32_t bits)
158 {
159 	struct kfd_event *ev;
160 
161 	if (!p->signal_page || id >= KFD_SIGNAL_EVENT_LIMIT)
162 		return NULL;
163 
164 	/* Fast path for the common case that @id is not a partial ID
165 	 * and we only need a single lookup.
166 	 */
167 	if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) {
168 		if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
169 			return NULL;
170 
171 		return idr_find(&p->event_idr, id);
172 	}
173 
174 	/* General case for partial IDs: Iterate over all matching IDs
175 	 * and find the first one that has signaled.
176 	 */
177 	for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) {
178 		if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)
179 			continue;
180 
181 		ev = idr_find(&p->event_idr, id);
182 	}
183 
184 	return ev;
185 }
186 
187 static int create_signal_event(struct file *devkfd, struct kfd_process *p,
188 				struct kfd_event *ev, const int *restore_id)
189 {
190 	int ret;
191 
192 	if (p->signal_mapped_size &&
193 	    p->signal_event_count == p->signal_mapped_size / 8) {
194 		if (!p->signal_event_limit_reached) {
195 			pr_debug("Signal event wasn't created because limit was reached\n");
196 			p->signal_event_limit_reached = true;
197 		}
198 		return -ENOSPC;
199 	}
200 
201 	ret = allocate_event_notification_slot(p, ev, restore_id);
202 	if (ret) {
203 		pr_warn("Signal event wasn't created because out of kernel memory\n");
204 		return ret;
205 	}
206 
207 	p->signal_event_count++;
208 
209 	ev->user_signal_address = &p->signal_page->user_address[ev->event_id];
210 	pr_debug("Signal event number %zu created with id %d, address %p\n",
211 			p->signal_event_count, ev->event_id,
212 			ev->user_signal_address);
213 
214 	return 0;
215 }
216 
217 static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id)
218 {
219 	int id;
220 
221 	if (restore_id)
222 		id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,
223 			GFP_KERNEL);
224 	else
225 		/* Cast KFD_LAST_NONSIGNAL_EVENT to uint32_t. This allows an
226 		 * intentional integer overflow to -1 without a compiler
227 		 * warning. idr_alloc treats a negative value as "maximum
228 		 * signed integer".
229 		 */
230 		id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,
231 				(uint32_t)KFD_LAST_NONSIGNAL_EVENT_ID + 1,
232 				GFP_KERNEL);
233 
234 	if (id < 0)
235 		return id;
236 	ev->event_id = id;
237 
238 	return 0;
239 }
240 
241 int kfd_event_init_process(struct kfd_process *p)
242 {
243 	int id;
244 
245 	mutex_init(&p->event_mutex);
246 	idr_init(&p->event_idr);
247 	p->signal_page = NULL;
248 	p->signal_event_count = 1;
249 	/* Allocate event ID 0. It is used for a fast path to ignore bogus events
250 	 * that are sent by the CP without a context ID
251 	 */
252 	id = idr_alloc(&p->event_idr, NULL, 0, 1, GFP_KERNEL);
253 	if (id < 0) {
254 		idr_destroy(&p->event_idr);
255 		mutex_destroy(&p->event_mutex);
256 		return id;
257 	}
258 	return 0;
259 }
260 
261 static void destroy_event(struct kfd_process *p, struct kfd_event *ev)
262 {
263 	struct kfd_event_waiter *waiter;
264 
265 	/* Wake up pending waiters. They will return failure */
266 	spin_lock(&ev->lock);
267 	list_for_each_entry(waiter, &ev->wq.head, wait.entry)
268 		WRITE_ONCE(waiter->event, NULL);
269 	wake_up_all(&ev->wq);
270 	spin_unlock(&ev->lock);
271 
272 	if (ev->type == KFD_EVENT_TYPE_SIGNAL ||
273 	    ev->type == KFD_EVENT_TYPE_DEBUG)
274 		p->signal_event_count--;
275 
276 	idr_remove(&p->event_idr, ev->event_id);
277 	kfree_rcu(ev, rcu);
278 }
279 
280 static void destroy_events(struct kfd_process *p)
281 {
282 	struct kfd_event *ev;
283 	uint32_t id;
284 
285 	idr_for_each_entry(&p->event_idr, ev, id)
286 		if (ev)
287 			destroy_event(p, ev);
288 	idr_destroy(&p->event_idr);
289 	mutex_destroy(&p->event_mutex);
290 }
291 
292 /*
293  * We assume that the process is being destroyed and there is no need to
294  * unmap the pages or keep bookkeeping data in order.
295  */
296 static void shutdown_signal_page(struct kfd_process *p)
297 {
298 	struct kfd_signal_page *page = p->signal_page;
299 
300 	if (page) {
301 		if (page->need_to_free_pages)
302 			free_pages((unsigned long)page->kernel_address,
303 				   get_order(KFD_SIGNAL_EVENT_LIMIT * 8));
304 		kfree(page);
305 	}
306 }
307 
308 void kfd_event_free_process(struct kfd_process *p)
309 {
310 	destroy_events(p);
311 	shutdown_signal_page(p);
312 }
313 
314 static bool event_can_be_gpu_signaled(const struct kfd_event *ev)
315 {
316 	return ev->type == KFD_EVENT_TYPE_SIGNAL ||
317 					ev->type == KFD_EVENT_TYPE_DEBUG;
318 }
319 
320 static bool event_can_be_cpu_signaled(const struct kfd_event *ev)
321 {
322 	return ev->type == KFD_EVENT_TYPE_SIGNAL;
323 }
324 
325 static int kfd_event_page_set(struct kfd_process *p, void *kernel_address,
326 		       uint64_t size, uint64_t user_handle)
327 {
328 	struct kfd_signal_page *page;
329 
330 	if (p->signal_page)
331 		return -EBUSY;
332 
333 	page = kzalloc(sizeof(*page), GFP_KERNEL);
334 	if (!page)
335 		return -ENOMEM;
336 
337 	/* Initialize all events to unsignaled */
338 	memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT,
339 	       KFD_SIGNAL_EVENT_LIMIT * 8);
340 
341 	page->kernel_address = kernel_address;
342 
343 	p->signal_page = page;
344 	p->signal_mapped_size = size;
345 	p->signal_handle = user_handle;
346 	return 0;
347 }
348 
349 int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset)
350 {
351 	struct kfd_dev *kfd;
352 	struct kfd_process_device *pdd;
353 	void *mem, *kern_addr;
354 	uint64_t size;
355 	int err = 0;
356 
357 	if (p->signal_page) {
358 		pr_err("Event page is already set\n");
359 		return -EINVAL;
360 	}
361 
362 	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(event_page_offset));
363 	if (!pdd) {
364 		pr_err("Getting device by id failed in %s\n", __func__);
365 		return -EINVAL;
366 	}
367 	kfd = pdd->dev;
368 
369 	pdd = kfd_bind_process_to_device(kfd, p);
370 	if (IS_ERR(pdd))
371 		return PTR_ERR(pdd);
372 
373 	mem = kfd_process_device_translate_handle(pdd,
374 			GET_IDR_HANDLE(event_page_offset));
375 	if (!mem) {
376 		pr_err("Can't find BO, offset is 0x%llx\n", event_page_offset);
377 		return -EINVAL;
378 	}
379 
380 	err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(mem, &kern_addr, &size);
381 	if (err) {
382 		pr_err("Failed to map event page to kernel\n");
383 		return err;
384 	}
385 
386 	err = kfd_event_page_set(p, kern_addr, size, event_page_offset);
387 	if (err) {
388 		pr_err("Failed to set event page\n");
389 		amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem);
390 		return err;
391 	}
392 	return err;
393 }
394 
395 int kfd_event_create(struct file *devkfd, struct kfd_process *p,
396 		     uint32_t event_type, bool auto_reset, uint32_t node_id,
397 		     uint32_t *event_id, uint32_t *event_trigger_data,
398 		     uint64_t *event_page_offset, uint32_t *event_slot_index)
399 {
400 	int ret = 0;
401 	struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL);
402 
403 	if (!ev)
404 		return -ENOMEM;
405 
406 	ev->type = event_type;
407 	ev->auto_reset = auto_reset;
408 	ev->signaled = false;
409 
410 	spin_lock_init(&ev->lock);
411 	init_waitqueue_head(&ev->wq);
412 
413 	*event_page_offset = 0;
414 
415 	mutex_lock(&p->event_mutex);
416 
417 	switch (event_type) {
418 	case KFD_EVENT_TYPE_SIGNAL:
419 	case KFD_EVENT_TYPE_DEBUG:
420 		ret = create_signal_event(devkfd, p, ev, NULL);
421 		if (!ret) {
422 			*event_page_offset = KFD_MMAP_TYPE_EVENTS;
423 			*event_slot_index = ev->event_id;
424 		}
425 		break;
426 	default:
427 		ret = create_other_event(p, ev, NULL);
428 		break;
429 	}
430 
431 	if (!ret) {
432 		*event_id = ev->event_id;
433 		*event_trigger_data = ev->event_id;
434 	} else {
435 		kfree(ev);
436 	}
437 
438 	mutex_unlock(&p->event_mutex);
439 
440 	return ret;
441 }
442 
443 int kfd_criu_restore_event(struct file *devkfd,
444 			   struct kfd_process *p,
445 			   uint8_t __user *user_priv_ptr,
446 			   uint64_t *priv_data_offset,
447 			   uint64_t max_priv_data_size)
448 {
449 	struct kfd_criu_event_priv_data *ev_priv;
450 	struct kfd_event *ev = NULL;
451 	int ret = 0;
452 
453 	ev_priv = kmalloc(sizeof(*ev_priv), GFP_KERNEL);
454 	if (!ev_priv)
455 		return -ENOMEM;
456 
457 	ev = kzalloc(sizeof(*ev), GFP_KERNEL);
458 	if (!ev) {
459 		ret = -ENOMEM;
460 		goto exit;
461 	}
462 
463 	if (*priv_data_offset + sizeof(*ev_priv) > max_priv_data_size) {
464 		ret = -EINVAL;
465 		goto exit;
466 	}
467 
468 	ret = copy_from_user(ev_priv, user_priv_ptr + *priv_data_offset, sizeof(*ev_priv));
469 	if (ret) {
470 		ret = -EFAULT;
471 		goto exit;
472 	}
473 	*priv_data_offset += sizeof(*ev_priv);
474 
475 	if (ev_priv->user_handle) {
476 		ret = kfd_kmap_event_page(p, ev_priv->user_handle);
477 		if (ret)
478 			goto exit;
479 	}
480 
481 	ev->type = ev_priv->type;
482 	ev->auto_reset = ev_priv->auto_reset;
483 	ev->signaled = ev_priv->signaled;
484 
485 	spin_lock_init(&ev->lock);
486 	init_waitqueue_head(&ev->wq);
487 
488 	mutex_lock(&p->event_mutex);
489 	switch (ev->type) {
490 	case KFD_EVENT_TYPE_SIGNAL:
491 	case KFD_EVENT_TYPE_DEBUG:
492 		ret = create_signal_event(devkfd, p, ev, &ev_priv->event_id);
493 		break;
494 	case KFD_EVENT_TYPE_MEMORY:
495 		memcpy(&ev->memory_exception_data,
496 			&ev_priv->memory_exception_data,
497 			sizeof(struct kfd_hsa_memory_exception_data));
498 
499 		ret = create_other_event(p, ev, &ev_priv->event_id);
500 		break;
501 	case KFD_EVENT_TYPE_HW_EXCEPTION:
502 		memcpy(&ev->hw_exception_data,
503 			&ev_priv->hw_exception_data,
504 			sizeof(struct kfd_hsa_hw_exception_data));
505 
506 		ret = create_other_event(p, ev, &ev_priv->event_id);
507 		break;
508 	}
509 
510 exit:
511 	if (ret)
512 		kfree(ev);
513 
514 	kfree(ev_priv);
515 
516 	mutex_unlock(&p->event_mutex);
517 
518 	return ret;
519 }
520 
521 int kfd_criu_checkpoint_events(struct kfd_process *p,
522 			 uint8_t __user *user_priv_data,
523 			 uint64_t *priv_data_offset)
524 {
525 	struct kfd_criu_event_priv_data *ev_privs;
526 	int i = 0;
527 	int ret =  0;
528 	struct kfd_event *ev;
529 	uint32_t ev_id;
530 
531 	uint32_t num_events = kfd_get_num_events(p);
532 
533 	if (!num_events)
534 		return 0;
535 
536 	ev_privs = kvzalloc(num_events * sizeof(*ev_privs), GFP_KERNEL);
537 	if (!ev_privs)
538 		return -ENOMEM;
539 
540 
541 	idr_for_each_entry(&p->event_idr, ev, ev_id) {
542 		struct kfd_criu_event_priv_data *ev_priv;
543 
544 		/*
545 		 * Currently, all events have same size of private_data, but the current ioctl's
546 		 * and CRIU plugin supports private_data of variable sizes
547 		 */
548 		ev_priv = &ev_privs[i];
549 
550 		ev_priv->object_type = KFD_CRIU_OBJECT_TYPE_EVENT;
551 
552 		/* We store the user_handle with the first event */
553 		if (i == 0 && p->signal_page)
554 			ev_priv->user_handle = p->signal_handle;
555 
556 		ev_priv->event_id = ev->event_id;
557 		ev_priv->auto_reset = ev->auto_reset;
558 		ev_priv->type = ev->type;
559 		ev_priv->signaled = ev->signaled;
560 
561 		if (ev_priv->type == KFD_EVENT_TYPE_MEMORY)
562 			memcpy(&ev_priv->memory_exception_data,
563 				&ev->memory_exception_data,
564 				sizeof(struct kfd_hsa_memory_exception_data));
565 		else if (ev_priv->type == KFD_EVENT_TYPE_HW_EXCEPTION)
566 			memcpy(&ev_priv->hw_exception_data,
567 				&ev->hw_exception_data,
568 				sizeof(struct kfd_hsa_hw_exception_data));
569 
570 		pr_debug("Checkpointed event[%d] id = 0x%08x auto_reset = %x type = %x signaled = %x\n",
571 			  i,
572 			  ev_priv->event_id,
573 			  ev_priv->auto_reset,
574 			  ev_priv->type,
575 			  ev_priv->signaled);
576 		i++;
577 	}
578 
579 	ret = copy_to_user(user_priv_data + *priv_data_offset,
580 			   ev_privs, num_events * sizeof(*ev_privs));
581 	if (ret) {
582 		pr_err("Failed to copy events priv to user\n");
583 		ret = -EFAULT;
584 	}
585 
586 	*priv_data_offset += num_events * sizeof(*ev_privs);
587 
588 	kvfree(ev_privs);
589 	return ret;
590 }
591 
592 int kfd_get_num_events(struct kfd_process *p)
593 {
594 	struct kfd_event *ev;
595 	uint32_t id;
596 	u32 num_events = 0;
597 
598 	idr_for_each_entry(&p->event_idr, ev, id)
599 		num_events++;
600 
601 	return num_events;
602 }
603 
604 /* Assumes that p is current. */
605 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id)
606 {
607 	struct kfd_event *ev;
608 	int ret = 0;
609 
610 	mutex_lock(&p->event_mutex);
611 
612 	ev = lookup_event_by_id(p, event_id);
613 
614 	if (ev)
615 		destroy_event(p, ev);
616 	else
617 		ret = -EINVAL;
618 
619 	mutex_unlock(&p->event_mutex);
620 	return ret;
621 }
622 
623 static void set_event(struct kfd_event *ev)
624 {
625 	struct kfd_event_waiter *waiter;
626 
627 	/* Auto reset if the list is non-empty and we're waking
628 	 * someone. waitqueue_active is safe here because we're
629 	 * protected by the ev->lock, which is also held when
630 	 * updating the wait queues in kfd_wait_on_events.
631 	 */
632 	ev->signaled = !ev->auto_reset || !waitqueue_active(&ev->wq);
633 
634 	list_for_each_entry(waiter, &ev->wq.head, wait.entry)
635 		WRITE_ONCE(waiter->activated, true);
636 
637 	wake_up_all(&ev->wq);
638 }
639 
640 /* Assumes that p is current. */
641 int kfd_set_event(struct kfd_process *p, uint32_t event_id)
642 {
643 	int ret = 0;
644 	struct kfd_event *ev;
645 
646 	rcu_read_lock();
647 
648 	ev = lookup_event_by_id(p, event_id);
649 	if (!ev) {
650 		ret = -EINVAL;
651 		goto unlock_rcu;
652 	}
653 	spin_lock(&ev->lock);
654 
655 	if (event_can_be_cpu_signaled(ev))
656 		set_event(ev);
657 	else
658 		ret = -EINVAL;
659 
660 	spin_unlock(&ev->lock);
661 unlock_rcu:
662 	rcu_read_unlock();
663 	return ret;
664 }
665 
666 static void reset_event(struct kfd_event *ev)
667 {
668 	ev->signaled = false;
669 }
670 
671 /* Assumes that p is current. */
672 int kfd_reset_event(struct kfd_process *p, uint32_t event_id)
673 {
674 	int ret = 0;
675 	struct kfd_event *ev;
676 
677 	rcu_read_lock();
678 
679 	ev = lookup_event_by_id(p, event_id);
680 	if (!ev) {
681 		ret = -EINVAL;
682 		goto unlock_rcu;
683 	}
684 	spin_lock(&ev->lock);
685 
686 	if (event_can_be_cpu_signaled(ev))
687 		reset_event(ev);
688 	else
689 		ret = -EINVAL;
690 
691 	spin_unlock(&ev->lock);
692 unlock_rcu:
693 	rcu_read_unlock();
694 	return ret;
695 
696 }
697 
698 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)
699 {
700 	WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT);
701 }
702 
703 static void set_event_from_interrupt(struct kfd_process *p,
704 					struct kfd_event *ev)
705 {
706 	if (ev && event_can_be_gpu_signaled(ev)) {
707 		acknowledge_signal(p, ev);
708 		spin_lock(&ev->lock);
709 		set_event(ev);
710 		spin_unlock(&ev->lock);
711 	}
712 }
713 
714 void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id,
715 				uint32_t valid_id_bits)
716 {
717 	struct kfd_event *ev = NULL;
718 
719 	/*
720 	 * Because we are called from arbitrary context (workqueue) as opposed
721 	 * to process context, kfd_process could attempt to exit while we are
722 	 * running so the lookup function increments the process ref count.
723 	 */
724 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
725 
726 	if (!p)
727 		return; /* Presumably process exited. */
728 
729 	rcu_read_lock();
730 
731 	if (valid_id_bits)
732 		ev = lookup_signaled_event_by_partial_id(p, partial_id,
733 							 valid_id_bits);
734 	if (ev) {
735 		set_event_from_interrupt(p, ev);
736 	} else if (p->signal_page) {
737 		/*
738 		 * Partial ID lookup failed. Assume that the event ID
739 		 * in the interrupt payload was invalid and do an
740 		 * exhaustive search of signaled events.
741 		 */
742 		uint64_t *slots = page_slots(p->signal_page);
743 		uint32_t id;
744 
745 		if (valid_id_bits)
746 			pr_debug_ratelimited("Partial ID invalid: %u (%u valid bits)\n",
747 					     partial_id, valid_id_bits);
748 
749 		if (p->signal_event_count < KFD_SIGNAL_EVENT_LIMIT / 64) {
750 			/* With relatively few events, it's faster to
751 			 * iterate over the event IDR
752 			 */
753 			idr_for_each_entry(&p->event_idr, ev, id) {
754 				if (id >= KFD_SIGNAL_EVENT_LIMIT)
755 					break;
756 
757 				if (READ_ONCE(slots[id]) != UNSIGNALED_EVENT_SLOT)
758 					set_event_from_interrupt(p, ev);
759 			}
760 		} else {
761 			/* With relatively many events, it's faster to
762 			 * iterate over the signal slots and lookup
763 			 * only signaled events from the IDR.
764 			 */
765 			for (id = 1; id < KFD_SIGNAL_EVENT_LIMIT; id++)
766 				if (READ_ONCE(slots[id]) != UNSIGNALED_EVENT_SLOT) {
767 					ev = lookup_event_by_id(p, id);
768 					set_event_from_interrupt(p, ev);
769 				}
770 		}
771 	}
772 
773 	rcu_read_unlock();
774 	kfd_unref_process(p);
775 }
776 
777 static struct kfd_event_waiter *alloc_event_waiters(uint32_t num_events)
778 {
779 	struct kfd_event_waiter *event_waiters;
780 	uint32_t i;
781 
782 	event_waiters = kmalloc_array(num_events,
783 					sizeof(struct kfd_event_waiter),
784 					GFP_KERNEL);
785 	if (!event_waiters)
786 		return NULL;
787 
788 	for (i = 0; (event_waiters) && (i < num_events) ; i++) {
789 		init_wait(&event_waiters[i].wait);
790 		event_waiters[i].activated = false;
791 	}
792 
793 	return event_waiters;
794 }
795 
796 static int init_event_waiter(struct kfd_process *p,
797 		struct kfd_event_waiter *waiter,
798 		uint32_t event_id)
799 {
800 	struct kfd_event *ev = lookup_event_by_id(p, event_id);
801 
802 	if (!ev)
803 		return -EINVAL;
804 
805 	spin_lock(&ev->lock);
806 	waiter->event = ev;
807 	waiter->activated = ev->signaled;
808 	ev->signaled = ev->signaled && !ev->auto_reset;
809 	if (!waiter->activated)
810 		add_wait_queue(&ev->wq, &waiter->wait);
811 	spin_unlock(&ev->lock);
812 
813 	return 0;
814 }
815 
816 /* test_event_condition - Test condition of events being waited for
817  * @all:           Return completion only if all events have signaled
818  * @num_events:    Number of events to wait for
819  * @event_waiters: Array of event waiters, one per event
820  *
821  * Returns KFD_IOC_WAIT_RESULT_COMPLETE if all (or one) event(s) have
822  * signaled. Returns KFD_IOC_WAIT_RESULT_TIMEOUT if no (or not all)
823  * events have signaled. Returns KFD_IOC_WAIT_RESULT_FAIL if any of
824  * the events have been destroyed.
825  */
826 static uint32_t test_event_condition(bool all, uint32_t num_events,
827 				struct kfd_event_waiter *event_waiters)
828 {
829 	uint32_t i;
830 	uint32_t activated_count = 0;
831 
832 	for (i = 0; i < num_events; i++) {
833 		if (!READ_ONCE(event_waiters[i].event))
834 			return KFD_IOC_WAIT_RESULT_FAIL;
835 
836 		if (READ_ONCE(event_waiters[i].activated)) {
837 			if (!all)
838 				return KFD_IOC_WAIT_RESULT_COMPLETE;
839 
840 			activated_count++;
841 		}
842 	}
843 
844 	return activated_count == num_events ?
845 		KFD_IOC_WAIT_RESULT_COMPLETE : KFD_IOC_WAIT_RESULT_TIMEOUT;
846 }
847 
848 /*
849  * Copy event specific data, if defined.
850  * Currently only memory exception events have additional data to copy to user
851  */
852 static int copy_signaled_event_data(uint32_t num_events,
853 		struct kfd_event_waiter *event_waiters,
854 		struct kfd_event_data __user *data)
855 {
856 	struct kfd_hsa_memory_exception_data *src;
857 	struct kfd_hsa_memory_exception_data __user *dst;
858 	struct kfd_event_waiter *waiter;
859 	struct kfd_event *event;
860 	uint32_t i;
861 
862 	for (i = 0; i < num_events; i++) {
863 		waiter = &event_waiters[i];
864 		event = waiter->event;
865 		if (!event)
866 			return -EINVAL; /* event was destroyed */
867 		if (waiter->activated && event->type == KFD_EVENT_TYPE_MEMORY) {
868 			dst = &data[i].memory_exception_data;
869 			src = &event->memory_exception_data;
870 			if (copy_to_user(dst, src,
871 				sizeof(struct kfd_hsa_memory_exception_data)))
872 				return -EFAULT;
873 		}
874 	}
875 
876 	return 0;
877 }
878 
879 static long user_timeout_to_jiffies(uint32_t user_timeout_ms)
880 {
881 	if (user_timeout_ms == KFD_EVENT_TIMEOUT_IMMEDIATE)
882 		return 0;
883 
884 	if (user_timeout_ms == KFD_EVENT_TIMEOUT_INFINITE)
885 		return MAX_SCHEDULE_TIMEOUT;
886 
887 	/*
888 	 * msecs_to_jiffies interprets all values above 2^31-1 as infinite,
889 	 * but we consider them finite.
890 	 * This hack is wrong, but nobody is likely to notice.
891 	 */
892 	user_timeout_ms = min_t(uint32_t, user_timeout_ms, 0x7FFFFFFF);
893 
894 	return msecs_to_jiffies(user_timeout_ms) + 1;
895 }
896 
897 static void free_waiters(uint32_t num_events, struct kfd_event_waiter *waiters)
898 {
899 	uint32_t i;
900 
901 	for (i = 0; i < num_events; i++)
902 		if (waiters[i].event) {
903 			spin_lock(&waiters[i].event->lock);
904 			remove_wait_queue(&waiters[i].event->wq,
905 					  &waiters[i].wait);
906 			spin_unlock(&waiters[i].event->lock);
907 		}
908 
909 	kfree(waiters);
910 }
911 
912 int kfd_wait_on_events(struct kfd_process *p,
913 		       uint32_t num_events, void __user *data,
914 		       bool all, uint32_t user_timeout_ms,
915 		       uint32_t *wait_result)
916 {
917 	struct kfd_event_data __user *events =
918 			(struct kfd_event_data __user *) data;
919 	uint32_t i;
920 	int ret = 0;
921 
922 	struct kfd_event_waiter *event_waiters = NULL;
923 	long timeout = user_timeout_to_jiffies(user_timeout_ms);
924 
925 	event_waiters = alloc_event_waiters(num_events);
926 	if (!event_waiters) {
927 		ret = -ENOMEM;
928 		goto out;
929 	}
930 
931 	/* Use p->event_mutex here to protect against concurrent creation and
932 	 * destruction of events while we initialize event_waiters.
933 	 */
934 	mutex_lock(&p->event_mutex);
935 
936 	for (i = 0; i < num_events; i++) {
937 		struct kfd_event_data event_data;
938 
939 		if (copy_from_user(&event_data, &events[i],
940 				sizeof(struct kfd_event_data))) {
941 			ret = -EFAULT;
942 			goto out_unlock;
943 		}
944 
945 		ret = init_event_waiter(p, &event_waiters[i],
946 					event_data.event_id);
947 		if (ret)
948 			goto out_unlock;
949 	}
950 
951 	/* Check condition once. */
952 	*wait_result = test_event_condition(all, num_events, event_waiters);
953 	if (*wait_result == KFD_IOC_WAIT_RESULT_COMPLETE) {
954 		ret = copy_signaled_event_data(num_events,
955 					       event_waiters, events);
956 		goto out_unlock;
957 	} else if (WARN_ON(*wait_result == KFD_IOC_WAIT_RESULT_FAIL)) {
958 		/* This should not happen. Events shouldn't be
959 		 * destroyed while we're holding the event_mutex
960 		 */
961 		goto out_unlock;
962 	}
963 
964 	mutex_unlock(&p->event_mutex);
965 
966 	while (true) {
967 		if (fatal_signal_pending(current)) {
968 			ret = -EINTR;
969 			break;
970 		}
971 
972 		if (signal_pending(current)) {
973 			/*
974 			 * This is wrong when a nonzero, non-infinite timeout
975 			 * is specified. We need to use
976 			 * ERESTARTSYS_RESTARTBLOCK, but struct restart_block
977 			 * contains a union with data for each user and it's
978 			 * in generic kernel code that I don't want to
979 			 * touch yet.
980 			 */
981 			ret = -ERESTARTSYS;
982 			break;
983 		}
984 
985 		/* Set task state to interruptible sleep before
986 		 * checking wake-up conditions. A concurrent wake-up
987 		 * will put the task back into runnable state. In that
988 		 * case schedule_timeout will not put the task to
989 		 * sleep and we'll get a chance to re-check the
990 		 * updated conditions almost immediately. Otherwise,
991 		 * this race condition would lead to a soft hang or a
992 		 * very long sleep.
993 		 */
994 		set_current_state(TASK_INTERRUPTIBLE);
995 
996 		*wait_result = test_event_condition(all, num_events,
997 						    event_waiters);
998 		if (*wait_result != KFD_IOC_WAIT_RESULT_TIMEOUT)
999 			break;
1000 
1001 		if (timeout <= 0)
1002 			break;
1003 
1004 		timeout = schedule_timeout(timeout);
1005 	}
1006 	__set_current_state(TASK_RUNNING);
1007 
1008 	mutex_lock(&p->event_mutex);
1009 	/* copy_signaled_event_data may sleep. So this has to happen
1010 	 * after the task state is set back to RUNNING.
1011 	 *
1012 	 * The event may also have been destroyed after signaling. So
1013 	 * copy_signaled_event_data also must confirm that the event
1014 	 * still exists. Therefore this must be under the p->event_mutex
1015 	 * which is also held when events are destroyed.
1016 	 */
1017 	if (!ret && *wait_result == KFD_IOC_WAIT_RESULT_COMPLETE)
1018 		ret = copy_signaled_event_data(num_events,
1019 					       event_waiters, events);
1020 
1021 out_unlock:
1022 	free_waiters(num_events, event_waiters);
1023 	mutex_unlock(&p->event_mutex);
1024 out:
1025 	if (ret)
1026 		*wait_result = KFD_IOC_WAIT_RESULT_FAIL;
1027 	else if (*wait_result == KFD_IOC_WAIT_RESULT_FAIL)
1028 		ret = -EIO;
1029 
1030 	return ret;
1031 }
1032 
1033 int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)
1034 {
1035 	unsigned long pfn;
1036 	struct kfd_signal_page *page;
1037 	int ret;
1038 
1039 	/* check required size doesn't exceed the allocated size */
1040 	if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) <
1041 			get_order(vma->vm_end - vma->vm_start)) {
1042 		pr_err("Event page mmap requested illegal size\n");
1043 		return -EINVAL;
1044 	}
1045 
1046 	page = p->signal_page;
1047 	if (!page) {
1048 		/* Probably KFD bug, but mmap is user-accessible. */
1049 		pr_debug("Signal page could not be found\n");
1050 		return -EINVAL;
1051 	}
1052 
1053 	pfn = __pa(page->kernel_address);
1054 	pfn >>= PAGE_SHIFT;
1055 
1056 	vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE
1057 		       | VM_DONTDUMP | VM_PFNMAP;
1058 
1059 	pr_debug("Mapping signal page\n");
1060 	pr_debug("     start user address  == 0x%08lx\n", vma->vm_start);
1061 	pr_debug("     end user address    == 0x%08lx\n", vma->vm_end);
1062 	pr_debug("     pfn                 == 0x%016lX\n", pfn);
1063 	pr_debug("     vm_flags            == 0x%08lX\n", vma->vm_flags);
1064 	pr_debug("     size                == 0x%08lX\n",
1065 			vma->vm_end - vma->vm_start);
1066 
1067 	page->user_address = (uint64_t __user *)vma->vm_start;
1068 
1069 	/* mapping the page to user process */
1070 	ret = remap_pfn_range(vma, vma->vm_start, pfn,
1071 			vma->vm_end - vma->vm_start, vma->vm_page_prot);
1072 	if (!ret)
1073 		p->signal_mapped_size = vma->vm_end - vma->vm_start;
1074 
1075 	return ret;
1076 }
1077 
1078 /*
1079  * Assumes that p is not going away.
1080  */
1081 static void lookup_events_by_type_and_signal(struct kfd_process *p,
1082 		int type, void *event_data)
1083 {
1084 	struct kfd_hsa_memory_exception_data *ev_data;
1085 	struct kfd_event *ev;
1086 	uint32_t id;
1087 	bool send_signal = true;
1088 
1089 	ev_data = (struct kfd_hsa_memory_exception_data *) event_data;
1090 
1091 	rcu_read_lock();
1092 
1093 	id = KFD_FIRST_NONSIGNAL_EVENT_ID;
1094 	idr_for_each_entry_continue(&p->event_idr, ev, id)
1095 		if (ev->type == type) {
1096 			send_signal = false;
1097 			dev_dbg(kfd_device,
1098 					"Event found: id %X type %d",
1099 					ev->event_id, ev->type);
1100 			spin_lock(&ev->lock);
1101 			set_event(ev);
1102 			if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data)
1103 				ev->memory_exception_data = *ev_data;
1104 			spin_unlock(&ev->lock);
1105 		}
1106 
1107 	if (type == KFD_EVENT_TYPE_MEMORY) {
1108 		dev_warn(kfd_device,
1109 			"Sending SIGSEGV to process %d (pasid 0x%x)",
1110 				p->lead_thread->pid, p->pasid);
1111 		send_sig(SIGSEGV, p->lead_thread, 0);
1112 	}
1113 
1114 	/* Send SIGTERM no event of type "type" has been found*/
1115 	if (send_signal) {
1116 		if (send_sigterm) {
1117 			dev_warn(kfd_device,
1118 				"Sending SIGTERM to process %d (pasid 0x%x)",
1119 					p->lead_thread->pid, p->pasid);
1120 			send_sig(SIGTERM, p->lead_thread, 0);
1121 		} else {
1122 			dev_err(kfd_device,
1123 				"Process %d (pasid 0x%x) got unhandled exception",
1124 				p->lead_thread->pid, p->pasid);
1125 		}
1126 	}
1127 
1128 	rcu_read_unlock();
1129 }
1130 
1131 #ifdef KFD_SUPPORT_IOMMU_V2
1132 void kfd_signal_iommu_event(struct kfd_dev *dev, u32 pasid,
1133 		unsigned long address, bool is_write_requested,
1134 		bool is_execute_requested)
1135 {
1136 	struct kfd_hsa_memory_exception_data memory_exception_data;
1137 	struct vm_area_struct *vma;
1138 	int user_gpu_id;
1139 
1140 	/*
1141 	 * Because we are called from arbitrary context (workqueue) as opposed
1142 	 * to process context, kfd_process could attempt to exit while we are
1143 	 * running so the lookup function increments the process ref count.
1144 	 */
1145 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1146 	struct mm_struct *mm;
1147 
1148 	if (!p)
1149 		return; /* Presumably process exited. */
1150 
1151 	/* Take a safe reference to the mm_struct, which may otherwise
1152 	 * disappear even while the kfd_process is still referenced.
1153 	 */
1154 	mm = get_task_mm(p->lead_thread);
1155 	if (!mm) {
1156 		kfd_unref_process(p);
1157 		return; /* Process is exiting */
1158 	}
1159 
1160 	user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id);
1161 	if (unlikely(user_gpu_id == -EINVAL)) {
1162 		WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id);
1163 		return;
1164 	}
1165 	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
1166 
1167 	mmap_read_lock(mm);
1168 	vma = find_vma(mm, address);
1169 
1170 	memory_exception_data.gpu_id = user_gpu_id;
1171 	memory_exception_data.va = address;
1172 	/* Set failure reason */
1173 	memory_exception_data.failure.NotPresent = 1;
1174 	memory_exception_data.failure.NoExecute = 0;
1175 	memory_exception_data.failure.ReadOnly = 0;
1176 	if (vma && address >= vma->vm_start) {
1177 		memory_exception_data.failure.NotPresent = 0;
1178 
1179 		if (is_write_requested && !(vma->vm_flags & VM_WRITE))
1180 			memory_exception_data.failure.ReadOnly = 1;
1181 		else
1182 			memory_exception_data.failure.ReadOnly = 0;
1183 
1184 		if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
1185 			memory_exception_data.failure.NoExecute = 1;
1186 		else
1187 			memory_exception_data.failure.NoExecute = 0;
1188 	}
1189 
1190 	mmap_read_unlock(mm);
1191 	mmput(mm);
1192 
1193 	pr_debug("notpresent %d, noexecute %d, readonly %d\n",
1194 			memory_exception_data.failure.NotPresent,
1195 			memory_exception_data.failure.NoExecute,
1196 			memory_exception_data.failure.ReadOnly);
1197 
1198 	/* Workaround on Raven to not kill the process when memory is freed
1199 	 * before IOMMU is able to finish processing all the excessive PPRs
1200 	 */
1201 
1202 	if (KFD_GC_VERSION(dev) != IP_VERSION(9, 1, 0) &&
1203 	    KFD_GC_VERSION(dev) != IP_VERSION(9, 2, 2) &&
1204 	    KFD_GC_VERSION(dev) != IP_VERSION(9, 3, 0))
1205 		lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,
1206 				&memory_exception_data);
1207 
1208 	kfd_unref_process(p);
1209 }
1210 #endif /* KFD_SUPPORT_IOMMU_V2 */
1211 
1212 void kfd_signal_hw_exception_event(u32 pasid)
1213 {
1214 	/*
1215 	 * Because we are called from arbitrary context (workqueue) as opposed
1216 	 * to process context, kfd_process could attempt to exit while we are
1217 	 * running so the lookup function increments the process ref count.
1218 	 */
1219 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1220 
1221 	if (!p)
1222 		return; /* Presumably process exited. */
1223 
1224 	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL);
1225 	kfd_unref_process(p);
1226 }
1227 
1228 void kfd_signal_vm_fault_event(struct kfd_dev *dev, u32 pasid,
1229 				struct kfd_vm_fault_info *info)
1230 {
1231 	struct kfd_event *ev;
1232 	uint32_t id;
1233 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1234 	struct kfd_hsa_memory_exception_data memory_exception_data;
1235 	int user_gpu_id;
1236 
1237 	if (!p)
1238 		return; /* Presumably process exited. */
1239 
1240 	user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id);
1241 	if (unlikely(user_gpu_id == -EINVAL)) {
1242 		WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id);
1243 		return;
1244 	}
1245 
1246 	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
1247 	memory_exception_data.gpu_id = user_gpu_id;
1248 	memory_exception_data.failure.imprecise = true;
1249 	/* Set failure reason */
1250 	if (info) {
1251 		memory_exception_data.va = (info->page_addr) << PAGE_SHIFT;
1252 		memory_exception_data.failure.NotPresent =
1253 			info->prot_valid ? 1 : 0;
1254 		memory_exception_data.failure.NoExecute =
1255 			info->prot_exec ? 1 : 0;
1256 		memory_exception_data.failure.ReadOnly =
1257 			info->prot_write ? 1 : 0;
1258 		memory_exception_data.failure.imprecise = 0;
1259 	}
1260 
1261 	rcu_read_lock();
1262 
1263 	id = KFD_FIRST_NONSIGNAL_EVENT_ID;
1264 	idr_for_each_entry_continue(&p->event_idr, ev, id)
1265 		if (ev->type == KFD_EVENT_TYPE_MEMORY) {
1266 			spin_lock(&ev->lock);
1267 			ev->memory_exception_data = memory_exception_data;
1268 			set_event(ev);
1269 			spin_unlock(&ev->lock);
1270 		}
1271 
1272 	rcu_read_unlock();
1273 	kfd_unref_process(p);
1274 }
1275 
1276 void kfd_signal_reset_event(struct kfd_dev *dev)
1277 {
1278 	struct kfd_hsa_hw_exception_data hw_exception_data;
1279 	struct kfd_hsa_memory_exception_data memory_exception_data;
1280 	struct kfd_process *p;
1281 	struct kfd_event *ev;
1282 	unsigned int temp;
1283 	uint32_t id, idx;
1284 	int reset_cause = atomic_read(&dev->sram_ecc_flag) ?
1285 			KFD_HW_EXCEPTION_ECC :
1286 			KFD_HW_EXCEPTION_GPU_HANG;
1287 
1288 	/* Whole gpu reset caused by GPU hang and memory is lost */
1289 	memset(&hw_exception_data, 0, sizeof(hw_exception_data));
1290 	hw_exception_data.memory_lost = 1;
1291 	hw_exception_data.reset_cause = reset_cause;
1292 
1293 	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
1294 	memory_exception_data.ErrorType = KFD_MEM_ERR_SRAM_ECC;
1295 	memory_exception_data.failure.imprecise = true;
1296 
1297 	idx = srcu_read_lock(&kfd_processes_srcu);
1298 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1299 		int user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id);
1300 
1301 		if (unlikely(user_gpu_id == -EINVAL)) {
1302 			WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id);
1303 			continue;
1304 		}
1305 
1306 		rcu_read_lock();
1307 
1308 		id = KFD_FIRST_NONSIGNAL_EVENT_ID;
1309 		idr_for_each_entry_continue(&p->event_idr, ev, id) {
1310 			if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
1311 				spin_lock(&ev->lock);
1312 				ev->hw_exception_data = hw_exception_data;
1313 				ev->hw_exception_data.gpu_id = user_gpu_id;
1314 				set_event(ev);
1315 				spin_unlock(&ev->lock);
1316 			}
1317 			if (ev->type == KFD_EVENT_TYPE_MEMORY &&
1318 			    reset_cause == KFD_HW_EXCEPTION_ECC) {
1319 				spin_lock(&ev->lock);
1320 				ev->memory_exception_data = memory_exception_data;
1321 				ev->memory_exception_data.gpu_id = user_gpu_id;
1322 				set_event(ev);
1323 				spin_unlock(&ev->lock);
1324 			}
1325 		}
1326 
1327 		rcu_read_unlock();
1328 	}
1329 	srcu_read_unlock(&kfd_processes_srcu, idx);
1330 }
1331 
1332 void kfd_signal_poison_consumed_event(struct kfd_dev *dev, u32 pasid)
1333 {
1334 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1335 	struct kfd_hsa_memory_exception_data memory_exception_data;
1336 	struct kfd_hsa_hw_exception_data hw_exception_data;
1337 	struct kfd_event *ev;
1338 	uint32_t id = KFD_FIRST_NONSIGNAL_EVENT_ID;
1339 	int user_gpu_id;
1340 
1341 	if (!p)
1342 		return; /* Presumably process exited. */
1343 
1344 	user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id);
1345 	if (unlikely(user_gpu_id == -EINVAL)) {
1346 		WARN_ONCE(1, "Could not get user_gpu_id from dev->id:%x\n", dev->id);
1347 		return;
1348 	}
1349 
1350 	memset(&hw_exception_data, 0, sizeof(hw_exception_data));
1351 	hw_exception_data.gpu_id = user_gpu_id;
1352 	hw_exception_data.memory_lost = 1;
1353 	hw_exception_data.reset_cause = KFD_HW_EXCEPTION_ECC;
1354 
1355 	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
1356 	memory_exception_data.ErrorType = KFD_MEM_ERR_POISON_CONSUMED;
1357 	memory_exception_data.gpu_id = user_gpu_id;
1358 	memory_exception_data.failure.imprecise = true;
1359 
1360 	rcu_read_lock();
1361 
1362 	idr_for_each_entry_continue(&p->event_idr, ev, id) {
1363 		if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
1364 			spin_lock(&ev->lock);
1365 			ev->hw_exception_data = hw_exception_data;
1366 			set_event(ev);
1367 			spin_unlock(&ev->lock);
1368 		}
1369 
1370 		if (ev->type == KFD_EVENT_TYPE_MEMORY) {
1371 			spin_lock(&ev->lock);
1372 			ev->memory_exception_data = memory_exception_data;
1373 			set_event(ev);
1374 			spin_unlock(&ev->lock);
1375 		}
1376 	}
1377 
1378 	rcu_read_unlock();
1379 
1380 	/* user application will handle SIGBUS signal */
1381 	send_sig(SIGBUS, p->lead_thread, 0);
1382 
1383 	kfd_unref_process(p);
1384 }
1385