Lines Matching refs:ev
95 struct kfd_event *ev, in allocate_event_notification_slot() argument
109 id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1, in allocate_event_notification_slot()
118 id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8, in allocate_event_notification_slot()
124 ev->event_id = id; in allocate_event_notification_slot()
159 struct kfd_event *ev; in lookup_signaled_event_by_partial_id() local
177 for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) { in lookup_signaled_event_by_partial_id()
181 ev = idr_find(&p->event_idr, id); in lookup_signaled_event_by_partial_id()
184 return ev; in lookup_signaled_event_by_partial_id()
188 struct kfd_event *ev, const int *restore_id) in create_signal_event() argument
201 ret = allocate_event_notification_slot(p, ev, restore_id); in create_signal_event()
209 ev->user_signal_address = &p->signal_page->user_address[ev->event_id]; in create_signal_event()
211 p->signal_event_count, ev->event_id, in create_signal_event()
212 ev->user_signal_address); in create_signal_event()
217 static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id) in create_other_event() argument
222 id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1, in create_other_event()
230 id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID, in create_other_event()
236 ev->event_id = id; in create_other_event()
261 static void destroy_event(struct kfd_process *p, struct kfd_event *ev) in destroy_event() argument
266 spin_lock(&ev->lock); in destroy_event()
267 list_for_each_entry(waiter, &ev->wq.head, wait.entry) in destroy_event()
269 wake_up_all(&ev->wq); in destroy_event()
270 spin_unlock(&ev->lock); in destroy_event()
272 if (ev->type == KFD_EVENT_TYPE_SIGNAL || in destroy_event()
273 ev->type == KFD_EVENT_TYPE_DEBUG) in destroy_event()
276 idr_remove(&p->event_idr, ev->event_id); in destroy_event()
277 kfree_rcu(ev, rcu); in destroy_event()
282 struct kfd_event *ev; in destroy_events() local
285 idr_for_each_entry(&p->event_idr, ev, id) in destroy_events()
286 if (ev) in destroy_events()
287 destroy_event(p, ev); in destroy_events()
314 static bool event_can_be_gpu_signaled(const struct kfd_event *ev) in event_can_be_gpu_signaled() argument
316 return ev->type == KFD_EVENT_TYPE_SIGNAL || in event_can_be_gpu_signaled()
317 ev->type == KFD_EVENT_TYPE_DEBUG; in event_can_be_gpu_signaled()
320 static bool event_can_be_cpu_signaled(const struct kfd_event *ev) in event_can_be_cpu_signaled() argument
322 return ev->type == KFD_EVENT_TYPE_SIGNAL; in event_can_be_cpu_signaled()
401 struct kfd_event *ev = kzalloc(sizeof(*ev), GFP_KERNEL); in kfd_event_create() local
403 if (!ev) in kfd_event_create()
406 ev->type = event_type; in kfd_event_create()
407 ev->auto_reset = auto_reset; in kfd_event_create()
408 ev->signaled = false; in kfd_event_create()
410 spin_lock_init(&ev->lock); in kfd_event_create()
411 init_waitqueue_head(&ev->wq); in kfd_event_create()
420 ret = create_signal_event(devkfd, p, ev, NULL); in kfd_event_create()
423 *event_slot_index = ev->event_id; in kfd_event_create()
427 ret = create_other_event(p, ev, NULL); in kfd_event_create()
432 *event_id = ev->event_id; in kfd_event_create()
433 *event_trigger_data = ev->event_id; in kfd_event_create()
434 ev->event_age = 1; in kfd_event_create()
436 kfree(ev); in kfd_event_create()
451 struct kfd_event *ev = NULL; in kfd_criu_restore_event() local
458 ev = kzalloc(sizeof(*ev), GFP_KERNEL); in kfd_criu_restore_event()
459 if (!ev) { in kfd_criu_restore_event()
482 ev->type = ev_priv->type; in kfd_criu_restore_event()
483 ev->auto_reset = ev_priv->auto_reset; in kfd_criu_restore_event()
484 ev->signaled = ev_priv->signaled; in kfd_criu_restore_event()
486 spin_lock_init(&ev->lock); in kfd_criu_restore_event()
487 init_waitqueue_head(&ev->wq); in kfd_criu_restore_event()
490 switch (ev->type) { in kfd_criu_restore_event()
493 ret = create_signal_event(devkfd, p, ev, &ev_priv->event_id); in kfd_criu_restore_event()
496 memcpy(&ev->memory_exception_data, in kfd_criu_restore_event()
500 ret = create_other_event(p, ev, &ev_priv->event_id); in kfd_criu_restore_event()
503 memcpy(&ev->hw_exception_data, in kfd_criu_restore_event()
507 ret = create_other_event(p, ev, &ev_priv->event_id); in kfd_criu_restore_event()
514 kfree(ev); in kfd_criu_restore_event()
528 struct kfd_event *ev; in kfd_criu_checkpoint_events() local
541 idr_for_each_entry(&p->event_idr, ev, ev_id) { in kfd_criu_checkpoint_events()
556 ev_priv->event_id = ev->event_id; in kfd_criu_checkpoint_events()
557 ev_priv->auto_reset = ev->auto_reset; in kfd_criu_checkpoint_events()
558 ev_priv->type = ev->type; in kfd_criu_checkpoint_events()
559 ev_priv->signaled = ev->signaled; in kfd_criu_checkpoint_events()
563 &ev->memory_exception_data, in kfd_criu_checkpoint_events()
567 &ev->hw_exception_data, in kfd_criu_checkpoint_events()
594 struct kfd_event *ev; in kfd_get_num_events() local
598 idr_for_each_entry(&p->event_idr, ev, id) in kfd_get_num_events()
607 struct kfd_event *ev; in kfd_event_destroy() local
612 ev = lookup_event_by_id(p, event_id); in kfd_event_destroy()
614 if (ev) in kfd_event_destroy()
615 destroy_event(p, ev); in kfd_event_destroy()
623 static void set_event(struct kfd_event *ev) in set_event() argument
632 ev->signaled = !ev->auto_reset || !waitqueue_active(&ev->wq); in set_event()
633 if (!(++ev->event_age)) { in set_event()
635 ev->event_age = 2; in set_event()
639 list_for_each_entry(waiter, &ev->wq.head, wait.entry) in set_event()
642 wake_up_all(&ev->wq); in set_event()
649 struct kfd_event *ev; in kfd_set_event() local
653 ev = lookup_event_by_id(p, event_id); in kfd_set_event()
654 if (!ev) { in kfd_set_event()
658 spin_lock(&ev->lock); in kfd_set_event()
660 if (event_can_be_cpu_signaled(ev)) in kfd_set_event()
661 set_event(ev); in kfd_set_event()
665 spin_unlock(&ev->lock); in kfd_set_event()
671 static void reset_event(struct kfd_event *ev) in reset_event() argument
673 ev->signaled = false; in reset_event()
680 struct kfd_event *ev; in kfd_reset_event() local
684 ev = lookup_event_by_id(p, event_id); in kfd_reset_event()
685 if (!ev) { in kfd_reset_event()
689 spin_lock(&ev->lock); in kfd_reset_event()
691 if (event_can_be_cpu_signaled(ev)) in kfd_reset_event()
692 reset_event(ev); in kfd_reset_event()
696 spin_unlock(&ev->lock); in kfd_reset_event()
703 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev) in acknowledge_signal() argument
705 WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT); in acknowledge_signal()
709 struct kfd_event *ev) in set_event_from_interrupt() argument
711 if (ev && event_can_be_gpu_signaled(ev)) { in set_event_from_interrupt()
712 acknowledge_signal(p, ev); in set_event_from_interrupt()
713 spin_lock(&ev->lock); in set_event_from_interrupt()
714 set_event(ev); in set_event_from_interrupt()
715 spin_unlock(&ev->lock); in set_event_from_interrupt()
722 struct kfd_event *ev = NULL; in kfd_signal_event_interrupt() local
737 ev = lookup_signaled_event_by_partial_id(p, partial_id, in kfd_signal_event_interrupt()
739 if (ev) { in kfd_signal_event_interrupt()
740 set_event_from_interrupt(p, ev); in kfd_signal_event_interrupt()
758 idr_for_each_entry(&p->event_idr, ev, id) { in kfd_signal_event_interrupt()
763 set_event_from_interrupt(p, ev); in kfd_signal_event_interrupt()
772 ev = lookup_event_by_id(p, id); in kfd_signal_event_interrupt()
773 set_event_from_interrupt(p, ev); in kfd_signal_event_interrupt()
802 struct kfd_event *ev = lookup_event_by_id(p, event_data->event_id); in init_event_waiter() local
804 if (!ev) in init_event_waiter()
807 spin_lock(&ev->lock); in init_event_waiter()
808 waiter->event = ev; in init_event_waiter()
809 waiter->activated = ev->signaled; in init_event_waiter()
810 ev->signaled = ev->signaled && !ev->auto_reset; in init_event_waiter()
816 if (ev->event_age != event_data->signal_event_data.last_event_age) in init_event_waiter()
821 add_wait_queue(&ev->wq, &waiter->wait); in init_event_waiter()
822 spin_unlock(&ev->lock); in init_event_waiter()
1102 struct kfd_event *ev; in lookup_events_by_type_and_signal() local
1111 idr_for_each_entry_continue(&p->event_idr, ev, id) in lookup_events_by_type_and_signal()
1112 if (ev->type == type) { in lookup_events_by_type_and_signal()
1116 ev->event_id, ev->type); in lookup_events_by_type_and_signal()
1117 spin_lock(&ev->lock); in lookup_events_by_type_and_signal()
1118 set_event(ev); in lookup_events_by_type_and_signal()
1119 if (ev->type == KFD_EVENT_TYPE_MEMORY && ev_data) in lookup_events_by_type_and_signal()
1120 ev->memory_exception_data = *ev_data; in lookup_events_by_type_and_signal()
1121 spin_unlock(&ev->lock); in lookup_events_by_type_and_signal()
1168 struct kfd_event *ev; in kfd_signal_vm_fault_event() local
1206 idr_for_each_entry_continue(&p->event_idr, ev, id) in kfd_signal_vm_fault_event()
1207 if (ev->type == KFD_EVENT_TYPE_MEMORY) { in kfd_signal_vm_fault_event()
1208 spin_lock(&ev->lock); in kfd_signal_vm_fault_event()
1209 ev->memory_exception_data = data ? *data : in kfd_signal_vm_fault_event()
1211 set_event(ev); in kfd_signal_vm_fault_event()
1212 spin_unlock(&ev->lock); in kfd_signal_vm_fault_event()
1224 struct kfd_event *ev; in kfd_signal_reset_event() local
1252 idr_for_each_entry_continue(&p->event_idr, ev, id) { in kfd_signal_reset_event()
1253 if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) { in kfd_signal_reset_event()
1254 spin_lock(&ev->lock); in kfd_signal_reset_event()
1255 ev->hw_exception_data = hw_exception_data; in kfd_signal_reset_event()
1256 ev->hw_exception_data.gpu_id = user_gpu_id; in kfd_signal_reset_event()
1257 set_event(ev); in kfd_signal_reset_event()
1258 spin_unlock(&ev->lock); in kfd_signal_reset_event()
1260 if (ev->type == KFD_EVENT_TYPE_MEMORY && in kfd_signal_reset_event()
1262 spin_lock(&ev->lock); in kfd_signal_reset_event()
1263 ev->memory_exception_data = memory_exception_data; in kfd_signal_reset_event()
1264 ev->memory_exception_data.gpu_id = user_gpu_id; in kfd_signal_reset_event()
1265 set_event(ev); in kfd_signal_reset_event()
1266 spin_unlock(&ev->lock); in kfd_signal_reset_event()
1280 struct kfd_event *ev; in kfd_signal_poison_consumed_event() local
1305 idr_for_each_entry_continue(&p->event_idr, ev, id) { in kfd_signal_poison_consumed_event()
1306 if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) { in kfd_signal_poison_consumed_event()
1307 spin_lock(&ev->lock); in kfd_signal_poison_consumed_event()
1308 ev->hw_exception_data = hw_exception_data; in kfd_signal_poison_consumed_event()
1309 set_event(ev); in kfd_signal_poison_consumed_event()
1310 spin_unlock(&ev->lock); in kfd_signal_poison_consumed_event()
1313 if (ev->type == KFD_EVENT_TYPE_MEMORY) { in kfd_signal_poison_consumed_event()
1314 spin_lock(&ev->lock); in kfd_signal_poison_consumed_event()
1315 ev->memory_exception_data = memory_exception_data; in kfd_signal_poison_consumed_event()
1316 set_event(ev); in kfd_signal_poison_consumed_event()
1317 spin_unlock(&ev->lock); in kfd_signal_poison_consumed_event()