64 static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p)  in allocate_signal_page()  argument
85 			page, p);  in allocate_signal_page()
94 static int allocate_event_notification_slot(struct kfd_process *p,  in allocate_event_notification_slot()  argument
100 	if (!p->signal_page) {  in allocate_event_notification_slot()
101 		p->signal_page = allocate_signal_page(p);  in allocate_event_notification_slot()
102 		if (!p->signal_page)  in allocate_event_notification_slot()
105 		p->signal_mapped_size = 256*8;  in allocate_event_notification_slot()
109 		id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,  in allocate_event_notification_slot()
118 		id = idr_alloc(&p->event_idr, ev, 0, p->signal_mapped_size / 8,  in allocate_event_notification_slot()
125 	page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT;  in allocate_event_notification_slot()  local
134 static struct kfd_event *lookup_event_by_id(struct kfd_process *p, uint32_t id)  in lookup_event_by_id()  argument
136 	return idr_find(&p->event_idr, id);  in lookup_event_by_id()
157 	struct kfd_process *p, uint32_t id, uint32_t bits)  in lookup_signaled_event_by_partial_id()  argument
161 	if (!p->signal_page || id >= KFD_SIGNAL_EVENT_LIMIT)  in lookup_signaled_event_by_partial_id()
168 		if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)  in lookup_signaled_event_by_partial_id()  local
171 		return idr_find(&p->event_idr, id);  in lookup_signaled_event_by_partial_id()
178 		if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT)  in lookup_signaled_event_by_partial_id()  local
181 		ev = idr_find(&p->event_idr, id);  in lookup_signaled_event_by_partial_id()
187 static int create_signal_event(struct file *devkfd, struct kfd_process *p,  in create_signal_event()  argument
192 	if (p->signal_mapped_size &&  in create_signal_event()
193 	    p->signal_event_count == p->signal_mapped_size / 8) {  in create_signal_event()
194 		if (!p->signal_event_limit_reached) {  in create_signal_event()
196 			p->signal_event_limit_reached = true;  in create_signal_event()
201 	ret = allocate_event_notification_slot(p, ev, restore_id);  in create_signal_event()
207 	p->signal_event_count++;  in create_signal_event()
209 	ev->user_signal_address = &p->signal_page->user_address[ev->event_id];  in create_signal_event()
211 			p->signal_event_count, ev->event_id,  in create_signal_event()
217 static int create_other_event(struct kfd_process *p, struct kfd_event *ev, const int *restore_id)  in create_other_event()  argument
222 		id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1,  in create_other_event()
230 		id = idr_alloc(&p->event_idr, ev, KFD_FIRST_NONSIGNAL_EVENT_ID,  in create_other_event()
241 int kfd_event_init_process(struct kfd_process *p)  in kfd_event_init_process()  argument
245 	mutex_init(&p->event_mutex);  in kfd_event_init_process()
246 	idr_init(&p->event_idr);  in kfd_event_init_process()
247 	p->signal_page = NULL;  in kfd_event_init_process()
248 	p->signal_event_count = 1;  in kfd_event_init_process()
252 	id = idr_alloc(&p->event_idr, NULL, 0, 1, GFP_KERNEL);  in kfd_event_init_process()
254 		idr_destroy(&p->event_idr);  in kfd_event_init_process()
255 		mutex_destroy(&p->event_mutex);  in kfd_event_init_process()
261 static void destroy_event(struct kfd_process *p, struct kfd_event *ev)  in destroy_event()  argument
274 		p->signal_event_count--;  in destroy_event()
276 	idr_remove(&p->event_idr, ev->event_id);  in destroy_event()
280 static void destroy_events(struct kfd_process *p)  in destroy_events()  argument
285 	idr_for_each_entry(&p->event_idr, ev, id)  in destroy_events()
287 			destroy_event(p, ev);  in destroy_events()
288 	idr_destroy(&p->event_idr);  in destroy_events()
289 	mutex_destroy(&p->event_mutex);  in destroy_events()
296 static void shutdown_signal_page(struct kfd_process *p)  in shutdown_signal_page()  argument
298 	struct kfd_signal_page *page = p->signal_page;  in shutdown_signal_page()
308 void kfd_event_free_process(struct kfd_process *p)  in kfd_event_free_process()  argument
310 	destroy_events(p);  in kfd_event_free_process()
311 	shutdown_signal_page(p);  in kfd_event_free_process()
325 static int kfd_event_page_set(struct kfd_process *p, void *kernel_address,  in kfd_event_page_set()  argument
330 	if (p->signal_page)  in kfd_event_page_set()
343 	p->signal_page = page;  in kfd_event_page_set()
344 	p->signal_mapped_size = size;  in kfd_event_page_set()
345 	p->signal_handle = user_handle;  in kfd_event_page_set()
349 int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset)  in kfd_kmap_event_page()  argument
357 	if (p->signal_page) {  in kfd_kmap_event_page()
362 	pdd = kfd_process_device_data_by_id(p, GET_GPU_ID(event_page_offset));  in kfd_kmap_event_page()
369 	pdd = kfd_bind_process_to_device(kfd, p);  in kfd_kmap_event_page()
386 	err = kfd_event_page_set(p, kern_addr, size, event_page_offset);  in kfd_kmap_event_page()
395 int kfd_event_create(struct file *devkfd, struct kfd_process *p,  in kfd_event_create()  argument
415 	mutex_lock(&p->event_mutex);  in kfd_event_create()
420 		ret = create_signal_event(devkfd, p, ev, NULL);  in kfd_event_create()
427 		ret = create_other_event(p, ev, NULL);  in kfd_event_create()
439 	mutex_unlock(&p->event_mutex);  in kfd_event_create()
445 			   struct kfd_process *p,  in kfd_criu_restore_event()  argument
477 		ret = kfd_kmap_event_page(p, ev_priv->user_handle);  in kfd_criu_restore_event()
489 	mutex_lock(&p->event_mutex);  in kfd_criu_restore_event()
493 		ret = create_signal_event(devkfd, p, ev, &ev_priv->event_id);  in kfd_criu_restore_event()
500 		ret = create_other_event(p, ev, &ev_priv->event_id);  in kfd_criu_restore_event()
507 		ret = create_other_event(p, ev, &ev_priv->event_id);  in kfd_criu_restore_event()
510 	mutex_unlock(&p->event_mutex);  in kfd_criu_restore_event()
521 int kfd_criu_checkpoint_events(struct kfd_process *p,  in kfd_criu_checkpoint_events()  argument
531 	uint32_t num_events = kfd_get_num_events(p);  in kfd_criu_checkpoint_events()
541 	idr_for_each_entry(&p->event_idr, ev, ev_id) {  in kfd_criu_checkpoint_events()
553 		if (i == 0 && p->signal_page)  in kfd_criu_checkpoint_events()
554 			ev_priv->user_handle = p->signal_handle;  in kfd_criu_checkpoint_events()
592 int kfd_get_num_events(struct kfd_process *p)  in kfd_get_num_events()  argument
598 	idr_for_each_entry(&p->event_idr, ev, id)  in kfd_get_num_events()
605 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id)  in kfd_event_destroy()  argument
610 	mutex_lock(&p->event_mutex);  in kfd_event_destroy()
612 	ev = lookup_event_by_id(p, event_id);  in kfd_event_destroy()
615 		destroy_event(p, ev);  in kfd_event_destroy()
619 	mutex_unlock(&p->event_mutex);  in kfd_event_destroy()
646 int kfd_set_event(struct kfd_process *p, uint32_t event_id)  in kfd_set_event()  argument
653 	ev = lookup_event_by_id(p, event_id);  in kfd_set_event()
677 int kfd_reset_event(struct kfd_process *p, uint32_t event_id)  in kfd_reset_event()  argument
684 	ev = lookup_event_by_id(p, event_id);  in kfd_reset_event()
703 static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev)  in acknowledge_signal()  argument
705 	WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT);  in acknowledge_signal()
708 static void set_event_from_interrupt(struct kfd_process *p,  in set_event_from_interrupt()  argument
712 		acknowledge_signal(p, ev);  in set_event_from_interrupt()
729 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);  in kfd_signal_event_interrupt()  local
731 	if (!p)  in kfd_signal_event_interrupt()
737 		ev = lookup_signaled_event_by_partial_id(p, partial_id,  in kfd_signal_event_interrupt()
740 		set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
741 	} else if (p->signal_page) {  in kfd_signal_event_interrupt()
747 		uint64_t *slots = page_slots(p->signal_page);  in kfd_signal_event_interrupt()
754 		if (p->signal_event_count < KFD_SIGNAL_EVENT_LIMIT / 64) {  in kfd_signal_event_interrupt()
758 			idr_for_each_entry(&p->event_idr, ev, id) {  in kfd_signal_event_interrupt()
763 					set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
772 					ev = lookup_event_by_id(p, id);  in kfd_signal_event_interrupt()
773 					set_event_from_interrupt(p, ev);  in kfd_signal_event_interrupt()
779 	kfd_unref_process(p);  in kfd_signal_event_interrupt()
798 static int init_event_waiter(struct kfd_process *p,  in init_event_waiter()  argument
802 	struct kfd_event *ev = lookup_event_by_id(p, event_data->event_id);  in init_event_waiter()
934 int kfd_wait_on_events(struct kfd_process *p,  in kfd_wait_on_events()  argument
956 	mutex_lock(&p->event_mutex);  in kfd_wait_on_events()
967 		ret = init_event_waiter(p, &event_waiters[i], &event_data);  in kfd_wait_on_events()
985 	mutex_unlock(&p->event_mutex);  in kfd_wait_on_events()
1025 	mutex_lock(&p->event_mutex);  in kfd_wait_on_events()
1040 	mutex_unlock(&p->event_mutex);  in kfd_wait_on_events()
1050 int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma)  in kfd_event_mmap()  argument
1063 	page = p->signal_page;  in kfd_event_mmap()
1090 		p->signal_mapped_size = vma->vm_end - vma->vm_start;  in kfd_event_mmap()
1098 static void lookup_events_by_type_and_signal(struct kfd_process *p,  in lookup_events_by_type_and_signal()  argument
1111 	idr_for_each_entry_continue(&p->event_idr, ev, id)  in lookup_events_by_type_and_signal()
1127 				p->lead_thread->pid, p->pasid);  in lookup_events_by_type_and_signal()
1128 		send_sig(SIGSEGV, p->lead_thread, 0);  in lookup_events_by_type_and_signal()
1136 					p->lead_thread->pid, p->pasid);  in lookup_events_by_type_and_signal()
1137 			send_sig(SIGTERM, p->lead_thread, 0);  in lookup_events_by_type_and_signal()
1141 				p->lead_thread->pid, p->pasid);  in lookup_events_by_type_and_signal()
1155 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);  in kfd_signal_hw_exception_event()  local
1157 	if (!p)  in kfd_signal_hw_exception_event()
1160 	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_HW_EXCEPTION, NULL);  in kfd_signal_hw_exception_event()
1161 	kfd_unref_process(p);  in kfd_signal_hw_exception_event()
1170 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);  in kfd_signal_vm_fault_event()  local
1174 	if (!p)  in kfd_signal_vm_fault_event()
1177 	user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id);  in kfd_signal_vm_fault_event()
1206 	idr_for_each_entry_continue(&p->event_idr, ev, id)  in kfd_signal_vm_fault_event()
1216 	kfd_unref_process(p);  in kfd_signal_vm_fault_event()
1223 	struct kfd_process *p;  in kfd_signal_reset_event()  local
1241 	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {  in kfd_signal_reset_event()
1242 		int user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id);  in kfd_signal_reset_event()
1252 		idr_for_each_entry_continue(&p->event_idr, ev, id) {  in kfd_signal_reset_event()
1277 	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);  in kfd_signal_poison_consumed_event()  local
1284 	if (!p)  in kfd_signal_poison_consumed_event()
1287 	user_gpu_id = kfd_process_get_user_gpu_id(p, dev->id);  in kfd_signal_poison_consumed_event()
1305 	idr_for_each_entry_continue(&p->event_idr, ev, id) {  in kfd_signal_poison_consumed_event()
1324 	send_sig(SIGBUS, p->lead_thread, 0);  in kfd_signal_poison_consumed_event()
1326 	kfd_unref_process(p);  in kfd_signal_poison_consumed_event()