44 	struct kvm_kernel_irqfd *irqfd =  in irqfd_inject()  local
46 	struct kvm *kvm = irqfd->kvm;  in irqfd_inject()
48 	if (!irqfd->resampler) {  in irqfd_inject()
49 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1,  in irqfd_inject()
51 		kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0,  in irqfd_inject()
55 			    irqfd->gsi, 1, false);  in irqfd_inject()
60 	struct kvm_kernel_irqfd *irqfd;  in irqfd_resampler_notify()  local
62 	list_for_each_entry_srcu(irqfd, &resampler->list, resampler_link,  in irqfd_resampler_notify()
64 		eventfd_signal(irqfd->resamplefd, 1);  in irqfd_resampler_notify()
92 irqfd_resampler_shutdown(struct kvm_kernel_irqfd *irqfd)  in irqfd_resampler_shutdown()  argument
94 	struct kvm_kernel_irqfd_resampler *resampler = irqfd->resampler;  in irqfd_resampler_shutdown()
99 	list_del_rcu(&irqfd->resampler_link);  in irqfd_resampler_shutdown()
123 	struct kvm_kernel_irqfd *irqfd =  in irqfd_shutdown()  local
125 	struct kvm *kvm = irqfd->kvm;  in irqfd_shutdown()
135 	eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt);  in irqfd_shutdown()
141 	flush_work(&irqfd->inject);  in irqfd_shutdown()
143 	if (irqfd->resampler) {  in irqfd_shutdown()
144 		irqfd_resampler_shutdown(irqfd);  in irqfd_shutdown()
145 		eventfd_ctx_put(irqfd->resamplefd);  in irqfd_shutdown()
152 	irq_bypass_unregister_consumer(&irqfd->consumer);  in irqfd_shutdown()
154 	eventfd_ctx_put(irqfd->eventfd);  in irqfd_shutdown()
155 	kfree(irqfd);  in irqfd_shutdown()
161 irqfd_is_active(struct kvm_kernel_irqfd *irqfd)  in irqfd_is_active()  argument
163 	return list_empty(&irqfd->list) ? false : true;  in irqfd_is_active()
172 irqfd_deactivate(struct kvm_kernel_irqfd *irqfd)  in irqfd_deactivate()  argument
174 	BUG_ON(!irqfd_is_active(irqfd));  in irqfd_deactivate()
176 	list_del_init(&irqfd->list);  in irqfd_deactivate()
178 	queue_work(irqfd_cleanup_wq, &irqfd->shutdown);  in irqfd_deactivate()
196 	struct kvm_kernel_irqfd *irqfd =  in irqfd_wakeup()  local
200 	struct kvm *kvm = irqfd->kvm;  in irqfd_wakeup()
207 		eventfd_ctx_do_read(irqfd->eventfd, &cnt);  in irqfd_wakeup()
211 			seq = read_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_wakeup()
212 			irq = irqfd->irq_entry;  in irqfd_wakeup()
213 		} while (read_seqcount_retry(&irqfd->irq_entry_sc, seq));  in irqfd_wakeup()
218 			schedule_work(&irqfd->inject);  in irqfd_wakeup()
238 		if (irqfd_is_active(irqfd))  in irqfd_wakeup()
239 			irqfd_deactivate(irqfd);  in irqfd_wakeup()
251 	struct kvm_kernel_irqfd *irqfd =  in irqfd_ptable_queue_proc()  local
253 	add_wait_queue_priority(wqh, &irqfd->wait);  in irqfd_ptable_queue_proc()
257 static void irqfd_update(struct kvm *kvm, struct kvm_kernel_irqfd *irqfd)  in irqfd_update()  argument
263 	n_entries = kvm_irq_map_gsi(kvm, entries, irqfd->gsi);  in irqfd_update()
265 	write_seqcount_begin(&irqfd->irq_entry_sc);  in irqfd_update()
269 		irqfd->irq_entry = *e;  in irqfd_update()
271 		irqfd->irq_entry.type = 0;  in irqfd_update()
273 	write_seqcount_end(&irqfd->irq_entry_sc);  in irqfd_update()
305 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_assign()  local
318 	irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL_ACCOUNT);  in kvm_irqfd_assign()
319 	if (!irqfd)  in kvm_irqfd_assign()
322 	irqfd->kvm = kvm;  in kvm_irqfd_assign()
323 	irqfd->gsi = args->gsi;  in kvm_irqfd_assign()
324 	INIT_LIST_HEAD(&irqfd->list);  in kvm_irqfd_assign()
325 	INIT_WORK(&irqfd->inject, irqfd_inject);  in kvm_irqfd_assign()
326 	INIT_WORK(&irqfd->shutdown, irqfd_shutdown);  in kvm_irqfd_assign()
327 	seqcount_spinlock_init(&irqfd->irq_entry_sc, &kvm->irqfds.lock);  in kvm_irqfd_assign()
341 	irqfd->eventfd = eventfd;  in kvm_irqfd_assign()
352 		irqfd->resamplefd = resamplefd;  in kvm_irqfd_assign()
353 		INIT_LIST_HEAD(&irqfd->resampler_link);  in kvm_irqfd_assign()
359 			if (resampler->notifier.gsi == irqfd->gsi) {  in kvm_irqfd_assign()
360 				irqfd->resampler = resampler;  in kvm_irqfd_assign()
365 		if (!irqfd->resampler) {  in kvm_irqfd_assign()
376 			resampler->notifier.gsi = irqfd->gsi;  in kvm_irqfd_assign()
383 			irqfd->resampler = resampler;  in kvm_irqfd_assign()
386 		list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list);  in kvm_irqfd_assign()
396 	init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);  in kvm_irqfd_assign()
397 	init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);  in kvm_irqfd_assign()
403 		if (irqfd->eventfd != tmp->eventfd)  in kvm_irqfd_assign()
412 	irqfd_update(kvm, irqfd);  in kvm_irqfd_assign()
414 	list_add_tail(&irqfd->list, &kvm->irqfds.items);  in kvm_irqfd_assign()
422 	events = vfs_poll(f.file, &irqfd->pt);  in kvm_irqfd_assign()
425 		schedule_work(&irqfd->inject);  in kvm_irqfd_assign()
429 		irqfd->consumer.token = (void *)irqfd->eventfd;  in kvm_irqfd_assign()
430 		irqfd->consumer.add_producer = kvm_arch_irq_bypass_add_producer;  in kvm_irqfd_assign()
431 		irqfd->consumer.del_producer = kvm_arch_irq_bypass_del_producer;  in kvm_irqfd_assign()
432 		irqfd->consumer.stop = kvm_arch_irq_bypass_stop;  in kvm_irqfd_assign()
433 		irqfd->consumer.start = kvm_arch_irq_bypass_start;  in kvm_irqfd_assign()
434 		ret = irq_bypass_register_consumer(&irqfd->consumer);  in kvm_irqfd_assign()
437 				irqfd->consumer.token, ret);  in kvm_irqfd_assign()
451 	if (irqfd->resampler)  in kvm_irqfd_assign()
452 		irqfd_resampler_shutdown(irqfd);  in kvm_irqfd_assign()
463 	kfree(irqfd);  in kvm_irqfd_assign()
550 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_deassign()  local
559 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) {  in kvm_irqfd_deassign()
560 		if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) {  in kvm_irqfd_deassign()
567 			write_seqcount_begin(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
568 			irqfd->irq_entry.type = 0;  in kvm_irqfd_deassign()
569 			write_seqcount_end(&irqfd->irq_entry_sc);  in kvm_irqfd_deassign()
570 			irqfd_deactivate(irqfd);  in kvm_irqfd_deassign()
606 	struct kvm_kernel_irqfd *irqfd, *tmp;  in kvm_irqfd_release()  local
610 	list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list)  in kvm_irqfd_release()
611 		irqfd_deactivate(irqfd);  in kvm_irqfd_release()
629 	struct kvm_kernel_irqfd *irqfd;  in kvm_irq_routing_update()  local
633 	list_for_each_entry(irqfd, &kvm->irqfds.items, list) {  in kvm_irq_routing_update()
636 		struct kvm_kernel_irq_routing_entry old = irqfd->irq_entry;  in kvm_irq_routing_update()
639 		irqfd_update(kvm, irqfd);  in kvm_irq_routing_update()
642 		if (irqfd->producer &&  in kvm_irq_routing_update()
643 		    kvm_arch_irqfd_route_changed(&old, &irqfd->irq_entry)) {  in kvm_irq_routing_update()
645 					irqfd->kvm, irqfd->producer->irq,  in kvm_irq_routing_update()
646 					irqfd->gsi, 1);  in kvm_irq_routing_update()