1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO generic eventfd code for IRQFD support. 4 * Derived from drivers/vfio/pci/vfio_pci_intrs.c 5 * 6 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 7 * Author: Alex Williamson <alex.williamson@redhat.com> 8 */ 9 10 #include <linux/vfio.h> 11 #include <linux/eventfd.h> 12 #include <linux/file.h> 13 #include <linux/module.h> 14 #include <linux/slab.h> 15 16 #define DRIVER_VERSION "0.1" 17 #define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 18 #define DRIVER_DESC "IRQFD support for VFIO bus drivers" 19 20 static struct workqueue_struct *vfio_irqfd_cleanup_wq; 21 static DEFINE_SPINLOCK(virqfd_lock); 22 23 static int __init vfio_virqfd_init(void) 24 { 25 vfio_irqfd_cleanup_wq = 26 create_singlethread_workqueue("vfio-irqfd-cleanup"); 27 if (!vfio_irqfd_cleanup_wq) 28 return -ENOMEM; 29 30 return 0; 31 } 32 33 static void __exit vfio_virqfd_exit(void) 34 { 35 destroy_workqueue(vfio_irqfd_cleanup_wq); 36 } 37 38 static void virqfd_deactivate(struct virqfd *virqfd) 39 { 40 queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown); 41 } 42 43 static int virqfd_wakeup(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) 44 { 45 struct virqfd *virqfd = container_of(wait, struct virqfd, wait); 46 __poll_t flags = key_to_poll(key); 47 48 if (flags & EPOLLIN) { 49 u64 cnt; 50 eventfd_ctx_do_read(virqfd->eventfd, &cnt); 51 52 /* An event has been signaled, call function */ 53 if ((!virqfd->handler || 54 virqfd->handler(virqfd->opaque, virqfd->data)) && 55 virqfd->thread) 56 schedule_work(&virqfd->inject); 57 } 58 59 if (flags & EPOLLHUP) { 60 unsigned long flags; 61 spin_lock_irqsave(&virqfd_lock, flags); 62 63 /* 64 * The eventfd is closing, if the virqfd has not yet been 65 * queued for release, as determined by testing whether the 66 * virqfd pointer to it is still valid, queue it now. As 67 * with kvm irqfds, we know we won't race against the virqfd 68 * going away because we hold the lock to get here. 69 */ 70 if (*(virqfd->pvirqfd) == virqfd) { 71 *(virqfd->pvirqfd) = NULL; 72 virqfd_deactivate(virqfd); 73 } 74 75 spin_unlock_irqrestore(&virqfd_lock, flags); 76 } 77 78 return 0; 79 } 80 81 static void virqfd_ptable_queue_proc(struct file *file, 82 wait_queue_head_t *wqh, poll_table *pt) 83 { 84 struct virqfd *virqfd = container_of(pt, struct virqfd, pt); 85 add_wait_queue(wqh, &virqfd->wait); 86 } 87 88 static void virqfd_shutdown(struct work_struct *work) 89 { 90 struct virqfd *virqfd = container_of(work, struct virqfd, shutdown); 91 u64 cnt; 92 93 eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt); 94 flush_work(&virqfd->inject); 95 eventfd_ctx_put(virqfd->eventfd); 96 97 kfree(virqfd); 98 } 99 100 static void virqfd_inject(struct work_struct *work) 101 { 102 struct virqfd *virqfd = container_of(work, struct virqfd, inject); 103 if (virqfd->thread) 104 virqfd->thread(virqfd->opaque, virqfd->data); 105 } 106 107 int vfio_virqfd_enable(void *opaque, 108 int (*handler)(void *, void *), 109 void (*thread)(void *, void *), 110 void *data, struct virqfd **pvirqfd, int fd) 111 { 112 struct fd irqfd; 113 struct eventfd_ctx *ctx; 114 struct virqfd *virqfd; 115 int ret = 0; 116 __poll_t events; 117 118 virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL); 119 if (!virqfd) 120 return -ENOMEM; 121 122 virqfd->pvirqfd = pvirqfd; 123 virqfd->opaque = opaque; 124 virqfd->handler = handler; 125 virqfd->thread = thread; 126 virqfd->data = data; 127 128 INIT_WORK(&virqfd->shutdown, virqfd_shutdown); 129 INIT_WORK(&virqfd->inject, virqfd_inject); 130 131 irqfd = fdget(fd); 132 if (!irqfd.file) { 133 ret = -EBADF; 134 goto err_fd; 135 } 136 137 ctx = eventfd_ctx_fileget(irqfd.file); 138 if (IS_ERR(ctx)) { 139 ret = PTR_ERR(ctx); 140 goto err_ctx; 141 } 142 143 virqfd->eventfd = ctx; 144 145 /* 146 * virqfds can be released by closing the eventfd or directly 147 * through ioctl. These are both done through a workqueue, so 148 * we update the pointer to the virqfd under lock to avoid 149 * pushing multiple jobs to release the same virqfd. 150 */ 151 spin_lock_irq(&virqfd_lock); 152 153 if (*pvirqfd) { 154 spin_unlock_irq(&virqfd_lock); 155 ret = -EBUSY; 156 goto err_busy; 157 } 158 *pvirqfd = virqfd; 159 160 spin_unlock_irq(&virqfd_lock); 161 162 /* 163 * Install our own custom wake-up handling so we are notified via 164 * a callback whenever someone signals the underlying eventfd. 165 */ 166 init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup); 167 init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc); 168 169 events = vfs_poll(irqfd.file, &virqfd->pt); 170 171 /* 172 * Check if there was an event already pending on the eventfd 173 * before we registered and trigger it as if we didn't miss it. 174 */ 175 if (events & EPOLLIN) { 176 if ((!handler || handler(opaque, data)) && thread) 177 schedule_work(&virqfd->inject); 178 } 179 180 /* 181 * Do not drop the file until the irqfd is fully initialized, 182 * otherwise we might race against the EPOLLHUP. 183 */ 184 fdput(irqfd); 185 186 return 0; 187 err_busy: 188 eventfd_ctx_put(ctx); 189 err_ctx: 190 fdput(irqfd); 191 err_fd: 192 kfree(virqfd); 193 194 return ret; 195 } 196 EXPORT_SYMBOL_GPL(vfio_virqfd_enable); 197 198 void vfio_virqfd_disable(struct virqfd **pvirqfd) 199 { 200 unsigned long flags; 201 202 spin_lock_irqsave(&virqfd_lock, flags); 203 204 if (*pvirqfd) { 205 virqfd_deactivate(*pvirqfd); 206 *pvirqfd = NULL; 207 } 208 209 spin_unlock_irqrestore(&virqfd_lock, flags); 210 211 /* 212 * Block until we know all outstanding shutdown jobs have completed. 213 * Even if we don't queue the job, flush the wq to be sure it's 214 * been released. 215 */ 216 flush_workqueue(vfio_irqfd_cleanup_wq); 217 } 218 EXPORT_SYMBOL_GPL(vfio_virqfd_disable); 219 220 module_init(vfio_virqfd_init); 221 module_exit(vfio_virqfd_exit); 222 223 MODULE_VERSION(DRIVER_VERSION); 224 MODULE_LICENSE("GPL v2"); 225 MODULE_AUTHOR(DRIVER_AUTHOR); 226 MODULE_DESCRIPTION(DRIVER_DESC); 227