1 /* 2 * VFIO PCI interrupt handling 3 * 4 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 5 * Author: Alex Williamson <alex.williamson@redhat.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * Derived from original vfio: 12 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 13 * Author: Tom Lyon, pugs@cisco.com 14 */ 15 16 #include <linux/device.h> 17 #include <linux/interrupt.h> 18 #include <linux/eventfd.h> 19 #include <linux/pci.h> 20 #include <linux/file.h> 21 #include <linux/poll.h> 22 #include <linux/vfio.h> 23 #include <linux/wait.h> 24 #include <linux/workqueue.h> 25 #include <linux/slab.h> 26 27 #include "vfio_pci_private.h" 28 29 /* 30 * IRQfd - generic 31 */ 32 struct virqfd { 33 struct vfio_pci_device *vdev; 34 struct eventfd_ctx *eventfd; 35 int (*handler)(struct vfio_pci_device *, void *); 36 void (*thread)(struct vfio_pci_device *, void *); 37 void *data; 38 struct work_struct inject; 39 wait_queue_t wait; 40 poll_table pt; 41 struct work_struct shutdown; 42 struct virqfd **pvirqfd; 43 }; 44 45 static struct workqueue_struct *vfio_irqfd_cleanup_wq; 46 47 int __init vfio_pci_virqfd_init(void) 48 { 49 vfio_irqfd_cleanup_wq = 50 create_singlethread_workqueue("vfio-irqfd-cleanup"); 51 if (!vfio_irqfd_cleanup_wq) 52 return -ENOMEM; 53 54 return 0; 55 } 56 57 void vfio_pci_virqfd_exit(void) 58 { 59 destroy_workqueue(vfio_irqfd_cleanup_wq); 60 } 61 62 static void virqfd_deactivate(struct virqfd *virqfd) 63 { 64 queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown); 65 } 66 67 static int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key) 68 { 69 struct virqfd *virqfd = container_of(wait, struct virqfd, wait); 70 unsigned long flags = (unsigned long)key; 71 72 if (flags & POLLIN) { 73 /* An event has been signaled, call function */ 74 if ((!virqfd->handler || 75 virqfd->handler(virqfd->vdev, virqfd->data)) && 76 virqfd->thread) 77 schedule_work(&virqfd->inject); 78 } 79 80 if (flags & POLLHUP) { 81 unsigned long flags; 82 spin_lock_irqsave(&virqfd->vdev->irqlock, flags); 83 84 /* 85 * The eventfd is closing, if the virqfd has not yet been 86 * queued for release, as determined by testing whether the 87 * vdev pointer to it is still valid, queue it now. As 88 * with kvm irqfds, we know we won't race against the virqfd 89 * going away because we hold wqh->lock to get here. 90 */ 91 if (*(virqfd->pvirqfd) == virqfd) { 92 *(virqfd->pvirqfd) = NULL; 93 virqfd_deactivate(virqfd); 94 } 95 96 spin_unlock_irqrestore(&virqfd->vdev->irqlock, flags); 97 } 98 99 return 0; 100 } 101 102 static void virqfd_ptable_queue_proc(struct file *file, 103 wait_queue_head_t *wqh, poll_table *pt) 104 { 105 struct virqfd *virqfd = container_of(pt, struct virqfd, pt); 106 add_wait_queue(wqh, &virqfd->wait); 107 } 108 109 static void virqfd_shutdown(struct work_struct *work) 110 { 111 struct virqfd *virqfd = container_of(work, struct virqfd, shutdown); 112 u64 cnt; 113 114 eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt); 115 flush_work(&virqfd->inject); 116 eventfd_ctx_put(virqfd->eventfd); 117 118 kfree(virqfd); 119 } 120 121 static void virqfd_inject(struct work_struct *work) 122 { 123 struct virqfd *virqfd = container_of(work, struct virqfd, inject); 124 if (virqfd->thread) 125 virqfd->thread(virqfd->vdev, virqfd->data); 126 } 127 128 static int virqfd_enable(struct vfio_pci_device *vdev, 129 int (*handler)(struct vfio_pci_device *, void *), 130 void (*thread)(struct vfio_pci_device *, void *), 131 void *data, struct virqfd **pvirqfd, int fd) 132 { 133 struct fd irqfd; 134 struct eventfd_ctx *ctx; 135 struct virqfd *virqfd; 136 int ret = 0; 137 unsigned int events; 138 139 virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL); 140 if (!virqfd) 141 return -ENOMEM; 142 143 virqfd->pvirqfd = pvirqfd; 144 virqfd->vdev = vdev; 145 virqfd->handler = handler; 146 virqfd->thread = thread; 147 virqfd->data = data; 148 149 INIT_WORK(&virqfd->shutdown, virqfd_shutdown); 150 INIT_WORK(&virqfd->inject, virqfd_inject); 151 152 irqfd = fdget(fd); 153 if (!irqfd.file) { 154 ret = -EBADF; 155 goto err_fd; 156 } 157 158 ctx = eventfd_ctx_fileget(irqfd.file); 159 if (IS_ERR(ctx)) { 160 ret = PTR_ERR(ctx); 161 goto err_ctx; 162 } 163 164 virqfd->eventfd = ctx; 165 166 /* 167 * virqfds can be released by closing the eventfd or directly 168 * through ioctl. These are both done through a workqueue, so 169 * we update the pointer to the virqfd under lock to avoid 170 * pushing multiple jobs to release the same virqfd. 171 */ 172 spin_lock_irq(&vdev->irqlock); 173 174 if (*pvirqfd) { 175 spin_unlock_irq(&vdev->irqlock); 176 ret = -EBUSY; 177 goto err_busy; 178 } 179 *pvirqfd = virqfd; 180 181 spin_unlock_irq(&vdev->irqlock); 182 183 /* 184 * Install our own custom wake-up handling so we are notified via 185 * a callback whenever someone signals the underlying eventfd. 186 */ 187 init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup); 188 init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc); 189 190 events = irqfd.file->f_op->poll(irqfd.file, &virqfd->pt); 191 192 /* 193 * Check if there was an event already pending on the eventfd 194 * before we registered and trigger it as if we didn't miss it. 195 */ 196 if (events & POLLIN) { 197 if ((!handler || handler(vdev, data)) && thread) 198 schedule_work(&virqfd->inject); 199 } 200 201 /* 202 * Do not drop the file until the irqfd is fully initialized, 203 * otherwise we might race against the POLLHUP. 204 */ 205 fdput(irqfd); 206 207 return 0; 208 err_busy: 209 eventfd_ctx_put(ctx); 210 err_ctx: 211 fdput(irqfd); 212 err_fd: 213 kfree(virqfd); 214 215 return ret; 216 } 217 218 static void virqfd_disable(struct vfio_pci_device *vdev, 219 struct virqfd **pvirqfd) 220 { 221 unsigned long flags; 222 223 spin_lock_irqsave(&vdev->irqlock, flags); 224 225 if (*pvirqfd) { 226 virqfd_deactivate(*pvirqfd); 227 *pvirqfd = NULL; 228 } 229 230 spin_unlock_irqrestore(&vdev->irqlock, flags); 231 232 /* 233 * Block until we know all outstanding shutdown jobs have completed. 234 * Even if we don't queue the job, flush the wq to be sure it's 235 * been released. 236 */ 237 flush_workqueue(vfio_irqfd_cleanup_wq); 238 } 239 240 /* 241 * INTx 242 */ 243 static void vfio_send_intx_eventfd(struct vfio_pci_device *vdev, void *unused) 244 { 245 if (likely(is_intx(vdev) && !vdev->virq_disabled)) 246 eventfd_signal(vdev->ctx[0].trigger, 1); 247 } 248 249 void vfio_pci_intx_mask(struct vfio_pci_device *vdev) 250 { 251 struct pci_dev *pdev = vdev->pdev; 252 unsigned long flags; 253 254 spin_lock_irqsave(&vdev->irqlock, flags); 255 256 /* 257 * Masking can come from interrupt, ioctl, or config space 258 * via INTx disable. The latter means this can get called 259 * even when not using intx delivery. In this case, just 260 * try to have the physical bit follow the virtual bit. 261 */ 262 if (unlikely(!is_intx(vdev))) { 263 if (vdev->pci_2_3) 264 pci_intx(pdev, 0); 265 } else if (!vdev->ctx[0].masked) { 266 /* 267 * Can't use check_and_mask here because we always want to 268 * mask, not just when something is pending. 269 */ 270 if (vdev->pci_2_3) 271 pci_intx(pdev, 0); 272 else 273 disable_irq_nosync(pdev->irq); 274 275 vdev->ctx[0].masked = true; 276 } 277 278 spin_unlock_irqrestore(&vdev->irqlock, flags); 279 } 280 281 /* 282 * If this is triggered by an eventfd, we can't call eventfd_signal 283 * or else we'll deadlock on the eventfd wait queue. Return >0 when 284 * a signal is necessary, which can then be handled via a work queue 285 * or directly depending on the caller. 286 */ 287 static int vfio_pci_intx_unmask_handler(struct vfio_pci_device *vdev, 288 void *unused) 289 { 290 struct pci_dev *pdev = vdev->pdev; 291 unsigned long flags; 292 int ret = 0; 293 294 spin_lock_irqsave(&vdev->irqlock, flags); 295 296 /* 297 * Unmasking comes from ioctl or config, so again, have the 298 * physical bit follow the virtual even when not using INTx. 299 */ 300 if (unlikely(!is_intx(vdev))) { 301 if (vdev->pci_2_3) 302 pci_intx(pdev, 1); 303 } else if (vdev->ctx[0].masked && !vdev->virq_disabled) { 304 /* 305 * A pending interrupt here would immediately trigger, 306 * but we can avoid that overhead by just re-sending 307 * the interrupt to the user. 308 */ 309 if (vdev->pci_2_3) { 310 if (!pci_check_and_unmask_intx(pdev)) 311 ret = 1; 312 } else 313 enable_irq(pdev->irq); 314 315 vdev->ctx[0].masked = (ret > 0); 316 } 317 318 spin_unlock_irqrestore(&vdev->irqlock, flags); 319 320 return ret; 321 } 322 323 void vfio_pci_intx_unmask(struct vfio_pci_device *vdev) 324 { 325 if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0) 326 vfio_send_intx_eventfd(vdev, NULL); 327 } 328 329 static irqreturn_t vfio_intx_handler(int irq, void *dev_id) 330 { 331 struct vfio_pci_device *vdev = dev_id; 332 unsigned long flags; 333 int ret = IRQ_NONE; 334 335 spin_lock_irqsave(&vdev->irqlock, flags); 336 337 if (!vdev->pci_2_3) { 338 disable_irq_nosync(vdev->pdev->irq); 339 vdev->ctx[0].masked = true; 340 ret = IRQ_HANDLED; 341 } else if (!vdev->ctx[0].masked && /* may be shared */ 342 pci_check_and_mask_intx(vdev->pdev)) { 343 vdev->ctx[0].masked = true; 344 ret = IRQ_HANDLED; 345 } 346 347 spin_unlock_irqrestore(&vdev->irqlock, flags); 348 349 if (ret == IRQ_HANDLED) 350 vfio_send_intx_eventfd(vdev, NULL); 351 352 return ret; 353 } 354 355 static int vfio_intx_enable(struct vfio_pci_device *vdev) 356 { 357 if (!is_irq_none(vdev)) 358 return -EINVAL; 359 360 if (!vdev->pdev->irq) 361 return -ENODEV; 362 363 vdev->ctx = kzalloc(sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL); 364 if (!vdev->ctx) 365 return -ENOMEM; 366 367 vdev->num_ctx = 1; 368 369 /* 370 * If the virtual interrupt is masked, restore it. Devices 371 * supporting DisINTx can be masked at the hardware level 372 * here, non-PCI-2.3 devices will have to wait until the 373 * interrupt is enabled. 374 */ 375 vdev->ctx[0].masked = vdev->virq_disabled; 376 if (vdev->pci_2_3) 377 pci_intx(vdev->pdev, !vdev->ctx[0].masked); 378 379 vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; 380 381 return 0; 382 } 383 384 static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd) 385 { 386 struct pci_dev *pdev = vdev->pdev; 387 unsigned long irqflags = IRQF_SHARED; 388 struct eventfd_ctx *trigger; 389 unsigned long flags; 390 int ret; 391 392 if (vdev->ctx[0].trigger) { 393 free_irq(pdev->irq, vdev); 394 kfree(vdev->ctx[0].name); 395 eventfd_ctx_put(vdev->ctx[0].trigger); 396 vdev->ctx[0].trigger = NULL; 397 } 398 399 if (fd < 0) /* Disable only */ 400 return 0; 401 402 vdev->ctx[0].name = kasprintf(GFP_KERNEL, "vfio-intx(%s)", 403 pci_name(pdev)); 404 if (!vdev->ctx[0].name) 405 return -ENOMEM; 406 407 trigger = eventfd_ctx_fdget(fd); 408 if (IS_ERR(trigger)) { 409 kfree(vdev->ctx[0].name); 410 return PTR_ERR(trigger); 411 } 412 413 vdev->ctx[0].trigger = trigger; 414 415 if (!vdev->pci_2_3) 416 irqflags = 0; 417 418 ret = request_irq(pdev->irq, vfio_intx_handler, 419 irqflags, vdev->ctx[0].name, vdev); 420 if (ret) { 421 vdev->ctx[0].trigger = NULL; 422 kfree(vdev->ctx[0].name); 423 eventfd_ctx_put(trigger); 424 return ret; 425 } 426 427 /* 428 * INTx disable will stick across the new irq setup, 429 * disable_irq won't. 430 */ 431 spin_lock_irqsave(&vdev->irqlock, flags); 432 if (!vdev->pci_2_3 && vdev->ctx[0].masked) 433 disable_irq_nosync(pdev->irq); 434 spin_unlock_irqrestore(&vdev->irqlock, flags); 435 436 return 0; 437 } 438 439 static void vfio_intx_disable(struct vfio_pci_device *vdev) 440 { 441 vfio_intx_set_signal(vdev, -1); 442 virqfd_disable(vdev, &vdev->ctx[0].unmask); 443 virqfd_disable(vdev, &vdev->ctx[0].mask); 444 vdev->irq_type = VFIO_PCI_NUM_IRQS; 445 vdev->num_ctx = 0; 446 kfree(vdev->ctx); 447 } 448 449 /* 450 * MSI/MSI-X 451 */ 452 static irqreturn_t vfio_msihandler(int irq, void *arg) 453 { 454 struct eventfd_ctx *trigger = arg; 455 456 eventfd_signal(trigger, 1); 457 return IRQ_HANDLED; 458 } 459 460 static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix) 461 { 462 struct pci_dev *pdev = vdev->pdev; 463 int ret; 464 465 if (!is_irq_none(vdev)) 466 return -EINVAL; 467 468 vdev->ctx = kzalloc(nvec * sizeof(struct vfio_pci_irq_ctx), GFP_KERNEL); 469 if (!vdev->ctx) 470 return -ENOMEM; 471 472 if (msix) { 473 int i; 474 475 vdev->msix = kzalloc(nvec * sizeof(struct msix_entry), 476 GFP_KERNEL); 477 if (!vdev->msix) { 478 kfree(vdev->ctx); 479 return -ENOMEM; 480 } 481 482 for (i = 0; i < nvec; i++) 483 vdev->msix[i].entry = i; 484 485 ret = pci_enable_msix_range(pdev, vdev->msix, 1, nvec); 486 if (ret < nvec) { 487 if (ret > 0) 488 pci_disable_msix(pdev); 489 kfree(vdev->msix); 490 kfree(vdev->ctx); 491 return ret; 492 } 493 } else { 494 ret = pci_enable_msi_range(pdev, 1, nvec); 495 if (ret < nvec) { 496 if (ret > 0) 497 pci_disable_msi(pdev); 498 kfree(vdev->ctx); 499 return ret; 500 } 501 } 502 503 vdev->num_ctx = nvec; 504 vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : 505 VFIO_PCI_MSI_IRQ_INDEX; 506 507 if (!msix) { 508 /* 509 * Compute the virtual hardware field for max msi vectors - 510 * it is the log base 2 of the number of vectors. 511 */ 512 vdev->msi_qmax = fls(nvec * 2 - 1) - 1; 513 } 514 515 return 0; 516 } 517 518 static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, 519 int vector, int fd, bool msix) 520 { 521 struct pci_dev *pdev = vdev->pdev; 522 int irq = msix ? vdev->msix[vector].vector : pdev->irq + vector; 523 char *name = msix ? "vfio-msix" : "vfio-msi"; 524 struct eventfd_ctx *trigger; 525 int ret; 526 527 if (vector >= vdev->num_ctx) 528 return -EINVAL; 529 530 if (vdev->ctx[vector].trigger) { 531 free_irq(irq, vdev->ctx[vector].trigger); 532 kfree(vdev->ctx[vector].name); 533 eventfd_ctx_put(vdev->ctx[vector].trigger); 534 vdev->ctx[vector].trigger = NULL; 535 } 536 537 if (fd < 0) 538 return 0; 539 540 vdev->ctx[vector].name = kasprintf(GFP_KERNEL, "%s[%d](%s)", 541 name, vector, pci_name(pdev)); 542 if (!vdev->ctx[vector].name) 543 return -ENOMEM; 544 545 trigger = eventfd_ctx_fdget(fd); 546 if (IS_ERR(trigger)) { 547 kfree(vdev->ctx[vector].name); 548 return PTR_ERR(trigger); 549 } 550 551 ret = request_irq(irq, vfio_msihandler, 0, 552 vdev->ctx[vector].name, trigger); 553 if (ret) { 554 kfree(vdev->ctx[vector].name); 555 eventfd_ctx_put(trigger); 556 return ret; 557 } 558 559 vdev->ctx[vector].trigger = trigger; 560 561 return 0; 562 } 563 564 static int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start, 565 unsigned count, int32_t *fds, bool msix) 566 { 567 int i, j, ret = 0; 568 569 if (start + count > vdev->num_ctx) 570 return -EINVAL; 571 572 for (i = 0, j = start; i < count && !ret; i++, j++) { 573 int fd = fds ? fds[i] : -1; 574 ret = vfio_msi_set_vector_signal(vdev, j, fd, msix); 575 } 576 577 if (ret) { 578 for (--j; j >= start; j--) 579 vfio_msi_set_vector_signal(vdev, j, -1, msix); 580 } 581 582 return ret; 583 } 584 585 static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix) 586 { 587 struct pci_dev *pdev = vdev->pdev; 588 int i; 589 590 vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix); 591 592 for (i = 0; i < vdev->num_ctx; i++) { 593 virqfd_disable(vdev, &vdev->ctx[i].unmask); 594 virqfd_disable(vdev, &vdev->ctx[i].mask); 595 } 596 597 if (msix) { 598 pci_disable_msix(vdev->pdev); 599 kfree(vdev->msix); 600 } else 601 pci_disable_msi(pdev); 602 603 vdev->irq_type = VFIO_PCI_NUM_IRQS; 604 vdev->num_ctx = 0; 605 kfree(vdev->ctx); 606 } 607 608 /* 609 * IOCTL support 610 */ 611 static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev, 612 unsigned index, unsigned start, 613 unsigned count, uint32_t flags, void *data) 614 { 615 if (!is_intx(vdev) || start != 0 || count != 1) 616 return -EINVAL; 617 618 if (flags & VFIO_IRQ_SET_DATA_NONE) { 619 vfio_pci_intx_unmask(vdev); 620 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 621 uint8_t unmask = *(uint8_t *)data; 622 if (unmask) 623 vfio_pci_intx_unmask(vdev); 624 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 625 int32_t fd = *(int32_t *)data; 626 if (fd >= 0) 627 return virqfd_enable(vdev, vfio_pci_intx_unmask_handler, 628 vfio_send_intx_eventfd, NULL, 629 &vdev->ctx[0].unmask, fd); 630 631 virqfd_disable(vdev, &vdev->ctx[0].unmask); 632 } 633 634 return 0; 635 } 636 637 static int vfio_pci_set_intx_mask(struct vfio_pci_device *vdev, 638 unsigned index, unsigned start, 639 unsigned count, uint32_t flags, void *data) 640 { 641 if (!is_intx(vdev) || start != 0 || count != 1) 642 return -EINVAL; 643 644 if (flags & VFIO_IRQ_SET_DATA_NONE) { 645 vfio_pci_intx_mask(vdev); 646 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 647 uint8_t mask = *(uint8_t *)data; 648 if (mask) 649 vfio_pci_intx_mask(vdev); 650 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 651 return -ENOTTY; /* XXX implement me */ 652 } 653 654 return 0; 655 } 656 657 static int vfio_pci_set_intx_trigger(struct vfio_pci_device *vdev, 658 unsigned index, unsigned start, 659 unsigned count, uint32_t flags, void *data) 660 { 661 if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 662 vfio_intx_disable(vdev); 663 return 0; 664 } 665 666 if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) 667 return -EINVAL; 668 669 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 670 int32_t fd = *(int32_t *)data; 671 int ret; 672 673 if (is_intx(vdev)) 674 return vfio_intx_set_signal(vdev, fd); 675 676 ret = vfio_intx_enable(vdev); 677 if (ret) 678 return ret; 679 680 ret = vfio_intx_set_signal(vdev, fd); 681 if (ret) 682 vfio_intx_disable(vdev); 683 684 return ret; 685 } 686 687 if (!is_intx(vdev)) 688 return -EINVAL; 689 690 if (flags & VFIO_IRQ_SET_DATA_NONE) { 691 vfio_send_intx_eventfd(vdev, NULL); 692 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 693 uint8_t trigger = *(uint8_t *)data; 694 if (trigger) 695 vfio_send_intx_eventfd(vdev, NULL); 696 } 697 return 0; 698 } 699 700 static int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev, 701 unsigned index, unsigned start, 702 unsigned count, uint32_t flags, void *data) 703 { 704 int i; 705 bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; 706 707 if (irq_is(vdev, index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 708 vfio_msi_disable(vdev, msix); 709 return 0; 710 } 711 712 if (!(irq_is(vdev, index) || is_irq_none(vdev))) 713 return -EINVAL; 714 715 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 716 int32_t *fds = data; 717 int ret; 718 719 if (vdev->irq_type == index) 720 return vfio_msi_set_block(vdev, start, count, 721 fds, msix); 722 723 ret = vfio_msi_enable(vdev, start + count, msix); 724 if (ret) 725 return ret; 726 727 ret = vfio_msi_set_block(vdev, start, count, fds, msix); 728 if (ret) 729 vfio_msi_disable(vdev, msix); 730 731 return ret; 732 } 733 734 if (!irq_is(vdev, index) || start + count > vdev->num_ctx) 735 return -EINVAL; 736 737 for (i = start; i < start + count; i++) { 738 if (!vdev->ctx[i].trigger) 739 continue; 740 if (flags & VFIO_IRQ_SET_DATA_NONE) { 741 eventfd_signal(vdev->ctx[i].trigger, 1); 742 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 743 uint8_t *bools = data; 744 if (bools[i - start]) 745 eventfd_signal(vdev->ctx[i].trigger, 1); 746 } 747 } 748 return 0; 749 } 750 751 static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev, 752 unsigned index, unsigned start, 753 unsigned count, uint32_t flags, void *data) 754 { 755 int32_t fd = *(int32_t *)data; 756 757 if ((index != VFIO_PCI_ERR_IRQ_INDEX) || 758 !(flags & VFIO_IRQ_SET_DATA_TYPE_MASK)) 759 return -EINVAL; 760 761 /* DATA_NONE/DATA_BOOL enables loopback testing */ 762 if (flags & VFIO_IRQ_SET_DATA_NONE) { 763 if (vdev->err_trigger) 764 eventfd_signal(vdev->err_trigger, 1); 765 return 0; 766 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 767 uint8_t trigger = *(uint8_t *)data; 768 if (trigger && vdev->err_trigger) 769 eventfd_signal(vdev->err_trigger, 1); 770 return 0; 771 } 772 773 /* Handle SET_DATA_EVENTFD */ 774 if (fd == -1) { 775 if (vdev->err_trigger) 776 eventfd_ctx_put(vdev->err_trigger); 777 vdev->err_trigger = NULL; 778 return 0; 779 } else if (fd >= 0) { 780 struct eventfd_ctx *efdctx; 781 efdctx = eventfd_ctx_fdget(fd); 782 if (IS_ERR(efdctx)) 783 return PTR_ERR(efdctx); 784 if (vdev->err_trigger) 785 eventfd_ctx_put(vdev->err_trigger); 786 vdev->err_trigger = efdctx; 787 return 0; 788 } else 789 return -EINVAL; 790 } 791 int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags, 792 unsigned index, unsigned start, unsigned count, 793 void *data) 794 { 795 int (*func)(struct vfio_pci_device *vdev, unsigned index, 796 unsigned start, unsigned count, uint32_t flags, 797 void *data) = NULL; 798 799 switch (index) { 800 case VFIO_PCI_INTX_IRQ_INDEX: 801 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 802 case VFIO_IRQ_SET_ACTION_MASK: 803 func = vfio_pci_set_intx_mask; 804 break; 805 case VFIO_IRQ_SET_ACTION_UNMASK: 806 func = vfio_pci_set_intx_unmask; 807 break; 808 case VFIO_IRQ_SET_ACTION_TRIGGER: 809 func = vfio_pci_set_intx_trigger; 810 break; 811 } 812 break; 813 case VFIO_PCI_MSI_IRQ_INDEX: 814 case VFIO_PCI_MSIX_IRQ_INDEX: 815 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 816 case VFIO_IRQ_SET_ACTION_MASK: 817 case VFIO_IRQ_SET_ACTION_UNMASK: 818 /* XXX Need masking support exported */ 819 break; 820 case VFIO_IRQ_SET_ACTION_TRIGGER: 821 func = vfio_pci_set_msi_trigger; 822 break; 823 } 824 break; 825 case VFIO_PCI_ERR_IRQ_INDEX: 826 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 827 case VFIO_IRQ_SET_ACTION_TRIGGER: 828 if (pci_is_pcie(vdev->pdev)) 829 func = vfio_pci_set_err_trigger; 830 break; 831 } 832 } 833 834 if (!func) 835 return -ENOTTY; 836 837 return func(vdev, index, start, count, flags, data); 838 } 839