1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO PCI interrupt handling 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/device.h> 14 #include <linux/interrupt.h> 15 #include <linux/eventfd.h> 16 #include <linux/msi.h> 17 #include <linux/pci.h> 18 #include <linux/file.h> 19 #include <linux/vfio.h> 20 #include <linux/wait.h> 21 #include <linux/slab.h> 22 23 #include "vfio_pci_priv.h" 24 25 struct vfio_pci_irq_ctx { 26 struct eventfd_ctx *trigger; 27 struct virqfd *unmask; 28 struct virqfd *mask; 29 char *name; 30 bool masked; 31 struct irq_bypass_producer producer; 32 }; 33 34 static bool irq_is(struct vfio_pci_core_device *vdev, int type) 35 { 36 return vdev->irq_type == type; 37 } 38 39 static bool is_intx(struct vfio_pci_core_device *vdev) 40 { 41 return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX; 42 } 43 44 static bool is_irq_none(struct vfio_pci_core_device *vdev) 45 { 46 return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX || 47 vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX || 48 vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX); 49 } 50 51 static 52 struct vfio_pci_irq_ctx *vfio_irq_ctx_get(struct vfio_pci_core_device *vdev, 53 unsigned long index) 54 { 55 return xa_load(&vdev->ctx, index); 56 } 57 58 static void vfio_irq_ctx_free(struct vfio_pci_core_device *vdev, 59 struct vfio_pci_irq_ctx *ctx, unsigned long index) 60 { 61 xa_erase(&vdev->ctx, index); 62 kfree(ctx); 63 } 64 65 static struct vfio_pci_irq_ctx * 66 vfio_irq_ctx_alloc(struct vfio_pci_core_device *vdev, unsigned long index) 67 { 68 struct vfio_pci_irq_ctx *ctx; 69 int ret; 70 71 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); 72 if (!ctx) 73 return NULL; 74 75 ret = xa_insert(&vdev->ctx, index, ctx, GFP_KERNEL_ACCOUNT); 76 if (ret) { 77 kfree(ctx); 78 return NULL; 79 } 80 81 return ctx; 82 } 83 84 /* 85 * INTx 86 */ 87 static void vfio_send_intx_eventfd(void *opaque, void *unused) 88 { 89 struct vfio_pci_core_device *vdev = opaque; 90 91 if (likely(is_intx(vdev) && !vdev->virq_disabled)) { 92 struct vfio_pci_irq_ctx *ctx; 93 94 ctx = vfio_irq_ctx_get(vdev, 0); 95 if (WARN_ON_ONCE(!ctx)) 96 return; 97 eventfd_signal(ctx->trigger, 1); 98 } 99 } 100 101 /* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */ 102 static bool __vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) 103 { 104 struct pci_dev *pdev = vdev->pdev; 105 struct vfio_pci_irq_ctx *ctx; 106 unsigned long flags; 107 bool masked_changed = false; 108 109 lockdep_assert_held(&vdev->igate); 110 111 spin_lock_irqsave(&vdev->irqlock, flags); 112 113 /* 114 * Masking can come from interrupt, ioctl, or config space 115 * via INTx disable. The latter means this can get called 116 * even when not using intx delivery. In this case, just 117 * try to have the physical bit follow the virtual bit. 118 */ 119 if (unlikely(!is_intx(vdev))) { 120 if (vdev->pci_2_3) 121 pci_intx(pdev, 0); 122 goto out_unlock; 123 } 124 125 ctx = vfio_irq_ctx_get(vdev, 0); 126 if (WARN_ON_ONCE(!ctx)) 127 goto out_unlock; 128 129 if (!ctx->masked) { 130 /* 131 * Can't use check_and_mask here because we always want to 132 * mask, not just when something is pending. 133 */ 134 if (vdev->pci_2_3) 135 pci_intx(pdev, 0); 136 else 137 disable_irq_nosync(pdev->irq); 138 139 ctx->masked = true; 140 masked_changed = true; 141 } 142 143 out_unlock: 144 spin_unlock_irqrestore(&vdev->irqlock, flags); 145 return masked_changed; 146 } 147 148 bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) 149 { 150 bool mask_changed; 151 152 mutex_lock(&vdev->igate); 153 mask_changed = __vfio_pci_intx_mask(vdev); 154 mutex_unlock(&vdev->igate); 155 156 return mask_changed; 157 } 158 159 /* 160 * If this is triggered by an eventfd, we can't call eventfd_signal 161 * or else we'll deadlock on the eventfd wait queue. Return >0 when 162 * a signal is necessary, which can then be handled via a work queue 163 * or directly depending on the caller. 164 */ 165 static int vfio_pci_intx_unmask_handler(void *opaque, void *unused) 166 { 167 struct vfio_pci_core_device *vdev = opaque; 168 struct pci_dev *pdev = vdev->pdev; 169 struct vfio_pci_irq_ctx *ctx; 170 unsigned long flags; 171 int ret = 0; 172 173 spin_lock_irqsave(&vdev->irqlock, flags); 174 175 /* 176 * Unmasking comes from ioctl or config, so again, have the 177 * physical bit follow the virtual even when not using INTx. 178 */ 179 if (unlikely(!is_intx(vdev))) { 180 if (vdev->pci_2_3) 181 pci_intx(pdev, 1); 182 goto out_unlock; 183 } 184 185 ctx = vfio_irq_ctx_get(vdev, 0); 186 if (WARN_ON_ONCE(!ctx)) 187 goto out_unlock; 188 189 if (ctx->masked && !vdev->virq_disabled) { 190 /* 191 * A pending interrupt here would immediately trigger, 192 * but we can avoid that overhead by just re-sending 193 * the interrupt to the user. 194 */ 195 if (vdev->pci_2_3) { 196 if (!pci_check_and_unmask_intx(pdev)) 197 ret = 1; 198 } else 199 enable_irq(pdev->irq); 200 201 ctx->masked = (ret > 0); 202 } 203 204 out_unlock: 205 spin_unlock_irqrestore(&vdev->irqlock, flags); 206 207 return ret; 208 } 209 210 static void __vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) 211 { 212 lockdep_assert_held(&vdev->igate); 213 214 if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0) 215 vfio_send_intx_eventfd(vdev, NULL); 216 } 217 218 void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) 219 { 220 mutex_lock(&vdev->igate); 221 __vfio_pci_intx_unmask(vdev); 222 mutex_unlock(&vdev->igate); 223 } 224 225 static irqreturn_t vfio_intx_handler(int irq, void *dev_id) 226 { 227 struct vfio_pci_core_device *vdev = dev_id; 228 struct vfio_pci_irq_ctx *ctx; 229 unsigned long flags; 230 int ret = IRQ_NONE; 231 232 ctx = vfio_irq_ctx_get(vdev, 0); 233 if (WARN_ON_ONCE(!ctx)) 234 return ret; 235 236 spin_lock_irqsave(&vdev->irqlock, flags); 237 238 if (!vdev->pci_2_3) { 239 disable_irq_nosync(vdev->pdev->irq); 240 ctx->masked = true; 241 ret = IRQ_HANDLED; 242 } else if (!ctx->masked && /* may be shared */ 243 pci_check_and_mask_intx(vdev->pdev)) { 244 ctx->masked = true; 245 ret = IRQ_HANDLED; 246 } 247 248 spin_unlock_irqrestore(&vdev->irqlock, flags); 249 250 if (ret == IRQ_HANDLED) 251 vfio_send_intx_eventfd(vdev, NULL); 252 253 return ret; 254 } 255 256 static int vfio_intx_enable(struct vfio_pci_core_device *vdev) 257 { 258 struct vfio_pci_irq_ctx *ctx; 259 260 if (!is_irq_none(vdev)) 261 return -EINVAL; 262 263 if (!vdev->pdev->irq) 264 return -ENODEV; 265 266 ctx = vfio_irq_ctx_alloc(vdev, 0); 267 if (!ctx) 268 return -ENOMEM; 269 270 /* 271 * If the virtual interrupt is masked, restore it. Devices 272 * supporting DisINTx can be masked at the hardware level 273 * here, non-PCI-2.3 devices will have to wait until the 274 * interrupt is enabled. 275 */ 276 ctx->masked = vdev->virq_disabled; 277 if (vdev->pci_2_3) 278 pci_intx(vdev->pdev, !ctx->masked); 279 280 vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; 281 282 return 0; 283 } 284 285 static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev, int fd) 286 { 287 struct pci_dev *pdev = vdev->pdev; 288 unsigned long irqflags = IRQF_SHARED; 289 struct vfio_pci_irq_ctx *ctx; 290 struct eventfd_ctx *trigger; 291 unsigned long flags; 292 int ret; 293 294 ctx = vfio_irq_ctx_get(vdev, 0); 295 if (WARN_ON_ONCE(!ctx)) 296 return -EINVAL; 297 298 if (ctx->trigger) { 299 free_irq(pdev->irq, vdev); 300 kfree(ctx->name); 301 eventfd_ctx_put(ctx->trigger); 302 ctx->trigger = NULL; 303 } 304 305 if (fd < 0) /* Disable only */ 306 return 0; 307 308 ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-intx(%s)", 309 pci_name(pdev)); 310 if (!ctx->name) 311 return -ENOMEM; 312 313 trigger = eventfd_ctx_fdget(fd); 314 if (IS_ERR(trigger)) { 315 kfree(ctx->name); 316 return PTR_ERR(trigger); 317 } 318 319 ctx->trigger = trigger; 320 321 /* 322 * Devices without DisINTx support require an exclusive interrupt, 323 * IRQ masking is performed at the IRQ chip. The masked status is 324 * protected by vdev->irqlock. Setup the IRQ without auto-enable and 325 * unmask as necessary below under lock. DisINTx is unmodified by 326 * the IRQ configuration and may therefore use auto-enable. 327 */ 328 if (!vdev->pci_2_3) 329 irqflags = IRQF_NO_AUTOEN; 330 331 ret = request_irq(pdev->irq, vfio_intx_handler, 332 irqflags, ctx->name, vdev); 333 if (ret) { 334 ctx->trigger = NULL; 335 kfree(ctx->name); 336 eventfd_ctx_put(trigger); 337 return ret; 338 } 339 340 spin_lock_irqsave(&vdev->irqlock, flags); 341 if (!vdev->pci_2_3 && !ctx->masked) 342 enable_irq(pdev->irq); 343 spin_unlock_irqrestore(&vdev->irqlock, flags); 344 345 return 0; 346 } 347 348 static void vfio_intx_disable(struct vfio_pci_core_device *vdev) 349 { 350 struct vfio_pci_irq_ctx *ctx; 351 352 ctx = vfio_irq_ctx_get(vdev, 0); 353 WARN_ON_ONCE(!ctx); 354 if (ctx) { 355 vfio_virqfd_disable(&ctx->unmask); 356 vfio_virqfd_disable(&ctx->mask); 357 } 358 vfio_intx_set_signal(vdev, -1); 359 vdev->irq_type = VFIO_PCI_NUM_IRQS; 360 vfio_irq_ctx_free(vdev, ctx, 0); 361 } 362 363 /* 364 * MSI/MSI-X 365 */ 366 static irqreturn_t vfio_msihandler(int irq, void *arg) 367 { 368 struct eventfd_ctx *trigger = arg; 369 370 eventfd_signal(trigger, 1); 371 return IRQ_HANDLED; 372 } 373 374 static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix) 375 { 376 struct pci_dev *pdev = vdev->pdev; 377 unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; 378 int ret; 379 u16 cmd; 380 381 if (!is_irq_none(vdev)) 382 return -EINVAL; 383 384 /* return the number of supported vectors if we can't get all: */ 385 cmd = vfio_pci_memory_lock_and_enable(vdev); 386 ret = pci_alloc_irq_vectors(pdev, 1, nvec, flag); 387 if (ret < nvec) { 388 if (ret > 0) 389 pci_free_irq_vectors(pdev); 390 vfio_pci_memory_unlock_and_restore(vdev, cmd); 391 return ret; 392 } 393 vfio_pci_memory_unlock_and_restore(vdev, cmd); 394 395 vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : 396 VFIO_PCI_MSI_IRQ_INDEX; 397 398 if (!msix) { 399 /* 400 * Compute the virtual hardware field for max msi vectors - 401 * it is the log base 2 of the number of vectors. 402 */ 403 vdev->msi_qmax = fls(nvec * 2 - 1) - 1; 404 } 405 406 return 0; 407 } 408 409 /* 410 * vfio_msi_alloc_irq() returns the Linux IRQ number of an MSI or MSI-X device 411 * interrupt vector. If a Linux IRQ number is not available then a new 412 * interrupt is allocated if dynamic MSI-X is supported. 413 * 414 * Where is vfio_msi_free_irq()? Allocated interrupts are maintained, 415 * essentially forming a cache that subsequent allocations can draw from. 416 * Interrupts are freed using pci_free_irq_vectors() when MSI/MSI-X is 417 * disabled. 418 */ 419 static int vfio_msi_alloc_irq(struct vfio_pci_core_device *vdev, 420 unsigned int vector, bool msix) 421 { 422 struct pci_dev *pdev = vdev->pdev; 423 struct msi_map map; 424 int irq; 425 u16 cmd; 426 427 irq = pci_irq_vector(pdev, vector); 428 if (WARN_ON_ONCE(irq == 0)) 429 return -EINVAL; 430 if (irq > 0 || !msix || !vdev->has_dyn_msix) 431 return irq; 432 433 cmd = vfio_pci_memory_lock_and_enable(vdev); 434 map = pci_msix_alloc_irq_at(pdev, vector, NULL); 435 vfio_pci_memory_unlock_and_restore(vdev, cmd); 436 437 return map.index < 0 ? map.index : map.virq; 438 } 439 440 static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev, 441 unsigned int vector, int fd, bool msix) 442 { 443 struct pci_dev *pdev = vdev->pdev; 444 struct vfio_pci_irq_ctx *ctx; 445 struct eventfd_ctx *trigger; 446 int irq = -EINVAL, ret; 447 u16 cmd; 448 449 ctx = vfio_irq_ctx_get(vdev, vector); 450 451 if (ctx) { 452 irq_bypass_unregister_producer(&ctx->producer); 453 irq = pci_irq_vector(pdev, vector); 454 cmd = vfio_pci_memory_lock_and_enable(vdev); 455 free_irq(irq, ctx->trigger); 456 vfio_pci_memory_unlock_and_restore(vdev, cmd); 457 /* Interrupt stays allocated, will be freed at MSI-X disable. */ 458 kfree(ctx->name); 459 eventfd_ctx_put(ctx->trigger); 460 vfio_irq_ctx_free(vdev, ctx, vector); 461 } 462 463 if (fd < 0) 464 return 0; 465 466 if (irq == -EINVAL) { 467 /* Interrupt stays allocated, will be freed at MSI-X disable. */ 468 irq = vfio_msi_alloc_irq(vdev, vector, msix); 469 if (irq < 0) 470 return irq; 471 } 472 473 ctx = vfio_irq_ctx_alloc(vdev, vector); 474 if (!ctx) 475 return -ENOMEM; 476 477 ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-msi%s[%d](%s)", 478 msix ? "x" : "", vector, pci_name(pdev)); 479 if (!ctx->name) { 480 ret = -ENOMEM; 481 goto out_free_ctx; 482 } 483 484 trigger = eventfd_ctx_fdget(fd); 485 if (IS_ERR(trigger)) { 486 ret = PTR_ERR(trigger); 487 goto out_free_name; 488 } 489 490 /* 491 * If the vector was previously allocated, refresh the on-device 492 * message data before enabling in case it had been cleared or 493 * corrupted (e.g. due to backdoor resets) since writing. 494 */ 495 cmd = vfio_pci_memory_lock_and_enable(vdev); 496 if (msix) { 497 struct msi_msg msg; 498 499 get_cached_msi_msg(irq, &msg); 500 pci_write_msi_msg(irq, &msg); 501 } 502 503 ret = request_irq(irq, vfio_msihandler, 0, ctx->name, trigger); 504 vfio_pci_memory_unlock_and_restore(vdev, cmd); 505 if (ret) 506 goto out_put_eventfd_ctx; 507 508 ctx->producer.token = trigger; 509 ctx->producer.irq = irq; 510 ret = irq_bypass_register_producer(&ctx->producer); 511 if (unlikely(ret)) { 512 dev_info(&pdev->dev, 513 "irq bypass producer (token %p) registration fails: %d\n", 514 ctx->producer.token, ret); 515 516 ctx->producer.token = NULL; 517 } 518 ctx->trigger = trigger; 519 520 return 0; 521 522 out_put_eventfd_ctx: 523 eventfd_ctx_put(trigger); 524 out_free_name: 525 kfree(ctx->name); 526 out_free_ctx: 527 vfio_irq_ctx_free(vdev, ctx, vector); 528 return ret; 529 } 530 531 static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start, 532 unsigned count, int32_t *fds, bool msix) 533 { 534 unsigned int i, j; 535 int ret = 0; 536 537 for (i = 0, j = start; i < count && !ret; i++, j++) { 538 int fd = fds ? fds[i] : -1; 539 ret = vfio_msi_set_vector_signal(vdev, j, fd, msix); 540 } 541 542 if (ret) { 543 for (i = start; i < j; i++) 544 vfio_msi_set_vector_signal(vdev, i, -1, msix); 545 } 546 547 return ret; 548 } 549 550 static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix) 551 { 552 struct pci_dev *pdev = vdev->pdev; 553 struct vfio_pci_irq_ctx *ctx; 554 unsigned long i; 555 u16 cmd; 556 557 xa_for_each(&vdev->ctx, i, ctx) { 558 vfio_virqfd_disable(&ctx->unmask); 559 vfio_virqfd_disable(&ctx->mask); 560 vfio_msi_set_vector_signal(vdev, i, -1, msix); 561 } 562 563 cmd = vfio_pci_memory_lock_and_enable(vdev); 564 pci_free_irq_vectors(pdev); 565 vfio_pci_memory_unlock_and_restore(vdev, cmd); 566 567 /* 568 * Both disable paths above use pci_intx_for_msi() to clear DisINTx 569 * via their shutdown paths. Restore for NoINTx devices. 570 */ 571 if (vdev->nointx) 572 pci_intx(pdev, 0); 573 574 vdev->irq_type = VFIO_PCI_NUM_IRQS; 575 } 576 577 /* 578 * IOCTL support 579 */ 580 static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev, 581 unsigned index, unsigned start, 582 unsigned count, uint32_t flags, void *data) 583 { 584 if (!is_intx(vdev) || start != 0 || count != 1) 585 return -EINVAL; 586 587 if (flags & VFIO_IRQ_SET_DATA_NONE) { 588 __vfio_pci_intx_unmask(vdev); 589 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 590 uint8_t unmask = *(uint8_t *)data; 591 if (unmask) 592 __vfio_pci_intx_unmask(vdev); 593 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 594 struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, 0); 595 int32_t fd = *(int32_t *)data; 596 597 if (WARN_ON_ONCE(!ctx)) 598 return -EINVAL; 599 if (fd >= 0) 600 return vfio_virqfd_enable((void *) vdev, 601 vfio_pci_intx_unmask_handler, 602 vfio_send_intx_eventfd, NULL, 603 &ctx->unmask, fd); 604 605 vfio_virqfd_disable(&ctx->unmask); 606 } 607 608 return 0; 609 } 610 611 static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev, 612 unsigned index, unsigned start, 613 unsigned count, uint32_t flags, void *data) 614 { 615 if (!is_intx(vdev) || start != 0 || count != 1) 616 return -EINVAL; 617 618 if (flags & VFIO_IRQ_SET_DATA_NONE) { 619 __vfio_pci_intx_mask(vdev); 620 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 621 uint8_t mask = *(uint8_t *)data; 622 if (mask) 623 __vfio_pci_intx_mask(vdev); 624 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 625 return -ENOTTY; /* XXX implement me */ 626 } 627 628 return 0; 629 } 630 631 static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev, 632 unsigned index, unsigned start, 633 unsigned count, uint32_t flags, void *data) 634 { 635 if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 636 vfio_intx_disable(vdev); 637 return 0; 638 } 639 640 if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) 641 return -EINVAL; 642 643 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 644 int32_t fd = *(int32_t *)data; 645 int ret; 646 647 if (is_intx(vdev)) 648 return vfio_intx_set_signal(vdev, fd); 649 650 ret = vfio_intx_enable(vdev); 651 if (ret) 652 return ret; 653 654 ret = vfio_intx_set_signal(vdev, fd); 655 if (ret) 656 vfio_intx_disable(vdev); 657 658 return ret; 659 } 660 661 if (!is_intx(vdev)) 662 return -EINVAL; 663 664 if (flags & VFIO_IRQ_SET_DATA_NONE) { 665 vfio_send_intx_eventfd(vdev, NULL); 666 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 667 uint8_t trigger = *(uint8_t *)data; 668 if (trigger) 669 vfio_send_intx_eventfd(vdev, NULL); 670 } 671 return 0; 672 } 673 674 static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, 675 unsigned index, unsigned start, 676 unsigned count, uint32_t flags, void *data) 677 { 678 struct vfio_pci_irq_ctx *ctx; 679 unsigned int i; 680 bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; 681 682 if (irq_is(vdev, index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 683 vfio_msi_disable(vdev, msix); 684 return 0; 685 } 686 687 if (!(irq_is(vdev, index) || is_irq_none(vdev))) 688 return -EINVAL; 689 690 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 691 int32_t *fds = data; 692 int ret; 693 694 if (vdev->irq_type == index) 695 return vfio_msi_set_block(vdev, start, count, 696 fds, msix); 697 698 ret = vfio_msi_enable(vdev, start + count, msix); 699 if (ret) 700 return ret; 701 702 ret = vfio_msi_set_block(vdev, start, count, fds, msix); 703 if (ret) 704 vfio_msi_disable(vdev, msix); 705 706 return ret; 707 } 708 709 if (!irq_is(vdev, index)) 710 return -EINVAL; 711 712 for (i = start; i < start + count; i++) { 713 ctx = vfio_irq_ctx_get(vdev, i); 714 if (!ctx) 715 continue; 716 if (flags & VFIO_IRQ_SET_DATA_NONE) { 717 eventfd_signal(ctx->trigger, 1); 718 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 719 uint8_t *bools = data; 720 if (bools[i - start]) 721 eventfd_signal(ctx->trigger, 1); 722 } 723 } 724 return 0; 725 } 726 727 static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, 728 unsigned int count, uint32_t flags, 729 void *data) 730 { 731 /* DATA_NONE/DATA_BOOL enables loopback testing */ 732 if (flags & VFIO_IRQ_SET_DATA_NONE) { 733 if (*ctx) { 734 if (count) { 735 eventfd_signal(*ctx, 1); 736 } else { 737 eventfd_ctx_put(*ctx); 738 *ctx = NULL; 739 } 740 return 0; 741 } 742 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 743 uint8_t trigger; 744 745 if (!count) 746 return -EINVAL; 747 748 trigger = *(uint8_t *)data; 749 if (trigger && *ctx) 750 eventfd_signal(*ctx, 1); 751 752 return 0; 753 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 754 int32_t fd; 755 756 if (!count) 757 return -EINVAL; 758 759 fd = *(int32_t *)data; 760 if (fd == -1) { 761 if (*ctx) 762 eventfd_ctx_put(*ctx); 763 *ctx = NULL; 764 } else if (fd >= 0) { 765 struct eventfd_ctx *efdctx; 766 767 efdctx = eventfd_ctx_fdget(fd); 768 if (IS_ERR(efdctx)) 769 return PTR_ERR(efdctx); 770 771 if (*ctx) 772 eventfd_ctx_put(*ctx); 773 774 *ctx = efdctx; 775 } 776 return 0; 777 } 778 779 return -EINVAL; 780 } 781 782 static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev, 783 unsigned index, unsigned start, 784 unsigned count, uint32_t flags, void *data) 785 { 786 if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1) 787 return -EINVAL; 788 789 return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger, 790 count, flags, data); 791 } 792 793 static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev, 794 unsigned index, unsigned start, 795 unsigned count, uint32_t flags, void *data) 796 { 797 if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1) 798 return -EINVAL; 799 800 return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger, 801 count, flags, data); 802 } 803 804 int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, 805 unsigned index, unsigned start, unsigned count, 806 void *data) 807 { 808 int (*func)(struct vfio_pci_core_device *vdev, unsigned index, 809 unsigned start, unsigned count, uint32_t flags, 810 void *data) = NULL; 811 812 switch (index) { 813 case VFIO_PCI_INTX_IRQ_INDEX: 814 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 815 case VFIO_IRQ_SET_ACTION_MASK: 816 func = vfio_pci_set_intx_mask; 817 break; 818 case VFIO_IRQ_SET_ACTION_UNMASK: 819 func = vfio_pci_set_intx_unmask; 820 break; 821 case VFIO_IRQ_SET_ACTION_TRIGGER: 822 func = vfio_pci_set_intx_trigger; 823 break; 824 } 825 break; 826 case VFIO_PCI_MSI_IRQ_INDEX: 827 case VFIO_PCI_MSIX_IRQ_INDEX: 828 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 829 case VFIO_IRQ_SET_ACTION_MASK: 830 case VFIO_IRQ_SET_ACTION_UNMASK: 831 /* XXX Need masking support exported */ 832 break; 833 case VFIO_IRQ_SET_ACTION_TRIGGER: 834 func = vfio_pci_set_msi_trigger; 835 break; 836 } 837 break; 838 case VFIO_PCI_ERR_IRQ_INDEX: 839 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 840 case VFIO_IRQ_SET_ACTION_TRIGGER: 841 if (pci_is_pcie(vdev->pdev)) 842 func = vfio_pci_set_err_trigger; 843 break; 844 } 845 break; 846 case VFIO_PCI_REQ_IRQ_INDEX: 847 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 848 case VFIO_IRQ_SET_ACTION_TRIGGER: 849 func = vfio_pci_set_req_trigger; 850 break; 851 } 852 break; 853 } 854 855 if (!func) 856 return -ENOTTY; 857 858 return func(vdev, index, start, count, flags, data); 859 } 860