1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VFIO PCI interrupt handling 4 * 5 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 6 * Author: Alex Williamson <alex.williamson@redhat.com> 7 * 8 * Derived from original vfio: 9 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 10 * Author: Tom Lyon, pugs@cisco.com 11 */ 12 13 #include <linux/device.h> 14 #include <linux/interrupt.h> 15 #include <linux/eventfd.h> 16 #include <linux/msi.h> 17 #include <linux/pci.h> 18 #include <linux/file.h> 19 #include <linux/vfio.h> 20 #include <linux/wait.h> 21 #include <linux/slab.h> 22 23 #include "vfio_pci_priv.h" 24 25 struct vfio_pci_irq_ctx { 26 struct eventfd_ctx *trigger; 27 struct virqfd *unmask; 28 struct virqfd *mask; 29 char *name; 30 bool masked; 31 struct irq_bypass_producer producer; 32 }; 33 34 static bool irq_is(struct vfio_pci_core_device *vdev, int type) 35 { 36 return vdev->irq_type == type; 37 } 38 39 static bool is_intx(struct vfio_pci_core_device *vdev) 40 { 41 return vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX; 42 } 43 44 static bool is_irq_none(struct vfio_pci_core_device *vdev) 45 { 46 return !(vdev->irq_type == VFIO_PCI_INTX_IRQ_INDEX || 47 vdev->irq_type == VFIO_PCI_MSI_IRQ_INDEX || 48 vdev->irq_type == VFIO_PCI_MSIX_IRQ_INDEX); 49 } 50 51 static 52 struct vfio_pci_irq_ctx *vfio_irq_ctx_get(struct vfio_pci_core_device *vdev, 53 unsigned long index) 54 { 55 return xa_load(&vdev->ctx, index); 56 } 57 58 static void vfio_irq_ctx_free(struct vfio_pci_core_device *vdev, 59 struct vfio_pci_irq_ctx *ctx, unsigned long index) 60 { 61 xa_erase(&vdev->ctx, index); 62 kfree(ctx); 63 } 64 65 static struct vfio_pci_irq_ctx * 66 vfio_irq_ctx_alloc(struct vfio_pci_core_device *vdev, unsigned long index) 67 { 68 struct vfio_pci_irq_ctx *ctx; 69 int ret; 70 71 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); 72 if (!ctx) 73 return NULL; 74 75 ret = xa_insert(&vdev->ctx, index, ctx, GFP_KERNEL_ACCOUNT); 76 if (ret) { 77 kfree(ctx); 78 return NULL; 79 } 80 81 return ctx; 82 } 83 84 /* 85 * INTx 86 */ 87 static void vfio_send_intx_eventfd(void *opaque, void *unused) 88 { 89 struct vfio_pci_core_device *vdev = opaque; 90 91 if (likely(is_intx(vdev) && !vdev->virq_disabled)) { 92 struct vfio_pci_irq_ctx *ctx; 93 struct eventfd_ctx *trigger; 94 95 ctx = vfio_irq_ctx_get(vdev, 0); 96 if (WARN_ON_ONCE(!ctx)) 97 return; 98 99 trigger = READ_ONCE(ctx->trigger); 100 if (likely(trigger)) 101 eventfd_signal(trigger, 1); 102 } 103 } 104 105 /* Returns true if the INTx vfio_pci_irq_ctx.masked value is changed. */ 106 static bool __vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) 107 { 108 struct pci_dev *pdev = vdev->pdev; 109 struct vfio_pci_irq_ctx *ctx; 110 unsigned long flags; 111 bool masked_changed = false; 112 113 lockdep_assert_held(&vdev->igate); 114 115 spin_lock_irqsave(&vdev->irqlock, flags); 116 117 /* 118 * Masking can come from interrupt, ioctl, or config space 119 * via INTx disable. The latter means this can get called 120 * even when not using intx delivery. In this case, just 121 * try to have the physical bit follow the virtual bit. 122 */ 123 if (unlikely(!is_intx(vdev))) { 124 if (vdev->pci_2_3) 125 pci_intx(pdev, 0); 126 goto out_unlock; 127 } 128 129 ctx = vfio_irq_ctx_get(vdev, 0); 130 if (WARN_ON_ONCE(!ctx)) 131 goto out_unlock; 132 133 if (!ctx->masked) { 134 /* 135 * Can't use check_and_mask here because we always want to 136 * mask, not just when something is pending. 137 */ 138 if (vdev->pci_2_3) 139 pci_intx(pdev, 0); 140 else 141 disable_irq_nosync(pdev->irq); 142 143 ctx->masked = true; 144 masked_changed = true; 145 } 146 147 out_unlock: 148 spin_unlock_irqrestore(&vdev->irqlock, flags); 149 return masked_changed; 150 } 151 152 bool vfio_pci_intx_mask(struct vfio_pci_core_device *vdev) 153 { 154 bool mask_changed; 155 156 mutex_lock(&vdev->igate); 157 mask_changed = __vfio_pci_intx_mask(vdev); 158 mutex_unlock(&vdev->igate); 159 160 return mask_changed; 161 } 162 163 /* 164 * If this is triggered by an eventfd, we can't call eventfd_signal 165 * or else we'll deadlock on the eventfd wait queue. Return >0 when 166 * a signal is necessary, which can then be handled via a work queue 167 * or directly depending on the caller. 168 */ 169 static int vfio_pci_intx_unmask_handler(void *opaque, void *unused) 170 { 171 struct vfio_pci_core_device *vdev = opaque; 172 struct pci_dev *pdev = vdev->pdev; 173 struct vfio_pci_irq_ctx *ctx; 174 unsigned long flags; 175 int ret = 0; 176 177 spin_lock_irqsave(&vdev->irqlock, flags); 178 179 /* 180 * Unmasking comes from ioctl or config, so again, have the 181 * physical bit follow the virtual even when not using INTx. 182 */ 183 if (unlikely(!is_intx(vdev))) { 184 if (vdev->pci_2_3) 185 pci_intx(pdev, 1); 186 goto out_unlock; 187 } 188 189 ctx = vfio_irq_ctx_get(vdev, 0); 190 if (WARN_ON_ONCE(!ctx)) 191 goto out_unlock; 192 193 if (ctx->masked && !vdev->virq_disabled) { 194 /* 195 * A pending interrupt here would immediately trigger, 196 * but we can avoid that overhead by just re-sending 197 * the interrupt to the user. 198 */ 199 if (vdev->pci_2_3) { 200 if (!pci_check_and_unmask_intx(pdev)) 201 ret = 1; 202 } else 203 enable_irq(pdev->irq); 204 205 ctx->masked = (ret > 0); 206 } 207 208 out_unlock: 209 spin_unlock_irqrestore(&vdev->irqlock, flags); 210 211 return ret; 212 } 213 214 static void __vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) 215 { 216 lockdep_assert_held(&vdev->igate); 217 218 if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0) 219 vfio_send_intx_eventfd(vdev, NULL); 220 } 221 222 void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev) 223 { 224 mutex_lock(&vdev->igate); 225 __vfio_pci_intx_unmask(vdev); 226 mutex_unlock(&vdev->igate); 227 } 228 229 static irqreturn_t vfio_intx_handler(int irq, void *dev_id) 230 { 231 struct vfio_pci_core_device *vdev = dev_id; 232 struct vfio_pci_irq_ctx *ctx; 233 unsigned long flags; 234 int ret = IRQ_NONE; 235 236 ctx = vfio_irq_ctx_get(vdev, 0); 237 if (WARN_ON_ONCE(!ctx)) 238 return ret; 239 240 spin_lock_irqsave(&vdev->irqlock, flags); 241 242 if (!vdev->pci_2_3) { 243 disable_irq_nosync(vdev->pdev->irq); 244 ctx->masked = true; 245 ret = IRQ_HANDLED; 246 } else if (!ctx->masked && /* may be shared */ 247 pci_check_and_mask_intx(vdev->pdev)) { 248 ctx->masked = true; 249 ret = IRQ_HANDLED; 250 } 251 252 spin_unlock_irqrestore(&vdev->irqlock, flags); 253 254 if (ret == IRQ_HANDLED) 255 vfio_send_intx_eventfd(vdev, NULL); 256 257 return ret; 258 } 259 260 static int vfio_intx_enable(struct vfio_pci_core_device *vdev, 261 struct eventfd_ctx *trigger) 262 { 263 struct pci_dev *pdev = vdev->pdev; 264 struct vfio_pci_irq_ctx *ctx; 265 unsigned long irqflags; 266 char *name; 267 int ret; 268 269 if (!is_irq_none(vdev)) 270 return -EINVAL; 271 272 if (!pdev->irq) 273 return -ENODEV; 274 275 name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-intx(%s)", pci_name(pdev)); 276 if (!name) 277 return -ENOMEM; 278 279 ctx = vfio_irq_ctx_alloc(vdev, 0); 280 if (!ctx) 281 return -ENOMEM; 282 283 ctx->name = name; 284 ctx->trigger = trigger; 285 286 /* 287 * Fill the initial masked state based on virq_disabled. After 288 * enable, changing the DisINTx bit in vconfig directly changes INTx 289 * masking. igate prevents races during setup, once running masked 290 * is protected via irqlock. 291 * 292 * Devices supporting DisINTx also reflect the current mask state in 293 * the physical DisINTx bit, which is not affected during IRQ setup. 294 * 295 * Devices without DisINTx support require an exclusive interrupt. 296 * IRQ masking is performed at the IRQ chip. Again, igate protects 297 * against races during setup and IRQ handlers and irqfds are not 298 * yet active, therefore masked is stable and can be used to 299 * conditionally auto-enable the IRQ. 300 * 301 * irq_type must be stable while the IRQ handler is registered, 302 * therefore it must be set before request_irq(). 303 */ 304 ctx->masked = vdev->virq_disabled; 305 if (vdev->pci_2_3) { 306 pci_intx(pdev, !ctx->masked); 307 irqflags = IRQF_SHARED; 308 } else { 309 irqflags = ctx->masked ? IRQF_NO_AUTOEN : 0; 310 } 311 312 vdev->irq_type = VFIO_PCI_INTX_IRQ_INDEX; 313 314 ret = request_irq(pdev->irq, vfio_intx_handler, 315 irqflags, ctx->name, vdev); 316 if (ret) { 317 vdev->irq_type = VFIO_PCI_NUM_IRQS; 318 kfree(name); 319 vfio_irq_ctx_free(vdev, ctx, 0); 320 return ret; 321 } 322 323 return 0; 324 } 325 326 static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev, 327 struct eventfd_ctx *trigger) 328 { 329 struct pci_dev *pdev = vdev->pdev; 330 struct vfio_pci_irq_ctx *ctx; 331 struct eventfd_ctx *old; 332 333 ctx = vfio_irq_ctx_get(vdev, 0); 334 if (WARN_ON_ONCE(!ctx)) 335 return -EINVAL; 336 337 old = ctx->trigger; 338 339 WRITE_ONCE(ctx->trigger, trigger); 340 341 /* Releasing an old ctx requires synchronizing in-flight users */ 342 if (old) { 343 synchronize_irq(pdev->irq); 344 vfio_virqfd_flush_thread(&ctx->unmask); 345 eventfd_ctx_put(old); 346 } 347 348 return 0; 349 } 350 351 static void vfio_intx_disable(struct vfio_pci_core_device *vdev) 352 { 353 struct pci_dev *pdev = vdev->pdev; 354 struct vfio_pci_irq_ctx *ctx; 355 356 ctx = vfio_irq_ctx_get(vdev, 0); 357 WARN_ON_ONCE(!ctx); 358 if (ctx) { 359 vfio_virqfd_disable(&ctx->unmask); 360 vfio_virqfd_disable(&ctx->mask); 361 free_irq(pdev->irq, vdev); 362 if (ctx->trigger) 363 eventfd_ctx_put(ctx->trigger); 364 kfree(ctx->name); 365 vfio_irq_ctx_free(vdev, ctx, 0); 366 } 367 vdev->irq_type = VFIO_PCI_NUM_IRQS; 368 } 369 370 /* 371 * MSI/MSI-X 372 */ 373 static irqreturn_t vfio_msihandler(int irq, void *arg) 374 { 375 struct eventfd_ctx *trigger = arg; 376 377 eventfd_signal(trigger, 1); 378 return IRQ_HANDLED; 379 } 380 381 static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix) 382 { 383 struct pci_dev *pdev = vdev->pdev; 384 unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI; 385 int ret; 386 u16 cmd; 387 388 if (!is_irq_none(vdev)) 389 return -EINVAL; 390 391 /* return the number of supported vectors if we can't get all: */ 392 cmd = vfio_pci_memory_lock_and_enable(vdev); 393 ret = pci_alloc_irq_vectors(pdev, 1, nvec, flag); 394 if (ret < nvec) { 395 if (ret > 0) 396 pci_free_irq_vectors(pdev); 397 vfio_pci_memory_unlock_and_restore(vdev, cmd); 398 return ret; 399 } 400 vfio_pci_memory_unlock_and_restore(vdev, cmd); 401 402 vdev->irq_type = msix ? VFIO_PCI_MSIX_IRQ_INDEX : 403 VFIO_PCI_MSI_IRQ_INDEX; 404 405 if (!msix) { 406 /* 407 * Compute the virtual hardware field for max msi vectors - 408 * it is the log base 2 of the number of vectors. 409 */ 410 vdev->msi_qmax = fls(nvec * 2 - 1) - 1; 411 } 412 413 return 0; 414 } 415 416 /* 417 * vfio_msi_alloc_irq() returns the Linux IRQ number of an MSI or MSI-X device 418 * interrupt vector. If a Linux IRQ number is not available then a new 419 * interrupt is allocated if dynamic MSI-X is supported. 420 * 421 * Where is vfio_msi_free_irq()? Allocated interrupts are maintained, 422 * essentially forming a cache that subsequent allocations can draw from. 423 * Interrupts are freed using pci_free_irq_vectors() when MSI/MSI-X is 424 * disabled. 425 */ 426 static int vfio_msi_alloc_irq(struct vfio_pci_core_device *vdev, 427 unsigned int vector, bool msix) 428 { 429 struct pci_dev *pdev = vdev->pdev; 430 struct msi_map map; 431 int irq; 432 u16 cmd; 433 434 irq = pci_irq_vector(pdev, vector); 435 if (WARN_ON_ONCE(irq == 0)) 436 return -EINVAL; 437 if (irq > 0 || !msix || !vdev->has_dyn_msix) 438 return irq; 439 440 cmd = vfio_pci_memory_lock_and_enable(vdev); 441 map = pci_msix_alloc_irq_at(pdev, vector, NULL); 442 vfio_pci_memory_unlock_and_restore(vdev, cmd); 443 444 return map.index < 0 ? map.index : map.virq; 445 } 446 447 static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev, 448 unsigned int vector, int fd, bool msix) 449 { 450 struct pci_dev *pdev = vdev->pdev; 451 struct vfio_pci_irq_ctx *ctx; 452 struct eventfd_ctx *trigger; 453 int irq = -EINVAL, ret; 454 u16 cmd; 455 456 ctx = vfio_irq_ctx_get(vdev, vector); 457 458 if (ctx) { 459 irq_bypass_unregister_producer(&ctx->producer); 460 irq = pci_irq_vector(pdev, vector); 461 cmd = vfio_pci_memory_lock_and_enable(vdev); 462 free_irq(irq, ctx->trigger); 463 vfio_pci_memory_unlock_and_restore(vdev, cmd); 464 /* Interrupt stays allocated, will be freed at MSI-X disable. */ 465 kfree(ctx->name); 466 eventfd_ctx_put(ctx->trigger); 467 vfio_irq_ctx_free(vdev, ctx, vector); 468 } 469 470 if (fd < 0) 471 return 0; 472 473 if (irq == -EINVAL) { 474 /* Interrupt stays allocated, will be freed at MSI-X disable. */ 475 irq = vfio_msi_alloc_irq(vdev, vector, msix); 476 if (irq < 0) 477 return irq; 478 } 479 480 ctx = vfio_irq_ctx_alloc(vdev, vector); 481 if (!ctx) 482 return -ENOMEM; 483 484 ctx->name = kasprintf(GFP_KERNEL_ACCOUNT, "vfio-msi%s[%d](%s)", 485 msix ? "x" : "", vector, pci_name(pdev)); 486 if (!ctx->name) { 487 ret = -ENOMEM; 488 goto out_free_ctx; 489 } 490 491 trigger = eventfd_ctx_fdget(fd); 492 if (IS_ERR(trigger)) { 493 ret = PTR_ERR(trigger); 494 goto out_free_name; 495 } 496 497 /* 498 * If the vector was previously allocated, refresh the on-device 499 * message data before enabling in case it had been cleared or 500 * corrupted (e.g. due to backdoor resets) since writing. 501 */ 502 cmd = vfio_pci_memory_lock_and_enable(vdev); 503 if (msix) { 504 struct msi_msg msg; 505 506 get_cached_msi_msg(irq, &msg); 507 pci_write_msi_msg(irq, &msg); 508 } 509 510 ret = request_irq(irq, vfio_msihandler, 0, ctx->name, trigger); 511 vfio_pci_memory_unlock_and_restore(vdev, cmd); 512 if (ret) 513 goto out_put_eventfd_ctx; 514 515 ctx->producer.token = trigger; 516 ctx->producer.irq = irq; 517 ret = irq_bypass_register_producer(&ctx->producer); 518 if (unlikely(ret)) { 519 dev_info(&pdev->dev, 520 "irq bypass producer (token %p) registration fails: %d\n", 521 ctx->producer.token, ret); 522 523 ctx->producer.token = NULL; 524 } 525 ctx->trigger = trigger; 526 527 return 0; 528 529 out_put_eventfd_ctx: 530 eventfd_ctx_put(trigger); 531 out_free_name: 532 kfree(ctx->name); 533 out_free_ctx: 534 vfio_irq_ctx_free(vdev, ctx, vector); 535 return ret; 536 } 537 538 static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start, 539 unsigned count, int32_t *fds, bool msix) 540 { 541 unsigned int i, j; 542 int ret = 0; 543 544 for (i = 0, j = start; i < count && !ret; i++, j++) { 545 int fd = fds ? fds[i] : -1; 546 ret = vfio_msi_set_vector_signal(vdev, j, fd, msix); 547 } 548 549 if (ret) { 550 for (i = start; i < j; i++) 551 vfio_msi_set_vector_signal(vdev, i, -1, msix); 552 } 553 554 return ret; 555 } 556 557 static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix) 558 { 559 struct pci_dev *pdev = vdev->pdev; 560 struct vfio_pci_irq_ctx *ctx; 561 unsigned long i; 562 u16 cmd; 563 564 xa_for_each(&vdev->ctx, i, ctx) { 565 vfio_virqfd_disable(&ctx->unmask); 566 vfio_virqfd_disable(&ctx->mask); 567 vfio_msi_set_vector_signal(vdev, i, -1, msix); 568 } 569 570 cmd = vfio_pci_memory_lock_and_enable(vdev); 571 pci_free_irq_vectors(pdev); 572 vfio_pci_memory_unlock_and_restore(vdev, cmd); 573 574 /* 575 * Both disable paths above use pci_intx_for_msi() to clear DisINTx 576 * via their shutdown paths. Restore for NoINTx devices. 577 */ 578 if (vdev->nointx) 579 pci_intx(pdev, 0); 580 581 vdev->irq_type = VFIO_PCI_NUM_IRQS; 582 } 583 584 /* 585 * IOCTL support 586 */ 587 static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev, 588 unsigned index, unsigned start, 589 unsigned count, uint32_t flags, void *data) 590 { 591 if (!is_intx(vdev) || start != 0 || count != 1) 592 return -EINVAL; 593 594 if (flags & VFIO_IRQ_SET_DATA_NONE) { 595 __vfio_pci_intx_unmask(vdev); 596 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 597 uint8_t unmask = *(uint8_t *)data; 598 if (unmask) 599 __vfio_pci_intx_unmask(vdev); 600 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 601 struct vfio_pci_irq_ctx *ctx = vfio_irq_ctx_get(vdev, 0); 602 int32_t fd = *(int32_t *)data; 603 604 if (WARN_ON_ONCE(!ctx)) 605 return -EINVAL; 606 if (fd >= 0) 607 return vfio_virqfd_enable((void *) vdev, 608 vfio_pci_intx_unmask_handler, 609 vfio_send_intx_eventfd, NULL, 610 &ctx->unmask, fd); 611 612 vfio_virqfd_disable(&ctx->unmask); 613 } 614 615 return 0; 616 } 617 618 static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev, 619 unsigned index, unsigned start, 620 unsigned count, uint32_t flags, void *data) 621 { 622 if (!is_intx(vdev) || start != 0 || count != 1) 623 return -EINVAL; 624 625 if (flags & VFIO_IRQ_SET_DATA_NONE) { 626 __vfio_pci_intx_mask(vdev); 627 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 628 uint8_t mask = *(uint8_t *)data; 629 if (mask) 630 __vfio_pci_intx_mask(vdev); 631 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 632 return -ENOTTY; /* XXX implement me */ 633 } 634 635 return 0; 636 } 637 638 static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev, 639 unsigned index, unsigned start, 640 unsigned count, uint32_t flags, void *data) 641 { 642 if (is_intx(vdev) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 643 vfio_intx_disable(vdev); 644 return 0; 645 } 646 647 if (!(is_intx(vdev) || is_irq_none(vdev)) || start != 0 || count != 1) 648 return -EINVAL; 649 650 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 651 struct eventfd_ctx *trigger = NULL; 652 int32_t fd = *(int32_t *)data; 653 int ret; 654 655 if (fd >= 0) { 656 trigger = eventfd_ctx_fdget(fd); 657 if (IS_ERR(trigger)) 658 return PTR_ERR(trigger); 659 } 660 661 if (is_intx(vdev)) 662 ret = vfio_intx_set_signal(vdev, trigger); 663 else 664 ret = vfio_intx_enable(vdev, trigger); 665 666 if (ret && trigger) 667 eventfd_ctx_put(trigger); 668 669 return ret; 670 } 671 672 if (!is_intx(vdev)) 673 return -EINVAL; 674 675 if (flags & VFIO_IRQ_SET_DATA_NONE) { 676 vfio_send_intx_eventfd(vdev, NULL); 677 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 678 uint8_t trigger = *(uint8_t *)data; 679 if (trigger) 680 vfio_send_intx_eventfd(vdev, NULL); 681 } 682 return 0; 683 } 684 685 static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev, 686 unsigned index, unsigned start, 687 unsigned count, uint32_t flags, void *data) 688 { 689 struct vfio_pci_irq_ctx *ctx; 690 unsigned int i; 691 bool msix = (index == VFIO_PCI_MSIX_IRQ_INDEX) ? true : false; 692 693 if (irq_is(vdev, index) && !count && (flags & VFIO_IRQ_SET_DATA_NONE)) { 694 vfio_msi_disable(vdev, msix); 695 return 0; 696 } 697 698 if (!(irq_is(vdev, index) || is_irq_none(vdev))) 699 return -EINVAL; 700 701 if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 702 int32_t *fds = data; 703 int ret; 704 705 if (vdev->irq_type == index) 706 return vfio_msi_set_block(vdev, start, count, 707 fds, msix); 708 709 ret = vfio_msi_enable(vdev, start + count, msix); 710 if (ret) 711 return ret; 712 713 ret = vfio_msi_set_block(vdev, start, count, fds, msix); 714 if (ret) 715 vfio_msi_disable(vdev, msix); 716 717 return ret; 718 } 719 720 if (!irq_is(vdev, index)) 721 return -EINVAL; 722 723 for (i = start; i < start + count; i++) { 724 ctx = vfio_irq_ctx_get(vdev, i); 725 if (!ctx) 726 continue; 727 if (flags & VFIO_IRQ_SET_DATA_NONE) { 728 eventfd_signal(ctx->trigger, 1); 729 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 730 uint8_t *bools = data; 731 if (bools[i - start]) 732 eventfd_signal(ctx->trigger, 1); 733 } 734 } 735 return 0; 736 } 737 738 static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, 739 unsigned int count, uint32_t flags, 740 void *data) 741 { 742 /* DATA_NONE/DATA_BOOL enables loopback testing */ 743 if (flags & VFIO_IRQ_SET_DATA_NONE) { 744 if (*ctx) { 745 if (count) { 746 eventfd_signal(*ctx, 1); 747 } else { 748 eventfd_ctx_put(*ctx); 749 *ctx = NULL; 750 } 751 return 0; 752 } 753 } else if (flags & VFIO_IRQ_SET_DATA_BOOL) { 754 uint8_t trigger; 755 756 if (!count) 757 return -EINVAL; 758 759 trigger = *(uint8_t *)data; 760 if (trigger && *ctx) 761 eventfd_signal(*ctx, 1); 762 763 return 0; 764 } else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) { 765 int32_t fd; 766 767 if (!count) 768 return -EINVAL; 769 770 fd = *(int32_t *)data; 771 if (fd == -1) { 772 if (*ctx) 773 eventfd_ctx_put(*ctx); 774 *ctx = NULL; 775 } else if (fd >= 0) { 776 struct eventfd_ctx *efdctx; 777 778 efdctx = eventfd_ctx_fdget(fd); 779 if (IS_ERR(efdctx)) 780 return PTR_ERR(efdctx); 781 782 if (*ctx) 783 eventfd_ctx_put(*ctx); 784 785 *ctx = efdctx; 786 } 787 return 0; 788 } 789 790 return -EINVAL; 791 } 792 793 static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev, 794 unsigned index, unsigned start, 795 unsigned count, uint32_t flags, void *data) 796 { 797 if (index != VFIO_PCI_ERR_IRQ_INDEX || start != 0 || count > 1) 798 return -EINVAL; 799 800 return vfio_pci_set_ctx_trigger_single(&vdev->err_trigger, 801 count, flags, data); 802 } 803 804 static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev, 805 unsigned index, unsigned start, 806 unsigned count, uint32_t flags, void *data) 807 { 808 if (index != VFIO_PCI_REQ_IRQ_INDEX || start != 0 || count > 1) 809 return -EINVAL; 810 811 return vfio_pci_set_ctx_trigger_single(&vdev->req_trigger, 812 count, flags, data); 813 } 814 815 int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags, 816 unsigned index, unsigned start, unsigned count, 817 void *data) 818 { 819 int (*func)(struct vfio_pci_core_device *vdev, unsigned index, 820 unsigned start, unsigned count, uint32_t flags, 821 void *data) = NULL; 822 823 switch (index) { 824 case VFIO_PCI_INTX_IRQ_INDEX: 825 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 826 case VFIO_IRQ_SET_ACTION_MASK: 827 func = vfio_pci_set_intx_mask; 828 break; 829 case VFIO_IRQ_SET_ACTION_UNMASK: 830 func = vfio_pci_set_intx_unmask; 831 break; 832 case VFIO_IRQ_SET_ACTION_TRIGGER: 833 func = vfio_pci_set_intx_trigger; 834 break; 835 } 836 break; 837 case VFIO_PCI_MSI_IRQ_INDEX: 838 case VFIO_PCI_MSIX_IRQ_INDEX: 839 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 840 case VFIO_IRQ_SET_ACTION_MASK: 841 case VFIO_IRQ_SET_ACTION_UNMASK: 842 /* XXX Need masking support exported */ 843 break; 844 case VFIO_IRQ_SET_ACTION_TRIGGER: 845 func = vfio_pci_set_msi_trigger; 846 break; 847 } 848 break; 849 case VFIO_PCI_ERR_IRQ_INDEX: 850 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 851 case VFIO_IRQ_SET_ACTION_TRIGGER: 852 if (pci_is_pcie(vdev->pdev)) 853 func = vfio_pci_set_err_trigger; 854 break; 855 } 856 break; 857 case VFIO_PCI_REQ_IRQ_INDEX: 858 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) { 859 case VFIO_IRQ_SET_ACTION_TRIGGER: 860 func = vfio_pci_set_req_trigger; 861 break; 862 } 863 break; 864 } 865 866 if (!func) 867 return -ENOTTY; 868 869 return func(vdev, index, start, count, flags, data); 870 } 871