1 /* 2 * kvm eventfd support - use eventfd objects to signal various KVM events 3 * 4 * Copyright 2009 Novell. All Rights Reserved. 5 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 6 * 7 * Author: 8 * Gregory Haskins <ghaskins@novell.com> 9 * 10 * This file is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License 12 * as published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software Foundation, 21 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. 22 */ 23 24 #include <linux/kvm_host.h> 25 #include <linux/kvm.h> 26 #include <linux/workqueue.h> 27 #include <linux/syscalls.h> 28 #include <linux/wait.h> 29 #include <linux/poll.h> 30 #include <linux/file.h> 31 #include <linux/list.h> 32 #include <linux/eventfd.h> 33 #include <linux/kernel.h> 34 #include <linux/slab.h> 35 36 #include "iodev.h" 37 38 #ifdef __KVM_HAVE_IOAPIC 39 /* 40 * -------------------------------------------------------------------- 41 * irqfd: Allows an fd to be used to inject an interrupt to the guest 42 * 43 * Credit goes to Avi Kivity for the original idea. 44 * -------------------------------------------------------------------- 45 */ 46 47 /* 48 * Resampling irqfds are a special variety of irqfds used to emulate 49 * level triggered interrupts. The interrupt is asserted on eventfd 50 * trigger. On acknowledgement through the irq ack notifier, the 51 * interrupt is de-asserted and userspace is notified through the 52 * resamplefd. All resamplers on the same gsi are de-asserted 53 * together, so we don't need to track the state of each individual 54 * user. We can also therefore share the same irq source ID. 55 */ 56 struct _irqfd_resampler { 57 struct kvm *kvm; 58 /* 59 * List of resampling struct _irqfd objects sharing this gsi. 60 * RCU list modified under kvm->irqfds.resampler_lock 61 */ 62 struct list_head list; 63 struct kvm_irq_ack_notifier notifier; 64 /* 65 * Entry in list of kvm->irqfd.resampler_list. Use for sharing 66 * resamplers among irqfds on the same gsi. 67 * Accessed and modified under kvm->irqfds.resampler_lock 68 */ 69 struct list_head link; 70 }; 71 72 struct _irqfd { 73 /* Used for MSI fast-path */ 74 struct kvm *kvm; 75 wait_queue_t wait; 76 /* Update side is protected by irqfds.lock */ 77 struct kvm_kernel_irq_routing_entry __rcu *irq_entry; 78 /* Used for level IRQ fast-path */ 79 int gsi; 80 struct work_struct inject; 81 /* The resampler used by this irqfd (resampler-only) */ 82 struct _irqfd_resampler *resampler; 83 /* Eventfd notified on resample (resampler-only) */ 84 struct eventfd_ctx *resamplefd; 85 /* Entry in list of irqfds for a resampler (resampler-only) */ 86 struct list_head resampler_link; 87 /* Used for setup/shutdown */ 88 struct eventfd_ctx *eventfd; 89 struct list_head list; 90 poll_table pt; 91 struct work_struct shutdown; 92 }; 93 94 static struct workqueue_struct *irqfd_cleanup_wq; 95 96 static void 97 irqfd_inject(struct work_struct *work) 98 { 99 struct _irqfd *irqfd = container_of(work, struct _irqfd, inject); 100 struct kvm *kvm = irqfd->kvm; 101 102 if (!irqfd->resampler) { 103 kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1); 104 kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0); 105 } else 106 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 107 irqfd->gsi, 1); 108 } 109 110 /* 111 * Since resampler irqfds share an IRQ source ID, we de-assert once 112 * then notify all of the resampler irqfds using this GSI. We can't 113 * do multiple de-asserts or we risk racing with incoming re-asserts. 114 */ 115 static void 116 irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian) 117 { 118 struct _irqfd_resampler *resampler; 119 struct _irqfd *irqfd; 120 121 resampler = container_of(kian, struct _irqfd_resampler, notifier); 122 123 kvm_set_irq(resampler->kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 124 resampler->notifier.gsi, 0); 125 126 rcu_read_lock(); 127 128 list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link) 129 eventfd_signal(irqfd->resamplefd, 1); 130 131 rcu_read_unlock(); 132 } 133 134 static void 135 irqfd_resampler_shutdown(struct _irqfd *irqfd) 136 { 137 struct _irqfd_resampler *resampler = irqfd->resampler; 138 struct kvm *kvm = resampler->kvm; 139 140 mutex_lock(&kvm->irqfds.resampler_lock); 141 142 list_del_rcu(&irqfd->resampler_link); 143 synchronize_rcu(); 144 145 if (list_empty(&resampler->list)) { 146 list_del(&resampler->link); 147 kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier); 148 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 149 resampler->notifier.gsi, 0); 150 kfree(resampler); 151 } 152 153 mutex_unlock(&kvm->irqfds.resampler_lock); 154 } 155 156 /* 157 * Race-free decouple logic (ordering is critical) 158 */ 159 static void 160 irqfd_shutdown(struct work_struct *work) 161 { 162 struct _irqfd *irqfd = container_of(work, struct _irqfd, shutdown); 163 u64 cnt; 164 165 /* 166 * Synchronize with the wait-queue and unhook ourselves to prevent 167 * further events. 168 */ 169 eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt); 170 171 /* 172 * We know no new events will be scheduled at this point, so block 173 * until all previously outstanding events have completed 174 */ 175 flush_work(&irqfd->inject); 176 177 if (irqfd->resampler) { 178 irqfd_resampler_shutdown(irqfd); 179 eventfd_ctx_put(irqfd->resamplefd); 180 } 181 182 /* 183 * It is now safe to release the object's resources 184 */ 185 eventfd_ctx_put(irqfd->eventfd); 186 kfree(irqfd); 187 } 188 189 190 /* assumes kvm->irqfds.lock is held */ 191 static bool 192 irqfd_is_active(struct _irqfd *irqfd) 193 { 194 return list_empty(&irqfd->list) ? false : true; 195 } 196 197 /* 198 * Mark the irqfd as inactive and schedule it for removal 199 * 200 * assumes kvm->irqfds.lock is held 201 */ 202 static void 203 irqfd_deactivate(struct _irqfd *irqfd) 204 { 205 BUG_ON(!irqfd_is_active(irqfd)); 206 207 list_del_init(&irqfd->list); 208 209 queue_work(irqfd_cleanup_wq, &irqfd->shutdown); 210 } 211 212 /* 213 * Called with wqh->lock held and interrupts disabled 214 */ 215 static int 216 irqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key) 217 { 218 struct _irqfd *irqfd = container_of(wait, struct _irqfd, wait); 219 unsigned long flags = (unsigned long)key; 220 struct kvm_kernel_irq_routing_entry *irq; 221 struct kvm *kvm = irqfd->kvm; 222 223 if (flags & POLLIN) { 224 rcu_read_lock(); 225 irq = rcu_dereference(irqfd->irq_entry); 226 /* An event has been signaled, inject an interrupt */ 227 if (irq) 228 kvm_set_msi(irq, kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1); 229 else 230 schedule_work(&irqfd->inject); 231 rcu_read_unlock(); 232 } 233 234 if (flags & POLLHUP) { 235 /* The eventfd is closing, detach from KVM */ 236 unsigned long flags; 237 238 spin_lock_irqsave(&kvm->irqfds.lock, flags); 239 240 /* 241 * We must check if someone deactivated the irqfd before 242 * we could acquire the irqfds.lock since the item is 243 * deactivated from the KVM side before it is unhooked from 244 * the wait-queue. If it is already deactivated, we can 245 * simply return knowing the other side will cleanup for us. 246 * We cannot race against the irqfd going away since the 247 * other side is required to acquire wqh->lock, which we hold 248 */ 249 if (irqfd_is_active(irqfd)) 250 irqfd_deactivate(irqfd); 251 252 spin_unlock_irqrestore(&kvm->irqfds.lock, flags); 253 } 254 255 return 0; 256 } 257 258 static void 259 irqfd_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, 260 poll_table *pt) 261 { 262 struct _irqfd *irqfd = container_of(pt, struct _irqfd, pt); 263 add_wait_queue(wqh, &irqfd->wait); 264 } 265 266 /* Must be called under irqfds.lock */ 267 static void irqfd_update(struct kvm *kvm, struct _irqfd *irqfd, 268 struct kvm_irq_routing_table *irq_rt) 269 { 270 struct kvm_kernel_irq_routing_entry *e; 271 272 if (irqfd->gsi >= irq_rt->nr_rt_entries) { 273 rcu_assign_pointer(irqfd->irq_entry, NULL); 274 return; 275 } 276 277 hlist_for_each_entry(e, &irq_rt->map[irqfd->gsi], link) { 278 /* Only fast-path MSI. */ 279 if (e->type == KVM_IRQ_ROUTING_MSI) 280 rcu_assign_pointer(irqfd->irq_entry, e); 281 else 282 rcu_assign_pointer(irqfd->irq_entry, NULL); 283 } 284 } 285 286 static int 287 kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args) 288 { 289 struct kvm_irq_routing_table *irq_rt; 290 struct _irqfd *irqfd, *tmp; 291 struct file *file = NULL; 292 struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL; 293 int ret; 294 unsigned int events; 295 296 irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL); 297 if (!irqfd) 298 return -ENOMEM; 299 300 irqfd->kvm = kvm; 301 irqfd->gsi = args->gsi; 302 INIT_LIST_HEAD(&irqfd->list); 303 INIT_WORK(&irqfd->inject, irqfd_inject); 304 INIT_WORK(&irqfd->shutdown, irqfd_shutdown); 305 306 file = eventfd_fget(args->fd); 307 if (IS_ERR(file)) { 308 ret = PTR_ERR(file); 309 goto fail; 310 } 311 312 eventfd = eventfd_ctx_fileget(file); 313 if (IS_ERR(eventfd)) { 314 ret = PTR_ERR(eventfd); 315 goto fail; 316 } 317 318 irqfd->eventfd = eventfd; 319 320 if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) { 321 struct _irqfd_resampler *resampler; 322 323 resamplefd = eventfd_ctx_fdget(args->resamplefd); 324 if (IS_ERR(resamplefd)) { 325 ret = PTR_ERR(resamplefd); 326 goto fail; 327 } 328 329 irqfd->resamplefd = resamplefd; 330 INIT_LIST_HEAD(&irqfd->resampler_link); 331 332 mutex_lock(&kvm->irqfds.resampler_lock); 333 334 list_for_each_entry(resampler, 335 &kvm->irqfds.resampler_list, link) { 336 if (resampler->notifier.gsi == irqfd->gsi) { 337 irqfd->resampler = resampler; 338 break; 339 } 340 } 341 342 if (!irqfd->resampler) { 343 resampler = kzalloc(sizeof(*resampler), GFP_KERNEL); 344 if (!resampler) { 345 ret = -ENOMEM; 346 mutex_unlock(&kvm->irqfds.resampler_lock); 347 goto fail; 348 } 349 350 resampler->kvm = kvm; 351 INIT_LIST_HEAD(&resampler->list); 352 resampler->notifier.gsi = irqfd->gsi; 353 resampler->notifier.irq_acked = irqfd_resampler_ack; 354 INIT_LIST_HEAD(&resampler->link); 355 356 list_add(&resampler->link, &kvm->irqfds.resampler_list); 357 kvm_register_irq_ack_notifier(kvm, 358 &resampler->notifier); 359 irqfd->resampler = resampler; 360 } 361 362 list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list); 363 synchronize_rcu(); 364 365 mutex_unlock(&kvm->irqfds.resampler_lock); 366 } 367 368 /* 369 * Install our own custom wake-up handling so we are notified via 370 * a callback whenever someone signals the underlying eventfd 371 */ 372 init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup); 373 init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc); 374 375 spin_lock_irq(&kvm->irqfds.lock); 376 377 ret = 0; 378 list_for_each_entry(tmp, &kvm->irqfds.items, list) { 379 if (irqfd->eventfd != tmp->eventfd) 380 continue; 381 /* This fd is used for another irq already. */ 382 ret = -EBUSY; 383 spin_unlock_irq(&kvm->irqfds.lock); 384 goto fail; 385 } 386 387 irq_rt = rcu_dereference_protected(kvm->irq_routing, 388 lockdep_is_held(&kvm->irqfds.lock)); 389 irqfd_update(kvm, irqfd, irq_rt); 390 391 events = file->f_op->poll(file, &irqfd->pt); 392 393 list_add_tail(&irqfd->list, &kvm->irqfds.items); 394 395 /* 396 * Check if there was an event already pending on the eventfd 397 * before we registered, and trigger it as if we didn't miss it. 398 */ 399 if (events & POLLIN) 400 schedule_work(&irqfd->inject); 401 402 spin_unlock_irq(&kvm->irqfds.lock); 403 404 /* 405 * do not drop the file until the irqfd is fully initialized, otherwise 406 * we might race against the POLLHUP 407 */ 408 fput(file); 409 410 return 0; 411 412 fail: 413 if (irqfd->resampler) 414 irqfd_resampler_shutdown(irqfd); 415 416 if (resamplefd && !IS_ERR(resamplefd)) 417 eventfd_ctx_put(resamplefd); 418 419 if (eventfd && !IS_ERR(eventfd)) 420 eventfd_ctx_put(eventfd); 421 422 if (!IS_ERR(file)) 423 fput(file); 424 425 kfree(irqfd); 426 return ret; 427 } 428 #endif 429 430 void 431 kvm_eventfd_init(struct kvm *kvm) 432 { 433 #ifdef __KVM_HAVE_IOAPIC 434 spin_lock_init(&kvm->irqfds.lock); 435 INIT_LIST_HEAD(&kvm->irqfds.items); 436 INIT_LIST_HEAD(&kvm->irqfds.resampler_list); 437 mutex_init(&kvm->irqfds.resampler_lock); 438 #endif 439 INIT_LIST_HEAD(&kvm->ioeventfds); 440 } 441 442 #ifdef __KVM_HAVE_IOAPIC 443 /* 444 * shutdown any irqfd's that match fd+gsi 445 */ 446 static int 447 kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args) 448 { 449 struct _irqfd *irqfd, *tmp; 450 struct eventfd_ctx *eventfd; 451 452 eventfd = eventfd_ctx_fdget(args->fd); 453 if (IS_ERR(eventfd)) 454 return PTR_ERR(eventfd); 455 456 spin_lock_irq(&kvm->irqfds.lock); 457 458 list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) { 459 if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) { 460 /* 461 * This rcu_assign_pointer is needed for when 462 * another thread calls kvm_irq_routing_update before 463 * we flush workqueue below (we synchronize with 464 * kvm_irq_routing_update using irqfds.lock). 465 * It is paired with synchronize_rcu done by caller 466 * of that function. 467 */ 468 rcu_assign_pointer(irqfd->irq_entry, NULL); 469 irqfd_deactivate(irqfd); 470 } 471 } 472 473 spin_unlock_irq(&kvm->irqfds.lock); 474 eventfd_ctx_put(eventfd); 475 476 /* 477 * Block until we know all outstanding shutdown jobs have completed 478 * so that we guarantee there will not be any more interrupts on this 479 * gsi once this deassign function returns. 480 */ 481 flush_workqueue(irqfd_cleanup_wq); 482 483 return 0; 484 } 485 486 int 487 kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 488 { 489 if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE)) 490 return -EINVAL; 491 492 if (args->flags & KVM_IRQFD_FLAG_DEASSIGN) 493 return kvm_irqfd_deassign(kvm, args); 494 495 return kvm_irqfd_assign(kvm, args); 496 } 497 498 /* 499 * This function is called as the kvm VM fd is being released. Shutdown all 500 * irqfds that still remain open 501 */ 502 void 503 kvm_irqfd_release(struct kvm *kvm) 504 { 505 struct _irqfd *irqfd, *tmp; 506 507 spin_lock_irq(&kvm->irqfds.lock); 508 509 list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) 510 irqfd_deactivate(irqfd); 511 512 spin_unlock_irq(&kvm->irqfds.lock); 513 514 /* 515 * Block until we know all outstanding shutdown jobs have completed 516 * since we do not take a kvm* reference. 517 */ 518 flush_workqueue(irqfd_cleanup_wq); 519 520 } 521 522 /* 523 * Change irq_routing and irqfd. 524 * Caller must invoke synchronize_rcu afterwards. 525 */ 526 void kvm_irq_routing_update(struct kvm *kvm, 527 struct kvm_irq_routing_table *irq_rt) 528 { 529 struct _irqfd *irqfd; 530 531 spin_lock_irq(&kvm->irqfds.lock); 532 533 rcu_assign_pointer(kvm->irq_routing, irq_rt); 534 535 list_for_each_entry(irqfd, &kvm->irqfds.items, list) 536 irqfd_update(kvm, irqfd, irq_rt); 537 538 spin_unlock_irq(&kvm->irqfds.lock); 539 } 540 541 /* 542 * create a host-wide workqueue for issuing deferred shutdown requests 543 * aggregated from all vm* instances. We need our own isolated single-thread 544 * queue to prevent deadlock against flushing the normal work-queue. 545 */ 546 static int __init irqfd_module_init(void) 547 { 548 irqfd_cleanup_wq = create_singlethread_workqueue("kvm-irqfd-cleanup"); 549 if (!irqfd_cleanup_wq) 550 return -ENOMEM; 551 552 return 0; 553 } 554 555 static void __exit irqfd_module_exit(void) 556 { 557 destroy_workqueue(irqfd_cleanup_wq); 558 } 559 560 module_init(irqfd_module_init); 561 module_exit(irqfd_module_exit); 562 #endif 563 564 /* 565 * -------------------------------------------------------------------- 566 * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal. 567 * 568 * userspace can register a PIO/MMIO address with an eventfd for receiving 569 * notification when the memory has been touched. 570 * -------------------------------------------------------------------- 571 */ 572 573 struct _ioeventfd { 574 struct list_head list; 575 u64 addr; 576 int length; 577 struct eventfd_ctx *eventfd; 578 u64 datamatch; 579 struct kvm_io_device dev; 580 bool wildcard; 581 }; 582 583 static inline struct _ioeventfd * 584 to_ioeventfd(struct kvm_io_device *dev) 585 { 586 return container_of(dev, struct _ioeventfd, dev); 587 } 588 589 static void 590 ioeventfd_release(struct _ioeventfd *p) 591 { 592 eventfd_ctx_put(p->eventfd); 593 list_del(&p->list); 594 kfree(p); 595 } 596 597 static bool 598 ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) 599 { 600 u64 _val; 601 602 if (!(addr == p->addr && len == p->length)) 603 /* address-range must be precise for a hit */ 604 return false; 605 606 if (p->wildcard) 607 /* all else equal, wildcard is always a hit */ 608 return true; 609 610 /* otherwise, we have to actually compare the data */ 611 612 BUG_ON(!IS_ALIGNED((unsigned long)val, len)); 613 614 switch (len) { 615 case 1: 616 _val = *(u8 *)val; 617 break; 618 case 2: 619 _val = *(u16 *)val; 620 break; 621 case 4: 622 _val = *(u32 *)val; 623 break; 624 case 8: 625 _val = *(u64 *)val; 626 break; 627 default: 628 return false; 629 } 630 631 return _val == p->datamatch ? true : false; 632 } 633 634 /* MMIO/PIO writes trigger an event if the addr/val match */ 635 static int 636 ioeventfd_write(struct kvm_io_device *this, gpa_t addr, int len, 637 const void *val) 638 { 639 struct _ioeventfd *p = to_ioeventfd(this); 640 641 if (!ioeventfd_in_range(p, addr, len, val)) 642 return -EOPNOTSUPP; 643 644 eventfd_signal(p->eventfd, 1); 645 return 0; 646 } 647 648 /* 649 * This function is called as KVM is completely shutting down. We do not 650 * need to worry about locking just nuke anything we have as quickly as possible 651 */ 652 static void 653 ioeventfd_destructor(struct kvm_io_device *this) 654 { 655 struct _ioeventfd *p = to_ioeventfd(this); 656 657 ioeventfd_release(p); 658 } 659 660 static const struct kvm_io_device_ops ioeventfd_ops = { 661 .write = ioeventfd_write, 662 .destructor = ioeventfd_destructor, 663 }; 664 665 /* assumes kvm->slots_lock held */ 666 static bool 667 ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p) 668 { 669 struct _ioeventfd *_p; 670 671 list_for_each_entry(_p, &kvm->ioeventfds, list) 672 if (_p->addr == p->addr && _p->length == p->length && 673 (_p->wildcard || p->wildcard || 674 _p->datamatch == p->datamatch)) 675 return true; 676 677 return false; 678 } 679 680 static int 681 kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 682 { 683 int pio = args->flags & KVM_IOEVENTFD_FLAG_PIO; 684 enum kvm_bus bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS; 685 struct _ioeventfd *p; 686 struct eventfd_ctx *eventfd; 687 int ret; 688 689 /* must be natural-word sized */ 690 switch (args->len) { 691 case 1: 692 case 2: 693 case 4: 694 case 8: 695 break; 696 default: 697 return -EINVAL; 698 } 699 700 /* check for range overflow */ 701 if (args->addr + args->len < args->addr) 702 return -EINVAL; 703 704 /* check for extra flags that we don't understand */ 705 if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK) 706 return -EINVAL; 707 708 eventfd = eventfd_ctx_fdget(args->fd); 709 if (IS_ERR(eventfd)) 710 return PTR_ERR(eventfd); 711 712 p = kzalloc(sizeof(*p), GFP_KERNEL); 713 if (!p) { 714 ret = -ENOMEM; 715 goto fail; 716 } 717 718 INIT_LIST_HEAD(&p->list); 719 p->addr = args->addr; 720 p->length = args->len; 721 p->eventfd = eventfd; 722 723 /* The datamatch feature is optional, otherwise this is a wildcard */ 724 if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH) 725 p->datamatch = args->datamatch; 726 else 727 p->wildcard = true; 728 729 mutex_lock(&kvm->slots_lock); 730 731 /* Verify that there isn't a match already */ 732 if (ioeventfd_check_collision(kvm, p)) { 733 ret = -EEXIST; 734 goto unlock_fail; 735 } 736 737 kvm_iodevice_init(&p->dev, &ioeventfd_ops); 738 739 ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length, 740 &p->dev); 741 if (ret < 0) 742 goto unlock_fail; 743 744 list_add_tail(&p->list, &kvm->ioeventfds); 745 746 mutex_unlock(&kvm->slots_lock); 747 748 return 0; 749 750 unlock_fail: 751 mutex_unlock(&kvm->slots_lock); 752 753 fail: 754 kfree(p); 755 eventfd_ctx_put(eventfd); 756 757 return ret; 758 } 759 760 static int 761 kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 762 { 763 int pio = args->flags & KVM_IOEVENTFD_FLAG_PIO; 764 enum kvm_bus bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS; 765 struct _ioeventfd *p, *tmp; 766 struct eventfd_ctx *eventfd; 767 int ret = -ENOENT; 768 769 eventfd = eventfd_ctx_fdget(args->fd); 770 if (IS_ERR(eventfd)) 771 return PTR_ERR(eventfd); 772 773 mutex_lock(&kvm->slots_lock); 774 775 list_for_each_entry_safe(p, tmp, &kvm->ioeventfds, list) { 776 bool wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH); 777 778 if (p->eventfd != eventfd || 779 p->addr != args->addr || 780 p->length != args->len || 781 p->wildcard != wildcard) 782 continue; 783 784 if (!p->wildcard && p->datamatch != args->datamatch) 785 continue; 786 787 kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev); 788 ioeventfd_release(p); 789 ret = 0; 790 break; 791 } 792 793 mutex_unlock(&kvm->slots_lock); 794 795 eventfd_ctx_put(eventfd); 796 797 return ret; 798 } 799 800 int 801 kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 802 { 803 if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN) 804 return kvm_deassign_ioeventfd(kvm, args); 805 806 return kvm_assign_ioeventfd(kvm, args); 807 } 808