1 /* 2 * kvm eventfd support - use eventfd objects to signal various KVM events 3 * 4 * Copyright 2009 Novell. All Rights Reserved. 5 * Copyright 2010 Red Hat, Inc. and/or its affiliates. 6 * 7 * Author: 8 * Gregory Haskins <ghaskins@novell.com> 9 * 10 * This file is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License 12 * as published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 * GNU General Public License for more details. 18 * 19 * You should have received a copy of the GNU General Public License 20 * along with this program; if not, write to the Free Software Foundation, 21 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. 22 */ 23 24 #include <linux/kvm_host.h> 25 #include <linux/kvm.h> 26 #include <linux/workqueue.h> 27 #include <linux/syscalls.h> 28 #include <linux/wait.h> 29 #include <linux/poll.h> 30 #include <linux/file.h> 31 #include <linux/list.h> 32 #include <linux/eventfd.h> 33 #include <linux/kernel.h> 34 #include <linux/slab.h> 35 36 #include "iodev.h" 37 38 #ifdef __KVM_HAVE_IOAPIC 39 /* 40 * -------------------------------------------------------------------- 41 * irqfd: Allows an fd to be used to inject an interrupt to the guest 42 * 43 * Credit goes to Avi Kivity for the original idea. 44 * -------------------------------------------------------------------- 45 */ 46 47 /* 48 * Resampling irqfds are a special variety of irqfds used to emulate 49 * level triggered interrupts. The interrupt is asserted on eventfd 50 * trigger. On acknowledgement through the irq ack notifier, the 51 * interrupt is de-asserted and userspace is notified through the 52 * resamplefd. All resamplers on the same gsi are de-asserted 53 * together, so we don't need to track the state of each individual 54 * user. We can also therefore share the same irq source ID. 55 */ 56 struct _irqfd_resampler { 57 struct kvm *kvm; 58 /* 59 * List of resampling struct _irqfd objects sharing this gsi. 60 * RCU list modified under kvm->irqfds.resampler_lock 61 */ 62 struct list_head list; 63 struct kvm_irq_ack_notifier notifier; 64 /* 65 * Entry in list of kvm->irqfd.resampler_list. Use for sharing 66 * resamplers among irqfds on the same gsi. 67 * Accessed and modified under kvm->irqfds.resampler_lock 68 */ 69 struct list_head link; 70 }; 71 72 struct _irqfd { 73 /* Used for MSI fast-path */ 74 struct kvm *kvm; 75 wait_queue_t wait; 76 /* Update side is protected by irqfds.lock */ 77 struct kvm_kernel_irq_routing_entry __rcu *irq_entry; 78 /* Used for level IRQ fast-path */ 79 int gsi; 80 struct work_struct inject; 81 /* The resampler used by this irqfd (resampler-only) */ 82 struct _irqfd_resampler *resampler; 83 /* Eventfd notified on resample (resampler-only) */ 84 struct eventfd_ctx *resamplefd; 85 /* Entry in list of irqfds for a resampler (resampler-only) */ 86 struct list_head resampler_link; 87 /* Used for setup/shutdown */ 88 struct eventfd_ctx *eventfd; 89 struct list_head list; 90 poll_table pt; 91 struct work_struct shutdown; 92 }; 93 94 static struct workqueue_struct *irqfd_cleanup_wq; 95 96 static void 97 irqfd_inject(struct work_struct *work) 98 { 99 struct _irqfd *irqfd = container_of(work, struct _irqfd, inject); 100 struct kvm *kvm = irqfd->kvm; 101 102 if (!irqfd->resampler) { 103 kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 1); 104 kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, irqfd->gsi, 0); 105 } else 106 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 107 irqfd->gsi, 1); 108 } 109 110 /* 111 * Since resampler irqfds share an IRQ source ID, we de-assert once 112 * then notify all of the resampler irqfds using this GSI. We can't 113 * do multiple de-asserts or we risk racing with incoming re-asserts. 114 */ 115 static void 116 irqfd_resampler_ack(struct kvm_irq_ack_notifier *kian) 117 { 118 struct _irqfd_resampler *resampler; 119 struct _irqfd *irqfd; 120 121 resampler = container_of(kian, struct _irqfd_resampler, notifier); 122 123 kvm_set_irq(resampler->kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 124 resampler->notifier.gsi, 0); 125 126 rcu_read_lock(); 127 128 list_for_each_entry_rcu(irqfd, &resampler->list, resampler_link) 129 eventfd_signal(irqfd->resamplefd, 1); 130 131 rcu_read_unlock(); 132 } 133 134 static void 135 irqfd_resampler_shutdown(struct _irqfd *irqfd) 136 { 137 struct _irqfd_resampler *resampler = irqfd->resampler; 138 struct kvm *kvm = resampler->kvm; 139 140 mutex_lock(&kvm->irqfds.resampler_lock); 141 142 list_del_rcu(&irqfd->resampler_link); 143 synchronize_rcu(); 144 145 if (list_empty(&resampler->list)) { 146 list_del(&resampler->link); 147 kvm_unregister_irq_ack_notifier(kvm, &resampler->notifier); 148 kvm_set_irq(kvm, KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID, 149 resampler->notifier.gsi, 0); 150 kfree(resampler); 151 } 152 153 mutex_unlock(&kvm->irqfds.resampler_lock); 154 } 155 156 /* 157 * Race-free decouple logic (ordering is critical) 158 */ 159 static void 160 irqfd_shutdown(struct work_struct *work) 161 { 162 struct _irqfd *irqfd = container_of(work, struct _irqfd, shutdown); 163 u64 cnt; 164 165 /* 166 * Synchronize with the wait-queue and unhook ourselves to prevent 167 * further events. 168 */ 169 eventfd_ctx_remove_wait_queue(irqfd->eventfd, &irqfd->wait, &cnt); 170 171 /* 172 * We know no new events will be scheduled at this point, so block 173 * until all previously outstanding events have completed 174 */ 175 flush_work(&irqfd->inject); 176 177 if (irqfd->resampler) { 178 irqfd_resampler_shutdown(irqfd); 179 eventfd_ctx_put(irqfd->resamplefd); 180 } 181 182 /* 183 * It is now safe to release the object's resources 184 */ 185 eventfd_ctx_put(irqfd->eventfd); 186 kfree(irqfd); 187 } 188 189 190 /* assumes kvm->irqfds.lock is held */ 191 static bool 192 irqfd_is_active(struct _irqfd *irqfd) 193 { 194 return list_empty(&irqfd->list) ? false : true; 195 } 196 197 /* 198 * Mark the irqfd as inactive and schedule it for removal 199 * 200 * assumes kvm->irqfds.lock is held 201 */ 202 static void 203 irqfd_deactivate(struct _irqfd *irqfd) 204 { 205 BUG_ON(!irqfd_is_active(irqfd)); 206 207 list_del_init(&irqfd->list); 208 209 queue_work(irqfd_cleanup_wq, &irqfd->shutdown); 210 } 211 212 /* 213 * Called with wqh->lock held and interrupts disabled 214 */ 215 static int 216 irqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key) 217 { 218 struct _irqfd *irqfd = container_of(wait, struct _irqfd, wait); 219 unsigned long flags = (unsigned long)key; 220 struct kvm_kernel_irq_routing_entry *irq; 221 struct kvm *kvm = irqfd->kvm; 222 223 if (flags & POLLIN) { 224 rcu_read_lock(); 225 irq = rcu_dereference(irqfd->irq_entry); 226 /* An event has been signaled, inject an interrupt */ 227 if (irq) 228 kvm_set_msi(irq, kvm, KVM_USERSPACE_IRQ_SOURCE_ID, 1); 229 else 230 schedule_work(&irqfd->inject); 231 rcu_read_unlock(); 232 } 233 234 if (flags & POLLHUP) { 235 /* The eventfd is closing, detach from KVM */ 236 unsigned long flags; 237 238 spin_lock_irqsave(&kvm->irqfds.lock, flags); 239 240 /* 241 * We must check if someone deactivated the irqfd before 242 * we could acquire the irqfds.lock since the item is 243 * deactivated from the KVM side before it is unhooked from 244 * the wait-queue. If it is already deactivated, we can 245 * simply return knowing the other side will cleanup for us. 246 * We cannot race against the irqfd going away since the 247 * other side is required to acquire wqh->lock, which we hold 248 */ 249 if (irqfd_is_active(irqfd)) 250 irqfd_deactivate(irqfd); 251 252 spin_unlock_irqrestore(&kvm->irqfds.lock, flags); 253 } 254 255 return 0; 256 } 257 258 static void 259 irqfd_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, 260 poll_table *pt) 261 { 262 struct _irqfd *irqfd = container_of(pt, struct _irqfd, pt); 263 add_wait_queue(wqh, &irqfd->wait); 264 } 265 266 /* Must be called under irqfds.lock */ 267 static void irqfd_update(struct kvm *kvm, struct _irqfd *irqfd, 268 struct kvm_irq_routing_table *irq_rt) 269 { 270 struct kvm_kernel_irq_routing_entry *e; 271 struct hlist_node *n; 272 273 if (irqfd->gsi >= irq_rt->nr_rt_entries) { 274 rcu_assign_pointer(irqfd->irq_entry, NULL); 275 return; 276 } 277 278 hlist_for_each_entry(e, n, &irq_rt->map[irqfd->gsi], link) { 279 /* Only fast-path MSI. */ 280 if (e->type == KVM_IRQ_ROUTING_MSI) 281 rcu_assign_pointer(irqfd->irq_entry, e); 282 else 283 rcu_assign_pointer(irqfd->irq_entry, NULL); 284 } 285 } 286 287 static int 288 kvm_irqfd_assign(struct kvm *kvm, struct kvm_irqfd *args) 289 { 290 struct kvm_irq_routing_table *irq_rt; 291 struct _irqfd *irqfd, *tmp; 292 struct file *file = NULL; 293 struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL; 294 int ret; 295 unsigned int events; 296 297 irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL); 298 if (!irqfd) 299 return -ENOMEM; 300 301 irqfd->kvm = kvm; 302 irqfd->gsi = args->gsi; 303 INIT_LIST_HEAD(&irqfd->list); 304 INIT_WORK(&irqfd->inject, irqfd_inject); 305 INIT_WORK(&irqfd->shutdown, irqfd_shutdown); 306 307 file = eventfd_fget(args->fd); 308 if (IS_ERR(file)) { 309 ret = PTR_ERR(file); 310 goto fail; 311 } 312 313 eventfd = eventfd_ctx_fileget(file); 314 if (IS_ERR(eventfd)) { 315 ret = PTR_ERR(eventfd); 316 goto fail; 317 } 318 319 irqfd->eventfd = eventfd; 320 321 if (args->flags & KVM_IRQFD_FLAG_RESAMPLE) { 322 struct _irqfd_resampler *resampler; 323 324 resamplefd = eventfd_ctx_fdget(args->resamplefd); 325 if (IS_ERR(resamplefd)) { 326 ret = PTR_ERR(resamplefd); 327 goto fail; 328 } 329 330 irqfd->resamplefd = resamplefd; 331 INIT_LIST_HEAD(&irqfd->resampler_link); 332 333 mutex_lock(&kvm->irqfds.resampler_lock); 334 335 list_for_each_entry(resampler, 336 &kvm->irqfds.resampler_list, link) { 337 if (resampler->notifier.gsi == irqfd->gsi) { 338 irqfd->resampler = resampler; 339 break; 340 } 341 } 342 343 if (!irqfd->resampler) { 344 resampler = kzalloc(sizeof(*resampler), GFP_KERNEL); 345 if (!resampler) { 346 ret = -ENOMEM; 347 mutex_unlock(&kvm->irqfds.resampler_lock); 348 goto fail; 349 } 350 351 resampler->kvm = kvm; 352 INIT_LIST_HEAD(&resampler->list); 353 resampler->notifier.gsi = irqfd->gsi; 354 resampler->notifier.irq_acked = irqfd_resampler_ack; 355 INIT_LIST_HEAD(&resampler->link); 356 357 list_add(&resampler->link, &kvm->irqfds.resampler_list); 358 kvm_register_irq_ack_notifier(kvm, 359 &resampler->notifier); 360 irqfd->resampler = resampler; 361 } 362 363 list_add_rcu(&irqfd->resampler_link, &irqfd->resampler->list); 364 synchronize_rcu(); 365 366 mutex_unlock(&kvm->irqfds.resampler_lock); 367 } 368 369 /* 370 * Install our own custom wake-up handling so we are notified via 371 * a callback whenever someone signals the underlying eventfd 372 */ 373 init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup); 374 init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc); 375 376 spin_lock_irq(&kvm->irqfds.lock); 377 378 ret = 0; 379 list_for_each_entry(tmp, &kvm->irqfds.items, list) { 380 if (irqfd->eventfd != tmp->eventfd) 381 continue; 382 /* This fd is used for another irq already. */ 383 ret = -EBUSY; 384 spin_unlock_irq(&kvm->irqfds.lock); 385 goto fail; 386 } 387 388 irq_rt = rcu_dereference_protected(kvm->irq_routing, 389 lockdep_is_held(&kvm->irqfds.lock)); 390 irqfd_update(kvm, irqfd, irq_rt); 391 392 events = file->f_op->poll(file, &irqfd->pt); 393 394 list_add_tail(&irqfd->list, &kvm->irqfds.items); 395 396 /* 397 * Check if there was an event already pending on the eventfd 398 * before we registered, and trigger it as if we didn't miss it. 399 */ 400 if (events & POLLIN) 401 schedule_work(&irqfd->inject); 402 403 spin_unlock_irq(&kvm->irqfds.lock); 404 405 /* 406 * do not drop the file until the irqfd is fully initialized, otherwise 407 * we might race against the POLLHUP 408 */ 409 fput(file); 410 411 return 0; 412 413 fail: 414 if (irqfd->resampler) 415 irqfd_resampler_shutdown(irqfd); 416 417 if (resamplefd && !IS_ERR(resamplefd)) 418 eventfd_ctx_put(resamplefd); 419 420 if (eventfd && !IS_ERR(eventfd)) 421 eventfd_ctx_put(eventfd); 422 423 if (!IS_ERR(file)) 424 fput(file); 425 426 kfree(irqfd); 427 return ret; 428 } 429 #endif 430 431 void 432 kvm_eventfd_init(struct kvm *kvm) 433 { 434 #ifdef __KVM_HAVE_IOAPIC 435 spin_lock_init(&kvm->irqfds.lock); 436 INIT_LIST_HEAD(&kvm->irqfds.items); 437 INIT_LIST_HEAD(&kvm->irqfds.resampler_list); 438 mutex_init(&kvm->irqfds.resampler_lock); 439 #endif 440 INIT_LIST_HEAD(&kvm->ioeventfds); 441 } 442 443 #ifdef __KVM_HAVE_IOAPIC 444 /* 445 * shutdown any irqfd's that match fd+gsi 446 */ 447 static int 448 kvm_irqfd_deassign(struct kvm *kvm, struct kvm_irqfd *args) 449 { 450 struct _irqfd *irqfd, *tmp; 451 struct eventfd_ctx *eventfd; 452 453 eventfd = eventfd_ctx_fdget(args->fd); 454 if (IS_ERR(eventfd)) 455 return PTR_ERR(eventfd); 456 457 spin_lock_irq(&kvm->irqfds.lock); 458 459 list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) { 460 if (irqfd->eventfd == eventfd && irqfd->gsi == args->gsi) { 461 /* 462 * This rcu_assign_pointer is needed for when 463 * another thread calls kvm_irq_routing_update before 464 * we flush workqueue below (we synchronize with 465 * kvm_irq_routing_update using irqfds.lock). 466 * It is paired with synchronize_rcu done by caller 467 * of that function. 468 */ 469 rcu_assign_pointer(irqfd->irq_entry, NULL); 470 irqfd_deactivate(irqfd); 471 } 472 } 473 474 spin_unlock_irq(&kvm->irqfds.lock); 475 eventfd_ctx_put(eventfd); 476 477 /* 478 * Block until we know all outstanding shutdown jobs have completed 479 * so that we guarantee there will not be any more interrupts on this 480 * gsi once this deassign function returns. 481 */ 482 flush_workqueue(irqfd_cleanup_wq); 483 484 return 0; 485 } 486 487 int 488 kvm_irqfd(struct kvm *kvm, struct kvm_irqfd *args) 489 { 490 if (args->flags & ~(KVM_IRQFD_FLAG_DEASSIGN | KVM_IRQFD_FLAG_RESAMPLE)) 491 return -EINVAL; 492 493 if (args->flags & KVM_IRQFD_FLAG_DEASSIGN) 494 return kvm_irqfd_deassign(kvm, args); 495 496 return kvm_irqfd_assign(kvm, args); 497 } 498 499 /* 500 * This function is called as the kvm VM fd is being released. Shutdown all 501 * irqfds that still remain open 502 */ 503 void 504 kvm_irqfd_release(struct kvm *kvm) 505 { 506 struct _irqfd *irqfd, *tmp; 507 508 spin_lock_irq(&kvm->irqfds.lock); 509 510 list_for_each_entry_safe(irqfd, tmp, &kvm->irqfds.items, list) 511 irqfd_deactivate(irqfd); 512 513 spin_unlock_irq(&kvm->irqfds.lock); 514 515 /* 516 * Block until we know all outstanding shutdown jobs have completed 517 * since we do not take a kvm* reference. 518 */ 519 flush_workqueue(irqfd_cleanup_wq); 520 521 } 522 523 /* 524 * Change irq_routing and irqfd. 525 * Caller must invoke synchronize_rcu afterwards. 526 */ 527 void kvm_irq_routing_update(struct kvm *kvm, 528 struct kvm_irq_routing_table *irq_rt) 529 { 530 struct _irqfd *irqfd; 531 532 spin_lock_irq(&kvm->irqfds.lock); 533 534 rcu_assign_pointer(kvm->irq_routing, irq_rt); 535 536 list_for_each_entry(irqfd, &kvm->irqfds.items, list) 537 irqfd_update(kvm, irqfd, irq_rt); 538 539 spin_unlock_irq(&kvm->irqfds.lock); 540 } 541 542 /* 543 * create a host-wide workqueue for issuing deferred shutdown requests 544 * aggregated from all vm* instances. We need our own isolated single-thread 545 * queue to prevent deadlock against flushing the normal work-queue. 546 */ 547 static int __init irqfd_module_init(void) 548 { 549 irqfd_cleanup_wq = create_singlethread_workqueue("kvm-irqfd-cleanup"); 550 if (!irqfd_cleanup_wq) 551 return -ENOMEM; 552 553 return 0; 554 } 555 556 static void __exit irqfd_module_exit(void) 557 { 558 destroy_workqueue(irqfd_cleanup_wq); 559 } 560 561 module_init(irqfd_module_init); 562 module_exit(irqfd_module_exit); 563 #endif 564 565 /* 566 * -------------------------------------------------------------------- 567 * ioeventfd: translate a PIO/MMIO memory write to an eventfd signal. 568 * 569 * userspace can register a PIO/MMIO address with an eventfd for receiving 570 * notification when the memory has been touched. 571 * -------------------------------------------------------------------- 572 */ 573 574 struct _ioeventfd { 575 struct list_head list; 576 u64 addr; 577 int length; 578 struct eventfd_ctx *eventfd; 579 u64 datamatch; 580 struct kvm_io_device dev; 581 bool wildcard; 582 }; 583 584 static inline struct _ioeventfd * 585 to_ioeventfd(struct kvm_io_device *dev) 586 { 587 return container_of(dev, struct _ioeventfd, dev); 588 } 589 590 static void 591 ioeventfd_release(struct _ioeventfd *p) 592 { 593 eventfd_ctx_put(p->eventfd); 594 list_del(&p->list); 595 kfree(p); 596 } 597 598 static bool 599 ioeventfd_in_range(struct _ioeventfd *p, gpa_t addr, int len, const void *val) 600 { 601 u64 _val; 602 603 if (!(addr == p->addr && len == p->length)) 604 /* address-range must be precise for a hit */ 605 return false; 606 607 if (p->wildcard) 608 /* all else equal, wildcard is always a hit */ 609 return true; 610 611 /* otherwise, we have to actually compare the data */ 612 613 BUG_ON(!IS_ALIGNED((unsigned long)val, len)); 614 615 switch (len) { 616 case 1: 617 _val = *(u8 *)val; 618 break; 619 case 2: 620 _val = *(u16 *)val; 621 break; 622 case 4: 623 _val = *(u32 *)val; 624 break; 625 case 8: 626 _val = *(u64 *)val; 627 break; 628 default: 629 return false; 630 } 631 632 return _val == p->datamatch ? true : false; 633 } 634 635 /* MMIO/PIO writes trigger an event if the addr/val match */ 636 static int 637 ioeventfd_write(struct kvm_io_device *this, gpa_t addr, int len, 638 const void *val) 639 { 640 struct _ioeventfd *p = to_ioeventfd(this); 641 642 if (!ioeventfd_in_range(p, addr, len, val)) 643 return -EOPNOTSUPP; 644 645 eventfd_signal(p->eventfd, 1); 646 return 0; 647 } 648 649 /* 650 * This function is called as KVM is completely shutting down. We do not 651 * need to worry about locking just nuke anything we have as quickly as possible 652 */ 653 static void 654 ioeventfd_destructor(struct kvm_io_device *this) 655 { 656 struct _ioeventfd *p = to_ioeventfd(this); 657 658 ioeventfd_release(p); 659 } 660 661 static const struct kvm_io_device_ops ioeventfd_ops = { 662 .write = ioeventfd_write, 663 .destructor = ioeventfd_destructor, 664 }; 665 666 /* assumes kvm->slots_lock held */ 667 static bool 668 ioeventfd_check_collision(struct kvm *kvm, struct _ioeventfd *p) 669 { 670 struct _ioeventfd *_p; 671 672 list_for_each_entry(_p, &kvm->ioeventfds, list) 673 if (_p->addr == p->addr && _p->length == p->length && 674 (_p->wildcard || p->wildcard || 675 _p->datamatch == p->datamatch)) 676 return true; 677 678 return false; 679 } 680 681 static int 682 kvm_assign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 683 { 684 int pio = args->flags & KVM_IOEVENTFD_FLAG_PIO; 685 enum kvm_bus bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS; 686 struct _ioeventfd *p; 687 struct eventfd_ctx *eventfd; 688 int ret; 689 690 /* must be natural-word sized */ 691 switch (args->len) { 692 case 1: 693 case 2: 694 case 4: 695 case 8: 696 break; 697 default: 698 return -EINVAL; 699 } 700 701 /* check for range overflow */ 702 if (args->addr + args->len < args->addr) 703 return -EINVAL; 704 705 /* check for extra flags that we don't understand */ 706 if (args->flags & ~KVM_IOEVENTFD_VALID_FLAG_MASK) 707 return -EINVAL; 708 709 eventfd = eventfd_ctx_fdget(args->fd); 710 if (IS_ERR(eventfd)) 711 return PTR_ERR(eventfd); 712 713 p = kzalloc(sizeof(*p), GFP_KERNEL); 714 if (!p) { 715 ret = -ENOMEM; 716 goto fail; 717 } 718 719 INIT_LIST_HEAD(&p->list); 720 p->addr = args->addr; 721 p->length = args->len; 722 p->eventfd = eventfd; 723 724 /* The datamatch feature is optional, otherwise this is a wildcard */ 725 if (args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH) 726 p->datamatch = args->datamatch; 727 else 728 p->wildcard = true; 729 730 mutex_lock(&kvm->slots_lock); 731 732 /* Verify that there isn't a match already */ 733 if (ioeventfd_check_collision(kvm, p)) { 734 ret = -EEXIST; 735 goto unlock_fail; 736 } 737 738 kvm_iodevice_init(&p->dev, &ioeventfd_ops); 739 740 ret = kvm_io_bus_register_dev(kvm, bus_idx, p->addr, p->length, 741 &p->dev); 742 if (ret < 0) 743 goto unlock_fail; 744 745 list_add_tail(&p->list, &kvm->ioeventfds); 746 747 mutex_unlock(&kvm->slots_lock); 748 749 return 0; 750 751 unlock_fail: 752 mutex_unlock(&kvm->slots_lock); 753 754 fail: 755 kfree(p); 756 eventfd_ctx_put(eventfd); 757 758 return ret; 759 } 760 761 static int 762 kvm_deassign_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 763 { 764 int pio = args->flags & KVM_IOEVENTFD_FLAG_PIO; 765 enum kvm_bus bus_idx = pio ? KVM_PIO_BUS : KVM_MMIO_BUS; 766 struct _ioeventfd *p, *tmp; 767 struct eventfd_ctx *eventfd; 768 int ret = -ENOENT; 769 770 eventfd = eventfd_ctx_fdget(args->fd); 771 if (IS_ERR(eventfd)) 772 return PTR_ERR(eventfd); 773 774 mutex_lock(&kvm->slots_lock); 775 776 list_for_each_entry_safe(p, tmp, &kvm->ioeventfds, list) { 777 bool wildcard = !(args->flags & KVM_IOEVENTFD_FLAG_DATAMATCH); 778 779 if (p->eventfd != eventfd || 780 p->addr != args->addr || 781 p->length != args->len || 782 p->wildcard != wildcard) 783 continue; 784 785 if (!p->wildcard && p->datamatch != args->datamatch) 786 continue; 787 788 kvm_io_bus_unregister_dev(kvm, bus_idx, &p->dev); 789 ioeventfd_release(p); 790 ret = 0; 791 break; 792 } 793 794 mutex_unlock(&kvm->slots_lock); 795 796 eventfd_ctx_put(eventfd); 797 798 return ret; 799 } 800 801 int 802 kvm_ioeventfd(struct kvm *kvm, struct kvm_ioeventfd *args) 803 { 804 if (args->flags & KVM_IOEVENTFD_FLAG_DEASSIGN) 805 return kvm_deassign_ioeventfd(kvm, args); 806 807 return kvm_assign_ioeventfd(kvm, args); 808 } 809