1 /* Copyright (C) 2009 Red Hat, Inc. 2 * Copyright (C) 2006 Rusty Russell IBM Corporation 3 * 4 * Author: Michael S. Tsirkin <mst@redhat.com> 5 * 6 * Inspiration, some code, and most witty comments come from 7 * Documentation/virtual/lguest/lguest.c, by Rusty Russell 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. 10 * 11 * Generic code for virtio server in host kernel. 12 */ 13 14 #include <linux/eventfd.h> 15 #include <linux/vhost.h> 16 #include <linux/uio.h> 17 #include <linux/mm.h> 18 #include <linux/mmu_context.h> 19 #include <linux/miscdevice.h> 20 #include <linux/mutex.h> 21 #include <linux/poll.h> 22 #include <linux/file.h> 23 #include <linux/highmem.h> 24 #include <linux/slab.h> 25 #include <linux/vmalloc.h> 26 #include <linux/kthread.h> 27 #include <linux/cgroup.h> 28 #include <linux/module.h> 29 #include <linux/sort.h> 30 #include <linux/sched/mm.h> 31 #include <linux/sched/signal.h> 32 #include <linux/interval_tree_generic.h> 33 34 #include "vhost.h" 35 36 static ushort max_mem_regions = 64; 37 module_param(max_mem_regions, ushort, 0444); 38 MODULE_PARM_DESC(max_mem_regions, 39 "Maximum number of memory regions in memory map. (default: 64)"); 40 static int max_iotlb_entries = 2048; 41 module_param(max_iotlb_entries, int, 0444); 42 MODULE_PARM_DESC(max_iotlb_entries, 43 "Maximum number of iotlb entries. (default: 2048)"); 44 45 enum { 46 VHOST_MEMORY_F_LOG = 0x1, 47 }; 48 49 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num]) 50 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) 51 52 INTERVAL_TREE_DEFINE(struct vhost_umem_node, 53 rb, __u64, __subtree_last, 54 START, LAST, static inline, vhost_umem_interval_tree); 55 56 #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY 57 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) 58 { 59 vq->user_be = !virtio_legacy_is_little_endian(); 60 } 61 62 static void vhost_enable_cross_endian_big(struct vhost_virtqueue *vq) 63 { 64 vq->user_be = true; 65 } 66 67 static void vhost_enable_cross_endian_little(struct vhost_virtqueue *vq) 68 { 69 vq->user_be = false; 70 } 71 72 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) 73 { 74 struct vhost_vring_state s; 75 76 if (vq->private_data) 77 return -EBUSY; 78 79 if (copy_from_user(&s, argp, sizeof(s))) 80 return -EFAULT; 81 82 if (s.num != VHOST_VRING_LITTLE_ENDIAN && 83 s.num != VHOST_VRING_BIG_ENDIAN) 84 return -EINVAL; 85 86 if (s.num == VHOST_VRING_BIG_ENDIAN) 87 vhost_enable_cross_endian_big(vq); 88 else 89 vhost_enable_cross_endian_little(vq); 90 91 return 0; 92 } 93 94 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, 95 int __user *argp) 96 { 97 struct vhost_vring_state s = { 98 .index = idx, 99 .num = vq->user_be 100 }; 101 102 if (copy_to_user(argp, &s, sizeof(s))) 103 return -EFAULT; 104 105 return 0; 106 } 107 108 static void vhost_init_is_le(struct vhost_virtqueue *vq) 109 { 110 /* Note for legacy virtio: user_be is initialized at reset time 111 * according to the host endianness. If userspace does not set an 112 * explicit endianness, the default behavior is native endian, as 113 * expected by legacy virtio. 114 */ 115 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) || !vq->user_be; 116 } 117 #else 118 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) 119 { 120 } 121 122 static long vhost_set_vring_endian(struct vhost_virtqueue *vq, int __user *argp) 123 { 124 return -ENOIOCTLCMD; 125 } 126 127 static long vhost_get_vring_endian(struct vhost_virtqueue *vq, u32 idx, 128 int __user *argp) 129 { 130 return -ENOIOCTLCMD; 131 } 132 133 static void vhost_init_is_le(struct vhost_virtqueue *vq) 134 { 135 vq->is_le = vhost_has_feature(vq, VIRTIO_F_VERSION_1) 136 || virtio_legacy_is_little_endian(); 137 } 138 #endif /* CONFIG_VHOST_CROSS_ENDIAN_LEGACY */ 139 140 static void vhost_reset_is_le(struct vhost_virtqueue *vq) 141 { 142 vhost_init_is_le(vq); 143 } 144 145 struct vhost_flush_struct { 146 struct vhost_work work; 147 struct completion wait_event; 148 }; 149 150 static void vhost_flush_work(struct vhost_work *work) 151 { 152 struct vhost_flush_struct *s; 153 154 s = container_of(work, struct vhost_flush_struct, work); 155 complete(&s->wait_event); 156 } 157 158 static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, 159 poll_table *pt) 160 { 161 struct vhost_poll *poll; 162 163 poll = container_of(pt, struct vhost_poll, table); 164 poll->wqh = wqh; 165 add_wait_queue(wqh, &poll->wait); 166 } 167 168 static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync, 169 void *key) 170 { 171 struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); 172 173 if (!((unsigned long)key & poll->mask)) 174 return 0; 175 176 vhost_poll_queue(poll); 177 return 0; 178 } 179 180 void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn) 181 { 182 clear_bit(VHOST_WORK_QUEUED, &work->flags); 183 work->fn = fn; 184 init_waitqueue_head(&work->done); 185 } 186 EXPORT_SYMBOL_GPL(vhost_work_init); 187 188 /* Init poll structure */ 189 void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 190 unsigned long mask, struct vhost_dev *dev) 191 { 192 init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup); 193 init_poll_funcptr(&poll->table, vhost_poll_func); 194 poll->mask = mask; 195 poll->dev = dev; 196 poll->wqh = NULL; 197 198 vhost_work_init(&poll->work, fn); 199 } 200 EXPORT_SYMBOL_GPL(vhost_poll_init); 201 202 /* Start polling a file. We add ourselves to file's wait queue. The caller must 203 * keep a reference to a file until after vhost_poll_stop is called. */ 204 int vhost_poll_start(struct vhost_poll *poll, struct file *file) 205 { 206 unsigned long mask; 207 int ret = 0; 208 209 if (poll->wqh) 210 return 0; 211 212 mask = file->f_op->poll(file, &poll->table); 213 if (mask) 214 vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask); 215 if (mask & POLLERR) { 216 if (poll->wqh) 217 remove_wait_queue(poll->wqh, &poll->wait); 218 ret = -EINVAL; 219 } 220 221 return ret; 222 } 223 EXPORT_SYMBOL_GPL(vhost_poll_start); 224 225 /* Stop polling a file. After this function returns, it becomes safe to drop the 226 * file reference. You must also flush afterwards. */ 227 void vhost_poll_stop(struct vhost_poll *poll) 228 { 229 if (poll->wqh) { 230 remove_wait_queue(poll->wqh, &poll->wait); 231 poll->wqh = NULL; 232 } 233 } 234 EXPORT_SYMBOL_GPL(vhost_poll_stop); 235 236 void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) 237 { 238 struct vhost_flush_struct flush; 239 240 if (dev->worker) { 241 init_completion(&flush.wait_event); 242 vhost_work_init(&flush.work, vhost_flush_work); 243 244 vhost_work_queue(dev, &flush.work); 245 wait_for_completion(&flush.wait_event); 246 } 247 } 248 EXPORT_SYMBOL_GPL(vhost_work_flush); 249 250 /* Flush any work that has been scheduled. When calling this, don't hold any 251 * locks that are also used by the callback. */ 252 void vhost_poll_flush(struct vhost_poll *poll) 253 { 254 vhost_work_flush(poll->dev, &poll->work); 255 } 256 EXPORT_SYMBOL_GPL(vhost_poll_flush); 257 258 void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) 259 { 260 if (!dev->worker) 261 return; 262 263 if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) { 264 /* We can only add the work to the list after we're 265 * sure it was not in the list. 266 * test_and_set_bit() implies a memory barrier. 267 */ 268 llist_add(&work->node, &dev->work_list); 269 wake_up_process(dev->worker); 270 } 271 } 272 EXPORT_SYMBOL_GPL(vhost_work_queue); 273 274 /* A lockless hint for busy polling code to exit the loop */ 275 bool vhost_has_work(struct vhost_dev *dev) 276 { 277 return !llist_empty(&dev->work_list); 278 } 279 EXPORT_SYMBOL_GPL(vhost_has_work); 280 281 void vhost_poll_queue(struct vhost_poll *poll) 282 { 283 vhost_work_queue(poll->dev, &poll->work); 284 } 285 EXPORT_SYMBOL_GPL(vhost_poll_queue); 286 287 static void vhost_vq_reset(struct vhost_dev *dev, 288 struct vhost_virtqueue *vq) 289 { 290 vq->num = 1; 291 vq->desc = NULL; 292 vq->avail = NULL; 293 vq->used = NULL; 294 vq->last_avail_idx = 0; 295 vq->last_used_event = 0; 296 vq->avail_idx = 0; 297 vq->last_used_idx = 0; 298 vq->signalled_used = 0; 299 vq->signalled_used_valid = false; 300 vq->used_flags = 0; 301 vq->log_used = false; 302 vq->log_addr = -1ull; 303 vq->private_data = NULL; 304 vq->acked_features = 0; 305 vq->log_base = NULL; 306 vq->error_ctx = NULL; 307 vq->error = NULL; 308 vq->kick = NULL; 309 vq->call_ctx = NULL; 310 vq->call = NULL; 311 vq->log_ctx = NULL; 312 vhost_reset_is_le(vq); 313 vhost_disable_cross_endian(vq); 314 vq->busyloop_timeout = 0; 315 vq->umem = NULL; 316 vq->iotlb = NULL; 317 } 318 319 static int vhost_worker(void *data) 320 { 321 struct vhost_dev *dev = data; 322 struct vhost_work *work, *work_next; 323 struct llist_node *node; 324 mm_segment_t oldfs = get_fs(); 325 326 set_fs(USER_DS); 327 use_mm(dev->mm); 328 329 for (;;) { 330 /* mb paired w/ kthread_stop */ 331 set_current_state(TASK_INTERRUPTIBLE); 332 333 if (kthread_should_stop()) { 334 __set_current_state(TASK_RUNNING); 335 break; 336 } 337 338 node = llist_del_all(&dev->work_list); 339 if (!node) 340 schedule(); 341 342 node = llist_reverse_order(node); 343 /* make sure flag is seen after deletion */ 344 smp_wmb(); 345 llist_for_each_entry_safe(work, work_next, node, node) { 346 clear_bit(VHOST_WORK_QUEUED, &work->flags); 347 __set_current_state(TASK_RUNNING); 348 work->fn(work); 349 if (need_resched()) 350 schedule(); 351 } 352 } 353 unuse_mm(dev->mm); 354 set_fs(oldfs); 355 return 0; 356 } 357 358 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) 359 { 360 kfree(vq->indirect); 361 vq->indirect = NULL; 362 kfree(vq->log); 363 vq->log = NULL; 364 kfree(vq->heads); 365 vq->heads = NULL; 366 } 367 368 /* Helper to allocate iovec buffers for all vqs. */ 369 static long vhost_dev_alloc_iovecs(struct vhost_dev *dev) 370 { 371 struct vhost_virtqueue *vq; 372 int i; 373 374 for (i = 0; i < dev->nvqs; ++i) { 375 vq = dev->vqs[i]; 376 vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV, 377 GFP_KERNEL); 378 vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL); 379 vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL); 380 if (!vq->indirect || !vq->log || !vq->heads) 381 goto err_nomem; 382 } 383 return 0; 384 385 err_nomem: 386 for (; i >= 0; --i) 387 vhost_vq_free_iovecs(dev->vqs[i]); 388 return -ENOMEM; 389 } 390 391 static void vhost_dev_free_iovecs(struct vhost_dev *dev) 392 { 393 int i; 394 395 for (i = 0; i < dev->nvqs; ++i) 396 vhost_vq_free_iovecs(dev->vqs[i]); 397 } 398 399 void vhost_dev_init(struct vhost_dev *dev, 400 struct vhost_virtqueue **vqs, int nvqs) 401 { 402 struct vhost_virtqueue *vq; 403 int i; 404 405 dev->vqs = vqs; 406 dev->nvqs = nvqs; 407 mutex_init(&dev->mutex); 408 dev->log_ctx = NULL; 409 dev->log_file = NULL; 410 dev->umem = NULL; 411 dev->iotlb = NULL; 412 dev->mm = NULL; 413 dev->worker = NULL; 414 init_llist_head(&dev->work_list); 415 init_waitqueue_head(&dev->wait); 416 INIT_LIST_HEAD(&dev->read_list); 417 INIT_LIST_HEAD(&dev->pending_list); 418 spin_lock_init(&dev->iotlb_lock); 419 420 421 for (i = 0; i < dev->nvqs; ++i) { 422 vq = dev->vqs[i]; 423 vq->log = NULL; 424 vq->indirect = NULL; 425 vq->heads = NULL; 426 vq->dev = dev; 427 mutex_init(&vq->mutex); 428 vhost_vq_reset(dev, vq); 429 if (vq->handle_kick) 430 vhost_poll_init(&vq->poll, vq->handle_kick, 431 POLLIN, dev); 432 } 433 } 434 EXPORT_SYMBOL_GPL(vhost_dev_init); 435 436 /* Caller should have device mutex */ 437 long vhost_dev_check_owner(struct vhost_dev *dev) 438 { 439 /* Are you the owner? If not, I don't think you mean to do that */ 440 return dev->mm == current->mm ? 0 : -EPERM; 441 } 442 EXPORT_SYMBOL_GPL(vhost_dev_check_owner); 443 444 struct vhost_attach_cgroups_struct { 445 struct vhost_work work; 446 struct task_struct *owner; 447 int ret; 448 }; 449 450 static void vhost_attach_cgroups_work(struct vhost_work *work) 451 { 452 struct vhost_attach_cgroups_struct *s; 453 454 s = container_of(work, struct vhost_attach_cgroups_struct, work); 455 s->ret = cgroup_attach_task_all(s->owner, current); 456 } 457 458 static int vhost_attach_cgroups(struct vhost_dev *dev) 459 { 460 struct vhost_attach_cgroups_struct attach; 461 462 attach.owner = current; 463 vhost_work_init(&attach.work, vhost_attach_cgroups_work); 464 vhost_work_queue(dev, &attach.work); 465 vhost_work_flush(dev, &attach.work); 466 return attach.ret; 467 } 468 469 /* Caller should have device mutex */ 470 bool vhost_dev_has_owner(struct vhost_dev *dev) 471 { 472 return dev->mm; 473 } 474 EXPORT_SYMBOL_GPL(vhost_dev_has_owner); 475 476 /* Caller should have device mutex */ 477 long vhost_dev_set_owner(struct vhost_dev *dev) 478 { 479 struct task_struct *worker; 480 int err; 481 482 /* Is there an owner already? */ 483 if (vhost_dev_has_owner(dev)) { 484 err = -EBUSY; 485 goto err_mm; 486 } 487 488 /* No owner, become one */ 489 dev->mm = get_task_mm(current); 490 worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid); 491 if (IS_ERR(worker)) { 492 err = PTR_ERR(worker); 493 goto err_worker; 494 } 495 496 dev->worker = worker; 497 wake_up_process(worker); /* avoid contributing to loadavg */ 498 499 err = vhost_attach_cgroups(dev); 500 if (err) 501 goto err_cgroup; 502 503 err = vhost_dev_alloc_iovecs(dev); 504 if (err) 505 goto err_cgroup; 506 507 return 0; 508 err_cgroup: 509 kthread_stop(worker); 510 dev->worker = NULL; 511 err_worker: 512 if (dev->mm) 513 mmput(dev->mm); 514 dev->mm = NULL; 515 err_mm: 516 return err; 517 } 518 EXPORT_SYMBOL_GPL(vhost_dev_set_owner); 519 520 static void *vhost_kvzalloc(unsigned long size) 521 { 522 void *n = kzalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 523 524 if (!n) 525 n = vzalloc(size); 526 return n; 527 } 528 529 struct vhost_umem *vhost_dev_reset_owner_prepare(void) 530 { 531 return vhost_kvzalloc(sizeof(struct vhost_umem)); 532 } 533 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); 534 535 /* Caller should have device mutex */ 536 void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem) 537 { 538 int i; 539 540 vhost_dev_cleanup(dev, true); 541 542 /* Restore memory to default empty mapping. */ 543 INIT_LIST_HEAD(&umem->umem_list); 544 dev->umem = umem; 545 /* We don't need VQ locks below since vhost_dev_cleanup makes sure 546 * VQs aren't running. 547 */ 548 for (i = 0; i < dev->nvqs; ++i) 549 dev->vqs[i]->umem = umem; 550 } 551 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner); 552 553 void vhost_dev_stop(struct vhost_dev *dev) 554 { 555 int i; 556 557 for (i = 0; i < dev->nvqs; ++i) { 558 if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) { 559 vhost_poll_stop(&dev->vqs[i]->poll); 560 vhost_poll_flush(&dev->vqs[i]->poll); 561 } 562 } 563 } 564 EXPORT_SYMBOL_GPL(vhost_dev_stop); 565 566 static void vhost_umem_free(struct vhost_umem *umem, 567 struct vhost_umem_node *node) 568 { 569 vhost_umem_interval_tree_remove(node, &umem->umem_tree); 570 list_del(&node->link); 571 kfree(node); 572 umem->numem--; 573 } 574 575 static void vhost_umem_clean(struct vhost_umem *umem) 576 { 577 struct vhost_umem_node *node, *tmp; 578 579 if (!umem) 580 return; 581 582 list_for_each_entry_safe(node, tmp, &umem->umem_list, link) 583 vhost_umem_free(umem, node); 584 585 kvfree(umem); 586 } 587 588 static void vhost_clear_msg(struct vhost_dev *dev) 589 { 590 struct vhost_msg_node *node, *n; 591 592 spin_lock(&dev->iotlb_lock); 593 594 list_for_each_entry_safe(node, n, &dev->read_list, node) { 595 list_del(&node->node); 596 kfree(node); 597 } 598 599 list_for_each_entry_safe(node, n, &dev->pending_list, node) { 600 list_del(&node->node); 601 kfree(node); 602 } 603 604 spin_unlock(&dev->iotlb_lock); 605 } 606 607 /* Caller should have device mutex if and only if locked is set */ 608 void vhost_dev_cleanup(struct vhost_dev *dev, bool locked) 609 { 610 int i; 611 612 for (i = 0; i < dev->nvqs; ++i) { 613 if (dev->vqs[i]->error_ctx) 614 eventfd_ctx_put(dev->vqs[i]->error_ctx); 615 if (dev->vqs[i]->error) 616 fput(dev->vqs[i]->error); 617 if (dev->vqs[i]->kick) 618 fput(dev->vqs[i]->kick); 619 if (dev->vqs[i]->call_ctx) 620 eventfd_ctx_put(dev->vqs[i]->call_ctx); 621 if (dev->vqs[i]->call) 622 fput(dev->vqs[i]->call); 623 vhost_vq_reset(dev, dev->vqs[i]); 624 } 625 vhost_dev_free_iovecs(dev); 626 if (dev->log_ctx) 627 eventfd_ctx_put(dev->log_ctx); 628 dev->log_ctx = NULL; 629 if (dev->log_file) 630 fput(dev->log_file); 631 dev->log_file = NULL; 632 /* No one will access memory at this point */ 633 vhost_umem_clean(dev->umem); 634 dev->umem = NULL; 635 vhost_umem_clean(dev->iotlb); 636 dev->iotlb = NULL; 637 vhost_clear_msg(dev); 638 wake_up_interruptible_poll(&dev->wait, POLLIN | POLLRDNORM); 639 WARN_ON(!llist_empty(&dev->work_list)); 640 if (dev->worker) { 641 kthread_stop(dev->worker); 642 dev->worker = NULL; 643 } 644 if (dev->mm) 645 mmput(dev->mm); 646 dev->mm = NULL; 647 } 648 EXPORT_SYMBOL_GPL(vhost_dev_cleanup); 649 650 static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz) 651 { 652 u64 a = addr / VHOST_PAGE_SIZE / 8; 653 654 /* Make sure 64 bit math will not overflow. */ 655 if (a > ULONG_MAX - (unsigned long)log_base || 656 a + (unsigned long)log_base > ULONG_MAX) 657 return 0; 658 659 return access_ok(VERIFY_WRITE, log_base + a, 660 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8); 661 } 662 663 static bool vhost_overflow(u64 uaddr, u64 size) 664 { 665 /* Make sure 64 bit math will not overflow. */ 666 return uaddr > ULONG_MAX || size > ULONG_MAX || uaddr > ULONG_MAX - size; 667 } 668 669 /* Caller should have vq mutex and device mutex. */ 670 static int vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem, 671 int log_all) 672 { 673 struct vhost_umem_node *node; 674 675 if (!umem) 676 return 0; 677 678 list_for_each_entry(node, &umem->umem_list, link) { 679 unsigned long a = node->userspace_addr; 680 681 if (vhost_overflow(node->userspace_addr, node->size)) 682 return 0; 683 684 685 if (!access_ok(VERIFY_WRITE, (void __user *)a, 686 node->size)) 687 return 0; 688 else if (log_all && !log_access_ok(log_base, 689 node->start, 690 node->size)) 691 return 0; 692 } 693 return 1; 694 } 695 696 /* Can we switch to this memory table? */ 697 /* Caller should have device mutex but not vq mutex */ 698 static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem, 699 int log_all) 700 { 701 int i; 702 703 for (i = 0; i < d->nvqs; ++i) { 704 int ok; 705 bool log; 706 707 mutex_lock(&d->vqs[i]->mutex); 708 log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL); 709 /* If ring is inactive, will check when it's enabled. */ 710 if (d->vqs[i]->private_data) 711 ok = vq_memory_access_ok(d->vqs[i]->log_base, 712 umem, log); 713 else 714 ok = 1; 715 mutex_unlock(&d->vqs[i]->mutex); 716 if (!ok) 717 return 0; 718 } 719 return 1; 720 } 721 722 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, 723 struct iovec iov[], int iov_size, int access); 724 725 static int vhost_copy_to_user(struct vhost_virtqueue *vq, void __user *to, 726 const void *from, unsigned size) 727 { 728 int ret; 729 730 if (!vq->iotlb) 731 return __copy_to_user(to, from, size); 732 else { 733 /* This function should be called after iotlb 734 * prefetch, which means we're sure that all vq 735 * could be access through iotlb. So -EAGAIN should 736 * not happen in this case. 737 */ 738 /* TODO: more fast path */ 739 struct iov_iter t; 740 ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov, 741 ARRAY_SIZE(vq->iotlb_iov), 742 VHOST_ACCESS_WO); 743 if (ret < 0) 744 goto out; 745 iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size); 746 ret = copy_to_iter(from, size, &t); 747 if (ret == size) 748 ret = 0; 749 } 750 out: 751 return ret; 752 } 753 754 static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to, 755 void __user *from, unsigned size) 756 { 757 int ret; 758 759 if (!vq->iotlb) 760 return __copy_from_user(to, from, size); 761 else { 762 /* This function should be called after iotlb 763 * prefetch, which means we're sure that vq 764 * could be access through iotlb. So -EAGAIN should 765 * not happen in this case. 766 */ 767 /* TODO: more fast path */ 768 struct iov_iter f; 769 ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov, 770 ARRAY_SIZE(vq->iotlb_iov), 771 VHOST_ACCESS_RO); 772 if (ret < 0) { 773 vq_err(vq, "IOTLB translation failure: uaddr " 774 "%p size 0x%llx\n", from, 775 (unsigned long long) size); 776 goto out; 777 } 778 iov_iter_init(&f, READ, vq->iotlb_iov, ret, size); 779 ret = copy_from_iter(to, size, &f); 780 if (ret == size) 781 ret = 0; 782 } 783 784 out: 785 return ret; 786 } 787 788 static void __user *__vhost_get_user(struct vhost_virtqueue *vq, 789 void __user *addr, unsigned size) 790 { 791 int ret; 792 793 /* This function should be called after iotlb 794 * prefetch, which means we're sure that vq 795 * could be access through iotlb. So -EAGAIN should 796 * not happen in this case. 797 */ 798 /* TODO: more fast path */ 799 ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov, 800 ARRAY_SIZE(vq->iotlb_iov), 801 VHOST_ACCESS_RO); 802 if (ret < 0) { 803 vq_err(vq, "IOTLB translation failure: uaddr " 804 "%p size 0x%llx\n", addr, 805 (unsigned long long) size); 806 return NULL; 807 } 808 809 if (ret != 1 || vq->iotlb_iov[0].iov_len != size) { 810 vq_err(vq, "Non atomic userspace memory access: uaddr " 811 "%p size 0x%llx\n", addr, 812 (unsigned long long) size); 813 return NULL; 814 } 815 816 return vq->iotlb_iov[0].iov_base; 817 } 818 819 #define vhost_put_user(vq, x, ptr) \ 820 ({ \ 821 int ret = -EFAULT; \ 822 if (!vq->iotlb) { \ 823 ret = __put_user(x, ptr); \ 824 } else { \ 825 __typeof__(ptr) to = \ 826 (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \ 827 if (to != NULL) \ 828 ret = __put_user(x, to); \ 829 else \ 830 ret = -EFAULT; \ 831 } \ 832 ret; \ 833 }) 834 835 #define vhost_get_user(vq, x, ptr) \ 836 ({ \ 837 int ret; \ 838 if (!vq->iotlb) { \ 839 ret = __get_user(x, ptr); \ 840 } else { \ 841 __typeof__(ptr) from = \ 842 (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \ 843 if (from != NULL) \ 844 ret = __get_user(x, from); \ 845 else \ 846 ret = -EFAULT; \ 847 } \ 848 ret; \ 849 }) 850 851 static void vhost_dev_lock_vqs(struct vhost_dev *d) 852 { 853 int i = 0; 854 for (i = 0; i < d->nvqs; ++i) 855 mutex_lock(&d->vqs[i]->mutex); 856 } 857 858 static void vhost_dev_unlock_vqs(struct vhost_dev *d) 859 { 860 int i = 0; 861 for (i = 0; i < d->nvqs; ++i) 862 mutex_unlock(&d->vqs[i]->mutex); 863 } 864 865 static int vhost_new_umem_range(struct vhost_umem *umem, 866 u64 start, u64 size, u64 end, 867 u64 userspace_addr, int perm) 868 { 869 struct vhost_umem_node *tmp, *node = kmalloc(sizeof(*node), GFP_ATOMIC); 870 871 if (!node) 872 return -ENOMEM; 873 874 if (umem->numem == max_iotlb_entries) { 875 tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link); 876 vhost_umem_free(umem, tmp); 877 } 878 879 node->start = start; 880 node->size = size; 881 node->last = end; 882 node->userspace_addr = userspace_addr; 883 node->perm = perm; 884 INIT_LIST_HEAD(&node->link); 885 list_add_tail(&node->link, &umem->umem_list); 886 vhost_umem_interval_tree_insert(node, &umem->umem_tree); 887 umem->numem++; 888 889 return 0; 890 } 891 892 static void vhost_del_umem_range(struct vhost_umem *umem, 893 u64 start, u64 end) 894 { 895 struct vhost_umem_node *node; 896 897 while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, 898 start, end))) 899 vhost_umem_free(umem, node); 900 } 901 902 static void vhost_iotlb_notify_vq(struct vhost_dev *d, 903 struct vhost_iotlb_msg *msg) 904 { 905 struct vhost_msg_node *node, *n; 906 907 spin_lock(&d->iotlb_lock); 908 909 list_for_each_entry_safe(node, n, &d->pending_list, node) { 910 struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb; 911 if (msg->iova <= vq_msg->iova && 912 msg->iova + msg->size - 1 > vq_msg->iova && 913 vq_msg->type == VHOST_IOTLB_MISS) { 914 vhost_poll_queue(&node->vq->poll); 915 list_del(&node->node); 916 kfree(node); 917 } 918 } 919 920 spin_unlock(&d->iotlb_lock); 921 } 922 923 static int umem_access_ok(u64 uaddr, u64 size, int access) 924 { 925 unsigned long a = uaddr; 926 927 /* Make sure 64 bit math will not overflow. */ 928 if (vhost_overflow(uaddr, size)) 929 return -EFAULT; 930 931 if ((access & VHOST_ACCESS_RO) && 932 !access_ok(VERIFY_READ, (void __user *)a, size)) 933 return -EFAULT; 934 if ((access & VHOST_ACCESS_WO) && 935 !access_ok(VERIFY_WRITE, (void __user *)a, size)) 936 return -EFAULT; 937 return 0; 938 } 939 940 static int vhost_process_iotlb_msg(struct vhost_dev *dev, 941 struct vhost_iotlb_msg *msg) 942 { 943 int ret = 0; 944 945 vhost_dev_lock_vqs(dev); 946 switch (msg->type) { 947 case VHOST_IOTLB_UPDATE: 948 if (!dev->iotlb) { 949 ret = -EFAULT; 950 break; 951 } 952 if (umem_access_ok(msg->uaddr, msg->size, msg->perm)) { 953 ret = -EFAULT; 954 break; 955 } 956 if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size, 957 msg->iova + msg->size - 1, 958 msg->uaddr, msg->perm)) { 959 ret = -ENOMEM; 960 break; 961 } 962 vhost_iotlb_notify_vq(dev, msg); 963 break; 964 case VHOST_IOTLB_INVALIDATE: 965 vhost_del_umem_range(dev->iotlb, msg->iova, 966 msg->iova + msg->size - 1); 967 break; 968 default: 969 ret = -EINVAL; 970 break; 971 } 972 973 vhost_dev_unlock_vqs(dev); 974 return ret; 975 } 976 ssize_t vhost_chr_write_iter(struct vhost_dev *dev, 977 struct iov_iter *from) 978 { 979 struct vhost_msg_node node; 980 unsigned size = sizeof(struct vhost_msg); 981 size_t ret; 982 int err; 983 984 if (iov_iter_count(from) < size) 985 return 0; 986 ret = copy_from_iter(&node.msg, size, from); 987 if (ret != size) 988 goto done; 989 990 switch (node.msg.type) { 991 case VHOST_IOTLB_MSG: 992 err = vhost_process_iotlb_msg(dev, &node.msg.iotlb); 993 if (err) 994 ret = err; 995 break; 996 default: 997 ret = -EINVAL; 998 break; 999 } 1000 1001 done: 1002 return ret; 1003 } 1004 EXPORT_SYMBOL(vhost_chr_write_iter); 1005 1006 unsigned int vhost_chr_poll(struct file *file, struct vhost_dev *dev, 1007 poll_table *wait) 1008 { 1009 unsigned int mask = 0; 1010 1011 poll_wait(file, &dev->wait, wait); 1012 1013 if (!list_empty(&dev->read_list)) 1014 mask |= POLLIN | POLLRDNORM; 1015 1016 return mask; 1017 } 1018 EXPORT_SYMBOL(vhost_chr_poll); 1019 1020 ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to, 1021 int noblock) 1022 { 1023 DEFINE_WAIT(wait); 1024 struct vhost_msg_node *node; 1025 ssize_t ret = 0; 1026 unsigned size = sizeof(struct vhost_msg); 1027 1028 if (iov_iter_count(to) < size) 1029 return 0; 1030 1031 while (1) { 1032 if (!noblock) 1033 prepare_to_wait(&dev->wait, &wait, 1034 TASK_INTERRUPTIBLE); 1035 1036 node = vhost_dequeue_msg(dev, &dev->read_list); 1037 if (node) 1038 break; 1039 if (noblock) { 1040 ret = -EAGAIN; 1041 break; 1042 } 1043 if (signal_pending(current)) { 1044 ret = -ERESTARTSYS; 1045 break; 1046 } 1047 if (!dev->iotlb) { 1048 ret = -EBADFD; 1049 break; 1050 } 1051 1052 schedule(); 1053 } 1054 1055 if (!noblock) 1056 finish_wait(&dev->wait, &wait); 1057 1058 if (node) { 1059 ret = copy_to_iter(&node->msg, size, to); 1060 1061 if (ret != size || node->msg.type != VHOST_IOTLB_MISS) { 1062 kfree(node); 1063 return ret; 1064 } 1065 1066 vhost_enqueue_msg(dev, &dev->pending_list, node); 1067 } 1068 1069 return ret; 1070 } 1071 EXPORT_SYMBOL_GPL(vhost_chr_read_iter); 1072 1073 static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access) 1074 { 1075 struct vhost_dev *dev = vq->dev; 1076 struct vhost_msg_node *node; 1077 struct vhost_iotlb_msg *msg; 1078 1079 node = vhost_new_msg(vq, VHOST_IOTLB_MISS); 1080 if (!node) 1081 return -ENOMEM; 1082 1083 msg = &node->msg.iotlb; 1084 msg->type = VHOST_IOTLB_MISS; 1085 msg->iova = iova; 1086 msg->perm = access; 1087 1088 vhost_enqueue_msg(dev, &dev->read_list, node); 1089 1090 return 0; 1091 } 1092 1093 static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, 1094 struct vring_desc __user *desc, 1095 struct vring_avail __user *avail, 1096 struct vring_used __user *used) 1097 1098 { 1099 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 1100 1101 return access_ok(VERIFY_READ, desc, num * sizeof *desc) && 1102 access_ok(VERIFY_READ, avail, 1103 sizeof *avail + num * sizeof *avail->ring + s) && 1104 access_ok(VERIFY_WRITE, used, 1105 sizeof *used + num * sizeof *used->ring + s); 1106 } 1107 1108 static int iotlb_access_ok(struct vhost_virtqueue *vq, 1109 int access, u64 addr, u64 len) 1110 { 1111 const struct vhost_umem_node *node; 1112 struct vhost_umem *umem = vq->iotlb; 1113 u64 s = 0, size; 1114 1115 while (len > s) { 1116 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, 1117 addr, 1118 addr + len - 1); 1119 if (node == NULL || node->start > addr) { 1120 vhost_iotlb_miss(vq, addr, access); 1121 return false; 1122 } else if (!(node->perm & access)) { 1123 /* Report the possible access violation by 1124 * request another translation from userspace. 1125 */ 1126 return false; 1127 } 1128 1129 size = node->size - addr + node->start; 1130 s += size; 1131 addr += size; 1132 } 1133 1134 return true; 1135 } 1136 1137 int vq_iotlb_prefetch(struct vhost_virtqueue *vq) 1138 { 1139 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 1140 unsigned int num = vq->num; 1141 1142 if (!vq->iotlb) 1143 return 1; 1144 1145 return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, 1146 num * sizeof *vq->desc) && 1147 iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail, 1148 sizeof *vq->avail + 1149 num * sizeof *vq->avail->ring + s) && 1150 iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used, 1151 sizeof *vq->used + 1152 num * sizeof *vq->used->ring + s); 1153 } 1154 EXPORT_SYMBOL_GPL(vq_iotlb_prefetch); 1155 1156 /* Can we log writes? */ 1157 /* Caller should have device mutex but not vq mutex */ 1158 int vhost_log_access_ok(struct vhost_dev *dev) 1159 { 1160 return memory_access_ok(dev, dev->umem, 1); 1161 } 1162 EXPORT_SYMBOL_GPL(vhost_log_access_ok); 1163 1164 /* Verify access for write logging. */ 1165 /* Caller should have vq mutex and device mutex */ 1166 static int vq_log_access_ok(struct vhost_virtqueue *vq, 1167 void __user *log_base) 1168 { 1169 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 1170 1171 return vq_memory_access_ok(log_base, vq->umem, 1172 vhost_has_feature(vq, VHOST_F_LOG_ALL)) && 1173 (!vq->log_used || log_access_ok(log_base, vq->log_addr, 1174 sizeof *vq->used + 1175 vq->num * sizeof *vq->used->ring + s)); 1176 } 1177 1178 /* Can we start vq? */ 1179 /* Caller should have vq mutex and device mutex */ 1180 int vhost_vq_access_ok(struct vhost_virtqueue *vq) 1181 { 1182 if (vq->iotlb) { 1183 /* When device IOTLB was used, the access validation 1184 * will be validated during prefetching. 1185 */ 1186 return 1; 1187 } 1188 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used) && 1189 vq_log_access_ok(vq, vq->log_base); 1190 } 1191 EXPORT_SYMBOL_GPL(vhost_vq_access_ok); 1192 1193 static struct vhost_umem *vhost_umem_alloc(void) 1194 { 1195 struct vhost_umem *umem = vhost_kvzalloc(sizeof(*umem)); 1196 1197 if (!umem) 1198 return NULL; 1199 1200 umem->umem_tree = RB_ROOT; 1201 umem->numem = 0; 1202 INIT_LIST_HEAD(&umem->umem_list); 1203 1204 return umem; 1205 } 1206 1207 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) 1208 { 1209 struct vhost_memory mem, *newmem; 1210 struct vhost_memory_region *region; 1211 struct vhost_umem *newumem, *oldumem; 1212 unsigned long size = offsetof(struct vhost_memory, regions); 1213 int i; 1214 1215 if (copy_from_user(&mem, m, size)) 1216 return -EFAULT; 1217 if (mem.padding) 1218 return -EOPNOTSUPP; 1219 if (mem.nregions > max_mem_regions) 1220 return -E2BIG; 1221 newmem = vhost_kvzalloc(size + mem.nregions * sizeof(*m->regions)); 1222 if (!newmem) 1223 return -ENOMEM; 1224 1225 memcpy(newmem, &mem, size); 1226 if (copy_from_user(newmem->regions, m->regions, 1227 mem.nregions * sizeof *m->regions)) { 1228 kvfree(newmem); 1229 return -EFAULT; 1230 } 1231 1232 newumem = vhost_umem_alloc(); 1233 if (!newumem) { 1234 kvfree(newmem); 1235 return -ENOMEM; 1236 } 1237 1238 for (region = newmem->regions; 1239 region < newmem->regions + mem.nregions; 1240 region++) { 1241 if (vhost_new_umem_range(newumem, 1242 region->guest_phys_addr, 1243 region->memory_size, 1244 region->guest_phys_addr + 1245 region->memory_size - 1, 1246 region->userspace_addr, 1247 VHOST_ACCESS_RW)) 1248 goto err; 1249 } 1250 1251 if (!memory_access_ok(d, newumem, 0)) 1252 goto err; 1253 1254 oldumem = d->umem; 1255 d->umem = newumem; 1256 1257 /* All memory accesses are done under some VQ mutex. */ 1258 for (i = 0; i < d->nvqs; ++i) { 1259 mutex_lock(&d->vqs[i]->mutex); 1260 d->vqs[i]->umem = newumem; 1261 mutex_unlock(&d->vqs[i]->mutex); 1262 } 1263 1264 kvfree(newmem); 1265 vhost_umem_clean(oldumem); 1266 return 0; 1267 1268 err: 1269 vhost_umem_clean(newumem); 1270 kvfree(newmem); 1271 return -EFAULT; 1272 } 1273 1274 long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp) 1275 { 1276 struct file *eventfp, *filep = NULL; 1277 bool pollstart = false, pollstop = false; 1278 struct eventfd_ctx *ctx = NULL; 1279 u32 __user *idxp = argp; 1280 struct vhost_virtqueue *vq; 1281 struct vhost_vring_state s; 1282 struct vhost_vring_file f; 1283 struct vhost_vring_addr a; 1284 u32 idx; 1285 long r; 1286 1287 r = get_user(idx, idxp); 1288 if (r < 0) 1289 return r; 1290 if (idx >= d->nvqs) 1291 return -ENOBUFS; 1292 1293 vq = d->vqs[idx]; 1294 1295 mutex_lock(&vq->mutex); 1296 1297 switch (ioctl) { 1298 case VHOST_SET_VRING_NUM: 1299 /* Resizing ring with an active backend? 1300 * You don't want to do that. */ 1301 if (vq->private_data) { 1302 r = -EBUSY; 1303 break; 1304 } 1305 if (copy_from_user(&s, argp, sizeof s)) { 1306 r = -EFAULT; 1307 break; 1308 } 1309 if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) { 1310 r = -EINVAL; 1311 break; 1312 } 1313 vq->num = s.num; 1314 break; 1315 case VHOST_SET_VRING_BASE: 1316 /* Moving base with an active backend? 1317 * You don't want to do that. */ 1318 if (vq->private_data) { 1319 r = -EBUSY; 1320 break; 1321 } 1322 if (copy_from_user(&s, argp, sizeof s)) { 1323 r = -EFAULT; 1324 break; 1325 } 1326 if (s.num > 0xffff) { 1327 r = -EINVAL; 1328 break; 1329 } 1330 vq->last_avail_idx = vq->last_used_event = s.num; 1331 /* Forget the cached index value. */ 1332 vq->avail_idx = vq->last_avail_idx; 1333 break; 1334 case VHOST_GET_VRING_BASE: 1335 s.index = idx; 1336 s.num = vq->last_avail_idx; 1337 if (copy_to_user(argp, &s, sizeof s)) 1338 r = -EFAULT; 1339 break; 1340 case VHOST_SET_VRING_ADDR: 1341 if (copy_from_user(&a, argp, sizeof a)) { 1342 r = -EFAULT; 1343 break; 1344 } 1345 if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) { 1346 r = -EOPNOTSUPP; 1347 break; 1348 } 1349 /* For 32bit, verify that the top 32bits of the user 1350 data are set to zero. */ 1351 if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr || 1352 (u64)(unsigned long)a.used_user_addr != a.used_user_addr || 1353 (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) { 1354 r = -EFAULT; 1355 break; 1356 } 1357 1358 /* Make sure it's safe to cast pointers to vring types. */ 1359 BUILD_BUG_ON(__alignof__ *vq->avail > VRING_AVAIL_ALIGN_SIZE); 1360 BUILD_BUG_ON(__alignof__ *vq->used > VRING_USED_ALIGN_SIZE); 1361 if ((a.avail_user_addr & (VRING_AVAIL_ALIGN_SIZE - 1)) || 1362 (a.used_user_addr & (VRING_USED_ALIGN_SIZE - 1)) || 1363 (a.log_guest_addr & (VRING_USED_ALIGN_SIZE - 1))) { 1364 r = -EINVAL; 1365 break; 1366 } 1367 1368 /* We only verify access here if backend is configured. 1369 * If it is not, we don't as size might not have been setup. 1370 * We will verify when backend is configured. */ 1371 if (vq->private_data) { 1372 if (!vq_access_ok(vq, vq->num, 1373 (void __user *)(unsigned long)a.desc_user_addr, 1374 (void __user *)(unsigned long)a.avail_user_addr, 1375 (void __user *)(unsigned long)a.used_user_addr)) { 1376 r = -EINVAL; 1377 break; 1378 } 1379 1380 /* Also validate log access for used ring if enabled. */ 1381 if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) && 1382 !log_access_ok(vq->log_base, a.log_guest_addr, 1383 sizeof *vq->used + 1384 vq->num * sizeof *vq->used->ring)) { 1385 r = -EINVAL; 1386 break; 1387 } 1388 } 1389 1390 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); 1391 vq->desc = (void __user *)(unsigned long)a.desc_user_addr; 1392 vq->avail = (void __user *)(unsigned long)a.avail_user_addr; 1393 vq->log_addr = a.log_guest_addr; 1394 vq->used = (void __user *)(unsigned long)a.used_user_addr; 1395 break; 1396 case VHOST_SET_VRING_KICK: 1397 if (copy_from_user(&f, argp, sizeof f)) { 1398 r = -EFAULT; 1399 break; 1400 } 1401 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd); 1402 if (IS_ERR(eventfp)) { 1403 r = PTR_ERR(eventfp); 1404 break; 1405 } 1406 if (eventfp != vq->kick) { 1407 pollstop = (filep = vq->kick) != NULL; 1408 pollstart = (vq->kick = eventfp) != NULL; 1409 } else 1410 filep = eventfp; 1411 break; 1412 case VHOST_SET_VRING_CALL: 1413 if (copy_from_user(&f, argp, sizeof f)) { 1414 r = -EFAULT; 1415 break; 1416 } 1417 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd); 1418 if (IS_ERR(eventfp)) { 1419 r = PTR_ERR(eventfp); 1420 break; 1421 } 1422 if (eventfp != vq->call) { 1423 filep = vq->call; 1424 ctx = vq->call_ctx; 1425 vq->call = eventfp; 1426 vq->call_ctx = eventfp ? 1427 eventfd_ctx_fileget(eventfp) : NULL; 1428 } else 1429 filep = eventfp; 1430 break; 1431 case VHOST_SET_VRING_ERR: 1432 if (copy_from_user(&f, argp, sizeof f)) { 1433 r = -EFAULT; 1434 break; 1435 } 1436 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd); 1437 if (IS_ERR(eventfp)) { 1438 r = PTR_ERR(eventfp); 1439 break; 1440 } 1441 if (eventfp != vq->error) { 1442 filep = vq->error; 1443 vq->error = eventfp; 1444 ctx = vq->error_ctx; 1445 vq->error_ctx = eventfp ? 1446 eventfd_ctx_fileget(eventfp) : NULL; 1447 } else 1448 filep = eventfp; 1449 break; 1450 case VHOST_SET_VRING_ENDIAN: 1451 r = vhost_set_vring_endian(vq, argp); 1452 break; 1453 case VHOST_GET_VRING_ENDIAN: 1454 r = vhost_get_vring_endian(vq, idx, argp); 1455 break; 1456 case VHOST_SET_VRING_BUSYLOOP_TIMEOUT: 1457 if (copy_from_user(&s, argp, sizeof(s))) { 1458 r = -EFAULT; 1459 break; 1460 } 1461 vq->busyloop_timeout = s.num; 1462 break; 1463 case VHOST_GET_VRING_BUSYLOOP_TIMEOUT: 1464 s.index = idx; 1465 s.num = vq->busyloop_timeout; 1466 if (copy_to_user(argp, &s, sizeof(s))) 1467 r = -EFAULT; 1468 break; 1469 default: 1470 r = -ENOIOCTLCMD; 1471 } 1472 1473 if (pollstop && vq->handle_kick) 1474 vhost_poll_stop(&vq->poll); 1475 1476 if (ctx) 1477 eventfd_ctx_put(ctx); 1478 if (filep) 1479 fput(filep); 1480 1481 if (pollstart && vq->handle_kick) 1482 r = vhost_poll_start(&vq->poll, vq->kick); 1483 1484 mutex_unlock(&vq->mutex); 1485 1486 if (pollstop && vq->handle_kick) 1487 vhost_poll_flush(&vq->poll); 1488 return r; 1489 } 1490 EXPORT_SYMBOL_GPL(vhost_vring_ioctl); 1491 1492 int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled) 1493 { 1494 struct vhost_umem *niotlb, *oiotlb; 1495 int i; 1496 1497 niotlb = vhost_umem_alloc(); 1498 if (!niotlb) 1499 return -ENOMEM; 1500 1501 oiotlb = d->iotlb; 1502 d->iotlb = niotlb; 1503 1504 for (i = 0; i < d->nvqs; ++i) { 1505 mutex_lock(&d->vqs[i]->mutex); 1506 d->vqs[i]->iotlb = niotlb; 1507 mutex_unlock(&d->vqs[i]->mutex); 1508 } 1509 1510 vhost_umem_clean(oiotlb); 1511 1512 return 0; 1513 } 1514 EXPORT_SYMBOL_GPL(vhost_init_device_iotlb); 1515 1516 /* Caller must have device mutex */ 1517 long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) 1518 { 1519 struct file *eventfp, *filep = NULL; 1520 struct eventfd_ctx *ctx = NULL; 1521 u64 p; 1522 long r; 1523 int i, fd; 1524 1525 /* If you are not the owner, you can become one */ 1526 if (ioctl == VHOST_SET_OWNER) { 1527 r = vhost_dev_set_owner(d); 1528 goto done; 1529 } 1530 1531 /* You must be the owner to do anything else */ 1532 r = vhost_dev_check_owner(d); 1533 if (r) 1534 goto done; 1535 1536 switch (ioctl) { 1537 case VHOST_SET_MEM_TABLE: 1538 r = vhost_set_memory(d, argp); 1539 break; 1540 case VHOST_SET_LOG_BASE: 1541 if (copy_from_user(&p, argp, sizeof p)) { 1542 r = -EFAULT; 1543 break; 1544 } 1545 if ((u64)(unsigned long)p != p) { 1546 r = -EFAULT; 1547 break; 1548 } 1549 for (i = 0; i < d->nvqs; ++i) { 1550 struct vhost_virtqueue *vq; 1551 void __user *base = (void __user *)(unsigned long)p; 1552 vq = d->vqs[i]; 1553 mutex_lock(&vq->mutex); 1554 /* If ring is inactive, will check when it's enabled. */ 1555 if (vq->private_data && !vq_log_access_ok(vq, base)) 1556 r = -EFAULT; 1557 else 1558 vq->log_base = base; 1559 mutex_unlock(&vq->mutex); 1560 } 1561 break; 1562 case VHOST_SET_LOG_FD: 1563 r = get_user(fd, (int __user *)argp); 1564 if (r < 0) 1565 break; 1566 eventfp = fd == -1 ? NULL : eventfd_fget(fd); 1567 if (IS_ERR(eventfp)) { 1568 r = PTR_ERR(eventfp); 1569 break; 1570 } 1571 if (eventfp != d->log_file) { 1572 filep = d->log_file; 1573 d->log_file = eventfp; 1574 ctx = d->log_ctx; 1575 d->log_ctx = eventfp ? 1576 eventfd_ctx_fileget(eventfp) : NULL; 1577 } else 1578 filep = eventfp; 1579 for (i = 0; i < d->nvqs; ++i) { 1580 mutex_lock(&d->vqs[i]->mutex); 1581 d->vqs[i]->log_ctx = d->log_ctx; 1582 mutex_unlock(&d->vqs[i]->mutex); 1583 } 1584 if (ctx) 1585 eventfd_ctx_put(ctx); 1586 if (filep) 1587 fput(filep); 1588 break; 1589 default: 1590 r = -ENOIOCTLCMD; 1591 break; 1592 } 1593 done: 1594 return r; 1595 } 1596 EXPORT_SYMBOL_GPL(vhost_dev_ioctl); 1597 1598 /* TODO: This is really inefficient. We need something like get_user() 1599 * (instruction directly accesses the data, with an exception table entry 1600 * returning -EFAULT). See Documentation/x86/exception-tables.txt. 1601 */ 1602 static int set_bit_to_user(int nr, void __user *addr) 1603 { 1604 unsigned long log = (unsigned long)addr; 1605 struct page *page; 1606 void *base; 1607 int bit = nr + (log % PAGE_SIZE) * 8; 1608 int r; 1609 1610 r = get_user_pages_fast(log, 1, 1, &page); 1611 if (r < 0) 1612 return r; 1613 BUG_ON(r != 1); 1614 base = kmap_atomic(page); 1615 set_bit(bit, base); 1616 kunmap_atomic(base); 1617 set_page_dirty_lock(page); 1618 put_page(page); 1619 return 0; 1620 } 1621 1622 static int log_write(void __user *log_base, 1623 u64 write_address, u64 write_length) 1624 { 1625 u64 write_page = write_address / VHOST_PAGE_SIZE; 1626 int r; 1627 1628 if (!write_length) 1629 return 0; 1630 write_length += write_address % VHOST_PAGE_SIZE; 1631 for (;;) { 1632 u64 base = (u64)(unsigned long)log_base; 1633 u64 log = base + write_page / 8; 1634 int bit = write_page % 8; 1635 if ((u64)(unsigned long)log != log) 1636 return -EFAULT; 1637 r = set_bit_to_user(bit, (void __user *)(unsigned long)log); 1638 if (r < 0) 1639 return r; 1640 if (write_length <= VHOST_PAGE_SIZE) 1641 break; 1642 write_length -= VHOST_PAGE_SIZE; 1643 write_page += 1; 1644 } 1645 return r; 1646 } 1647 1648 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 1649 unsigned int log_num, u64 len) 1650 { 1651 int i, r; 1652 1653 /* Make sure data written is seen before log. */ 1654 smp_wmb(); 1655 for (i = 0; i < log_num; ++i) { 1656 u64 l = min(log[i].len, len); 1657 r = log_write(vq->log_base, log[i].addr, l); 1658 if (r < 0) 1659 return r; 1660 len -= l; 1661 if (!len) { 1662 if (vq->log_ctx) 1663 eventfd_signal(vq->log_ctx, 1); 1664 return 0; 1665 } 1666 } 1667 /* Length written exceeds what we have stored. This is a bug. */ 1668 BUG(); 1669 return 0; 1670 } 1671 EXPORT_SYMBOL_GPL(vhost_log_write); 1672 1673 static int vhost_update_used_flags(struct vhost_virtqueue *vq) 1674 { 1675 void __user *used; 1676 if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), 1677 &vq->used->flags) < 0) 1678 return -EFAULT; 1679 if (unlikely(vq->log_used)) { 1680 /* Make sure the flag is seen before log. */ 1681 smp_wmb(); 1682 /* Log used flag write. */ 1683 used = &vq->used->flags; 1684 log_write(vq->log_base, vq->log_addr + 1685 (used - (void __user *)vq->used), 1686 sizeof vq->used->flags); 1687 if (vq->log_ctx) 1688 eventfd_signal(vq->log_ctx, 1); 1689 } 1690 return 0; 1691 } 1692 1693 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) 1694 { 1695 if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), 1696 vhost_avail_event(vq))) 1697 return -EFAULT; 1698 if (unlikely(vq->log_used)) { 1699 void __user *used; 1700 /* Make sure the event is seen before log. */ 1701 smp_wmb(); 1702 /* Log avail event write */ 1703 used = vhost_avail_event(vq); 1704 log_write(vq->log_base, vq->log_addr + 1705 (used - (void __user *)vq->used), 1706 sizeof *vhost_avail_event(vq)); 1707 if (vq->log_ctx) 1708 eventfd_signal(vq->log_ctx, 1); 1709 } 1710 return 0; 1711 } 1712 1713 int vhost_vq_init_access(struct vhost_virtqueue *vq) 1714 { 1715 __virtio16 last_used_idx; 1716 int r; 1717 bool is_le = vq->is_le; 1718 1719 if (!vq->private_data) 1720 return 0; 1721 1722 vhost_init_is_le(vq); 1723 1724 r = vhost_update_used_flags(vq); 1725 if (r) 1726 goto err; 1727 vq->signalled_used_valid = false; 1728 if (!vq->iotlb && 1729 !access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx)) { 1730 r = -EFAULT; 1731 goto err; 1732 } 1733 r = vhost_get_user(vq, last_used_idx, &vq->used->idx); 1734 if (r) { 1735 vq_err(vq, "Can't access used idx at %p\n", 1736 &vq->used->idx); 1737 goto err; 1738 } 1739 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx); 1740 return 0; 1741 1742 err: 1743 vq->is_le = is_le; 1744 return r; 1745 } 1746 EXPORT_SYMBOL_GPL(vhost_vq_init_access); 1747 1748 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, 1749 struct iovec iov[], int iov_size, int access) 1750 { 1751 const struct vhost_umem_node *node; 1752 struct vhost_dev *dev = vq->dev; 1753 struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem; 1754 struct iovec *_iov; 1755 u64 s = 0; 1756 int ret = 0; 1757 1758 while ((u64)len > s) { 1759 u64 size; 1760 if (unlikely(ret >= iov_size)) { 1761 ret = -ENOBUFS; 1762 break; 1763 } 1764 1765 node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, 1766 addr, addr + len - 1); 1767 if (node == NULL || node->start > addr) { 1768 if (umem != dev->iotlb) { 1769 ret = -EFAULT; 1770 break; 1771 } 1772 ret = -EAGAIN; 1773 break; 1774 } else if (!(node->perm & access)) { 1775 ret = -EPERM; 1776 break; 1777 } 1778 1779 _iov = iov + ret; 1780 size = node->size - addr + node->start; 1781 _iov->iov_len = min((u64)len - s, size); 1782 _iov->iov_base = (void __user *)(unsigned long) 1783 (node->userspace_addr + addr - node->start); 1784 s += size; 1785 addr += size; 1786 ++ret; 1787 } 1788 1789 if (ret == -EAGAIN) 1790 vhost_iotlb_miss(vq, addr, access); 1791 return ret; 1792 } 1793 1794 /* Each buffer in the virtqueues is actually a chain of descriptors. This 1795 * function returns the next descriptor in the chain, 1796 * or -1U if we're at the end. */ 1797 static unsigned next_desc(struct vhost_virtqueue *vq, struct vring_desc *desc) 1798 { 1799 unsigned int next; 1800 1801 /* If this descriptor says it doesn't chain, we're done. */ 1802 if (!(desc->flags & cpu_to_vhost16(vq, VRING_DESC_F_NEXT))) 1803 return -1U; 1804 1805 /* Check they're not leading us off end of descriptors. */ 1806 next = vhost16_to_cpu(vq, desc->next); 1807 /* Make sure compiler knows to grab that: we don't want it changing! */ 1808 /* We will use the result as an index in an array, so most 1809 * architectures only need a compiler barrier here. */ 1810 read_barrier_depends(); 1811 1812 return next; 1813 } 1814 1815 static int get_indirect(struct vhost_virtqueue *vq, 1816 struct iovec iov[], unsigned int iov_size, 1817 unsigned int *out_num, unsigned int *in_num, 1818 struct vhost_log *log, unsigned int *log_num, 1819 struct vring_desc *indirect) 1820 { 1821 struct vring_desc desc; 1822 unsigned int i = 0, count, found = 0; 1823 u32 len = vhost32_to_cpu(vq, indirect->len); 1824 struct iov_iter from; 1825 int ret, access; 1826 1827 /* Sanity check */ 1828 if (unlikely(len % sizeof desc)) { 1829 vq_err(vq, "Invalid length in indirect descriptor: " 1830 "len 0x%llx not multiple of 0x%zx\n", 1831 (unsigned long long)len, 1832 sizeof desc); 1833 return -EINVAL; 1834 } 1835 1836 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect, 1837 UIO_MAXIOV, VHOST_ACCESS_RO); 1838 if (unlikely(ret < 0)) { 1839 if (ret != -EAGAIN) 1840 vq_err(vq, "Translation failure %d in indirect.\n", ret); 1841 return ret; 1842 } 1843 iov_iter_init(&from, READ, vq->indirect, ret, len); 1844 1845 /* We will use the result as an address to read from, so most 1846 * architectures only need a compiler barrier here. */ 1847 read_barrier_depends(); 1848 1849 count = len / sizeof desc; 1850 /* Buffers are chained via a 16 bit next field, so 1851 * we can have at most 2^16 of these. */ 1852 if (unlikely(count > USHRT_MAX + 1)) { 1853 vq_err(vq, "Indirect buffer length too big: %d\n", 1854 indirect->len); 1855 return -E2BIG; 1856 } 1857 1858 do { 1859 unsigned iov_count = *in_num + *out_num; 1860 if (unlikely(++found > count)) { 1861 vq_err(vq, "Loop detected: last one at %u " 1862 "indirect size %u\n", 1863 i, count); 1864 return -EINVAL; 1865 } 1866 if (unlikely(!copy_from_iter_full(&desc, sizeof(desc), &from))) { 1867 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", 1868 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); 1869 return -EINVAL; 1870 } 1871 if (unlikely(desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT))) { 1872 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", 1873 i, (size_t)vhost64_to_cpu(vq, indirect->addr) + i * sizeof desc); 1874 return -EINVAL; 1875 } 1876 1877 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) 1878 access = VHOST_ACCESS_WO; 1879 else 1880 access = VHOST_ACCESS_RO; 1881 1882 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), 1883 vhost32_to_cpu(vq, desc.len), iov + iov_count, 1884 iov_size - iov_count, access); 1885 if (unlikely(ret < 0)) { 1886 if (ret != -EAGAIN) 1887 vq_err(vq, "Translation failure %d indirect idx %d\n", 1888 ret, i); 1889 return ret; 1890 } 1891 /* If this is an input descriptor, increment that count. */ 1892 if (access == VHOST_ACCESS_WO) { 1893 *in_num += ret; 1894 if (unlikely(log)) { 1895 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); 1896 log[*log_num].len = vhost32_to_cpu(vq, desc.len); 1897 ++*log_num; 1898 } 1899 } else { 1900 /* If it's an output descriptor, they're all supposed 1901 * to come before any input descriptors. */ 1902 if (unlikely(*in_num)) { 1903 vq_err(vq, "Indirect descriptor " 1904 "has out after in: idx %d\n", i); 1905 return -EINVAL; 1906 } 1907 *out_num += ret; 1908 } 1909 } while ((i = next_desc(vq, &desc)) != -1); 1910 return 0; 1911 } 1912 1913 /* This looks in the virtqueue and for the first available buffer, and converts 1914 * it to an iovec for convenient access. Since descriptors consist of some 1915 * number of output then some number of input descriptors, it's actually two 1916 * iovecs, but we pack them into one and note how many of each there were. 1917 * 1918 * This function returns the descriptor number found, or vq->num (which is 1919 * never a valid descriptor number) if none was found. A negative code is 1920 * returned on error. */ 1921 int vhost_get_vq_desc(struct vhost_virtqueue *vq, 1922 struct iovec iov[], unsigned int iov_size, 1923 unsigned int *out_num, unsigned int *in_num, 1924 struct vhost_log *log, unsigned int *log_num) 1925 { 1926 struct vring_desc desc; 1927 unsigned int i, head, found = 0; 1928 u16 last_avail_idx; 1929 __virtio16 avail_idx; 1930 __virtio16 ring_head; 1931 int ret, access; 1932 1933 /* Check it isn't doing very strange things with descriptor numbers. */ 1934 last_avail_idx = vq->last_avail_idx; 1935 if (unlikely(vhost_get_user(vq, avail_idx, &vq->avail->idx))) { 1936 vq_err(vq, "Failed to access avail idx at %p\n", 1937 &vq->avail->idx); 1938 return -EFAULT; 1939 } 1940 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); 1941 1942 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { 1943 vq_err(vq, "Guest moved used index from %u to %u", 1944 last_avail_idx, vq->avail_idx); 1945 return -EFAULT; 1946 } 1947 1948 /* If there's nothing new since last we looked, return invalid. */ 1949 if (vq->avail_idx == last_avail_idx) 1950 return vq->num; 1951 1952 /* Only get avail ring entries after they have been exposed by guest. */ 1953 smp_rmb(); 1954 1955 /* Grab the next descriptor number they're advertising, and increment 1956 * the index we've seen. */ 1957 if (unlikely(vhost_get_user(vq, ring_head, 1958 &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) { 1959 vq_err(vq, "Failed to read head: idx %d address %p\n", 1960 last_avail_idx, 1961 &vq->avail->ring[last_avail_idx % vq->num]); 1962 return -EFAULT; 1963 } 1964 1965 head = vhost16_to_cpu(vq, ring_head); 1966 1967 /* If their number is silly, that's an error. */ 1968 if (unlikely(head >= vq->num)) { 1969 vq_err(vq, "Guest says index %u > %u is available", 1970 head, vq->num); 1971 return -EINVAL; 1972 } 1973 1974 /* When we start there are none of either input nor output. */ 1975 *out_num = *in_num = 0; 1976 if (unlikely(log)) 1977 *log_num = 0; 1978 1979 i = head; 1980 do { 1981 unsigned iov_count = *in_num + *out_num; 1982 if (unlikely(i >= vq->num)) { 1983 vq_err(vq, "Desc index is %u > %u, head = %u", 1984 i, vq->num, head); 1985 return -EINVAL; 1986 } 1987 if (unlikely(++found > vq->num)) { 1988 vq_err(vq, "Loop detected: last one at %u " 1989 "vq size %u head %u\n", 1990 i, vq->num, head); 1991 return -EINVAL; 1992 } 1993 ret = vhost_copy_from_user(vq, &desc, vq->desc + i, 1994 sizeof desc); 1995 if (unlikely(ret)) { 1996 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", 1997 i, vq->desc + i); 1998 return -EFAULT; 1999 } 2000 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_INDIRECT)) { 2001 ret = get_indirect(vq, iov, iov_size, 2002 out_num, in_num, 2003 log, log_num, &desc); 2004 if (unlikely(ret < 0)) { 2005 if (ret != -EAGAIN) 2006 vq_err(vq, "Failure detected " 2007 "in indirect descriptor at idx %d\n", i); 2008 return ret; 2009 } 2010 continue; 2011 } 2012 2013 if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) 2014 access = VHOST_ACCESS_WO; 2015 else 2016 access = VHOST_ACCESS_RO; 2017 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), 2018 vhost32_to_cpu(vq, desc.len), iov + iov_count, 2019 iov_size - iov_count, access); 2020 if (unlikely(ret < 0)) { 2021 if (ret != -EAGAIN) 2022 vq_err(vq, "Translation failure %d descriptor idx %d\n", 2023 ret, i); 2024 return ret; 2025 } 2026 if (access == VHOST_ACCESS_WO) { 2027 /* If this is an input descriptor, 2028 * increment that count. */ 2029 *in_num += ret; 2030 if (unlikely(log)) { 2031 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); 2032 log[*log_num].len = vhost32_to_cpu(vq, desc.len); 2033 ++*log_num; 2034 } 2035 } else { 2036 /* If it's an output descriptor, they're all supposed 2037 * to come before any input descriptors. */ 2038 if (unlikely(*in_num)) { 2039 vq_err(vq, "Descriptor has out after in: " 2040 "idx %d\n", i); 2041 return -EINVAL; 2042 } 2043 *out_num += ret; 2044 } 2045 } while ((i = next_desc(vq, &desc)) != -1); 2046 2047 /* On success, increment avail index. */ 2048 vq->last_avail_idx++; 2049 2050 /* Assume notifications from guest are disabled at this point, 2051 * if they aren't we would need to update avail_event index. */ 2052 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); 2053 return head; 2054 } 2055 EXPORT_SYMBOL_GPL(vhost_get_vq_desc); 2056 2057 /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */ 2058 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) 2059 { 2060 vq->last_avail_idx -= n; 2061 } 2062 EXPORT_SYMBOL_GPL(vhost_discard_vq_desc); 2063 2064 /* After we've used one of their buffers, we tell them about it. We'll then 2065 * want to notify the guest, using eventfd. */ 2066 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) 2067 { 2068 struct vring_used_elem heads = { 2069 cpu_to_vhost32(vq, head), 2070 cpu_to_vhost32(vq, len) 2071 }; 2072 2073 return vhost_add_used_n(vq, &heads, 1); 2074 } 2075 EXPORT_SYMBOL_GPL(vhost_add_used); 2076 2077 static int __vhost_add_used_n(struct vhost_virtqueue *vq, 2078 struct vring_used_elem *heads, 2079 unsigned count) 2080 { 2081 struct vring_used_elem __user *used; 2082 u16 old, new; 2083 int start; 2084 2085 start = vq->last_used_idx & (vq->num - 1); 2086 used = vq->used->ring + start; 2087 if (count == 1) { 2088 if (vhost_put_user(vq, heads[0].id, &used->id)) { 2089 vq_err(vq, "Failed to write used id"); 2090 return -EFAULT; 2091 } 2092 if (vhost_put_user(vq, heads[0].len, &used->len)) { 2093 vq_err(vq, "Failed to write used len"); 2094 return -EFAULT; 2095 } 2096 } else if (vhost_copy_to_user(vq, used, heads, count * sizeof *used)) { 2097 vq_err(vq, "Failed to write used"); 2098 return -EFAULT; 2099 } 2100 if (unlikely(vq->log_used)) { 2101 /* Make sure data is seen before log. */ 2102 smp_wmb(); 2103 /* Log used ring entry write. */ 2104 log_write(vq->log_base, 2105 vq->log_addr + 2106 ((void __user *)used - (void __user *)vq->used), 2107 count * sizeof *used); 2108 } 2109 old = vq->last_used_idx; 2110 new = (vq->last_used_idx += count); 2111 /* If the driver never bothers to signal in a very long while, 2112 * used index might wrap around. If that happens, invalidate 2113 * signalled_used index we stored. TODO: make sure driver 2114 * signals at least once in 2^16 and remove this. */ 2115 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) 2116 vq->signalled_used_valid = false; 2117 return 0; 2118 } 2119 2120 /* After we've used one of their buffers, we tell them about it. We'll then 2121 * want to notify the guest, using eventfd. */ 2122 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, 2123 unsigned count) 2124 { 2125 int start, n, r; 2126 2127 start = vq->last_used_idx & (vq->num - 1); 2128 n = vq->num - start; 2129 if (n < count) { 2130 r = __vhost_add_used_n(vq, heads, n); 2131 if (r < 0) 2132 return r; 2133 heads += n; 2134 count -= n; 2135 } 2136 r = __vhost_add_used_n(vq, heads, count); 2137 2138 /* Make sure buffer is written before we update index. */ 2139 smp_wmb(); 2140 if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), 2141 &vq->used->idx)) { 2142 vq_err(vq, "Failed to increment used idx"); 2143 return -EFAULT; 2144 } 2145 if (unlikely(vq->log_used)) { 2146 /* Log used index update. */ 2147 log_write(vq->log_base, 2148 vq->log_addr + offsetof(struct vring_used, idx), 2149 sizeof vq->used->idx); 2150 if (vq->log_ctx) 2151 eventfd_signal(vq->log_ctx, 1); 2152 } 2153 return r; 2154 } 2155 EXPORT_SYMBOL_GPL(vhost_add_used_n); 2156 2157 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2158 { 2159 __u16 old, new; 2160 __virtio16 event; 2161 bool v; 2162 2163 if (vhost_has_feature(vq, VIRTIO_F_NOTIFY_ON_EMPTY) && 2164 unlikely(vq->avail_idx == vq->last_avail_idx)) 2165 return true; 2166 2167 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2168 __virtio16 flags; 2169 /* Flush out used index updates. This is paired 2170 * with the barrier that the Guest executes when enabling 2171 * interrupts. */ 2172 smp_mb(); 2173 if (vhost_get_user(vq, flags, &vq->avail->flags)) { 2174 vq_err(vq, "Failed to get flags"); 2175 return true; 2176 } 2177 return !(flags & cpu_to_vhost16(vq, VRING_AVAIL_F_NO_INTERRUPT)); 2178 } 2179 old = vq->signalled_used; 2180 v = vq->signalled_used_valid; 2181 new = vq->signalled_used = vq->last_used_idx; 2182 vq->signalled_used_valid = true; 2183 2184 if (unlikely(!v)) 2185 return true; 2186 2187 /* We're sure if the following conditions are met, there's no 2188 * need to notify guest: 2189 * 1) cached used event is ahead of new 2190 * 2) old to new updating does not cross cached used event. */ 2191 if (vring_need_event(vq->last_used_event, new + vq->num, new) && 2192 !vring_need_event(vq->last_used_event, new, old)) 2193 return false; 2194 2195 /* Flush out used index updates. This is paired 2196 * with the barrier that the Guest executes when enabling 2197 * interrupts. */ 2198 smp_mb(); 2199 2200 if (vhost_get_user(vq, event, vhost_used_event(vq))) { 2201 vq_err(vq, "Failed to get used event idx"); 2202 return true; 2203 } 2204 vq->last_used_event = vhost16_to_cpu(vq, event); 2205 2206 return vring_need_event(vq->last_used_event, new, old); 2207 } 2208 2209 /* This actually signals the guest, using eventfd. */ 2210 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2211 { 2212 /* Signal the Guest tell them we used something up. */ 2213 if (vq->call_ctx && vhost_notify(dev, vq)) 2214 eventfd_signal(vq->call_ctx, 1); 2215 } 2216 EXPORT_SYMBOL_GPL(vhost_signal); 2217 2218 /* And here's the combo meal deal. Supersize me! */ 2219 void vhost_add_used_and_signal(struct vhost_dev *dev, 2220 struct vhost_virtqueue *vq, 2221 unsigned int head, int len) 2222 { 2223 vhost_add_used(vq, head, len); 2224 vhost_signal(dev, vq); 2225 } 2226 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal); 2227 2228 /* multi-buffer version of vhost_add_used_and_signal */ 2229 void vhost_add_used_and_signal_n(struct vhost_dev *dev, 2230 struct vhost_virtqueue *vq, 2231 struct vring_used_elem *heads, unsigned count) 2232 { 2233 vhost_add_used_n(vq, heads, count); 2234 vhost_signal(dev, vq); 2235 } 2236 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n); 2237 2238 /* return true if we're sure that avaiable ring is empty */ 2239 bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2240 { 2241 __virtio16 avail_idx; 2242 int r; 2243 2244 if (vq->avail_idx != vq->last_avail_idx) 2245 return false; 2246 2247 r = vhost_get_user(vq, avail_idx, &vq->avail->idx); 2248 if (unlikely(r)) 2249 return false; 2250 vq->avail_idx = vhost16_to_cpu(vq, avail_idx); 2251 2252 return vq->avail_idx == vq->last_avail_idx; 2253 } 2254 EXPORT_SYMBOL_GPL(vhost_vq_avail_empty); 2255 2256 /* OK, now we need to know about added descriptors. */ 2257 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2258 { 2259 __virtio16 avail_idx; 2260 int r; 2261 2262 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) 2263 return false; 2264 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; 2265 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2266 r = vhost_update_used_flags(vq); 2267 if (r) { 2268 vq_err(vq, "Failed to enable notification at %p: %d\n", 2269 &vq->used->flags, r); 2270 return false; 2271 } 2272 } else { 2273 r = vhost_update_avail_event(vq, vq->avail_idx); 2274 if (r) { 2275 vq_err(vq, "Failed to update avail event index at %p: %d\n", 2276 vhost_avail_event(vq), r); 2277 return false; 2278 } 2279 } 2280 /* They could have slipped one in as we were doing that: make 2281 * sure it's written, then check again. */ 2282 smp_mb(); 2283 r = vhost_get_user(vq, avail_idx, &vq->avail->idx); 2284 if (r) { 2285 vq_err(vq, "Failed to check avail idx at %p: %d\n", 2286 &vq->avail->idx, r); 2287 return false; 2288 } 2289 2290 return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx; 2291 } 2292 EXPORT_SYMBOL_GPL(vhost_enable_notify); 2293 2294 /* We don't need to be notified again. */ 2295 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 2296 { 2297 int r; 2298 2299 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) 2300 return; 2301 vq->used_flags |= VRING_USED_F_NO_NOTIFY; 2302 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2303 r = vhost_update_used_flags(vq); 2304 if (r) 2305 vq_err(vq, "Failed to enable notification at %p: %d\n", 2306 &vq->used->flags, r); 2307 } 2308 } 2309 EXPORT_SYMBOL_GPL(vhost_disable_notify); 2310 2311 /* Create a new message. */ 2312 struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type) 2313 { 2314 struct vhost_msg_node *node = kmalloc(sizeof *node, GFP_KERNEL); 2315 if (!node) 2316 return NULL; 2317 node->vq = vq; 2318 node->msg.type = type; 2319 return node; 2320 } 2321 EXPORT_SYMBOL_GPL(vhost_new_msg); 2322 2323 void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head, 2324 struct vhost_msg_node *node) 2325 { 2326 spin_lock(&dev->iotlb_lock); 2327 list_add_tail(&node->node, head); 2328 spin_unlock(&dev->iotlb_lock); 2329 2330 wake_up_interruptible_poll(&dev->wait, POLLIN | POLLRDNORM); 2331 } 2332 EXPORT_SYMBOL_GPL(vhost_enqueue_msg); 2333 2334 struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev, 2335 struct list_head *head) 2336 { 2337 struct vhost_msg_node *node = NULL; 2338 2339 spin_lock(&dev->iotlb_lock); 2340 if (!list_empty(head)) { 2341 node = list_first_entry(head, struct vhost_msg_node, 2342 node); 2343 list_del(&node->node); 2344 } 2345 spin_unlock(&dev->iotlb_lock); 2346 2347 return node; 2348 } 2349 EXPORT_SYMBOL_GPL(vhost_dequeue_msg); 2350 2351 2352 static int __init vhost_init(void) 2353 { 2354 return 0; 2355 } 2356 2357 static void __exit vhost_exit(void) 2358 { 2359 } 2360 2361 module_init(vhost_init); 2362 module_exit(vhost_exit); 2363 2364 MODULE_VERSION("0.0.1"); 2365 MODULE_LICENSE("GPL v2"); 2366 MODULE_AUTHOR("Michael S. Tsirkin"); 2367 MODULE_DESCRIPTION("Host kernel accelerator for virtio"); 2368