1 /* Copyright (C) 2009 Red Hat, Inc. 2 * Copyright (C) 2006 Rusty Russell IBM Corporation 3 * 4 * Author: Michael S. Tsirkin <mst@redhat.com> 5 * 6 * Inspiration, some code, and most witty comments come from 7 * Documentation/virtual/lguest/lguest.c, by Rusty Russell 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2. 10 * 11 * Generic code for virtio server in host kernel. 12 */ 13 14 #include <linux/eventfd.h> 15 #include <linux/vhost.h> 16 #include <linux/uio.h> 17 #include <linux/mm.h> 18 #include <linux/mmu_context.h> 19 #include <linux/miscdevice.h> 20 #include <linux/mutex.h> 21 #include <linux/rcupdate.h> 22 #include <linux/poll.h> 23 #include <linux/file.h> 24 #include <linux/highmem.h> 25 #include <linux/slab.h> 26 #include <linux/kthread.h> 27 #include <linux/cgroup.h> 28 #include <linux/module.h> 29 30 #include "vhost.h" 31 32 enum { 33 VHOST_MEMORY_MAX_NREGIONS = 64, 34 VHOST_MEMORY_F_LOG = 0x1, 35 }; 36 37 #define vhost_used_event(vq) ((u16 __user *)&vq->avail->ring[vq->num]) 38 #define vhost_avail_event(vq) ((u16 __user *)&vq->used->ring[vq->num]) 39 40 static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, 41 poll_table *pt) 42 { 43 struct vhost_poll *poll; 44 45 poll = container_of(pt, struct vhost_poll, table); 46 poll->wqh = wqh; 47 add_wait_queue(wqh, &poll->wait); 48 } 49 50 static int vhost_poll_wakeup(wait_queue_t *wait, unsigned mode, int sync, 51 void *key) 52 { 53 struct vhost_poll *poll = container_of(wait, struct vhost_poll, wait); 54 55 if (!((unsigned long)key & poll->mask)) 56 return 0; 57 58 vhost_poll_queue(poll); 59 return 0; 60 } 61 62 void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn) 63 { 64 INIT_LIST_HEAD(&work->node); 65 work->fn = fn; 66 init_waitqueue_head(&work->done); 67 work->flushing = 0; 68 work->queue_seq = work->done_seq = 0; 69 } 70 EXPORT_SYMBOL_GPL(vhost_work_init); 71 72 /* Init poll structure */ 73 void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 74 unsigned long mask, struct vhost_dev *dev) 75 { 76 init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup); 77 init_poll_funcptr(&poll->table, vhost_poll_func); 78 poll->mask = mask; 79 poll->dev = dev; 80 poll->wqh = NULL; 81 82 vhost_work_init(&poll->work, fn); 83 } 84 EXPORT_SYMBOL_GPL(vhost_poll_init); 85 86 /* Start polling a file. We add ourselves to file's wait queue. The caller must 87 * keep a reference to a file until after vhost_poll_stop is called. */ 88 int vhost_poll_start(struct vhost_poll *poll, struct file *file) 89 { 90 unsigned long mask; 91 int ret = 0; 92 93 if (poll->wqh) 94 return 0; 95 96 mask = file->f_op->poll(file, &poll->table); 97 if (mask) 98 vhost_poll_wakeup(&poll->wait, 0, 0, (void *)mask); 99 if (mask & POLLERR) { 100 if (poll->wqh) 101 remove_wait_queue(poll->wqh, &poll->wait); 102 ret = -EINVAL; 103 } 104 105 return ret; 106 } 107 EXPORT_SYMBOL_GPL(vhost_poll_start); 108 109 /* Stop polling a file. After this function returns, it becomes safe to drop the 110 * file reference. You must also flush afterwards. */ 111 void vhost_poll_stop(struct vhost_poll *poll) 112 { 113 if (poll->wqh) { 114 remove_wait_queue(poll->wqh, &poll->wait); 115 poll->wqh = NULL; 116 } 117 } 118 EXPORT_SYMBOL_GPL(vhost_poll_stop); 119 120 static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work, 121 unsigned seq) 122 { 123 int left; 124 125 spin_lock_irq(&dev->work_lock); 126 left = seq - work->done_seq; 127 spin_unlock_irq(&dev->work_lock); 128 return left <= 0; 129 } 130 131 void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) 132 { 133 unsigned seq; 134 int flushing; 135 136 spin_lock_irq(&dev->work_lock); 137 seq = work->queue_seq; 138 work->flushing++; 139 spin_unlock_irq(&dev->work_lock); 140 wait_event(work->done, vhost_work_seq_done(dev, work, seq)); 141 spin_lock_irq(&dev->work_lock); 142 flushing = --work->flushing; 143 spin_unlock_irq(&dev->work_lock); 144 BUG_ON(flushing < 0); 145 } 146 EXPORT_SYMBOL_GPL(vhost_work_flush); 147 148 /* Flush any work that has been scheduled. When calling this, don't hold any 149 * locks that are also used by the callback. */ 150 void vhost_poll_flush(struct vhost_poll *poll) 151 { 152 vhost_work_flush(poll->dev, &poll->work); 153 } 154 EXPORT_SYMBOL_GPL(vhost_poll_flush); 155 156 void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) 157 { 158 unsigned long flags; 159 160 spin_lock_irqsave(&dev->work_lock, flags); 161 if (list_empty(&work->node)) { 162 list_add_tail(&work->node, &dev->work_list); 163 work->queue_seq++; 164 spin_unlock_irqrestore(&dev->work_lock, flags); 165 wake_up_process(dev->worker); 166 } else { 167 spin_unlock_irqrestore(&dev->work_lock, flags); 168 } 169 } 170 EXPORT_SYMBOL_GPL(vhost_work_queue); 171 172 void vhost_poll_queue(struct vhost_poll *poll) 173 { 174 vhost_work_queue(poll->dev, &poll->work); 175 } 176 EXPORT_SYMBOL_GPL(vhost_poll_queue); 177 178 static void vhost_vq_reset(struct vhost_dev *dev, 179 struct vhost_virtqueue *vq) 180 { 181 vq->num = 1; 182 vq->desc = NULL; 183 vq->avail = NULL; 184 vq->used = NULL; 185 vq->last_avail_idx = 0; 186 vq->avail_idx = 0; 187 vq->last_used_idx = 0; 188 vq->signalled_used = 0; 189 vq->signalled_used_valid = false; 190 vq->used_flags = 0; 191 vq->log_used = false; 192 vq->log_addr = -1ull; 193 vq->private_data = NULL; 194 vq->log_base = NULL; 195 vq->error_ctx = NULL; 196 vq->error = NULL; 197 vq->kick = NULL; 198 vq->call_ctx = NULL; 199 vq->call = NULL; 200 vq->log_ctx = NULL; 201 } 202 203 static int vhost_worker(void *data) 204 { 205 struct vhost_dev *dev = data; 206 struct vhost_work *work = NULL; 207 unsigned uninitialized_var(seq); 208 mm_segment_t oldfs = get_fs(); 209 210 set_fs(USER_DS); 211 use_mm(dev->mm); 212 213 for (;;) { 214 /* mb paired w/ kthread_stop */ 215 set_current_state(TASK_INTERRUPTIBLE); 216 217 spin_lock_irq(&dev->work_lock); 218 if (work) { 219 work->done_seq = seq; 220 if (work->flushing) 221 wake_up_all(&work->done); 222 } 223 224 if (kthread_should_stop()) { 225 spin_unlock_irq(&dev->work_lock); 226 __set_current_state(TASK_RUNNING); 227 break; 228 } 229 if (!list_empty(&dev->work_list)) { 230 work = list_first_entry(&dev->work_list, 231 struct vhost_work, node); 232 list_del_init(&work->node); 233 seq = work->queue_seq; 234 } else 235 work = NULL; 236 spin_unlock_irq(&dev->work_lock); 237 238 if (work) { 239 __set_current_state(TASK_RUNNING); 240 work->fn(work); 241 if (need_resched()) 242 schedule(); 243 } else 244 schedule(); 245 246 } 247 unuse_mm(dev->mm); 248 set_fs(oldfs); 249 return 0; 250 } 251 252 static void vhost_vq_free_iovecs(struct vhost_virtqueue *vq) 253 { 254 kfree(vq->indirect); 255 vq->indirect = NULL; 256 kfree(vq->log); 257 vq->log = NULL; 258 kfree(vq->heads); 259 vq->heads = NULL; 260 } 261 262 /* Helper to allocate iovec buffers for all vqs. */ 263 static long vhost_dev_alloc_iovecs(struct vhost_dev *dev) 264 { 265 struct vhost_virtqueue *vq; 266 int i; 267 268 for (i = 0; i < dev->nvqs; ++i) { 269 vq = dev->vqs[i]; 270 vq->indirect = kmalloc(sizeof *vq->indirect * UIO_MAXIOV, 271 GFP_KERNEL); 272 vq->log = kmalloc(sizeof *vq->log * UIO_MAXIOV, GFP_KERNEL); 273 vq->heads = kmalloc(sizeof *vq->heads * UIO_MAXIOV, GFP_KERNEL); 274 if (!vq->indirect || !vq->log || !vq->heads) 275 goto err_nomem; 276 } 277 return 0; 278 279 err_nomem: 280 for (; i >= 0; --i) 281 vhost_vq_free_iovecs(dev->vqs[i]); 282 return -ENOMEM; 283 } 284 285 static void vhost_dev_free_iovecs(struct vhost_dev *dev) 286 { 287 int i; 288 289 for (i = 0; i < dev->nvqs; ++i) 290 vhost_vq_free_iovecs(dev->vqs[i]); 291 } 292 293 long vhost_dev_init(struct vhost_dev *dev, 294 struct vhost_virtqueue **vqs, int nvqs) 295 { 296 struct vhost_virtqueue *vq; 297 int i; 298 299 dev->vqs = vqs; 300 dev->nvqs = nvqs; 301 mutex_init(&dev->mutex); 302 dev->log_ctx = NULL; 303 dev->log_file = NULL; 304 dev->memory = NULL; 305 dev->mm = NULL; 306 spin_lock_init(&dev->work_lock); 307 INIT_LIST_HEAD(&dev->work_list); 308 dev->worker = NULL; 309 310 for (i = 0; i < dev->nvqs; ++i) { 311 vq = dev->vqs[i]; 312 vq->log = NULL; 313 vq->indirect = NULL; 314 vq->heads = NULL; 315 vq->dev = dev; 316 mutex_init(&vq->mutex); 317 vhost_vq_reset(dev, vq); 318 if (vq->handle_kick) 319 vhost_poll_init(&vq->poll, vq->handle_kick, 320 POLLIN, dev); 321 } 322 323 return 0; 324 } 325 EXPORT_SYMBOL_GPL(vhost_dev_init); 326 327 /* Caller should have device mutex */ 328 long vhost_dev_check_owner(struct vhost_dev *dev) 329 { 330 /* Are you the owner? If not, I don't think you mean to do that */ 331 return dev->mm == current->mm ? 0 : -EPERM; 332 } 333 EXPORT_SYMBOL_GPL(vhost_dev_check_owner); 334 335 struct vhost_attach_cgroups_struct { 336 struct vhost_work work; 337 struct task_struct *owner; 338 int ret; 339 }; 340 341 static void vhost_attach_cgroups_work(struct vhost_work *work) 342 { 343 struct vhost_attach_cgroups_struct *s; 344 345 s = container_of(work, struct vhost_attach_cgroups_struct, work); 346 s->ret = cgroup_attach_task_all(s->owner, current); 347 } 348 349 static int vhost_attach_cgroups(struct vhost_dev *dev) 350 { 351 struct vhost_attach_cgroups_struct attach; 352 353 attach.owner = current; 354 vhost_work_init(&attach.work, vhost_attach_cgroups_work); 355 vhost_work_queue(dev, &attach.work); 356 vhost_work_flush(dev, &attach.work); 357 return attach.ret; 358 } 359 360 /* Caller should have device mutex */ 361 bool vhost_dev_has_owner(struct vhost_dev *dev) 362 { 363 return dev->mm; 364 } 365 EXPORT_SYMBOL_GPL(vhost_dev_has_owner); 366 367 /* Caller should have device mutex */ 368 long vhost_dev_set_owner(struct vhost_dev *dev) 369 { 370 struct task_struct *worker; 371 int err; 372 373 /* Is there an owner already? */ 374 if (vhost_dev_has_owner(dev)) { 375 err = -EBUSY; 376 goto err_mm; 377 } 378 379 /* No owner, become one */ 380 dev->mm = get_task_mm(current); 381 worker = kthread_create(vhost_worker, dev, "vhost-%d", current->pid); 382 if (IS_ERR(worker)) { 383 err = PTR_ERR(worker); 384 goto err_worker; 385 } 386 387 dev->worker = worker; 388 wake_up_process(worker); /* avoid contributing to loadavg */ 389 390 err = vhost_attach_cgroups(dev); 391 if (err) 392 goto err_cgroup; 393 394 err = vhost_dev_alloc_iovecs(dev); 395 if (err) 396 goto err_cgroup; 397 398 return 0; 399 err_cgroup: 400 kthread_stop(worker); 401 dev->worker = NULL; 402 err_worker: 403 if (dev->mm) 404 mmput(dev->mm); 405 dev->mm = NULL; 406 err_mm: 407 return err; 408 } 409 EXPORT_SYMBOL_GPL(vhost_dev_set_owner); 410 411 struct vhost_memory *vhost_dev_reset_owner_prepare(void) 412 { 413 return kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL); 414 } 415 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); 416 417 /* Caller should have device mutex */ 418 void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory) 419 { 420 vhost_dev_cleanup(dev, true); 421 422 /* Restore memory to default empty mapping. */ 423 memory->nregions = 0; 424 RCU_INIT_POINTER(dev->memory, memory); 425 } 426 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner); 427 428 void vhost_dev_stop(struct vhost_dev *dev) 429 { 430 int i; 431 432 for (i = 0; i < dev->nvqs; ++i) { 433 if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick) { 434 vhost_poll_stop(&dev->vqs[i]->poll); 435 vhost_poll_flush(&dev->vqs[i]->poll); 436 } 437 } 438 } 439 EXPORT_SYMBOL_GPL(vhost_dev_stop); 440 441 /* Caller should have device mutex if and only if locked is set */ 442 void vhost_dev_cleanup(struct vhost_dev *dev, bool locked) 443 { 444 int i; 445 446 for (i = 0; i < dev->nvqs; ++i) { 447 if (dev->vqs[i]->error_ctx) 448 eventfd_ctx_put(dev->vqs[i]->error_ctx); 449 if (dev->vqs[i]->error) 450 fput(dev->vqs[i]->error); 451 if (dev->vqs[i]->kick) 452 fput(dev->vqs[i]->kick); 453 if (dev->vqs[i]->call_ctx) 454 eventfd_ctx_put(dev->vqs[i]->call_ctx); 455 if (dev->vqs[i]->call) 456 fput(dev->vqs[i]->call); 457 vhost_vq_reset(dev, dev->vqs[i]); 458 } 459 vhost_dev_free_iovecs(dev); 460 if (dev->log_ctx) 461 eventfd_ctx_put(dev->log_ctx); 462 dev->log_ctx = NULL; 463 if (dev->log_file) 464 fput(dev->log_file); 465 dev->log_file = NULL; 466 /* No one will access memory at this point */ 467 kfree(rcu_dereference_protected(dev->memory, 468 locked == 469 lockdep_is_held(&dev->mutex))); 470 RCU_INIT_POINTER(dev->memory, NULL); 471 WARN_ON(!list_empty(&dev->work_list)); 472 if (dev->worker) { 473 kthread_stop(dev->worker); 474 dev->worker = NULL; 475 } 476 if (dev->mm) 477 mmput(dev->mm); 478 dev->mm = NULL; 479 } 480 EXPORT_SYMBOL_GPL(vhost_dev_cleanup); 481 482 static int log_access_ok(void __user *log_base, u64 addr, unsigned long sz) 483 { 484 u64 a = addr / VHOST_PAGE_SIZE / 8; 485 486 /* Make sure 64 bit math will not overflow. */ 487 if (a > ULONG_MAX - (unsigned long)log_base || 488 a + (unsigned long)log_base > ULONG_MAX) 489 return 0; 490 491 return access_ok(VERIFY_WRITE, log_base + a, 492 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8); 493 } 494 495 /* Caller should have vq mutex and device mutex. */ 496 static int vq_memory_access_ok(void __user *log_base, struct vhost_memory *mem, 497 int log_all) 498 { 499 int i; 500 501 if (!mem) 502 return 0; 503 504 for (i = 0; i < mem->nregions; ++i) { 505 struct vhost_memory_region *m = mem->regions + i; 506 unsigned long a = m->userspace_addr; 507 if (m->memory_size > ULONG_MAX) 508 return 0; 509 else if (!access_ok(VERIFY_WRITE, (void __user *)a, 510 m->memory_size)) 511 return 0; 512 else if (log_all && !log_access_ok(log_base, 513 m->guest_phys_addr, 514 m->memory_size)) 515 return 0; 516 } 517 return 1; 518 } 519 520 /* Can we switch to this memory table? */ 521 /* Caller should have device mutex but not vq mutex */ 522 static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem, 523 int log_all) 524 { 525 int i; 526 527 for (i = 0; i < d->nvqs; ++i) { 528 int ok; 529 mutex_lock(&d->vqs[i]->mutex); 530 /* If ring is inactive, will check when it's enabled. */ 531 if (d->vqs[i]->private_data) 532 ok = vq_memory_access_ok(d->vqs[i]->log_base, mem, 533 log_all); 534 else 535 ok = 1; 536 mutex_unlock(&d->vqs[i]->mutex); 537 if (!ok) 538 return 0; 539 } 540 return 1; 541 } 542 543 static int vq_access_ok(struct vhost_dev *d, unsigned int num, 544 struct vring_desc __user *desc, 545 struct vring_avail __user *avail, 546 struct vring_used __user *used) 547 { 548 size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 549 return access_ok(VERIFY_READ, desc, num * sizeof *desc) && 550 access_ok(VERIFY_READ, avail, 551 sizeof *avail + num * sizeof *avail->ring + s) && 552 access_ok(VERIFY_WRITE, used, 553 sizeof *used + num * sizeof *used->ring + s); 554 } 555 556 /* Can we log writes? */ 557 /* Caller should have device mutex but not vq mutex */ 558 int vhost_log_access_ok(struct vhost_dev *dev) 559 { 560 struct vhost_memory *mp; 561 562 mp = rcu_dereference_protected(dev->memory, 563 lockdep_is_held(&dev->mutex)); 564 return memory_access_ok(dev, mp, 1); 565 } 566 EXPORT_SYMBOL_GPL(vhost_log_access_ok); 567 568 /* Verify access for write logging. */ 569 /* Caller should have vq mutex and device mutex */ 570 static int vq_log_access_ok(struct vhost_dev *d, struct vhost_virtqueue *vq, 571 void __user *log_base) 572 { 573 struct vhost_memory *mp; 574 size_t s = vhost_has_feature(d, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 575 576 mp = rcu_dereference_protected(vq->dev->memory, 577 lockdep_is_held(&vq->mutex)); 578 return vq_memory_access_ok(log_base, mp, 579 vhost_has_feature(vq->dev, VHOST_F_LOG_ALL)) && 580 (!vq->log_used || log_access_ok(log_base, vq->log_addr, 581 sizeof *vq->used + 582 vq->num * sizeof *vq->used->ring + s)); 583 } 584 585 /* Can we start vq? */ 586 /* Caller should have vq mutex and device mutex */ 587 int vhost_vq_access_ok(struct vhost_virtqueue *vq) 588 { 589 return vq_access_ok(vq->dev, vq->num, vq->desc, vq->avail, vq->used) && 590 vq_log_access_ok(vq->dev, vq, vq->log_base); 591 } 592 EXPORT_SYMBOL_GPL(vhost_vq_access_ok); 593 594 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) 595 { 596 struct vhost_memory mem, *newmem, *oldmem; 597 unsigned long size = offsetof(struct vhost_memory, regions); 598 599 if (copy_from_user(&mem, m, size)) 600 return -EFAULT; 601 if (mem.padding) 602 return -EOPNOTSUPP; 603 if (mem.nregions > VHOST_MEMORY_MAX_NREGIONS) 604 return -E2BIG; 605 newmem = kmalloc(size + mem.nregions * sizeof *m->regions, GFP_KERNEL); 606 if (!newmem) 607 return -ENOMEM; 608 609 memcpy(newmem, &mem, size); 610 if (copy_from_user(newmem->regions, m->regions, 611 mem.nregions * sizeof *m->regions)) { 612 kfree(newmem); 613 return -EFAULT; 614 } 615 616 if (!memory_access_ok(d, newmem, 617 vhost_has_feature(d, VHOST_F_LOG_ALL))) { 618 kfree(newmem); 619 return -EFAULT; 620 } 621 oldmem = rcu_dereference_protected(d->memory, 622 lockdep_is_held(&d->mutex)); 623 rcu_assign_pointer(d->memory, newmem); 624 synchronize_rcu(); 625 kfree(oldmem); 626 return 0; 627 } 628 629 long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp) 630 { 631 struct file *eventfp, *filep = NULL; 632 bool pollstart = false, pollstop = false; 633 struct eventfd_ctx *ctx = NULL; 634 u32 __user *idxp = argp; 635 struct vhost_virtqueue *vq; 636 struct vhost_vring_state s; 637 struct vhost_vring_file f; 638 struct vhost_vring_addr a; 639 u32 idx; 640 long r; 641 642 r = get_user(idx, idxp); 643 if (r < 0) 644 return r; 645 if (idx >= d->nvqs) 646 return -ENOBUFS; 647 648 vq = d->vqs[idx]; 649 650 mutex_lock(&vq->mutex); 651 652 switch (ioctl) { 653 case VHOST_SET_VRING_NUM: 654 /* Resizing ring with an active backend? 655 * You don't want to do that. */ 656 if (vq->private_data) { 657 r = -EBUSY; 658 break; 659 } 660 if (copy_from_user(&s, argp, sizeof s)) { 661 r = -EFAULT; 662 break; 663 } 664 if (!s.num || s.num > 0xffff || (s.num & (s.num - 1))) { 665 r = -EINVAL; 666 break; 667 } 668 vq->num = s.num; 669 break; 670 case VHOST_SET_VRING_BASE: 671 /* Moving base with an active backend? 672 * You don't want to do that. */ 673 if (vq->private_data) { 674 r = -EBUSY; 675 break; 676 } 677 if (copy_from_user(&s, argp, sizeof s)) { 678 r = -EFAULT; 679 break; 680 } 681 if (s.num > 0xffff) { 682 r = -EINVAL; 683 break; 684 } 685 vq->last_avail_idx = s.num; 686 /* Forget the cached index value. */ 687 vq->avail_idx = vq->last_avail_idx; 688 break; 689 case VHOST_GET_VRING_BASE: 690 s.index = idx; 691 s.num = vq->last_avail_idx; 692 if (copy_to_user(argp, &s, sizeof s)) 693 r = -EFAULT; 694 break; 695 case VHOST_SET_VRING_ADDR: 696 if (copy_from_user(&a, argp, sizeof a)) { 697 r = -EFAULT; 698 break; 699 } 700 if (a.flags & ~(0x1 << VHOST_VRING_F_LOG)) { 701 r = -EOPNOTSUPP; 702 break; 703 } 704 /* For 32bit, verify that the top 32bits of the user 705 data are set to zero. */ 706 if ((u64)(unsigned long)a.desc_user_addr != a.desc_user_addr || 707 (u64)(unsigned long)a.used_user_addr != a.used_user_addr || 708 (u64)(unsigned long)a.avail_user_addr != a.avail_user_addr) { 709 r = -EFAULT; 710 break; 711 } 712 if ((a.avail_user_addr & (sizeof *vq->avail->ring - 1)) || 713 (a.used_user_addr & (sizeof *vq->used->ring - 1)) || 714 (a.log_guest_addr & (sizeof *vq->used->ring - 1))) { 715 r = -EINVAL; 716 break; 717 } 718 719 /* We only verify access here if backend is configured. 720 * If it is not, we don't as size might not have been setup. 721 * We will verify when backend is configured. */ 722 if (vq->private_data) { 723 if (!vq_access_ok(d, vq->num, 724 (void __user *)(unsigned long)a.desc_user_addr, 725 (void __user *)(unsigned long)a.avail_user_addr, 726 (void __user *)(unsigned long)a.used_user_addr)) { 727 r = -EINVAL; 728 break; 729 } 730 731 /* Also validate log access for used ring if enabled. */ 732 if ((a.flags & (0x1 << VHOST_VRING_F_LOG)) && 733 !log_access_ok(vq->log_base, a.log_guest_addr, 734 sizeof *vq->used + 735 vq->num * sizeof *vq->used->ring)) { 736 r = -EINVAL; 737 break; 738 } 739 } 740 741 vq->log_used = !!(a.flags & (0x1 << VHOST_VRING_F_LOG)); 742 vq->desc = (void __user *)(unsigned long)a.desc_user_addr; 743 vq->avail = (void __user *)(unsigned long)a.avail_user_addr; 744 vq->log_addr = a.log_guest_addr; 745 vq->used = (void __user *)(unsigned long)a.used_user_addr; 746 break; 747 case VHOST_SET_VRING_KICK: 748 if (copy_from_user(&f, argp, sizeof f)) { 749 r = -EFAULT; 750 break; 751 } 752 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd); 753 if (IS_ERR(eventfp)) { 754 r = PTR_ERR(eventfp); 755 break; 756 } 757 if (eventfp != vq->kick) { 758 pollstop = (filep = vq->kick) != NULL; 759 pollstart = (vq->kick = eventfp) != NULL; 760 } else 761 filep = eventfp; 762 break; 763 case VHOST_SET_VRING_CALL: 764 if (copy_from_user(&f, argp, sizeof f)) { 765 r = -EFAULT; 766 break; 767 } 768 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd); 769 if (IS_ERR(eventfp)) { 770 r = PTR_ERR(eventfp); 771 break; 772 } 773 if (eventfp != vq->call) { 774 filep = vq->call; 775 ctx = vq->call_ctx; 776 vq->call = eventfp; 777 vq->call_ctx = eventfp ? 778 eventfd_ctx_fileget(eventfp) : NULL; 779 } else 780 filep = eventfp; 781 break; 782 case VHOST_SET_VRING_ERR: 783 if (copy_from_user(&f, argp, sizeof f)) { 784 r = -EFAULT; 785 break; 786 } 787 eventfp = f.fd == -1 ? NULL : eventfd_fget(f.fd); 788 if (IS_ERR(eventfp)) { 789 r = PTR_ERR(eventfp); 790 break; 791 } 792 if (eventfp != vq->error) { 793 filep = vq->error; 794 vq->error = eventfp; 795 ctx = vq->error_ctx; 796 vq->error_ctx = eventfp ? 797 eventfd_ctx_fileget(eventfp) : NULL; 798 } else 799 filep = eventfp; 800 break; 801 default: 802 r = -ENOIOCTLCMD; 803 } 804 805 if (pollstop && vq->handle_kick) 806 vhost_poll_stop(&vq->poll); 807 808 if (ctx) 809 eventfd_ctx_put(ctx); 810 if (filep) 811 fput(filep); 812 813 if (pollstart && vq->handle_kick) 814 r = vhost_poll_start(&vq->poll, vq->kick); 815 816 mutex_unlock(&vq->mutex); 817 818 if (pollstop && vq->handle_kick) 819 vhost_poll_flush(&vq->poll); 820 return r; 821 } 822 EXPORT_SYMBOL_GPL(vhost_vring_ioctl); 823 824 /* Caller must have device mutex */ 825 long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) 826 { 827 struct file *eventfp, *filep = NULL; 828 struct eventfd_ctx *ctx = NULL; 829 u64 p; 830 long r; 831 int i, fd; 832 833 /* If you are not the owner, you can become one */ 834 if (ioctl == VHOST_SET_OWNER) { 835 r = vhost_dev_set_owner(d); 836 goto done; 837 } 838 839 /* You must be the owner to do anything else */ 840 r = vhost_dev_check_owner(d); 841 if (r) 842 goto done; 843 844 switch (ioctl) { 845 case VHOST_SET_MEM_TABLE: 846 r = vhost_set_memory(d, argp); 847 break; 848 case VHOST_SET_LOG_BASE: 849 if (copy_from_user(&p, argp, sizeof p)) { 850 r = -EFAULT; 851 break; 852 } 853 if ((u64)(unsigned long)p != p) { 854 r = -EFAULT; 855 break; 856 } 857 for (i = 0; i < d->nvqs; ++i) { 858 struct vhost_virtqueue *vq; 859 void __user *base = (void __user *)(unsigned long)p; 860 vq = d->vqs[i]; 861 mutex_lock(&vq->mutex); 862 /* If ring is inactive, will check when it's enabled. */ 863 if (vq->private_data && !vq_log_access_ok(d, vq, base)) 864 r = -EFAULT; 865 else 866 vq->log_base = base; 867 mutex_unlock(&vq->mutex); 868 } 869 break; 870 case VHOST_SET_LOG_FD: 871 r = get_user(fd, (int __user *)argp); 872 if (r < 0) 873 break; 874 eventfp = fd == -1 ? NULL : eventfd_fget(fd); 875 if (IS_ERR(eventfp)) { 876 r = PTR_ERR(eventfp); 877 break; 878 } 879 if (eventfp != d->log_file) { 880 filep = d->log_file; 881 ctx = d->log_ctx; 882 d->log_ctx = eventfp ? 883 eventfd_ctx_fileget(eventfp) : NULL; 884 } else 885 filep = eventfp; 886 for (i = 0; i < d->nvqs; ++i) { 887 mutex_lock(&d->vqs[i]->mutex); 888 d->vqs[i]->log_ctx = d->log_ctx; 889 mutex_unlock(&d->vqs[i]->mutex); 890 } 891 if (ctx) 892 eventfd_ctx_put(ctx); 893 if (filep) 894 fput(filep); 895 break; 896 default: 897 r = -ENOIOCTLCMD; 898 break; 899 } 900 done: 901 return r; 902 } 903 EXPORT_SYMBOL_GPL(vhost_dev_ioctl); 904 905 static const struct vhost_memory_region *find_region(struct vhost_memory *mem, 906 __u64 addr, __u32 len) 907 { 908 struct vhost_memory_region *reg; 909 int i; 910 911 /* linear search is not brilliant, but we really have on the order of 6 912 * regions in practice */ 913 for (i = 0; i < mem->nregions; ++i) { 914 reg = mem->regions + i; 915 if (reg->guest_phys_addr <= addr && 916 reg->guest_phys_addr + reg->memory_size - 1 >= addr) 917 return reg; 918 } 919 return NULL; 920 } 921 922 /* TODO: This is really inefficient. We need something like get_user() 923 * (instruction directly accesses the data, with an exception table entry 924 * returning -EFAULT). See Documentation/x86/exception-tables.txt. 925 */ 926 static int set_bit_to_user(int nr, void __user *addr) 927 { 928 unsigned long log = (unsigned long)addr; 929 struct page *page; 930 void *base; 931 int bit = nr + (log % PAGE_SIZE) * 8; 932 int r; 933 934 r = get_user_pages_fast(log, 1, 1, &page); 935 if (r < 0) 936 return r; 937 BUG_ON(r != 1); 938 base = kmap_atomic(page); 939 set_bit(bit, base); 940 kunmap_atomic(base); 941 set_page_dirty_lock(page); 942 put_page(page); 943 return 0; 944 } 945 946 static int log_write(void __user *log_base, 947 u64 write_address, u64 write_length) 948 { 949 u64 write_page = write_address / VHOST_PAGE_SIZE; 950 int r; 951 952 if (!write_length) 953 return 0; 954 write_length += write_address % VHOST_PAGE_SIZE; 955 for (;;) { 956 u64 base = (u64)(unsigned long)log_base; 957 u64 log = base + write_page / 8; 958 int bit = write_page % 8; 959 if ((u64)(unsigned long)log != log) 960 return -EFAULT; 961 r = set_bit_to_user(bit, (void __user *)(unsigned long)log); 962 if (r < 0) 963 return r; 964 if (write_length <= VHOST_PAGE_SIZE) 965 break; 966 write_length -= VHOST_PAGE_SIZE; 967 write_page += 1; 968 } 969 return r; 970 } 971 972 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 973 unsigned int log_num, u64 len) 974 { 975 int i, r; 976 977 /* Make sure data written is seen before log. */ 978 smp_wmb(); 979 for (i = 0; i < log_num; ++i) { 980 u64 l = min(log[i].len, len); 981 r = log_write(vq->log_base, log[i].addr, l); 982 if (r < 0) 983 return r; 984 len -= l; 985 if (!len) { 986 if (vq->log_ctx) 987 eventfd_signal(vq->log_ctx, 1); 988 return 0; 989 } 990 } 991 /* Length written exceeds what we have stored. This is a bug. */ 992 BUG(); 993 return 0; 994 } 995 EXPORT_SYMBOL_GPL(vhost_log_write); 996 997 static int vhost_update_used_flags(struct vhost_virtqueue *vq) 998 { 999 void __user *used; 1000 if (__put_user(vq->used_flags, &vq->used->flags) < 0) 1001 return -EFAULT; 1002 if (unlikely(vq->log_used)) { 1003 /* Make sure the flag is seen before log. */ 1004 smp_wmb(); 1005 /* Log used flag write. */ 1006 used = &vq->used->flags; 1007 log_write(vq->log_base, vq->log_addr + 1008 (used - (void __user *)vq->used), 1009 sizeof vq->used->flags); 1010 if (vq->log_ctx) 1011 eventfd_signal(vq->log_ctx, 1); 1012 } 1013 return 0; 1014 } 1015 1016 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) 1017 { 1018 if (__put_user(vq->avail_idx, vhost_avail_event(vq))) 1019 return -EFAULT; 1020 if (unlikely(vq->log_used)) { 1021 void __user *used; 1022 /* Make sure the event is seen before log. */ 1023 smp_wmb(); 1024 /* Log avail event write */ 1025 used = vhost_avail_event(vq); 1026 log_write(vq->log_base, vq->log_addr + 1027 (used - (void __user *)vq->used), 1028 sizeof *vhost_avail_event(vq)); 1029 if (vq->log_ctx) 1030 eventfd_signal(vq->log_ctx, 1); 1031 } 1032 return 0; 1033 } 1034 1035 int vhost_init_used(struct vhost_virtqueue *vq) 1036 { 1037 int r; 1038 if (!vq->private_data) 1039 return 0; 1040 1041 r = vhost_update_used_flags(vq); 1042 if (r) 1043 return r; 1044 vq->signalled_used_valid = false; 1045 return get_user(vq->last_used_idx, &vq->used->idx); 1046 } 1047 EXPORT_SYMBOL_GPL(vhost_init_used); 1048 1049 static int translate_desc(struct vhost_dev *dev, u64 addr, u32 len, 1050 struct iovec iov[], int iov_size) 1051 { 1052 const struct vhost_memory_region *reg; 1053 struct vhost_memory *mem; 1054 struct iovec *_iov; 1055 u64 s = 0; 1056 int ret = 0; 1057 1058 rcu_read_lock(); 1059 1060 mem = rcu_dereference(dev->memory); 1061 while ((u64)len > s) { 1062 u64 size; 1063 if (unlikely(ret >= iov_size)) { 1064 ret = -ENOBUFS; 1065 break; 1066 } 1067 reg = find_region(mem, addr, len); 1068 if (unlikely(!reg)) { 1069 ret = -EFAULT; 1070 break; 1071 } 1072 _iov = iov + ret; 1073 size = reg->memory_size - addr + reg->guest_phys_addr; 1074 _iov->iov_len = min((u64)len - s, size); 1075 _iov->iov_base = (void __user *)(unsigned long) 1076 (reg->userspace_addr + addr - reg->guest_phys_addr); 1077 s += size; 1078 addr += size; 1079 ++ret; 1080 } 1081 1082 rcu_read_unlock(); 1083 return ret; 1084 } 1085 1086 /* Each buffer in the virtqueues is actually a chain of descriptors. This 1087 * function returns the next descriptor in the chain, 1088 * or -1U if we're at the end. */ 1089 static unsigned next_desc(struct vring_desc *desc) 1090 { 1091 unsigned int next; 1092 1093 /* If this descriptor says it doesn't chain, we're done. */ 1094 if (!(desc->flags & VRING_DESC_F_NEXT)) 1095 return -1U; 1096 1097 /* Check they're not leading us off end of descriptors. */ 1098 next = desc->next; 1099 /* Make sure compiler knows to grab that: we don't want it changing! */ 1100 /* We will use the result as an index in an array, so most 1101 * architectures only need a compiler barrier here. */ 1102 read_barrier_depends(); 1103 1104 return next; 1105 } 1106 1107 static int get_indirect(struct vhost_dev *dev, struct vhost_virtqueue *vq, 1108 struct iovec iov[], unsigned int iov_size, 1109 unsigned int *out_num, unsigned int *in_num, 1110 struct vhost_log *log, unsigned int *log_num, 1111 struct vring_desc *indirect) 1112 { 1113 struct vring_desc desc; 1114 unsigned int i = 0, count, found = 0; 1115 int ret; 1116 1117 /* Sanity check */ 1118 if (unlikely(indirect->len % sizeof desc)) { 1119 vq_err(vq, "Invalid length in indirect descriptor: " 1120 "len 0x%llx not multiple of 0x%zx\n", 1121 (unsigned long long)indirect->len, 1122 sizeof desc); 1123 return -EINVAL; 1124 } 1125 1126 ret = translate_desc(dev, indirect->addr, indirect->len, vq->indirect, 1127 UIO_MAXIOV); 1128 if (unlikely(ret < 0)) { 1129 vq_err(vq, "Translation failure %d in indirect.\n", ret); 1130 return ret; 1131 } 1132 1133 /* We will use the result as an address to read from, so most 1134 * architectures only need a compiler barrier here. */ 1135 read_barrier_depends(); 1136 1137 count = indirect->len / sizeof desc; 1138 /* Buffers are chained via a 16 bit next field, so 1139 * we can have at most 2^16 of these. */ 1140 if (unlikely(count > USHRT_MAX + 1)) { 1141 vq_err(vq, "Indirect buffer length too big: %d\n", 1142 indirect->len); 1143 return -E2BIG; 1144 } 1145 1146 do { 1147 unsigned iov_count = *in_num + *out_num; 1148 if (unlikely(++found > count)) { 1149 vq_err(vq, "Loop detected: last one at %u " 1150 "indirect size %u\n", 1151 i, count); 1152 return -EINVAL; 1153 } 1154 if (unlikely(memcpy_fromiovec((unsigned char *)&desc, 1155 vq->indirect, sizeof desc))) { 1156 vq_err(vq, "Failed indirect descriptor: idx %d, %zx\n", 1157 i, (size_t)indirect->addr + i * sizeof desc); 1158 return -EINVAL; 1159 } 1160 if (unlikely(desc.flags & VRING_DESC_F_INDIRECT)) { 1161 vq_err(vq, "Nested indirect descriptor: idx %d, %zx\n", 1162 i, (size_t)indirect->addr + i * sizeof desc); 1163 return -EINVAL; 1164 } 1165 1166 ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count, 1167 iov_size - iov_count); 1168 if (unlikely(ret < 0)) { 1169 vq_err(vq, "Translation failure %d indirect idx %d\n", 1170 ret, i); 1171 return ret; 1172 } 1173 /* If this is an input descriptor, increment that count. */ 1174 if (desc.flags & VRING_DESC_F_WRITE) { 1175 *in_num += ret; 1176 if (unlikely(log)) { 1177 log[*log_num].addr = desc.addr; 1178 log[*log_num].len = desc.len; 1179 ++*log_num; 1180 } 1181 } else { 1182 /* If it's an output descriptor, they're all supposed 1183 * to come before any input descriptors. */ 1184 if (unlikely(*in_num)) { 1185 vq_err(vq, "Indirect descriptor " 1186 "has out after in: idx %d\n", i); 1187 return -EINVAL; 1188 } 1189 *out_num += ret; 1190 } 1191 } while ((i = next_desc(&desc)) != -1); 1192 return 0; 1193 } 1194 1195 /* This looks in the virtqueue and for the first available buffer, and converts 1196 * it to an iovec for convenient access. Since descriptors consist of some 1197 * number of output then some number of input descriptors, it's actually two 1198 * iovecs, but we pack them into one and note how many of each there were. 1199 * 1200 * This function returns the descriptor number found, or vq->num (which is 1201 * never a valid descriptor number) if none was found. A negative code is 1202 * returned on error. */ 1203 int vhost_get_vq_desc(struct vhost_dev *dev, struct vhost_virtqueue *vq, 1204 struct iovec iov[], unsigned int iov_size, 1205 unsigned int *out_num, unsigned int *in_num, 1206 struct vhost_log *log, unsigned int *log_num) 1207 { 1208 struct vring_desc desc; 1209 unsigned int i, head, found = 0; 1210 u16 last_avail_idx; 1211 int ret; 1212 1213 /* Check it isn't doing very strange things with descriptor numbers. */ 1214 last_avail_idx = vq->last_avail_idx; 1215 if (unlikely(__get_user(vq->avail_idx, &vq->avail->idx))) { 1216 vq_err(vq, "Failed to access avail idx at %p\n", 1217 &vq->avail->idx); 1218 return -EFAULT; 1219 } 1220 1221 if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) { 1222 vq_err(vq, "Guest moved used index from %u to %u", 1223 last_avail_idx, vq->avail_idx); 1224 return -EFAULT; 1225 } 1226 1227 /* If there's nothing new since last we looked, return invalid. */ 1228 if (vq->avail_idx == last_avail_idx) 1229 return vq->num; 1230 1231 /* Only get avail ring entries after they have been exposed by guest. */ 1232 smp_rmb(); 1233 1234 /* Grab the next descriptor number they're advertising, and increment 1235 * the index we've seen. */ 1236 if (unlikely(__get_user(head, 1237 &vq->avail->ring[last_avail_idx % vq->num]))) { 1238 vq_err(vq, "Failed to read head: idx %d address %p\n", 1239 last_avail_idx, 1240 &vq->avail->ring[last_avail_idx % vq->num]); 1241 return -EFAULT; 1242 } 1243 1244 /* If their number is silly, that's an error. */ 1245 if (unlikely(head >= vq->num)) { 1246 vq_err(vq, "Guest says index %u > %u is available", 1247 head, vq->num); 1248 return -EINVAL; 1249 } 1250 1251 /* When we start there are none of either input nor output. */ 1252 *out_num = *in_num = 0; 1253 if (unlikely(log)) 1254 *log_num = 0; 1255 1256 i = head; 1257 do { 1258 unsigned iov_count = *in_num + *out_num; 1259 if (unlikely(i >= vq->num)) { 1260 vq_err(vq, "Desc index is %u > %u, head = %u", 1261 i, vq->num, head); 1262 return -EINVAL; 1263 } 1264 if (unlikely(++found > vq->num)) { 1265 vq_err(vq, "Loop detected: last one at %u " 1266 "vq size %u head %u\n", 1267 i, vq->num, head); 1268 return -EINVAL; 1269 } 1270 ret = __copy_from_user(&desc, vq->desc + i, sizeof desc); 1271 if (unlikely(ret)) { 1272 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", 1273 i, vq->desc + i); 1274 return -EFAULT; 1275 } 1276 if (desc.flags & VRING_DESC_F_INDIRECT) { 1277 ret = get_indirect(dev, vq, iov, iov_size, 1278 out_num, in_num, 1279 log, log_num, &desc); 1280 if (unlikely(ret < 0)) { 1281 vq_err(vq, "Failure detected " 1282 "in indirect descriptor at idx %d\n", i); 1283 return ret; 1284 } 1285 continue; 1286 } 1287 1288 ret = translate_desc(dev, desc.addr, desc.len, iov + iov_count, 1289 iov_size - iov_count); 1290 if (unlikely(ret < 0)) { 1291 vq_err(vq, "Translation failure %d descriptor idx %d\n", 1292 ret, i); 1293 return ret; 1294 } 1295 if (desc.flags & VRING_DESC_F_WRITE) { 1296 /* If this is an input descriptor, 1297 * increment that count. */ 1298 *in_num += ret; 1299 if (unlikely(log)) { 1300 log[*log_num].addr = desc.addr; 1301 log[*log_num].len = desc.len; 1302 ++*log_num; 1303 } 1304 } else { 1305 /* If it's an output descriptor, they're all supposed 1306 * to come before any input descriptors. */ 1307 if (unlikely(*in_num)) { 1308 vq_err(vq, "Descriptor has out after in: " 1309 "idx %d\n", i); 1310 return -EINVAL; 1311 } 1312 *out_num += ret; 1313 } 1314 } while ((i = next_desc(&desc)) != -1); 1315 1316 /* On success, increment avail index. */ 1317 vq->last_avail_idx++; 1318 1319 /* Assume notifications from guest are disabled at this point, 1320 * if they aren't we would need to update avail_event index. */ 1321 BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY)); 1322 return head; 1323 } 1324 EXPORT_SYMBOL_GPL(vhost_get_vq_desc); 1325 1326 /* Reverse the effect of vhost_get_vq_desc. Useful for error handling. */ 1327 void vhost_discard_vq_desc(struct vhost_virtqueue *vq, int n) 1328 { 1329 vq->last_avail_idx -= n; 1330 } 1331 EXPORT_SYMBOL_GPL(vhost_discard_vq_desc); 1332 1333 /* After we've used one of their buffers, we tell them about it. We'll then 1334 * want to notify the guest, using eventfd. */ 1335 int vhost_add_used(struct vhost_virtqueue *vq, unsigned int head, int len) 1336 { 1337 struct vring_used_elem heads = { head, len }; 1338 1339 return vhost_add_used_n(vq, &heads, 1); 1340 } 1341 EXPORT_SYMBOL_GPL(vhost_add_used); 1342 1343 static int __vhost_add_used_n(struct vhost_virtqueue *vq, 1344 struct vring_used_elem *heads, 1345 unsigned count) 1346 { 1347 struct vring_used_elem __user *used; 1348 u16 old, new; 1349 int start; 1350 1351 start = vq->last_used_idx % vq->num; 1352 used = vq->used->ring + start; 1353 if (count == 1) { 1354 if (__put_user(heads[0].id, &used->id)) { 1355 vq_err(vq, "Failed to write used id"); 1356 return -EFAULT; 1357 } 1358 if (__put_user(heads[0].len, &used->len)) { 1359 vq_err(vq, "Failed to write used len"); 1360 return -EFAULT; 1361 } 1362 } else if (__copy_to_user(used, heads, count * sizeof *used)) { 1363 vq_err(vq, "Failed to write used"); 1364 return -EFAULT; 1365 } 1366 if (unlikely(vq->log_used)) { 1367 /* Make sure data is seen before log. */ 1368 smp_wmb(); 1369 /* Log used ring entry write. */ 1370 log_write(vq->log_base, 1371 vq->log_addr + 1372 ((void __user *)used - (void __user *)vq->used), 1373 count * sizeof *used); 1374 } 1375 old = vq->last_used_idx; 1376 new = (vq->last_used_idx += count); 1377 /* If the driver never bothers to signal in a very long while, 1378 * used index might wrap around. If that happens, invalidate 1379 * signalled_used index we stored. TODO: make sure driver 1380 * signals at least once in 2^16 and remove this. */ 1381 if (unlikely((u16)(new - vq->signalled_used) < (u16)(new - old))) 1382 vq->signalled_used_valid = false; 1383 return 0; 1384 } 1385 1386 /* After we've used one of their buffers, we tell them about it. We'll then 1387 * want to notify the guest, using eventfd. */ 1388 int vhost_add_used_n(struct vhost_virtqueue *vq, struct vring_used_elem *heads, 1389 unsigned count) 1390 { 1391 int start, n, r; 1392 1393 start = vq->last_used_idx % vq->num; 1394 n = vq->num - start; 1395 if (n < count) { 1396 r = __vhost_add_used_n(vq, heads, n); 1397 if (r < 0) 1398 return r; 1399 heads += n; 1400 count -= n; 1401 } 1402 r = __vhost_add_used_n(vq, heads, count); 1403 1404 /* Make sure buffer is written before we update index. */ 1405 smp_wmb(); 1406 if (put_user(vq->last_used_idx, &vq->used->idx)) { 1407 vq_err(vq, "Failed to increment used idx"); 1408 return -EFAULT; 1409 } 1410 if (unlikely(vq->log_used)) { 1411 /* Log used index update. */ 1412 log_write(vq->log_base, 1413 vq->log_addr + offsetof(struct vring_used, idx), 1414 sizeof vq->used->idx); 1415 if (vq->log_ctx) 1416 eventfd_signal(vq->log_ctx, 1); 1417 } 1418 return r; 1419 } 1420 EXPORT_SYMBOL_GPL(vhost_add_used_n); 1421 1422 static bool vhost_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1423 { 1424 __u16 old, new, event; 1425 bool v; 1426 /* Flush out used index updates. This is paired 1427 * with the barrier that the Guest executes when enabling 1428 * interrupts. */ 1429 smp_mb(); 1430 1431 if (vhost_has_feature(dev, VIRTIO_F_NOTIFY_ON_EMPTY) && 1432 unlikely(vq->avail_idx == vq->last_avail_idx)) 1433 return true; 1434 1435 if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) { 1436 __u16 flags; 1437 if (__get_user(flags, &vq->avail->flags)) { 1438 vq_err(vq, "Failed to get flags"); 1439 return true; 1440 } 1441 return !(flags & VRING_AVAIL_F_NO_INTERRUPT); 1442 } 1443 old = vq->signalled_used; 1444 v = vq->signalled_used_valid; 1445 new = vq->signalled_used = vq->last_used_idx; 1446 vq->signalled_used_valid = true; 1447 1448 if (unlikely(!v)) 1449 return true; 1450 1451 if (get_user(event, vhost_used_event(vq))) { 1452 vq_err(vq, "Failed to get used event idx"); 1453 return true; 1454 } 1455 return vring_need_event(event, new, old); 1456 } 1457 1458 /* This actually signals the guest, using eventfd. */ 1459 void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1460 { 1461 /* Signal the Guest tell them we used something up. */ 1462 if (vq->call_ctx && vhost_notify(dev, vq)) 1463 eventfd_signal(vq->call_ctx, 1); 1464 } 1465 EXPORT_SYMBOL_GPL(vhost_signal); 1466 1467 /* And here's the combo meal deal. Supersize me! */ 1468 void vhost_add_used_and_signal(struct vhost_dev *dev, 1469 struct vhost_virtqueue *vq, 1470 unsigned int head, int len) 1471 { 1472 vhost_add_used(vq, head, len); 1473 vhost_signal(dev, vq); 1474 } 1475 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal); 1476 1477 /* multi-buffer version of vhost_add_used_and_signal */ 1478 void vhost_add_used_and_signal_n(struct vhost_dev *dev, 1479 struct vhost_virtqueue *vq, 1480 struct vring_used_elem *heads, unsigned count) 1481 { 1482 vhost_add_used_n(vq, heads, count); 1483 vhost_signal(dev, vq); 1484 } 1485 EXPORT_SYMBOL_GPL(vhost_add_used_and_signal_n); 1486 1487 /* OK, now we need to know about added descriptors. */ 1488 bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1489 { 1490 u16 avail_idx; 1491 int r; 1492 1493 if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY)) 1494 return false; 1495 vq->used_flags &= ~VRING_USED_F_NO_NOTIFY; 1496 if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) { 1497 r = vhost_update_used_flags(vq); 1498 if (r) { 1499 vq_err(vq, "Failed to enable notification at %p: %d\n", 1500 &vq->used->flags, r); 1501 return false; 1502 } 1503 } else { 1504 r = vhost_update_avail_event(vq, vq->avail_idx); 1505 if (r) { 1506 vq_err(vq, "Failed to update avail event index at %p: %d\n", 1507 vhost_avail_event(vq), r); 1508 return false; 1509 } 1510 } 1511 /* They could have slipped one in as we were doing that: make 1512 * sure it's written, then check again. */ 1513 smp_mb(); 1514 r = __get_user(avail_idx, &vq->avail->idx); 1515 if (r) { 1516 vq_err(vq, "Failed to check avail idx at %p: %d\n", 1517 &vq->avail->idx, r); 1518 return false; 1519 } 1520 1521 return avail_idx != vq->avail_idx; 1522 } 1523 EXPORT_SYMBOL_GPL(vhost_enable_notify); 1524 1525 /* We don't need to be notified again. */ 1526 void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq) 1527 { 1528 int r; 1529 1530 if (vq->used_flags & VRING_USED_F_NO_NOTIFY) 1531 return; 1532 vq->used_flags |= VRING_USED_F_NO_NOTIFY; 1533 if (!vhost_has_feature(dev, VIRTIO_RING_F_EVENT_IDX)) { 1534 r = vhost_update_used_flags(vq); 1535 if (r) 1536 vq_err(vq, "Failed to enable notification at %p: %d\n", 1537 &vq->used->flags, r); 1538 } 1539 } 1540 EXPORT_SYMBOL_GPL(vhost_disable_notify); 1541 1542 static int __init vhost_init(void) 1543 { 1544 return 0; 1545 } 1546 1547 static void __exit vhost_exit(void) 1548 { 1549 } 1550 1551 module_init(vhost_init); 1552 module_exit(vhost_exit); 1553 1554 MODULE_VERSION("0.0.1"); 1555 MODULE_LICENSE("GPL v2"); 1556 MODULE_AUTHOR("Michael S. Tsirkin"); 1557 MODULE_DESCRIPTION("Host kernel accelerator for virtio"); 1558