1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDUSE: vDPA Device in Userspace 4 * 5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved. 6 * 7 * Author: Xie Yongji <xieyongji@bytedance.com> 8 * 9 */ 10 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/eventfd.h> 16 #include <linux/slab.h> 17 #include <linux/wait.h> 18 #include <linux/dma-map-ops.h> 19 #include <linux/poll.h> 20 #include <linux/file.h> 21 #include <linux/uio.h> 22 #include <linux/vdpa.h> 23 #include <linux/nospec.h> 24 #include <linux/vmalloc.h> 25 #include <linux/sched/mm.h> 26 #include <uapi/linux/vduse.h> 27 #include <uapi/linux/vdpa.h> 28 #include <uapi/linux/virtio_config.h> 29 #include <uapi/linux/virtio_ids.h> 30 #include <uapi/linux/virtio_blk.h> 31 #include <linux/mod_devicetable.h> 32 33 #include "iova_domain.h" 34 35 #define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>" 36 #define DRV_DESC "vDPA Device in Userspace" 37 #define DRV_LICENSE "GPL v2" 38 39 #define VDUSE_DEV_MAX (1U << MINORBITS) 40 #define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024) 41 #define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024) 42 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024) 43 /* 128 MB reserved for virtqueue creation */ 44 #define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024) 45 #define VDUSE_MSG_DEFAULT_TIMEOUT 30 46 47 #define IRQ_UNBOUND -1 48 49 struct vduse_virtqueue { 50 u16 index; 51 u16 num_max; 52 u32 num; 53 u64 desc_addr; 54 u64 driver_addr; 55 u64 device_addr; 56 struct vdpa_vq_state state; 57 bool ready; 58 bool kicked; 59 spinlock_t kick_lock; 60 spinlock_t irq_lock; 61 struct eventfd_ctx *kickfd; 62 struct vdpa_callback cb; 63 struct work_struct inject; 64 struct work_struct kick; 65 int irq_effective_cpu; 66 struct cpumask irq_affinity; 67 struct kobject kobj; 68 }; 69 70 struct vduse_dev; 71 72 struct vduse_vdpa { 73 struct vdpa_device vdpa; 74 struct vduse_dev *dev; 75 }; 76 77 struct vduse_umem { 78 unsigned long iova; 79 unsigned long npages; 80 struct page **pages; 81 struct mm_struct *mm; 82 }; 83 84 struct vduse_dev { 85 struct vduse_vdpa *vdev; 86 struct device *dev; 87 struct vduse_virtqueue **vqs; 88 struct vduse_iova_domain *domain; 89 char *name; 90 struct mutex lock; 91 spinlock_t msg_lock; 92 u64 msg_unique; 93 u32 msg_timeout; 94 wait_queue_head_t waitq; 95 struct list_head send_list; 96 struct list_head recv_list; 97 struct vdpa_callback config_cb; 98 struct work_struct inject; 99 spinlock_t irq_lock; 100 struct rw_semaphore rwsem; 101 int minor; 102 bool broken; 103 bool connected; 104 u64 api_version; 105 u64 device_features; 106 u64 driver_features; 107 u32 device_id; 108 u32 vendor_id; 109 u32 generation; 110 u32 config_size; 111 void *config; 112 u8 status; 113 u32 vq_num; 114 u32 vq_align; 115 struct vduse_umem *umem; 116 struct mutex mem_lock; 117 unsigned int bounce_size; 118 struct mutex domain_lock; 119 }; 120 121 struct vduse_dev_msg { 122 struct vduse_dev_request req; 123 struct vduse_dev_response resp; 124 struct list_head list; 125 wait_queue_head_t waitq; 126 bool completed; 127 }; 128 129 struct vduse_control { 130 u64 api_version; 131 }; 132 133 static DEFINE_MUTEX(vduse_lock); 134 static DEFINE_IDR(vduse_idr); 135 136 static dev_t vduse_major; 137 static struct class *vduse_class; 138 static struct cdev vduse_ctrl_cdev; 139 static struct cdev vduse_cdev; 140 static struct workqueue_struct *vduse_irq_wq; 141 static struct workqueue_struct *vduse_irq_bound_wq; 142 143 static u32 allowed_device_id[] = { 144 VIRTIO_ID_BLOCK, 145 }; 146 147 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa) 148 { 149 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa); 150 151 return vdev->dev; 152 } 153 154 static inline struct vduse_dev *dev_to_vduse(struct device *dev) 155 { 156 struct vdpa_device *vdpa = dev_to_vdpa(dev); 157 158 return vdpa_to_vduse(vdpa); 159 } 160 161 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head, 162 uint32_t request_id) 163 { 164 struct vduse_dev_msg *msg; 165 166 list_for_each_entry(msg, head, list) { 167 if (msg->req.request_id == request_id) { 168 list_del(&msg->list); 169 return msg; 170 } 171 } 172 173 return NULL; 174 } 175 176 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head) 177 { 178 struct vduse_dev_msg *msg = NULL; 179 180 if (!list_empty(head)) { 181 msg = list_first_entry(head, struct vduse_dev_msg, list); 182 list_del(&msg->list); 183 } 184 185 return msg; 186 } 187 188 static void vduse_enqueue_msg(struct list_head *head, 189 struct vduse_dev_msg *msg) 190 { 191 list_add_tail(&msg->list, head); 192 } 193 194 static void vduse_dev_broken(struct vduse_dev *dev) 195 { 196 struct vduse_dev_msg *msg, *tmp; 197 198 if (unlikely(dev->broken)) 199 return; 200 201 list_splice_init(&dev->recv_list, &dev->send_list); 202 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) { 203 list_del(&msg->list); 204 msg->completed = 1; 205 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 206 wake_up(&msg->waitq); 207 } 208 dev->broken = true; 209 wake_up(&dev->waitq); 210 } 211 212 static int vduse_dev_msg_sync(struct vduse_dev *dev, 213 struct vduse_dev_msg *msg) 214 { 215 int ret; 216 217 if (unlikely(dev->broken)) 218 return -EIO; 219 220 init_waitqueue_head(&msg->waitq); 221 spin_lock(&dev->msg_lock); 222 if (unlikely(dev->broken)) { 223 spin_unlock(&dev->msg_lock); 224 return -EIO; 225 } 226 msg->req.request_id = dev->msg_unique++; 227 vduse_enqueue_msg(&dev->send_list, msg); 228 wake_up(&dev->waitq); 229 spin_unlock(&dev->msg_lock); 230 if (dev->msg_timeout) 231 ret = wait_event_killable_timeout(msg->waitq, msg->completed, 232 (long)dev->msg_timeout * HZ); 233 else 234 ret = wait_event_killable(msg->waitq, msg->completed); 235 236 spin_lock(&dev->msg_lock); 237 if (!msg->completed) { 238 list_del(&msg->list); 239 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 240 /* Mark the device as malfunction when there is a timeout */ 241 if (!ret) 242 vduse_dev_broken(dev); 243 } 244 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO; 245 spin_unlock(&dev->msg_lock); 246 247 return ret; 248 } 249 250 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev, 251 struct vduse_virtqueue *vq, 252 struct vdpa_vq_state_packed *packed) 253 { 254 struct vduse_dev_msg msg = { 0 }; 255 int ret; 256 257 msg.req.type = VDUSE_GET_VQ_STATE; 258 msg.req.vq_state.index = vq->index; 259 260 ret = vduse_dev_msg_sync(dev, &msg); 261 if (ret) 262 return ret; 263 264 packed->last_avail_counter = 265 msg.resp.vq_state.packed.last_avail_counter & 0x0001; 266 packed->last_avail_idx = 267 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF; 268 packed->last_used_counter = 269 msg.resp.vq_state.packed.last_used_counter & 0x0001; 270 packed->last_used_idx = 271 msg.resp.vq_state.packed.last_used_idx & 0x7FFF; 272 273 return 0; 274 } 275 276 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev, 277 struct vduse_virtqueue *vq, 278 struct vdpa_vq_state_split *split) 279 { 280 struct vduse_dev_msg msg = { 0 }; 281 int ret; 282 283 msg.req.type = VDUSE_GET_VQ_STATE; 284 msg.req.vq_state.index = vq->index; 285 286 ret = vduse_dev_msg_sync(dev, &msg); 287 if (ret) 288 return ret; 289 290 split->avail_index = msg.resp.vq_state.split.avail_index; 291 292 return 0; 293 } 294 295 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status) 296 { 297 struct vduse_dev_msg msg = { 0 }; 298 299 msg.req.type = VDUSE_SET_STATUS; 300 msg.req.s.status = status; 301 302 return vduse_dev_msg_sync(dev, &msg); 303 } 304 305 static int vduse_dev_update_iotlb(struct vduse_dev *dev, 306 u64 start, u64 last) 307 { 308 struct vduse_dev_msg msg = { 0 }; 309 310 if (last < start) 311 return -EINVAL; 312 313 msg.req.type = VDUSE_UPDATE_IOTLB; 314 msg.req.iova.start = start; 315 msg.req.iova.last = last; 316 317 return vduse_dev_msg_sync(dev, &msg); 318 } 319 320 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) 321 { 322 struct file *file = iocb->ki_filp; 323 struct vduse_dev *dev = file->private_data; 324 struct vduse_dev_msg *msg; 325 int size = sizeof(struct vduse_dev_request); 326 ssize_t ret; 327 328 if (iov_iter_count(to) < size) 329 return -EINVAL; 330 331 spin_lock(&dev->msg_lock); 332 while (1) { 333 msg = vduse_dequeue_msg(&dev->send_list); 334 if (msg) 335 break; 336 337 ret = -EAGAIN; 338 if (file->f_flags & O_NONBLOCK) 339 goto unlock; 340 341 spin_unlock(&dev->msg_lock); 342 ret = wait_event_interruptible_exclusive(dev->waitq, 343 !list_empty(&dev->send_list)); 344 if (ret) 345 return ret; 346 347 spin_lock(&dev->msg_lock); 348 } 349 spin_unlock(&dev->msg_lock); 350 ret = copy_to_iter(&msg->req, size, to); 351 spin_lock(&dev->msg_lock); 352 if (ret != size) { 353 ret = -EFAULT; 354 vduse_enqueue_msg(&dev->send_list, msg); 355 goto unlock; 356 } 357 vduse_enqueue_msg(&dev->recv_list, msg); 358 unlock: 359 spin_unlock(&dev->msg_lock); 360 361 return ret; 362 } 363 364 static bool is_mem_zero(const char *ptr, int size) 365 { 366 int i; 367 368 for (i = 0; i < size; i++) { 369 if (ptr[i]) 370 return false; 371 } 372 return true; 373 } 374 375 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from) 376 { 377 struct file *file = iocb->ki_filp; 378 struct vduse_dev *dev = file->private_data; 379 struct vduse_dev_response resp; 380 struct vduse_dev_msg *msg; 381 size_t ret; 382 383 ret = copy_from_iter(&resp, sizeof(resp), from); 384 if (ret != sizeof(resp)) 385 return -EINVAL; 386 387 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved))) 388 return -EINVAL; 389 390 spin_lock(&dev->msg_lock); 391 msg = vduse_find_msg(&dev->recv_list, resp.request_id); 392 if (!msg) { 393 ret = -ENOENT; 394 goto unlock; 395 } 396 397 memcpy(&msg->resp, &resp, sizeof(resp)); 398 msg->completed = 1; 399 wake_up(&msg->waitq); 400 unlock: 401 spin_unlock(&dev->msg_lock); 402 403 return ret; 404 } 405 406 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait) 407 { 408 struct vduse_dev *dev = file->private_data; 409 __poll_t mask = 0; 410 411 poll_wait(file, &dev->waitq, wait); 412 413 spin_lock(&dev->msg_lock); 414 415 if (unlikely(dev->broken)) 416 mask |= EPOLLERR; 417 if (!list_empty(&dev->send_list)) 418 mask |= EPOLLIN | EPOLLRDNORM; 419 if (!list_empty(&dev->recv_list)) 420 mask |= EPOLLOUT | EPOLLWRNORM; 421 422 spin_unlock(&dev->msg_lock); 423 424 return mask; 425 } 426 427 static void vduse_dev_reset(struct vduse_dev *dev) 428 { 429 int i; 430 struct vduse_iova_domain *domain = dev->domain; 431 432 /* The coherent mappings are handled in vduse_dev_free_coherent() */ 433 if (domain && domain->bounce_map) 434 vduse_domain_reset_bounce_map(domain); 435 436 down_write(&dev->rwsem); 437 438 dev->status = 0; 439 dev->driver_features = 0; 440 dev->generation++; 441 spin_lock(&dev->irq_lock); 442 dev->config_cb.callback = NULL; 443 dev->config_cb.private = NULL; 444 spin_unlock(&dev->irq_lock); 445 flush_work(&dev->inject); 446 447 for (i = 0; i < dev->vq_num; i++) { 448 struct vduse_virtqueue *vq = dev->vqs[i]; 449 450 vq->ready = false; 451 vq->desc_addr = 0; 452 vq->driver_addr = 0; 453 vq->device_addr = 0; 454 vq->num = 0; 455 memset(&vq->state, 0, sizeof(vq->state)); 456 457 spin_lock(&vq->kick_lock); 458 vq->kicked = false; 459 if (vq->kickfd) 460 eventfd_ctx_put(vq->kickfd); 461 vq->kickfd = NULL; 462 spin_unlock(&vq->kick_lock); 463 464 spin_lock(&vq->irq_lock); 465 vq->cb.callback = NULL; 466 vq->cb.private = NULL; 467 vq->cb.trigger = NULL; 468 spin_unlock(&vq->irq_lock); 469 flush_work(&vq->inject); 470 flush_work(&vq->kick); 471 } 472 473 up_write(&dev->rwsem); 474 } 475 476 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx, 477 u64 desc_area, u64 driver_area, 478 u64 device_area) 479 { 480 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 481 struct vduse_virtqueue *vq = dev->vqs[idx]; 482 483 vq->desc_addr = desc_area; 484 vq->driver_addr = driver_area; 485 vq->device_addr = device_area; 486 487 return 0; 488 } 489 490 static void vduse_vq_kick(struct vduse_virtqueue *vq) 491 { 492 spin_lock(&vq->kick_lock); 493 if (!vq->ready) 494 goto unlock; 495 496 if (vq->kickfd) 497 eventfd_signal(vq->kickfd, 1); 498 else 499 vq->kicked = true; 500 unlock: 501 spin_unlock(&vq->kick_lock); 502 } 503 504 static void vduse_vq_kick_work(struct work_struct *work) 505 { 506 struct vduse_virtqueue *vq = container_of(work, 507 struct vduse_virtqueue, kick); 508 509 vduse_vq_kick(vq); 510 } 511 512 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx) 513 { 514 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 515 struct vduse_virtqueue *vq = dev->vqs[idx]; 516 517 if (!eventfd_signal_allowed()) { 518 schedule_work(&vq->kick); 519 return; 520 } 521 vduse_vq_kick(vq); 522 } 523 524 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 525 struct vdpa_callback *cb) 526 { 527 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 528 struct vduse_virtqueue *vq = dev->vqs[idx]; 529 530 spin_lock(&vq->irq_lock); 531 vq->cb.callback = cb->callback; 532 vq->cb.private = cb->private; 533 vq->cb.trigger = cb->trigger; 534 spin_unlock(&vq->irq_lock); 535 } 536 537 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 538 { 539 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 540 struct vduse_virtqueue *vq = dev->vqs[idx]; 541 542 vq->num = num; 543 } 544 545 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa, 546 u16 idx, bool ready) 547 { 548 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 549 struct vduse_virtqueue *vq = dev->vqs[idx]; 550 551 vq->ready = ready; 552 } 553 554 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 555 { 556 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 557 struct vduse_virtqueue *vq = dev->vqs[idx]; 558 559 return vq->ready; 560 } 561 562 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx, 563 const struct vdpa_vq_state *state) 564 { 565 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 566 struct vduse_virtqueue *vq = dev->vqs[idx]; 567 568 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 569 vq->state.packed.last_avail_counter = 570 state->packed.last_avail_counter; 571 vq->state.packed.last_avail_idx = state->packed.last_avail_idx; 572 vq->state.packed.last_used_counter = 573 state->packed.last_used_counter; 574 vq->state.packed.last_used_idx = state->packed.last_used_idx; 575 } else 576 vq->state.split.avail_index = state->split.avail_index; 577 578 return 0; 579 } 580 581 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx, 582 struct vdpa_vq_state *state) 583 { 584 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 585 struct vduse_virtqueue *vq = dev->vqs[idx]; 586 587 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) 588 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed); 589 590 return vduse_dev_get_vq_state_split(dev, vq, &state->split); 591 } 592 593 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa) 594 { 595 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 596 597 return dev->vq_align; 598 } 599 600 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa) 601 { 602 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 603 604 return dev->device_features; 605 } 606 607 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) 608 { 609 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 610 611 dev->driver_features = features; 612 return 0; 613 } 614 615 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa) 616 { 617 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 618 619 return dev->driver_features; 620 } 621 622 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa, 623 struct vdpa_callback *cb) 624 { 625 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 626 627 spin_lock(&dev->irq_lock); 628 dev->config_cb.callback = cb->callback; 629 dev->config_cb.private = cb->private; 630 spin_unlock(&dev->irq_lock); 631 } 632 633 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa) 634 { 635 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 636 u16 num_max = 0; 637 int i; 638 639 for (i = 0; i < dev->vq_num; i++) 640 if (num_max < dev->vqs[i]->num_max) 641 num_max = dev->vqs[i]->num_max; 642 643 return num_max; 644 } 645 646 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa) 647 { 648 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 649 650 return dev->device_id; 651 } 652 653 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa) 654 { 655 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 656 657 return dev->vendor_id; 658 } 659 660 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa) 661 { 662 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 663 664 return dev->status; 665 } 666 667 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status) 668 { 669 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 670 671 if (vduse_dev_set_status(dev, status)) 672 return; 673 674 dev->status = status; 675 } 676 677 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa) 678 { 679 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 680 681 return dev->config_size; 682 } 683 684 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset, 685 void *buf, unsigned int len) 686 { 687 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 688 689 /* Initialize the buffer in case of partial copy. */ 690 memset(buf, 0, len); 691 692 if (offset > dev->config_size) 693 return; 694 695 if (len > dev->config_size - offset) 696 len = dev->config_size - offset; 697 698 memcpy(buf, dev->config + offset, len); 699 } 700 701 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset, 702 const void *buf, unsigned int len) 703 { 704 /* Now we only support read-only configuration space */ 705 } 706 707 static int vduse_vdpa_reset(struct vdpa_device *vdpa) 708 { 709 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 710 int ret = vduse_dev_set_status(dev, 0); 711 712 vduse_dev_reset(dev); 713 714 return ret; 715 } 716 717 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa) 718 { 719 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 720 721 return dev->generation; 722 } 723 724 static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx, 725 const struct cpumask *cpu_mask) 726 { 727 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 728 729 cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); 730 return 0; 731 } 732 733 static const struct cpumask * 734 vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx) 735 { 736 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 737 738 return &dev->vqs[idx]->irq_affinity; 739 } 740 741 static int vduse_vdpa_set_map(struct vdpa_device *vdpa, 742 unsigned int asid, 743 struct vhost_iotlb *iotlb) 744 { 745 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 746 int ret; 747 748 ret = vduse_domain_set_map(dev->domain, iotlb); 749 if (ret) 750 return ret; 751 752 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX); 753 if (ret) { 754 vduse_domain_clear_map(dev->domain, iotlb); 755 return ret; 756 } 757 758 return 0; 759 } 760 761 static void vduse_vdpa_free(struct vdpa_device *vdpa) 762 { 763 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 764 765 dev->vdev = NULL; 766 } 767 768 static const struct vdpa_config_ops vduse_vdpa_config_ops = { 769 .set_vq_address = vduse_vdpa_set_vq_address, 770 .kick_vq = vduse_vdpa_kick_vq, 771 .set_vq_cb = vduse_vdpa_set_vq_cb, 772 .set_vq_num = vduse_vdpa_set_vq_num, 773 .set_vq_ready = vduse_vdpa_set_vq_ready, 774 .get_vq_ready = vduse_vdpa_get_vq_ready, 775 .set_vq_state = vduse_vdpa_set_vq_state, 776 .get_vq_state = vduse_vdpa_get_vq_state, 777 .get_vq_align = vduse_vdpa_get_vq_align, 778 .get_device_features = vduse_vdpa_get_device_features, 779 .set_driver_features = vduse_vdpa_set_driver_features, 780 .get_driver_features = vduse_vdpa_get_driver_features, 781 .set_config_cb = vduse_vdpa_set_config_cb, 782 .get_vq_num_max = vduse_vdpa_get_vq_num_max, 783 .get_device_id = vduse_vdpa_get_device_id, 784 .get_vendor_id = vduse_vdpa_get_vendor_id, 785 .get_status = vduse_vdpa_get_status, 786 .set_status = vduse_vdpa_set_status, 787 .get_config_size = vduse_vdpa_get_config_size, 788 .get_config = vduse_vdpa_get_config, 789 .set_config = vduse_vdpa_set_config, 790 .get_generation = vduse_vdpa_get_generation, 791 .set_vq_affinity = vduse_vdpa_set_vq_affinity, 792 .get_vq_affinity = vduse_vdpa_get_vq_affinity, 793 .reset = vduse_vdpa_reset, 794 .set_map = vduse_vdpa_set_map, 795 .free = vduse_vdpa_free, 796 }; 797 798 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, 799 unsigned long offset, size_t size, 800 enum dma_data_direction dir, 801 unsigned long attrs) 802 { 803 struct vduse_dev *vdev = dev_to_vduse(dev); 804 struct vduse_iova_domain *domain = vdev->domain; 805 806 return vduse_domain_map_page(domain, page, offset, size, dir, attrs); 807 } 808 809 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr, 810 size_t size, enum dma_data_direction dir, 811 unsigned long attrs) 812 { 813 struct vduse_dev *vdev = dev_to_vduse(dev); 814 struct vduse_iova_domain *domain = vdev->domain; 815 816 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs); 817 } 818 819 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size, 820 dma_addr_t *dma_addr, gfp_t flag, 821 unsigned long attrs) 822 { 823 struct vduse_dev *vdev = dev_to_vduse(dev); 824 struct vduse_iova_domain *domain = vdev->domain; 825 unsigned long iova; 826 void *addr; 827 828 *dma_addr = DMA_MAPPING_ERROR; 829 addr = vduse_domain_alloc_coherent(domain, size, 830 (dma_addr_t *)&iova, flag, attrs); 831 if (!addr) 832 return NULL; 833 834 *dma_addr = (dma_addr_t)iova; 835 836 return addr; 837 } 838 839 static void vduse_dev_free_coherent(struct device *dev, size_t size, 840 void *vaddr, dma_addr_t dma_addr, 841 unsigned long attrs) 842 { 843 struct vduse_dev *vdev = dev_to_vduse(dev); 844 struct vduse_iova_domain *domain = vdev->domain; 845 846 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs); 847 } 848 849 static size_t vduse_dev_max_mapping_size(struct device *dev) 850 { 851 struct vduse_dev *vdev = dev_to_vduse(dev); 852 struct vduse_iova_domain *domain = vdev->domain; 853 854 return domain->bounce_size; 855 } 856 857 static const struct dma_map_ops vduse_dev_dma_ops = { 858 .map_page = vduse_dev_map_page, 859 .unmap_page = vduse_dev_unmap_page, 860 .alloc = vduse_dev_alloc_coherent, 861 .free = vduse_dev_free_coherent, 862 .max_mapping_size = vduse_dev_max_mapping_size, 863 }; 864 865 static unsigned int perm_to_file_flags(u8 perm) 866 { 867 unsigned int flags = 0; 868 869 switch (perm) { 870 case VDUSE_ACCESS_WO: 871 flags |= O_WRONLY; 872 break; 873 case VDUSE_ACCESS_RO: 874 flags |= O_RDONLY; 875 break; 876 case VDUSE_ACCESS_RW: 877 flags |= O_RDWR; 878 break; 879 default: 880 WARN(1, "invalidate vhost IOTLB permission\n"); 881 break; 882 } 883 884 return flags; 885 } 886 887 static int vduse_kickfd_setup(struct vduse_dev *dev, 888 struct vduse_vq_eventfd *eventfd) 889 { 890 struct eventfd_ctx *ctx = NULL; 891 struct vduse_virtqueue *vq; 892 u32 index; 893 894 if (eventfd->index >= dev->vq_num) 895 return -EINVAL; 896 897 index = array_index_nospec(eventfd->index, dev->vq_num); 898 vq = dev->vqs[index]; 899 if (eventfd->fd >= 0) { 900 ctx = eventfd_ctx_fdget(eventfd->fd); 901 if (IS_ERR(ctx)) 902 return PTR_ERR(ctx); 903 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN) 904 return 0; 905 906 spin_lock(&vq->kick_lock); 907 if (vq->kickfd) 908 eventfd_ctx_put(vq->kickfd); 909 vq->kickfd = ctx; 910 if (vq->ready && vq->kicked && vq->kickfd) { 911 eventfd_signal(vq->kickfd, 1); 912 vq->kicked = false; 913 } 914 spin_unlock(&vq->kick_lock); 915 916 return 0; 917 } 918 919 static bool vduse_dev_is_ready(struct vduse_dev *dev) 920 { 921 int i; 922 923 for (i = 0; i < dev->vq_num; i++) 924 if (!dev->vqs[i]->num_max) 925 return false; 926 927 return true; 928 } 929 930 static void vduse_dev_irq_inject(struct work_struct *work) 931 { 932 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject); 933 934 spin_lock_irq(&dev->irq_lock); 935 if (dev->config_cb.callback) 936 dev->config_cb.callback(dev->config_cb.private); 937 spin_unlock_irq(&dev->irq_lock); 938 } 939 940 static void vduse_vq_irq_inject(struct work_struct *work) 941 { 942 struct vduse_virtqueue *vq = container_of(work, 943 struct vduse_virtqueue, inject); 944 945 spin_lock_irq(&vq->irq_lock); 946 if (vq->ready && vq->cb.callback) 947 vq->cb.callback(vq->cb.private); 948 spin_unlock_irq(&vq->irq_lock); 949 } 950 951 static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq) 952 { 953 bool signal = false; 954 955 if (!vq->cb.trigger) 956 return false; 957 958 spin_lock_irq(&vq->irq_lock); 959 if (vq->ready && vq->cb.trigger) { 960 eventfd_signal(vq->cb.trigger, 1); 961 signal = true; 962 } 963 spin_unlock_irq(&vq->irq_lock); 964 965 return signal; 966 } 967 968 static int vduse_dev_queue_irq_work(struct vduse_dev *dev, 969 struct work_struct *irq_work, 970 int irq_effective_cpu) 971 { 972 int ret = -EINVAL; 973 974 down_read(&dev->rwsem); 975 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 976 goto unlock; 977 978 ret = 0; 979 if (irq_effective_cpu == IRQ_UNBOUND) 980 queue_work(vduse_irq_wq, irq_work); 981 else 982 queue_work_on(irq_effective_cpu, 983 vduse_irq_bound_wq, irq_work); 984 unlock: 985 up_read(&dev->rwsem); 986 987 return ret; 988 } 989 990 static int vduse_dev_dereg_umem(struct vduse_dev *dev, 991 u64 iova, u64 size) 992 { 993 int ret; 994 995 mutex_lock(&dev->mem_lock); 996 ret = -ENOENT; 997 if (!dev->umem) 998 goto unlock; 999 1000 ret = -EINVAL; 1001 if (!dev->domain) 1002 goto unlock; 1003 1004 if (dev->umem->iova != iova || size != dev->domain->bounce_size) 1005 goto unlock; 1006 1007 vduse_domain_remove_user_bounce_pages(dev->domain); 1008 unpin_user_pages_dirty_lock(dev->umem->pages, 1009 dev->umem->npages, true); 1010 atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm); 1011 mmdrop(dev->umem->mm); 1012 vfree(dev->umem->pages); 1013 kfree(dev->umem); 1014 dev->umem = NULL; 1015 ret = 0; 1016 unlock: 1017 mutex_unlock(&dev->mem_lock); 1018 return ret; 1019 } 1020 1021 static int vduse_dev_reg_umem(struct vduse_dev *dev, 1022 u64 iova, u64 uaddr, u64 size) 1023 { 1024 struct page **page_list = NULL; 1025 struct vduse_umem *umem = NULL; 1026 long pinned = 0; 1027 unsigned long npages, lock_limit; 1028 int ret; 1029 1030 if (!dev->domain || !dev->domain->bounce_map || 1031 size != dev->domain->bounce_size || 1032 iova != 0 || uaddr & ~PAGE_MASK) 1033 return -EINVAL; 1034 1035 mutex_lock(&dev->mem_lock); 1036 ret = -EEXIST; 1037 if (dev->umem) 1038 goto unlock; 1039 1040 ret = -ENOMEM; 1041 npages = size >> PAGE_SHIFT; 1042 page_list = __vmalloc(array_size(npages, sizeof(struct page *)), 1043 GFP_KERNEL_ACCOUNT); 1044 umem = kzalloc(sizeof(*umem), GFP_KERNEL); 1045 if (!page_list || !umem) 1046 goto unlock; 1047 1048 mmap_read_lock(current->mm); 1049 1050 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK)); 1051 if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit) 1052 goto out; 1053 1054 pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE, 1055 page_list); 1056 if (pinned != npages) { 1057 ret = pinned < 0 ? pinned : -ENOMEM; 1058 goto out; 1059 } 1060 1061 ret = vduse_domain_add_user_bounce_pages(dev->domain, 1062 page_list, pinned); 1063 if (ret) 1064 goto out; 1065 1066 atomic64_add(npages, ¤t->mm->pinned_vm); 1067 1068 umem->pages = page_list; 1069 umem->npages = pinned; 1070 umem->iova = iova; 1071 umem->mm = current->mm; 1072 mmgrab(current->mm); 1073 1074 dev->umem = umem; 1075 out: 1076 if (ret && pinned > 0) 1077 unpin_user_pages(page_list, pinned); 1078 1079 mmap_read_unlock(current->mm); 1080 unlock: 1081 if (ret) { 1082 vfree(page_list); 1083 kfree(umem); 1084 } 1085 mutex_unlock(&dev->mem_lock); 1086 return ret; 1087 } 1088 1089 static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq) 1090 { 1091 int curr_cpu = vq->irq_effective_cpu; 1092 1093 while (true) { 1094 curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity); 1095 if (cpu_online(curr_cpu)) 1096 break; 1097 1098 if (curr_cpu >= nr_cpu_ids) 1099 curr_cpu = IRQ_UNBOUND; 1100 } 1101 1102 vq->irq_effective_cpu = curr_cpu; 1103 } 1104 1105 static long vduse_dev_ioctl(struct file *file, unsigned int cmd, 1106 unsigned long arg) 1107 { 1108 struct vduse_dev *dev = file->private_data; 1109 void __user *argp = (void __user *)arg; 1110 int ret; 1111 1112 if (unlikely(dev->broken)) 1113 return -EPERM; 1114 1115 switch (cmd) { 1116 case VDUSE_IOTLB_GET_FD: { 1117 struct vduse_iotlb_entry entry; 1118 struct vhost_iotlb_map *map; 1119 struct vdpa_map_file *map_file; 1120 struct file *f = NULL; 1121 1122 ret = -EFAULT; 1123 if (copy_from_user(&entry, argp, sizeof(entry))) 1124 break; 1125 1126 ret = -EINVAL; 1127 if (entry.start > entry.last) 1128 break; 1129 1130 mutex_lock(&dev->domain_lock); 1131 if (!dev->domain) { 1132 mutex_unlock(&dev->domain_lock); 1133 break; 1134 } 1135 spin_lock(&dev->domain->iotlb_lock); 1136 map = vhost_iotlb_itree_first(dev->domain->iotlb, 1137 entry.start, entry.last); 1138 if (map) { 1139 map_file = (struct vdpa_map_file *)map->opaque; 1140 f = get_file(map_file->file); 1141 entry.offset = map_file->offset; 1142 entry.start = map->start; 1143 entry.last = map->last; 1144 entry.perm = map->perm; 1145 } 1146 spin_unlock(&dev->domain->iotlb_lock); 1147 mutex_unlock(&dev->domain_lock); 1148 ret = -EINVAL; 1149 if (!f) 1150 break; 1151 1152 ret = -EFAULT; 1153 if (copy_to_user(argp, &entry, sizeof(entry))) { 1154 fput(f); 1155 break; 1156 } 1157 ret = receive_fd(f, perm_to_file_flags(entry.perm)); 1158 fput(f); 1159 break; 1160 } 1161 case VDUSE_DEV_GET_FEATURES: 1162 /* 1163 * Just mirror what driver wrote here. 1164 * The driver is expected to check FEATURE_OK later. 1165 */ 1166 ret = put_user(dev->driver_features, (u64 __user *)argp); 1167 break; 1168 case VDUSE_DEV_SET_CONFIG: { 1169 struct vduse_config_data config; 1170 unsigned long size = offsetof(struct vduse_config_data, 1171 buffer); 1172 1173 ret = -EFAULT; 1174 if (copy_from_user(&config, argp, size)) 1175 break; 1176 1177 ret = -EINVAL; 1178 if (config.offset > dev->config_size || 1179 config.length == 0 || 1180 config.length > dev->config_size - config.offset) 1181 break; 1182 1183 ret = -EFAULT; 1184 if (copy_from_user(dev->config + config.offset, argp + size, 1185 config.length)) 1186 break; 1187 1188 ret = 0; 1189 break; 1190 } 1191 case VDUSE_DEV_INJECT_CONFIG_IRQ: 1192 ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND); 1193 break; 1194 case VDUSE_VQ_SETUP: { 1195 struct vduse_vq_config config; 1196 u32 index; 1197 1198 ret = -EFAULT; 1199 if (copy_from_user(&config, argp, sizeof(config))) 1200 break; 1201 1202 ret = -EINVAL; 1203 if (config.index >= dev->vq_num) 1204 break; 1205 1206 if (!is_mem_zero((const char *)config.reserved, 1207 sizeof(config.reserved))) 1208 break; 1209 1210 index = array_index_nospec(config.index, dev->vq_num); 1211 dev->vqs[index]->num_max = config.max_size; 1212 ret = 0; 1213 break; 1214 } 1215 case VDUSE_VQ_GET_INFO: { 1216 struct vduse_vq_info vq_info; 1217 struct vduse_virtqueue *vq; 1218 u32 index; 1219 1220 ret = -EFAULT; 1221 if (copy_from_user(&vq_info, argp, sizeof(vq_info))) 1222 break; 1223 1224 ret = -EINVAL; 1225 if (vq_info.index >= dev->vq_num) 1226 break; 1227 1228 index = array_index_nospec(vq_info.index, dev->vq_num); 1229 vq = dev->vqs[index]; 1230 vq_info.desc_addr = vq->desc_addr; 1231 vq_info.driver_addr = vq->driver_addr; 1232 vq_info.device_addr = vq->device_addr; 1233 vq_info.num = vq->num; 1234 1235 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 1236 vq_info.packed.last_avail_counter = 1237 vq->state.packed.last_avail_counter; 1238 vq_info.packed.last_avail_idx = 1239 vq->state.packed.last_avail_idx; 1240 vq_info.packed.last_used_counter = 1241 vq->state.packed.last_used_counter; 1242 vq_info.packed.last_used_idx = 1243 vq->state.packed.last_used_idx; 1244 } else 1245 vq_info.split.avail_index = 1246 vq->state.split.avail_index; 1247 1248 vq_info.ready = vq->ready; 1249 1250 ret = -EFAULT; 1251 if (copy_to_user(argp, &vq_info, sizeof(vq_info))) 1252 break; 1253 1254 ret = 0; 1255 break; 1256 } 1257 case VDUSE_VQ_SETUP_KICKFD: { 1258 struct vduse_vq_eventfd eventfd; 1259 1260 ret = -EFAULT; 1261 if (copy_from_user(&eventfd, argp, sizeof(eventfd))) 1262 break; 1263 1264 ret = vduse_kickfd_setup(dev, &eventfd); 1265 break; 1266 } 1267 case VDUSE_VQ_INJECT_IRQ: { 1268 u32 index; 1269 1270 ret = -EFAULT; 1271 if (get_user(index, (u32 __user *)argp)) 1272 break; 1273 1274 ret = -EINVAL; 1275 if (index >= dev->vq_num) 1276 break; 1277 1278 ret = 0; 1279 index = array_index_nospec(index, dev->vq_num); 1280 if (!vduse_vq_signal_irqfd(dev->vqs[index])) { 1281 vduse_vq_update_effective_cpu(dev->vqs[index]); 1282 ret = vduse_dev_queue_irq_work(dev, 1283 &dev->vqs[index]->inject, 1284 dev->vqs[index]->irq_effective_cpu); 1285 } 1286 break; 1287 } 1288 case VDUSE_IOTLB_REG_UMEM: { 1289 struct vduse_iova_umem umem; 1290 1291 ret = -EFAULT; 1292 if (copy_from_user(&umem, argp, sizeof(umem))) 1293 break; 1294 1295 ret = -EINVAL; 1296 if (!is_mem_zero((const char *)umem.reserved, 1297 sizeof(umem.reserved))) 1298 break; 1299 1300 mutex_lock(&dev->domain_lock); 1301 ret = vduse_dev_reg_umem(dev, umem.iova, 1302 umem.uaddr, umem.size); 1303 mutex_unlock(&dev->domain_lock); 1304 break; 1305 } 1306 case VDUSE_IOTLB_DEREG_UMEM: { 1307 struct vduse_iova_umem umem; 1308 1309 ret = -EFAULT; 1310 if (copy_from_user(&umem, argp, sizeof(umem))) 1311 break; 1312 1313 ret = -EINVAL; 1314 if (!is_mem_zero((const char *)umem.reserved, 1315 sizeof(umem.reserved))) 1316 break; 1317 mutex_lock(&dev->domain_lock); 1318 ret = vduse_dev_dereg_umem(dev, umem.iova, 1319 umem.size); 1320 mutex_unlock(&dev->domain_lock); 1321 break; 1322 } 1323 case VDUSE_IOTLB_GET_INFO: { 1324 struct vduse_iova_info info; 1325 struct vhost_iotlb_map *map; 1326 1327 ret = -EFAULT; 1328 if (copy_from_user(&info, argp, sizeof(info))) 1329 break; 1330 1331 ret = -EINVAL; 1332 if (info.start > info.last) 1333 break; 1334 1335 if (!is_mem_zero((const char *)info.reserved, 1336 sizeof(info.reserved))) 1337 break; 1338 1339 mutex_lock(&dev->domain_lock); 1340 if (!dev->domain) { 1341 mutex_unlock(&dev->domain_lock); 1342 break; 1343 } 1344 spin_lock(&dev->domain->iotlb_lock); 1345 map = vhost_iotlb_itree_first(dev->domain->iotlb, 1346 info.start, info.last); 1347 if (map) { 1348 info.start = map->start; 1349 info.last = map->last; 1350 info.capability = 0; 1351 if (dev->domain->bounce_map && map->start == 0 && 1352 map->last == dev->domain->bounce_size - 1) 1353 info.capability |= VDUSE_IOVA_CAP_UMEM; 1354 } 1355 spin_unlock(&dev->domain->iotlb_lock); 1356 mutex_unlock(&dev->domain_lock); 1357 if (!map) 1358 break; 1359 1360 ret = -EFAULT; 1361 if (copy_to_user(argp, &info, sizeof(info))) 1362 break; 1363 1364 ret = 0; 1365 break; 1366 } 1367 default: 1368 ret = -ENOIOCTLCMD; 1369 break; 1370 } 1371 1372 return ret; 1373 } 1374 1375 static int vduse_dev_release(struct inode *inode, struct file *file) 1376 { 1377 struct vduse_dev *dev = file->private_data; 1378 1379 mutex_lock(&dev->domain_lock); 1380 if (dev->domain) 1381 vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size); 1382 mutex_unlock(&dev->domain_lock); 1383 spin_lock(&dev->msg_lock); 1384 /* Make sure the inflight messages can processed after reconncection */ 1385 list_splice_init(&dev->recv_list, &dev->send_list); 1386 spin_unlock(&dev->msg_lock); 1387 dev->connected = false; 1388 1389 return 0; 1390 } 1391 1392 static struct vduse_dev *vduse_dev_get_from_minor(int minor) 1393 { 1394 struct vduse_dev *dev; 1395 1396 mutex_lock(&vduse_lock); 1397 dev = idr_find(&vduse_idr, minor); 1398 mutex_unlock(&vduse_lock); 1399 1400 return dev; 1401 } 1402 1403 static int vduse_dev_open(struct inode *inode, struct file *file) 1404 { 1405 int ret; 1406 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode)); 1407 1408 if (!dev) 1409 return -ENODEV; 1410 1411 ret = -EBUSY; 1412 mutex_lock(&dev->lock); 1413 if (dev->connected) 1414 goto unlock; 1415 1416 ret = 0; 1417 dev->connected = true; 1418 file->private_data = dev; 1419 unlock: 1420 mutex_unlock(&dev->lock); 1421 1422 return ret; 1423 } 1424 1425 static const struct file_operations vduse_dev_fops = { 1426 .owner = THIS_MODULE, 1427 .open = vduse_dev_open, 1428 .release = vduse_dev_release, 1429 .read_iter = vduse_dev_read_iter, 1430 .write_iter = vduse_dev_write_iter, 1431 .poll = vduse_dev_poll, 1432 .unlocked_ioctl = vduse_dev_ioctl, 1433 .compat_ioctl = compat_ptr_ioctl, 1434 .llseek = noop_llseek, 1435 }; 1436 1437 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf) 1438 { 1439 return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity)); 1440 } 1441 1442 static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq, 1443 const char *buf, size_t count) 1444 { 1445 cpumask_var_t new_value; 1446 int ret; 1447 1448 if (!zalloc_cpumask_var(&new_value, GFP_KERNEL)) 1449 return -ENOMEM; 1450 1451 ret = cpumask_parse(buf, new_value); 1452 if (ret) 1453 goto free_mask; 1454 1455 ret = -EINVAL; 1456 if (!cpumask_intersects(new_value, cpu_online_mask)) 1457 goto free_mask; 1458 1459 cpumask_copy(&vq->irq_affinity, new_value); 1460 ret = count; 1461 free_mask: 1462 free_cpumask_var(new_value); 1463 return ret; 1464 } 1465 1466 struct vq_sysfs_entry { 1467 struct attribute attr; 1468 ssize_t (*show)(struct vduse_virtqueue *vq, char *buf); 1469 ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf, 1470 size_t count); 1471 }; 1472 1473 static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity); 1474 1475 static struct attribute *vq_attrs[] = { 1476 &irq_cb_affinity_attr.attr, 1477 NULL, 1478 }; 1479 ATTRIBUTE_GROUPS(vq); 1480 1481 static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr, 1482 char *buf) 1483 { 1484 struct vduse_virtqueue *vq = container_of(kobj, 1485 struct vduse_virtqueue, kobj); 1486 struct vq_sysfs_entry *entry = container_of(attr, 1487 struct vq_sysfs_entry, attr); 1488 1489 if (!entry->show) 1490 return -EIO; 1491 1492 return entry->show(vq, buf); 1493 } 1494 1495 static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr, 1496 const char *buf, size_t count) 1497 { 1498 struct vduse_virtqueue *vq = container_of(kobj, 1499 struct vduse_virtqueue, kobj); 1500 struct vq_sysfs_entry *entry = container_of(attr, 1501 struct vq_sysfs_entry, attr); 1502 1503 if (!entry->store) 1504 return -EIO; 1505 1506 return entry->store(vq, buf, count); 1507 } 1508 1509 static const struct sysfs_ops vq_sysfs_ops = { 1510 .show = vq_attr_show, 1511 .store = vq_attr_store, 1512 }; 1513 1514 static void vq_release(struct kobject *kobj) 1515 { 1516 struct vduse_virtqueue *vq = container_of(kobj, 1517 struct vduse_virtqueue, kobj); 1518 kfree(vq); 1519 } 1520 1521 static const struct kobj_type vq_type = { 1522 .release = vq_release, 1523 .sysfs_ops = &vq_sysfs_ops, 1524 .default_groups = vq_groups, 1525 }; 1526 1527 static void vduse_dev_deinit_vqs(struct vduse_dev *dev) 1528 { 1529 int i; 1530 1531 if (!dev->vqs) 1532 return; 1533 1534 for (i = 0; i < dev->vq_num; i++) 1535 kobject_put(&dev->vqs[i]->kobj); 1536 kfree(dev->vqs); 1537 } 1538 1539 static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) 1540 { 1541 int ret, i; 1542 1543 dev->vq_align = vq_align; 1544 dev->vq_num = vq_num; 1545 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL); 1546 if (!dev->vqs) 1547 return -ENOMEM; 1548 1549 for (i = 0; i < vq_num; i++) { 1550 dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL); 1551 if (!dev->vqs[i]) { 1552 ret = -ENOMEM; 1553 goto err; 1554 } 1555 1556 dev->vqs[i]->index = i; 1557 dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND; 1558 INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject); 1559 INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work); 1560 spin_lock_init(&dev->vqs[i]->kick_lock); 1561 spin_lock_init(&dev->vqs[i]->irq_lock); 1562 cpumask_setall(&dev->vqs[i]->irq_affinity); 1563 1564 kobject_init(&dev->vqs[i]->kobj, &vq_type); 1565 ret = kobject_add(&dev->vqs[i]->kobj, 1566 &dev->dev->kobj, "vq%d", i); 1567 if (ret) { 1568 kfree(dev->vqs[i]); 1569 goto err; 1570 } 1571 } 1572 1573 return 0; 1574 err: 1575 while (i--) 1576 kobject_put(&dev->vqs[i]->kobj); 1577 kfree(dev->vqs); 1578 dev->vqs = NULL; 1579 return ret; 1580 } 1581 1582 static struct vduse_dev *vduse_dev_create(void) 1583 { 1584 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1585 1586 if (!dev) 1587 return NULL; 1588 1589 mutex_init(&dev->lock); 1590 mutex_init(&dev->mem_lock); 1591 mutex_init(&dev->domain_lock); 1592 spin_lock_init(&dev->msg_lock); 1593 INIT_LIST_HEAD(&dev->send_list); 1594 INIT_LIST_HEAD(&dev->recv_list); 1595 spin_lock_init(&dev->irq_lock); 1596 init_rwsem(&dev->rwsem); 1597 1598 INIT_WORK(&dev->inject, vduse_dev_irq_inject); 1599 init_waitqueue_head(&dev->waitq); 1600 1601 return dev; 1602 } 1603 1604 static void vduse_dev_destroy(struct vduse_dev *dev) 1605 { 1606 kfree(dev); 1607 } 1608 1609 static struct vduse_dev *vduse_find_dev(const char *name) 1610 { 1611 struct vduse_dev *dev; 1612 int id; 1613 1614 idr_for_each_entry(&vduse_idr, dev, id) 1615 if (!strcmp(dev->name, name)) 1616 return dev; 1617 1618 return NULL; 1619 } 1620 1621 static int vduse_destroy_dev(char *name) 1622 { 1623 struct vduse_dev *dev = vduse_find_dev(name); 1624 1625 if (!dev) 1626 return -EINVAL; 1627 1628 mutex_lock(&dev->lock); 1629 if (dev->vdev || dev->connected) { 1630 mutex_unlock(&dev->lock); 1631 return -EBUSY; 1632 } 1633 dev->connected = true; 1634 mutex_unlock(&dev->lock); 1635 1636 vduse_dev_reset(dev); 1637 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 1638 idr_remove(&vduse_idr, dev->minor); 1639 kvfree(dev->config); 1640 vduse_dev_deinit_vqs(dev); 1641 if (dev->domain) 1642 vduse_domain_destroy(dev->domain); 1643 kfree(dev->name); 1644 vduse_dev_destroy(dev); 1645 module_put(THIS_MODULE); 1646 1647 return 0; 1648 } 1649 1650 static bool device_is_allowed(u32 device_id) 1651 { 1652 int i; 1653 1654 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++) 1655 if (allowed_device_id[i] == device_id) 1656 return true; 1657 1658 return false; 1659 } 1660 1661 static bool features_is_valid(u64 features) 1662 { 1663 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) 1664 return false; 1665 1666 /* Now we only support read-only configuration space */ 1667 if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE)) 1668 return false; 1669 1670 return true; 1671 } 1672 1673 static bool vduse_validate_config(struct vduse_dev_config *config) 1674 { 1675 if (!is_mem_zero((const char *)config->reserved, 1676 sizeof(config->reserved))) 1677 return false; 1678 1679 if (config->vq_align > PAGE_SIZE) 1680 return false; 1681 1682 if (config->config_size > PAGE_SIZE) 1683 return false; 1684 1685 if (config->vq_num > 0xffff) 1686 return false; 1687 1688 if (!device_is_allowed(config->device_id)) 1689 return false; 1690 1691 if (!features_is_valid(config->features)) 1692 return false; 1693 1694 return true; 1695 } 1696 1697 static ssize_t msg_timeout_show(struct device *device, 1698 struct device_attribute *attr, char *buf) 1699 { 1700 struct vduse_dev *dev = dev_get_drvdata(device); 1701 1702 return sysfs_emit(buf, "%u\n", dev->msg_timeout); 1703 } 1704 1705 static ssize_t msg_timeout_store(struct device *device, 1706 struct device_attribute *attr, 1707 const char *buf, size_t count) 1708 { 1709 struct vduse_dev *dev = dev_get_drvdata(device); 1710 int ret; 1711 1712 ret = kstrtouint(buf, 10, &dev->msg_timeout); 1713 if (ret < 0) 1714 return ret; 1715 1716 return count; 1717 } 1718 1719 static DEVICE_ATTR_RW(msg_timeout); 1720 1721 static ssize_t bounce_size_show(struct device *device, 1722 struct device_attribute *attr, char *buf) 1723 { 1724 struct vduse_dev *dev = dev_get_drvdata(device); 1725 1726 return sysfs_emit(buf, "%u\n", dev->bounce_size); 1727 } 1728 1729 static ssize_t bounce_size_store(struct device *device, 1730 struct device_attribute *attr, 1731 const char *buf, size_t count) 1732 { 1733 struct vduse_dev *dev = dev_get_drvdata(device); 1734 unsigned int bounce_size; 1735 int ret; 1736 1737 ret = -EPERM; 1738 mutex_lock(&dev->domain_lock); 1739 if (dev->domain) 1740 goto unlock; 1741 1742 ret = kstrtouint(buf, 10, &bounce_size); 1743 if (ret < 0) 1744 goto unlock; 1745 1746 ret = -EINVAL; 1747 if (bounce_size > VDUSE_MAX_BOUNCE_SIZE || 1748 bounce_size < VDUSE_MIN_BOUNCE_SIZE) 1749 goto unlock; 1750 1751 dev->bounce_size = bounce_size & PAGE_MASK; 1752 ret = count; 1753 unlock: 1754 mutex_unlock(&dev->domain_lock); 1755 return ret; 1756 } 1757 1758 static DEVICE_ATTR_RW(bounce_size); 1759 1760 static struct attribute *vduse_dev_attrs[] = { 1761 &dev_attr_msg_timeout.attr, 1762 &dev_attr_bounce_size.attr, 1763 NULL 1764 }; 1765 1766 ATTRIBUTE_GROUPS(vduse_dev); 1767 1768 static int vduse_create_dev(struct vduse_dev_config *config, 1769 void *config_buf, u64 api_version) 1770 { 1771 int ret; 1772 struct vduse_dev *dev; 1773 1774 ret = -EEXIST; 1775 if (vduse_find_dev(config->name)) 1776 goto err; 1777 1778 ret = -ENOMEM; 1779 dev = vduse_dev_create(); 1780 if (!dev) 1781 goto err; 1782 1783 dev->api_version = api_version; 1784 dev->device_features = config->features; 1785 dev->device_id = config->device_id; 1786 dev->vendor_id = config->vendor_id; 1787 dev->name = kstrdup(config->name, GFP_KERNEL); 1788 if (!dev->name) 1789 goto err_str; 1790 1791 dev->bounce_size = VDUSE_BOUNCE_SIZE; 1792 dev->config = config_buf; 1793 dev->config_size = config->config_size; 1794 1795 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL); 1796 if (ret < 0) 1797 goto err_idr; 1798 1799 dev->minor = ret; 1800 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT; 1801 dev->dev = device_create_with_groups(vduse_class, NULL, 1802 MKDEV(MAJOR(vduse_major), dev->minor), 1803 dev, vduse_dev_groups, "%s", config->name); 1804 if (IS_ERR(dev->dev)) { 1805 ret = PTR_ERR(dev->dev); 1806 goto err_dev; 1807 } 1808 1809 ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num); 1810 if (ret) 1811 goto err_vqs; 1812 1813 __module_get(THIS_MODULE); 1814 1815 return 0; 1816 err_vqs: 1817 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 1818 err_dev: 1819 idr_remove(&vduse_idr, dev->minor); 1820 err_idr: 1821 kfree(dev->name); 1822 err_str: 1823 vduse_dev_destroy(dev); 1824 err: 1825 return ret; 1826 } 1827 1828 static long vduse_ioctl(struct file *file, unsigned int cmd, 1829 unsigned long arg) 1830 { 1831 int ret; 1832 void __user *argp = (void __user *)arg; 1833 struct vduse_control *control = file->private_data; 1834 1835 mutex_lock(&vduse_lock); 1836 switch (cmd) { 1837 case VDUSE_GET_API_VERSION: 1838 ret = put_user(control->api_version, (u64 __user *)argp); 1839 break; 1840 case VDUSE_SET_API_VERSION: { 1841 u64 api_version; 1842 1843 ret = -EFAULT; 1844 if (get_user(api_version, (u64 __user *)argp)) 1845 break; 1846 1847 ret = -EINVAL; 1848 if (api_version > VDUSE_API_VERSION) 1849 break; 1850 1851 ret = 0; 1852 control->api_version = api_version; 1853 break; 1854 } 1855 case VDUSE_CREATE_DEV: { 1856 struct vduse_dev_config config; 1857 unsigned long size = offsetof(struct vduse_dev_config, config); 1858 void *buf; 1859 1860 ret = -EFAULT; 1861 if (copy_from_user(&config, argp, size)) 1862 break; 1863 1864 ret = -EINVAL; 1865 if (vduse_validate_config(&config) == false) 1866 break; 1867 1868 buf = vmemdup_user(argp + size, config.config_size); 1869 if (IS_ERR(buf)) { 1870 ret = PTR_ERR(buf); 1871 break; 1872 } 1873 config.name[VDUSE_NAME_MAX - 1] = '\0'; 1874 ret = vduse_create_dev(&config, buf, control->api_version); 1875 if (ret) 1876 kvfree(buf); 1877 break; 1878 } 1879 case VDUSE_DESTROY_DEV: { 1880 char name[VDUSE_NAME_MAX]; 1881 1882 ret = -EFAULT; 1883 if (copy_from_user(name, argp, VDUSE_NAME_MAX)) 1884 break; 1885 1886 name[VDUSE_NAME_MAX - 1] = '\0'; 1887 ret = vduse_destroy_dev(name); 1888 break; 1889 } 1890 default: 1891 ret = -EINVAL; 1892 break; 1893 } 1894 mutex_unlock(&vduse_lock); 1895 1896 return ret; 1897 } 1898 1899 static int vduse_release(struct inode *inode, struct file *file) 1900 { 1901 struct vduse_control *control = file->private_data; 1902 1903 kfree(control); 1904 return 0; 1905 } 1906 1907 static int vduse_open(struct inode *inode, struct file *file) 1908 { 1909 struct vduse_control *control; 1910 1911 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL); 1912 if (!control) 1913 return -ENOMEM; 1914 1915 control->api_version = VDUSE_API_VERSION; 1916 file->private_data = control; 1917 1918 return 0; 1919 } 1920 1921 static const struct file_operations vduse_ctrl_fops = { 1922 .owner = THIS_MODULE, 1923 .open = vduse_open, 1924 .release = vduse_release, 1925 .unlocked_ioctl = vduse_ioctl, 1926 .compat_ioctl = compat_ptr_ioctl, 1927 .llseek = noop_llseek, 1928 }; 1929 1930 static char *vduse_devnode(const struct device *dev, umode_t *mode) 1931 { 1932 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev)); 1933 } 1934 1935 struct vduse_mgmt_dev { 1936 struct vdpa_mgmt_dev mgmt_dev; 1937 struct device dev; 1938 }; 1939 1940 static struct vduse_mgmt_dev *vduse_mgmt; 1941 1942 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) 1943 { 1944 struct vduse_vdpa *vdev; 1945 int ret; 1946 1947 if (dev->vdev) 1948 return -EEXIST; 1949 1950 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev, 1951 &vduse_vdpa_config_ops, 1, 1, name, true); 1952 if (IS_ERR(vdev)) 1953 return PTR_ERR(vdev); 1954 1955 dev->vdev = vdev; 1956 vdev->dev = dev; 1957 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask; 1958 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64)); 1959 if (ret) { 1960 put_device(&vdev->vdpa.dev); 1961 return ret; 1962 } 1963 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops); 1964 vdev->vdpa.dma_dev = &vdev->vdpa.dev; 1965 vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev; 1966 1967 return 0; 1968 } 1969 1970 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 1971 const struct vdpa_dev_set_config *config) 1972 { 1973 struct vduse_dev *dev; 1974 int ret; 1975 1976 mutex_lock(&vduse_lock); 1977 dev = vduse_find_dev(name); 1978 if (!dev || !vduse_dev_is_ready(dev)) { 1979 mutex_unlock(&vduse_lock); 1980 return -EINVAL; 1981 } 1982 ret = vduse_dev_init_vdpa(dev, name); 1983 mutex_unlock(&vduse_lock); 1984 if (ret) 1985 return ret; 1986 1987 mutex_lock(&dev->domain_lock); 1988 if (!dev->domain) 1989 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1, 1990 dev->bounce_size); 1991 mutex_unlock(&dev->domain_lock); 1992 if (!dev->domain) { 1993 put_device(&dev->vdev->vdpa.dev); 1994 return -ENOMEM; 1995 } 1996 1997 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); 1998 if (ret) { 1999 put_device(&dev->vdev->vdpa.dev); 2000 mutex_lock(&dev->domain_lock); 2001 vduse_domain_destroy(dev->domain); 2002 dev->domain = NULL; 2003 mutex_unlock(&dev->domain_lock); 2004 return ret; 2005 } 2006 2007 return 0; 2008 } 2009 2010 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) 2011 { 2012 _vdpa_unregister_device(dev); 2013 } 2014 2015 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = { 2016 .dev_add = vdpa_dev_add, 2017 .dev_del = vdpa_dev_del, 2018 }; 2019 2020 static struct virtio_device_id id_table[] = { 2021 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 2022 { 0 }, 2023 }; 2024 2025 static void vduse_mgmtdev_release(struct device *dev) 2026 { 2027 struct vduse_mgmt_dev *mgmt_dev; 2028 2029 mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev); 2030 kfree(mgmt_dev); 2031 } 2032 2033 static int vduse_mgmtdev_init(void) 2034 { 2035 int ret; 2036 2037 vduse_mgmt = kzalloc(sizeof(*vduse_mgmt), GFP_KERNEL); 2038 if (!vduse_mgmt) 2039 return -ENOMEM; 2040 2041 ret = dev_set_name(&vduse_mgmt->dev, "vduse"); 2042 if (ret) { 2043 kfree(vduse_mgmt); 2044 return ret; 2045 } 2046 2047 vduse_mgmt->dev.release = vduse_mgmtdev_release; 2048 2049 ret = device_register(&vduse_mgmt->dev); 2050 if (ret) 2051 goto dev_reg_err; 2052 2053 vduse_mgmt->mgmt_dev.id_table = id_table; 2054 vduse_mgmt->mgmt_dev.ops = &vdpa_dev_mgmtdev_ops; 2055 vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev; 2056 ret = vdpa_mgmtdev_register(&vduse_mgmt->mgmt_dev); 2057 if (ret) 2058 device_unregister(&vduse_mgmt->dev); 2059 2060 return ret; 2061 2062 dev_reg_err: 2063 put_device(&vduse_mgmt->dev); 2064 return ret; 2065 } 2066 2067 static void vduse_mgmtdev_exit(void) 2068 { 2069 vdpa_mgmtdev_unregister(&vduse_mgmt->mgmt_dev); 2070 device_unregister(&vduse_mgmt->dev); 2071 } 2072 2073 static int vduse_init(void) 2074 { 2075 int ret; 2076 struct device *dev; 2077 2078 vduse_class = class_create("vduse"); 2079 if (IS_ERR(vduse_class)) 2080 return PTR_ERR(vduse_class); 2081 2082 vduse_class->devnode = vduse_devnode; 2083 2084 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse"); 2085 if (ret) 2086 goto err_chardev_region; 2087 2088 /* /dev/vduse/control */ 2089 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops); 2090 vduse_ctrl_cdev.owner = THIS_MODULE; 2091 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1); 2092 if (ret) 2093 goto err_ctrl_cdev; 2094 2095 dev = device_create(vduse_class, NULL, vduse_major, NULL, "control"); 2096 if (IS_ERR(dev)) { 2097 ret = PTR_ERR(dev); 2098 goto err_device; 2099 } 2100 2101 /* /dev/vduse/$DEVICE */ 2102 cdev_init(&vduse_cdev, &vduse_dev_fops); 2103 vduse_cdev.owner = THIS_MODULE; 2104 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1), 2105 VDUSE_DEV_MAX - 1); 2106 if (ret) 2107 goto err_cdev; 2108 2109 ret = -ENOMEM; 2110 vduse_irq_wq = alloc_workqueue("vduse-irq", 2111 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0); 2112 if (!vduse_irq_wq) 2113 goto err_wq; 2114 2115 vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0); 2116 if (!vduse_irq_bound_wq) 2117 goto err_bound_wq; 2118 2119 ret = vduse_domain_init(); 2120 if (ret) 2121 goto err_domain; 2122 2123 ret = vduse_mgmtdev_init(); 2124 if (ret) 2125 goto err_mgmtdev; 2126 2127 return 0; 2128 err_mgmtdev: 2129 vduse_domain_exit(); 2130 err_domain: 2131 destroy_workqueue(vduse_irq_bound_wq); 2132 err_bound_wq: 2133 destroy_workqueue(vduse_irq_wq); 2134 err_wq: 2135 cdev_del(&vduse_cdev); 2136 err_cdev: 2137 device_destroy(vduse_class, vduse_major); 2138 err_device: 2139 cdev_del(&vduse_ctrl_cdev); 2140 err_ctrl_cdev: 2141 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 2142 err_chardev_region: 2143 class_destroy(vduse_class); 2144 return ret; 2145 } 2146 module_init(vduse_init); 2147 2148 static void vduse_exit(void) 2149 { 2150 vduse_mgmtdev_exit(); 2151 vduse_domain_exit(); 2152 destroy_workqueue(vduse_irq_bound_wq); 2153 destroy_workqueue(vduse_irq_wq); 2154 cdev_del(&vduse_cdev); 2155 device_destroy(vduse_class, vduse_major); 2156 cdev_del(&vduse_ctrl_cdev); 2157 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 2158 class_destroy(vduse_class); 2159 } 2160 module_exit(vduse_exit); 2161 2162 MODULE_LICENSE(DRV_LICENSE); 2163 MODULE_AUTHOR(DRV_AUTHOR); 2164 MODULE_DESCRIPTION(DRV_DESC); 2165