1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDUSE: vDPA Device in Userspace 4 * 5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved. 6 * 7 * Author: Xie Yongji <xieyongji@bytedance.com> 8 * 9 */ 10 11 #include "linux/virtio_net.h" 12 #include <linux/init.h> 13 #include <linux/module.h> 14 #include <linux/cdev.h> 15 #include <linux/device.h> 16 #include <linux/eventfd.h> 17 #include <linux/slab.h> 18 #include <linux/wait.h> 19 #include <linux/dma-map-ops.h> 20 #include <linux/poll.h> 21 #include <linux/file.h> 22 #include <linux/uio.h> 23 #include <linux/vdpa.h> 24 #include <linux/nospec.h> 25 #include <linux/vmalloc.h> 26 #include <linux/sched/mm.h> 27 #include <uapi/linux/vduse.h> 28 #include <uapi/linux/vdpa.h> 29 #include <uapi/linux/virtio_config.h> 30 #include <uapi/linux/virtio_ids.h> 31 #include <uapi/linux/virtio_blk.h> 32 #include <uapi/linux/virtio_ring.h> 33 #include <linux/mod_devicetable.h> 34 35 #include "iova_domain.h" 36 37 #define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>" 38 #define DRV_DESC "vDPA Device in Userspace" 39 #define DRV_LICENSE "GPL v2" 40 41 #define VDUSE_DEV_MAX (1U << MINORBITS) 42 #define VDUSE_MAX_BOUNCE_SIZE (1024 * 1024 * 1024) 43 #define VDUSE_MIN_BOUNCE_SIZE (1024 * 1024) 44 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024) 45 /* 128 MB reserved for virtqueue creation */ 46 #define VDUSE_IOVA_SIZE (VDUSE_MAX_BOUNCE_SIZE + 128 * 1024 * 1024) 47 #define VDUSE_MSG_DEFAULT_TIMEOUT 30 48 49 #define IRQ_UNBOUND -1 50 51 struct vduse_virtqueue { 52 u16 index; 53 u16 num_max; 54 u32 num; 55 u64 desc_addr; 56 u64 driver_addr; 57 u64 device_addr; 58 struct vdpa_vq_state state; 59 bool ready; 60 bool kicked; 61 spinlock_t kick_lock; 62 spinlock_t irq_lock; 63 struct eventfd_ctx *kickfd; 64 struct vdpa_callback cb; 65 struct work_struct inject; 66 struct work_struct kick; 67 int irq_effective_cpu; 68 struct cpumask irq_affinity; 69 struct kobject kobj; 70 }; 71 72 struct vduse_dev; 73 74 struct vduse_vdpa { 75 struct vdpa_device vdpa; 76 struct vduse_dev *dev; 77 }; 78 79 struct vduse_umem { 80 unsigned long iova; 81 unsigned long npages; 82 struct page **pages; 83 struct mm_struct *mm; 84 }; 85 86 struct vduse_dev { 87 struct vduse_vdpa *vdev; 88 struct device *dev; 89 struct vduse_virtqueue **vqs; 90 struct vduse_iova_domain *domain; 91 char *name; 92 struct mutex lock; 93 spinlock_t msg_lock; 94 u64 msg_unique; 95 u32 msg_timeout; 96 wait_queue_head_t waitq; 97 struct list_head send_list; 98 struct list_head recv_list; 99 struct vdpa_callback config_cb; 100 struct work_struct inject; 101 spinlock_t irq_lock; 102 struct rw_semaphore rwsem; 103 int minor; 104 bool broken; 105 bool connected; 106 u64 api_version; 107 u64 device_features; 108 u64 driver_features; 109 u32 device_id; 110 u32 vendor_id; 111 u32 generation; 112 u32 config_size; 113 void *config; 114 u8 status; 115 u32 vq_num; 116 u32 vq_align; 117 struct vduse_umem *umem; 118 struct mutex mem_lock; 119 unsigned int bounce_size; 120 struct mutex domain_lock; 121 }; 122 123 struct vduse_dev_msg { 124 struct vduse_dev_request req; 125 struct vduse_dev_response resp; 126 struct list_head list; 127 wait_queue_head_t waitq; 128 bool completed; 129 }; 130 131 struct vduse_control { 132 u64 api_version; 133 }; 134 135 static DEFINE_MUTEX(vduse_lock); 136 static DEFINE_IDR(vduse_idr); 137 138 static dev_t vduse_major; 139 static struct class *vduse_class; 140 static struct cdev vduse_ctrl_cdev; 141 static struct cdev vduse_cdev; 142 static struct workqueue_struct *vduse_irq_wq; 143 static struct workqueue_struct *vduse_irq_bound_wq; 144 145 static u32 allowed_device_id[] = { 146 VIRTIO_ID_BLOCK, 147 }; 148 149 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa) 150 { 151 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa); 152 153 return vdev->dev; 154 } 155 156 static inline struct vduse_dev *dev_to_vduse(struct device *dev) 157 { 158 struct vdpa_device *vdpa = dev_to_vdpa(dev); 159 160 return vdpa_to_vduse(vdpa); 161 } 162 163 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head, 164 uint32_t request_id) 165 { 166 struct vduse_dev_msg *msg; 167 168 list_for_each_entry(msg, head, list) { 169 if (msg->req.request_id == request_id) { 170 list_del(&msg->list); 171 return msg; 172 } 173 } 174 175 return NULL; 176 } 177 178 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head) 179 { 180 struct vduse_dev_msg *msg = NULL; 181 182 if (!list_empty(head)) { 183 msg = list_first_entry(head, struct vduse_dev_msg, list); 184 list_del(&msg->list); 185 } 186 187 return msg; 188 } 189 190 static void vduse_enqueue_msg(struct list_head *head, 191 struct vduse_dev_msg *msg) 192 { 193 list_add_tail(&msg->list, head); 194 } 195 196 static void vduse_dev_broken(struct vduse_dev *dev) 197 { 198 struct vduse_dev_msg *msg, *tmp; 199 200 if (unlikely(dev->broken)) 201 return; 202 203 list_splice_init(&dev->recv_list, &dev->send_list); 204 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) { 205 list_del(&msg->list); 206 msg->completed = 1; 207 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 208 wake_up(&msg->waitq); 209 } 210 dev->broken = true; 211 wake_up(&dev->waitq); 212 } 213 214 static int vduse_dev_msg_sync(struct vduse_dev *dev, 215 struct vduse_dev_msg *msg) 216 { 217 int ret; 218 219 if (unlikely(dev->broken)) 220 return -EIO; 221 222 init_waitqueue_head(&msg->waitq); 223 spin_lock(&dev->msg_lock); 224 if (unlikely(dev->broken)) { 225 spin_unlock(&dev->msg_lock); 226 return -EIO; 227 } 228 msg->req.request_id = dev->msg_unique++; 229 vduse_enqueue_msg(&dev->send_list, msg); 230 wake_up(&dev->waitq); 231 spin_unlock(&dev->msg_lock); 232 if (dev->msg_timeout) 233 ret = wait_event_killable_timeout(msg->waitq, msg->completed, 234 (long)dev->msg_timeout * HZ); 235 else 236 ret = wait_event_killable(msg->waitq, msg->completed); 237 238 spin_lock(&dev->msg_lock); 239 if (!msg->completed) { 240 list_del(&msg->list); 241 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 242 /* Mark the device as malfunction when there is a timeout */ 243 if (!ret) 244 vduse_dev_broken(dev); 245 } 246 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO; 247 spin_unlock(&dev->msg_lock); 248 249 return ret; 250 } 251 252 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev, 253 struct vduse_virtqueue *vq, 254 struct vdpa_vq_state_packed *packed) 255 { 256 struct vduse_dev_msg msg = { 0 }; 257 int ret; 258 259 msg.req.type = VDUSE_GET_VQ_STATE; 260 msg.req.vq_state.index = vq->index; 261 262 ret = vduse_dev_msg_sync(dev, &msg); 263 if (ret) 264 return ret; 265 266 packed->last_avail_counter = 267 msg.resp.vq_state.packed.last_avail_counter & 0x0001; 268 packed->last_avail_idx = 269 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF; 270 packed->last_used_counter = 271 msg.resp.vq_state.packed.last_used_counter & 0x0001; 272 packed->last_used_idx = 273 msg.resp.vq_state.packed.last_used_idx & 0x7FFF; 274 275 return 0; 276 } 277 278 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev, 279 struct vduse_virtqueue *vq, 280 struct vdpa_vq_state_split *split) 281 { 282 struct vduse_dev_msg msg = { 0 }; 283 int ret; 284 285 msg.req.type = VDUSE_GET_VQ_STATE; 286 msg.req.vq_state.index = vq->index; 287 288 ret = vduse_dev_msg_sync(dev, &msg); 289 if (ret) 290 return ret; 291 292 split->avail_index = msg.resp.vq_state.split.avail_index; 293 294 return 0; 295 } 296 297 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status) 298 { 299 struct vduse_dev_msg msg = { 0 }; 300 301 msg.req.type = VDUSE_SET_STATUS; 302 msg.req.s.status = status; 303 304 return vduse_dev_msg_sync(dev, &msg); 305 } 306 307 static int vduse_dev_update_iotlb(struct vduse_dev *dev, 308 u64 start, u64 last) 309 { 310 struct vduse_dev_msg msg = { 0 }; 311 312 if (last < start) 313 return -EINVAL; 314 315 msg.req.type = VDUSE_UPDATE_IOTLB; 316 msg.req.iova.start = start; 317 msg.req.iova.last = last; 318 319 return vduse_dev_msg_sync(dev, &msg); 320 } 321 322 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) 323 { 324 struct file *file = iocb->ki_filp; 325 struct vduse_dev *dev = file->private_data; 326 struct vduse_dev_msg *msg; 327 int size = sizeof(struct vduse_dev_request); 328 ssize_t ret; 329 330 if (iov_iter_count(to) < size) 331 return -EINVAL; 332 333 spin_lock(&dev->msg_lock); 334 while (1) { 335 msg = vduse_dequeue_msg(&dev->send_list); 336 if (msg) 337 break; 338 339 ret = -EAGAIN; 340 if (file->f_flags & O_NONBLOCK) 341 goto unlock; 342 343 spin_unlock(&dev->msg_lock); 344 ret = wait_event_interruptible_exclusive(dev->waitq, 345 !list_empty(&dev->send_list)); 346 if (ret) 347 return ret; 348 349 spin_lock(&dev->msg_lock); 350 } 351 spin_unlock(&dev->msg_lock); 352 ret = copy_to_iter(&msg->req, size, to); 353 spin_lock(&dev->msg_lock); 354 if (ret != size) { 355 ret = -EFAULT; 356 vduse_enqueue_msg(&dev->send_list, msg); 357 goto unlock; 358 } 359 vduse_enqueue_msg(&dev->recv_list, msg); 360 unlock: 361 spin_unlock(&dev->msg_lock); 362 363 return ret; 364 } 365 366 static bool is_mem_zero(const char *ptr, int size) 367 { 368 int i; 369 370 for (i = 0; i < size; i++) { 371 if (ptr[i]) 372 return false; 373 } 374 return true; 375 } 376 377 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from) 378 { 379 struct file *file = iocb->ki_filp; 380 struct vduse_dev *dev = file->private_data; 381 struct vduse_dev_response resp; 382 struct vduse_dev_msg *msg; 383 size_t ret; 384 385 ret = copy_from_iter(&resp, sizeof(resp), from); 386 if (ret != sizeof(resp)) 387 return -EINVAL; 388 389 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved))) 390 return -EINVAL; 391 392 spin_lock(&dev->msg_lock); 393 msg = vduse_find_msg(&dev->recv_list, resp.request_id); 394 if (!msg) { 395 ret = -ENOENT; 396 goto unlock; 397 } 398 399 memcpy(&msg->resp, &resp, sizeof(resp)); 400 msg->completed = 1; 401 wake_up(&msg->waitq); 402 unlock: 403 spin_unlock(&dev->msg_lock); 404 405 return ret; 406 } 407 408 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait) 409 { 410 struct vduse_dev *dev = file->private_data; 411 __poll_t mask = 0; 412 413 poll_wait(file, &dev->waitq, wait); 414 415 spin_lock(&dev->msg_lock); 416 417 if (unlikely(dev->broken)) 418 mask |= EPOLLERR; 419 if (!list_empty(&dev->send_list)) 420 mask |= EPOLLIN | EPOLLRDNORM; 421 if (!list_empty(&dev->recv_list)) 422 mask |= EPOLLOUT | EPOLLWRNORM; 423 424 spin_unlock(&dev->msg_lock); 425 426 return mask; 427 } 428 429 static void vduse_dev_reset(struct vduse_dev *dev) 430 { 431 int i; 432 struct vduse_iova_domain *domain = dev->domain; 433 434 /* The coherent mappings are handled in vduse_dev_free_coherent() */ 435 if (domain && domain->bounce_map) 436 vduse_domain_reset_bounce_map(domain); 437 438 down_write(&dev->rwsem); 439 440 dev->status = 0; 441 dev->driver_features = 0; 442 dev->generation++; 443 spin_lock(&dev->irq_lock); 444 dev->config_cb.callback = NULL; 445 dev->config_cb.private = NULL; 446 spin_unlock(&dev->irq_lock); 447 flush_work(&dev->inject); 448 449 for (i = 0; i < dev->vq_num; i++) { 450 struct vduse_virtqueue *vq = dev->vqs[i]; 451 452 vq->ready = false; 453 vq->desc_addr = 0; 454 vq->driver_addr = 0; 455 vq->device_addr = 0; 456 vq->num = 0; 457 memset(&vq->state, 0, sizeof(vq->state)); 458 459 spin_lock(&vq->kick_lock); 460 vq->kicked = false; 461 if (vq->kickfd) 462 eventfd_ctx_put(vq->kickfd); 463 vq->kickfd = NULL; 464 spin_unlock(&vq->kick_lock); 465 466 spin_lock(&vq->irq_lock); 467 vq->cb.callback = NULL; 468 vq->cb.private = NULL; 469 vq->cb.trigger = NULL; 470 spin_unlock(&vq->irq_lock); 471 flush_work(&vq->inject); 472 flush_work(&vq->kick); 473 } 474 475 up_write(&dev->rwsem); 476 } 477 478 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx, 479 u64 desc_area, u64 driver_area, 480 u64 device_area) 481 { 482 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 483 struct vduse_virtqueue *vq = dev->vqs[idx]; 484 485 vq->desc_addr = desc_area; 486 vq->driver_addr = driver_area; 487 vq->device_addr = device_area; 488 489 return 0; 490 } 491 492 static void vduse_vq_kick(struct vduse_virtqueue *vq) 493 { 494 spin_lock(&vq->kick_lock); 495 if (!vq->ready) 496 goto unlock; 497 498 if (vq->kickfd) 499 eventfd_signal(vq->kickfd, 1); 500 else 501 vq->kicked = true; 502 unlock: 503 spin_unlock(&vq->kick_lock); 504 } 505 506 static void vduse_vq_kick_work(struct work_struct *work) 507 { 508 struct vduse_virtqueue *vq = container_of(work, 509 struct vduse_virtqueue, kick); 510 511 vduse_vq_kick(vq); 512 } 513 514 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx) 515 { 516 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 517 struct vduse_virtqueue *vq = dev->vqs[idx]; 518 519 if (!eventfd_signal_allowed()) { 520 schedule_work(&vq->kick); 521 return; 522 } 523 vduse_vq_kick(vq); 524 } 525 526 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 527 struct vdpa_callback *cb) 528 { 529 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 530 struct vduse_virtqueue *vq = dev->vqs[idx]; 531 532 spin_lock(&vq->irq_lock); 533 vq->cb.callback = cb->callback; 534 vq->cb.private = cb->private; 535 vq->cb.trigger = cb->trigger; 536 spin_unlock(&vq->irq_lock); 537 } 538 539 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 540 { 541 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 542 struct vduse_virtqueue *vq = dev->vqs[idx]; 543 544 vq->num = num; 545 } 546 547 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa, 548 u16 idx, bool ready) 549 { 550 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 551 struct vduse_virtqueue *vq = dev->vqs[idx]; 552 553 vq->ready = ready; 554 } 555 556 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 557 { 558 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 559 struct vduse_virtqueue *vq = dev->vqs[idx]; 560 561 return vq->ready; 562 } 563 564 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx, 565 const struct vdpa_vq_state *state) 566 { 567 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 568 struct vduse_virtqueue *vq = dev->vqs[idx]; 569 570 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 571 vq->state.packed.last_avail_counter = 572 state->packed.last_avail_counter; 573 vq->state.packed.last_avail_idx = state->packed.last_avail_idx; 574 vq->state.packed.last_used_counter = 575 state->packed.last_used_counter; 576 vq->state.packed.last_used_idx = state->packed.last_used_idx; 577 } else 578 vq->state.split.avail_index = state->split.avail_index; 579 580 return 0; 581 } 582 583 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx, 584 struct vdpa_vq_state *state) 585 { 586 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 587 struct vduse_virtqueue *vq = dev->vqs[idx]; 588 589 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) 590 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed); 591 592 return vduse_dev_get_vq_state_split(dev, vq, &state->split); 593 } 594 595 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa) 596 { 597 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 598 599 return dev->vq_align; 600 } 601 602 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa) 603 { 604 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 605 606 return dev->device_features; 607 } 608 609 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features) 610 { 611 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 612 613 dev->driver_features = features; 614 return 0; 615 } 616 617 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa) 618 { 619 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 620 621 return dev->driver_features; 622 } 623 624 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa, 625 struct vdpa_callback *cb) 626 { 627 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 628 629 spin_lock(&dev->irq_lock); 630 dev->config_cb.callback = cb->callback; 631 dev->config_cb.private = cb->private; 632 spin_unlock(&dev->irq_lock); 633 } 634 635 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa) 636 { 637 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 638 u16 num_max = 0; 639 int i; 640 641 for (i = 0; i < dev->vq_num; i++) 642 if (num_max < dev->vqs[i]->num_max) 643 num_max = dev->vqs[i]->num_max; 644 645 return num_max; 646 } 647 648 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa) 649 { 650 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 651 652 return dev->device_id; 653 } 654 655 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa) 656 { 657 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 658 659 return dev->vendor_id; 660 } 661 662 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa) 663 { 664 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 665 666 return dev->status; 667 } 668 669 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status) 670 { 671 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 672 673 if (vduse_dev_set_status(dev, status)) 674 return; 675 676 dev->status = status; 677 } 678 679 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa) 680 { 681 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 682 683 return dev->config_size; 684 } 685 686 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset, 687 void *buf, unsigned int len) 688 { 689 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 690 691 /* Initialize the buffer in case of partial copy. */ 692 memset(buf, 0, len); 693 694 if (offset > dev->config_size) 695 return; 696 697 if (len > dev->config_size - offset) 698 len = dev->config_size - offset; 699 700 memcpy(buf, dev->config + offset, len); 701 } 702 703 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset, 704 const void *buf, unsigned int len) 705 { 706 /* Now we only support read-only configuration space */ 707 } 708 709 static int vduse_vdpa_reset(struct vdpa_device *vdpa) 710 { 711 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 712 int ret = vduse_dev_set_status(dev, 0); 713 714 vduse_dev_reset(dev); 715 716 return ret; 717 } 718 719 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa) 720 { 721 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 722 723 return dev->generation; 724 } 725 726 static int vduse_vdpa_set_vq_affinity(struct vdpa_device *vdpa, u16 idx, 727 const struct cpumask *cpu_mask) 728 { 729 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 730 731 if (cpu_mask) 732 cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); 733 else 734 cpumask_setall(&dev->vqs[idx]->irq_affinity); 735 736 return 0; 737 } 738 739 static const struct cpumask * 740 vduse_vdpa_get_vq_affinity(struct vdpa_device *vdpa, u16 idx) 741 { 742 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 743 744 return &dev->vqs[idx]->irq_affinity; 745 } 746 747 static int vduse_vdpa_set_map(struct vdpa_device *vdpa, 748 unsigned int asid, 749 struct vhost_iotlb *iotlb) 750 { 751 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 752 int ret; 753 754 ret = vduse_domain_set_map(dev->domain, iotlb); 755 if (ret) 756 return ret; 757 758 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX); 759 if (ret) { 760 vduse_domain_clear_map(dev->domain, iotlb); 761 return ret; 762 } 763 764 return 0; 765 } 766 767 static void vduse_vdpa_free(struct vdpa_device *vdpa) 768 { 769 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 770 771 dev->vdev = NULL; 772 } 773 774 static const struct vdpa_config_ops vduse_vdpa_config_ops = { 775 .set_vq_address = vduse_vdpa_set_vq_address, 776 .kick_vq = vduse_vdpa_kick_vq, 777 .set_vq_cb = vduse_vdpa_set_vq_cb, 778 .set_vq_num = vduse_vdpa_set_vq_num, 779 .set_vq_ready = vduse_vdpa_set_vq_ready, 780 .get_vq_ready = vduse_vdpa_get_vq_ready, 781 .set_vq_state = vduse_vdpa_set_vq_state, 782 .get_vq_state = vduse_vdpa_get_vq_state, 783 .get_vq_align = vduse_vdpa_get_vq_align, 784 .get_device_features = vduse_vdpa_get_device_features, 785 .set_driver_features = vduse_vdpa_set_driver_features, 786 .get_driver_features = vduse_vdpa_get_driver_features, 787 .set_config_cb = vduse_vdpa_set_config_cb, 788 .get_vq_num_max = vduse_vdpa_get_vq_num_max, 789 .get_device_id = vduse_vdpa_get_device_id, 790 .get_vendor_id = vduse_vdpa_get_vendor_id, 791 .get_status = vduse_vdpa_get_status, 792 .set_status = vduse_vdpa_set_status, 793 .get_config_size = vduse_vdpa_get_config_size, 794 .get_config = vduse_vdpa_get_config, 795 .set_config = vduse_vdpa_set_config, 796 .get_generation = vduse_vdpa_get_generation, 797 .set_vq_affinity = vduse_vdpa_set_vq_affinity, 798 .get_vq_affinity = vduse_vdpa_get_vq_affinity, 799 .reset = vduse_vdpa_reset, 800 .set_map = vduse_vdpa_set_map, 801 .free = vduse_vdpa_free, 802 }; 803 804 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, 805 unsigned long offset, size_t size, 806 enum dma_data_direction dir, 807 unsigned long attrs) 808 { 809 struct vduse_dev *vdev = dev_to_vduse(dev); 810 struct vduse_iova_domain *domain = vdev->domain; 811 812 return vduse_domain_map_page(domain, page, offset, size, dir, attrs); 813 } 814 815 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr, 816 size_t size, enum dma_data_direction dir, 817 unsigned long attrs) 818 { 819 struct vduse_dev *vdev = dev_to_vduse(dev); 820 struct vduse_iova_domain *domain = vdev->domain; 821 822 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs); 823 } 824 825 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size, 826 dma_addr_t *dma_addr, gfp_t flag, 827 unsigned long attrs) 828 { 829 struct vduse_dev *vdev = dev_to_vduse(dev); 830 struct vduse_iova_domain *domain = vdev->domain; 831 unsigned long iova; 832 void *addr; 833 834 *dma_addr = DMA_MAPPING_ERROR; 835 addr = vduse_domain_alloc_coherent(domain, size, 836 (dma_addr_t *)&iova, flag, attrs); 837 if (!addr) 838 return NULL; 839 840 *dma_addr = (dma_addr_t)iova; 841 842 return addr; 843 } 844 845 static void vduse_dev_free_coherent(struct device *dev, size_t size, 846 void *vaddr, dma_addr_t dma_addr, 847 unsigned long attrs) 848 { 849 struct vduse_dev *vdev = dev_to_vduse(dev); 850 struct vduse_iova_domain *domain = vdev->domain; 851 852 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs); 853 } 854 855 static size_t vduse_dev_max_mapping_size(struct device *dev) 856 { 857 struct vduse_dev *vdev = dev_to_vduse(dev); 858 struct vduse_iova_domain *domain = vdev->domain; 859 860 return domain->bounce_size; 861 } 862 863 static const struct dma_map_ops vduse_dev_dma_ops = { 864 .map_page = vduse_dev_map_page, 865 .unmap_page = vduse_dev_unmap_page, 866 .alloc = vduse_dev_alloc_coherent, 867 .free = vduse_dev_free_coherent, 868 .max_mapping_size = vduse_dev_max_mapping_size, 869 }; 870 871 static unsigned int perm_to_file_flags(u8 perm) 872 { 873 unsigned int flags = 0; 874 875 switch (perm) { 876 case VDUSE_ACCESS_WO: 877 flags |= O_WRONLY; 878 break; 879 case VDUSE_ACCESS_RO: 880 flags |= O_RDONLY; 881 break; 882 case VDUSE_ACCESS_RW: 883 flags |= O_RDWR; 884 break; 885 default: 886 WARN(1, "invalidate vhost IOTLB permission\n"); 887 break; 888 } 889 890 return flags; 891 } 892 893 static int vduse_kickfd_setup(struct vduse_dev *dev, 894 struct vduse_vq_eventfd *eventfd) 895 { 896 struct eventfd_ctx *ctx = NULL; 897 struct vduse_virtqueue *vq; 898 u32 index; 899 900 if (eventfd->index >= dev->vq_num) 901 return -EINVAL; 902 903 index = array_index_nospec(eventfd->index, dev->vq_num); 904 vq = dev->vqs[index]; 905 if (eventfd->fd >= 0) { 906 ctx = eventfd_ctx_fdget(eventfd->fd); 907 if (IS_ERR(ctx)) 908 return PTR_ERR(ctx); 909 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN) 910 return 0; 911 912 spin_lock(&vq->kick_lock); 913 if (vq->kickfd) 914 eventfd_ctx_put(vq->kickfd); 915 vq->kickfd = ctx; 916 if (vq->ready && vq->kicked && vq->kickfd) { 917 eventfd_signal(vq->kickfd, 1); 918 vq->kicked = false; 919 } 920 spin_unlock(&vq->kick_lock); 921 922 return 0; 923 } 924 925 static bool vduse_dev_is_ready(struct vduse_dev *dev) 926 { 927 int i; 928 929 for (i = 0; i < dev->vq_num; i++) 930 if (!dev->vqs[i]->num_max) 931 return false; 932 933 return true; 934 } 935 936 static void vduse_dev_irq_inject(struct work_struct *work) 937 { 938 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject); 939 940 spin_lock_bh(&dev->irq_lock); 941 if (dev->config_cb.callback) 942 dev->config_cb.callback(dev->config_cb.private); 943 spin_unlock_bh(&dev->irq_lock); 944 } 945 946 static void vduse_vq_irq_inject(struct work_struct *work) 947 { 948 struct vduse_virtqueue *vq = container_of(work, 949 struct vduse_virtqueue, inject); 950 951 spin_lock_bh(&vq->irq_lock); 952 if (vq->ready && vq->cb.callback) 953 vq->cb.callback(vq->cb.private); 954 spin_unlock_bh(&vq->irq_lock); 955 } 956 957 static bool vduse_vq_signal_irqfd(struct vduse_virtqueue *vq) 958 { 959 bool signal = false; 960 961 if (!vq->cb.trigger) 962 return false; 963 964 spin_lock_irq(&vq->irq_lock); 965 if (vq->ready && vq->cb.trigger) { 966 eventfd_signal(vq->cb.trigger, 1); 967 signal = true; 968 } 969 spin_unlock_irq(&vq->irq_lock); 970 971 return signal; 972 } 973 974 static int vduse_dev_queue_irq_work(struct vduse_dev *dev, 975 struct work_struct *irq_work, 976 int irq_effective_cpu) 977 { 978 int ret = -EINVAL; 979 980 down_read(&dev->rwsem); 981 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 982 goto unlock; 983 984 ret = 0; 985 if (irq_effective_cpu == IRQ_UNBOUND) 986 queue_work(vduse_irq_wq, irq_work); 987 else 988 queue_work_on(irq_effective_cpu, 989 vduse_irq_bound_wq, irq_work); 990 unlock: 991 up_read(&dev->rwsem); 992 993 return ret; 994 } 995 996 static int vduse_dev_dereg_umem(struct vduse_dev *dev, 997 u64 iova, u64 size) 998 { 999 int ret; 1000 1001 mutex_lock(&dev->mem_lock); 1002 ret = -ENOENT; 1003 if (!dev->umem) 1004 goto unlock; 1005 1006 ret = -EINVAL; 1007 if (!dev->domain) 1008 goto unlock; 1009 1010 if (dev->umem->iova != iova || size != dev->domain->bounce_size) 1011 goto unlock; 1012 1013 vduse_domain_remove_user_bounce_pages(dev->domain); 1014 unpin_user_pages_dirty_lock(dev->umem->pages, 1015 dev->umem->npages, true); 1016 atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm); 1017 mmdrop(dev->umem->mm); 1018 vfree(dev->umem->pages); 1019 kfree(dev->umem); 1020 dev->umem = NULL; 1021 ret = 0; 1022 unlock: 1023 mutex_unlock(&dev->mem_lock); 1024 return ret; 1025 } 1026 1027 static int vduse_dev_reg_umem(struct vduse_dev *dev, 1028 u64 iova, u64 uaddr, u64 size) 1029 { 1030 struct page **page_list = NULL; 1031 struct vduse_umem *umem = NULL; 1032 long pinned = 0; 1033 unsigned long npages, lock_limit; 1034 int ret; 1035 1036 if (!dev->domain || !dev->domain->bounce_map || 1037 size != dev->domain->bounce_size || 1038 iova != 0 || uaddr & ~PAGE_MASK) 1039 return -EINVAL; 1040 1041 mutex_lock(&dev->mem_lock); 1042 ret = -EEXIST; 1043 if (dev->umem) 1044 goto unlock; 1045 1046 ret = -ENOMEM; 1047 npages = size >> PAGE_SHIFT; 1048 page_list = __vmalloc(array_size(npages, sizeof(struct page *)), 1049 GFP_KERNEL_ACCOUNT); 1050 umem = kzalloc(sizeof(*umem), GFP_KERNEL); 1051 if (!page_list || !umem) 1052 goto unlock; 1053 1054 mmap_read_lock(current->mm); 1055 1056 lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK)); 1057 if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit) 1058 goto out; 1059 1060 pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE, 1061 page_list); 1062 if (pinned != npages) { 1063 ret = pinned < 0 ? pinned : -ENOMEM; 1064 goto out; 1065 } 1066 1067 ret = vduse_domain_add_user_bounce_pages(dev->domain, 1068 page_list, pinned); 1069 if (ret) 1070 goto out; 1071 1072 atomic64_add(npages, ¤t->mm->pinned_vm); 1073 1074 umem->pages = page_list; 1075 umem->npages = pinned; 1076 umem->iova = iova; 1077 umem->mm = current->mm; 1078 mmgrab(current->mm); 1079 1080 dev->umem = umem; 1081 out: 1082 if (ret && pinned > 0) 1083 unpin_user_pages(page_list, pinned); 1084 1085 mmap_read_unlock(current->mm); 1086 unlock: 1087 if (ret) { 1088 vfree(page_list); 1089 kfree(umem); 1090 } 1091 mutex_unlock(&dev->mem_lock); 1092 return ret; 1093 } 1094 1095 static void vduse_vq_update_effective_cpu(struct vduse_virtqueue *vq) 1096 { 1097 int curr_cpu = vq->irq_effective_cpu; 1098 1099 while (true) { 1100 curr_cpu = cpumask_next(curr_cpu, &vq->irq_affinity); 1101 if (cpu_online(curr_cpu)) 1102 break; 1103 1104 if (curr_cpu >= nr_cpu_ids) 1105 curr_cpu = IRQ_UNBOUND; 1106 } 1107 1108 vq->irq_effective_cpu = curr_cpu; 1109 } 1110 1111 static long vduse_dev_ioctl(struct file *file, unsigned int cmd, 1112 unsigned long arg) 1113 { 1114 struct vduse_dev *dev = file->private_data; 1115 void __user *argp = (void __user *)arg; 1116 int ret; 1117 1118 if (unlikely(dev->broken)) 1119 return -EPERM; 1120 1121 switch (cmd) { 1122 case VDUSE_IOTLB_GET_FD: { 1123 struct vduse_iotlb_entry entry; 1124 struct vhost_iotlb_map *map; 1125 struct vdpa_map_file *map_file; 1126 struct file *f = NULL; 1127 1128 ret = -EFAULT; 1129 if (copy_from_user(&entry, argp, sizeof(entry))) 1130 break; 1131 1132 ret = -EINVAL; 1133 if (entry.start > entry.last) 1134 break; 1135 1136 mutex_lock(&dev->domain_lock); 1137 if (!dev->domain) { 1138 mutex_unlock(&dev->domain_lock); 1139 break; 1140 } 1141 spin_lock(&dev->domain->iotlb_lock); 1142 map = vhost_iotlb_itree_first(dev->domain->iotlb, 1143 entry.start, entry.last); 1144 if (map) { 1145 map_file = (struct vdpa_map_file *)map->opaque; 1146 f = get_file(map_file->file); 1147 entry.offset = map_file->offset; 1148 entry.start = map->start; 1149 entry.last = map->last; 1150 entry.perm = map->perm; 1151 } 1152 spin_unlock(&dev->domain->iotlb_lock); 1153 mutex_unlock(&dev->domain_lock); 1154 ret = -EINVAL; 1155 if (!f) 1156 break; 1157 1158 ret = -EFAULT; 1159 if (copy_to_user(argp, &entry, sizeof(entry))) { 1160 fput(f); 1161 break; 1162 } 1163 ret = receive_fd(f, perm_to_file_flags(entry.perm)); 1164 fput(f); 1165 break; 1166 } 1167 case VDUSE_DEV_GET_FEATURES: 1168 /* 1169 * Just mirror what driver wrote here. 1170 * The driver is expected to check FEATURE_OK later. 1171 */ 1172 ret = put_user(dev->driver_features, (u64 __user *)argp); 1173 break; 1174 case VDUSE_DEV_SET_CONFIG: { 1175 struct vduse_config_data config; 1176 unsigned long size = offsetof(struct vduse_config_data, 1177 buffer); 1178 1179 ret = -EFAULT; 1180 if (copy_from_user(&config, argp, size)) 1181 break; 1182 1183 ret = -EINVAL; 1184 if (config.offset > dev->config_size || 1185 config.length == 0 || 1186 config.length > dev->config_size - config.offset) 1187 break; 1188 1189 ret = -EFAULT; 1190 if (copy_from_user(dev->config + config.offset, argp + size, 1191 config.length)) 1192 break; 1193 1194 ret = 0; 1195 break; 1196 } 1197 case VDUSE_DEV_INJECT_CONFIG_IRQ: 1198 ret = vduse_dev_queue_irq_work(dev, &dev->inject, IRQ_UNBOUND); 1199 break; 1200 case VDUSE_VQ_SETUP: { 1201 struct vduse_vq_config config; 1202 u32 index; 1203 1204 ret = -EFAULT; 1205 if (copy_from_user(&config, argp, sizeof(config))) 1206 break; 1207 1208 ret = -EINVAL; 1209 if (config.index >= dev->vq_num) 1210 break; 1211 1212 if (!is_mem_zero((const char *)config.reserved, 1213 sizeof(config.reserved))) 1214 break; 1215 1216 index = array_index_nospec(config.index, dev->vq_num); 1217 dev->vqs[index]->num_max = config.max_size; 1218 ret = 0; 1219 break; 1220 } 1221 case VDUSE_VQ_GET_INFO: { 1222 struct vduse_vq_info vq_info; 1223 struct vduse_virtqueue *vq; 1224 u32 index; 1225 1226 ret = -EFAULT; 1227 if (copy_from_user(&vq_info, argp, sizeof(vq_info))) 1228 break; 1229 1230 ret = -EINVAL; 1231 if (vq_info.index >= dev->vq_num) 1232 break; 1233 1234 index = array_index_nospec(vq_info.index, dev->vq_num); 1235 vq = dev->vqs[index]; 1236 vq_info.desc_addr = vq->desc_addr; 1237 vq_info.driver_addr = vq->driver_addr; 1238 vq_info.device_addr = vq->device_addr; 1239 vq_info.num = vq->num; 1240 1241 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 1242 vq_info.packed.last_avail_counter = 1243 vq->state.packed.last_avail_counter; 1244 vq_info.packed.last_avail_idx = 1245 vq->state.packed.last_avail_idx; 1246 vq_info.packed.last_used_counter = 1247 vq->state.packed.last_used_counter; 1248 vq_info.packed.last_used_idx = 1249 vq->state.packed.last_used_idx; 1250 } else 1251 vq_info.split.avail_index = 1252 vq->state.split.avail_index; 1253 1254 vq_info.ready = vq->ready; 1255 1256 ret = -EFAULT; 1257 if (copy_to_user(argp, &vq_info, sizeof(vq_info))) 1258 break; 1259 1260 ret = 0; 1261 break; 1262 } 1263 case VDUSE_VQ_SETUP_KICKFD: { 1264 struct vduse_vq_eventfd eventfd; 1265 1266 ret = -EFAULT; 1267 if (copy_from_user(&eventfd, argp, sizeof(eventfd))) 1268 break; 1269 1270 ret = vduse_kickfd_setup(dev, &eventfd); 1271 break; 1272 } 1273 case VDUSE_VQ_INJECT_IRQ: { 1274 u32 index; 1275 1276 ret = -EFAULT; 1277 if (get_user(index, (u32 __user *)argp)) 1278 break; 1279 1280 ret = -EINVAL; 1281 if (index >= dev->vq_num) 1282 break; 1283 1284 ret = 0; 1285 index = array_index_nospec(index, dev->vq_num); 1286 if (!vduse_vq_signal_irqfd(dev->vqs[index])) { 1287 vduse_vq_update_effective_cpu(dev->vqs[index]); 1288 ret = vduse_dev_queue_irq_work(dev, 1289 &dev->vqs[index]->inject, 1290 dev->vqs[index]->irq_effective_cpu); 1291 } 1292 break; 1293 } 1294 case VDUSE_IOTLB_REG_UMEM: { 1295 struct vduse_iova_umem umem; 1296 1297 ret = -EFAULT; 1298 if (copy_from_user(&umem, argp, sizeof(umem))) 1299 break; 1300 1301 ret = -EINVAL; 1302 if (!is_mem_zero((const char *)umem.reserved, 1303 sizeof(umem.reserved))) 1304 break; 1305 1306 mutex_lock(&dev->domain_lock); 1307 ret = vduse_dev_reg_umem(dev, umem.iova, 1308 umem.uaddr, umem.size); 1309 mutex_unlock(&dev->domain_lock); 1310 break; 1311 } 1312 case VDUSE_IOTLB_DEREG_UMEM: { 1313 struct vduse_iova_umem umem; 1314 1315 ret = -EFAULT; 1316 if (copy_from_user(&umem, argp, sizeof(umem))) 1317 break; 1318 1319 ret = -EINVAL; 1320 if (!is_mem_zero((const char *)umem.reserved, 1321 sizeof(umem.reserved))) 1322 break; 1323 mutex_lock(&dev->domain_lock); 1324 ret = vduse_dev_dereg_umem(dev, umem.iova, 1325 umem.size); 1326 mutex_unlock(&dev->domain_lock); 1327 break; 1328 } 1329 case VDUSE_IOTLB_GET_INFO: { 1330 struct vduse_iova_info info; 1331 struct vhost_iotlb_map *map; 1332 1333 ret = -EFAULT; 1334 if (copy_from_user(&info, argp, sizeof(info))) 1335 break; 1336 1337 ret = -EINVAL; 1338 if (info.start > info.last) 1339 break; 1340 1341 if (!is_mem_zero((const char *)info.reserved, 1342 sizeof(info.reserved))) 1343 break; 1344 1345 mutex_lock(&dev->domain_lock); 1346 if (!dev->domain) { 1347 mutex_unlock(&dev->domain_lock); 1348 break; 1349 } 1350 spin_lock(&dev->domain->iotlb_lock); 1351 map = vhost_iotlb_itree_first(dev->domain->iotlb, 1352 info.start, info.last); 1353 if (map) { 1354 info.start = map->start; 1355 info.last = map->last; 1356 info.capability = 0; 1357 if (dev->domain->bounce_map && map->start == 0 && 1358 map->last == dev->domain->bounce_size - 1) 1359 info.capability |= VDUSE_IOVA_CAP_UMEM; 1360 } 1361 spin_unlock(&dev->domain->iotlb_lock); 1362 mutex_unlock(&dev->domain_lock); 1363 if (!map) 1364 break; 1365 1366 ret = -EFAULT; 1367 if (copy_to_user(argp, &info, sizeof(info))) 1368 break; 1369 1370 ret = 0; 1371 break; 1372 } 1373 default: 1374 ret = -ENOIOCTLCMD; 1375 break; 1376 } 1377 1378 return ret; 1379 } 1380 1381 static int vduse_dev_release(struct inode *inode, struct file *file) 1382 { 1383 struct vduse_dev *dev = file->private_data; 1384 1385 mutex_lock(&dev->domain_lock); 1386 if (dev->domain) 1387 vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size); 1388 mutex_unlock(&dev->domain_lock); 1389 spin_lock(&dev->msg_lock); 1390 /* Make sure the inflight messages can processed after reconncection */ 1391 list_splice_init(&dev->recv_list, &dev->send_list); 1392 spin_unlock(&dev->msg_lock); 1393 dev->connected = false; 1394 1395 return 0; 1396 } 1397 1398 static struct vduse_dev *vduse_dev_get_from_minor(int minor) 1399 { 1400 struct vduse_dev *dev; 1401 1402 mutex_lock(&vduse_lock); 1403 dev = idr_find(&vduse_idr, minor); 1404 mutex_unlock(&vduse_lock); 1405 1406 return dev; 1407 } 1408 1409 static int vduse_dev_open(struct inode *inode, struct file *file) 1410 { 1411 int ret; 1412 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode)); 1413 1414 if (!dev) 1415 return -ENODEV; 1416 1417 ret = -EBUSY; 1418 mutex_lock(&dev->lock); 1419 if (dev->connected) 1420 goto unlock; 1421 1422 ret = 0; 1423 dev->connected = true; 1424 file->private_data = dev; 1425 unlock: 1426 mutex_unlock(&dev->lock); 1427 1428 return ret; 1429 } 1430 1431 static const struct file_operations vduse_dev_fops = { 1432 .owner = THIS_MODULE, 1433 .open = vduse_dev_open, 1434 .release = vduse_dev_release, 1435 .read_iter = vduse_dev_read_iter, 1436 .write_iter = vduse_dev_write_iter, 1437 .poll = vduse_dev_poll, 1438 .unlocked_ioctl = vduse_dev_ioctl, 1439 .compat_ioctl = compat_ptr_ioctl, 1440 .llseek = noop_llseek, 1441 }; 1442 1443 static ssize_t irq_cb_affinity_show(struct vduse_virtqueue *vq, char *buf) 1444 { 1445 return sprintf(buf, "%*pb\n", cpumask_pr_args(&vq->irq_affinity)); 1446 } 1447 1448 static ssize_t irq_cb_affinity_store(struct vduse_virtqueue *vq, 1449 const char *buf, size_t count) 1450 { 1451 cpumask_var_t new_value; 1452 int ret; 1453 1454 if (!zalloc_cpumask_var(&new_value, GFP_KERNEL)) 1455 return -ENOMEM; 1456 1457 ret = cpumask_parse(buf, new_value); 1458 if (ret) 1459 goto free_mask; 1460 1461 ret = -EINVAL; 1462 if (!cpumask_intersects(new_value, cpu_online_mask)) 1463 goto free_mask; 1464 1465 cpumask_copy(&vq->irq_affinity, new_value); 1466 ret = count; 1467 free_mask: 1468 free_cpumask_var(new_value); 1469 return ret; 1470 } 1471 1472 struct vq_sysfs_entry { 1473 struct attribute attr; 1474 ssize_t (*show)(struct vduse_virtqueue *vq, char *buf); 1475 ssize_t (*store)(struct vduse_virtqueue *vq, const char *buf, 1476 size_t count); 1477 }; 1478 1479 static struct vq_sysfs_entry irq_cb_affinity_attr = __ATTR_RW(irq_cb_affinity); 1480 1481 static struct attribute *vq_attrs[] = { 1482 &irq_cb_affinity_attr.attr, 1483 NULL, 1484 }; 1485 ATTRIBUTE_GROUPS(vq); 1486 1487 static ssize_t vq_attr_show(struct kobject *kobj, struct attribute *attr, 1488 char *buf) 1489 { 1490 struct vduse_virtqueue *vq = container_of(kobj, 1491 struct vduse_virtqueue, kobj); 1492 struct vq_sysfs_entry *entry = container_of(attr, 1493 struct vq_sysfs_entry, attr); 1494 1495 if (!entry->show) 1496 return -EIO; 1497 1498 return entry->show(vq, buf); 1499 } 1500 1501 static ssize_t vq_attr_store(struct kobject *kobj, struct attribute *attr, 1502 const char *buf, size_t count) 1503 { 1504 struct vduse_virtqueue *vq = container_of(kobj, 1505 struct vduse_virtqueue, kobj); 1506 struct vq_sysfs_entry *entry = container_of(attr, 1507 struct vq_sysfs_entry, attr); 1508 1509 if (!entry->store) 1510 return -EIO; 1511 1512 return entry->store(vq, buf, count); 1513 } 1514 1515 static const struct sysfs_ops vq_sysfs_ops = { 1516 .show = vq_attr_show, 1517 .store = vq_attr_store, 1518 }; 1519 1520 static void vq_release(struct kobject *kobj) 1521 { 1522 struct vduse_virtqueue *vq = container_of(kobj, 1523 struct vduse_virtqueue, kobj); 1524 kfree(vq); 1525 } 1526 1527 static const struct kobj_type vq_type = { 1528 .release = vq_release, 1529 .sysfs_ops = &vq_sysfs_ops, 1530 .default_groups = vq_groups, 1531 }; 1532 1533 static void vduse_dev_deinit_vqs(struct vduse_dev *dev) 1534 { 1535 int i; 1536 1537 if (!dev->vqs) 1538 return; 1539 1540 for (i = 0; i < dev->vq_num; i++) 1541 kobject_put(&dev->vqs[i]->kobj); 1542 kfree(dev->vqs); 1543 } 1544 1545 static int vduse_dev_init_vqs(struct vduse_dev *dev, u32 vq_align, u32 vq_num) 1546 { 1547 int ret, i; 1548 1549 dev->vq_align = vq_align; 1550 dev->vq_num = vq_num; 1551 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL); 1552 if (!dev->vqs) 1553 return -ENOMEM; 1554 1555 for (i = 0; i < vq_num; i++) { 1556 dev->vqs[i] = kzalloc(sizeof(*dev->vqs[i]), GFP_KERNEL); 1557 if (!dev->vqs[i]) { 1558 ret = -ENOMEM; 1559 goto err; 1560 } 1561 1562 dev->vqs[i]->index = i; 1563 dev->vqs[i]->irq_effective_cpu = IRQ_UNBOUND; 1564 INIT_WORK(&dev->vqs[i]->inject, vduse_vq_irq_inject); 1565 INIT_WORK(&dev->vqs[i]->kick, vduse_vq_kick_work); 1566 spin_lock_init(&dev->vqs[i]->kick_lock); 1567 spin_lock_init(&dev->vqs[i]->irq_lock); 1568 cpumask_setall(&dev->vqs[i]->irq_affinity); 1569 1570 kobject_init(&dev->vqs[i]->kobj, &vq_type); 1571 ret = kobject_add(&dev->vqs[i]->kobj, 1572 &dev->dev->kobj, "vq%d", i); 1573 if (ret) { 1574 kfree(dev->vqs[i]); 1575 goto err; 1576 } 1577 } 1578 1579 return 0; 1580 err: 1581 while (i--) 1582 kobject_put(&dev->vqs[i]->kobj); 1583 kfree(dev->vqs); 1584 dev->vqs = NULL; 1585 return ret; 1586 } 1587 1588 static struct vduse_dev *vduse_dev_create(void) 1589 { 1590 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1591 1592 if (!dev) 1593 return NULL; 1594 1595 mutex_init(&dev->lock); 1596 mutex_init(&dev->mem_lock); 1597 mutex_init(&dev->domain_lock); 1598 spin_lock_init(&dev->msg_lock); 1599 INIT_LIST_HEAD(&dev->send_list); 1600 INIT_LIST_HEAD(&dev->recv_list); 1601 spin_lock_init(&dev->irq_lock); 1602 init_rwsem(&dev->rwsem); 1603 1604 INIT_WORK(&dev->inject, vduse_dev_irq_inject); 1605 init_waitqueue_head(&dev->waitq); 1606 1607 return dev; 1608 } 1609 1610 static void vduse_dev_destroy(struct vduse_dev *dev) 1611 { 1612 kfree(dev); 1613 } 1614 1615 static struct vduse_dev *vduse_find_dev(const char *name) 1616 { 1617 struct vduse_dev *dev; 1618 int id; 1619 1620 idr_for_each_entry(&vduse_idr, dev, id) 1621 if (!strcmp(dev->name, name)) 1622 return dev; 1623 1624 return NULL; 1625 } 1626 1627 static int vduse_destroy_dev(char *name) 1628 { 1629 struct vduse_dev *dev = vduse_find_dev(name); 1630 1631 if (!dev) 1632 return -EINVAL; 1633 1634 mutex_lock(&dev->lock); 1635 if (dev->vdev || dev->connected) { 1636 mutex_unlock(&dev->lock); 1637 return -EBUSY; 1638 } 1639 dev->connected = true; 1640 mutex_unlock(&dev->lock); 1641 1642 vduse_dev_reset(dev); 1643 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 1644 idr_remove(&vduse_idr, dev->minor); 1645 kvfree(dev->config); 1646 vduse_dev_deinit_vqs(dev); 1647 if (dev->domain) 1648 vduse_domain_destroy(dev->domain); 1649 kfree(dev->name); 1650 vduse_dev_destroy(dev); 1651 module_put(THIS_MODULE); 1652 1653 return 0; 1654 } 1655 1656 static bool device_is_allowed(u32 device_id) 1657 { 1658 int i; 1659 1660 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++) 1661 if (allowed_device_id[i] == device_id) 1662 return true; 1663 1664 return false; 1665 } 1666 1667 static bool features_is_valid(struct vduse_dev_config *config) 1668 { 1669 if (!(config->features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM))) 1670 return false; 1671 1672 /* Now we only support read-only configuration space */ 1673 if ((config->device_id == VIRTIO_ID_BLOCK) && 1674 (config->features & BIT_ULL(VIRTIO_BLK_F_CONFIG_WCE))) 1675 return false; 1676 else if ((config->device_id == VIRTIO_ID_NET) && 1677 (config->features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) 1678 return false; 1679 1680 return true; 1681 } 1682 1683 static bool vduse_validate_config(struct vduse_dev_config *config) 1684 { 1685 if (!is_mem_zero((const char *)config->reserved, 1686 sizeof(config->reserved))) 1687 return false; 1688 1689 if (config->vq_align > PAGE_SIZE) 1690 return false; 1691 1692 if (config->config_size > PAGE_SIZE) 1693 return false; 1694 1695 if (config->vq_num > 0xffff) 1696 return false; 1697 1698 if (!config->name[0]) 1699 return false; 1700 1701 if (!device_is_allowed(config->device_id)) 1702 return false; 1703 1704 if (!features_is_valid(config)) 1705 return false; 1706 1707 return true; 1708 } 1709 1710 static ssize_t msg_timeout_show(struct device *device, 1711 struct device_attribute *attr, char *buf) 1712 { 1713 struct vduse_dev *dev = dev_get_drvdata(device); 1714 1715 return sysfs_emit(buf, "%u\n", dev->msg_timeout); 1716 } 1717 1718 static ssize_t msg_timeout_store(struct device *device, 1719 struct device_attribute *attr, 1720 const char *buf, size_t count) 1721 { 1722 struct vduse_dev *dev = dev_get_drvdata(device); 1723 int ret; 1724 1725 ret = kstrtouint(buf, 10, &dev->msg_timeout); 1726 if (ret < 0) 1727 return ret; 1728 1729 return count; 1730 } 1731 1732 static DEVICE_ATTR_RW(msg_timeout); 1733 1734 static ssize_t bounce_size_show(struct device *device, 1735 struct device_attribute *attr, char *buf) 1736 { 1737 struct vduse_dev *dev = dev_get_drvdata(device); 1738 1739 return sysfs_emit(buf, "%u\n", dev->bounce_size); 1740 } 1741 1742 static ssize_t bounce_size_store(struct device *device, 1743 struct device_attribute *attr, 1744 const char *buf, size_t count) 1745 { 1746 struct vduse_dev *dev = dev_get_drvdata(device); 1747 unsigned int bounce_size; 1748 int ret; 1749 1750 ret = -EPERM; 1751 mutex_lock(&dev->domain_lock); 1752 if (dev->domain) 1753 goto unlock; 1754 1755 ret = kstrtouint(buf, 10, &bounce_size); 1756 if (ret < 0) 1757 goto unlock; 1758 1759 ret = -EINVAL; 1760 if (bounce_size > VDUSE_MAX_BOUNCE_SIZE || 1761 bounce_size < VDUSE_MIN_BOUNCE_SIZE) 1762 goto unlock; 1763 1764 dev->bounce_size = bounce_size & PAGE_MASK; 1765 ret = count; 1766 unlock: 1767 mutex_unlock(&dev->domain_lock); 1768 return ret; 1769 } 1770 1771 static DEVICE_ATTR_RW(bounce_size); 1772 1773 static struct attribute *vduse_dev_attrs[] = { 1774 &dev_attr_msg_timeout.attr, 1775 &dev_attr_bounce_size.attr, 1776 NULL 1777 }; 1778 1779 ATTRIBUTE_GROUPS(vduse_dev); 1780 1781 static int vduse_create_dev(struct vduse_dev_config *config, 1782 void *config_buf, u64 api_version) 1783 { 1784 int ret; 1785 struct vduse_dev *dev; 1786 1787 ret = -EEXIST; 1788 if (vduse_find_dev(config->name)) 1789 goto err; 1790 1791 ret = -ENOMEM; 1792 dev = vduse_dev_create(); 1793 if (!dev) 1794 goto err; 1795 1796 dev->api_version = api_version; 1797 dev->device_features = config->features; 1798 dev->device_id = config->device_id; 1799 dev->vendor_id = config->vendor_id; 1800 dev->name = kstrdup(config->name, GFP_KERNEL); 1801 if (!dev->name) 1802 goto err_str; 1803 1804 dev->bounce_size = VDUSE_BOUNCE_SIZE; 1805 dev->config = config_buf; 1806 dev->config_size = config->config_size; 1807 1808 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL); 1809 if (ret < 0) 1810 goto err_idr; 1811 1812 dev->minor = ret; 1813 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT; 1814 dev->dev = device_create_with_groups(vduse_class, NULL, 1815 MKDEV(MAJOR(vduse_major), dev->minor), 1816 dev, vduse_dev_groups, "%s", config->name); 1817 if (IS_ERR(dev->dev)) { 1818 ret = PTR_ERR(dev->dev); 1819 goto err_dev; 1820 } 1821 1822 ret = vduse_dev_init_vqs(dev, config->vq_align, config->vq_num); 1823 if (ret) 1824 goto err_vqs; 1825 1826 __module_get(THIS_MODULE); 1827 1828 return 0; 1829 err_vqs: 1830 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 1831 err_dev: 1832 idr_remove(&vduse_idr, dev->minor); 1833 err_idr: 1834 kfree(dev->name); 1835 err_str: 1836 vduse_dev_destroy(dev); 1837 err: 1838 return ret; 1839 } 1840 1841 static long vduse_ioctl(struct file *file, unsigned int cmd, 1842 unsigned long arg) 1843 { 1844 int ret; 1845 void __user *argp = (void __user *)arg; 1846 struct vduse_control *control = file->private_data; 1847 1848 mutex_lock(&vduse_lock); 1849 switch (cmd) { 1850 case VDUSE_GET_API_VERSION: 1851 ret = put_user(control->api_version, (u64 __user *)argp); 1852 break; 1853 case VDUSE_SET_API_VERSION: { 1854 u64 api_version; 1855 1856 ret = -EFAULT; 1857 if (get_user(api_version, (u64 __user *)argp)) 1858 break; 1859 1860 ret = -EINVAL; 1861 if (api_version > VDUSE_API_VERSION) 1862 break; 1863 1864 ret = 0; 1865 control->api_version = api_version; 1866 break; 1867 } 1868 case VDUSE_CREATE_DEV: { 1869 struct vduse_dev_config config; 1870 unsigned long size = offsetof(struct vduse_dev_config, config); 1871 void *buf; 1872 1873 ret = -EFAULT; 1874 if (copy_from_user(&config, argp, size)) 1875 break; 1876 1877 ret = -EINVAL; 1878 if (vduse_validate_config(&config) == false) 1879 break; 1880 1881 buf = vmemdup_user(argp + size, config.config_size); 1882 if (IS_ERR(buf)) { 1883 ret = PTR_ERR(buf); 1884 break; 1885 } 1886 config.name[VDUSE_NAME_MAX - 1] = '\0'; 1887 ret = vduse_create_dev(&config, buf, control->api_version); 1888 if (ret) 1889 kvfree(buf); 1890 break; 1891 } 1892 case VDUSE_DESTROY_DEV: { 1893 char name[VDUSE_NAME_MAX]; 1894 1895 ret = -EFAULT; 1896 if (copy_from_user(name, argp, VDUSE_NAME_MAX)) 1897 break; 1898 1899 name[VDUSE_NAME_MAX - 1] = '\0'; 1900 ret = vduse_destroy_dev(name); 1901 break; 1902 } 1903 default: 1904 ret = -EINVAL; 1905 break; 1906 } 1907 mutex_unlock(&vduse_lock); 1908 1909 return ret; 1910 } 1911 1912 static int vduse_release(struct inode *inode, struct file *file) 1913 { 1914 struct vduse_control *control = file->private_data; 1915 1916 kfree(control); 1917 return 0; 1918 } 1919 1920 static int vduse_open(struct inode *inode, struct file *file) 1921 { 1922 struct vduse_control *control; 1923 1924 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL); 1925 if (!control) 1926 return -ENOMEM; 1927 1928 control->api_version = VDUSE_API_VERSION; 1929 file->private_data = control; 1930 1931 return 0; 1932 } 1933 1934 static const struct file_operations vduse_ctrl_fops = { 1935 .owner = THIS_MODULE, 1936 .open = vduse_open, 1937 .release = vduse_release, 1938 .unlocked_ioctl = vduse_ioctl, 1939 .compat_ioctl = compat_ptr_ioctl, 1940 .llseek = noop_llseek, 1941 }; 1942 1943 static char *vduse_devnode(const struct device *dev, umode_t *mode) 1944 { 1945 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev)); 1946 } 1947 1948 struct vduse_mgmt_dev { 1949 struct vdpa_mgmt_dev mgmt_dev; 1950 struct device dev; 1951 }; 1952 1953 static struct vduse_mgmt_dev *vduse_mgmt; 1954 1955 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) 1956 { 1957 struct vduse_vdpa *vdev; 1958 int ret; 1959 1960 if (dev->vdev) 1961 return -EEXIST; 1962 1963 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev, 1964 &vduse_vdpa_config_ops, 1, 1, name, true); 1965 if (IS_ERR(vdev)) 1966 return PTR_ERR(vdev); 1967 1968 dev->vdev = vdev; 1969 vdev->dev = dev; 1970 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask; 1971 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64)); 1972 if (ret) { 1973 put_device(&vdev->vdpa.dev); 1974 return ret; 1975 } 1976 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops); 1977 vdev->vdpa.dma_dev = &vdev->vdpa.dev; 1978 vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev; 1979 1980 return 0; 1981 } 1982 1983 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 1984 const struct vdpa_dev_set_config *config) 1985 { 1986 struct vduse_dev *dev; 1987 int ret; 1988 1989 mutex_lock(&vduse_lock); 1990 dev = vduse_find_dev(name); 1991 if (!dev || !vduse_dev_is_ready(dev)) { 1992 mutex_unlock(&vduse_lock); 1993 return -EINVAL; 1994 } 1995 ret = vduse_dev_init_vdpa(dev, name); 1996 mutex_unlock(&vduse_lock); 1997 if (ret) 1998 return ret; 1999 2000 mutex_lock(&dev->domain_lock); 2001 if (!dev->domain) 2002 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1, 2003 dev->bounce_size); 2004 mutex_unlock(&dev->domain_lock); 2005 if (!dev->domain) { 2006 put_device(&dev->vdev->vdpa.dev); 2007 return -ENOMEM; 2008 } 2009 2010 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); 2011 if (ret) { 2012 put_device(&dev->vdev->vdpa.dev); 2013 mutex_lock(&dev->domain_lock); 2014 vduse_domain_destroy(dev->domain); 2015 dev->domain = NULL; 2016 mutex_unlock(&dev->domain_lock); 2017 return ret; 2018 } 2019 2020 return 0; 2021 } 2022 2023 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) 2024 { 2025 _vdpa_unregister_device(dev); 2026 } 2027 2028 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = { 2029 .dev_add = vdpa_dev_add, 2030 .dev_del = vdpa_dev_del, 2031 }; 2032 2033 static struct virtio_device_id id_table[] = { 2034 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 2035 { 0 }, 2036 }; 2037 2038 static void vduse_mgmtdev_release(struct device *dev) 2039 { 2040 struct vduse_mgmt_dev *mgmt_dev; 2041 2042 mgmt_dev = container_of(dev, struct vduse_mgmt_dev, dev); 2043 kfree(mgmt_dev); 2044 } 2045 2046 static int vduse_mgmtdev_init(void) 2047 { 2048 int ret; 2049 2050 vduse_mgmt = kzalloc(sizeof(*vduse_mgmt), GFP_KERNEL); 2051 if (!vduse_mgmt) 2052 return -ENOMEM; 2053 2054 ret = dev_set_name(&vduse_mgmt->dev, "vduse"); 2055 if (ret) { 2056 kfree(vduse_mgmt); 2057 return ret; 2058 } 2059 2060 vduse_mgmt->dev.release = vduse_mgmtdev_release; 2061 2062 ret = device_register(&vduse_mgmt->dev); 2063 if (ret) 2064 goto dev_reg_err; 2065 2066 vduse_mgmt->mgmt_dev.id_table = id_table; 2067 vduse_mgmt->mgmt_dev.ops = &vdpa_dev_mgmtdev_ops; 2068 vduse_mgmt->mgmt_dev.device = &vduse_mgmt->dev; 2069 ret = vdpa_mgmtdev_register(&vduse_mgmt->mgmt_dev); 2070 if (ret) 2071 device_unregister(&vduse_mgmt->dev); 2072 2073 return ret; 2074 2075 dev_reg_err: 2076 put_device(&vduse_mgmt->dev); 2077 return ret; 2078 } 2079 2080 static void vduse_mgmtdev_exit(void) 2081 { 2082 vdpa_mgmtdev_unregister(&vduse_mgmt->mgmt_dev); 2083 device_unregister(&vduse_mgmt->dev); 2084 } 2085 2086 static int vduse_init(void) 2087 { 2088 int ret; 2089 struct device *dev; 2090 2091 vduse_class = class_create("vduse"); 2092 if (IS_ERR(vduse_class)) 2093 return PTR_ERR(vduse_class); 2094 2095 vduse_class->devnode = vduse_devnode; 2096 2097 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse"); 2098 if (ret) 2099 goto err_chardev_region; 2100 2101 /* /dev/vduse/control */ 2102 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops); 2103 vduse_ctrl_cdev.owner = THIS_MODULE; 2104 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1); 2105 if (ret) 2106 goto err_ctrl_cdev; 2107 2108 dev = device_create(vduse_class, NULL, vduse_major, NULL, "control"); 2109 if (IS_ERR(dev)) { 2110 ret = PTR_ERR(dev); 2111 goto err_device; 2112 } 2113 2114 /* /dev/vduse/$DEVICE */ 2115 cdev_init(&vduse_cdev, &vduse_dev_fops); 2116 vduse_cdev.owner = THIS_MODULE; 2117 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1), 2118 VDUSE_DEV_MAX - 1); 2119 if (ret) 2120 goto err_cdev; 2121 2122 ret = -ENOMEM; 2123 vduse_irq_wq = alloc_workqueue("vduse-irq", 2124 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0); 2125 if (!vduse_irq_wq) 2126 goto err_wq; 2127 2128 vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0); 2129 if (!vduse_irq_bound_wq) 2130 goto err_bound_wq; 2131 2132 ret = vduse_domain_init(); 2133 if (ret) 2134 goto err_domain; 2135 2136 ret = vduse_mgmtdev_init(); 2137 if (ret) 2138 goto err_mgmtdev; 2139 2140 return 0; 2141 err_mgmtdev: 2142 vduse_domain_exit(); 2143 err_domain: 2144 destroy_workqueue(vduse_irq_bound_wq); 2145 err_bound_wq: 2146 destroy_workqueue(vduse_irq_wq); 2147 err_wq: 2148 cdev_del(&vduse_cdev); 2149 err_cdev: 2150 device_destroy(vduse_class, vduse_major); 2151 err_device: 2152 cdev_del(&vduse_ctrl_cdev); 2153 err_ctrl_cdev: 2154 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 2155 err_chardev_region: 2156 class_destroy(vduse_class); 2157 return ret; 2158 } 2159 module_init(vduse_init); 2160 2161 static void vduse_exit(void) 2162 { 2163 vduse_mgmtdev_exit(); 2164 vduse_domain_exit(); 2165 destroy_workqueue(vduse_irq_bound_wq); 2166 destroy_workqueue(vduse_irq_wq); 2167 cdev_del(&vduse_cdev); 2168 device_destroy(vduse_class, vduse_major); 2169 cdev_del(&vduse_ctrl_cdev); 2170 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 2171 class_destroy(vduse_class); 2172 } 2173 module_exit(vduse_exit); 2174 2175 MODULE_LICENSE(DRV_LICENSE); 2176 MODULE_AUTHOR(DRV_AUTHOR); 2177 MODULE_DESCRIPTION(DRV_DESC); 2178