1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDUSE: vDPA Device in Userspace 4 * 5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved. 6 * 7 * Author: Xie Yongji <xieyongji@bytedance.com> 8 * 9 */ 10 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/eventfd.h> 16 #include <linux/slab.h> 17 #include <linux/wait.h> 18 #include <linux/dma-map-ops.h> 19 #include <linux/poll.h> 20 #include <linux/file.h> 21 #include <linux/uio.h> 22 #include <linux/vdpa.h> 23 #include <linux/nospec.h> 24 #include <uapi/linux/vduse.h> 25 #include <uapi/linux/vdpa.h> 26 #include <uapi/linux/virtio_config.h> 27 #include <uapi/linux/virtio_ids.h> 28 #include <uapi/linux/virtio_blk.h> 29 #include <linux/mod_devicetable.h> 30 31 #include "iova_domain.h" 32 33 #define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>" 34 #define DRV_DESC "vDPA Device in Userspace" 35 #define DRV_LICENSE "GPL v2" 36 37 #define VDUSE_DEV_MAX (1U << MINORBITS) 38 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024) 39 #define VDUSE_IOVA_SIZE (128 * 1024 * 1024) 40 #define VDUSE_MSG_DEFAULT_TIMEOUT 30 41 42 struct vduse_virtqueue { 43 u16 index; 44 u16 num_max; 45 u32 num; 46 u64 desc_addr; 47 u64 driver_addr; 48 u64 device_addr; 49 struct vdpa_vq_state state; 50 bool ready; 51 bool kicked; 52 spinlock_t kick_lock; 53 spinlock_t irq_lock; 54 struct eventfd_ctx *kickfd; 55 struct vdpa_callback cb; 56 struct work_struct inject; 57 struct work_struct kick; 58 }; 59 60 struct vduse_dev; 61 62 struct vduse_vdpa { 63 struct vdpa_device vdpa; 64 struct vduse_dev *dev; 65 }; 66 67 struct vduse_dev { 68 struct vduse_vdpa *vdev; 69 struct device *dev; 70 struct vduse_virtqueue *vqs; 71 struct vduse_iova_domain *domain; 72 char *name; 73 struct mutex lock; 74 spinlock_t msg_lock; 75 u64 msg_unique; 76 u32 msg_timeout; 77 wait_queue_head_t waitq; 78 struct list_head send_list; 79 struct list_head recv_list; 80 struct vdpa_callback config_cb; 81 struct work_struct inject; 82 spinlock_t irq_lock; 83 int minor; 84 bool broken; 85 bool connected; 86 u64 api_version; 87 u64 device_features; 88 u64 driver_features; 89 u32 device_id; 90 u32 vendor_id; 91 u32 generation; 92 u32 config_size; 93 void *config; 94 u8 status; 95 u32 vq_num; 96 u32 vq_align; 97 }; 98 99 struct vduse_dev_msg { 100 struct vduse_dev_request req; 101 struct vduse_dev_response resp; 102 struct list_head list; 103 wait_queue_head_t waitq; 104 bool completed; 105 }; 106 107 struct vduse_control { 108 u64 api_version; 109 }; 110 111 static DEFINE_MUTEX(vduse_lock); 112 static DEFINE_IDR(vduse_idr); 113 114 static dev_t vduse_major; 115 static struct class *vduse_class; 116 static struct cdev vduse_ctrl_cdev; 117 static struct cdev vduse_cdev; 118 static struct workqueue_struct *vduse_irq_wq; 119 120 static u32 allowed_device_id[] = { 121 VIRTIO_ID_BLOCK, 122 }; 123 124 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa) 125 { 126 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa); 127 128 return vdev->dev; 129 } 130 131 static inline struct vduse_dev *dev_to_vduse(struct device *dev) 132 { 133 struct vdpa_device *vdpa = dev_to_vdpa(dev); 134 135 return vdpa_to_vduse(vdpa); 136 } 137 138 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head, 139 uint32_t request_id) 140 { 141 struct vduse_dev_msg *msg; 142 143 list_for_each_entry(msg, head, list) { 144 if (msg->req.request_id == request_id) { 145 list_del(&msg->list); 146 return msg; 147 } 148 } 149 150 return NULL; 151 } 152 153 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head) 154 { 155 struct vduse_dev_msg *msg = NULL; 156 157 if (!list_empty(head)) { 158 msg = list_first_entry(head, struct vduse_dev_msg, list); 159 list_del(&msg->list); 160 } 161 162 return msg; 163 } 164 165 static void vduse_enqueue_msg(struct list_head *head, 166 struct vduse_dev_msg *msg) 167 { 168 list_add_tail(&msg->list, head); 169 } 170 171 static void vduse_dev_broken(struct vduse_dev *dev) 172 { 173 struct vduse_dev_msg *msg, *tmp; 174 175 if (unlikely(dev->broken)) 176 return; 177 178 list_splice_init(&dev->recv_list, &dev->send_list); 179 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) { 180 list_del(&msg->list); 181 msg->completed = 1; 182 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 183 wake_up(&msg->waitq); 184 } 185 dev->broken = true; 186 wake_up(&dev->waitq); 187 } 188 189 static int vduse_dev_msg_sync(struct vduse_dev *dev, 190 struct vduse_dev_msg *msg) 191 { 192 int ret; 193 194 if (unlikely(dev->broken)) 195 return -EIO; 196 197 init_waitqueue_head(&msg->waitq); 198 spin_lock(&dev->msg_lock); 199 if (unlikely(dev->broken)) { 200 spin_unlock(&dev->msg_lock); 201 return -EIO; 202 } 203 msg->req.request_id = dev->msg_unique++; 204 vduse_enqueue_msg(&dev->send_list, msg); 205 wake_up(&dev->waitq); 206 spin_unlock(&dev->msg_lock); 207 if (dev->msg_timeout) 208 ret = wait_event_killable_timeout(msg->waitq, msg->completed, 209 (long)dev->msg_timeout * HZ); 210 else 211 ret = wait_event_killable(msg->waitq, msg->completed); 212 213 spin_lock(&dev->msg_lock); 214 if (!msg->completed) { 215 list_del(&msg->list); 216 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 217 /* Mark the device as malfunction when there is a timeout */ 218 if (!ret) 219 vduse_dev_broken(dev); 220 } 221 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO; 222 spin_unlock(&dev->msg_lock); 223 224 return ret; 225 } 226 227 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev, 228 struct vduse_virtqueue *vq, 229 struct vdpa_vq_state_packed *packed) 230 { 231 struct vduse_dev_msg msg = { 0 }; 232 int ret; 233 234 msg.req.type = VDUSE_GET_VQ_STATE; 235 msg.req.vq_state.index = vq->index; 236 237 ret = vduse_dev_msg_sync(dev, &msg); 238 if (ret) 239 return ret; 240 241 packed->last_avail_counter = 242 msg.resp.vq_state.packed.last_avail_counter & 0x0001; 243 packed->last_avail_idx = 244 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF; 245 packed->last_used_counter = 246 msg.resp.vq_state.packed.last_used_counter & 0x0001; 247 packed->last_used_idx = 248 msg.resp.vq_state.packed.last_used_idx & 0x7FFF; 249 250 return 0; 251 } 252 253 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev, 254 struct vduse_virtqueue *vq, 255 struct vdpa_vq_state_split *split) 256 { 257 struct vduse_dev_msg msg = { 0 }; 258 int ret; 259 260 msg.req.type = VDUSE_GET_VQ_STATE; 261 msg.req.vq_state.index = vq->index; 262 263 ret = vduse_dev_msg_sync(dev, &msg); 264 if (ret) 265 return ret; 266 267 split->avail_index = msg.resp.vq_state.split.avail_index; 268 269 return 0; 270 } 271 272 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status) 273 { 274 struct vduse_dev_msg msg = { 0 }; 275 276 msg.req.type = VDUSE_SET_STATUS; 277 msg.req.s.status = status; 278 279 return vduse_dev_msg_sync(dev, &msg); 280 } 281 282 static int vduse_dev_update_iotlb(struct vduse_dev *dev, 283 u64 start, u64 last) 284 { 285 struct vduse_dev_msg msg = { 0 }; 286 287 if (last < start) 288 return -EINVAL; 289 290 msg.req.type = VDUSE_UPDATE_IOTLB; 291 msg.req.iova.start = start; 292 msg.req.iova.last = last; 293 294 return vduse_dev_msg_sync(dev, &msg); 295 } 296 297 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) 298 { 299 struct file *file = iocb->ki_filp; 300 struct vduse_dev *dev = file->private_data; 301 struct vduse_dev_msg *msg; 302 int size = sizeof(struct vduse_dev_request); 303 ssize_t ret; 304 305 if (iov_iter_count(to) < size) 306 return -EINVAL; 307 308 spin_lock(&dev->msg_lock); 309 while (1) { 310 msg = vduse_dequeue_msg(&dev->send_list); 311 if (msg) 312 break; 313 314 ret = -EAGAIN; 315 if (file->f_flags & O_NONBLOCK) 316 goto unlock; 317 318 spin_unlock(&dev->msg_lock); 319 ret = wait_event_interruptible_exclusive(dev->waitq, 320 !list_empty(&dev->send_list)); 321 if (ret) 322 return ret; 323 324 spin_lock(&dev->msg_lock); 325 } 326 spin_unlock(&dev->msg_lock); 327 ret = copy_to_iter(&msg->req, size, to); 328 spin_lock(&dev->msg_lock); 329 if (ret != size) { 330 ret = -EFAULT; 331 vduse_enqueue_msg(&dev->send_list, msg); 332 goto unlock; 333 } 334 vduse_enqueue_msg(&dev->recv_list, msg); 335 unlock: 336 spin_unlock(&dev->msg_lock); 337 338 return ret; 339 } 340 341 static bool is_mem_zero(const char *ptr, int size) 342 { 343 int i; 344 345 for (i = 0; i < size; i++) { 346 if (ptr[i]) 347 return false; 348 } 349 return true; 350 } 351 352 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from) 353 { 354 struct file *file = iocb->ki_filp; 355 struct vduse_dev *dev = file->private_data; 356 struct vduse_dev_response resp; 357 struct vduse_dev_msg *msg; 358 size_t ret; 359 360 ret = copy_from_iter(&resp, sizeof(resp), from); 361 if (ret != sizeof(resp)) 362 return -EINVAL; 363 364 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved))) 365 return -EINVAL; 366 367 spin_lock(&dev->msg_lock); 368 msg = vduse_find_msg(&dev->recv_list, resp.request_id); 369 if (!msg) { 370 ret = -ENOENT; 371 goto unlock; 372 } 373 374 memcpy(&msg->resp, &resp, sizeof(resp)); 375 msg->completed = 1; 376 wake_up(&msg->waitq); 377 unlock: 378 spin_unlock(&dev->msg_lock); 379 380 return ret; 381 } 382 383 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait) 384 { 385 struct vduse_dev *dev = file->private_data; 386 __poll_t mask = 0; 387 388 poll_wait(file, &dev->waitq, wait); 389 390 spin_lock(&dev->msg_lock); 391 392 if (unlikely(dev->broken)) 393 mask |= EPOLLERR; 394 if (!list_empty(&dev->send_list)) 395 mask |= EPOLLIN | EPOLLRDNORM; 396 if (!list_empty(&dev->recv_list)) 397 mask |= EPOLLOUT | EPOLLWRNORM; 398 399 spin_unlock(&dev->msg_lock); 400 401 return mask; 402 } 403 404 static void vduse_dev_reset(struct vduse_dev *dev) 405 { 406 int i; 407 struct vduse_iova_domain *domain = dev->domain; 408 409 /* The coherent mappings are handled in vduse_dev_free_coherent() */ 410 if (domain->bounce_map) 411 vduse_domain_reset_bounce_map(domain); 412 413 dev->status = 0; 414 dev->driver_features = 0; 415 dev->generation++; 416 spin_lock(&dev->irq_lock); 417 dev->config_cb.callback = NULL; 418 dev->config_cb.private = NULL; 419 spin_unlock(&dev->irq_lock); 420 flush_work(&dev->inject); 421 422 for (i = 0; i < dev->vq_num; i++) { 423 struct vduse_virtqueue *vq = &dev->vqs[i]; 424 425 vq->ready = false; 426 vq->desc_addr = 0; 427 vq->driver_addr = 0; 428 vq->device_addr = 0; 429 vq->num = 0; 430 memset(&vq->state, 0, sizeof(vq->state)); 431 432 spin_lock(&vq->kick_lock); 433 vq->kicked = false; 434 if (vq->kickfd) 435 eventfd_ctx_put(vq->kickfd); 436 vq->kickfd = NULL; 437 spin_unlock(&vq->kick_lock); 438 439 spin_lock(&vq->irq_lock); 440 vq->cb.callback = NULL; 441 vq->cb.private = NULL; 442 spin_unlock(&vq->irq_lock); 443 flush_work(&vq->inject); 444 flush_work(&vq->kick); 445 } 446 } 447 448 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx, 449 u64 desc_area, u64 driver_area, 450 u64 device_area) 451 { 452 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 453 struct vduse_virtqueue *vq = &dev->vqs[idx]; 454 455 vq->desc_addr = desc_area; 456 vq->driver_addr = driver_area; 457 vq->device_addr = device_area; 458 459 return 0; 460 } 461 462 static void vduse_vq_kick(struct vduse_virtqueue *vq) 463 { 464 spin_lock(&vq->kick_lock); 465 if (!vq->ready) 466 goto unlock; 467 468 if (vq->kickfd) 469 eventfd_signal(vq->kickfd, 1); 470 else 471 vq->kicked = true; 472 unlock: 473 spin_unlock(&vq->kick_lock); 474 } 475 476 static void vduse_vq_kick_work(struct work_struct *work) 477 { 478 struct vduse_virtqueue *vq = container_of(work, 479 struct vduse_virtqueue, kick); 480 481 vduse_vq_kick(vq); 482 } 483 484 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx) 485 { 486 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 487 struct vduse_virtqueue *vq = &dev->vqs[idx]; 488 489 if (!eventfd_signal_allowed()) { 490 schedule_work(&vq->kick); 491 return; 492 } 493 vduse_vq_kick(vq); 494 } 495 496 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 497 struct vdpa_callback *cb) 498 { 499 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 500 struct vduse_virtqueue *vq = &dev->vqs[idx]; 501 502 spin_lock(&vq->irq_lock); 503 vq->cb.callback = cb->callback; 504 vq->cb.private = cb->private; 505 spin_unlock(&vq->irq_lock); 506 } 507 508 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 509 { 510 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 511 struct vduse_virtqueue *vq = &dev->vqs[idx]; 512 513 vq->num = num; 514 } 515 516 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa, 517 u16 idx, bool ready) 518 { 519 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 520 struct vduse_virtqueue *vq = &dev->vqs[idx]; 521 522 vq->ready = ready; 523 } 524 525 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 526 { 527 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 528 struct vduse_virtqueue *vq = &dev->vqs[idx]; 529 530 return vq->ready; 531 } 532 533 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx, 534 const struct vdpa_vq_state *state) 535 { 536 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 537 struct vduse_virtqueue *vq = &dev->vqs[idx]; 538 539 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 540 vq->state.packed.last_avail_counter = 541 state->packed.last_avail_counter; 542 vq->state.packed.last_avail_idx = state->packed.last_avail_idx; 543 vq->state.packed.last_used_counter = 544 state->packed.last_used_counter; 545 vq->state.packed.last_used_idx = state->packed.last_used_idx; 546 } else 547 vq->state.split.avail_index = state->split.avail_index; 548 549 return 0; 550 } 551 552 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx, 553 struct vdpa_vq_state *state) 554 { 555 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 556 struct vduse_virtqueue *vq = &dev->vqs[idx]; 557 558 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) 559 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed); 560 561 return vduse_dev_get_vq_state_split(dev, vq, &state->split); 562 } 563 564 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa) 565 { 566 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 567 568 return dev->vq_align; 569 } 570 571 static u64 vduse_vdpa_get_features(struct vdpa_device *vdpa) 572 { 573 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 574 575 return dev->device_features; 576 } 577 578 static int vduse_vdpa_set_features(struct vdpa_device *vdpa, u64 features) 579 { 580 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 581 582 dev->driver_features = features; 583 return 0; 584 } 585 586 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa, 587 struct vdpa_callback *cb) 588 { 589 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 590 591 spin_lock(&dev->irq_lock); 592 dev->config_cb.callback = cb->callback; 593 dev->config_cb.private = cb->private; 594 spin_unlock(&dev->irq_lock); 595 } 596 597 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa) 598 { 599 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 600 u16 num_max = 0; 601 int i; 602 603 for (i = 0; i < dev->vq_num; i++) 604 if (num_max < dev->vqs[i].num_max) 605 num_max = dev->vqs[i].num_max; 606 607 return num_max; 608 } 609 610 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa) 611 { 612 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 613 614 return dev->device_id; 615 } 616 617 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa) 618 { 619 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 620 621 return dev->vendor_id; 622 } 623 624 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa) 625 { 626 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 627 628 return dev->status; 629 } 630 631 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status) 632 { 633 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 634 635 if (vduse_dev_set_status(dev, status)) 636 return; 637 638 dev->status = status; 639 } 640 641 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa) 642 { 643 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 644 645 return dev->config_size; 646 } 647 648 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset, 649 void *buf, unsigned int len) 650 { 651 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 652 653 if (len > dev->config_size - offset) 654 return; 655 656 memcpy(buf, dev->config + offset, len); 657 } 658 659 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset, 660 const void *buf, unsigned int len) 661 { 662 /* Now we only support read-only configuration space */ 663 } 664 665 static int vduse_vdpa_reset(struct vdpa_device *vdpa) 666 { 667 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 668 669 if (vduse_dev_set_status(dev, 0)) 670 return -EIO; 671 672 vduse_dev_reset(dev); 673 674 return 0; 675 } 676 677 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa) 678 { 679 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 680 681 return dev->generation; 682 } 683 684 static int vduse_vdpa_set_map(struct vdpa_device *vdpa, 685 struct vhost_iotlb *iotlb) 686 { 687 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 688 int ret; 689 690 ret = vduse_domain_set_map(dev->domain, iotlb); 691 if (ret) 692 return ret; 693 694 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX); 695 if (ret) { 696 vduse_domain_clear_map(dev->domain, iotlb); 697 return ret; 698 } 699 700 return 0; 701 } 702 703 static void vduse_vdpa_free(struct vdpa_device *vdpa) 704 { 705 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 706 707 dev->vdev = NULL; 708 } 709 710 static const struct vdpa_config_ops vduse_vdpa_config_ops = { 711 .set_vq_address = vduse_vdpa_set_vq_address, 712 .kick_vq = vduse_vdpa_kick_vq, 713 .set_vq_cb = vduse_vdpa_set_vq_cb, 714 .set_vq_num = vduse_vdpa_set_vq_num, 715 .set_vq_ready = vduse_vdpa_set_vq_ready, 716 .get_vq_ready = vduse_vdpa_get_vq_ready, 717 .set_vq_state = vduse_vdpa_set_vq_state, 718 .get_vq_state = vduse_vdpa_get_vq_state, 719 .get_vq_align = vduse_vdpa_get_vq_align, 720 .get_features = vduse_vdpa_get_features, 721 .set_features = vduse_vdpa_set_features, 722 .set_config_cb = vduse_vdpa_set_config_cb, 723 .get_vq_num_max = vduse_vdpa_get_vq_num_max, 724 .get_device_id = vduse_vdpa_get_device_id, 725 .get_vendor_id = vduse_vdpa_get_vendor_id, 726 .get_status = vduse_vdpa_get_status, 727 .set_status = vduse_vdpa_set_status, 728 .get_config_size = vduse_vdpa_get_config_size, 729 .get_config = vduse_vdpa_get_config, 730 .set_config = vduse_vdpa_set_config, 731 .get_generation = vduse_vdpa_get_generation, 732 .reset = vduse_vdpa_reset, 733 .set_map = vduse_vdpa_set_map, 734 .free = vduse_vdpa_free, 735 }; 736 737 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, 738 unsigned long offset, size_t size, 739 enum dma_data_direction dir, 740 unsigned long attrs) 741 { 742 struct vduse_dev *vdev = dev_to_vduse(dev); 743 struct vduse_iova_domain *domain = vdev->domain; 744 745 return vduse_domain_map_page(domain, page, offset, size, dir, attrs); 746 } 747 748 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr, 749 size_t size, enum dma_data_direction dir, 750 unsigned long attrs) 751 { 752 struct vduse_dev *vdev = dev_to_vduse(dev); 753 struct vduse_iova_domain *domain = vdev->domain; 754 755 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs); 756 } 757 758 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size, 759 dma_addr_t *dma_addr, gfp_t flag, 760 unsigned long attrs) 761 { 762 struct vduse_dev *vdev = dev_to_vduse(dev); 763 struct vduse_iova_domain *domain = vdev->domain; 764 unsigned long iova; 765 void *addr; 766 767 *dma_addr = DMA_MAPPING_ERROR; 768 addr = vduse_domain_alloc_coherent(domain, size, 769 (dma_addr_t *)&iova, flag, attrs); 770 if (!addr) 771 return NULL; 772 773 *dma_addr = (dma_addr_t)iova; 774 775 return addr; 776 } 777 778 static void vduse_dev_free_coherent(struct device *dev, size_t size, 779 void *vaddr, dma_addr_t dma_addr, 780 unsigned long attrs) 781 { 782 struct vduse_dev *vdev = dev_to_vduse(dev); 783 struct vduse_iova_domain *domain = vdev->domain; 784 785 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs); 786 } 787 788 static size_t vduse_dev_max_mapping_size(struct device *dev) 789 { 790 struct vduse_dev *vdev = dev_to_vduse(dev); 791 struct vduse_iova_domain *domain = vdev->domain; 792 793 return domain->bounce_size; 794 } 795 796 static const struct dma_map_ops vduse_dev_dma_ops = { 797 .map_page = vduse_dev_map_page, 798 .unmap_page = vduse_dev_unmap_page, 799 .alloc = vduse_dev_alloc_coherent, 800 .free = vduse_dev_free_coherent, 801 .max_mapping_size = vduse_dev_max_mapping_size, 802 }; 803 804 static unsigned int perm_to_file_flags(u8 perm) 805 { 806 unsigned int flags = 0; 807 808 switch (perm) { 809 case VDUSE_ACCESS_WO: 810 flags |= O_WRONLY; 811 break; 812 case VDUSE_ACCESS_RO: 813 flags |= O_RDONLY; 814 break; 815 case VDUSE_ACCESS_RW: 816 flags |= O_RDWR; 817 break; 818 default: 819 WARN(1, "invalidate vhost IOTLB permission\n"); 820 break; 821 } 822 823 return flags; 824 } 825 826 static int vduse_kickfd_setup(struct vduse_dev *dev, 827 struct vduse_vq_eventfd *eventfd) 828 { 829 struct eventfd_ctx *ctx = NULL; 830 struct vduse_virtqueue *vq; 831 u32 index; 832 833 if (eventfd->index >= dev->vq_num) 834 return -EINVAL; 835 836 index = array_index_nospec(eventfd->index, dev->vq_num); 837 vq = &dev->vqs[index]; 838 if (eventfd->fd >= 0) { 839 ctx = eventfd_ctx_fdget(eventfd->fd); 840 if (IS_ERR(ctx)) 841 return PTR_ERR(ctx); 842 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN) 843 return 0; 844 845 spin_lock(&vq->kick_lock); 846 if (vq->kickfd) 847 eventfd_ctx_put(vq->kickfd); 848 vq->kickfd = ctx; 849 if (vq->ready && vq->kicked && vq->kickfd) { 850 eventfd_signal(vq->kickfd, 1); 851 vq->kicked = false; 852 } 853 spin_unlock(&vq->kick_lock); 854 855 return 0; 856 } 857 858 static bool vduse_dev_is_ready(struct vduse_dev *dev) 859 { 860 int i; 861 862 for (i = 0; i < dev->vq_num; i++) 863 if (!dev->vqs[i].num_max) 864 return false; 865 866 return true; 867 } 868 869 static void vduse_dev_irq_inject(struct work_struct *work) 870 { 871 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject); 872 873 spin_lock_irq(&dev->irq_lock); 874 if (dev->config_cb.callback) 875 dev->config_cb.callback(dev->config_cb.private); 876 spin_unlock_irq(&dev->irq_lock); 877 } 878 879 static void vduse_vq_irq_inject(struct work_struct *work) 880 { 881 struct vduse_virtqueue *vq = container_of(work, 882 struct vduse_virtqueue, inject); 883 884 spin_lock_irq(&vq->irq_lock); 885 if (vq->ready && vq->cb.callback) 886 vq->cb.callback(vq->cb.private); 887 spin_unlock_irq(&vq->irq_lock); 888 } 889 890 static long vduse_dev_ioctl(struct file *file, unsigned int cmd, 891 unsigned long arg) 892 { 893 struct vduse_dev *dev = file->private_data; 894 void __user *argp = (void __user *)arg; 895 int ret; 896 897 if (unlikely(dev->broken)) 898 return -EPERM; 899 900 switch (cmd) { 901 case VDUSE_IOTLB_GET_FD: { 902 struct vduse_iotlb_entry entry; 903 struct vhost_iotlb_map *map; 904 struct vdpa_map_file *map_file; 905 struct vduse_iova_domain *domain = dev->domain; 906 struct file *f = NULL; 907 908 ret = -EFAULT; 909 if (copy_from_user(&entry, argp, sizeof(entry))) 910 break; 911 912 ret = -EINVAL; 913 if (entry.start > entry.last) 914 break; 915 916 spin_lock(&domain->iotlb_lock); 917 map = vhost_iotlb_itree_first(domain->iotlb, 918 entry.start, entry.last); 919 if (map) { 920 map_file = (struct vdpa_map_file *)map->opaque; 921 f = get_file(map_file->file); 922 entry.offset = map_file->offset; 923 entry.start = map->start; 924 entry.last = map->last; 925 entry.perm = map->perm; 926 } 927 spin_unlock(&domain->iotlb_lock); 928 ret = -EINVAL; 929 if (!f) 930 break; 931 932 ret = -EFAULT; 933 if (copy_to_user(argp, &entry, sizeof(entry))) { 934 fput(f); 935 break; 936 } 937 ret = receive_fd(f, perm_to_file_flags(entry.perm)); 938 fput(f); 939 break; 940 } 941 case VDUSE_DEV_GET_FEATURES: 942 /* 943 * Just mirror what driver wrote here. 944 * The driver is expected to check FEATURE_OK later. 945 */ 946 ret = put_user(dev->driver_features, (u64 __user *)argp); 947 break; 948 case VDUSE_DEV_SET_CONFIG: { 949 struct vduse_config_data config; 950 unsigned long size = offsetof(struct vduse_config_data, 951 buffer); 952 953 ret = -EFAULT; 954 if (copy_from_user(&config, argp, size)) 955 break; 956 957 ret = -EINVAL; 958 if (config.length == 0 || 959 config.length > dev->config_size - config.offset) 960 break; 961 962 ret = -EFAULT; 963 if (copy_from_user(dev->config + config.offset, argp + size, 964 config.length)) 965 break; 966 967 ret = 0; 968 break; 969 } 970 case VDUSE_DEV_INJECT_CONFIG_IRQ: 971 ret = 0; 972 queue_work(vduse_irq_wq, &dev->inject); 973 break; 974 case VDUSE_VQ_SETUP: { 975 struct vduse_vq_config config; 976 u32 index; 977 978 ret = -EFAULT; 979 if (copy_from_user(&config, argp, sizeof(config))) 980 break; 981 982 ret = -EINVAL; 983 if (config.index >= dev->vq_num) 984 break; 985 986 if (!is_mem_zero((const char *)config.reserved, 987 sizeof(config.reserved))) 988 break; 989 990 index = array_index_nospec(config.index, dev->vq_num); 991 dev->vqs[index].num_max = config.max_size; 992 ret = 0; 993 break; 994 } 995 case VDUSE_VQ_GET_INFO: { 996 struct vduse_vq_info vq_info; 997 struct vduse_virtqueue *vq; 998 u32 index; 999 1000 ret = -EFAULT; 1001 if (copy_from_user(&vq_info, argp, sizeof(vq_info))) 1002 break; 1003 1004 ret = -EINVAL; 1005 if (vq_info.index >= dev->vq_num) 1006 break; 1007 1008 index = array_index_nospec(vq_info.index, dev->vq_num); 1009 vq = &dev->vqs[index]; 1010 vq_info.desc_addr = vq->desc_addr; 1011 vq_info.driver_addr = vq->driver_addr; 1012 vq_info.device_addr = vq->device_addr; 1013 vq_info.num = vq->num; 1014 1015 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 1016 vq_info.packed.last_avail_counter = 1017 vq->state.packed.last_avail_counter; 1018 vq_info.packed.last_avail_idx = 1019 vq->state.packed.last_avail_idx; 1020 vq_info.packed.last_used_counter = 1021 vq->state.packed.last_used_counter; 1022 vq_info.packed.last_used_idx = 1023 vq->state.packed.last_used_idx; 1024 } else 1025 vq_info.split.avail_index = 1026 vq->state.split.avail_index; 1027 1028 vq_info.ready = vq->ready; 1029 1030 ret = -EFAULT; 1031 if (copy_to_user(argp, &vq_info, sizeof(vq_info))) 1032 break; 1033 1034 ret = 0; 1035 break; 1036 } 1037 case VDUSE_VQ_SETUP_KICKFD: { 1038 struct vduse_vq_eventfd eventfd; 1039 1040 ret = -EFAULT; 1041 if (copy_from_user(&eventfd, argp, sizeof(eventfd))) 1042 break; 1043 1044 ret = vduse_kickfd_setup(dev, &eventfd); 1045 break; 1046 } 1047 case VDUSE_VQ_INJECT_IRQ: { 1048 u32 index; 1049 1050 ret = -EFAULT; 1051 if (get_user(index, (u32 __user *)argp)) 1052 break; 1053 1054 ret = -EINVAL; 1055 if (index >= dev->vq_num) 1056 break; 1057 1058 ret = 0; 1059 index = array_index_nospec(index, dev->vq_num); 1060 queue_work(vduse_irq_wq, &dev->vqs[index].inject); 1061 break; 1062 } 1063 default: 1064 ret = -ENOIOCTLCMD; 1065 break; 1066 } 1067 1068 return ret; 1069 } 1070 1071 static int vduse_dev_release(struct inode *inode, struct file *file) 1072 { 1073 struct vduse_dev *dev = file->private_data; 1074 1075 spin_lock(&dev->msg_lock); 1076 /* Make sure the inflight messages can processed after reconncection */ 1077 list_splice_init(&dev->recv_list, &dev->send_list); 1078 spin_unlock(&dev->msg_lock); 1079 dev->connected = false; 1080 1081 return 0; 1082 } 1083 1084 static struct vduse_dev *vduse_dev_get_from_minor(int minor) 1085 { 1086 struct vduse_dev *dev; 1087 1088 mutex_lock(&vduse_lock); 1089 dev = idr_find(&vduse_idr, minor); 1090 mutex_unlock(&vduse_lock); 1091 1092 return dev; 1093 } 1094 1095 static int vduse_dev_open(struct inode *inode, struct file *file) 1096 { 1097 int ret; 1098 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode)); 1099 1100 if (!dev) 1101 return -ENODEV; 1102 1103 ret = -EBUSY; 1104 mutex_lock(&dev->lock); 1105 if (dev->connected) 1106 goto unlock; 1107 1108 ret = 0; 1109 dev->connected = true; 1110 file->private_data = dev; 1111 unlock: 1112 mutex_unlock(&dev->lock); 1113 1114 return ret; 1115 } 1116 1117 static const struct file_operations vduse_dev_fops = { 1118 .owner = THIS_MODULE, 1119 .open = vduse_dev_open, 1120 .release = vduse_dev_release, 1121 .read_iter = vduse_dev_read_iter, 1122 .write_iter = vduse_dev_write_iter, 1123 .poll = vduse_dev_poll, 1124 .unlocked_ioctl = vduse_dev_ioctl, 1125 .compat_ioctl = compat_ptr_ioctl, 1126 .llseek = noop_llseek, 1127 }; 1128 1129 static struct vduse_dev *vduse_dev_create(void) 1130 { 1131 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1132 1133 if (!dev) 1134 return NULL; 1135 1136 mutex_init(&dev->lock); 1137 spin_lock_init(&dev->msg_lock); 1138 INIT_LIST_HEAD(&dev->send_list); 1139 INIT_LIST_HEAD(&dev->recv_list); 1140 spin_lock_init(&dev->irq_lock); 1141 1142 INIT_WORK(&dev->inject, vduse_dev_irq_inject); 1143 init_waitqueue_head(&dev->waitq); 1144 1145 return dev; 1146 } 1147 1148 static void vduse_dev_destroy(struct vduse_dev *dev) 1149 { 1150 kfree(dev); 1151 } 1152 1153 static struct vduse_dev *vduse_find_dev(const char *name) 1154 { 1155 struct vduse_dev *dev; 1156 int id; 1157 1158 idr_for_each_entry(&vduse_idr, dev, id) 1159 if (!strcmp(dev->name, name)) 1160 return dev; 1161 1162 return NULL; 1163 } 1164 1165 static int vduse_destroy_dev(char *name) 1166 { 1167 struct vduse_dev *dev = vduse_find_dev(name); 1168 1169 if (!dev) 1170 return -EINVAL; 1171 1172 mutex_lock(&dev->lock); 1173 if (dev->vdev || dev->connected) { 1174 mutex_unlock(&dev->lock); 1175 return -EBUSY; 1176 } 1177 dev->connected = true; 1178 mutex_unlock(&dev->lock); 1179 1180 vduse_dev_reset(dev); 1181 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 1182 idr_remove(&vduse_idr, dev->minor); 1183 kvfree(dev->config); 1184 kfree(dev->vqs); 1185 vduse_domain_destroy(dev->domain); 1186 kfree(dev->name); 1187 vduse_dev_destroy(dev); 1188 module_put(THIS_MODULE); 1189 1190 return 0; 1191 } 1192 1193 static bool device_is_allowed(u32 device_id) 1194 { 1195 int i; 1196 1197 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++) 1198 if (allowed_device_id[i] == device_id) 1199 return true; 1200 1201 return false; 1202 } 1203 1204 static bool features_is_valid(u64 features) 1205 { 1206 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) 1207 return false; 1208 1209 /* Now we only support read-only configuration space */ 1210 if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE)) 1211 return false; 1212 1213 return true; 1214 } 1215 1216 static bool vduse_validate_config(struct vduse_dev_config *config) 1217 { 1218 if (!is_mem_zero((const char *)config->reserved, 1219 sizeof(config->reserved))) 1220 return false; 1221 1222 if (config->vq_align > PAGE_SIZE) 1223 return false; 1224 1225 if (config->config_size > PAGE_SIZE) 1226 return false; 1227 1228 if (!device_is_allowed(config->device_id)) 1229 return false; 1230 1231 if (!features_is_valid(config->features)) 1232 return false; 1233 1234 return true; 1235 } 1236 1237 static ssize_t msg_timeout_show(struct device *device, 1238 struct device_attribute *attr, char *buf) 1239 { 1240 struct vduse_dev *dev = dev_get_drvdata(device); 1241 1242 return sysfs_emit(buf, "%u\n", dev->msg_timeout); 1243 } 1244 1245 static ssize_t msg_timeout_store(struct device *device, 1246 struct device_attribute *attr, 1247 const char *buf, size_t count) 1248 { 1249 struct vduse_dev *dev = dev_get_drvdata(device); 1250 int ret; 1251 1252 ret = kstrtouint(buf, 10, &dev->msg_timeout); 1253 if (ret < 0) 1254 return ret; 1255 1256 return count; 1257 } 1258 1259 static DEVICE_ATTR_RW(msg_timeout); 1260 1261 static struct attribute *vduse_dev_attrs[] = { 1262 &dev_attr_msg_timeout.attr, 1263 NULL 1264 }; 1265 1266 ATTRIBUTE_GROUPS(vduse_dev); 1267 1268 static int vduse_create_dev(struct vduse_dev_config *config, 1269 void *config_buf, u64 api_version) 1270 { 1271 int i, ret; 1272 struct vduse_dev *dev; 1273 1274 ret = -EEXIST; 1275 if (vduse_find_dev(config->name)) 1276 goto err; 1277 1278 ret = -ENOMEM; 1279 dev = vduse_dev_create(); 1280 if (!dev) 1281 goto err; 1282 1283 dev->api_version = api_version; 1284 dev->device_features = config->features; 1285 dev->device_id = config->device_id; 1286 dev->vendor_id = config->vendor_id; 1287 dev->name = kstrdup(config->name, GFP_KERNEL); 1288 if (!dev->name) 1289 goto err_str; 1290 1291 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1, 1292 VDUSE_BOUNCE_SIZE); 1293 if (!dev->domain) 1294 goto err_domain; 1295 1296 dev->config = config_buf; 1297 dev->config_size = config->config_size; 1298 dev->vq_align = config->vq_align; 1299 dev->vq_num = config->vq_num; 1300 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL); 1301 if (!dev->vqs) 1302 goto err_vqs; 1303 1304 for (i = 0; i < dev->vq_num; i++) { 1305 dev->vqs[i].index = i; 1306 INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject); 1307 INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work); 1308 spin_lock_init(&dev->vqs[i].kick_lock); 1309 spin_lock_init(&dev->vqs[i].irq_lock); 1310 } 1311 1312 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL); 1313 if (ret < 0) 1314 goto err_idr; 1315 1316 dev->minor = ret; 1317 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT; 1318 dev->dev = device_create(vduse_class, NULL, 1319 MKDEV(MAJOR(vduse_major), dev->minor), 1320 dev, "%s", config->name); 1321 if (IS_ERR(dev->dev)) { 1322 ret = PTR_ERR(dev->dev); 1323 goto err_dev; 1324 } 1325 __module_get(THIS_MODULE); 1326 1327 return 0; 1328 err_dev: 1329 idr_remove(&vduse_idr, dev->minor); 1330 err_idr: 1331 kfree(dev->vqs); 1332 err_vqs: 1333 vduse_domain_destroy(dev->domain); 1334 err_domain: 1335 kfree(dev->name); 1336 err_str: 1337 vduse_dev_destroy(dev); 1338 err: 1339 kvfree(config_buf); 1340 return ret; 1341 } 1342 1343 static long vduse_ioctl(struct file *file, unsigned int cmd, 1344 unsigned long arg) 1345 { 1346 int ret; 1347 void __user *argp = (void __user *)arg; 1348 struct vduse_control *control = file->private_data; 1349 1350 mutex_lock(&vduse_lock); 1351 switch (cmd) { 1352 case VDUSE_GET_API_VERSION: 1353 ret = put_user(control->api_version, (u64 __user *)argp); 1354 break; 1355 case VDUSE_SET_API_VERSION: { 1356 u64 api_version; 1357 1358 ret = -EFAULT; 1359 if (get_user(api_version, (u64 __user *)argp)) 1360 break; 1361 1362 ret = -EINVAL; 1363 if (api_version > VDUSE_API_VERSION) 1364 break; 1365 1366 ret = 0; 1367 control->api_version = api_version; 1368 break; 1369 } 1370 case VDUSE_CREATE_DEV: { 1371 struct vduse_dev_config config; 1372 unsigned long size = offsetof(struct vduse_dev_config, config); 1373 void *buf; 1374 1375 ret = -EFAULT; 1376 if (copy_from_user(&config, argp, size)) 1377 break; 1378 1379 ret = -EINVAL; 1380 if (vduse_validate_config(&config) == false) 1381 break; 1382 1383 buf = vmemdup_user(argp + size, config.config_size); 1384 if (IS_ERR(buf)) { 1385 ret = PTR_ERR(buf); 1386 break; 1387 } 1388 config.name[VDUSE_NAME_MAX - 1] = '\0'; 1389 ret = vduse_create_dev(&config, buf, control->api_version); 1390 break; 1391 } 1392 case VDUSE_DESTROY_DEV: { 1393 char name[VDUSE_NAME_MAX]; 1394 1395 ret = -EFAULT; 1396 if (copy_from_user(name, argp, VDUSE_NAME_MAX)) 1397 break; 1398 1399 name[VDUSE_NAME_MAX - 1] = '\0'; 1400 ret = vduse_destroy_dev(name); 1401 break; 1402 } 1403 default: 1404 ret = -EINVAL; 1405 break; 1406 } 1407 mutex_unlock(&vduse_lock); 1408 1409 return ret; 1410 } 1411 1412 static int vduse_release(struct inode *inode, struct file *file) 1413 { 1414 struct vduse_control *control = file->private_data; 1415 1416 kfree(control); 1417 return 0; 1418 } 1419 1420 static int vduse_open(struct inode *inode, struct file *file) 1421 { 1422 struct vduse_control *control; 1423 1424 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL); 1425 if (!control) 1426 return -ENOMEM; 1427 1428 control->api_version = VDUSE_API_VERSION; 1429 file->private_data = control; 1430 1431 return 0; 1432 } 1433 1434 static const struct file_operations vduse_ctrl_fops = { 1435 .owner = THIS_MODULE, 1436 .open = vduse_open, 1437 .release = vduse_release, 1438 .unlocked_ioctl = vduse_ioctl, 1439 .compat_ioctl = compat_ptr_ioctl, 1440 .llseek = noop_llseek, 1441 }; 1442 1443 static char *vduse_devnode(struct device *dev, umode_t *mode) 1444 { 1445 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev)); 1446 } 1447 1448 static void vduse_mgmtdev_release(struct device *dev) 1449 { 1450 } 1451 1452 static struct device vduse_mgmtdev = { 1453 .init_name = "vduse", 1454 .release = vduse_mgmtdev_release, 1455 }; 1456 1457 static struct vdpa_mgmt_dev mgmt_dev; 1458 1459 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) 1460 { 1461 struct vduse_vdpa *vdev; 1462 int ret; 1463 1464 if (dev->vdev) 1465 return -EEXIST; 1466 1467 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev, 1468 &vduse_vdpa_config_ops, name, true); 1469 if (IS_ERR(vdev)) 1470 return PTR_ERR(vdev); 1471 1472 dev->vdev = vdev; 1473 vdev->dev = dev; 1474 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask; 1475 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64)); 1476 if (ret) { 1477 put_device(&vdev->vdpa.dev); 1478 return ret; 1479 } 1480 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops); 1481 vdev->vdpa.dma_dev = &vdev->vdpa.dev; 1482 vdev->vdpa.mdev = &mgmt_dev; 1483 1484 return 0; 1485 } 1486 1487 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name) 1488 { 1489 struct vduse_dev *dev; 1490 int ret; 1491 1492 mutex_lock(&vduse_lock); 1493 dev = vduse_find_dev(name); 1494 if (!dev || !vduse_dev_is_ready(dev)) { 1495 mutex_unlock(&vduse_lock); 1496 return -EINVAL; 1497 } 1498 ret = vduse_dev_init_vdpa(dev, name); 1499 mutex_unlock(&vduse_lock); 1500 if (ret) 1501 return ret; 1502 1503 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); 1504 if (ret) { 1505 put_device(&dev->vdev->vdpa.dev); 1506 return ret; 1507 } 1508 1509 return 0; 1510 } 1511 1512 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) 1513 { 1514 _vdpa_unregister_device(dev); 1515 } 1516 1517 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = { 1518 .dev_add = vdpa_dev_add, 1519 .dev_del = vdpa_dev_del, 1520 }; 1521 1522 static struct virtio_device_id id_table[] = { 1523 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 1524 { 0 }, 1525 }; 1526 1527 static struct vdpa_mgmt_dev mgmt_dev = { 1528 .device = &vduse_mgmtdev, 1529 .id_table = id_table, 1530 .ops = &vdpa_dev_mgmtdev_ops, 1531 }; 1532 1533 static int vduse_mgmtdev_init(void) 1534 { 1535 int ret; 1536 1537 ret = device_register(&vduse_mgmtdev); 1538 if (ret) 1539 return ret; 1540 1541 ret = vdpa_mgmtdev_register(&mgmt_dev); 1542 if (ret) 1543 goto err; 1544 1545 return 0; 1546 err: 1547 device_unregister(&vduse_mgmtdev); 1548 return ret; 1549 } 1550 1551 static void vduse_mgmtdev_exit(void) 1552 { 1553 vdpa_mgmtdev_unregister(&mgmt_dev); 1554 device_unregister(&vduse_mgmtdev); 1555 } 1556 1557 static int vduse_init(void) 1558 { 1559 int ret; 1560 struct device *dev; 1561 1562 vduse_class = class_create(THIS_MODULE, "vduse"); 1563 if (IS_ERR(vduse_class)) 1564 return PTR_ERR(vduse_class); 1565 1566 vduse_class->devnode = vduse_devnode; 1567 vduse_class->dev_groups = vduse_dev_groups; 1568 1569 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse"); 1570 if (ret) 1571 goto err_chardev_region; 1572 1573 /* /dev/vduse/control */ 1574 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops); 1575 vduse_ctrl_cdev.owner = THIS_MODULE; 1576 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1); 1577 if (ret) 1578 goto err_ctrl_cdev; 1579 1580 dev = device_create(vduse_class, NULL, vduse_major, NULL, "control"); 1581 if (IS_ERR(dev)) { 1582 ret = PTR_ERR(dev); 1583 goto err_device; 1584 } 1585 1586 /* /dev/vduse/$DEVICE */ 1587 cdev_init(&vduse_cdev, &vduse_dev_fops); 1588 vduse_cdev.owner = THIS_MODULE; 1589 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1), 1590 VDUSE_DEV_MAX - 1); 1591 if (ret) 1592 goto err_cdev; 1593 1594 vduse_irq_wq = alloc_workqueue("vduse-irq", 1595 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0); 1596 if (!vduse_irq_wq) 1597 goto err_wq; 1598 1599 ret = vduse_domain_init(); 1600 if (ret) 1601 goto err_domain; 1602 1603 ret = vduse_mgmtdev_init(); 1604 if (ret) 1605 goto err_mgmtdev; 1606 1607 return 0; 1608 err_mgmtdev: 1609 vduse_domain_exit(); 1610 err_domain: 1611 destroy_workqueue(vduse_irq_wq); 1612 err_wq: 1613 cdev_del(&vduse_cdev); 1614 err_cdev: 1615 device_destroy(vduse_class, vduse_major); 1616 err_device: 1617 cdev_del(&vduse_ctrl_cdev); 1618 err_ctrl_cdev: 1619 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 1620 err_chardev_region: 1621 class_destroy(vduse_class); 1622 return ret; 1623 } 1624 module_init(vduse_init); 1625 1626 static void vduse_exit(void) 1627 { 1628 vduse_mgmtdev_exit(); 1629 vduse_domain_exit(); 1630 destroy_workqueue(vduse_irq_wq); 1631 cdev_del(&vduse_cdev); 1632 device_destroy(vduse_class, vduse_major); 1633 cdev_del(&vduse_ctrl_cdev); 1634 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 1635 class_destroy(vduse_class); 1636 } 1637 module_exit(vduse_exit); 1638 1639 MODULE_LICENSE(DRV_LICENSE); 1640 MODULE_AUTHOR(DRV_AUTHOR); 1641 MODULE_DESCRIPTION(DRV_DESC); 1642