1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDUSE: vDPA Device in Userspace 4 * 5 * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved. 6 * 7 * Author: Xie Yongji <xieyongji@bytedance.com> 8 * 9 */ 10 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/cdev.h> 14 #include <linux/device.h> 15 #include <linux/eventfd.h> 16 #include <linux/slab.h> 17 #include <linux/wait.h> 18 #include <linux/dma-map-ops.h> 19 #include <linux/poll.h> 20 #include <linux/file.h> 21 #include <linux/uio.h> 22 #include <linux/vdpa.h> 23 #include <linux/nospec.h> 24 #include <uapi/linux/vduse.h> 25 #include <uapi/linux/vdpa.h> 26 #include <uapi/linux/virtio_config.h> 27 #include <uapi/linux/virtio_ids.h> 28 #include <uapi/linux/virtio_blk.h> 29 #include <linux/mod_devicetable.h> 30 31 #include "iova_domain.h" 32 33 #define DRV_AUTHOR "Yongji Xie <xieyongji@bytedance.com>" 34 #define DRV_DESC "vDPA Device in Userspace" 35 #define DRV_LICENSE "GPL v2" 36 37 #define VDUSE_DEV_MAX (1U << MINORBITS) 38 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024) 39 #define VDUSE_IOVA_SIZE (128 * 1024 * 1024) 40 #define VDUSE_MSG_DEFAULT_TIMEOUT 30 41 42 struct vduse_virtqueue { 43 u16 index; 44 u16 num_max; 45 u32 num; 46 u64 desc_addr; 47 u64 driver_addr; 48 u64 device_addr; 49 struct vdpa_vq_state state; 50 bool ready; 51 bool kicked; 52 spinlock_t kick_lock; 53 spinlock_t irq_lock; 54 struct eventfd_ctx *kickfd; 55 struct vdpa_callback cb; 56 struct work_struct inject; 57 struct work_struct kick; 58 }; 59 60 struct vduse_dev; 61 62 struct vduse_vdpa { 63 struct vdpa_device vdpa; 64 struct vduse_dev *dev; 65 }; 66 67 struct vduse_dev { 68 struct vduse_vdpa *vdev; 69 struct device *dev; 70 struct vduse_virtqueue *vqs; 71 struct vduse_iova_domain *domain; 72 char *name; 73 struct mutex lock; 74 spinlock_t msg_lock; 75 u64 msg_unique; 76 u32 msg_timeout; 77 wait_queue_head_t waitq; 78 struct list_head send_list; 79 struct list_head recv_list; 80 struct vdpa_callback config_cb; 81 struct work_struct inject; 82 spinlock_t irq_lock; 83 struct rw_semaphore rwsem; 84 int minor; 85 bool broken; 86 bool connected; 87 u64 api_version; 88 u64 device_features; 89 u64 driver_features; 90 u32 device_id; 91 u32 vendor_id; 92 u32 generation; 93 u32 config_size; 94 void *config; 95 u8 status; 96 u32 vq_num; 97 u32 vq_align; 98 }; 99 100 struct vduse_dev_msg { 101 struct vduse_dev_request req; 102 struct vduse_dev_response resp; 103 struct list_head list; 104 wait_queue_head_t waitq; 105 bool completed; 106 }; 107 108 struct vduse_control { 109 u64 api_version; 110 }; 111 112 static DEFINE_MUTEX(vduse_lock); 113 static DEFINE_IDR(vduse_idr); 114 115 static dev_t vduse_major; 116 static struct class *vduse_class; 117 static struct cdev vduse_ctrl_cdev; 118 static struct cdev vduse_cdev; 119 static struct workqueue_struct *vduse_irq_wq; 120 121 static u32 allowed_device_id[] = { 122 VIRTIO_ID_BLOCK, 123 }; 124 125 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa) 126 { 127 struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa); 128 129 return vdev->dev; 130 } 131 132 static inline struct vduse_dev *dev_to_vduse(struct device *dev) 133 { 134 struct vdpa_device *vdpa = dev_to_vdpa(dev); 135 136 return vdpa_to_vduse(vdpa); 137 } 138 139 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head, 140 uint32_t request_id) 141 { 142 struct vduse_dev_msg *msg; 143 144 list_for_each_entry(msg, head, list) { 145 if (msg->req.request_id == request_id) { 146 list_del(&msg->list); 147 return msg; 148 } 149 } 150 151 return NULL; 152 } 153 154 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head) 155 { 156 struct vduse_dev_msg *msg = NULL; 157 158 if (!list_empty(head)) { 159 msg = list_first_entry(head, struct vduse_dev_msg, list); 160 list_del(&msg->list); 161 } 162 163 return msg; 164 } 165 166 static void vduse_enqueue_msg(struct list_head *head, 167 struct vduse_dev_msg *msg) 168 { 169 list_add_tail(&msg->list, head); 170 } 171 172 static void vduse_dev_broken(struct vduse_dev *dev) 173 { 174 struct vduse_dev_msg *msg, *tmp; 175 176 if (unlikely(dev->broken)) 177 return; 178 179 list_splice_init(&dev->recv_list, &dev->send_list); 180 list_for_each_entry_safe(msg, tmp, &dev->send_list, list) { 181 list_del(&msg->list); 182 msg->completed = 1; 183 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 184 wake_up(&msg->waitq); 185 } 186 dev->broken = true; 187 wake_up(&dev->waitq); 188 } 189 190 static int vduse_dev_msg_sync(struct vduse_dev *dev, 191 struct vduse_dev_msg *msg) 192 { 193 int ret; 194 195 if (unlikely(dev->broken)) 196 return -EIO; 197 198 init_waitqueue_head(&msg->waitq); 199 spin_lock(&dev->msg_lock); 200 if (unlikely(dev->broken)) { 201 spin_unlock(&dev->msg_lock); 202 return -EIO; 203 } 204 msg->req.request_id = dev->msg_unique++; 205 vduse_enqueue_msg(&dev->send_list, msg); 206 wake_up(&dev->waitq); 207 spin_unlock(&dev->msg_lock); 208 if (dev->msg_timeout) 209 ret = wait_event_killable_timeout(msg->waitq, msg->completed, 210 (long)dev->msg_timeout * HZ); 211 else 212 ret = wait_event_killable(msg->waitq, msg->completed); 213 214 spin_lock(&dev->msg_lock); 215 if (!msg->completed) { 216 list_del(&msg->list); 217 msg->resp.result = VDUSE_REQ_RESULT_FAILED; 218 /* Mark the device as malfunction when there is a timeout */ 219 if (!ret) 220 vduse_dev_broken(dev); 221 } 222 ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO; 223 spin_unlock(&dev->msg_lock); 224 225 return ret; 226 } 227 228 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev, 229 struct vduse_virtqueue *vq, 230 struct vdpa_vq_state_packed *packed) 231 { 232 struct vduse_dev_msg msg = { 0 }; 233 int ret; 234 235 msg.req.type = VDUSE_GET_VQ_STATE; 236 msg.req.vq_state.index = vq->index; 237 238 ret = vduse_dev_msg_sync(dev, &msg); 239 if (ret) 240 return ret; 241 242 packed->last_avail_counter = 243 msg.resp.vq_state.packed.last_avail_counter & 0x0001; 244 packed->last_avail_idx = 245 msg.resp.vq_state.packed.last_avail_idx & 0x7FFF; 246 packed->last_used_counter = 247 msg.resp.vq_state.packed.last_used_counter & 0x0001; 248 packed->last_used_idx = 249 msg.resp.vq_state.packed.last_used_idx & 0x7FFF; 250 251 return 0; 252 } 253 254 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev, 255 struct vduse_virtqueue *vq, 256 struct vdpa_vq_state_split *split) 257 { 258 struct vduse_dev_msg msg = { 0 }; 259 int ret; 260 261 msg.req.type = VDUSE_GET_VQ_STATE; 262 msg.req.vq_state.index = vq->index; 263 264 ret = vduse_dev_msg_sync(dev, &msg); 265 if (ret) 266 return ret; 267 268 split->avail_index = msg.resp.vq_state.split.avail_index; 269 270 return 0; 271 } 272 273 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status) 274 { 275 struct vduse_dev_msg msg = { 0 }; 276 277 msg.req.type = VDUSE_SET_STATUS; 278 msg.req.s.status = status; 279 280 return vduse_dev_msg_sync(dev, &msg); 281 } 282 283 static int vduse_dev_update_iotlb(struct vduse_dev *dev, 284 u64 start, u64 last) 285 { 286 struct vduse_dev_msg msg = { 0 }; 287 288 if (last < start) 289 return -EINVAL; 290 291 msg.req.type = VDUSE_UPDATE_IOTLB; 292 msg.req.iova.start = start; 293 msg.req.iova.last = last; 294 295 return vduse_dev_msg_sync(dev, &msg); 296 } 297 298 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to) 299 { 300 struct file *file = iocb->ki_filp; 301 struct vduse_dev *dev = file->private_data; 302 struct vduse_dev_msg *msg; 303 int size = sizeof(struct vduse_dev_request); 304 ssize_t ret; 305 306 if (iov_iter_count(to) < size) 307 return -EINVAL; 308 309 spin_lock(&dev->msg_lock); 310 while (1) { 311 msg = vduse_dequeue_msg(&dev->send_list); 312 if (msg) 313 break; 314 315 ret = -EAGAIN; 316 if (file->f_flags & O_NONBLOCK) 317 goto unlock; 318 319 spin_unlock(&dev->msg_lock); 320 ret = wait_event_interruptible_exclusive(dev->waitq, 321 !list_empty(&dev->send_list)); 322 if (ret) 323 return ret; 324 325 spin_lock(&dev->msg_lock); 326 } 327 spin_unlock(&dev->msg_lock); 328 ret = copy_to_iter(&msg->req, size, to); 329 spin_lock(&dev->msg_lock); 330 if (ret != size) { 331 ret = -EFAULT; 332 vduse_enqueue_msg(&dev->send_list, msg); 333 goto unlock; 334 } 335 vduse_enqueue_msg(&dev->recv_list, msg); 336 unlock: 337 spin_unlock(&dev->msg_lock); 338 339 return ret; 340 } 341 342 static bool is_mem_zero(const char *ptr, int size) 343 { 344 int i; 345 346 for (i = 0; i < size; i++) { 347 if (ptr[i]) 348 return false; 349 } 350 return true; 351 } 352 353 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from) 354 { 355 struct file *file = iocb->ki_filp; 356 struct vduse_dev *dev = file->private_data; 357 struct vduse_dev_response resp; 358 struct vduse_dev_msg *msg; 359 size_t ret; 360 361 ret = copy_from_iter(&resp, sizeof(resp), from); 362 if (ret != sizeof(resp)) 363 return -EINVAL; 364 365 if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved))) 366 return -EINVAL; 367 368 spin_lock(&dev->msg_lock); 369 msg = vduse_find_msg(&dev->recv_list, resp.request_id); 370 if (!msg) { 371 ret = -ENOENT; 372 goto unlock; 373 } 374 375 memcpy(&msg->resp, &resp, sizeof(resp)); 376 msg->completed = 1; 377 wake_up(&msg->waitq); 378 unlock: 379 spin_unlock(&dev->msg_lock); 380 381 return ret; 382 } 383 384 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait) 385 { 386 struct vduse_dev *dev = file->private_data; 387 __poll_t mask = 0; 388 389 poll_wait(file, &dev->waitq, wait); 390 391 spin_lock(&dev->msg_lock); 392 393 if (unlikely(dev->broken)) 394 mask |= EPOLLERR; 395 if (!list_empty(&dev->send_list)) 396 mask |= EPOLLIN | EPOLLRDNORM; 397 if (!list_empty(&dev->recv_list)) 398 mask |= EPOLLOUT | EPOLLWRNORM; 399 400 spin_unlock(&dev->msg_lock); 401 402 return mask; 403 } 404 405 static void vduse_dev_reset(struct vduse_dev *dev) 406 { 407 int i; 408 struct vduse_iova_domain *domain = dev->domain; 409 410 /* The coherent mappings are handled in vduse_dev_free_coherent() */ 411 if (domain->bounce_map) 412 vduse_domain_reset_bounce_map(domain); 413 414 down_write(&dev->rwsem); 415 416 dev->status = 0; 417 dev->driver_features = 0; 418 dev->generation++; 419 spin_lock(&dev->irq_lock); 420 dev->config_cb.callback = NULL; 421 dev->config_cb.private = NULL; 422 spin_unlock(&dev->irq_lock); 423 flush_work(&dev->inject); 424 425 for (i = 0; i < dev->vq_num; i++) { 426 struct vduse_virtqueue *vq = &dev->vqs[i]; 427 428 vq->ready = false; 429 vq->desc_addr = 0; 430 vq->driver_addr = 0; 431 vq->device_addr = 0; 432 vq->num = 0; 433 memset(&vq->state, 0, sizeof(vq->state)); 434 435 spin_lock(&vq->kick_lock); 436 vq->kicked = false; 437 if (vq->kickfd) 438 eventfd_ctx_put(vq->kickfd); 439 vq->kickfd = NULL; 440 spin_unlock(&vq->kick_lock); 441 442 spin_lock(&vq->irq_lock); 443 vq->cb.callback = NULL; 444 vq->cb.private = NULL; 445 spin_unlock(&vq->irq_lock); 446 flush_work(&vq->inject); 447 flush_work(&vq->kick); 448 } 449 450 up_write(&dev->rwsem); 451 } 452 453 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx, 454 u64 desc_area, u64 driver_area, 455 u64 device_area) 456 { 457 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 458 struct vduse_virtqueue *vq = &dev->vqs[idx]; 459 460 vq->desc_addr = desc_area; 461 vq->driver_addr = driver_area; 462 vq->device_addr = device_area; 463 464 return 0; 465 } 466 467 static void vduse_vq_kick(struct vduse_virtqueue *vq) 468 { 469 spin_lock(&vq->kick_lock); 470 if (!vq->ready) 471 goto unlock; 472 473 if (vq->kickfd) 474 eventfd_signal(vq->kickfd, 1); 475 else 476 vq->kicked = true; 477 unlock: 478 spin_unlock(&vq->kick_lock); 479 } 480 481 static void vduse_vq_kick_work(struct work_struct *work) 482 { 483 struct vduse_virtqueue *vq = container_of(work, 484 struct vduse_virtqueue, kick); 485 486 vduse_vq_kick(vq); 487 } 488 489 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx) 490 { 491 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 492 struct vduse_virtqueue *vq = &dev->vqs[idx]; 493 494 if (!eventfd_signal_allowed()) { 495 schedule_work(&vq->kick); 496 return; 497 } 498 vduse_vq_kick(vq); 499 } 500 501 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx, 502 struct vdpa_callback *cb) 503 { 504 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 505 struct vduse_virtqueue *vq = &dev->vqs[idx]; 506 507 spin_lock(&vq->irq_lock); 508 vq->cb.callback = cb->callback; 509 vq->cb.private = cb->private; 510 spin_unlock(&vq->irq_lock); 511 } 512 513 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num) 514 { 515 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 516 struct vduse_virtqueue *vq = &dev->vqs[idx]; 517 518 vq->num = num; 519 } 520 521 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa, 522 u16 idx, bool ready) 523 { 524 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 525 struct vduse_virtqueue *vq = &dev->vqs[idx]; 526 527 vq->ready = ready; 528 } 529 530 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx) 531 { 532 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 533 struct vduse_virtqueue *vq = &dev->vqs[idx]; 534 535 return vq->ready; 536 } 537 538 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx, 539 const struct vdpa_vq_state *state) 540 { 541 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 542 struct vduse_virtqueue *vq = &dev->vqs[idx]; 543 544 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 545 vq->state.packed.last_avail_counter = 546 state->packed.last_avail_counter; 547 vq->state.packed.last_avail_idx = state->packed.last_avail_idx; 548 vq->state.packed.last_used_counter = 549 state->packed.last_used_counter; 550 vq->state.packed.last_used_idx = state->packed.last_used_idx; 551 } else 552 vq->state.split.avail_index = state->split.avail_index; 553 554 return 0; 555 } 556 557 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx, 558 struct vdpa_vq_state *state) 559 { 560 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 561 struct vduse_virtqueue *vq = &dev->vqs[idx]; 562 563 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) 564 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed); 565 566 return vduse_dev_get_vq_state_split(dev, vq, &state->split); 567 } 568 569 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa) 570 { 571 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 572 573 return dev->vq_align; 574 } 575 576 static u64 vduse_vdpa_get_features(struct vdpa_device *vdpa) 577 { 578 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 579 580 return dev->device_features; 581 } 582 583 static int vduse_vdpa_set_features(struct vdpa_device *vdpa, u64 features) 584 { 585 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 586 587 dev->driver_features = features; 588 return 0; 589 } 590 591 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa, 592 struct vdpa_callback *cb) 593 { 594 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 595 596 spin_lock(&dev->irq_lock); 597 dev->config_cb.callback = cb->callback; 598 dev->config_cb.private = cb->private; 599 spin_unlock(&dev->irq_lock); 600 } 601 602 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa) 603 { 604 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 605 u16 num_max = 0; 606 int i; 607 608 for (i = 0; i < dev->vq_num; i++) 609 if (num_max < dev->vqs[i].num_max) 610 num_max = dev->vqs[i].num_max; 611 612 return num_max; 613 } 614 615 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa) 616 { 617 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 618 619 return dev->device_id; 620 } 621 622 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa) 623 { 624 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 625 626 return dev->vendor_id; 627 } 628 629 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa) 630 { 631 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 632 633 return dev->status; 634 } 635 636 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status) 637 { 638 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 639 640 if (vduse_dev_set_status(dev, status)) 641 return; 642 643 dev->status = status; 644 } 645 646 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa) 647 { 648 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 649 650 return dev->config_size; 651 } 652 653 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset, 654 void *buf, unsigned int len) 655 { 656 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 657 658 if (offset > dev->config_size || 659 len > dev->config_size - offset) 660 return; 661 662 memcpy(buf, dev->config + offset, len); 663 } 664 665 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset, 666 const void *buf, unsigned int len) 667 { 668 /* Now we only support read-only configuration space */ 669 } 670 671 static int vduse_vdpa_reset(struct vdpa_device *vdpa) 672 { 673 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 674 int ret = vduse_dev_set_status(dev, 0); 675 676 vduse_dev_reset(dev); 677 678 return ret; 679 } 680 681 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa) 682 { 683 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 684 685 return dev->generation; 686 } 687 688 static int vduse_vdpa_set_map(struct vdpa_device *vdpa, 689 struct vhost_iotlb *iotlb) 690 { 691 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 692 int ret; 693 694 ret = vduse_domain_set_map(dev->domain, iotlb); 695 if (ret) 696 return ret; 697 698 ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX); 699 if (ret) { 700 vduse_domain_clear_map(dev->domain, iotlb); 701 return ret; 702 } 703 704 return 0; 705 } 706 707 static void vduse_vdpa_free(struct vdpa_device *vdpa) 708 { 709 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 710 711 dev->vdev = NULL; 712 } 713 714 static const struct vdpa_config_ops vduse_vdpa_config_ops = { 715 .set_vq_address = vduse_vdpa_set_vq_address, 716 .kick_vq = vduse_vdpa_kick_vq, 717 .set_vq_cb = vduse_vdpa_set_vq_cb, 718 .set_vq_num = vduse_vdpa_set_vq_num, 719 .set_vq_ready = vduse_vdpa_set_vq_ready, 720 .get_vq_ready = vduse_vdpa_get_vq_ready, 721 .set_vq_state = vduse_vdpa_set_vq_state, 722 .get_vq_state = vduse_vdpa_get_vq_state, 723 .get_vq_align = vduse_vdpa_get_vq_align, 724 .get_features = vduse_vdpa_get_features, 725 .set_features = vduse_vdpa_set_features, 726 .set_config_cb = vduse_vdpa_set_config_cb, 727 .get_vq_num_max = vduse_vdpa_get_vq_num_max, 728 .get_device_id = vduse_vdpa_get_device_id, 729 .get_vendor_id = vduse_vdpa_get_vendor_id, 730 .get_status = vduse_vdpa_get_status, 731 .set_status = vduse_vdpa_set_status, 732 .get_config_size = vduse_vdpa_get_config_size, 733 .get_config = vduse_vdpa_get_config, 734 .set_config = vduse_vdpa_set_config, 735 .get_generation = vduse_vdpa_get_generation, 736 .reset = vduse_vdpa_reset, 737 .set_map = vduse_vdpa_set_map, 738 .free = vduse_vdpa_free, 739 }; 740 741 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, 742 unsigned long offset, size_t size, 743 enum dma_data_direction dir, 744 unsigned long attrs) 745 { 746 struct vduse_dev *vdev = dev_to_vduse(dev); 747 struct vduse_iova_domain *domain = vdev->domain; 748 749 return vduse_domain_map_page(domain, page, offset, size, dir, attrs); 750 } 751 752 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr, 753 size_t size, enum dma_data_direction dir, 754 unsigned long attrs) 755 { 756 struct vduse_dev *vdev = dev_to_vduse(dev); 757 struct vduse_iova_domain *domain = vdev->domain; 758 759 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs); 760 } 761 762 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size, 763 dma_addr_t *dma_addr, gfp_t flag, 764 unsigned long attrs) 765 { 766 struct vduse_dev *vdev = dev_to_vduse(dev); 767 struct vduse_iova_domain *domain = vdev->domain; 768 unsigned long iova; 769 void *addr; 770 771 *dma_addr = DMA_MAPPING_ERROR; 772 addr = vduse_domain_alloc_coherent(domain, size, 773 (dma_addr_t *)&iova, flag, attrs); 774 if (!addr) 775 return NULL; 776 777 *dma_addr = (dma_addr_t)iova; 778 779 return addr; 780 } 781 782 static void vduse_dev_free_coherent(struct device *dev, size_t size, 783 void *vaddr, dma_addr_t dma_addr, 784 unsigned long attrs) 785 { 786 struct vduse_dev *vdev = dev_to_vduse(dev); 787 struct vduse_iova_domain *domain = vdev->domain; 788 789 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs); 790 } 791 792 static size_t vduse_dev_max_mapping_size(struct device *dev) 793 { 794 struct vduse_dev *vdev = dev_to_vduse(dev); 795 struct vduse_iova_domain *domain = vdev->domain; 796 797 return domain->bounce_size; 798 } 799 800 static const struct dma_map_ops vduse_dev_dma_ops = { 801 .map_page = vduse_dev_map_page, 802 .unmap_page = vduse_dev_unmap_page, 803 .alloc = vduse_dev_alloc_coherent, 804 .free = vduse_dev_free_coherent, 805 .max_mapping_size = vduse_dev_max_mapping_size, 806 }; 807 808 static unsigned int perm_to_file_flags(u8 perm) 809 { 810 unsigned int flags = 0; 811 812 switch (perm) { 813 case VDUSE_ACCESS_WO: 814 flags |= O_WRONLY; 815 break; 816 case VDUSE_ACCESS_RO: 817 flags |= O_RDONLY; 818 break; 819 case VDUSE_ACCESS_RW: 820 flags |= O_RDWR; 821 break; 822 default: 823 WARN(1, "invalidate vhost IOTLB permission\n"); 824 break; 825 } 826 827 return flags; 828 } 829 830 static int vduse_kickfd_setup(struct vduse_dev *dev, 831 struct vduse_vq_eventfd *eventfd) 832 { 833 struct eventfd_ctx *ctx = NULL; 834 struct vduse_virtqueue *vq; 835 u32 index; 836 837 if (eventfd->index >= dev->vq_num) 838 return -EINVAL; 839 840 index = array_index_nospec(eventfd->index, dev->vq_num); 841 vq = &dev->vqs[index]; 842 if (eventfd->fd >= 0) { 843 ctx = eventfd_ctx_fdget(eventfd->fd); 844 if (IS_ERR(ctx)) 845 return PTR_ERR(ctx); 846 } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN) 847 return 0; 848 849 spin_lock(&vq->kick_lock); 850 if (vq->kickfd) 851 eventfd_ctx_put(vq->kickfd); 852 vq->kickfd = ctx; 853 if (vq->ready && vq->kicked && vq->kickfd) { 854 eventfd_signal(vq->kickfd, 1); 855 vq->kicked = false; 856 } 857 spin_unlock(&vq->kick_lock); 858 859 return 0; 860 } 861 862 static bool vduse_dev_is_ready(struct vduse_dev *dev) 863 { 864 int i; 865 866 for (i = 0; i < dev->vq_num; i++) 867 if (!dev->vqs[i].num_max) 868 return false; 869 870 return true; 871 } 872 873 static void vduse_dev_irq_inject(struct work_struct *work) 874 { 875 struct vduse_dev *dev = container_of(work, struct vduse_dev, inject); 876 877 spin_lock_irq(&dev->irq_lock); 878 if (dev->config_cb.callback) 879 dev->config_cb.callback(dev->config_cb.private); 880 spin_unlock_irq(&dev->irq_lock); 881 } 882 883 static void vduse_vq_irq_inject(struct work_struct *work) 884 { 885 struct vduse_virtqueue *vq = container_of(work, 886 struct vduse_virtqueue, inject); 887 888 spin_lock_irq(&vq->irq_lock); 889 if (vq->ready && vq->cb.callback) 890 vq->cb.callback(vq->cb.private); 891 spin_unlock_irq(&vq->irq_lock); 892 } 893 894 static int vduse_dev_queue_irq_work(struct vduse_dev *dev, 895 struct work_struct *irq_work) 896 { 897 int ret = -EINVAL; 898 899 down_read(&dev->rwsem); 900 if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK)) 901 goto unlock; 902 903 ret = 0; 904 queue_work(vduse_irq_wq, irq_work); 905 unlock: 906 up_read(&dev->rwsem); 907 908 return ret; 909 } 910 911 static long vduse_dev_ioctl(struct file *file, unsigned int cmd, 912 unsigned long arg) 913 { 914 struct vduse_dev *dev = file->private_data; 915 void __user *argp = (void __user *)arg; 916 int ret; 917 918 if (unlikely(dev->broken)) 919 return -EPERM; 920 921 switch (cmd) { 922 case VDUSE_IOTLB_GET_FD: { 923 struct vduse_iotlb_entry entry; 924 struct vhost_iotlb_map *map; 925 struct vdpa_map_file *map_file; 926 struct vduse_iova_domain *domain = dev->domain; 927 struct file *f = NULL; 928 929 ret = -EFAULT; 930 if (copy_from_user(&entry, argp, sizeof(entry))) 931 break; 932 933 ret = -EINVAL; 934 if (entry.start > entry.last) 935 break; 936 937 spin_lock(&domain->iotlb_lock); 938 map = vhost_iotlb_itree_first(domain->iotlb, 939 entry.start, entry.last); 940 if (map) { 941 map_file = (struct vdpa_map_file *)map->opaque; 942 f = get_file(map_file->file); 943 entry.offset = map_file->offset; 944 entry.start = map->start; 945 entry.last = map->last; 946 entry.perm = map->perm; 947 } 948 spin_unlock(&domain->iotlb_lock); 949 ret = -EINVAL; 950 if (!f) 951 break; 952 953 ret = -EFAULT; 954 if (copy_to_user(argp, &entry, sizeof(entry))) { 955 fput(f); 956 break; 957 } 958 ret = receive_fd(f, perm_to_file_flags(entry.perm)); 959 fput(f); 960 break; 961 } 962 case VDUSE_DEV_GET_FEATURES: 963 /* 964 * Just mirror what driver wrote here. 965 * The driver is expected to check FEATURE_OK later. 966 */ 967 ret = put_user(dev->driver_features, (u64 __user *)argp); 968 break; 969 case VDUSE_DEV_SET_CONFIG: { 970 struct vduse_config_data config; 971 unsigned long size = offsetof(struct vduse_config_data, 972 buffer); 973 974 ret = -EFAULT; 975 if (copy_from_user(&config, argp, size)) 976 break; 977 978 ret = -EINVAL; 979 if (config.offset > dev->config_size || 980 config.length == 0 || 981 config.length > dev->config_size - config.offset) 982 break; 983 984 ret = -EFAULT; 985 if (copy_from_user(dev->config + config.offset, argp + size, 986 config.length)) 987 break; 988 989 ret = 0; 990 break; 991 } 992 case VDUSE_DEV_INJECT_CONFIG_IRQ: 993 ret = vduse_dev_queue_irq_work(dev, &dev->inject); 994 break; 995 case VDUSE_VQ_SETUP: { 996 struct vduse_vq_config config; 997 u32 index; 998 999 ret = -EFAULT; 1000 if (copy_from_user(&config, argp, sizeof(config))) 1001 break; 1002 1003 ret = -EINVAL; 1004 if (config.index >= dev->vq_num) 1005 break; 1006 1007 if (!is_mem_zero((const char *)config.reserved, 1008 sizeof(config.reserved))) 1009 break; 1010 1011 index = array_index_nospec(config.index, dev->vq_num); 1012 dev->vqs[index].num_max = config.max_size; 1013 ret = 0; 1014 break; 1015 } 1016 case VDUSE_VQ_GET_INFO: { 1017 struct vduse_vq_info vq_info; 1018 struct vduse_virtqueue *vq; 1019 u32 index; 1020 1021 ret = -EFAULT; 1022 if (copy_from_user(&vq_info, argp, sizeof(vq_info))) 1023 break; 1024 1025 ret = -EINVAL; 1026 if (vq_info.index >= dev->vq_num) 1027 break; 1028 1029 index = array_index_nospec(vq_info.index, dev->vq_num); 1030 vq = &dev->vqs[index]; 1031 vq_info.desc_addr = vq->desc_addr; 1032 vq_info.driver_addr = vq->driver_addr; 1033 vq_info.device_addr = vq->device_addr; 1034 vq_info.num = vq->num; 1035 1036 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 1037 vq_info.packed.last_avail_counter = 1038 vq->state.packed.last_avail_counter; 1039 vq_info.packed.last_avail_idx = 1040 vq->state.packed.last_avail_idx; 1041 vq_info.packed.last_used_counter = 1042 vq->state.packed.last_used_counter; 1043 vq_info.packed.last_used_idx = 1044 vq->state.packed.last_used_idx; 1045 } else 1046 vq_info.split.avail_index = 1047 vq->state.split.avail_index; 1048 1049 vq_info.ready = vq->ready; 1050 1051 ret = -EFAULT; 1052 if (copy_to_user(argp, &vq_info, sizeof(vq_info))) 1053 break; 1054 1055 ret = 0; 1056 break; 1057 } 1058 case VDUSE_VQ_SETUP_KICKFD: { 1059 struct vduse_vq_eventfd eventfd; 1060 1061 ret = -EFAULT; 1062 if (copy_from_user(&eventfd, argp, sizeof(eventfd))) 1063 break; 1064 1065 ret = vduse_kickfd_setup(dev, &eventfd); 1066 break; 1067 } 1068 case VDUSE_VQ_INJECT_IRQ: { 1069 u32 index; 1070 1071 ret = -EFAULT; 1072 if (get_user(index, (u32 __user *)argp)) 1073 break; 1074 1075 ret = -EINVAL; 1076 if (index >= dev->vq_num) 1077 break; 1078 1079 index = array_index_nospec(index, dev->vq_num); 1080 ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject); 1081 break; 1082 } 1083 default: 1084 ret = -ENOIOCTLCMD; 1085 break; 1086 } 1087 1088 return ret; 1089 } 1090 1091 static int vduse_dev_release(struct inode *inode, struct file *file) 1092 { 1093 struct vduse_dev *dev = file->private_data; 1094 1095 spin_lock(&dev->msg_lock); 1096 /* Make sure the inflight messages can processed after reconncection */ 1097 list_splice_init(&dev->recv_list, &dev->send_list); 1098 spin_unlock(&dev->msg_lock); 1099 dev->connected = false; 1100 1101 return 0; 1102 } 1103 1104 static struct vduse_dev *vduse_dev_get_from_minor(int minor) 1105 { 1106 struct vduse_dev *dev; 1107 1108 mutex_lock(&vduse_lock); 1109 dev = idr_find(&vduse_idr, minor); 1110 mutex_unlock(&vduse_lock); 1111 1112 return dev; 1113 } 1114 1115 static int vduse_dev_open(struct inode *inode, struct file *file) 1116 { 1117 int ret; 1118 struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode)); 1119 1120 if (!dev) 1121 return -ENODEV; 1122 1123 ret = -EBUSY; 1124 mutex_lock(&dev->lock); 1125 if (dev->connected) 1126 goto unlock; 1127 1128 ret = 0; 1129 dev->connected = true; 1130 file->private_data = dev; 1131 unlock: 1132 mutex_unlock(&dev->lock); 1133 1134 return ret; 1135 } 1136 1137 static const struct file_operations vduse_dev_fops = { 1138 .owner = THIS_MODULE, 1139 .open = vduse_dev_open, 1140 .release = vduse_dev_release, 1141 .read_iter = vduse_dev_read_iter, 1142 .write_iter = vduse_dev_write_iter, 1143 .poll = vduse_dev_poll, 1144 .unlocked_ioctl = vduse_dev_ioctl, 1145 .compat_ioctl = compat_ptr_ioctl, 1146 .llseek = noop_llseek, 1147 }; 1148 1149 static struct vduse_dev *vduse_dev_create(void) 1150 { 1151 struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1152 1153 if (!dev) 1154 return NULL; 1155 1156 mutex_init(&dev->lock); 1157 spin_lock_init(&dev->msg_lock); 1158 INIT_LIST_HEAD(&dev->send_list); 1159 INIT_LIST_HEAD(&dev->recv_list); 1160 spin_lock_init(&dev->irq_lock); 1161 init_rwsem(&dev->rwsem); 1162 1163 INIT_WORK(&dev->inject, vduse_dev_irq_inject); 1164 init_waitqueue_head(&dev->waitq); 1165 1166 return dev; 1167 } 1168 1169 static void vduse_dev_destroy(struct vduse_dev *dev) 1170 { 1171 kfree(dev); 1172 } 1173 1174 static struct vduse_dev *vduse_find_dev(const char *name) 1175 { 1176 struct vduse_dev *dev; 1177 int id; 1178 1179 idr_for_each_entry(&vduse_idr, dev, id) 1180 if (!strcmp(dev->name, name)) 1181 return dev; 1182 1183 return NULL; 1184 } 1185 1186 static int vduse_destroy_dev(char *name) 1187 { 1188 struct vduse_dev *dev = vduse_find_dev(name); 1189 1190 if (!dev) 1191 return -EINVAL; 1192 1193 mutex_lock(&dev->lock); 1194 if (dev->vdev || dev->connected) { 1195 mutex_unlock(&dev->lock); 1196 return -EBUSY; 1197 } 1198 dev->connected = true; 1199 mutex_unlock(&dev->lock); 1200 1201 vduse_dev_reset(dev); 1202 device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor)); 1203 idr_remove(&vduse_idr, dev->minor); 1204 kvfree(dev->config); 1205 kfree(dev->vqs); 1206 vduse_domain_destroy(dev->domain); 1207 kfree(dev->name); 1208 vduse_dev_destroy(dev); 1209 module_put(THIS_MODULE); 1210 1211 return 0; 1212 } 1213 1214 static bool device_is_allowed(u32 device_id) 1215 { 1216 int i; 1217 1218 for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++) 1219 if (allowed_device_id[i] == device_id) 1220 return true; 1221 1222 return false; 1223 } 1224 1225 static bool features_is_valid(u64 features) 1226 { 1227 if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM))) 1228 return false; 1229 1230 /* Now we only support read-only configuration space */ 1231 if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE)) 1232 return false; 1233 1234 return true; 1235 } 1236 1237 static bool vduse_validate_config(struct vduse_dev_config *config) 1238 { 1239 if (!is_mem_zero((const char *)config->reserved, 1240 sizeof(config->reserved))) 1241 return false; 1242 1243 if (config->vq_align > PAGE_SIZE) 1244 return false; 1245 1246 if (config->config_size > PAGE_SIZE) 1247 return false; 1248 1249 if (!device_is_allowed(config->device_id)) 1250 return false; 1251 1252 if (!features_is_valid(config->features)) 1253 return false; 1254 1255 return true; 1256 } 1257 1258 static ssize_t msg_timeout_show(struct device *device, 1259 struct device_attribute *attr, char *buf) 1260 { 1261 struct vduse_dev *dev = dev_get_drvdata(device); 1262 1263 return sysfs_emit(buf, "%u\n", dev->msg_timeout); 1264 } 1265 1266 static ssize_t msg_timeout_store(struct device *device, 1267 struct device_attribute *attr, 1268 const char *buf, size_t count) 1269 { 1270 struct vduse_dev *dev = dev_get_drvdata(device); 1271 int ret; 1272 1273 ret = kstrtouint(buf, 10, &dev->msg_timeout); 1274 if (ret < 0) 1275 return ret; 1276 1277 return count; 1278 } 1279 1280 static DEVICE_ATTR_RW(msg_timeout); 1281 1282 static struct attribute *vduse_dev_attrs[] = { 1283 &dev_attr_msg_timeout.attr, 1284 NULL 1285 }; 1286 1287 ATTRIBUTE_GROUPS(vduse_dev); 1288 1289 static int vduse_create_dev(struct vduse_dev_config *config, 1290 void *config_buf, u64 api_version) 1291 { 1292 int i, ret; 1293 struct vduse_dev *dev; 1294 1295 ret = -EEXIST; 1296 if (vduse_find_dev(config->name)) 1297 goto err; 1298 1299 ret = -ENOMEM; 1300 dev = vduse_dev_create(); 1301 if (!dev) 1302 goto err; 1303 1304 dev->api_version = api_version; 1305 dev->device_features = config->features; 1306 dev->device_id = config->device_id; 1307 dev->vendor_id = config->vendor_id; 1308 dev->name = kstrdup(config->name, GFP_KERNEL); 1309 if (!dev->name) 1310 goto err_str; 1311 1312 dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1, 1313 VDUSE_BOUNCE_SIZE); 1314 if (!dev->domain) 1315 goto err_domain; 1316 1317 dev->config = config_buf; 1318 dev->config_size = config->config_size; 1319 dev->vq_align = config->vq_align; 1320 dev->vq_num = config->vq_num; 1321 dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL); 1322 if (!dev->vqs) 1323 goto err_vqs; 1324 1325 for (i = 0; i < dev->vq_num; i++) { 1326 dev->vqs[i].index = i; 1327 INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject); 1328 INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work); 1329 spin_lock_init(&dev->vqs[i].kick_lock); 1330 spin_lock_init(&dev->vqs[i].irq_lock); 1331 } 1332 1333 ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL); 1334 if (ret < 0) 1335 goto err_idr; 1336 1337 dev->minor = ret; 1338 dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT; 1339 dev->dev = device_create(vduse_class, NULL, 1340 MKDEV(MAJOR(vduse_major), dev->minor), 1341 dev, "%s", config->name); 1342 if (IS_ERR(dev->dev)) { 1343 ret = PTR_ERR(dev->dev); 1344 goto err_dev; 1345 } 1346 __module_get(THIS_MODULE); 1347 1348 return 0; 1349 err_dev: 1350 idr_remove(&vduse_idr, dev->minor); 1351 err_idr: 1352 kfree(dev->vqs); 1353 err_vqs: 1354 vduse_domain_destroy(dev->domain); 1355 err_domain: 1356 kfree(dev->name); 1357 err_str: 1358 vduse_dev_destroy(dev); 1359 err: 1360 kvfree(config_buf); 1361 return ret; 1362 } 1363 1364 static long vduse_ioctl(struct file *file, unsigned int cmd, 1365 unsigned long arg) 1366 { 1367 int ret; 1368 void __user *argp = (void __user *)arg; 1369 struct vduse_control *control = file->private_data; 1370 1371 mutex_lock(&vduse_lock); 1372 switch (cmd) { 1373 case VDUSE_GET_API_VERSION: 1374 ret = put_user(control->api_version, (u64 __user *)argp); 1375 break; 1376 case VDUSE_SET_API_VERSION: { 1377 u64 api_version; 1378 1379 ret = -EFAULT; 1380 if (get_user(api_version, (u64 __user *)argp)) 1381 break; 1382 1383 ret = -EINVAL; 1384 if (api_version > VDUSE_API_VERSION) 1385 break; 1386 1387 ret = 0; 1388 control->api_version = api_version; 1389 break; 1390 } 1391 case VDUSE_CREATE_DEV: { 1392 struct vduse_dev_config config; 1393 unsigned long size = offsetof(struct vduse_dev_config, config); 1394 void *buf; 1395 1396 ret = -EFAULT; 1397 if (copy_from_user(&config, argp, size)) 1398 break; 1399 1400 ret = -EINVAL; 1401 if (vduse_validate_config(&config) == false) 1402 break; 1403 1404 buf = vmemdup_user(argp + size, config.config_size); 1405 if (IS_ERR(buf)) { 1406 ret = PTR_ERR(buf); 1407 break; 1408 } 1409 config.name[VDUSE_NAME_MAX - 1] = '\0'; 1410 ret = vduse_create_dev(&config, buf, control->api_version); 1411 break; 1412 } 1413 case VDUSE_DESTROY_DEV: { 1414 char name[VDUSE_NAME_MAX]; 1415 1416 ret = -EFAULT; 1417 if (copy_from_user(name, argp, VDUSE_NAME_MAX)) 1418 break; 1419 1420 name[VDUSE_NAME_MAX - 1] = '\0'; 1421 ret = vduse_destroy_dev(name); 1422 break; 1423 } 1424 default: 1425 ret = -EINVAL; 1426 break; 1427 } 1428 mutex_unlock(&vduse_lock); 1429 1430 return ret; 1431 } 1432 1433 static int vduse_release(struct inode *inode, struct file *file) 1434 { 1435 struct vduse_control *control = file->private_data; 1436 1437 kfree(control); 1438 return 0; 1439 } 1440 1441 static int vduse_open(struct inode *inode, struct file *file) 1442 { 1443 struct vduse_control *control; 1444 1445 control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL); 1446 if (!control) 1447 return -ENOMEM; 1448 1449 control->api_version = VDUSE_API_VERSION; 1450 file->private_data = control; 1451 1452 return 0; 1453 } 1454 1455 static const struct file_operations vduse_ctrl_fops = { 1456 .owner = THIS_MODULE, 1457 .open = vduse_open, 1458 .release = vduse_release, 1459 .unlocked_ioctl = vduse_ioctl, 1460 .compat_ioctl = compat_ptr_ioctl, 1461 .llseek = noop_llseek, 1462 }; 1463 1464 static char *vduse_devnode(struct device *dev, umode_t *mode) 1465 { 1466 return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev)); 1467 } 1468 1469 static void vduse_mgmtdev_release(struct device *dev) 1470 { 1471 } 1472 1473 static struct device vduse_mgmtdev = { 1474 .init_name = "vduse", 1475 .release = vduse_mgmtdev_release, 1476 }; 1477 1478 static struct vdpa_mgmt_dev mgmt_dev; 1479 1480 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) 1481 { 1482 struct vduse_vdpa *vdev; 1483 int ret; 1484 1485 if (dev->vdev) 1486 return -EEXIST; 1487 1488 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev, 1489 &vduse_vdpa_config_ops, name, true); 1490 if (IS_ERR(vdev)) 1491 return PTR_ERR(vdev); 1492 1493 dev->vdev = vdev; 1494 vdev->dev = dev; 1495 vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask; 1496 ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64)); 1497 if (ret) { 1498 put_device(&vdev->vdpa.dev); 1499 return ret; 1500 } 1501 set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops); 1502 vdev->vdpa.dma_dev = &vdev->vdpa.dev; 1503 vdev->vdpa.mdev = &mgmt_dev; 1504 1505 return 0; 1506 } 1507 1508 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 1509 const struct vdpa_dev_set_config *config) 1510 { 1511 struct vduse_dev *dev; 1512 int ret; 1513 1514 mutex_lock(&vduse_lock); 1515 dev = vduse_find_dev(name); 1516 if (!dev || !vduse_dev_is_ready(dev)) { 1517 mutex_unlock(&vduse_lock); 1518 return -EINVAL; 1519 } 1520 ret = vduse_dev_init_vdpa(dev, name); 1521 mutex_unlock(&vduse_lock); 1522 if (ret) 1523 return ret; 1524 1525 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); 1526 if (ret) { 1527 put_device(&dev->vdev->vdpa.dev); 1528 return ret; 1529 } 1530 1531 return 0; 1532 } 1533 1534 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) 1535 { 1536 _vdpa_unregister_device(dev); 1537 } 1538 1539 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = { 1540 .dev_add = vdpa_dev_add, 1541 .dev_del = vdpa_dev_del, 1542 }; 1543 1544 static struct virtio_device_id id_table[] = { 1545 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID }, 1546 { 0 }, 1547 }; 1548 1549 static struct vdpa_mgmt_dev mgmt_dev = { 1550 .device = &vduse_mgmtdev, 1551 .id_table = id_table, 1552 .ops = &vdpa_dev_mgmtdev_ops, 1553 }; 1554 1555 static int vduse_mgmtdev_init(void) 1556 { 1557 int ret; 1558 1559 ret = device_register(&vduse_mgmtdev); 1560 if (ret) 1561 return ret; 1562 1563 ret = vdpa_mgmtdev_register(&mgmt_dev); 1564 if (ret) 1565 goto err; 1566 1567 return 0; 1568 err: 1569 device_unregister(&vduse_mgmtdev); 1570 return ret; 1571 } 1572 1573 static void vduse_mgmtdev_exit(void) 1574 { 1575 vdpa_mgmtdev_unregister(&mgmt_dev); 1576 device_unregister(&vduse_mgmtdev); 1577 } 1578 1579 static int vduse_init(void) 1580 { 1581 int ret; 1582 struct device *dev; 1583 1584 vduse_class = class_create(THIS_MODULE, "vduse"); 1585 if (IS_ERR(vduse_class)) 1586 return PTR_ERR(vduse_class); 1587 1588 vduse_class->devnode = vduse_devnode; 1589 vduse_class->dev_groups = vduse_dev_groups; 1590 1591 ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse"); 1592 if (ret) 1593 goto err_chardev_region; 1594 1595 /* /dev/vduse/control */ 1596 cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops); 1597 vduse_ctrl_cdev.owner = THIS_MODULE; 1598 ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1); 1599 if (ret) 1600 goto err_ctrl_cdev; 1601 1602 dev = device_create(vduse_class, NULL, vduse_major, NULL, "control"); 1603 if (IS_ERR(dev)) { 1604 ret = PTR_ERR(dev); 1605 goto err_device; 1606 } 1607 1608 /* /dev/vduse/$DEVICE */ 1609 cdev_init(&vduse_cdev, &vduse_dev_fops); 1610 vduse_cdev.owner = THIS_MODULE; 1611 ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1), 1612 VDUSE_DEV_MAX - 1); 1613 if (ret) 1614 goto err_cdev; 1615 1616 vduse_irq_wq = alloc_workqueue("vduse-irq", 1617 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0); 1618 if (!vduse_irq_wq) { 1619 ret = -ENOMEM; 1620 goto err_wq; 1621 } 1622 1623 ret = vduse_domain_init(); 1624 if (ret) 1625 goto err_domain; 1626 1627 ret = vduse_mgmtdev_init(); 1628 if (ret) 1629 goto err_mgmtdev; 1630 1631 return 0; 1632 err_mgmtdev: 1633 vduse_domain_exit(); 1634 err_domain: 1635 destroy_workqueue(vduse_irq_wq); 1636 err_wq: 1637 cdev_del(&vduse_cdev); 1638 err_cdev: 1639 device_destroy(vduse_class, vduse_major); 1640 err_device: 1641 cdev_del(&vduse_ctrl_cdev); 1642 err_ctrl_cdev: 1643 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 1644 err_chardev_region: 1645 class_destroy(vduse_class); 1646 return ret; 1647 } 1648 module_init(vduse_init); 1649 1650 static void vduse_exit(void) 1651 { 1652 vduse_mgmtdev_exit(); 1653 vduse_domain_exit(); 1654 destroy_workqueue(vduse_irq_wq); 1655 cdev_del(&vduse_cdev); 1656 device_destroy(vduse_class, vduse_major); 1657 cdev_del(&vduse_ctrl_cdev); 1658 unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX); 1659 class_destroy(vduse_class); 1660 } 1661 module_exit(vduse_exit); 1662 1663 MODULE_LICENSE(DRV_LICENSE); 1664 MODULE_AUTHOR(DRV_AUTHOR); 1665 MODULE_DESCRIPTION(DRV_DESC); 1666