1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2018-2020 Intel Corporation. 4 * Copyright (C) 2020 Red Hat, Inc. 5 * 6 * Author: Tiwei Bie <tiwei.bie@intel.com> 7 * Jason Wang <jasowang@redhat.com> 8 * 9 * Thanks Michael S. Tsirkin for the valuable comments and 10 * suggestions. And thanks to Cunming Liang and Zhihong Wang for all 11 * their supports. 12 */ 13 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/cdev.h> 17 #include <linux/device.h> 18 #include <linux/iommu.h> 19 #include <linux/uuid.h> 20 #include <linux/vdpa.h> 21 #include <linux/nospec.h> 22 #include <linux/vhost.h> 23 #include <linux/virtio_net.h> 24 25 #include "vhost.h" 26 27 enum { 28 VHOST_VDPA_FEATURES = 29 (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | 30 (1ULL << VIRTIO_F_ANY_LAYOUT) | 31 (1ULL << VIRTIO_F_VERSION_1) | 32 (1ULL << VIRTIO_F_IOMMU_PLATFORM) | 33 (1ULL << VIRTIO_F_RING_PACKED) | 34 (1ULL << VIRTIO_F_ORDER_PLATFORM) | 35 (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | 36 (1ULL << VIRTIO_RING_F_EVENT_IDX), 37 38 VHOST_VDPA_NET_FEATURES = VHOST_VDPA_FEATURES | 39 (1ULL << VIRTIO_NET_F_CSUM) | 40 (1ULL << VIRTIO_NET_F_GUEST_CSUM) | 41 (1ULL << VIRTIO_NET_F_MTU) | 42 (1ULL << VIRTIO_NET_F_MAC) | 43 (1ULL << VIRTIO_NET_F_GUEST_TSO4) | 44 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | 45 (1ULL << VIRTIO_NET_F_GUEST_ECN) | 46 (1ULL << VIRTIO_NET_F_GUEST_UFO) | 47 (1ULL << VIRTIO_NET_F_HOST_TSO4) | 48 (1ULL << VIRTIO_NET_F_HOST_TSO6) | 49 (1ULL << VIRTIO_NET_F_HOST_ECN) | 50 (1ULL << VIRTIO_NET_F_HOST_UFO) | 51 (1ULL << VIRTIO_NET_F_MRG_RXBUF) | 52 (1ULL << VIRTIO_NET_F_STATUS) | 53 (1ULL << VIRTIO_NET_F_SPEED_DUPLEX), 54 }; 55 56 /* Currently, only network backend w/o multiqueue is supported. */ 57 #define VHOST_VDPA_VQ_MAX 2 58 59 #define VHOST_VDPA_DEV_MAX (1U << MINORBITS) 60 61 struct vhost_vdpa { 62 struct vhost_dev vdev; 63 struct iommu_domain *domain; 64 struct vhost_virtqueue *vqs; 65 struct completion completion; 66 struct vdpa_device *vdpa; 67 struct device dev; 68 struct cdev cdev; 69 atomic_t opened; 70 int nvqs; 71 int virtio_id; 72 int minor; 73 }; 74 75 static DEFINE_IDA(vhost_vdpa_ida); 76 77 static dev_t vhost_vdpa_major; 78 79 static const u64 vhost_vdpa_features[] = { 80 [VIRTIO_ID_NET] = VHOST_VDPA_NET_FEATURES, 81 }; 82 83 static void handle_vq_kick(struct vhost_work *work) 84 { 85 struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 86 poll.work); 87 struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev); 88 const struct vdpa_config_ops *ops = v->vdpa->config; 89 90 ops->kick_vq(v->vdpa, vq - v->vqs); 91 } 92 93 static irqreturn_t vhost_vdpa_virtqueue_cb(void *private) 94 { 95 struct vhost_virtqueue *vq = private; 96 struct eventfd_ctx *call_ctx = vq->call_ctx; 97 98 if (call_ctx) 99 eventfd_signal(call_ctx, 1); 100 101 return IRQ_HANDLED; 102 } 103 104 static void vhost_vdpa_reset(struct vhost_vdpa *v) 105 { 106 struct vdpa_device *vdpa = v->vdpa; 107 const struct vdpa_config_ops *ops = vdpa->config; 108 109 ops->set_status(vdpa, 0); 110 } 111 112 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp) 113 { 114 struct vdpa_device *vdpa = v->vdpa; 115 const struct vdpa_config_ops *ops = vdpa->config; 116 u32 device_id; 117 118 device_id = ops->get_device_id(vdpa); 119 120 if (copy_to_user(argp, &device_id, sizeof(device_id))) 121 return -EFAULT; 122 123 return 0; 124 } 125 126 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp) 127 { 128 struct vdpa_device *vdpa = v->vdpa; 129 const struct vdpa_config_ops *ops = vdpa->config; 130 u8 status; 131 132 status = ops->get_status(vdpa); 133 134 if (copy_to_user(statusp, &status, sizeof(status))) 135 return -EFAULT; 136 137 return 0; 138 } 139 140 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp) 141 { 142 struct vdpa_device *vdpa = v->vdpa; 143 const struct vdpa_config_ops *ops = vdpa->config; 144 u8 status; 145 146 if (copy_from_user(&status, statusp, sizeof(status))) 147 return -EFAULT; 148 149 /* 150 * Userspace shouldn't remove status bits unless reset the 151 * status to 0. 152 */ 153 if (status != 0 && (ops->get_status(vdpa) & ~status) != 0) 154 return -EINVAL; 155 156 ops->set_status(vdpa, status); 157 158 return 0; 159 } 160 161 static int vhost_vdpa_config_validate(struct vhost_vdpa *v, 162 struct vhost_vdpa_config *c) 163 { 164 long size = 0; 165 166 switch (v->virtio_id) { 167 case VIRTIO_ID_NET: 168 size = sizeof(struct virtio_net_config); 169 break; 170 } 171 172 if (c->len == 0) 173 return -EINVAL; 174 175 if (c->len > size - c->off) 176 return -E2BIG; 177 178 return 0; 179 } 180 181 static long vhost_vdpa_get_config(struct vhost_vdpa *v, 182 struct vhost_vdpa_config __user *c) 183 { 184 struct vdpa_device *vdpa = v->vdpa; 185 const struct vdpa_config_ops *ops = vdpa->config; 186 struct vhost_vdpa_config config; 187 unsigned long size = offsetof(struct vhost_vdpa_config, buf); 188 u8 *buf; 189 190 if (copy_from_user(&config, c, size)) 191 return -EFAULT; 192 if (vhost_vdpa_config_validate(v, &config)) 193 return -EINVAL; 194 buf = kvzalloc(config.len, GFP_KERNEL); 195 if (!buf) 196 return -ENOMEM; 197 198 ops->get_config(vdpa, config.off, buf, config.len); 199 200 if (copy_to_user(c->buf, buf, config.len)) { 201 kvfree(buf); 202 return -EFAULT; 203 } 204 205 kvfree(buf); 206 return 0; 207 } 208 209 static long vhost_vdpa_set_config(struct vhost_vdpa *v, 210 struct vhost_vdpa_config __user *c) 211 { 212 struct vdpa_device *vdpa = v->vdpa; 213 const struct vdpa_config_ops *ops = vdpa->config; 214 struct vhost_vdpa_config config; 215 unsigned long size = offsetof(struct vhost_vdpa_config, buf); 216 u8 *buf; 217 218 if (copy_from_user(&config, c, size)) 219 return -EFAULT; 220 if (vhost_vdpa_config_validate(v, &config)) 221 return -EINVAL; 222 buf = kvzalloc(config.len, GFP_KERNEL); 223 if (!buf) 224 return -ENOMEM; 225 226 if (copy_from_user(buf, c->buf, config.len)) { 227 kvfree(buf); 228 return -EFAULT; 229 } 230 231 ops->set_config(vdpa, config.off, buf, config.len); 232 233 kvfree(buf); 234 return 0; 235 } 236 237 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) 238 { 239 struct vdpa_device *vdpa = v->vdpa; 240 const struct vdpa_config_ops *ops = vdpa->config; 241 u64 features; 242 243 features = ops->get_features(vdpa); 244 features &= vhost_vdpa_features[v->virtio_id]; 245 246 if (copy_to_user(featurep, &features, sizeof(features))) 247 return -EFAULT; 248 249 return 0; 250 } 251 252 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep) 253 { 254 struct vdpa_device *vdpa = v->vdpa; 255 const struct vdpa_config_ops *ops = vdpa->config; 256 u64 features; 257 258 /* 259 * It's not allowed to change the features after they have 260 * been negotiated. 261 */ 262 if (ops->get_status(vdpa) & VIRTIO_CONFIG_S_FEATURES_OK) 263 return -EBUSY; 264 265 if (copy_from_user(&features, featurep, sizeof(features))) 266 return -EFAULT; 267 268 if (features & ~vhost_vdpa_features[v->virtio_id]) 269 return -EINVAL; 270 271 if (ops->set_features(vdpa, features)) 272 return -EINVAL; 273 274 return 0; 275 } 276 277 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp) 278 { 279 struct vdpa_device *vdpa = v->vdpa; 280 const struct vdpa_config_ops *ops = vdpa->config; 281 u16 num; 282 283 num = ops->get_vq_num_max(vdpa); 284 285 if (copy_to_user(argp, &num, sizeof(num))) 286 return -EFAULT; 287 288 return 0; 289 } 290 291 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, 292 void __user *argp) 293 { 294 struct vdpa_device *vdpa = v->vdpa; 295 const struct vdpa_config_ops *ops = vdpa->config; 296 struct vdpa_callback cb; 297 struct vhost_virtqueue *vq; 298 struct vhost_vring_state s; 299 u32 idx; 300 long r; 301 302 r = get_user(idx, (u32 __user *)argp); 303 if (r < 0) 304 return r; 305 306 if (idx >= v->nvqs) 307 return -ENOBUFS; 308 309 idx = array_index_nospec(idx, v->nvqs); 310 vq = &v->vqs[idx]; 311 312 if (cmd == VHOST_VDPA_SET_VRING_ENABLE) { 313 if (copy_from_user(&s, argp, sizeof(s))) 314 return -EFAULT; 315 ops->set_vq_ready(vdpa, idx, s.num); 316 return 0; 317 } 318 319 if (cmd == VHOST_GET_VRING_BASE) 320 vq->last_avail_idx = ops->get_vq_state(v->vdpa, idx); 321 322 r = vhost_vring_ioctl(&v->vdev, cmd, argp); 323 if (r) 324 return r; 325 326 switch (cmd) { 327 case VHOST_SET_VRING_ADDR: 328 if (ops->set_vq_address(vdpa, idx, 329 (u64)(uintptr_t)vq->desc, 330 (u64)(uintptr_t)vq->avail, 331 (u64)(uintptr_t)vq->used)) 332 r = -EINVAL; 333 break; 334 335 case VHOST_SET_VRING_BASE: 336 if (ops->set_vq_state(vdpa, idx, vq->last_avail_idx)) 337 r = -EINVAL; 338 break; 339 340 case VHOST_SET_VRING_CALL: 341 if (vq->call_ctx) { 342 cb.callback = vhost_vdpa_virtqueue_cb; 343 cb.private = vq; 344 } else { 345 cb.callback = NULL; 346 cb.private = NULL; 347 } 348 ops->set_vq_cb(vdpa, idx, &cb); 349 break; 350 351 case VHOST_SET_VRING_NUM: 352 ops->set_vq_num(vdpa, idx, vq->num); 353 break; 354 } 355 356 return r; 357 } 358 359 static long vhost_vdpa_unlocked_ioctl(struct file *filep, 360 unsigned int cmd, unsigned long arg) 361 { 362 struct vhost_vdpa *v = filep->private_data; 363 struct vhost_dev *d = &v->vdev; 364 void __user *argp = (void __user *)arg; 365 long r; 366 367 mutex_lock(&d->mutex); 368 369 switch (cmd) { 370 case VHOST_VDPA_GET_DEVICE_ID: 371 r = vhost_vdpa_get_device_id(v, argp); 372 break; 373 case VHOST_VDPA_GET_STATUS: 374 r = vhost_vdpa_get_status(v, argp); 375 break; 376 case VHOST_VDPA_SET_STATUS: 377 r = vhost_vdpa_set_status(v, argp); 378 break; 379 case VHOST_VDPA_GET_CONFIG: 380 r = vhost_vdpa_get_config(v, argp); 381 break; 382 case VHOST_VDPA_SET_CONFIG: 383 r = vhost_vdpa_set_config(v, argp); 384 break; 385 case VHOST_GET_FEATURES: 386 r = vhost_vdpa_get_features(v, argp); 387 break; 388 case VHOST_SET_FEATURES: 389 r = vhost_vdpa_set_features(v, argp); 390 break; 391 case VHOST_VDPA_GET_VRING_NUM: 392 r = vhost_vdpa_get_vring_num(v, argp); 393 break; 394 case VHOST_SET_LOG_BASE: 395 case VHOST_SET_LOG_FD: 396 r = -ENOIOCTLCMD; 397 break; 398 default: 399 r = vhost_dev_ioctl(&v->vdev, cmd, argp); 400 if (r == -ENOIOCTLCMD) 401 r = vhost_vdpa_vring_ioctl(v, cmd, argp); 402 break; 403 } 404 405 mutex_unlock(&d->mutex); 406 return r; 407 } 408 409 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, u64 start, u64 last) 410 { 411 struct vhost_dev *dev = &v->vdev; 412 struct vhost_iotlb *iotlb = dev->iotlb; 413 struct vhost_iotlb_map *map; 414 struct page *page; 415 unsigned long pfn, pinned; 416 417 while ((map = vhost_iotlb_itree_first(iotlb, start, last)) != NULL) { 418 pinned = map->size >> PAGE_SHIFT; 419 for (pfn = map->addr >> PAGE_SHIFT; 420 pinned > 0; pfn++, pinned--) { 421 page = pfn_to_page(pfn); 422 if (map->perm & VHOST_ACCESS_WO) 423 set_page_dirty_lock(page); 424 unpin_user_page(page); 425 } 426 atomic64_sub(map->size >> PAGE_SHIFT, &dev->mm->pinned_vm); 427 vhost_iotlb_map_free(iotlb, map); 428 } 429 } 430 431 static void vhost_vdpa_iotlb_free(struct vhost_vdpa *v) 432 { 433 struct vhost_dev *dev = &v->vdev; 434 435 vhost_vdpa_iotlb_unmap(v, 0ULL, 0ULL - 1); 436 kfree(dev->iotlb); 437 dev->iotlb = NULL; 438 } 439 440 static int perm_to_iommu_flags(u32 perm) 441 { 442 int flags = 0; 443 444 switch (perm) { 445 case VHOST_ACCESS_WO: 446 flags |= IOMMU_WRITE; 447 break; 448 case VHOST_ACCESS_RO: 449 flags |= IOMMU_READ; 450 break; 451 case VHOST_ACCESS_RW: 452 flags |= (IOMMU_WRITE | IOMMU_READ); 453 break; 454 default: 455 WARN(1, "invalidate vhost IOTLB permission\n"); 456 break; 457 } 458 459 return flags | IOMMU_CACHE; 460 } 461 462 static int vhost_vdpa_map(struct vhost_vdpa *v, 463 u64 iova, u64 size, u64 pa, u32 perm) 464 { 465 struct vhost_dev *dev = &v->vdev; 466 struct vdpa_device *vdpa = v->vdpa; 467 const struct vdpa_config_ops *ops = vdpa->config; 468 int r = 0; 469 470 r = vhost_iotlb_add_range(dev->iotlb, iova, iova + size - 1, 471 pa, perm); 472 if (r) 473 return r; 474 475 if (ops->dma_map) 476 r = ops->dma_map(vdpa, iova, size, pa, perm); 477 else if (ops->set_map) 478 r = ops->set_map(vdpa, dev->iotlb); 479 else 480 r = iommu_map(v->domain, iova, pa, size, 481 perm_to_iommu_flags(perm)); 482 483 return r; 484 } 485 486 static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size) 487 { 488 struct vhost_dev *dev = &v->vdev; 489 struct vdpa_device *vdpa = v->vdpa; 490 const struct vdpa_config_ops *ops = vdpa->config; 491 492 vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1); 493 494 if (ops->dma_map) 495 ops->dma_unmap(vdpa, iova, size); 496 else if (ops->set_map) 497 ops->set_map(vdpa, dev->iotlb); 498 else 499 iommu_unmap(v->domain, iova, size); 500 } 501 502 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v, 503 struct vhost_iotlb_msg *msg) 504 { 505 struct vhost_dev *dev = &v->vdev; 506 struct vhost_iotlb *iotlb = dev->iotlb; 507 struct page **page_list; 508 unsigned long list_size = PAGE_SIZE / sizeof(struct page *); 509 unsigned int gup_flags = FOLL_LONGTERM; 510 unsigned long npages, cur_base, map_pfn, last_pfn = 0; 511 unsigned long locked, lock_limit, pinned, i; 512 u64 iova = msg->iova; 513 int ret = 0; 514 515 if (vhost_iotlb_itree_first(iotlb, msg->iova, 516 msg->iova + msg->size - 1)) 517 return -EEXIST; 518 519 page_list = (struct page **) __get_free_page(GFP_KERNEL); 520 if (!page_list) 521 return -ENOMEM; 522 523 if (msg->perm & VHOST_ACCESS_WO) 524 gup_flags |= FOLL_WRITE; 525 526 npages = PAGE_ALIGN(msg->size + (iova & ~PAGE_MASK)) >> PAGE_SHIFT; 527 if (!npages) 528 return -EINVAL; 529 530 down_read(&dev->mm->mmap_sem); 531 532 locked = atomic64_add_return(npages, &dev->mm->pinned_vm); 533 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; 534 535 if (locked > lock_limit) { 536 ret = -ENOMEM; 537 goto out; 538 } 539 540 cur_base = msg->uaddr & PAGE_MASK; 541 iova &= PAGE_MASK; 542 543 while (npages) { 544 pinned = min_t(unsigned long, npages, list_size); 545 ret = pin_user_pages(cur_base, pinned, 546 gup_flags, page_list, NULL); 547 if (ret != pinned) 548 goto out; 549 550 if (!last_pfn) 551 map_pfn = page_to_pfn(page_list[0]); 552 553 for (i = 0; i < ret; i++) { 554 unsigned long this_pfn = page_to_pfn(page_list[i]); 555 u64 csize; 556 557 if (last_pfn && (this_pfn != last_pfn + 1)) { 558 /* Pin a contiguous chunk of memory */ 559 csize = (last_pfn - map_pfn + 1) << PAGE_SHIFT; 560 if (vhost_vdpa_map(v, iova, csize, 561 map_pfn << PAGE_SHIFT, 562 msg->perm)) 563 goto out; 564 map_pfn = this_pfn; 565 iova += csize; 566 } 567 568 last_pfn = this_pfn; 569 } 570 571 cur_base += ret << PAGE_SHIFT; 572 npages -= ret; 573 } 574 575 /* Pin the rest chunk */ 576 ret = vhost_vdpa_map(v, iova, (last_pfn - map_pfn + 1) << PAGE_SHIFT, 577 map_pfn << PAGE_SHIFT, msg->perm); 578 out: 579 if (ret) { 580 vhost_vdpa_unmap(v, msg->iova, msg->size); 581 atomic64_sub(npages, &dev->mm->pinned_vm); 582 } 583 up_read(&dev->mm->mmap_sem); 584 free_page((unsigned long)page_list); 585 return ret; 586 } 587 588 static int vhost_vdpa_process_iotlb_msg(struct vhost_dev *dev, 589 struct vhost_iotlb_msg *msg) 590 { 591 struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev); 592 int r = 0; 593 594 r = vhost_dev_check_owner(dev); 595 if (r) 596 return r; 597 598 switch (msg->type) { 599 case VHOST_IOTLB_UPDATE: 600 r = vhost_vdpa_process_iotlb_update(v, msg); 601 break; 602 case VHOST_IOTLB_INVALIDATE: 603 vhost_vdpa_unmap(v, msg->iova, msg->size); 604 break; 605 default: 606 r = -EINVAL; 607 break; 608 } 609 610 return r; 611 } 612 613 static ssize_t vhost_vdpa_chr_write_iter(struct kiocb *iocb, 614 struct iov_iter *from) 615 { 616 struct file *file = iocb->ki_filp; 617 struct vhost_vdpa *v = file->private_data; 618 struct vhost_dev *dev = &v->vdev; 619 620 return vhost_chr_write_iter(dev, from); 621 } 622 623 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v) 624 { 625 struct vdpa_device *vdpa = v->vdpa; 626 const struct vdpa_config_ops *ops = vdpa->config; 627 struct device *dma_dev = vdpa_get_dma_dev(vdpa); 628 struct bus_type *bus; 629 int ret; 630 631 /* Device want to do DMA by itself */ 632 if (ops->set_map || ops->dma_map) 633 return 0; 634 635 bus = dma_dev->bus; 636 if (!bus) 637 return -EFAULT; 638 639 if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY)) 640 return -ENOTSUPP; 641 642 v->domain = iommu_domain_alloc(bus); 643 if (!v->domain) 644 return -EIO; 645 646 ret = iommu_attach_device(v->domain, dma_dev); 647 if (ret) 648 goto err_attach; 649 650 return 0; 651 652 err_attach: 653 iommu_domain_free(v->domain); 654 return ret; 655 } 656 657 static void vhost_vdpa_free_domain(struct vhost_vdpa *v) 658 { 659 struct vdpa_device *vdpa = v->vdpa; 660 struct device *dma_dev = vdpa_get_dma_dev(vdpa); 661 662 if (v->domain) { 663 iommu_detach_device(v->domain, dma_dev); 664 iommu_domain_free(v->domain); 665 } 666 667 v->domain = NULL; 668 } 669 670 static int vhost_vdpa_open(struct inode *inode, struct file *filep) 671 { 672 struct vhost_vdpa *v; 673 struct vhost_dev *dev; 674 struct vhost_virtqueue **vqs; 675 int nvqs, i, r, opened; 676 677 v = container_of(inode->i_cdev, struct vhost_vdpa, cdev); 678 679 opened = atomic_cmpxchg(&v->opened, 0, 1); 680 if (opened) 681 return -EBUSY; 682 683 nvqs = v->nvqs; 684 vhost_vdpa_reset(v); 685 686 vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL); 687 if (!vqs) { 688 r = -ENOMEM; 689 goto err; 690 } 691 692 dev = &v->vdev; 693 for (i = 0; i < nvqs; i++) { 694 vqs[i] = &v->vqs[i]; 695 vqs[i]->handle_kick = handle_vq_kick; 696 } 697 vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, 698 vhost_vdpa_process_iotlb_msg); 699 700 dev->iotlb = vhost_iotlb_alloc(0, 0); 701 if (!dev->iotlb) { 702 r = -ENOMEM; 703 goto err_init_iotlb; 704 } 705 706 r = vhost_vdpa_alloc_domain(v); 707 if (r) 708 goto err_init_iotlb; 709 710 filep->private_data = v; 711 712 return 0; 713 714 err_init_iotlb: 715 vhost_dev_cleanup(&v->vdev); 716 err: 717 atomic_dec(&v->opened); 718 return r; 719 } 720 721 static int vhost_vdpa_release(struct inode *inode, struct file *filep) 722 { 723 struct vhost_vdpa *v = filep->private_data; 724 struct vhost_dev *d = &v->vdev; 725 726 mutex_lock(&d->mutex); 727 filep->private_data = NULL; 728 vhost_vdpa_reset(v); 729 vhost_dev_stop(&v->vdev); 730 vhost_vdpa_iotlb_free(v); 731 vhost_vdpa_free_domain(v); 732 vhost_dev_cleanup(&v->vdev); 733 kfree(v->vdev.vqs); 734 mutex_unlock(&d->mutex); 735 736 atomic_dec(&v->opened); 737 complete(&v->completion); 738 739 return 0; 740 } 741 742 static const struct file_operations vhost_vdpa_fops = { 743 .owner = THIS_MODULE, 744 .open = vhost_vdpa_open, 745 .release = vhost_vdpa_release, 746 .write_iter = vhost_vdpa_chr_write_iter, 747 .unlocked_ioctl = vhost_vdpa_unlocked_ioctl, 748 .compat_ioctl = compat_ptr_ioctl, 749 }; 750 751 static void vhost_vdpa_release_dev(struct device *device) 752 { 753 struct vhost_vdpa *v = 754 container_of(device, struct vhost_vdpa, dev); 755 756 ida_simple_remove(&vhost_vdpa_ida, v->minor); 757 kfree(v->vqs); 758 kfree(v); 759 } 760 761 static int vhost_vdpa_probe(struct vdpa_device *vdpa) 762 { 763 const struct vdpa_config_ops *ops = vdpa->config; 764 struct vhost_vdpa *v; 765 int minor, nvqs = VHOST_VDPA_VQ_MAX; 766 int r; 767 768 /* Currently, we only accept the network devices. */ 769 if (ops->get_device_id(vdpa) != VIRTIO_ID_NET) 770 return -ENOTSUPP; 771 772 v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL); 773 if (!v) 774 return -ENOMEM; 775 776 minor = ida_simple_get(&vhost_vdpa_ida, 0, 777 VHOST_VDPA_DEV_MAX, GFP_KERNEL); 778 if (minor < 0) { 779 kfree(v); 780 return minor; 781 } 782 783 atomic_set(&v->opened, 0); 784 v->minor = minor; 785 v->vdpa = vdpa; 786 v->nvqs = nvqs; 787 v->virtio_id = ops->get_device_id(vdpa); 788 789 device_initialize(&v->dev); 790 v->dev.release = vhost_vdpa_release_dev; 791 v->dev.parent = &vdpa->dev; 792 v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor); 793 v->vqs = kmalloc_array(nvqs, sizeof(struct vhost_virtqueue), 794 GFP_KERNEL); 795 if (!v->vqs) { 796 r = -ENOMEM; 797 goto err; 798 } 799 800 r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor); 801 if (r) 802 goto err; 803 804 cdev_init(&v->cdev, &vhost_vdpa_fops); 805 v->cdev.owner = THIS_MODULE; 806 807 r = cdev_device_add(&v->cdev, &v->dev); 808 if (r) 809 goto err; 810 811 init_completion(&v->completion); 812 vdpa_set_drvdata(vdpa, v); 813 814 return 0; 815 816 err: 817 put_device(&v->dev); 818 return r; 819 } 820 821 static void vhost_vdpa_remove(struct vdpa_device *vdpa) 822 { 823 struct vhost_vdpa *v = vdpa_get_drvdata(vdpa); 824 int opened; 825 826 cdev_device_del(&v->cdev, &v->dev); 827 828 do { 829 opened = atomic_cmpxchg(&v->opened, 0, 1); 830 if (!opened) 831 break; 832 wait_for_completion(&v->completion); 833 } while (1); 834 835 put_device(&v->dev); 836 } 837 838 static struct vdpa_driver vhost_vdpa_driver = { 839 .driver = { 840 .name = "vhost_vdpa", 841 }, 842 .probe = vhost_vdpa_probe, 843 .remove = vhost_vdpa_remove, 844 }; 845 846 static int __init vhost_vdpa_init(void) 847 { 848 int r; 849 850 r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX, 851 "vhost-vdpa"); 852 if (r) 853 goto err_alloc_chrdev; 854 855 r = vdpa_register_driver(&vhost_vdpa_driver); 856 if (r) 857 goto err_vdpa_register_driver; 858 859 return 0; 860 861 err_vdpa_register_driver: 862 unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX); 863 err_alloc_chrdev: 864 return r; 865 } 866 module_init(vhost_vdpa_init); 867 868 static void __exit vhost_vdpa_exit(void) 869 { 870 vdpa_unregister_driver(&vhost_vdpa_driver); 871 unregister_chrdev_region(vhost_vdpa_major, VHOST_VDPA_DEV_MAX); 872 } 873 module_exit(vhost_vdpa_exit); 874 875 MODULE_VERSION("0.0.1"); 876 MODULE_LICENSE("GPL v2"); 877 MODULE_AUTHOR("Intel Corporation"); 878 MODULE_DESCRIPTION("vDPA-based vhost backend for virtio"); 879