1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * virtio-fs: Virtio Filesystem 4 * Copyright (C) 2018 Red Hat, Inc. 5 */ 6 7 #include <linux/fs.h> 8 #include <linux/dax.h> 9 #include <linux/pci.h> 10 #include <linux/pfn_t.h> 11 #include <linux/module.h> 12 #include <linux/virtio.h> 13 #include <linux/virtio_fs.h> 14 #include <linux/delay.h> 15 #include <linux/fs_context.h> 16 #include <linux/fs_parser.h> 17 #include <linux/highmem.h> 18 #include <linux/uio.h> 19 #include "fuse_i.h" 20 21 /* List of virtio-fs device instances and a lock for the list. Also provides 22 * mutual exclusion in device removal and mounting path 23 */ 24 static DEFINE_MUTEX(virtio_fs_mutex); 25 static LIST_HEAD(virtio_fs_instances); 26 27 enum { 28 VQ_HIPRIO, 29 VQ_REQUEST 30 }; 31 32 #define VQ_NAME_LEN 24 33 34 /* Per-virtqueue state */ 35 struct virtio_fs_vq { 36 spinlock_t lock; 37 struct virtqueue *vq; /* protected by ->lock */ 38 struct work_struct done_work; 39 struct list_head queued_reqs; 40 struct list_head end_reqs; /* End these requests */ 41 struct delayed_work dispatch_work; 42 struct fuse_dev *fud; 43 bool connected; 44 long in_flight; 45 struct completion in_flight_zero; /* No inflight requests */ 46 char name[VQ_NAME_LEN]; 47 } ____cacheline_aligned_in_smp; 48 49 /* A virtio-fs device instance */ 50 struct virtio_fs { 51 struct kref refcount; 52 struct list_head list; /* on virtio_fs_instances */ 53 char *tag; 54 struct virtio_fs_vq *vqs; 55 unsigned int nvqs; /* number of virtqueues */ 56 unsigned int num_request_queues; /* number of request queues */ 57 struct dax_device *dax_dev; 58 59 /* DAX memory window where file contents are mapped */ 60 void *window_kaddr; 61 phys_addr_t window_phys_addr; 62 size_t window_len; 63 }; 64 65 struct virtio_fs_forget_req { 66 struct fuse_in_header ih; 67 struct fuse_forget_in arg; 68 }; 69 70 struct virtio_fs_forget { 71 /* This request can be temporarily queued on virt queue */ 72 struct list_head list; 73 struct virtio_fs_forget_req req; 74 }; 75 76 struct virtio_fs_req_work { 77 struct fuse_req *req; 78 struct virtio_fs_vq *fsvq; 79 struct work_struct done_work; 80 }; 81 82 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 83 struct fuse_req *req, bool in_flight); 84 85 enum { 86 OPT_DAX, 87 }; 88 89 static const struct fs_parameter_spec virtio_fs_parameters[] = { 90 fsparam_flag("dax", OPT_DAX), 91 {} 92 }; 93 94 static int virtio_fs_parse_param(struct fs_context *fc, 95 struct fs_parameter *param) 96 { 97 struct fs_parse_result result; 98 struct fuse_fs_context *ctx = fc->fs_private; 99 int opt; 100 101 opt = fs_parse(fc, virtio_fs_parameters, param, &result); 102 if (opt < 0) 103 return opt; 104 105 switch (opt) { 106 case OPT_DAX: 107 ctx->dax = 1; 108 break; 109 default: 110 return -EINVAL; 111 } 112 113 return 0; 114 } 115 116 static void virtio_fs_free_fc(struct fs_context *fc) 117 { 118 struct fuse_fs_context *ctx = fc->fs_private; 119 120 kfree(ctx); 121 } 122 123 static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) 124 { 125 struct virtio_fs *fs = vq->vdev->priv; 126 127 return &fs->vqs[vq->index]; 128 } 129 130 static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq) 131 { 132 return &vq_to_fsvq(vq)->fud->pq; 133 } 134 135 /* Should be called with fsvq->lock held. */ 136 static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq) 137 { 138 fsvq->in_flight++; 139 } 140 141 /* Should be called with fsvq->lock held. */ 142 static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) 143 { 144 WARN_ON(fsvq->in_flight <= 0); 145 fsvq->in_flight--; 146 if (!fsvq->in_flight) 147 complete(&fsvq->in_flight_zero); 148 } 149 150 static void release_virtio_fs_obj(struct kref *ref) 151 { 152 struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount); 153 154 kfree(vfs->vqs); 155 kfree(vfs); 156 } 157 158 /* Make sure virtiofs_mutex is held */ 159 static void virtio_fs_put(struct virtio_fs *fs) 160 { 161 kref_put(&fs->refcount, release_virtio_fs_obj); 162 } 163 164 static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) 165 { 166 struct virtio_fs *vfs = fiq->priv; 167 168 mutex_lock(&virtio_fs_mutex); 169 virtio_fs_put(vfs); 170 mutex_unlock(&virtio_fs_mutex); 171 } 172 173 static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) 174 { 175 WARN_ON(fsvq->in_flight < 0); 176 177 /* Wait for in flight requests to finish.*/ 178 spin_lock(&fsvq->lock); 179 if (fsvq->in_flight) { 180 /* We are holding virtio_fs_mutex. There should not be any 181 * waiters waiting for completion. 182 */ 183 reinit_completion(&fsvq->in_flight_zero); 184 spin_unlock(&fsvq->lock); 185 wait_for_completion(&fsvq->in_flight_zero); 186 } else { 187 spin_unlock(&fsvq->lock); 188 } 189 190 flush_work(&fsvq->done_work); 191 flush_delayed_work(&fsvq->dispatch_work); 192 } 193 194 static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs) 195 { 196 struct virtio_fs_vq *fsvq; 197 int i; 198 199 for (i = 0; i < fs->nvqs; i++) { 200 fsvq = &fs->vqs[i]; 201 virtio_fs_drain_queue(fsvq); 202 } 203 } 204 205 static void virtio_fs_drain_all_queues(struct virtio_fs *fs) 206 { 207 /* Provides mutual exclusion between ->remove and ->kill_sb 208 * paths. We don't want both of these draining queue at the 209 * same time. Current completion logic reinits completion 210 * and that means there should not be any other thread 211 * doing reinit or waiting for completion already. 212 */ 213 mutex_lock(&virtio_fs_mutex); 214 virtio_fs_drain_all_queues_locked(fs); 215 mutex_unlock(&virtio_fs_mutex); 216 } 217 218 static void virtio_fs_start_all_queues(struct virtio_fs *fs) 219 { 220 struct virtio_fs_vq *fsvq; 221 int i; 222 223 for (i = 0; i < fs->nvqs; i++) { 224 fsvq = &fs->vqs[i]; 225 spin_lock(&fsvq->lock); 226 fsvq->connected = true; 227 spin_unlock(&fsvq->lock); 228 } 229 } 230 231 /* Add a new instance to the list or return -EEXIST if tag name exists*/ 232 static int virtio_fs_add_instance(struct virtio_fs *fs) 233 { 234 struct virtio_fs *fs2; 235 bool duplicate = false; 236 237 mutex_lock(&virtio_fs_mutex); 238 239 list_for_each_entry(fs2, &virtio_fs_instances, list) { 240 if (strcmp(fs->tag, fs2->tag) == 0) 241 duplicate = true; 242 } 243 244 if (!duplicate) 245 list_add_tail(&fs->list, &virtio_fs_instances); 246 247 mutex_unlock(&virtio_fs_mutex); 248 249 if (duplicate) 250 return -EEXIST; 251 return 0; 252 } 253 254 /* Return the virtio_fs with a given tag, or NULL */ 255 static struct virtio_fs *virtio_fs_find_instance(const char *tag) 256 { 257 struct virtio_fs *fs; 258 259 mutex_lock(&virtio_fs_mutex); 260 261 list_for_each_entry(fs, &virtio_fs_instances, list) { 262 if (strcmp(fs->tag, tag) == 0) { 263 kref_get(&fs->refcount); 264 goto found; 265 } 266 } 267 268 fs = NULL; /* not found */ 269 270 found: 271 mutex_unlock(&virtio_fs_mutex); 272 273 return fs; 274 } 275 276 static void virtio_fs_free_devs(struct virtio_fs *fs) 277 { 278 unsigned int i; 279 280 for (i = 0; i < fs->nvqs; i++) { 281 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 282 283 if (!fsvq->fud) 284 continue; 285 286 fuse_dev_free(fsvq->fud); 287 fsvq->fud = NULL; 288 } 289 } 290 291 /* Read filesystem name from virtio config into fs->tag (must kfree()). */ 292 static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs) 293 { 294 char tag_buf[sizeof_field(struct virtio_fs_config, tag)]; 295 char *end; 296 size_t len; 297 298 virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag), 299 &tag_buf, sizeof(tag_buf)); 300 end = memchr(tag_buf, '\0', sizeof(tag_buf)); 301 if (end == tag_buf) 302 return -EINVAL; /* empty tag */ 303 if (!end) 304 end = &tag_buf[sizeof(tag_buf)]; 305 306 len = end - tag_buf; 307 fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); 308 if (!fs->tag) 309 return -ENOMEM; 310 memcpy(fs->tag, tag_buf, len); 311 fs->tag[len] = '\0'; 312 return 0; 313 } 314 315 /* Work function for hiprio completion */ 316 static void virtio_fs_hiprio_done_work(struct work_struct *work) 317 { 318 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 319 done_work); 320 struct virtqueue *vq = fsvq->vq; 321 322 /* Free completed FUSE_FORGET requests */ 323 spin_lock(&fsvq->lock); 324 do { 325 unsigned int len; 326 void *req; 327 328 virtqueue_disable_cb(vq); 329 330 while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 331 kfree(req); 332 dec_in_flight_req(fsvq); 333 } 334 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 335 spin_unlock(&fsvq->lock); 336 } 337 338 static void virtio_fs_request_dispatch_work(struct work_struct *work) 339 { 340 struct fuse_req *req; 341 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 342 dispatch_work.work); 343 int ret; 344 345 pr_debug("virtio-fs: worker %s called.\n", __func__); 346 while (1) { 347 spin_lock(&fsvq->lock); 348 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, 349 list); 350 if (!req) { 351 spin_unlock(&fsvq->lock); 352 break; 353 } 354 355 list_del_init(&req->list); 356 spin_unlock(&fsvq->lock); 357 fuse_request_end(req); 358 } 359 360 /* Dispatch pending requests */ 361 while (1) { 362 spin_lock(&fsvq->lock); 363 req = list_first_entry_or_null(&fsvq->queued_reqs, 364 struct fuse_req, list); 365 if (!req) { 366 spin_unlock(&fsvq->lock); 367 return; 368 } 369 list_del_init(&req->list); 370 spin_unlock(&fsvq->lock); 371 372 ret = virtio_fs_enqueue_req(fsvq, req, true); 373 if (ret < 0) { 374 if (ret == -ENOMEM || ret == -ENOSPC) { 375 spin_lock(&fsvq->lock); 376 list_add_tail(&req->list, &fsvq->queued_reqs); 377 schedule_delayed_work(&fsvq->dispatch_work, 378 msecs_to_jiffies(1)); 379 spin_unlock(&fsvq->lock); 380 return; 381 } 382 req->out.h.error = ret; 383 spin_lock(&fsvq->lock); 384 dec_in_flight_req(fsvq); 385 spin_unlock(&fsvq->lock); 386 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", 387 ret); 388 fuse_request_end(req); 389 } 390 } 391 } 392 393 /* 394 * Returns 1 if queue is full and sender should wait a bit before sending 395 * next request, 0 otherwise. 396 */ 397 static int send_forget_request(struct virtio_fs_vq *fsvq, 398 struct virtio_fs_forget *forget, 399 bool in_flight) 400 { 401 struct scatterlist sg; 402 struct virtqueue *vq; 403 int ret = 0; 404 bool notify; 405 struct virtio_fs_forget_req *req = &forget->req; 406 407 spin_lock(&fsvq->lock); 408 if (!fsvq->connected) { 409 if (in_flight) 410 dec_in_flight_req(fsvq); 411 kfree(forget); 412 goto out; 413 } 414 415 sg_init_one(&sg, req, sizeof(*req)); 416 vq = fsvq->vq; 417 dev_dbg(&vq->vdev->dev, "%s\n", __func__); 418 419 ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); 420 if (ret < 0) { 421 if (ret == -ENOMEM || ret == -ENOSPC) { 422 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", 423 ret); 424 list_add_tail(&forget->list, &fsvq->queued_reqs); 425 schedule_delayed_work(&fsvq->dispatch_work, 426 msecs_to_jiffies(1)); 427 if (!in_flight) 428 inc_in_flight_req(fsvq); 429 /* Queue is full */ 430 ret = 1; 431 } else { 432 pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", 433 ret); 434 kfree(forget); 435 if (in_flight) 436 dec_in_flight_req(fsvq); 437 } 438 goto out; 439 } 440 441 if (!in_flight) 442 inc_in_flight_req(fsvq); 443 notify = virtqueue_kick_prepare(vq); 444 spin_unlock(&fsvq->lock); 445 446 if (notify) 447 virtqueue_notify(vq); 448 return ret; 449 out: 450 spin_unlock(&fsvq->lock); 451 return ret; 452 } 453 454 static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) 455 { 456 struct virtio_fs_forget *forget; 457 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 458 dispatch_work.work); 459 pr_debug("virtio-fs: worker %s called.\n", __func__); 460 while (1) { 461 spin_lock(&fsvq->lock); 462 forget = list_first_entry_or_null(&fsvq->queued_reqs, 463 struct virtio_fs_forget, list); 464 if (!forget) { 465 spin_unlock(&fsvq->lock); 466 return; 467 } 468 469 list_del(&forget->list); 470 spin_unlock(&fsvq->lock); 471 if (send_forget_request(fsvq, forget, true)) 472 return; 473 } 474 } 475 476 /* Allocate and copy args into req->argbuf */ 477 static int copy_args_to_argbuf(struct fuse_req *req) 478 { 479 struct fuse_args *args = req->args; 480 unsigned int offset = 0; 481 unsigned int num_in; 482 unsigned int num_out; 483 unsigned int len; 484 unsigned int i; 485 486 num_in = args->in_numargs - args->in_pages; 487 num_out = args->out_numargs - args->out_pages; 488 len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + 489 fuse_len_args(num_out, args->out_args); 490 491 req->argbuf = kmalloc(len, GFP_ATOMIC); 492 if (!req->argbuf) 493 return -ENOMEM; 494 495 for (i = 0; i < num_in; i++) { 496 memcpy(req->argbuf + offset, 497 args->in_args[i].value, 498 args->in_args[i].size); 499 offset += args->in_args[i].size; 500 } 501 502 return 0; 503 } 504 505 /* Copy args out of and free req->argbuf */ 506 static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req) 507 { 508 unsigned int remaining; 509 unsigned int offset; 510 unsigned int num_in; 511 unsigned int num_out; 512 unsigned int i; 513 514 remaining = req->out.h.len - sizeof(req->out.h); 515 num_in = args->in_numargs - args->in_pages; 516 num_out = args->out_numargs - args->out_pages; 517 offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); 518 519 for (i = 0; i < num_out; i++) { 520 unsigned int argsize = args->out_args[i].size; 521 522 if (args->out_argvar && 523 i == args->out_numargs - 1 && 524 argsize > remaining) { 525 argsize = remaining; 526 } 527 528 memcpy(args->out_args[i].value, req->argbuf + offset, argsize); 529 offset += argsize; 530 531 if (i != args->out_numargs - 1) 532 remaining -= argsize; 533 } 534 535 /* Store the actual size of the variable-length arg */ 536 if (args->out_argvar) 537 args->out_args[args->out_numargs - 1].size = remaining; 538 539 kfree(req->argbuf); 540 req->argbuf = NULL; 541 } 542 543 /* Work function for request completion */ 544 static void virtio_fs_request_complete(struct fuse_req *req, 545 struct virtio_fs_vq *fsvq) 546 { 547 struct fuse_pqueue *fpq = &fsvq->fud->pq; 548 struct fuse_args *args; 549 struct fuse_args_pages *ap; 550 unsigned int len, i, thislen; 551 struct page *page; 552 553 /* 554 * TODO verify that server properly follows FUSE protocol 555 * (oh.uniq, oh.len) 556 */ 557 args = req->args; 558 copy_args_from_argbuf(args, req); 559 560 if (args->out_pages && args->page_zeroing) { 561 len = args->out_args[args->out_numargs - 1].size; 562 ap = container_of(args, typeof(*ap), args); 563 for (i = 0; i < ap->num_pages; i++) { 564 thislen = ap->descs[i].length; 565 if (len < thislen) { 566 WARN_ON(ap->descs[i].offset); 567 page = ap->pages[i]; 568 zero_user_segment(page, len, thislen); 569 len = 0; 570 } else { 571 len -= thislen; 572 } 573 } 574 } 575 576 spin_lock(&fpq->lock); 577 clear_bit(FR_SENT, &req->flags); 578 spin_unlock(&fpq->lock); 579 580 fuse_request_end(req); 581 spin_lock(&fsvq->lock); 582 dec_in_flight_req(fsvq); 583 spin_unlock(&fsvq->lock); 584 } 585 586 static void virtio_fs_complete_req_work(struct work_struct *work) 587 { 588 struct virtio_fs_req_work *w = 589 container_of(work, typeof(*w), done_work); 590 591 virtio_fs_request_complete(w->req, w->fsvq); 592 kfree(w); 593 } 594 595 static void virtio_fs_requests_done_work(struct work_struct *work) 596 { 597 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 598 done_work); 599 struct fuse_pqueue *fpq = &fsvq->fud->pq; 600 struct virtqueue *vq = fsvq->vq; 601 struct fuse_req *req; 602 struct fuse_req *next; 603 unsigned int len; 604 LIST_HEAD(reqs); 605 606 /* Collect completed requests off the virtqueue */ 607 spin_lock(&fsvq->lock); 608 do { 609 virtqueue_disable_cb(vq); 610 611 while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 612 spin_lock(&fpq->lock); 613 list_move_tail(&req->list, &reqs); 614 spin_unlock(&fpq->lock); 615 } 616 } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 617 spin_unlock(&fsvq->lock); 618 619 /* End requests */ 620 list_for_each_entry_safe(req, next, &reqs, list) { 621 list_del_init(&req->list); 622 623 /* blocking async request completes in a worker context */ 624 if (req->args->may_block) { 625 struct virtio_fs_req_work *w; 626 627 w = kzalloc(sizeof(*w), GFP_NOFS | __GFP_NOFAIL); 628 INIT_WORK(&w->done_work, virtio_fs_complete_req_work); 629 w->fsvq = fsvq; 630 w->req = req; 631 schedule_work(&w->done_work); 632 } else { 633 virtio_fs_request_complete(req, fsvq); 634 } 635 } 636 } 637 638 /* Virtqueue interrupt handler */ 639 static void virtio_fs_vq_done(struct virtqueue *vq) 640 { 641 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq); 642 643 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); 644 645 schedule_work(&fsvq->done_work); 646 } 647 648 static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name, 649 int vq_type) 650 { 651 strncpy(fsvq->name, name, VQ_NAME_LEN); 652 spin_lock_init(&fsvq->lock); 653 INIT_LIST_HEAD(&fsvq->queued_reqs); 654 INIT_LIST_HEAD(&fsvq->end_reqs); 655 init_completion(&fsvq->in_flight_zero); 656 657 if (vq_type == VQ_REQUEST) { 658 INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work); 659 INIT_DELAYED_WORK(&fsvq->dispatch_work, 660 virtio_fs_request_dispatch_work); 661 } else { 662 INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work); 663 INIT_DELAYED_WORK(&fsvq->dispatch_work, 664 virtio_fs_hiprio_dispatch_work); 665 } 666 } 667 668 /* Initialize virtqueues */ 669 static int virtio_fs_setup_vqs(struct virtio_device *vdev, 670 struct virtio_fs *fs) 671 { 672 struct virtqueue **vqs; 673 vq_callback_t **callbacks; 674 const char **names; 675 unsigned int i; 676 int ret = 0; 677 678 virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues, 679 &fs->num_request_queues); 680 if (fs->num_request_queues == 0) 681 return -EINVAL; 682 683 fs->nvqs = VQ_REQUEST + fs->num_request_queues; 684 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); 685 if (!fs->vqs) 686 return -ENOMEM; 687 688 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); 689 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]), 690 GFP_KERNEL); 691 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL); 692 if (!vqs || !callbacks || !names) { 693 ret = -ENOMEM; 694 goto out; 695 } 696 697 /* Initialize the hiprio/forget request virtqueue */ 698 callbacks[VQ_HIPRIO] = virtio_fs_vq_done; 699 virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO); 700 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; 701 702 /* Initialize the requests virtqueues */ 703 for (i = VQ_REQUEST; i < fs->nvqs; i++) { 704 char vq_name[VQ_NAME_LEN]; 705 706 snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST); 707 virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST); 708 callbacks[i] = virtio_fs_vq_done; 709 names[i] = fs->vqs[i].name; 710 } 711 712 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL); 713 if (ret < 0) 714 goto out; 715 716 for (i = 0; i < fs->nvqs; i++) 717 fs->vqs[i].vq = vqs[i]; 718 719 virtio_fs_start_all_queues(fs); 720 out: 721 kfree(names); 722 kfree(callbacks); 723 kfree(vqs); 724 if (ret) 725 kfree(fs->vqs); 726 return ret; 727 } 728 729 /* Free virtqueues (device must already be reset) */ 730 static void virtio_fs_cleanup_vqs(struct virtio_device *vdev, 731 struct virtio_fs *fs) 732 { 733 vdev->config->del_vqs(vdev); 734 } 735 736 /* Map a window offset to a page frame number. The window offset will have 737 * been produced by .iomap_begin(), which maps a file offset to a window 738 * offset. 739 */ 740 static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 741 long nr_pages, void **kaddr, pfn_t *pfn) 742 { 743 struct virtio_fs *fs = dax_get_private(dax_dev); 744 phys_addr_t offset = PFN_PHYS(pgoff); 745 size_t max_nr_pages = fs->window_len/PAGE_SIZE - pgoff; 746 747 if (kaddr) 748 *kaddr = fs->window_kaddr + offset; 749 if (pfn) 750 *pfn = phys_to_pfn_t(fs->window_phys_addr + offset, 751 PFN_DEV | PFN_MAP); 752 return nr_pages > max_nr_pages ? max_nr_pages : nr_pages; 753 } 754 755 static size_t virtio_fs_copy_from_iter(struct dax_device *dax_dev, 756 pgoff_t pgoff, void *addr, 757 size_t bytes, struct iov_iter *i) 758 { 759 return copy_from_iter(addr, bytes, i); 760 } 761 762 static size_t virtio_fs_copy_to_iter(struct dax_device *dax_dev, 763 pgoff_t pgoff, void *addr, 764 size_t bytes, struct iov_iter *i) 765 { 766 return copy_to_iter(addr, bytes, i); 767 } 768 769 static int virtio_fs_zero_page_range(struct dax_device *dax_dev, 770 pgoff_t pgoff, size_t nr_pages) 771 { 772 long rc; 773 void *kaddr; 774 775 rc = dax_direct_access(dax_dev, pgoff, nr_pages, &kaddr, NULL); 776 if (rc < 0) 777 return rc; 778 memset(kaddr, 0, nr_pages << PAGE_SHIFT); 779 dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT); 780 return 0; 781 } 782 783 static const struct dax_operations virtio_fs_dax_ops = { 784 .direct_access = virtio_fs_direct_access, 785 .copy_from_iter = virtio_fs_copy_from_iter, 786 .copy_to_iter = virtio_fs_copy_to_iter, 787 .zero_page_range = virtio_fs_zero_page_range, 788 }; 789 790 static void virtio_fs_cleanup_dax(void *data) 791 { 792 struct dax_device *dax_dev = data; 793 794 kill_dax(dax_dev); 795 put_dax(dax_dev); 796 } 797 798 static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs) 799 { 800 struct virtio_shm_region cache_reg; 801 struct dev_pagemap *pgmap; 802 bool have_cache; 803 804 if (!IS_ENABLED(CONFIG_FUSE_DAX)) 805 return 0; 806 807 /* Get cache region */ 808 have_cache = virtio_get_shm_region(vdev, &cache_reg, 809 (u8)VIRTIO_FS_SHMCAP_ID_CACHE); 810 if (!have_cache) { 811 dev_notice(&vdev->dev, "%s: No cache capability\n", __func__); 812 return 0; 813 } 814 815 if (!devm_request_mem_region(&vdev->dev, cache_reg.addr, cache_reg.len, 816 dev_name(&vdev->dev))) { 817 dev_warn(&vdev->dev, "could not reserve region addr=0x%llx len=0x%llx\n", 818 cache_reg.addr, cache_reg.len); 819 return -EBUSY; 820 } 821 822 dev_notice(&vdev->dev, "Cache len: 0x%llx @ 0x%llx\n", cache_reg.len, 823 cache_reg.addr); 824 825 pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL); 826 if (!pgmap) 827 return -ENOMEM; 828 829 pgmap->type = MEMORY_DEVICE_FS_DAX; 830 831 /* Ideally we would directly use the PCI BAR resource but 832 * devm_memremap_pages() wants its own copy in pgmap. So 833 * initialize a struct resource from scratch (only the start 834 * and end fields will be used). 835 */ 836 pgmap->range = (struct range) { 837 .start = (phys_addr_t) cache_reg.addr, 838 .end = (phys_addr_t) cache_reg.addr + cache_reg.len - 1, 839 }; 840 pgmap->nr_range = 1; 841 842 fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap); 843 if (IS_ERR(fs->window_kaddr)) 844 return PTR_ERR(fs->window_kaddr); 845 846 fs->window_phys_addr = (phys_addr_t) cache_reg.addr; 847 fs->window_len = (phys_addr_t) cache_reg.len; 848 849 dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n", 850 __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len); 851 852 fs->dax_dev = alloc_dax(fs, NULL, &virtio_fs_dax_ops, 0); 853 if (IS_ERR(fs->dax_dev)) 854 return PTR_ERR(fs->dax_dev); 855 856 return devm_add_action_or_reset(&vdev->dev, virtio_fs_cleanup_dax, 857 fs->dax_dev); 858 } 859 860 static int virtio_fs_probe(struct virtio_device *vdev) 861 { 862 struct virtio_fs *fs; 863 int ret; 864 865 fs = kzalloc(sizeof(*fs), GFP_KERNEL); 866 if (!fs) 867 return -ENOMEM; 868 kref_init(&fs->refcount); 869 vdev->priv = fs; 870 871 ret = virtio_fs_read_tag(vdev, fs); 872 if (ret < 0) 873 goto out; 874 875 ret = virtio_fs_setup_vqs(vdev, fs); 876 if (ret < 0) 877 goto out; 878 879 /* TODO vq affinity */ 880 881 ret = virtio_fs_setup_dax(vdev, fs); 882 if (ret < 0) 883 goto out_vqs; 884 885 /* Bring the device online in case the filesystem is mounted and 886 * requests need to be sent before we return. 887 */ 888 virtio_device_ready(vdev); 889 890 ret = virtio_fs_add_instance(fs); 891 if (ret < 0) 892 goto out_vqs; 893 894 return 0; 895 896 out_vqs: 897 vdev->config->reset(vdev); 898 virtio_fs_cleanup_vqs(vdev, fs); 899 900 out: 901 vdev->priv = NULL; 902 kfree(fs); 903 return ret; 904 } 905 906 static void virtio_fs_stop_all_queues(struct virtio_fs *fs) 907 { 908 struct virtio_fs_vq *fsvq; 909 int i; 910 911 for (i = 0; i < fs->nvqs; i++) { 912 fsvq = &fs->vqs[i]; 913 spin_lock(&fsvq->lock); 914 fsvq->connected = false; 915 spin_unlock(&fsvq->lock); 916 } 917 } 918 919 static void virtio_fs_remove(struct virtio_device *vdev) 920 { 921 struct virtio_fs *fs = vdev->priv; 922 923 mutex_lock(&virtio_fs_mutex); 924 /* This device is going away. No one should get new reference */ 925 list_del_init(&fs->list); 926 virtio_fs_stop_all_queues(fs); 927 virtio_fs_drain_all_queues_locked(fs); 928 vdev->config->reset(vdev); 929 virtio_fs_cleanup_vqs(vdev, fs); 930 931 vdev->priv = NULL; 932 /* Put device reference on virtio_fs object */ 933 virtio_fs_put(fs); 934 mutex_unlock(&virtio_fs_mutex); 935 } 936 937 #ifdef CONFIG_PM_SLEEP 938 static int virtio_fs_freeze(struct virtio_device *vdev) 939 { 940 /* TODO need to save state here */ 941 pr_warn("virtio-fs: suspend/resume not yet supported\n"); 942 return -EOPNOTSUPP; 943 } 944 945 static int virtio_fs_restore(struct virtio_device *vdev) 946 { 947 /* TODO need to restore state here */ 948 return 0; 949 } 950 #endif /* CONFIG_PM_SLEEP */ 951 952 static const struct virtio_device_id id_table[] = { 953 { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, 954 {}, 955 }; 956 957 static const unsigned int feature_table[] = {}; 958 959 static struct virtio_driver virtio_fs_driver = { 960 .driver.name = KBUILD_MODNAME, 961 .driver.owner = THIS_MODULE, 962 .id_table = id_table, 963 .feature_table = feature_table, 964 .feature_table_size = ARRAY_SIZE(feature_table), 965 .probe = virtio_fs_probe, 966 .remove = virtio_fs_remove, 967 #ifdef CONFIG_PM_SLEEP 968 .freeze = virtio_fs_freeze, 969 .restore = virtio_fs_restore, 970 #endif 971 }; 972 973 static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq) 974 __releases(fiq->lock) 975 { 976 struct fuse_forget_link *link; 977 struct virtio_fs_forget *forget; 978 struct virtio_fs_forget_req *req; 979 struct virtio_fs *fs; 980 struct virtio_fs_vq *fsvq; 981 u64 unique; 982 983 link = fuse_dequeue_forget(fiq, 1, NULL); 984 unique = fuse_get_unique(fiq); 985 986 fs = fiq->priv; 987 fsvq = &fs->vqs[VQ_HIPRIO]; 988 spin_unlock(&fiq->lock); 989 990 /* Allocate a buffer for the request */ 991 forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL); 992 req = &forget->req; 993 994 req->ih = (struct fuse_in_header){ 995 .opcode = FUSE_FORGET, 996 .nodeid = link->forget_one.nodeid, 997 .unique = unique, 998 .len = sizeof(*req), 999 }; 1000 req->arg = (struct fuse_forget_in){ 1001 .nlookup = link->forget_one.nlookup, 1002 }; 1003 1004 send_forget_request(fsvq, forget, false); 1005 kfree(link); 1006 } 1007 1008 static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq) 1009 __releases(fiq->lock) 1010 { 1011 /* 1012 * TODO interrupts. 1013 * 1014 * Normal fs operations on a local filesystems aren't interruptible. 1015 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW) 1016 * with shared lock between host and guest. 1017 */ 1018 spin_unlock(&fiq->lock); 1019 } 1020 1021 /* Count number of scatter-gather elements required */ 1022 static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs, 1023 unsigned int num_pages, 1024 unsigned int total_len) 1025 { 1026 unsigned int i; 1027 unsigned int this_len; 1028 1029 for (i = 0; i < num_pages && total_len; i++) { 1030 this_len = min(page_descs[i].length, total_len); 1031 total_len -= this_len; 1032 } 1033 1034 return i; 1035 } 1036 1037 /* Return the number of scatter-gather list elements required */ 1038 static unsigned int sg_count_fuse_req(struct fuse_req *req) 1039 { 1040 struct fuse_args *args = req->args; 1041 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); 1042 unsigned int size, total_sgs = 1 /* fuse_in_header */; 1043 1044 if (args->in_numargs - args->in_pages) 1045 total_sgs += 1; 1046 1047 if (args->in_pages) { 1048 size = args->in_args[args->in_numargs - 1].size; 1049 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, 1050 size); 1051 } 1052 1053 if (!test_bit(FR_ISREPLY, &req->flags)) 1054 return total_sgs; 1055 1056 total_sgs += 1 /* fuse_out_header */; 1057 1058 if (args->out_numargs - args->out_pages) 1059 total_sgs += 1; 1060 1061 if (args->out_pages) { 1062 size = args->out_args[args->out_numargs - 1].size; 1063 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages, 1064 size); 1065 } 1066 1067 return total_sgs; 1068 } 1069 1070 /* Add pages to scatter-gather list and return number of elements used */ 1071 static unsigned int sg_init_fuse_pages(struct scatterlist *sg, 1072 struct page **pages, 1073 struct fuse_page_desc *page_descs, 1074 unsigned int num_pages, 1075 unsigned int total_len) 1076 { 1077 unsigned int i; 1078 unsigned int this_len; 1079 1080 for (i = 0; i < num_pages && total_len; i++) { 1081 sg_init_table(&sg[i], 1); 1082 this_len = min(page_descs[i].length, total_len); 1083 sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); 1084 total_len -= this_len; 1085 } 1086 1087 return i; 1088 } 1089 1090 /* Add args to scatter-gather list and return number of elements used */ 1091 static unsigned int sg_init_fuse_args(struct scatterlist *sg, 1092 struct fuse_req *req, 1093 struct fuse_arg *args, 1094 unsigned int numargs, 1095 bool argpages, 1096 void *argbuf, 1097 unsigned int *len_used) 1098 { 1099 struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); 1100 unsigned int total_sgs = 0; 1101 unsigned int len; 1102 1103 len = fuse_len_args(numargs - argpages, args); 1104 if (len) 1105 sg_init_one(&sg[total_sgs++], argbuf, len); 1106 1107 if (argpages) 1108 total_sgs += sg_init_fuse_pages(&sg[total_sgs], 1109 ap->pages, ap->descs, 1110 ap->num_pages, 1111 args[numargs - 1].size); 1112 1113 if (len_used) 1114 *len_used = len; 1115 1116 return total_sgs; 1117 } 1118 1119 /* Add a request to a virtqueue and kick the device */ 1120 static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 1121 struct fuse_req *req, bool in_flight) 1122 { 1123 /* requests need at least 4 elements */ 1124 struct scatterlist *stack_sgs[6]; 1125 struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)]; 1126 struct scatterlist **sgs = stack_sgs; 1127 struct scatterlist *sg = stack_sg; 1128 struct virtqueue *vq; 1129 struct fuse_args *args = req->args; 1130 unsigned int argbuf_used = 0; 1131 unsigned int out_sgs = 0; 1132 unsigned int in_sgs = 0; 1133 unsigned int total_sgs; 1134 unsigned int i; 1135 int ret; 1136 bool notify; 1137 struct fuse_pqueue *fpq; 1138 1139 /* Does the sglist fit on the stack? */ 1140 total_sgs = sg_count_fuse_req(req); 1141 if (total_sgs > ARRAY_SIZE(stack_sgs)) { 1142 sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); 1143 sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); 1144 if (!sgs || !sg) { 1145 ret = -ENOMEM; 1146 goto out; 1147 } 1148 } 1149 1150 /* Use a bounce buffer since stack args cannot be mapped */ 1151 ret = copy_args_to_argbuf(req); 1152 if (ret < 0) 1153 goto out; 1154 1155 /* Request elements */ 1156 sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); 1157 out_sgs += sg_init_fuse_args(&sg[out_sgs], req, 1158 (struct fuse_arg *)args->in_args, 1159 args->in_numargs, args->in_pages, 1160 req->argbuf, &argbuf_used); 1161 1162 /* Reply elements */ 1163 if (test_bit(FR_ISREPLY, &req->flags)) { 1164 sg_init_one(&sg[out_sgs + in_sgs++], 1165 &req->out.h, sizeof(req->out.h)); 1166 in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req, 1167 args->out_args, args->out_numargs, 1168 args->out_pages, 1169 req->argbuf + argbuf_used, NULL); 1170 } 1171 1172 WARN_ON(out_sgs + in_sgs != total_sgs); 1173 1174 for (i = 0; i < total_sgs; i++) 1175 sgs[i] = &sg[i]; 1176 1177 spin_lock(&fsvq->lock); 1178 1179 if (!fsvq->connected) { 1180 spin_unlock(&fsvq->lock); 1181 ret = -ENOTCONN; 1182 goto out; 1183 } 1184 1185 vq = fsvq->vq; 1186 ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC); 1187 if (ret < 0) { 1188 spin_unlock(&fsvq->lock); 1189 goto out; 1190 } 1191 1192 /* Request successfully sent. */ 1193 fpq = &fsvq->fud->pq; 1194 spin_lock(&fpq->lock); 1195 list_add_tail(&req->list, fpq->processing); 1196 spin_unlock(&fpq->lock); 1197 set_bit(FR_SENT, &req->flags); 1198 /* matches barrier in request_wait_answer() */ 1199 smp_mb__after_atomic(); 1200 1201 if (!in_flight) 1202 inc_in_flight_req(fsvq); 1203 notify = virtqueue_kick_prepare(vq); 1204 1205 spin_unlock(&fsvq->lock); 1206 1207 if (notify) 1208 virtqueue_notify(vq); 1209 1210 out: 1211 if (ret < 0 && req->argbuf) { 1212 kfree(req->argbuf); 1213 req->argbuf = NULL; 1214 } 1215 if (sgs != stack_sgs) { 1216 kfree(sgs); 1217 kfree(sg); 1218 } 1219 1220 return ret; 1221 } 1222 1223 static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq) 1224 __releases(fiq->lock) 1225 { 1226 unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */ 1227 struct virtio_fs *fs; 1228 struct fuse_req *req; 1229 struct virtio_fs_vq *fsvq; 1230 int ret; 1231 1232 WARN_ON(list_empty(&fiq->pending)); 1233 req = list_last_entry(&fiq->pending, struct fuse_req, list); 1234 clear_bit(FR_PENDING, &req->flags); 1235 list_del_init(&req->list); 1236 WARN_ON(!list_empty(&fiq->pending)); 1237 spin_unlock(&fiq->lock); 1238 1239 fs = fiq->priv; 1240 1241 pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n", 1242 __func__, req->in.h.opcode, req->in.h.unique, 1243 req->in.h.nodeid, req->in.h.len, 1244 fuse_len_args(req->args->out_numargs, req->args->out_args)); 1245 1246 fsvq = &fs->vqs[queue_id]; 1247 ret = virtio_fs_enqueue_req(fsvq, req, false); 1248 if (ret < 0) { 1249 if (ret == -ENOMEM || ret == -ENOSPC) { 1250 /* 1251 * Virtqueue full. Retry submission from worker 1252 * context as we might be holding fc->bg_lock. 1253 */ 1254 spin_lock(&fsvq->lock); 1255 list_add_tail(&req->list, &fsvq->queued_reqs); 1256 inc_in_flight_req(fsvq); 1257 schedule_delayed_work(&fsvq->dispatch_work, 1258 msecs_to_jiffies(1)); 1259 spin_unlock(&fsvq->lock); 1260 return; 1261 } 1262 req->out.h.error = ret; 1263 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); 1264 1265 /* Can't end request in submission context. Use a worker */ 1266 spin_lock(&fsvq->lock); 1267 list_add_tail(&req->list, &fsvq->end_reqs); 1268 schedule_delayed_work(&fsvq->dispatch_work, 0); 1269 spin_unlock(&fsvq->lock); 1270 return; 1271 } 1272 } 1273 1274 static const struct fuse_iqueue_ops virtio_fs_fiq_ops = { 1275 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock, 1276 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock, 1277 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock, 1278 .release = virtio_fs_fiq_release, 1279 }; 1280 1281 static inline void virtio_fs_ctx_set_defaults(struct fuse_fs_context *ctx) 1282 { 1283 ctx->rootmode = S_IFDIR; 1284 ctx->default_permissions = 1; 1285 ctx->allow_other = 1; 1286 ctx->max_read = UINT_MAX; 1287 ctx->blksize = 512; 1288 ctx->destroy = true; 1289 ctx->no_control = true; 1290 ctx->no_force_umount = true; 1291 } 1292 1293 static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc) 1294 { 1295 struct fuse_mount *fm = get_fuse_mount_super(sb); 1296 struct fuse_conn *fc = fm->fc; 1297 struct virtio_fs *fs = fc->iq.priv; 1298 struct fuse_fs_context *ctx = fsc->fs_private; 1299 unsigned int i; 1300 int err; 1301 1302 virtio_fs_ctx_set_defaults(ctx); 1303 mutex_lock(&virtio_fs_mutex); 1304 1305 /* After holding mutex, make sure virtiofs device is still there. 1306 * Though we are holding a reference to it, drive ->remove might 1307 * still have cleaned up virtual queues. In that case bail out. 1308 */ 1309 err = -EINVAL; 1310 if (list_empty(&fs->list)) { 1311 pr_info("virtio-fs: tag <%s> not found\n", fs->tag); 1312 goto err; 1313 } 1314 1315 err = -ENOMEM; 1316 /* Allocate fuse_dev for hiprio and notification queues */ 1317 for (i = 0; i < fs->nvqs; i++) { 1318 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1319 1320 fsvq->fud = fuse_dev_alloc(); 1321 if (!fsvq->fud) 1322 goto err_free_fuse_devs; 1323 } 1324 1325 /* virtiofs allocates and installs its own fuse devices */ 1326 ctx->fudptr = NULL; 1327 if (ctx->dax) 1328 ctx->dax_dev = fs->dax_dev; 1329 err = fuse_fill_super_common(sb, ctx); 1330 if (err < 0) 1331 goto err_free_fuse_devs; 1332 1333 for (i = 0; i < fs->nvqs; i++) { 1334 struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1335 1336 fuse_dev_install(fsvq->fud, fc); 1337 } 1338 1339 /* Previous unmount will stop all queues. Start these again */ 1340 virtio_fs_start_all_queues(fs); 1341 fuse_send_init(fm); 1342 mutex_unlock(&virtio_fs_mutex); 1343 return 0; 1344 1345 err_free_fuse_devs: 1346 virtio_fs_free_devs(fs); 1347 err: 1348 mutex_unlock(&virtio_fs_mutex); 1349 return err; 1350 } 1351 1352 static void virtio_fs_conn_destroy(struct fuse_mount *fm) 1353 { 1354 struct fuse_conn *fc = fm->fc; 1355 struct virtio_fs *vfs = fc->iq.priv; 1356 struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO]; 1357 1358 /* Stop dax worker. Soon evict_inodes() will be called which 1359 * will free all memory ranges belonging to all inodes. 1360 */ 1361 if (IS_ENABLED(CONFIG_FUSE_DAX)) 1362 fuse_dax_cancel_work(fc); 1363 1364 /* Stop forget queue. Soon destroy will be sent */ 1365 spin_lock(&fsvq->lock); 1366 fsvq->connected = false; 1367 spin_unlock(&fsvq->lock); 1368 virtio_fs_drain_all_queues(vfs); 1369 1370 fuse_conn_destroy(fm); 1371 1372 /* fuse_conn_destroy() must have sent destroy. Stop all queues 1373 * and drain one more time and free fuse devices. Freeing fuse 1374 * devices will drop their reference on fuse_conn and that in 1375 * turn will drop its reference on virtio_fs object. 1376 */ 1377 virtio_fs_stop_all_queues(vfs); 1378 virtio_fs_drain_all_queues(vfs); 1379 virtio_fs_free_devs(vfs); 1380 } 1381 1382 static void virtio_kill_sb(struct super_block *sb) 1383 { 1384 struct fuse_mount *fm = get_fuse_mount_super(sb); 1385 bool last; 1386 1387 /* If mount failed, we can still be called without any fc */ 1388 if (fm) { 1389 last = fuse_mount_remove(fm); 1390 if (last) 1391 virtio_fs_conn_destroy(fm); 1392 } 1393 kill_anon_super(sb); 1394 } 1395 1396 static int virtio_fs_test_super(struct super_block *sb, 1397 struct fs_context *fsc) 1398 { 1399 struct fuse_mount *fsc_fm = fsc->s_fs_info; 1400 struct fuse_mount *sb_fm = get_fuse_mount_super(sb); 1401 1402 return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv; 1403 } 1404 1405 static int virtio_fs_set_super(struct super_block *sb, 1406 struct fs_context *fsc) 1407 { 1408 int err; 1409 1410 err = get_anon_bdev(&sb->s_dev); 1411 if (!err) 1412 fuse_mount_get(fsc->s_fs_info); 1413 1414 return err; 1415 } 1416 1417 static int virtio_fs_get_tree(struct fs_context *fsc) 1418 { 1419 struct virtio_fs *fs; 1420 struct super_block *sb; 1421 struct fuse_conn *fc; 1422 struct fuse_mount *fm; 1423 int err; 1424 1425 /* This gets a reference on virtio_fs object. This ptr gets installed 1426 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() 1427 * to drop the reference to this object. 1428 */ 1429 fs = virtio_fs_find_instance(fsc->source); 1430 if (!fs) { 1431 pr_info("virtio-fs: tag <%s> not found\n", fsc->source); 1432 return -EINVAL; 1433 } 1434 1435 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); 1436 if (!fc) { 1437 mutex_lock(&virtio_fs_mutex); 1438 virtio_fs_put(fs); 1439 mutex_unlock(&virtio_fs_mutex); 1440 return -ENOMEM; 1441 } 1442 1443 fm = kzalloc(sizeof(struct fuse_mount), GFP_KERNEL); 1444 if (!fm) { 1445 mutex_lock(&virtio_fs_mutex); 1446 virtio_fs_put(fs); 1447 mutex_unlock(&virtio_fs_mutex); 1448 kfree(fc); 1449 return -ENOMEM; 1450 } 1451 1452 fuse_conn_init(fc, fm, get_user_ns(current_user_ns()), 1453 &virtio_fs_fiq_ops, fs); 1454 fc->release = fuse_free_conn; 1455 fc->delete_stale = true; 1456 fc->auto_submounts = true; 1457 1458 fsc->s_fs_info = fm; 1459 sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super); 1460 fuse_mount_put(fm); 1461 if (IS_ERR(sb)) 1462 return PTR_ERR(sb); 1463 1464 if (!sb->s_root) { 1465 err = virtio_fs_fill_super(sb, fsc); 1466 if (err) { 1467 deactivate_locked_super(sb); 1468 return err; 1469 } 1470 1471 sb->s_flags |= SB_ACTIVE; 1472 } 1473 1474 WARN_ON(fsc->root); 1475 fsc->root = dget(sb->s_root); 1476 return 0; 1477 } 1478 1479 static const struct fs_context_operations virtio_fs_context_ops = { 1480 .free = virtio_fs_free_fc, 1481 .parse_param = virtio_fs_parse_param, 1482 .get_tree = virtio_fs_get_tree, 1483 }; 1484 1485 static int virtio_fs_init_fs_context(struct fs_context *fsc) 1486 { 1487 struct fuse_fs_context *ctx; 1488 1489 ctx = kzalloc(sizeof(struct fuse_fs_context), GFP_KERNEL); 1490 if (!ctx) 1491 return -ENOMEM; 1492 fsc->fs_private = ctx; 1493 fsc->ops = &virtio_fs_context_ops; 1494 return 0; 1495 } 1496 1497 static struct file_system_type virtio_fs_type = { 1498 .owner = THIS_MODULE, 1499 .name = "virtiofs", 1500 .init_fs_context = virtio_fs_init_fs_context, 1501 .kill_sb = virtio_kill_sb, 1502 }; 1503 1504 static int __init virtio_fs_init(void) 1505 { 1506 int ret; 1507 1508 ret = register_virtio_driver(&virtio_fs_driver); 1509 if (ret < 0) 1510 return ret; 1511 1512 ret = register_filesystem(&virtio_fs_type); 1513 if (ret < 0) { 1514 unregister_virtio_driver(&virtio_fs_driver); 1515 return ret; 1516 } 1517 1518 return 0; 1519 } 1520 module_init(virtio_fs_init); 1521 1522 static void __exit virtio_fs_exit(void) 1523 { 1524 unregister_filesystem(&virtio_fs_type); 1525 unregister_virtio_driver(&virtio_fs_driver); 1526 } 1527 module_exit(virtio_fs_exit); 1528 1529 MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>"); 1530 MODULE_DESCRIPTION("Virtio Filesystem"); 1531 MODULE_LICENSE("GPL"); 1532 MODULE_ALIAS_FS(KBUILD_MODNAME); 1533 MODULE_DEVICE_TABLE(virtio, id_table); 1534