virtio_fs.c (62b31a045757eac81fed94b19df47418a0818528) virtio_fs.c (694565356c2e06224d94774a42709cc8dfab49ee)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * virtio-fs: Virtio Filesystem
4 * Copyright (C) 2018 Red Hat, Inc.
5 */
6
7#include <linux/fs.h>
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * virtio-fs: Virtio Filesystem
4 * Copyright (C) 2018 Red Hat, Inc.
5 */
6
7#include <linux/fs.h>
8#include <linux/dax.h>
9#include <linux/pci.h>
10#include <linux/pfn_t.h>
8#include <linux/module.h>
9#include <linux/virtio.h>
10#include <linux/virtio_fs.h>
11#include <linux/delay.h>
12#include <linux/fs_context.h>
11#include <linux/module.h>
12#include <linux/virtio.h>
13#include <linux/virtio_fs.h>
14#include <linux/delay.h>
15#include <linux/fs_context.h>
16#include <linux/fs_parser.h>
13#include <linux/highmem.h>
17#include <linux/highmem.h>
18#include <linux/uio.h>
14#include "fuse_i.h"
15
16/* List of virtio-fs device instances and a lock for the list. Also provides
17 * mutual exclusion in device removal and mounting path
18 */
19static DEFINE_MUTEX(virtio_fs_mutex);
20static LIST_HEAD(virtio_fs_instances);
21
22enum {
23 VQ_HIPRIO,
24 VQ_REQUEST
25};
26
19#include "fuse_i.h"
20
21/* List of virtio-fs device instances and a lock for the list. Also provides
22 * mutual exclusion in device removal and mounting path
23 */
24static DEFINE_MUTEX(virtio_fs_mutex);
25static LIST_HEAD(virtio_fs_instances);
26
27enum {
28 VQ_HIPRIO,
29 VQ_REQUEST
30};
31
32#define VQ_NAME_LEN 24
33
27/* Per-virtqueue state */
28struct virtio_fs_vq {
29 spinlock_t lock;
30 struct virtqueue *vq; /* protected by ->lock */
31 struct work_struct done_work;
32 struct list_head queued_reqs;
33 struct list_head end_reqs; /* End these requests */
34 struct delayed_work dispatch_work;
35 struct fuse_dev *fud;
36 bool connected;
37 long in_flight;
38 struct completion in_flight_zero; /* No inflight requests */
34/* Per-virtqueue state */
35struct virtio_fs_vq {
36 spinlock_t lock;
37 struct virtqueue *vq; /* protected by ->lock */
38 struct work_struct done_work;
39 struct list_head queued_reqs;
40 struct list_head end_reqs; /* End these requests */
41 struct delayed_work dispatch_work;
42 struct fuse_dev *fud;
43 bool connected;
44 long in_flight;
45 struct completion in_flight_zero; /* No inflight requests */
39 char name[24];
46 char name[VQ_NAME_LEN];
40} ____cacheline_aligned_in_smp;
41
42/* A virtio-fs device instance */
43struct virtio_fs {
44 struct kref refcount;
45 struct list_head list; /* on virtio_fs_instances */
46 char *tag;
47 struct virtio_fs_vq *vqs;
48 unsigned int nvqs; /* number of virtqueues */
49 unsigned int num_request_queues; /* number of request queues */
47} ____cacheline_aligned_in_smp;
48
49/* A virtio-fs device instance */
50struct virtio_fs {
51 struct kref refcount;
52 struct list_head list; /* on virtio_fs_instances */
53 char *tag;
54 struct virtio_fs_vq *vqs;
55 unsigned int nvqs; /* number of virtqueues */
56 unsigned int num_request_queues; /* number of request queues */
57 struct dax_device *dax_dev;
58
59 /* DAX memory window where file contents are mapped */
60 void *window_kaddr;
61 phys_addr_t window_phys_addr;
62 size_t window_len;
50};
51
52struct virtio_fs_forget_req {
53 struct fuse_in_header ih;
54 struct fuse_forget_in arg;
55};
56
57struct virtio_fs_forget {

--- 6 unchanged lines hidden (view full) ---

64 struct fuse_req *req;
65 struct virtio_fs_vq *fsvq;
66 struct work_struct done_work;
67};
68
69static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
70 struct fuse_req *req, bool in_flight);
71
63};
64
65struct virtio_fs_forget_req {
66 struct fuse_in_header ih;
67 struct fuse_forget_in arg;
68};
69
70struct virtio_fs_forget {

--- 6 unchanged lines hidden (view full) ---

77 struct fuse_req *req;
78 struct virtio_fs_vq *fsvq;
79 struct work_struct done_work;
80};
81
82static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
83 struct fuse_req *req, bool in_flight);
84
85enum {
86 OPT_DAX,
87};
88
89static const struct fs_parameter_spec virtio_fs_parameters[] = {
90 fsparam_flag("dax", OPT_DAX),
91 {}
92};
93
94static int virtio_fs_parse_param(struct fs_context *fc,
95 struct fs_parameter *param)
96{
97 struct fs_parse_result result;
98 struct fuse_fs_context *ctx = fc->fs_private;
99 int opt;
100
101 opt = fs_parse(fc, virtio_fs_parameters, param, &result);
102 if (opt < 0)
103 return opt;
104
105 switch (opt) {
106 case OPT_DAX:
107 ctx->dax = 1;
108 break;
109 default:
110 return -EINVAL;
111 }
112
113 return 0;
114}
115
116static void virtio_fs_free_fc(struct fs_context *fc)
117{
118 struct fuse_fs_context *ctx = fc->fs_private;
119
120 kfree(ctx);
121}
122
72static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
73{
74 struct virtio_fs *fs = vq->vdev->priv;
75
76 return &fs->vqs[vq->index];
77}
78
79static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)

--- 204 unchanged lines hidden (view full) ---

284 spin_unlock(&fsvq->lock);
285}
286
287static void virtio_fs_request_dispatch_work(struct work_struct *work)
288{
289 struct fuse_req *req;
290 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
291 dispatch_work.work);
123static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
124{
125 struct virtio_fs *fs = vq->vdev->priv;
126
127 return &fs->vqs[vq->index];
128}
129
130static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq)

--- 204 unchanged lines hidden (view full) ---

335 spin_unlock(&fsvq->lock);
336}
337
338static void virtio_fs_request_dispatch_work(struct work_struct *work)
339{
340 struct fuse_req *req;
341 struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
342 dispatch_work.work);
292 struct fuse_conn *fc = fsvq->fud->fc;
293 int ret;
294
295 pr_debug("virtio-fs: worker %s called.\n", __func__);
296 while (1) {
297 spin_lock(&fsvq->lock);
298 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,
299 list);
300 if (!req) {
301 spin_unlock(&fsvq->lock);
302 break;
303 }
304
305 list_del_init(&req->list);
306 spin_unlock(&fsvq->lock);
343 int ret;
344
345 pr_debug("virtio-fs: worker %s called.\n", __func__);
346 while (1) {
347 spin_lock(&fsvq->lock);
348 req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,
349 list);
350 if (!req) {
351 spin_unlock(&fsvq->lock);
352 break;
353 }
354
355 list_del_init(&req->list);
356 spin_unlock(&fsvq->lock);
307 fuse_request_end(fc, req);
357 fuse_request_end(req);
308 }
309
310 /* Dispatch pending requests */
311 while (1) {
312 spin_lock(&fsvq->lock);
313 req = list_first_entry_or_null(&fsvq->queued_reqs,
314 struct fuse_req, list);
315 if (!req) {

--- 14 unchanged lines hidden (view full) ---

330 return;
331 }
332 req->out.h.error = ret;
333 spin_lock(&fsvq->lock);
334 dec_in_flight_req(fsvq);
335 spin_unlock(&fsvq->lock);
336 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n",
337 ret);
358 }
359
360 /* Dispatch pending requests */
361 while (1) {
362 spin_lock(&fsvq->lock);
363 req = list_first_entry_or_null(&fsvq->queued_reqs,
364 struct fuse_req, list);
365 if (!req) {

--- 14 unchanged lines hidden (view full) ---

380 return;
381 }
382 req->out.h.error = ret;
383 spin_lock(&fsvq->lock);
384 dec_in_flight_req(fsvq);
385 spin_unlock(&fsvq->lock);
386 pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n",
387 ret);
338 fuse_request_end(fc, req);
388 fuse_request_end(req);
339 }
340 }
341}
342
343/*
344 * Returns 1 if queue is full and sender should wait a bit before sending
345 * next request, 0 otherwise.
346 */

--- 143 unchanged lines hidden (view full) ---

490 req->argbuf = NULL;
491}
492
493/* Work function for request completion */
494static void virtio_fs_request_complete(struct fuse_req *req,
495 struct virtio_fs_vq *fsvq)
496{
497 struct fuse_pqueue *fpq = &fsvq->fud->pq;
389 }
390 }
391}
392
393/*
394 * Returns 1 if queue is full and sender should wait a bit before sending
395 * next request, 0 otherwise.
396 */

--- 143 unchanged lines hidden (view full) ---

540 req->argbuf = NULL;
541}
542
543/* Work function for request completion */
544static void virtio_fs_request_complete(struct fuse_req *req,
545 struct virtio_fs_vq *fsvq)
546{
547 struct fuse_pqueue *fpq = &fsvq->fud->pq;
498 struct fuse_conn *fc = fsvq->fud->fc;
499 struct fuse_args *args;
500 struct fuse_args_pages *ap;
501 unsigned int len, i, thislen;
502 struct page *page;
503
504 /*
505 * TODO verify that server properly follows FUSE protocol
506 * (oh.uniq, oh.len)

--- 16 unchanged lines hidden (view full) ---

523 }
524 }
525 }
526
527 spin_lock(&fpq->lock);
528 clear_bit(FR_SENT, &req->flags);
529 spin_unlock(&fpq->lock);
530
548 struct fuse_args *args;
549 struct fuse_args_pages *ap;
550 unsigned int len, i, thislen;
551 struct page *page;
552
553 /*
554 * TODO verify that server properly follows FUSE protocol
555 * (oh.uniq, oh.len)

--- 16 unchanged lines hidden (view full) ---

572 }
573 }
574 }
575
576 spin_lock(&fpq->lock);
577 clear_bit(FR_SENT, &req->flags);
578 spin_unlock(&fpq->lock);
579
531 fuse_request_end(fc, req);
580 fuse_request_end(req);
532 spin_lock(&fsvq->lock);
533 dec_in_flight_req(fsvq);
534 spin_unlock(&fsvq->lock);
535}
536
537static void virtio_fs_complete_req_work(struct work_struct *work)
538{
539 struct virtio_fs_req_work *w =

--- 51 unchanged lines hidden (view full) ---

591{
592 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
593
594 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
595
596 schedule_work(&fsvq->done_work);
597}
598
581 spin_lock(&fsvq->lock);
582 dec_in_flight_req(fsvq);
583 spin_unlock(&fsvq->lock);
584}
585
586static void virtio_fs_complete_req_work(struct work_struct *work)
587{
588 struct virtio_fs_req_work *w =

--- 51 unchanged lines hidden (view full) ---

640{
641 struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
642
643 dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
644
645 schedule_work(&fsvq->done_work);
646}
647
648static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name,
649 int vq_type)
650{
651 strncpy(fsvq->name, name, VQ_NAME_LEN);
652 spin_lock_init(&fsvq->lock);
653 INIT_LIST_HEAD(&fsvq->queued_reqs);
654 INIT_LIST_HEAD(&fsvq->end_reqs);
655 init_completion(&fsvq->in_flight_zero);
656
657 if (vq_type == VQ_REQUEST) {
658 INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work);
659 INIT_DELAYED_WORK(&fsvq->dispatch_work,
660 virtio_fs_request_dispatch_work);
661 } else {
662 INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work);
663 INIT_DELAYED_WORK(&fsvq->dispatch_work,
664 virtio_fs_hiprio_dispatch_work);
665 }
666}
667
599/* Initialize virtqueues */
600static int virtio_fs_setup_vqs(struct virtio_device *vdev,
601 struct virtio_fs *fs)
602{
603 struct virtqueue **vqs;
604 vq_callback_t **callbacks;
605 const char **names;
606 unsigned int i;
607 int ret = 0;
608
609 virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues,
610 &fs->num_request_queues);
611 if (fs->num_request_queues == 0)
612 return -EINVAL;
613
668/* Initialize virtqueues */
669static int virtio_fs_setup_vqs(struct virtio_device *vdev,
670 struct virtio_fs *fs)
671{
672 struct virtqueue **vqs;
673 vq_callback_t **callbacks;
674 const char **names;
675 unsigned int i;
676 int ret = 0;
677
678 virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues,
679 &fs->num_request_queues);
680 if (fs->num_request_queues == 0)
681 return -EINVAL;
682
614 fs->nvqs = 1 + fs->num_request_queues;
683 fs->nvqs = VQ_REQUEST + fs->num_request_queues;
615 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
616 if (!fs->vqs)
617 return -ENOMEM;
618
619 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
620 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
621 GFP_KERNEL);
622 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
623 if (!vqs || !callbacks || !names) {
624 ret = -ENOMEM;
625 goto out;
626 }
627
684 fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
685 if (!fs->vqs)
686 return -ENOMEM;
687
688 vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
689 callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
690 GFP_KERNEL);
691 names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
692 if (!vqs || !callbacks || !names) {
693 ret = -ENOMEM;
694 goto out;
695 }
696
697 /* Initialize the hiprio/forget request virtqueue */
628 callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
698 callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
629 snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name),
630 "hiprio");
699 virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO);
631 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
700 names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
632 INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work);
633 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs);
634 INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs);
635 INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work,
636 virtio_fs_hiprio_dispatch_work);
637 init_completion(&fs->vqs[VQ_HIPRIO].in_flight_zero);
638 spin_lock_init(&fs->vqs[VQ_HIPRIO].lock);
639
640 /* Initialize the requests virtqueues */
641 for (i = VQ_REQUEST; i < fs->nvqs; i++) {
701
702 /* Initialize the requests virtqueues */
703 for (i = VQ_REQUEST; i < fs->nvqs; i++) {
642 spin_lock_init(&fs->vqs[i].lock);
643 INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work);
644 INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work,
645 virtio_fs_request_dispatch_work);
646 INIT_LIST_HEAD(&fs->vqs[i].queued_reqs);
647 INIT_LIST_HEAD(&fs->vqs[i].end_reqs);
648 init_completion(&fs->vqs[i].in_flight_zero);
649 snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name),
650 "requests.%u", i - VQ_REQUEST);
704 char vq_name[VQ_NAME_LEN];
705
706 snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST);
707 virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST);
651 callbacks[i] = virtio_fs_vq_done;
652 names[i] = fs->vqs[i].name;
653 }
654
655 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
656 if (ret < 0)
657 goto out;
658

--- 12 unchanged lines hidden (view full) ---

671
672/* Free virtqueues (device must already be reset) */
673static void virtio_fs_cleanup_vqs(struct virtio_device *vdev,
674 struct virtio_fs *fs)
675{
676 vdev->config->del_vqs(vdev);
677}
678
708 callbacks[i] = virtio_fs_vq_done;
709 names[i] = fs->vqs[i].name;
710 }
711
712 ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
713 if (ret < 0)
714 goto out;
715

--- 12 unchanged lines hidden (view full) ---

728
729/* Free virtqueues (device must already be reset) */
730static void virtio_fs_cleanup_vqs(struct virtio_device *vdev,
731 struct virtio_fs *fs)
732{
733 vdev->config->del_vqs(vdev);
734}
735
736/* Map a window offset to a page frame number. The window offset will have
737 * been produced by .iomap_begin(), which maps a file offset to a window
738 * offset.
739 */
740static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
741 long nr_pages, void **kaddr, pfn_t *pfn)
742{
743 struct virtio_fs *fs = dax_get_private(dax_dev);
744 phys_addr_t offset = PFN_PHYS(pgoff);
745 size_t max_nr_pages = fs->window_len/PAGE_SIZE - pgoff;
746
747 if (kaddr)
748 *kaddr = fs->window_kaddr + offset;
749 if (pfn)
750 *pfn = phys_to_pfn_t(fs->window_phys_addr + offset,
751 PFN_DEV | PFN_MAP);
752 return nr_pages > max_nr_pages ? max_nr_pages : nr_pages;
753}
754
755static size_t virtio_fs_copy_from_iter(struct dax_device *dax_dev,
756 pgoff_t pgoff, void *addr,
757 size_t bytes, struct iov_iter *i)
758{
759 return copy_from_iter(addr, bytes, i);
760}
761
762static size_t virtio_fs_copy_to_iter(struct dax_device *dax_dev,
763 pgoff_t pgoff, void *addr,
764 size_t bytes, struct iov_iter *i)
765{
766 return copy_to_iter(addr, bytes, i);
767}
768
769static int virtio_fs_zero_page_range(struct dax_device *dax_dev,
770 pgoff_t pgoff, size_t nr_pages)
771{
772 long rc;
773 void *kaddr;
774
775 rc = dax_direct_access(dax_dev, pgoff, nr_pages, &kaddr, NULL);
776 if (rc < 0)
777 return rc;
778 memset(kaddr, 0, nr_pages << PAGE_SHIFT);
779 dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT);
780 return 0;
781}
782
783static const struct dax_operations virtio_fs_dax_ops = {
784 .direct_access = virtio_fs_direct_access,
785 .copy_from_iter = virtio_fs_copy_from_iter,
786 .copy_to_iter = virtio_fs_copy_to_iter,
787 .zero_page_range = virtio_fs_zero_page_range,
788};
789
790static void virtio_fs_cleanup_dax(void *data)
791{
792 struct dax_device *dax_dev = data;
793
794 kill_dax(dax_dev);
795 put_dax(dax_dev);
796}
797
798static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs)
799{
800 struct virtio_shm_region cache_reg;
801 struct dev_pagemap *pgmap;
802 bool have_cache;
803
804 if (!IS_ENABLED(CONFIG_FUSE_DAX))
805 return 0;
806
807 /* Get cache region */
808 have_cache = virtio_get_shm_region(vdev, &cache_reg,
809 (u8)VIRTIO_FS_SHMCAP_ID_CACHE);
810 if (!have_cache) {
811 dev_notice(&vdev->dev, "%s: No cache capability\n", __func__);
812 return 0;
813 }
814
815 if (!devm_request_mem_region(&vdev->dev, cache_reg.addr, cache_reg.len,
816 dev_name(&vdev->dev))) {
817 dev_warn(&vdev->dev, "could not reserve region addr=0x%llx len=0x%llx\n",
818 cache_reg.addr, cache_reg.len);
819 return -EBUSY;
820 }
821
822 dev_notice(&vdev->dev, "Cache len: 0x%llx @ 0x%llx\n", cache_reg.len,
823 cache_reg.addr);
824
825 pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL);
826 if (!pgmap)
827 return -ENOMEM;
828
829 pgmap->type = MEMORY_DEVICE_FS_DAX;
830
831 /* Ideally we would directly use the PCI BAR resource but
832 * devm_memremap_pages() wants its own copy in pgmap. So
833 * initialize a struct resource from scratch (only the start
834 * and end fields will be used).
835 */
836 pgmap->range = (struct range) {
837 .start = (phys_addr_t) cache_reg.addr,
838 .end = (phys_addr_t) cache_reg.addr + cache_reg.len - 1,
839 };
840 pgmap->nr_range = 1;
841
842 fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap);
843 if (IS_ERR(fs->window_kaddr))
844 return PTR_ERR(fs->window_kaddr);
845
846 fs->window_phys_addr = (phys_addr_t) cache_reg.addr;
847 fs->window_len = (phys_addr_t) cache_reg.len;
848
849 dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n",
850 __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len);
851
852 fs->dax_dev = alloc_dax(fs, NULL, &virtio_fs_dax_ops, 0);
853 if (IS_ERR(fs->dax_dev))
854 return PTR_ERR(fs->dax_dev);
855
856 return devm_add_action_or_reset(&vdev->dev, virtio_fs_cleanup_dax,
857 fs->dax_dev);
858}
859
679static int virtio_fs_probe(struct virtio_device *vdev)
680{
681 struct virtio_fs *fs;
682 int ret;
683
684 fs = kzalloc(sizeof(*fs), GFP_KERNEL);
685 if (!fs)
686 return -ENOMEM;

--- 5 unchanged lines hidden (view full) ---

692 goto out;
693
694 ret = virtio_fs_setup_vqs(vdev, fs);
695 if (ret < 0)
696 goto out;
697
698 /* TODO vq affinity */
699
860static int virtio_fs_probe(struct virtio_device *vdev)
861{
862 struct virtio_fs *fs;
863 int ret;
864
865 fs = kzalloc(sizeof(*fs), GFP_KERNEL);
866 if (!fs)
867 return -ENOMEM;

--- 5 unchanged lines hidden (view full) ---

873 goto out;
874
875 ret = virtio_fs_setup_vqs(vdev, fs);
876 if (ret < 0)
877 goto out;
878
879 /* TODO vq affinity */
880
881 ret = virtio_fs_setup_dax(vdev, fs);
882 if (ret < 0)
883 goto out_vqs;
884
700 /* Bring the device online in case the filesystem is mounted and
701 * requests need to be sent before we return.
702 */
703 virtio_device_ready(vdev);
704
705 ret = virtio_fs_add_instance(fs);
706 if (ret < 0)
707 goto out_vqs;

--- 120 unchanged lines hidden (view full) ---

828 *
829 * Normal fs operations on a local filesystems aren't interruptible.
830 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
831 * with shared lock between host and guest.
832 */
833 spin_unlock(&fiq->lock);
834}
835
885 /* Bring the device online in case the filesystem is mounted and
886 * requests need to be sent before we return.
887 */
888 virtio_device_ready(vdev);
889
890 ret = virtio_fs_add_instance(fs);
891 if (ret < 0)
892 goto out_vqs;

--- 120 unchanged lines hidden (view full) ---

1013 *
1014 * Normal fs operations on a local filesystems aren't interruptible.
1015 * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
1016 * with shared lock between host and guest.
1017 */
1018 spin_unlock(&fiq->lock);
1019}
1020
1021/* Count number of scatter-gather elements required */
1022static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs,
1023 unsigned int num_pages,
1024 unsigned int total_len)
1025{
1026 unsigned int i;
1027 unsigned int this_len;
1028
1029 for (i = 0; i < num_pages && total_len; i++) {
1030 this_len = min(page_descs[i].length, total_len);
1031 total_len -= this_len;
1032 }
1033
1034 return i;
1035}
1036
836/* Return the number of scatter-gather list elements required */
837static unsigned int sg_count_fuse_req(struct fuse_req *req)
838{
839 struct fuse_args *args = req->args;
840 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
1037/* Return the number of scatter-gather list elements required */
1038static unsigned int sg_count_fuse_req(struct fuse_req *req)
1039{
1040 struct fuse_args *args = req->args;
1041 struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
841 unsigned int total_sgs = 1 /* fuse_in_header */;
1042 unsigned int size, total_sgs = 1 /* fuse_in_header */;
842
843 if (args->in_numargs - args->in_pages)
844 total_sgs += 1;
845
1043
1044 if (args->in_numargs - args->in_pages)
1045 total_sgs += 1;
1046
846 if (args->in_pages)
847 total_sgs += ap->num_pages;
1047 if (args->in_pages) {
1048 size = args->in_args[args->in_numargs - 1].size;
1049 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages,
1050 size);
1051 }
848
849 if (!test_bit(FR_ISREPLY, &req->flags))
850 return total_sgs;
851
852 total_sgs += 1 /* fuse_out_header */;
853
854 if (args->out_numargs - args->out_pages)
855 total_sgs += 1;
856
1052
1053 if (!test_bit(FR_ISREPLY, &req->flags))
1054 return total_sgs;
1055
1056 total_sgs += 1 /* fuse_out_header */;
1057
1058 if (args->out_numargs - args->out_pages)
1059 total_sgs += 1;
1060
857 if (args->out_pages)
858 total_sgs += ap->num_pages;
1061 if (args->out_pages) {
1062 size = args->out_args[args->out_numargs - 1].size;
1063 total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages,
1064 size);
1065 }
859
860 return total_sgs;
861}
862
863/* Add pages to scatter-gather list and return number of elements used */
864static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
865 struct page **pages,
866 struct fuse_page_desc *page_descs,

--- 199 unchanged lines hidden (view full) ---

1066
1067static const struct fuse_iqueue_ops virtio_fs_fiq_ops = {
1068 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock,
1069 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock,
1070 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock,
1071 .release = virtio_fs_fiq_release,
1072};
1073
1066
1067 return total_sgs;
1068}
1069
1070/* Add pages to scatter-gather list and return number of elements used */
1071static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
1072 struct page **pages,
1073 struct fuse_page_desc *page_descs,

--- 199 unchanged lines hidden (view full) ---

1273
1274static const struct fuse_iqueue_ops virtio_fs_fiq_ops = {
1275 .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock,
1276 .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock,
1277 .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock,
1278 .release = virtio_fs_fiq_release,
1279};
1280
1074static int virtio_fs_fill_super(struct super_block *sb)
1281static inline void virtio_fs_ctx_set_defaults(struct fuse_fs_context *ctx)
1075{
1282{
1076 struct fuse_conn *fc = get_fuse_conn_super(sb);
1283 ctx->rootmode = S_IFDIR;
1284 ctx->default_permissions = 1;
1285 ctx->allow_other = 1;
1286 ctx->max_read = UINT_MAX;
1287 ctx->blksize = 512;
1288 ctx->destroy = true;
1289 ctx->no_control = true;
1290 ctx->no_force_umount = true;
1291}
1292
1293static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
1294{
1295 struct fuse_mount *fm = get_fuse_mount_super(sb);
1296 struct fuse_conn *fc = fm->fc;
1077 struct virtio_fs *fs = fc->iq.priv;
1297 struct virtio_fs *fs = fc->iq.priv;
1298 struct fuse_fs_context *ctx = fsc->fs_private;
1078 unsigned int i;
1079 int err;
1299 unsigned int i;
1300 int err;
1080 struct fuse_fs_context ctx = {
1081 .rootmode = S_IFDIR,
1082 .default_permissions = 1,
1083 .allow_other = 1,
1084 .max_read = UINT_MAX,
1085 .blksize = 512,
1086 .destroy = true,
1087 .no_control = true,
1088 .no_force_umount = true,
1089 .no_mount_options = true,
1090 };
1091
1301
1302 virtio_fs_ctx_set_defaults(ctx);
1092 mutex_lock(&virtio_fs_mutex);
1093
1094 /* After holding mutex, make sure virtiofs device is still there.
1095 * Though we are holding a reference to it, drive ->remove might
1096 * still have cleaned up virtual queues. In that case bail out.
1097 */
1098 err = -EINVAL;
1099 if (list_empty(&fs->list)) {

--- 7 unchanged lines hidden (view full) ---

1107 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1108
1109 fsvq->fud = fuse_dev_alloc();
1110 if (!fsvq->fud)
1111 goto err_free_fuse_devs;
1112 }
1113
1114 /* virtiofs allocates and installs its own fuse devices */
1303 mutex_lock(&virtio_fs_mutex);
1304
1305 /* After holding mutex, make sure virtiofs device is still there.
1306 * Though we are holding a reference to it, drive ->remove might
1307 * still have cleaned up virtual queues. In that case bail out.
1308 */
1309 err = -EINVAL;
1310 if (list_empty(&fs->list)) {

--- 7 unchanged lines hidden (view full) ---

1318 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1319
1320 fsvq->fud = fuse_dev_alloc();
1321 if (!fsvq->fud)
1322 goto err_free_fuse_devs;
1323 }
1324
1325 /* virtiofs allocates and installs its own fuse devices */
1115 ctx.fudptr = NULL;
1116 err = fuse_fill_super_common(sb, &ctx);
1326 ctx->fudptr = NULL;
1327 if (ctx->dax)
1328 ctx->dax_dev = fs->dax_dev;
1329 err = fuse_fill_super_common(sb, ctx);
1117 if (err < 0)
1118 goto err_free_fuse_devs;
1119
1120 for (i = 0; i < fs->nvqs; i++) {
1121 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1122
1123 fuse_dev_install(fsvq->fud, fc);
1124 }
1125
1126 /* Previous unmount will stop all queues. Start these again */
1127 virtio_fs_start_all_queues(fs);
1330 if (err < 0)
1331 goto err_free_fuse_devs;
1332
1333 for (i = 0; i < fs->nvqs; i++) {
1334 struct virtio_fs_vq *fsvq = &fs->vqs[i];
1335
1336 fuse_dev_install(fsvq->fud, fc);
1337 }
1338
1339 /* Previous unmount will stop all queues. Start these again */
1340 virtio_fs_start_all_queues(fs);
1128 fuse_send_init(fc);
1341 fuse_send_init(fm);
1129 mutex_unlock(&virtio_fs_mutex);
1130 return 0;
1131
1132err_free_fuse_devs:
1133 virtio_fs_free_devs(fs);
1134err:
1135 mutex_unlock(&virtio_fs_mutex);
1136 return err;
1137}
1138
1342 mutex_unlock(&virtio_fs_mutex);
1343 return 0;
1344
1345err_free_fuse_devs:
1346 virtio_fs_free_devs(fs);
1347err:
1348 mutex_unlock(&virtio_fs_mutex);
1349 return err;
1350}
1351
1139static void virtio_kill_sb(struct super_block *sb)
1352static void virtio_fs_conn_destroy(struct fuse_mount *fm)
1140{
1353{
1141 struct fuse_conn *fc = get_fuse_conn_super(sb);
1142 struct virtio_fs *vfs;
1143 struct virtio_fs_vq *fsvq;
1354 struct fuse_conn *fc = fm->fc;
1355 struct virtio_fs *vfs = fc->iq.priv;
1356 struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO];
1144
1357
1145 /* If mount failed, we can still be called without any fc */
1146 if (!fc)
1147 return fuse_kill_sb_anon(sb);
1358 /* Stop dax worker. Soon evict_inodes() will be called which
1359 * will free all memory ranges belonging to all inodes.
1360 */
1361 if (IS_ENABLED(CONFIG_FUSE_DAX))
1362 fuse_dax_cancel_work(fc);
1148
1363
1149 vfs = fc->iq.priv;
1150 fsvq = &vfs->vqs[VQ_HIPRIO];
1151
1152 /* Stop forget queue. Soon destroy will be sent */
1153 spin_lock(&fsvq->lock);
1154 fsvq->connected = false;
1155 spin_unlock(&fsvq->lock);
1156 virtio_fs_drain_all_queues(vfs);
1157
1364 /* Stop forget queue. Soon destroy will be sent */
1365 spin_lock(&fsvq->lock);
1366 fsvq->connected = false;
1367 spin_unlock(&fsvq->lock);
1368 virtio_fs_drain_all_queues(vfs);
1369
1158 fuse_kill_sb_anon(sb);
1370 fuse_conn_destroy(fm);
1159
1371
1160 /* fuse_kill_sb_anon() must have sent destroy. Stop all queues
1372 /* fuse_conn_destroy() must have sent destroy. Stop all queues
1161 * and drain one more time and free fuse devices. Freeing fuse
1162 * devices will drop their reference on fuse_conn and that in
1163 * turn will drop its reference on virtio_fs object.
1164 */
1165 virtio_fs_stop_all_queues(vfs);
1166 virtio_fs_drain_all_queues(vfs);
1167 virtio_fs_free_devs(vfs);
1168}
1169
1373 * and drain one more time and free fuse devices. Freeing fuse
1374 * devices will drop their reference on fuse_conn and that in
1375 * turn will drop its reference on virtio_fs object.
1376 */
1377 virtio_fs_stop_all_queues(vfs);
1378 virtio_fs_drain_all_queues(vfs);
1379 virtio_fs_free_devs(vfs);
1380}
1381
1382static void virtio_kill_sb(struct super_block *sb)
1383{
1384 struct fuse_mount *fm = get_fuse_mount_super(sb);
1385 bool last;
1386
1387 /* If mount failed, we can still be called without any fc */
1388 if (fm) {
1389 last = fuse_mount_remove(fm);
1390 if (last)
1391 virtio_fs_conn_destroy(fm);
1392 }
1393 kill_anon_super(sb);
1394}
1395
1170static int virtio_fs_test_super(struct super_block *sb,
1171 struct fs_context *fsc)
1172{
1396static int virtio_fs_test_super(struct super_block *sb,
1397 struct fs_context *fsc)
1398{
1173 struct fuse_conn *fc = fsc->s_fs_info;
1399 struct fuse_mount *fsc_fm = fsc->s_fs_info;
1400 struct fuse_mount *sb_fm = get_fuse_mount_super(sb);
1174
1401
1175 return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv;
1402 return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv;
1176}
1177
1178static int virtio_fs_set_super(struct super_block *sb,
1179 struct fs_context *fsc)
1180{
1181 int err;
1182
1183 err = get_anon_bdev(&sb->s_dev);
1184 if (!err)
1403}
1404
1405static int virtio_fs_set_super(struct super_block *sb,
1406 struct fs_context *fsc)
1407{
1408 int err;
1409
1410 err = get_anon_bdev(&sb->s_dev);
1411 if (!err)
1185 fuse_conn_get(fsc->s_fs_info);
1412 fuse_mount_get(fsc->s_fs_info);
1186
1187 return err;
1188}
1189
1190static int virtio_fs_get_tree(struct fs_context *fsc)
1191{
1192 struct virtio_fs *fs;
1193 struct super_block *sb;
1194 struct fuse_conn *fc;
1413
1414 return err;
1415}
1416
1417static int virtio_fs_get_tree(struct fs_context *fsc)
1418{
1419 struct virtio_fs *fs;
1420 struct super_block *sb;
1421 struct fuse_conn *fc;
1422 struct fuse_mount *fm;
1195 int err;
1196
1197 /* This gets a reference on virtio_fs object. This ptr gets installed
1198 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
1199 * to drop the reference to this object.
1200 */
1201 fs = virtio_fs_find_instance(fsc->source);
1202 if (!fs) {

--- 4 unchanged lines hidden (view full) ---

1207 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
1208 if (!fc) {
1209 mutex_lock(&virtio_fs_mutex);
1210 virtio_fs_put(fs);
1211 mutex_unlock(&virtio_fs_mutex);
1212 return -ENOMEM;
1213 }
1214
1423 int err;
1424
1425 /* This gets a reference on virtio_fs object. This ptr gets installed
1426 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
1427 * to drop the reference to this object.
1428 */
1429 fs = virtio_fs_find_instance(fsc->source);
1430 if (!fs) {

--- 4 unchanged lines hidden (view full) ---

1435 fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
1436 if (!fc) {
1437 mutex_lock(&virtio_fs_mutex);
1438 virtio_fs_put(fs);
1439 mutex_unlock(&virtio_fs_mutex);
1440 return -ENOMEM;
1441 }
1442
1215 fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops,
1216 fs);
1443 fm = kzalloc(sizeof(struct fuse_mount), GFP_KERNEL);
1444 if (!fm) {
1445 mutex_lock(&virtio_fs_mutex);
1446 virtio_fs_put(fs);
1447 mutex_unlock(&virtio_fs_mutex);
1448 kfree(fc);
1449 return -ENOMEM;
1450 }
1451
1452 fuse_conn_init(fc, fm, get_user_ns(current_user_ns()),
1453 &virtio_fs_fiq_ops, fs);
1217 fc->release = fuse_free_conn;
1218 fc->delete_stale = true;
1454 fc->release = fuse_free_conn;
1455 fc->delete_stale = true;
1456 fc->auto_submounts = true;
1219
1457
1220 fsc->s_fs_info = fc;
1458 fsc->s_fs_info = fm;
1221 sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super);
1459 sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super);
1222 fuse_conn_put(fc);
1460 fuse_mount_put(fm);
1223 if (IS_ERR(sb))
1224 return PTR_ERR(sb);
1225
1226 if (!sb->s_root) {
1461 if (IS_ERR(sb))
1462 return PTR_ERR(sb);
1463
1464 if (!sb->s_root) {
1227 err = virtio_fs_fill_super(sb);
1465 err = virtio_fs_fill_super(sb, fsc);
1228 if (err) {
1229 deactivate_locked_super(sb);
1230 return err;
1231 }
1232
1233 sb->s_flags |= SB_ACTIVE;
1234 }
1235
1236 WARN_ON(fsc->root);
1237 fsc->root = dget(sb->s_root);
1238 return 0;
1239}
1240
1241static const struct fs_context_operations virtio_fs_context_ops = {
1466 if (err) {
1467 deactivate_locked_super(sb);
1468 return err;
1469 }
1470
1471 sb->s_flags |= SB_ACTIVE;
1472 }
1473
1474 WARN_ON(fsc->root);
1475 fsc->root = dget(sb->s_root);
1476 return 0;
1477}
1478
1479static const struct fs_context_operations virtio_fs_context_ops = {
1480 .free = virtio_fs_free_fc,
1481 .parse_param = virtio_fs_parse_param,
1242 .get_tree = virtio_fs_get_tree,
1243};
1244
1245static int virtio_fs_init_fs_context(struct fs_context *fsc)
1246{
1482 .get_tree = virtio_fs_get_tree,
1483};
1484
1485static int virtio_fs_init_fs_context(struct fs_context *fsc)
1486{
1487 struct fuse_fs_context *ctx;
1488
1489 ctx = kzalloc(sizeof(struct fuse_fs_context), GFP_KERNEL);
1490 if (!ctx)
1491 return -ENOMEM;
1492 fsc->fs_private = ctx;
1247 fsc->ops = &virtio_fs_context_ops;
1248 return 0;
1249}
1250
1251static struct file_system_type virtio_fs_type = {
1252 .owner = THIS_MODULE,
1253 .name = "virtiofs",
1254 .init_fs_context = virtio_fs_init_fs_context,

--- 33 unchanged lines hidden --- 		1493 fsc->ops = &virtio_fs_context_ops;
1494 return 0;
1495}
1496
1497static struct file_system_type virtio_fs_type = {
1498 .owner = THIS_MODULE,
1499 .name = "virtiofs",
1500 .init_fs_context = virtio_fs_init_fs_context,

--- 33 unchanged lines hidden ---