1a62a8ef9SStefan Hajnoczi // SPDX-License-Identifier: GPL-2.0
2a62a8ef9SStefan Hajnoczi /*
3a62a8ef9SStefan Hajnoczi * virtio-fs: Virtio Filesystem
4a62a8ef9SStefan Hajnoczi * Copyright (C) 2018 Red Hat, Inc.
5a62a8ef9SStefan Hajnoczi */
6a62a8ef9SStefan Hajnoczi
7a62a8ef9SStefan Hajnoczi #include <linux/fs.h>
822f3787eSStefan Hajnoczi #include <linux/dax.h>
922f3787eSStefan Hajnoczi #include <linux/pci.h>
1022f3787eSStefan Hajnoczi #include <linux/pfn_t.h>
11dc90f084SChristoph Hellwig #include <linux/memremap.h>
12a62a8ef9SStefan Hajnoczi #include <linux/module.h>
13a62a8ef9SStefan Hajnoczi #include <linux/virtio.h>
14a62a8ef9SStefan Hajnoczi #include <linux/virtio_fs.h>
15a62a8ef9SStefan Hajnoczi #include <linux/delay.h>
16a62a8ef9SStefan Hajnoczi #include <linux/fs_context.h>
171dd53957SVivek Goyal #include <linux/fs_parser.h>
18a62a8ef9SStefan Hajnoczi #include <linux/highmem.h>
1922f3787eSStefan Hajnoczi #include <linux/uio.h>
20a62a8ef9SStefan Hajnoczi #include "fuse_i.h"
21a62a8ef9SStefan Hajnoczi
22a7f0d7aaSConnor Kuehl /* Used to help calculate the FUSE connection's max_pages limit for a request's
23a7f0d7aaSConnor Kuehl * size. Parts of the struct fuse_req are sliced into scattergather lists in
24a7f0d7aaSConnor Kuehl * addition to the pages used, so this can help account for that overhead.
25a7f0d7aaSConnor Kuehl */
26a7f0d7aaSConnor Kuehl #define FUSE_HEADER_OVERHEAD 4
27a7f0d7aaSConnor Kuehl
28a62a8ef9SStefan Hajnoczi /* List of virtio-fs device instances and a lock for the list. Also provides
29a62a8ef9SStefan Hajnoczi * mutual exclusion in device removal and mounting path
30a62a8ef9SStefan Hajnoczi */
31a62a8ef9SStefan Hajnoczi static DEFINE_MUTEX(virtio_fs_mutex);
32a62a8ef9SStefan Hajnoczi static LIST_HEAD(virtio_fs_instances);
33a62a8ef9SStefan Hajnoczi
34a62a8ef9SStefan Hajnoczi enum {
35a62a8ef9SStefan Hajnoczi VQ_HIPRIO,
36a62a8ef9SStefan Hajnoczi VQ_REQUEST
37a62a8ef9SStefan Hajnoczi };
38a62a8ef9SStefan Hajnoczi
39b43b7e81SVivek Goyal #define VQ_NAME_LEN 24
40b43b7e81SVivek Goyal
41a62a8ef9SStefan Hajnoczi /* Per-virtqueue state */
42a62a8ef9SStefan Hajnoczi struct virtio_fs_vq {
43a62a8ef9SStefan Hajnoczi spinlock_t lock;
44a62a8ef9SStefan Hajnoczi struct virtqueue *vq; /* protected by ->lock */
45a62a8ef9SStefan Hajnoczi struct work_struct done_work;
46a62a8ef9SStefan Hajnoczi struct list_head queued_reqs;
4751fecdd2SVivek Goyal struct list_head end_reqs; /* End these requests */
48a62a8ef9SStefan Hajnoczi struct delayed_work dispatch_work;
49a62a8ef9SStefan Hajnoczi struct fuse_dev *fud;
50a62a8ef9SStefan Hajnoczi bool connected;
51a62a8ef9SStefan Hajnoczi long in_flight;
52724c15a4SVivek Goyal struct completion in_flight_zero; /* No inflight requests */
53b43b7e81SVivek Goyal char name[VQ_NAME_LEN];
54a62a8ef9SStefan Hajnoczi } ____cacheline_aligned_in_smp;
55a62a8ef9SStefan Hajnoczi
56a62a8ef9SStefan Hajnoczi /* A virtio-fs device instance */
57a62a8ef9SStefan Hajnoczi struct virtio_fs {
58a62a8ef9SStefan Hajnoczi struct kref refcount;
59a62a8ef9SStefan Hajnoczi struct list_head list; /* on virtio_fs_instances */
60a62a8ef9SStefan Hajnoczi char *tag;
61a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *vqs;
62a62a8ef9SStefan Hajnoczi unsigned int nvqs; /* number of virtqueues */
63a62a8ef9SStefan Hajnoczi unsigned int num_request_queues; /* number of request queues */
6422f3787eSStefan Hajnoczi struct dax_device *dax_dev;
6522f3787eSStefan Hajnoczi
6622f3787eSStefan Hajnoczi /* DAX memory window where file contents are mapped */
6722f3787eSStefan Hajnoczi void *window_kaddr;
6822f3787eSStefan Hajnoczi phys_addr_t window_phys_addr;
6922f3787eSStefan Hajnoczi size_t window_len;
70a62a8ef9SStefan Hajnoczi };
71a62a8ef9SStefan Hajnoczi
721efcf39eSVivek Goyal struct virtio_fs_forget_req {
73a62a8ef9SStefan Hajnoczi struct fuse_in_header ih;
74a62a8ef9SStefan Hajnoczi struct fuse_forget_in arg;
751efcf39eSVivek Goyal };
761efcf39eSVivek Goyal
771efcf39eSVivek Goyal struct virtio_fs_forget {
78a62a8ef9SStefan Hajnoczi /* This request can be temporarily queued on virt queue */
79a62a8ef9SStefan Hajnoczi struct list_head list;
801efcf39eSVivek Goyal struct virtio_fs_forget_req req;
81a62a8ef9SStefan Hajnoczi };
82a62a8ef9SStefan Hajnoczi
83bb737bbeSVivek Goyal struct virtio_fs_req_work {
84bb737bbeSVivek Goyal struct fuse_req *req;
85bb737bbeSVivek Goyal struct virtio_fs_vq *fsvq;
86bb737bbeSVivek Goyal struct work_struct done_work;
87bb737bbeSVivek Goyal };
88bb737bbeSVivek Goyal
89a9bfd9ddSVivek Goyal static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
90a9bfd9ddSVivek Goyal struct fuse_req *req, bool in_flight);
91a9bfd9ddSVivek Goyal
92780b1b95SJeffle Xu static const struct constant_table dax_param_enums[] = {
93780b1b95SJeffle Xu {"always", FUSE_DAX_ALWAYS },
94780b1b95SJeffle Xu {"never", FUSE_DAX_NEVER },
95780b1b95SJeffle Xu {"inode", FUSE_DAX_INODE_USER },
96780b1b95SJeffle Xu {}
97780b1b95SJeffle Xu };
98780b1b95SJeffle Xu
991dd53957SVivek Goyal enum {
1001dd53957SVivek Goyal OPT_DAX,
101780b1b95SJeffle Xu OPT_DAX_ENUM,
1021dd53957SVivek Goyal };
1031dd53957SVivek Goyal
1041dd53957SVivek Goyal static const struct fs_parameter_spec virtio_fs_parameters[] = {
1051dd53957SVivek Goyal fsparam_flag("dax", OPT_DAX),
106780b1b95SJeffle Xu fsparam_enum("dax", OPT_DAX_ENUM, dax_param_enums),
1071dd53957SVivek Goyal {}
1081dd53957SVivek Goyal };
1091dd53957SVivek Goyal
virtio_fs_parse_param(struct fs_context * fsc,struct fs_parameter * param)11084c21507SMiklos Szeredi static int virtio_fs_parse_param(struct fs_context *fsc,
1111dd53957SVivek Goyal struct fs_parameter *param)
1121dd53957SVivek Goyal {
1131dd53957SVivek Goyal struct fs_parse_result result;
11484c21507SMiklos Szeredi struct fuse_fs_context *ctx = fsc->fs_private;
1151dd53957SVivek Goyal int opt;
1161dd53957SVivek Goyal
11784c21507SMiklos Szeredi opt = fs_parse(fsc, virtio_fs_parameters, param, &result);
1181dd53957SVivek Goyal if (opt < 0)
1191dd53957SVivek Goyal return opt;
1201dd53957SVivek Goyal
1211dd53957SVivek Goyal switch (opt) {
1221dd53957SVivek Goyal case OPT_DAX:
123780b1b95SJeffle Xu ctx->dax_mode = FUSE_DAX_ALWAYS;
124780b1b95SJeffle Xu break;
125780b1b95SJeffle Xu case OPT_DAX_ENUM:
126780b1b95SJeffle Xu ctx->dax_mode = result.uint_32;
1271dd53957SVivek Goyal break;
1281dd53957SVivek Goyal default:
1291dd53957SVivek Goyal return -EINVAL;
1301dd53957SVivek Goyal }
1311dd53957SVivek Goyal
1321dd53957SVivek Goyal return 0;
1331dd53957SVivek Goyal }
1341dd53957SVivek Goyal
virtio_fs_free_fsc(struct fs_context * fsc)13584c21507SMiklos Szeredi static void virtio_fs_free_fsc(struct fs_context *fsc)
1361dd53957SVivek Goyal {
13784c21507SMiklos Szeredi struct fuse_fs_context *ctx = fsc->fs_private;
1381dd53957SVivek Goyal
1391dd53957SVivek Goyal kfree(ctx);
1401dd53957SVivek Goyal }
1411dd53957SVivek Goyal
vq_to_fsvq(struct virtqueue * vq)142a62a8ef9SStefan Hajnoczi static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq)
143a62a8ef9SStefan Hajnoczi {
144a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vq->vdev->priv;
145a62a8ef9SStefan Hajnoczi
146a62a8ef9SStefan Hajnoczi return &fs->vqs[vq->index];
147a62a8ef9SStefan Hajnoczi }
148a62a8ef9SStefan Hajnoczi
149c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */
inc_in_flight_req(struct virtio_fs_vq * fsvq)150c17ea009SVivek Goyal static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq)
151c17ea009SVivek Goyal {
152c17ea009SVivek Goyal fsvq->in_flight++;
153c17ea009SVivek Goyal }
154c17ea009SVivek Goyal
155c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */
dec_in_flight_req(struct virtio_fs_vq * fsvq)156c17ea009SVivek Goyal static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq)
157c17ea009SVivek Goyal {
158c17ea009SVivek Goyal WARN_ON(fsvq->in_flight <= 0);
159c17ea009SVivek Goyal fsvq->in_flight--;
160724c15a4SVivek Goyal if (!fsvq->in_flight)
161724c15a4SVivek Goyal complete(&fsvq->in_flight_zero);
162c17ea009SVivek Goyal }
163c17ea009SVivek Goyal
release_virtio_fs_obj(struct kref * ref)164a62a8ef9SStefan Hajnoczi static void release_virtio_fs_obj(struct kref *ref)
165a62a8ef9SStefan Hajnoczi {
166a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount);
167a62a8ef9SStefan Hajnoczi
168a62a8ef9SStefan Hajnoczi kfree(vfs->vqs);
169a62a8ef9SStefan Hajnoczi kfree(vfs);
170a62a8ef9SStefan Hajnoczi }
171a62a8ef9SStefan Hajnoczi
172a62a8ef9SStefan Hajnoczi /* Make sure virtiofs_mutex is held */
virtio_fs_put(struct virtio_fs * fs)173a62a8ef9SStefan Hajnoczi static void virtio_fs_put(struct virtio_fs *fs)
174a62a8ef9SStefan Hajnoczi {
175a62a8ef9SStefan Hajnoczi kref_put(&fs->refcount, release_virtio_fs_obj);
176a62a8ef9SStefan Hajnoczi }
177a62a8ef9SStefan Hajnoczi
virtio_fs_fiq_release(struct fuse_iqueue * fiq)178a62a8ef9SStefan Hajnoczi static void virtio_fs_fiq_release(struct fuse_iqueue *fiq)
179a62a8ef9SStefan Hajnoczi {
180a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = fiq->priv;
181a62a8ef9SStefan Hajnoczi
182a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex);
183a62a8ef9SStefan Hajnoczi virtio_fs_put(vfs);
184a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex);
185a62a8ef9SStefan Hajnoczi }
186a62a8ef9SStefan Hajnoczi
virtio_fs_drain_queue(struct virtio_fs_vq * fsvq)187a62a8ef9SStefan Hajnoczi static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
188a62a8ef9SStefan Hajnoczi {
189a62a8ef9SStefan Hajnoczi WARN_ON(fsvq->in_flight < 0);
190a62a8ef9SStefan Hajnoczi
191a62a8ef9SStefan Hajnoczi /* Wait for in flight requests to finish.*/
192a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock);
193724c15a4SVivek Goyal if (fsvq->in_flight) {
194724c15a4SVivek Goyal /* We are holding virtio_fs_mutex. There should not be any
195724c15a4SVivek Goyal * waiters waiting for completion.
196724c15a4SVivek Goyal */
197724c15a4SVivek Goyal reinit_completion(&fsvq->in_flight_zero);
198a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
199724c15a4SVivek Goyal wait_for_completion(&fsvq->in_flight_zero);
200724c15a4SVivek Goyal } else {
201a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
202a62a8ef9SStefan Hajnoczi }
203a62a8ef9SStefan Hajnoczi
204a62a8ef9SStefan Hajnoczi flush_work(&fsvq->done_work);
205a62a8ef9SStefan Hajnoczi flush_delayed_work(&fsvq->dispatch_work);
206a62a8ef9SStefan Hajnoczi }
207a62a8ef9SStefan Hajnoczi
virtio_fs_drain_all_queues_locked(struct virtio_fs * fs)208724c15a4SVivek Goyal static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs)
209a62a8ef9SStefan Hajnoczi {
210a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq;
211a62a8ef9SStefan Hajnoczi int i;
212a62a8ef9SStefan Hajnoczi
213a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) {
214a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i];
215a62a8ef9SStefan Hajnoczi virtio_fs_drain_queue(fsvq);
216a62a8ef9SStefan Hajnoczi }
217a62a8ef9SStefan Hajnoczi }
218a62a8ef9SStefan Hajnoczi
virtio_fs_drain_all_queues(struct virtio_fs * fs)219724c15a4SVivek Goyal static void virtio_fs_drain_all_queues(struct virtio_fs *fs)
220724c15a4SVivek Goyal {
221724c15a4SVivek Goyal /* Provides mutual exclusion between ->remove and ->kill_sb
222724c15a4SVivek Goyal * paths. We don't want both of these draining queue at the
223724c15a4SVivek Goyal * same time. Current completion logic reinits completion
224724c15a4SVivek Goyal * and that means there should not be any other thread
225724c15a4SVivek Goyal * doing reinit or waiting for completion already.
226724c15a4SVivek Goyal */
227724c15a4SVivek Goyal mutex_lock(&virtio_fs_mutex);
228724c15a4SVivek Goyal virtio_fs_drain_all_queues_locked(fs);
229724c15a4SVivek Goyal mutex_unlock(&virtio_fs_mutex);
230724c15a4SVivek Goyal }
231724c15a4SVivek Goyal
virtio_fs_start_all_queues(struct virtio_fs * fs)232a62a8ef9SStefan Hajnoczi static void virtio_fs_start_all_queues(struct virtio_fs *fs)
233a62a8ef9SStefan Hajnoczi {
234a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq;
235a62a8ef9SStefan Hajnoczi int i;
236a62a8ef9SStefan Hajnoczi
237a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) {
238a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i];
239a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock);
240a62a8ef9SStefan Hajnoczi fsvq->connected = true;
241a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
242a62a8ef9SStefan Hajnoczi }
243a62a8ef9SStefan Hajnoczi }
244a62a8ef9SStefan Hajnoczi
245a62a8ef9SStefan Hajnoczi /* Add a new instance to the list or return -EEXIST if tag name exists*/
virtio_fs_add_instance(struct virtio_fs * fs)246a62a8ef9SStefan Hajnoczi static int virtio_fs_add_instance(struct virtio_fs *fs)
247a62a8ef9SStefan Hajnoczi {
248a62a8ef9SStefan Hajnoczi struct virtio_fs *fs2;
249a62a8ef9SStefan Hajnoczi bool duplicate = false;
250a62a8ef9SStefan Hajnoczi
251a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex);
252a62a8ef9SStefan Hajnoczi
253a62a8ef9SStefan Hajnoczi list_for_each_entry(fs2, &virtio_fs_instances, list) {
254a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, fs2->tag) == 0)
255a62a8ef9SStefan Hajnoczi duplicate = true;
256a62a8ef9SStefan Hajnoczi }
257a62a8ef9SStefan Hajnoczi
258a62a8ef9SStefan Hajnoczi if (!duplicate)
259a62a8ef9SStefan Hajnoczi list_add_tail(&fs->list, &virtio_fs_instances);
260a62a8ef9SStefan Hajnoczi
261a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex);
262a62a8ef9SStefan Hajnoczi
263a62a8ef9SStefan Hajnoczi if (duplicate)
264a62a8ef9SStefan Hajnoczi return -EEXIST;
265a62a8ef9SStefan Hajnoczi return 0;
266a62a8ef9SStefan Hajnoczi }
267a62a8ef9SStefan Hajnoczi
268a62a8ef9SStefan Hajnoczi /* Return the virtio_fs with a given tag, or NULL */
virtio_fs_find_instance(const char * tag)269a62a8ef9SStefan Hajnoczi static struct virtio_fs *virtio_fs_find_instance(const char *tag)
270a62a8ef9SStefan Hajnoczi {
271a62a8ef9SStefan Hajnoczi struct virtio_fs *fs;
272a62a8ef9SStefan Hajnoczi
273a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex);
274a62a8ef9SStefan Hajnoczi
275a62a8ef9SStefan Hajnoczi list_for_each_entry(fs, &virtio_fs_instances, list) {
276a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, tag) == 0) {
277a62a8ef9SStefan Hajnoczi kref_get(&fs->refcount);
278a62a8ef9SStefan Hajnoczi goto found;
279a62a8ef9SStefan Hajnoczi }
280a62a8ef9SStefan Hajnoczi }
281a62a8ef9SStefan Hajnoczi
282a62a8ef9SStefan Hajnoczi fs = NULL; /* not found */
283a62a8ef9SStefan Hajnoczi
284a62a8ef9SStefan Hajnoczi found:
285a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex);
286a62a8ef9SStefan Hajnoczi
287a62a8ef9SStefan Hajnoczi return fs;
288a62a8ef9SStefan Hajnoczi }
289a62a8ef9SStefan Hajnoczi
virtio_fs_free_devs(struct virtio_fs * fs)290a62a8ef9SStefan Hajnoczi static void virtio_fs_free_devs(struct virtio_fs *fs)
291a62a8ef9SStefan Hajnoczi {
292a62a8ef9SStefan Hajnoczi unsigned int i;
293a62a8ef9SStefan Hajnoczi
294a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) {
295a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i];
296a62a8ef9SStefan Hajnoczi
297a62a8ef9SStefan Hajnoczi if (!fsvq->fud)
298a62a8ef9SStefan Hajnoczi continue;
299a62a8ef9SStefan Hajnoczi
300a62a8ef9SStefan Hajnoczi fuse_dev_free(fsvq->fud);
301a62a8ef9SStefan Hajnoczi fsvq->fud = NULL;
302a62a8ef9SStefan Hajnoczi }
303a62a8ef9SStefan Hajnoczi }
304a62a8ef9SStefan Hajnoczi
305a62a8ef9SStefan Hajnoczi /* Read filesystem name from virtio config into fs->tag (must kfree()). */
virtio_fs_read_tag(struct virtio_device * vdev,struct virtio_fs * fs)306a62a8ef9SStefan Hajnoczi static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs)
307a62a8ef9SStefan Hajnoczi {
308a62a8ef9SStefan Hajnoczi char tag_buf[sizeof_field(struct virtio_fs_config, tag)];
309a62a8ef9SStefan Hajnoczi char *end;
310a62a8ef9SStefan Hajnoczi size_t len;
311a62a8ef9SStefan Hajnoczi
312a62a8ef9SStefan Hajnoczi virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag),
313a62a8ef9SStefan Hajnoczi &tag_buf, sizeof(tag_buf));
314a62a8ef9SStefan Hajnoczi end = memchr(tag_buf, '\0', sizeof(tag_buf));
315a62a8ef9SStefan Hajnoczi if (end == tag_buf)
316a62a8ef9SStefan Hajnoczi return -EINVAL; /* empty tag */
317a62a8ef9SStefan Hajnoczi if (!end)
318a62a8ef9SStefan Hajnoczi end = &tag_buf[sizeof(tag_buf)];
319a62a8ef9SStefan Hajnoczi
320a62a8ef9SStefan Hajnoczi len = end - tag_buf;
321a62a8ef9SStefan Hajnoczi fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL);
322a62a8ef9SStefan Hajnoczi if (!fs->tag)
323a62a8ef9SStefan Hajnoczi return -ENOMEM;
324a62a8ef9SStefan Hajnoczi memcpy(fs->tag, tag_buf, len);
325a62a8ef9SStefan Hajnoczi fs->tag[len] = '\0';
326a62a8ef9SStefan Hajnoczi return 0;
327a62a8ef9SStefan Hajnoczi }
328a62a8ef9SStefan Hajnoczi
329a62a8ef9SStefan Hajnoczi /* Work function for hiprio completion */
virtio_fs_hiprio_done_work(struct work_struct * work)330a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_done_work(struct work_struct *work)
331a62a8ef9SStefan Hajnoczi {
332a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
333a62a8ef9SStefan Hajnoczi done_work);
334a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq;
335a62a8ef9SStefan Hajnoczi
336a62a8ef9SStefan Hajnoczi /* Free completed FUSE_FORGET requests */
337a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock);
338a62a8ef9SStefan Hajnoczi do {
339a62a8ef9SStefan Hajnoczi unsigned int len;
340a62a8ef9SStefan Hajnoczi void *req;
341a62a8ef9SStefan Hajnoczi
342a62a8ef9SStefan Hajnoczi virtqueue_disable_cb(vq);
343a62a8ef9SStefan Hajnoczi
344a62a8ef9SStefan Hajnoczi while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
345a62a8ef9SStefan Hajnoczi kfree(req);
346c17ea009SVivek Goyal dec_in_flight_req(fsvq);
347a62a8ef9SStefan Hajnoczi }
348a62a8ef9SStefan Hajnoczi } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
349a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
350a62a8ef9SStefan Hajnoczi }
351a62a8ef9SStefan Hajnoczi
virtio_fs_request_dispatch_work(struct work_struct * work)35251fecdd2SVivek Goyal static void virtio_fs_request_dispatch_work(struct work_struct *work)
353a62a8ef9SStefan Hajnoczi {
35451fecdd2SVivek Goyal struct fuse_req *req;
35551fecdd2SVivek Goyal struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
35651fecdd2SVivek Goyal dispatch_work.work);
357a9bfd9ddSVivek Goyal int ret;
35851fecdd2SVivek Goyal
35951fecdd2SVivek Goyal pr_debug("virtio-fs: worker %s called.\n", __func__);
36051fecdd2SVivek Goyal while (1) {
36151fecdd2SVivek Goyal spin_lock(&fsvq->lock);
36251fecdd2SVivek Goyal req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req,
36351fecdd2SVivek Goyal list);
36451fecdd2SVivek Goyal if (!req) {
36551fecdd2SVivek Goyal spin_unlock(&fsvq->lock);
366a9bfd9ddSVivek Goyal break;
36751fecdd2SVivek Goyal }
36851fecdd2SVivek Goyal
36951fecdd2SVivek Goyal list_del_init(&req->list);
37051fecdd2SVivek Goyal spin_unlock(&fsvq->lock);
3718f622e94SMax Reitz fuse_request_end(req);
37251fecdd2SVivek Goyal }
373a9bfd9ddSVivek Goyal
374a9bfd9ddSVivek Goyal /* Dispatch pending requests */
375a9bfd9ddSVivek Goyal while (1) {
376a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock);
377a9bfd9ddSVivek Goyal req = list_first_entry_or_null(&fsvq->queued_reqs,
378a9bfd9ddSVivek Goyal struct fuse_req, list);
379a9bfd9ddSVivek Goyal if (!req) {
380a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock);
381a9bfd9ddSVivek Goyal return;
382a9bfd9ddSVivek Goyal }
383a9bfd9ddSVivek Goyal list_del_init(&req->list);
384a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock);
385a9bfd9ddSVivek Goyal
386a9bfd9ddSVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req, true);
387a9bfd9ddSVivek Goyal if (ret < 0) {
388a9bfd9ddSVivek Goyal if (ret == -ENOMEM || ret == -ENOSPC) {
389a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock);
390a9bfd9ddSVivek Goyal list_add_tail(&req->list, &fsvq->queued_reqs);
391a9bfd9ddSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work,
392a9bfd9ddSVivek Goyal msecs_to_jiffies(1));
393a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock);
394a9bfd9ddSVivek Goyal return;
395a9bfd9ddSVivek Goyal }
396a9bfd9ddSVivek Goyal req->out.h.error = ret;
397a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock);
398a9bfd9ddSVivek Goyal dec_in_flight_req(fsvq);
399a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock);
400a9bfd9ddSVivek Goyal pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n",
401a9bfd9ddSVivek Goyal ret);
4028f622e94SMax Reitz fuse_request_end(req);
403a9bfd9ddSVivek Goyal }
404a9bfd9ddSVivek Goyal }
405a62a8ef9SStefan Hajnoczi }
406a62a8ef9SStefan Hajnoczi
40758ada94fSVivek Goyal /*
40858ada94fSVivek Goyal * Returns 1 if queue is full and sender should wait a bit before sending
40958ada94fSVivek Goyal * next request, 0 otherwise.
41058ada94fSVivek Goyal */
send_forget_request(struct virtio_fs_vq * fsvq,struct virtio_fs_forget * forget,bool in_flight)41158ada94fSVivek Goyal static int send_forget_request(struct virtio_fs_vq *fsvq,
41258ada94fSVivek Goyal struct virtio_fs_forget *forget,
41358ada94fSVivek Goyal bool in_flight)
41458ada94fSVivek Goyal {
41558ada94fSVivek Goyal struct scatterlist sg;
41658ada94fSVivek Goyal struct virtqueue *vq;
41758ada94fSVivek Goyal int ret = 0;
41858ada94fSVivek Goyal bool notify;
4191efcf39eSVivek Goyal struct virtio_fs_forget_req *req = &forget->req;
42058ada94fSVivek Goyal
42158ada94fSVivek Goyal spin_lock(&fsvq->lock);
42258ada94fSVivek Goyal if (!fsvq->connected) {
42358ada94fSVivek Goyal if (in_flight)
42458ada94fSVivek Goyal dec_in_flight_req(fsvq);
42558ada94fSVivek Goyal kfree(forget);
42658ada94fSVivek Goyal goto out;
42758ada94fSVivek Goyal }
42858ada94fSVivek Goyal
4291efcf39eSVivek Goyal sg_init_one(&sg, req, sizeof(*req));
43058ada94fSVivek Goyal vq = fsvq->vq;
43158ada94fSVivek Goyal dev_dbg(&vq->vdev->dev, "%s\n", __func__);
43258ada94fSVivek Goyal
43358ada94fSVivek Goyal ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC);
43458ada94fSVivek Goyal if (ret < 0) {
43558ada94fSVivek Goyal if (ret == -ENOMEM || ret == -ENOSPC) {
43658ada94fSVivek Goyal pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n",
43758ada94fSVivek Goyal ret);
43858ada94fSVivek Goyal list_add_tail(&forget->list, &fsvq->queued_reqs);
43958ada94fSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work,
44058ada94fSVivek Goyal msecs_to_jiffies(1));
44158ada94fSVivek Goyal if (!in_flight)
44258ada94fSVivek Goyal inc_in_flight_req(fsvq);
44358ada94fSVivek Goyal /* Queue is full */
44458ada94fSVivek Goyal ret = 1;
44558ada94fSVivek Goyal } else {
44658ada94fSVivek Goyal pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n",
44758ada94fSVivek Goyal ret);
44858ada94fSVivek Goyal kfree(forget);
44958ada94fSVivek Goyal if (in_flight)
45058ada94fSVivek Goyal dec_in_flight_req(fsvq);
45158ada94fSVivek Goyal }
45258ada94fSVivek Goyal goto out;
45358ada94fSVivek Goyal }
45458ada94fSVivek Goyal
45558ada94fSVivek Goyal if (!in_flight)
45658ada94fSVivek Goyal inc_in_flight_req(fsvq);
45758ada94fSVivek Goyal notify = virtqueue_kick_prepare(vq);
45858ada94fSVivek Goyal spin_unlock(&fsvq->lock);
45958ada94fSVivek Goyal
46058ada94fSVivek Goyal if (notify)
46158ada94fSVivek Goyal virtqueue_notify(vq);
46258ada94fSVivek Goyal return ret;
46358ada94fSVivek Goyal out:
46458ada94fSVivek Goyal spin_unlock(&fsvq->lock);
46558ada94fSVivek Goyal return ret;
46658ada94fSVivek Goyal }
46758ada94fSVivek Goyal
virtio_fs_hiprio_dispatch_work(struct work_struct * work)468a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_dispatch_work(struct work_struct *work)
469a62a8ef9SStefan Hajnoczi {
470a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget;
471a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
472a62a8ef9SStefan Hajnoczi dispatch_work.work);
473a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: worker %s called.\n", __func__);
474a62a8ef9SStefan Hajnoczi while (1) {
475a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock);
476a62a8ef9SStefan Hajnoczi forget = list_first_entry_or_null(&fsvq->queued_reqs,
477a62a8ef9SStefan Hajnoczi struct virtio_fs_forget, list);
478a62a8ef9SStefan Hajnoczi if (!forget) {
479a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
480a62a8ef9SStefan Hajnoczi return;
481a62a8ef9SStefan Hajnoczi }
482a62a8ef9SStefan Hajnoczi
483a62a8ef9SStefan Hajnoczi list_del(&forget->list);
484a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
48558ada94fSVivek Goyal if (send_forget_request(fsvq, forget, true))
486a62a8ef9SStefan Hajnoczi return;
487a62a8ef9SStefan Hajnoczi }
488a62a8ef9SStefan Hajnoczi }
489a62a8ef9SStefan Hajnoczi
490a62a8ef9SStefan Hajnoczi /* Allocate and copy args into req->argbuf */
copy_args_to_argbuf(struct fuse_req * req)491a62a8ef9SStefan Hajnoczi static int copy_args_to_argbuf(struct fuse_req *req)
492a62a8ef9SStefan Hajnoczi {
493a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args;
494a62a8ef9SStefan Hajnoczi unsigned int offset = 0;
495a62a8ef9SStefan Hajnoczi unsigned int num_in;
496a62a8ef9SStefan Hajnoczi unsigned int num_out;
497a62a8ef9SStefan Hajnoczi unsigned int len;
498a62a8ef9SStefan Hajnoczi unsigned int i;
499a62a8ef9SStefan Hajnoczi
500a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages;
501a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages;
502a62a8ef9SStefan Hajnoczi len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) +
503a62a8ef9SStefan Hajnoczi fuse_len_args(num_out, args->out_args);
504a62a8ef9SStefan Hajnoczi
505a62a8ef9SStefan Hajnoczi req->argbuf = kmalloc(len, GFP_ATOMIC);
506a62a8ef9SStefan Hajnoczi if (!req->argbuf)
507a62a8ef9SStefan Hajnoczi return -ENOMEM;
508a62a8ef9SStefan Hajnoczi
509a62a8ef9SStefan Hajnoczi for (i = 0; i < num_in; i++) {
510a62a8ef9SStefan Hajnoczi memcpy(req->argbuf + offset,
511a62a8ef9SStefan Hajnoczi args->in_args[i].value,
512a62a8ef9SStefan Hajnoczi args->in_args[i].size);
513a62a8ef9SStefan Hajnoczi offset += args->in_args[i].size;
514a62a8ef9SStefan Hajnoczi }
515a62a8ef9SStefan Hajnoczi
516a62a8ef9SStefan Hajnoczi return 0;
517a62a8ef9SStefan Hajnoczi }
518a62a8ef9SStefan Hajnoczi
519a62a8ef9SStefan Hajnoczi /* Copy args out of and free req->argbuf */
copy_args_from_argbuf(struct fuse_args * args,struct fuse_req * req)520a62a8ef9SStefan Hajnoczi static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req)
521a62a8ef9SStefan Hajnoczi {
522a62a8ef9SStefan Hajnoczi unsigned int remaining;
523a62a8ef9SStefan Hajnoczi unsigned int offset;
524a62a8ef9SStefan Hajnoczi unsigned int num_in;
525a62a8ef9SStefan Hajnoczi unsigned int num_out;
526a62a8ef9SStefan Hajnoczi unsigned int i;
527a62a8ef9SStefan Hajnoczi
528a62a8ef9SStefan Hajnoczi remaining = req->out.h.len - sizeof(req->out.h);
529a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages;
530a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages;
531a62a8ef9SStefan Hajnoczi offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
532a62a8ef9SStefan Hajnoczi
533a62a8ef9SStefan Hajnoczi for (i = 0; i < num_out; i++) {
534a62a8ef9SStefan Hajnoczi unsigned int argsize = args->out_args[i].size;
535a62a8ef9SStefan Hajnoczi
536a62a8ef9SStefan Hajnoczi if (args->out_argvar &&
537a62a8ef9SStefan Hajnoczi i == args->out_numargs - 1 &&
538a62a8ef9SStefan Hajnoczi argsize > remaining) {
539a62a8ef9SStefan Hajnoczi argsize = remaining;
540a62a8ef9SStefan Hajnoczi }
541a62a8ef9SStefan Hajnoczi
542a62a8ef9SStefan Hajnoczi memcpy(args->out_args[i].value, req->argbuf + offset, argsize);
543a62a8ef9SStefan Hajnoczi offset += argsize;
544a62a8ef9SStefan Hajnoczi
545a62a8ef9SStefan Hajnoczi if (i != args->out_numargs - 1)
546a62a8ef9SStefan Hajnoczi remaining -= argsize;
547a62a8ef9SStefan Hajnoczi }
548a62a8ef9SStefan Hajnoczi
549a62a8ef9SStefan Hajnoczi /* Store the actual size of the variable-length arg */
550a62a8ef9SStefan Hajnoczi if (args->out_argvar)
551a62a8ef9SStefan Hajnoczi args->out_args[args->out_numargs - 1].size = remaining;
552a62a8ef9SStefan Hajnoczi
553a62a8ef9SStefan Hajnoczi kfree(req->argbuf);
554a62a8ef9SStefan Hajnoczi req->argbuf = NULL;
555a62a8ef9SStefan Hajnoczi }
556a62a8ef9SStefan Hajnoczi
557a62a8ef9SStefan Hajnoczi /* Work function for request completion */
virtio_fs_request_complete(struct fuse_req * req,struct virtio_fs_vq * fsvq)558bb737bbeSVivek Goyal static void virtio_fs_request_complete(struct fuse_req *req,
559bb737bbeSVivek Goyal struct virtio_fs_vq *fsvq)
560a62a8ef9SStefan Hajnoczi {
561a62a8ef9SStefan Hajnoczi struct fuse_pqueue *fpq = &fsvq->fud->pq;
562a62a8ef9SStefan Hajnoczi struct fuse_args *args;
563bb737bbeSVivek Goyal struct fuse_args_pages *ap;
564a62a8ef9SStefan Hajnoczi unsigned int len, i, thislen;
565a62a8ef9SStefan Hajnoczi struct page *page;
566a62a8ef9SStefan Hajnoczi
567a62a8ef9SStefan Hajnoczi /*
568a62a8ef9SStefan Hajnoczi * TODO verify that server properly follows FUSE protocol
569a62a8ef9SStefan Hajnoczi * (oh.uniq, oh.len)
570a62a8ef9SStefan Hajnoczi */
571a62a8ef9SStefan Hajnoczi args = req->args;
572a62a8ef9SStefan Hajnoczi copy_args_from_argbuf(args, req);
573a62a8ef9SStefan Hajnoczi
574a62a8ef9SStefan Hajnoczi if (args->out_pages && args->page_zeroing) {
575a62a8ef9SStefan Hajnoczi len = args->out_args[args->out_numargs - 1].size;
576a62a8ef9SStefan Hajnoczi ap = container_of(args, typeof(*ap), args);
577a62a8ef9SStefan Hajnoczi for (i = 0; i < ap->num_pages; i++) {
578a62a8ef9SStefan Hajnoczi thislen = ap->descs[i].length;
579a62a8ef9SStefan Hajnoczi if (len < thislen) {
580a62a8ef9SStefan Hajnoczi WARN_ON(ap->descs[i].offset);
581a62a8ef9SStefan Hajnoczi page = ap->pages[i];
582a62a8ef9SStefan Hajnoczi zero_user_segment(page, len, thislen);
583a62a8ef9SStefan Hajnoczi len = 0;
584a62a8ef9SStefan Hajnoczi } else {
585a62a8ef9SStefan Hajnoczi len -= thislen;
586a62a8ef9SStefan Hajnoczi }
587a62a8ef9SStefan Hajnoczi }
588a62a8ef9SStefan Hajnoczi }
589a62a8ef9SStefan Hajnoczi
590a62a8ef9SStefan Hajnoczi spin_lock(&fpq->lock);
591a62a8ef9SStefan Hajnoczi clear_bit(FR_SENT, &req->flags);
592a62a8ef9SStefan Hajnoczi spin_unlock(&fpq->lock);
593a62a8ef9SStefan Hajnoczi
5948f622e94SMax Reitz fuse_request_end(req);
595a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock);
596c17ea009SVivek Goyal dec_in_flight_req(fsvq);
597a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
598a62a8ef9SStefan Hajnoczi }
599bb737bbeSVivek Goyal
virtio_fs_complete_req_work(struct work_struct * work)600bb737bbeSVivek Goyal static void virtio_fs_complete_req_work(struct work_struct *work)
601bb737bbeSVivek Goyal {
602bb737bbeSVivek Goyal struct virtio_fs_req_work *w =
603bb737bbeSVivek Goyal container_of(work, typeof(*w), done_work);
604bb737bbeSVivek Goyal
605bb737bbeSVivek Goyal virtio_fs_request_complete(w->req, w->fsvq);
606bb737bbeSVivek Goyal kfree(w);
607bb737bbeSVivek Goyal }
608bb737bbeSVivek Goyal
virtio_fs_requests_done_work(struct work_struct * work)609bb737bbeSVivek Goyal static void virtio_fs_requests_done_work(struct work_struct *work)
610bb737bbeSVivek Goyal {
611bb737bbeSVivek Goyal struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq,
612bb737bbeSVivek Goyal done_work);
613bb737bbeSVivek Goyal struct fuse_pqueue *fpq = &fsvq->fud->pq;
614bb737bbeSVivek Goyal struct virtqueue *vq = fsvq->vq;
615bb737bbeSVivek Goyal struct fuse_req *req;
616bb737bbeSVivek Goyal struct fuse_req *next;
617bb737bbeSVivek Goyal unsigned int len;
618bb737bbeSVivek Goyal LIST_HEAD(reqs);
619bb737bbeSVivek Goyal
620bb737bbeSVivek Goyal /* Collect completed requests off the virtqueue */
621bb737bbeSVivek Goyal spin_lock(&fsvq->lock);
622bb737bbeSVivek Goyal do {
623bb737bbeSVivek Goyal virtqueue_disable_cb(vq);
624bb737bbeSVivek Goyal
625bb737bbeSVivek Goyal while ((req = virtqueue_get_buf(vq, &len)) != NULL) {
626bb737bbeSVivek Goyal spin_lock(&fpq->lock);
627bb737bbeSVivek Goyal list_move_tail(&req->list, &reqs);
628bb737bbeSVivek Goyal spin_unlock(&fpq->lock);
629bb737bbeSVivek Goyal }
630bb737bbeSVivek Goyal } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq)));
631bb737bbeSVivek Goyal spin_unlock(&fsvq->lock);
632bb737bbeSVivek Goyal
633bb737bbeSVivek Goyal /* End requests */
634bb737bbeSVivek Goyal list_for_each_entry_safe(req, next, &reqs, list) {
635bb737bbeSVivek Goyal list_del_init(&req->list);
636bb737bbeSVivek Goyal
637bb737bbeSVivek Goyal /* blocking async request completes in a worker context */
638bb737bbeSVivek Goyal if (req->args->may_block) {
639bb737bbeSVivek Goyal struct virtio_fs_req_work *w;
640bb737bbeSVivek Goyal
641bb737bbeSVivek Goyal w = kzalloc(sizeof(*w), GFP_NOFS | __GFP_NOFAIL);
642bb737bbeSVivek Goyal INIT_WORK(&w->done_work, virtio_fs_complete_req_work);
643bb737bbeSVivek Goyal w->fsvq = fsvq;
644bb737bbeSVivek Goyal w->req = req;
645bb737bbeSVivek Goyal schedule_work(&w->done_work);
646bb737bbeSVivek Goyal } else {
647bb737bbeSVivek Goyal virtio_fs_request_complete(req, fsvq);
648bb737bbeSVivek Goyal }
649bb737bbeSVivek Goyal }
650a62a8ef9SStefan Hajnoczi }
651a62a8ef9SStefan Hajnoczi
652a62a8ef9SStefan Hajnoczi /* Virtqueue interrupt handler */
virtio_fs_vq_done(struct virtqueue * vq)653a62a8ef9SStefan Hajnoczi static void virtio_fs_vq_done(struct virtqueue *vq)
654a62a8ef9SStefan Hajnoczi {
655a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = vq_to_fsvq(vq);
656a62a8ef9SStefan Hajnoczi
657a62a8ef9SStefan Hajnoczi dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name);
658a62a8ef9SStefan Hajnoczi
659a62a8ef9SStefan Hajnoczi schedule_work(&fsvq->done_work);
660a62a8ef9SStefan Hajnoczi }
661a62a8ef9SStefan Hajnoczi
virtio_fs_init_vq(struct virtio_fs_vq * fsvq,char * name,int vq_type)662b43b7e81SVivek Goyal static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name,
663b43b7e81SVivek Goyal int vq_type)
664b43b7e81SVivek Goyal {
6657c594bbdSMiklos Szeredi strscpy(fsvq->name, name, VQ_NAME_LEN);
666b43b7e81SVivek Goyal spin_lock_init(&fsvq->lock);
667b43b7e81SVivek Goyal INIT_LIST_HEAD(&fsvq->queued_reqs);
668b43b7e81SVivek Goyal INIT_LIST_HEAD(&fsvq->end_reqs);
669b43b7e81SVivek Goyal init_completion(&fsvq->in_flight_zero);
670b43b7e81SVivek Goyal
671b43b7e81SVivek Goyal if (vq_type == VQ_REQUEST) {
672b43b7e81SVivek Goyal INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work);
673b43b7e81SVivek Goyal INIT_DELAYED_WORK(&fsvq->dispatch_work,
674b43b7e81SVivek Goyal virtio_fs_request_dispatch_work);
675b43b7e81SVivek Goyal } else {
676b43b7e81SVivek Goyal INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work);
677b43b7e81SVivek Goyal INIT_DELAYED_WORK(&fsvq->dispatch_work,
678b43b7e81SVivek Goyal virtio_fs_hiprio_dispatch_work);
679b43b7e81SVivek Goyal }
680b43b7e81SVivek Goyal }
681b43b7e81SVivek Goyal
682a62a8ef9SStefan Hajnoczi /* Initialize virtqueues */
virtio_fs_setup_vqs(struct virtio_device * vdev,struct virtio_fs * fs)683a62a8ef9SStefan Hajnoczi static int virtio_fs_setup_vqs(struct virtio_device *vdev,
684a62a8ef9SStefan Hajnoczi struct virtio_fs *fs)
685a62a8ef9SStefan Hajnoczi {
686a62a8ef9SStefan Hajnoczi struct virtqueue **vqs;
687a62a8ef9SStefan Hajnoczi vq_callback_t **callbacks;
688a62a8ef9SStefan Hajnoczi const char **names;
689a62a8ef9SStefan Hajnoczi unsigned int i;
690a62a8ef9SStefan Hajnoczi int ret = 0;
691a62a8ef9SStefan Hajnoczi
6922c0349ecSMichael S. Tsirkin virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues,
693a62a8ef9SStefan Hajnoczi &fs->num_request_queues);
694a62a8ef9SStefan Hajnoczi if (fs->num_request_queues == 0)
695a62a8ef9SStefan Hajnoczi return -EINVAL;
696a62a8ef9SStefan Hajnoczi
697b43b7e81SVivek Goyal fs->nvqs = VQ_REQUEST + fs->num_request_queues;
698a62a8ef9SStefan Hajnoczi fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL);
699a62a8ef9SStefan Hajnoczi if (!fs->vqs)
700a62a8ef9SStefan Hajnoczi return -ENOMEM;
701a62a8ef9SStefan Hajnoczi
702a62a8ef9SStefan Hajnoczi vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL);
703a62a8ef9SStefan Hajnoczi callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]),
704a62a8ef9SStefan Hajnoczi GFP_KERNEL);
705a62a8ef9SStefan Hajnoczi names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL);
706a62a8ef9SStefan Hajnoczi if (!vqs || !callbacks || !names) {
707a62a8ef9SStefan Hajnoczi ret = -ENOMEM;
708a62a8ef9SStefan Hajnoczi goto out;
709a62a8ef9SStefan Hajnoczi }
710a62a8ef9SStefan Hajnoczi
711b43b7e81SVivek Goyal /* Initialize the hiprio/forget request virtqueue */
712a62a8ef9SStefan Hajnoczi callbacks[VQ_HIPRIO] = virtio_fs_vq_done;
713b43b7e81SVivek Goyal virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO);
714a62a8ef9SStefan Hajnoczi names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name;
715a62a8ef9SStefan Hajnoczi
716a62a8ef9SStefan Hajnoczi /* Initialize the requests virtqueues */
717a62a8ef9SStefan Hajnoczi for (i = VQ_REQUEST; i < fs->nvqs; i++) {
718b43b7e81SVivek Goyal char vq_name[VQ_NAME_LEN];
719b43b7e81SVivek Goyal
720b43b7e81SVivek Goyal snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST);
721b43b7e81SVivek Goyal virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST);
722a62a8ef9SStefan Hajnoczi callbacks[i] = virtio_fs_vq_done;
723a62a8ef9SStefan Hajnoczi names[i] = fs->vqs[i].name;
724a62a8ef9SStefan Hajnoczi }
725a62a8ef9SStefan Hajnoczi
726a62a8ef9SStefan Hajnoczi ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL);
727a62a8ef9SStefan Hajnoczi if (ret < 0)
728a62a8ef9SStefan Hajnoczi goto out;
729a62a8ef9SStefan Hajnoczi
730a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++)
731a62a8ef9SStefan Hajnoczi fs->vqs[i].vq = vqs[i];
732a62a8ef9SStefan Hajnoczi
733a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs);
734a62a8ef9SStefan Hajnoczi out:
735a62a8ef9SStefan Hajnoczi kfree(names);
736a62a8ef9SStefan Hajnoczi kfree(callbacks);
737a62a8ef9SStefan Hajnoczi kfree(vqs);
738a62a8ef9SStefan Hajnoczi if (ret)
739a62a8ef9SStefan Hajnoczi kfree(fs->vqs);
740a62a8ef9SStefan Hajnoczi return ret;
741a62a8ef9SStefan Hajnoczi }
742a62a8ef9SStefan Hajnoczi
743a62a8ef9SStefan Hajnoczi /* Free virtqueues (device must already be reset) */
virtio_fs_cleanup_vqs(struct virtio_device * vdev)7441e5b9e04SDeming Wang static void virtio_fs_cleanup_vqs(struct virtio_device *vdev)
745a62a8ef9SStefan Hajnoczi {
746a62a8ef9SStefan Hajnoczi vdev->config->del_vqs(vdev);
747a62a8ef9SStefan Hajnoczi }
748a62a8ef9SStefan Hajnoczi
74922f3787eSStefan Hajnoczi /* Map a window offset to a page frame number. The window offset will have
75022f3787eSStefan Hajnoczi * been produced by .iomap_begin(), which maps a file offset to a window
75122f3787eSStefan Hajnoczi * offset.
75222f3787eSStefan Hajnoczi */
virtio_fs_direct_access(struct dax_device * dax_dev,pgoff_t pgoff,long nr_pages,enum dax_access_mode mode,void ** kaddr,pfn_t * pfn)75322f3787eSStefan Hajnoczi static long virtio_fs_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
754e511c4a3SJane Chu long nr_pages, enum dax_access_mode mode,
755e511c4a3SJane Chu void **kaddr, pfn_t *pfn)
75622f3787eSStefan Hajnoczi {
75722f3787eSStefan Hajnoczi struct virtio_fs *fs = dax_get_private(dax_dev);
75822f3787eSStefan Hajnoczi phys_addr_t offset = PFN_PHYS(pgoff);
75922f3787eSStefan Hajnoczi size_t max_nr_pages = fs->window_len / PAGE_SIZE - pgoff;
76022f3787eSStefan Hajnoczi
76122f3787eSStefan Hajnoczi if (kaddr)
76222f3787eSStefan Hajnoczi *kaddr = fs->window_kaddr + offset;
76322f3787eSStefan Hajnoczi if (pfn)
76422f3787eSStefan Hajnoczi *pfn = phys_to_pfn_t(fs->window_phys_addr + offset,
76522f3787eSStefan Hajnoczi PFN_DEV | PFN_MAP);
76622f3787eSStefan Hajnoczi return nr_pages > max_nr_pages ? max_nr_pages : nr_pages;
76722f3787eSStefan Hajnoczi }
76822f3787eSStefan Hajnoczi
virtio_fs_zero_page_range(struct dax_device * dax_dev,pgoff_t pgoff,size_t nr_pages)76922f3787eSStefan Hajnoczi static int virtio_fs_zero_page_range(struct dax_device *dax_dev,
77022f3787eSStefan Hajnoczi pgoff_t pgoff, size_t nr_pages)
77122f3787eSStefan Hajnoczi {
77222f3787eSStefan Hajnoczi long rc;
77322f3787eSStefan Hajnoczi void *kaddr;
77422f3787eSStefan Hajnoczi
775e511c4a3SJane Chu rc = dax_direct_access(dax_dev, pgoff, nr_pages, DAX_ACCESS, &kaddr,
776e511c4a3SJane Chu NULL);
77722f3787eSStefan Hajnoczi if (rc < 0)
778*1ea7ca1bSJane Chu return dax_mem2blk_err(rc);
779*1ea7ca1bSJane Chu
78022f3787eSStefan Hajnoczi memset(kaddr, 0, nr_pages << PAGE_SHIFT);
78122f3787eSStefan Hajnoczi dax_flush(dax_dev, kaddr, nr_pages << PAGE_SHIFT);
78222f3787eSStefan Hajnoczi return 0;
78322f3787eSStefan Hajnoczi }
78422f3787eSStefan Hajnoczi
78522f3787eSStefan Hajnoczi static const struct dax_operations virtio_fs_dax_ops = {
78622f3787eSStefan Hajnoczi .direct_access = virtio_fs_direct_access,
78722f3787eSStefan Hajnoczi .zero_page_range = virtio_fs_zero_page_range,
78822f3787eSStefan Hajnoczi };
78922f3787eSStefan Hajnoczi
virtio_fs_cleanup_dax(void * data)79022f3787eSStefan Hajnoczi static void virtio_fs_cleanup_dax(void *data)
79122f3787eSStefan Hajnoczi {
79222f3787eSStefan Hajnoczi struct dax_device *dax_dev = data;
79322f3787eSStefan Hajnoczi
79422f3787eSStefan Hajnoczi kill_dax(dax_dev);
79522f3787eSStefan Hajnoczi put_dax(dax_dev);
79622f3787eSStefan Hajnoczi }
79722f3787eSStefan Hajnoczi
virtio_fs_setup_dax(struct virtio_device * vdev,struct virtio_fs * fs)79822f3787eSStefan Hajnoczi static int virtio_fs_setup_dax(struct virtio_device *vdev, struct virtio_fs *fs)
79922f3787eSStefan Hajnoczi {
80022f3787eSStefan Hajnoczi struct virtio_shm_region cache_reg;
80122f3787eSStefan Hajnoczi struct dev_pagemap *pgmap;
80222f3787eSStefan Hajnoczi bool have_cache;
80322f3787eSStefan Hajnoczi
80422f3787eSStefan Hajnoczi if (!IS_ENABLED(CONFIG_FUSE_DAX))
80522f3787eSStefan Hajnoczi return 0;
80622f3787eSStefan Hajnoczi
80722f3787eSStefan Hajnoczi /* Get cache region */
80822f3787eSStefan Hajnoczi have_cache = virtio_get_shm_region(vdev, &cache_reg,
80922f3787eSStefan Hajnoczi (u8)VIRTIO_FS_SHMCAP_ID_CACHE);
81022f3787eSStefan Hajnoczi if (!have_cache) {
81122f3787eSStefan Hajnoczi dev_notice(&vdev->dev, "%s: No cache capability\n", __func__);
81222f3787eSStefan Hajnoczi return 0;
81322f3787eSStefan Hajnoczi }
81422f3787eSStefan Hajnoczi
81522f3787eSStefan Hajnoczi if (!devm_request_mem_region(&vdev->dev, cache_reg.addr, cache_reg.len,
81622f3787eSStefan Hajnoczi dev_name(&vdev->dev))) {
81722f3787eSStefan Hajnoczi dev_warn(&vdev->dev, "could not reserve region addr=0x%llx len=0x%llx\n",
81822f3787eSStefan Hajnoczi cache_reg.addr, cache_reg.len);
81922f3787eSStefan Hajnoczi return -EBUSY;
82022f3787eSStefan Hajnoczi }
82122f3787eSStefan Hajnoczi
82222f3787eSStefan Hajnoczi dev_notice(&vdev->dev, "Cache len: 0x%llx @ 0x%llx\n", cache_reg.len,
82322f3787eSStefan Hajnoczi cache_reg.addr);
82422f3787eSStefan Hajnoczi
82522f3787eSStefan Hajnoczi pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL);
82622f3787eSStefan Hajnoczi if (!pgmap)
82722f3787eSStefan Hajnoczi return -ENOMEM;
82822f3787eSStefan Hajnoczi
82922f3787eSStefan Hajnoczi pgmap->type = MEMORY_DEVICE_FS_DAX;
83022f3787eSStefan Hajnoczi
83122f3787eSStefan Hajnoczi /* Ideally we would directly use the PCI BAR resource but
83222f3787eSStefan Hajnoczi * devm_memremap_pages() wants its own copy in pgmap. So
83322f3787eSStefan Hajnoczi * initialize a struct resource from scratch (only the start
83422f3787eSStefan Hajnoczi * and end fields will be used).
83522f3787eSStefan Hajnoczi */
83669456535SLinus Torvalds pgmap->range = (struct range) {
83722f3787eSStefan Hajnoczi .start = (phys_addr_t) cache_reg.addr,
83822f3787eSStefan Hajnoczi .end = (phys_addr_t) cache_reg.addr + cache_reg.len - 1,
83922f3787eSStefan Hajnoczi };
84069456535SLinus Torvalds pgmap->nr_range = 1;
84122f3787eSStefan Hajnoczi
84222f3787eSStefan Hajnoczi fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap);
84322f3787eSStefan Hajnoczi if (IS_ERR(fs->window_kaddr))
84422f3787eSStefan Hajnoczi return PTR_ERR(fs->window_kaddr);
84522f3787eSStefan Hajnoczi
84622f3787eSStefan Hajnoczi fs->window_phys_addr = (phys_addr_t) cache_reg.addr;
84722f3787eSStefan Hajnoczi fs->window_len = (phys_addr_t) cache_reg.len;
84822f3787eSStefan Hajnoczi
84922f3787eSStefan Hajnoczi dev_dbg(&vdev->dev, "%s: window kaddr 0x%px phys_addr 0x%llx len 0x%llx\n",
85022f3787eSStefan Hajnoczi __func__, fs->window_kaddr, cache_reg.addr, cache_reg.len);
85122f3787eSStefan Hajnoczi
85230c6828aSChristoph Hellwig fs->dax_dev = alloc_dax(fs, &virtio_fs_dax_ops);
85322f3787eSStefan Hajnoczi if (IS_ERR(fs->dax_dev))
85422f3787eSStefan Hajnoczi return PTR_ERR(fs->dax_dev);
85522f3787eSStefan Hajnoczi
85622f3787eSStefan Hajnoczi return devm_add_action_or_reset(&vdev->dev, virtio_fs_cleanup_dax,
85722f3787eSStefan Hajnoczi fs->dax_dev);
85822f3787eSStefan Hajnoczi }
85922f3787eSStefan Hajnoczi
virtio_fs_probe(struct virtio_device * vdev)860a62a8ef9SStefan Hajnoczi static int virtio_fs_probe(struct virtio_device *vdev)
861a62a8ef9SStefan Hajnoczi {
862a62a8ef9SStefan Hajnoczi struct virtio_fs *fs;
863a62a8ef9SStefan Hajnoczi int ret;
864a62a8ef9SStefan Hajnoczi
865a62a8ef9SStefan Hajnoczi fs = kzalloc(sizeof(*fs), GFP_KERNEL);
866a62a8ef9SStefan Hajnoczi if (!fs)
867a62a8ef9SStefan Hajnoczi return -ENOMEM;
868a62a8ef9SStefan Hajnoczi kref_init(&fs->refcount);
869a62a8ef9SStefan Hajnoczi vdev->priv = fs;
870a62a8ef9SStefan Hajnoczi
871a62a8ef9SStefan Hajnoczi ret = virtio_fs_read_tag(vdev, fs);
872a62a8ef9SStefan Hajnoczi if (ret < 0)
873a62a8ef9SStefan Hajnoczi goto out;
874a62a8ef9SStefan Hajnoczi
875a62a8ef9SStefan Hajnoczi ret = virtio_fs_setup_vqs(vdev, fs);
876a62a8ef9SStefan Hajnoczi if (ret < 0)
877a62a8ef9SStefan Hajnoczi goto out;
878a62a8ef9SStefan Hajnoczi
879a62a8ef9SStefan Hajnoczi /* TODO vq affinity */
880a62a8ef9SStefan Hajnoczi
88122f3787eSStefan Hajnoczi ret = virtio_fs_setup_dax(vdev, fs);
88222f3787eSStefan Hajnoczi if (ret < 0)
88322f3787eSStefan Hajnoczi goto out_vqs;
88422f3787eSStefan Hajnoczi
885a62a8ef9SStefan Hajnoczi /* Bring the device online in case the filesystem is mounted and
886a62a8ef9SStefan Hajnoczi * requests need to be sent before we return.
887a62a8ef9SStefan Hajnoczi */
888a62a8ef9SStefan Hajnoczi virtio_device_ready(vdev);
889a62a8ef9SStefan Hajnoczi
890a62a8ef9SStefan Hajnoczi ret = virtio_fs_add_instance(fs);
891a62a8ef9SStefan Hajnoczi if (ret < 0)
892a62a8ef9SStefan Hajnoczi goto out_vqs;
893a62a8ef9SStefan Hajnoczi
894a62a8ef9SStefan Hajnoczi return 0;
895a62a8ef9SStefan Hajnoczi
896a62a8ef9SStefan Hajnoczi out_vqs:
897d9679d00SMichael S. Tsirkin virtio_reset_device(vdev);
8981e5b9e04SDeming Wang virtio_fs_cleanup_vqs(vdev);
899c79c5e01SLuis Henriques kfree(fs->vqs);
900a62a8ef9SStefan Hajnoczi
901a62a8ef9SStefan Hajnoczi out:
902a62a8ef9SStefan Hajnoczi vdev->priv = NULL;
903a62a8ef9SStefan Hajnoczi kfree(fs);
904a62a8ef9SStefan Hajnoczi return ret;
905a62a8ef9SStefan Hajnoczi }
906a62a8ef9SStefan Hajnoczi
virtio_fs_stop_all_queues(struct virtio_fs * fs)907a62a8ef9SStefan Hajnoczi static void virtio_fs_stop_all_queues(struct virtio_fs *fs)
908a62a8ef9SStefan Hajnoczi {
909a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq;
910a62a8ef9SStefan Hajnoczi int i;
911a62a8ef9SStefan Hajnoczi
912a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) {
913a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i];
914a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock);
915a62a8ef9SStefan Hajnoczi fsvq->connected = false;
916a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
917a62a8ef9SStefan Hajnoczi }
918a62a8ef9SStefan Hajnoczi }
919a62a8ef9SStefan Hajnoczi
virtio_fs_remove(struct virtio_device * vdev)920a62a8ef9SStefan Hajnoczi static void virtio_fs_remove(struct virtio_device *vdev)
921a62a8ef9SStefan Hajnoczi {
922a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vdev->priv;
923a62a8ef9SStefan Hajnoczi
924a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex);
925a62a8ef9SStefan Hajnoczi /* This device is going away. No one should get new reference */
926a62a8ef9SStefan Hajnoczi list_del_init(&fs->list);
927a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(fs);
928724c15a4SVivek Goyal virtio_fs_drain_all_queues_locked(fs);
929d9679d00SMichael S. Tsirkin virtio_reset_device(vdev);
9301e5b9e04SDeming Wang virtio_fs_cleanup_vqs(vdev);
931a62a8ef9SStefan Hajnoczi
932a62a8ef9SStefan Hajnoczi vdev->priv = NULL;
933a62a8ef9SStefan Hajnoczi /* Put device reference on virtio_fs object */
934a62a8ef9SStefan Hajnoczi virtio_fs_put(fs);
935a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex);
936a62a8ef9SStefan Hajnoczi }
937a62a8ef9SStefan Hajnoczi
938a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP
virtio_fs_freeze(struct virtio_device * vdev)939a62a8ef9SStefan Hajnoczi static int virtio_fs_freeze(struct virtio_device *vdev)
940a62a8ef9SStefan Hajnoczi {
941a62a8ef9SStefan Hajnoczi /* TODO need to save state here */
942a62a8ef9SStefan Hajnoczi pr_warn("virtio-fs: suspend/resume not yet supported\n");
943a62a8ef9SStefan Hajnoczi return -EOPNOTSUPP;
944a62a8ef9SStefan Hajnoczi }
945a62a8ef9SStefan Hajnoczi
virtio_fs_restore(struct virtio_device * vdev)946a62a8ef9SStefan Hajnoczi static int virtio_fs_restore(struct virtio_device *vdev)
947a62a8ef9SStefan Hajnoczi {
948a62a8ef9SStefan Hajnoczi /* TODO need to restore state here */
949a62a8ef9SStefan Hajnoczi return 0;
950a62a8ef9SStefan Hajnoczi }
951a62a8ef9SStefan Hajnoczi #endif /* CONFIG_PM_SLEEP */
952a62a8ef9SStefan Hajnoczi
95300929447SYueHaibing static const struct virtio_device_id id_table[] = {
954a62a8ef9SStefan Hajnoczi { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID },
955a62a8ef9SStefan Hajnoczi {},
956a62a8ef9SStefan Hajnoczi };
957a62a8ef9SStefan Hajnoczi
95800929447SYueHaibing static const unsigned int feature_table[] = {};
959a62a8ef9SStefan Hajnoczi
960a62a8ef9SStefan Hajnoczi static struct virtio_driver virtio_fs_driver = {
961a62a8ef9SStefan Hajnoczi .driver.name = KBUILD_MODNAME,
962a62a8ef9SStefan Hajnoczi .driver.owner = THIS_MODULE,
963a62a8ef9SStefan Hajnoczi .id_table = id_table,
964a62a8ef9SStefan Hajnoczi .feature_table = feature_table,
965a62a8ef9SStefan Hajnoczi .feature_table_size = ARRAY_SIZE(feature_table),
966a62a8ef9SStefan Hajnoczi .probe = virtio_fs_probe,
967a62a8ef9SStefan Hajnoczi .remove = virtio_fs_remove,
968a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP
969a62a8ef9SStefan Hajnoczi .freeze = virtio_fs_freeze,
970a62a8ef9SStefan Hajnoczi .restore = virtio_fs_restore,
971a62a8ef9SStefan Hajnoczi #endif
972a62a8ef9SStefan Hajnoczi };
973a62a8ef9SStefan Hajnoczi
virtio_fs_wake_forget_and_unlock(struct fuse_iqueue * fiq)974a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq)
975a62a8ef9SStefan Hajnoczi __releases(fiq->lock)
976a62a8ef9SStefan Hajnoczi {
977a62a8ef9SStefan Hajnoczi struct fuse_forget_link *link;
978a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget;
9791efcf39eSVivek Goyal struct virtio_fs_forget_req *req;
980a62a8ef9SStefan Hajnoczi struct virtio_fs *fs;
981a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq;
982a62a8ef9SStefan Hajnoczi u64 unique;
983a62a8ef9SStefan Hajnoczi
984a62a8ef9SStefan Hajnoczi link = fuse_dequeue_forget(fiq, 1, NULL);
985a62a8ef9SStefan Hajnoczi unique = fuse_get_unique(fiq);
986a62a8ef9SStefan Hajnoczi
987a62a8ef9SStefan Hajnoczi fs = fiq->priv;
988a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[VQ_HIPRIO];
989a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock);
990a62a8ef9SStefan Hajnoczi
991a62a8ef9SStefan Hajnoczi /* Allocate a buffer for the request */
992a62a8ef9SStefan Hajnoczi forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL);
9931efcf39eSVivek Goyal req = &forget->req;
994a62a8ef9SStefan Hajnoczi
9951efcf39eSVivek Goyal req->ih = (struct fuse_in_header){
996a62a8ef9SStefan Hajnoczi .opcode = FUSE_FORGET,
997a62a8ef9SStefan Hajnoczi .nodeid = link->forget_one.nodeid,
998a62a8ef9SStefan Hajnoczi .unique = unique,
9991efcf39eSVivek Goyal .len = sizeof(*req),
1000a62a8ef9SStefan Hajnoczi };
10011efcf39eSVivek Goyal req->arg = (struct fuse_forget_in){
1002a62a8ef9SStefan Hajnoczi .nlookup = link->forget_one.nlookup,
1003a62a8ef9SStefan Hajnoczi };
1004a62a8ef9SStefan Hajnoczi
100558ada94fSVivek Goyal send_forget_request(fsvq, forget, false);
1006a62a8ef9SStefan Hajnoczi kfree(link);
1007a62a8ef9SStefan Hajnoczi }
1008a62a8ef9SStefan Hajnoczi
virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue * fiq)1009a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq)
1010a62a8ef9SStefan Hajnoczi __releases(fiq->lock)
1011a62a8ef9SStefan Hajnoczi {
1012a62a8ef9SStefan Hajnoczi /*
1013a62a8ef9SStefan Hajnoczi * TODO interrupts.
1014a62a8ef9SStefan Hajnoczi *
1015a62a8ef9SStefan Hajnoczi * Normal fs operations on a local filesystems aren't interruptible.
1016a62a8ef9SStefan Hajnoczi * Exceptions are blocking lock operations; for example fcntl(F_SETLKW)
1017a62a8ef9SStefan Hajnoczi * with shared lock between host and guest.
1018a62a8ef9SStefan Hajnoczi */
1019a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock);
1020a62a8ef9SStefan Hajnoczi }
1021a62a8ef9SStefan Hajnoczi
102242d3e2d0SVivek Goyal /* Count number of scatter-gather elements required */
sg_count_fuse_pages(struct fuse_page_desc * page_descs,unsigned int num_pages,unsigned int total_len)102342d3e2d0SVivek Goyal static unsigned int sg_count_fuse_pages(struct fuse_page_desc *page_descs,
102442d3e2d0SVivek Goyal unsigned int num_pages,
102542d3e2d0SVivek Goyal unsigned int total_len)
102642d3e2d0SVivek Goyal {
102742d3e2d0SVivek Goyal unsigned int i;
102842d3e2d0SVivek Goyal unsigned int this_len;
102942d3e2d0SVivek Goyal
103042d3e2d0SVivek Goyal for (i = 0; i < num_pages && total_len; i++) {
103142d3e2d0SVivek Goyal this_len = min(page_descs[i].length, total_len);
103242d3e2d0SVivek Goyal total_len -= this_len;
103342d3e2d0SVivek Goyal }
103442d3e2d0SVivek Goyal
103542d3e2d0SVivek Goyal return i;
103642d3e2d0SVivek Goyal }
103742d3e2d0SVivek Goyal
1038a62a8ef9SStefan Hajnoczi /* Return the number of scatter-gather list elements required */
sg_count_fuse_req(struct fuse_req * req)1039a62a8ef9SStefan Hajnoczi static unsigned int sg_count_fuse_req(struct fuse_req *req)
1040a62a8ef9SStefan Hajnoczi {
1041a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args;
1042a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(args, typeof(*ap), args);
104342d3e2d0SVivek Goyal unsigned int size, total_sgs = 1 /* fuse_in_header */;
1044a62a8ef9SStefan Hajnoczi
1045a62a8ef9SStefan Hajnoczi if (args->in_numargs - args->in_pages)
1046a62a8ef9SStefan Hajnoczi total_sgs += 1;
1047a62a8ef9SStefan Hajnoczi
104842d3e2d0SVivek Goyal if (args->in_pages) {
104942d3e2d0SVivek Goyal size = args->in_args[args->in_numargs - 1].size;
105042d3e2d0SVivek Goyal total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages,
105142d3e2d0SVivek Goyal size);
105242d3e2d0SVivek Goyal }
1053a62a8ef9SStefan Hajnoczi
1054a62a8ef9SStefan Hajnoczi if (!test_bit(FR_ISREPLY, &req->flags))
1055a62a8ef9SStefan Hajnoczi return total_sgs;
1056a62a8ef9SStefan Hajnoczi
1057a62a8ef9SStefan Hajnoczi total_sgs += 1 /* fuse_out_header */;
1058a62a8ef9SStefan Hajnoczi
1059a62a8ef9SStefan Hajnoczi if (args->out_numargs - args->out_pages)
1060a62a8ef9SStefan Hajnoczi total_sgs += 1;
1061a62a8ef9SStefan Hajnoczi
106242d3e2d0SVivek Goyal if (args->out_pages) {
106342d3e2d0SVivek Goyal size = args->out_args[args->out_numargs - 1].size;
106442d3e2d0SVivek Goyal total_sgs += sg_count_fuse_pages(ap->descs, ap->num_pages,
106542d3e2d0SVivek Goyal size);
106642d3e2d0SVivek Goyal }
1067a62a8ef9SStefan Hajnoczi
1068a62a8ef9SStefan Hajnoczi return total_sgs;
1069a62a8ef9SStefan Hajnoczi }
1070a62a8ef9SStefan Hajnoczi
1071a62a8ef9SStefan Hajnoczi /* Add pages to scatter-gather list and return number of elements used */
sg_init_fuse_pages(struct scatterlist * sg,struct page ** pages,struct fuse_page_desc * page_descs,unsigned int num_pages,unsigned int total_len)1072a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_pages(struct scatterlist *sg,
1073a62a8ef9SStefan Hajnoczi struct page **pages,
1074a62a8ef9SStefan Hajnoczi struct fuse_page_desc *page_descs,
1075a62a8ef9SStefan Hajnoczi unsigned int num_pages,
1076a62a8ef9SStefan Hajnoczi unsigned int total_len)
1077a62a8ef9SStefan Hajnoczi {
1078a62a8ef9SStefan Hajnoczi unsigned int i;
1079a62a8ef9SStefan Hajnoczi unsigned int this_len;
1080a62a8ef9SStefan Hajnoczi
1081a62a8ef9SStefan Hajnoczi for (i = 0; i < num_pages && total_len; i++) {
1082a62a8ef9SStefan Hajnoczi sg_init_table(&sg[i], 1);
1083a62a8ef9SStefan Hajnoczi this_len = min(page_descs[i].length, total_len);
1084a62a8ef9SStefan Hajnoczi sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset);
1085a62a8ef9SStefan Hajnoczi total_len -= this_len;
1086a62a8ef9SStefan Hajnoczi }
1087a62a8ef9SStefan Hajnoczi
1088a62a8ef9SStefan Hajnoczi return i;
1089a62a8ef9SStefan Hajnoczi }
1090a62a8ef9SStefan Hajnoczi
1091a62a8ef9SStefan Hajnoczi /* Add args to scatter-gather list and return number of elements used */
sg_init_fuse_args(struct scatterlist * sg,struct fuse_req * req,struct fuse_arg * args,unsigned int numargs,bool argpages,void * argbuf,unsigned int * len_used)1092a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_args(struct scatterlist *sg,
1093a62a8ef9SStefan Hajnoczi struct fuse_req *req,
1094a62a8ef9SStefan Hajnoczi struct fuse_arg *args,
1095a62a8ef9SStefan Hajnoczi unsigned int numargs,
1096a62a8ef9SStefan Hajnoczi bool argpages,
1097a62a8ef9SStefan Hajnoczi void *argbuf,
1098a62a8ef9SStefan Hajnoczi unsigned int *len_used)
1099a62a8ef9SStefan Hajnoczi {
1100a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
1101a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 0;
1102a62a8ef9SStefan Hajnoczi unsigned int len;
1103a62a8ef9SStefan Hajnoczi
1104a62a8ef9SStefan Hajnoczi len = fuse_len_args(numargs - argpages, args);
1105a62a8ef9SStefan Hajnoczi if (len)
1106a62a8ef9SStefan Hajnoczi sg_init_one(&sg[total_sgs++], argbuf, len);
1107a62a8ef9SStefan Hajnoczi
1108a62a8ef9SStefan Hajnoczi if (argpages)
1109a62a8ef9SStefan Hajnoczi total_sgs += sg_init_fuse_pages(&sg[total_sgs],
1110a62a8ef9SStefan Hajnoczi ap->pages, ap->descs,
1111a62a8ef9SStefan Hajnoczi ap->num_pages,
1112a62a8ef9SStefan Hajnoczi args[numargs - 1].size);
1113a62a8ef9SStefan Hajnoczi
1114a62a8ef9SStefan Hajnoczi if (len_used)
1115a62a8ef9SStefan Hajnoczi *len_used = len;
1116a62a8ef9SStefan Hajnoczi
1117a62a8ef9SStefan Hajnoczi return total_sgs;
1118a62a8ef9SStefan Hajnoczi }
1119a62a8ef9SStefan Hajnoczi
1120a62a8ef9SStefan Hajnoczi /* Add a request to a virtqueue and kick the device */
virtio_fs_enqueue_req(struct virtio_fs_vq * fsvq,struct fuse_req * req,bool in_flight)1121a62a8ef9SStefan Hajnoczi static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
1122a9bfd9ddSVivek Goyal struct fuse_req *req, bool in_flight)
1123a62a8ef9SStefan Hajnoczi {
1124a62a8ef9SStefan Hajnoczi /* requests need at least 4 elements */
1125a62a8ef9SStefan Hajnoczi struct scatterlist *stack_sgs[6];
1126a62a8ef9SStefan Hajnoczi struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)];
1127a62a8ef9SStefan Hajnoczi struct scatterlist **sgs = stack_sgs;
1128a62a8ef9SStefan Hajnoczi struct scatterlist *sg = stack_sg;
1129a62a8ef9SStefan Hajnoczi struct virtqueue *vq;
1130a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args;
1131a62a8ef9SStefan Hajnoczi unsigned int argbuf_used = 0;
1132a62a8ef9SStefan Hajnoczi unsigned int out_sgs = 0;
1133a62a8ef9SStefan Hajnoczi unsigned int in_sgs = 0;
1134a62a8ef9SStefan Hajnoczi unsigned int total_sgs;
1135a62a8ef9SStefan Hajnoczi unsigned int i;
1136a62a8ef9SStefan Hajnoczi int ret;
1137a62a8ef9SStefan Hajnoczi bool notify;
11385dbe190fSVivek Goyal struct fuse_pqueue *fpq;
1139a62a8ef9SStefan Hajnoczi
1140a62a8ef9SStefan Hajnoczi /* Does the sglist fit on the stack? */
1141a62a8ef9SStefan Hajnoczi total_sgs = sg_count_fuse_req(req);
1142a62a8ef9SStefan Hajnoczi if (total_sgs > ARRAY_SIZE(stack_sgs)) {
1143a62a8ef9SStefan Hajnoczi sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC);
1144a62a8ef9SStefan Hajnoczi sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC);
1145a62a8ef9SStefan Hajnoczi if (!sgs || !sg) {
1146a62a8ef9SStefan Hajnoczi ret = -ENOMEM;
1147a62a8ef9SStefan Hajnoczi goto out;
1148a62a8ef9SStefan Hajnoczi }
1149a62a8ef9SStefan Hajnoczi }
1150a62a8ef9SStefan Hajnoczi
1151a62a8ef9SStefan Hajnoczi /* Use a bounce buffer since stack args cannot be mapped */
1152a62a8ef9SStefan Hajnoczi ret = copy_args_to_argbuf(req);
1153a62a8ef9SStefan Hajnoczi if (ret < 0)
1154a62a8ef9SStefan Hajnoczi goto out;
1155a62a8ef9SStefan Hajnoczi
1156a62a8ef9SStefan Hajnoczi /* Request elements */
1157a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h));
1158a62a8ef9SStefan Hajnoczi out_sgs += sg_init_fuse_args(&sg[out_sgs], req,
1159a62a8ef9SStefan Hajnoczi (struct fuse_arg *)args->in_args,
1160a62a8ef9SStefan Hajnoczi args->in_numargs, args->in_pages,
1161a62a8ef9SStefan Hajnoczi req->argbuf, &argbuf_used);
1162a62a8ef9SStefan Hajnoczi
1163a62a8ef9SStefan Hajnoczi /* Reply elements */
1164a62a8ef9SStefan Hajnoczi if (test_bit(FR_ISREPLY, &req->flags)) {
1165a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs + in_sgs++],
1166a62a8ef9SStefan Hajnoczi &req->out.h, sizeof(req->out.h));
1167a62a8ef9SStefan Hajnoczi in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req,
1168a62a8ef9SStefan Hajnoczi args->out_args, args->out_numargs,
1169a62a8ef9SStefan Hajnoczi args->out_pages,
1170a62a8ef9SStefan Hajnoczi req->argbuf + argbuf_used, NULL);
1171a62a8ef9SStefan Hajnoczi }
1172a62a8ef9SStefan Hajnoczi
1173a62a8ef9SStefan Hajnoczi WARN_ON(out_sgs + in_sgs != total_sgs);
1174a62a8ef9SStefan Hajnoczi
1175a62a8ef9SStefan Hajnoczi for (i = 0; i < total_sgs; i++)
1176a62a8ef9SStefan Hajnoczi sgs[i] = &sg[i];
1177a62a8ef9SStefan Hajnoczi
1178a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock);
1179a62a8ef9SStefan Hajnoczi
1180a62a8ef9SStefan Hajnoczi if (!fsvq->connected) {
1181a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
1182a62a8ef9SStefan Hajnoczi ret = -ENOTCONN;
1183a62a8ef9SStefan Hajnoczi goto out;
1184a62a8ef9SStefan Hajnoczi }
1185a62a8ef9SStefan Hajnoczi
1186a62a8ef9SStefan Hajnoczi vq = fsvq->vq;
1187a62a8ef9SStefan Hajnoczi ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC);
1188a62a8ef9SStefan Hajnoczi if (ret < 0) {
1189a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
1190a62a8ef9SStefan Hajnoczi goto out;
1191a62a8ef9SStefan Hajnoczi }
1192a62a8ef9SStefan Hajnoczi
11935dbe190fSVivek Goyal /* Request successfully sent. */
11945dbe190fSVivek Goyal fpq = &fsvq->fud->pq;
11955dbe190fSVivek Goyal spin_lock(&fpq->lock);
11965dbe190fSVivek Goyal list_add_tail(&req->list, fpq->processing);
11975dbe190fSVivek Goyal spin_unlock(&fpq->lock);
11985dbe190fSVivek Goyal set_bit(FR_SENT, &req->flags);
11995dbe190fSVivek Goyal /* matches barrier in request_wait_answer() */
12005dbe190fSVivek Goyal smp_mb__after_atomic();
12015dbe190fSVivek Goyal
1202a9bfd9ddSVivek Goyal if (!in_flight)
1203c17ea009SVivek Goyal inc_in_flight_req(fsvq);
1204a62a8ef9SStefan Hajnoczi notify = virtqueue_kick_prepare(vq);
1205a62a8ef9SStefan Hajnoczi
1206a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
1207a62a8ef9SStefan Hajnoczi
1208a62a8ef9SStefan Hajnoczi if (notify)
1209a62a8ef9SStefan Hajnoczi virtqueue_notify(vq);
1210a62a8ef9SStefan Hajnoczi
1211a62a8ef9SStefan Hajnoczi out:
1212a62a8ef9SStefan Hajnoczi if (ret < 0 && req->argbuf) {
1213a62a8ef9SStefan Hajnoczi kfree(req->argbuf);
1214a62a8ef9SStefan Hajnoczi req->argbuf = NULL;
1215a62a8ef9SStefan Hajnoczi }
1216a62a8ef9SStefan Hajnoczi if (sgs != stack_sgs) {
1217a62a8ef9SStefan Hajnoczi kfree(sgs);
1218a62a8ef9SStefan Hajnoczi kfree(sg);
1219a62a8ef9SStefan Hajnoczi }
1220a62a8ef9SStefan Hajnoczi
1221a62a8ef9SStefan Hajnoczi return ret;
1222a62a8ef9SStefan Hajnoczi }
1223a62a8ef9SStefan Hajnoczi
virtio_fs_wake_pending_and_unlock(struct fuse_iqueue * fiq)1224a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq)
1225a62a8ef9SStefan Hajnoczi __releases(fiq->lock)
1226a62a8ef9SStefan Hajnoczi {
1227a62a8ef9SStefan Hajnoczi unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */
1228a62a8ef9SStefan Hajnoczi struct virtio_fs *fs;
1229a62a8ef9SStefan Hajnoczi struct fuse_req *req;
123051fecdd2SVivek Goyal struct virtio_fs_vq *fsvq;
1231a62a8ef9SStefan Hajnoczi int ret;
1232a62a8ef9SStefan Hajnoczi
1233a62a8ef9SStefan Hajnoczi WARN_ON(list_empty(&fiq->pending));
1234a62a8ef9SStefan Hajnoczi req = list_last_entry(&fiq->pending, struct fuse_req, list);
1235a62a8ef9SStefan Hajnoczi clear_bit(FR_PENDING, &req->flags);
1236a62a8ef9SStefan Hajnoczi list_del_init(&req->list);
1237a62a8ef9SStefan Hajnoczi WARN_ON(!list_empty(&fiq->pending));
1238a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock);
1239a62a8ef9SStefan Hajnoczi
1240a62a8ef9SStefan Hajnoczi fs = fiq->priv;
1241a62a8ef9SStefan Hajnoczi
1242a62a8ef9SStefan Hajnoczi pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n",
1243a62a8ef9SStefan Hajnoczi __func__, req->in.h.opcode, req->in.h.unique,
1244a62a8ef9SStefan Hajnoczi req->in.h.nodeid, req->in.h.len,
1245a62a8ef9SStefan Hajnoczi fuse_len_args(req->args->out_numargs, req->args->out_args));
1246a62a8ef9SStefan Hajnoczi
124751fecdd2SVivek Goyal fsvq = &fs->vqs[queue_id];
1248a9bfd9ddSVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req, false);
1249a62a8ef9SStefan Hajnoczi if (ret < 0) {
1250a62a8ef9SStefan Hajnoczi if (ret == -ENOMEM || ret == -ENOSPC) {
1251a9bfd9ddSVivek Goyal /*
1252a9bfd9ddSVivek Goyal * Virtqueue full. Retry submission from worker
1253a9bfd9ddSVivek Goyal * context as we might be holding fc->bg_lock.
1254a9bfd9ddSVivek Goyal */
1255a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock);
1256a9bfd9ddSVivek Goyal list_add_tail(&req->list, &fsvq->queued_reqs);
1257a9bfd9ddSVivek Goyal inc_in_flight_req(fsvq);
1258a9bfd9ddSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work,
1259a9bfd9ddSVivek Goyal msecs_to_jiffies(1));
1260a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock);
1261a9bfd9ddSVivek Goyal return;
1262a62a8ef9SStefan Hajnoczi }
1263a62a8ef9SStefan Hajnoczi req->out.h.error = ret;
1264a62a8ef9SStefan Hajnoczi pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret);
126551fecdd2SVivek Goyal
126651fecdd2SVivek Goyal /* Can't end request in submission context. Use a worker */
126751fecdd2SVivek Goyal spin_lock(&fsvq->lock);
126851fecdd2SVivek Goyal list_add_tail(&req->list, &fsvq->end_reqs);
126951fecdd2SVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 0);
127051fecdd2SVivek Goyal spin_unlock(&fsvq->lock);
1271a62a8ef9SStefan Hajnoczi return;
1272a62a8ef9SStefan Hajnoczi }
1273a62a8ef9SStefan Hajnoczi }
1274a62a8ef9SStefan Hajnoczi
127500929447SYueHaibing static const struct fuse_iqueue_ops virtio_fs_fiq_ops = {
1276a62a8ef9SStefan Hajnoczi .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock,
1277a62a8ef9SStefan Hajnoczi .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock,
1278a62a8ef9SStefan Hajnoczi .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock,
1279a62a8ef9SStefan Hajnoczi .release = virtio_fs_fiq_release,
1280a62a8ef9SStefan Hajnoczi };
1281a62a8ef9SStefan Hajnoczi
virtio_fs_ctx_set_defaults(struct fuse_fs_context * ctx)12821dd53957SVivek Goyal static inline void virtio_fs_ctx_set_defaults(struct fuse_fs_context *ctx)
1283a62a8ef9SStefan Hajnoczi {
12841dd53957SVivek Goyal ctx->rootmode = S_IFDIR;
12851dd53957SVivek Goyal ctx->default_permissions = 1;
12861dd53957SVivek Goyal ctx->allow_other = 1;
12871dd53957SVivek Goyal ctx->max_read = UINT_MAX;
12881dd53957SVivek Goyal ctx->blksize = 512;
12891dd53957SVivek Goyal ctx->destroy = true;
12901dd53957SVivek Goyal ctx->no_control = true;
12911dd53957SVivek Goyal ctx->no_force_umount = true;
12921dd53957SVivek Goyal }
12931dd53957SVivek Goyal
virtio_fs_fill_super(struct super_block * sb,struct fs_context * fsc)12941dd53957SVivek Goyal static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
1295a62a8ef9SStefan Hajnoczi {
1296fcee216bSMax Reitz struct fuse_mount *fm = get_fuse_mount_super(sb);
1297fcee216bSMax Reitz struct fuse_conn *fc = fm->fc;
1298a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = fc->iq.priv;
12991dd53957SVivek Goyal struct fuse_fs_context *ctx = fsc->fs_private;
1300a62a8ef9SStefan Hajnoczi unsigned int i;
1301a62a8ef9SStefan Hajnoczi int err;
1302a62a8ef9SStefan Hajnoczi
13031dd53957SVivek Goyal virtio_fs_ctx_set_defaults(ctx);
1304a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex);
1305a62a8ef9SStefan Hajnoczi
1306a62a8ef9SStefan Hajnoczi /* After holding mutex, make sure virtiofs device is still there.
1307a62a8ef9SStefan Hajnoczi * Though we are holding a reference to it, drive ->remove might
1308a62a8ef9SStefan Hajnoczi * still have cleaned up virtual queues. In that case bail out.
1309a62a8ef9SStefan Hajnoczi */
1310a62a8ef9SStefan Hajnoczi err = -EINVAL;
1311a62a8ef9SStefan Hajnoczi if (list_empty(&fs->list)) {
1312a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fs->tag);
1313a62a8ef9SStefan Hajnoczi goto err;
1314a62a8ef9SStefan Hajnoczi }
1315a62a8ef9SStefan Hajnoczi
1316a62a8ef9SStefan Hajnoczi err = -ENOMEM;
1317a62a8ef9SStefan Hajnoczi /* Allocate fuse_dev for hiprio and notification queues */
13187fd3abfaSVivek Goyal for (i = 0; i < fs->nvqs; i++) {
1319a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i];
1320a62a8ef9SStefan Hajnoczi
1321a62a8ef9SStefan Hajnoczi fsvq->fud = fuse_dev_alloc();
1322a62a8ef9SStefan Hajnoczi if (!fsvq->fud)
1323a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs;
1324a62a8ef9SStefan Hajnoczi }
1325a62a8ef9SStefan Hajnoczi
13267fd3abfaSVivek Goyal /* virtiofs allocates and installs its own fuse devices */
13271dd53957SVivek Goyal ctx->fudptr = NULL;
1328780b1b95SJeffle Xu if (ctx->dax_mode != FUSE_DAX_NEVER) {
1329780b1b95SJeffle Xu if (ctx->dax_mode == FUSE_DAX_ALWAYS && !fs->dax_dev) {
13303f9b9efdSVivek Goyal err = -EINVAL;
13313f9b9efdSVivek Goyal pr_err("virtio-fs: dax can't be enabled as filesystem"
13323f9b9efdSVivek Goyal " device does not support it.\n");
13333f9b9efdSVivek Goyal goto err_free_fuse_devs;
13343f9b9efdSVivek Goyal }
13351dd53957SVivek Goyal ctx->dax_dev = fs->dax_dev;
13363f9b9efdSVivek Goyal }
13371dd53957SVivek Goyal err = fuse_fill_super_common(sb, ctx);
1338a62a8ef9SStefan Hajnoczi if (err < 0)
1339a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs;
1340a62a8ef9SStefan Hajnoczi
1341a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) {
1342a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i];
1343a62a8ef9SStefan Hajnoczi
1344a62a8ef9SStefan Hajnoczi fuse_dev_install(fsvq->fud, fc);
1345a62a8ef9SStefan Hajnoczi }
1346a62a8ef9SStefan Hajnoczi
1347a62a8ef9SStefan Hajnoczi /* Previous unmount will stop all queues. Start these again */
1348a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs);
1349fcee216bSMax Reitz fuse_send_init(fm);
1350a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex);
1351a62a8ef9SStefan Hajnoczi return 0;
1352a62a8ef9SStefan Hajnoczi
1353a62a8ef9SStefan Hajnoczi err_free_fuse_devs:
1354a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(fs);
1355a62a8ef9SStefan Hajnoczi err:
1356a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex);
1357a62a8ef9SStefan Hajnoczi return err;
1358a62a8ef9SStefan Hajnoczi }
1359a62a8ef9SStefan Hajnoczi
virtio_fs_conn_destroy(struct fuse_mount * fm)1360fcee216bSMax Reitz static void virtio_fs_conn_destroy(struct fuse_mount *fm)
1361a62a8ef9SStefan Hajnoczi {
1362fcee216bSMax Reitz struct fuse_conn *fc = fm->fc;
1363fcee216bSMax Reitz struct virtio_fs *vfs = fc->iq.priv;
1364fcee216bSMax Reitz struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO];
1365a62a8ef9SStefan Hajnoczi
1366fcee216bSMax Reitz /* Stop dax worker. Soon evict_inodes() will be called which
1367fcee216bSMax Reitz * will free all memory ranges belonging to all inodes.
13689a752d18SVivek Goyal */
13699a752d18SVivek Goyal if (IS_ENABLED(CONFIG_FUSE_DAX))
13709a752d18SVivek Goyal fuse_dax_cancel_work(fc);
1371a62a8ef9SStefan Hajnoczi
1372a62a8ef9SStefan Hajnoczi /* Stop forget queue. Soon destroy will be sent */
1373a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock);
1374a62a8ef9SStefan Hajnoczi fsvq->connected = false;
1375a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock);
1376a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs);
1377a62a8ef9SStefan Hajnoczi
1378fcee216bSMax Reitz fuse_conn_destroy(fm);
1379a62a8ef9SStefan Hajnoczi
1380fcee216bSMax Reitz /* fuse_conn_destroy() must have sent destroy. Stop all queues
1381a62a8ef9SStefan Hajnoczi * and drain one more time and free fuse devices. Freeing fuse
1382a62a8ef9SStefan Hajnoczi * devices will drop their reference on fuse_conn and that in
1383a62a8ef9SStefan Hajnoczi * turn will drop its reference on virtio_fs object.
1384a62a8ef9SStefan Hajnoczi */
1385a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(vfs);
1386a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs);
1387a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(vfs);
1388a62a8ef9SStefan Hajnoczi }
1389a62a8ef9SStefan Hajnoczi
virtio_kill_sb(struct super_block * sb)1390fcee216bSMax Reitz static void virtio_kill_sb(struct super_block *sb)
1391fcee216bSMax Reitz {
1392fcee216bSMax Reitz struct fuse_mount *fm = get_fuse_mount_super(sb);
1393fcee216bSMax Reitz bool last;
1394fcee216bSMax Reitz
1395fcee216bSMax Reitz /* If mount failed, we can still be called without any fc */
1396d534d31dSMiklos Szeredi if (sb->s_root) {
1397fcee216bSMax Reitz last = fuse_mount_remove(fm);
1398fcee216bSMax Reitz if (last)
1399fcee216bSMax Reitz virtio_fs_conn_destroy(fm);
1400fcee216bSMax Reitz }
1401fcee216bSMax Reitz kill_anon_super(sb);
1402a27c061aSMiklos Szeredi fuse_mount_destroy(fm);
1403fcee216bSMax Reitz }
1404fcee216bSMax Reitz
virtio_fs_test_super(struct super_block * sb,struct fs_context * fsc)1405a62a8ef9SStefan Hajnoczi static int virtio_fs_test_super(struct super_block *sb,
1406a62a8ef9SStefan Hajnoczi struct fs_context *fsc)
1407a62a8ef9SStefan Hajnoczi {
1408fcee216bSMax Reitz struct fuse_mount *fsc_fm = fsc->s_fs_info;
1409fcee216bSMax Reitz struct fuse_mount *sb_fm = get_fuse_mount_super(sb);
1410a62a8ef9SStefan Hajnoczi
1411fcee216bSMax Reitz return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv;
1412a62a8ef9SStefan Hajnoczi }
1413a62a8ef9SStefan Hajnoczi
virtio_fs_get_tree(struct fs_context * fsc)1414a62a8ef9SStefan Hajnoczi static int virtio_fs_get_tree(struct fs_context *fsc)
1415a62a8ef9SStefan Hajnoczi {
1416a62a8ef9SStefan Hajnoczi struct virtio_fs *fs;
1417a62a8ef9SStefan Hajnoczi struct super_block *sb;
1418a7f0d7aaSConnor Kuehl struct fuse_conn *fc = NULL;
1419fcee216bSMax Reitz struct fuse_mount *fm;
1420a7f0d7aaSConnor Kuehl unsigned int virtqueue_size;
1421a7f0d7aaSConnor Kuehl int err = -EIO;
1422a62a8ef9SStefan Hajnoczi
1423a62a8ef9SStefan Hajnoczi /* This gets a reference on virtio_fs object. This ptr gets installed
1424a62a8ef9SStefan Hajnoczi * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
1425a62a8ef9SStefan Hajnoczi * to drop the reference to this object.
1426a62a8ef9SStefan Hajnoczi */
1427a62a8ef9SStefan Hajnoczi fs = virtio_fs_find_instance(fsc->source);
1428a62a8ef9SStefan Hajnoczi if (!fs) {
1429a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fsc->source);
1430a62a8ef9SStefan Hajnoczi return -EINVAL;
1431a62a8ef9SStefan Hajnoczi }
1432a62a8ef9SStefan Hajnoczi
1433a7f0d7aaSConnor Kuehl virtqueue_size = virtqueue_get_vring_size(fs->vqs[VQ_REQUEST].vq);
1434a7f0d7aaSConnor Kuehl if (WARN_ON(virtqueue_size <= FUSE_HEADER_OVERHEAD))
1435a7f0d7aaSConnor Kuehl goto out_err;
1436a7f0d7aaSConnor Kuehl
1437833c5a42SMiklos Szeredi err = -ENOMEM;
1438a62a8ef9SStefan Hajnoczi fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL);
1439833c5a42SMiklos Szeredi if (!fc)
1440833c5a42SMiklos Szeredi goto out_err;
1441a62a8ef9SStefan Hajnoczi
1442fcee216bSMax Reitz fm = kzalloc(sizeof(struct fuse_mount), GFP_KERNEL);
1443833c5a42SMiklos Szeredi if (!fm)
1444833c5a42SMiklos Szeredi goto out_err;
1445fcee216bSMax Reitz
14460a7419c6SMiklos Szeredi fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs);
1447a62a8ef9SStefan Hajnoczi fc->release = fuse_free_conn;
1448a62a8ef9SStefan Hajnoczi fc->delete_stale = true;
1449bf109c64SMax Reitz fc->auto_submounts = true;
14502d82ab25SGreg Kurz fc->sync_fs = true;
1451a62a8ef9SStefan Hajnoczi
1452a7f0d7aaSConnor Kuehl /* Tell FUSE to split requests that exceed the virtqueue's size */
1453a7f0d7aaSConnor Kuehl fc->max_pages_limit = min_t(unsigned int, fc->max_pages_limit,
1454a7f0d7aaSConnor Kuehl virtqueue_size - FUSE_HEADER_OVERHEAD);
1455a7f0d7aaSConnor Kuehl
1456fcee216bSMax Reitz fsc->s_fs_info = fm;
1457b19d3d00SMiklos Szeredi sb = sget_fc(fsc, virtio_fs_test_super, set_anon_super_fc);
1458c191cd07SMiklos Szeredi if (fsc->s_fs_info)
1459c191cd07SMiklos Szeredi fuse_mount_destroy(fm);
1460a62a8ef9SStefan Hajnoczi if (IS_ERR(sb))
1461a62a8ef9SStefan Hajnoczi return PTR_ERR(sb);
1462a62a8ef9SStefan Hajnoczi
1463a62a8ef9SStefan Hajnoczi if (!sb->s_root) {
14641dd53957SVivek Goyal err = virtio_fs_fill_super(sb, fsc);
1465a62a8ef9SStefan Hajnoczi if (err) {
1466a62a8ef9SStefan Hajnoczi deactivate_locked_super(sb);
1467a62a8ef9SStefan Hajnoczi return err;
1468a62a8ef9SStefan Hajnoczi }
1469a62a8ef9SStefan Hajnoczi
1470a62a8ef9SStefan Hajnoczi sb->s_flags |= SB_ACTIVE;
1471a62a8ef9SStefan Hajnoczi }
1472a62a8ef9SStefan Hajnoczi
1473a62a8ef9SStefan Hajnoczi WARN_ON(fsc->root);
1474a62a8ef9SStefan Hajnoczi fsc->root = dget(sb->s_root);
1475a62a8ef9SStefan Hajnoczi return 0;
1476833c5a42SMiklos Szeredi
1477833c5a42SMiklos Szeredi out_err:
1478833c5a42SMiklos Szeredi kfree(fc);
1479833c5a42SMiklos Szeredi mutex_lock(&virtio_fs_mutex);
1480833c5a42SMiklos Szeredi virtio_fs_put(fs);
1481833c5a42SMiklos Szeredi mutex_unlock(&virtio_fs_mutex);
1482833c5a42SMiklos Szeredi return err;
1483a62a8ef9SStefan Hajnoczi }
1484a62a8ef9SStefan Hajnoczi
1485a62a8ef9SStefan Hajnoczi static const struct fs_context_operations virtio_fs_context_ops = {
148684c21507SMiklos Szeredi .free = virtio_fs_free_fsc,
14871dd53957SVivek Goyal .parse_param = virtio_fs_parse_param,
1488a62a8ef9SStefan Hajnoczi .get_tree = virtio_fs_get_tree,
1489a62a8ef9SStefan Hajnoczi };
1490a62a8ef9SStefan Hajnoczi
virtio_fs_init_fs_context(struct fs_context * fsc)1491a62a8ef9SStefan Hajnoczi static int virtio_fs_init_fs_context(struct fs_context *fsc)
1492a62a8ef9SStefan Hajnoczi {
14931dd53957SVivek Goyal struct fuse_fs_context *ctx;
14941dd53957SVivek Goyal
1495fe0a7bd8SGreg Kurz if (fsc->purpose == FS_CONTEXT_FOR_SUBMOUNT)
1496fe0a7bd8SGreg Kurz return fuse_init_fs_context_submount(fsc);
1497fe0a7bd8SGreg Kurz
14981dd53957SVivek Goyal ctx = kzalloc(sizeof(struct fuse_fs_context), GFP_KERNEL);
14991dd53957SVivek Goyal if (!ctx)
15001dd53957SVivek Goyal return -ENOMEM;
15011dd53957SVivek Goyal fsc->fs_private = ctx;
1502a62a8ef9SStefan Hajnoczi fsc->ops = &virtio_fs_context_ops;
1503a62a8ef9SStefan Hajnoczi return 0;
1504a62a8ef9SStefan Hajnoczi }
1505a62a8ef9SStefan Hajnoczi
1506a62a8ef9SStefan Hajnoczi static struct file_system_type virtio_fs_type = {
1507a62a8ef9SStefan Hajnoczi .owner = THIS_MODULE,
1508a62a8ef9SStefan Hajnoczi .name = "virtiofs",
1509a62a8ef9SStefan Hajnoczi .init_fs_context = virtio_fs_init_fs_context,
1510a62a8ef9SStefan Hajnoczi .kill_sb = virtio_kill_sb,
1511a62a8ef9SStefan Hajnoczi };
1512a62a8ef9SStefan Hajnoczi
virtio_fs_init(void)1513a62a8ef9SStefan Hajnoczi static int __init virtio_fs_init(void)
1514a62a8ef9SStefan Hajnoczi {
1515a62a8ef9SStefan Hajnoczi int ret;
1516a62a8ef9SStefan Hajnoczi
1517a62a8ef9SStefan Hajnoczi ret = register_virtio_driver(&virtio_fs_driver);
1518a62a8ef9SStefan Hajnoczi if (ret < 0)
1519a62a8ef9SStefan Hajnoczi return ret;
1520a62a8ef9SStefan Hajnoczi
1521a62a8ef9SStefan Hajnoczi ret = register_filesystem(&virtio_fs_type);
1522a62a8ef9SStefan Hajnoczi if (ret < 0) {
1523a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver);
1524a62a8ef9SStefan Hajnoczi return ret;
1525a62a8ef9SStefan Hajnoczi }
1526a62a8ef9SStefan Hajnoczi
1527a62a8ef9SStefan Hajnoczi return 0;
1528a62a8ef9SStefan Hajnoczi }
1529a62a8ef9SStefan Hajnoczi module_init(virtio_fs_init);
1530a62a8ef9SStefan Hajnoczi
virtio_fs_exit(void)1531a62a8ef9SStefan Hajnoczi static void __exit virtio_fs_exit(void)
1532a62a8ef9SStefan Hajnoczi {
1533a62a8ef9SStefan Hajnoczi unregister_filesystem(&virtio_fs_type);
1534a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver);
1535a62a8ef9SStefan Hajnoczi }
1536a62a8ef9SStefan Hajnoczi module_exit(virtio_fs_exit);
1537a62a8ef9SStefan Hajnoczi
1538a62a8ef9SStefan Hajnoczi MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>");
1539a62a8ef9SStefan Hajnoczi MODULE_DESCRIPTION("Virtio Filesystem");
1540a62a8ef9SStefan Hajnoczi MODULE_LICENSE("GPL");
1541a62a8ef9SStefan Hajnoczi MODULE_ALIAS_FS(KBUILD_MODNAME);
1542a62a8ef9SStefan Hajnoczi MODULE_DEVICE_TABLE(virtio, id_table);
1543