1a62a8ef9SStefan Hajnoczi // SPDX-License-Identifier: GPL-2.0 2a62a8ef9SStefan Hajnoczi /* 3a62a8ef9SStefan Hajnoczi * virtio-fs: Virtio Filesystem 4a62a8ef9SStefan Hajnoczi * Copyright (C) 2018 Red Hat, Inc. 5a62a8ef9SStefan Hajnoczi */ 6a62a8ef9SStefan Hajnoczi 7a62a8ef9SStefan Hajnoczi #include <linux/fs.h> 8a62a8ef9SStefan Hajnoczi #include <linux/module.h> 9a62a8ef9SStefan Hajnoczi #include <linux/virtio.h> 10a62a8ef9SStefan Hajnoczi #include <linux/virtio_fs.h> 11a62a8ef9SStefan Hajnoczi #include <linux/delay.h> 12a62a8ef9SStefan Hajnoczi #include <linux/fs_context.h> 13a62a8ef9SStefan Hajnoczi #include <linux/highmem.h> 14a62a8ef9SStefan Hajnoczi #include "fuse_i.h" 15a62a8ef9SStefan Hajnoczi 16a62a8ef9SStefan Hajnoczi /* List of virtio-fs device instances and a lock for the list. Also provides 17a62a8ef9SStefan Hajnoczi * mutual exclusion in device removal and mounting path 18a62a8ef9SStefan Hajnoczi */ 19a62a8ef9SStefan Hajnoczi static DEFINE_MUTEX(virtio_fs_mutex); 20a62a8ef9SStefan Hajnoczi static LIST_HEAD(virtio_fs_instances); 21a62a8ef9SStefan Hajnoczi 22a62a8ef9SStefan Hajnoczi enum { 23a62a8ef9SStefan Hajnoczi VQ_HIPRIO, 24a62a8ef9SStefan Hajnoczi VQ_REQUEST 25a62a8ef9SStefan Hajnoczi }; 26a62a8ef9SStefan Hajnoczi 27a62a8ef9SStefan Hajnoczi /* Per-virtqueue state */ 28a62a8ef9SStefan Hajnoczi struct virtio_fs_vq { 29a62a8ef9SStefan Hajnoczi spinlock_t lock; 30a62a8ef9SStefan Hajnoczi struct virtqueue *vq; /* protected by ->lock */ 31a62a8ef9SStefan Hajnoczi struct work_struct done_work; 32a62a8ef9SStefan Hajnoczi struct list_head queued_reqs; 3351fecdd2SVivek Goyal struct list_head end_reqs; /* End these requests */ 34a62a8ef9SStefan Hajnoczi struct delayed_work dispatch_work; 35a62a8ef9SStefan Hajnoczi struct fuse_dev *fud; 36a62a8ef9SStefan Hajnoczi bool connected; 37a62a8ef9SStefan Hajnoczi long in_flight; 38a62a8ef9SStefan Hajnoczi char name[24]; 39a62a8ef9SStefan Hajnoczi } ____cacheline_aligned_in_smp; 40a62a8ef9SStefan Hajnoczi 41a62a8ef9SStefan Hajnoczi /* A virtio-fs device instance */ 42a62a8ef9SStefan Hajnoczi struct virtio_fs { 43a62a8ef9SStefan Hajnoczi struct kref refcount; 44a62a8ef9SStefan Hajnoczi struct list_head list; /* on virtio_fs_instances */ 45a62a8ef9SStefan Hajnoczi char *tag; 46a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *vqs; 47a62a8ef9SStefan Hajnoczi unsigned int nvqs; /* number of virtqueues */ 48a62a8ef9SStefan Hajnoczi unsigned int num_request_queues; /* number of request queues */ 49a62a8ef9SStefan Hajnoczi }; 50a62a8ef9SStefan Hajnoczi 51a62a8ef9SStefan Hajnoczi struct virtio_fs_forget { 52a62a8ef9SStefan Hajnoczi struct fuse_in_header ih; 53a62a8ef9SStefan Hajnoczi struct fuse_forget_in arg; 54a62a8ef9SStefan Hajnoczi /* This request can be temporarily queued on virt queue */ 55a62a8ef9SStefan Hajnoczi struct list_head list; 56a62a8ef9SStefan Hajnoczi }; 57a62a8ef9SStefan Hajnoczi 58a62a8ef9SStefan Hajnoczi static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) 59a62a8ef9SStefan Hajnoczi { 60a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vq->vdev->priv; 61a62a8ef9SStefan Hajnoczi 62a62a8ef9SStefan Hajnoczi return &fs->vqs[vq->index]; 63a62a8ef9SStefan Hajnoczi } 64a62a8ef9SStefan Hajnoczi 65a62a8ef9SStefan Hajnoczi static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq) 66a62a8ef9SStefan Hajnoczi { 67a62a8ef9SStefan Hajnoczi return &vq_to_fsvq(vq)->fud->pq; 68a62a8ef9SStefan Hajnoczi } 69a62a8ef9SStefan Hajnoczi 70*c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */ 71*c17ea009SVivek Goyal static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq) 72*c17ea009SVivek Goyal { 73*c17ea009SVivek Goyal fsvq->in_flight++; 74*c17ea009SVivek Goyal } 75*c17ea009SVivek Goyal 76*c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */ 77*c17ea009SVivek Goyal static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) 78*c17ea009SVivek Goyal { 79*c17ea009SVivek Goyal WARN_ON(fsvq->in_flight <= 0); 80*c17ea009SVivek Goyal fsvq->in_flight--; 81*c17ea009SVivek Goyal } 82*c17ea009SVivek Goyal 83a62a8ef9SStefan Hajnoczi static void release_virtio_fs_obj(struct kref *ref) 84a62a8ef9SStefan Hajnoczi { 85a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount); 86a62a8ef9SStefan Hajnoczi 87a62a8ef9SStefan Hajnoczi kfree(vfs->vqs); 88a62a8ef9SStefan Hajnoczi kfree(vfs); 89a62a8ef9SStefan Hajnoczi } 90a62a8ef9SStefan Hajnoczi 91a62a8ef9SStefan Hajnoczi /* Make sure virtiofs_mutex is held */ 92a62a8ef9SStefan Hajnoczi static void virtio_fs_put(struct virtio_fs *fs) 93a62a8ef9SStefan Hajnoczi { 94a62a8ef9SStefan Hajnoczi kref_put(&fs->refcount, release_virtio_fs_obj); 95a62a8ef9SStefan Hajnoczi } 96a62a8ef9SStefan Hajnoczi 97a62a8ef9SStefan Hajnoczi static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) 98a62a8ef9SStefan Hajnoczi { 99a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = fiq->priv; 100a62a8ef9SStefan Hajnoczi 101a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 102a62a8ef9SStefan Hajnoczi virtio_fs_put(vfs); 103a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 104a62a8ef9SStefan Hajnoczi } 105a62a8ef9SStefan Hajnoczi 106a62a8ef9SStefan Hajnoczi static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) 107a62a8ef9SStefan Hajnoczi { 108a62a8ef9SStefan Hajnoczi WARN_ON(fsvq->in_flight < 0); 109a62a8ef9SStefan Hajnoczi 110a62a8ef9SStefan Hajnoczi /* Wait for in flight requests to finish.*/ 111a62a8ef9SStefan Hajnoczi while (1) { 112a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 113a62a8ef9SStefan Hajnoczi if (!fsvq->in_flight) { 114a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 115a62a8ef9SStefan Hajnoczi break; 116a62a8ef9SStefan Hajnoczi } 117a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 118a62a8ef9SStefan Hajnoczi /* TODO use completion instead of timeout */ 119a62a8ef9SStefan Hajnoczi usleep_range(1000, 2000); 120a62a8ef9SStefan Hajnoczi } 121a62a8ef9SStefan Hajnoczi 122a62a8ef9SStefan Hajnoczi flush_work(&fsvq->done_work); 123a62a8ef9SStefan Hajnoczi flush_delayed_work(&fsvq->dispatch_work); 124a62a8ef9SStefan Hajnoczi } 125a62a8ef9SStefan Hajnoczi 126a62a8ef9SStefan Hajnoczi static void virtio_fs_drain_all_queues(struct virtio_fs *fs) 127a62a8ef9SStefan Hajnoczi { 128a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 129a62a8ef9SStefan Hajnoczi int i; 130a62a8ef9SStefan Hajnoczi 131a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 132a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 133a62a8ef9SStefan Hajnoczi virtio_fs_drain_queue(fsvq); 134a62a8ef9SStefan Hajnoczi } 135a62a8ef9SStefan Hajnoczi } 136a62a8ef9SStefan Hajnoczi 137a62a8ef9SStefan Hajnoczi static void virtio_fs_start_all_queues(struct virtio_fs *fs) 138a62a8ef9SStefan Hajnoczi { 139a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 140a62a8ef9SStefan Hajnoczi int i; 141a62a8ef9SStefan Hajnoczi 142a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 143a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 144a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 145a62a8ef9SStefan Hajnoczi fsvq->connected = true; 146a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 147a62a8ef9SStefan Hajnoczi } 148a62a8ef9SStefan Hajnoczi } 149a62a8ef9SStefan Hajnoczi 150a62a8ef9SStefan Hajnoczi /* Add a new instance to the list or return -EEXIST if tag name exists*/ 151a62a8ef9SStefan Hajnoczi static int virtio_fs_add_instance(struct virtio_fs *fs) 152a62a8ef9SStefan Hajnoczi { 153a62a8ef9SStefan Hajnoczi struct virtio_fs *fs2; 154a62a8ef9SStefan Hajnoczi bool duplicate = false; 155a62a8ef9SStefan Hajnoczi 156a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 157a62a8ef9SStefan Hajnoczi 158a62a8ef9SStefan Hajnoczi list_for_each_entry(fs2, &virtio_fs_instances, list) { 159a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, fs2->tag) == 0) 160a62a8ef9SStefan Hajnoczi duplicate = true; 161a62a8ef9SStefan Hajnoczi } 162a62a8ef9SStefan Hajnoczi 163a62a8ef9SStefan Hajnoczi if (!duplicate) 164a62a8ef9SStefan Hajnoczi list_add_tail(&fs->list, &virtio_fs_instances); 165a62a8ef9SStefan Hajnoczi 166a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 167a62a8ef9SStefan Hajnoczi 168a62a8ef9SStefan Hajnoczi if (duplicate) 169a62a8ef9SStefan Hajnoczi return -EEXIST; 170a62a8ef9SStefan Hajnoczi return 0; 171a62a8ef9SStefan Hajnoczi } 172a62a8ef9SStefan Hajnoczi 173a62a8ef9SStefan Hajnoczi /* Return the virtio_fs with a given tag, or NULL */ 174a62a8ef9SStefan Hajnoczi static struct virtio_fs *virtio_fs_find_instance(const char *tag) 175a62a8ef9SStefan Hajnoczi { 176a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 177a62a8ef9SStefan Hajnoczi 178a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 179a62a8ef9SStefan Hajnoczi 180a62a8ef9SStefan Hajnoczi list_for_each_entry(fs, &virtio_fs_instances, list) { 181a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, tag) == 0) { 182a62a8ef9SStefan Hajnoczi kref_get(&fs->refcount); 183a62a8ef9SStefan Hajnoczi goto found; 184a62a8ef9SStefan Hajnoczi } 185a62a8ef9SStefan Hajnoczi } 186a62a8ef9SStefan Hajnoczi 187a62a8ef9SStefan Hajnoczi fs = NULL; /* not found */ 188a62a8ef9SStefan Hajnoczi 189a62a8ef9SStefan Hajnoczi found: 190a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 191a62a8ef9SStefan Hajnoczi 192a62a8ef9SStefan Hajnoczi return fs; 193a62a8ef9SStefan Hajnoczi } 194a62a8ef9SStefan Hajnoczi 195a62a8ef9SStefan Hajnoczi static void virtio_fs_free_devs(struct virtio_fs *fs) 196a62a8ef9SStefan Hajnoczi { 197a62a8ef9SStefan Hajnoczi unsigned int i; 198a62a8ef9SStefan Hajnoczi 199a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 200a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 201a62a8ef9SStefan Hajnoczi 202a62a8ef9SStefan Hajnoczi if (!fsvq->fud) 203a62a8ef9SStefan Hajnoczi continue; 204a62a8ef9SStefan Hajnoczi 205a62a8ef9SStefan Hajnoczi fuse_dev_free(fsvq->fud); 206a62a8ef9SStefan Hajnoczi fsvq->fud = NULL; 207a62a8ef9SStefan Hajnoczi } 208a62a8ef9SStefan Hajnoczi } 209a62a8ef9SStefan Hajnoczi 210a62a8ef9SStefan Hajnoczi /* Read filesystem name from virtio config into fs->tag (must kfree()). */ 211a62a8ef9SStefan Hajnoczi static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs) 212a62a8ef9SStefan Hajnoczi { 213a62a8ef9SStefan Hajnoczi char tag_buf[sizeof_field(struct virtio_fs_config, tag)]; 214a62a8ef9SStefan Hajnoczi char *end; 215a62a8ef9SStefan Hajnoczi size_t len; 216a62a8ef9SStefan Hajnoczi 217a62a8ef9SStefan Hajnoczi virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag), 218a62a8ef9SStefan Hajnoczi &tag_buf, sizeof(tag_buf)); 219a62a8ef9SStefan Hajnoczi end = memchr(tag_buf, '\0', sizeof(tag_buf)); 220a62a8ef9SStefan Hajnoczi if (end == tag_buf) 221a62a8ef9SStefan Hajnoczi return -EINVAL; /* empty tag */ 222a62a8ef9SStefan Hajnoczi if (!end) 223a62a8ef9SStefan Hajnoczi end = &tag_buf[sizeof(tag_buf)]; 224a62a8ef9SStefan Hajnoczi 225a62a8ef9SStefan Hajnoczi len = end - tag_buf; 226a62a8ef9SStefan Hajnoczi fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); 227a62a8ef9SStefan Hajnoczi if (!fs->tag) 228a62a8ef9SStefan Hajnoczi return -ENOMEM; 229a62a8ef9SStefan Hajnoczi memcpy(fs->tag, tag_buf, len); 230a62a8ef9SStefan Hajnoczi fs->tag[len] = '\0'; 231a62a8ef9SStefan Hajnoczi return 0; 232a62a8ef9SStefan Hajnoczi } 233a62a8ef9SStefan Hajnoczi 234a62a8ef9SStefan Hajnoczi /* Work function for hiprio completion */ 235a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_done_work(struct work_struct *work) 236a62a8ef9SStefan Hajnoczi { 237a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 238a62a8ef9SStefan Hajnoczi done_work); 239a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq; 240a62a8ef9SStefan Hajnoczi 241a62a8ef9SStefan Hajnoczi /* Free completed FUSE_FORGET requests */ 242a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 243a62a8ef9SStefan Hajnoczi do { 244a62a8ef9SStefan Hajnoczi unsigned int len; 245a62a8ef9SStefan Hajnoczi void *req; 246a62a8ef9SStefan Hajnoczi 247a62a8ef9SStefan Hajnoczi virtqueue_disable_cb(vq); 248a62a8ef9SStefan Hajnoczi 249a62a8ef9SStefan Hajnoczi while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 250a62a8ef9SStefan Hajnoczi kfree(req); 251*c17ea009SVivek Goyal dec_in_flight_req(fsvq); 252a62a8ef9SStefan Hajnoczi } 253a62a8ef9SStefan Hajnoczi } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 254a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 255a62a8ef9SStefan Hajnoczi } 256a62a8ef9SStefan Hajnoczi 25751fecdd2SVivek Goyal static void virtio_fs_request_dispatch_work(struct work_struct *work) 258a62a8ef9SStefan Hajnoczi { 25951fecdd2SVivek Goyal struct fuse_req *req; 26051fecdd2SVivek Goyal struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 26151fecdd2SVivek Goyal dispatch_work.work); 26251fecdd2SVivek Goyal struct fuse_conn *fc = fsvq->fud->fc; 26351fecdd2SVivek Goyal 26451fecdd2SVivek Goyal pr_debug("virtio-fs: worker %s called.\n", __func__); 26551fecdd2SVivek Goyal while (1) { 26651fecdd2SVivek Goyal spin_lock(&fsvq->lock); 26751fecdd2SVivek Goyal req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, 26851fecdd2SVivek Goyal list); 26951fecdd2SVivek Goyal if (!req) { 27051fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 27151fecdd2SVivek Goyal return; 27251fecdd2SVivek Goyal } 27351fecdd2SVivek Goyal 27451fecdd2SVivek Goyal list_del_init(&req->list); 27551fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 27651fecdd2SVivek Goyal fuse_request_end(fc, req); 27751fecdd2SVivek Goyal } 278a62a8ef9SStefan Hajnoczi } 279a62a8ef9SStefan Hajnoczi 280a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) 281a62a8ef9SStefan Hajnoczi { 282a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget; 283a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 284a62a8ef9SStefan Hajnoczi dispatch_work.work); 285a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq; 286a62a8ef9SStefan Hajnoczi struct scatterlist sg; 287a62a8ef9SStefan Hajnoczi struct scatterlist *sgs[] = {&sg}; 288a62a8ef9SStefan Hajnoczi bool notify; 289a62a8ef9SStefan Hajnoczi int ret; 290a62a8ef9SStefan Hajnoczi 291a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: worker %s called.\n", __func__); 292a62a8ef9SStefan Hajnoczi while (1) { 293a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 294a62a8ef9SStefan Hajnoczi forget = list_first_entry_or_null(&fsvq->queued_reqs, 295a62a8ef9SStefan Hajnoczi struct virtio_fs_forget, list); 296a62a8ef9SStefan Hajnoczi if (!forget) { 297a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 298a62a8ef9SStefan Hajnoczi return; 299a62a8ef9SStefan Hajnoczi } 300a62a8ef9SStefan Hajnoczi 301a62a8ef9SStefan Hajnoczi list_del(&forget->list); 302a62a8ef9SStefan Hajnoczi if (!fsvq->connected) { 303*c17ea009SVivek Goyal dec_in_flight_req(fsvq); 304a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 305a62a8ef9SStefan Hajnoczi kfree(forget); 306a62a8ef9SStefan Hajnoczi continue; 307a62a8ef9SStefan Hajnoczi } 308a62a8ef9SStefan Hajnoczi 309a62a8ef9SStefan Hajnoczi sg_init_one(&sg, forget, sizeof(*forget)); 310a62a8ef9SStefan Hajnoczi 311a62a8ef9SStefan Hajnoczi /* Enqueue the request */ 312a62a8ef9SStefan Hajnoczi dev_dbg(&vq->vdev->dev, "%s\n", __func__); 313a62a8ef9SStefan Hajnoczi ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC); 314a62a8ef9SStefan Hajnoczi if (ret < 0) { 315a62a8ef9SStefan Hajnoczi if (ret == -ENOMEM || ret == -ENOSPC) { 316a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", 317a62a8ef9SStefan Hajnoczi ret); 318a62a8ef9SStefan Hajnoczi list_add_tail(&forget->list, 319a62a8ef9SStefan Hajnoczi &fsvq->queued_reqs); 320a62a8ef9SStefan Hajnoczi schedule_delayed_work(&fsvq->dispatch_work, 321a62a8ef9SStefan Hajnoczi msecs_to_jiffies(1)); 322a62a8ef9SStefan Hajnoczi } else { 323a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", 324a62a8ef9SStefan Hajnoczi ret); 325*c17ea009SVivek Goyal dec_in_flight_req(fsvq); 326a62a8ef9SStefan Hajnoczi kfree(forget); 327a62a8ef9SStefan Hajnoczi } 328a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 329a62a8ef9SStefan Hajnoczi return; 330a62a8ef9SStefan Hajnoczi } 331a62a8ef9SStefan Hajnoczi 332a62a8ef9SStefan Hajnoczi notify = virtqueue_kick_prepare(vq); 333a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 334a62a8ef9SStefan Hajnoczi 335a62a8ef9SStefan Hajnoczi if (notify) 336a62a8ef9SStefan Hajnoczi virtqueue_notify(vq); 337a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: worker %s dispatched one forget request.\n", 338a62a8ef9SStefan Hajnoczi __func__); 339a62a8ef9SStefan Hajnoczi } 340a62a8ef9SStefan Hajnoczi } 341a62a8ef9SStefan Hajnoczi 342a62a8ef9SStefan Hajnoczi /* Allocate and copy args into req->argbuf */ 343a62a8ef9SStefan Hajnoczi static int copy_args_to_argbuf(struct fuse_req *req) 344a62a8ef9SStefan Hajnoczi { 345a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 346a62a8ef9SStefan Hajnoczi unsigned int offset = 0; 347a62a8ef9SStefan Hajnoczi unsigned int num_in; 348a62a8ef9SStefan Hajnoczi unsigned int num_out; 349a62a8ef9SStefan Hajnoczi unsigned int len; 350a62a8ef9SStefan Hajnoczi unsigned int i; 351a62a8ef9SStefan Hajnoczi 352a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages; 353a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages; 354a62a8ef9SStefan Hajnoczi len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + 355a62a8ef9SStefan Hajnoczi fuse_len_args(num_out, args->out_args); 356a62a8ef9SStefan Hajnoczi 357a62a8ef9SStefan Hajnoczi req->argbuf = kmalloc(len, GFP_ATOMIC); 358a62a8ef9SStefan Hajnoczi if (!req->argbuf) 359a62a8ef9SStefan Hajnoczi return -ENOMEM; 360a62a8ef9SStefan Hajnoczi 361a62a8ef9SStefan Hajnoczi for (i = 0; i < num_in; i++) { 362a62a8ef9SStefan Hajnoczi memcpy(req->argbuf + offset, 363a62a8ef9SStefan Hajnoczi args->in_args[i].value, 364a62a8ef9SStefan Hajnoczi args->in_args[i].size); 365a62a8ef9SStefan Hajnoczi offset += args->in_args[i].size; 366a62a8ef9SStefan Hajnoczi } 367a62a8ef9SStefan Hajnoczi 368a62a8ef9SStefan Hajnoczi return 0; 369a62a8ef9SStefan Hajnoczi } 370a62a8ef9SStefan Hajnoczi 371a62a8ef9SStefan Hajnoczi /* Copy args out of and free req->argbuf */ 372a62a8ef9SStefan Hajnoczi static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req) 373a62a8ef9SStefan Hajnoczi { 374a62a8ef9SStefan Hajnoczi unsigned int remaining; 375a62a8ef9SStefan Hajnoczi unsigned int offset; 376a62a8ef9SStefan Hajnoczi unsigned int num_in; 377a62a8ef9SStefan Hajnoczi unsigned int num_out; 378a62a8ef9SStefan Hajnoczi unsigned int i; 379a62a8ef9SStefan Hajnoczi 380a62a8ef9SStefan Hajnoczi remaining = req->out.h.len - sizeof(req->out.h); 381a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages; 382a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages; 383a62a8ef9SStefan Hajnoczi offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); 384a62a8ef9SStefan Hajnoczi 385a62a8ef9SStefan Hajnoczi for (i = 0; i < num_out; i++) { 386a62a8ef9SStefan Hajnoczi unsigned int argsize = args->out_args[i].size; 387a62a8ef9SStefan Hajnoczi 388a62a8ef9SStefan Hajnoczi if (args->out_argvar && 389a62a8ef9SStefan Hajnoczi i == args->out_numargs - 1 && 390a62a8ef9SStefan Hajnoczi argsize > remaining) { 391a62a8ef9SStefan Hajnoczi argsize = remaining; 392a62a8ef9SStefan Hajnoczi } 393a62a8ef9SStefan Hajnoczi 394a62a8ef9SStefan Hajnoczi memcpy(args->out_args[i].value, req->argbuf + offset, argsize); 395a62a8ef9SStefan Hajnoczi offset += argsize; 396a62a8ef9SStefan Hajnoczi 397a62a8ef9SStefan Hajnoczi if (i != args->out_numargs - 1) 398a62a8ef9SStefan Hajnoczi remaining -= argsize; 399a62a8ef9SStefan Hajnoczi } 400a62a8ef9SStefan Hajnoczi 401a62a8ef9SStefan Hajnoczi /* Store the actual size of the variable-length arg */ 402a62a8ef9SStefan Hajnoczi if (args->out_argvar) 403a62a8ef9SStefan Hajnoczi args->out_args[args->out_numargs - 1].size = remaining; 404a62a8ef9SStefan Hajnoczi 405a62a8ef9SStefan Hajnoczi kfree(req->argbuf); 406a62a8ef9SStefan Hajnoczi req->argbuf = NULL; 407a62a8ef9SStefan Hajnoczi } 408a62a8ef9SStefan Hajnoczi 409a62a8ef9SStefan Hajnoczi /* Work function for request completion */ 410a62a8ef9SStefan Hajnoczi static void virtio_fs_requests_done_work(struct work_struct *work) 411a62a8ef9SStefan Hajnoczi { 412a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 413a62a8ef9SStefan Hajnoczi done_work); 414a62a8ef9SStefan Hajnoczi struct fuse_pqueue *fpq = &fsvq->fud->pq; 415a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = fsvq->fud->fc; 416a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq; 417a62a8ef9SStefan Hajnoczi struct fuse_req *req; 418a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap; 419a62a8ef9SStefan Hajnoczi struct fuse_req *next; 420a62a8ef9SStefan Hajnoczi struct fuse_args *args; 421a62a8ef9SStefan Hajnoczi unsigned int len, i, thislen; 422a62a8ef9SStefan Hajnoczi struct page *page; 423a62a8ef9SStefan Hajnoczi LIST_HEAD(reqs); 424a62a8ef9SStefan Hajnoczi 425a62a8ef9SStefan Hajnoczi /* Collect completed requests off the virtqueue */ 426a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 427a62a8ef9SStefan Hajnoczi do { 428a62a8ef9SStefan Hajnoczi virtqueue_disable_cb(vq); 429a62a8ef9SStefan Hajnoczi 430a62a8ef9SStefan Hajnoczi while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 431a62a8ef9SStefan Hajnoczi spin_lock(&fpq->lock); 432a62a8ef9SStefan Hajnoczi list_move_tail(&req->list, &reqs); 433a62a8ef9SStefan Hajnoczi spin_unlock(&fpq->lock); 434a62a8ef9SStefan Hajnoczi } 435a62a8ef9SStefan Hajnoczi } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 436a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 437a62a8ef9SStefan Hajnoczi 438a62a8ef9SStefan Hajnoczi /* End requests */ 439a62a8ef9SStefan Hajnoczi list_for_each_entry_safe(req, next, &reqs, list) { 440a62a8ef9SStefan Hajnoczi /* 441a62a8ef9SStefan Hajnoczi * TODO verify that server properly follows FUSE protocol 442a62a8ef9SStefan Hajnoczi * (oh.uniq, oh.len) 443a62a8ef9SStefan Hajnoczi */ 444a62a8ef9SStefan Hajnoczi args = req->args; 445a62a8ef9SStefan Hajnoczi copy_args_from_argbuf(args, req); 446a62a8ef9SStefan Hajnoczi 447a62a8ef9SStefan Hajnoczi if (args->out_pages && args->page_zeroing) { 448a62a8ef9SStefan Hajnoczi len = args->out_args[args->out_numargs - 1].size; 449a62a8ef9SStefan Hajnoczi ap = container_of(args, typeof(*ap), args); 450a62a8ef9SStefan Hajnoczi for (i = 0; i < ap->num_pages; i++) { 451a62a8ef9SStefan Hajnoczi thislen = ap->descs[i].length; 452a62a8ef9SStefan Hajnoczi if (len < thislen) { 453a62a8ef9SStefan Hajnoczi WARN_ON(ap->descs[i].offset); 454a62a8ef9SStefan Hajnoczi page = ap->pages[i]; 455a62a8ef9SStefan Hajnoczi zero_user_segment(page, len, thislen); 456a62a8ef9SStefan Hajnoczi len = 0; 457a62a8ef9SStefan Hajnoczi } else { 458a62a8ef9SStefan Hajnoczi len -= thislen; 459a62a8ef9SStefan Hajnoczi } 460a62a8ef9SStefan Hajnoczi } 461a62a8ef9SStefan Hajnoczi } 462a62a8ef9SStefan Hajnoczi 463a62a8ef9SStefan Hajnoczi spin_lock(&fpq->lock); 464a62a8ef9SStefan Hajnoczi clear_bit(FR_SENT, &req->flags); 465a62a8ef9SStefan Hajnoczi list_del_init(&req->list); 466a62a8ef9SStefan Hajnoczi spin_unlock(&fpq->lock); 467a62a8ef9SStefan Hajnoczi 468a62a8ef9SStefan Hajnoczi fuse_request_end(fc, req); 469a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 470*c17ea009SVivek Goyal dec_in_flight_req(fsvq); 471a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 472a62a8ef9SStefan Hajnoczi } 473a62a8ef9SStefan Hajnoczi } 474a62a8ef9SStefan Hajnoczi 475a62a8ef9SStefan Hajnoczi /* Virtqueue interrupt handler */ 476a62a8ef9SStefan Hajnoczi static void virtio_fs_vq_done(struct virtqueue *vq) 477a62a8ef9SStefan Hajnoczi { 478a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = vq_to_fsvq(vq); 479a62a8ef9SStefan Hajnoczi 480a62a8ef9SStefan Hajnoczi dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); 481a62a8ef9SStefan Hajnoczi 482a62a8ef9SStefan Hajnoczi schedule_work(&fsvq->done_work); 483a62a8ef9SStefan Hajnoczi } 484a62a8ef9SStefan Hajnoczi 485a62a8ef9SStefan Hajnoczi /* Initialize virtqueues */ 486a62a8ef9SStefan Hajnoczi static int virtio_fs_setup_vqs(struct virtio_device *vdev, 487a62a8ef9SStefan Hajnoczi struct virtio_fs *fs) 488a62a8ef9SStefan Hajnoczi { 489a62a8ef9SStefan Hajnoczi struct virtqueue **vqs; 490a62a8ef9SStefan Hajnoczi vq_callback_t **callbacks; 491a62a8ef9SStefan Hajnoczi const char **names; 492a62a8ef9SStefan Hajnoczi unsigned int i; 493a62a8ef9SStefan Hajnoczi int ret = 0; 494a62a8ef9SStefan Hajnoczi 495a62a8ef9SStefan Hajnoczi virtio_cread(vdev, struct virtio_fs_config, num_request_queues, 496a62a8ef9SStefan Hajnoczi &fs->num_request_queues); 497a62a8ef9SStefan Hajnoczi if (fs->num_request_queues == 0) 498a62a8ef9SStefan Hajnoczi return -EINVAL; 499a62a8ef9SStefan Hajnoczi 500a62a8ef9SStefan Hajnoczi fs->nvqs = 1 + fs->num_request_queues; 501a62a8ef9SStefan Hajnoczi fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); 502a62a8ef9SStefan Hajnoczi if (!fs->vqs) 503a62a8ef9SStefan Hajnoczi return -ENOMEM; 504a62a8ef9SStefan Hajnoczi 505a62a8ef9SStefan Hajnoczi vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); 506a62a8ef9SStefan Hajnoczi callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]), 507a62a8ef9SStefan Hajnoczi GFP_KERNEL); 508a62a8ef9SStefan Hajnoczi names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL); 509a62a8ef9SStefan Hajnoczi if (!vqs || !callbacks || !names) { 510a62a8ef9SStefan Hajnoczi ret = -ENOMEM; 511a62a8ef9SStefan Hajnoczi goto out; 512a62a8ef9SStefan Hajnoczi } 513a62a8ef9SStefan Hajnoczi 514a62a8ef9SStefan Hajnoczi callbacks[VQ_HIPRIO] = virtio_fs_vq_done; 515a62a8ef9SStefan Hajnoczi snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name), 516a62a8ef9SStefan Hajnoczi "hiprio"); 517a62a8ef9SStefan Hajnoczi names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; 518a62a8ef9SStefan Hajnoczi INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work); 519a62a8ef9SStefan Hajnoczi INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs); 52051fecdd2SVivek Goyal INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs); 521a62a8ef9SStefan Hajnoczi INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work, 522a62a8ef9SStefan Hajnoczi virtio_fs_hiprio_dispatch_work); 523a62a8ef9SStefan Hajnoczi spin_lock_init(&fs->vqs[VQ_HIPRIO].lock); 524a62a8ef9SStefan Hajnoczi 525a62a8ef9SStefan Hajnoczi /* Initialize the requests virtqueues */ 526a62a8ef9SStefan Hajnoczi for (i = VQ_REQUEST; i < fs->nvqs; i++) { 527a62a8ef9SStefan Hajnoczi spin_lock_init(&fs->vqs[i].lock); 528a62a8ef9SStefan Hajnoczi INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work); 529a62a8ef9SStefan Hajnoczi INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work, 53051fecdd2SVivek Goyal virtio_fs_request_dispatch_work); 531a62a8ef9SStefan Hajnoczi INIT_LIST_HEAD(&fs->vqs[i].queued_reqs); 53251fecdd2SVivek Goyal INIT_LIST_HEAD(&fs->vqs[i].end_reqs); 533a62a8ef9SStefan Hajnoczi snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name), 534a62a8ef9SStefan Hajnoczi "requests.%u", i - VQ_REQUEST); 535a62a8ef9SStefan Hajnoczi callbacks[i] = virtio_fs_vq_done; 536a62a8ef9SStefan Hajnoczi names[i] = fs->vqs[i].name; 537a62a8ef9SStefan Hajnoczi } 538a62a8ef9SStefan Hajnoczi 539a62a8ef9SStefan Hajnoczi ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL); 540a62a8ef9SStefan Hajnoczi if (ret < 0) 541a62a8ef9SStefan Hajnoczi goto out; 542a62a8ef9SStefan Hajnoczi 543a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) 544a62a8ef9SStefan Hajnoczi fs->vqs[i].vq = vqs[i]; 545a62a8ef9SStefan Hajnoczi 546a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs); 547a62a8ef9SStefan Hajnoczi out: 548a62a8ef9SStefan Hajnoczi kfree(names); 549a62a8ef9SStefan Hajnoczi kfree(callbacks); 550a62a8ef9SStefan Hajnoczi kfree(vqs); 551a62a8ef9SStefan Hajnoczi if (ret) 552a62a8ef9SStefan Hajnoczi kfree(fs->vqs); 553a62a8ef9SStefan Hajnoczi return ret; 554a62a8ef9SStefan Hajnoczi } 555a62a8ef9SStefan Hajnoczi 556a62a8ef9SStefan Hajnoczi /* Free virtqueues (device must already be reset) */ 557a62a8ef9SStefan Hajnoczi static void virtio_fs_cleanup_vqs(struct virtio_device *vdev, 558a62a8ef9SStefan Hajnoczi struct virtio_fs *fs) 559a62a8ef9SStefan Hajnoczi { 560a62a8ef9SStefan Hajnoczi vdev->config->del_vqs(vdev); 561a62a8ef9SStefan Hajnoczi } 562a62a8ef9SStefan Hajnoczi 563a62a8ef9SStefan Hajnoczi static int virtio_fs_probe(struct virtio_device *vdev) 564a62a8ef9SStefan Hajnoczi { 565a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 566a62a8ef9SStefan Hajnoczi int ret; 567a62a8ef9SStefan Hajnoczi 568a62a8ef9SStefan Hajnoczi fs = kzalloc(sizeof(*fs), GFP_KERNEL); 569a62a8ef9SStefan Hajnoczi if (!fs) 570a62a8ef9SStefan Hajnoczi return -ENOMEM; 571a62a8ef9SStefan Hajnoczi kref_init(&fs->refcount); 572a62a8ef9SStefan Hajnoczi vdev->priv = fs; 573a62a8ef9SStefan Hajnoczi 574a62a8ef9SStefan Hajnoczi ret = virtio_fs_read_tag(vdev, fs); 575a62a8ef9SStefan Hajnoczi if (ret < 0) 576a62a8ef9SStefan Hajnoczi goto out; 577a62a8ef9SStefan Hajnoczi 578a62a8ef9SStefan Hajnoczi ret = virtio_fs_setup_vqs(vdev, fs); 579a62a8ef9SStefan Hajnoczi if (ret < 0) 580a62a8ef9SStefan Hajnoczi goto out; 581a62a8ef9SStefan Hajnoczi 582a62a8ef9SStefan Hajnoczi /* TODO vq affinity */ 583a62a8ef9SStefan Hajnoczi 584a62a8ef9SStefan Hajnoczi /* Bring the device online in case the filesystem is mounted and 585a62a8ef9SStefan Hajnoczi * requests need to be sent before we return. 586a62a8ef9SStefan Hajnoczi */ 587a62a8ef9SStefan Hajnoczi virtio_device_ready(vdev); 588a62a8ef9SStefan Hajnoczi 589a62a8ef9SStefan Hajnoczi ret = virtio_fs_add_instance(fs); 590a62a8ef9SStefan Hajnoczi if (ret < 0) 591a62a8ef9SStefan Hajnoczi goto out_vqs; 592a62a8ef9SStefan Hajnoczi 593a62a8ef9SStefan Hajnoczi return 0; 594a62a8ef9SStefan Hajnoczi 595a62a8ef9SStefan Hajnoczi out_vqs: 596a62a8ef9SStefan Hajnoczi vdev->config->reset(vdev); 597a62a8ef9SStefan Hajnoczi virtio_fs_cleanup_vqs(vdev, fs); 598a62a8ef9SStefan Hajnoczi 599a62a8ef9SStefan Hajnoczi out: 600a62a8ef9SStefan Hajnoczi vdev->priv = NULL; 601a62a8ef9SStefan Hajnoczi kfree(fs); 602a62a8ef9SStefan Hajnoczi return ret; 603a62a8ef9SStefan Hajnoczi } 604a62a8ef9SStefan Hajnoczi 605a62a8ef9SStefan Hajnoczi static void virtio_fs_stop_all_queues(struct virtio_fs *fs) 606a62a8ef9SStefan Hajnoczi { 607a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 608a62a8ef9SStefan Hajnoczi int i; 609a62a8ef9SStefan Hajnoczi 610a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 611a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 612a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 613a62a8ef9SStefan Hajnoczi fsvq->connected = false; 614a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 615a62a8ef9SStefan Hajnoczi } 616a62a8ef9SStefan Hajnoczi } 617a62a8ef9SStefan Hajnoczi 618a62a8ef9SStefan Hajnoczi static void virtio_fs_remove(struct virtio_device *vdev) 619a62a8ef9SStefan Hajnoczi { 620a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vdev->priv; 621a62a8ef9SStefan Hajnoczi 622a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 623a62a8ef9SStefan Hajnoczi /* This device is going away. No one should get new reference */ 624a62a8ef9SStefan Hajnoczi list_del_init(&fs->list); 625a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(fs); 626a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(fs); 627a62a8ef9SStefan Hajnoczi vdev->config->reset(vdev); 628a62a8ef9SStefan Hajnoczi virtio_fs_cleanup_vqs(vdev, fs); 629a62a8ef9SStefan Hajnoczi 630a62a8ef9SStefan Hajnoczi vdev->priv = NULL; 631a62a8ef9SStefan Hajnoczi /* Put device reference on virtio_fs object */ 632a62a8ef9SStefan Hajnoczi virtio_fs_put(fs); 633a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 634a62a8ef9SStefan Hajnoczi } 635a62a8ef9SStefan Hajnoczi 636a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP 637a62a8ef9SStefan Hajnoczi static int virtio_fs_freeze(struct virtio_device *vdev) 638a62a8ef9SStefan Hajnoczi { 639a62a8ef9SStefan Hajnoczi /* TODO need to save state here */ 640a62a8ef9SStefan Hajnoczi pr_warn("virtio-fs: suspend/resume not yet supported\n"); 641a62a8ef9SStefan Hajnoczi return -EOPNOTSUPP; 642a62a8ef9SStefan Hajnoczi } 643a62a8ef9SStefan Hajnoczi 644a62a8ef9SStefan Hajnoczi static int virtio_fs_restore(struct virtio_device *vdev) 645a62a8ef9SStefan Hajnoczi { 646a62a8ef9SStefan Hajnoczi /* TODO need to restore state here */ 647a62a8ef9SStefan Hajnoczi return 0; 648a62a8ef9SStefan Hajnoczi } 649a62a8ef9SStefan Hajnoczi #endif /* CONFIG_PM_SLEEP */ 650a62a8ef9SStefan Hajnoczi 651a62a8ef9SStefan Hajnoczi const static struct virtio_device_id id_table[] = { 652a62a8ef9SStefan Hajnoczi { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, 653a62a8ef9SStefan Hajnoczi {}, 654a62a8ef9SStefan Hajnoczi }; 655a62a8ef9SStefan Hajnoczi 656a62a8ef9SStefan Hajnoczi const static unsigned int feature_table[] = {}; 657a62a8ef9SStefan Hajnoczi 658a62a8ef9SStefan Hajnoczi static struct virtio_driver virtio_fs_driver = { 659a62a8ef9SStefan Hajnoczi .driver.name = KBUILD_MODNAME, 660a62a8ef9SStefan Hajnoczi .driver.owner = THIS_MODULE, 661a62a8ef9SStefan Hajnoczi .id_table = id_table, 662a62a8ef9SStefan Hajnoczi .feature_table = feature_table, 663a62a8ef9SStefan Hajnoczi .feature_table_size = ARRAY_SIZE(feature_table), 664a62a8ef9SStefan Hajnoczi .probe = virtio_fs_probe, 665a62a8ef9SStefan Hajnoczi .remove = virtio_fs_remove, 666a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP 667a62a8ef9SStefan Hajnoczi .freeze = virtio_fs_freeze, 668a62a8ef9SStefan Hajnoczi .restore = virtio_fs_restore, 669a62a8ef9SStefan Hajnoczi #endif 670a62a8ef9SStefan Hajnoczi }; 671a62a8ef9SStefan Hajnoczi 672a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq) 673a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 674a62a8ef9SStefan Hajnoczi { 675a62a8ef9SStefan Hajnoczi struct fuse_forget_link *link; 676a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget; 677a62a8ef9SStefan Hajnoczi struct scatterlist sg; 678a62a8ef9SStefan Hajnoczi struct scatterlist *sgs[] = {&sg}; 679a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 680a62a8ef9SStefan Hajnoczi struct virtqueue *vq; 681a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 682a62a8ef9SStefan Hajnoczi bool notify; 683a62a8ef9SStefan Hajnoczi u64 unique; 684a62a8ef9SStefan Hajnoczi int ret; 685a62a8ef9SStefan Hajnoczi 686a62a8ef9SStefan Hajnoczi link = fuse_dequeue_forget(fiq, 1, NULL); 687a62a8ef9SStefan Hajnoczi unique = fuse_get_unique(fiq); 688a62a8ef9SStefan Hajnoczi 689a62a8ef9SStefan Hajnoczi fs = fiq->priv; 690a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[VQ_HIPRIO]; 691a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 692a62a8ef9SStefan Hajnoczi 693a62a8ef9SStefan Hajnoczi /* Allocate a buffer for the request */ 694a62a8ef9SStefan Hajnoczi forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL); 695a62a8ef9SStefan Hajnoczi 696a62a8ef9SStefan Hajnoczi forget->ih = (struct fuse_in_header){ 697a62a8ef9SStefan Hajnoczi .opcode = FUSE_FORGET, 698a62a8ef9SStefan Hajnoczi .nodeid = link->forget_one.nodeid, 699a62a8ef9SStefan Hajnoczi .unique = unique, 700a62a8ef9SStefan Hajnoczi .len = sizeof(*forget), 701a62a8ef9SStefan Hajnoczi }; 702a62a8ef9SStefan Hajnoczi forget->arg = (struct fuse_forget_in){ 703a62a8ef9SStefan Hajnoczi .nlookup = link->forget_one.nlookup, 704a62a8ef9SStefan Hajnoczi }; 705a62a8ef9SStefan Hajnoczi 706a62a8ef9SStefan Hajnoczi sg_init_one(&sg, forget, sizeof(*forget)); 707a62a8ef9SStefan Hajnoczi 708a62a8ef9SStefan Hajnoczi /* Enqueue the request */ 709a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 710a62a8ef9SStefan Hajnoczi 711a62a8ef9SStefan Hajnoczi if (!fsvq->connected) { 712a62a8ef9SStefan Hajnoczi kfree(forget); 713a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 714a62a8ef9SStefan Hajnoczi goto out; 715a62a8ef9SStefan Hajnoczi } 716a62a8ef9SStefan Hajnoczi 717a62a8ef9SStefan Hajnoczi vq = fsvq->vq; 718a62a8ef9SStefan Hajnoczi dev_dbg(&vq->vdev->dev, "%s\n", __func__); 719a62a8ef9SStefan Hajnoczi 720a62a8ef9SStefan Hajnoczi ret = virtqueue_add_sgs(vq, sgs, 1, 0, forget, GFP_ATOMIC); 721a62a8ef9SStefan Hajnoczi if (ret < 0) { 722a62a8ef9SStefan Hajnoczi if (ret == -ENOMEM || ret == -ENOSPC) { 723a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later.\n", 724a62a8ef9SStefan Hajnoczi ret); 725a62a8ef9SStefan Hajnoczi list_add_tail(&forget->list, &fsvq->queued_reqs); 726a62a8ef9SStefan Hajnoczi schedule_delayed_work(&fsvq->dispatch_work, 727a62a8ef9SStefan Hajnoczi msecs_to_jiffies(1)); 728*c17ea009SVivek Goyal inc_in_flight_req(fsvq); 729a62a8ef9SStefan Hajnoczi } else { 730a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", 731a62a8ef9SStefan Hajnoczi ret); 732a62a8ef9SStefan Hajnoczi kfree(forget); 733a62a8ef9SStefan Hajnoczi } 734a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 735a62a8ef9SStefan Hajnoczi goto out; 736a62a8ef9SStefan Hajnoczi } 737a62a8ef9SStefan Hajnoczi 738*c17ea009SVivek Goyal inc_in_flight_req(fsvq); 739a62a8ef9SStefan Hajnoczi notify = virtqueue_kick_prepare(vq); 740a62a8ef9SStefan Hajnoczi 741a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 742a62a8ef9SStefan Hajnoczi 743a62a8ef9SStefan Hajnoczi if (notify) 744a62a8ef9SStefan Hajnoczi virtqueue_notify(vq); 745a62a8ef9SStefan Hajnoczi out: 746a62a8ef9SStefan Hajnoczi kfree(link); 747a62a8ef9SStefan Hajnoczi } 748a62a8ef9SStefan Hajnoczi 749a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq) 750a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 751a62a8ef9SStefan Hajnoczi { 752a62a8ef9SStefan Hajnoczi /* 753a62a8ef9SStefan Hajnoczi * TODO interrupts. 754a62a8ef9SStefan Hajnoczi * 755a62a8ef9SStefan Hajnoczi * Normal fs operations on a local filesystems aren't interruptible. 756a62a8ef9SStefan Hajnoczi * Exceptions are blocking lock operations; for example fcntl(F_SETLKW) 757a62a8ef9SStefan Hajnoczi * with shared lock between host and guest. 758a62a8ef9SStefan Hajnoczi */ 759a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 760a62a8ef9SStefan Hajnoczi } 761a62a8ef9SStefan Hajnoczi 762a62a8ef9SStefan Hajnoczi /* Return the number of scatter-gather list elements required */ 763a62a8ef9SStefan Hajnoczi static unsigned int sg_count_fuse_req(struct fuse_req *req) 764a62a8ef9SStefan Hajnoczi { 765a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 766a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); 767a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 1 /* fuse_in_header */; 768a62a8ef9SStefan Hajnoczi 769a62a8ef9SStefan Hajnoczi if (args->in_numargs - args->in_pages) 770a62a8ef9SStefan Hajnoczi total_sgs += 1; 771a62a8ef9SStefan Hajnoczi 772a62a8ef9SStefan Hajnoczi if (args->in_pages) 773a62a8ef9SStefan Hajnoczi total_sgs += ap->num_pages; 774a62a8ef9SStefan Hajnoczi 775a62a8ef9SStefan Hajnoczi if (!test_bit(FR_ISREPLY, &req->flags)) 776a62a8ef9SStefan Hajnoczi return total_sgs; 777a62a8ef9SStefan Hajnoczi 778a62a8ef9SStefan Hajnoczi total_sgs += 1 /* fuse_out_header */; 779a62a8ef9SStefan Hajnoczi 780a62a8ef9SStefan Hajnoczi if (args->out_numargs - args->out_pages) 781a62a8ef9SStefan Hajnoczi total_sgs += 1; 782a62a8ef9SStefan Hajnoczi 783a62a8ef9SStefan Hajnoczi if (args->out_pages) 784a62a8ef9SStefan Hajnoczi total_sgs += ap->num_pages; 785a62a8ef9SStefan Hajnoczi 786a62a8ef9SStefan Hajnoczi return total_sgs; 787a62a8ef9SStefan Hajnoczi } 788a62a8ef9SStefan Hajnoczi 789a62a8ef9SStefan Hajnoczi /* Add pages to scatter-gather list and return number of elements used */ 790a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_pages(struct scatterlist *sg, 791a62a8ef9SStefan Hajnoczi struct page **pages, 792a62a8ef9SStefan Hajnoczi struct fuse_page_desc *page_descs, 793a62a8ef9SStefan Hajnoczi unsigned int num_pages, 794a62a8ef9SStefan Hajnoczi unsigned int total_len) 795a62a8ef9SStefan Hajnoczi { 796a62a8ef9SStefan Hajnoczi unsigned int i; 797a62a8ef9SStefan Hajnoczi unsigned int this_len; 798a62a8ef9SStefan Hajnoczi 799a62a8ef9SStefan Hajnoczi for (i = 0; i < num_pages && total_len; i++) { 800a62a8ef9SStefan Hajnoczi sg_init_table(&sg[i], 1); 801a62a8ef9SStefan Hajnoczi this_len = min(page_descs[i].length, total_len); 802a62a8ef9SStefan Hajnoczi sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); 803a62a8ef9SStefan Hajnoczi total_len -= this_len; 804a62a8ef9SStefan Hajnoczi } 805a62a8ef9SStefan Hajnoczi 806a62a8ef9SStefan Hajnoczi return i; 807a62a8ef9SStefan Hajnoczi } 808a62a8ef9SStefan Hajnoczi 809a62a8ef9SStefan Hajnoczi /* Add args to scatter-gather list and return number of elements used */ 810a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_args(struct scatterlist *sg, 811a62a8ef9SStefan Hajnoczi struct fuse_req *req, 812a62a8ef9SStefan Hajnoczi struct fuse_arg *args, 813a62a8ef9SStefan Hajnoczi unsigned int numargs, 814a62a8ef9SStefan Hajnoczi bool argpages, 815a62a8ef9SStefan Hajnoczi void *argbuf, 816a62a8ef9SStefan Hajnoczi unsigned int *len_used) 817a62a8ef9SStefan Hajnoczi { 818a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); 819a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 0; 820a62a8ef9SStefan Hajnoczi unsigned int len; 821a62a8ef9SStefan Hajnoczi 822a62a8ef9SStefan Hajnoczi len = fuse_len_args(numargs - argpages, args); 823a62a8ef9SStefan Hajnoczi if (len) 824a62a8ef9SStefan Hajnoczi sg_init_one(&sg[total_sgs++], argbuf, len); 825a62a8ef9SStefan Hajnoczi 826a62a8ef9SStefan Hajnoczi if (argpages) 827a62a8ef9SStefan Hajnoczi total_sgs += sg_init_fuse_pages(&sg[total_sgs], 828a62a8ef9SStefan Hajnoczi ap->pages, ap->descs, 829a62a8ef9SStefan Hajnoczi ap->num_pages, 830a62a8ef9SStefan Hajnoczi args[numargs - 1].size); 831a62a8ef9SStefan Hajnoczi 832a62a8ef9SStefan Hajnoczi if (len_used) 833a62a8ef9SStefan Hajnoczi *len_used = len; 834a62a8ef9SStefan Hajnoczi 835a62a8ef9SStefan Hajnoczi return total_sgs; 836a62a8ef9SStefan Hajnoczi } 837a62a8ef9SStefan Hajnoczi 838a62a8ef9SStefan Hajnoczi /* Add a request to a virtqueue and kick the device */ 839a62a8ef9SStefan Hajnoczi static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 840a62a8ef9SStefan Hajnoczi struct fuse_req *req) 841a62a8ef9SStefan Hajnoczi { 842a62a8ef9SStefan Hajnoczi /* requests need at least 4 elements */ 843a62a8ef9SStefan Hajnoczi struct scatterlist *stack_sgs[6]; 844a62a8ef9SStefan Hajnoczi struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)]; 845a62a8ef9SStefan Hajnoczi struct scatterlist **sgs = stack_sgs; 846a62a8ef9SStefan Hajnoczi struct scatterlist *sg = stack_sg; 847a62a8ef9SStefan Hajnoczi struct virtqueue *vq; 848a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 849a62a8ef9SStefan Hajnoczi unsigned int argbuf_used = 0; 850a62a8ef9SStefan Hajnoczi unsigned int out_sgs = 0; 851a62a8ef9SStefan Hajnoczi unsigned int in_sgs = 0; 852a62a8ef9SStefan Hajnoczi unsigned int total_sgs; 853a62a8ef9SStefan Hajnoczi unsigned int i; 854a62a8ef9SStefan Hajnoczi int ret; 855a62a8ef9SStefan Hajnoczi bool notify; 8565dbe190fSVivek Goyal struct fuse_pqueue *fpq; 857a62a8ef9SStefan Hajnoczi 858a62a8ef9SStefan Hajnoczi /* Does the sglist fit on the stack? */ 859a62a8ef9SStefan Hajnoczi total_sgs = sg_count_fuse_req(req); 860a62a8ef9SStefan Hajnoczi if (total_sgs > ARRAY_SIZE(stack_sgs)) { 861a62a8ef9SStefan Hajnoczi sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); 862a62a8ef9SStefan Hajnoczi sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); 863a62a8ef9SStefan Hajnoczi if (!sgs || !sg) { 864a62a8ef9SStefan Hajnoczi ret = -ENOMEM; 865a62a8ef9SStefan Hajnoczi goto out; 866a62a8ef9SStefan Hajnoczi } 867a62a8ef9SStefan Hajnoczi } 868a62a8ef9SStefan Hajnoczi 869a62a8ef9SStefan Hajnoczi /* Use a bounce buffer since stack args cannot be mapped */ 870a62a8ef9SStefan Hajnoczi ret = copy_args_to_argbuf(req); 871a62a8ef9SStefan Hajnoczi if (ret < 0) 872a62a8ef9SStefan Hajnoczi goto out; 873a62a8ef9SStefan Hajnoczi 874a62a8ef9SStefan Hajnoczi /* Request elements */ 875a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); 876a62a8ef9SStefan Hajnoczi out_sgs += sg_init_fuse_args(&sg[out_sgs], req, 877a62a8ef9SStefan Hajnoczi (struct fuse_arg *)args->in_args, 878a62a8ef9SStefan Hajnoczi args->in_numargs, args->in_pages, 879a62a8ef9SStefan Hajnoczi req->argbuf, &argbuf_used); 880a62a8ef9SStefan Hajnoczi 881a62a8ef9SStefan Hajnoczi /* Reply elements */ 882a62a8ef9SStefan Hajnoczi if (test_bit(FR_ISREPLY, &req->flags)) { 883a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs + in_sgs++], 884a62a8ef9SStefan Hajnoczi &req->out.h, sizeof(req->out.h)); 885a62a8ef9SStefan Hajnoczi in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req, 886a62a8ef9SStefan Hajnoczi args->out_args, args->out_numargs, 887a62a8ef9SStefan Hajnoczi args->out_pages, 888a62a8ef9SStefan Hajnoczi req->argbuf + argbuf_used, NULL); 889a62a8ef9SStefan Hajnoczi } 890a62a8ef9SStefan Hajnoczi 891a62a8ef9SStefan Hajnoczi WARN_ON(out_sgs + in_sgs != total_sgs); 892a62a8ef9SStefan Hajnoczi 893a62a8ef9SStefan Hajnoczi for (i = 0; i < total_sgs; i++) 894a62a8ef9SStefan Hajnoczi sgs[i] = &sg[i]; 895a62a8ef9SStefan Hajnoczi 896a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 897a62a8ef9SStefan Hajnoczi 898a62a8ef9SStefan Hajnoczi if (!fsvq->connected) { 899a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 900a62a8ef9SStefan Hajnoczi ret = -ENOTCONN; 901a62a8ef9SStefan Hajnoczi goto out; 902a62a8ef9SStefan Hajnoczi } 903a62a8ef9SStefan Hajnoczi 904a62a8ef9SStefan Hajnoczi vq = fsvq->vq; 905a62a8ef9SStefan Hajnoczi ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC); 906a62a8ef9SStefan Hajnoczi if (ret < 0) { 907a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 908a62a8ef9SStefan Hajnoczi goto out; 909a62a8ef9SStefan Hajnoczi } 910a62a8ef9SStefan Hajnoczi 9115dbe190fSVivek Goyal /* Request successfully sent. */ 9125dbe190fSVivek Goyal fpq = &fsvq->fud->pq; 9135dbe190fSVivek Goyal spin_lock(&fpq->lock); 9145dbe190fSVivek Goyal list_add_tail(&req->list, fpq->processing); 9155dbe190fSVivek Goyal spin_unlock(&fpq->lock); 9165dbe190fSVivek Goyal set_bit(FR_SENT, &req->flags); 9175dbe190fSVivek Goyal /* matches barrier in request_wait_answer() */ 9185dbe190fSVivek Goyal smp_mb__after_atomic(); 9195dbe190fSVivek Goyal 920*c17ea009SVivek Goyal inc_in_flight_req(fsvq); 921a62a8ef9SStefan Hajnoczi notify = virtqueue_kick_prepare(vq); 922a62a8ef9SStefan Hajnoczi 923a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 924a62a8ef9SStefan Hajnoczi 925a62a8ef9SStefan Hajnoczi if (notify) 926a62a8ef9SStefan Hajnoczi virtqueue_notify(vq); 927a62a8ef9SStefan Hajnoczi 928a62a8ef9SStefan Hajnoczi out: 929a62a8ef9SStefan Hajnoczi if (ret < 0 && req->argbuf) { 930a62a8ef9SStefan Hajnoczi kfree(req->argbuf); 931a62a8ef9SStefan Hajnoczi req->argbuf = NULL; 932a62a8ef9SStefan Hajnoczi } 933a62a8ef9SStefan Hajnoczi if (sgs != stack_sgs) { 934a62a8ef9SStefan Hajnoczi kfree(sgs); 935a62a8ef9SStefan Hajnoczi kfree(sg); 936a62a8ef9SStefan Hajnoczi } 937a62a8ef9SStefan Hajnoczi 938a62a8ef9SStefan Hajnoczi return ret; 939a62a8ef9SStefan Hajnoczi } 940a62a8ef9SStefan Hajnoczi 941a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq) 942a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 943a62a8ef9SStefan Hajnoczi { 944a62a8ef9SStefan Hajnoczi unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */ 945a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 946a62a8ef9SStefan Hajnoczi struct fuse_conn *fc; 947a62a8ef9SStefan Hajnoczi struct fuse_req *req; 94851fecdd2SVivek Goyal struct virtio_fs_vq *fsvq; 949a62a8ef9SStefan Hajnoczi int ret; 950a62a8ef9SStefan Hajnoczi 951a62a8ef9SStefan Hajnoczi WARN_ON(list_empty(&fiq->pending)); 952a62a8ef9SStefan Hajnoczi req = list_last_entry(&fiq->pending, struct fuse_req, list); 953a62a8ef9SStefan Hajnoczi clear_bit(FR_PENDING, &req->flags); 954a62a8ef9SStefan Hajnoczi list_del_init(&req->list); 955a62a8ef9SStefan Hajnoczi WARN_ON(!list_empty(&fiq->pending)); 956a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 957a62a8ef9SStefan Hajnoczi 958a62a8ef9SStefan Hajnoczi fs = fiq->priv; 959a62a8ef9SStefan Hajnoczi fc = fs->vqs[queue_id].fud->fc; 960a62a8ef9SStefan Hajnoczi 961a62a8ef9SStefan Hajnoczi pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n", 962a62a8ef9SStefan Hajnoczi __func__, req->in.h.opcode, req->in.h.unique, 963a62a8ef9SStefan Hajnoczi req->in.h.nodeid, req->in.h.len, 964a62a8ef9SStefan Hajnoczi fuse_len_args(req->args->out_numargs, req->args->out_args)); 965a62a8ef9SStefan Hajnoczi 966a62a8ef9SStefan Hajnoczi retry: 96751fecdd2SVivek Goyal fsvq = &fs->vqs[queue_id]; 96851fecdd2SVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req); 969a62a8ef9SStefan Hajnoczi if (ret < 0) { 970a62a8ef9SStefan Hajnoczi if (ret == -ENOMEM || ret == -ENOSPC) { 971a62a8ef9SStefan Hajnoczi /* Virtqueue full. Retry submission */ 972a62a8ef9SStefan Hajnoczi /* TODO use completion instead of timeout */ 973a62a8ef9SStefan Hajnoczi usleep_range(20, 30); 974a62a8ef9SStefan Hajnoczi goto retry; 975a62a8ef9SStefan Hajnoczi } 976a62a8ef9SStefan Hajnoczi req->out.h.error = ret; 977a62a8ef9SStefan Hajnoczi pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); 97851fecdd2SVivek Goyal 97951fecdd2SVivek Goyal /* Can't end request in submission context. Use a worker */ 98051fecdd2SVivek Goyal spin_lock(&fsvq->lock); 98151fecdd2SVivek Goyal list_add_tail(&req->list, &fsvq->end_reqs); 98251fecdd2SVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 0); 98351fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 984a62a8ef9SStefan Hajnoczi return; 985a62a8ef9SStefan Hajnoczi } 986a62a8ef9SStefan Hajnoczi } 987a62a8ef9SStefan Hajnoczi 988a62a8ef9SStefan Hajnoczi const static struct fuse_iqueue_ops virtio_fs_fiq_ops = { 989a62a8ef9SStefan Hajnoczi .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock, 990a62a8ef9SStefan Hajnoczi .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock, 991a62a8ef9SStefan Hajnoczi .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock, 992a62a8ef9SStefan Hajnoczi .release = virtio_fs_fiq_release, 993a62a8ef9SStefan Hajnoczi }; 994a62a8ef9SStefan Hajnoczi 995a62a8ef9SStefan Hajnoczi static int virtio_fs_fill_super(struct super_block *sb) 996a62a8ef9SStefan Hajnoczi { 997a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = get_fuse_conn_super(sb); 998a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = fc->iq.priv; 999a62a8ef9SStefan Hajnoczi unsigned int i; 1000a62a8ef9SStefan Hajnoczi int err; 1001a62a8ef9SStefan Hajnoczi struct fuse_fs_context ctx = { 1002a62a8ef9SStefan Hajnoczi .rootmode = S_IFDIR, 1003a62a8ef9SStefan Hajnoczi .default_permissions = 1, 1004a62a8ef9SStefan Hajnoczi .allow_other = 1, 1005a62a8ef9SStefan Hajnoczi .max_read = UINT_MAX, 1006a62a8ef9SStefan Hajnoczi .blksize = 512, 1007a62a8ef9SStefan Hajnoczi .destroy = true, 1008a62a8ef9SStefan Hajnoczi .no_control = true, 1009a62a8ef9SStefan Hajnoczi .no_force_umount = true, 10103f22c746SMiklos Szeredi .no_mount_options = true, 1011a62a8ef9SStefan Hajnoczi }; 1012a62a8ef9SStefan Hajnoczi 1013a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 1014a62a8ef9SStefan Hajnoczi 1015a62a8ef9SStefan Hajnoczi /* After holding mutex, make sure virtiofs device is still there. 1016a62a8ef9SStefan Hajnoczi * Though we are holding a reference to it, drive ->remove might 1017a62a8ef9SStefan Hajnoczi * still have cleaned up virtual queues. In that case bail out. 1018a62a8ef9SStefan Hajnoczi */ 1019a62a8ef9SStefan Hajnoczi err = -EINVAL; 1020a62a8ef9SStefan Hajnoczi if (list_empty(&fs->list)) { 1021a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fs->tag); 1022a62a8ef9SStefan Hajnoczi goto err; 1023a62a8ef9SStefan Hajnoczi } 1024a62a8ef9SStefan Hajnoczi 1025a62a8ef9SStefan Hajnoczi err = -ENOMEM; 1026a62a8ef9SStefan Hajnoczi /* Allocate fuse_dev for hiprio and notification queues */ 1027a62a8ef9SStefan Hajnoczi for (i = 0; i < VQ_REQUEST; i++) { 1028a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1029a62a8ef9SStefan Hajnoczi 1030a62a8ef9SStefan Hajnoczi fsvq->fud = fuse_dev_alloc(); 1031a62a8ef9SStefan Hajnoczi if (!fsvq->fud) 1032a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs; 1033a62a8ef9SStefan Hajnoczi } 1034a62a8ef9SStefan Hajnoczi 1035a62a8ef9SStefan Hajnoczi ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud; 1036a62a8ef9SStefan Hajnoczi err = fuse_fill_super_common(sb, &ctx); 1037a62a8ef9SStefan Hajnoczi if (err < 0) 1038a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs; 1039a62a8ef9SStefan Hajnoczi 1040a62a8ef9SStefan Hajnoczi fc = fs->vqs[VQ_REQUEST].fud->fc; 1041a62a8ef9SStefan Hajnoczi 1042a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 1043a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1044a62a8ef9SStefan Hajnoczi 1045a62a8ef9SStefan Hajnoczi if (i == VQ_REQUEST) 1046a62a8ef9SStefan Hajnoczi continue; /* already initialized */ 1047a62a8ef9SStefan Hajnoczi fuse_dev_install(fsvq->fud, fc); 1048a62a8ef9SStefan Hajnoczi } 1049a62a8ef9SStefan Hajnoczi 1050a62a8ef9SStefan Hajnoczi /* Previous unmount will stop all queues. Start these again */ 1051a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs); 1052a62a8ef9SStefan Hajnoczi fuse_send_init(fc); 1053a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1054a62a8ef9SStefan Hajnoczi return 0; 1055a62a8ef9SStefan Hajnoczi 1056a62a8ef9SStefan Hajnoczi err_free_fuse_devs: 1057a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(fs); 1058a62a8ef9SStefan Hajnoczi err: 1059a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1060a62a8ef9SStefan Hajnoczi return err; 1061a62a8ef9SStefan Hajnoczi } 1062a62a8ef9SStefan Hajnoczi 1063a62a8ef9SStefan Hajnoczi static void virtio_kill_sb(struct super_block *sb) 1064a62a8ef9SStefan Hajnoczi { 1065a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = get_fuse_conn_super(sb); 1066a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs; 1067a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 1068a62a8ef9SStefan Hajnoczi 1069a62a8ef9SStefan Hajnoczi /* If mount failed, we can still be called without any fc */ 1070a62a8ef9SStefan Hajnoczi if (!fc) 1071a62a8ef9SStefan Hajnoczi return fuse_kill_sb_anon(sb); 1072a62a8ef9SStefan Hajnoczi 1073a62a8ef9SStefan Hajnoczi vfs = fc->iq.priv; 1074a62a8ef9SStefan Hajnoczi fsvq = &vfs->vqs[VQ_HIPRIO]; 1075a62a8ef9SStefan Hajnoczi 1076a62a8ef9SStefan Hajnoczi /* Stop forget queue. Soon destroy will be sent */ 1077a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 1078a62a8ef9SStefan Hajnoczi fsvq->connected = false; 1079a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 1080a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs); 1081a62a8ef9SStefan Hajnoczi 1082a62a8ef9SStefan Hajnoczi fuse_kill_sb_anon(sb); 1083a62a8ef9SStefan Hajnoczi 1084a62a8ef9SStefan Hajnoczi /* fuse_kill_sb_anon() must have sent destroy. Stop all queues 1085a62a8ef9SStefan Hajnoczi * and drain one more time and free fuse devices. Freeing fuse 1086a62a8ef9SStefan Hajnoczi * devices will drop their reference on fuse_conn and that in 1087a62a8ef9SStefan Hajnoczi * turn will drop its reference on virtio_fs object. 1088a62a8ef9SStefan Hajnoczi */ 1089a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(vfs); 1090a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs); 1091a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(vfs); 1092a62a8ef9SStefan Hajnoczi } 1093a62a8ef9SStefan Hajnoczi 1094a62a8ef9SStefan Hajnoczi static int virtio_fs_test_super(struct super_block *sb, 1095a62a8ef9SStefan Hajnoczi struct fs_context *fsc) 1096a62a8ef9SStefan Hajnoczi { 1097a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = fsc->s_fs_info; 1098a62a8ef9SStefan Hajnoczi 1099a62a8ef9SStefan Hajnoczi return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv; 1100a62a8ef9SStefan Hajnoczi } 1101a62a8ef9SStefan Hajnoczi 1102a62a8ef9SStefan Hajnoczi static int virtio_fs_set_super(struct super_block *sb, 1103a62a8ef9SStefan Hajnoczi struct fs_context *fsc) 1104a62a8ef9SStefan Hajnoczi { 1105a62a8ef9SStefan Hajnoczi int err; 1106a62a8ef9SStefan Hajnoczi 1107a62a8ef9SStefan Hajnoczi err = get_anon_bdev(&sb->s_dev); 1108a62a8ef9SStefan Hajnoczi if (!err) 1109a62a8ef9SStefan Hajnoczi fuse_conn_get(fsc->s_fs_info); 1110a62a8ef9SStefan Hajnoczi 1111a62a8ef9SStefan Hajnoczi return err; 1112a62a8ef9SStefan Hajnoczi } 1113a62a8ef9SStefan Hajnoczi 1114a62a8ef9SStefan Hajnoczi static int virtio_fs_get_tree(struct fs_context *fsc) 1115a62a8ef9SStefan Hajnoczi { 1116a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 1117a62a8ef9SStefan Hajnoczi struct super_block *sb; 1118a62a8ef9SStefan Hajnoczi struct fuse_conn *fc; 1119a62a8ef9SStefan Hajnoczi int err; 1120a62a8ef9SStefan Hajnoczi 1121a62a8ef9SStefan Hajnoczi /* This gets a reference on virtio_fs object. This ptr gets installed 1122a62a8ef9SStefan Hajnoczi * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() 1123a62a8ef9SStefan Hajnoczi * to drop the reference to this object. 1124a62a8ef9SStefan Hajnoczi */ 1125a62a8ef9SStefan Hajnoczi fs = virtio_fs_find_instance(fsc->source); 1126a62a8ef9SStefan Hajnoczi if (!fs) { 1127a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fsc->source); 1128a62a8ef9SStefan Hajnoczi return -EINVAL; 1129a62a8ef9SStefan Hajnoczi } 1130a62a8ef9SStefan Hajnoczi 1131a62a8ef9SStefan Hajnoczi fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); 1132a62a8ef9SStefan Hajnoczi if (!fc) { 1133a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 1134a62a8ef9SStefan Hajnoczi virtio_fs_put(fs); 1135a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1136a62a8ef9SStefan Hajnoczi return -ENOMEM; 1137a62a8ef9SStefan Hajnoczi } 1138a62a8ef9SStefan Hajnoczi 1139a62a8ef9SStefan Hajnoczi fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops, 1140a62a8ef9SStefan Hajnoczi fs); 1141a62a8ef9SStefan Hajnoczi fc->release = fuse_free_conn; 1142a62a8ef9SStefan Hajnoczi fc->delete_stale = true; 1143a62a8ef9SStefan Hajnoczi 1144a62a8ef9SStefan Hajnoczi fsc->s_fs_info = fc; 1145a62a8ef9SStefan Hajnoczi sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super); 1146a62a8ef9SStefan Hajnoczi fuse_conn_put(fc); 1147a62a8ef9SStefan Hajnoczi if (IS_ERR(sb)) 1148a62a8ef9SStefan Hajnoczi return PTR_ERR(sb); 1149a62a8ef9SStefan Hajnoczi 1150a62a8ef9SStefan Hajnoczi if (!sb->s_root) { 1151a62a8ef9SStefan Hajnoczi err = virtio_fs_fill_super(sb); 1152a62a8ef9SStefan Hajnoczi if (err) { 1153a62a8ef9SStefan Hajnoczi deactivate_locked_super(sb); 1154a62a8ef9SStefan Hajnoczi return err; 1155a62a8ef9SStefan Hajnoczi } 1156a62a8ef9SStefan Hajnoczi 1157a62a8ef9SStefan Hajnoczi sb->s_flags |= SB_ACTIVE; 1158a62a8ef9SStefan Hajnoczi } 1159a62a8ef9SStefan Hajnoczi 1160a62a8ef9SStefan Hajnoczi WARN_ON(fsc->root); 1161a62a8ef9SStefan Hajnoczi fsc->root = dget(sb->s_root); 1162a62a8ef9SStefan Hajnoczi return 0; 1163a62a8ef9SStefan Hajnoczi } 1164a62a8ef9SStefan Hajnoczi 1165a62a8ef9SStefan Hajnoczi static const struct fs_context_operations virtio_fs_context_ops = { 1166a62a8ef9SStefan Hajnoczi .get_tree = virtio_fs_get_tree, 1167a62a8ef9SStefan Hajnoczi }; 1168a62a8ef9SStefan Hajnoczi 1169a62a8ef9SStefan Hajnoczi static int virtio_fs_init_fs_context(struct fs_context *fsc) 1170a62a8ef9SStefan Hajnoczi { 1171a62a8ef9SStefan Hajnoczi fsc->ops = &virtio_fs_context_ops; 1172a62a8ef9SStefan Hajnoczi return 0; 1173a62a8ef9SStefan Hajnoczi } 1174a62a8ef9SStefan Hajnoczi 1175a62a8ef9SStefan Hajnoczi static struct file_system_type virtio_fs_type = { 1176a62a8ef9SStefan Hajnoczi .owner = THIS_MODULE, 1177a62a8ef9SStefan Hajnoczi .name = "virtiofs", 1178a62a8ef9SStefan Hajnoczi .init_fs_context = virtio_fs_init_fs_context, 1179a62a8ef9SStefan Hajnoczi .kill_sb = virtio_kill_sb, 1180a62a8ef9SStefan Hajnoczi }; 1181a62a8ef9SStefan Hajnoczi 1182a62a8ef9SStefan Hajnoczi static int __init virtio_fs_init(void) 1183a62a8ef9SStefan Hajnoczi { 1184a62a8ef9SStefan Hajnoczi int ret; 1185a62a8ef9SStefan Hajnoczi 1186a62a8ef9SStefan Hajnoczi ret = register_virtio_driver(&virtio_fs_driver); 1187a62a8ef9SStefan Hajnoczi if (ret < 0) 1188a62a8ef9SStefan Hajnoczi return ret; 1189a62a8ef9SStefan Hajnoczi 1190a62a8ef9SStefan Hajnoczi ret = register_filesystem(&virtio_fs_type); 1191a62a8ef9SStefan Hajnoczi if (ret < 0) { 1192a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver); 1193a62a8ef9SStefan Hajnoczi return ret; 1194a62a8ef9SStefan Hajnoczi } 1195a62a8ef9SStefan Hajnoczi 1196a62a8ef9SStefan Hajnoczi return 0; 1197a62a8ef9SStefan Hajnoczi } 1198a62a8ef9SStefan Hajnoczi module_init(virtio_fs_init); 1199a62a8ef9SStefan Hajnoczi 1200a62a8ef9SStefan Hajnoczi static void __exit virtio_fs_exit(void) 1201a62a8ef9SStefan Hajnoczi { 1202a62a8ef9SStefan Hajnoczi unregister_filesystem(&virtio_fs_type); 1203a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver); 1204a62a8ef9SStefan Hajnoczi } 1205a62a8ef9SStefan Hajnoczi module_exit(virtio_fs_exit); 1206a62a8ef9SStefan Hajnoczi 1207a62a8ef9SStefan Hajnoczi MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>"); 1208a62a8ef9SStefan Hajnoczi MODULE_DESCRIPTION("Virtio Filesystem"); 1209a62a8ef9SStefan Hajnoczi MODULE_LICENSE("GPL"); 1210a62a8ef9SStefan Hajnoczi MODULE_ALIAS_FS(KBUILD_MODNAME); 1211a62a8ef9SStefan Hajnoczi MODULE_DEVICE_TABLE(virtio, id_table); 1212