1a62a8ef9SStefan Hajnoczi // SPDX-License-Identifier: GPL-2.0 2a62a8ef9SStefan Hajnoczi /* 3a62a8ef9SStefan Hajnoczi * virtio-fs: Virtio Filesystem 4a62a8ef9SStefan Hajnoczi * Copyright (C) 2018 Red Hat, Inc. 5a62a8ef9SStefan Hajnoczi */ 6a62a8ef9SStefan Hajnoczi 7a62a8ef9SStefan Hajnoczi #include <linux/fs.h> 8a62a8ef9SStefan Hajnoczi #include <linux/module.h> 9a62a8ef9SStefan Hajnoczi #include <linux/virtio.h> 10a62a8ef9SStefan Hajnoczi #include <linux/virtio_fs.h> 11a62a8ef9SStefan Hajnoczi #include <linux/delay.h> 12a62a8ef9SStefan Hajnoczi #include <linux/fs_context.h> 13a62a8ef9SStefan Hajnoczi #include <linux/highmem.h> 14a62a8ef9SStefan Hajnoczi #include "fuse_i.h" 15a62a8ef9SStefan Hajnoczi 16a62a8ef9SStefan Hajnoczi /* List of virtio-fs device instances and a lock for the list. Also provides 17a62a8ef9SStefan Hajnoczi * mutual exclusion in device removal and mounting path 18a62a8ef9SStefan Hajnoczi */ 19a62a8ef9SStefan Hajnoczi static DEFINE_MUTEX(virtio_fs_mutex); 20a62a8ef9SStefan Hajnoczi static LIST_HEAD(virtio_fs_instances); 21a62a8ef9SStefan Hajnoczi 22a62a8ef9SStefan Hajnoczi enum { 23a62a8ef9SStefan Hajnoczi VQ_HIPRIO, 24a62a8ef9SStefan Hajnoczi VQ_REQUEST 25a62a8ef9SStefan Hajnoczi }; 26a62a8ef9SStefan Hajnoczi 27a62a8ef9SStefan Hajnoczi /* Per-virtqueue state */ 28a62a8ef9SStefan Hajnoczi struct virtio_fs_vq { 29a62a8ef9SStefan Hajnoczi spinlock_t lock; 30a62a8ef9SStefan Hajnoczi struct virtqueue *vq; /* protected by ->lock */ 31a62a8ef9SStefan Hajnoczi struct work_struct done_work; 32a62a8ef9SStefan Hajnoczi struct list_head queued_reqs; 3351fecdd2SVivek Goyal struct list_head end_reqs; /* End these requests */ 34a62a8ef9SStefan Hajnoczi struct delayed_work dispatch_work; 35a62a8ef9SStefan Hajnoczi struct fuse_dev *fud; 36a62a8ef9SStefan Hajnoczi bool connected; 37a62a8ef9SStefan Hajnoczi long in_flight; 38a62a8ef9SStefan Hajnoczi char name[24]; 39a62a8ef9SStefan Hajnoczi } ____cacheline_aligned_in_smp; 40a62a8ef9SStefan Hajnoczi 41a62a8ef9SStefan Hajnoczi /* A virtio-fs device instance */ 42a62a8ef9SStefan Hajnoczi struct virtio_fs { 43a62a8ef9SStefan Hajnoczi struct kref refcount; 44a62a8ef9SStefan Hajnoczi struct list_head list; /* on virtio_fs_instances */ 45a62a8ef9SStefan Hajnoczi char *tag; 46a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *vqs; 47a62a8ef9SStefan Hajnoczi unsigned int nvqs; /* number of virtqueues */ 48a62a8ef9SStefan Hajnoczi unsigned int num_request_queues; /* number of request queues */ 49a62a8ef9SStefan Hajnoczi }; 50a62a8ef9SStefan Hajnoczi 51*1efcf39eSVivek Goyal struct virtio_fs_forget_req { 52a62a8ef9SStefan Hajnoczi struct fuse_in_header ih; 53a62a8ef9SStefan Hajnoczi struct fuse_forget_in arg; 54*1efcf39eSVivek Goyal }; 55*1efcf39eSVivek Goyal 56*1efcf39eSVivek Goyal struct virtio_fs_forget { 57a62a8ef9SStefan Hajnoczi /* This request can be temporarily queued on virt queue */ 58a62a8ef9SStefan Hajnoczi struct list_head list; 59*1efcf39eSVivek Goyal struct virtio_fs_forget_req req; 60a62a8ef9SStefan Hajnoczi }; 61a62a8ef9SStefan Hajnoczi 62a9bfd9ddSVivek Goyal static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 63a9bfd9ddSVivek Goyal struct fuse_req *req, bool in_flight); 64a9bfd9ddSVivek Goyal 65a62a8ef9SStefan Hajnoczi static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) 66a62a8ef9SStefan Hajnoczi { 67a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vq->vdev->priv; 68a62a8ef9SStefan Hajnoczi 69a62a8ef9SStefan Hajnoczi return &fs->vqs[vq->index]; 70a62a8ef9SStefan Hajnoczi } 71a62a8ef9SStefan Hajnoczi 72a62a8ef9SStefan Hajnoczi static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq) 73a62a8ef9SStefan Hajnoczi { 74a62a8ef9SStefan Hajnoczi return &vq_to_fsvq(vq)->fud->pq; 75a62a8ef9SStefan Hajnoczi } 76a62a8ef9SStefan Hajnoczi 77c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */ 78c17ea009SVivek Goyal static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq) 79c17ea009SVivek Goyal { 80c17ea009SVivek Goyal fsvq->in_flight++; 81c17ea009SVivek Goyal } 82c17ea009SVivek Goyal 83c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */ 84c17ea009SVivek Goyal static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) 85c17ea009SVivek Goyal { 86c17ea009SVivek Goyal WARN_ON(fsvq->in_flight <= 0); 87c17ea009SVivek Goyal fsvq->in_flight--; 88c17ea009SVivek Goyal } 89c17ea009SVivek Goyal 90a62a8ef9SStefan Hajnoczi static void release_virtio_fs_obj(struct kref *ref) 91a62a8ef9SStefan Hajnoczi { 92a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount); 93a62a8ef9SStefan Hajnoczi 94a62a8ef9SStefan Hajnoczi kfree(vfs->vqs); 95a62a8ef9SStefan Hajnoczi kfree(vfs); 96a62a8ef9SStefan Hajnoczi } 97a62a8ef9SStefan Hajnoczi 98a62a8ef9SStefan Hajnoczi /* Make sure virtiofs_mutex is held */ 99a62a8ef9SStefan Hajnoczi static void virtio_fs_put(struct virtio_fs *fs) 100a62a8ef9SStefan Hajnoczi { 101a62a8ef9SStefan Hajnoczi kref_put(&fs->refcount, release_virtio_fs_obj); 102a62a8ef9SStefan Hajnoczi } 103a62a8ef9SStefan Hajnoczi 104a62a8ef9SStefan Hajnoczi static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) 105a62a8ef9SStefan Hajnoczi { 106a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = fiq->priv; 107a62a8ef9SStefan Hajnoczi 108a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 109a62a8ef9SStefan Hajnoczi virtio_fs_put(vfs); 110a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 111a62a8ef9SStefan Hajnoczi } 112a62a8ef9SStefan Hajnoczi 113a62a8ef9SStefan Hajnoczi static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) 114a62a8ef9SStefan Hajnoczi { 115a62a8ef9SStefan Hajnoczi WARN_ON(fsvq->in_flight < 0); 116a62a8ef9SStefan Hajnoczi 117a62a8ef9SStefan Hajnoczi /* Wait for in flight requests to finish.*/ 118a62a8ef9SStefan Hajnoczi while (1) { 119a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 120a62a8ef9SStefan Hajnoczi if (!fsvq->in_flight) { 121a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 122a62a8ef9SStefan Hajnoczi break; 123a62a8ef9SStefan Hajnoczi } 124a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 125a62a8ef9SStefan Hajnoczi /* TODO use completion instead of timeout */ 126a62a8ef9SStefan Hajnoczi usleep_range(1000, 2000); 127a62a8ef9SStefan Hajnoczi } 128a62a8ef9SStefan Hajnoczi 129a62a8ef9SStefan Hajnoczi flush_work(&fsvq->done_work); 130a62a8ef9SStefan Hajnoczi flush_delayed_work(&fsvq->dispatch_work); 131a62a8ef9SStefan Hajnoczi } 132a62a8ef9SStefan Hajnoczi 133a62a8ef9SStefan Hajnoczi static void virtio_fs_drain_all_queues(struct virtio_fs *fs) 134a62a8ef9SStefan Hajnoczi { 135a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 136a62a8ef9SStefan Hajnoczi int i; 137a62a8ef9SStefan Hajnoczi 138a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 139a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 140a62a8ef9SStefan Hajnoczi virtio_fs_drain_queue(fsvq); 141a62a8ef9SStefan Hajnoczi } 142a62a8ef9SStefan Hajnoczi } 143a62a8ef9SStefan Hajnoczi 144a62a8ef9SStefan Hajnoczi static void virtio_fs_start_all_queues(struct virtio_fs *fs) 145a62a8ef9SStefan Hajnoczi { 146a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 147a62a8ef9SStefan Hajnoczi int i; 148a62a8ef9SStefan Hajnoczi 149a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 150a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 151a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 152a62a8ef9SStefan Hajnoczi fsvq->connected = true; 153a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 154a62a8ef9SStefan Hajnoczi } 155a62a8ef9SStefan Hajnoczi } 156a62a8ef9SStefan Hajnoczi 157a62a8ef9SStefan Hajnoczi /* Add a new instance to the list or return -EEXIST if tag name exists*/ 158a62a8ef9SStefan Hajnoczi static int virtio_fs_add_instance(struct virtio_fs *fs) 159a62a8ef9SStefan Hajnoczi { 160a62a8ef9SStefan Hajnoczi struct virtio_fs *fs2; 161a62a8ef9SStefan Hajnoczi bool duplicate = false; 162a62a8ef9SStefan Hajnoczi 163a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 164a62a8ef9SStefan Hajnoczi 165a62a8ef9SStefan Hajnoczi list_for_each_entry(fs2, &virtio_fs_instances, list) { 166a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, fs2->tag) == 0) 167a62a8ef9SStefan Hajnoczi duplicate = true; 168a62a8ef9SStefan Hajnoczi } 169a62a8ef9SStefan Hajnoczi 170a62a8ef9SStefan Hajnoczi if (!duplicate) 171a62a8ef9SStefan Hajnoczi list_add_tail(&fs->list, &virtio_fs_instances); 172a62a8ef9SStefan Hajnoczi 173a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 174a62a8ef9SStefan Hajnoczi 175a62a8ef9SStefan Hajnoczi if (duplicate) 176a62a8ef9SStefan Hajnoczi return -EEXIST; 177a62a8ef9SStefan Hajnoczi return 0; 178a62a8ef9SStefan Hajnoczi } 179a62a8ef9SStefan Hajnoczi 180a62a8ef9SStefan Hajnoczi /* Return the virtio_fs with a given tag, or NULL */ 181a62a8ef9SStefan Hajnoczi static struct virtio_fs *virtio_fs_find_instance(const char *tag) 182a62a8ef9SStefan Hajnoczi { 183a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 184a62a8ef9SStefan Hajnoczi 185a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 186a62a8ef9SStefan Hajnoczi 187a62a8ef9SStefan Hajnoczi list_for_each_entry(fs, &virtio_fs_instances, list) { 188a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, tag) == 0) { 189a62a8ef9SStefan Hajnoczi kref_get(&fs->refcount); 190a62a8ef9SStefan Hajnoczi goto found; 191a62a8ef9SStefan Hajnoczi } 192a62a8ef9SStefan Hajnoczi } 193a62a8ef9SStefan Hajnoczi 194a62a8ef9SStefan Hajnoczi fs = NULL; /* not found */ 195a62a8ef9SStefan Hajnoczi 196a62a8ef9SStefan Hajnoczi found: 197a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 198a62a8ef9SStefan Hajnoczi 199a62a8ef9SStefan Hajnoczi return fs; 200a62a8ef9SStefan Hajnoczi } 201a62a8ef9SStefan Hajnoczi 202a62a8ef9SStefan Hajnoczi static void virtio_fs_free_devs(struct virtio_fs *fs) 203a62a8ef9SStefan Hajnoczi { 204a62a8ef9SStefan Hajnoczi unsigned int i; 205a62a8ef9SStefan Hajnoczi 206a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 207a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 208a62a8ef9SStefan Hajnoczi 209a62a8ef9SStefan Hajnoczi if (!fsvq->fud) 210a62a8ef9SStefan Hajnoczi continue; 211a62a8ef9SStefan Hajnoczi 212a62a8ef9SStefan Hajnoczi fuse_dev_free(fsvq->fud); 213a62a8ef9SStefan Hajnoczi fsvq->fud = NULL; 214a62a8ef9SStefan Hajnoczi } 215a62a8ef9SStefan Hajnoczi } 216a62a8ef9SStefan Hajnoczi 217a62a8ef9SStefan Hajnoczi /* Read filesystem name from virtio config into fs->tag (must kfree()). */ 218a62a8ef9SStefan Hajnoczi static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs) 219a62a8ef9SStefan Hajnoczi { 220a62a8ef9SStefan Hajnoczi char tag_buf[sizeof_field(struct virtio_fs_config, tag)]; 221a62a8ef9SStefan Hajnoczi char *end; 222a62a8ef9SStefan Hajnoczi size_t len; 223a62a8ef9SStefan Hajnoczi 224a62a8ef9SStefan Hajnoczi virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag), 225a62a8ef9SStefan Hajnoczi &tag_buf, sizeof(tag_buf)); 226a62a8ef9SStefan Hajnoczi end = memchr(tag_buf, '\0', sizeof(tag_buf)); 227a62a8ef9SStefan Hajnoczi if (end == tag_buf) 228a62a8ef9SStefan Hajnoczi return -EINVAL; /* empty tag */ 229a62a8ef9SStefan Hajnoczi if (!end) 230a62a8ef9SStefan Hajnoczi end = &tag_buf[sizeof(tag_buf)]; 231a62a8ef9SStefan Hajnoczi 232a62a8ef9SStefan Hajnoczi len = end - tag_buf; 233a62a8ef9SStefan Hajnoczi fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); 234a62a8ef9SStefan Hajnoczi if (!fs->tag) 235a62a8ef9SStefan Hajnoczi return -ENOMEM; 236a62a8ef9SStefan Hajnoczi memcpy(fs->tag, tag_buf, len); 237a62a8ef9SStefan Hajnoczi fs->tag[len] = '\0'; 238a62a8ef9SStefan Hajnoczi return 0; 239a62a8ef9SStefan Hajnoczi } 240a62a8ef9SStefan Hajnoczi 241a62a8ef9SStefan Hajnoczi /* Work function for hiprio completion */ 242a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_done_work(struct work_struct *work) 243a62a8ef9SStefan Hajnoczi { 244a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 245a62a8ef9SStefan Hajnoczi done_work); 246a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq; 247a62a8ef9SStefan Hajnoczi 248a62a8ef9SStefan Hajnoczi /* Free completed FUSE_FORGET requests */ 249a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 250a62a8ef9SStefan Hajnoczi do { 251a62a8ef9SStefan Hajnoczi unsigned int len; 252a62a8ef9SStefan Hajnoczi void *req; 253a62a8ef9SStefan Hajnoczi 254a62a8ef9SStefan Hajnoczi virtqueue_disable_cb(vq); 255a62a8ef9SStefan Hajnoczi 256a62a8ef9SStefan Hajnoczi while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 257a62a8ef9SStefan Hajnoczi kfree(req); 258c17ea009SVivek Goyal dec_in_flight_req(fsvq); 259a62a8ef9SStefan Hajnoczi } 260a62a8ef9SStefan Hajnoczi } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 261a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 262a62a8ef9SStefan Hajnoczi } 263a62a8ef9SStefan Hajnoczi 26451fecdd2SVivek Goyal static void virtio_fs_request_dispatch_work(struct work_struct *work) 265a62a8ef9SStefan Hajnoczi { 26651fecdd2SVivek Goyal struct fuse_req *req; 26751fecdd2SVivek Goyal struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 26851fecdd2SVivek Goyal dispatch_work.work); 26951fecdd2SVivek Goyal struct fuse_conn *fc = fsvq->fud->fc; 270a9bfd9ddSVivek Goyal int ret; 27151fecdd2SVivek Goyal 27251fecdd2SVivek Goyal pr_debug("virtio-fs: worker %s called.\n", __func__); 27351fecdd2SVivek Goyal while (1) { 27451fecdd2SVivek Goyal spin_lock(&fsvq->lock); 27551fecdd2SVivek Goyal req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, 27651fecdd2SVivek Goyal list); 27751fecdd2SVivek Goyal if (!req) { 27851fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 279a9bfd9ddSVivek Goyal break; 28051fecdd2SVivek Goyal } 28151fecdd2SVivek Goyal 28251fecdd2SVivek Goyal list_del_init(&req->list); 28351fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 28451fecdd2SVivek Goyal fuse_request_end(fc, req); 28551fecdd2SVivek Goyal } 286a9bfd9ddSVivek Goyal 287a9bfd9ddSVivek Goyal /* Dispatch pending requests */ 288a9bfd9ddSVivek Goyal while (1) { 289a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 290a9bfd9ddSVivek Goyal req = list_first_entry_or_null(&fsvq->queued_reqs, 291a9bfd9ddSVivek Goyal struct fuse_req, list); 292a9bfd9ddSVivek Goyal if (!req) { 293a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 294a9bfd9ddSVivek Goyal return; 295a9bfd9ddSVivek Goyal } 296a9bfd9ddSVivek Goyal list_del_init(&req->list); 297a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 298a9bfd9ddSVivek Goyal 299a9bfd9ddSVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req, true); 300a9bfd9ddSVivek Goyal if (ret < 0) { 301a9bfd9ddSVivek Goyal if (ret == -ENOMEM || ret == -ENOSPC) { 302a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 303a9bfd9ddSVivek Goyal list_add_tail(&req->list, &fsvq->queued_reqs); 304a9bfd9ddSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 305a9bfd9ddSVivek Goyal msecs_to_jiffies(1)); 306a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 307a9bfd9ddSVivek Goyal return; 308a9bfd9ddSVivek Goyal } 309a9bfd9ddSVivek Goyal req->out.h.error = ret; 310a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 311a9bfd9ddSVivek Goyal dec_in_flight_req(fsvq); 312a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 313a9bfd9ddSVivek Goyal pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", 314a9bfd9ddSVivek Goyal ret); 315a9bfd9ddSVivek Goyal fuse_request_end(fc, req); 316a9bfd9ddSVivek Goyal } 317a9bfd9ddSVivek Goyal } 318a62a8ef9SStefan Hajnoczi } 319a62a8ef9SStefan Hajnoczi 32058ada94fSVivek Goyal /* 32158ada94fSVivek Goyal * Returns 1 if queue is full and sender should wait a bit before sending 32258ada94fSVivek Goyal * next request, 0 otherwise. 32358ada94fSVivek Goyal */ 32458ada94fSVivek Goyal static int send_forget_request(struct virtio_fs_vq *fsvq, 32558ada94fSVivek Goyal struct virtio_fs_forget *forget, 32658ada94fSVivek Goyal bool in_flight) 32758ada94fSVivek Goyal { 32858ada94fSVivek Goyal struct scatterlist sg; 32958ada94fSVivek Goyal struct virtqueue *vq; 33058ada94fSVivek Goyal int ret = 0; 33158ada94fSVivek Goyal bool notify; 332*1efcf39eSVivek Goyal struct virtio_fs_forget_req *req = &forget->req; 33358ada94fSVivek Goyal 33458ada94fSVivek Goyal spin_lock(&fsvq->lock); 33558ada94fSVivek Goyal if (!fsvq->connected) { 33658ada94fSVivek Goyal if (in_flight) 33758ada94fSVivek Goyal dec_in_flight_req(fsvq); 33858ada94fSVivek Goyal kfree(forget); 33958ada94fSVivek Goyal goto out; 34058ada94fSVivek Goyal } 34158ada94fSVivek Goyal 342*1efcf39eSVivek Goyal sg_init_one(&sg, req, sizeof(*req)); 34358ada94fSVivek Goyal vq = fsvq->vq; 34458ada94fSVivek Goyal dev_dbg(&vq->vdev->dev, "%s\n", __func__); 34558ada94fSVivek Goyal 34658ada94fSVivek Goyal ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); 34758ada94fSVivek Goyal if (ret < 0) { 34858ada94fSVivek Goyal if (ret == -ENOMEM || ret == -ENOSPC) { 34958ada94fSVivek Goyal pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", 35058ada94fSVivek Goyal ret); 35158ada94fSVivek Goyal list_add_tail(&forget->list, &fsvq->queued_reqs); 35258ada94fSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 35358ada94fSVivek Goyal msecs_to_jiffies(1)); 35458ada94fSVivek Goyal if (!in_flight) 35558ada94fSVivek Goyal inc_in_flight_req(fsvq); 35658ada94fSVivek Goyal /* Queue is full */ 35758ada94fSVivek Goyal ret = 1; 35858ada94fSVivek Goyal } else { 35958ada94fSVivek Goyal pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", 36058ada94fSVivek Goyal ret); 36158ada94fSVivek Goyal kfree(forget); 36258ada94fSVivek Goyal if (in_flight) 36358ada94fSVivek Goyal dec_in_flight_req(fsvq); 36458ada94fSVivek Goyal } 36558ada94fSVivek Goyal goto out; 36658ada94fSVivek Goyal } 36758ada94fSVivek Goyal 36858ada94fSVivek Goyal if (!in_flight) 36958ada94fSVivek Goyal inc_in_flight_req(fsvq); 37058ada94fSVivek Goyal notify = virtqueue_kick_prepare(vq); 37158ada94fSVivek Goyal spin_unlock(&fsvq->lock); 37258ada94fSVivek Goyal 37358ada94fSVivek Goyal if (notify) 37458ada94fSVivek Goyal virtqueue_notify(vq); 37558ada94fSVivek Goyal return ret; 37658ada94fSVivek Goyal out: 37758ada94fSVivek Goyal spin_unlock(&fsvq->lock); 37858ada94fSVivek Goyal return ret; 37958ada94fSVivek Goyal } 38058ada94fSVivek Goyal 381a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) 382a62a8ef9SStefan Hajnoczi { 383a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget; 384a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 385a62a8ef9SStefan Hajnoczi dispatch_work.work); 386a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: worker %s called.\n", __func__); 387a62a8ef9SStefan Hajnoczi while (1) { 388a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 389a62a8ef9SStefan Hajnoczi forget = list_first_entry_or_null(&fsvq->queued_reqs, 390a62a8ef9SStefan Hajnoczi struct virtio_fs_forget, list); 391a62a8ef9SStefan Hajnoczi if (!forget) { 392a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 393a62a8ef9SStefan Hajnoczi return; 394a62a8ef9SStefan Hajnoczi } 395a62a8ef9SStefan Hajnoczi 396a62a8ef9SStefan Hajnoczi list_del(&forget->list); 397a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 39858ada94fSVivek Goyal if (send_forget_request(fsvq, forget, true)) 399a62a8ef9SStefan Hajnoczi return; 400a62a8ef9SStefan Hajnoczi } 401a62a8ef9SStefan Hajnoczi } 402a62a8ef9SStefan Hajnoczi 403a62a8ef9SStefan Hajnoczi /* Allocate and copy args into req->argbuf */ 404a62a8ef9SStefan Hajnoczi static int copy_args_to_argbuf(struct fuse_req *req) 405a62a8ef9SStefan Hajnoczi { 406a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 407a62a8ef9SStefan Hajnoczi unsigned int offset = 0; 408a62a8ef9SStefan Hajnoczi unsigned int num_in; 409a62a8ef9SStefan Hajnoczi unsigned int num_out; 410a62a8ef9SStefan Hajnoczi unsigned int len; 411a62a8ef9SStefan Hajnoczi unsigned int i; 412a62a8ef9SStefan Hajnoczi 413a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages; 414a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages; 415a62a8ef9SStefan Hajnoczi len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + 416a62a8ef9SStefan Hajnoczi fuse_len_args(num_out, args->out_args); 417a62a8ef9SStefan Hajnoczi 418a62a8ef9SStefan Hajnoczi req->argbuf = kmalloc(len, GFP_ATOMIC); 419a62a8ef9SStefan Hajnoczi if (!req->argbuf) 420a62a8ef9SStefan Hajnoczi return -ENOMEM; 421a62a8ef9SStefan Hajnoczi 422a62a8ef9SStefan Hajnoczi for (i = 0; i < num_in; i++) { 423a62a8ef9SStefan Hajnoczi memcpy(req->argbuf + offset, 424a62a8ef9SStefan Hajnoczi args->in_args[i].value, 425a62a8ef9SStefan Hajnoczi args->in_args[i].size); 426a62a8ef9SStefan Hajnoczi offset += args->in_args[i].size; 427a62a8ef9SStefan Hajnoczi } 428a62a8ef9SStefan Hajnoczi 429a62a8ef9SStefan Hajnoczi return 0; 430a62a8ef9SStefan Hajnoczi } 431a62a8ef9SStefan Hajnoczi 432a62a8ef9SStefan Hajnoczi /* Copy args out of and free req->argbuf */ 433a62a8ef9SStefan Hajnoczi static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req) 434a62a8ef9SStefan Hajnoczi { 435a62a8ef9SStefan Hajnoczi unsigned int remaining; 436a62a8ef9SStefan Hajnoczi unsigned int offset; 437a62a8ef9SStefan Hajnoczi unsigned int num_in; 438a62a8ef9SStefan Hajnoczi unsigned int num_out; 439a62a8ef9SStefan Hajnoczi unsigned int i; 440a62a8ef9SStefan Hajnoczi 441a62a8ef9SStefan Hajnoczi remaining = req->out.h.len - sizeof(req->out.h); 442a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages; 443a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages; 444a62a8ef9SStefan Hajnoczi offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); 445a62a8ef9SStefan Hajnoczi 446a62a8ef9SStefan Hajnoczi for (i = 0; i < num_out; i++) { 447a62a8ef9SStefan Hajnoczi unsigned int argsize = args->out_args[i].size; 448a62a8ef9SStefan Hajnoczi 449a62a8ef9SStefan Hajnoczi if (args->out_argvar && 450a62a8ef9SStefan Hajnoczi i == args->out_numargs - 1 && 451a62a8ef9SStefan Hajnoczi argsize > remaining) { 452a62a8ef9SStefan Hajnoczi argsize = remaining; 453a62a8ef9SStefan Hajnoczi } 454a62a8ef9SStefan Hajnoczi 455a62a8ef9SStefan Hajnoczi memcpy(args->out_args[i].value, req->argbuf + offset, argsize); 456a62a8ef9SStefan Hajnoczi offset += argsize; 457a62a8ef9SStefan Hajnoczi 458a62a8ef9SStefan Hajnoczi if (i != args->out_numargs - 1) 459a62a8ef9SStefan Hajnoczi remaining -= argsize; 460a62a8ef9SStefan Hajnoczi } 461a62a8ef9SStefan Hajnoczi 462a62a8ef9SStefan Hajnoczi /* Store the actual size of the variable-length arg */ 463a62a8ef9SStefan Hajnoczi if (args->out_argvar) 464a62a8ef9SStefan Hajnoczi args->out_args[args->out_numargs - 1].size = remaining; 465a62a8ef9SStefan Hajnoczi 466a62a8ef9SStefan Hajnoczi kfree(req->argbuf); 467a62a8ef9SStefan Hajnoczi req->argbuf = NULL; 468a62a8ef9SStefan Hajnoczi } 469a62a8ef9SStefan Hajnoczi 470a62a8ef9SStefan Hajnoczi /* Work function for request completion */ 471a62a8ef9SStefan Hajnoczi static void virtio_fs_requests_done_work(struct work_struct *work) 472a62a8ef9SStefan Hajnoczi { 473a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 474a62a8ef9SStefan Hajnoczi done_work); 475a62a8ef9SStefan Hajnoczi struct fuse_pqueue *fpq = &fsvq->fud->pq; 476a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = fsvq->fud->fc; 477a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq; 478a62a8ef9SStefan Hajnoczi struct fuse_req *req; 479a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap; 480a62a8ef9SStefan Hajnoczi struct fuse_req *next; 481a62a8ef9SStefan Hajnoczi struct fuse_args *args; 482a62a8ef9SStefan Hajnoczi unsigned int len, i, thislen; 483a62a8ef9SStefan Hajnoczi struct page *page; 484a62a8ef9SStefan Hajnoczi LIST_HEAD(reqs); 485a62a8ef9SStefan Hajnoczi 486a62a8ef9SStefan Hajnoczi /* Collect completed requests off the virtqueue */ 487a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 488a62a8ef9SStefan Hajnoczi do { 489a62a8ef9SStefan Hajnoczi virtqueue_disable_cb(vq); 490a62a8ef9SStefan Hajnoczi 491a62a8ef9SStefan Hajnoczi while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 492a62a8ef9SStefan Hajnoczi spin_lock(&fpq->lock); 493a62a8ef9SStefan Hajnoczi list_move_tail(&req->list, &reqs); 494a62a8ef9SStefan Hajnoczi spin_unlock(&fpq->lock); 495a62a8ef9SStefan Hajnoczi } 496a62a8ef9SStefan Hajnoczi } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 497a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 498a62a8ef9SStefan Hajnoczi 499a62a8ef9SStefan Hajnoczi /* End requests */ 500a62a8ef9SStefan Hajnoczi list_for_each_entry_safe(req, next, &reqs, list) { 501a62a8ef9SStefan Hajnoczi /* 502a62a8ef9SStefan Hajnoczi * TODO verify that server properly follows FUSE protocol 503a62a8ef9SStefan Hajnoczi * (oh.uniq, oh.len) 504a62a8ef9SStefan Hajnoczi */ 505a62a8ef9SStefan Hajnoczi args = req->args; 506a62a8ef9SStefan Hajnoczi copy_args_from_argbuf(args, req); 507a62a8ef9SStefan Hajnoczi 508a62a8ef9SStefan Hajnoczi if (args->out_pages && args->page_zeroing) { 509a62a8ef9SStefan Hajnoczi len = args->out_args[args->out_numargs - 1].size; 510a62a8ef9SStefan Hajnoczi ap = container_of(args, typeof(*ap), args); 511a62a8ef9SStefan Hajnoczi for (i = 0; i < ap->num_pages; i++) { 512a62a8ef9SStefan Hajnoczi thislen = ap->descs[i].length; 513a62a8ef9SStefan Hajnoczi if (len < thislen) { 514a62a8ef9SStefan Hajnoczi WARN_ON(ap->descs[i].offset); 515a62a8ef9SStefan Hajnoczi page = ap->pages[i]; 516a62a8ef9SStefan Hajnoczi zero_user_segment(page, len, thislen); 517a62a8ef9SStefan Hajnoczi len = 0; 518a62a8ef9SStefan Hajnoczi } else { 519a62a8ef9SStefan Hajnoczi len -= thislen; 520a62a8ef9SStefan Hajnoczi } 521a62a8ef9SStefan Hajnoczi } 522a62a8ef9SStefan Hajnoczi } 523a62a8ef9SStefan Hajnoczi 524a62a8ef9SStefan Hajnoczi spin_lock(&fpq->lock); 525a62a8ef9SStefan Hajnoczi clear_bit(FR_SENT, &req->flags); 526a62a8ef9SStefan Hajnoczi list_del_init(&req->list); 527a62a8ef9SStefan Hajnoczi spin_unlock(&fpq->lock); 528a62a8ef9SStefan Hajnoczi 529a62a8ef9SStefan Hajnoczi fuse_request_end(fc, req); 530a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 531c17ea009SVivek Goyal dec_in_flight_req(fsvq); 532a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 533a62a8ef9SStefan Hajnoczi } 534a62a8ef9SStefan Hajnoczi } 535a62a8ef9SStefan Hajnoczi 536a62a8ef9SStefan Hajnoczi /* Virtqueue interrupt handler */ 537a62a8ef9SStefan Hajnoczi static void virtio_fs_vq_done(struct virtqueue *vq) 538a62a8ef9SStefan Hajnoczi { 539a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = vq_to_fsvq(vq); 540a62a8ef9SStefan Hajnoczi 541a62a8ef9SStefan Hajnoczi dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); 542a62a8ef9SStefan Hajnoczi 543a62a8ef9SStefan Hajnoczi schedule_work(&fsvq->done_work); 544a62a8ef9SStefan Hajnoczi } 545a62a8ef9SStefan Hajnoczi 546a62a8ef9SStefan Hajnoczi /* Initialize virtqueues */ 547a62a8ef9SStefan Hajnoczi static int virtio_fs_setup_vqs(struct virtio_device *vdev, 548a62a8ef9SStefan Hajnoczi struct virtio_fs *fs) 549a62a8ef9SStefan Hajnoczi { 550a62a8ef9SStefan Hajnoczi struct virtqueue **vqs; 551a62a8ef9SStefan Hajnoczi vq_callback_t **callbacks; 552a62a8ef9SStefan Hajnoczi const char **names; 553a62a8ef9SStefan Hajnoczi unsigned int i; 554a62a8ef9SStefan Hajnoczi int ret = 0; 555a62a8ef9SStefan Hajnoczi 556a62a8ef9SStefan Hajnoczi virtio_cread(vdev, struct virtio_fs_config, num_request_queues, 557a62a8ef9SStefan Hajnoczi &fs->num_request_queues); 558a62a8ef9SStefan Hajnoczi if (fs->num_request_queues == 0) 559a62a8ef9SStefan Hajnoczi return -EINVAL; 560a62a8ef9SStefan Hajnoczi 561a62a8ef9SStefan Hajnoczi fs->nvqs = 1 + fs->num_request_queues; 562a62a8ef9SStefan Hajnoczi fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); 563a62a8ef9SStefan Hajnoczi if (!fs->vqs) 564a62a8ef9SStefan Hajnoczi return -ENOMEM; 565a62a8ef9SStefan Hajnoczi 566a62a8ef9SStefan Hajnoczi vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); 567a62a8ef9SStefan Hajnoczi callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]), 568a62a8ef9SStefan Hajnoczi GFP_KERNEL); 569a62a8ef9SStefan Hajnoczi names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL); 570a62a8ef9SStefan Hajnoczi if (!vqs || !callbacks || !names) { 571a62a8ef9SStefan Hajnoczi ret = -ENOMEM; 572a62a8ef9SStefan Hajnoczi goto out; 573a62a8ef9SStefan Hajnoczi } 574a62a8ef9SStefan Hajnoczi 575a62a8ef9SStefan Hajnoczi callbacks[VQ_HIPRIO] = virtio_fs_vq_done; 576a62a8ef9SStefan Hajnoczi snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name), 577a62a8ef9SStefan Hajnoczi "hiprio"); 578a62a8ef9SStefan Hajnoczi names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; 579a62a8ef9SStefan Hajnoczi INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work); 580a62a8ef9SStefan Hajnoczi INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs); 58151fecdd2SVivek Goyal INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs); 582a62a8ef9SStefan Hajnoczi INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work, 583a62a8ef9SStefan Hajnoczi virtio_fs_hiprio_dispatch_work); 584a62a8ef9SStefan Hajnoczi spin_lock_init(&fs->vqs[VQ_HIPRIO].lock); 585a62a8ef9SStefan Hajnoczi 586a62a8ef9SStefan Hajnoczi /* Initialize the requests virtqueues */ 587a62a8ef9SStefan Hajnoczi for (i = VQ_REQUEST; i < fs->nvqs; i++) { 588a62a8ef9SStefan Hajnoczi spin_lock_init(&fs->vqs[i].lock); 589a62a8ef9SStefan Hajnoczi INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work); 590a62a8ef9SStefan Hajnoczi INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work, 59151fecdd2SVivek Goyal virtio_fs_request_dispatch_work); 592a62a8ef9SStefan Hajnoczi INIT_LIST_HEAD(&fs->vqs[i].queued_reqs); 59351fecdd2SVivek Goyal INIT_LIST_HEAD(&fs->vqs[i].end_reqs); 594a62a8ef9SStefan Hajnoczi snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name), 595a62a8ef9SStefan Hajnoczi "requests.%u", i - VQ_REQUEST); 596a62a8ef9SStefan Hajnoczi callbacks[i] = virtio_fs_vq_done; 597a62a8ef9SStefan Hajnoczi names[i] = fs->vqs[i].name; 598a62a8ef9SStefan Hajnoczi } 599a62a8ef9SStefan Hajnoczi 600a62a8ef9SStefan Hajnoczi ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL); 601a62a8ef9SStefan Hajnoczi if (ret < 0) 602a62a8ef9SStefan Hajnoczi goto out; 603a62a8ef9SStefan Hajnoczi 604a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) 605a62a8ef9SStefan Hajnoczi fs->vqs[i].vq = vqs[i]; 606a62a8ef9SStefan Hajnoczi 607a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs); 608a62a8ef9SStefan Hajnoczi out: 609a62a8ef9SStefan Hajnoczi kfree(names); 610a62a8ef9SStefan Hajnoczi kfree(callbacks); 611a62a8ef9SStefan Hajnoczi kfree(vqs); 612a62a8ef9SStefan Hajnoczi if (ret) 613a62a8ef9SStefan Hajnoczi kfree(fs->vqs); 614a62a8ef9SStefan Hajnoczi return ret; 615a62a8ef9SStefan Hajnoczi } 616a62a8ef9SStefan Hajnoczi 617a62a8ef9SStefan Hajnoczi /* Free virtqueues (device must already be reset) */ 618a62a8ef9SStefan Hajnoczi static void virtio_fs_cleanup_vqs(struct virtio_device *vdev, 619a62a8ef9SStefan Hajnoczi struct virtio_fs *fs) 620a62a8ef9SStefan Hajnoczi { 621a62a8ef9SStefan Hajnoczi vdev->config->del_vqs(vdev); 622a62a8ef9SStefan Hajnoczi } 623a62a8ef9SStefan Hajnoczi 624a62a8ef9SStefan Hajnoczi static int virtio_fs_probe(struct virtio_device *vdev) 625a62a8ef9SStefan Hajnoczi { 626a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 627a62a8ef9SStefan Hajnoczi int ret; 628a62a8ef9SStefan Hajnoczi 629a62a8ef9SStefan Hajnoczi fs = kzalloc(sizeof(*fs), GFP_KERNEL); 630a62a8ef9SStefan Hajnoczi if (!fs) 631a62a8ef9SStefan Hajnoczi return -ENOMEM; 632a62a8ef9SStefan Hajnoczi kref_init(&fs->refcount); 633a62a8ef9SStefan Hajnoczi vdev->priv = fs; 634a62a8ef9SStefan Hajnoczi 635a62a8ef9SStefan Hajnoczi ret = virtio_fs_read_tag(vdev, fs); 636a62a8ef9SStefan Hajnoczi if (ret < 0) 637a62a8ef9SStefan Hajnoczi goto out; 638a62a8ef9SStefan Hajnoczi 639a62a8ef9SStefan Hajnoczi ret = virtio_fs_setup_vqs(vdev, fs); 640a62a8ef9SStefan Hajnoczi if (ret < 0) 641a62a8ef9SStefan Hajnoczi goto out; 642a62a8ef9SStefan Hajnoczi 643a62a8ef9SStefan Hajnoczi /* TODO vq affinity */ 644a62a8ef9SStefan Hajnoczi 645a62a8ef9SStefan Hajnoczi /* Bring the device online in case the filesystem is mounted and 646a62a8ef9SStefan Hajnoczi * requests need to be sent before we return. 647a62a8ef9SStefan Hajnoczi */ 648a62a8ef9SStefan Hajnoczi virtio_device_ready(vdev); 649a62a8ef9SStefan Hajnoczi 650a62a8ef9SStefan Hajnoczi ret = virtio_fs_add_instance(fs); 651a62a8ef9SStefan Hajnoczi if (ret < 0) 652a62a8ef9SStefan Hajnoczi goto out_vqs; 653a62a8ef9SStefan Hajnoczi 654a62a8ef9SStefan Hajnoczi return 0; 655a62a8ef9SStefan Hajnoczi 656a62a8ef9SStefan Hajnoczi out_vqs: 657a62a8ef9SStefan Hajnoczi vdev->config->reset(vdev); 658a62a8ef9SStefan Hajnoczi virtio_fs_cleanup_vqs(vdev, fs); 659a62a8ef9SStefan Hajnoczi 660a62a8ef9SStefan Hajnoczi out: 661a62a8ef9SStefan Hajnoczi vdev->priv = NULL; 662a62a8ef9SStefan Hajnoczi kfree(fs); 663a62a8ef9SStefan Hajnoczi return ret; 664a62a8ef9SStefan Hajnoczi } 665a62a8ef9SStefan Hajnoczi 666a62a8ef9SStefan Hajnoczi static void virtio_fs_stop_all_queues(struct virtio_fs *fs) 667a62a8ef9SStefan Hajnoczi { 668a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 669a62a8ef9SStefan Hajnoczi int i; 670a62a8ef9SStefan Hajnoczi 671a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 672a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 673a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 674a62a8ef9SStefan Hajnoczi fsvq->connected = false; 675a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 676a62a8ef9SStefan Hajnoczi } 677a62a8ef9SStefan Hajnoczi } 678a62a8ef9SStefan Hajnoczi 679a62a8ef9SStefan Hajnoczi static void virtio_fs_remove(struct virtio_device *vdev) 680a62a8ef9SStefan Hajnoczi { 681a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vdev->priv; 682a62a8ef9SStefan Hajnoczi 683a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 684a62a8ef9SStefan Hajnoczi /* This device is going away. No one should get new reference */ 685a62a8ef9SStefan Hajnoczi list_del_init(&fs->list); 686a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(fs); 687a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(fs); 688a62a8ef9SStefan Hajnoczi vdev->config->reset(vdev); 689a62a8ef9SStefan Hajnoczi virtio_fs_cleanup_vqs(vdev, fs); 690a62a8ef9SStefan Hajnoczi 691a62a8ef9SStefan Hajnoczi vdev->priv = NULL; 692a62a8ef9SStefan Hajnoczi /* Put device reference on virtio_fs object */ 693a62a8ef9SStefan Hajnoczi virtio_fs_put(fs); 694a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 695a62a8ef9SStefan Hajnoczi } 696a62a8ef9SStefan Hajnoczi 697a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP 698a62a8ef9SStefan Hajnoczi static int virtio_fs_freeze(struct virtio_device *vdev) 699a62a8ef9SStefan Hajnoczi { 700a62a8ef9SStefan Hajnoczi /* TODO need to save state here */ 701a62a8ef9SStefan Hajnoczi pr_warn("virtio-fs: suspend/resume not yet supported\n"); 702a62a8ef9SStefan Hajnoczi return -EOPNOTSUPP; 703a62a8ef9SStefan Hajnoczi } 704a62a8ef9SStefan Hajnoczi 705a62a8ef9SStefan Hajnoczi static int virtio_fs_restore(struct virtio_device *vdev) 706a62a8ef9SStefan Hajnoczi { 707a62a8ef9SStefan Hajnoczi /* TODO need to restore state here */ 708a62a8ef9SStefan Hajnoczi return 0; 709a62a8ef9SStefan Hajnoczi } 710a62a8ef9SStefan Hajnoczi #endif /* CONFIG_PM_SLEEP */ 711a62a8ef9SStefan Hajnoczi 71200929447SYueHaibing static const struct virtio_device_id id_table[] = { 713a62a8ef9SStefan Hajnoczi { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, 714a62a8ef9SStefan Hajnoczi {}, 715a62a8ef9SStefan Hajnoczi }; 716a62a8ef9SStefan Hajnoczi 71700929447SYueHaibing static const unsigned int feature_table[] = {}; 718a62a8ef9SStefan Hajnoczi 719a62a8ef9SStefan Hajnoczi static struct virtio_driver virtio_fs_driver = { 720a62a8ef9SStefan Hajnoczi .driver.name = KBUILD_MODNAME, 721a62a8ef9SStefan Hajnoczi .driver.owner = THIS_MODULE, 722a62a8ef9SStefan Hajnoczi .id_table = id_table, 723a62a8ef9SStefan Hajnoczi .feature_table = feature_table, 724a62a8ef9SStefan Hajnoczi .feature_table_size = ARRAY_SIZE(feature_table), 725a62a8ef9SStefan Hajnoczi .probe = virtio_fs_probe, 726a62a8ef9SStefan Hajnoczi .remove = virtio_fs_remove, 727a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP 728a62a8ef9SStefan Hajnoczi .freeze = virtio_fs_freeze, 729a62a8ef9SStefan Hajnoczi .restore = virtio_fs_restore, 730a62a8ef9SStefan Hajnoczi #endif 731a62a8ef9SStefan Hajnoczi }; 732a62a8ef9SStefan Hajnoczi 733a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq) 734a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 735a62a8ef9SStefan Hajnoczi { 736a62a8ef9SStefan Hajnoczi struct fuse_forget_link *link; 737a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget; 738*1efcf39eSVivek Goyal struct virtio_fs_forget_req *req; 739a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 740a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 741a62a8ef9SStefan Hajnoczi u64 unique; 742a62a8ef9SStefan Hajnoczi 743a62a8ef9SStefan Hajnoczi link = fuse_dequeue_forget(fiq, 1, NULL); 744a62a8ef9SStefan Hajnoczi unique = fuse_get_unique(fiq); 745a62a8ef9SStefan Hajnoczi 746a62a8ef9SStefan Hajnoczi fs = fiq->priv; 747a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[VQ_HIPRIO]; 748a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 749a62a8ef9SStefan Hajnoczi 750a62a8ef9SStefan Hajnoczi /* Allocate a buffer for the request */ 751a62a8ef9SStefan Hajnoczi forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL); 752*1efcf39eSVivek Goyal req = &forget->req; 753a62a8ef9SStefan Hajnoczi 754*1efcf39eSVivek Goyal req->ih = (struct fuse_in_header){ 755a62a8ef9SStefan Hajnoczi .opcode = FUSE_FORGET, 756a62a8ef9SStefan Hajnoczi .nodeid = link->forget_one.nodeid, 757a62a8ef9SStefan Hajnoczi .unique = unique, 758*1efcf39eSVivek Goyal .len = sizeof(*req), 759a62a8ef9SStefan Hajnoczi }; 760*1efcf39eSVivek Goyal req->arg = (struct fuse_forget_in){ 761a62a8ef9SStefan Hajnoczi .nlookup = link->forget_one.nlookup, 762a62a8ef9SStefan Hajnoczi }; 763a62a8ef9SStefan Hajnoczi 76458ada94fSVivek Goyal send_forget_request(fsvq, forget, false); 765a62a8ef9SStefan Hajnoczi kfree(link); 766a62a8ef9SStefan Hajnoczi } 767a62a8ef9SStefan Hajnoczi 768a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq) 769a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 770a62a8ef9SStefan Hajnoczi { 771a62a8ef9SStefan Hajnoczi /* 772a62a8ef9SStefan Hajnoczi * TODO interrupts. 773a62a8ef9SStefan Hajnoczi * 774a62a8ef9SStefan Hajnoczi * Normal fs operations on a local filesystems aren't interruptible. 775a62a8ef9SStefan Hajnoczi * Exceptions are blocking lock operations; for example fcntl(F_SETLKW) 776a62a8ef9SStefan Hajnoczi * with shared lock between host and guest. 777a62a8ef9SStefan Hajnoczi */ 778a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 779a62a8ef9SStefan Hajnoczi } 780a62a8ef9SStefan Hajnoczi 781a62a8ef9SStefan Hajnoczi /* Return the number of scatter-gather list elements required */ 782a62a8ef9SStefan Hajnoczi static unsigned int sg_count_fuse_req(struct fuse_req *req) 783a62a8ef9SStefan Hajnoczi { 784a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 785a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); 786a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 1 /* fuse_in_header */; 787a62a8ef9SStefan Hajnoczi 788a62a8ef9SStefan Hajnoczi if (args->in_numargs - args->in_pages) 789a62a8ef9SStefan Hajnoczi total_sgs += 1; 790a62a8ef9SStefan Hajnoczi 791a62a8ef9SStefan Hajnoczi if (args->in_pages) 792a62a8ef9SStefan Hajnoczi total_sgs += ap->num_pages; 793a62a8ef9SStefan Hajnoczi 794a62a8ef9SStefan Hajnoczi if (!test_bit(FR_ISREPLY, &req->flags)) 795a62a8ef9SStefan Hajnoczi return total_sgs; 796a62a8ef9SStefan Hajnoczi 797a62a8ef9SStefan Hajnoczi total_sgs += 1 /* fuse_out_header */; 798a62a8ef9SStefan Hajnoczi 799a62a8ef9SStefan Hajnoczi if (args->out_numargs - args->out_pages) 800a62a8ef9SStefan Hajnoczi total_sgs += 1; 801a62a8ef9SStefan Hajnoczi 802a62a8ef9SStefan Hajnoczi if (args->out_pages) 803a62a8ef9SStefan Hajnoczi total_sgs += ap->num_pages; 804a62a8ef9SStefan Hajnoczi 805a62a8ef9SStefan Hajnoczi return total_sgs; 806a62a8ef9SStefan Hajnoczi } 807a62a8ef9SStefan Hajnoczi 808a62a8ef9SStefan Hajnoczi /* Add pages to scatter-gather list and return number of elements used */ 809a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_pages(struct scatterlist *sg, 810a62a8ef9SStefan Hajnoczi struct page **pages, 811a62a8ef9SStefan Hajnoczi struct fuse_page_desc *page_descs, 812a62a8ef9SStefan Hajnoczi unsigned int num_pages, 813a62a8ef9SStefan Hajnoczi unsigned int total_len) 814a62a8ef9SStefan Hajnoczi { 815a62a8ef9SStefan Hajnoczi unsigned int i; 816a62a8ef9SStefan Hajnoczi unsigned int this_len; 817a62a8ef9SStefan Hajnoczi 818a62a8ef9SStefan Hajnoczi for (i = 0; i < num_pages && total_len; i++) { 819a62a8ef9SStefan Hajnoczi sg_init_table(&sg[i], 1); 820a62a8ef9SStefan Hajnoczi this_len = min(page_descs[i].length, total_len); 821a62a8ef9SStefan Hajnoczi sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); 822a62a8ef9SStefan Hajnoczi total_len -= this_len; 823a62a8ef9SStefan Hajnoczi } 824a62a8ef9SStefan Hajnoczi 825a62a8ef9SStefan Hajnoczi return i; 826a62a8ef9SStefan Hajnoczi } 827a62a8ef9SStefan Hajnoczi 828a62a8ef9SStefan Hajnoczi /* Add args to scatter-gather list and return number of elements used */ 829a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_args(struct scatterlist *sg, 830a62a8ef9SStefan Hajnoczi struct fuse_req *req, 831a62a8ef9SStefan Hajnoczi struct fuse_arg *args, 832a62a8ef9SStefan Hajnoczi unsigned int numargs, 833a62a8ef9SStefan Hajnoczi bool argpages, 834a62a8ef9SStefan Hajnoczi void *argbuf, 835a62a8ef9SStefan Hajnoczi unsigned int *len_used) 836a62a8ef9SStefan Hajnoczi { 837a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); 838a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 0; 839a62a8ef9SStefan Hajnoczi unsigned int len; 840a62a8ef9SStefan Hajnoczi 841a62a8ef9SStefan Hajnoczi len = fuse_len_args(numargs - argpages, args); 842a62a8ef9SStefan Hajnoczi if (len) 843a62a8ef9SStefan Hajnoczi sg_init_one(&sg[total_sgs++], argbuf, len); 844a62a8ef9SStefan Hajnoczi 845a62a8ef9SStefan Hajnoczi if (argpages) 846a62a8ef9SStefan Hajnoczi total_sgs += sg_init_fuse_pages(&sg[total_sgs], 847a62a8ef9SStefan Hajnoczi ap->pages, ap->descs, 848a62a8ef9SStefan Hajnoczi ap->num_pages, 849a62a8ef9SStefan Hajnoczi args[numargs - 1].size); 850a62a8ef9SStefan Hajnoczi 851a62a8ef9SStefan Hajnoczi if (len_used) 852a62a8ef9SStefan Hajnoczi *len_used = len; 853a62a8ef9SStefan Hajnoczi 854a62a8ef9SStefan Hajnoczi return total_sgs; 855a62a8ef9SStefan Hajnoczi } 856a62a8ef9SStefan Hajnoczi 857a62a8ef9SStefan Hajnoczi /* Add a request to a virtqueue and kick the device */ 858a62a8ef9SStefan Hajnoczi static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 859a9bfd9ddSVivek Goyal struct fuse_req *req, bool in_flight) 860a62a8ef9SStefan Hajnoczi { 861a62a8ef9SStefan Hajnoczi /* requests need at least 4 elements */ 862a62a8ef9SStefan Hajnoczi struct scatterlist *stack_sgs[6]; 863a62a8ef9SStefan Hajnoczi struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)]; 864a62a8ef9SStefan Hajnoczi struct scatterlist **sgs = stack_sgs; 865a62a8ef9SStefan Hajnoczi struct scatterlist *sg = stack_sg; 866a62a8ef9SStefan Hajnoczi struct virtqueue *vq; 867a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 868a62a8ef9SStefan Hajnoczi unsigned int argbuf_used = 0; 869a62a8ef9SStefan Hajnoczi unsigned int out_sgs = 0; 870a62a8ef9SStefan Hajnoczi unsigned int in_sgs = 0; 871a62a8ef9SStefan Hajnoczi unsigned int total_sgs; 872a62a8ef9SStefan Hajnoczi unsigned int i; 873a62a8ef9SStefan Hajnoczi int ret; 874a62a8ef9SStefan Hajnoczi bool notify; 8755dbe190fSVivek Goyal struct fuse_pqueue *fpq; 876a62a8ef9SStefan Hajnoczi 877a62a8ef9SStefan Hajnoczi /* Does the sglist fit on the stack? */ 878a62a8ef9SStefan Hajnoczi total_sgs = sg_count_fuse_req(req); 879a62a8ef9SStefan Hajnoczi if (total_sgs > ARRAY_SIZE(stack_sgs)) { 880a62a8ef9SStefan Hajnoczi sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); 881a62a8ef9SStefan Hajnoczi sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); 882a62a8ef9SStefan Hajnoczi if (!sgs || !sg) { 883a62a8ef9SStefan Hajnoczi ret = -ENOMEM; 884a62a8ef9SStefan Hajnoczi goto out; 885a62a8ef9SStefan Hajnoczi } 886a62a8ef9SStefan Hajnoczi } 887a62a8ef9SStefan Hajnoczi 888a62a8ef9SStefan Hajnoczi /* Use a bounce buffer since stack args cannot be mapped */ 889a62a8ef9SStefan Hajnoczi ret = copy_args_to_argbuf(req); 890a62a8ef9SStefan Hajnoczi if (ret < 0) 891a62a8ef9SStefan Hajnoczi goto out; 892a62a8ef9SStefan Hajnoczi 893a62a8ef9SStefan Hajnoczi /* Request elements */ 894a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); 895a62a8ef9SStefan Hajnoczi out_sgs += sg_init_fuse_args(&sg[out_sgs], req, 896a62a8ef9SStefan Hajnoczi (struct fuse_arg *)args->in_args, 897a62a8ef9SStefan Hajnoczi args->in_numargs, args->in_pages, 898a62a8ef9SStefan Hajnoczi req->argbuf, &argbuf_used); 899a62a8ef9SStefan Hajnoczi 900a62a8ef9SStefan Hajnoczi /* Reply elements */ 901a62a8ef9SStefan Hajnoczi if (test_bit(FR_ISREPLY, &req->flags)) { 902a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs + in_sgs++], 903a62a8ef9SStefan Hajnoczi &req->out.h, sizeof(req->out.h)); 904a62a8ef9SStefan Hajnoczi in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req, 905a62a8ef9SStefan Hajnoczi args->out_args, args->out_numargs, 906a62a8ef9SStefan Hajnoczi args->out_pages, 907a62a8ef9SStefan Hajnoczi req->argbuf + argbuf_used, NULL); 908a62a8ef9SStefan Hajnoczi } 909a62a8ef9SStefan Hajnoczi 910a62a8ef9SStefan Hajnoczi WARN_ON(out_sgs + in_sgs != total_sgs); 911a62a8ef9SStefan Hajnoczi 912a62a8ef9SStefan Hajnoczi for (i = 0; i < total_sgs; i++) 913a62a8ef9SStefan Hajnoczi sgs[i] = &sg[i]; 914a62a8ef9SStefan Hajnoczi 915a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 916a62a8ef9SStefan Hajnoczi 917a62a8ef9SStefan Hajnoczi if (!fsvq->connected) { 918a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 919a62a8ef9SStefan Hajnoczi ret = -ENOTCONN; 920a62a8ef9SStefan Hajnoczi goto out; 921a62a8ef9SStefan Hajnoczi } 922a62a8ef9SStefan Hajnoczi 923a62a8ef9SStefan Hajnoczi vq = fsvq->vq; 924a62a8ef9SStefan Hajnoczi ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC); 925a62a8ef9SStefan Hajnoczi if (ret < 0) { 926a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 927a62a8ef9SStefan Hajnoczi goto out; 928a62a8ef9SStefan Hajnoczi } 929a62a8ef9SStefan Hajnoczi 9305dbe190fSVivek Goyal /* Request successfully sent. */ 9315dbe190fSVivek Goyal fpq = &fsvq->fud->pq; 9325dbe190fSVivek Goyal spin_lock(&fpq->lock); 9335dbe190fSVivek Goyal list_add_tail(&req->list, fpq->processing); 9345dbe190fSVivek Goyal spin_unlock(&fpq->lock); 9355dbe190fSVivek Goyal set_bit(FR_SENT, &req->flags); 9365dbe190fSVivek Goyal /* matches barrier in request_wait_answer() */ 9375dbe190fSVivek Goyal smp_mb__after_atomic(); 9385dbe190fSVivek Goyal 939a9bfd9ddSVivek Goyal if (!in_flight) 940c17ea009SVivek Goyal inc_in_flight_req(fsvq); 941a62a8ef9SStefan Hajnoczi notify = virtqueue_kick_prepare(vq); 942a62a8ef9SStefan Hajnoczi 943a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 944a62a8ef9SStefan Hajnoczi 945a62a8ef9SStefan Hajnoczi if (notify) 946a62a8ef9SStefan Hajnoczi virtqueue_notify(vq); 947a62a8ef9SStefan Hajnoczi 948a62a8ef9SStefan Hajnoczi out: 949a62a8ef9SStefan Hajnoczi if (ret < 0 && req->argbuf) { 950a62a8ef9SStefan Hajnoczi kfree(req->argbuf); 951a62a8ef9SStefan Hajnoczi req->argbuf = NULL; 952a62a8ef9SStefan Hajnoczi } 953a62a8ef9SStefan Hajnoczi if (sgs != stack_sgs) { 954a62a8ef9SStefan Hajnoczi kfree(sgs); 955a62a8ef9SStefan Hajnoczi kfree(sg); 956a62a8ef9SStefan Hajnoczi } 957a62a8ef9SStefan Hajnoczi 958a62a8ef9SStefan Hajnoczi return ret; 959a62a8ef9SStefan Hajnoczi } 960a62a8ef9SStefan Hajnoczi 961a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq) 962a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 963a62a8ef9SStefan Hajnoczi { 964a62a8ef9SStefan Hajnoczi unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */ 965a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 966a62a8ef9SStefan Hajnoczi struct fuse_req *req; 96751fecdd2SVivek Goyal struct virtio_fs_vq *fsvq; 968a62a8ef9SStefan Hajnoczi int ret; 969a62a8ef9SStefan Hajnoczi 970a62a8ef9SStefan Hajnoczi WARN_ON(list_empty(&fiq->pending)); 971a62a8ef9SStefan Hajnoczi req = list_last_entry(&fiq->pending, struct fuse_req, list); 972a62a8ef9SStefan Hajnoczi clear_bit(FR_PENDING, &req->flags); 973a62a8ef9SStefan Hajnoczi list_del_init(&req->list); 974a62a8ef9SStefan Hajnoczi WARN_ON(!list_empty(&fiq->pending)); 975a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 976a62a8ef9SStefan Hajnoczi 977a62a8ef9SStefan Hajnoczi fs = fiq->priv; 978a62a8ef9SStefan Hajnoczi 979a62a8ef9SStefan Hajnoczi pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n", 980a62a8ef9SStefan Hajnoczi __func__, req->in.h.opcode, req->in.h.unique, 981a62a8ef9SStefan Hajnoczi req->in.h.nodeid, req->in.h.len, 982a62a8ef9SStefan Hajnoczi fuse_len_args(req->args->out_numargs, req->args->out_args)); 983a62a8ef9SStefan Hajnoczi 98451fecdd2SVivek Goyal fsvq = &fs->vqs[queue_id]; 985a9bfd9ddSVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req, false); 986a62a8ef9SStefan Hajnoczi if (ret < 0) { 987a62a8ef9SStefan Hajnoczi if (ret == -ENOMEM || ret == -ENOSPC) { 988a9bfd9ddSVivek Goyal /* 989a9bfd9ddSVivek Goyal * Virtqueue full. Retry submission from worker 990a9bfd9ddSVivek Goyal * context as we might be holding fc->bg_lock. 991a9bfd9ddSVivek Goyal */ 992a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 993a9bfd9ddSVivek Goyal list_add_tail(&req->list, &fsvq->queued_reqs); 994a9bfd9ddSVivek Goyal inc_in_flight_req(fsvq); 995a9bfd9ddSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 996a9bfd9ddSVivek Goyal msecs_to_jiffies(1)); 997a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 998a9bfd9ddSVivek Goyal return; 999a62a8ef9SStefan Hajnoczi } 1000a62a8ef9SStefan Hajnoczi req->out.h.error = ret; 1001a62a8ef9SStefan Hajnoczi pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); 100251fecdd2SVivek Goyal 100351fecdd2SVivek Goyal /* Can't end request in submission context. Use a worker */ 100451fecdd2SVivek Goyal spin_lock(&fsvq->lock); 100551fecdd2SVivek Goyal list_add_tail(&req->list, &fsvq->end_reqs); 100651fecdd2SVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 0); 100751fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 1008a62a8ef9SStefan Hajnoczi return; 1009a62a8ef9SStefan Hajnoczi } 1010a62a8ef9SStefan Hajnoczi } 1011a62a8ef9SStefan Hajnoczi 101200929447SYueHaibing static const struct fuse_iqueue_ops virtio_fs_fiq_ops = { 1013a62a8ef9SStefan Hajnoczi .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock, 1014a62a8ef9SStefan Hajnoczi .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock, 1015a62a8ef9SStefan Hajnoczi .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock, 1016a62a8ef9SStefan Hajnoczi .release = virtio_fs_fiq_release, 1017a62a8ef9SStefan Hajnoczi }; 1018a62a8ef9SStefan Hajnoczi 1019a62a8ef9SStefan Hajnoczi static int virtio_fs_fill_super(struct super_block *sb) 1020a62a8ef9SStefan Hajnoczi { 1021a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = get_fuse_conn_super(sb); 1022a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = fc->iq.priv; 1023a62a8ef9SStefan Hajnoczi unsigned int i; 1024a62a8ef9SStefan Hajnoczi int err; 1025a62a8ef9SStefan Hajnoczi struct fuse_fs_context ctx = { 1026a62a8ef9SStefan Hajnoczi .rootmode = S_IFDIR, 1027a62a8ef9SStefan Hajnoczi .default_permissions = 1, 1028a62a8ef9SStefan Hajnoczi .allow_other = 1, 1029a62a8ef9SStefan Hajnoczi .max_read = UINT_MAX, 1030a62a8ef9SStefan Hajnoczi .blksize = 512, 1031a62a8ef9SStefan Hajnoczi .destroy = true, 1032a62a8ef9SStefan Hajnoczi .no_control = true, 1033a62a8ef9SStefan Hajnoczi .no_force_umount = true, 10343f22c746SMiklos Szeredi .no_mount_options = true, 1035a62a8ef9SStefan Hajnoczi }; 1036a62a8ef9SStefan Hajnoczi 1037a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 1038a62a8ef9SStefan Hajnoczi 1039a62a8ef9SStefan Hajnoczi /* After holding mutex, make sure virtiofs device is still there. 1040a62a8ef9SStefan Hajnoczi * Though we are holding a reference to it, drive ->remove might 1041a62a8ef9SStefan Hajnoczi * still have cleaned up virtual queues. In that case bail out. 1042a62a8ef9SStefan Hajnoczi */ 1043a62a8ef9SStefan Hajnoczi err = -EINVAL; 1044a62a8ef9SStefan Hajnoczi if (list_empty(&fs->list)) { 1045a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fs->tag); 1046a62a8ef9SStefan Hajnoczi goto err; 1047a62a8ef9SStefan Hajnoczi } 1048a62a8ef9SStefan Hajnoczi 1049a62a8ef9SStefan Hajnoczi err = -ENOMEM; 1050a62a8ef9SStefan Hajnoczi /* Allocate fuse_dev for hiprio and notification queues */ 1051a62a8ef9SStefan Hajnoczi for (i = 0; i < VQ_REQUEST; i++) { 1052a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1053a62a8ef9SStefan Hajnoczi 1054a62a8ef9SStefan Hajnoczi fsvq->fud = fuse_dev_alloc(); 1055a62a8ef9SStefan Hajnoczi if (!fsvq->fud) 1056a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs; 1057a62a8ef9SStefan Hajnoczi } 1058a62a8ef9SStefan Hajnoczi 1059a62a8ef9SStefan Hajnoczi ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud; 1060a62a8ef9SStefan Hajnoczi err = fuse_fill_super_common(sb, &ctx); 1061a62a8ef9SStefan Hajnoczi if (err < 0) 1062a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs; 1063a62a8ef9SStefan Hajnoczi 1064a62a8ef9SStefan Hajnoczi fc = fs->vqs[VQ_REQUEST].fud->fc; 1065a62a8ef9SStefan Hajnoczi 1066a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 1067a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1068a62a8ef9SStefan Hajnoczi 1069a62a8ef9SStefan Hajnoczi if (i == VQ_REQUEST) 1070a62a8ef9SStefan Hajnoczi continue; /* already initialized */ 1071a62a8ef9SStefan Hajnoczi fuse_dev_install(fsvq->fud, fc); 1072a62a8ef9SStefan Hajnoczi } 1073a62a8ef9SStefan Hajnoczi 1074a62a8ef9SStefan Hajnoczi /* Previous unmount will stop all queues. Start these again */ 1075a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs); 1076a62a8ef9SStefan Hajnoczi fuse_send_init(fc); 1077a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1078a62a8ef9SStefan Hajnoczi return 0; 1079a62a8ef9SStefan Hajnoczi 1080a62a8ef9SStefan Hajnoczi err_free_fuse_devs: 1081a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(fs); 1082a62a8ef9SStefan Hajnoczi err: 1083a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1084a62a8ef9SStefan Hajnoczi return err; 1085a62a8ef9SStefan Hajnoczi } 1086a62a8ef9SStefan Hajnoczi 1087a62a8ef9SStefan Hajnoczi static void virtio_kill_sb(struct super_block *sb) 1088a62a8ef9SStefan Hajnoczi { 1089a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = get_fuse_conn_super(sb); 1090a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs; 1091a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 1092a62a8ef9SStefan Hajnoczi 1093a62a8ef9SStefan Hajnoczi /* If mount failed, we can still be called without any fc */ 1094a62a8ef9SStefan Hajnoczi if (!fc) 1095a62a8ef9SStefan Hajnoczi return fuse_kill_sb_anon(sb); 1096a62a8ef9SStefan Hajnoczi 1097a62a8ef9SStefan Hajnoczi vfs = fc->iq.priv; 1098a62a8ef9SStefan Hajnoczi fsvq = &vfs->vqs[VQ_HIPRIO]; 1099a62a8ef9SStefan Hajnoczi 1100a62a8ef9SStefan Hajnoczi /* Stop forget queue. Soon destroy will be sent */ 1101a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 1102a62a8ef9SStefan Hajnoczi fsvq->connected = false; 1103a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 1104a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs); 1105a62a8ef9SStefan Hajnoczi 1106a62a8ef9SStefan Hajnoczi fuse_kill_sb_anon(sb); 1107a62a8ef9SStefan Hajnoczi 1108a62a8ef9SStefan Hajnoczi /* fuse_kill_sb_anon() must have sent destroy. Stop all queues 1109a62a8ef9SStefan Hajnoczi * and drain one more time and free fuse devices. Freeing fuse 1110a62a8ef9SStefan Hajnoczi * devices will drop their reference on fuse_conn and that in 1111a62a8ef9SStefan Hajnoczi * turn will drop its reference on virtio_fs object. 1112a62a8ef9SStefan Hajnoczi */ 1113a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(vfs); 1114a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs); 1115a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(vfs); 1116a62a8ef9SStefan Hajnoczi } 1117a62a8ef9SStefan Hajnoczi 1118a62a8ef9SStefan Hajnoczi static int virtio_fs_test_super(struct super_block *sb, 1119a62a8ef9SStefan Hajnoczi struct fs_context *fsc) 1120a62a8ef9SStefan Hajnoczi { 1121a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = fsc->s_fs_info; 1122a62a8ef9SStefan Hajnoczi 1123a62a8ef9SStefan Hajnoczi return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv; 1124a62a8ef9SStefan Hajnoczi } 1125a62a8ef9SStefan Hajnoczi 1126a62a8ef9SStefan Hajnoczi static int virtio_fs_set_super(struct super_block *sb, 1127a62a8ef9SStefan Hajnoczi struct fs_context *fsc) 1128a62a8ef9SStefan Hajnoczi { 1129a62a8ef9SStefan Hajnoczi int err; 1130a62a8ef9SStefan Hajnoczi 1131a62a8ef9SStefan Hajnoczi err = get_anon_bdev(&sb->s_dev); 1132a62a8ef9SStefan Hajnoczi if (!err) 1133a62a8ef9SStefan Hajnoczi fuse_conn_get(fsc->s_fs_info); 1134a62a8ef9SStefan Hajnoczi 1135a62a8ef9SStefan Hajnoczi return err; 1136a62a8ef9SStefan Hajnoczi } 1137a62a8ef9SStefan Hajnoczi 1138a62a8ef9SStefan Hajnoczi static int virtio_fs_get_tree(struct fs_context *fsc) 1139a62a8ef9SStefan Hajnoczi { 1140a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 1141a62a8ef9SStefan Hajnoczi struct super_block *sb; 1142a62a8ef9SStefan Hajnoczi struct fuse_conn *fc; 1143a62a8ef9SStefan Hajnoczi int err; 1144a62a8ef9SStefan Hajnoczi 1145a62a8ef9SStefan Hajnoczi /* This gets a reference on virtio_fs object. This ptr gets installed 1146a62a8ef9SStefan Hajnoczi * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() 1147a62a8ef9SStefan Hajnoczi * to drop the reference to this object. 1148a62a8ef9SStefan Hajnoczi */ 1149a62a8ef9SStefan Hajnoczi fs = virtio_fs_find_instance(fsc->source); 1150a62a8ef9SStefan Hajnoczi if (!fs) { 1151a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fsc->source); 1152a62a8ef9SStefan Hajnoczi return -EINVAL; 1153a62a8ef9SStefan Hajnoczi } 1154a62a8ef9SStefan Hajnoczi 1155a62a8ef9SStefan Hajnoczi fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); 1156a62a8ef9SStefan Hajnoczi if (!fc) { 1157a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 1158a62a8ef9SStefan Hajnoczi virtio_fs_put(fs); 1159a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1160a62a8ef9SStefan Hajnoczi return -ENOMEM; 1161a62a8ef9SStefan Hajnoczi } 1162a62a8ef9SStefan Hajnoczi 1163a62a8ef9SStefan Hajnoczi fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops, 1164a62a8ef9SStefan Hajnoczi fs); 1165a62a8ef9SStefan Hajnoczi fc->release = fuse_free_conn; 1166a62a8ef9SStefan Hajnoczi fc->delete_stale = true; 1167a62a8ef9SStefan Hajnoczi 1168a62a8ef9SStefan Hajnoczi fsc->s_fs_info = fc; 1169a62a8ef9SStefan Hajnoczi sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super); 1170a62a8ef9SStefan Hajnoczi fuse_conn_put(fc); 1171a62a8ef9SStefan Hajnoczi if (IS_ERR(sb)) 1172a62a8ef9SStefan Hajnoczi return PTR_ERR(sb); 1173a62a8ef9SStefan Hajnoczi 1174a62a8ef9SStefan Hajnoczi if (!sb->s_root) { 1175a62a8ef9SStefan Hajnoczi err = virtio_fs_fill_super(sb); 1176a62a8ef9SStefan Hajnoczi if (err) { 1177a62a8ef9SStefan Hajnoczi deactivate_locked_super(sb); 1178a62a8ef9SStefan Hajnoczi return err; 1179a62a8ef9SStefan Hajnoczi } 1180a62a8ef9SStefan Hajnoczi 1181a62a8ef9SStefan Hajnoczi sb->s_flags |= SB_ACTIVE; 1182a62a8ef9SStefan Hajnoczi } 1183a62a8ef9SStefan Hajnoczi 1184a62a8ef9SStefan Hajnoczi WARN_ON(fsc->root); 1185a62a8ef9SStefan Hajnoczi fsc->root = dget(sb->s_root); 1186a62a8ef9SStefan Hajnoczi return 0; 1187a62a8ef9SStefan Hajnoczi } 1188a62a8ef9SStefan Hajnoczi 1189a62a8ef9SStefan Hajnoczi static const struct fs_context_operations virtio_fs_context_ops = { 1190a62a8ef9SStefan Hajnoczi .get_tree = virtio_fs_get_tree, 1191a62a8ef9SStefan Hajnoczi }; 1192a62a8ef9SStefan Hajnoczi 1193a62a8ef9SStefan Hajnoczi static int virtio_fs_init_fs_context(struct fs_context *fsc) 1194a62a8ef9SStefan Hajnoczi { 1195a62a8ef9SStefan Hajnoczi fsc->ops = &virtio_fs_context_ops; 1196a62a8ef9SStefan Hajnoczi return 0; 1197a62a8ef9SStefan Hajnoczi } 1198a62a8ef9SStefan Hajnoczi 1199a62a8ef9SStefan Hajnoczi static struct file_system_type virtio_fs_type = { 1200a62a8ef9SStefan Hajnoczi .owner = THIS_MODULE, 1201a62a8ef9SStefan Hajnoczi .name = "virtiofs", 1202a62a8ef9SStefan Hajnoczi .init_fs_context = virtio_fs_init_fs_context, 1203a62a8ef9SStefan Hajnoczi .kill_sb = virtio_kill_sb, 1204a62a8ef9SStefan Hajnoczi }; 1205a62a8ef9SStefan Hajnoczi 1206a62a8ef9SStefan Hajnoczi static int __init virtio_fs_init(void) 1207a62a8ef9SStefan Hajnoczi { 1208a62a8ef9SStefan Hajnoczi int ret; 1209a62a8ef9SStefan Hajnoczi 1210a62a8ef9SStefan Hajnoczi ret = register_virtio_driver(&virtio_fs_driver); 1211a62a8ef9SStefan Hajnoczi if (ret < 0) 1212a62a8ef9SStefan Hajnoczi return ret; 1213a62a8ef9SStefan Hajnoczi 1214a62a8ef9SStefan Hajnoczi ret = register_filesystem(&virtio_fs_type); 1215a62a8ef9SStefan Hajnoczi if (ret < 0) { 1216a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver); 1217a62a8ef9SStefan Hajnoczi return ret; 1218a62a8ef9SStefan Hajnoczi } 1219a62a8ef9SStefan Hajnoczi 1220a62a8ef9SStefan Hajnoczi return 0; 1221a62a8ef9SStefan Hajnoczi } 1222a62a8ef9SStefan Hajnoczi module_init(virtio_fs_init); 1223a62a8ef9SStefan Hajnoczi 1224a62a8ef9SStefan Hajnoczi static void __exit virtio_fs_exit(void) 1225a62a8ef9SStefan Hajnoczi { 1226a62a8ef9SStefan Hajnoczi unregister_filesystem(&virtio_fs_type); 1227a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver); 1228a62a8ef9SStefan Hajnoczi } 1229a62a8ef9SStefan Hajnoczi module_exit(virtio_fs_exit); 1230a62a8ef9SStefan Hajnoczi 1231a62a8ef9SStefan Hajnoczi MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>"); 1232a62a8ef9SStefan Hajnoczi MODULE_DESCRIPTION("Virtio Filesystem"); 1233a62a8ef9SStefan Hajnoczi MODULE_LICENSE("GPL"); 1234a62a8ef9SStefan Hajnoczi MODULE_ALIAS_FS(KBUILD_MODNAME); 1235a62a8ef9SStefan Hajnoczi MODULE_DEVICE_TABLE(virtio, id_table); 1236