1a62a8ef9SStefan Hajnoczi // SPDX-License-Identifier: GPL-2.0 2a62a8ef9SStefan Hajnoczi /* 3a62a8ef9SStefan Hajnoczi * virtio-fs: Virtio Filesystem 4a62a8ef9SStefan Hajnoczi * Copyright (C) 2018 Red Hat, Inc. 5a62a8ef9SStefan Hajnoczi */ 6a62a8ef9SStefan Hajnoczi 7a62a8ef9SStefan Hajnoczi #include <linux/fs.h> 8a62a8ef9SStefan Hajnoczi #include <linux/module.h> 9a62a8ef9SStefan Hajnoczi #include <linux/virtio.h> 10a62a8ef9SStefan Hajnoczi #include <linux/virtio_fs.h> 11a62a8ef9SStefan Hajnoczi #include <linux/delay.h> 12a62a8ef9SStefan Hajnoczi #include <linux/fs_context.h> 13a62a8ef9SStefan Hajnoczi #include <linux/highmem.h> 14a62a8ef9SStefan Hajnoczi #include "fuse_i.h" 15a62a8ef9SStefan Hajnoczi 16a62a8ef9SStefan Hajnoczi /* List of virtio-fs device instances and a lock for the list. Also provides 17a62a8ef9SStefan Hajnoczi * mutual exclusion in device removal and mounting path 18a62a8ef9SStefan Hajnoczi */ 19a62a8ef9SStefan Hajnoczi static DEFINE_MUTEX(virtio_fs_mutex); 20a62a8ef9SStefan Hajnoczi static LIST_HEAD(virtio_fs_instances); 21a62a8ef9SStefan Hajnoczi 22a62a8ef9SStefan Hajnoczi enum { 23a62a8ef9SStefan Hajnoczi VQ_HIPRIO, 24a62a8ef9SStefan Hajnoczi VQ_REQUEST 25a62a8ef9SStefan Hajnoczi }; 26a62a8ef9SStefan Hajnoczi 27*b43b7e81SVivek Goyal #define VQ_NAME_LEN 24 28*b43b7e81SVivek Goyal 29a62a8ef9SStefan Hajnoczi /* Per-virtqueue state */ 30a62a8ef9SStefan Hajnoczi struct virtio_fs_vq { 31a62a8ef9SStefan Hajnoczi spinlock_t lock; 32a62a8ef9SStefan Hajnoczi struct virtqueue *vq; /* protected by ->lock */ 33a62a8ef9SStefan Hajnoczi struct work_struct done_work; 34a62a8ef9SStefan Hajnoczi struct list_head queued_reqs; 3551fecdd2SVivek Goyal struct list_head end_reqs; /* End these requests */ 36a62a8ef9SStefan Hajnoczi struct delayed_work dispatch_work; 37a62a8ef9SStefan Hajnoczi struct fuse_dev *fud; 38a62a8ef9SStefan Hajnoczi bool connected; 39a62a8ef9SStefan Hajnoczi long in_flight; 40724c15a4SVivek Goyal struct completion in_flight_zero; /* No inflight requests */ 41*b43b7e81SVivek Goyal char name[VQ_NAME_LEN]; 42a62a8ef9SStefan Hajnoczi } ____cacheline_aligned_in_smp; 43a62a8ef9SStefan Hajnoczi 44a62a8ef9SStefan Hajnoczi /* A virtio-fs device instance */ 45a62a8ef9SStefan Hajnoczi struct virtio_fs { 46a62a8ef9SStefan Hajnoczi struct kref refcount; 47a62a8ef9SStefan Hajnoczi struct list_head list; /* on virtio_fs_instances */ 48a62a8ef9SStefan Hajnoczi char *tag; 49a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *vqs; 50a62a8ef9SStefan Hajnoczi unsigned int nvqs; /* number of virtqueues */ 51a62a8ef9SStefan Hajnoczi unsigned int num_request_queues; /* number of request queues */ 52a62a8ef9SStefan Hajnoczi }; 53a62a8ef9SStefan Hajnoczi 541efcf39eSVivek Goyal struct virtio_fs_forget_req { 55a62a8ef9SStefan Hajnoczi struct fuse_in_header ih; 56a62a8ef9SStefan Hajnoczi struct fuse_forget_in arg; 571efcf39eSVivek Goyal }; 581efcf39eSVivek Goyal 591efcf39eSVivek Goyal struct virtio_fs_forget { 60a62a8ef9SStefan Hajnoczi /* This request can be temporarily queued on virt queue */ 61a62a8ef9SStefan Hajnoczi struct list_head list; 621efcf39eSVivek Goyal struct virtio_fs_forget_req req; 63a62a8ef9SStefan Hajnoczi }; 64a62a8ef9SStefan Hajnoczi 65bb737bbeSVivek Goyal struct virtio_fs_req_work { 66bb737bbeSVivek Goyal struct fuse_req *req; 67bb737bbeSVivek Goyal struct virtio_fs_vq *fsvq; 68bb737bbeSVivek Goyal struct work_struct done_work; 69bb737bbeSVivek Goyal }; 70bb737bbeSVivek Goyal 71a9bfd9ddSVivek Goyal static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 72a9bfd9ddSVivek Goyal struct fuse_req *req, bool in_flight); 73a9bfd9ddSVivek Goyal 74a62a8ef9SStefan Hajnoczi static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) 75a62a8ef9SStefan Hajnoczi { 76a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vq->vdev->priv; 77a62a8ef9SStefan Hajnoczi 78a62a8ef9SStefan Hajnoczi return &fs->vqs[vq->index]; 79a62a8ef9SStefan Hajnoczi } 80a62a8ef9SStefan Hajnoczi 81a62a8ef9SStefan Hajnoczi static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq) 82a62a8ef9SStefan Hajnoczi { 83a62a8ef9SStefan Hajnoczi return &vq_to_fsvq(vq)->fud->pq; 84a62a8ef9SStefan Hajnoczi } 85a62a8ef9SStefan Hajnoczi 86c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */ 87c17ea009SVivek Goyal static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq) 88c17ea009SVivek Goyal { 89c17ea009SVivek Goyal fsvq->in_flight++; 90c17ea009SVivek Goyal } 91c17ea009SVivek Goyal 92c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */ 93c17ea009SVivek Goyal static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) 94c17ea009SVivek Goyal { 95c17ea009SVivek Goyal WARN_ON(fsvq->in_flight <= 0); 96c17ea009SVivek Goyal fsvq->in_flight--; 97724c15a4SVivek Goyal if (!fsvq->in_flight) 98724c15a4SVivek Goyal complete(&fsvq->in_flight_zero); 99c17ea009SVivek Goyal } 100c17ea009SVivek Goyal 101a62a8ef9SStefan Hajnoczi static void release_virtio_fs_obj(struct kref *ref) 102a62a8ef9SStefan Hajnoczi { 103a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount); 104a62a8ef9SStefan Hajnoczi 105a62a8ef9SStefan Hajnoczi kfree(vfs->vqs); 106a62a8ef9SStefan Hajnoczi kfree(vfs); 107a62a8ef9SStefan Hajnoczi } 108a62a8ef9SStefan Hajnoczi 109a62a8ef9SStefan Hajnoczi /* Make sure virtiofs_mutex is held */ 110a62a8ef9SStefan Hajnoczi static void virtio_fs_put(struct virtio_fs *fs) 111a62a8ef9SStefan Hajnoczi { 112a62a8ef9SStefan Hajnoczi kref_put(&fs->refcount, release_virtio_fs_obj); 113a62a8ef9SStefan Hajnoczi } 114a62a8ef9SStefan Hajnoczi 115a62a8ef9SStefan Hajnoczi static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) 116a62a8ef9SStefan Hajnoczi { 117a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = fiq->priv; 118a62a8ef9SStefan Hajnoczi 119a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 120a62a8ef9SStefan Hajnoczi virtio_fs_put(vfs); 121a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 122a62a8ef9SStefan Hajnoczi } 123a62a8ef9SStefan Hajnoczi 124a62a8ef9SStefan Hajnoczi static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) 125a62a8ef9SStefan Hajnoczi { 126a62a8ef9SStefan Hajnoczi WARN_ON(fsvq->in_flight < 0); 127a62a8ef9SStefan Hajnoczi 128a62a8ef9SStefan Hajnoczi /* Wait for in flight requests to finish.*/ 129a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 130724c15a4SVivek Goyal if (fsvq->in_flight) { 131724c15a4SVivek Goyal /* We are holding virtio_fs_mutex. There should not be any 132724c15a4SVivek Goyal * waiters waiting for completion. 133724c15a4SVivek Goyal */ 134724c15a4SVivek Goyal reinit_completion(&fsvq->in_flight_zero); 135a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 136724c15a4SVivek Goyal wait_for_completion(&fsvq->in_flight_zero); 137724c15a4SVivek Goyal } else { 138a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 139a62a8ef9SStefan Hajnoczi } 140a62a8ef9SStefan Hajnoczi 141a62a8ef9SStefan Hajnoczi flush_work(&fsvq->done_work); 142a62a8ef9SStefan Hajnoczi flush_delayed_work(&fsvq->dispatch_work); 143a62a8ef9SStefan Hajnoczi } 144a62a8ef9SStefan Hajnoczi 145724c15a4SVivek Goyal static void virtio_fs_drain_all_queues_locked(struct virtio_fs *fs) 146a62a8ef9SStefan Hajnoczi { 147a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 148a62a8ef9SStefan Hajnoczi int i; 149a62a8ef9SStefan Hajnoczi 150a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 151a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 152a62a8ef9SStefan Hajnoczi virtio_fs_drain_queue(fsvq); 153a62a8ef9SStefan Hajnoczi } 154a62a8ef9SStefan Hajnoczi } 155a62a8ef9SStefan Hajnoczi 156724c15a4SVivek Goyal static void virtio_fs_drain_all_queues(struct virtio_fs *fs) 157724c15a4SVivek Goyal { 158724c15a4SVivek Goyal /* Provides mutual exclusion between ->remove and ->kill_sb 159724c15a4SVivek Goyal * paths. We don't want both of these draining queue at the 160724c15a4SVivek Goyal * same time. Current completion logic reinits completion 161724c15a4SVivek Goyal * and that means there should not be any other thread 162724c15a4SVivek Goyal * doing reinit or waiting for completion already. 163724c15a4SVivek Goyal */ 164724c15a4SVivek Goyal mutex_lock(&virtio_fs_mutex); 165724c15a4SVivek Goyal virtio_fs_drain_all_queues_locked(fs); 166724c15a4SVivek Goyal mutex_unlock(&virtio_fs_mutex); 167724c15a4SVivek Goyal } 168724c15a4SVivek Goyal 169a62a8ef9SStefan Hajnoczi static void virtio_fs_start_all_queues(struct virtio_fs *fs) 170a62a8ef9SStefan Hajnoczi { 171a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 172a62a8ef9SStefan Hajnoczi int i; 173a62a8ef9SStefan Hajnoczi 174a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 175a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 176a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 177a62a8ef9SStefan Hajnoczi fsvq->connected = true; 178a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 179a62a8ef9SStefan Hajnoczi } 180a62a8ef9SStefan Hajnoczi } 181a62a8ef9SStefan Hajnoczi 182a62a8ef9SStefan Hajnoczi /* Add a new instance to the list or return -EEXIST if tag name exists*/ 183a62a8ef9SStefan Hajnoczi static int virtio_fs_add_instance(struct virtio_fs *fs) 184a62a8ef9SStefan Hajnoczi { 185a62a8ef9SStefan Hajnoczi struct virtio_fs *fs2; 186a62a8ef9SStefan Hajnoczi bool duplicate = false; 187a62a8ef9SStefan Hajnoczi 188a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 189a62a8ef9SStefan Hajnoczi 190a62a8ef9SStefan Hajnoczi list_for_each_entry(fs2, &virtio_fs_instances, list) { 191a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, fs2->tag) == 0) 192a62a8ef9SStefan Hajnoczi duplicate = true; 193a62a8ef9SStefan Hajnoczi } 194a62a8ef9SStefan Hajnoczi 195a62a8ef9SStefan Hajnoczi if (!duplicate) 196a62a8ef9SStefan Hajnoczi list_add_tail(&fs->list, &virtio_fs_instances); 197a62a8ef9SStefan Hajnoczi 198a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 199a62a8ef9SStefan Hajnoczi 200a62a8ef9SStefan Hajnoczi if (duplicate) 201a62a8ef9SStefan Hajnoczi return -EEXIST; 202a62a8ef9SStefan Hajnoczi return 0; 203a62a8ef9SStefan Hajnoczi } 204a62a8ef9SStefan Hajnoczi 205a62a8ef9SStefan Hajnoczi /* Return the virtio_fs with a given tag, or NULL */ 206a62a8ef9SStefan Hajnoczi static struct virtio_fs *virtio_fs_find_instance(const char *tag) 207a62a8ef9SStefan Hajnoczi { 208a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 209a62a8ef9SStefan Hajnoczi 210a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 211a62a8ef9SStefan Hajnoczi 212a62a8ef9SStefan Hajnoczi list_for_each_entry(fs, &virtio_fs_instances, list) { 213a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, tag) == 0) { 214a62a8ef9SStefan Hajnoczi kref_get(&fs->refcount); 215a62a8ef9SStefan Hajnoczi goto found; 216a62a8ef9SStefan Hajnoczi } 217a62a8ef9SStefan Hajnoczi } 218a62a8ef9SStefan Hajnoczi 219a62a8ef9SStefan Hajnoczi fs = NULL; /* not found */ 220a62a8ef9SStefan Hajnoczi 221a62a8ef9SStefan Hajnoczi found: 222a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 223a62a8ef9SStefan Hajnoczi 224a62a8ef9SStefan Hajnoczi return fs; 225a62a8ef9SStefan Hajnoczi } 226a62a8ef9SStefan Hajnoczi 227a62a8ef9SStefan Hajnoczi static void virtio_fs_free_devs(struct virtio_fs *fs) 228a62a8ef9SStefan Hajnoczi { 229a62a8ef9SStefan Hajnoczi unsigned int i; 230a62a8ef9SStefan Hajnoczi 231a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 232a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 233a62a8ef9SStefan Hajnoczi 234a62a8ef9SStefan Hajnoczi if (!fsvq->fud) 235a62a8ef9SStefan Hajnoczi continue; 236a62a8ef9SStefan Hajnoczi 237a62a8ef9SStefan Hajnoczi fuse_dev_free(fsvq->fud); 238a62a8ef9SStefan Hajnoczi fsvq->fud = NULL; 239a62a8ef9SStefan Hajnoczi } 240a62a8ef9SStefan Hajnoczi } 241a62a8ef9SStefan Hajnoczi 242a62a8ef9SStefan Hajnoczi /* Read filesystem name from virtio config into fs->tag (must kfree()). */ 243a62a8ef9SStefan Hajnoczi static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs) 244a62a8ef9SStefan Hajnoczi { 245a62a8ef9SStefan Hajnoczi char tag_buf[sizeof_field(struct virtio_fs_config, tag)]; 246a62a8ef9SStefan Hajnoczi char *end; 247a62a8ef9SStefan Hajnoczi size_t len; 248a62a8ef9SStefan Hajnoczi 249a62a8ef9SStefan Hajnoczi virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag), 250a62a8ef9SStefan Hajnoczi &tag_buf, sizeof(tag_buf)); 251a62a8ef9SStefan Hajnoczi end = memchr(tag_buf, '\0', sizeof(tag_buf)); 252a62a8ef9SStefan Hajnoczi if (end == tag_buf) 253a62a8ef9SStefan Hajnoczi return -EINVAL; /* empty tag */ 254a62a8ef9SStefan Hajnoczi if (!end) 255a62a8ef9SStefan Hajnoczi end = &tag_buf[sizeof(tag_buf)]; 256a62a8ef9SStefan Hajnoczi 257a62a8ef9SStefan Hajnoczi len = end - tag_buf; 258a62a8ef9SStefan Hajnoczi fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); 259a62a8ef9SStefan Hajnoczi if (!fs->tag) 260a62a8ef9SStefan Hajnoczi return -ENOMEM; 261a62a8ef9SStefan Hajnoczi memcpy(fs->tag, tag_buf, len); 262a62a8ef9SStefan Hajnoczi fs->tag[len] = '\0'; 263a62a8ef9SStefan Hajnoczi return 0; 264a62a8ef9SStefan Hajnoczi } 265a62a8ef9SStefan Hajnoczi 266a62a8ef9SStefan Hajnoczi /* Work function for hiprio completion */ 267a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_done_work(struct work_struct *work) 268a62a8ef9SStefan Hajnoczi { 269a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 270a62a8ef9SStefan Hajnoczi done_work); 271a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq; 272a62a8ef9SStefan Hajnoczi 273a62a8ef9SStefan Hajnoczi /* Free completed FUSE_FORGET requests */ 274a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 275a62a8ef9SStefan Hajnoczi do { 276a62a8ef9SStefan Hajnoczi unsigned int len; 277a62a8ef9SStefan Hajnoczi void *req; 278a62a8ef9SStefan Hajnoczi 279a62a8ef9SStefan Hajnoczi virtqueue_disable_cb(vq); 280a62a8ef9SStefan Hajnoczi 281a62a8ef9SStefan Hajnoczi while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 282a62a8ef9SStefan Hajnoczi kfree(req); 283c17ea009SVivek Goyal dec_in_flight_req(fsvq); 284a62a8ef9SStefan Hajnoczi } 285a62a8ef9SStefan Hajnoczi } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 286a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 287a62a8ef9SStefan Hajnoczi } 288a62a8ef9SStefan Hajnoczi 28951fecdd2SVivek Goyal static void virtio_fs_request_dispatch_work(struct work_struct *work) 290a62a8ef9SStefan Hajnoczi { 29151fecdd2SVivek Goyal struct fuse_req *req; 29251fecdd2SVivek Goyal struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 29351fecdd2SVivek Goyal dispatch_work.work); 29451fecdd2SVivek Goyal struct fuse_conn *fc = fsvq->fud->fc; 295a9bfd9ddSVivek Goyal int ret; 29651fecdd2SVivek Goyal 29751fecdd2SVivek Goyal pr_debug("virtio-fs: worker %s called.\n", __func__); 29851fecdd2SVivek Goyal while (1) { 29951fecdd2SVivek Goyal spin_lock(&fsvq->lock); 30051fecdd2SVivek Goyal req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, 30151fecdd2SVivek Goyal list); 30251fecdd2SVivek Goyal if (!req) { 30351fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 304a9bfd9ddSVivek Goyal break; 30551fecdd2SVivek Goyal } 30651fecdd2SVivek Goyal 30751fecdd2SVivek Goyal list_del_init(&req->list); 30851fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 30951fecdd2SVivek Goyal fuse_request_end(fc, req); 31051fecdd2SVivek Goyal } 311a9bfd9ddSVivek Goyal 312a9bfd9ddSVivek Goyal /* Dispatch pending requests */ 313a9bfd9ddSVivek Goyal while (1) { 314a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 315a9bfd9ddSVivek Goyal req = list_first_entry_or_null(&fsvq->queued_reqs, 316a9bfd9ddSVivek Goyal struct fuse_req, list); 317a9bfd9ddSVivek Goyal if (!req) { 318a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 319a9bfd9ddSVivek Goyal return; 320a9bfd9ddSVivek Goyal } 321a9bfd9ddSVivek Goyal list_del_init(&req->list); 322a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 323a9bfd9ddSVivek Goyal 324a9bfd9ddSVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req, true); 325a9bfd9ddSVivek Goyal if (ret < 0) { 326a9bfd9ddSVivek Goyal if (ret == -ENOMEM || ret == -ENOSPC) { 327a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 328a9bfd9ddSVivek Goyal list_add_tail(&req->list, &fsvq->queued_reqs); 329a9bfd9ddSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 330a9bfd9ddSVivek Goyal msecs_to_jiffies(1)); 331a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 332a9bfd9ddSVivek Goyal return; 333a9bfd9ddSVivek Goyal } 334a9bfd9ddSVivek Goyal req->out.h.error = ret; 335a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 336a9bfd9ddSVivek Goyal dec_in_flight_req(fsvq); 337a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 338a9bfd9ddSVivek Goyal pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", 339a9bfd9ddSVivek Goyal ret); 340a9bfd9ddSVivek Goyal fuse_request_end(fc, req); 341a9bfd9ddSVivek Goyal } 342a9bfd9ddSVivek Goyal } 343a62a8ef9SStefan Hajnoczi } 344a62a8ef9SStefan Hajnoczi 34558ada94fSVivek Goyal /* 34658ada94fSVivek Goyal * Returns 1 if queue is full and sender should wait a bit before sending 34758ada94fSVivek Goyal * next request, 0 otherwise. 34858ada94fSVivek Goyal */ 34958ada94fSVivek Goyal static int send_forget_request(struct virtio_fs_vq *fsvq, 35058ada94fSVivek Goyal struct virtio_fs_forget *forget, 35158ada94fSVivek Goyal bool in_flight) 35258ada94fSVivek Goyal { 35358ada94fSVivek Goyal struct scatterlist sg; 35458ada94fSVivek Goyal struct virtqueue *vq; 35558ada94fSVivek Goyal int ret = 0; 35658ada94fSVivek Goyal bool notify; 3571efcf39eSVivek Goyal struct virtio_fs_forget_req *req = &forget->req; 35858ada94fSVivek Goyal 35958ada94fSVivek Goyal spin_lock(&fsvq->lock); 36058ada94fSVivek Goyal if (!fsvq->connected) { 36158ada94fSVivek Goyal if (in_flight) 36258ada94fSVivek Goyal dec_in_flight_req(fsvq); 36358ada94fSVivek Goyal kfree(forget); 36458ada94fSVivek Goyal goto out; 36558ada94fSVivek Goyal } 36658ada94fSVivek Goyal 3671efcf39eSVivek Goyal sg_init_one(&sg, req, sizeof(*req)); 36858ada94fSVivek Goyal vq = fsvq->vq; 36958ada94fSVivek Goyal dev_dbg(&vq->vdev->dev, "%s\n", __func__); 37058ada94fSVivek Goyal 37158ada94fSVivek Goyal ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); 37258ada94fSVivek Goyal if (ret < 0) { 37358ada94fSVivek Goyal if (ret == -ENOMEM || ret == -ENOSPC) { 37458ada94fSVivek Goyal pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", 37558ada94fSVivek Goyal ret); 37658ada94fSVivek Goyal list_add_tail(&forget->list, &fsvq->queued_reqs); 37758ada94fSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 37858ada94fSVivek Goyal msecs_to_jiffies(1)); 37958ada94fSVivek Goyal if (!in_flight) 38058ada94fSVivek Goyal inc_in_flight_req(fsvq); 38158ada94fSVivek Goyal /* Queue is full */ 38258ada94fSVivek Goyal ret = 1; 38358ada94fSVivek Goyal } else { 38458ada94fSVivek Goyal pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", 38558ada94fSVivek Goyal ret); 38658ada94fSVivek Goyal kfree(forget); 38758ada94fSVivek Goyal if (in_flight) 38858ada94fSVivek Goyal dec_in_flight_req(fsvq); 38958ada94fSVivek Goyal } 39058ada94fSVivek Goyal goto out; 39158ada94fSVivek Goyal } 39258ada94fSVivek Goyal 39358ada94fSVivek Goyal if (!in_flight) 39458ada94fSVivek Goyal inc_in_flight_req(fsvq); 39558ada94fSVivek Goyal notify = virtqueue_kick_prepare(vq); 39658ada94fSVivek Goyal spin_unlock(&fsvq->lock); 39758ada94fSVivek Goyal 39858ada94fSVivek Goyal if (notify) 39958ada94fSVivek Goyal virtqueue_notify(vq); 40058ada94fSVivek Goyal return ret; 40158ada94fSVivek Goyal out: 40258ada94fSVivek Goyal spin_unlock(&fsvq->lock); 40358ada94fSVivek Goyal return ret; 40458ada94fSVivek Goyal } 40558ada94fSVivek Goyal 406a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) 407a62a8ef9SStefan Hajnoczi { 408a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget; 409a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 410a62a8ef9SStefan Hajnoczi dispatch_work.work); 411a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: worker %s called.\n", __func__); 412a62a8ef9SStefan Hajnoczi while (1) { 413a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 414a62a8ef9SStefan Hajnoczi forget = list_first_entry_or_null(&fsvq->queued_reqs, 415a62a8ef9SStefan Hajnoczi struct virtio_fs_forget, list); 416a62a8ef9SStefan Hajnoczi if (!forget) { 417a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 418a62a8ef9SStefan Hajnoczi return; 419a62a8ef9SStefan Hajnoczi } 420a62a8ef9SStefan Hajnoczi 421a62a8ef9SStefan Hajnoczi list_del(&forget->list); 422a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 42358ada94fSVivek Goyal if (send_forget_request(fsvq, forget, true)) 424a62a8ef9SStefan Hajnoczi return; 425a62a8ef9SStefan Hajnoczi } 426a62a8ef9SStefan Hajnoczi } 427a62a8ef9SStefan Hajnoczi 428a62a8ef9SStefan Hajnoczi /* Allocate and copy args into req->argbuf */ 429a62a8ef9SStefan Hajnoczi static int copy_args_to_argbuf(struct fuse_req *req) 430a62a8ef9SStefan Hajnoczi { 431a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 432a62a8ef9SStefan Hajnoczi unsigned int offset = 0; 433a62a8ef9SStefan Hajnoczi unsigned int num_in; 434a62a8ef9SStefan Hajnoczi unsigned int num_out; 435a62a8ef9SStefan Hajnoczi unsigned int len; 436a62a8ef9SStefan Hajnoczi unsigned int i; 437a62a8ef9SStefan Hajnoczi 438a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages; 439a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages; 440a62a8ef9SStefan Hajnoczi len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + 441a62a8ef9SStefan Hajnoczi fuse_len_args(num_out, args->out_args); 442a62a8ef9SStefan Hajnoczi 443a62a8ef9SStefan Hajnoczi req->argbuf = kmalloc(len, GFP_ATOMIC); 444a62a8ef9SStefan Hajnoczi if (!req->argbuf) 445a62a8ef9SStefan Hajnoczi return -ENOMEM; 446a62a8ef9SStefan Hajnoczi 447a62a8ef9SStefan Hajnoczi for (i = 0; i < num_in; i++) { 448a62a8ef9SStefan Hajnoczi memcpy(req->argbuf + offset, 449a62a8ef9SStefan Hajnoczi args->in_args[i].value, 450a62a8ef9SStefan Hajnoczi args->in_args[i].size); 451a62a8ef9SStefan Hajnoczi offset += args->in_args[i].size; 452a62a8ef9SStefan Hajnoczi } 453a62a8ef9SStefan Hajnoczi 454a62a8ef9SStefan Hajnoczi return 0; 455a62a8ef9SStefan Hajnoczi } 456a62a8ef9SStefan Hajnoczi 457a62a8ef9SStefan Hajnoczi /* Copy args out of and free req->argbuf */ 458a62a8ef9SStefan Hajnoczi static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req) 459a62a8ef9SStefan Hajnoczi { 460a62a8ef9SStefan Hajnoczi unsigned int remaining; 461a62a8ef9SStefan Hajnoczi unsigned int offset; 462a62a8ef9SStefan Hajnoczi unsigned int num_in; 463a62a8ef9SStefan Hajnoczi unsigned int num_out; 464a62a8ef9SStefan Hajnoczi unsigned int i; 465a62a8ef9SStefan Hajnoczi 466a62a8ef9SStefan Hajnoczi remaining = req->out.h.len - sizeof(req->out.h); 467a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages; 468a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages; 469a62a8ef9SStefan Hajnoczi offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); 470a62a8ef9SStefan Hajnoczi 471a62a8ef9SStefan Hajnoczi for (i = 0; i < num_out; i++) { 472a62a8ef9SStefan Hajnoczi unsigned int argsize = args->out_args[i].size; 473a62a8ef9SStefan Hajnoczi 474a62a8ef9SStefan Hajnoczi if (args->out_argvar && 475a62a8ef9SStefan Hajnoczi i == args->out_numargs - 1 && 476a62a8ef9SStefan Hajnoczi argsize > remaining) { 477a62a8ef9SStefan Hajnoczi argsize = remaining; 478a62a8ef9SStefan Hajnoczi } 479a62a8ef9SStefan Hajnoczi 480a62a8ef9SStefan Hajnoczi memcpy(args->out_args[i].value, req->argbuf + offset, argsize); 481a62a8ef9SStefan Hajnoczi offset += argsize; 482a62a8ef9SStefan Hajnoczi 483a62a8ef9SStefan Hajnoczi if (i != args->out_numargs - 1) 484a62a8ef9SStefan Hajnoczi remaining -= argsize; 485a62a8ef9SStefan Hajnoczi } 486a62a8ef9SStefan Hajnoczi 487a62a8ef9SStefan Hajnoczi /* Store the actual size of the variable-length arg */ 488a62a8ef9SStefan Hajnoczi if (args->out_argvar) 489a62a8ef9SStefan Hajnoczi args->out_args[args->out_numargs - 1].size = remaining; 490a62a8ef9SStefan Hajnoczi 491a62a8ef9SStefan Hajnoczi kfree(req->argbuf); 492a62a8ef9SStefan Hajnoczi req->argbuf = NULL; 493a62a8ef9SStefan Hajnoczi } 494a62a8ef9SStefan Hajnoczi 495a62a8ef9SStefan Hajnoczi /* Work function for request completion */ 496bb737bbeSVivek Goyal static void virtio_fs_request_complete(struct fuse_req *req, 497bb737bbeSVivek Goyal struct virtio_fs_vq *fsvq) 498a62a8ef9SStefan Hajnoczi { 499a62a8ef9SStefan Hajnoczi struct fuse_pqueue *fpq = &fsvq->fud->pq; 500a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = fsvq->fud->fc; 501a62a8ef9SStefan Hajnoczi struct fuse_args *args; 502bb737bbeSVivek Goyal struct fuse_args_pages *ap; 503a62a8ef9SStefan Hajnoczi unsigned int len, i, thislen; 504a62a8ef9SStefan Hajnoczi struct page *page; 505a62a8ef9SStefan Hajnoczi 506a62a8ef9SStefan Hajnoczi /* 507a62a8ef9SStefan Hajnoczi * TODO verify that server properly follows FUSE protocol 508a62a8ef9SStefan Hajnoczi * (oh.uniq, oh.len) 509a62a8ef9SStefan Hajnoczi */ 510a62a8ef9SStefan Hajnoczi args = req->args; 511a62a8ef9SStefan Hajnoczi copy_args_from_argbuf(args, req); 512a62a8ef9SStefan Hajnoczi 513a62a8ef9SStefan Hajnoczi if (args->out_pages && args->page_zeroing) { 514a62a8ef9SStefan Hajnoczi len = args->out_args[args->out_numargs - 1].size; 515a62a8ef9SStefan Hajnoczi ap = container_of(args, typeof(*ap), args); 516a62a8ef9SStefan Hajnoczi for (i = 0; i < ap->num_pages; i++) { 517a62a8ef9SStefan Hajnoczi thislen = ap->descs[i].length; 518a62a8ef9SStefan Hajnoczi if (len < thislen) { 519a62a8ef9SStefan Hajnoczi WARN_ON(ap->descs[i].offset); 520a62a8ef9SStefan Hajnoczi page = ap->pages[i]; 521a62a8ef9SStefan Hajnoczi zero_user_segment(page, len, thislen); 522a62a8ef9SStefan Hajnoczi len = 0; 523a62a8ef9SStefan Hajnoczi } else { 524a62a8ef9SStefan Hajnoczi len -= thislen; 525a62a8ef9SStefan Hajnoczi } 526a62a8ef9SStefan Hajnoczi } 527a62a8ef9SStefan Hajnoczi } 528a62a8ef9SStefan Hajnoczi 529a62a8ef9SStefan Hajnoczi spin_lock(&fpq->lock); 530a62a8ef9SStefan Hajnoczi clear_bit(FR_SENT, &req->flags); 531a62a8ef9SStefan Hajnoczi spin_unlock(&fpq->lock); 532a62a8ef9SStefan Hajnoczi 533a62a8ef9SStefan Hajnoczi fuse_request_end(fc, req); 534a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 535c17ea009SVivek Goyal dec_in_flight_req(fsvq); 536a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 537a62a8ef9SStefan Hajnoczi } 538bb737bbeSVivek Goyal 539bb737bbeSVivek Goyal static void virtio_fs_complete_req_work(struct work_struct *work) 540bb737bbeSVivek Goyal { 541bb737bbeSVivek Goyal struct virtio_fs_req_work *w = 542bb737bbeSVivek Goyal container_of(work, typeof(*w), done_work); 543bb737bbeSVivek Goyal 544bb737bbeSVivek Goyal virtio_fs_request_complete(w->req, w->fsvq); 545bb737bbeSVivek Goyal kfree(w); 546bb737bbeSVivek Goyal } 547bb737bbeSVivek Goyal 548bb737bbeSVivek Goyal static void virtio_fs_requests_done_work(struct work_struct *work) 549bb737bbeSVivek Goyal { 550bb737bbeSVivek Goyal struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 551bb737bbeSVivek Goyal done_work); 552bb737bbeSVivek Goyal struct fuse_pqueue *fpq = &fsvq->fud->pq; 553bb737bbeSVivek Goyal struct virtqueue *vq = fsvq->vq; 554bb737bbeSVivek Goyal struct fuse_req *req; 555bb737bbeSVivek Goyal struct fuse_req *next; 556bb737bbeSVivek Goyal unsigned int len; 557bb737bbeSVivek Goyal LIST_HEAD(reqs); 558bb737bbeSVivek Goyal 559bb737bbeSVivek Goyal /* Collect completed requests off the virtqueue */ 560bb737bbeSVivek Goyal spin_lock(&fsvq->lock); 561bb737bbeSVivek Goyal do { 562bb737bbeSVivek Goyal virtqueue_disable_cb(vq); 563bb737bbeSVivek Goyal 564bb737bbeSVivek Goyal while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 565bb737bbeSVivek Goyal spin_lock(&fpq->lock); 566bb737bbeSVivek Goyal list_move_tail(&req->list, &reqs); 567bb737bbeSVivek Goyal spin_unlock(&fpq->lock); 568bb737bbeSVivek Goyal } 569bb737bbeSVivek Goyal } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 570bb737bbeSVivek Goyal spin_unlock(&fsvq->lock); 571bb737bbeSVivek Goyal 572bb737bbeSVivek Goyal /* End requests */ 573bb737bbeSVivek Goyal list_for_each_entry_safe(req, next, &reqs, list) { 574bb737bbeSVivek Goyal list_del_init(&req->list); 575bb737bbeSVivek Goyal 576bb737bbeSVivek Goyal /* blocking async request completes in a worker context */ 577bb737bbeSVivek Goyal if (req->args->may_block) { 578bb737bbeSVivek Goyal struct virtio_fs_req_work *w; 579bb737bbeSVivek Goyal 580bb737bbeSVivek Goyal w = kzalloc(sizeof(*w), GFP_NOFS | __GFP_NOFAIL); 581bb737bbeSVivek Goyal INIT_WORK(&w->done_work, virtio_fs_complete_req_work); 582bb737bbeSVivek Goyal w->fsvq = fsvq; 583bb737bbeSVivek Goyal w->req = req; 584bb737bbeSVivek Goyal schedule_work(&w->done_work); 585bb737bbeSVivek Goyal } else { 586bb737bbeSVivek Goyal virtio_fs_request_complete(req, fsvq); 587bb737bbeSVivek Goyal } 588bb737bbeSVivek Goyal } 589a62a8ef9SStefan Hajnoczi } 590a62a8ef9SStefan Hajnoczi 591a62a8ef9SStefan Hajnoczi /* Virtqueue interrupt handler */ 592a62a8ef9SStefan Hajnoczi static void virtio_fs_vq_done(struct virtqueue *vq) 593a62a8ef9SStefan Hajnoczi { 594a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = vq_to_fsvq(vq); 595a62a8ef9SStefan Hajnoczi 596a62a8ef9SStefan Hajnoczi dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); 597a62a8ef9SStefan Hajnoczi 598a62a8ef9SStefan Hajnoczi schedule_work(&fsvq->done_work); 599a62a8ef9SStefan Hajnoczi } 600a62a8ef9SStefan Hajnoczi 601*b43b7e81SVivek Goyal static void virtio_fs_init_vq(struct virtio_fs_vq *fsvq, char *name, 602*b43b7e81SVivek Goyal int vq_type) 603*b43b7e81SVivek Goyal { 604*b43b7e81SVivek Goyal strncpy(fsvq->name, name, VQ_NAME_LEN); 605*b43b7e81SVivek Goyal spin_lock_init(&fsvq->lock); 606*b43b7e81SVivek Goyal INIT_LIST_HEAD(&fsvq->queued_reqs); 607*b43b7e81SVivek Goyal INIT_LIST_HEAD(&fsvq->end_reqs); 608*b43b7e81SVivek Goyal init_completion(&fsvq->in_flight_zero); 609*b43b7e81SVivek Goyal 610*b43b7e81SVivek Goyal if (vq_type == VQ_REQUEST) { 611*b43b7e81SVivek Goyal INIT_WORK(&fsvq->done_work, virtio_fs_requests_done_work); 612*b43b7e81SVivek Goyal INIT_DELAYED_WORK(&fsvq->dispatch_work, 613*b43b7e81SVivek Goyal virtio_fs_request_dispatch_work); 614*b43b7e81SVivek Goyal } else { 615*b43b7e81SVivek Goyal INIT_WORK(&fsvq->done_work, virtio_fs_hiprio_done_work); 616*b43b7e81SVivek Goyal INIT_DELAYED_WORK(&fsvq->dispatch_work, 617*b43b7e81SVivek Goyal virtio_fs_hiprio_dispatch_work); 618*b43b7e81SVivek Goyal } 619*b43b7e81SVivek Goyal } 620*b43b7e81SVivek Goyal 621a62a8ef9SStefan Hajnoczi /* Initialize virtqueues */ 622a62a8ef9SStefan Hajnoczi static int virtio_fs_setup_vqs(struct virtio_device *vdev, 623a62a8ef9SStefan Hajnoczi struct virtio_fs *fs) 624a62a8ef9SStefan Hajnoczi { 625a62a8ef9SStefan Hajnoczi struct virtqueue **vqs; 626a62a8ef9SStefan Hajnoczi vq_callback_t **callbacks; 627a62a8ef9SStefan Hajnoczi const char **names; 628a62a8ef9SStefan Hajnoczi unsigned int i; 629a62a8ef9SStefan Hajnoczi int ret = 0; 630a62a8ef9SStefan Hajnoczi 6312c0349ecSMichael S. Tsirkin virtio_cread_le(vdev, struct virtio_fs_config, num_request_queues, 632a62a8ef9SStefan Hajnoczi &fs->num_request_queues); 633a62a8ef9SStefan Hajnoczi if (fs->num_request_queues == 0) 634a62a8ef9SStefan Hajnoczi return -EINVAL; 635a62a8ef9SStefan Hajnoczi 636*b43b7e81SVivek Goyal fs->nvqs = VQ_REQUEST + fs->num_request_queues; 637a62a8ef9SStefan Hajnoczi fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); 638a62a8ef9SStefan Hajnoczi if (!fs->vqs) 639a62a8ef9SStefan Hajnoczi return -ENOMEM; 640a62a8ef9SStefan Hajnoczi 641a62a8ef9SStefan Hajnoczi vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); 642a62a8ef9SStefan Hajnoczi callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]), 643a62a8ef9SStefan Hajnoczi GFP_KERNEL); 644a62a8ef9SStefan Hajnoczi names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL); 645a62a8ef9SStefan Hajnoczi if (!vqs || !callbacks || !names) { 646a62a8ef9SStefan Hajnoczi ret = -ENOMEM; 647a62a8ef9SStefan Hajnoczi goto out; 648a62a8ef9SStefan Hajnoczi } 649a62a8ef9SStefan Hajnoczi 650*b43b7e81SVivek Goyal /* Initialize the hiprio/forget request virtqueue */ 651a62a8ef9SStefan Hajnoczi callbacks[VQ_HIPRIO] = virtio_fs_vq_done; 652*b43b7e81SVivek Goyal virtio_fs_init_vq(&fs->vqs[VQ_HIPRIO], "hiprio", VQ_HIPRIO); 653a62a8ef9SStefan Hajnoczi names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; 654a62a8ef9SStefan Hajnoczi 655a62a8ef9SStefan Hajnoczi /* Initialize the requests virtqueues */ 656a62a8ef9SStefan Hajnoczi for (i = VQ_REQUEST; i < fs->nvqs; i++) { 657*b43b7e81SVivek Goyal char vq_name[VQ_NAME_LEN]; 658*b43b7e81SVivek Goyal 659*b43b7e81SVivek Goyal snprintf(vq_name, VQ_NAME_LEN, "requests.%u", i - VQ_REQUEST); 660*b43b7e81SVivek Goyal virtio_fs_init_vq(&fs->vqs[i], vq_name, VQ_REQUEST); 661a62a8ef9SStefan Hajnoczi callbacks[i] = virtio_fs_vq_done; 662a62a8ef9SStefan Hajnoczi names[i] = fs->vqs[i].name; 663a62a8ef9SStefan Hajnoczi } 664a62a8ef9SStefan Hajnoczi 665a62a8ef9SStefan Hajnoczi ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL); 666a62a8ef9SStefan Hajnoczi if (ret < 0) 667a62a8ef9SStefan Hajnoczi goto out; 668a62a8ef9SStefan Hajnoczi 669a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) 670a62a8ef9SStefan Hajnoczi fs->vqs[i].vq = vqs[i]; 671a62a8ef9SStefan Hajnoczi 672a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs); 673a62a8ef9SStefan Hajnoczi out: 674a62a8ef9SStefan Hajnoczi kfree(names); 675a62a8ef9SStefan Hajnoczi kfree(callbacks); 676a62a8ef9SStefan Hajnoczi kfree(vqs); 677a62a8ef9SStefan Hajnoczi if (ret) 678a62a8ef9SStefan Hajnoczi kfree(fs->vqs); 679a62a8ef9SStefan Hajnoczi return ret; 680a62a8ef9SStefan Hajnoczi } 681a62a8ef9SStefan Hajnoczi 682a62a8ef9SStefan Hajnoczi /* Free virtqueues (device must already be reset) */ 683a62a8ef9SStefan Hajnoczi static void virtio_fs_cleanup_vqs(struct virtio_device *vdev, 684a62a8ef9SStefan Hajnoczi struct virtio_fs *fs) 685a62a8ef9SStefan Hajnoczi { 686a62a8ef9SStefan Hajnoczi vdev->config->del_vqs(vdev); 687a62a8ef9SStefan Hajnoczi } 688a62a8ef9SStefan Hajnoczi 689a62a8ef9SStefan Hajnoczi static int virtio_fs_probe(struct virtio_device *vdev) 690a62a8ef9SStefan Hajnoczi { 691a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 692a62a8ef9SStefan Hajnoczi int ret; 693a62a8ef9SStefan Hajnoczi 694a62a8ef9SStefan Hajnoczi fs = kzalloc(sizeof(*fs), GFP_KERNEL); 695a62a8ef9SStefan Hajnoczi if (!fs) 696a62a8ef9SStefan Hajnoczi return -ENOMEM; 697a62a8ef9SStefan Hajnoczi kref_init(&fs->refcount); 698a62a8ef9SStefan Hajnoczi vdev->priv = fs; 699a62a8ef9SStefan Hajnoczi 700a62a8ef9SStefan Hajnoczi ret = virtio_fs_read_tag(vdev, fs); 701a62a8ef9SStefan Hajnoczi if (ret < 0) 702a62a8ef9SStefan Hajnoczi goto out; 703a62a8ef9SStefan Hajnoczi 704a62a8ef9SStefan Hajnoczi ret = virtio_fs_setup_vqs(vdev, fs); 705a62a8ef9SStefan Hajnoczi if (ret < 0) 706a62a8ef9SStefan Hajnoczi goto out; 707a62a8ef9SStefan Hajnoczi 708a62a8ef9SStefan Hajnoczi /* TODO vq affinity */ 709a62a8ef9SStefan Hajnoczi 710a62a8ef9SStefan Hajnoczi /* Bring the device online in case the filesystem is mounted and 711a62a8ef9SStefan Hajnoczi * requests need to be sent before we return. 712a62a8ef9SStefan Hajnoczi */ 713a62a8ef9SStefan Hajnoczi virtio_device_ready(vdev); 714a62a8ef9SStefan Hajnoczi 715a62a8ef9SStefan Hajnoczi ret = virtio_fs_add_instance(fs); 716a62a8ef9SStefan Hajnoczi if (ret < 0) 717a62a8ef9SStefan Hajnoczi goto out_vqs; 718a62a8ef9SStefan Hajnoczi 719a62a8ef9SStefan Hajnoczi return 0; 720a62a8ef9SStefan Hajnoczi 721a62a8ef9SStefan Hajnoczi out_vqs: 722a62a8ef9SStefan Hajnoczi vdev->config->reset(vdev); 723a62a8ef9SStefan Hajnoczi virtio_fs_cleanup_vqs(vdev, fs); 724a62a8ef9SStefan Hajnoczi 725a62a8ef9SStefan Hajnoczi out: 726a62a8ef9SStefan Hajnoczi vdev->priv = NULL; 727a62a8ef9SStefan Hajnoczi kfree(fs); 728a62a8ef9SStefan Hajnoczi return ret; 729a62a8ef9SStefan Hajnoczi } 730a62a8ef9SStefan Hajnoczi 731a62a8ef9SStefan Hajnoczi static void virtio_fs_stop_all_queues(struct virtio_fs *fs) 732a62a8ef9SStefan Hajnoczi { 733a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 734a62a8ef9SStefan Hajnoczi int i; 735a62a8ef9SStefan Hajnoczi 736a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 737a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 738a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 739a62a8ef9SStefan Hajnoczi fsvq->connected = false; 740a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 741a62a8ef9SStefan Hajnoczi } 742a62a8ef9SStefan Hajnoczi } 743a62a8ef9SStefan Hajnoczi 744a62a8ef9SStefan Hajnoczi static void virtio_fs_remove(struct virtio_device *vdev) 745a62a8ef9SStefan Hajnoczi { 746a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vdev->priv; 747a62a8ef9SStefan Hajnoczi 748a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 749a62a8ef9SStefan Hajnoczi /* This device is going away. No one should get new reference */ 750a62a8ef9SStefan Hajnoczi list_del_init(&fs->list); 751a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(fs); 752724c15a4SVivek Goyal virtio_fs_drain_all_queues_locked(fs); 753a62a8ef9SStefan Hajnoczi vdev->config->reset(vdev); 754a62a8ef9SStefan Hajnoczi virtio_fs_cleanup_vqs(vdev, fs); 755a62a8ef9SStefan Hajnoczi 756a62a8ef9SStefan Hajnoczi vdev->priv = NULL; 757a62a8ef9SStefan Hajnoczi /* Put device reference on virtio_fs object */ 758a62a8ef9SStefan Hajnoczi virtio_fs_put(fs); 759a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 760a62a8ef9SStefan Hajnoczi } 761a62a8ef9SStefan Hajnoczi 762a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP 763a62a8ef9SStefan Hajnoczi static int virtio_fs_freeze(struct virtio_device *vdev) 764a62a8ef9SStefan Hajnoczi { 765a62a8ef9SStefan Hajnoczi /* TODO need to save state here */ 766a62a8ef9SStefan Hajnoczi pr_warn("virtio-fs: suspend/resume not yet supported\n"); 767a62a8ef9SStefan Hajnoczi return -EOPNOTSUPP; 768a62a8ef9SStefan Hajnoczi } 769a62a8ef9SStefan Hajnoczi 770a62a8ef9SStefan Hajnoczi static int virtio_fs_restore(struct virtio_device *vdev) 771a62a8ef9SStefan Hajnoczi { 772a62a8ef9SStefan Hajnoczi /* TODO need to restore state here */ 773a62a8ef9SStefan Hajnoczi return 0; 774a62a8ef9SStefan Hajnoczi } 775a62a8ef9SStefan Hajnoczi #endif /* CONFIG_PM_SLEEP */ 776a62a8ef9SStefan Hajnoczi 77700929447SYueHaibing static const struct virtio_device_id id_table[] = { 778a62a8ef9SStefan Hajnoczi { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, 779a62a8ef9SStefan Hajnoczi {}, 780a62a8ef9SStefan Hajnoczi }; 781a62a8ef9SStefan Hajnoczi 78200929447SYueHaibing static const unsigned int feature_table[] = {}; 783a62a8ef9SStefan Hajnoczi 784a62a8ef9SStefan Hajnoczi static struct virtio_driver virtio_fs_driver = { 785a62a8ef9SStefan Hajnoczi .driver.name = KBUILD_MODNAME, 786a62a8ef9SStefan Hajnoczi .driver.owner = THIS_MODULE, 787a62a8ef9SStefan Hajnoczi .id_table = id_table, 788a62a8ef9SStefan Hajnoczi .feature_table = feature_table, 789a62a8ef9SStefan Hajnoczi .feature_table_size = ARRAY_SIZE(feature_table), 790a62a8ef9SStefan Hajnoczi .probe = virtio_fs_probe, 791a62a8ef9SStefan Hajnoczi .remove = virtio_fs_remove, 792a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP 793a62a8ef9SStefan Hajnoczi .freeze = virtio_fs_freeze, 794a62a8ef9SStefan Hajnoczi .restore = virtio_fs_restore, 795a62a8ef9SStefan Hajnoczi #endif 796a62a8ef9SStefan Hajnoczi }; 797a62a8ef9SStefan Hajnoczi 798a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq) 799a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 800a62a8ef9SStefan Hajnoczi { 801a62a8ef9SStefan Hajnoczi struct fuse_forget_link *link; 802a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget; 8031efcf39eSVivek Goyal struct virtio_fs_forget_req *req; 804a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 805a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 806a62a8ef9SStefan Hajnoczi u64 unique; 807a62a8ef9SStefan Hajnoczi 808a62a8ef9SStefan Hajnoczi link = fuse_dequeue_forget(fiq, 1, NULL); 809a62a8ef9SStefan Hajnoczi unique = fuse_get_unique(fiq); 810a62a8ef9SStefan Hajnoczi 811a62a8ef9SStefan Hajnoczi fs = fiq->priv; 812a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[VQ_HIPRIO]; 813a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 814a62a8ef9SStefan Hajnoczi 815a62a8ef9SStefan Hajnoczi /* Allocate a buffer for the request */ 816a62a8ef9SStefan Hajnoczi forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL); 8171efcf39eSVivek Goyal req = &forget->req; 818a62a8ef9SStefan Hajnoczi 8191efcf39eSVivek Goyal req->ih = (struct fuse_in_header){ 820a62a8ef9SStefan Hajnoczi .opcode = FUSE_FORGET, 821a62a8ef9SStefan Hajnoczi .nodeid = link->forget_one.nodeid, 822a62a8ef9SStefan Hajnoczi .unique = unique, 8231efcf39eSVivek Goyal .len = sizeof(*req), 824a62a8ef9SStefan Hajnoczi }; 8251efcf39eSVivek Goyal req->arg = (struct fuse_forget_in){ 826a62a8ef9SStefan Hajnoczi .nlookup = link->forget_one.nlookup, 827a62a8ef9SStefan Hajnoczi }; 828a62a8ef9SStefan Hajnoczi 82958ada94fSVivek Goyal send_forget_request(fsvq, forget, false); 830a62a8ef9SStefan Hajnoczi kfree(link); 831a62a8ef9SStefan Hajnoczi } 832a62a8ef9SStefan Hajnoczi 833a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq) 834a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 835a62a8ef9SStefan Hajnoczi { 836a62a8ef9SStefan Hajnoczi /* 837a62a8ef9SStefan Hajnoczi * TODO interrupts. 838a62a8ef9SStefan Hajnoczi * 839a62a8ef9SStefan Hajnoczi * Normal fs operations on a local filesystems aren't interruptible. 840a62a8ef9SStefan Hajnoczi * Exceptions are blocking lock operations; for example fcntl(F_SETLKW) 841a62a8ef9SStefan Hajnoczi * with shared lock between host and guest. 842a62a8ef9SStefan Hajnoczi */ 843a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 844a62a8ef9SStefan Hajnoczi } 845a62a8ef9SStefan Hajnoczi 846a62a8ef9SStefan Hajnoczi /* Return the number of scatter-gather list elements required */ 847a62a8ef9SStefan Hajnoczi static unsigned int sg_count_fuse_req(struct fuse_req *req) 848a62a8ef9SStefan Hajnoczi { 849a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 850a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); 851a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 1 /* fuse_in_header */; 852a62a8ef9SStefan Hajnoczi 853a62a8ef9SStefan Hajnoczi if (args->in_numargs - args->in_pages) 854a62a8ef9SStefan Hajnoczi total_sgs += 1; 855a62a8ef9SStefan Hajnoczi 856a62a8ef9SStefan Hajnoczi if (args->in_pages) 857a62a8ef9SStefan Hajnoczi total_sgs += ap->num_pages; 858a62a8ef9SStefan Hajnoczi 859a62a8ef9SStefan Hajnoczi if (!test_bit(FR_ISREPLY, &req->flags)) 860a62a8ef9SStefan Hajnoczi return total_sgs; 861a62a8ef9SStefan Hajnoczi 862a62a8ef9SStefan Hajnoczi total_sgs += 1 /* fuse_out_header */; 863a62a8ef9SStefan Hajnoczi 864a62a8ef9SStefan Hajnoczi if (args->out_numargs - args->out_pages) 865a62a8ef9SStefan Hajnoczi total_sgs += 1; 866a62a8ef9SStefan Hajnoczi 867a62a8ef9SStefan Hajnoczi if (args->out_pages) 868a62a8ef9SStefan Hajnoczi total_sgs += ap->num_pages; 869a62a8ef9SStefan Hajnoczi 870a62a8ef9SStefan Hajnoczi return total_sgs; 871a62a8ef9SStefan Hajnoczi } 872a62a8ef9SStefan Hajnoczi 873a62a8ef9SStefan Hajnoczi /* Add pages to scatter-gather list and return number of elements used */ 874a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_pages(struct scatterlist *sg, 875a62a8ef9SStefan Hajnoczi struct page **pages, 876a62a8ef9SStefan Hajnoczi struct fuse_page_desc *page_descs, 877a62a8ef9SStefan Hajnoczi unsigned int num_pages, 878a62a8ef9SStefan Hajnoczi unsigned int total_len) 879a62a8ef9SStefan Hajnoczi { 880a62a8ef9SStefan Hajnoczi unsigned int i; 881a62a8ef9SStefan Hajnoczi unsigned int this_len; 882a62a8ef9SStefan Hajnoczi 883a62a8ef9SStefan Hajnoczi for (i = 0; i < num_pages && total_len; i++) { 884a62a8ef9SStefan Hajnoczi sg_init_table(&sg[i], 1); 885a62a8ef9SStefan Hajnoczi this_len = min(page_descs[i].length, total_len); 886a62a8ef9SStefan Hajnoczi sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); 887a62a8ef9SStefan Hajnoczi total_len -= this_len; 888a62a8ef9SStefan Hajnoczi } 889a62a8ef9SStefan Hajnoczi 890a62a8ef9SStefan Hajnoczi return i; 891a62a8ef9SStefan Hajnoczi } 892a62a8ef9SStefan Hajnoczi 893a62a8ef9SStefan Hajnoczi /* Add args to scatter-gather list and return number of elements used */ 894a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_args(struct scatterlist *sg, 895a62a8ef9SStefan Hajnoczi struct fuse_req *req, 896a62a8ef9SStefan Hajnoczi struct fuse_arg *args, 897a62a8ef9SStefan Hajnoczi unsigned int numargs, 898a62a8ef9SStefan Hajnoczi bool argpages, 899a62a8ef9SStefan Hajnoczi void *argbuf, 900a62a8ef9SStefan Hajnoczi unsigned int *len_used) 901a62a8ef9SStefan Hajnoczi { 902a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); 903a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 0; 904a62a8ef9SStefan Hajnoczi unsigned int len; 905a62a8ef9SStefan Hajnoczi 906a62a8ef9SStefan Hajnoczi len = fuse_len_args(numargs - argpages, args); 907a62a8ef9SStefan Hajnoczi if (len) 908a62a8ef9SStefan Hajnoczi sg_init_one(&sg[total_sgs++], argbuf, len); 909a62a8ef9SStefan Hajnoczi 910a62a8ef9SStefan Hajnoczi if (argpages) 911a62a8ef9SStefan Hajnoczi total_sgs += sg_init_fuse_pages(&sg[total_sgs], 912a62a8ef9SStefan Hajnoczi ap->pages, ap->descs, 913a62a8ef9SStefan Hajnoczi ap->num_pages, 914a62a8ef9SStefan Hajnoczi args[numargs - 1].size); 915a62a8ef9SStefan Hajnoczi 916a62a8ef9SStefan Hajnoczi if (len_used) 917a62a8ef9SStefan Hajnoczi *len_used = len; 918a62a8ef9SStefan Hajnoczi 919a62a8ef9SStefan Hajnoczi return total_sgs; 920a62a8ef9SStefan Hajnoczi } 921a62a8ef9SStefan Hajnoczi 922a62a8ef9SStefan Hajnoczi /* Add a request to a virtqueue and kick the device */ 923a62a8ef9SStefan Hajnoczi static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 924a9bfd9ddSVivek Goyal struct fuse_req *req, bool in_flight) 925a62a8ef9SStefan Hajnoczi { 926a62a8ef9SStefan Hajnoczi /* requests need at least 4 elements */ 927a62a8ef9SStefan Hajnoczi struct scatterlist *stack_sgs[6]; 928a62a8ef9SStefan Hajnoczi struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)]; 929a62a8ef9SStefan Hajnoczi struct scatterlist **sgs = stack_sgs; 930a62a8ef9SStefan Hajnoczi struct scatterlist *sg = stack_sg; 931a62a8ef9SStefan Hajnoczi struct virtqueue *vq; 932a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 933a62a8ef9SStefan Hajnoczi unsigned int argbuf_used = 0; 934a62a8ef9SStefan Hajnoczi unsigned int out_sgs = 0; 935a62a8ef9SStefan Hajnoczi unsigned int in_sgs = 0; 936a62a8ef9SStefan Hajnoczi unsigned int total_sgs; 937a62a8ef9SStefan Hajnoczi unsigned int i; 938a62a8ef9SStefan Hajnoczi int ret; 939a62a8ef9SStefan Hajnoczi bool notify; 9405dbe190fSVivek Goyal struct fuse_pqueue *fpq; 941a62a8ef9SStefan Hajnoczi 942a62a8ef9SStefan Hajnoczi /* Does the sglist fit on the stack? */ 943a62a8ef9SStefan Hajnoczi total_sgs = sg_count_fuse_req(req); 944a62a8ef9SStefan Hajnoczi if (total_sgs > ARRAY_SIZE(stack_sgs)) { 945a62a8ef9SStefan Hajnoczi sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); 946a62a8ef9SStefan Hajnoczi sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); 947a62a8ef9SStefan Hajnoczi if (!sgs || !sg) { 948a62a8ef9SStefan Hajnoczi ret = -ENOMEM; 949a62a8ef9SStefan Hajnoczi goto out; 950a62a8ef9SStefan Hajnoczi } 951a62a8ef9SStefan Hajnoczi } 952a62a8ef9SStefan Hajnoczi 953a62a8ef9SStefan Hajnoczi /* Use a bounce buffer since stack args cannot be mapped */ 954a62a8ef9SStefan Hajnoczi ret = copy_args_to_argbuf(req); 955a62a8ef9SStefan Hajnoczi if (ret < 0) 956a62a8ef9SStefan Hajnoczi goto out; 957a62a8ef9SStefan Hajnoczi 958a62a8ef9SStefan Hajnoczi /* Request elements */ 959a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); 960a62a8ef9SStefan Hajnoczi out_sgs += sg_init_fuse_args(&sg[out_sgs], req, 961a62a8ef9SStefan Hajnoczi (struct fuse_arg *)args->in_args, 962a62a8ef9SStefan Hajnoczi args->in_numargs, args->in_pages, 963a62a8ef9SStefan Hajnoczi req->argbuf, &argbuf_used); 964a62a8ef9SStefan Hajnoczi 965a62a8ef9SStefan Hajnoczi /* Reply elements */ 966a62a8ef9SStefan Hajnoczi if (test_bit(FR_ISREPLY, &req->flags)) { 967a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs + in_sgs++], 968a62a8ef9SStefan Hajnoczi &req->out.h, sizeof(req->out.h)); 969a62a8ef9SStefan Hajnoczi in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req, 970a62a8ef9SStefan Hajnoczi args->out_args, args->out_numargs, 971a62a8ef9SStefan Hajnoczi args->out_pages, 972a62a8ef9SStefan Hajnoczi req->argbuf + argbuf_used, NULL); 973a62a8ef9SStefan Hajnoczi } 974a62a8ef9SStefan Hajnoczi 975a62a8ef9SStefan Hajnoczi WARN_ON(out_sgs + in_sgs != total_sgs); 976a62a8ef9SStefan Hajnoczi 977a62a8ef9SStefan Hajnoczi for (i = 0; i < total_sgs; i++) 978a62a8ef9SStefan Hajnoczi sgs[i] = &sg[i]; 979a62a8ef9SStefan Hajnoczi 980a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 981a62a8ef9SStefan Hajnoczi 982a62a8ef9SStefan Hajnoczi if (!fsvq->connected) { 983a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 984a62a8ef9SStefan Hajnoczi ret = -ENOTCONN; 985a62a8ef9SStefan Hajnoczi goto out; 986a62a8ef9SStefan Hajnoczi } 987a62a8ef9SStefan Hajnoczi 988a62a8ef9SStefan Hajnoczi vq = fsvq->vq; 989a62a8ef9SStefan Hajnoczi ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC); 990a62a8ef9SStefan Hajnoczi if (ret < 0) { 991a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 992a62a8ef9SStefan Hajnoczi goto out; 993a62a8ef9SStefan Hajnoczi } 994a62a8ef9SStefan Hajnoczi 9955dbe190fSVivek Goyal /* Request successfully sent. */ 9965dbe190fSVivek Goyal fpq = &fsvq->fud->pq; 9975dbe190fSVivek Goyal spin_lock(&fpq->lock); 9985dbe190fSVivek Goyal list_add_tail(&req->list, fpq->processing); 9995dbe190fSVivek Goyal spin_unlock(&fpq->lock); 10005dbe190fSVivek Goyal set_bit(FR_SENT, &req->flags); 10015dbe190fSVivek Goyal /* matches barrier in request_wait_answer() */ 10025dbe190fSVivek Goyal smp_mb__after_atomic(); 10035dbe190fSVivek Goyal 1004a9bfd9ddSVivek Goyal if (!in_flight) 1005c17ea009SVivek Goyal inc_in_flight_req(fsvq); 1006a62a8ef9SStefan Hajnoczi notify = virtqueue_kick_prepare(vq); 1007a62a8ef9SStefan Hajnoczi 1008a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 1009a62a8ef9SStefan Hajnoczi 1010a62a8ef9SStefan Hajnoczi if (notify) 1011a62a8ef9SStefan Hajnoczi virtqueue_notify(vq); 1012a62a8ef9SStefan Hajnoczi 1013a62a8ef9SStefan Hajnoczi out: 1014a62a8ef9SStefan Hajnoczi if (ret < 0 && req->argbuf) { 1015a62a8ef9SStefan Hajnoczi kfree(req->argbuf); 1016a62a8ef9SStefan Hajnoczi req->argbuf = NULL; 1017a62a8ef9SStefan Hajnoczi } 1018a62a8ef9SStefan Hajnoczi if (sgs != stack_sgs) { 1019a62a8ef9SStefan Hajnoczi kfree(sgs); 1020a62a8ef9SStefan Hajnoczi kfree(sg); 1021a62a8ef9SStefan Hajnoczi } 1022a62a8ef9SStefan Hajnoczi 1023a62a8ef9SStefan Hajnoczi return ret; 1024a62a8ef9SStefan Hajnoczi } 1025a62a8ef9SStefan Hajnoczi 1026a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq) 1027a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 1028a62a8ef9SStefan Hajnoczi { 1029a62a8ef9SStefan Hajnoczi unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */ 1030a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 1031a62a8ef9SStefan Hajnoczi struct fuse_req *req; 103251fecdd2SVivek Goyal struct virtio_fs_vq *fsvq; 1033a62a8ef9SStefan Hajnoczi int ret; 1034a62a8ef9SStefan Hajnoczi 1035a62a8ef9SStefan Hajnoczi WARN_ON(list_empty(&fiq->pending)); 1036a62a8ef9SStefan Hajnoczi req = list_last_entry(&fiq->pending, struct fuse_req, list); 1037a62a8ef9SStefan Hajnoczi clear_bit(FR_PENDING, &req->flags); 1038a62a8ef9SStefan Hajnoczi list_del_init(&req->list); 1039a62a8ef9SStefan Hajnoczi WARN_ON(!list_empty(&fiq->pending)); 1040a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 1041a62a8ef9SStefan Hajnoczi 1042a62a8ef9SStefan Hajnoczi fs = fiq->priv; 1043a62a8ef9SStefan Hajnoczi 1044a62a8ef9SStefan Hajnoczi pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n", 1045a62a8ef9SStefan Hajnoczi __func__, req->in.h.opcode, req->in.h.unique, 1046a62a8ef9SStefan Hajnoczi req->in.h.nodeid, req->in.h.len, 1047a62a8ef9SStefan Hajnoczi fuse_len_args(req->args->out_numargs, req->args->out_args)); 1048a62a8ef9SStefan Hajnoczi 104951fecdd2SVivek Goyal fsvq = &fs->vqs[queue_id]; 1050a9bfd9ddSVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req, false); 1051a62a8ef9SStefan Hajnoczi if (ret < 0) { 1052a62a8ef9SStefan Hajnoczi if (ret == -ENOMEM || ret == -ENOSPC) { 1053a9bfd9ddSVivek Goyal /* 1054a9bfd9ddSVivek Goyal * Virtqueue full. Retry submission from worker 1055a9bfd9ddSVivek Goyal * context as we might be holding fc->bg_lock. 1056a9bfd9ddSVivek Goyal */ 1057a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 1058a9bfd9ddSVivek Goyal list_add_tail(&req->list, &fsvq->queued_reqs); 1059a9bfd9ddSVivek Goyal inc_in_flight_req(fsvq); 1060a9bfd9ddSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 1061a9bfd9ddSVivek Goyal msecs_to_jiffies(1)); 1062a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 1063a9bfd9ddSVivek Goyal return; 1064a62a8ef9SStefan Hajnoczi } 1065a62a8ef9SStefan Hajnoczi req->out.h.error = ret; 1066a62a8ef9SStefan Hajnoczi pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); 106751fecdd2SVivek Goyal 106851fecdd2SVivek Goyal /* Can't end request in submission context. Use a worker */ 106951fecdd2SVivek Goyal spin_lock(&fsvq->lock); 107051fecdd2SVivek Goyal list_add_tail(&req->list, &fsvq->end_reqs); 107151fecdd2SVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 0); 107251fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 1073a62a8ef9SStefan Hajnoczi return; 1074a62a8ef9SStefan Hajnoczi } 1075a62a8ef9SStefan Hajnoczi } 1076a62a8ef9SStefan Hajnoczi 107700929447SYueHaibing static const struct fuse_iqueue_ops virtio_fs_fiq_ops = { 1078a62a8ef9SStefan Hajnoczi .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock, 1079a62a8ef9SStefan Hajnoczi .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock, 1080a62a8ef9SStefan Hajnoczi .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock, 1081a62a8ef9SStefan Hajnoczi .release = virtio_fs_fiq_release, 1082a62a8ef9SStefan Hajnoczi }; 1083a62a8ef9SStefan Hajnoczi 1084a62a8ef9SStefan Hajnoczi static int virtio_fs_fill_super(struct super_block *sb) 1085a62a8ef9SStefan Hajnoczi { 1086a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = get_fuse_conn_super(sb); 1087a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = fc->iq.priv; 1088a62a8ef9SStefan Hajnoczi unsigned int i; 1089a62a8ef9SStefan Hajnoczi int err; 1090a62a8ef9SStefan Hajnoczi struct fuse_fs_context ctx = { 1091a62a8ef9SStefan Hajnoczi .rootmode = S_IFDIR, 1092a62a8ef9SStefan Hajnoczi .default_permissions = 1, 1093a62a8ef9SStefan Hajnoczi .allow_other = 1, 1094a62a8ef9SStefan Hajnoczi .max_read = UINT_MAX, 1095a62a8ef9SStefan Hajnoczi .blksize = 512, 1096a62a8ef9SStefan Hajnoczi .destroy = true, 1097a62a8ef9SStefan Hajnoczi .no_control = true, 1098a62a8ef9SStefan Hajnoczi .no_force_umount = true, 10993f22c746SMiklos Szeredi .no_mount_options = true, 1100a62a8ef9SStefan Hajnoczi }; 1101a62a8ef9SStefan Hajnoczi 1102a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 1103a62a8ef9SStefan Hajnoczi 1104a62a8ef9SStefan Hajnoczi /* After holding mutex, make sure virtiofs device is still there. 1105a62a8ef9SStefan Hajnoczi * Though we are holding a reference to it, drive ->remove might 1106a62a8ef9SStefan Hajnoczi * still have cleaned up virtual queues. In that case bail out. 1107a62a8ef9SStefan Hajnoczi */ 1108a62a8ef9SStefan Hajnoczi err = -EINVAL; 1109a62a8ef9SStefan Hajnoczi if (list_empty(&fs->list)) { 1110a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fs->tag); 1111a62a8ef9SStefan Hajnoczi goto err; 1112a62a8ef9SStefan Hajnoczi } 1113a62a8ef9SStefan Hajnoczi 1114a62a8ef9SStefan Hajnoczi err = -ENOMEM; 1115a62a8ef9SStefan Hajnoczi /* Allocate fuse_dev for hiprio and notification queues */ 11167fd3abfaSVivek Goyal for (i = 0; i < fs->nvqs; i++) { 1117a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1118a62a8ef9SStefan Hajnoczi 1119a62a8ef9SStefan Hajnoczi fsvq->fud = fuse_dev_alloc(); 1120a62a8ef9SStefan Hajnoczi if (!fsvq->fud) 1121a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs; 1122a62a8ef9SStefan Hajnoczi } 1123a62a8ef9SStefan Hajnoczi 11247fd3abfaSVivek Goyal /* virtiofs allocates and installs its own fuse devices */ 11257fd3abfaSVivek Goyal ctx.fudptr = NULL; 1126a62a8ef9SStefan Hajnoczi err = fuse_fill_super_common(sb, &ctx); 1127a62a8ef9SStefan Hajnoczi if (err < 0) 1128a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs; 1129a62a8ef9SStefan Hajnoczi 1130a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 1131a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1132a62a8ef9SStefan Hajnoczi 1133a62a8ef9SStefan Hajnoczi fuse_dev_install(fsvq->fud, fc); 1134a62a8ef9SStefan Hajnoczi } 1135a62a8ef9SStefan Hajnoczi 1136a62a8ef9SStefan Hajnoczi /* Previous unmount will stop all queues. Start these again */ 1137a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs); 1138a62a8ef9SStefan Hajnoczi fuse_send_init(fc); 1139a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1140a62a8ef9SStefan Hajnoczi return 0; 1141a62a8ef9SStefan Hajnoczi 1142a62a8ef9SStefan Hajnoczi err_free_fuse_devs: 1143a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(fs); 1144a62a8ef9SStefan Hajnoczi err: 1145a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1146a62a8ef9SStefan Hajnoczi return err; 1147a62a8ef9SStefan Hajnoczi } 1148a62a8ef9SStefan Hajnoczi 1149a62a8ef9SStefan Hajnoczi static void virtio_kill_sb(struct super_block *sb) 1150a62a8ef9SStefan Hajnoczi { 1151a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = get_fuse_conn_super(sb); 1152a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs; 1153a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 1154a62a8ef9SStefan Hajnoczi 1155a62a8ef9SStefan Hajnoczi /* If mount failed, we can still be called without any fc */ 1156a62a8ef9SStefan Hajnoczi if (!fc) 1157a62a8ef9SStefan Hajnoczi return fuse_kill_sb_anon(sb); 1158a62a8ef9SStefan Hajnoczi 1159a62a8ef9SStefan Hajnoczi vfs = fc->iq.priv; 1160a62a8ef9SStefan Hajnoczi fsvq = &vfs->vqs[VQ_HIPRIO]; 1161a62a8ef9SStefan Hajnoczi 1162a62a8ef9SStefan Hajnoczi /* Stop forget queue. Soon destroy will be sent */ 1163a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 1164a62a8ef9SStefan Hajnoczi fsvq->connected = false; 1165a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 1166a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs); 1167a62a8ef9SStefan Hajnoczi 1168a62a8ef9SStefan Hajnoczi fuse_kill_sb_anon(sb); 1169a62a8ef9SStefan Hajnoczi 1170a62a8ef9SStefan Hajnoczi /* fuse_kill_sb_anon() must have sent destroy. Stop all queues 1171a62a8ef9SStefan Hajnoczi * and drain one more time and free fuse devices. Freeing fuse 1172a62a8ef9SStefan Hajnoczi * devices will drop their reference on fuse_conn and that in 1173a62a8ef9SStefan Hajnoczi * turn will drop its reference on virtio_fs object. 1174a62a8ef9SStefan Hajnoczi */ 1175a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(vfs); 1176a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs); 1177a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(vfs); 1178a62a8ef9SStefan Hajnoczi } 1179a62a8ef9SStefan Hajnoczi 1180a62a8ef9SStefan Hajnoczi static int virtio_fs_test_super(struct super_block *sb, 1181a62a8ef9SStefan Hajnoczi struct fs_context *fsc) 1182a62a8ef9SStefan Hajnoczi { 1183a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = fsc->s_fs_info; 1184a62a8ef9SStefan Hajnoczi 1185a62a8ef9SStefan Hajnoczi return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv; 1186a62a8ef9SStefan Hajnoczi } 1187a62a8ef9SStefan Hajnoczi 1188a62a8ef9SStefan Hajnoczi static int virtio_fs_set_super(struct super_block *sb, 1189a62a8ef9SStefan Hajnoczi struct fs_context *fsc) 1190a62a8ef9SStefan Hajnoczi { 1191a62a8ef9SStefan Hajnoczi int err; 1192a62a8ef9SStefan Hajnoczi 1193a62a8ef9SStefan Hajnoczi err = get_anon_bdev(&sb->s_dev); 1194a62a8ef9SStefan Hajnoczi if (!err) 1195a62a8ef9SStefan Hajnoczi fuse_conn_get(fsc->s_fs_info); 1196a62a8ef9SStefan Hajnoczi 1197a62a8ef9SStefan Hajnoczi return err; 1198a62a8ef9SStefan Hajnoczi } 1199a62a8ef9SStefan Hajnoczi 1200a62a8ef9SStefan Hajnoczi static int virtio_fs_get_tree(struct fs_context *fsc) 1201a62a8ef9SStefan Hajnoczi { 1202a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 1203a62a8ef9SStefan Hajnoczi struct super_block *sb; 1204a62a8ef9SStefan Hajnoczi struct fuse_conn *fc; 1205a62a8ef9SStefan Hajnoczi int err; 1206a62a8ef9SStefan Hajnoczi 1207a62a8ef9SStefan Hajnoczi /* This gets a reference on virtio_fs object. This ptr gets installed 1208a62a8ef9SStefan Hajnoczi * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() 1209a62a8ef9SStefan Hajnoczi * to drop the reference to this object. 1210a62a8ef9SStefan Hajnoczi */ 1211a62a8ef9SStefan Hajnoczi fs = virtio_fs_find_instance(fsc->source); 1212a62a8ef9SStefan Hajnoczi if (!fs) { 1213a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fsc->source); 1214a62a8ef9SStefan Hajnoczi return -EINVAL; 1215a62a8ef9SStefan Hajnoczi } 1216a62a8ef9SStefan Hajnoczi 1217a62a8ef9SStefan Hajnoczi fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); 1218a62a8ef9SStefan Hajnoczi if (!fc) { 1219a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 1220a62a8ef9SStefan Hajnoczi virtio_fs_put(fs); 1221a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1222a62a8ef9SStefan Hajnoczi return -ENOMEM; 1223a62a8ef9SStefan Hajnoczi } 1224a62a8ef9SStefan Hajnoczi 1225a62a8ef9SStefan Hajnoczi fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops, 1226a62a8ef9SStefan Hajnoczi fs); 1227a62a8ef9SStefan Hajnoczi fc->release = fuse_free_conn; 1228a62a8ef9SStefan Hajnoczi fc->delete_stale = true; 1229a62a8ef9SStefan Hajnoczi 1230a62a8ef9SStefan Hajnoczi fsc->s_fs_info = fc; 1231a62a8ef9SStefan Hajnoczi sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super); 1232a62a8ef9SStefan Hajnoczi fuse_conn_put(fc); 1233a62a8ef9SStefan Hajnoczi if (IS_ERR(sb)) 1234a62a8ef9SStefan Hajnoczi return PTR_ERR(sb); 1235a62a8ef9SStefan Hajnoczi 1236a62a8ef9SStefan Hajnoczi if (!sb->s_root) { 1237a62a8ef9SStefan Hajnoczi err = virtio_fs_fill_super(sb); 1238a62a8ef9SStefan Hajnoczi if (err) { 1239a62a8ef9SStefan Hajnoczi deactivate_locked_super(sb); 1240a62a8ef9SStefan Hajnoczi return err; 1241a62a8ef9SStefan Hajnoczi } 1242a62a8ef9SStefan Hajnoczi 1243a62a8ef9SStefan Hajnoczi sb->s_flags |= SB_ACTIVE; 1244a62a8ef9SStefan Hajnoczi } 1245a62a8ef9SStefan Hajnoczi 1246a62a8ef9SStefan Hajnoczi WARN_ON(fsc->root); 1247a62a8ef9SStefan Hajnoczi fsc->root = dget(sb->s_root); 1248a62a8ef9SStefan Hajnoczi return 0; 1249a62a8ef9SStefan Hajnoczi } 1250a62a8ef9SStefan Hajnoczi 1251a62a8ef9SStefan Hajnoczi static const struct fs_context_operations virtio_fs_context_ops = { 1252a62a8ef9SStefan Hajnoczi .get_tree = virtio_fs_get_tree, 1253a62a8ef9SStefan Hajnoczi }; 1254a62a8ef9SStefan Hajnoczi 1255a62a8ef9SStefan Hajnoczi static int virtio_fs_init_fs_context(struct fs_context *fsc) 1256a62a8ef9SStefan Hajnoczi { 1257a62a8ef9SStefan Hajnoczi fsc->ops = &virtio_fs_context_ops; 1258a62a8ef9SStefan Hajnoczi return 0; 1259a62a8ef9SStefan Hajnoczi } 1260a62a8ef9SStefan Hajnoczi 1261a62a8ef9SStefan Hajnoczi static struct file_system_type virtio_fs_type = { 1262a62a8ef9SStefan Hajnoczi .owner = THIS_MODULE, 1263a62a8ef9SStefan Hajnoczi .name = "virtiofs", 1264a62a8ef9SStefan Hajnoczi .init_fs_context = virtio_fs_init_fs_context, 1265a62a8ef9SStefan Hajnoczi .kill_sb = virtio_kill_sb, 1266a62a8ef9SStefan Hajnoczi }; 1267a62a8ef9SStefan Hajnoczi 1268a62a8ef9SStefan Hajnoczi static int __init virtio_fs_init(void) 1269a62a8ef9SStefan Hajnoczi { 1270a62a8ef9SStefan Hajnoczi int ret; 1271a62a8ef9SStefan Hajnoczi 1272a62a8ef9SStefan Hajnoczi ret = register_virtio_driver(&virtio_fs_driver); 1273a62a8ef9SStefan Hajnoczi if (ret < 0) 1274a62a8ef9SStefan Hajnoczi return ret; 1275a62a8ef9SStefan Hajnoczi 1276a62a8ef9SStefan Hajnoczi ret = register_filesystem(&virtio_fs_type); 1277a62a8ef9SStefan Hajnoczi if (ret < 0) { 1278a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver); 1279a62a8ef9SStefan Hajnoczi return ret; 1280a62a8ef9SStefan Hajnoczi } 1281a62a8ef9SStefan Hajnoczi 1282a62a8ef9SStefan Hajnoczi return 0; 1283a62a8ef9SStefan Hajnoczi } 1284a62a8ef9SStefan Hajnoczi module_init(virtio_fs_init); 1285a62a8ef9SStefan Hajnoczi 1286a62a8ef9SStefan Hajnoczi static void __exit virtio_fs_exit(void) 1287a62a8ef9SStefan Hajnoczi { 1288a62a8ef9SStefan Hajnoczi unregister_filesystem(&virtio_fs_type); 1289a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver); 1290a62a8ef9SStefan Hajnoczi } 1291a62a8ef9SStefan Hajnoczi module_exit(virtio_fs_exit); 1292a62a8ef9SStefan Hajnoczi 1293a62a8ef9SStefan Hajnoczi MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>"); 1294a62a8ef9SStefan Hajnoczi MODULE_DESCRIPTION("Virtio Filesystem"); 1295a62a8ef9SStefan Hajnoczi MODULE_LICENSE("GPL"); 1296a62a8ef9SStefan Hajnoczi MODULE_ALIAS_FS(KBUILD_MODNAME); 1297a62a8ef9SStefan Hajnoczi MODULE_DEVICE_TABLE(virtio, id_table); 1298