1a62a8ef9SStefan Hajnoczi // SPDX-License-Identifier: GPL-2.0 2a62a8ef9SStefan Hajnoczi /* 3a62a8ef9SStefan Hajnoczi * virtio-fs: Virtio Filesystem 4a62a8ef9SStefan Hajnoczi * Copyright (C) 2018 Red Hat, Inc. 5a62a8ef9SStefan Hajnoczi */ 6a62a8ef9SStefan Hajnoczi 7a62a8ef9SStefan Hajnoczi #include <linux/fs.h> 8a62a8ef9SStefan Hajnoczi #include <linux/module.h> 9a62a8ef9SStefan Hajnoczi #include <linux/virtio.h> 10a62a8ef9SStefan Hajnoczi #include <linux/virtio_fs.h> 11a62a8ef9SStefan Hajnoczi #include <linux/delay.h> 12a62a8ef9SStefan Hajnoczi #include <linux/fs_context.h> 13a62a8ef9SStefan Hajnoczi #include <linux/highmem.h> 14a62a8ef9SStefan Hajnoczi #include "fuse_i.h" 15a62a8ef9SStefan Hajnoczi 16a62a8ef9SStefan Hajnoczi /* List of virtio-fs device instances and a lock for the list. Also provides 17a62a8ef9SStefan Hajnoczi * mutual exclusion in device removal and mounting path 18a62a8ef9SStefan Hajnoczi */ 19a62a8ef9SStefan Hajnoczi static DEFINE_MUTEX(virtio_fs_mutex); 20a62a8ef9SStefan Hajnoczi static LIST_HEAD(virtio_fs_instances); 21a62a8ef9SStefan Hajnoczi 22a62a8ef9SStefan Hajnoczi enum { 23a62a8ef9SStefan Hajnoczi VQ_HIPRIO, 24a62a8ef9SStefan Hajnoczi VQ_REQUEST 25a62a8ef9SStefan Hajnoczi }; 26a62a8ef9SStefan Hajnoczi 27a62a8ef9SStefan Hajnoczi /* Per-virtqueue state */ 28a62a8ef9SStefan Hajnoczi struct virtio_fs_vq { 29a62a8ef9SStefan Hajnoczi spinlock_t lock; 30a62a8ef9SStefan Hajnoczi struct virtqueue *vq; /* protected by ->lock */ 31a62a8ef9SStefan Hajnoczi struct work_struct done_work; 32a62a8ef9SStefan Hajnoczi struct list_head queued_reqs; 3351fecdd2SVivek Goyal struct list_head end_reqs; /* End these requests */ 34a62a8ef9SStefan Hajnoczi struct delayed_work dispatch_work; 35a62a8ef9SStefan Hajnoczi struct fuse_dev *fud; 36a62a8ef9SStefan Hajnoczi bool connected; 37a62a8ef9SStefan Hajnoczi long in_flight; 38a62a8ef9SStefan Hajnoczi char name[24]; 39a62a8ef9SStefan Hajnoczi } ____cacheline_aligned_in_smp; 40a62a8ef9SStefan Hajnoczi 41a62a8ef9SStefan Hajnoczi /* A virtio-fs device instance */ 42a62a8ef9SStefan Hajnoczi struct virtio_fs { 43a62a8ef9SStefan Hajnoczi struct kref refcount; 44a62a8ef9SStefan Hajnoczi struct list_head list; /* on virtio_fs_instances */ 45a62a8ef9SStefan Hajnoczi char *tag; 46a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *vqs; 47a62a8ef9SStefan Hajnoczi unsigned int nvqs; /* number of virtqueues */ 48a62a8ef9SStefan Hajnoczi unsigned int num_request_queues; /* number of request queues */ 49a62a8ef9SStefan Hajnoczi }; 50a62a8ef9SStefan Hajnoczi 51a62a8ef9SStefan Hajnoczi struct virtio_fs_forget { 52a62a8ef9SStefan Hajnoczi struct fuse_in_header ih; 53a62a8ef9SStefan Hajnoczi struct fuse_forget_in arg; 54a62a8ef9SStefan Hajnoczi /* This request can be temporarily queued on virt queue */ 55a62a8ef9SStefan Hajnoczi struct list_head list; 56a62a8ef9SStefan Hajnoczi }; 57a62a8ef9SStefan Hajnoczi 58a9bfd9ddSVivek Goyal static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 59a9bfd9ddSVivek Goyal struct fuse_req *req, bool in_flight); 60a9bfd9ddSVivek Goyal 61a62a8ef9SStefan Hajnoczi static inline struct virtio_fs_vq *vq_to_fsvq(struct virtqueue *vq) 62a62a8ef9SStefan Hajnoczi { 63a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vq->vdev->priv; 64a62a8ef9SStefan Hajnoczi 65a62a8ef9SStefan Hajnoczi return &fs->vqs[vq->index]; 66a62a8ef9SStefan Hajnoczi } 67a62a8ef9SStefan Hajnoczi 68a62a8ef9SStefan Hajnoczi static inline struct fuse_pqueue *vq_to_fpq(struct virtqueue *vq) 69a62a8ef9SStefan Hajnoczi { 70a62a8ef9SStefan Hajnoczi return &vq_to_fsvq(vq)->fud->pq; 71a62a8ef9SStefan Hajnoczi } 72a62a8ef9SStefan Hajnoczi 73c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */ 74c17ea009SVivek Goyal static inline void inc_in_flight_req(struct virtio_fs_vq *fsvq) 75c17ea009SVivek Goyal { 76c17ea009SVivek Goyal fsvq->in_flight++; 77c17ea009SVivek Goyal } 78c17ea009SVivek Goyal 79c17ea009SVivek Goyal /* Should be called with fsvq->lock held. */ 80c17ea009SVivek Goyal static inline void dec_in_flight_req(struct virtio_fs_vq *fsvq) 81c17ea009SVivek Goyal { 82c17ea009SVivek Goyal WARN_ON(fsvq->in_flight <= 0); 83c17ea009SVivek Goyal fsvq->in_flight--; 84c17ea009SVivek Goyal } 85c17ea009SVivek Goyal 86a62a8ef9SStefan Hajnoczi static void release_virtio_fs_obj(struct kref *ref) 87a62a8ef9SStefan Hajnoczi { 88a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = container_of(ref, struct virtio_fs, refcount); 89a62a8ef9SStefan Hajnoczi 90a62a8ef9SStefan Hajnoczi kfree(vfs->vqs); 91a62a8ef9SStefan Hajnoczi kfree(vfs); 92a62a8ef9SStefan Hajnoczi } 93a62a8ef9SStefan Hajnoczi 94a62a8ef9SStefan Hajnoczi /* Make sure virtiofs_mutex is held */ 95a62a8ef9SStefan Hajnoczi static void virtio_fs_put(struct virtio_fs *fs) 96a62a8ef9SStefan Hajnoczi { 97a62a8ef9SStefan Hajnoczi kref_put(&fs->refcount, release_virtio_fs_obj); 98a62a8ef9SStefan Hajnoczi } 99a62a8ef9SStefan Hajnoczi 100a62a8ef9SStefan Hajnoczi static void virtio_fs_fiq_release(struct fuse_iqueue *fiq) 101a62a8ef9SStefan Hajnoczi { 102a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs = fiq->priv; 103a62a8ef9SStefan Hajnoczi 104a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 105a62a8ef9SStefan Hajnoczi virtio_fs_put(vfs); 106a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 107a62a8ef9SStefan Hajnoczi } 108a62a8ef9SStefan Hajnoczi 109a62a8ef9SStefan Hajnoczi static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq) 110a62a8ef9SStefan Hajnoczi { 111a62a8ef9SStefan Hajnoczi WARN_ON(fsvq->in_flight < 0); 112a62a8ef9SStefan Hajnoczi 113a62a8ef9SStefan Hajnoczi /* Wait for in flight requests to finish.*/ 114a62a8ef9SStefan Hajnoczi while (1) { 115a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 116a62a8ef9SStefan Hajnoczi if (!fsvq->in_flight) { 117a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 118a62a8ef9SStefan Hajnoczi break; 119a62a8ef9SStefan Hajnoczi } 120a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 121a62a8ef9SStefan Hajnoczi /* TODO use completion instead of timeout */ 122a62a8ef9SStefan Hajnoczi usleep_range(1000, 2000); 123a62a8ef9SStefan Hajnoczi } 124a62a8ef9SStefan Hajnoczi 125a62a8ef9SStefan Hajnoczi flush_work(&fsvq->done_work); 126a62a8ef9SStefan Hajnoczi flush_delayed_work(&fsvq->dispatch_work); 127a62a8ef9SStefan Hajnoczi } 128a62a8ef9SStefan Hajnoczi 129a62a8ef9SStefan Hajnoczi static void virtio_fs_drain_all_queues(struct virtio_fs *fs) 130a62a8ef9SStefan Hajnoczi { 131a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 132a62a8ef9SStefan Hajnoczi int i; 133a62a8ef9SStefan Hajnoczi 134a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 135a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 136a62a8ef9SStefan Hajnoczi virtio_fs_drain_queue(fsvq); 137a62a8ef9SStefan Hajnoczi } 138a62a8ef9SStefan Hajnoczi } 139a62a8ef9SStefan Hajnoczi 140a62a8ef9SStefan Hajnoczi static void virtio_fs_start_all_queues(struct virtio_fs *fs) 141a62a8ef9SStefan Hajnoczi { 142a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 143a62a8ef9SStefan Hajnoczi int i; 144a62a8ef9SStefan Hajnoczi 145a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 146a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 147a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 148a62a8ef9SStefan Hajnoczi fsvq->connected = true; 149a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 150a62a8ef9SStefan Hajnoczi } 151a62a8ef9SStefan Hajnoczi } 152a62a8ef9SStefan Hajnoczi 153a62a8ef9SStefan Hajnoczi /* Add a new instance to the list or return -EEXIST if tag name exists*/ 154a62a8ef9SStefan Hajnoczi static int virtio_fs_add_instance(struct virtio_fs *fs) 155a62a8ef9SStefan Hajnoczi { 156a62a8ef9SStefan Hajnoczi struct virtio_fs *fs2; 157a62a8ef9SStefan Hajnoczi bool duplicate = false; 158a62a8ef9SStefan Hajnoczi 159a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 160a62a8ef9SStefan Hajnoczi 161a62a8ef9SStefan Hajnoczi list_for_each_entry(fs2, &virtio_fs_instances, list) { 162a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, fs2->tag) == 0) 163a62a8ef9SStefan Hajnoczi duplicate = true; 164a62a8ef9SStefan Hajnoczi } 165a62a8ef9SStefan Hajnoczi 166a62a8ef9SStefan Hajnoczi if (!duplicate) 167a62a8ef9SStefan Hajnoczi list_add_tail(&fs->list, &virtio_fs_instances); 168a62a8ef9SStefan Hajnoczi 169a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 170a62a8ef9SStefan Hajnoczi 171a62a8ef9SStefan Hajnoczi if (duplicate) 172a62a8ef9SStefan Hajnoczi return -EEXIST; 173a62a8ef9SStefan Hajnoczi return 0; 174a62a8ef9SStefan Hajnoczi } 175a62a8ef9SStefan Hajnoczi 176a62a8ef9SStefan Hajnoczi /* Return the virtio_fs with a given tag, or NULL */ 177a62a8ef9SStefan Hajnoczi static struct virtio_fs *virtio_fs_find_instance(const char *tag) 178a62a8ef9SStefan Hajnoczi { 179a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 180a62a8ef9SStefan Hajnoczi 181a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 182a62a8ef9SStefan Hajnoczi 183a62a8ef9SStefan Hajnoczi list_for_each_entry(fs, &virtio_fs_instances, list) { 184a62a8ef9SStefan Hajnoczi if (strcmp(fs->tag, tag) == 0) { 185a62a8ef9SStefan Hajnoczi kref_get(&fs->refcount); 186a62a8ef9SStefan Hajnoczi goto found; 187a62a8ef9SStefan Hajnoczi } 188a62a8ef9SStefan Hajnoczi } 189a62a8ef9SStefan Hajnoczi 190a62a8ef9SStefan Hajnoczi fs = NULL; /* not found */ 191a62a8ef9SStefan Hajnoczi 192a62a8ef9SStefan Hajnoczi found: 193a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 194a62a8ef9SStefan Hajnoczi 195a62a8ef9SStefan Hajnoczi return fs; 196a62a8ef9SStefan Hajnoczi } 197a62a8ef9SStefan Hajnoczi 198a62a8ef9SStefan Hajnoczi static void virtio_fs_free_devs(struct virtio_fs *fs) 199a62a8ef9SStefan Hajnoczi { 200a62a8ef9SStefan Hajnoczi unsigned int i; 201a62a8ef9SStefan Hajnoczi 202a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 203a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 204a62a8ef9SStefan Hajnoczi 205a62a8ef9SStefan Hajnoczi if (!fsvq->fud) 206a62a8ef9SStefan Hajnoczi continue; 207a62a8ef9SStefan Hajnoczi 208a62a8ef9SStefan Hajnoczi fuse_dev_free(fsvq->fud); 209a62a8ef9SStefan Hajnoczi fsvq->fud = NULL; 210a62a8ef9SStefan Hajnoczi } 211a62a8ef9SStefan Hajnoczi } 212a62a8ef9SStefan Hajnoczi 213a62a8ef9SStefan Hajnoczi /* Read filesystem name from virtio config into fs->tag (must kfree()). */ 214a62a8ef9SStefan Hajnoczi static int virtio_fs_read_tag(struct virtio_device *vdev, struct virtio_fs *fs) 215a62a8ef9SStefan Hajnoczi { 216a62a8ef9SStefan Hajnoczi char tag_buf[sizeof_field(struct virtio_fs_config, tag)]; 217a62a8ef9SStefan Hajnoczi char *end; 218a62a8ef9SStefan Hajnoczi size_t len; 219a62a8ef9SStefan Hajnoczi 220a62a8ef9SStefan Hajnoczi virtio_cread_bytes(vdev, offsetof(struct virtio_fs_config, tag), 221a62a8ef9SStefan Hajnoczi &tag_buf, sizeof(tag_buf)); 222a62a8ef9SStefan Hajnoczi end = memchr(tag_buf, '\0', sizeof(tag_buf)); 223a62a8ef9SStefan Hajnoczi if (end == tag_buf) 224a62a8ef9SStefan Hajnoczi return -EINVAL; /* empty tag */ 225a62a8ef9SStefan Hajnoczi if (!end) 226a62a8ef9SStefan Hajnoczi end = &tag_buf[sizeof(tag_buf)]; 227a62a8ef9SStefan Hajnoczi 228a62a8ef9SStefan Hajnoczi len = end - tag_buf; 229a62a8ef9SStefan Hajnoczi fs->tag = devm_kmalloc(&vdev->dev, len + 1, GFP_KERNEL); 230a62a8ef9SStefan Hajnoczi if (!fs->tag) 231a62a8ef9SStefan Hajnoczi return -ENOMEM; 232a62a8ef9SStefan Hajnoczi memcpy(fs->tag, tag_buf, len); 233a62a8ef9SStefan Hajnoczi fs->tag[len] = '\0'; 234a62a8ef9SStefan Hajnoczi return 0; 235a62a8ef9SStefan Hajnoczi } 236a62a8ef9SStefan Hajnoczi 237a62a8ef9SStefan Hajnoczi /* Work function for hiprio completion */ 238a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_done_work(struct work_struct *work) 239a62a8ef9SStefan Hajnoczi { 240a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 241a62a8ef9SStefan Hajnoczi done_work); 242a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq; 243a62a8ef9SStefan Hajnoczi 244a62a8ef9SStefan Hajnoczi /* Free completed FUSE_FORGET requests */ 245a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 246a62a8ef9SStefan Hajnoczi do { 247a62a8ef9SStefan Hajnoczi unsigned int len; 248a62a8ef9SStefan Hajnoczi void *req; 249a62a8ef9SStefan Hajnoczi 250a62a8ef9SStefan Hajnoczi virtqueue_disable_cb(vq); 251a62a8ef9SStefan Hajnoczi 252a62a8ef9SStefan Hajnoczi while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 253a62a8ef9SStefan Hajnoczi kfree(req); 254c17ea009SVivek Goyal dec_in_flight_req(fsvq); 255a62a8ef9SStefan Hajnoczi } 256a62a8ef9SStefan Hajnoczi } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 257a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 258a62a8ef9SStefan Hajnoczi } 259a62a8ef9SStefan Hajnoczi 26051fecdd2SVivek Goyal static void virtio_fs_request_dispatch_work(struct work_struct *work) 261a62a8ef9SStefan Hajnoczi { 26251fecdd2SVivek Goyal struct fuse_req *req; 26351fecdd2SVivek Goyal struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 26451fecdd2SVivek Goyal dispatch_work.work); 26551fecdd2SVivek Goyal struct fuse_conn *fc = fsvq->fud->fc; 266a9bfd9ddSVivek Goyal int ret; 26751fecdd2SVivek Goyal 26851fecdd2SVivek Goyal pr_debug("virtio-fs: worker %s called.\n", __func__); 26951fecdd2SVivek Goyal while (1) { 27051fecdd2SVivek Goyal spin_lock(&fsvq->lock); 27151fecdd2SVivek Goyal req = list_first_entry_or_null(&fsvq->end_reqs, struct fuse_req, 27251fecdd2SVivek Goyal list); 27351fecdd2SVivek Goyal if (!req) { 27451fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 275a9bfd9ddSVivek Goyal break; 27651fecdd2SVivek Goyal } 27751fecdd2SVivek Goyal 27851fecdd2SVivek Goyal list_del_init(&req->list); 27951fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 28051fecdd2SVivek Goyal fuse_request_end(fc, req); 28151fecdd2SVivek Goyal } 282a9bfd9ddSVivek Goyal 283a9bfd9ddSVivek Goyal /* Dispatch pending requests */ 284a9bfd9ddSVivek Goyal while (1) { 285a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 286a9bfd9ddSVivek Goyal req = list_first_entry_or_null(&fsvq->queued_reqs, 287a9bfd9ddSVivek Goyal struct fuse_req, list); 288a9bfd9ddSVivek Goyal if (!req) { 289a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 290a9bfd9ddSVivek Goyal return; 291a9bfd9ddSVivek Goyal } 292a9bfd9ddSVivek Goyal list_del_init(&req->list); 293a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 294a9bfd9ddSVivek Goyal 295a9bfd9ddSVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req, true); 296a9bfd9ddSVivek Goyal if (ret < 0) { 297a9bfd9ddSVivek Goyal if (ret == -ENOMEM || ret == -ENOSPC) { 298a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 299a9bfd9ddSVivek Goyal list_add_tail(&req->list, &fsvq->queued_reqs); 300a9bfd9ddSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 301a9bfd9ddSVivek Goyal msecs_to_jiffies(1)); 302a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 303a9bfd9ddSVivek Goyal return; 304a9bfd9ddSVivek Goyal } 305a9bfd9ddSVivek Goyal req->out.h.error = ret; 306a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 307a9bfd9ddSVivek Goyal dec_in_flight_req(fsvq); 308a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 309a9bfd9ddSVivek Goyal pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", 310a9bfd9ddSVivek Goyal ret); 311a9bfd9ddSVivek Goyal fuse_request_end(fc, req); 312a9bfd9ddSVivek Goyal } 313a9bfd9ddSVivek Goyal } 314a62a8ef9SStefan Hajnoczi } 315a62a8ef9SStefan Hajnoczi 316*58ada94fSVivek Goyal /* 317*58ada94fSVivek Goyal * Returns 1 if queue is full and sender should wait a bit before sending 318*58ada94fSVivek Goyal * next request, 0 otherwise. 319*58ada94fSVivek Goyal */ 320*58ada94fSVivek Goyal static int send_forget_request(struct virtio_fs_vq *fsvq, 321*58ada94fSVivek Goyal struct virtio_fs_forget *forget, 322*58ada94fSVivek Goyal bool in_flight) 323*58ada94fSVivek Goyal { 324*58ada94fSVivek Goyal struct scatterlist sg; 325*58ada94fSVivek Goyal struct virtqueue *vq; 326*58ada94fSVivek Goyal int ret = 0; 327*58ada94fSVivek Goyal bool notify; 328*58ada94fSVivek Goyal 329*58ada94fSVivek Goyal spin_lock(&fsvq->lock); 330*58ada94fSVivek Goyal if (!fsvq->connected) { 331*58ada94fSVivek Goyal if (in_flight) 332*58ada94fSVivek Goyal dec_in_flight_req(fsvq); 333*58ada94fSVivek Goyal kfree(forget); 334*58ada94fSVivek Goyal goto out; 335*58ada94fSVivek Goyal } 336*58ada94fSVivek Goyal 337*58ada94fSVivek Goyal sg_init_one(&sg, forget, sizeof(*forget)); 338*58ada94fSVivek Goyal vq = fsvq->vq; 339*58ada94fSVivek Goyal dev_dbg(&vq->vdev->dev, "%s\n", __func__); 340*58ada94fSVivek Goyal 341*58ada94fSVivek Goyal ret = virtqueue_add_outbuf(vq, &sg, 1, forget, GFP_ATOMIC); 342*58ada94fSVivek Goyal if (ret < 0) { 343*58ada94fSVivek Goyal if (ret == -ENOMEM || ret == -ENOSPC) { 344*58ada94fSVivek Goyal pr_debug("virtio-fs: Could not queue FORGET: err=%d. Will try later\n", 345*58ada94fSVivek Goyal ret); 346*58ada94fSVivek Goyal list_add_tail(&forget->list, &fsvq->queued_reqs); 347*58ada94fSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 348*58ada94fSVivek Goyal msecs_to_jiffies(1)); 349*58ada94fSVivek Goyal if (!in_flight) 350*58ada94fSVivek Goyal inc_in_flight_req(fsvq); 351*58ada94fSVivek Goyal /* Queue is full */ 352*58ada94fSVivek Goyal ret = 1; 353*58ada94fSVivek Goyal } else { 354*58ada94fSVivek Goyal pr_debug("virtio-fs: Could not queue FORGET: err=%d. Dropping it.\n", 355*58ada94fSVivek Goyal ret); 356*58ada94fSVivek Goyal kfree(forget); 357*58ada94fSVivek Goyal if (in_flight) 358*58ada94fSVivek Goyal dec_in_flight_req(fsvq); 359*58ada94fSVivek Goyal } 360*58ada94fSVivek Goyal goto out; 361*58ada94fSVivek Goyal } 362*58ada94fSVivek Goyal 363*58ada94fSVivek Goyal if (!in_flight) 364*58ada94fSVivek Goyal inc_in_flight_req(fsvq); 365*58ada94fSVivek Goyal notify = virtqueue_kick_prepare(vq); 366*58ada94fSVivek Goyal spin_unlock(&fsvq->lock); 367*58ada94fSVivek Goyal 368*58ada94fSVivek Goyal if (notify) 369*58ada94fSVivek Goyal virtqueue_notify(vq); 370*58ada94fSVivek Goyal return ret; 371*58ada94fSVivek Goyal out: 372*58ada94fSVivek Goyal spin_unlock(&fsvq->lock); 373*58ada94fSVivek Goyal return ret; 374*58ada94fSVivek Goyal } 375*58ada94fSVivek Goyal 376a62a8ef9SStefan Hajnoczi static void virtio_fs_hiprio_dispatch_work(struct work_struct *work) 377a62a8ef9SStefan Hajnoczi { 378a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget; 379a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 380a62a8ef9SStefan Hajnoczi dispatch_work.work); 381a62a8ef9SStefan Hajnoczi pr_debug("virtio-fs: worker %s called.\n", __func__); 382a62a8ef9SStefan Hajnoczi while (1) { 383a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 384a62a8ef9SStefan Hajnoczi forget = list_first_entry_or_null(&fsvq->queued_reqs, 385a62a8ef9SStefan Hajnoczi struct virtio_fs_forget, list); 386a62a8ef9SStefan Hajnoczi if (!forget) { 387a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 388a62a8ef9SStefan Hajnoczi return; 389a62a8ef9SStefan Hajnoczi } 390a62a8ef9SStefan Hajnoczi 391a62a8ef9SStefan Hajnoczi list_del(&forget->list); 392a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 393*58ada94fSVivek Goyal if (send_forget_request(fsvq, forget, true)) 394a62a8ef9SStefan Hajnoczi return; 395a62a8ef9SStefan Hajnoczi } 396a62a8ef9SStefan Hajnoczi } 397a62a8ef9SStefan Hajnoczi 398a62a8ef9SStefan Hajnoczi /* Allocate and copy args into req->argbuf */ 399a62a8ef9SStefan Hajnoczi static int copy_args_to_argbuf(struct fuse_req *req) 400a62a8ef9SStefan Hajnoczi { 401a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 402a62a8ef9SStefan Hajnoczi unsigned int offset = 0; 403a62a8ef9SStefan Hajnoczi unsigned int num_in; 404a62a8ef9SStefan Hajnoczi unsigned int num_out; 405a62a8ef9SStefan Hajnoczi unsigned int len; 406a62a8ef9SStefan Hajnoczi unsigned int i; 407a62a8ef9SStefan Hajnoczi 408a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages; 409a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages; 410a62a8ef9SStefan Hajnoczi len = fuse_len_args(num_in, (struct fuse_arg *) args->in_args) + 411a62a8ef9SStefan Hajnoczi fuse_len_args(num_out, args->out_args); 412a62a8ef9SStefan Hajnoczi 413a62a8ef9SStefan Hajnoczi req->argbuf = kmalloc(len, GFP_ATOMIC); 414a62a8ef9SStefan Hajnoczi if (!req->argbuf) 415a62a8ef9SStefan Hajnoczi return -ENOMEM; 416a62a8ef9SStefan Hajnoczi 417a62a8ef9SStefan Hajnoczi for (i = 0; i < num_in; i++) { 418a62a8ef9SStefan Hajnoczi memcpy(req->argbuf + offset, 419a62a8ef9SStefan Hajnoczi args->in_args[i].value, 420a62a8ef9SStefan Hajnoczi args->in_args[i].size); 421a62a8ef9SStefan Hajnoczi offset += args->in_args[i].size; 422a62a8ef9SStefan Hajnoczi } 423a62a8ef9SStefan Hajnoczi 424a62a8ef9SStefan Hajnoczi return 0; 425a62a8ef9SStefan Hajnoczi } 426a62a8ef9SStefan Hajnoczi 427a62a8ef9SStefan Hajnoczi /* Copy args out of and free req->argbuf */ 428a62a8ef9SStefan Hajnoczi static void copy_args_from_argbuf(struct fuse_args *args, struct fuse_req *req) 429a62a8ef9SStefan Hajnoczi { 430a62a8ef9SStefan Hajnoczi unsigned int remaining; 431a62a8ef9SStefan Hajnoczi unsigned int offset; 432a62a8ef9SStefan Hajnoczi unsigned int num_in; 433a62a8ef9SStefan Hajnoczi unsigned int num_out; 434a62a8ef9SStefan Hajnoczi unsigned int i; 435a62a8ef9SStefan Hajnoczi 436a62a8ef9SStefan Hajnoczi remaining = req->out.h.len - sizeof(req->out.h); 437a62a8ef9SStefan Hajnoczi num_in = args->in_numargs - args->in_pages; 438a62a8ef9SStefan Hajnoczi num_out = args->out_numargs - args->out_pages; 439a62a8ef9SStefan Hajnoczi offset = fuse_len_args(num_in, (struct fuse_arg *)args->in_args); 440a62a8ef9SStefan Hajnoczi 441a62a8ef9SStefan Hajnoczi for (i = 0; i < num_out; i++) { 442a62a8ef9SStefan Hajnoczi unsigned int argsize = args->out_args[i].size; 443a62a8ef9SStefan Hajnoczi 444a62a8ef9SStefan Hajnoczi if (args->out_argvar && 445a62a8ef9SStefan Hajnoczi i == args->out_numargs - 1 && 446a62a8ef9SStefan Hajnoczi argsize > remaining) { 447a62a8ef9SStefan Hajnoczi argsize = remaining; 448a62a8ef9SStefan Hajnoczi } 449a62a8ef9SStefan Hajnoczi 450a62a8ef9SStefan Hajnoczi memcpy(args->out_args[i].value, req->argbuf + offset, argsize); 451a62a8ef9SStefan Hajnoczi offset += argsize; 452a62a8ef9SStefan Hajnoczi 453a62a8ef9SStefan Hajnoczi if (i != args->out_numargs - 1) 454a62a8ef9SStefan Hajnoczi remaining -= argsize; 455a62a8ef9SStefan Hajnoczi } 456a62a8ef9SStefan Hajnoczi 457a62a8ef9SStefan Hajnoczi /* Store the actual size of the variable-length arg */ 458a62a8ef9SStefan Hajnoczi if (args->out_argvar) 459a62a8ef9SStefan Hajnoczi args->out_args[args->out_numargs - 1].size = remaining; 460a62a8ef9SStefan Hajnoczi 461a62a8ef9SStefan Hajnoczi kfree(req->argbuf); 462a62a8ef9SStefan Hajnoczi req->argbuf = NULL; 463a62a8ef9SStefan Hajnoczi } 464a62a8ef9SStefan Hajnoczi 465a62a8ef9SStefan Hajnoczi /* Work function for request completion */ 466a62a8ef9SStefan Hajnoczi static void virtio_fs_requests_done_work(struct work_struct *work) 467a62a8ef9SStefan Hajnoczi { 468a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = container_of(work, struct virtio_fs_vq, 469a62a8ef9SStefan Hajnoczi done_work); 470a62a8ef9SStefan Hajnoczi struct fuse_pqueue *fpq = &fsvq->fud->pq; 471a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = fsvq->fud->fc; 472a62a8ef9SStefan Hajnoczi struct virtqueue *vq = fsvq->vq; 473a62a8ef9SStefan Hajnoczi struct fuse_req *req; 474a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap; 475a62a8ef9SStefan Hajnoczi struct fuse_req *next; 476a62a8ef9SStefan Hajnoczi struct fuse_args *args; 477a62a8ef9SStefan Hajnoczi unsigned int len, i, thislen; 478a62a8ef9SStefan Hajnoczi struct page *page; 479a62a8ef9SStefan Hajnoczi LIST_HEAD(reqs); 480a62a8ef9SStefan Hajnoczi 481a62a8ef9SStefan Hajnoczi /* Collect completed requests off the virtqueue */ 482a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 483a62a8ef9SStefan Hajnoczi do { 484a62a8ef9SStefan Hajnoczi virtqueue_disable_cb(vq); 485a62a8ef9SStefan Hajnoczi 486a62a8ef9SStefan Hajnoczi while ((req = virtqueue_get_buf(vq, &len)) != NULL) { 487a62a8ef9SStefan Hajnoczi spin_lock(&fpq->lock); 488a62a8ef9SStefan Hajnoczi list_move_tail(&req->list, &reqs); 489a62a8ef9SStefan Hajnoczi spin_unlock(&fpq->lock); 490a62a8ef9SStefan Hajnoczi } 491a62a8ef9SStefan Hajnoczi } while (!virtqueue_enable_cb(vq) && likely(!virtqueue_is_broken(vq))); 492a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 493a62a8ef9SStefan Hajnoczi 494a62a8ef9SStefan Hajnoczi /* End requests */ 495a62a8ef9SStefan Hajnoczi list_for_each_entry_safe(req, next, &reqs, list) { 496a62a8ef9SStefan Hajnoczi /* 497a62a8ef9SStefan Hajnoczi * TODO verify that server properly follows FUSE protocol 498a62a8ef9SStefan Hajnoczi * (oh.uniq, oh.len) 499a62a8ef9SStefan Hajnoczi */ 500a62a8ef9SStefan Hajnoczi args = req->args; 501a62a8ef9SStefan Hajnoczi copy_args_from_argbuf(args, req); 502a62a8ef9SStefan Hajnoczi 503a62a8ef9SStefan Hajnoczi if (args->out_pages && args->page_zeroing) { 504a62a8ef9SStefan Hajnoczi len = args->out_args[args->out_numargs - 1].size; 505a62a8ef9SStefan Hajnoczi ap = container_of(args, typeof(*ap), args); 506a62a8ef9SStefan Hajnoczi for (i = 0; i < ap->num_pages; i++) { 507a62a8ef9SStefan Hajnoczi thislen = ap->descs[i].length; 508a62a8ef9SStefan Hajnoczi if (len < thislen) { 509a62a8ef9SStefan Hajnoczi WARN_ON(ap->descs[i].offset); 510a62a8ef9SStefan Hajnoczi page = ap->pages[i]; 511a62a8ef9SStefan Hajnoczi zero_user_segment(page, len, thislen); 512a62a8ef9SStefan Hajnoczi len = 0; 513a62a8ef9SStefan Hajnoczi } else { 514a62a8ef9SStefan Hajnoczi len -= thislen; 515a62a8ef9SStefan Hajnoczi } 516a62a8ef9SStefan Hajnoczi } 517a62a8ef9SStefan Hajnoczi } 518a62a8ef9SStefan Hajnoczi 519a62a8ef9SStefan Hajnoczi spin_lock(&fpq->lock); 520a62a8ef9SStefan Hajnoczi clear_bit(FR_SENT, &req->flags); 521a62a8ef9SStefan Hajnoczi list_del_init(&req->list); 522a62a8ef9SStefan Hajnoczi spin_unlock(&fpq->lock); 523a62a8ef9SStefan Hajnoczi 524a62a8ef9SStefan Hajnoczi fuse_request_end(fc, req); 525a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 526c17ea009SVivek Goyal dec_in_flight_req(fsvq); 527a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 528a62a8ef9SStefan Hajnoczi } 529a62a8ef9SStefan Hajnoczi } 530a62a8ef9SStefan Hajnoczi 531a62a8ef9SStefan Hajnoczi /* Virtqueue interrupt handler */ 532a62a8ef9SStefan Hajnoczi static void virtio_fs_vq_done(struct virtqueue *vq) 533a62a8ef9SStefan Hajnoczi { 534a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = vq_to_fsvq(vq); 535a62a8ef9SStefan Hajnoczi 536a62a8ef9SStefan Hajnoczi dev_dbg(&vq->vdev->dev, "%s %s\n", __func__, fsvq->name); 537a62a8ef9SStefan Hajnoczi 538a62a8ef9SStefan Hajnoczi schedule_work(&fsvq->done_work); 539a62a8ef9SStefan Hajnoczi } 540a62a8ef9SStefan Hajnoczi 541a62a8ef9SStefan Hajnoczi /* Initialize virtqueues */ 542a62a8ef9SStefan Hajnoczi static int virtio_fs_setup_vqs(struct virtio_device *vdev, 543a62a8ef9SStefan Hajnoczi struct virtio_fs *fs) 544a62a8ef9SStefan Hajnoczi { 545a62a8ef9SStefan Hajnoczi struct virtqueue **vqs; 546a62a8ef9SStefan Hajnoczi vq_callback_t **callbacks; 547a62a8ef9SStefan Hajnoczi const char **names; 548a62a8ef9SStefan Hajnoczi unsigned int i; 549a62a8ef9SStefan Hajnoczi int ret = 0; 550a62a8ef9SStefan Hajnoczi 551a62a8ef9SStefan Hajnoczi virtio_cread(vdev, struct virtio_fs_config, num_request_queues, 552a62a8ef9SStefan Hajnoczi &fs->num_request_queues); 553a62a8ef9SStefan Hajnoczi if (fs->num_request_queues == 0) 554a62a8ef9SStefan Hajnoczi return -EINVAL; 555a62a8ef9SStefan Hajnoczi 556a62a8ef9SStefan Hajnoczi fs->nvqs = 1 + fs->num_request_queues; 557a62a8ef9SStefan Hajnoczi fs->vqs = kcalloc(fs->nvqs, sizeof(fs->vqs[VQ_HIPRIO]), GFP_KERNEL); 558a62a8ef9SStefan Hajnoczi if (!fs->vqs) 559a62a8ef9SStefan Hajnoczi return -ENOMEM; 560a62a8ef9SStefan Hajnoczi 561a62a8ef9SStefan Hajnoczi vqs = kmalloc_array(fs->nvqs, sizeof(vqs[VQ_HIPRIO]), GFP_KERNEL); 562a62a8ef9SStefan Hajnoczi callbacks = kmalloc_array(fs->nvqs, sizeof(callbacks[VQ_HIPRIO]), 563a62a8ef9SStefan Hajnoczi GFP_KERNEL); 564a62a8ef9SStefan Hajnoczi names = kmalloc_array(fs->nvqs, sizeof(names[VQ_HIPRIO]), GFP_KERNEL); 565a62a8ef9SStefan Hajnoczi if (!vqs || !callbacks || !names) { 566a62a8ef9SStefan Hajnoczi ret = -ENOMEM; 567a62a8ef9SStefan Hajnoczi goto out; 568a62a8ef9SStefan Hajnoczi } 569a62a8ef9SStefan Hajnoczi 570a62a8ef9SStefan Hajnoczi callbacks[VQ_HIPRIO] = virtio_fs_vq_done; 571a62a8ef9SStefan Hajnoczi snprintf(fs->vqs[VQ_HIPRIO].name, sizeof(fs->vqs[VQ_HIPRIO].name), 572a62a8ef9SStefan Hajnoczi "hiprio"); 573a62a8ef9SStefan Hajnoczi names[VQ_HIPRIO] = fs->vqs[VQ_HIPRIO].name; 574a62a8ef9SStefan Hajnoczi INIT_WORK(&fs->vqs[VQ_HIPRIO].done_work, virtio_fs_hiprio_done_work); 575a62a8ef9SStefan Hajnoczi INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].queued_reqs); 57651fecdd2SVivek Goyal INIT_LIST_HEAD(&fs->vqs[VQ_HIPRIO].end_reqs); 577a62a8ef9SStefan Hajnoczi INIT_DELAYED_WORK(&fs->vqs[VQ_HIPRIO].dispatch_work, 578a62a8ef9SStefan Hajnoczi virtio_fs_hiprio_dispatch_work); 579a62a8ef9SStefan Hajnoczi spin_lock_init(&fs->vqs[VQ_HIPRIO].lock); 580a62a8ef9SStefan Hajnoczi 581a62a8ef9SStefan Hajnoczi /* Initialize the requests virtqueues */ 582a62a8ef9SStefan Hajnoczi for (i = VQ_REQUEST; i < fs->nvqs; i++) { 583a62a8ef9SStefan Hajnoczi spin_lock_init(&fs->vqs[i].lock); 584a62a8ef9SStefan Hajnoczi INIT_WORK(&fs->vqs[i].done_work, virtio_fs_requests_done_work); 585a62a8ef9SStefan Hajnoczi INIT_DELAYED_WORK(&fs->vqs[i].dispatch_work, 58651fecdd2SVivek Goyal virtio_fs_request_dispatch_work); 587a62a8ef9SStefan Hajnoczi INIT_LIST_HEAD(&fs->vqs[i].queued_reqs); 58851fecdd2SVivek Goyal INIT_LIST_HEAD(&fs->vqs[i].end_reqs); 589a62a8ef9SStefan Hajnoczi snprintf(fs->vqs[i].name, sizeof(fs->vqs[i].name), 590a62a8ef9SStefan Hajnoczi "requests.%u", i - VQ_REQUEST); 591a62a8ef9SStefan Hajnoczi callbacks[i] = virtio_fs_vq_done; 592a62a8ef9SStefan Hajnoczi names[i] = fs->vqs[i].name; 593a62a8ef9SStefan Hajnoczi } 594a62a8ef9SStefan Hajnoczi 595a62a8ef9SStefan Hajnoczi ret = virtio_find_vqs(vdev, fs->nvqs, vqs, callbacks, names, NULL); 596a62a8ef9SStefan Hajnoczi if (ret < 0) 597a62a8ef9SStefan Hajnoczi goto out; 598a62a8ef9SStefan Hajnoczi 599a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) 600a62a8ef9SStefan Hajnoczi fs->vqs[i].vq = vqs[i]; 601a62a8ef9SStefan Hajnoczi 602a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs); 603a62a8ef9SStefan Hajnoczi out: 604a62a8ef9SStefan Hajnoczi kfree(names); 605a62a8ef9SStefan Hajnoczi kfree(callbacks); 606a62a8ef9SStefan Hajnoczi kfree(vqs); 607a62a8ef9SStefan Hajnoczi if (ret) 608a62a8ef9SStefan Hajnoczi kfree(fs->vqs); 609a62a8ef9SStefan Hajnoczi return ret; 610a62a8ef9SStefan Hajnoczi } 611a62a8ef9SStefan Hajnoczi 612a62a8ef9SStefan Hajnoczi /* Free virtqueues (device must already be reset) */ 613a62a8ef9SStefan Hajnoczi static void virtio_fs_cleanup_vqs(struct virtio_device *vdev, 614a62a8ef9SStefan Hajnoczi struct virtio_fs *fs) 615a62a8ef9SStefan Hajnoczi { 616a62a8ef9SStefan Hajnoczi vdev->config->del_vqs(vdev); 617a62a8ef9SStefan Hajnoczi } 618a62a8ef9SStefan Hajnoczi 619a62a8ef9SStefan Hajnoczi static int virtio_fs_probe(struct virtio_device *vdev) 620a62a8ef9SStefan Hajnoczi { 621a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 622a62a8ef9SStefan Hajnoczi int ret; 623a62a8ef9SStefan Hajnoczi 624a62a8ef9SStefan Hajnoczi fs = kzalloc(sizeof(*fs), GFP_KERNEL); 625a62a8ef9SStefan Hajnoczi if (!fs) 626a62a8ef9SStefan Hajnoczi return -ENOMEM; 627a62a8ef9SStefan Hajnoczi kref_init(&fs->refcount); 628a62a8ef9SStefan Hajnoczi vdev->priv = fs; 629a62a8ef9SStefan Hajnoczi 630a62a8ef9SStefan Hajnoczi ret = virtio_fs_read_tag(vdev, fs); 631a62a8ef9SStefan Hajnoczi if (ret < 0) 632a62a8ef9SStefan Hajnoczi goto out; 633a62a8ef9SStefan Hajnoczi 634a62a8ef9SStefan Hajnoczi ret = virtio_fs_setup_vqs(vdev, fs); 635a62a8ef9SStefan Hajnoczi if (ret < 0) 636a62a8ef9SStefan Hajnoczi goto out; 637a62a8ef9SStefan Hajnoczi 638a62a8ef9SStefan Hajnoczi /* TODO vq affinity */ 639a62a8ef9SStefan Hajnoczi 640a62a8ef9SStefan Hajnoczi /* Bring the device online in case the filesystem is mounted and 641a62a8ef9SStefan Hajnoczi * requests need to be sent before we return. 642a62a8ef9SStefan Hajnoczi */ 643a62a8ef9SStefan Hajnoczi virtio_device_ready(vdev); 644a62a8ef9SStefan Hajnoczi 645a62a8ef9SStefan Hajnoczi ret = virtio_fs_add_instance(fs); 646a62a8ef9SStefan Hajnoczi if (ret < 0) 647a62a8ef9SStefan Hajnoczi goto out_vqs; 648a62a8ef9SStefan Hajnoczi 649a62a8ef9SStefan Hajnoczi return 0; 650a62a8ef9SStefan Hajnoczi 651a62a8ef9SStefan Hajnoczi out_vqs: 652a62a8ef9SStefan Hajnoczi vdev->config->reset(vdev); 653a62a8ef9SStefan Hajnoczi virtio_fs_cleanup_vqs(vdev, fs); 654a62a8ef9SStefan Hajnoczi 655a62a8ef9SStefan Hajnoczi out: 656a62a8ef9SStefan Hajnoczi vdev->priv = NULL; 657a62a8ef9SStefan Hajnoczi kfree(fs); 658a62a8ef9SStefan Hajnoczi return ret; 659a62a8ef9SStefan Hajnoczi } 660a62a8ef9SStefan Hajnoczi 661a62a8ef9SStefan Hajnoczi static void virtio_fs_stop_all_queues(struct virtio_fs *fs) 662a62a8ef9SStefan Hajnoczi { 663a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 664a62a8ef9SStefan Hajnoczi int i; 665a62a8ef9SStefan Hajnoczi 666a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 667a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[i]; 668a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 669a62a8ef9SStefan Hajnoczi fsvq->connected = false; 670a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 671a62a8ef9SStefan Hajnoczi } 672a62a8ef9SStefan Hajnoczi } 673a62a8ef9SStefan Hajnoczi 674a62a8ef9SStefan Hajnoczi static void virtio_fs_remove(struct virtio_device *vdev) 675a62a8ef9SStefan Hajnoczi { 676a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = vdev->priv; 677a62a8ef9SStefan Hajnoczi 678a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 679a62a8ef9SStefan Hajnoczi /* This device is going away. No one should get new reference */ 680a62a8ef9SStefan Hajnoczi list_del_init(&fs->list); 681a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(fs); 682a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(fs); 683a62a8ef9SStefan Hajnoczi vdev->config->reset(vdev); 684a62a8ef9SStefan Hajnoczi virtio_fs_cleanup_vqs(vdev, fs); 685a62a8ef9SStefan Hajnoczi 686a62a8ef9SStefan Hajnoczi vdev->priv = NULL; 687a62a8ef9SStefan Hajnoczi /* Put device reference on virtio_fs object */ 688a62a8ef9SStefan Hajnoczi virtio_fs_put(fs); 689a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 690a62a8ef9SStefan Hajnoczi } 691a62a8ef9SStefan Hajnoczi 692a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP 693a62a8ef9SStefan Hajnoczi static int virtio_fs_freeze(struct virtio_device *vdev) 694a62a8ef9SStefan Hajnoczi { 695a62a8ef9SStefan Hajnoczi /* TODO need to save state here */ 696a62a8ef9SStefan Hajnoczi pr_warn("virtio-fs: suspend/resume not yet supported\n"); 697a62a8ef9SStefan Hajnoczi return -EOPNOTSUPP; 698a62a8ef9SStefan Hajnoczi } 699a62a8ef9SStefan Hajnoczi 700a62a8ef9SStefan Hajnoczi static int virtio_fs_restore(struct virtio_device *vdev) 701a62a8ef9SStefan Hajnoczi { 702a62a8ef9SStefan Hajnoczi /* TODO need to restore state here */ 703a62a8ef9SStefan Hajnoczi return 0; 704a62a8ef9SStefan Hajnoczi } 705a62a8ef9SStefan Hajnoczi #endif /* CONFIG_PM_SLEEP */ 706a62a8ef9SStefan Hajnoczi 70700929447SYueHaibing static const struct virtio_device_id id_table[] = { 708a62a8ef9SStefan Hajnoczi { VIRTIO_ID_FS, VIRTIO_DEV_ANY_ID }, 709a62a8ef9SStefan Hajnoczi {}, 710a62a8ef9SStefan Hajnoczi }; 711a62a8ef9SStefan Hajnoczi 71200929447SYueHaibing static const unsigned int feature_table[] = {}; 713a62a8ef9SStefan Hajnoczi 714a62a8ef9SStefan Hajnoczi static struct virtio_driver virtio_fs_driver = { 715a62a8ef9SStefan Hajnoczi .driver.name = KBUILD_MODNAME, 716a62a8ef9SStefan Hajnoczi .driver.owner = THIS_MODULE, 717a62a8ef9SStefan Hajnoczi .id_table = id_table, 718a62a8ef9SStefan Hajnoczi .feature_table = feature_table, 719a62a8ef9SStefan Hajnoczi .feature_table_size = ARRAY_SIZE(feature_table), 720a62a8ef9SStefan Hajnoczi .probe = virtio_fs_probe, 721a62a8ef9SStefan Hajnoczi .remove = virtio_fs_remove, 722a62a8ef9SStefan Hajnoczi #ifdef CONFIG_PM_SLEEP 723a62a8ef9SStefan Hajnoczi .freeze = virtio_fs_freeze, 724a62a8ef9SStefan Hajnoczi .restore = virtio_fs_restore, 725a62a8ef9SStefan Hajnoczi #endif 726a62a8ef9SStefan Hajnoczi }; 727a62a8ef9SStefan Hajnoczi 728a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_forget_and_unlock(struct fuse_iqueue *fiq) 729a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 730a62a8ef9SStefan Hajnoczi { 731a62a8ef9SStefan Hajnoczi struct fuse_forget_link *link; 732a62a8ef9SStefan Hajnoczi struct virtio_fs_forget *forget; 733a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 734a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 735a62a8ef9SStefan Hajnoczi u64 unique; 736a62a8ef9SStefan Hajnoczi 737a62a8ef9SStefan Hajnoczi link = fuse_dequeue_forget(fiq, 1, NULL); 738a62a8ef9SStefan Hajnoczi unique = fuse_get_unique(fiq); 739a62a8ef9SStefan Hajnoczi 740a62a8ef9SStefan Hajnoczi fs = fiq->priv; 741a62a8ef9SStefan Hajnoczi fsvq = &fs->vqs[VQ_HIPRIO]; 742a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 743a62a8ef9SStefan Hajnoczi 744a62a8ef9SStefan Hajnoczi /* Allocate a buffer for the request */ 745a62a8ef9SStefan Hajnoczi forget = kmalloc(sizeof(*forget), GFP_NOFS | __GFP_NOFAIL); 746a62a8ef9SStefan Hajnoczi 747a62a8ef9SStefan Hajnoczi forget->ih = (struct fuse_in_header){ 748a62a8ef9SStefan Hajnoczi .opcode = FUSE_FORGET, 749a62a8ef9SStefan Hajnoczi .nodeid = link->forget_one.nodeid, 750a62a8ef9SStefan Hajnoczi .unique = unique, 751a62a8ef9SStefan Hajnoczi .len = sizeof(*forget), 752a62a8ef9SStefan Hajnoczi }; 753a62a8ef9SStefan Hajnoczi forget->arg = (struct fuse_forget_in){ 754a62a8ef9SStefan Hajnoczi .nlookup = link->forget_one.nlookup, 755a62a8ef9SStefan Hajnoczi }; 756a62a8ef9SStefan Hajnoczi 757*58ada94fSVivek Goyal send_forget_request(fsvq, forget, false); 758a62a8ef9SStefan Hajnoczi kfree(link); 759a62a8ef9SStefan Hajnoczi } 760a62a8ef9SStefan Hajnoczi 761a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_interrupt_and_unlock(struct fuse_iqueue *fiq) 762a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 763a62a8ef9SStefan Hajnoczi { 764a62a8ef9SStefan Hajnoczi /* 765a62a8ef9SStefan Hajnoczi * TODO interrupts. 766a62a8ef9SStefan Hajnoczi * 767a62a8ef9SStefan Hajnoczi * Normal fs operations on a local filesystems aren't interruptible. 768a62a8ef9SStefan Hajnoczi * Exceptions are blocking lock operations; for example fcntl(F_SETLKW) 769a62a8ef9SStefan Hajnoczi * with shared lock between host and guest. 770a62a8ef9SStefan Hajnoczi */ 771a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 772a62a8ef9SStefan Hajnoczi } 773a62a8ef9SStefan Hajnoczi 774a62a8ef9SStefan Hajnoczi /* Return the number of scatter-gather list elements required */ 775a62a8ef9SStefan Hajnoczi static unsigned int sg_count_fuse_req(struct fuse_req *req) 776a62a8ef9SStefan Hajnoczi { 777a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 778a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(args, typeof(*ap), args); 779a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 1 /* fuse_in_header */; 780a62a8ef9SStefan Hajnoczi 781a62a8ef9SStefan Hajnoczi if (args->in_numargs - args->in_pages) 782a62a8ef9SStefan Hajnoczi total_sgs += 1; 783a62a8ef9SStefan Hajnoczi 784a62a8ef9SStefan Hajnoczi if (args->in_pages) 785a62a8ef9SStefan Hajnoczi total_sgs += ap->num_pages; 786a62a8ef9SStefan Hajnoczi 787a62a8ef9SStefan Hajnoczi if (!test_bit(FR_ISREPLY, &req->flags)) 788a62a8ef9SStefan Hajnoczi return total_sgs; 789a62a8ef9SStefan Hajnoczi 790a62a8ef9SStefan Hajnoczi total_sgs += 1 /* fuse_out_header */; 791a62a8ef9SStefan Hajnoczi 792a62a8ef9SStefan Hajnoczi if (args->out_numargs - args->out_pages) 793a62a8ef9SStefan Hajnoczi total_sgs += 1; 794a62a8ef9SStefan Hajnoczi 795a62a8ef9SStefan Hajnoczi if (args->out_pages) 796a62a8ef9SStefan Hajnoczi total_sgs += ap->num_pages; 797a62a8ef9SStefan Hajnoczi 798a62a8ef9SStefan Hajnoczi return total_sgs; 799a62a8ef9SStefan Hajnoczi } 800a62a8ef9SStefan Hajnoczi 801a62a8ef9SStefan Hajnoczi /* Add pages to scatter-gather list and return number of elements used */ 802a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_pages(struct scatterlist *sg, 803a62a8ef9SStefan Hajnoczi struct page **pages, 804a62a8ef9SStefan Hajnoczi struct fuse_page_desc *page_descs, 805a62a8ef9SStefan Hajnoczi unsigned int num_pages, 806a62a8ef9SStefan Hajnoczi unsigned int total_len) 807a62a8ef9SStefan Hajnoczi { 808a62a8ef9SStefan Hajnoczi unsigned int i; 809a62a8ef9SStefan Hajnoczi unsigned int this_len; 810a62a8ef9SStefan Hajnoczi 811a62a8ef9SStefan Hajnoczi for (i = 0; i < num_pages && total_len; i++) { 812a62a8ef9SStefan Hajnoczi sg_init_table(&sg[i], 1); 813a62a8ef9SStefan Hajnoczi this_len = min(page_descs[i].length, total_len); 814a62a8ef9SStefan Hajnoczi sg_set_page(&sg[i], pages[i], this_len, page_descs[i].offset); 815a62a8ef9SStefan Hajnoczi total_len -= this_len; 816a62a8ef9SStefan Hajnoczi } 817a62a8ef9SStefan Hajnoczi 818a62a8ef9SStefan Hajnoczi return i; 819a62a8ef9SStefan Hajnoczi } 820a62a8ef9SStefan Hajnoczi 821a62a8ef9SStefan Hajnoczi /* Add args to scatter-gather list and return number of elements used */ 822a62a8ef9SStefan Hajnoczi static unsigned int sg_init_fuse_args(struct scatterlist *sg, 823a62a8ef9SStefan Hajnoczi struct fuse_req *req, 824a62a8ef9SStefan Hajnoczi struct fuse_arg *args, 825a62a8ef9SStefan Hajnoczi unsigned int numargs, 826a62a8ef9SStefan Hajnoczi bool argpages, 827a62a8ef9SStefan Hajnoczi void *argbuf, 828a62a8ef9SStefan Hajnoczi unsigned int *len_used) 829a62a8ef9SStefan Hajnoczi { 830a62a8ef9SStefan Hajnoczi struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args); 831a62a8ef9SStefan Hajnoczi unsigned int total_sgs = 0; 832a62a8ef9SStefan Hajnoczi unsigned int len; 833a62a8ef9SStefan Hajnoczi 834a62a8ef9SStefan Hajnoczi len = fuse_len_args(numargs - argpages, args); 835a62a8ef9SStefan Hajnoczi if (len) 836a62a8ef9SStefan Hajnoczi sg_init_one(&sg[total_sgs++], argbuf, len); 837a62a8ef9SStefan Hajnoczi 838a62a8ef9SStefan Hajnoczi if (argpages) 839a62a8ef9SStefan Hajnoczi total_sgs += sg_init_fuse_pages(&sg[total_sgs], 840a62a8ef9SStefan Hajnoczi ap->pages, ap->descs, 841a62a8ef9SStefan Hajnoczi ap->num_pages, 842a62a8ef9SStefan Hajnoczi args[numargs - 1].size); 843a62a8ef9SStefan Hajnoczi 844a62a8ef9SStefan Hajnoczi if (len_used) 845a62a8ef9SStefan Hajnoczi *len_used = len; 846a62a8ef9SStefan Hajnoczi 847a62a8ef9SStefan Hajnoczi return total_sgs; 848a62a8ef9SStefan Hajnoczi } 849a62a8ef9SStefan Hajnoczi 850a62a8ef9SStefan Hajnoczi /* Add a request to a virtqueue and kick the device */ 851a62a8ef9SStefan Hajnoczi static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq, 852a9bfd9ddSVivek Goyal struct fuse_req *req, bool in_flight) 853a62a8ef9SStefan Hajnoczi { 854a62a8ef9SStefan Hajnoczi /* requests need at least 4 elements */ 855a62a8ef9SStefan Hajnoczi struct scatterlist *stack_sgs[6]; 856a62a8ef9SStefan Hajnoczi struct scatterlist stack_sg[ARRAY_SIZE(stack_sgs)]; 857a62a8ef9SStefan Hajnoczi struct scatterlist **sgs = stack_sgs; 858a62a8ef9SStefan Hajnoczi struct scatterlist *sg = stack_sg; 859a62a8ef9SStefan Hajnoczi struct virtqueue *vq; 860a62a8ef9SStefan Hajnoczi struct fuse_args *args = req->args; 861a62a8ef9SStefan Hajnoczi unsigned int argbuf_used = 0; 862a62a8ef9SStefan Hajnoczi unsigned int out_sgs = 0; 863a62a8ef9SStefan Hajnoczi unsigned int in_sgs = 0; 864a62a8ef9SStefan Hajnoczi unsigned int total_sgs; 865a62a8ef9SStefan Hajnoczi unsigned int i; 866a62a8ef9SStefan Hajnoczi int ret; 867a62a8ef9SStefan Hajnoczi bool notify; 8685dbe190fSVivek Goyal struct fuse_pqueue *fpq; 869a62a8ef9SStefan Hajnoczi 870a62a8ef9SStefan Hajnoczi /* Does the sglist fit on the stack? */ 871a62a8ef9SStefan Hajnoczi total_sgs = sg_count_fuse_req(req); 872a62a8ef9SStefan Hajnoczi if (total_sgs > ARRAY_SIZE(stack_sgs)) { 873a62a8ef9SStefan Hajnoczi sgs = kmalloc_array(total_sgs, sizeof(sgs[0]), GFP_ATOMIC); 874a62a8ef9SStefan Hajnoczi sg = kmalloc_array(total_sgs, sizeof(sg[0]), GFP_ATOMIC); 875a62a8ef9SStefan Hajnoczi if (!sgs || !sg) { 876a62a8ef9SStefan Hajnoczi ret = -ENOMEM; 877a62a8ef9SStefan Hajnoczi goto out; 878a62a8ef9SStefan Hajnoczi } 879a62a8ef9SStefan Hajnoczi } 880a62a8ef9SStefan Hajnoczi 881a62a8ef9SStefan Hajnoczi /* Use a bounce buffer since stack args cannot be mapped */ 882a62a8ef9SStefan Hajnoczi ret = copy_args_to_argbuf(req); 883a62a8ef9SStefan Hajnoczi if (ret < 0) 884a62a8ef9SStefan Hajnoczi goto out; 885a62a8ef9SStefan Hajnoczi 886a62a8ef9SStefan Hajnoczi /* Request elements */ 887a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs++], &req->in.h, sizeof(req->in.h)); 888a62a8ef9SStefan Hajnoczi out_sgs += sg_init_fuse_args(&sg[out_sgs], req, 889a62a8ef9SStefan Hajnoczi (struct fuse_arg *)args->in_args, 890a62a8ef9SStefan Hajnoczi args->in_numargs, args->in_pages, 891a62a8ef9SStefan Hajnoczi req->argbuf, &argbuf_used); 892a62a8ef9SStefan Hajnoczi 893a62a8ef9SStefan Hajnoczi /* Reply elements */ 894a62a8ef9SStefan Hajnoczi if (test_bit(FR_ISREPLY, &req->flags)) { 895a62a8ef9SStefan Hajnoczi sg_init_one(&sg[out_sgs + in_sgs++], 896a62a8ef9SStefan Hajnoczi &req->out.h, sizeof(req->out.h)); 897a62a8ef9SStefan Hajnoczi in_sgs += sg_init_fuse_args(&sg[out_sgs + in_sgs], req, 898a62a8ef9SStefan Hajnoczi args->out_args, args->out_numargs, 899a62a8ef9SStefan Hajnoczi args->out_pages, 900a62a8ef9SStefan Hajnoczi req->argbuf + argbuf_used, NULL); 901a62a8ef9SStefan Hajnoczi } 902a62a8ef9SStefan Hajnoczi 903a62a8ef9SStefan Hajnoczi WARN_ON(out_sgs + in_sgs != total_sgs); 904a62a8ef9SStefan Hajnoczi 905a62a8ef9SStefan Hajnoczi for (i = 0; i < total_sgs; i++) 906a62a8ef9SStefan Hajnoczi sgs[i] = &sg[i]; 907a62a8ef9SStefan Hajnoczi 908a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 909a62a8ef9SStefan Hajnoczi 910a62a8ef9SStefan Hajnoczi if (!fsvq->connected) { 911a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 912a62a8ef9SStefan Hajnoczi ret = -ENOTCONN; 913a62a8ef9SStefan Hajnoczi goto out; 914a62a8ef9SStefan Hajnoczi } 915a62a8ef9SStefan Hajnoczi 916a62a8ef9SStefan Hajnoczi vq = fsvq->vq; 917a62a8ef9SStefan Hajnoczi ret = virtqueue_add_sgs(vq, sgs, out_sgs, in_sgs, req, GFP_ATOMIC); 918a62a8ef9SStefan Hajnoczi if (ret < 0) { 919a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 920a62a8ef9SStefan Hajnoczi goto out; 921a62a8ef9SStefan Hajnoczi } 922a62a8ef9SStefan Hajnoczi 9235dbe190fSVivek Goyal /* Request successfully sent. */ 9245dbe190fSVivek Goyal fpq = &fsvq->fud->pq; 9255dbe190fSVivek Goyal spin_lock(&fpq->lock); 9265dbe190fSVivek Goyal list_add_tail(&req->list, fpq->processing); 9275dbe190fSVivek Goyal spin_unlock(&fpq->lock); 9285dbe190fSVivek Goyal set_bit(FR_SENT, &req->flags); 9295dbe190fSVivek Goyal /* matches barrier in request_wait_answer() */ 9305dbe190fSVivek Goyal smp_mb__after_atomic(); 9315dbe190fSVivek Goyal 932a9bfd9ddSVivek Goyal if (!in_flight) 933c17ea009SVivek Goyal inc_in_flight_req(fsvq); 934a62a8ef9SStefan Hajnoczi notify = virtqueue_kick_prepare(vq); 935a62a8ef9SStefan Hajnoczi 936a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 937a62a8ef9SStefan Hajnoczi 938a62a8ef9SStefan Hajnoczi if (notify) 939a62a8ef9SStefan Hajnoczi virtqueue_notify(vq); 940a62a8ef9SStefan Hajnoczi 941a62a8ef9SStefan Hajnoczi out: 942a62a8ef9SStefan Hajnoczi if (ret < 0 && req->argbuf) { 943a62a8ef9SStefan Hajnoczi kfree(req->argbuf); 944a62a8ef9SStefan Hajnoczi req->argbuf = NULL; 945a62a8ef9SStefan Hajnoczi } 946a62a8ef9SStefan Hajnoczi if (sgs != stack_sgs) { 947a62a8ef9SStefan Hajnoczi kfree(sgs); 948a62a8ef9SStefan Hajnoczi kfree(sg); 949a62a8ef9SStefan Hajnoczi } 950a62a8ef9SStefan Hajnoczi 951a62a8ef9SStefan Hajnoczi return ret; 952a62a8ef9SStefan Hajnoczi } 953a62a8ef9SStefan Hajnoczi 954a62a8ef9SStefan Hajnoczi static void virtio_fs_wake_pending_and_unlock(struct fuse_iqueue *fiq) 955a62a8ef9SStefan Hajnoczi __releases(fiq->lock) 956a62a8ef9SStefan Hajnoczi { 957a62a8ef9SStefan Hajnoczi unsigned int queue_id = VQ_REQUEST; /* TODO multiqueue */ 958a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 959a62a8ef9SStefan Hajnoczi struct fuse_req *req; 96051fecdd2SVivek Goyal struct virtio_fs_vq *fsvq; 961a62a8ef9SStefan Hajnoczi int ret; 962a62a8ef9SStefan Hajnoczi 963a62a8ef9SStefan Hajnoczi WARN_ON(list_empty(&fiq->pending)); 964a62a8ef9SStefan Hajnoczi req = list_last_entry(&fiq->pending, struct fuse_req, list); 965a62a8ef9SStefan Hajnoczi clear_bit(FR_PENDING, &req->flags); 966a62a8ef9SStefan Hajnoczi list_del_init(&req->list); 967a62a8ef9SStefan Hajnoczi WARN_ON(!list_empty(&fiq->pending)); 968a62a8ef9SStefan Hajnoczi spin_unlock(&fiq->lock); 969a62a8ef9SStefan Hajnoczi 970a62a8ef9SStefan Hajnoczi fs = fiq->priv; 971a62a8ef9SStefan Hajnoczi 972a62a8ef9SStefan Hajnoczi pr_debug("%s: opcode %u unique %#llx nodeid %#llx in.len %u out.len %u\n", 973a62a8ef9SStefan Hajnoczi __func__, req->in.h.opcode, req->in.h.unique, 974a62a8ef9SStefan Hajnoczi req->in.h.nodeid, req->in.h.len, 975a62a8ef9SStefan Hajnoczi fuse_len_args(req->args->out_numargs, req->args->out_args)); 976a62a8ef9SStefan Hajnoczi 97751fecdd2SVivek Goyal fsvq = &fs->vqs[queue_id]; 978a9bfd9ddSVivek Goyal ret = virtio_fs_enqueue_req(fsvq, req, false); 979a62a8ef9SStefan Hajnoczi if (ret < 0) { 980a62a8ef9SStefan Hajnoczi if (ret == -ENOMEM || ret == -ENOSPC) { 981a9bfd9ddSVivek Goyal /* 982a9bfd9ddSVivek Goyal * Virtqueue full. Retry submission from worker 983a9bfd9ddSVivek Goyal * context as we might be holding fc->bg_lock. 984a9bfd9ddSVivek Goyal */ 985a9bfd9ddSVivek Goyal spin_lock(&fsvq->lock); 986a9bfd9ddSVivek Goyal list_add_tail(&req->list, &fsvq->queued_reqs); 987a9bfd9ddSVivek Goyal inc_in_flight_req(fsvq); 988a9bfd9ddSVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 989a9bfd9ddSVivek Goyal msecs_to_jiffies(1)); 990a9bfd9ddSVivek Goyal spin_unlock(&fsvq->lock); 991a9bfd9ddSVivek Goyal return; 992a62a8ef9SStefan Hajnoczi } 993a62a8ef9SStefan Hajnoczi req->out.h.error = ret; 994a62a8ef9SStefan Hajnoczi pr_err("virtio-fs: virtio_fs_enqueue_req() failed %d\n", ret); 99551fecdd2SVivek Goyal 99651fecdd2SVivek Goyal /* Can't end request in submission context. Use a worker */ 99751fecdd2SVivek Goyal spin_lock(&fsvq->lock); 99851fecdd2SVivek Goyal list_add_tail(&req->list, &fsvq->end_reqs); 99951fecdd2SVivek Goyal schedule_delayed_work(&fsvq->dispatch_work, 0); 100051fecdd2SVivek Goyal spin_unlock(&fsvq->lock); 1001a62a8ef9SStefan Hajnoczi return; 1002a62a8ef9SStefan Hajnoczi } 1003a62a8ef9SStefan Hajnoczi } 1004a62a8ef9SStefan Hajnoczi 100500929447SYueHaibing static const struct fuse_iqueue_ops virtio_fs_fiq_ops = { 1006a62a8ef9SStefan Hajnoczi .wake_forget_and_unlock = virtio_fs_wake_forget_and_unlock, 1007a62a8ef9SStefan Hajnoczi .wake_interrupt_and_unlock = virtio_fs_wake_interrupt_and_unlock, 1008a62a8ef9SStefan Hajnoczi .wake_pending_and_unlock = virtio_fs_wake_pending_and_unlock, 1009a62a8ef9SStefan Hajnoczi .release = virtio_fs_fiq_release, 1010a62a8ef9SStefan Hajnoczi }; 1011a62a8ef9SStefan Hajnoczi 1012a62a8ef9SStefan Hajnoczi static int virtio_fs_fill_super(struct super_block *sb) 1013a62a8ef9SStefan Hajnoczi { 1014a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = get_fuse_conn_super(sb); 1015a62a8ef9SStefan Hajnoczi struct virtio_fs *fs = fc->iq.priv; 1016a62a8ef9SStefan Hajnoczi unsigned int i; 1017a62a8ef9SStefan Hajnoczi int err; 1018a62a8ef9SStefan Hajnoczi struct fuse_fs_context ctx = { 1019a62a8ef9SStefan Hajnoczi .rootmode = S_IFDIR, 1020a62a8ef9SStefan Hajnoczi .default_permissions = 1, 1021a62a8ef9SStefan Hajnoczi .allow_other = 1, 1022a62a8ef9SStefan Hajnoczi .max_read = UINT_MAX, 1023a62a8ef9SStefan Hajnoczi .blksize = 512, 1024a62a8ef9SStefan Hajnoczi .destroy = true, 1025a62a8ef9SStefan Hajnoczi .no_control = true, 1026a62a8ef9SStefan Hajnoczi .no_force_umount = true, 10273f22c746SMiklos Szeredi .no_mount_options = true, 1028a62a8ef9SStefan Hajnoczi }; 1029a62a8ef9SStefan Hajnoczi 1030a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 1031a62a8ef9SStefan Hajnoczi 1032a62a8ef9SStefan Hajnoczi /* After holding mutex, make sure virtiofs device is still there. 1033a62a8ef9SStefan Hajnoczi * Though we are holding a reference to it, drive ->remove might 1034a62a8ef9SStefan Hajnoczi * still have cleaned up virtual queues. In that case bail out. 1035a62a8ef9SStefan Hajnoczi */ 1036a62a8ef9SStefan Hajnoczi err = -EINVAL; 1037a62a8ef9SStefan Hajnoczi if (list_empty(&fs->list)) { 1038a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fs->tag); 1039a62a8ef9SStefan Hajnoczi goto err; 1040a62a8ef9SStefan Hajnoczi } 1041a62a8ef9SStefan Hajnoczi 1042a62a8ef9SStefan Hajnoczi err = -ENOMEM; 1043a62a8ef9SStefan Hajnoczi /* Allocate fuse_dev for hiprio and notification queues */ 1044a62a8ef9SStefan Hajnoczi for (i = 0; i < VQ_REQUEST; i++) { 1045a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1046a62a8ef9SStefan Hajnoczi 1047a62a8ef9SStefan Hajnoczi fsvq->fud = fuse_dev_alloc(); 1048a62a8ef9SStefan Hajnoczi if (!fsvq->fud) 1049a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs; 1050a62a8ef9SStefan Hajnoczi } 1051a62a8ef9SStefan Hajnoczi 1052a62a8ef9SStefan Hajnoczi ctx.fudptr = (void **)&fs->vqs[VQ_REQUEST].fud; 1053a62a8ef9SStefan Hajnoczi err = fuse_fill_super_common(sb, &ctx); 1054a62a8ef9SStefan Hajnoczi if (err < 0) 1055a62a8ef9SStefan Hajnoczi goto err_free_fuse_devs; 1056a62a8ef9SStefan Hajnoczi 1057a62a8ef9SStefan Hajnoczi fc = fs->vqs[VQ_REQUEST].fud->fc; 1058a62a8ef9SStefan Hajnoczi 1059a62a8ef9SStefan Hajnoczi for (i = 0; i < fs->nvqs; i++) { 1060a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq = &fs->vqs[i]; 1061a62a8ef9SStefan Hajnoczi 1062a62a8ef9SStefan Hajnoczi if (i == VQ_REQUEST) 1063a62a8ef9SStefan Hajnoczi continue; /* already initialized */ 1064a62a8ef9SStefan Hajnoczi fuse_dev_install(fsvq->fud, fc); 1065a62a8ef9SStefan Hajnoczi } 1066a62a8ef9SStefan Hajnoczi 1067a62a8ef9SStefan Hajnoczi /* Previous unmount will stop all queues. Start these again */ 1068a62a8ef9SStefan Hajnoczi virtio_fs_start_all_queues(fs); 1069a62a8ef9SStefan Hajnoczi fuse_send_init(fc); 1070a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1071a62a8ef9SStefan Hajnoczi return 0; 1072a62a8ef9SStefan Hajnoczi 1073a62a8ef9SStefan Hajnoczi err_free_fuse_devs: 1074a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(fs); 1075a62a8ef9SStefan Hajnoczi err: 1076a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1077a62a8ef9SStefan Hajnoczi return err; 1078a62a8ef9SStefan Hajnoczi } 1079a62a8ef9SStefan Hajnoczi 1080a62a8ef9SStefan Hajnoczi static void virtio_kill_sb(struct super_block *sb) 1081a62a8ef9SStefan Hajnoczi { 1082a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = get_fuse_conn_super(sb); 1083a62a8ef9SStefan Hajnoczi struct virtio_fs *vfs; 1084a62a8ef9SStefan Hajnoczi struct virtio_fs_vq *fsvq; 1085a62a8ef9SStefan Hajnoczi 1086a62a8ef9SStefan Hajnoczi /* If mount failed, we can still be called without any fc */ 1087a62a8ef9SStefan Hajnoczi if (!fc) 1088a62a8ef9SStefan Hajnoczi return fuse_kill_sb_anon(sb); 1089a62a8ef9SStefan Hajnoczi 1090a62a8ef9SStefan Hajnoczi vfs = fc->iq.priv; 1091a62a8ef9SStefan Hajnoczi fsvq = &vfs->vqs[VQ_HIPRIO]; 1092a62a8ef9SStefan Hajnoczi 1093a62a8ef9SStefan Hajnoczi /* Stop forget queue. Soon destroy will be sent */ 1094a62a8ef9SStefan Hajnoczi spin_lock(&fsvq->lock); 1095a62a8ef9SStefan Hajnoczi fsvq->connected = false; 1096a62a8ef9SStefan Hajnoczi spin_unlock(&fsvq->lock); 1097a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs); 1098a62a8ef9SStefan Hajnoczi 1099a62a8ef9SStefan Hajnoczi fuse_kill_sb_anon(sb); 1100a62a8ef9SStefan Hajnoczi 1101a62a8ef9SStefan Hajnoczi /* fuse_kill_sb_anon() must have sent destroy. Stop all queues 1102a62a8ef9SStefan Hajnoczi * and drain one more time and free fuse devices. Freeing fuse 1103a62a8ef9SStefan Hajnoczi * devices will drop their reference on fuse_conn and that in 1104a62a8ef9SStefan Hajnoczi * turn will drop its reference on virtio_fs object. 1105a62a8ef9SStefan Hajnoczi */ 1106a62a8ef9SStefan Hajnoczi virtio_fs_stop_all_queues(vfs); 1107a62a8ef9SStefan Hajnoczi virtio_fs_drain_all_queues(vfs); 1108a62a8ef9SStefan Hajnoczi virtio_fs_free_devs(vfs); 1109a62a8ef9SStefan Hajnoczi } 1110a62a8ef9SStefan Hajnoczi 1111a62a8ef9SStefan Hajnoczi static int virtio_fs_test_super(struct super_block *sb, 1112a62a8ef9SStefan Hajnoczi struct fs_context *fsc) 1113a62a8ef9SStefan Hajnoczi { 1114a62a8ef9SStefan Hajnoczi struct fuse_conn *fc = fsc->s_fs_info; 1115a62a8ef9SStefan Hajnoczi 1116a62a8ef9SStefan Hajnoczi return fc->iq.priv == get_fuse_conn_super(sb)->iq.priv; 1117a62a8ef9SStefan Hajnoczi } 1118a62a8ef9SStefan Hajnoczi 1119a62a8ef9SStefan Hajnoczi static int virtio_fs_set_super(struct super_block *sb, 1120a62a8ef9SStefan Hajnoczi struct fs_context *fsc) 1121a62a8ef9SStefan Hajnoczi { 1122a62a8ef9SStefan Hajnoczi int err; 1123a62a8ef9SStefan Hajnoczi 1124a62a8ef9SStefan Hajnoczi err = get_anon_bdev(&sb->s_dev); 1125a62a8ef9SStefan Hajnoczi if (!err) 1126a62a8ef9SStefan Hajnoczi fuse_conn_get(fsc->s_fs_info); 1127a62a8ef9SStefan Hajnoczi 1128a62a8ef9SStefan Hajnoczi return err; 1129a62a8ef9SStefan Hajnoczi } 1130a62a8ef9SStefan Hajnoczi 1131a62a8ef9SStefan Hajnoczi static int virtio_fs_get_tree(struct fs_context *fsc) 1132a62a8ef9SStefan Hajnoczi { 1133a62a8ef9SStefan Hajnoczi struct virtio_fs *fs; 1134a62a8ef9SStefan Hajnoczi struct super_block *sb; 1135a62a8ef9SStefan Hajnoczi struct fuse_conn *fc; 1136a62a8ef9SStefan Hajnoczi int err; 1137a62a8ef9SStefan Hajnoczi 1138a62a8ef9SStefan Hajnoczi /* This gets a reference on virtio_fs object. This ptr gets installed 1139a62a8ef9SStefan Hajnoczi * in fc->iq->priv. Once fuse_conn is going away, it calls ->put() 1140a62a8ef9SStefan Hajnoczi * to drop the reference to this object. 1141a62a8ef9SStefan Hajnoczi */ 1142a62a8ef9SStefan Hajnoczi fs = virtio_fs_find_instance(fsc->source); 1143a62a8ef9SStefan Hajnoczi if (!fs) { 1144a62a8ef9SStefan Hajnoczi pr_info("virtio-fs: tag <%s> not found\n", fsc->source); 1145a62a8ef9SStefan Hajnoczi return -EINVAL; 1146a62a8ef9SStefan Hajnoczi } 1147a62a8ef9SStefan Hajnoczi 1148a62a8ef9SStefan Hajnoczi fc = kzalloc(sizeof(struct fuse_conn), GFP_KERNEL); 1149a62a8ef9SStefan Hajnoczi if (!fc) { 1150a62a8ef9SStefan Hajnoczi mutex_lock(&virtio_fs_mutex); 1151a62a8ef9SStefan Hajnoczi virtio_fs_put(fs); 1152a62a8ef9SStefan Hajnoczi mutex_unlock(&virtio_fs_mutex); 1153a62a8ef9SStefan Hajnoczi return -ENOMEM; 1154a62a8ef9SStefan Hajnoczi } 1155a62a8ef9SStefan Hajnoczi 1156a62a8ef9SStefan Hajnoczi fuse_conn_init(fc, get_user_ns(current_user_ns()), &virtio_fs_fiq_ops, 1157a62a8ef9SStefan Hajnoczi fs); 1158a62a8ef9SStefan Hajnoczi fc->release = fuse_free_conn; 1159a62a8ef9SStefan Hajnoczi fc->delete_stale = true; 1160a62a8ef9SStefan Hajnoczi 1161a62a8ef9SStefan Hajnoczi fsc->s_fs_info = fc; 1162a62a8ef9SStefan Hajnoczi sb = sget_fc(fsc, virtio_fs_test_super, virtio_fs_set_super); 1163a62a8ef9SStefan Hajnoczi fuse_conn_put(fc); 1164a62a8ef9SStefan Hajnoczi if (IS_ERR(sb)) 1165a62a8ef9SStefan Hajnoczi return PTR_ERR(sb); 1166a62a8ef9SStefan Hajnoczi 1167a62a8ef9SStefan Hajnoczi if (!sb->s_root) { 1168a62a8ef9SStefan Hajnoczi err = virtio_fs_fill_super(sb); 1169a62a8ef9SStefan Hajnoczi if (err) { 1170a62a8ef9SStefan Hajnoczi deactivate_locked_super(sb); 1171a62a8ef9SStefan Hajnoczi return err; 1172a62a8ef9SStefan Hajnoczi } 1173a62a8ef9SStefan Hajnoczi 1174a62a8ef9SStefan Hajnoczi sb->s_flags |= SB_ACTIVE; 1175a62a8ef9SStefan Hajnoczi } 1176a62a8ef9SStefan Hajnoczi 1177a62a8ef9SStefan Hajnoczi WARN_ON(fsc->root); 1178a62a8ef9SStefan Hajnoczi fsc->root = dget(sb->s_root); 1179a62a8ef9SStefan Hajnoczi return 0; 1180a62a8ef9SStefan Hajnoczi } 1181a62a8ef9SStefan Hajnoczi 1182a62a8ef9SStefan Hajnoczi static const struct fs_context_operations virtio_fs_context_ops = { 1183a62a8ef9SStefan Hajnoczi .get_tree = virtio_fs_get_tree, 1184a62a8ef9SStefan Hajnoczi }; 1185a62a8ef9SStefan Hajnoczi 1186a62a8ef9SStefan Hajnoczi static int virtio_fs_init_fs_context(struct fs_context *fsc) 1187a62a8ef9SStefan Hajnoczi { 1188a62a8ef9SStefan Hajnoczi fsc->ops = &virtio_fs_context_ops; 1189a62a8ef9SStefan Hajnoczi return 0; 1190a62a8ef9SStefan Hajnoczi } 1191a62a8ef9SStefan Hajnoczi 1192a62a8ef9SStefan Hajnoczi static struct file_system_type virtio_fs_type = { 1193a62a8ef9SStefan Hajnoczi .owner = THIS_MODULE, 1194a62a8ef9SStefan Hajnoczi .name = "virtiofs", 1195a62a8ef9SStefan Hajnoczi .init_fs_context = virtio_fs_init_fs_context, 1196a62a8ef9SStefan Hajnoczi .kill_sb = virtio_kill_sb, 1197a62a8ef9SStefan Hajnoczi }; 1198a62a8ef9SStefan Hajnoczi 1199a62a8ef9SStefan Hajnoczi static int __init virtio_fs_init(void) 1200a62a8ef9SStefan Hajnoczi { 1201a62a8ef9SStefan Hajnoczi int ret; 1202a62a8ef9SStefan Hajnoczi 1203a62a8ef9SStefan Hajnoczi ret = register_virtio_driver(&virtio_fs_driver); 1204a62a8ef9SStefan Hajnoczi if (ret < 0) 1205a62a8ef9SStefan Hajnoczi return ret; 1206a62a8ef9SStefan Hajnoczi 1207a62a8ef9SStefan Hajnoczi ret = register_filesystem(&virtio_fs_type); 1208a62a8ef9SStefan Hajnoczi if (ret < 0) { 1209a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver); 1210a62a8ef9SStefan Hajnoczi return ret; 1211a62a8ef9SStefan Hajnoczi } 1212a62a8ef9SStefan Hajnoczi 1213a62a8ef9SStefan Hajnoczi return 0; 1214a62a8ef9SStefan Hajnoczi } 1215a62a8ef9SStefan Hajnoczi module_init(virtio_fs_init); 1216a62a8ef9SStefan Hajnoczi 1217a62a8ef9SStefan Hajnoczi static void __exit virtio_fs_exit(void) 1218a62a8ef9SStefan Hajnoczi { 1219a62a8ef9SStefan Hajnoczi unregister_filesystem(&virtio_fs_type); 1220a62a8ef9SStefan Hajnoczi unregister_virtio_driver(&virtio_fs_driver); 1221a62a8ef9SStefan Hajnoczi } 1222a62a8ef9SStefan Hajnoczi module_exit(virtio_fs_exit); 1223a62a8ef9SStefan Hajnoczi 1224a62a8ef9SStefan Hajnoczi MODULE_AUTHOR("Stefan Hajnoczi <stefanha@redhat.com>"); 1225a62a8ef9SStefan Hajnoczi MODULE_DESCRIPTION("Virtio Filesystem"); 1226a62a8ef9SStefan Hajnoczi MODULE_LICENSE("GPL"); 1227a62a8ef9SStefan Hajnoczi MODULE_ALIAS_FS(KBUILD_MODNAME); 1228a62a8ef9SStefan Hajnoczi MODULE_DEVICE_TABLE(virtio, id_table); 1229