xref: /openbmc/linux/fs/fuse/dev.c (revision c819e2cf)
1 /*
2   FUSE: Filesystem in Userspace
3   Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
4 
5   This program can be distributed under the terms of the GNU GPL.
6   See the file COPYING.
7 */
8 
9 #include "fuse_i.h"
10 
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/poll.h>
14 #include <linux/uio.h>
15 #include <linux/miscdevice.h>
16 #include <linux/pagemap.h>
17 #include <linux/file.h>
18 #include <linux/slab.h>
19 #include <linux/pipe_fs_i.h>
20 #include <linux/swap.h>
21 #include <linux/splice.h>
22 #include <linux/aio.h>
23 
24 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
25 MODULE_ALIAS("devname:fuse");
26 
27 static struct kmem_cache *fuse_req_cachep;
28 
29 static struct fuse_conn *fuse_get_conn(struct file *file)
30 {
31 	/*
32 	 * Lockless access is OK, because file->private data is set
33 	 * once during mount and is valid until the file is released.
34 	 */
35 	return file->private_data;
36 }
37 
38 static void fuse_request_init(struct fuse_req *req, struct page **pages,
39 			      struct fuse_page_desc *page_descs,
40 			      unsigned npages)
41 {
42 	memset(req, 0, sizeof(*req));
43 	memset(pages, 0, sizeof(*pages) * npages);
44 	memset(page_descs, 0, sizeof(*page_descs) * npages);
45 	INIT_LIST_HEAD(&req->list);
46 	INIT_LIST_HEAD(&req->intr_entry);
47 	init_waitqueue_head(&req->waitq);
48 	atomic_set(&req->count, 1);
49 	req->pages = pages;
50 	req->page_descs = page_descs;
51 	req->max_pages = npages;
52 }
53 
54 static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
55 {
56 	struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
57 	if (req) {
58 		struct page **pages;
59 		struct fuse_page_desc *page_descs;
60 
61 		if (npages <= FUSE_REQ_INLINE_PAGES) {
62 			pages = req->inline_pages;
63 			page_descs = req->inline_page_descs;
64 		} else {
65 			pages = kmalloc(sizeof(struct page *) * npages, flags);
66 			page_descs = kmalloc(sizeof(struct fuse_page_desc) *
67 					     npages, flags);
68 		}
69 
70 		if (!pages || !page_descs) {
71 			kfree(pages);
72 			kfree(page_descs);
73 			kmem_cache_free(fuse_req_cachep, req);
74 			return NULL;
75 		}
76 
77 		fuse_request_init(req, pages, page_descs, npages);
78 	}
79 	return req;
80 }
81 
82 struct fuse_req *fuse_request_alloc(unsigned npages)
83 {
84 	return __fuse_request_alloc(npages, GFP_KERNEL);
85 }
86 EXPORT_SYMBOL_GPL(fuse_request_alloc);
87 
88 struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
89 {
90 	return __fuse_request_alloc(npages, GFP_NOFS);
91 }
92 
93 void fuse_request_free(struct fuse_req *req)
94 {
95 	if (req->pages != req->inline_pages) {
96 		kfree(req->pages);
97 		kfree(req->page_descs);
98 	}
99 	kmem_cache_free(fuse_req_cachep, req);
100 }
101 
102 static void block_sigs(sigset_t *oldset)
103 {
104 	sigset_t mask;
105 
106 	siginitsetinv(&mask, sigmask(SIGKILL));
107 	sigprocmask(SIG_BLOCK, &mask, oldset);
108 }
109 
110 static void restore_sigs(sigset_t *oldset)
111 {
112 	sigprocmask(SIG_SETMASK, oldset, NULL);
113 }
114 
115 void __fuse_get_request(struct fuse_req *req)
116 {
117 	atomic_inc(&req->count);
118 }
119 
120 /* Must be called with > 1 refcount */
121 static void __fuse_put_request(struct fuse_req *req)
122 {
123 	BUG_ON(atomic_read(&req->count) < 2);
124 	atomic_dec(&req->count);
125 }
126 
127 static void fuse_req_init_context(struct fuse_req *req)
128 {
129 	req->in.h.uid = from_kuid_munged(&init_user_ns, current_fsuid());
130 	req->in.h.gid = from_kgid_munged(&init_user_ns, current_fsgid());
131 	req->in.h.pid = current->pid;
132 }
133 
134 void fuse_set_initialized(struct fuse_conn *fc)
135 {
136 	/* Make sure stores before this are seen on another CPU */
137 	smp_wmb();
138 	fc->initialized = 1;
139 }
140 
141 static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
142 {
143 	return !fc->initialized || (for_background && fc->blocked);
144 }
145 
146 static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
147 				       bool for_background)
148 {
149 	struct fuse_req *req;
150 	int err;
151 	atomic_inc(&fc->num_waiting);
152 
153 	if (fuse_block_alloc(fc, for_background)) {
154 		sigset_t oldset;
155 		int intr;
156 
157 		block_sigs(&oldset);
158 		intr = wait_event_interruptible_exclusive(fc->blocked_waitq,
159 				!fuse_block_alloc(fc, for_background));
160 		restore_sigs(&oldset);
161 		err = -EINTR;
162 		if (intr)
163 			goto out;
164 	}
165 	/* Matches smp_wmb() in fuse_set_initialized() */
166 	smp_rmb();
167 
168 	err = -ENOTCONN;
169 	if (!fc->connected)
170 		goto out;
171 
172 	req = fuse_request_alloc(npages);
173 	err = -ENOMEM;
174 	if (!req) {
175 		if (for_background)
176 			wake_up(&fc->blocked_waitq);
177 		goto out;
178 	}
179 
180 	fuse_req_init_context(req);
181 	req->waiting = 1;
182 	req->background = for_background;
183 	return req;
184 
185  out:
186 	atomic_dec(&fc->num_waiting);
187 	return ERR_PTR(err);
188 }
189 
190 struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
191 {
192 	return __fuse_get_req(fc, npages, false);
193 }
194 EXPORT_SYMBOL_GPL(fuse_get_req);
195 
196 struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
197 					     unsigned npages)
198 {
199 	return __fuse_get_req(fc, npages, true);
200 }
201 EXPORT_SYMBOL_GPL(fuse_get_req_for_background);
202 
203 /*
204  * Return request in fuse_file->reserved_req.  However that may
205  * currently be in use.  If that is the case, wait for it to become
206  * available.
207  */
208 static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
209 					 struct file *file)
210 {
211 	struct fuse_req *req = NULL;
212 	struct fuse_file *ff = file->private_data;
213 
214 	do {
215 		wait_event(fc->reserved_req_waitq, ff->reserved_req);
216 		spin_lock(&fc->lock);
217 		if (ff->reserved_req) {
218 			req = ff->reserved_req;
219 			ff->reserved_req = NULL;
220 			req->stolen_file = get_file(file);
221 		}
222 		spin_unlock(&fc->lock);
223 	} while (!req);
224 
225 	return req;
226 }
227 
228 /*
229  * Put stolen request back into fuse_file->reserved_req
230  */
231 static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
232 {
233 	struct file *file = req->stolen_file;
234 	struct fuse_file *ff = file->private_data;
235 
236 	spin_lock(&fc->lock);
237 	fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
238 	BUG_ON(ff->reserved_req);
239 	ff->reserved_req = req;
240 	wake_up_all(&fc->reserved_req_waitq);
241 	spin_unlock(&fc->lock);
242 	fput(file);
243 }
244 
245 /*
246  * Gets a requests for a file operation, always succeeds
247  *
248  * This is used for sending the FLUSH request, which must get to
249  * userspace, due to POSIX locks which may need to be unlocked.
250  *
251  * If allocation fails due to OOM, use the reserved request in
252  * fuse_file.
253  *
254  * This is very unlikely to deadlock accidentally, since the
255  * filesystem should not have it's own file open.  If deadlock is
256  * intentional, it can still be broken by "aborting" the filesystem.
257  */
258 struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
259 					     struct file *file)
260 {
261 	struct fuse_req *req;
262 
263 	atomic_inc(&fc->num_waiting);
264 	wait_event(fc->blocked_waitq, fc->initialized);
265 	/* Matches smp_wmb() in fuse_set_initialized() */
266 	smp_rmb();
267 	req = fuse_request_alloc(0);
268 	if (!req)
269 		req = get_reserved_req(fc, file);
270 
271 	fuse_req_init_context(req);
272 	req->waiting = 1;
273 	req->background = 0;
274 	return req;
275 }
276 
277 void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
278 {
279 	if (atomic_dec_and_test(&req->count)) {
280 		if (unlikely(req->background)) {
281 			/*
282 			 * We get here in the unlikely case that a background
283 			 * request was allocated but not sent
284 			 */
285 			spin_lock(&fc->lock);
286 			if (!fc->blocked)
287 				wake_up(&fc->blocked_waitq);
288 			spin_unlock(&fc->lock);
289 		}
290 
291 		if (req->waiting)
292 			atomic_dec(&fc->num_waiting);
293 
294 		if (req->stolen_file)
295 			put_reserved_req(fc, req);
296 		else
297 			fuse_request_free(req);
298 	}
299 }
300 EXPORT_SYMBOL_GPL(fuse_put_request);
301 
302 static unsigned len_args(unsigned numargs, struct fuse_arg *args)
303 {
304 	unsigned nbytes = 0;
305 	unsigned i;
306 
307 	for (i = 0; i < numargs; i++)
308 		nbytes += args[i].size;
309 
310 	return nbytes;
311 }
312 
313 static u64 fuse_get_unique(struct fuse_conn *fc)
314 {
315 	fc->reqctr++;
316 	/* zero is special */
317 	if (fc->reqctr == 0)
318 		fc->reqctr = 1;
319 
320 	return fc->reqctr;
321 }
322 
323 static void queue_request(struct fuse_conn *fc, struct fuse_req *req)
324 {
325 	req->in.h.len = sizeof(struct fuse_in_header) +
326 		len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
327 	list_add_tail(&req->list, &fc->pending);
328 	req->state = FUSE_REQ_PENDING;
329 	if (!req->waiting) {
330 		req->waiting = 1;
331 		atomic_inc(&fc->num_waiting);
332 	}
333 	wake_up(&fc->waitq);
334 	kill_fasync(&fc->fasync, SIGIO, POLL_IN);
335 }
336 
337 void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
338 		       u64 nodeid, u64 nlookup)
339 {
340 	forget->forget_one.nodeid = nodeid;
341 	forget->forget_one.nlookup = nlookup;
342 
343 	spin_lock(&fc->lock);
344 	if (fc->connected) {
345 		fc->forget_list_tail->next = forget;
346 		fc->forget_list_tail = forget;
347 		wake_up(&fc->waitq);
348 		kill_fasync(&fc->fasync, SIGIO, POLL_IN);
349 	} else {
350 		kfree(forget);
351 	}
352 	spin_unlock(&fc->lock);
353 }
354 
355 static void flush_bg_queue(struct fuse_conn *fc)
356 {
357 	while (fc->active_background < fc->max_background &&
358 	       !list_empty(&fc->bg_queue)) {
359 		struct fuse_req *req;
360 
361 		req = list_entry(fc->bg_queue.next, struct fuse_req, list);
362 		list_del(&req->list);
363 		fc->active_background++;
364 		req->in.h.unique = fuse_get_unique(fc);
365 		queue_request(fc, req);
366 	}
367 }
368 
369 /*
370  * This function is called when a request is finished.  Either a reply
371  * has arrived or it was aborted (and not yet sent) or some error
372  * occurred during communication with userspace, or the device file
373  * was closed.  The requester thread is woken up (if still waiting),
374  * the 'end' callback is called if given, else the reference to the
375  * request is released
376  *
377  * Called with fc->lock, unlocks it
378  */
379 static void request_end(struct fuse_conn *fc, struct fuse_req *req)
380 __releases(fc->lock)
381 {
382 	void (*end) (struct fuse_conn *, struct fuse_req *) = req->end;
383 	req->end = NULL;
384 	list_del(&req->list);
385 	list_del(&req->intr_entry);
386 	req->state = FUSE_REQ_FINISHED;
387 	if (req->background) {
388 		req->background = 0;
389 
390 		if (fc->num_background == fc->max_background)
391 			fc->blocked = 0;
392 
393 		/* Wake up next waiter, if any */
394 		if (!fc->blocked && waitqueue_active(&fc->blocked_waitq))
395 			wake_up(&fc->blocked_waitq);
396 
397 		if (fc->num_background == fc->congestion_threshold &&
398 		    fc->connected && fc->bdi_initialized) {
399 			clear_bdi_congested(&fc->bdi, BLK_RW_SYNC);
400 			clear_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
401 		}
402 		fc->num_background--;
403 		fc->active_background--;
404 		flush_bg_queue(fc);
405 	}
406 	spin_unlock(&fc->lock);
407 	wake_up(&req->waitq);
408 	if (end)
409 		end(fc, req);
410 	fuse_put_request(fc, req);
411 }
412 
413 static void wait_answer_interruptible(struct fuse_conn *fc,
414 				      struct fuse_req *req)
415 __releases(fc->lock)
416 __acquires(fc->lock)
417 {
418 	if (signal_pending(current))
419 		return;
420 
421 	spin_unlock(&fc->lock);
422 	wait_event_interruptible(req->waitq, req->state == FUSE_REQ_FINISHED);
423 	spin_lock(&fc->lock);
424 }
425 
426 static void queue_interrupt(struct fuse_conn *fc, struct fuse_req *req)
427 {
428 	list_add_tail(&req->intr_entry, &fc->interrupts);
429 	wake_up(&fc->waitq);
430 	kill_fasync(&fc->fasync, SIGIO, POLL_IN);
431 }
432 
433 static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
434 __releases(fc->lock)
435 __acquires(fc->lock)
436 {
437 	if (!fc->no_interrupt) {
438 		/* Any signal may interrupt this */
439 		wait_answer_interruptible(fc, req);
440 
441 		if (req->aborted)
442 			goto aborted;
443 		if (req->state == FUSE_REQ_FINISHED)
444 			return;
445 
446 		req->interrupted = 1;
447 		if (req->state == FUSE_REQ_SENT)
448 			queue_interrupt(fc, req);
449 	}
450 
451 	if (!req->force) {
452 		sigset_t oldset;
453 
454 		/* Only fatal signals may interrupt this */
455 		block_sigs(&oldset);
456 		wait_answer_interruptible(fc, req);
457 		restore_sigs(&oldset);
458 
459 		if (req->aborted)
460 			goto aborted;
461 		if (req->state == FUSE_REQ_FINISHED)
462 			return;
463 
464 		/* Request is not yet in userspace, bail out */
465 		if (req->state == FUSE_REQ_PENDING) {
466 			list_del(&req->list);
467 			__fuse_put_request(req);
468 			req->out.h.error = -EINTR;
469 			return;
470 		}
471 	}
472 
473 	/*
474 	 * Either request is already in userspace, or it was forced.
475 	 * Wait it out.
476 	 */
477 	spin_unlock(&fc->lock);
478 	wait_event(req->waitq, req->state == FUSE_REQ_FINISHED);
479 	spin_lock(&fc->lock);
480 
481 	if (!req->aborted)
482 		return;
483 
484  aborted:
485 	BUG_ON(req->state != FUSE_REQ_FINISHED);
486 	if (req->locked) {
487 		/* This is uninterruptible sleep, because data is
488 		   being copied to/from the buffers of req.  During
489 		   locked state, there mustn't be any filesystem
490 		   operation (e.g. page fault), since that could lead
491 		   to deadlock */
492 		spin_unlock(&fc->lock);
493 		wait_event(req->waitq, !req->locked);
494 		spin_lock(&fc->lock);
495 	}
496 }
497 
498 static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
499 {
500 	BUG_ON(req->background);
501 	spin_lock(&fc->lock);
502 	if (!fc->connected)
503 		req->out.h.error = -ENOTCONN;
504 	else if (fc->conn_error)
505 		req->out.h.error = -ECONNREFUSED;
506 	else {
507 		req->in.h.unique = fuse_get_unique(fc);
508 		queue_request(fc, req);
509 		/* acquire extra reference, since request is still needed
510 		   after request_end() */
511 		__fuse_get_request(req);
512 
513 		request_wait_answer(fc, req);
514 	}
515 	spin_unlock(&fc->lock);
516 }
517 
518 void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
519 {
520 	req->isreply = 1;
521 	__fuse_request_send(fc, req);
522 }
523 EXPORT_SYMBOL_GPL(fuse_request_send);
524 
525 static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
526 {
527 	if (fc->minor < 4 && args->in.h.opcode == FUSE_STATFS)
528 		args->out.args[0].size = FUSE_COMPAT_STATFS_SIZE;
529 
530 	if (fc->minor < 9) {
531 		switch (args->in.h.opcode) {
532 		case FUSE_LOOKUP:
533 		case FUSE_CREATE:
534 		case FUSE_MKNOD:
535 		case FUSE_MKDIR:
536 		case FUSE_SYMLINK:
537 		case FUSE_LINK:
538 			args->out.args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
539 			break;
540 		case FUSE_GETATTR:
541 		case FUSE_SETATTR:
542 			args->out.args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
543 			break;
544 		}
545 	}
546 	if (fc->minor < 12) {
547 		switch (args->in.h.opcode) {
548 		case FUSE_CREATE:
549 			args->in.args[0].size = sizeof(struct fuse_open_in);
550 			break;
551 		case FUSE_MKNOD:
552 			args->in.args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
553 			break;
554 		}
555 	}
556 }
557 
558 ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
559 {
560 	struct fuse_req *req;
561 	ssize_t ret;
562 
563 	req = fuse_get_req(fc, 0);
564 	if (IS_ERR(req))
565 		return PTR_ERR(req);
566 
567 	/* Needs to be done after fuse_get_req() so that fc->minor is valid */
568 	fuse_adjust_compat(fc, args);
569 
570 	req->in.h.opcode = args->in.h.opcode;
571 	req->in.h.nodeid = args->in.h.nodeid;
572 	req->in.numargs = args->in.numargs;
573 	memcpy(req->in.args, args->in.args,
574 	       args->in.numargs * sizeof(struct fuse_in_arg));
575 	req->out.argvar = args->out.argvar;
576 	req->out.numargs = args->out.numargs;
577 	memcpy(req->out.args, args->out.args,
578 	       args->out.numargs * sizeof(struct fuse_arg));
579 	fuse_request_send(fc, req);
580 	ret = req->out.h.error;
581 	if (!ret && args->out.argvar) {
582 		BUG_ON(args->out.numargs != 1);
583 		ret = req->out.args[0].size;
584 	}
585 	fuse_put_request(fc, req);
586 
587 	return ret;
588 }
589 
590 static void fuse_request_send_nowait_locked(struct fuse_conn *fc,
591 					    struct fuse_req *req)
592 {
593 	BUG_ON(!req->background);
594 	fc->num_background++;
595 	if (fc->num_background == fc->max_background)
596 		fc->blocked = 1;
597 	if (fc->num_background == fc->congestion_threshold &&
598 	    fc->bdi_initialized) {
599 		set_bdi_congested(&fc->bdi, BLK_RW_SYNC);
600 		set_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
601 	}
602 	list_add_tail(&req->list, &fc->bg_queue);
603 	flush_bg_queue(fc);
604 }
605 
606 static void fuse_request_send_nowait(struct fuse_conn *fc, struct fuse_req *req)
607 {
608 	spin_lock(&fc->lock);
609 	if (fc->connected) {
610 		fuse_request_send_nowait_locked(fc, req);
611 		spin_unlock(&fc->lock);
612 	} else {
613 		req->out.h.error = -ENOTCONN;
614 		request_end(fc, req);
615 	}
616 }
617 
618 void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
619 {
620 	req->isreply = 1;
621 	fuse_request_send_nowait(fc, req);
622 }
623 EXPORT_SYMBOL_GPL(fuse_request_send_background);
624 
625 static int fuse_request_send_notify_reply(struct fuse_conn *fc,
626 					  struct fuse_req *req, u64 unique)
627 {
628 	int err = -ENODEV;
629 
630 	req->isreply = 0;
631 	req->in.h.unique = unique;
632 	spin_lock(&fc->lock);
633 	if (fc->connected) {
634 		queue_request(fc, req);
635 		err = 0;
636 	}
637 	spin_unlock(&fc->lock);
638 
639 	return err;
640 }
641 
642 /*
643  * Called under fc->lock
644  *
645  * fc->connected must have been checked previously
646  */
647 void fuse_request_send_background_locked(struct fuse_conn *fc,
648 					 struct fuse_req *req)
649 {
650 	req->isreply = 1;
651 	fuse_request_send_nowait_locked(fc, req);
652 }
653 
654 void fuse_force_forget(struct file *file, u64 nodeid)
655 {
656 	struct inode *inode = file_inode(file);
657 	struct fuse_conn *fc = get_fuse_conn(inode);
658 	struct fuse_req *req;
659 	struct fuse_forget_in inarg;
660 
661 	memset(&inarg, 0, sizeof(inarg));
662 	inarg.nlookup = 1;
663 	req = fuse_get_req_nofail_nopages(fc, file);
664 	req->in.h.opcode = FUSE_FORGET;
665 	req->in.h.nodeid = nodeid;
666 	req->in.numargs = 1;
667 	req->in.args[0].size = sizeof(inarg);
668 	req->in.args[0].value = &inarg;
669 	req->isreply = 0;
670 	__fuse_request_send(fc, req);
671 	/* ignore errors */
672 	fuse_put_request(fc, req);
673 }
674 
675 /*
676  * Lock the request.  Up to the next unlock_request() there mustn't be
677  * anything that could cause a page-fault.  If the request was already
678  * aborted bail out.
679  */
680 static int lock_request(struct fuse_conn *fc, struct fuse_req *req)
681 {
682 	int err = 0;
683 	if (req) {
684 		spin_lock(&fc->lock);
685 		if (req->aborted)
686 			err = -ENOENT;
687 		else
688 			req->locked = 1;
689 		spin_unlock(&fc->lock);
690 	}
691 	return err;
692 }
693 
694 /*
695  * Unlock request.  If it was aborted during being locked, the
696  * requester thread is currently waiting for it to be unlocked, so
697  * wake it up.
698  */
699 static void unlock_request(struct fuse_conn *fc, struct fuse_req *req)
700 {
701 	if (req) {
702 		spin_lock(&fc->lock);
703 		req->locked = 0;
704 		if (req->aborted)
705 			wake_up(&req->waitq);
706 		spin_unlock(&fc->lock);
707 	}
708 }
709 
710 struct fuse_copy_state {
711 	struct fuse_conn *fc;
712 	int write;
713 	struct fuse_req *req;
714 	const struct iovec *iov;
715 	struct pipe_buffer *pipebufs;
716 	struct pipe_buffer *currbuf;
717 	struct pipe_inode_info *pipe;
718 	unsigned long nr_segs;
719 	unsigned long seglen;
720 	unsigned long addr;
721 	struct page *pg;
722 	unsigned len;
723 	unsigned offset;
724 	unsigned move_pages:1;
725 };
726 
727 static void fuse_copy_init(struct fuse_copy_state *cs, struct fuse_conn *fc,
728 			   int write,
729 			   const struct iovec *iov, unsigned long nr_segs)
730 {
731 	memset(cs, 0, sizeof(*cs));
732 	cs->fc = fc;
733 	cs->write = write;
734 	cs->iov = iov;
735 	cs->nr_segs = nr_segs;
736 }
737 
738 /* Unmap and put previous page of userspace buffer */
739 static void fuse_copy_finish(struct fuse_copy_state *cs)
740 {
741 	if (cs->currbuf) {
742 		struct pipe_buffer *buf = cs->currbuf;
743 
744 		if (cs->write)
745 			buf->len = PAGE_SIZE - cs->len;
746 		cs->currbuf = NULL;
747 	} else if (cs->pg) {
748 		if (cs->write) {
749 			flush_dcache_page(cs->pg);
750 			set_page_dirty_lock(cs->pg);
751 		}
752 		put_page(cs->pg);
753 	}
754 	cs->pg = NULL;
755 }
756 
757 /*
758  * Get another pagefull of userspace buffer, and map it to kernel
759  * address space, and lock request
760  */
761 static int fuse_copy_fill(struct fuse_copy_state *cs)
762 {
763 	struct page *page;
764 	int err;
765 
766 	unlock_request(cs->fc, cs->req);
767 	fuse_copy_finish(cs);
768 	if (cs->pipebufs) {
769 		struct pipe_buffer *buf = cs->pipebufs;
770 
771 		if (!cs->write) {
772 			err = buf->ops->confirm(cs->pipe, buf);
773 			if (err)
774 				return err;
775 
776 			BUG_ON(!cs->nr_segs);
777 			cs->currbuf = buf;
778 			cs->pg = buf->page;
779 			cs->offset = buf->offset;
780 			cs->len = buf->len;
781 			cs->pipebufs++;
782 			cs->nr_segs--;
783 		} else {
784 			if (cs->nr_segs == cs->pipe->buffers)
785 				return -EIO;
786 
787 			page = alloc_page(GFP_HIGHUSER);
788 			if (!page)
789 				return -ENOMEM;
790 
791 			buf->page = page;
792 			buf->offset = 0;
793 			buf->len = 0;
794 
795 			cs->currbuf = buf;
796 			cs->pg = page;
797 			cs->offset = 0;
798 			cs->len = PAGE_SIZE;
799 			cs->pipebufs++;
800 			cs->nr_segs++;
801 		}
802 	} else {
803 		if (!cs->seglen) {
804 			BUG_ON(!cs->nr_segs);
805 			cs->seglen = cs->iov[0].iov_len;
806 			cs->addr = (unsigned long) cs->iov[0].iov_base;
807 			cs->iov++;
808 			cs->nr_segs--;
809 		}
810 		err = get_user_pages_fast(cs->addr, 1, cs->write, &page);
811 		if (err < 0)
812 			return err;
813 		BUG_ON(err != 1);
814 		cs->pg = page;
815 		cs->offset = cs->addr % PAGE_SIZE;
816 		cs->len = min(PAGE_SIZE - cs->offset, cs->seglen);
817 		cs->seglen -= cs->len;
818 		cs->addr += cs->len;
819 	}
820 
821 	return lock_request(cs->fc, cs->req);
822 }
823 
824 /* Do as much copy to/from userspace buffer as we can */
825 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
826 {
827 	unsigned ncpy = min(*size, cs->len);
828 	if (val) {
829 		void *pgaddr = kmap_atomic(cs->pg);
830 		void *buf = pgaddr + cs->offset;
831 
832 		if (cs->write)
833 			memcpy(buf, *val, ncpy);
834 		else
835 			memcpy(*val, buf, ncpy);
836 
837 		kunmap_atomic(pgaddr);
838 		*val += ncpy;
839 	}
840 	*size -= ncpy;
841 	cs->len -= ncpy;
842 	cs->offset += ncpy;
843 	return ncpy;
844 }
845 
846 static int fuse_check_page(struct page *page)
847 {
848 	if (page_mapcount(page) ||
849 	    page->mapping != NULL ||
850 	    page_count(page) != 1 ||
851 	    (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
852 	     ~(1 << PG_locked |
853 	       1 << PG_referenced |
854 	       1 << PG_uptodate |
855 	       1 << PG_lru |
856 	       1 << PG_active |
857 	       1 << PG_reclaim))) {
858 		printk(KERN_WARNING "fuse: trying to steal weird page\n");
859 		printk(KERN_WARNING "  page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
860 		return 1;
861 	}
862 	return 0;
863 }
864 
865 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
866 {
867 	int err;
868 	struct page *oldpage = *pagep;
869 	struct page *newpage;
870 	struct pipe_buffer *buf = cs->pipebufs;
871 
872 	unlock_request(cs->fc, cs->req);
873 	fuse_copy_finish(cs);
874 
875 	err = buf->ops->confirm(cs->pipe, buf);
876 	if (err)
877 		return err;
878 
879 	BUG_ON(!cs->nr_segs);
880 	cs->currbuf = buf;
881 	cs->len = buf->len;
882 	cs->pipebufs++;
883 	cs->nr_segs--;
884 
885 	if (cs->len != PAGE_SIZE)
886 		goto out_fallback;
887 
888 	if (buf->ops->steal(cs->pipe, buf) != 0)
889 		goto out_fallback;
890 
891 	newpage = buf->page;
892 
893 	if (WARN_ON(!PageUptodate(newpage)))
894 		return -EIO;
895 
896 	ClearPageMappedToDisk(newpage);
897 
898 	if (fuse_check_page(newpage) != 0)
899 		goto out_fallback_unlock;
900 
901 	/*
902 	 * This is a new and locked page, it shouldn't be mapped or
903 	 * have any special flags on it
904 	 */
905 	if (WARN_ON(page_mapped(oldpage)))
906 		goto out_fallback_unlock;
907 	if (WARN_ON(page_has_private(oldpage)))
908 		goto out_fallback_unlock;
909 	if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
910 		goto out_fallback_unlock;
911 	if (WARN_ON(PageMlocked(oldpage)))
912 		goto out_fallback_unlock;
913 
914 	err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
915 	if (err) {
916 		unlock_page(newpage);
917 		return err;
918 	}
919 
920 	page_cache_get(newpage);
921 
922 	if (!(buf->flags & PIPE_BUF_FLAG_LRU))
923 		lru_cache_add_file(newpage);
924 
925 	err = 0;
926 	spin_lock(&cs->fc->lock);
927 	if (cs->req->aborted)
928 		err = -ENOENT;
929 	else
930 		*pagep = newpage;
931 	spin_unlock(&cs->fc->lock);
932 
933 	if (err) {
934 		unlock_page(newpage);
935 		page_cache_release(newpage);
936 		return err;
937 	}
938 
939 	unlock_page(oldpage);
940 	page_cache_release(oldpage);
941 	cs->len = 0;
942 
943 	return 0;
944 
945 out_fallback_unlock:
946 	unlock_page(newpage);
947 out_fallback:
948 	cs->pg = buf->page;
949 	cs->offset = buf->offset;
950 
951 	err = lock_request(cs->fc, cs->req);
952 	if (err)
953 		return err;
954 
955 	return 1;
956 }
957 
958 static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
959 			 unsigned offset, unsigned count)
960 {
961 	struct pipe_buffer *buf;
962 
963 	if (cs->nr_segs == cs->pipe->buffers)
964 		return -EIO;
965 
966 	unlock_request(cs->fc, cs->req);
967 	fuse_copy_finish(cs);
968 
969 	buf = cs->pipebufs;
970 	page_cache_get(page);
971 	buf->page = page;
972 	buf->offset = offset;
973 	buf->len = count;
974 
975 	cs->pipebufs++;
976 	cs->nr_segs++;
977 	cs->len = 0;
978 
979 	return 0;
980 }
981 
982 /*
983  * Copy a page in the request to/from the userspace buffer.  Must be
984  * done atomically
985  */
986 static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
987 			  unsigned offset, unsigned count, int zeroing)
988 {
989 	int err;
990 	struct page *page = *pagep;
991 
992 	if (page && zeroing && count < PAGE_SIZE)
993 		clear_highpage(page);
994 
995 	while (count) {
996 		if (cs->write && cs->pipebufs && page) {
997 			return fuse_ref_page(cs, page, offset, count);
998 		} else if (!cs->len) {
999 			if (cs->move_pages && page &&
1000 			    offset == 0 && count == PAGE_SIZE) {
1001 				err = fuse_try_move_page(cs, pagep);
1002 				if (err <= 0)
1003 					return err;
1004 			} else {
1005 				err = fuse_copy_fill(cs);
1006 				if (err)
1007 					return err;
1008 			}
1009 		}
1010 		if (page) {
1011 			void *mapaddr = kmap_atomic(page);
1012 			void *buf = mapaddr + offset;
1013 			offset += fuse_copy_do(cs, &buf, &count);
1014 			kunmap_atomic(mapaddr);
1015 		} else
1016 			offset += fuse_copy_do(cs, NULL, &count);
1017 	}
1018 	if (page && !cs->write)
1019 		flush_dcache_page(page);
1020 	return 0;
1021 }
1022 
1023 /* Copy pages in the request to/from userspace buffer */
1024 static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
1025 			   int zeroing)
1026 {
1027 	unsigned i;
1028 	struct fuse_req *req = cs->req;
1029 
1030 	for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
1031 		int err;
1032 		unsigned offset = req->page_descs[i].offset;
1033 		unsigned count = min(nbytes, req->page_descs[i].length);
1034 
1035 		err = fuse_copy_page(cs, &req->pages[i], offset, count,
1036 				     zeroing);
1037 		if (err)
1038 			return err;
1039 
1040 		nbytes -= count;
1041 	}
1042 	return 0;
1043 }
1044 
1045 /* Copy a single argument in the request to/from userspace buffer */
1046 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1047 {
1048 	while (size) {
1049 		if (!cs->len) {
1050 			int err = fuse_copy_fill(cs);
1051 			if (err)
1052 				return err;
1053 		}
1054 		fuse_copy_do(cs, &val, &size);
1055 	}
1056 	return 0;
1057 }
1058 
1059 /* Copy request arguments to/from userspace buffer */
1060 static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1061 			  unsigned argpages, struct fuse_arg *args,
1062 			  int zeroing)
1063 {
1064 	int err = 0;
1065 	unsigned i;
1066 
1067 	for (i = 0; !err && i < numargs; i++)  {
1068 		struct fuse_arg *arg = &args[i];
1069 		if (i == numargs - 1 && argpages)
1070 			err = fuse_copy_pages(cs, arg->size, zeroing);
1071 		else
1072 			err = fuse_copy_one(cs, arg->value, arg->size);
1073 	}
1074 	return err;
1075 }
1076 
1077 static int forget_pending(struct fuse_conn *fc)
1078 {
1079 	return fc->forget_list_head.next != NULL;
1080 }
1081 
1082 static int request_pending(struct fuse_conn *fc)
1083 {
1084 	return !list_empty(&fc->pending) || !list_empty(&fc->interrupts) ||
1085 		forget_pending(fc);
1086 }
1087 
1088 /* Wait until a request is available on the pending list */
1089 static void request_wait(struct fuse_conn *fc)
1090 __releases(fc->lock)
1091 __acquires(fc->lock)
1092 {
1093 	DECLARE_WAITQUEUE(wait, current);
1094 
1095 	add_wait_queue_exclusive(&fc->waitq, &wait);
1096 	while (fc->connected && !request_pending(fc)) {
1097 		set_current_state(TASK_INTERRUPTIBLE);
1098 		if (signal_pending(current))
1099 			break;
1100 
1101 		spin_unlock(&fc->lock);
1102 		schedule();
1103 		spin_lock(&fc->lock);
1104 	}
1105 	set_current_state(TASK_RUNNING);
1106 	remove_wait_queue(&fc->waitq, &wait);
1107 }
1108 
1109 /*
1110  * Transfer an interrupt request to userspace
1111  *
1112  * Unlike other requests this is assembled on demand, without a need
1113  * to allocate a separate fuse_req structure.
1114  *
1115  * Called with fc->lock held, releases it
1116  */
1117 static int fuse_read_interrupt(struct fuse_conn *fc, struct fuse_copy_state *cs,
1118 			       size_t nbytes, struct fuse_req *req)
1119 __releases(fc->lock)
1120 {
1121 	struct fuse_in_header ih;
1122 	struct fuse_interrupt_in arg;
1123 	unsigned reqsize = sizeof(ih) + sizeof(arg);
1124 	int err;
1125 
1126 	list_del_init(&req->intr_entry);
1127 	req->intr_unique = fuse_get_unique(fc);
1128 	memset(&ih, 0, sizeof(ih));
1129 	memset(&arg, 0, sizeof(arg));
1130 	ih.len = reqsize;
1131 	ih.opcode = FUSE_INTERRUPT;
1132 	ih.unique = req->intr_unique;
1133 	arg.unique = req->in.h.unique;
1134 
1135 	spin_unlock(&fc->lock);
1136 	if (nbytes < reqsize)
1137 		return -EINVAL;
1138 
1139 	err = fuse_copy_one(cs, &ih, sizeof(ih));
1140 	if (!err)
1141 		err = fuse_copy_one(cs, &arg, sizeof(arg));
1142 	fuse_copy_finish(cs);
1143 
1144 	return err ? err : reqsize;
1145 }
1146 
1147 static struct fuse_forget_link *dequeue_forget(struct fuse_conn *fc,
1148 					       unsigned max,
1149 					       unsigned *countp)
1150 {
1151 	struct fuse_forget_link *head = fc->forget_list_head.next;
1152 	struct fuse_forget_link **newhead = &head;
1153 	unsigned count;
1154 
1155 	for (count = 0; *newhead != NULL && count < max; count++)
1156 		newhead = &(*newhead)->next;
1157 
1158 	fc->forget_list_head.next = *newhead;
1159 	*newhead = NULL;
1160 	if (fc->forget_list_head.next == NULL)
1161 		fc->forget_list_tail = &fc->forget_list_head;
1162 
1163 	if (countp != NULL)
1164 		*countp = count;
1165 
1166 	return head;
1167 }
1168 
1169 static int fuse_read_single_forget(struct fuse_conn *fc,
1170 				   struct fuse_copy_state *cs,
1171 				   size_t nbytes)
1172 __releases(fc->lock)
1173 {
1174 	int err;
1175 	struct fuse_forget_link *forget = dequeue_forget(fc, 1, NULL);
1176 	struct fuse_forget_in arg = {
1177 		.nlookup = forget->forget_one.nlookup,
1178 	};
1179 	struct fuse_in_header ih = {
1180 		.opcode = FUSE_FORGET,
1181 		.nodeid = forget->forget_one.nodeid,
1182 		.unique = fuse_get_unique(fc),
1183 		.len = sizeof(ih) + sizeof(arg),
1184 	};
1185 
1186 	spin_unlock(&fc->lock);
1187 	kfree(forget);
1188 	if (nbytes < ih.len)
1189 		return -EINVAL;
1190 
1191 	err = fuse_copy_one(cs, &ih, sizeof(ih));
1192 	if (!err)
1193 		err = fuse_copy_one(cs, &arg, sizeof(arg));
1194 	fuse_copy_finish(cs);
1195 
1196 	if (err)
1197 		return err;
1198 
1199 	return ih.len;
1200 }
1201 
1202 static int fuse_read_batch_forget(struct fuse_conn *fc,
1203 				   struct fuse_copy_state *cs, size_t nbytes)
1204 __releases(fc->lock)
1205 {
1206 	int err;
1207 	unsigned max_forgets;
1208 	unsigned count;
1209 	struct fuse_forget_link *head;
1210 	struct fuse_batch_forget_in arg = { .count = 0 };
1211 	struct fuse_in_header ih = {
1212 		.opcode = FUSE_BATCH_FORGET,
1213 		.unique = fuse_get_unique(fc),
1214 		.len = sizeof(ih) + sizeof(arg),
1215 	};
1216 
1217 	if (nbytes < ih.len) {
1218 		spin_unlock(&fc->lock);
1219 		return -EINVAL;
1220 	}
1221 
1222 	max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
1223 	head = dequeue_forget(fc, max_forgets, &count);
1224 	spin_unlock(&fc->lock);
1225 
1226 	arg.count = count;
1227 	ih.len += count * sizeof(struct fuse_forget_one);
1228 	err = fuse_copy_one(cs, &ih, sizeof(ih));
1229 	if (!err)
1230 		err = fuse_copy_one(cs, &arg, sizeof(arg));
1231 
1232 	while (head) {
1233 		struct fuse_forget_link *forget = head;
1234 
1235 		if (!err) {
1236 			err = fuse_copy_one(cs, &forget->forget_one,
1237 					    sizeof(forget->forget_one));
1238 		}
1239 		head = forget->next;
1240 		kfree(forget);
1241 	}
1242 
1243 	fuse_copy_finish(cs);
1244 
1245 	if (err)
1246 		return err;
1247 
1248 	return ih.len;
1249 }
1250 
1251 static int fuse_read_forget(struct fuse_conn *fc, struct fuse_copy_state *cs,
1252 			    size_t nbytes)
1253 __releases(fc->lock)
1254 {
1255 	if (fc->minor < 16 || fc->forget_list_head.next->next == NULL)
1256 		return fuse_read_single_forget(fc, cs, nbytes);
1257 	else
1258 		return fuse_read_batch_forget(fc, cs, nbytes);
1259 }
1260 
1261 /*
1262  * Read a single request into the userspace filesystem's buffer.  This
1263  * function waits until a request is available, then removes it from
1264  * the pending list and copies request data to userspace buffer.  If
1265  * no reply is needed (FORGET) or request has been aborted or there
1266  * was an error during the copying then it's finished by calling
1267  * request_end().  Otherwise add it to the processing list, and set
1268  * the 'sent' flag.
1269  */
1270 static ssize_t fuse_dev_do_read(struct fuse_conn *fc, struct file *file,
1271 				struct fuse_copy_state *cs, size_t nbytes)
1272 {
1273 	int err;
1274 	struct fuse_req *req;
1275 	struct fuse_in *in;
1276 	unsigned reqsize;
1277 
1278  restart:
1279 	spin_lock(&fc->lock);
1280 	err = -EAGAIN;
1281 	if ((file->f_flags & O_NONBLOCK) && fc->connected &&
1282 	    !request_pending(fc))
1283 		goto err_unlock;
1284 
1285 	request_wait(fc);
1286 	err = -ENODEV;
1287 	if (!fc->connected)
1288 		goto err_unlock;
1289 	err = -ERESTARTSYS;
1290 	if (!request_pending(fc))
1291 		goto err_unlock;
1292 
1293 	if (!list_empty(&fc->interrupts)) {
1294 		req = list_entry(fc->interrupts.next, struct fuse_req,
1295 				 intr_entry);
1296 		return fuse_read_interrupt(fc, cs, nbytes, req);
1297 	}
1298 
1299 	if (forget_pending(fc)) {
1300 		if (list_empty(&fc->pending) || fc->forget_batch-- > 0)
1301 			return fuse_read_forget(fc, cs, nbytes);
1302 
1303 		if (fc->forget_batch <= -8)
1304 			fc->forget_batch = 16;
1305 	}
1306 
1307 	req = list_entry(fc->pending.next, struct fuse_req, list);
1308 	req->state = FUSE_REQ_READING;
1309 	list_move(&req->list, &fc->io);
1310 
1311 	in = &req->in;
1312 	reqsize = in->h.len;
1313 	/* If request is too large, reply with an error and restart the read */
1314 	if (nbytes < reqsize) {
1315 		req->out.h.error = -EIO;
1316 		/* SETXATTR is special, since it may contain too large data */
1317 		if (in->h.opcode == FUSE_SETXATTR)
1318 			req->out.h.error = -E2BIG;
1319 		request_end(fc, req);
1320 		goto restart;
1321 	}
1322 	spin_unlock(&fc->lock);
1323 	cs->req = req;
1324 	err = fuse_copy_one(cs, &in->h, sizeof(in->h));
1325 	if (!err)
1326 		err = fuse_copy_args(cs, in->numargs, in->argpages,
1327 				     (struct fuse_arg *) in->args, 0);
1328 	fuse_copy_finish(cs);
1329 	spin_lock(&fc->lock);
1330 	req->locked = 0;
1331 	if (req->aborted) {
1332 		request_end(fc, req);
1333 		return -ENODEV;
1334 	}
1335 	if (err) {
1336 		req->out.h.error = -EIO;
1337 		request_end(fc, req);
1338 		return err;
1339 	}
1340 	if (!req->isreply)
1341 		request_end(fc, req);
1342 	else {
1343 		req->state = FUSE_REQ_SENT;
1344 		list_move_tail(&req->list, &fc->processing);
1345 		if (req->interrupted)
1346 			queue_interrupt(fc, req);
1347 		spin_unlock(&fc->lock);
1348 	}
1349 	return reqsize;
1350 
1351  err_unlock:
1352 	spin_unlock(&fc->lock);
1353 	return err;
1354 }
1355 
1356 static ssize_t fuse_dev_read(struct kiocb *iocb, const struct iovec *iov,
1357 			      unsigned long nr_segs, loff_t pos)
1358 {
1359 	struct fuse_copy_state cs;
1360 	struct file *file = iocb->ki_filp;
1361 	struct fuse_conn *fc = fuse_get_conn(file);
1362 	if (!fc)
1363 		return -EPERM;
1364 
1365 	fuse_copy_init(&cs, fc, 1, iov, nr_segs);
1366 
1367 	return fuse_dev_do_read(fc, file, &cs, iov_length(iov, nr_segs));
1368 }
1369 
1370 static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1371 				    struct pipe_inode_info *pipe,
1372 				    size_t len, unsigned int flags)
1373 {
1374 	int ret;
1375 	int page_nr = 0;
1376 	int do_wakeup = 0;
1377 	struct pipe_buffer *bufs;
1378 	struct fuse_copy_state cs;
1379 	struct fuse_conn *fc = fuse_get_conn(in);
1380 	if (!fc)
1381 		return -EPERM;
1382 
1383 	bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
1384 	if (!bufs)
1385 		return -ENOMEM;
1386 
1387 	fuse_copy_init(&cs, fc, 1, NULL, 0);
1388 	cs.pipebufs = bufs;
1389 	cs.pipe = pipe;
1390 	ret = fuse_dev_do_read(fc, in, &cs, len);
1391 	if (ret < 0)
1392 		goto out;
1393 
1394 	ret = 0;
1395 	pipe_lock(pipe);
1396 
1397 	if (!pipe->readers) {
1398 		send_sig(SIGPIPE, current, 0);
1399 		if (!ret)
1400 			ret = -EPIPE;
1401 		goto out_unlock;
1402 	}
1403 
1404 	if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
1405 		ret = -EIO;
1406 		goto out_unlock;
1407 	}
1408 
1409 	while (page_nr < cs.nr_segs) {
1410 		int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
1411 		struct pipe_buffer *buf = pipe->bufs + newbuf;
1412 
1413 		buf->page = bufs[page_nr].page;
1414 		buf->offset = bufs[page_nr].offset;
1415 		buf->len = bufs[page_nr].len;
1416 		/*
1417 		 * Need to be careful about this.  Having buf->ops in module
1418 		 * code can Oops if the buffer persists after module unload.
1419 		 */
1420 		buf->ops = &nosteal_pipe_buf_ops;
1421 
1422 		pipe->nrbufs++;
1423 		page_nr++;
1424 		ret += buf->len;
1425 
1426 		if (pipe->files)
1427 			do_wakeup = 1;
1428 	}
1429 
1430 out_unlock:
1431 	pipe_unlock(pipe);
1432 
1433 	if (do_wakeup) {
1434 		smp_mb();
1435 		if (waitqueue_active(&pipe->wait))
1436 			wake_up_interruptible(&pipe->wait);
1437 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
1438 	}
1439 
1440 out:
1441 	for (; page_nr < cs.nr_segs; page_nr++)
1442 		page_cache_release(bufs[page_nr].page);
1443 
1444 	kfree(bufs);
1445 	return ret;
1446 }
1447 
1448 static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1449 			    struct fuse_copy_state *cs)
1450 {
1451 	struct fuse_notify_poll_wakeup_out outarg;
1452 	int err = -EINVAL;
1453 
1454 	if (size != sizeof(outarg))
1455 		goto err;
1456 
1457 	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1458 	if (err)
1459 		goto err;
1460 
1461 	fuse_copy_finish(cs);
1462 	return fuse_notify_poll_wakeup(fc, &outarg);
1463 
1464 err:
1465 	fuse_copy_finish(cs);
1466 	return err;
1467 }
1468 
1469 static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1470 				   struct fuse_copy_state *cs)
1471 {
1472 	struct fuse_notify_inval_inode_out outarg;
1473 	int err = -EINVAL;
1474 
1475 	if (size != sizeof(outarg))
1476 		goto err;
1477 
1478 	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1479 	if (err)
1480 		goto err;
1481 	fuse_copy_finish(cs);
1482 
1483 	down_read(&fc->killsb);
1484 	err = -ENOENT;
1485 	if (fc->sb) {
1486 		err = fuse_reverse_inval_inode(fc->sb, outarg.ino,
1487 					       outarg.off, outarg.len);
1488 	}
1489 	up_read(&fc->killsb);
1490 	return err;
1491 
1492 err:
1493 	fuse_copy_finish(cs);
1494 	return err;
1495 }
1496 
1497 static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1498 				   struct fuse_copy_state *cs)
1499 {
1500 	struct fuse_notify_inval_entry_out outarg;
1501 	int err = -ENOMEM;
1502 	char *buf;
1503 	struct qstr name;
1504 
1505 	buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1506 	if (!buf)
1507 		goto err;
1508 
1509 	err = -EINVAL;
1510 	if (size < sizeof(outarg))
1511 		goto err;
1512 
1513 	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1514 	if (err)
1515 		goto err;
1516 
1517 	err = -ENAMETOOLONG;
1518 	if (outarg.namelen > FUSE_NAME_MAX)
1519 		goto err;
1520 
1521 	err = -EINVAL;
1522 	if (size != sizeof(outarg) + outarg.namelen + 1)
1523 		goto err;
1524 
1525 	name.name = buf;
1526 	name.len = outarg.namelen;
1527 	err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1528 	if (err)
1529 		goto err;
1530 	fuse_copy_finish(cs);
1531 	buf[outarg.namelen] = 0;
1532 	name.hash = full_name_hash(name.name, name.len);
1533 
1534 	down_read(&fc->killsb);
1535 	err = -ENOENT;
1536 	if (fc->sb)
1537 		err = fuse_reverse_inval_entry(fc->sb, outarg.parent, 0, &name);
1538 	up_read(&fc->killsb);
1539 	kfree(buf);
1540 	return err;
1541 
1542 err:
1543 	kfree(buf);
1544 	fuse_copy_finish(cs);
1545 	return err;
1546 }
1547 
1548 static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1549 			      struct fuse_copy_state *cs)
1550 {
1551 	struct fuse_notify_delete_out outarg;
1552 	int err = -ENOMEM;
1553 	char *buf;
1554 	struct qstr name;
1555 
1556 	buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1557 	if (!buf)
1558 		goto err;
1559 
1560 	err = -EINVAL;
1561 	if (size < sizeof(outarg))
1562 		goto err;
1563 
1564 	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1565 	if (err)
1566 		goto err;
1567 
1568 	err = -ENAMETOOLONG;
1569 	if (outarg.namelen > FUSE_NAME_MAX)
1570 		goto err;
1571 
1572 	err = -EINVAL;
1573 	if (size != sizeof(outarg) + outarg.namelen + 1)
1574 		goto err;
1575 
1576 	name.name = buf;
1577 	name.len = outarg.namelen;
1578 	err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1579 	if (err)
1580 		goto err;
1581 	fuse_copy_finish(cs);
1582 	buf[outarg.namelen] = 0;
1583 	name.hash = full_name_hash(name.name, name.len);
1584 
1585 	down_read(&fc->killsb);
1586 	err = -ENOENT;
1587 	if (fc->sb)
1588 		err = fuse_reverse_inval_entry(fc->sb, outarg.parent,
1589 					       outarg.child, &name);
1590 	up_read(&fc->killsb);
1591 	kfree(buf);
1592 	return err;
1593 
1594 err:
1595 	kfree(buf);
1596 	fuse_copy_finish(cs);
1597 	return err;
1598 }
1599 
1600 static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1601 			     struct fuse_copy_state *cs)
1602 {
1603 	struct fuse_notify_store_out outarg;
1604 	struct inode *inode;
1605 	struct address_space *mapping;
1606 	u64 nodeid;
1607 	int err;
1608 	pgoff_t index;
1609 	unsigned int offset;
1610 	unsigned int num;
1611 	loff_t file_size;
1612 	loff_t end;
1613 
1614 	err = -EINVAL;
1615 	if (size < sizeof(outarg))
1616 		goto out_finish;
1617 
1618 	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1619 	if (err)
1620 		goto out_finish;
1621 
1622 	err = -EINVAL;
1623 	if (size - sizeof(outarg) != outarg.size)
1624 		goto out_finish;
1625 
1626 	nodeid = outarg.nodeid;
1627 
1628 	down_read(&fc->killsb);
1629 
1630 	err = -ENOENT;
1631 	if (!fc->sb)
1632 		goto out_up_killsb;
1633 
1634 	inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1635 	if (!inode)
1636 		goto out_up_killsb;
1637 
1638 	mapping = inode->i_mapping;
1639 	index = outarg.offset >> PAGE_CACHE_SHIFT;
1640 	offset = outarg.offset & ~PAGE_CACHE_MASK;
1641 	file_size = i_size_read(inode);
1642 	end = outarg.offset + outarg.size;
1643 	if (end > file_size) {
1644 		file_size = end;
1645 		fuse_write_update_size(inode, file_size);
1646 	}
1647 
1648 	num = outarg.size;
1649 	while (num) {
1650 		struct page *page;
1651 		unsigned int this_num;
1652 
1653 		err = -ENOMEM;
1654 		page = find_or_create_page(mapping, index,
1655 					   mapping_gfp_mask(mapping));
1656 		if (!page)
1657 			goto out_iput;
1658 
1659 		this_num = min_t(unsigned, num, PAGE_CACHE_SIZE - offset);
1660 		err = fuse_copy_page(cs, &page, offset, this_num, 0);
1661 		if (!err && offset == 0 &&
1662 		    (this_num == PAGE_CACHE_SIZE || file_size == end))
1663 			SetPageUptodate(page);
1664 		unlock_page(page);
1665 		page_cache_release(page);
1666 
1667 		if (err)
1668 			goto out_iput;
1669 
1670 		num -= this_num;
1671 		offset = 0;
1672 		index++;
1673 	}
1674 
1675 	err = 0;
1676 
1677 out_iput:
1678 	iput(inode);
1679 out_up_killsb:
1680 	up_read(&fc->killsb);
1681 out_finish:
1682 	fuse_copy_finish(cs);
1683 	return err;
1684 }
1685 
1686 static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_req *req)
1687 {
1688 	release_pages(req->pages, req->num_pages, false);
1689 }
1690 
1691 static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode,
1692 			 struct fuse_notify_retrieve_out *outarg)
1693 {
1694 	int err;
1695 	struct address_space *mapping = inode->i_mapping;
1696 	struct fuse_req *req;
1697 	pgoff_t index;
1698 	loff_t file_size;
1699 	unsigned int num;
1700 	unsigned int offset;
1701 	size_t total_len = 0;
1702 	int num_pages;
1703 
1704 	offset = outarg->offset & ~PAGE_CACHE_MASK;
1705 	file_size = i_size_read(inode);
1706 
1707 	num = outarg->size;
1708 	if (outarg->offset > file_size)
1709 		num = 0;
1710 	else if (outarg->offset + num > file_size)
1711 		num = file_size - outarg->offset;
1712 
1713 	num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1714 	num_pages = min(num_pages, FUSE_MAX_PAGES_PER_REQ);
1715 
1716 	req = fuse_get_req(fc, num_pages);
1717 	if (IS_ERR(req))
1718 		return PTR_ERR(req);
1719 
1720 	req->in.h.opcode = FUSE_NOTIFY_REPLY;
1721 	req->in.h.nodeid = outarg->nodeid;
1722 	req->in.numargs = 2;
1723 	req->in.argpages = 1;
1724 	req->page_descs[0].offset = offset;
1725 	req->end = fuse_retrieve_end;
1726 
1727 	index = outarg->offset >> PAGE_CACHE_SHIFT;
1728 
1729 	while (num && req->num_pages < num_pages) {
1730 		struct page *page;
1731 		unsigned int this_num;
1732 
1733 		page = find_get_page(mapping, index);
1734 		if (!page)
1735 			break;
1736 
1737 		this_num = min_t(unsigned, num, PAGE_CACHE_SIZE - offset);
1738 		req->pages[req->num_pages] = page;
1739 		req->page_descs[req->num_pages].length = this_num;
1740 		req->num_pages++;
1741 
1742 		offset = 0;
1743 		num -= this_num;
1744 		total_len += this_num;
1745 		index++;
1746 	}
1747 	req->misc.retrieve_in.offset = outarg->offset;
1748 	req->misc.retrieve_in.size = total_len;
1749 	req->in.args[0].size = sizeof(req->misc.retrieve_in);
1750 	req->in.args[0].value = &req->misc.retrieve_in;
1751 	req->in.args[1].size = total_len;
1752 
1753 	err = fuse_request_send_notify_reply(fc, req, outarg->notify_unique);
1754 	if (err)
1755 		fuse_retrieve_end(fc, req);
1756 
1757 	return err;
1758 }
1759 
1760 static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1761 				struct fuse_copy_state *cs)
1762 {
1763 	struct fuse_notify_retrieve_out outarg;
1764 	struct inode *inode;
1765 	int err;
1766 
1767 	err = -EINVAL;
1768 	if (size != sizeof(outarg))
1769 		goto copy_finish;
1770 
1771 	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1772 	if (err)
1773 		goto copy_finish;
1774 
1775 	fuse_copy_finish(cs);
1776 
1777 	down_read(&fc->killsb);
1778 	err = -ENOENT;
1779 	if (fc->sb) {
1780 		u64 nodeid = outarg.nodeid;
1781 
1782 		inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1783 		if (inode) {
1784 			err = fuse_retrieve(fc, inode, &outarg);
1785 			iput(inode);
1786 		}
1787 	}
1788 	up_read(&fc->killsb);
1789 
1790 	return err;
1791 
1792 copy_finish:
1793 	fuse_copy_finish(cs);
1794 	return err;
1795 }
1796 
1797 static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1798 		       unsigned int size, struct fuse_copy_state *cs)
1799 {
1800 	switch (code) {
1801 	case FUSE_NOTIFY_POLL:
1802 		return fuse_notify_poll(fc, size, cs);
1803 
1804 	case FUSE_NOTIFY_INVAL_INODE:
1805 		return fuse_notify_inval_inode(fc, size, cs);
1806 
1807 	case FUSE_NOTIFY_INVAL_ENTRY:
1808 		return fuse_notify_inval_entry(fc, size, cs);
1809 
1810 	case FUSE_NOTIFY_STORE:
1811 		return fuse_notify_store(fc, size, cs);
1812 
1813 	case FUSE_NOTIFY_RETRIEVE:
1814 		return fuse_notify_retrieve(fc, size, cs);
1815 
1816 	case FUSE_NOTIFY_DELETE:
1817 		return fuse_notify_delete(fc, size, cs);
1818 
1819 	default:
1820 		fuse_copy_finish(cs);
1821 		return -EINVAL;
1822 	}
1823 }
1824 
1825 /* Look up request on processing list by unique ID */
1826 static struct fuse_req *request_find(struct fuse_conn *fc, u64 unique)
1827 {
1828 	struct fuse_req *req;
1829 
1830 	list_for_each_entry(req, &fc->processing, list) {
1831 		if (req->in.h.unique == unique || req->intr_unique == unique)
1832 			return req;
1833 	}
1834 	return NULL;
1835 }
1836 
1837 static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out,
1838 			 unsigned nbytes)
1839 {
1840 	unsigned reqsize = sizeof(struct fuse_out_header);
1841 
1842 	if (out->h.error)
1843 		return nbytes != reqsize ? -EINVAL : 0;
1844 
1845 	reqsize += len_args(out->numargs, out->args);
1846 
1847 	if (reqsize < nbytes || (reqsize > nbytes && !out->argvar))
1848 		return -EINVAL;
1849 	else if (reqsize > nbytes) {
1850 		struct fuse_arg *lastarg = &out->args[out->numargs-1];
1851 		unsigned diffsize = reqsize - nbytes;
1852 		if (diffsize > lastarg->size)
1853 			return -EINVAL;
1854 		lastarg->size -= diffsize;
1855 	}
1856 	return fuse_copy_args(cs, out->numargs, out->argpages, out->args,
1857 			      out->page_zeroing);
1858 }
1859 
1860 /*
1861  * Write a single reply to a request.  First the header is copied from
1862  * the write buffer.  The request is then searched on the processing
1863  * list by the unique ID found in the header.  If found, then remove
1864  * it from the list and copy the rest of the buffer to the request.
1865  * The request is finished by calling request_end()
1866  */
1867 static ssize_t fuse_dev_do_write(struct fuse_conn *fc,
1868 				 struct fuse_copy_state *cs, size_t nbytes)
1869 {
1870 	int err;
1871 	struct fuse_req *req;
1872 	struct fuse_out_header oh;
1873 
1874 	if (nbytes < sizeof(struct fuse_out_header))
1875 		return -EINVAL;
1876 
1877 	err = fuse_copy_one(cs, &oh, sizeof(oh));
1878 	if (err)
1879 		goto err_finish;
1880 
1881 	err = -EINVAL;
1882 	if (oh.len != nbytes)
1883 		goto err_finish;
1884 
1885 	/*
1886 	 * Zero oh.unique indicates unsolicited notification message
1887 	 * and error contains notification code.
1888 	 */
1889 	if (!oh.unique) {
1890 		err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
1891 		return err ? err : nbytes;
1892 	}
1893 
1894 	err = -EINVAL;
1895 	if (oh.error <= -1000 || oh.error > 0)
1896 		goto err_finish;
1897 
1898 	spin_lock(&fc->lock);
1899 	err = -ENOENT;
1900 	if (!fc->connected)
1901 		goto err_unlock;
1902 
1903 	req = request_find(fc, oh.unique);
1904 	if (!req)
1905 		goto err_unlock;
1906 
1907 	if (req->aborted) {
1908 		spin_unlock(&fc->lock);
1909 		fuse_copy_finish(cs);
1910 		spin_lock(&fc->lock);
1911 		request_end(fc, req);
1912 		return -ENOENT;
1913 	}
1914 	/* Is it an interrupt reply? */
1915 	if (req->intr_unique == oh.unique) {
1916 		err = -EINVAL;
1917 		if (nbytes != sizeof(struct fuse_out_header))
1918 			goto err_unlock;
1919 
1920 		if (oh.error == -ENOSYS)
1921 			fc->no_interrupt = 1;
1922 		else if (oh.error == -EAGAIN)
1923 			queue_interrupt(fc, req);
1924 
1925 		spin_unlock(&fc->lock);
1926 		fuse_copy_finish(cs);
1927 		return nbytes;
1928 	}
1929 
1930 	req->state = FUSE_REQ_WRITING;
1931 	list_move(&req->list, &fc->io);
1932 	req->out.h = oh;
1933 	req->locked = 1;
1934 	cs->req = req;
1935 	if (!req->out.page_replace)
1936 		cs->move_pages = 0;
1937 	spin_unlock(&fc->lock);
1938 
1939 	err = copy_out_args(cs, &req->out, nbytes);
1940 	fuse_copy_finish(cs);
1941 
1942 	spin_lock(&fc->lock);
1943 	req->locked = 0;
1944 	if (!err) {
1945 		if (req->aborted)
1946 			err = -ENOENT;
1947 	} else if (!req->aborted)
1948 		req->out.h.error = -EIO;
1949 	request_end(fc, req);
1950 
1951 	return err ? err : nbytes;
1952 
1953  err_unlock:
1954 	spin_unlock(&fc->lock);
1955  err_finish:
1956 	fuse_copy_finish(cs);
1957 	return err;
1958 }
1959 
1960 static ssize_t fuse_dev_write(struct kiocb *iocb, const struct iovec *iov,
1961 			      unsigned long nr_segs, loff_t pos)
1962 {
1963 	struct fuse_copy_state cs;
1964 	struct fuse_conn *fc = fuse_get_conn(iocb->ki_filp);
1965 	if (!fc)
1966 		return -EPERM;
1967 
1968 	fuse_copy_init(&cs, fc, 0, iov, nr_segs);
1969 
1970 	return fuse_dev_do_write(fc, &cs, iov_length(iov, nr_segs));
1971 }
1972 
1973 static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
1974 				     struct file *out, loff_t *ppos,
1975 				     size_t len, unsigned int flags)
1976 {
1977 	unsigned nbuf;
1978 	unsigned idx;
1979 	struct pipe_buffer *bufs;
1980 	struct fuse_copy_state cs;
1981 	struct fuse_conn *fc;
1982 	size_t rem;
1983 	ssize_t ret;
1984 
1985 	fc = fuse_get_conn(out);
1986 	if (!fc)
1987 		return -EPERM;
1988 
1989 	bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
1990 	if (!bufs)
1991 		return -ENOMEM;
1992 
1993 	pipe_lock(pipe);
1994 	nbuf = 0;
1995 	rem = 0;
1996 	for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
1997 		rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;
1998 
1999 	ret = -EINVAL;
2000 	if (rem < len) {
2001 		pipe_unlock(pipe);
2002 		goto out;
2003 	}
2004 
2005 	rem = len;
2006 	while (rem) {
2007 		struct pipe_buffer *ibuf;
2008 		struct pipe_buffer *obuf;
2009 
2010 		BUG_ON(nbuf >= pipe->buffers);
2011 		BUG_ON(!pipe->nrbufs);
2012 		ibuf = &pipe->bufs[pipe->curbuf];
2013 		obuf = &bufs[nbuf];
2014 
2015 		if (rem >= ibuf->len) {
2016 			*obuf = *ibuf;
2017 			ibuf->ops = NULL;
2018 			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
2019 			pipe->nrbufs--;
2020 		} else {
2021 			ibuf->ops->get(pipe, ibuf);
2022 			*obuf = *ibuf;
2023 			obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
2024 			obuf->len = rem;
2025 			ibuf->offset += obuf->len;
2026 			ibuf->len -= obuf->len;
2027 		}
2028 		nbuf++;
2029 		rem -= obuf->len;
2030 	}
2031 	pipe_unlock(pipe);
2032 
2033 	fuse_copy_init(&cs, fc, 0, NULL, nbuf);
2034 	cs.pipebufs = bufs;
2035 	cs.pipe = pipe;
2036 
2037 	if (flags & SPLICE_F_MOVE)
2038 		cs.move_pages = 1;
2039 
2040 	ret = fuse_dev_do_write(fc, &cs, len);
2041 
2042 	for (idx = 0; idx < nbuf; idx++) {
2043 		struct pipe_buffer *buf = &bufs[idx];
2044 		buf->ops->release(pipe, buf);
2045 	}
2046 out:
2047 	kfree(bufs);
2048 	return ret;
2049 }
2050 
2051 static unsigned fuse_dev_poll(struct file *file, poll_table *wait)
2052 {
2053 	unsigned mask = POLLOUT | POLLWRNORM;
2054 	struct fuse_conn *fc = fuse_get_conn(file);
2055 	if (!fc)
2056 		return POLLERR;
2057 
2058 	poll_wait(file, &fc->waitq, wait);
2059 
2060 	spin_lock(&fc->lock);
2061 	if (!fc->connected)
2062 		mask = POLLERR;
2063 	else if (request_pending(fc))
2064 		mask |= POLLIN | POLLRDNORM;
2065 	spin_unlock(&fc->lock);
2066 
2067 	return mask;
2068 }
2069 
2070 /*
2071  * Abort all requests on the given list (pending or processing)
2072  *
2073  * This function releases and reacquires fc->lock
2074  */
2075 static void end_requests(struct fuse_conn *fc, struct list_head *head)
2076 __releases(fc->lock)
2077 __acquires(fc->lock)
2078 {
2079 	while (!list_empty(head)) {
2080 		struct fuse_req *req;
2081 		req = list_entry(head->next, struct fuse_req, list);
2082 		req->out.h.error = -ECONNABORTED;
2083 		request_end(fc, req);
2084 		spin_lock(&fc->lock);
2085 	}
2086 }
2087 
2088 /*
2089  * Abort requests under I/O
2090  *
2091  * The requests are set to aborted and finished, and the request
2092  * waiter is woken up.  This will make request_wait_answer() wait
2093  * until the request is unlocked and then return.
2094  *
2095  * If the request is asynchronous, then the end function needs to be
2096  * called after waiting for the request to be unlocked (if it was
2097  * locked).
2098  */
2099 static void end_io_requests(struct fuse_conn *fc)
2100 __releases(fc->lock)
2101 __acquires(fc->lock)
2102 {
2103 	while (!list_empty(&fc->io)) {
2104 		struct fuse_req *req =
2105 			list_entry(fc->io.next, struct fuse_req, list);
2106 		void (*end) (struct fuse_conn *, struct fuse_req *) = req->end;
2107 
2108 		req->aborted = 1;
2109 		req->out.h.error = -ECONNABORTED;
2110 		req->state = FUSE_REQ_FINISHED;
2111 		list_del_init(&req->list);
2112 		wake_up(&req->waitq);
2113 		if (end) {
2114 			req->end = NULL;
2115 			__fuse_get_request(req);
2116 			spin_unlock(&fc->lock);
2117 			wait_event(req->waitq, !req->locked);
2118 			end(fc, req);
2119 			fuse_put_request(fc, req);
2120 			spin_lock(&fc->lock);
2121 		}
2122 	}
2123 }
2124 
2125 static void end_queued_requests(struct fuse_conn *fc)
2126 __releases(fc->lock)
2127 __acquires(fc->lock)
2128 {
2129 	fc->max_background = UINT_MAX;
2130 	flush_bg_queue(fc);
2131 	end_requests(fc, &fc->pending);
2132 	end_requests(fc, &fc->processing);
2133 	while (forget_pending(fc))
2134 		kfree(dequeue_forget(fc, 1, NULL));
2135 }
2136 
2137 static void end_polls(struct fuse_conn *fc)
2138 {
2139 	struct rb_node *p;
2140 
2141 	p = rb_first(&fc->polled_files);
2142 
2143 	while (p) {
2144 		struct fuse_file *ff;
2145 		ff = rb_entry(p, struct fuse_file, polled_node);
2146 		wake_up_interruptible_all(&ff->poll_wait);
2147 
2148 		p = rb_next(p);
2149 	}
2150 }
2151 
2152 /*
2153  * Abort all requests.
2154  *
2155  * Emergency exit in case of a malicious or accidental deadlock, or
2156  * just a hung filesystem.
2157  *
2158  * The same effect is usually achievable through killing the
2159  * filesystem daemon and all users of the filesystem.  The exception
2160  * is the combination of an asynchronous request and the tricky
2161  * deadlock (see Documentation/filesystems/fuse.txt).
2162  *
2163  * During the aborting, progression of requests from the pending and
2164  * processing lists onto the io list, and progression of new requests
2165  * onto the pending list is prevented by req->connected being false.
2166  *
2167  * Progression of requests under I/O to the processing list is
2168  * prevented by the req->aborted flag being true for these requests.
2169  * For this reason requests on the io list must be aborted first.
2170  */
2171 void fuse_abort_conn(struct fuse_conn *fc)
2172 {
2173 	spin_lock(&fc->lock);
2174 	if (fc->connected) {
2175 		fc->connected = 0;
2176 		fc->blocked = 0;
2177 		fuse_set_initialized(fc);
2178 		end_io_requests(fc);
2179 		end_queued_requests(fc);
2180 		end_polls(fc);
2181 		wake_up_all(&fc->waitq);
2182 		wake_up_all(&fc->blocked_waitq);
2183 		kill_fasync(&fc->fasync, SIGIO, POLL_IN);
2184 	}
2185 	spin_unlock(&fc->lock);
2186 }
2187 EXPORT_SYMBOL_GPL(fuse_abort_conn);
2188 
2189 int fuse_dev_release(struct inode *inode, struct file *file)
2190 {
2191 	struct fuse_conn *fc = fuse_get_conn(file);
2192 	if (fc) {
2193 		spin_lock(&fc->lock);
2194 		fc->connected = 0;
2195 		fc->blocked = 0;
2196 		fuse_set_initialized(fc);
2197 		end_queued_requests(fc);
2198 		end_polls(fc);
2199 		wake_up_all(&fc->blocked_waitq);
2200 		spin_unlock(&fc->lock);
2201 		fuse_conn_put(fc);
2202 	}
2203 
2204 	return 0;
2205 }
2206 EXPORT_SYMBOL_GPL(fuse_dev_release);
2207 
2208 static int fuse_dev_fasync(int fd, struct file *file, int on)
2209 {
2210 	struct fuse_conn *fc = fuse_get_conn(file);
2211 	if (!fc)
2212 		return -EPERM;
2213 
2214 	/* No locking - fasync_helper does its own locking */
2215 	return fasync_helper(fd, file, on, &fc->fasync);
2216 }
2217 
2218 const struct file_operations fuse_dev_operations = {
2219 	.owner		= THIS_MODULE,
2220 	.llseek		= no_llseek,
2221 	.read		= do_sync_read,
2222 	.aio_read	= fuse_dev_read,
2223 	.splice_read	= fuse_dev_splice_read,
2224 	.write		= do_sync_write,
2225 	.aio_write	= fuse_dev_write,
2226 	.splice_write	= fuse_dev_splice_write,
2227 	.poll		= fuse_dev_poll,
2228 	.release	= fuse_dev_release,
2229 	.fasync		= fuse_dev_fasync,
2230 };
2231 EXPORT_SYMBOL_GPL(fuse_dev_operations);
2232 
2233 static struct miscdevice fuse_miscdevice = {
2234 	.minor = FUSE_MINOR,
2235 	.name  = "fuse",
2236 	.fops = &fuse_dev_operations,
2237 };
2238 
2239 int __init fuse_dev_init(void)
2240 {
2241 	int err = -ENOMEM;
2242 	fuse_req_cachep = kmem_cache_create("fuse_request",
2243 					    sizeof(struct fuse_req),
2244 					    0, 0, NULL);
2245 	if (!fuse_req_cachep)
2246 		goto out;
2247 
2248 	err = misc_register(&fuse_miscdevice);
2249 	if (err)
2250 		goto out_cache_clean;
2251 
2252 	return 0;
2253 
2254  out_cache_clean:
2255 	kmem_cache_destroy(fuse_req_cachep);
2256  out:
2257 	return err;
2258 }
2259 
2260 void fuse_dev_cleanup(void)
2261 {
2262 	misc_deregister(&fuse_miscdevice);
2263 	kmem_cache_destroy(fuse_req_cachep);
2264 }
2265