xref: /openbmc/linux/fs/pipe.c (revision c21b37f6)
1 /*
2  *  linux/fs/pipe.c
3  *
4  *  Copyright (C) 1991, 1992, 1999  Linus Torvalds
5  */
6 
7 #include <linux/mm.h>
8 #include <linux/file.h>
9 #include <linux/poll.h>
10 #include <linux/slab.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/fs.h>
14 #include <linux/mount.h>
15 #include <linux/pipe_fs_i.h>
16 #include <linux/uio.h>
17 #include <linux/highmem.h>
18 #include <linux/pagemap.h>
19 #include <linux/audit.h>
20 
21 #include <asm/uaccess.h>
22 #include <asm/ioctls.h>
23 
24 /*
25  * We use a start+len construction, which provides full use of the
26  * allocated memory.
27  * -- Florian Coosmann (FGC)
28  *
29  * Reads with count = 0 should always return 0.
30  * -- Julian Bradfield 1999-06-07.
31  *
32  * FIFOs and Pipes now generate SIGIO for both readers and writers.
33  * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16
34  *
35  * pipe_read & write cleanup
36  * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09
37  */
38 
39 /* Drop the inode semaphore and wait for a pipe event, atomically */
40 void pipe_wait(struct pipe_inode_info *pipe)
41 {
42 	DEFINE_WAIT(wait);
43 
44 	/*
45 	 * Pipes are system-local resources, so sleeping on them
46 	 * is considered a noninteractive wait:
47 	 */
48 	prepare_to_wait(&pipe->wait, &wait,
49 			TASK_INTERRUPTIBLE | TASK_NONINTERACTIVE);
50 	if (pipe->inode)
51 		mutex_unlock(&pipe->inode->i_mutex);
52 	schedule();
53 	finish_wait(&pipe->wait, &wait);
54 	if (pipe->inode)
55 		mutex_lock(&pipe->inode->i_mutex);
56 }
57 
58 static int
59 pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len,
60 			int atomic)
61 {
62 	unsigned long copy;
63 
64 	while (len > 0) {
65 		while (!iov->iov_len)
66 			iov++;
67 		copy = min_t(unsigned long, len, iov->iov_len);
68 
69 		if (atomic) {
70 			if (__copy_from_user_inatomic(to, iov->iov_base, copy))
71 				return -EFAULT;
72 		} else {
73 			if (copy_from_user(to, iov->iov_base, copy))
74 				return -EFAULT;
75 		}
76 		to += copy;
77 		len -= copy;
78 		iov->iov_base += copy;
79 		iov->iov_len -= copy;
80 	}
81 	return 0;
82 }
83 
84 static int
85 pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len,
86 		      int atomic)
87 {
88 	unsigned long copy;
89 
90 	while (len > 0) {
91 		while (!iov->iov_len)
92 			iov++;
93 		copy = min_t(unsigned long, len, iov->iov_len);
94 
95 		if (atomic) {
96 			if (__copy_to_user_inatomic(iov->iov_base, from, copy))
97 				return -EFAULT;
98 		} else {
99 			if (copy_to_user(iov->iov_base, from, copy))
100 				return -EFAULT;
101 		}
102 		from += copy;
103 		len -= copy;
104 		iov->iov_base += copy;
105 		iov->iov_len -= copy;
106 	}
107 	return 0;
108 }
109 
110 /*
111  * Attempt to pre-fault in the user memory, so we can use atomic copies.
112  * Returns the number of bytes not faulted in.
113  */
114 static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len)
115 {
116 	while (!iov->iov_len)
117 		iov++;
118 
119 	while (len > 0) {
120 		unsigned long this_len;
121 
122 		this_len = min_t(unsigned long, len, iov->iov_len);
123 		if (fault_in_pages_writeable(iov->iov_base, this_len))
124 			break;
125 
126 		len -= this_len;
127 		iov++;
128 	}
129 
130 	return len;
131 }
132 
133 /*
134  * Pre-fault in the user memory, so we can use atomic copies.
135  */
136 static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len)
137 {
138 	while (!iov->iov_len)
139 		iov++;
140 
141 	while (len > 0) {
142 		unsigned long this_len;
143 
144 		this_len = min_t(unsigned long, len, iov->iov_len);
145 		fault_in_pages_readable(iov->iov_base, this_len);
146 		len -= this_len;
147 		iov++;
148 	}
149 }
150 
151 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
152 				  struct pipe_buffer *buf)
153 {
154 	struct page *page = buf->page;
155 
156 	/*
157 	 * If nobody else uses this page, and we don't already have a
158 	 * temporary page, let's keep track of it as a one-deep
159 	 * allocation cache. (Otherwise just release our reference to it)
160 	 */
161 	if (page_count(page) == 1 && !pipe->tmp_page)
162 		pipe->tmp_page = page;
163 	else
164 		page_cache_release(page);
165 }
166 
167 /**
168  * generic_pipe_buf_map - virtually map a pipe buffer
169  * @pipe:	the pipe that the buffer belongs to
170  * @buf:	the buffer that should be mapped
171  * @atomic:	whether to use an atomic map
172  *
173  * Description:
174  *	This function returns a kernel virtual address mapping for the
175  *	passed in @pipe_buffer. If @atomic is set, an atomic map is provided
176  *	and the caller has to be careful not to fault before calling
177  *	the unmap function.
178  *
179  *	Note that this function occupies KM_USER0 if @atomic != 0.
180  */
181 void *generic_pipe_buf_map(struct pipe_inode_info *pipe,
182 			   struct pipe_buffer *buf, int atomic)
183 {
184 	if (atomic) {
185 		buf->flags |= PIPE_BUF_FLAG_ATOMIC;
186 		return kmap_atomic(buf->page, KM_USER0);
187 	}
188 
189 	return kmap(buf->page);
190 }
191 
192 /**
193  * generic_pipe_buf_unmap - unmap a previously mapped pipe buffer
194  * @pipe:	the pipe that the buffer belongs to
195  * @buf:	the buffer that should be unmapped
196  * @map_data:	the data that the mapping function returned
197  *
198  * Description:
199  *	This function undoes the mapping that ->map() provided.
200  */
201 void generic_pipe_buf_unmap(struct pipe_inode_info *pipe,
202 			    struct pipe_buffer *buf, void *map_data)
203 {
204 	if (buf->flags & PIPE_BUF_FLAG_ATOMIC) {
205 		buf->flags &= ~PIPE_BUF_FLAG_ATOMIC;
206 		kunmap_atomic(map_data, KM_USER0);
207 	} else
208 		kunmap(buf->page);
209 }
210 
211 /**
212  * generic_pipe_buf_steal - attempt to take ownership of a @pipe_buffer
213  * @pipe:	the pipe that the buffer belongs to
214  * @buf:	the buffer to attempt to steal
215  *
216  * Description:
217  *	This function attempts to steal the @struct page attached to
218  *	@buf. If successful, this function returns 0 and returns with
219  *	the page locked. The caller may then reuse the page for whatever
220  *	he wishes, the typical use is insertion into a different file
221  *	page cache.
222  */
223 int generic_pipe_buf_steal(struct pipe_inode_info *pipe,
224 			   struct pipe_buffer *buf)
225 {
226 	struct page *page = buf->page;
227 
228 	/*
229 	 * A reference of one is golden, that means that the owner of this
230 	 * page is the only one holding a reference to it. lock the page
231 	 * and return OK.
232 	 */
233 	if (page_count(page) == 1) {
234 		lock_page(page);
235 		return 0;
236 	}
237 
238 	return 1;
239 }
240 
241 /**
242  * generic_pipe_buf_get - get a reference to a @struct pipe_buffer
243  * @pipe:	the pipe that the buffer belongs to
244  * @buf:	the buffer to get a reference to
245  *
246  * Description:
247  *	This function grabs an extra reference to @buf. It's used in
248  *	in the tee() system call, when we duplicate the buffers in one
249  *	pipe into another.
250  */
251 void generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
252 {
253 	page_cache_get(buf->page);
254 }
255 
256 /**
257  * generic_pipe_buf_confirm - verify contents of the pipe buffer
258  * @info:	the pipe that the buffer belongs to
259  * @buf:	the buffer to confirm
260  *
261  * Description:
262  *	This function does nothing, because the generic pipe code uses
263  *	pages that are always good when inserted into the pipe.
264  */
265 int generic_pipe_buf_confirm(struct pipe_inode_info *info,
266 			     struct pipe_buffer *buf)
267 {
268 	return 0;
269 }
270 
271 static const struct pipe_buf_operations anon_pipe_buf_ops = {
272 	.can_merge = 1,
273 	.map = generic_pipe_buf_map,
274 	.unmap = generic_pipe_buf_unmap,
275 	.confirm = generic_pipe_buf_confirm,
276 	.release = anon_pipe_buf_release,
277 	.steal = generic_pipe_buf_steal,
278 	.get = generic_pipe_buf_get,
279 };
280 
281 static ssize_t
282 pipe_read(struct kiocb *iocb, const struct iovec *_iov,
283 	   unsigned long nr_segs, loff_t pos)
284 {
285 	struct file *filp = iocb->ki_filp;
286 	struct inode *inode = filp->f_path.dentry->d_inode;
287 	struct pipe_inode_info *pipe;
288 	int do_wakeup;
289 	ssize_t ret;
290 	struct iovec *iov = (struct iovec *)_iov;
291 	size_t total_len;
292 
293 	total_len = iov_length(iov, nr_segs);
294 	/* Null read succeeds. */
295 	if (unlikely(total_len == 0))
296 		return 0;
297 
298 	do_wakeup = 0;
299 	ret = 0;
300 	mutex_lock(&inode->i_mutex);
301 	pipe = inode->i_pipe;
302 	for (;;) {
303 		int bufs = pipe->nrbufs;
304 		if (bufs) {
305 			int curbuf = pipe->curbuf;
306 			struct pipe_buffer *buf = pipe->bufs + curbuf;
307 			const struct pipe_buf_operations *ops = buf->ops;
308 			void *addr;
309 			size_t chars = buf->len;
310 			int error, atomic;
311 
312 			if (chars > total_len)
313 				chars = total_len;
314 
315 			error = ops->confirm(pipe, buf);
316 			if (error) {
317 				if (!ret)
318 					error = ret;
319 				break;
320 			}
321 
322 			atomic = !iov_fault_in_pages_write(iov, chars);
323 redo:
324 			addr = ops->map(pipe, buf, atomic);
325 			error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic);
326 			ops->unmap(pipe, buf, addr);
327 			if (unlikely(error)) {
328 				/*
329 				 * Just retry with the slow path if we failed.
330 				 */
331 				if (atomic) {
332 					atomic = 0;
333 					goto redo;
334 				}
335 				if (!ret)
336 					ret = error;
337 				break;
338 			}
339 			ret += chars;
340 			buf->offset += chars;
341 			buf->len -= chars;
342 			if (!buf->len) {
343 				buf->ops = NULL;
344 				ops->release(pipe, buf);
345 				curbuf = (curbuf + 1) & (PIPE_BUFFERS-1);
346 				pipe->curbuf = curbuf;
347 				pipe->nrbufs = --bufs;
348 				do_wakeup = 1;
349 			}
350 			total_len -= chars;
351 			if (!total_len)
352 				break;	/* common path: read succeeded */
353 		}
354 		if (bufs)	/* More to do? */
355 			continue;
356 		if (!pipe->writers)
357 			break;
358 		if (!pipe->waiting_writers) {
359 			/* syscall merging: Usually we must not sleep
360 			 * if O_NONBLOCK is set, or if we got some data.
361 			 * But if a writer sleeps in kernel space, then
362 			 * we can wait for that data without violating POSIX.
363 			 */
364 			if (ret)
365 				break;
366 			if (filp->f_flags & O_NONBLOCK) {
367 				ret = -EAGAIN;
368 				break;
369 			}
370 		}
371 		if (signal_pending(current)) {
372 			if (!ret)
373 				ret = -ERESTARTSYS;
374 			break;
375 		}
376 		if (do_wakeup) {
377 			wake_up_interruptible_sync(&pipe->wait);
378  			kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
379 		}
380 		pipe_wait(pipe);
381 	}
382 	mutex_unlock(&inode->i_mutex);
383 
384 	/* Signal writers asynchronously that there is more room. */
385 	if (do_wakeup) {
386 		wake_up_interruptible(&pipe->wait);
387 		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
388 	}
389 	if (ret > 0)
390 		file_accessed(filp);
391 	return ret;
392 }
393 
394 static ssize_t
395 pipe_write(struct kiocb *iocb, const struct iovec *_iov,
396 	    unsigned long nr_segs, loff_t ppos)
397 {
398 	struct file *filp = iocb->ki_filp;
399 	struct inode *inode = filp->f_path.dentry->d_inode;
400 	struct pipe_inode_info *pipe;
401 	ssize_t ret;
402 	int do_wakeup;
403 	struct iovec *iov = (struct iovec *)_iov;
404 	size_t total_len;
405 	ssize_t chars;
406 
407 	total_len = iov_length(iov, nr_segs);
408 	/* Null write succeeds. */
409 	if (unlikely(total_len == 0))
410 		return 0;
411 
412 	do_wakeup = 0;
413 	ret = 0;
414 	mutex_lock(&inode->i_mutex);
415 	pipe = inode->i_pipe;
416 
417 	if (!pipe->readers) {
418 		send_sig(SIGPIPE, current, 0);
419 		ret = -EPIPE;
420 		goto out;
421 	}
422 
423 	/* We try to merge small writes */
424 	chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */
425 	if (pipe->nrbufs && chars != 0) {
426 		int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) &
427 							(PIPE_BUFFERS-1);
428 		struct pipe_buffer *buf = pipe->bufs + lastbuf;
429 		const struct pipe_buf_operations *ops = buf->ops;
430 		int offset = buf->offset + buf->len;
431 
432 		if (ops->can_merge && offset + chars <= PAGE_SIZE) {
433 			int error, atomic = 1;
434 			void *addr;
435 
436 			error = ops->confirm(pipe, buf);
437 			if (error)
438 				goto out;
439 
440 			iov_fault_in_pages_read(iov, chars);
441 redo1:
442 			addr = ops->map(pipe, buf, atomic);
443 			error = pipe_iov_copy_from_user(offset + addr, iov,
444 							chars, atomic);
445 			ops->unmap(pipe, buf, addr);
446 			ret = error;
447 			do_wakeup = 1;
448 			if (error) {
449 				if (atomic) {
450 					atomic = 0;
451 					goto redo1;
452 				}
453 				goto out;
454 			}
455 			buf->len += chars;
456 			total_len -= chars;
457 			ret = chars;
458 			if (!total_len)
459 				goto out;
460 		}
461 	}
462 
463 	for (;;) {
464 		int bufs;
465 
466 		if (!pipe->readers) {
467 			send_sig(SIGPIPE, current, 0);
468 			if (!ret)
469 				ret = -EPIPE;
470 			break;
471 		}
472 		bufs = pipe->nrbufs;
473 		if (bufs < PIPE_BUFFERS) {
474 			int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS-1);
475 			struct pipe_buffer *buf = pipe->bufs + newbuf;
476 			struct page *page = pipe->tmp_page;
477 			char *src;
478 			int error, atomic = 1;
479 
480 			if (!page) {
481 				page = alloc_page(GFP_HIGHUSER);
482 				if (unlikely(!page)) {
483 					ret = ret ? : -ENOMEM;
484 					break;
485 				}
486 				pipe->tmp_page = page;
487 			}
488 			/* Always wake up, even if the copy fails. Otherwise
489 			 * we lock up (O_NONBLOCK-)readers that sleep due to
490 			 * syscall merging.
491 			 * FIXME! Is this really true?
492 			 */
493 			do_wakeup = 1;
494 			chars = PAGE_SIZE;
495 			if (chars > total_len)
496 				chars = total_len;
497 
498 			iov_fault_in_pages_read(iov, chars);
499 redo2:
500 			if (atomic)
501 				src = kmap_atomic(page, KM_USER0);
502 			else
503 				src = kmap(page);
504 
505 			error = pipe_iov_copy_from_user(src, iov, chars,
506 							atomic);
507 			if (atomic)
508 				kunmap_atomic(src, KM_USER0);
509 			else
510 				kunmap(page);
511 
512 			if (unlikely(error)) {
513 				if (atomic) {
514 					atomic = 0;
515 					goto redo2;
516 				}
517 				if (!ret)
518 					ret = error;
519 				break;
520 			}
521 			ret += chars;
522 
523 			/* Insert it into the buffer array */
524 			buf->page = page;
525 			buf->ops = &anon_pipe_buf_ops;
526 			buf->offset = 0;
527 			buf->len = chars;
528 			pipe->nrbufs = ++bufs;
529 			pipe->tmp_page = NULL;
530 
531 			total_len -= chars;
532 			if (!total_len)
533 				break;
534 		}
535 		if (bufs < PIPE_BUFFERS)
536 			continue;
537 		if (filp->f_flags & O_NONBLOCK) {
538 			if (!ret)
539 				ret = -EAGAIN;
540 			break;
541 		}
542 		if (signal_pending(current)) {
543 			if (!ret)
544 				ret = -ERESTARTSYS;
545 			break;
546 		}
547 		if (do_wakeup) {
548 			wake_up_interruptible_sync(&pipe->wait);
549 			kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
550 			do_wakeup = 0;
551 		}
552 		pipe->waiting_writers++;
553 		pipe_wait(pipe);
554 		pipe->waiting_writers--;
555 	}
556 out:
557 	mutex_unlock(&inode->i_mutex);
558 	if (do_wakeup) {
559 		wake_up_interruptible(&pipe->wait);
560 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
561 	}
562 	if (ret > 0)
563 		file_update_time(filp);
564 	return ret;
565 }
566 
567 static ssize_t
568 bad_pipe_r(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
569 {
570 	return -EBADF;
571 }
572 
573 static ssize_t
574 bad_pipe_w(struct file *filp, const char __user *buf, size_t count,
575 	   loff_t *ppos)
576 {
577 	return -EBADF;
578 }
579 
580 static int
581 pipe_ioctl(struct inode *pino, struct file *filp,
582 	   unsigned int cmd, unsigned long arg)
583 {
584 	struct inode *inode = filp->f_path.dentry->d_inode;
585 	struct pipe_inode_info *pipe;
586 	int count, buf, nrbufs;
587 
588 	switch (cmd) {
589 		case FIONREAD:
590 			mutex_lock(&inode->i_mutex);
591 			pipe = inode->i_pipe;
592 			count = 0;
593 			buf = pipe->curbuf;
594 			nrbufs = pipe->nrbufs;
595 			while (--nrbufs >= 0) {
596 				count += pipe->bufs[buf].len;
597 				buf = (buf+1) & (PIPE_BUFFERS-1);
598 			}
599 			mutex_unlock(&inode->i_mutex);
600 
601 			return put_user(count, (int __user *)arg);
602 		default:
603 			return -EINVAL;
604 	}
605 }
606 
607 /* No kernel lock held - fine */
608 static unsigned int
609 pipe_poll(struct file *filp, poll_table *wait)
610 {
611 	unsigned int mask;
612 	struct inode *inode = filp->f_path.dentry->d_inode;
613 	struct pipe_inode_info *pipe = inode->i_pipe;
614 	int nrbufs;
615 
616 	poll_wait(filp, &pipe->wait, wait);
617 
618 	/* Reading only -- no need for acquiring the semaphore.  */
619 	nrbufs = pipe->nrbufs;
620 	mask = 0;
621 	if (filp->f_mode & FMODE_READ) {
622 		mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0;
623 		if (!pipe->writers && filp->f_version != pipe->w_counter)
624 			mask |= POLLHUP;
625 	}
626 
627 	if (filp->f_mode & FMODE_WRITE) {
628 		mask |= (nrbufs < PIPE_BUFFERS) ? POLLOUT | POLLWRNORM : 0;
629 		/*
630 		 * Most Unices do not set POLLERR for FIFOs but on Linux they
631 		 * behave exactly like pipes for poll().
632 		 */
633 		if (!pipe->readers)
634 			mask |= POLLERR;
635 	}
636 
637 	return mask;
638 }
639 
640 static int
641 pipe_release(struct inode *inode, int decr, int decw)
642 {
643 	struct pipe_inode_info *pipe;
644 
645 	mutex_lock(&inode->i_mutex);
646 	pipe = inode->i_pipe;
647 	pipe->readers -= decr;
648 	pipe->writers -= decw;
649 
650 	if (!pipe->readers && !pipe->writers) {
651 		free_pipe_info(inode);
652 	} else {
653 		wake_up_interruptible(&pipe->wait);
654 		kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
655 		kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
656 	}
657 	mutex_unlock(&inode->i_mutex);
658 
659 	return 0;
660 }
661 
662 static int
663 pipe_read_fasync(int fd, struct file *filp, int on)
664 {
665 	struct inode *inode = filp->f_path.dentry->d_inode;
666 	int retval;
667 
668 	mutex_lock(&inode->i_mutex);
669 	retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_readers);
670 	mutex_unlock(&inode->i_mutex);
671 
672 	if (retval < 0)
673 		return retval;
674 
675 	return 0;
676 }
677 
678 
679 static int
680 pipe_write_fasync(int fd, struct file *filp, int on)
681 {
682 	struct inode *inode = filp->f_path.dentry->d_inode;
683 	int retval;
684 
685 	mutex_lock(&inode->i_mutex);
686 	retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_writers);
687 	mutex_unlock(&inode->i_mutex);
688 
689 	if (retval < 0)
690 		return retval;
691 
692 	return 0;
693 }
694 
695 
696 static int
697 pipe_rdwr_fasync(int fd, struct file *filp, int on)
698 {
699 	struct inode *inode = filp->f_path.dentry->d_inode;
700 	struct pipe_inode_info *pipe = inode->i_pipe;
701 	int retval;
702 
703 	mutex_lock(&inode->i_mutex);
704 
705 	retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
706 
707 	if (retval >= 0)
708 		retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
709 
710 	mutex_unlock(&inode->i_mutex);
711 
712 	if (retval < 0)
713 		return retval;
714 
715 	return 0;
716 }
717 
718 
719 static int
720 pipe_read_release(struct inode *inode, struct file *filp)
721 {
722 	pipe_read_fasync(-1, filp, 0);
723 	return pipe_release(inode, 1, 0);
724 }
725 
726 static int
727 pipe_write_release(struct inode *inode, struct file *filp)
728 {
729 	pipe_write_fasync(-1, filp, 0);
730 	return pipe_release(inode, 0, 1);
731 }
732 
733 static int
734 pipe_rdwr_release(struct inode *inode, struct file *filp)
735 {
736 	int decr, decw;
737 
738 	pipe_rdwr_fasync(-1, filp, 0);
739 	decr = (filp->f_mode & FMODE_READ) != 0;
740 	decw = (filp->f_mode & FMODE_WRITE) != 0;
741 	return pipe_release(inode, decr, decw);
742 }
743 
744 static int
745 pipe_read_open(struct inode *inode, struct file *filp)
746 {
747 	/* We could have perhaps used atomic_t, but this and friends
748 	   below are the only places.  So it doesn't seem worthwhile.  */
749 	mutex_lock(&inode->i_mutex);
750 	inode->i_pipe->readers++;
751 	mutex_unlock(&inode->i_mutex);
752 
753 	return 0;
754 }
755 
756 static int
757 pipe_write_open(struct inode *inode, struct file *filp)
758 {
759 	mutex_lock(&inode->i_mutex);
760 	inode->i_pipe->writers++;
761 	mutex_unlock(&inode->i_mutex);
762 
763 	return 0;
764 }
765 
766 static int
767 pipe_rdwr_open(struct inode *inode, struct file *filp)
768 {
769 	mutex_lock(&inode->i_mutex);
770 	if (filp->f_mode & FMODE_READ)
771 		inode->i_pipe->readers++;
772 	if (filp->f_mode & FMODE_WRITE)
773 		inode->i_pipe->writers++;
774 	mutex_unlock(&inode->i_mutex);
775 
776 	return 0;
777 }
778 
779 /*
780  * The file_operations structs are not static because they
781  * are also used in linux/fs/fifo.c to do operations on FIFOs.
782  */
783 const struct file_operations read_fifo_fops = {
784 	.llseek		= no_llseek,
785 	.read		= do_sync_read,
786 	.aio_read	= pipe_read,
787 	.write		= bad_pipe_w,
788 	.poll		= pipe_poll,
789 	.ioctl		= pipe_ioctl,
790 	.open		= pipe_read_open,
791 	.release	= pipe_read_release,
792 	.fasync		= pipe_read_fasync,
793 };
794 
795 const struct file_operations write_fifo_fops = {
796 	.llseek		= no_llseek,
797 	.read		= bad_pipe_r,
798 	.write		= do_sync_write,
799 	.aio_write	= pipe_write,
800 	.poll		= pipe_poll,
801 	.ioctl		= pipe_ioctl,
802 	.open		= pipe_write_open,
803 	.release	= pipe_write_release,
804 	.fasync		= pipe_write_fasync,
805 };
806 
807 const struct file_operations rdwr_fifo_fops = {
808 	.llseek		= no_llseek,
809 	.read		= do_sync_read,
810 	.aio_read	= pipe_read,
811 	.write		= do_sync_write,
812 	.aio_write	= pipe_write,
813 	.poll		= pipe_poll,
814 	.ioctl		= pipe_ioctl,
815 	.open		= pipe_rdwr_open,
816 	.release	= pipe_rdwr_release,
817 	.fasync		= pipe_rdwr_fasync,
818 };
819 
820 static const struct file_operations read_pipe_fops = {
821 	.llseek		= no_llseek,
822 	.read		= do_sync_read,
823 	.aio_read	= pipe_read,
824 	.write		= bad_pipe_w,
825 	.poll		= pipe_poll,
826 	.ioctl		= pipe_ioctl,
827 	.open		= pipe_read_open,
828 	.release	= pipe_read_release,
829 	.fasync		= pipe_read_fasync,
830 };
831 
832 static const struct file_operations write_pipe_fops = {
833 	.llseek		= no_llseek,
834 	.read		= bad_pipe_r,
835 	.write		= do_sync_write,
836 	.aio_write	= pipe_write,
837 	.poll		= pipe_poll,
838 	.ioctl		= pipe_ioctl,
839 	.open		= pipe_write_open,
840 	.release	= pipe_write_release,
841 	.fasync		= pipe_write_fasync,
842 };
843 
844 static const struct file_operations rdwr_pipe_fops = {
845 	.llseek		= no_llseek,
846 	.read		= do_sync_read,
847 	.aio_read	= pipe_read,
848 	.write		= do_sync_write,
849 	.aio_write	= pipe_write,
850 	.poll		= pipe_poll,
851 	.ioctl		= pipe_ioctl,
852 	.open		= pipe_rdwr_open,
853 	.release	= pipe_rdwr_release,
854 	.fasync		= pipe_rdwr_fasync,
855 };
856 
857 struct pipe_inode_info * alloc_pipe_info(struct inode *inode)
858 {
859 	struct pipe_inode_info *pipe;
860 
861 	pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL);
862 	if (pipe) {
863 		init_waitqueue_head(&pipe->wait);
864 		pipe->r_counter = pipe->w_counter = 1;
865 		pipe->inode = inode;
866 	}
867 
868 	return pipe;
869 }
870 
871 void __free_pipe_info(struct pipe_inode_info *pipe)
872 {
873 	int i;
874 
875 	for (i = 0; i < PIPE_BUFFERS; i++) {
876 		struct pipe_buffer *buf = pipe->bufs + i;
877 		if (buf->ops)
878 			buf->ops->release(pipe, buf);
879 	}
880 	if (pipe->tmp_page)
881 		__free_page(pipe->tmp_page);
882 	kfree(pipe);
883 }
884 
885 void free_pipe_info(struct inode *inode)
886 {
887 	__free_pipe_info(inode->i_pipe);
888 	inode->i_pipe = NULL;
889 }
890 
891 static struct vfsmount *pipe_mnt __read_mostly;
892 static int pipefs_delete_dentry(struct dentry *dentry)
893 {
894 	/*
895 	 * At creation time, we pretended this dentry was hashed
896 	 * (by clearing DCACHE_UNHASHED bit in d_flags)
897 	 * At delete time, we restore the truth : not hashed.
898 	 * (so that dput() can proceed correctly)
899 	 */
900 	dentry->d_flags |= DCACHE_UNHASHED;
901 	return 0;
902 }
903 
904 /*
905  * pipefs_dname() is called from d_path().
906  */
907 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen)
908 {
909 	return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
910 				dentry->d_inode->i_ino);
911 }
912 
913 static struct dentry_operations pipefs_dentry_operations = {
914 	.d_delete	= pipefs_delete_dentry,
915 	.d_dname	= pipefs_dname,
916 };
917 
918 static struct inode * get_pipe_inode(void)
919 {
920 	struct inode *inode = new_inode(pipe_mnt->mnt_sb);
921 	struct pipe_inode_info *pipe;
922 
923 	if (!inode)
924 		goto fail_inode;
925 
926 	pipe = alloc_pipe_info(inode);
927 	if (!pipe)
928 		goto fail_iput;
929 	inode->i_pipe = pipe;
930 
931 	pipe->readers = pipe->writers = 1;
932 	inode->i_fop = &rdwr_pipe_fops;
933 
934 	/*
935 	 * Mark the inode dirty from the very beginning,
936 	 * that way it will never be moved to the dirty
937 	 * list because "mark_inode_dirty()" will think
938 	 * that it already _is_ on the dirty list.
939 	 */
940 	inode->i_state = I_DIRTY;
941 	inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR;
942 	inode->i_uid = current->fsuid;
943 	inode->i_gid = current->fsgid;
944 	inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
945 
946 	return inode;
947 
948 fail_iput:
949 	iput(inode);
950 
951 fail_inode:
952 	return NULL;
953 }
954 
955 struct file *create_write_pipe(void)
956 {
957 	int err;
958 	struct inode *inode;
959 	struct file *f;
960 	struct dentry *dentry;
961 	struct qstr name = { .name = "" };
962 
963 	f = get_empty_filp();
964 	if (!f)
965 		return ERR_PTR(-ENFILE);
966 	err = -ENFILE;
967 	inode = get_pipe_inode();
968 	if (!inode)
969 		goto err_file;
970 
971 	err = -ENOMEM;
972 	dentry = d_alloc(pipe_mnt->mnt_sb->s_root, &name);
973 	if (!dentry)
974 		goto err_inode;
975 
976 	dentry->d_op = &pipefs_dentry_operations;
977 	/*
978 	 * We dont want to publish this dentry into global dentry hash table.
979 	 * We pretend dentry is already hashed, by unsetting DCACHE_UNHASHED
980 	 * This permits a working /proc/$pid/fd/XXX on pipes
981 	 */
982 	dentry->d_flags &= ~DCACHE_UNHASHED;
983 	d_instantiate(dentry, inode);
984 	f->f_path.mnt = mntget(pipe_mnt);
985 	f->f_path.dentry = dentry;
986 	f->f_mapping = inode->i_mapping;
987 
988 	f->f_flags = O_WRONLY;
989 	f->f_op = &write_pipe_fops;
990 	f->f_mode = FMODE_WRITE;
991 	f->f_version = 0;
992 
993 	return f;
994 
995  err_inode:
996 	free_pipe_info(inode);
997 	iput(inode);
998  err_file:
999 	put_filp(f);
1000 	return ERR_PTR(err);
1001 }
1002 
1003 void free_write_pipe(struct file *f)
1004 {
1005 	free_pipe_info(f->f_dentry->d_inode);
1006 	dput(f->f_path.dentry);
1007 	mntput(f->f_path.mnt);
1008 	put_filp(f);
1009 }
1010 
1011 struct file *create_read_pipe(struct file *wrf)
1012 {
1013 	struct file *f = get_empty_filp();
1014 	if (!f)
1015 		return ERR_PTR(-ENFILE);
1016 
1017 	/* Grab pipe from the writer */
1018 	f->f_path.mnt = mntget(wrf->f_path.mnt);
1019 	f->f_path.dentry = dget(wrf->f_path.dentry);
1020 	f->f_mapping = wrf->f_path.dentry->d_inode->i_mapping;
1021 
1022 	f->f_pos = 0;
1023 	f->f_flags = O_RDONLY;
1024 	f->f_op = &read_pipe_fops;
1025 	f->f_mode = FMODE_READ;
1026 	f->f_version = 0;
1027 
1028 	return f;
1029 }
1030 
1031 int do_pipe(int *fd)
1032 {
1033 	struct file *fw, *fr;
1034 	int error;
1035 	int fdw, fdr;
1036 
1037 	fw = create_write_pipe();
1038 	if (IS_ERR(fw))
1039 		return PTR_ERR(fw);
1040 	fr = create_read_pipe(fw);
1041 	error = PTR_ERR(fr);
1042 	if (IS_ERR(fr))
1043 		goto err_write_pipe;
1044 
1045 	error = get_unused_fd();
1046 	if (error < 0)
1047 		goto err_read_pipe;
1048 	fdr = error;
1049 
1050 	error = get_unused_fd();
1051 	if (error < 0)
1052 		goto err_fdr;
1053 	fdw = error;
1054 
1055 	error = audit_fd_pair(fdr, fdw);
1056 	if (error < 0)
1057 		goto err_fdw;
1058 
1059 	fd_install(fdr, fr);
1060 	fd_install(fdw, fw);
1061 	fd[0] = fdr;
1062 	fd[1] = fdw;
1063 
1064 	return 0;
1065 
1066  err_fdw:
1067 	put_unused_fd(fdw);
1068  err_fdr:
1069 	put_unused_fd(fdr);
1070  err_read_pipe:
1071 	dput(fr->f_dentry);
1072 	mntput(fr->f_vfsmnt);
1073 	put_filp(fr);
1074  err_write_pipe:
1075 	free_write_pipe(fw);
1076 	return error;
1077 }
1078 
1079 /*
1080  * pipefs should _never_ be mounted by userland - too much of security hassle,
1081  * no real gain from having the whole whorehouse mounted. So we don't need
1082  * any operations on the root directory. However, we need a non-trivial
1083  * d_name - pipe: will go nicely and kill the special-casing in procfs.
1084  */
1085 static int pipefs_get_sb(struct file_system_type *fs_type,
1086 			 int flags, const char *dev_name, void *data,
1087 			 struct vfsmount *mnt)
1088 {
1089 	return get_sb_pseudo(fs_type, "pipe:", NULL, PIPEFS_MAGIC, mnt);
1090 }
1091 
1092 static struct file_system_type pipe_fs_type = {
1093 	.name		= "pipefs",
1094 	.get_sb		= pipefs_get_sb,
1095 	.kill_sb	= kill_anon_super,
1096 };
1097 
1098 static int __init init_pipe_fs(void)
1099 {
1100 	int err = register_filesystem(&pipe_fs_type);
1101 
1102 	if (!err) {
1103 		pipe_mnt = kern_mount(&pipe_fs_type);
1104 		if (IS_ERR(pipe_mnt)) {
1105 			err = PTR_ERR(pipe_mnt);
1106 			unregister_filesystem(&pipe_fs_type);
1107 		}
1108 	}
1109 	return err;
1110 }
1111 
1112 static void __exit exit_pipe_fs(void)
1113 {
1114 	unregister_filesystem(&pipe_fs_type);
1115 	mntput(pipe_mnt);
1116 }
1117 
1118 fs_initcall(init_pipe_fs);
1119 module_exit(exit_pipe_fs);
1120