1 /* 2 * linux/fs/pipe.c 3 * 4 * Copyright (C) 1991, 1992, 1999 Linus Torvalds 5 */ 6 7 #include <linux/mm.h> 8 #include <linux/file.h> 9 #include <linux/poll.h> 10 #include <linux/slab.h> 11 #include <linux/module.h> 12 #include <linux/init.h> 13 #include <linux/fs.h> 14 #include <linux/mount.h> 15 #include <linux/pipe_fs_i.h> 16 #include <linux/uio.h> 17 #include <linux/highmem.h> 18 #include <linux/pagemap.h> 19 #include <linux/audit.h> 20 #include <linux/syscalls.h> 21 22 #include <asm/uaccess.h> 23 #include <asm/ioctls.h> 24 25 /* 26 * We use a start+len construction, which provides full use of the 27 * allocated memory. 28 * -- Florian Coosmann (FGC) 29 * 30 * Reads with count = 0 should always return 0. 31 * -- Julian Bradfield 1999-06-07. 32 * 33 * FIFOs and Pipes now generate SIGIO for both readers and writers. 34 * -- Jeremy Elson <jelson@circlemud.org> 2001-08-16 35 * 36 * pipe_read & write cleanup 37 * -- Manfred Spraul <manfred@colorfullife.com> 2002-05-09 38 */ 39 40 /* Drop the inode semaphore and wait for a pipe event, atomically */ 41 void pipe_wait(struct pipe_inode_info *pipe) 42 { 43 DEFINE_WAIT(wait); 44 45 /* 46 * Pipes are system-local resources, so sleeping on them 47 * is considered a noninteractive wait: 48 */ 49 prepare_to_wait(&pipe->wait, &wait, TASK_INTERRUPTIBLE); 50 if (pipe->inode) 51 mutex_unlock(&pipe->inode->i_mutex); 52 schedule(); 53 finish_wait(&pipe->wait, &wait); 54 if (pipe->inode) 55 mutex_lock(&pipe->inode->i_mutex); 56 } 57 58 static int 59 pipe_iov_copy_from_user(void *to, struct iovec *iov, unsigned long len, 60 int atomic) 61 { 62 unsigned long copy; 63 64 while (len > 0) { 65 while (!iov->iov_len) 66 iov++; 67 copy = min_t(unsigned long, len, iov->iov_len); 68 69 if (atomic) { 70 if (__copy_from_user_inatomic(to, iov->iov_base, copy)) 71 return -EFAULT; 72 } else { 73 if (copy_from_user(to, iov->iov_base, copy)) 74 return -EFAULT; 75 } 76 to += copy; 77 len -= copy; 78 iov->iov_base += copy; 79 iov->iov_len -= copy; 80 } 81 return 0; 82 } 83 84 static int 85 pipe_iov_copy_to_user(struct iovec *iov, const void *from, unsigned long len, 86 int atomic) 87 { 88 unsigned long copy; 89 90 while (len > 0) { 91 while (!iov->iov_len) 92 iov++; 93 copy = min_t(unsigned long, len, iov->iov_len); 94 95 if (atomic) { 96 if (__copy_to_user_inatomic(iov->iov_base, from, copy)) 97 return -EFAULT; 98 } else { 99 if (copy_to_user(iov->iov_base, from, copy)) 100 return -EFAULT; 101 } 102 from += copy; 103 len -= copy; 104 iov->iov_base += copy; 105 iov->iov_len -= copy; 106 } 107 return 0; 108 } 109 110 /* 111 * Attempt to pre-fault in the user memory, so we can use atomic copies. 112 * Returns the number of bytes not faulted in. 113 */ 114 static int iov_fault_in_pages_write(struct iovec *iov, unsigned long len) 115 { 116 while (!iov->iov_len) 117 iov++; 118 119 while (len > 0) { 120 unsigned long this_len; 121 122 this_len = min_t(unsigned long, len, iov->iov_len); 123 if (fault_in_pages_writeable(iov->iov_base, this_len)) 124 break; 125 126 len -= this_len; 127 iov++; 128 } 129 130 return len; 131 } 132 133 /* 134 * Pre-fault in the user memory, so we can use atomic copies. 135 */ 136 static void iov_fault_in_pages_read(struct iovec *iov, unsigned long len) 137 { 138 while (!iov->iov_len) 139 iov++; 140 141 while (len > 0) { 142 unsigned long this_len; 143 144 this_len = min_t(unsigned long, len, iov->iov_len); 145 fault_in_pages_readable(iov->iov_base, this_len); 146 len -= this_len; 147 iov++; 148 } 149 } 150 151 static void anon_pipe_buf_release(struct pipe_inode_info *pipe, 152 struct pipe_buffer *buf) 153 { 154 struct page *page = buf->page; 155 156 /* 157 * If nobody else uses this page, and we don't already have a 158 * temporary page, let's keep track of it as a one-deep 159 * allocation cache. (Otherwise just release our reference to it) 160 */ 161 if (page_count(page) == 1 && !pipe->tmp_page) 162 pipe->tmp_page = page; 163 else 164 page_cache_release(page); 165 } 166 167 /** 168 * generic_pipe_buf_map - virtually map a pipe buffer 169 * @pipe: the pipe that the buffer belongs to 170 * @buf: the buffer that should be mapped 171 * @atomic: whether to use an atomic map 172 * 173 * Description: 174 * This function returns a kernel virtual address mapping for the 175 * pipe_buffer passed in @buf. If @atomic is set, an atomic map is provided 176 * and the caller has to be careful not to fault before calling 177 * the unmap function. 178 * 179 * Note that this function occupies KM_USER0 if @atomic != 0. 180 */ 181 void *generic_pipe_buf_map(struct pipe_inode_info *pipe, 182 struct pipe_buffer *buf, int atomic) 183 { 184 if (atomic) { 185 buf->flags |= PIPE_BUF_FLAG_ATOMIC; 186 return kmap_atomic(buf->page, KM_USER0); 187 } 188 189 return kmap(buf->page); 190 } 191 192 /** 193 * generic_pipe_buf_unmap - unmap a previously mapped pipe buffer 194 * @pipe: the pipe that the buffer belongs to 195 * @buf: the buffer that should be unmapped 196 * @map_data: the data that the mapping function returned 197 * 198 * Description: 199 * This function undoes the mapping that ->map() provided. 200 */ 201 void generic_pipe_buf_unmap(struct pipe_inode_info *pipe, 202 struct pipe_buffer *buf, void *map_data) 203 { 204 if (buf->flags & PIPE_BUF_FLAG_ATOMIC) { 205 buf->flags &= ~PIPE_BUF_FLAG_ATOMIC; 206 kunmap_atomic(map_data, KM_USER0); 207 } else 208 kunmap(buf->page); 209 } 210 211 /** 212 * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer 213 * @pipe: the pipe that the buffer belongs to 214 * @buf: the buffer to attempt to steal 215 * 216 * Description: 217 * This function attempts to steal the &struct page attached to 218 * @buf. If successful, this function returns 0 and returns with 219 * the page locked. The caller may then reuse the page for whatever 220 * he wishes; the typical use is insertion into a different file 221 * page cache. 222 */ 223 int generic_pipe_buf_steal(struct pipe_inode_info *pipe, 224 struct pipe_buffer *buf) 225 { 226 struct page *page = buf->page; 227 228 /* 229 * A reference of one is golden, that means that the owner of this 230 * page is the only one holding a reference to it. lock the page 231 * and return OK. 232 */ 233 if (page_count(page) == 1) { 234 lock_page(page); 235 return 0; 236 } 237 238 return 1; 239 } 240 241 /** 242 * generic_pipe_buf_get - get a reference to a &struct pipe_buffer 243 * @pipe: the pipe that the buffer belongs to 244 * @buf: the buffer to get a reference to 245 * 246 * Description: 247 * This function grabs an extra reference to @buf. It's used in 248 * in the tee() system call, when we duplicate the buffers in one 249 * pipe into another. 250 */ 251 void generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 252 { 253 page_cache_get(buf->page); 254 } 255 256 /** 257 * generic_pipe_buf_confirm - verify contents of the pipe buffer 258 * @info: the pipe that the buffer belongs to 259 * @buf: the buffer to confirm 260 * 261 * Description: 262 * This function does nothing, because the generic pipe code uses 263 * pages that are always good when inserted into the pipe. 264 */ 265 int generic_pipe_buf_confirm(struct pipe_inode_info *info, 266 struct pipe_buffer *buf) 267 { 268 return 0; 269 } 270 271 static const struct pipe_buf_operations anon_pipe_buf_ops = { 272 .can_merge = 1, 273 .map = generic_pipe_buf_map, 274 .unmap = generic_pipe_buf_unmap, 275 .confirm = generic_pipe_buf_confirm, 276 .release = anon_pipe_buf_release, 277 .steal = generic_pipe_buf_steal, 278 .get = generic_pipe_buf_get, 279 }; 280 281 static ssize_t 282 pipe_read(struct kiocb *iocb, const struct iovec *_iov, 283 unsigned long nr_segs, loff_t pos) 284 { 285 struct file *filp = iocb->ki_filp; 286 struct inode *inode = filp->f_path.dentry->d_inode; 287 struct pipe_inode_info *pipe; 288 int do_wakeup; 289 ssize_t ret; 290 struct iovec *iov = (struct iovec *)_iov; 291 size_t total_len; 292 293 total_len = iov_length(iov, nr_segs); 294 /* Null read succeeds. */ 295 if (unlikely(total_len == 0)) 296 return 0; 297 298 do_wakeup = 0; 299 ret = 0; 300 mutex_lock(&inode->i_mutex); 301 pipe = inode->i_pipe; 302 for (;;) { 303 int bufs = pipe->nrbufs; 304 if (bufs) { 305 int curbuf = pipe->curbuf; 306 struct pipe_buffer *buf = pipe->bufs + curbuf; 307 const struct pipe_buf_operations *ops = buf->ops; 308 void *addr; 309 size_t chars = buf->len; 310 int error, atomic; 311 312 if (chars > total_len) 313 chars = total_len; 314 315 error = ops->confirm(pipe, buf); 316 if (error) { 317 if (!ret) 318 error = ret; 319 break; 320 } 321 322 atomic = !iov_fault_in_pages_write(iov, chars); 323 redo: 324 addr = ops->map(pipe, buf, atomic); 325 error = pipe_iov_copy_to_user(iov, addr + buf->offset, chars, atomic); 326 ops->unmap(pipe, buf, addr); 327 if (unlikely(error)) { 328 /* 329 * Just retry with the slow path if we failed. 330 */ 331 if (atomic) { 332 atomic = 0; 333 goto redo; 334 } 335 if (!ret) 336 ret = error; 337 break; 338 } 339 ret += chars; 340 buf->offset += chars; 341 buf->len -= chars; 342 if (!buf->len) { 343 buf->ops = NULL; 344 ops->release(pipe, buf); 345 curbuf = (curbuf + 1) & (PIPE_BUFFERS-1); 346 pipe->curbuf = curbuf; 347 pipe->nrbufs = --bufs; 348 do_wakeup = 1; 349 } 350 total_len -= chars; 351 if (!total_len) 352 break; /* common path: read succeeded */ 353 } 354 if (bufs) /* More to do? */ 355 continue; 356 if (!pipe->writers) 357 break; 358 if (!pipe->waiting_writers) { 359 /* syscall merging: Usually we must not sleep 360 * if O_NONBLOCK is set, or if we got some data. 361 * But if a writer sleeps in kernel space, then 362 * we can wait for that data without violating POSIX. 363 */ 364 if (ret) 365 break; 366 if (filp->f_flags & O_NONBLOCK) { 367 ret = -EAGAIN; 368 break; 369 } 370 } 371 if (signal_pending(current)) { 372 if (!ret) 373 ret = -ERESTARTSYS; 374 break; 375 } 376 if (do_wakeup) { 377 wake_up_interruptible_sync(&pipe->wait); 378 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 379 } 380 pipe_wait(pipe); 381 } 382 mutex_unlock(&inode->i_mutex); 383 384 /* Signal writers asynchronously that there is more room. */ 385 if (do_wakeup) { 386 wake_up_interruptible_sync(&pipe->wait); 387 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 388 } 389 if (ret > 0) 390 file_accessed(filp); 391 return ret; 392 } 393 394 static ssize_t 395 pipe_write(struct kiocb *iocb, const struct iovec *_iov, 396 unsigned long nr_segs, loff_t ppos) 397 { 398 struct file *filp = iocb->ki_filp; 399 struct inode *inode = filp->f_path.dentry->d_inode; 400 struct pipe_inode_info *pipe; 401 ssize_t ret; 402 int do_wakeup; 403 struct iovec *iov = (struct iovec *)_iov; 404 size_t total_len; 405 ssize_t chars; 406 407 total_len = iov_length(iov, nr_segs); 408 /* Null write succeeds. */ 409 if (unlikely(total_len == 0)) 410 return 0; 411 412 do_wakeup = 0; 413 ret = 0; 414 mutex_lock(&inode->i_mutex); 415 pipe = inode->i_pipe; 416 417 if (!pipe->readers) { 418 send_sig(SIGPIPE, current, 0); 419 ret = -EPIPE; 420 goto out; 421 } 422 423 /* We try to merge small writes */ 424 chars = total_len & (PAGE_SIZE-1); /* size of the last buffer */ 425 if (pipe->nrbufs && chars != 0) { 426 int lastbuf = (pipe->curbuf + pipe->nrbufs - 1) & 427 (PIPE_BUFFERS-1); 428 struct pipe_buffer *buf = pipe->bufs + lastbuf; 429 const struct pipe_buf_operations *ops = buf->ops; 430 int offset = buf->offset + buf->len; 431 432 if (ops->can_merge && offset + chars <= PAGE_SIZE) { 433 int error, atomic = 1; 434 void *addr; 435 436 error = ops->confirm(pipe, buf); 437 if (error) 438 goto out; 439 440 iov_fault_in_pages_read(iov, chars); 441 redo1: 442 addr = ops->map(pipe, buf, atomic); 443 error = pipe_iov_copy_from_user(offset + addr, iov, 444 chars, atomic); 445 ops->unmap(pipe, buf, addr); 446 ret = error; 447 do_wakeup = 1; 448 if (error) { 449 if (atomic) { 450 atomic = 0; 451 goto redo1; 452 } 453 goto out; 454 } 455 buf->len += chars; 456 total_len -= chars; 457 ret = chars; 458 if (!total_len) 459 goto out; 460 } 461 } 462 463 for (;;) { 464 int bufs; 465 466 if (!pipe->readers) { 467 send_sig(SIGPIPE, current, 0); 468 if (!ret) 469 ret = -EPIPE; 470 break; 471 } 472 bufs = pipe->nrbufs; 473 if (bufs < PIPE_BUFFERS) { 474 int newbuf = (pipe->curbuf + bufs) & (PIPE_BUFFERS-1); 475 struct pipe_buffer *buf = pipe->bufs + newbuf; 476 struct page *page = pipe->tmp_page; 477 char *src; 478 int error, atomic = 1; 479 480 if (!page) { 481 page = alloc_page(GFP_HIGHUSER); 482 if (unlikely(!page)) { 483 ret = ret ? : -ENOMEM; 484 break; 485 } 486 pipe->tmp_page = page; 487 } 488 /* Always wake up, even if the copy fails. Otherwise 489 * we lock up (O_NONBLOCK-)readers that sleep due to 490 * syscall merging. 491 * FIXME! Is this really true? 492 */ 493 do_wakeup = 1; 494 chars = PAGE_SIZE; 495 if (chars > total_len) 496 chars = total_len; 497 498 iov_fault_in_pages_read(iov, chars); 499 redo2: 500 if (atomic) 501 src = kmap_atomic(page, KM_USER0); 502 else 503 src = kmap(page); 504 505 error = pipe_iov_copy_from_user(src, iov, chars, 506 atomic); 507 if (atomic) 508 kunmap_atomic(src, KM_USER0); 509 else 510 kunmap(page); 511 512 if (unlikely(error)) { 513 if (atomic) { 514 atomic = 0; 515 goto redo2; 516 } 517 if (!ret) 518 ret = error; 519 break; 520 } 521 ret += chars; 522 523 /* Insert it into the buffer array */ 524 buf->page = page; 525 buf->ops = &anon_pipe_buf_ops; 526 buf->offset = 0; 527 buf->len = chars; 528 pipe->nrbufs = ++bufs; 529 pipe->tmp_page = NULL; 530 531 total_len -= chars; 532 if (!total_len) 533 break; 534 } 535 if (bufs < PIPE_BUFFERS) 536 continue; 537 if (filp->f_flags & O_NONBLOCK) { 538 if (!ret) 539 ret = -EAGAIN; 540 break; 541 } 542 if (signal_pending(current)) { 543 if (!ret) 544 ret = -ERESTARTSYS; 545 break; 546 } 547 if (do_wakeup) { 548 wake_up_interruptible_sync(&pipe->wait); 549 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 550 do_wakeup = 0; 551 } 552 pipe->waiting_writers++; 553 pipe_wait(pipe); 554 pipe->waiting_writers--; 555 } 556 out: 557 mutex_unlock(&inode->i_mutex); 558 if (do_wakeup) { 559 wake_up_interruptible_sync(&pipe->wait); 560 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 561 } 562 if (ret > 0) 563 file_update_time(filp); 564 return ret; 565 } 566 567 static ssize_t 568 bad_pipe_r(struct file *filp, char __user *buf, size_t count, loff_t *ppos) 569 { 570 return -EBADF; 571 } 572 573 static ssize_t 574 bad_pipe_w(struct file *filp, const char __user *buf, size_t count, 575 loff_t *ppos) 576 { 577 return -EBADF; 578 } 579 580 static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) 581 { 582 struct inode *inode = filp->f_path.dentry->d_inode; 583 struct pipe_inode_info *pipe; 584 int count, buf, nrbufs; 585 586 switch (cmd) { 587 case FIONREAD: 588 mutex_lock(&inode->i_mutex); 589 pipe = inode->i_pipe; 590 count = 0; 591 buf = pipe->curbuf; 592 nrbufs = pipe->nrbufs; 593 while (--nrbufs >= 0) { 594 count += pipe->bufs[buf].len; 595 buf = (buf+1) & (PIPE_BUFFERS-1); 596 } 597 mutex_unlock(&inode->i_mutex); 598 599 return put_user(count, (int __user *)arg); 600 default: 601 return -EINVAL; 602 } 603 } 604 605 /* No kernel lock held - fine */ 606 static unsigned int 607 pipe_poll(struct file *filp, poll_table *wait) 608 { 609 unsigned int mask; 610 struct inode *inode = filp->f_path.dentry->d_inode; 611 struct pipe_inode_info *pipe = inode->i_pipe; 612 int nrbufs; 613 614 poll_wait(filp, &pipe->wait, wait); 615 616 /* Reading only -- no need for acquiring the semaphore. */ 617 nrbufs = pipe->nrbufs; 618 mask = 0; 619 if (filp->f_mode & FMODE_READ) { 620 mask = (nrbufs > 0) ? POLLIN | POLLRDNORM : 0; 621 if (!pipe->writers && filp->f_version != pipe->w_counter) 622 mask |= POLLHUP; 623 } 624 625 if (filp->f_mode & FMODE_WRITE) { 626 mask |= (nrbufs < PIPE_BUFFERS) ? POLLOUT | POLLWRNORM : 0; 627 /* 628 * Most Unices do not set POLLERR for FIFOs but on Linux they 629 * behave exactly like pipes for poll(). 630 */ 631 if (!pipe->readers) 632 mask |= POLLERR; 633 } 634 635 return mask; 636 } 637 638 static int 639 pipe_release(struct inode *inode, int decr, int decw) 640 { 641 struct pipe_inode_info *pipe; 642 643 mutex_lock(&inode->i_mutex); 644 pipe = inode->i_pipe; 645 pipe->readers -= decr; 646 pipe->writers -= decw; 647 648 if (!pipe->readers && !pipe->writers) { 649 free_pipe_info(inode); 650 } else { 651 wake_up_interruptible_sync(&pipe->wait); 652 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 653 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 654 } 655 mutex_unlock(&inode->i_mutex); 656 657 return 0; 658 } 659 660 static int 661 pipe_read_fasync(int fd, struct file *filp, int on) 662 { 663 struct inode *inode = filp->f_path.dentry->d_inode; 664 int retval; 665 666 mutex_lock(&inode->i_mutex); 667 retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_readers); 668 mutex_unlock(&inode->i_mutex); 669 670 if (retval < 0) 671 return retval; 672 673 return 0; 674 } 675 676 677 static int 678 pipe_write_fasync(int fd, struct file *filp, int on) 679 { 680 struct inode *inode = filp->f_path.dentry->d_inode; 681 int retval; 682 683 mutex_lock(&inode->i_mutex); 684 retval = fasync_helper(fd, filp, on, &inode->i_pipe->fasync_writers); 685 mutex_unlock(&inode->i_mutex); 686 687 if (retval < 0) 688 return retval; 689 690 return 0; 691 } 692 693 694 static int 695 pipe_rdwr_fasync(int fd, struct file *filp, int on) 696 { 697 struct inode *inode = filp->f_path.dentry->d_inode; 698 struct pipe_inode_info *pipe = inode->i_pipe; 699 int retval; 700 701 mutex_lock(&inode->i_mutex); 702 703 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers); 704 705 if (retval >= 0) 706 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers); 707 708 mutex_unlock(&inode->i_mutex); 709 710 if (retval < 0) 711 return retval; 712 713 return 0; 714 } 715 716 717 static int 718 pipe_read_release(struct inode *inode, struct file *filp) 719 { 720 pipe_read_fasync(-1, filp, 0); 721 return pipe_release(inode, 1, 0); 722 } 723 724 static int 725 pipe_write_release(struct inode *inode, struct file *filp) 726 { 727 pipe_write_fasync(-1, filp, 0); 728 return pipe_release(inode, 0, 1); 729 } 730 731 static int 732 pipe_rdwr_release(struct inode *inode, struct file *filp) 733 { 734 int decr, decw; 735 736 pipe_rdwr_fasync(-1, filp, 0); 737 decr = (filp->f_mode & FMODE_READ) != 0; 738 decw = (filp->f_mode & FMODE_WRITE) != 0; 739 return pipe_release(inode, decr, decw); 740 } 741 742 static int 743 pipe_read_open(struct inode *inode, struct file *filp) 744 { 745 /* We could have perhaps used atomic_t, but this and friends 746 below are the only places. So it doesn't seem worthwhile. */ 747 mutex_lock(&inode->i_mutex); 748 inode->i_pipe->readers++; 749 mutex_unlock(&inode->i_mutex); 750 751 return 0; 752 } 753 754 static int 755 pipe_write_open(struct inode *inode, struct file *filp) 756 { 757 mutex_lock(&inode->i_mutex); 758 inode->i_pipe->writers++; 759 mutex_unlock(&inode->i_mutex); 760 761 return 0; 762 } 763 764 static int 765 pipe_rdwr_open(struct inode *inode, struct file *filp) 766 { 767 mutex_lock(&inode->i_mutex); 768 if (filp->f_mode & FMODE_READ) 769 inode->i_pipe->readers++; 770 if (filp->f_mode & FMODE_WRITE) 771 inode->i_pipe->writers++; 772 mutex_unlock(&inode->i_mutex); 773 774 return 0; 775 } 776 777 /* 778 * The file_operations structs are not static because they 779 * are also used in linux/fs/fifo.c to do operations on FIFOs. 780 */ 781 const struct file_operations read_fifo_fops = { 782 .llseek = no_llseek, 783 .read = do_sync_read, 784 .aio_read = pipe_read, 785 .write = bad_pipe_w, 786 .poll = pipe_poll, 787 .unlocked_ioctl = pipe_ioctl, 788 .open = pipe_read_open, 789 .release = pipe_read_release, 790 .fasync = pipe_read_fasync, 791 }; 792 793 const struct file_operations write_fifo_fops = { 794 .llseek = no_llseek, 795 .read = bad_pipe_r, 796 .write = do_sync_write, 797 .aio_write = pipe_write, 798 .poll = pipe_poll, 799 .unlocked_ioctl = pipe_ioctl, 800 .open = pipe_write_open, 801 .release = pipe_write_release, 802 .fasync = pipe_write_fasync, 803 }; 804 805 const struct file_operations rdwr_fifo_fops = { 806 .llseek = no_llseek, 807 .read = do_sync_read, 808 .aio_read = pipe_read, 809 .write = do_sync_write, 810 .aio_write = pipe_write, 811 .poll = pipe_poll, 812 .unlocked_ioctl = pipe_ioctl, 813 .open = pipe_rdwr_open, 814 .release = pipe_rdwr_release, 815 .fasync = pipe_rdwr_fasync, 816 }; 817 818 static const struct file_operations read_pipe_fops = { 819 .llseek = no_llseek, 820 .read = do_sync_read, 821 .aio_read = pipe_read, 822 .write = bad_pipe_w, 823 .poll = pipe_poll, 824 .unlocked_ioctl = pipe_ioctl, 825 .open = pipe_read_open, 826 .release = pipe_read_release, 827 .fasync = pipe_read_fasync, 828 }; 829 830 static const struct file_operations write_pipe_fops = { 831 .llseek = no_llseek, 832 .read = bad_pipe_r, 833 .write = do_sync_write, 834 .aio_write = pipe_write, 835 .poll = pipe_poll, 836 .unlocked_ioctl = pipe_ioctl, 837 .open = pipe_write_open, 838 .release = pipe_write_release, 839 .fasync = pipe_write_fasync, 840 }; 841 842 static const struct file_operations rdwr_pipe_fops = { 843 .llseek = no_llseek, 844 .read = do_sync_read, 845 .aio_read = pipe_read, 846 .write = do_sync_write, 847 .aio_write = pipe_write, 848 .poll = pipe_poll, 849 .unlocked_ioctl = pipe_ioctl, 850 .open = pipe_rdwr_open, 851 .release = pipe_rdwr_release, 852 .fasync = pipe_rdwr_fasync, 853 }; 854 855 struct pipe_inode_info * alloc_pipe_info(struct inode *inode) 856 { 857 struct pipe_inode_info *pipe; 858 859 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL); 860 if (pipe) { 861 init_waitqueue_head(&pipe->wait); 862 pipe->r_counter = pipe->w_counter = 1; 863 pipe->inode = inode; 864 } 865 866 return pipe; 867 } 868 869 void __free_pipe_info(struct pipe_inode_info *pipe) 870 { 871 int i; 872 873 for (i = 0; i < PIPE_BUFFERS; i++) { 874 struct pipe_buffer *buf = pipe->bufs + i; 875 if (buf->ops) 876 buf->ops->release(pipe, buf); 877 } 878 if (pipe->tmp_page) 879 __free_page(pipe->tmp_page); 880 kfree(pipe); 881 } 882 883 void free_pipe_info(struct inode *inode) 884 { 885 __free_pipe_info(inode->i_pipe); 886 inode->i_pipe = NULL; 887 } 888 889 static struct vfsmount *pipe_mnt __read_mostly; 890 static int pipefs_delete_dentry(struct dentry *dentry) 891 { 892 /* 893 * At creation time, we pretended this dentry was hashed 894 * (by clearing DCACHE_UNHASHED bit in d_flags) 895 * At delete time, we restore the truth : not hashed. 896 * (so that dput() can proceed correctly) 897 */ 898 dentry->d_flags |= DCACHE_UNHASHED; 899 return 0; 900 } 901 902 /* 903 * pipefs_dname() is called from d_path(). 904 */ 905 static char *pipefs_dname(struct dentry *dentry, char *buffer, int buflen) 906 { 907 return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]", 908 dentry->d_inode->i_ino); 909 } 910 911 static struct dentry_operations pipefs_dentry_operations = { 912 .d_delete = pipefs_delete_dentry, 913 .d_dname = pipefs_dname, 914 }; 915 916 static struct inode * get_pipe_inode(void) 917 { 918 struct inode *inode = new_inode(pipe_mnt->mnt_sb); 919 struct pipe_inode_info *pipe; 920 921 if (!inode) 922 goto fail_inode; 923 924 pipe = alloc_pipe_info(inode); 925 if (!pipe) 926 goto fail_iput; 927 inode->i_pipe = pipe; 928 929 pipe->readers = pipe->writers = 1; 930 inode->i_fop = &rdwr_pipe_fops; 931 932 /* 933 * Mark the inode dirty from the very beginning, 934 * that way it will never be moved to the dirty 935 * list because "mark_inode_dirty()" will think 936 * that it already _is_ on the dirty list. 937 */ 938 inode->i_state = I_DIRTY; 939 inode->i_mode = S_IFIFO | S_IRUSR | S_IWUSR; 940 inode->i_uid = current->fsuid; 941 inode->i_gid = current->fsgid; 942 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 943 944 return inode; 945 946 fail_iput: 947 iput(inode); 948 949 fail_inode: 950 return NULL; 951 } 952 953 struct file *create_write_pipe(void) 954 { 955 int err; 956 struct inode *inode; 957 struct file *f; 958 struct dentry *dentry; 959 struct qstr name = { .name = "" }; 960 961 err = -ENFILE; 962 inode = get_pipe_inode(); 963 if (!inode) 964 goto err; 965 966 err = -ENOMEM; 967 dentry = d_alloc(pipe_mnt->mnt_sb->s_root, &name); 968 if (!dentry) 969 goto err_inode; 970 971 dentry->d_op = &pipefs_dentry_operations; 972 /* 973 * We dont want to publish this dentry into global dentry hash table. 974 * We pretend dentry is already hashed, by unsetting DCACHE_UNHASHED 975 * This permits a working /proc/$pid/fd/XXX on pipes 976 */ 977 dentry->d_flags &= ~DCACHE_UNHASHED; 978 d_instantiate(dentry, inode); 979 980 err = -ENFILE; 981 f = alloc_file(pipe_mnt, dentry, FMODE_WRITE, &write_pipe_fops); 982 if (!f) 983 goto err_dentry; 984 f->f_mapping = inode->i_mapping; 985 986 f->f_flags = O_WRONLY; 987 f->f_version = 0; 988 989 return f; 990 991 err_dentry: 992 free_pipe_info(inode); 993 dput(dentry); 994 return ERR_PTR(err); 995 996 err_inode: 997 free_pipe_info(inode); 998 iput(inode); 999 err: 1000 return ERR_PTR(err); 1001 } 1002 1003 void free_write_pipe(struct file *f) 1004 { 1005 free_pipe_info(f->f_dentry->d_inode); 1006 dput(f->f_path.dentry); 1007 mntput(f->f_path.mnt); 1008 put_filp(f); 1009 } 1010 1011 struct file *create_read_pipe(struct file *wrf) 1012 { 1013 struct file *f = get_empty_filp(); 1014 if (!f) 1015 return ERR_PTR(-ENFILE); 1016 1017 /* Grab pipe from the writer */ 1018 f->f_path.mnt = mntget(wrf->f_path.mnt); 1019 f->f_path.dentry = dget(wrf->f_path.dentry); 1020 f->f_mapping = wrf->f_path.dentry->d_inode->i_mapping; 1021 1022 f->f_pos = 0; 1023 f->f_flags = O_RDONLY; 1024 f->f_op = &read_pipe_fops; 1025 f->f_mode = FMODE_READ; 1026 f->f_version = 0; 1027 1028 return f; 1029 } 1030 1031 int do_pipe(int *fd) 1032 { 1033 struct file *fw, *fr; 1034 int error; 1035 int fdw, fdr; 1036 1037 fw = create_write_pipe(); 1038 if (IS_ERR(fw)) 1039 return PTR_ERR(fw); 1040 fr = create_read_pipe(fw); 1041 error = PTR_ERR(fr); 1042 if (IS_ERR(fr)) 1043 goto err_write_pipe; 1044 1045 error = get_unused_fd(); 1046 if (error < 0) 1047 goto err_read_pipe; 1048 fdr = error; 1049 1050 error = get_unused_fd(); 1051 if (error < 0) 1052 goto err_fdr; 1053 fdw = error; 1054 1055 error = audit_fd_pair(fdr, fdw); 1056 if (error < 0) 1057 goto err_fdw; 1058 1059 fd_install(fdr, fr); 1060 fd_install(fdw, fw); 1061 fd[0] = fdr; 1062 fd[1] = fdw; 1063 1064 return 0; 1065 1066 err_fdw: 1067 put_unused_fd(fdw); 1068 err_fdr: 1069 put_unused_fd(fdr); 1070 err_read_pipe: 1071 dput(fr->f_dentry); 1072 mntput(fr->f_vfsmnt); 1073 put_filp(fr); 1074 err_write_pipe: 1075 free_write_pipe(fw); 1076 return error; 1077 } 1078 1079 /* 1080 * sys_pipe() is the normal C calling standard for creating 1081 * a pipe. It's not the way Unix traditionally does this, though. 1082 */ 1083 asmlinkage long __weak sys_pipe(int __user *fildes) 1084 { 1085 int fd[2]; 1086 int error; 1087 1088 error = do_pipe(fd); 1089 if (!error) { 1090 if (copy_to_user(fildes, fd, sizeof(fd))) { 1091 sys_close(fd[0]); 1092 sys_close(fd[1]); 1093 error = -EFAULT; 1094 } 1095 } 1096 return error; 1097 } 1098 1099 /* 1100 * pipefs should _never_ be mounted by userland - too much of security hassle, 1101 * no real gain from having the whole whorehouse mounted. So we don't need 1102 * any operations on the root directory. However, we need a non-trivial 1103 * d_name - pipe: will go nicely and kill the special-casing in procfs. 1104 */ 1105 static int pipefs_get_sb(struct file_system_type *fs_type, 1106 int flags, const char *dev_name, void *data, 1107 struct vfsmount *mnt) 1108 { 1109 return get_sb_pseudo(fs_type, "pipe:", NULL, PIPEFS_MAGIC, mnt); 1110 } 1111 1112 static struct file_system_type pipe_fs_type = { 1113 .name = "pipefs", 1114 .get_sb = pipefs_get_sb, 1115 .kill_sb = kill_anon_super, 1116 }; 1117 1118 static int __init init_pipe_fs(void) 1119 { 1120 int err = register_filesystem(&pipe_fs_type); 1121 1122 if (!err) { 1123 pipe_mnt = kern_mount(&pipe_fs_type); 1124 if (IS_ERR(pipe_mnt)) { 1125 err = PTR_ERR(pipe_mnt); 1126 unregister_filesystem(&pipe_fs_type); 1127 } 1128 } 1129 return err; 1130 } 1131 1132 static void __exit exit_pipe_fs(void) 1133 { 1134 unregister_filesystem(&pipe_fs_type); 1135 mntput(pipe_mnt); 1136 } 1137 1138 fs_initcall(init_pipe_fs); 1139 module_exit(exit_pipe_fs); 1140