1 /* 2 * "splice": joining two ropes together by interweaving their strands. 3 * 4 * This is the "extended pipe" functionality, where a pipe is used as 5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 6 * buffer that you can use to transfer data from one end to the other. 7 * 8 * The traditional unix read/write is extended with a "splice()" operation 9 * that transfers data buffers to or from a pipe buffer. 10 * 11 * Named by Larry McVoy, original implementation from Linus, extended by 12 * Jens to support splicing to files, network, direct splicing, etc and 13 * fixing lots of bugs. 14 * 15 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 18 * 19 */ 20 #include <linux/fs.h> 21 #include <linux/file.h> 22 #include <linux/pagemap.h> 23 #include <linux/splice.h> 24 #include <linux/mm_inline.h> 25 #include <linux/swap.h> 26 #include <linux/writeback.h> 27 #include <linux/buffer_head.h> 28 #include <linux/module.h> 29 #include <linux/syscalls.h> 30 #include <linux/uio.h> 31 #include <linux/security.h> 32 33 /* 34 * Attempt to steal a page from a pipe buffer. This should perhaps go into 35 * a vm helper function, it's already simplified quite a bit by the 36 * addition of remove_mapping(). If success is returned, the caller may 37 * attempt to reuse this page for another destination. 38 */ 39 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, 40 struct pipe_buffer *buf) 41 { 42 struct page *page = buf->page; 43 struct address_space *mapping; 44 45 lock_page(page); 46 47 mapping = page_mapping(page); 48 if (mapping) { 49 WARN_ON(!PageUptodate(page)); 50 51 /* 52 * At least for ext2 with nobh option, we need to wait on 53 * writeback completing on this page, since we'll remove it 54 * from the pagecache. Otherwise truncate wont wait on the 55 * page, allowing the disk blocks to be reused by someone else 56 * before we actually wrote our data to them. fs corruption 57 * ensues. 58 */ 59 wait_on_page_writeback(page); 60 61 if (PagePrivate(page)) 62 try_to_release_page(page, GFP_KERNEL); 63 64 /* 65 * If we succeeded in removing the mapping, set LRU flag 66 * and return good. 67 */ 68 if (remove_mapping(mapping, page)) { 69 buf->flags |= PIPE_BUF_FLAG_LRU; 70 return 0; 71 } 72 } 73 74 /* 75 * Raced with truncate or failed to remove page from current 76 * address space, unlock and return failure. 77 */ 78 unlock_page(page); 79 return 1; 80 } 81 82 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, 83 struct pipe_buffer *buf) 84 { 85 page_cache_release(buf->page); 86 buf->flags &= ~PIPE_BUF_FLAG_LRU; 87 } 88 89 /* 90 * Check whether the contents of buf is OK to access. Since the content 91 * is a page cache page, IO may be in flight. 92 */ 93 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 94 struct pipe_buffer *buf) 95 { 96 struct page *page = buf->page; 97 int err; 98 99 if (!PageUptodate(page)) { 100 lock_page(page); 101 102 /* 103 * Page got truncated/unhashed. This will cause a 0-byte 104 * splice, if this is the first page. 105 */ 106 if (!page->mapping) { 107 err = -ENODATA; 108 goto error; 109 } 110 111 /* 112 * Uh oh, read-error from disk. 113 */ 114 if (!PageUptodate(page)) { 115 err = -EIO; 116 goto error; 117 } 118 119 /* 120 * Page is ok afterall, we are done. 121 */ 122 unlock_page(page); 123 } 124 125 return 0; 126 error: 127 unlock_page(page); 128 return err; 129 } 130 131 static const struct pipe_buf_operations page_cache_pipe_buf_ops = { 132 .can_merge = 0, 133 .map = generic_pipe_buf_map, 134 .unmap = generic_pipe_buf_unmap, 135 .confirm = page_cache_pipe_buf_confirm, 136 .release = page_cache_pipe_buf_release, 137 .steal = page_cache_pipe_buf_steal, 138 .get = generic_pipe_buf_get, 139 }; 140 141 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, 142 struct pipe_buffer *buf) 143 { 144 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) 145 return 1; 146 147 buf->flags |= PIPE_BUF_FLAG_LRU; 148 return generic_pipe_buf_steal(pipe, buf); 149 } 150 151 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 152 .can_merge = 0, 153 .map = generic_pipe_buf_map, 154 .unmap = generic_pipe_buf_unmap, 155 .confirm = generic_pipe_buf_confirm, 156 .release = page_cache_pipe_buf_release, 157 .steal = user_page_pipe_buf_steal, 158 .get = generic_pipe_buf_get, 159 }; 160 161 /** 162 * splice_to_pipe - fill passed data into a pipe 163 * @pipe: pipe to fill 164 * @spd: data to fill 165 * 166 * Description: 167 * @spd contains a map of pages and len/offset tuples, along with 168 * the struct pipe_buf_operations associated with these pages. This 169 * function will link that data to the pipe. 170 * 171 */ 172 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 173 struct splice_pipe_desc *spd) 174 { 175 unsigned int spd_pages = spd->nr_pages; 176 int ret, do_wakeup, page_nr; 177 178 ret = 0; 179 do_wakeup = 0; 180 page_nr = 0; 181 182 if (pipe->inode) 183 mutex_lock(&pipe->inode->i_mutex); 184 185 for (;;) { 186 if (!pipe->readers) { 187 send_sig(SIGPIPE, current, 0); 188 if (!ret) 189 ret = -EPIPE; 190 break; 191 } 192 193 if (pipe->nrbufs < PIPE_BUFFERS) { 194 int newbuf = (pipe->curbuf + pipe->nrbufs) & (PIPE_BUFFERS - 1); 195 struct pipe_buffer *buf = pipe->bufs + newbuf; 196 197 buf->page = spd->pages[page_nr]; 198 buf->offset = spd->partial[page_nr].offset; 199 buf->len = spd->partial[page_nr].len; 200 buf->private = spd->partial[page_nr].private; 201 buf->ops = spd->ops; 202 if (spd->flags & SPLICE_F_GIFT) 203 buf->flags |= PIPE_BUF_FLAG_GIFT; 204 205 pipe->nrbufs++; 206 page_nr++; 207 ret += buf->len; 208 209 if (pipe->inode) 210 do_wakeup = 1; 211 212 if (!--spd->nr_pages) 213 break; 214 if (pipe->nrbufs < PIPE_BUFFERS) 215 continue; 216 217 break; 218 } 219 220 if (spd->flags & SPLICE_F_NONBLOCK) { 221 if (!ret) 222 ret = -EAGAIN; 223 break; 224 } 225 226 if (signal_pending(current)) { 227 if (!ret) 228 ret = -ERESTARTSYS; 229 break; 230 } 231 232 if (do_wakeup) { 233 smp_mb(); 234 if (waitqueue_active(&pipe->wait)) 235 wake_up_interruptible_sync(&pipe->wait); 236 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 237 do_wakeup = 0; 238 } 239 240 pipe->waiting_writers++; 241 pipe_wait(pipe); 242 pipe->waiting_writers--; 243 } 244 245 if (pipe->inode) { 246 mutex_unlock(&pipe->inode->i_mutex); 247 248 if (do_wakeup) { 249 smp_mb(); 250 if (waitqueue_active(&pipe->wait)) 251 wake_up_interruptible(&pipe->wait); 252 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 253 } 254 } 255 256 while (page_nr < spd_pages) 257 spd->spd_release(spd, page_nr++); 258 259 return ret; 260 } 261 262 static void spd_release_page(struct splice_pipe_desc *spd, unsigned int i) 263 { 264 page_cache_release(spd->pages[i]); 265 } 266 267 static int 268 __generic_file_splice_read(struct file *in, loff_t *ppos, 269 struct pipe_inode_info *pipe, size_t len, 270 unsigned int flags) 271 { 272 struct address_space *mapping = in->f_mapping; 273 unsigned int loff, nr_pages, req_pages; 274 struct page *pages[PIPE_BUFFERS]; 275 struct partial_page partial[PIPE_BUFFERS]; 276 struct page *page; 277 pgoff_t index, end_index; 278 loff_t isize; 279 int error, page_nr; 280 struct splice_pipe_desc spd = { 281 .pages = pages, 282 .partial = partial, 283 .flags = flags, 284 .ops = &page_cache_pipe_buf_ops, 285 .spd_release = spd_release_page, 286 }; 287 288 index = *ppos >> PAGE_CACHE_SHIFT; 289 loff = *ppos & ~PAGE_CACHE_MASK; 290 req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 291 nr_pages = min(req_pages, (unsigned)PIPE_BUFFERS); 292 293 /* 294 * Lookup the (hopefully) full range of pages we need. 295 */ 296 spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, pages); 297 index += spd.nr_pages; 298 299 /* 300 * If find_get_pages_contig() returned fewer pages than we needed, 301 * readahead/allocate the rest and fill in the holes. 302 */ 303 if (spd.nr_pages < nr_pages) 304 page_cache_sync_readahead(mapping, &in->f_ra, in, 305 index, req_pages - spd.nr_pages); 306 307 error = 0; 308 while (spd.nr_pages < nr_pages) { 309 /* 310 * Page could be there, find_get_pages_contig() breaks on 311 * the first hole. 312 */ 313 page = find_get_page(mapping, index); 314 if (!page) { 315 /* 316 * page didn't exist, allocate one. 317 */ 318 page = page_cache_alloc_cold(mapping); 319 if (!page) 320 break; 321 322 error = add_to_page_cache_lru(page, mapping, index, 323 GFP_KERNEL); 324 if (unlikely(error)) { 325 page_cache_release(page); 326 if (error == -EEXIST) 327 continue; 328 break; 329 } 330 /* 331 * add_to_page_cache() locks the page, unlock it 332 * to avoid convoluting the logic below even more. 333 */ 334 unlock_page(page); 335 } 336 337 pages[spd.nr_pages++] = page; 338 index++; 339 } 340 341 /* 342 * Now loop over the map and see if we need to start IO on any 343 * pages, fill in the partial map, etc. 344 */ 345 index = *ppos >> PAGE_CACHE_SHIFT; 346 nr_pages = spd.nr_pages; 347 spd.nr_pages = 0; 348 for (page_nr = 0; page_nr < nr_pages; page_nr++) { 349 unsigned int this_len; 350 351 if (!len) 352 break; 353 354 /* 355 * this_len is the max we'll use from this page 356 */ 357 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff); 358 page = pages[page_nr]; 359 360 if (PageReadahead(page)) 361 page_cache_async_readahead(mapping, &in->f_ra, in, 362 page, index, req_pages - page_nr); 363 364 /* 365 * If the page isn't uptodate, we may need to start io on it 366 */ 367 if (!PageUptodate(page)) { 368 /* 369 * If in nonblock mode then dont block on waiting 370 * for an in-flight io page 371 */ 372 if (flags & SPLICE_F_NONBLOCK) { 373 if (TestSetPageLocked(page)) 374 break; 375 } else 376 lock_page(page); 377 378 /* 379 * page was truncated, stop here. if this isn't the 380 * first page, we'll just complete what we already 381 * added 382 */ 383 if (!page->mapping) { 384 unlock_page(page); 385 break; 386 } 387 /* 388 * page was already under io and is now done, great 389 */ 390 if (PageUptodate(page)) { 391 unlock_page(page); 392 goto fill_it; 393 } 394 395 /* 396 * need to read in the page 397 */ 398 error = mapping->a_ops->readpage(in, page); 399 if (unlikely(error)) { 400 /* 401 * We really should re-lookup the page here, 402 * but it complicates things a lot. Instead 403 * lets just do what we already stored, and 404 * we'll get it the next time we are called. 405 */ 406 if (error == AOP_TRUNCATED_PAGE) 407 error = 0; 408 409 break; 410 } 411 } 412 fill_it: 413 /* 414 * i_size must be checked after PageUptodate. 415 */ 416 isize = i_size_read(mapping->host); 417 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 418 if (unlikely(!isize || index > end_index)) 419 break; 420 421 /* 422 * if this is the last page, see if we need to shrink 423 * the length and stop 424 */ 425 if (end_index == index) { 426 unsigned int plen; 427 428 /* 429 * max good bytes in this page 430 */ 431 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 432 if (plen <= loff) 433 break; 434 435 /* 436 * force quit after adding this page 437 */ 438 this_len = min(this_len, plen - loff); 439 len = this_len; 440 } 441 442 partial[page_nr].offset = loff; 443 partial[page_nr].len = this_len; 444 len -= this_len; 445 loff = 0; 446 spd.nr_pages++; 447 index++; 448 } 449 450 /* 451 * Release any pages at the end, if we quit early. 'page_nr' is how far 452 * we got, 'nr_pages' is how many pages are in the map. 453 */ 454 while (page_nr < nr_pages) 455 page_cache_release(pages[page_nr++]); 456 in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT; 457 458 if (spd.nr_pages) 459 return splice_to_pipe(pipe, &spd); 460 461 return error; 462 } 463 464 /** 465 * generic_file_splice_read - splice data from file to a pipe 466 * @in: file to splice from 467 * @ppos: position in @in 468 * @pipe: pipe to splice to 469 * @len: number of bytes to splice 470 * @flags: splice modifier flags 471 * 472 * Description: 473 * Will read pages from given file and fill them into a pipe. Can be 474 * used as long as the address_space operations for the source implements 475 * a readpage() hook. 476 * 477 */ 478 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, 479 struct pipe_inode_info *pipe, size_t len, 480 unsigned int flags) 481 { 482 ssize_t spliced; 483 int ret; 484 loff_t isize, left; 485 486 isize = i_size_read(in->f_mapping->host); 487 if (unlikely(*ppos >= isize)) 488 return 0; 489 490 left = isize - *ppos; 491 if (unlikely(left < len)) 492 len = left; 493 494 ret = 0; 495 spliced = 0; 496 while (len && !spliced) { 497 ret = __generic_file_splice_read(in, ppos, pipe, len, flags); 498 499 if (ret < 0) 500 break; 501 else if (!ret) { 502 if (spliced) 503 break; 504 if (flags & SPLICE_F_NONBLOCK) { 505 ret = -EAGAIN; 506 break; 507 } 508 } 509 510 *ppos += ret; 511 len -= ret; 512 spliced += ret; 513 } 514 515 if (spliced) 516 return spliced; 517 518 return ret; 519 } 520 521 EXPORT_SYMBOL(generic_file_splice_read); 522 523 /* 524 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 525 * using sendpage(). Return the number of bytes sent. 526 */ 527 static int pipe_to_sendpage(struct pipe_inode_info *pipe, 528 struct pipe_buffer *buf, struct splice_desc *sd) 529 { 530 struct file *file = sd->u.file; 531 loff_t pos = sd->pos; 532 int ret, more; 533 534 ret = buf->ops->confirm(pipe, buf); 535 if (!ret) { 536 more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len; 537 538 ret = file->f_op->sendpage(file, buf->page, buf->offset, 539 sd->len, &pos, more); 540 } 541 542 return ret; 543 } 544 545 /* 546 * This is a little more tricky than the file -> pipe splicing. There are 547 * basically three cases: 548 * 549 * - Destination page already exists in the address space and there 550 * are users of it. For that case we have no other option that 551 * copying the data. Tough luck. 552 * - Destination page already exists in the address space, but there 553 * are no users of it. Make sure it's uptodate, then drop it. Fall 554 * through to last case. 555 * - Destination page does not exist, we can add the pipe page to 556 * the page cache and avoid the copy. 557 * 558 * If asked to move pages to the output file (SPLICE_F_MOVE is set in 559 * sd->flags), we attempt to migrate pages from the pipe to the output 560 * file address space page cache. This is possible if no one else has 561 * the pipe page referenced outside of the pipe and page cache. If 562 * SPLICE_F_MOVE isn't set, or we cannot move the page, we simply create 563 * a new page in the output file page cache and fill/dirty that. 564 */ 565 static int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 566 struct splice_desc *sd) 567 { 568 struct file *file = sd->u.file; 569 struct address_space *mapping = file->f_mapping; 570 unsigned int offset, this_len; 571 struct page *page; 572 void *fsdata; 573 int ret; 574 575 /* 576 * make sure the data in this buffer is uptodate 577 */ 578 ret = buf->ops->confirm(pipe, buf); 579 if (unlikely(ret)) 580 return ret; 581 582 offset = sd->pos & ~PAGE_CACHE_MASK; 583 584 this_len = sd->len; 585 if (this_len + offset > PAGE_CACHE_SIZE) 586 this_len = PAGE_CACHE_SIZE - offset; 587 588 ret = pagecache_write_begin(file, mapping, sd->pos, this_len, 589 AOP_FLAG_UNINTERRUPTIBLE, &page, &fsdata); 590 if (unlikely(ret)) 591 goto out; 592 593 if (buf->page != page) { 594 /* 595 * Careful, ->map() uses KM_USER0! 596 */ 597 char *src = buf->ops->map(pipe, buf, 1); 598 char *dst = kmap_atomic(page, KM_USER1); 599 600 memcpy(dst + offset, src + buf->offset, this_len); 601 flush_dcache_page(page); 602 kunmap_atomic(dst, KM_USER1); 603 buf->ops->unmap(pipe, buf, src); 604 } 605 ret = pagecache_write_end(file, mapping, sd->pos, this_len, this_len, 606 page, fsdata); 607 out: 608 return ret; 609 } 610 611 /** 612 * __splice_from_pipe - splice data from a pipe to given actor 613 * @pipe: pipe to splice from 614 * @sd: information to @actor 615 * @actor: handler that splices the data 616 * 617 * Description: 618 * This function does little more than loop over the pipe and call 619 * @actor to do the actual moving of a single struct pipe_buffer to 620 * the desired destination. See pipe_to_file, pipe_to_sendpage, or 621 * pipe_to_user. 622 * 623 */ 624 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 625 splice_actor *actor) 626 { 627 int ret, do_wakeup, err; 628 629 ret = 0; 630 do_wakeup = 0; 631 632 for (;;) { 633 if (pipe->nrbufs) { 634 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 635 const struct pipe_buf_operations *ops = buf->ops; 636 637 sd->len = buf->len; 638 if (sd->len > sd->total_len) 639 sd->len = sd->total_len; 640 641 err = actor(pipe, buf, sd); 642 if (err <= 0) { 643 if (!ret && err != -ENODATA) 644 ret = err; 645 646 break; 647 } 648 649 ret += err; 650 buf->offset += err; 651 buf->len -= err; 652 653 sd->len -= err; 654 sd->pos += err; 655 sd->total_len -= err; 656 if (sd->len) 657 continue; 658 659 if (!buf->len) { 660 buf->ops = NULL; 661 ops->release(pipe, buf); 662 pipe->curbuf = (pipe->curbuf + 1) & (PIPE_BUFFERS - 1); 663 pipe->nrbufs--; 664 if (pipe->inode) 665 do_wakeup = 1; 666 } 667 668 if (!sd->total_len) 669 break; 670 } 671 672 if (pipe->nrbufs) 673 continue; 674 if (!pipe->writers) 675 break; 676 if (!pipe->waiting_writers) { 677 if (ret) 678 break; 679 } 680 681 if (sd->flags & SPLICE_F_NONBLOCK) { 682 if (!ret) 683 ret = -EAGAIN; 684 break; 685 } 686 687 if (signal_pending(current)) { 688 if (!ret) 689 ret = -ERESTARTSYS; 690 break; 691 } 692 693 if (do_wakeup) { 694 smp_mb(); 695 if (waitqueue_active(&pipe->wait)) 696 wake_up_interruptible_sync(&pipe->wait); 697 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 698 do_wakeup = 0; 699 } 700 701 pipe_wait(pipe); 702 } 703 704 if (do_wakeup) { 705 smp_mb(); 706 if (waitqueue_active(&pipe->wait)) 707 wake_up_interruptible(&pipe->wait); 708 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 709 } 710 711 return ret; 712 } 713 EXPORT_SYMBOL(__splice_from_pipe); 714 715 /** 716 * splice_from_pipe - splice data from a pipe to a file 717 * @pipe: pipe to splice from 718 * @out: file to splice to 719 * @ppos: position in @out 720 * @len: how many bytes to splice 721 * @flags: splice modifier flags 722 * @actor: handler that splices the data 723 * 724 * Description: 725 * See __splice_from_pipe. This function locks the input and output inodes, 726 * otherwise it's identical to __splice_from_pipe(). 727 * 728 */ 729 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 730 loff_t *ppos, size_t len, unsigned int flags, 731 splice_actor *actor) 732 { 733 ssize_t ret; 734 struct inode *inode = out->f_mapping->host; 735 struct splice_desc sd = { 736 .total_len = len, 737 .flags = flags, 738 .pos = *ppos, 739 .u.file = out, 740 }; 741 742 /* 743 * The actor worker might be calling ->prepare_write and 744 * ->commit_write. Most of the time, these expect i_mutex to 745 * be held. Since this may result in an ABBA deadlock with 746 * pipe->inode, we have to order lock acquiry here. 747 */ 748 inode_double_lock(inode, pipe->inode); 749 ret = __splice_from_pipe(pipe, &sd, actor); 750 inode_double_unlock(inode, pipe->inode); 751 752 return ret; 753 } 754 755 /** 756 * generic_file_splice_write_nolock - generic_file_splice_write without mutexes 757 * @pipe: pipe info 758 * @out: file to write to 759 * @ppos: position in @out 760 * @len: number of bytes to splice 761 * @flags: splice modifier flags 762 * 763 * Description: 764 * Will either move or copy pages (determined by @flags options) from 765 * the given pipe inode to the given file. The caller is responsible 766 * for acquiring i_mutex on both inodes. 767 * 768 */ 769 ssize_t 770 generic_file_splice_write_nolock(struct pipe_inode_info *pipe, struct file *out, 771 loff_t *ppos, size_t len, unsigned int flags) 772 { 773 struct address_space *mapping = out->f_mapping; 774 struct inode *inode = mapping->host; 775 struct splice_desc sd = { 776 .total_len = len, 777 .flags = flags, 778 .pos = *ppos, 779 .u.file = out, 780 }; 781 ssize_t ret; 782 int err; 783 784 err = remove_suid(out->f_path.dentry); 785 if (unlikely(err)) 786 return err; 787 788 ret = __splice_from_pipe(pipe, &sd, pipe_to_file); 789 if (ret > 0) { 790 unsigned long nr_pages; 791 792 *ppos += ret; 793 nr_pages = (ret + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 794 795 /* 796 * If file or inode is SYNC and we actually wrote some data, 797 * sync it. 798 */ 799 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) { 800 err = generic_osync_inode(inode, mapping, 801 OSYNC_METADATA|OSYNC_DATA); 802 803 if (err) 804 ret = err; 805 } 806 balance_dirty_pages_ratelimited_nr(mapping, nr_pages); 807 } 808 809 return ret; 810 } 811 812 EXPORT_SYMBOL(generic_file_splice_write_nolock); 813 814 /** 815 * generic_file_splice_write - splice data from a pipe to a file 816 * @pipe: pipe info 817 * @out: file to write to 818 * @ppos: position in @out 819 * @len: number of bytes to splice 820 * @flags: splice modifier flags 821 * 822 * Description: 823 * Will either move or copy pages (determined by @flags options) from 824 * the given pipe inode to the given file. 825 * 826 */ 827 ssize_t 828 generic_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 829 loff_t *ppos, size_t len, unsigned int flags) 830 { 831 struct address_space *mapping = out->f_mapping; 832 struct inode *inode = mapping->host; 833 int killsuid, killpriv; 834 ssize_t ret; 835 int err = 0; 836 837 killpriv = security_inode_need_killpriv(out->f_path.dentry); 838 killsuid = should_remove_suid(out->f_path.dentry); 839 if (unlikely(killsuid || killpriv)) { 840 mutex_lock(&inode->i_mutex); 841 if (killpriv) 842 err = security_inode_killpriv(out->f_path.dentry); 843 if (!err && killsuid) 844 err = __remove_suid(out->f_path.dentry, killsuid); 845 mutex_unlock(&inode->i_mutex); 846 if (err) 847 return err; 848 } 849 850 ret = splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_file); 851 if (ret > 0) { 852 unsigned long nr_pages; 853 854 *ppos += ret; 855 nr_pages = (ret + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 856 857 /* 858 * If file or inode is SYNC and we actually wrote some data, 859 * sync it. 860 */ 861 if (unlikely((out->f_flags & O_SYNC) || IS_SYNC(inode))) { 862 mutex_lock(&inode->i_mutex); 863 err = generic_osync_inode(inode, mapping, 864 OSYNC_METADATA|OSYNC_DATA); 865 mutex_unlock(&inode->i_mutex); 866 867 if (err) 868 ret = err; 869 } 870 balance_dirty_pages_ratelimited_nr(mapping, nr_pages); 871 } 872 873 return ret; 874 } 875 876 EXPORT_SYMBOL(generic_file_splice_write); 877 878 /** 879 * generic_splice_sendpage - splice data from a pipe to a socket 880 * @pipe: pipe to splice from 881 * @out: socket to write to 882 * @ppos: position in @out 883 * @len: number of bytes to splice 884 * @flags: splice modifier flags 885 * 886 * Description: 887 * Will send @len bytes from the pipe to a network socket. No data copying 888 * is involved. 889 * 890 */ 891 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 892 loff_t *ppos, size_t len, unsigned int flags) 893 { 894 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); 895 } 896 897 EXPORT_SYMBOL(generic_splice_sendpage); 898 899 /* 900 * Attempt to initiate a splice from pipe to file. 901 */ 902 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 903 loff_t *ppos, size_t len, unsigned int flags) 904 { 905 int ret; 906 907 if (unlikely(!out->f_op || !out->f_op->splice_write)) 908 return -EINVAL; 909 910 if (unlikely(!(out->f_mode & FMODE_WRITE))) 911 return -EBADF; 912 913 ret = rw_verify_area(WRITE, out, ppos, len); 914 if (unlikely(ret < 0)) 915 return ret; 916 917 return out->f_op->splice_write(pipe, out, ppos, len, flags); 918 } 919 920 /* 921 * Attempt to initiate a splice from a file to a pipe. 922 */ 923 static long do_splice_to(struct file *in, loff_t *ppos, 924 struct pipe_inode_info *pipe, size_t len, 925 unsigned int flags) 926 { 927 int ret; 928 929 if (unlikely(!in->f_op || !in->f_op->splice_read)) 930 return -EINVAL; 931 932 if (unlikely(!(in->f_mode & FMODE_READ))) 933 return -EBADF; 934 935 ret = rw_verify_area(READ, in, ppos, len); 936 if (unlikely(ret < 0)) 937 return ret; 938 939 return in->f_op->splice_read(in, ppos, pipe, len, flags); 940 } 941 942 /** 943 * splice_direct_to_actor - splices data directly between two non-pipes 944 * @in: file to splice from 945 * @sd: actor information on where to splice to 946 * @actor: handles the data splicing 947 * 948 * Description: 949 * This is a special case helper to splice directly between two 950 * points, without requiring an explicit pipe. Internally an allocated 951 * pipe is cached in the process, and reused during the lifetime of 952 * that process. 953 * 954 */ 955 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 956 splice_direct_actor *actor) 957 { 958 struct pipe_inode_info *pipe; 959 long ret, bytes; 960 umode_t i_mode; 961 size_t len; 962 int i, flags; 963 964 /* 965 * We require the input being a regular file, as we don't want to 966 * randomly drop data for eg socket -> socket splicing. Use the 967 * piped splicing for that! 968 */ 969 i_mode = in->f_path.dentry->d_inode->i_mode; 970 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) 971 return -EINVAL; 972 973 /* 974 * neither in nor out is a pipe, setup an internal pipe attached to 975 * 'out' and transfer the wanted data from 'in' to 'out' through that 976 */ 977 pipe = current->splice_pipe; 978 if (unlikely(!pipe)) { 979 pipe = alloc_pipe_info(NULL); 980 if (!pipe) 981 return -ENOMEM; 982 983 /* 984 * We don't have an immediate reader, but we'll read the stuff 985 * out of the pipe right after the splice_to_pipe(). So set 986 * PIPE_READERS appropriately. 987 */ 988 pipe->readers = 1; 989 990 current->splice_pipe = pipe; 991 } 992 993 /* 994 * Do the splice. 995 */ 996 ret = 0; 997 bytes = 0; 998 len = sd->total_len; 999 flags = sd->flags; 1000 1001 /* 1002 * Don't block on output, we have to drain the direct pipe. 1003 */ 1004 sd->flags &= ~SPLICE_F_NONBLOCK; 1005 1006 while (len) { 1007 size_t read_len; 1008 loff_t pos = sd->pos; 1009 1010 ret = do_splice_to(in, &pos, pipe, len, flags); 1011 if (unlikely(ret <= 0)) 1012 goto out_release; 1013 1014 read_len = ret; 1015 sd->total_len = read_len; 1016 1017 /* 1018 * NOTE: nonblocking mode only applies to the input. We 1019 * must not do the output in nonblocking mode as then we 1020 * could get stuck data in the internal pipe: 1021 */ 1022 ret = actor(pipe, sd); 1023 if (unlikely(ret <= 0)) 1024 goto out_release; 1025 1026 bytes += ret; 1027 len -= ret; 1028 sd->pos = pos; 1029 1030 if (ret < read_len) 1031 goto out_release; 1032 } 1033 1034 done: 1035 pipe->nrbufs = pipe->curbuf = 0; 1036 file_accessed(in); 1037 return bytes; 1038 1039 out_release: 1040 /* 1041 * If we did an incomplete transfer we must release 1042 * the pipe buffers in question: 1043 */ 1044 for (i = 0; i < PIPE_BUFFERS; i++) { 1045 struct pipe_buffer *buf = pipe->bufs + i; 1046 1047 if (buf->ops) { 1048 buf->ops->release(pipe, buf); 1049 buf->ops = NULL; 1050 } 1051 } 1052 1053 if (!bytes) 1054 bytes = ret; 1055 1056 goto done; 1057 } 1058 EXPORT_SYMBOL(splice_direct_to_actor); 1059 1060 static int direct_splice_actor(struct pipe_inode_info *pipe, 1061 struct splice_desc *sd) 1062 { 1063 struct file *file = sd->u.file; 1064 1065 return do_splice_from(pipe, file, &sd->pos, sd->total_len, sd->flags); 1066 } 1067 1068 /** 1069 * do_splice_direct - splices data directly between two files 1070 * @in: file to splice from 1071 * @ppos: input file offset 1072 * @out: file to splice to 1073 * @len: number of bytes to splice 1074 * @flags: splice modifier flags 1075 * 1076 * Description: 1077 * For use by do_sendfile(). splice can easily emulate sendfile, but 1078 * doing it in the application would incur an extra system call 1079 * (splice in + splice out, as compared to just sendfile()). So this helper 1080 * can splice directly through a process-private pipe. 1081 * 1082 */ 1083 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 1084 size_t len, unsigned int flags) 1085 { 1086 struct splice_desc sd = { 1087 .len = len, 1088 .total_len = len, 1089 .flags = flags, 1090 .pos = *ppos, 1091 .u.file = out, 1092 }; 1093 long ret; 1094 1095 ret = splice_direct_to_actor(in, &sd, direct_splice_actor); 1096 if (ret > 0) 1097 *ppos += ret; 1098 1099 return ret; 1100 } 1101 1102 /* 1103 * After the inode slimming patch, i_pipe/i_bdev/i_cdev share the same 1104 * location, so checking ->i_pipe is not enough to verify that this is a 1105 * pipe. 1106 */ 1107 static inline struct pipe_inode_info *pipe_info(struct inode *inode) 1108 { 1109 if (S_ISFIFO(inode->i_mode)) 1110 return inode->i_pipe; 1111 1112 return NULL; 1113 } 1114 1115 /* 1116 * Determine where to splice to/from. 1117 */ 1118 static long do_splice(struct file *in, loff_t __user *off_in, 1119 struct file *out, loff_t __user *off_out, 1120 size_t len, unsigned int flags) 1121 { 1122 struct pipe_inode_info *pipe; 1123 loff_t offset, *off; 1124 long ret; 1125 1126 pipe = pipe_info(in->f_path.dentry->d_inode); 1127 if (pipe) { 1128 if (off_in) 1129 return -ESPIPE; 1130 if (off_out) { 1131 if (out->f_op->llseek == no_llseek) 1132 return -EINVAL; 1133 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1134 return -EFAULT; 1135 off = &offset; 1136 } else 1137 off = &out->f_pos; 1138 1139 ret = do_splice_from(pipe, out, off, len, flags); 1140 1141 if (off_out && copy_to_user(off_out, off, sizeof(loff_t))) 1142 ret = -EFAULT; 1143 1144 return ret; 1145 } 1146 1147 pipe = pipe_info(out->f_path.dentry->d_inode); 1148 if (pipe) { 1149 if (off_out) 1150 return -ESPIPE; 1151 if (off_in) { 1152 if (in->f_op->llseek == no_llseek) 1153 return -EINVAL; 1154 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1155 return -EFAULT; 1156 off = &offset; 1157 } else 1158 off = &in->f_pos; 1159 1160 ret = do_splice_to(in, off, pipe, len, flags); 1161 1162 if (off_in && copy_to_user(off_in, off, sizeof(loff_t))) 1163 ret = -EFAULT; 1164 1165 return ret; 1166 } 1167 1168 return -EINVAL; 1169 } 1170 1171 /* 1172 * Do a copy-from-user while holding the mmap_semaphore for reading, in a 1173 * manner safe from deadlocking with simultaneous mmap() (grabbing mmap_sem 1174 * for writing) and page faulting on the user memory pointed to by src. 1175 * This assumes that we will very rarely hit the partial != 0 path, or this 1176 * will not be a win. 1177 */ 1178 static int copy_from_user_mmap_sem(void *dst, const void __user *src, size_t n) 1179 { 1180 int partial; 1181 1182 pagefault_disable(); 1183 partial = __copy_from_user_inatomic(dst, src, n); 1184 pagefault_enable(); 1185 1186 /* 1187 * Didn't copy everything, drop the mmap_sem and do a faulting copy 1188 */ 1189 if (unlikely(partial)) { 1190 up_read(¤t->mm->mmap_sem); 1191 partial = copy_from_user(dst, src, n); 1192 down_read(¤t->mm->mmap_sem); 1193 } 1194 1195 return partial; 1196 } 1197 1198 /* 1199 * Map an iov into an array of pages and offset/length tupples. With the 1200 * partial_page structure, we can map several non-contiguous ranges into 1201 * our ones pages[] map instead of splitting that operation into pieces. 1202 * Could easily be exported as a generic helper for other users, in which 1203 * case one would probably want to add a 'max_nr_pages' parameter as well. 1204 */ 1205 static int get_iovec_page_array(const struct iovec __user *iov, 1206 unsigned int nr_vecs, struct page **pages, 1207 struct partial_page *partial, int aligned) 1208 { 1209 int buffers = 0, error = 0; 1210 1211 down_read(¤t->mm->mmap_sem); 1212 1213 while (nr_vecs) { 1214 unsigned long off, npages; 1215 struct iovec entry; 1216 void __user *base; 1217 size_t len; 1218 int i; 1219 1220 error = -EFAULT; 1221 if (copy_from_user_mmap_sem(&entry, iov, sizeof(entry))) 1222 break; 1223 1224 base = entry.iov_base; 1225 len = entry.iov_len; 1226 1227 /* 1228 * Sanity check this iovec. 0 read succeeds. 1229 */ 1230 error = 0; 1231 if (unlikely(!len)) 1232 break; 1233 error = -EFAULT; 1234 if (unlikely(!base)) 1235 break; 1236 1237 /* 1238 * Get this base offset and number of pages, then map 1239 * in the user pages. 1240 */ 1241 off = (unsigned long) base & ~PAGE_MASK; 1242 1243 /* 1244 * If asked for alignment, the offset must be zero and the 1245 * length a multiple of the PAGE_SIZE. 1246 */ 1247 error = -EINVAL; 1248 if (aligned && (off || len & ~PAGE_MASK)) 1249 break; 1250 1251 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1252 if (npages > PIPE_BUFFERS - buffers) 1253 npages = PIPE_BUFFERS - buffers; 1254 1255 error = get_user_pages(current, current->mm, 1256 (unsigned long) base, npages, 0, 0, 1257 &pages[buffers], NULL); 1258 1259 if (unlikely(error <= 0)) 1260 break; 1261 1262 /* 1263 * Fill this contiguous range into the partial page map. 1264 */ 1265 for (i = 0; i < error; i++) { 1266 const int plen = min_t(size_t, len, PAGE_SIZE - off); 1267 1268 partial[buffers].offset = off; 1269 partial[buffers].len = plen; 1270 1271 off = 0; 1272 len -= plen; 1273 buffers++; 1274 } 1275 1276 /* 1277 * We didn't complete this iov, stop here since it probably 1278 * means we have to move some of this into a pipe to 1279 * be able to continue. 1280 */ 1281 if (len) 1282 break; 1283 1284 /* 1285 * Don't continue if we mapped fewer pages than we asked for, 1286 * or if we mapped the max number of pages that we have 1287 * room for. 1288 */ 1289 if (error < npages || buffers == PIPE_BUFFERS) 1290 break; 1291 1292 nr_vecs--; 1293 iov++; 1294 } 1295 1296 up_read(¤t->mm->mmap_sem); 1297 1298 if (buffers) 1299 return buffers; 1300 1301 return error; 1302 } 1303 1304 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1305 struct splice_desc *sd) 1306 { 1307 char *src; 1308 int ret; 1309 1310 ret = buf->ops->confirm(pipe, buf); 1311 if (unlikely(ret)) 1312 return ret; 1313 1314 /* 1315 * See if we can use the atomic maps, by prefaulting in the 1316 * pages and doing an atomic copy 1317 */ 1318 if (!fault_in_pages_writeable(sd->u.userptr, sd->len)) { 1319 src = buf->ops->map(pipe, buf, 1); 1320 ret = __copy_to_user_inatomic(sd->u.userptr, src + buf->offset, 1321 sd->len); 1322 buf->ops->unmap(pipe, buf, src); 1323 if (!ret) { 1324 ret = sd->len; 1325 goto out; 1326 } 1327 } 1328 1329 /* 1330 * No dice, use slow non-atomic map and copy 1331 */ 1332 src = buf->ops->map(pipe, buf, 0); 1333 1334 ret = sd->len; 1335 if (copy_to_user(sd->u.userptr, src + buf->offset, sd->len)) 1336 ret = -EFAULT; 1337 1338 buf->ops->unmap(pipe, buf, src); 1339 out: 1340 if (ret > 0) 1341 sd->u.userptr += ret; 1342 return ret; 1343 } 1344 1345 /* 1346 * For lack of a better implementation, implement vmsplice() to userspace 1347 * as a simple copy of the pipes pages to the user iov. 1348 */ 1349 static long vmsplice_to_user(struct file *file, const struct iovec __user *iov, 1350 unsigned long nr_segs, unsigned int flags) 1351 { 1352 struct pipe_inode_info *pipe; 1353 struct splice_desc sd; 1354 ssize_t size; 1355 int error; 1356 long ret; 1357 1358 pipe = pipe_info(file->f_path.dentry->d_inode); 1359 if (!pipe) 1360 return -EBADF; 1361 1362 if (pipe->inode) 1363 mutex_lock(&pipe->inode->i_mutex); 1364 1365 error = ret = 0; 1366 while (nr_segs) { 1367 void __user *base; 1368 size_t len; 1369 1370 /* 1371 * Get user address base and length for this iovec. 1372 */ 1373 error = get_user(base, &iov->iov_base); 1374 if (unlikely(error)) 1375 break; 1376 error = get_user(len, &iov->iov_len); 1377 if (unlikely(error)) 1378 break; 1379 1380 /* 1381 * Sanity check this iovec. 0 read succeeds. 1382 */ 1383 if (unlikely(!len)) 1384 break; 1385 if (unlikely(!base)) { 1386 error = -EFAULT; 1387 break; 1388 } 1389 1390 sd.len = 0; 1391 sd.total_len = len; 1392 sd.flags = flags; 1393 sd.u.userptr = base; 1394 sd.pos = 0; 1395 1396 size = __splice_from_pipe(pipe, &sd, pipe_to_user); 1397 if (size < 0) { 1398 if (!ret) 1399 ret = size; 1400 1401 break; 1402 } 1403 1404 ret += size; 1405 1406 if (size < len) 1407 break; 1408 1409 nr_segs--; 1410 iov++; 1411 } 1412 1413 if (pipe->inode) 1414 mutex_unlock(&pipe->inode->i_mutex); 1415 1416 if (!ret) 1417 ret = error; 1418 1419 return ret; 1420 } 1421 1422 /* 1423 * vmsplice splices a user address range into a pipe. It can be thought of 1424 * as splice-from-memory, where the regular splice is splice-from-file (or 1425 * to file). In both cases the output is a pipe, naturally. 1426 */ 1427 static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov, 1428 unsigned long nr_segs, unsigned int flags) 1429 { 1430 struct pipe_inode_info *pipe; 1431 struct page *pages[PIPE_BUFFERS]; 1432 struct partial_page partial[PIPE_BUFFERS]; 1433 struct splice_pipe_desc spd = { 1434 .pages = pages, 1435 .partial = partial, 1436 .flags = flags, 1437 .ops = &user_page_pipe_buf_ops, 1438 .spd_release = spd_release_page, 1439 }; 1440 1441 pipe = pipe_info(file->f_path.dentry->d_inode); 1442 if (!pipe) 1443 return -EBADF; 1444 1445 spd.nr_pages = get_iovec_page_array(iov, nr_segs, pages, partial, 1446 flags & SPLICE_F_GIFT); 1447 if (spd.nr_pages <= 0) 1448 return spd.nr_pages; 1449 1450 return splice_to_pipe(pipe, &spd); 1451 } 1452 1453 /* 1454 * Note that vmsplice only really supports true splicing _from_ user memory 1455 * to a pipe, not the other way around. Splicing from user memory is a simple 1456 * operation that can be supported without any funky alignment restrictions 1457 * or nasty vm tricks. We simply map in the user memory and fill them into 1458 * a pipe. The reverse isn't quite as easy, though. There are two possible 1459 * solutions for that: 1460 * 1461 * - memcpy() the data internally, at which point we might as well just 1462 * do a regular read() on the buffer anyway. 1463 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1464 * has restriction limitations on both ends of the pipe). 1465 * 1466 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1467 * 1468 */ 1469 asmlinkage long sys_vmsplice(int fd, const struct iovec __user *iov, 1470 unsigned long nr_segs, unsigned int flags) 1471 { 1472 struct file *file; 1473 long error; 1474 int fput; 1475 1476 if (unlikely(nr_segs > UIO_MAXIOV)) 1477 return -EINVAL; 1478 else if (unlikely(!nr_segs)) 1479 return 0; 1480 1481 error = -EBADF; 1482 file = fget_light(fd, &fput); 1483 if (file) { 1484 if (file->f_mode & FMODE_WRITE) 1485 error = vmsplice_to_pipe(file, iov, nr_segs, flags); 1486 else if (file->f_mode & FMODE_READ) 1487 error = vmsplice_to_user(file, iov, nr_segs, flags); 1488 1489 fput_light(file, fput); 1490 } 1491 1492 return error; 1493 } 1494 1495 asmlinkage long sys_splice(int fd_in, loff_t __user *off_in, 1496 int fd_out, loff_t __user *off_out, 1497 size_t len, unsigned int flags) 1498 { 1499 long error; 1500 struct file *in, *out; 1501 int fput_in, fput_out; 1502 1503 if (unlikely(!len)) 1504 return 0; 1505 1506 error = -EBADF; 1507 in = fget_light(fd_in, &fput_in); 1508 if (in) { 1509 if (in->f_mode & FMODE_READ) { 1510 out = fget_light(fd_out, &fput_out); 1511 if (out) { 1512 if (out->f_mode & FMODE_WRITE) 1513 error = do_splice(in, off_in, 1514 out, off_out, 1515 len, flags); 1516 fput_light(out, fput_out); 1517 } 1518 } 1519 1520 fput_light(in, fput_in); 1521 } 1522 1523 return error; 1524 } 1525 1526 /* 1527 * Make sure there's data to read. Wait for input if we can, otherwise 1528 * return an appropriate error. 1529 */ 1530 static int link_ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1531 { 1532 int ret; 1533 1534 /* 1535 * Check ->nrbufs without the inode lock first. This function 1536 * is speculative anyways, so missing one is ok. 1537 */ 1538 if (pipe->nrbufs) 1539 return 0; 1540 1541 ret = 0; 1542 mutex_lock(&pipe->inode->i_mutex); 1543 1544 while (!pipe->nrbufs) { 1545 if (signal_pending(current)) { 1546 ret = -ERESTARTSYS; 1547 break; 1548 } 1549 if (!pipe->writers) 1550 break; 1551 if (!pipe->waiting_writers) { 1552 if (flags & SPLICE_F_NONBLOCK) { 1553 ret = -EAGAIN; 1554 break; 1555 } 1556 } 1557 pipe_wait(pipe); 1558 } 1559 1560 mutex_unlock(&pipe->inode->i_mutex); 1561 return ret; 1562 } 1563 1564 /* 1565 * Make sure there's writeable room. Wait for room if we can, otherwise 1566 * return an appropriate error. 1567 */ 1568 static int link_opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1569 { 1570 int ret; 1571 1572 /* 1573 * Check ->nrbufs without the inode lock first. This function 1574 * is speculative anyways, so missing one is ok. 1575 */ 1576 if (pipe->nrbufs < PIPE_BUFFERS) 1577 return 0; 1578 1579 ret = 0; 1580 mutex_lock(&pipe->inode->i_mutex); 1581 1582 while (pipe->nrbufs >= PIPE_BUFFERS) { 1583 if (!pipe->readers) { 1584 send_sig(SIGPIPE, current, 0); 1585 ret = -EPIPE; 1586 break; 1587 } 1588 if (flags & SPLICE_F_NONBLOCK) { 1589 ret = -EAGAIN; 1590 break; 1591 } 1592 if (signal_pending(current)) { 1593 ret = -ERESTARTSYS; 1594 break; 1595 } 1596 pipe->waiting_writers++; 1597 pipe_wait(pipe); 1598 pipe->waiting_writers--; 1599 } 1600 1601 mutex_unlock(&pipe->inode->i_mutex); 1602 return ret; 1603 } 1604 1605 /* 1606 * Link contents of ipipe to opipe. 1607 */ 1608 static int link_pipe(struct pipe_inode_info *ipipe, 1609 struct pipe_inode_info *opipe, 1610 size_t len, unsigned int flags) 1611 { 1612 struct pipe_buffer *ibuf, *obuf; 1613 int ret = 0, i = 0, nbuf; 1614 1615 /* 1616 * Potential ABBA deadlock, work around it by ordering lock 1617 * grabbing by inode address. Otherwise two different processes 1618 * could deadlock (one doing tee from A -> B, the other from B -> A). 1619 */ 1620 inode_double_lock(ipipe->inode, opipe->inode); 1621 1622 do { 1623 if (!opipe->readers) { 1624 send_sig(SIGPIPE, current, 0); 1625 if (!ret) 1626 ret = -EPIPE; 1627 break; 1628 } 1629 1630 /* 1631 * If we have iterated all input buffers or ran out of 1632 * output room, break. 1633 */ 1634 if (i >= ipipe->nrbufs || opipe->nrbufs >= PIPE_BUFFERS) 1635 break; 1636 1637 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (PIPE_BUFFERS - 1)); 1638 nbuf = (opipe->curbuf + opipe->nrbufs) & (PIPE_BUFFERS - 1); 1639 1640 /* 1641 * Get a reference to this pipe buffer, 1642 * so we can copy the contents over. 1643 */ 1644 ibuf->ops->get(ipipe, ibuf); 1645 1646 obuf = opipe->bufs + nbuf; 1647 *obuf = *ibuf; 1648 1649 /* 1650 * Don't inherit the gift flag, we need to 1651 * prevent multiple steals of this page. 1652 */ 1653 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1654 1655 if (obuf->len > len) 1656 obuf->len = len; 1657 1658 opipe->nrbufs++; 1659 ret += obuf->len; 1660 len -= obuf->len; 1661 i++; 1662 } while (len); 1663 1664 inode_double_unlock(ipipe->inode, opipe->inode); 1665 1666 /* 1667 * If we put data in the output pipe, wakeup any potential readers. 1668 */ 1669 if (ret > 0) { 1670 smp_mb(); 1671 if (waitqueue_active(&opipe->wait)) 1672 wake_up_interruptible(&opipe->wait); 1673 kill_fasync(&opipe->fasync_readers, SIGIO, POLL_IN); 1674 } 1675 1676 return ret; 1677 } 1678 1679 /* 1680 * This is a tee(1) implementation that works on pipes. It doesn't copy 1681 * any data, it simply references the 'in' pages on the 'out' pipe. 1682 * The 'flags' used are the SPLICE_F_* variants, currently the only 1683 * applicable one is SPLICE_F_NONBLOCK. 1684 */ 1685 static long do_tee(struct file *in, struct file *out, size_t len, 1686 unsigned int flags) 1687 { 1688 struct pipe_inode_info *ipipe = pipe_info(in->f_path.dentry->d_inode); 1689 struct pipe_inode_info *opipe = pipe_info(out->f_path.dentry->d_inode); 1690 int ret = -EINVAL; 1691 1692 /* 1693 * Duplicate the contents of ipipe to opipe without actually 1694 * copying the data. 1695 */ 1696 if (ipipe && opipe && ipipe != opipe) { 1697 /* 1698 * Keep going, unless we encounter an error. The ipipe/opipe 1699 * ordering doesn't really matter. 1700 */ 1701 ret = link_ipipe_prep(ipipe, flags); 1702 if (!ret) { 1703 ret = link_opipe_prep(opipe, flags); 1704 if (!ret) { 1705 ret = link_pipe(ipipe, opipe, len, flags); 1706 if (!ret && (flags & SPLICE_F_NONBLOCK)) 1707 ret = -EAGAIN; 1708 } 1709 } 1710 } 1711 1712 return ret; 1713 } 1714 1715 asmlinkage long sys_tee(int fdin, int fdout, size_t len, unsigned int flags) 1716 { 1717 struct file *in; 1718 int error, fput_in; 1719 1720 if (unlikely(!len)) 1721 return 0; 1722 1723 error = -EBADF; 1724 in = fget_light(fdin, &fput_in); 1725 if (in) { 1726 if (in->f_mode & FMODE_READ) { 1727 int fput_out; 1728 struct file *out = fget_light(fdout, &fput_out); 1729 1730 if (out) { 1731 if (out->f_mode & FMODE_WRITE) 1732 error = do_tee(in, out, len, flags); 1733 fput_light(out, fput_out); 1734 } 1735 } 1736 fput_light(in, fput_in); 1737 } 1738 1739 return error; 1740 } 1741