1 /* 2 * "splice": joining two ropes together by interweaving their strands. 3 * 4 * This is the "extended pipe" functionality, where a pipe is used as 5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 6 * buffer that you can use to transfer data from one end to the other. 7 * 8 * The traditional unix read/write is extended with a "splice()" operation 9 * that transfers data buffers to or from a pipe buffer. 10 * 11 * Named by Larry McVoy, original implementation from Linus, extended by 12 * Jens to support splicing to files, network, direct splicing, etc and 13 * fixing lots of bugs. 14 * 15 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 18 * 19 */ 20 #include <linux/bvec.h> 21 #include <linux/fs.h> 22 #include <linux/file.h> 23 #include <linux/pagemap.h> 24 #include <linux/splice.h> 25 #include <linux/memcontrol.h> 26 #include <linux/mm_inline.h> 27 #include <linux/swap.h> 28 #include <linux/writeback.h> 29 #include <linux/export.h> 30 #include <linux/syscalls.h> 31 #include <linux/uio.h> 32 #include <linux/security.h> 33 #include <linux/gfp.h> 34 #include <linux/socket.h> 35 #include <linux/compat.h> 36 #include "internal.h" 37 38 /* 39 * Attempt to steal a page from a pipe buffer. This should perhaps go into 40 * a vm helper function, it's already simplified quite a bit by the 41 * addition of remove_mapping(). If success is returned, the caller may 42 * attempt to reuse this page for another destination. 43 */ 44 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, 45 struct pipe_buffer *buf) 46 { 47 struct page *page = buf->page; 48 struct address_space *mapping; 49 50 lock_page(page); 51 52 mapping = page_mapping(page); 53 if (mapping) { 54 WARN_ON(!PageUptodate(page)); 55 56 /* 57 * At least for ext2 with nobh option, we need to wait on 58 * writeback completing on this page, since we'll remove it 59 * from the pagecache. Otherwise truncate wont wait on the 60 * page, allowing the disk blocks to be reused by someone else 61 * before we actually wrote our data to them. fs corruption 62 * ensues. 63 */ 64 wait_on_page_writeback(page); 65 66 if (page_has_private(page) && 67 !try_to_release_page(page, GFP_KERNEL)) 68 goto out_unlock; 69 70 /* 71 * If we succeeded in removing the mapping, set LRU flag 72 * and return good. 73 */ 74 if (remove_mapping(mapping, page)) { 75 buf->flags |= PIPE_BUF_FLAG_LRU; 76 return 0; 77 } 78 } 79 80 /* 81 * Raced with truncate or failed to remove page from current 82 * address space, unlock and return failure. 83 */ 84 out_unlock: 85 unlock_page(page); 86 return 1; 87 } 88 89 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, 90 struct pipe_buffer *buf) 91 { 92 put_page(buf->page); 93 buf->flags &= ~PIPE_BUF_FLAG_LRU; 94 } 95 96 /* 97 * Check whether the contents of buf is OK to access. Since the content 98 * is a page cache page, IO may be in flight. 99 */ 100 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 101 struct pipe_buffer *buf) 102 { 103 struct page *page = buf->page; 104 int err; 105 106 if (!PageUptodate(page)) { 107 lock_page(page); 108 109 /* 110 * Page got truncated/unhashed. This will cause a 0-byte 111 * splice, if this is the first page. 112 */ 113 if (!page->mapping) { 114 err = -ENODATA; 115 goto error; 116 } 117 118 /* 119 * Uh oh, read-error from disk. 120 */ 121 if (!PageUptodate(page)) { 122 err = -EIO; 123 goto error; 124 } 125 126 /* 127 * Page is ok afterall, we are done. 128 */ 129 unlock_page(page); 130 } 131 132 return 0; 133 error: 134 unlock_page(page); 135 return err; 136 } 137 138 const struct pipe_buf_operations page_cache_pipe_buf_ops = { 139 .can_merge = 0, 140 .confirm = page_cache_pipe_buf_confirm, 141 .release = page_cache_pipe_buf_release, 142 .steal = page_cache_pipe_buf_steal, 143 .get = generic_pipe_buf_get, 144 }; 145 146 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, 147 struct pipe_buffer *buf) 148 { 149 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) 150 return 1; 151 152 buf->flags |= PIPE_BUF_FLAG_LRU; 153 return generic_pipe_buf_steal(pipe, buf); 154 } 155 156 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 157 .can_merge = 0, 158 .confirm = generic_pipe_buf_confirm, 159 .release = page_cache_pipe_buf_release, 160 .steal = user_page_pipe_buf_steal, 161 .get = generic_pipe_buf_get, 162 }; 163 164 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) 165 { 166 smp_mb(); 167 if (waitqueue_active(&pipe->wait)) 168 wake_up_interruptible(&pipe->wait); 169 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 170 } 171 172 /** 173 * splice_to_pipe - fill passed data into a pipe 174 * @pipe: pipe to fill 175 * @spd: data to fill 176 * 177 * Description: 178 * @spd contains a map of pages and len/offset tuples, along with 179 * the struct pipe_buf_operations associated with these pages. This 180 * function will link that data to the pipe. 181 * 182 */ 183 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 184 struct splice_pipe_desc *spd) 185 { 186 unsigned int spd_pages = spd->nr_pages; 187 int ret = 0, page_nr = 0; 188 189 if (!spd_pages) 190 return 0; 191 192 if (unlikely(!pipe->readers)) { 193 send_sig(SIGPIPE, current, 0); 194 ret = -EPIPE; 195 goto out; 196 } 197 198 while (pipe->nrbufs < pipe->buffers) { 199 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 200 struct pipe_buffer *buf = pipe->bufs + newbuf; 201 202 buf->page = spd->pages[page_nr]; 203 buf->offset = spd->partial[page_nr].offset; 204 buf->len = spd->partial[page_nr].len; 205 buf->private = spd->partial[page_nr].private; 206 buf->ops = spd->ops; 207 208 pipe->nrbufs++; 209 page_nr++; 210 ret += buf->len; 211 212 if (!--spd->nr_pages) 213 break; 214 } 215 216 if (!ret) 217 ret = -EAGAIN; 218 219 out: 220 while (page_nr < spd_pages) 221 spd->spd_release(spd, page_nr++); 222 223 return ret; 224 } 225 EXPORT_SYMBOL_GPL(splice_to_pipe); 226 227 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 228 { 229 int ret; 230 231 if (unlikely(!pipe->readers)) { 232 send_sig(SIGPIPE, current, 0); 233 ret = -EPIPE; 234 } else if (pipe->nrbufs == pipe->buffers) { 235 ret = -EAGAIN; 236 } else { 237 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 238 pipe->bufs[newbuf] = *buf; 239 pipe->nrbufs++; 240 return buf->len; 241 } 242 pipe_buf_release(pipe, buf); 243 return ret; 244 } 245 EXPORT_SYMBOL(add_to_pipe); 246 247 void spd_release_page(struct splice_pipe_desc *spd, unsigned int i) 248 { 249 put_page(spd->pages[i]); 250 } 251 252 /* 253 * Check if we need to grow the arrays holding pages and partial page 254 * descriptions. 255 */ 256 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) 257 { 258 unsigned int buffers = ACCESS_ONCE(pipe->buffers); 259 260 spd->nr_pages_max = buffers; 261 if (buffers <= PIPE_DEF_BUFFERS) 262 return 0; 263 264 spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL); 265 spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL); 266 267 if (spd->pages && spd->partial) 268 return 0; 269 270 kfree(spd->pages); 271 kfree(spd->partial); 272 return -ENOMEM; 273 } 274 275 void splice_shrink_spd(struct splice_pipe_desc *spd) 276 { 277 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) 278 return; 279 280 kfree(spd->pages); 281 kfree(spd->partial); 282 } 283 284 /** 285 * generic_file_splice_read - splice data from file to a pipe 286 * @in: file to splice from 287 * @ppos: position in @in 288 * @pipe: pipe to splice to 289 * @len: number of bytes to splice 290 * @flags: splice modifier flags 291 * 292 * Description: 293 * Will read pages from given file and fill them into a pipe. Can be 294 * used as long as it has more or less sane ->read_iter(). 295 * 296 */ 297 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, 298 struct pipe_inode_info *pipe, size_t len, 299 unsigned int flags) 300 { 301 struct iov_iter to; 302 struct kiocb kiocb; 303 int idx, ret; 304 305 iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len); 306 idx = to.idx; 307 init_sync_kiocb(&kiocb, in); 308 kiocb.ki_pos = *ppos; 309 ret = in->f_op->read_iter(&kiocb, &to); 310 if (ret > 0) { 311 *ppos = kiocb.ki_pos; 312 file_accessed(in); 313 } else if (ret < 0) { 314 to.idx = idx; 315 to.iov_offset = 0; 316 iov_iter_advance(&to, 0); /* to free what was emitted */ 317 /* 318 * callers of ->splice_read() expect -EAGAIN on 319 * "can't put anything in there", rather than -EFAULT. 320 */ 321 if (ret == -EFAULT) 322 ret = -EAGAIN; 323 } 324 325 return ret; 326 } 327 EXPORT_SYMBOL(generic_file_splice_read); 328 329 const struct pipe_buf_operations default_pipe_buf_ops = { 330 .can_merge = 0, 331 .confirm = generic_pipe_buf_confirm, 332 .release = generic_pipe_buf_release, 333 .steal = generic_pipe_buf_steal, 334 .get = generic_pipe_buf_get, 335 }; 336 337 static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe, 338 struct pipe_buffer *buf) 339 { 340 return 1; 341 } 342 343 /* Pipe buffer operations for a socket and similar. */ 344 const struct pipe_buf_operations nosteal_pipe_buf_ops = { 345 .can_merge = 0, 346 .confirm = generic_pipe_buf_confirm, 347 .release = generic_pipe_buf_release, 348 .steal = generic_pipe_buf_nosteal, 349 .get = generic_pipe_buf_get, 350 }; 351 EXPORT_SYMBOL(nosteal_pipe_buf_ops); 352 353 static ssize_t kernel_readv(struct file *file, const struct kvec *vec, 354 unsigned long vlen, loff_t offset) 355 { 356 mm_segment_t old_fs; 357 loff_t pos = offset; 358 ssize_t res; 359 360 old_fs = get_fs(); 361 set_fs(get_ds()); 362 /* The cast to a user pointer is valid due to the set_fs() */ 363 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0); 364 set_fs(old_fs); 365 366 return res; 367 } 368 369 ssize_t kernel_write(struct file *file, const char *buf, size_t count, 370 loff_t pos) 371 { 372 mm_segment_t old_fs; 373 ssize_t res; 374 375 old_fs = get_fs(); 376 set_fs(get_ds()); 377 /* The cast to a user pointer is valid due to the set_fs() */ 378 res = vfs_write(file, (__force const char __user *)buf, count, &pos); 379 set_fs(old_fs); 380 381 return res; 382 } 383 EXPORT_SYMBOL(kernel_write); 384 385 static ssize_t default_file_splice_read(struct file *in, loff_t *ppos, 386 struct pipe_inode_info *pipe, size_t len, 387 unsigned int flags) 388 { 389 struct kvec *vec, __vec[PIPE_DEF_BUFFERS]; 390 struct iov_iter to; 391 struct page **pages; 392 unsigned int nr_pages; 393 size_t offset, dummy, copied = 0; 394 ssize_t res; 395 int i; 396 397 if (pipe->nrbufs == pipe->buffers) 398 return -EAGAIN; 399 400 /* 401 * Try to keep page boundaries matching to source pagecache ones - 402 * it probably won't be much help, but... 403 */ 404 offset = *ppos & ~PAGE_MASK; 405 406 iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len + offset); 407 408 res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &dummy); 409 if (res <= 0) 410 return -ENOMEM; 411 412 BUG_ON(dummy); 413 nr_pages = DIV_ROUND_UP(res, PAGE_SIZE); 414 415 vec = __vec; 416 if (nr_pages > PIPE_DEF_BUFFERS) { 417 vec = kmalloc(nr_pages * sizeof(struct kvec), GFP_KERNEL); 418 if (unlikely(!vec)) { 419 res = -ENOMEM; 420 goto out; 421 } 422 } 423 424 pipe->bufs[to.idx].offset = offset; 425 pipe->bufs[to.idx].len -= offset; 426 427 for (i = 0; i < nr_pages; i++) { 428 size_t this_len = min_t(size_t, len, PAGE_SIZE - offset); 429 vec[i].iov_base = page_address(pages[i]) + offset; 430 vec[i].iov_len = this_len; 431 len -= this_len; 432 offset = 0; 433 } 434 435 res = kernel_readv(in, vec, nr_pages, *ppos); 436 if (res > 0) { 437 copied = res; 438 *ppos += res; 439 } 440 441 if (vec != __vec) 442 kfree(vec); 443 out: 444 for (i = 0; i < nr_pages; i++) 445 put_page(pages[i]); 446 kvfree(pages); 447 iov_iter_advance(&to, copied); /* truncates and discards */ 448 return res; 449 } 450 451 /* 452 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 453 * using sendpage(). Return the number of bytes sent. 454 */ 455 static int pipe_to_sendpage(struct pipe_inode_info *pipe, 456 struct pipe_buffer *buf, struct splice_desc *sd) 457 { 458 struct file *file = sd->u.file; 459 loff_t pos = sd->pos; 460 int more; 461 462 if (!likely(file->f_op->sendpage)) 463 return -EINVAL; 464 465 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; 466 467 if (sd->len < sd->total_len && pipe->nrbufs > 1) 468 more |= MSG_SENDPAGE_NOTLAST; 469 470 return file->f_op->sendpage(file, buf->page, buf->offset, 471 sd->len, &pos, more); 472 } 473 474 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) 475 { 476 smp_mb(); 477 if (waitqueue_active(&pipe->wait)) 478 wake_up_interruptible(&pipe->wait); 479 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 480 } 481 482 /** 483 * splice_from_pipe_feed - feed available data from a pipe to a file 484 * @pipe: pipe to splice from 485 * @sd: information to @actor 486 * @actor: handler that splices the data 487 * 488 * Description: 489 * This function loops over the pipe and calls @actor to do the 490 * actual moving of a single struct pipe_buffer to the desired 491 * destination. It returns when there's no more buffers left in 492 * the pipe or if the requested number of bytes (@sd->total_len) 493 * have been copied. It returns a positive number (one) if the 494 * pipe needs to be filled with more data, zero if the required 495 * number of bytes have been copied and -errno on error. 496 * 497 * This, together with splice_from_pipe_{begin,end,next}, may be 498 * used to implement the functionality of __splice_from_pipe() when 499 * locking is required around copying the pipe buffers to the 500 * destination. 501 */ 502 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, 503 splice_actor *actor) 504 { 505 int ret; 506 507 while (pipe->nrbufs) { 508 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 509 510 sd->len = buf->len; 511 if (sd->len > sd->total_len) 512 sd->len = sd->total_len; 513 514 ret = pipe_buf_confirm(pipe, buf); 515 if (unlikely(ret)) { 516 if (ret == -ENODATA) 517 ret = 0; 518 return ret; 519 } 520 521 ret = actor(pipe, buf, sd); 522 if (ret <= 0) 523 return ret; 524 525 buf->offset += ret; 526 buf->len -= ret; 527 528 sd->num_spliced += ret; 529 sd->len -= ret; 530 sd->pos += ret; 531 sd->total_len -= ret; 532 533 if (!buf->len) { 534 pipe_buf_release(pipe, buf); 535 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 536 pipe->nrbufs--; 537 if (pipe->files) 538 sd->need_wakeup = true; 539 } 540 541 if (!sd->total_len) 542 return 0; 543 } 544 545 return 1; 546 } 547 548 /** 549 * splice_from_pipe_next - wait for some data to splice from 550 * @pipe: pipe to splice from 551 * @sd: information about the splice operation 552 * 553 * Description: 554 * This function will wait for some data and return a positive 555 * value (one) if pipe buffers are available. It will return zero 556 * or -errno if no more data needs to be spliced. 557 */ 558 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) 559 { 560 /* 561 * Check for signal early to make process killable when there are 562 * always buffers available 563 */ 564 if (signal_pending(current)) 565 return -ERESTARTSYS; 566 567 while (!pipe->nrbufs) { 568 if (!pipe->writers) 569 return 0; 570 571 if (!pipe->waiting_writers && sd->num_spliced) 572 return 0; 573 574 if (sd->flags & SPLICE_F_NONBLOCK) 575 return -EAGAIN; 576 577 if (signal_pending(current)) 578 return -ERESTARTSYS; 579 580 if (sd->need_wakeup) { 581 wakeup_pipe_writers(pipe); 582 sd->need_wakeup = false; 583 } 584 585 pipe_wait(pipe); 586 } 587 588 return 1; 589 } 590 591 /** 592 * splice_from_pipe_begin - start splicing from pipe 593 * @sd: information about the splice operation 594 * 595 * Description: 596 * This function should be called before a loop containing 597 * splice_from_pipe_next() and splice_from_pipe_feed() to 598 * initialize the necessary fields of @sd. 599 */ 600 static void splice_from_pipe_begin(struct splice_desc *sd) 601 { 602 sd->num_spliced = 0; 603 sd->need_wakeup = false; 604 } 605 606 /** 607 * splice_from_pipe_end - finish splicing from pipe 608 * @pipe: pipe to splice from 609 * @sd: information about the splice operation 610 * 611 * Description: 612 * This function will wake up pipe writers if necessary. It should 613 * be called after a loop containing splice_from_pipe_next() and 614 * splice_from_pipe_feed(). 615 */ 616 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) 617 { 618 if (sd->need_wakeup) 619 wakeup_pipe_writers(pipe); 620 } 621 622 /** 623 * __splice_from_pipe - splice data from a pipe to given actor 624 * @pipe: pipe to splice from 625 * @sd: information to @actor 626 * @actor: handler that splices the data 627 * 628 * Description: 629 * This function does little more than loop over the pipe and call 630 * @actor to do the actual moving of a single struct pipe_buffer to 631 * the desired destination. See pipe_to_file, pipe_to_sendpage, or 632 * pipe_to_user. 633 * 634 */ 635 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 636 splice_actor *actor) 637 { 638 int ret; 639 640 splice_from_pipe_begin(sd); 641 do { 642 cond_resched(); 643 ret = splice_from_pipe_next(pipe, sd); 644 if (ret > 0) 645 ret = splice_from_pipe_feed(pipe, sd, actor); 646 } while (ret > 0); 647 splice_from_pipe_end(pipe, sd); 648 649 return sd->num_spliced ? sd->num_spliced : ret; 650 } 651 EXPORT_SYMBOL(__splice_from_pipe); 652 653 /** 654 * splice_from_pipe - splice data from a pipe to a file 655 * @pipe: pipe to splice from 656 * @out: file to splice to 657 * @ppos: position in @out 658 * @len: how many bytes to splice 659 * @flags: splice modifier flags 660 * @actor: handler that splices the data 661 * 662 * Description: 663 * See __splice_from_pipe. This function locks the pipe inode, 664 * otherwise it's identical to __splice_from_pipe(). 665 * 666 */ 667 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 668 loff_t *ppos, size_t len, unsigned int flags, 669 splice_actor *actor) 670 { 671 ssize_t ret; 672 struct splice_desc sd = { 673 .total_len = len, 674 .flags = flags, 675 .pos = *ppos, 676 .u.file = out, 677 }; 678 679 pipe_lock(pipe); 680 ret = __splice_from_pipe(pipe, &sd, actor); 681 pipe_unlock(pipe); 682 683 return ret; 684 } 685 686 /** 687 * iter_file_splice_write - splice data from a pipe to a file 688 * @pipe: pipe info 689 * @out: file to write to 690 * @ppos: position in @out 691 * @len: number of bytes to splice 692 * @flags: splice modifier flags 693 * 694 * Description: 695 * Will either move or copy pages (determined by @flags options) from 696 * the given pipe inode to the given file. 697 * This one is ->write_iter-based. 698 * 699 */ 700 ssize_t 701 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 702 loff_t *ppos, size_t len, unsigned int flags) 703 { 704 struct splice_desc sd = { 705 .total_len = len, 706 .flags = flags, 707 .pos = *ppos, 708 .u.file = out, 709 }; 710 int nbufs = pipe->buffers; 711 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec), 712 GFP_KERNEL); 713 ssize_t ret; 714 715 if (unlikely(!array)) 716 return -ENOMEM; 717 718 pipe_lock(pipe); 719 720 splice_from_pipe_begin(&sd); 721 while (sd.total_len) { 722 struct iov_iter from; 723 size_t left; 724 int n, idx; 725 726 ret = splice_from_pipe_next(pipe, &sd); 727 if (ret <= 0) 728 break; 729 730 if (unlikely(nbufs < pipe->buffers)) { 731 kfree(array); 732 nbufs = pipe->buffers; 733 array = kcalloc(nbufs, sizeof(struct bio_vec), 734 GFP_KERNEL); 735 if (!array) { 736 ret = -ENOMEM; 737 break; 738 } 739 } 740 741 /* build the vector */ 742 left = sd.total_len; 743 for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) { 744 struct pipe_buffer *buf = pipe->bufs + idx; 745 size_t this_len = buf->len; 746 747 if (this_len > left) 748 this_len = left; 749 750 if (idx == pipe->buffers - 1) 751 idx = -1; 752 753 ret = pipe_buf_confirm(pipe, buf); 754 if (unlikely(ret)) { 755 if (ret == -ENODATA) 756 ret = 0; 757 goto done; 758 } 759 760 array[n].bv_page = buf->page; 761 array[n].bv_len = this_len; 762 array[n].bv_offset = buf->offset; 763 left -= this_len; 764 } 765 766 iov_iter_bvec(&from, ITER_BVEC | WRITE, array, n, 767 sd.total_len - left); 768 ret = vfs_iter_write(out, &from, &sd.pos); 769 if (ret <= 0) 770 break; 771 772 sd.num_spliced += ret; 773 sd.total_len -= ret; 774 *ppos = sd.pos; 775 776 /* dismiss the fully eaten buffers, adjust the partial one */ 777 while (ret) { 778 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 779 if (ret >= buf->len) { 780 ret -= buf->len; 781 buf->len = 0; 782 pipe_buf_release(pipe, buf); 783 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 784 pipe->nrbufs--; 785 if (pipe->files) 786 sd.need_wakeup = true; 787 } else { 788 buf->offset += ret; 789 buf->len -= ret; 790 ret = 0; 791 } 792 } 793 } 794 done: 795 kfree(array); 796 splice_from_pipe_end(pipe, &sd); 797 798 pipe_unlock(pipe); 799 800 if (sd.num_spliced) 801 ret = sd.num_spliced; 802 803 return ret; 804 } 805 806 EXPORT_SYMBOL(iter_file_splice_write); 807 808 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 809 struct splice_desc *sd) 810 { 811 int ret; 812 void *data; 813 loff_t tmp = sd->pos; 814 815 data = kmap(buf->page); 816 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp); 817 kunmap(buf->page); 818 819 return ret; 820 } 821 822 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, 823 struct file *out, loff_t *ppos, 824 size_t len, unsigned int flags) 825 { 826 ssize_t ret; 827 828 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); 829 if (ret > 0) 830 *ppos += ret; 831 832 return ret; 833 } 834 835 /** 836 * generic_splice_sendpage - splice data from a pipe to a socket 837 * @pipe: pipe to splice from 838 * @out: socket to write to 839 * @ppos: position in @out 840 * @len: number of bytes to splice 841 * @flags: splice modifier flags 842 * 843 * Description: 844 * Will send @len bytes from the pipe to a network socket. No data copying 845 * is involved. 846 * 847 */ 848 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 849 loff_t *ppos, size_t len, unsigned int flags) 850 { 851 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); 852 } 853 854 EXPORT_SYMBOL(generic_splice_sendpage); 855 856 /* 857 * Attempt to initiate a splice from pipe to file. 858 */ 859 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 860 loff_t *ppos, size_t len, unsigned int flags) 861 { 862 ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, 863 loff_t *, size_t, unsigned int); 864 865 if (out->f_op->splice_write) 866 splice_write = out->f_op->splice_write; 867 else 868 splice_write = default_file_splice_write; 869 870 return splice_write(pipe, out, ppos, len, flags); 871 } 872 873 /* 874 * Attempt to initiate a splice from a file to a pipe. 875 */ 876 static long do_splice_to(struct file *in, loff_t *ppos, 877 struct pipe_inode_info *pipe, size_t len, 878 unsigned int flags) 879 { 880 ssize_t (*splice_read)(struct file *, loff_t *, 881 struct pipe_inode_info *, size_t, unsigned int); 882 int ret; 883 884 if (unlikely(!(in->f_mode & FMODE_READ))) 885 return -EBADF; 886 887 ret = rw_verify_area(READ, in, ppos, len); 888 if (unlikely(ret < 0)) 889 return ret; 890 891 if (unlikely(len > MAX_RW_COUNT)) 892 len = MAX_RW_COUNT; 893 894 if (in->f_op->splice_read) 895 splice_read = in->f_op->splice_read; 896 else 897 splice_read = default_file_splice_read; 898 899 return splice_read(in, ppos, pipe, len, flags); 900 } 901 902 /** 903 * splice_direct_to_actor - splices data directly between two non-pipes 904 * @in: file to splice from 905 * @sd: actor information on where to splice to 906 * @actor: handles the data splicing 907 * 908 * Description: 909 * This is a special case helper to splice directly between two 910 * points, without requiring an explicit pipe. Internally an allocated 911 * pipe is cached in the process, and reused during the lifetime of 912 * that process. 913 * 914 */ 915 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 916 splice_direct_actor *actor) 917 { 918 struct pipe_inode_info *pipe; 919 long ret, bytes; 920 umode_t i_mode; 921 size_t len; 922 int i, flags, more; 923 924 /* 925 * We require the input being a regular file, as we don't want to 926 * randomly drop data for eg socket -> socket splicing. Use the 927 * piped splicing for that! 928 */ 929 i_mode = file_inode(in)->i_mode; 930 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) 931 return -EINVAL; 932 933 /* 934 * neither in nor out is a pipe, setup an internal pipe attached to 935 * 'out' and transfer the wanted data from 'in' to 'out' through that 936 */ 937 pipe = current->splice_pipe; 938 if (unlikely(!pipe)) { 939 pipe = alloc_pipe_info(); 940 if (!pipe) 941 return -ENOMEM; 942 943 /* 944 * We don't have an immediate reader, but we'll read the stuff 945 * out of the pipe right after the splice_to_pipe(). So set 946 * PIPE_READERS appropriately. 947 */ 948 pipe->readers = 1; 949 950 current->splice_pipe = pipe; 951 } 952 953 /* 954 * Do the splice. 955 */ 956 ret = 0; 957 bytes = 0; 958 len = sd->total_len; 959 flags = sd->flags; 960 961 /* 962 * Don't block on output, we have to drain the direct pipe. 963 */ 964 sd->flags &= ~SPLICE_F_NONBLOCK; 965 more = sd->flags & SPLICE_F_MORE; 966 967 while (len) { 968 size_t read_len; 969 loff_t pos = sd->pos, prev_pos = pos; 970 971 ret = do_splice_to(in, &pos, pipe, len, flags); 972 if (unlikely(ret <= 0)) 973 goto out_release; 974 975 read_len = ret; 976 sd->total_len = read_len; 977 978 /* 979 * If more data is pending, set SPLICE_F_MORE 980 * If this is the last data and SPLICE_F_MORE was not set 981 * initially, clears it. 982 */ 983 if (read_len < len) 984 sd->flags |= SPLICE_F_MORE; 985 else if (!more) 986 sd->flags &= ~SPLICE_F_MORE; 987 /* 988 * NOTE: nonblocking mode only applies to the input. We 989 * must not do the output in nonblocking mode as then we 990 * could get stuck data in the internal pipe: 991 */ 992 ret = actor(pipe, sd); 993 if (unlikely(ret <= 0)) { 994 sd->pos = prev_pos; 995 goto out_release; 996 } 997 998 bytes += ret; 999 len -= ret; 1000 sd->pos = pos; 1001 1002 if (ret < read_len) { 1003 sd->pos = prev_pos + ret; 1004 goto out_release; 1005 } 1006 } 1007 1008 done: 1009 pipe->nrbufs = pipe->curbuf = 0; 1010 file_accessed(in); 1011 return bytes; 1012 1013 out_release: 1014 /* 1015 * If we did an incomplete transfer we must release 1016 * the pipe buffers in question: 1017 */ 1018 for (i = 0; i < pipe->buffers; i++) { 1019 struct pipe_buffer *buf = pipe->bufs + i; 1020 1021 if (buf->ops) 1022 pipe_buf_release(pipe, buf); 1023 } 1024 1025 if (!bytes) 1026 bytes = ret; 1027 1028 goto done; 1029 } 1030 EXPORT_SYMBOL(splice_direct_to_actor); 1031 1032 static int direct_splice_actor(struct pipe_inode_info *pipe, 1033 struct splice_desc *sd) 1034 { 1035 struct file *file = sd->u.file; 1036 1037 return do_splice_from(pipe, file, sd->opos, sd->total_len, 1038 sd->flags); 1039 } 1040 1041 /** 1042 * do_splice_direct - splices data directly between two files 1043 * @in: file to splice from 1044 * @ppos: input file offset 1045 * @out: file to splice to 1046 * @opos: output file offset 1047 * @len: number of bytes to splice 1048 * @flags: splice modifier flags 1049 * 1050 * Description: 1051 * For use by do_sendfile(). splice can easily emulate sendfile, but 1052 * doing it in the application would incur an extra system call 1053 * (splice in + splice out, as compared to just sendfile()). So this helper 1054 * can splice directly through a process-private pipe. 1055 * 1056 */ 1057 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 1058 loff_t *opos, size_t len, unsigned int flags) 1059 { 1060 struct splice_desc sd = { 1061 .len = len, 1062 .total_len = len, 1063 .flags = flags, 1064 .pos = *ppos, 1065 .u.file = out, 1066 .opos = opos, 1067 }; 1068 long ret; 1069 1070 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1071 return -EBADF; 1072 1073 if (unlikely(out->f_flags & O_APPEND)) 1074 return -EINVAL; 1075 1076 ret = rw_verify_area(WRITE, out, opos, len); 1077 if (unlikely(ret < 0)) 1078 return ret; 1079 1080 ret = splice_direct_to_actor(in, &sd, direct_splice_actor); 1081 if (ret > 0) 1082 *ppos = sd.pos; 1083 1084 return ret; 1085 } 1086 EXPORT_SYMBOL(do_splice_direct); 1087 1088 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) 1089 { 1090 for (;;) { 1091 if (unlikely(!pipe->readers)) { 1092 send_sig(SIGPIPE, current, 0); 1093 return -EPIPE; 1094 } 1095 if (pipe->nrbufs != pipe->buffers) 1096 return 0; 1097 if (flags & SPLICE_F_NONBLOCK) 1098 return -EAGAIN; 1099 if (signal_pending(current)) 1100 return -ERESTARTSYS; 1101 pipe->waiting_writers++; 1102 pipe_wait(pipe); 1103 pipe->waiting_writers--; 1104 } 1105 } 1106 1107 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1108 struct pipe_inode_info *opipe, 1109 size_t len, unsigned int flags); 1110 1111 /* 1112 * Determine where to splice to/from. 1113 */ 1114 static long do_splice(struct file *in, loff_t __user *off_in, 1115 struct file *out, loff_t __user *off_out, 1116 size_t len, unsigned int flags) 1117 { 1118 struct pipe_inode_info *ipipe; 1119 struct pipe_inode_info *opipe; 1120 loff_t offset; 1121 long ret; 1122 1123 ipipe = get_pipe_info(in); 1124 opipe = get_pipe_info(out); 1125 1126 if (ipipe && opipe) { 1127 if (off_in || off_out) 1128 return -ESPIPE; 1129 1130 if (!(in->f_mode & FMODE_READ)) 1131 return -EBADF; 1132 1133 if (!(out->f_mode & FMODE_WRITE)) 1134 return -EBADF; 1135 1136 /* Splicing to self would be fun, but... */ 1137 if (ipipe == opipe) 1138 return -EINVAL; 1139 1140 return splice_pipe_to_pipe(ipipe, opipe, len, flags); 1141 } 1142 1143 if (ipipe) { 1144 if (off_in) 1145 return -ESPIPE; 1146 if (off_out) { 1147 if (!(out->f_mode & FMODE_PWRITE)) 1148 return -EINVAL; 1149 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1150 return -EFAULT; 1151 } else { 1152 offset = out->f_pos; 1153 } 1154 1155 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1156 return -EBADF; 1157 1158 if (unlikely(out->f_flags & O_APPEND)) 1159 return -EINVAL; 1160 1161 ret = rw_verify_area(WRITE, out, &offset, len); 1162 if (unlikely(ret < 0)) 1163 return ret; 1164 1165 file_start_write(out); 1166 ret = do_splice_from(ipipe, out, &offset, len, flags); 1167 file_end_write(out); 1168 1169 if (!off_out) 1170 out->f_pos = offset; 1171 else if (copy_to_user(off_out, &offset, sizeof(loff_t))) 1172 ret = -EFAULT; 1173 1174 return ret; 1175 } 1176 1177 if (opipe) { 1178 if (off_out) 1179 return -ESPIPE; 1180 if (off_in) { 1181 if (!(in->f_mode & FMODE_PREAD)) 1182 return -EINVAL; 1183 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1184 return -EFAULT; 1185 } else { 1186 offset = in->f_pos; 1187 } 1188 1189 pipe_lock(opipe); 1190 ret = wait_for_space(opipe, flags); 1191 if (!ret) 1192 ret = do_splice_to(in, &offset, opipe, len, flags); 1193 pipe_unlock(opipe); 1194 if (ret > 0) 1195 wakeup_pipe_readers(opipe); 1196 if (!off_in) 1197 in->f_pos = offset; 1198 else if (copy_to_user(off_in, &offset, sizeof(loff_t))) 1199 ret = -EFAULT; 1200 1201 return ret; 1202 } 1203 1204 return -EINVAL; 1205 } 1206 1207 static int iter_to_pipe(struct iov_iter *from, 1208 struct pipe_inode_info *pipe, 1209 unsigned flags) 1210 { 1211 struct pipe_buffer buf = { 1212 .ops = &user_page_pipe_buf_ops, 1213 .flags = flags 1214 }; 1215 size_t total = 0; 1216 int ret = 0; 1217 bool failed = false; 1218 1219 while (iov_iter_count(from) && !failed) { 1220 struct page *pages[16]; 1221 ssize_t copied; 1222 size_t start; 1223 int n; 1224 1225 copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start); 1226 if (copied <= 0) { 1227 ret = copied; 1228 break; 1229 } 1230 1231 for (n = 0; copied; n++, start = 0) { 1232 int size = min_t(int, copied, PAGE_SIZE - start); 1233 if (!failed) { 1234 buf.page = pages[n]; 1235 buf.offset = start; 1236 buf.len = size; 1237 ret = add_to_pipe(pipe, &buf); 1238 if (unlikely(ret < 0)) { 1239 failed = true; 1240 } else { 1241 iov_iter_advance(from, ret); 1242 total += ret; 1243 } 1244 } else { 1245 put_page(pages[n]); 1246 } 1247 copied -= size; 1248 } 1249 } 1250 return total ? total : ret; 1251 } 1252 1253 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1254 struct splice_desc *sd) 1255 { 1256 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); 1257 return n == sd->len ? n : -EFAULT; 1258 } 1259 1260 /* 1261 * For lack of a better implementation, implement vmsplice() to userspace 1262 * as a simple copy of the pipes pages to the user iov. 1263 */ 1264 static long vmsplice_to_user(struct file *file, const struct iovec __user *uiov, 1265 unsigned long nr_segs, unsigned int flags) 1266 { 1267 struct pipe_inode_info *pipe; 1268 struct splice_desc sd; 1269 long ret; 1270 struct iovec iovstack[UIO_FASTIOV]; 1271 struct iovec *iov = iovstack; 1272 struct iov_iter iter; 1273 1274 pipe = get_pipe_info(file); 1275 if (!pipe) 1276 return -EBADF; 1277 1278 ret = import_iovec(READ, uiov, nr_segs, 1279 ARRAY_SIZE(iovstack), &iov, &iter); 1280 if (ret < 0) 1281 return ret; 1282 1283 sd.total_len = iov_iter_count(&iter); 1284 sd.len = 0; 1285 sd.flags = flags; 1286 sd.u.data = &iter; 1287 sd.pos = 0; 1288 1289 if (sd.total_len) { 1290 pipe_lock(pipe); 1291 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); 1292 pipe_unlock(pipe); 1293 } 1294 1295 kfree(iov); 1296 return ret; 1297 } 1298 1299 /* 1300 * vmsplice splices a user address range into a pipe. It can be thought of 1301 * as splice-from-memory, where the regular splice is splice-from-file (or 1302 * to file). In both cases the output is a pipe, naturally. 1303 */ 1304 static long vmsplice_to_pipe(struct file *file, const struct iovec __user *uiov, 1305 unsigned long nr_segs, unsigned int flags) 1306 { 1307 struct pipe_inode_info *pipe; 1308 struct iovec iovstack[UIO_FASTIOV]; 1309 struct iovec *iov = iovstack; 1310 struct iov_iter from; 1311 long ret; 1312 unsigned buf_flag = 0; 1313 1314 if (flags & SPLICE_F_GIFT) 1315 buf_flag = PIPE_BUF_FLAG_GIFT; 1316 1317 pipe = get_pipe_info(file); 1318 if (!pipe) 1319 return -EBADF; 1320 1321 ret = import_iovec(WRITE, uiov, nr_segs, 1322 ARRAY_SIZE(iovstack), &iov, &from); 1323 if (ret < 0) 1324 return ret; 1325 1326 pipe_lock(pipe); 1327 ret = wait_for_space(pipe, flags); 1328 if (!ret) 1329 ret = iter_to_pipe(&from, pipe, buf_flag); 1330 pipe_unlock(pipe); 1331 if (ret > 0) 1332 wakeup_pipe_readers(pipe); 1333 kfree(iov); 1334 return ret; 1335 } 1336 1337 /* 1338 * Note that vmsplice only really supports true splicing _from_ user memory 1339 * to a pipe, not the other way around. Splicing from user memory is a simple 1340 * operation that can be supported without any funky alignment restrictions 1341 * or nasty vm tricks. We simply map in the user memory and fill them into 1342 * a pipe. The reverse isn't quite as easy, though. There are two possible 1343 * solutions for that: 1344 * 1345 * - memcpy() the data internally, at which point we might as well just 1346 * do a regular read() on the buffer anyway. 1347 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1348 * has restriction limitations on both ends of the pipe). 1349 * 1350 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1351 * 1352 */ 1353 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov, 1354 unsigned long, nr_segs, unsigned int, flags) 1355 { 1356 struct fd f; 1357 long error; 1358 1359 if (unlikely(nr_segs > UIO_MAXIOV)) 1360 return -EINVAL; 1361 else if (unlikely(!nr_segs)) 1362 return 0; 1363 1364 error = -EBADF; 1365 f = fdget(fd); 1366 if (f.file) { 1367 if (f.file->f_mode & FMODE_WRITE) 1368 error = vmsplice_to_pipe(f.file, iov, nr_segs, flags); 1369 else if (f.file->f_mode & FMODE_READ) 1370 error = vmsplice_to_user(f.file, iov, nr_segs, flags); 1371 1372 fdput(f); 1373 } 1374 1375 return error; 1376 } 1377 1378 #ifdef CONFIG_COMPAT 1379 COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32, 1380 unsigned int, nr_segs, unsigned int, flags) 1381 { 1382 unsigned i; 1383 struct iovec __user *iov; 1384 if (nr_segs > UIO_MAXIOV) 1385 return -EINVAL; 1386 iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec)); 1387 for (i = 0; i < nr_segs; i++) { 1388 struct compat_iovec v; 1389 if (get_user(v.iov_base, &iov32[i].iov_base) || 1390 get_user(v.iov_len, &iov32[i].iov_len) || 1391 put_user(compat_ptr(v.iov_base), &iov[i].iov_base) || 1392 put_user(v.iov_len, &iov[i].iov_len)) 1393 return -EFAULT; 1394 } 1395 return sys_vmsplice(fd, iov, nr_segs, flags); 1396 } 1397 #endif 1398 1399 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, 1400 int, fd_out, loff_t __user *, off_out, 1401 size_t, len, unsigned int, flags) 1402 { 1403 struct fd in, out; 1404 long error; 1405 1406 if (unlikely(!len)) 1407 return 0; 1408 1409 error = -EBADF; 1410 in = fdget(fd_in); 1411 if (in.file) { 1412 if (in.file->f_mode & FMODE_READ) { 1413 out = fdget(fd_out); 1414 if (out.file) { 1415 if (out.file->f_mode & FMODE_WRITE) 1416 error = do_splice(in.file, off_in, 1417 out.file, off_out, 1418 len, flags); 1419 fdput(out); 1420 } 1421 } 1422 fdput(in); 1423 } 1424 return error; 1425 } 1426 1427 /* 1428 * Make sure there's data to read. Wait for input if we can, otherwise 1429 * return an appropriate error. 1430 */ 1431 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1432 { 1433 int ret; 1434 1435 /* 1436 * Check ->nrbufs without the inode lock first. This function 1437 * is speculative anyways, so missing one is ok. 1438 */ 1439 if (pipe->nrbufs) 1440 return 0; 1441 1442 ret = 0; 1443 pipe_lock(pipe); 1444 1445 while (!pipe->nrbufs) { 1446 if (signal_pending(current)) { 1447 ret = -ERESTARTSYS; 1448 break; 1449 } 1450 if (!pipe->writers) 1451 break; 1452 if (!pipe->waiting_writers) { 1453 if (flags & SPLICE_F_NONBLOCK) { 1454 ret = -EAGAIN; 1455 break; 1456 } 1457 } 1458 pipe_wait(pipe); 1459 } 1460 1461 pipe_unlock(pipe); 1462 return ret; 1463 } 1464 1465 /* 1466 * Make sure there's writeable room. Wait for room if we can, otherwise 1467 * return an appropriate error. 1468 */ 1469 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1470 { 1471 int ret; 1472 1473 /* 1474 * Check ->nrbufs without the inode lock first. This function 1475 * is speculative anyways, so missing one is ok. 1476 */ 1477 if (pipe->nrbufs < pipe->buffers) 1478 return 0; 1479 1480 ret = 0; 1481 pipe_lock(pipe); 1482 1483 while (pipe->nrbufs >= pipe->buffers) { 1484 if (!pipe->readers) { 1485 send_sig(SIGPIPE, current, 0); 1486 ret = -EPIPE; 1487 break; 1488 } 1489 if (flags & SPLICE_F_NONBLOCK) { 1490 ret = -EAGAIN; 1491 break; 1492 } 1493 if (signal_pending(current)) { 1494 ret = -ERESTARTSYS; 1495 break; 1496 } 1497 pipe->waiting_writers++; 1498 pipe_wait(pipe); 1499 pipe->waiting_writers--; 1500 } 1501 1502 pipe_unlock(pipe); 1503 return ret; 1504 } 1505 1506 /* 1507 * Splice contents of ipipe to opipe. 1508 */ 1509 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1510 struct pipe_inode_info *opipe, 1511 size_t len, unsigned int flags) 1512 { 1513 struct pipe_buffer *ibuf, *obuf; 1514 int ret = 0, nbuf; 1515 bool input_wakeup = false; 1516 1517 1518 retry: 1519 ret = ipipe_prep(ipipe, flags); 1520 if (ret) 1521 return ret; 1522 1523 ret = opipe_prep(opipe, flags); 1524 if (ret) 1525 return ret; 1526 1527 /* 1528 * Potential ABBA deadlock, work around it by ordering lock 1529 * grabbing by pipe info address. Otherwise two different processes 1530 * could deadlock (one doing tee from A -> B, the other from B -> A). 1531 */ 1532 pipe_double_lock(ipipe, opipe); 1533 1534 do { 1535 if (!opipe->readers) { 1536 send_sig(SIGPIPE, current, 0); 1537 if (!ret) 1538 ret = -EPIPE; 1539 break; 1540 } 1541 1542 if (!ipipe->nrbufs && !ipipe->writers) 1543 break; 1544 1545 /* 1546 * Cannot make any progress, because either the input 1547 * pipe is empty or the output pipe is full. 1548 */ 1549 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) { 1550 /* Already processed some buffers, break */ 1551 if (ret) 1552 break; 1553 1554 if (flags & SPLICE_F_NONBLOCK) { 1555 ret = -EAGAIN; 1556 break; 1557 } 1558 1559 /* 1560 * We raced with another reader/writer and haven't 1561 * managed to process any buffers. A zero return 1562 * value means EOF, so retry instead. 1563 */ 1564 pipe_unlock(ipipe); 1565 pipe_unlock(opipe); 1566 goto retry; 1567 } 1568 1569 ibuf = ipipe->bufs + ipipe->curbuf; 1570 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1571 obuf = opipe->bufs + nbuf; 1572 1573 if (len >= ibuf->len) { 1574 /* 1575 * Simply move the whole buffer from ipipe to opipe 1576 */ 1577 *obuf = *ibuf; 1578 ibuf->ops = NULL; 1579 opipe->nrbufs++; 1580 ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1); 1581 ipipe->nrbufs--; 1582 input_wakeup = true; 1583 } else { 1584 /* 1585 * Get a reference to this pipe buffer, 1586 * so we can copy the contents over. 1587 */ 1588 pipe_buf_get(ipipe, ibuf); 1589 *obuf = *ibuf; 1590 1591 /* 1592 * Don't inherit the gift flag, we need to 1593 * prevent multiple steals of this page. 1594 */ 1595 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1596 1597 obuf->len = len; 1598 opipe->nrbufs++; 1599 ibuf->offset += obuf->len; 1600 ibuf->len -= obuf->len; 1601 } 1602 ret += obuf->len; 1603 len -= obuf->len; 1604 } while (len); 1605 1606 pipe_unlock(ipipe); 1607 pipe_unlock(opipe); 1608 1609 /* 1610 * If we put data in the output pipe, wakeup any potential readers. 1611 */ 1612 if (ret > 0) 1613 wakeup_pipe_readers(opipe); 1614 1615 if (input_wakeup) 1616 wakeup_pipe_writers(ipipe); 1617 1618 return ret; 1619 } 1620 1621 /* 1622 * Link contents of ipipe to opipe. 1623 */ 1624 static int link_pipe(struct pipe_inode_info *ipipe, 1625 struct pipe_inode_info *opipe, 1626 size_t len, unsigned int flags) 1627 { 1628 struct pipe_buffer *ibuf, *obuf; 1629 int ret = 0, i = 0, nbuf; 1630 1631 /* 1632 * Potential ABBA deadlock, work around it by ordering lock 1633 * grabbing by pipe info address. Otherwise two different processes 1634 * could deadlock (one doing tee from A -> B, the other from B -> A). 1635 */ 1636 pipe_double_lock(ipipe, opipe); 1637 1638 do { 1639 if (!opipe->readers) { 1640 send_sig(SIGPIPE, current, 0); 1641 if (!ret) 1642 ret = -EPIPE; 1643 break; 1644 } 1645 1646 /* 1647 * If we have iterated all input buffers or ran out of 1648 * output room, break. 1649 */ 1650 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) 1651 break; 1652 1653 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1)); 1654 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1655 1656 /* 1657 * Get a reference to this pipe buffer, 1658 * so we can copy the contents over. 1659 */ 1660 pipe_buf_get(ipipe, ibuf); 1661 1662 obuf = opipe->bufs + nbuf; 1663 *obuf = *ibuf; 1664 1665 /* 1666 * Don't inherit the gift flag, we need to 1667 * prevent multiple steals of this page. 1668 */ 1669 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1670 1671 if (obuf->len > len) 1672 obuf->len = len; 1673 1674 opipe->nrbufs++; 1675 ret += obuf->len; 1676 len -= obuf->len; 1677 i++; 1678 } while (len); 1679 1680 /* 1681 * return EAGAIN if we have the potential of some data in the 1682 * future, otherwise just return 0 1683 */ 1684 if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK)) 1685 ret = -EAGAIN; 1686 1687 pipe_unlock(ipipe); 1688 pipe_unlock(opipe); 1689 1690 /* 1691 * If we put data in the output pipe, wakeup any potential readers. 1692 */ 1693 if (ret > 0) 1694 wakeup_pipe_readers(opipe); 1695 1696 return ret; 1697 } 1698 1699 /* 1700 * This is a tee(1) implementation that works on pipes. It doesn't copy 1701 * any data, it simply references the 'in' pages on the 'out' pipe. 1702 * The 'flags' used are the SPLICE_F_* variants, currently the only 1703 * applicable one is SPLICE_F_NONBLOCK. 1704 */ 1705 static long do_tee(struct file *in, struct file *out, size_t len, 1706 unsigned int flags) 1707 { 1708 struct pipe_inode_info *ipipe = get_pipe_info(in); 1709 struct pipe_inode_info *opipe = get_pipe_info(out); 1710 int ret = -EINVAL; 1711 1712 /* 1713 * Duplicate the contents of ipipe to opipe without actually 1714 * copying the data. 1715 */ 1716 if (ipipe && opipe && ipipe != opipe) { 1717 /* 1718 * Keep going, unless we encounter an error. The ipipe/opipe 1719 * ordering doesn't really matter. 1720 */ 1721 ret = ipipe_prep(ipipe, flags); 1722 if (!ret) { 1723 ret = opipe_prep(opipe, flags); 1724 if (!ret) 1725 ret = link_pipe(ipipe, opipe, len, flags); 1726 } 1727 } 1728 1729 return ret; 1730 } 1731 1732 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) 1733 { 1734 struct fd in; 1735 int error; 1736 1737 if (unlikely(!len)) 1738 return 0; 1739 1740 error = -EBADF; 1741 in = fdget(fdin); 1742 if (in.file) { 1743 if (in.file->f_mode & FMODE_READ) { 1744 struct fd out = fdget(fdout); 1745 if (out.file) { 1746 if (out.file->f_mode & FMODE_WRITE) 1747 error = do_tee(in.file, out.file, 1748 len, flags); 1749 fdput(out); 1750 } 1751 } 1752 fdput(in); 1753 } 1754 1755 return error; 1756 } 1757