1 /* 2 * "splice": joining two ropes together by interweaving their strands. 3 * 4 * This is the "extended pipe" functionality, where a pipe is used as 5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 6 * buffer that you can use to transfer data from one end to the other. 7 * 8 * The traditional unix read/write is extended with a "splice()" operation 9 * that transfers data buffers to or from a pipe buffer. 10 * 11 * Named by Larry McVoy, original implementation from Linus, extended by 12 * Jens to support splicing to files, network, direct splicing, etc and 13 * fixing lots of bugs. 14 * 15 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 18 * 19 */ 20 #include <linux/bvec.h> 21 #include <linux/fs.h> 22 #include <linux/file.h> 23 #include <linux/pagemap.h> 24 #include <linux/splice.h> 25 #include <linux/memcontrol.h> 26 #include <linux/mm_inline.h> 27 #include <linux/swap.h> 28 #include <linux/writeback.h> 29 #include <linux/export.h> 30 #include <linux/syscalls.h> 31 #include <linux/uio.h> 32 #include <linux/security.h> 33 #include <linux/gfp.h> 34 #include <linux/socket.h> 35 #include <linux/compat.h> 36 #include "internal.h" 37 38 /* 39 * Attempt to steal a page from a pipe buffer. This should perhaps go into 40 * a vm helper function, it's already simplified quite a bit by the 41 * addition of remove_mapping(). If success is returned, the caller may 42 * attempt to reuse this page for another destination. 43 */ 44 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, 45 struct pipe_buffer *buf) 46 { 47 struct page *page = buf->page; 48 struct address_space *mapping; 49 50 lock_page(page); 51 52 mapping = page_mapping(page); 53 if (mapping) { 54 WARN_ON(!PageUptodate(page)); 55 56 /* 57 * At least for ext2 with nobh option, we need to wait on 58 * writeback completing on this page, since we'll remove it 59 * from the pagecache. Otherwise truncate wont wait on the 60 * page, allowing the disk blocks to be reused by someone else 61 * before we actually wrote our data to them. fs corruption 62 * ensues. 63 */ 64 wait_on_page_writeback(page); 65 66 if (page_has_private(page) && 67 !try_to_release_page(page, GFP_KERNEL)) 68 goto out_unlock; 69 70 /* 71 * If we succeeded in removing the mapping, set LRU flag 72 * and return good. 73 */ 74 if (remove_mapping(mapping, page)) { 75 buf->flags |= PIPE_BUF_FLAG_LRU; 76 return 0; 77 } 78 } 79 80 /* 81 * Raced with truncate or failed to remove page from current 82 * address space, unlock and return failure. 83 */ 84 out_unlock: 85 unlock_page(page); 86 return 1; 87 } 88 89 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, 90 struct pipe_buffer *buf) 91 { 92 put_page(buf->page); 93 buf->flags &= ~PIPE_BUF_FLAG_LRU; 94 } 95 96 /* 97 * Check whether the contents of buf is OK to access. Since the content 98 * is a page cache page, IO may be in flight. 99 */ 100 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 101 struct pipe_buffer *buf) 102 { 103 struct page *page = buf->page; 104 int err; 105 106 if (!PageUptodate(page)) { 107 lock_page(page); 108 109 /* 110 * Page got truncated/unhashed. This will cause a 0-byte 111 * splice, if this is the first page. 112 */ 113 if (!page->mapping) { 114 err = -ENODATA; 115 goto error; 116 } 117 118 /* 119 * Uh oh, read-error from disk. 120 */ 121 if (!PageUptodate(page)) { 122 err = -EIO; 123 goto error; 124 } 125 126 /* 127 * Page is ok afterall, we are done. 128 */ 129 unlock_page(page); 130 } 131 132 return 0; 133 error: 134 unlock_page(page); 135 return err; 136 } 137 138 const struct pipe_buf_operations page_cache_pipe_buf_ops = { 139 .can_merge = 0, 140 .confirm = page_cache_pipe_buf_confirm, 141 .release = page_cache_pipe_buf_release, 142 .steal = page_cache_pipe_buf_steal, 143 .get = generic_pipe_buf_get, 144 }; 145 146 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, 147 struct pipe_buffer *buf) 148 { 149 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) 150 return 1; 151 152 buf->flags |= PIPE_BUF_FLAG_LRU; 153 return generic_pipe_buf_steal(pipe, buf); 154 } 155 156 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 157 .can_merge = 0, 158 .confirm = generic_pipe_buf_confirm, 159 .release = page_cache_pipe_buf_release, 160 .steal = user_page_pipe_buf_steal, 161 .get = generic_pipe_buf_get, 162 }; 163 164 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) 165 { 166 smp_mb(); 167 if (waitqueue_active(&pipe->wait)) 168 wake_up_interruptible(&pipe->wait); 169 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 170 } 171 172 /** 173 * splice_to_pipe - fill passed data into a pipe 174 * @pipe: pipe to fill 175 * @spd: data to fill 176 * 177 * Description: 178 * @spd contains a map of pages and len/offset tuples, along with 179 * the struct pipe_buf_operations associated with these pages. This 180 * function will link that data to the pipe. 181 * 182 */ 183 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 184 struct splice_pipe_desc *spd) 185 { 186 unsigned int spd_pages = spd->nr_pages; 187 int ret = 0, page_nr = 0; 188 189 if (!spd_pages) 190 return 0; 191 192 if (unlikely(!pipe->readers)) { 193 send_sig(SIGPIPE, current, 0); 194 ret = -EPIPE; 195 goto out; 196 } 197 198 while (pipe->nrbufs < pipe->buffers) { 199 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 200 struct pipe_buffer *buf = pipe->bufs + newbuf; 201 202 buf->page = spd->pages[page_nr]; 203 buf->offset = spd->partial[page_nr].offset; 204 buf->len = spd->partial[page_nr].len; 205 buf->private = spd->partial[page_nr].private; 206 buf->ops = spd->ops; 207 208 pipe->nrbufs++; 209 page_nr++; 210 ret += buf->len; 211 212 if (!--spd->nr_pages) 213 break; 214 } 215 216 if (!ret) 217 ret = -EAGAIN; 218 219 out: 220 while (page_nr < spd_pages) 221 spd->spd_release(spd, page_nr++); 222 223 return ret; 224 } 225 EXPORT_SYMBOL_GPL(splice_to_pipe); 226 227 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 228 { 229 int ret; 230 231 if (unlikely(!pipe->readers)) { 232 send_sig(SIGPIPE, current, 0); 233 ret = -EPIPE; 234 } else if (pipe->nrbufs == pipe->buffers) { 235 ret = -EAGAIN; 236 } else { 237 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 238 pipe->bufs[newbuf] = *buf; 239 pipe->nrbufs++; 240 return buf->len; 241 } 242 pipe_buf_release(pipe, buf); 243 return ret; 244 } 245 EXPORT_SYMBOL(add_to_pipe); 246 247 void spd_release_page(struct splice_pipe_desc *spd, unsigned int i) 248 { 249 put_page(spd->pages[i]); 250 } 251 252 /* 253 * Check if we need to grow the arrays holding pages and partial page 254 * descriptions. 255 */ 256 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) 257 { 258 unsigned int buffers = ACCESS_ONCE(pipe->buffers); 259 260 spd->nr_pages_max = buffers; 261 if (buffers <= PIPE_DEF_BUFFERS) 262 return 0; 263 264 spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL); 265 spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL); 266 267 if (spd->pages && spd->partial) 268 return 0; 269 270 kfree(spd->pages); 271 kfree(spd->partial); 272 return -ENOMEM; 273 } 274 275 void splice_shrink_spd(struct splice_pipe_desc *spd) 276 { 277 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) 278 return; 279 280 kfree(spd->pages); 281 kfree(spd->partial); 282 } 283 284 /** 285 * generic_file_splice_read - splice data from file to a pipe 286 * @in: file to splice from 287 * @ppos: position in @in 288 * @pipe: pipe to splice to 289 * @len: number of bytes to splice 290 * @flags: splice modifier flags 291 * 292 * Description: 293 * Will read pages from given file and fill them into a pipe. Can be 294 * used as long as it has more or less sane ->read_iter(). 295 * 296 */ 297 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, 298 struct pipe_inode_info *pipe, size_t len, 299 unsigned int flags) 300 { 301 struct iov_iter to; 302 struct kiocb kiocb; 303 int idx, ret; 304 305 iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len); 306 idx = to.idx; 307 init_sync_kiocb(&kiocb, in); 308 kiocb.ki_pos = *ppos; 309 ret = in->f_op->read_iter(&kiocb, &to); 310 if (ret > 0) { 311 *ppos = kiocb.ki_pos; 312 file_accessed(in); 313 } else if (ret < 0) { 314 to.idx = idx; 315 to.iov_offset = 0; 316 iov_iter_advance(&to, 0); /* to free what was emitted */ 317 /* 318 * callers of ->splice_read() expect -EAGAIN on 319 * "can't put anything in there", rather than -EFAULT. 320 */ 321 if (ret == -EFAULT) 322 ret = -EAGAIN; 323 } 324 325 return ret; 326 } 327 EXPORT_SYMBOL(generic_file_splice_read); 328 329 const struct pipe_buf_operations default_pipe_buf_ops = { 330 .can_merge = 0, 331 .confirm = generic_pipe_buf_confirm, 332 .release = generic_pipe_buf_release, 333 .steal = generic_pipe_buf_steal, 334 .get = generic_pipe_buf_get, 335 }; 336 337 static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe, 338 struct pipe_buffer *buf) 339 { 340 return 1; 341 } 342 343 /* Pipe buffer operations for a socket and similar. */ 344 const struct pipe_buf_operations nosteal_pipe_buf_ops = { 345 .can_merge = 0, 346 .confirm = generic_pipe_buf_confirm, 347 .release = generic_pipe_buf_release, 348 .steal = generic_pipe_buf_nosteal, 349 .get = generic_pipe_buf_get, 350 }; 351 EXPORT_SYMBOL(nosteal_pipe_buf_ops); 352 353 static ssize_t kernel_readv(struct file *file, const struct kvec *vec, 354 unsigned long vlen, loff_t offset) 355 { 356 mm_segment_t old_fs; 357 loff_t pos = offset; 358 ssize_t res; 359 360 old_fs = get_fs(); 361 set_fs(get_ds()); 362 /* The cast to a user pointer is valid due to the set_fs() */ 363 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0); 364 set_fs(old_fs); 365 366 return res; 367 } 368 369 ssize_t kernel_write(struct file *file, const char *buf, size_t count, 370 loff_t pos) 371 { 372 mm_segment_t old_fs; 373 ssize_t res; 374 375 old_fs = get_fs(); 376 set_fs(get_ds()); 377 /* The cast to a user pointer is valid due to the set_fs() */ 378 res = vfs_write(file, (__force const char __user *)buf, count, &pos); 379 set_fs(old_fs); 380 381 return res; 382 } 383 EXPORT_SYMBOL(kernel_write); 384 385 static ssize_t default_file_splice_read(struct file *in, loff_t *ppos, 386 struct pipe_inode_info *pipe, size_t len, 387 unsigned int flags) 388 { 389 struct kvec *vec, __vec[PIPE_DEF_BUFFERS]; 390 struct iov_iter to; 391 struct page **pages; 392 unsigned int nr_pages; 393 size_t offset, dummy, copied = 0; 394 ssize_t res; 395 int i; 396 397 if (pipe->nrbufs == pipe->buffers) 398 return -EAGAIN; 399 400 /* 401 * Try to keep page boundaries matching to source pagecache ones - 402 * it probably won't be much help, but... 403 */ 404 offset = *ppos & ~PAGE_MASK; 405 406 iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len + offset); 407 408 res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &dummy); 409 if (res <= 0) 410 return -ENOMEM; 411 412 BUG_ON(dummy); 413 nr_pages = DIV_ROUND_UP(res, PAGE_SIZE); 414 415 vec = __vec; 416 if (nr_pages > PIPE_DEF_BUFFERS) { 417 vec = kmalloc(nr_pages * sizeof(struct kvec), GFP_KERNEL); 418 if (unlikely(!vec)) { 419 res = -ENOMEM; 420 goto out; 421 } 422 } 423 424 pipe->bufs[to.idx].offset = offset; 425 pipe->bufs[to.idx].len -= offset; 426 427 for (i = 0; i < nr_pages; i++) { 428 size_t this_len = min_t(size_t, len, PAGE_SIZE - offset); 429 vec[i].iov_base = page_address(pages[i]) + offset; 430 vec[i].iov_len = this_len; 431 len -= this_len; 432 offset = 0; 433 } 434 435 res = kernel_readv(in, vec, nr_pages, *ppos); 436 if (res > 0) { 437 copied = res; 438 *ppos += res; 439 } 440 441 if (vec != __vec) 442 kfree(vec); 443 out: 444 for (i = 0; i < nr_pages; i++) 445 put_page(pages[i]); 446 kvfree(pages); 447 iov_iter_advance(&to, copied); /* truncates and discards */ 448 return res; 449 } 450 451 /* 452 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 453 * using sendpage(). Return the number of bytes sent. 454 */ 455 static int pipe_to_sendpage(struct pipe_inode_info *pipe, 456 struct pipe_buffer *buf, struct splice_desc *sd) 457 { 458 struct file *file = sd->u.file; 459 loff_t pos = sd->pos; 460 int more; 461 462 if (!likely(file->f_op->sendpage)) 463 return -EINVAL; 464 465 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; 466 467 if (sd->len < sd->total_len && pipe->nrbufs > 1) 468 more |= MSG_SENDPAGE_NOTLAST; 469 470 return file->f_op->sendpage(file, buf->page, buf->offset, 471 sd->len, &pos, more); 472 } 473 474 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) 475 { 476 smp_mb(); 477 if (waitqueue_active(&pipe->wait)) 478 wake_up_interruptible(&pipe->wait); 479 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 480 } 481 482 /** 483 * splice_from_pipe_feed - feed available data from a pipe to a file 484 * @pipe: pipe to splice from 485 * @sd: information to @actor 486 * @actor: handler that splices the data 487 * 488 * Description: 489 * This function loops over the pipe and calls @actor to do the 490 * actual moving of a single struct pipe_buffer to the desired 491 * destination. It returns when there's no more buffers left in 492 * the pipe or if the requested number of bytes (@sd->total_len) 493 * have been copied. It returns a positive number (one) if the 494 * pipe needs to be filled with more data, zero if the required 495 * number of bytes have been copied and -errno on error. 496 * 497 * This, together with splice_from_pipe_{begin,end,next}, may be 498 * used to implement the functionality of __splice_from_pipe() when 499 * locking is required around copying the pipe buffers to the 500 * destination. 501 */ 502 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, 503 splice_actor *actor) 504 { 505 int ret; 506 507 while (pipe->nrbufs) { 508 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 509 510 sd->len = buf->len; 511 if (sd->len > sd->total_len) 512 sd->len = sd->total_len; 513 514 ret = pipe_buf_confirm(pipe, buf); 515 if (unlikely(ret)) { 516 if (ret == -ENODATA) 517 ret = 0; 518 return ret; 519 } 520 521 ret = actor(pipe, buf, sd); 522 if (ret <= 0) 523 return ret; 524 525 buf->offset += ret; 526 buf->len -= ret; 527 528 sd->num_spliced += ret; 529 sd->len -= ret; 530 sd->pos += ret; 531 sd->total_len -= ret; 532 533 if (!buf->len) { 534 pipe_buf_release(pipe, buf); 535 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 536 pipe->nrbufs--; 537 if (pipe->files) 538 sd->need_wakeup = true; 539 } 540 541 if (!sd->total_len) 542 return 0; 543 } 544 545 return 1; 546 } 547 548 /** 549 * splice_from_pipe_next - wait for some data to splice from 550 * @pipe: pipe to splice from 551 * @sd: information about the splice operation 552 * 553 * Description: 554 * This function will wait for some data and return a positive 555 * value (one) if pipe buffers are available. It will return zero 556 * or -errno if no more data needs to be spliced. 557 */ 558 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) 559 { 560 /* 561 * Check for signal early to make process killable when there are 562 * always buffers available 563 */ 564 if (signal_pending(current)) 565 return -ERESTARTSYS; 566 567 while (!pipe->nrbufs) { 568 if (!pipe->writers) 569 return 0; 570 571 if (!pipe->waiting_writers && sd->num_spliced) 572 return 0; 573 574 if (sd->flags & SPLICE_F_NONBLOCK) 575 return -EAGAIN; 576 577 if (signal_pending(current)) 578 return -ERESTARTSYS; 579 580 if (sd->need_wakeup) { 581 wakeup_pipe_writers(pipe); 582 sd->need_wakeup = false; 583 } 584 585 pipe_wait(pipe); 586 } 587 588 return 1; 589 } 590 591 /** 592 * splice_from_pipe_begin - start splicing from pipe 593 * @sd: information about the splice operation 594 * 595 * Description: 596 * This function should be called before a loop containing 597 * splice_from_pipe_next() and splice_from_pipe_feed() to 598 * initialize the necessary fields of @sd. 599 */ 600 static void splice_from_pipe_begin(struct splice_desc *sd) 601 { 602 sd->num_spliced = 0; 603 sd->need_wakeup = false; 604 } 605 606 /** 607 * splice_from_pipe_end - finish splicing from pipe 608 * @pipe: pipe to splice from 609 * @sd: information about the splice operation 610 * 611 * Description: 612 * This function will wake up pipe writers if necessary. It should 613 * be called after a loop containing splice_from_pipe_next() and 614 * splice_from_pipe_feed(). 615 */ 616 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) 617 { 618 if (sd->need_wakeup) 619 wakeup_pipe_writers(pipe); 620 } 621 622 /** 623 * __splice_from_pipe - splice data from a pipe to given actor 624 * @pipe: pipe to splice from 625 * @sd: information to @actor 626 * @actor: handler that splices the data 627 * 628 * Description: 629 * This function does little more than loop over the pipe and call 630 * @actor to do the actual moving of a single struct pipe_buffer to 631 * the desired destination. See pipe_to_file, pipe_to_sendpage, or 632 * pipe_to_user. 633 * 634 */ 635 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 636 splice_actor *actor) 637 { 638 int ret; 639 640 splice_from_pipe_begin(sd); 641 do { 642 cond_resched(); 643 ret = splice_from_pipe_next(pipe, sd); 644 if (ret > 0) 645 ret = splice_from_pipe_feed(pipe, sd, actor); 646 } while (ret > 0); 647 splice_from_pipe_end(pipe, sd); 648 649 return sd->num_spliced ? sd->num_spliced : ret; 650 } 651 EXPORT_SYMBOL(__splice_from_pipe); 652 653 /** 654 * splice_from_pipe - splice data from a pipe to a file 655 * @pipe: pipe to splice from 656 * @out: file to splice to 657 * @ppos: position in @out 658 * @len: how many bytes to splice 659 * @flags: splice modifier flags 660 * @actor: handler that splices the data 661 * 662 * Description: 663 * See __splice_from_pipe. This function locks the pipe inode, 664 * otherwise it's identical to __splice_from_pipe(). 665 * 666 */ 667 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 668 loff_t *ppos, size_t len, unsigned int flags, 669 splice_actor *actor) 670 { 671 ssize_t ret; 672 struct splice_desc sd = { 673 .total_len = len, 674 .flags = flags, 675 .pos = *ppos, 676 .u.file = out, 677 }; 678 679 pipe_lock(pipe); 680 ret = __splice_from_pipe(pipe, &sd, actor); 681 pipe_unlock(pipe); 682 683 return ret; 684 } 685 686 /** 687 * iter_file_splice_write - splice data from a pipe to a file 688 * @pipe: pipe info 689 * @out: file to write to 690 * @ppos: position in @out 691 * @len: number of bytes to splice 692 * @flags: splice modifier flags 693 * 694 * Description: 695 * Will either move or copy pages (determined by @flags options) from 696 * the given pipe inode to the given file. 697 * This one is ->write_iter-based. 698 * 699 */ 700 ssize_t 701 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 702 loff_t *ppos, size_t len, unsigned int flags) 703 { 704 struct splice_desc sd = { 705 .total_len = len, 706 .flags = flags, 707 .pos = *ppos, 708 .u.file = out, 709 }; 710 int nbufs = pipe->buffers; 711 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec), 712 GFP_KERNEL); 713 ssize_t ret; 714 715 if (unlikely(!array)) 716 return -ENOMEM; 717 718 pipe_lock(pipe); 719 720 splice_from_pipe_begin(&sd); 721 while (sd.total_len) { 722 struct iov_iter from; 723 size_t left; 724 int n, idx; 725 726 ret = splice_from_pipe_next(pipe, &sd); 727 if (ret <= 0) 728 break; 729 730 if (unlikely(nbufs < pipe->buffers)) { 731 kfree(array); 732 nbufs = pipe->buffers; 733 array = kcalloc(nbufs, sizeof(struct bio_vec), 734 GFP_KERNEL); 735 if (!array) { 736 ret = -ENOMEM; 737 break; 738 } 739 } 740 741 /* build the vector */ 742 left = sd.total_len; 743 for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) { 744 struct pipe_buffer *buf = pipe->bufs + idx; 745 size_t this_len = buf->len; 746 747 if (this_len > left) 748 this_len = left; 749 750 if (idx == pipe->buffers - 1) 751 idx = -1; 752 753 ret = pipe_buf_confirm(pipe, buf); 754 if (unlikely(ret)) { 755 if (ret == -ENODATA) 756 ret = 0; 757 goto done; 758 } 759 760 array[n].bv_page = buf->page; 761 array[n].bv_len = this_len; 762 array[n].bv_offset = buf->offset; 763 left -= this_len; 764 } 765 766 iov_iter_bvec(&from, ITER_BVEC | WRITE, array, n, 767 sd.total_len - left); 768 ret = vfs_iter_write(out, &from, &sd.pos); 769 if (ret <= 0) 770 break; 771 772 sd.num_spliced += ret; 773 sd.total_len -= ret; 774 *ppos = sd.pos; 775 776 /* dismiss the fully eaten buffers, adjust the partial one */ 777 while (ret) { 778 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 779 if (ret >= buf->len) { 780 ret -= buf->len; 781 buf->len = 0; 782 pipe_buf_release(pipe, buf); 783 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 784 pipe->nrbufs--; 785 if (pipe->files) 786 sd.need_wakeup = true; 787 } else { 788 buf->offset += ret; 789 buf->len -= ret; 790 ret = 0; 791 } 792 } 793 } 794 done: 795 kfree(array); 796 splice_from_pipe_end(pipe, &sd); 797 798 pipe_unlock(pipe); 799 800 if (sd.num_spliced) 801 ret = sd.num_spliced; 802 803 return ret; 804 } 805 806 EXPORT_SYMBOL(iter_file_splice_write); 807 808 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 809 struct splice_desc *sd) 810 { 811 int ret; 812 void *data; 813 loff_t tmp = sd->pos; 814 815 data = kmap(buf->page); 816 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp); 817 kunmap(buf->page); 818 819 return ret; 820 } 821 822 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, 823 struct file *out, loff_t *ppos, 824 size_t len, unsigned int flags) 825 { 826 ssize_t ret; 827 828 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); 829 if (ret > 0) 830 *ppos += ret; 831 832 return ret; 833 } 834 835 /** 836 * generic_splice_sendpage - splice data from a pipe to a socket 837 * @pipe: pipe to splice from 838 * @out: socket to write to 839 * @ppos: position in @out 840 * @len: number of bytes to splice 841 * @flags: splice modifier flags 842 * 843 * Description: 844 * Will send @len bytes from the pipe to a network socket. No data copying 845 * is involved. 846 * 847 */ 848 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 849 loff_t *ppos, size_t len, unsigned int flags) 850 { 851 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); 852 } 853 854 EXPORT_SYMBOL(generic_splice_sendpage); 855 856 /* 857 * Attempt to initiate a splice from pipe to file. 858 */ 859 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 860 loff_t *ppos, size_t len, unsigned int flags) 861 { 862 ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, 863 loff_t *, size_t, unsigned int); 864 865 if (out->f_op->splice_write) 866 splice_write = out->f_op->splice_write; 867 else 868 splice_write = default_file_splice_write; 869 870 return splice_write(pipe, out, ppos, len, flags); 871 } 872 873 /* 874 * Attempt to initiate a splice from a file to a pipe. 875 */ 876 static long do_splice_to(struct file *in, loff_t *ppos, 877 struct pipe_inode_info *pipe, size_t len, 878 unsigned int flags) 879 { 880 ssize_t (*splice_read)(struct file *, loff_t *, 881 struct pipe_inode_info *, size_t, unsigned int); 882 int ret; 883 884 if (unlikely(!(in->f_mode & FMODE_READ))) 885 return -EBADF; 886 887 ret = rw_verify_area(READ, in, ppos, len); 888 if (unlikely(ret < 0)) 889 return ret; 890 891 if (unlikely(len > MAX_RW_COUNT)) 892 len = MAX_RW_COUNT; 893 894 if (in->f_op->splice_read) 895 splice_read = in->f_op->splice_read; 896 else 897 splice_read = default_file_splice_read; 898 899 return splice_read(in, ppos, pipe, len, flags); 900 } 901 902 /** 903 * splice_direct_to_actor - splices data directly between two non-pipes 904 * @in: file to splice from 905 * @sd: actor information on where to splice to 906 * @actor: handles the data splicing 907 * 908 * Description: 909 * This is a special case helper to splice directly between two 910 * points, without requiring an explicit pipe. Internally an allocated 911 * pipe is cached in the process, and reused during the lifetime of 912 * that process. 913 * 914 */ 915 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 916 splice_direct_actor *actor) 917 { 918 struct pipe_inode_info *pipe; 919 long ret, bytes; 920 umode_t i_mode; 921 size_t len; 922 int i, flags, more; 923 924 /* 925 * We require the input being a regular file, as we don't want to 926 * randomly drop data for eg socket -> socket splicing. Use the 927 * piped splicing for that! 928 */ 929 i_mode = file_inode(in)->i_mode; 930 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) 931 return -EINVAL; 932 933 /* 934 * neither in nor out is a pipe, setup an internal pipe attached to 935 * 'out' and transfer the wanted data from 'in' to 'out' through that 936 */ 937 pipe = current->splice_pipe; 938 if (unlikely(!pipe)) { 939 pipe = alloc_pipe_info(); 940 if (!pipe) 941 return -ENOMEM; 942 943 /* 944 * We don't have an immediate reader, but we'll read the stuff 945 * out of the pipe right after the splice_to_pipe(). So set 946 * PIPE_READERS appropriately. 947 */ 948 pipe->readers = 1; 949 950 current->splice_pipe = pipe; 951 } 952 953 /* 954 * Do the splice. 955 */ 956 ret = 0; 957 bytes = 0; 958 len = sd->total_len; 959 flags = sd->flags; 960 961 /* 962 * Don't block on output, we have to drain the direct pipe. 963 */ 964 sd->flags &= ~SPLICE_F_NONBLOCK; 965 more = sd->flags & SPLICE_F_MORE; 966 967 while (len) { 968 size_t read_len; 969 loff_t pos = sd->pos, prev_pos = pos; 970 971 ret = do_splice_to(in, &pos, pipe, len, flags); 972 if (unlikely(ret <= 0)) 973 goto out_release; 974 975 read_len = ret; 976 sd->total_len = read_len; 977 978 /* 979 * If more data is pending, set SPLICE_F_MORE 980 * If this is the last data and SPLICE_F_MORE was not set 981 * initially, clears it. 982 */ 983 if (read_len < len) 984 sd->flags |= SPLICE_F_MORE; 985 else if (!more) 986 sd->flags &= ~SPLICE_F_MORE; 987 /* 988 * NOTE: nonblocking mode only applies to the input. We 989 * must not do the output in nonblocking mode as then we 990 * could get stuck data in the internal pipe: 991 */ 992 ret = actor(pipe, sd); 993 if (unlikely(ret <= 0)) { 994 sd->pos = prev_pos; 995 goto out_release; 996 } 997 998 bytes += ret; 999 len -= ret; 1000 sd->pos = pos; 1001 1002 if (ret < read_len) { 1003 sd->pos = prev_pos + ret; 1004 goto out_release; 1005 } 1006 } 1007 1008 done: 1009 pipe->nrbufs = pipe->curbuf = 0; 1010 file_accessed(in); 1011 return bytes; 1012 1013 out_release: 1014 /* 1015 * If we did an incomplete transfer we must release 1016 * the pipe buffers in question: 1017 */ 1018 for (i = 0; i < pipe->buffers; i++) { 1019 struct pipe_buffer *buf = pipe->bufs + i; 1020 1021 if (buf->ops) 1022 pipe_buf_release(pipe, buf); 1023 } 1024 1025 if (!bytes) 1026 bytes = ret; 1027 1028 goto done; 1029 } 1030 EXPORT_SYMBOL(splice_direct_to_actor); 1031 1032 static int direct_splice_actor(struct pipe_inode_info *pipe, 1033 struct splice_desc *sd) 1034 { 1035 struct file *file = sd->u.file; 1036 1037 return do_splice_from(pipe, file, sd->opos, sd->total_len, 1038 sd->flags); 1039 } 1040 1041 /** 1042 * do_splice_direct - splices data directly between two files 1043 * @in: file to splice from 1044 * @ppos: input file offset 1045 * @out: file to splice to 1046 * @opos: output file offset 1047 * @len: number of bytes to splice 1048 * @flags: splice modifier flags 1049 * 1050 * Description: 1051 * For use by do_sendfile(). splice can easily emulate sendfile, but 1052 * doing it in the application would incur an extra system call 1053 * (splice in + splice out, as compared to just sendfile()). So this helper 1054 * can splice directly through a process-private pipe. 1055 * 1056 */ 1057 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 1058 loff_t *opos, size_t len, unsigned int flags) 1059 { 1060 struct splice_desc sd = { 1061 .len = len, 1062 .total_len = len, 1063 .flags = flags, 1064 .pos = *ppos, 1065 .u.file = out, 1066 .opos = opos, 1067 }; 1068 long ret; 1069 1070 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1071 return -EBADF; 1072 1073 if (unlikely(out->f_flags & O_APPEND)) 1074 return -EINVAL; 1075 1076 ret = rw_verify_area(WRITE, out, opos, len); 1077 if (unlikely(ret < 0)) 1078 return ret; 1079 1080 ret = splice_direct_to_actor(in, &sd, direct_splice_actor); 1081 if (ret > 0) 1082 *ppos = sd.pos; 1083 1084 return ret; 1085 } 1086 EXPORT_SYMBOL(do_splice_direct); 1087 1088 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) 1089 { 1090 while (pipe->nrbufs == pipe->buffers) { 1091 if (flags & SPLICE_F_NONBLOCK) 1092 return -EAGAIN; 1093 if (signal_pending(current)) 1094 return -ERESTARTSYS; 1095 pipe->waiting_writers++; 1096 pipe_wait(pipe); 1097 pipe->waiting_writers--; 1098 } 1099 return 0; 1100 } 1101 1102 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1103 struct pipe_inode_info *opipe, 1104 size_t len, unsigned int flags); 1105 1106 /* 1107 * Determine where to splice to/from. 1108 */ 1109 static long do_splice(struct file *in, loff_t __user *off_in, 1110 struct file *out, loff_t __user *off_out, 1111 size_t len, unsigned int flags) 1112 { 1113 struct pipe_inode_info *ipipe; 1114 struct pipe_inode_info *opipe; 1115 loff_t offset; 1116 long ret; 1117 1118 ipipe = get_pipe_info(in); 1119 opipe = get_pipe_info(out); 1120 1121 if (ipipe && opipe) { 1122 if (off_in || off_out) 1123 return -ESPIPE; 1124 1125 if (!(in->f_mode & FMODE_READ)) 1126 return -EBADF; 1127 1128 if (!(out->f_mode & FMODE_WRITE)) 1129 return -EBADF; 1130 1131 /* Splicing to self would be fun, but... */ 1132 if (ipipe == opipe) 1133 return -EINVAL; 1134 1135 return splice_pipe_to_pipe(ipipe, opipe, len, flags); 1136 } 1137 1138 if (ipipe) { 1139 if (off_in) 1140 return -ESPIPE; 1141 if (off_out) { 1142 if (!(out->f_mode & FMODE_PWRITE)) 1143 return -EINVAL; 1144 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1145 return -EFAULT; 1146 } else { 1147 offset = out->f_pos; 1148 } 1149 1150 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1151 return -EBADF; 1152 1153 if (unlikely(out->f_flags & O_APPEND)) 1154 return -EINVAL; 1155 1156 ret = rw_verify_area(WRITE, out, &offset, len); 1157 if (unlikely(ret < 0)) 1158 return ret; 1159 1160 file_start_write(out); 1161 ret = do_splice_from(ipipe, out, &offset, len, flags); 1162 file_end_write(out); 1163 1164 if (!off_out) 1165 out->f_pos = offset; 1166 else if (copy_to_user(off_out, &offset, sizeof(loff_t))) 1167 ret = -EFAULT; 1168 1169 return ret; 1170 } 1171 1172 if (opipe) { 1173 if (off_out) 1174 return -ESPIPE; 1175 if (off_in) { 1176 if (!(in->f_mode & FMODE_PREAD)) 1177 return -EINVAL; 1178 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1179 return -EFAULT; 1180 } else { 1181 offset = in->f_pos; 1182 } 1183 1184 pipe_lock(opipe); 1185 ret = wait_for_space(opipe, flags); 1186 if (!ret) 1187 ret = do_splice_to(in, &offset, opipe, len, flags); 1188 pipe_unlock(opipe); 1189 if (ret > 0) 1190 wakeup_pipe_readers(opipe); 1191 if (!off_in) 1192 in->f_pos = offset; 1193 else if (copy_to_user(off_in, &offset, sizeof(loff_t))) 1194 ret = -EFAULT; 1195 1196 return ret; 1197 } 1198 1199 return -EINVAL; 1200 } 1201 1202 static int iter_to_pipe(struct iov_iter *from, 1203 struct pipe_inode_info *pipe, 1204 unsigned flags) 1205 { 1206 struct pipe_buffer buf = { 1207 .ops = &user_page_pipe_buf_ops, 1208 .flags = flags 1209 }; 1210 size_t total = 0; 1211 int ret = 0; 1212 bool failed = false; 1213 1214 while (iov_iter_count(from) && !failed) { 1215 struct page *pages[16]; 1216 ssize_t copied; 1217 size_t start; 1218 int n; 1219 1220 copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start); 1221 if (copied <= 0) { 1222 ret = copied; 1223 break; 1224 } 1225 1226 for (n = 0; copied; n++, start = 0) { 1227 int size = min_t(int, copied, PAGE_SIZE - start); 1228 if (!failed) { 1229 buf.page = pages[n]; 1230 buf.offset = start; 1231 buf.len = size; 1232 ret = add_to_pipe(pipe, &buf); 1233 if (unlikely(ret < 0)) { 1234 failed = true; 1235 } else { 1236 iov_iter_advance(from, ret); 1237 total += ret; 1238 } 1239 } else { 1240 put_page(pages[n]); 1241 } 1242 copied -= size; 1243 } 1244 } 1245 return total ? total : ret; 1246 } 1247 1248 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1249 struct splice_desc *sd) 1250 { 1251 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); 1252 return n == sd->len ? n : -EFAULT; 1253 } 1254 1255 /* 1256 * For lack of a better implementation, implement vmsplice() to userspace 1257 * as a simple copy of the pipes pages to the user iov. 1258 */ 1259 static long vmsplice_to_user(struct file *file, const struct iovec __user *uiov, 1260 unsigned long nr_segs, unsigned int flags) 1261 { 1262 struct pipe_inode_info *pipe; 1263 struct splice_desc sd; 1264 long ret; 1265 struct iovec iovstack[UIO_FASTIOV]; 1266 struct iovec *iov = iovstack; 1267 struct iov_iter iter; 1268 1269 pipe = get_pipe_info(file); 1270 if (!pipe) 1271 return -EBADF; 1272 1273 ret = import_iovec(READ, uiov, nr_segs, 1274 ARRAY_SIZE(iovstack), &iov, &iter); 1275 if (ret < 0) 1276 return ret; 1277 1278 sd.total_len = iov_iter_count(&iter); 1279 sd.len = 0; 1280 sd.flags = flags; 1281 sd.u.data = &iter; 1282 sd.pos = 0; 1283 1284 if (sd.total_len) { 1285 pipe_lock(pipe); 1286 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); 1287 pipe_unlock(pipe); 1288 } 1289 1290 kfree(iov); 1291 return ret; 1292 } 1293 1294 /* 1295 * vmsplice splices a user address range into a pipe. It can be thought of 1296 * as splice-from-memory, where the regular splice is splice-from-file (or 1297 * to file). In both cases the output is a pipe, naturally. 1298 */ 1299 static long vmsplice_to_pipe(struct file *file, const struct iovec __user *uiov, 1300 unsigned long nr_segs, unsigned int flags) 1301 { 1302 struct pipe_inode_info *pipe; 1303 struct iovec iovstack[UIO_FASTIOV]; 1304 struct iovec *iov = iovstack; 1305 struct iov_iter from; 1306 long ret; 1307 unsigned buf_flag = 0; 1308 1309 if (flags & SPLICE_F_GIFT) 1310 buf_flag = PIPE_BUF_FLAG_GIFT; 1311 1312 pipe = get_pipe_info(file); 1313 if (!pipe) 1314 return -EBADF; 1315 1316 ret = import_iovec(WRITE, uiov, nr_segs, 1317 ARRAY_SIZE(iovstack), &iov, &from); 1318 if (ret < 0) 1319 return ret; 1320 1321 pipe_lock(pipe); 1322 ret = wait_for_space(pipe, flags); 1323 if (!ret) 1324 ret = iter_to_pipe(&from, pipe, buf_flag); 1325 pipe_unlock(pipe); 1326 if (ret > 0) 1327 wakeup_pipe_readers(pipe); 1328 kfree(iov); 1329 return ret; 1330 } 1331 1332 /* 1333 * Note that vmsplice only really supports true splicing _from_ user memory 1334 * to a pipe, not the other way around. Splicing from user memory is a simple 1335 * operation that can be supported without any funky alignment restrictions 1336 * or nasty vm tricks. We simply map in the user memory and fill them into 1337 * a pipe. The reverse isn't quite as easy, though. There are two possible 1338 * solutions for that: 1339 * 1340 * - memcpy() the data internally, at which point we might as well just 1341 * do a regular read() on the buffer anyway. 1342 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1343 * has restriction limitations on both ends of the pipe). 1344 * 1345 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1346 * 1347 */ 1348 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov, 1349 unsigned long, nr_segs, unsigned int, flags) 1350 { 1351 struct fd f; 1352 long error; 1353 1354 if (unlikely(nr_segs > UIO_MAXIOV)) 1355 return -EINVAL; 1356 else if (unlikely(!nr_segs)) 1357 return 0; 1358 1359 error = -EBADF; 1360 f = fdget(fd); 1361 if (f.file) { 1362 if (f.file->f_mode & FMODE_WRITE) 1363 error = vmsplice_to_pipe(f.file, iov, nr_segs, flags); 1364 else if (f.file->f_mode & FMODE_READ) 1365 error = vmsplice_to_user(f.file, iov, nr_segs, flags); 1366 1367 fdput(f); 1368 } 1369 1370 return error; 1371 } 1372 1373 #ifdef CONFIG_COMPAT 1374 COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32, 1375 unsigned int, nr_segs, unsigned int, flags) 1376 { 1377 unsigned i; 1378 struct iovec __user *iov; 1379 if (nr_segs > UIO_MAXIOV) 1380 return -EINVAL; 1381 iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec)); 1382 for (i = 0; i < nr_segs; i++) { 1383 struct compat_iovec v; 1384 if (get_user(v.iov_base, &iov32[i].iov_base) || 1385 get_user(v.iov_len, &iov32[i].iov_len) || 1386 put_user(compat_ptr(v.iov_base), &iov[i].iov_base) || 1387 put_user(v.iov_len, &iov[i].iov_len)) 1388 return -EFAULT; 1389 } 1390 return sys_vmsplice(fd, iov, nr_segs, flags); 1391 } 1392 #endif 1393 1394 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, 1395 int, fd_out, loff_t __user *, off_out, 1396 size_t, len, unsigned int, flags) 1397 { 1398 struct fd in, out; 1399 long error; 1400 1401 if (unlikely(!len)) 1402 return 0; 1403 1404 error = -EBADF; 1405 in = fdget(fd_in); 1406 if (in.file) { 1407 if (in.file->f_mode & FMODE_READ) { 1408 out = fdget(fd_out); 1409 if (out.file) { 1410 if (out.file->f_mode & FMODE_WRITE) 1411 error = do_splice(in.file, off_in, 1412 out.file, off_out, 1413 len, flags); 1414 fdput(out); 1415 } 1416 } 1417 fdput(in); 1418 } 1419 return error; 1420 } 1421 1422 /* 1423 * Make sure there's data to read. Wait for input if we can, otherwise 1424 * return an appropriate error. 1425 */ 1426 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1427 { 1428 int ret; 1429 1430 /* 1431 * Check ->nrbufs without the inode lock first. This function 1432 * is speculative anyways, so missing one is ok. 1433 */ 1434 if (pipe->nrbufs) 1435 return 0; 1436 1437 ret = 0; 1438 pipe_lock(pipe); 1439 1440 while (!pipe->nrbufs) { 1441 if (signal_pending(current)) { 1442 ret = -ERESTARTSYS; 1443 break; 1444 } 1445 if (!pipe->writers) 1446 break; 1447 if (!pipe->waiting_writers) { 1448 if (flags & SPLICE_F_NONBLOCK) { 1449 ret = -EAGAIN; 1450 break; 1451 } 1452 } 1453 pipe_wait(pipe); 1454 } 1455 1456 pipe_unlock(pipe); 1457 return ret; 1458 } 1459 1460 /* 1461 * Make sure there's writeable room. Wait for room if we can, otherwise 1462 * return an appropriate error. 1463 */ 1464 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1465 { 1466 int ret; 1467 1468 /* 1469 * Check ->nrbufs without the inode lock first. This function 1470 * is speculative anyways, so missing one is ok. 1471 */ 1472 if (pipe->nrbufs < pipe->buffers) 1473 return 0; 1474 1475 ret = 0; 1476 pipe_lock(pipe); 1477 1478 while (pipe->nrbufs >= pipe->buffers) { 1479 if (!pipe->readers) { 1480 send_sig(SIGPIPE, current, 0); 1481 ret = -EPIPE; 1482 break; 1483 } 1484 if (flags & SPLICE_F_NONBLOCK) { 1485 ret = -EAGAIN; 1486 break; 1487 } 1488 if (signal_pending(current)) { 1489 ret = -ERESTARTSYS; 1490 break; 1491 } 1492 pipe->waiting_writers++; 1493 pipe_wait(pipe); 1494 pipe->waiting_writers--; 1495 } 1496 1497 pipe_unlock(pipe); 1498 return ret; 1499 } 1500 1501 /* 1502 * Splice contents of ipipe to opipe. 1503 */ 1504 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1505 struct pipe_inode_info *opipe, 1506 size_t len, unsigned int flags) 1507 { 1508 struct pipe_buffer *ibuf, *obuf; 1509 int ret = 0, nbuf; 1510 bool input_wakeup = false; 1511 1512 1513 retry: 1514 ret = ipipe_prep(ipipe, flags); 1515 if (ret) 1516 return ret; 1517 1518 ret = opipe_prep(opipe, flags); 1519 if (ret) 1520 return ret; 1521 1522 /* 1523 * Potential ABBA deadlock, work around it by ordering lock 1524 * grabbing by pipe info address. Otherwise two different processes 1525 * could deadlock (one doing tee from A -> B, the other from B -> A). 1526 */ 1527 pipe_double_lock(ipipe, opipe); 1528 1529 do { 1530 if (!opipe->readers) { 1531 send_sig(SIGPIPE, current, 0); 1532 if (!ret) 1533 ret = -EPIPE; 1534 break; 1535 } 1536 1537 if (!ipipe->nrbufs && !ipipe->writers) 1538 break; 1539 1540 /* 1541 * Cannot make any progress, because either the input 1542 * pipe is empty or the output pipe is full. 1543 */ 1544 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) { 1545 /* Already processed some buffers, break */ 1546 if (ret) 1547 break; 1548 1549 if (flags & SPLICE_F_NONBLOCK) { 1550 ret = -EAGAIN; 1551 break; 1552 } 1553 1554 /* 1555 * We raced with another reader/writer and haven't 1556 * managed to process any buffers. A zero return 1557 * value means EOF, so retry instead. 1558 */ 1559 pipe_unlock(ipipe); 1560 pipe_unlock(opipe); 1561 goto retry; 1562 } 1563 1564 ibuf = ipipe->bufs + ipipe->curbuf; 1565 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1566 obuf = opipe->bufs + nbuf; 1567 1568 if (len >= ibuf->len) { 1569 /* 1570 * Simply move the whole buffer from ipipe to opipe 1571 */ 1572 *obuf = *ibuf; 1573 ibuf->ops = NULL; 1574 opipe->nrbufs++; 1575 ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1); 1576 ipipe->nrbufs--; 1577 input_wakeup = true; 1578 } else { 1579 /* 1580 * Get a reference to this pipe buffer, 1581 * so we can copy the contents over. 1582 */ 1583 pipe_buf_get(ipipe, ibuf); 1584 *obuf = *ibuf; 1585 1586 /* 1587 * Don't inherit the gift flag, we need to 1588 * prevent multiple steals of this page. 1589 */ 1590 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1591 1592 obuf->len = len; 1593 opipe->nrbufs++; 1594 ibuf->offset += obuf->len; 1595 ibuf->len -= obuf->len; 1596 } 1597 ret += obuf->len; 1598 len -= obuf->len; 1599 } while (len); 1600 1601 pipe_unlock(ipipe); 1602 pipe_unlock(opipe); 1603 1604 /* 1605 * If we put data in the output pipe, wakeup any potential readers. 1606 */ 1607 if (ret > 0) 1608 wakeup_pipe_readers(opipe); 1609 1610 if (input_wakeup) 1611 wakeup_pipe_writers(ipipe); 1612 1613 return ret; 1614 } 1615 1616 /* 1617 * Link contents of ipipe to opipe. 1618 */ 1619 static int link_pipe(struct pipe_inode_info *ipipe, 1620 struct pipe_inode_info *opipe, 1621 size_t len, unsigned int flags) 1622 { 1623 struct pipe_buffer *ibuf, *obuf; 1624 int ret = 0, i = 0, nbuf; 1625 1626 /* 1627 * Potential ABBA deadlock, work around it by ordering lock 1628 * grabbing by pipe info address. Otherwise two different processes 1629 * could deadlock (one doing tee from A -> B, the other from B -> A). 1630 */ 1631 pipe_double_lock(ipipe, opipe); 1632 1633 do { 1634 if (!opipe->readers) { 1635 send_sig(SIGPIPE, current, 0); 1636 if (!ret) 1637 ret = -EPIPE; 1638 break; 1639 } 1640 1641 /* 1642 * If we have iterated all input buffers or ran out of 1643 * output room, break. 1644 */ 1645 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) 1646 break; 1647 1648 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1)); 1649 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1650 1651 /* 1652 * Get a reference to this pipe buffer, 1653 * so we can copy the contents over. 1654 */ 1655 pipe_buf_get(ipipe, ibuf); 1656 1657 obuf = opipe->bufs + nbuf; 1658 *obuf = *ibuf; 1659 1660 /* 1661 * Don't inherit the gift flag, we need to 1662 * prevent multiple steals of this page. 1663 */ 1664 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1665 1666 if (obuf->len > len) 1667 obuf->len = len; 1668 1669 opipe->nrbufs++; 1670 ret += obuf->len; 1671 len -= obuf->len; 1672 i++; 1673 } while (len); 1674 1675 /* 1676 * return EAGAIN if we have the potential of some data in the 1677 * future, otherwise just return 0 1678 */ 1679 if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK)) 1680 ret = -EAGAIN; 1681 1682 pipe_unlock(ipipe); 1683 pipe_unlock(opipe); 1684 1685 /* 1686 * If we put data in the output pipe, wakeup any potential readers. 1687 */ 1688 if (ret > 0) 1689 wakeup_pipe_readers(opipe); 1690 1691 return ret; 1692 } 1693 1694 /* 1695 * This is a tee(1) implementation that works on pipes. It doesn't copy 1696 * any data, it simply references the 'in' pages on the 'out' pipe. 1697 * The 'flags' used are the SPLICE_F_* variants, currently the only 1698 * applicable one is SPLICE_F_NONBLOCK. 1699 */ 1700 static long do_tee(struct file *in, struct file *out, size_t len, 1701 unsigned int flags) 1702 { 1703 struct pipe_inode_info *ipipe = get_pipe_info(in); 1704 struct pipe_inode_info *opipe = get_pipe_info(out); 1705 int ret = -EINVAL; 1706 1707 /* 1708 * Duplicate the contents of ipipe to opipe without actually 1709 * copying the data. 1710 */ 1711 if (ipipe && opipe && ipipe != opipe) { 1712 /* 1713 * Keep going, unless we encounter an error. The ipipe/opipe 1714 * ordering doesn't really matter. 1715 */ 1716 ret = ipipe_prep(ipipe, flags); 1717 if (!ret) { 1718 ret = opipe_prep(opipe, flags); 1719 if (!ret) 1720 ret = link_pipe(ipipe, opipe, len, flags); 1721 } 1722 } 1723 1724 return ret; 1725 } 1726 1727 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) 1728 { 1729 struct fd in; 1730 int error; 1731 1732 if (unlikely(!len)) 1733 return 0; 1734 1735 error = -EBADF; 1736 in = fdget(fdin); 1737 if (in.file) { 1738 if (in.file->f_mode & FMODE_READ) { 1739 struct fd out = fdget(fdout); 1740 if (out.file) { 1741 if (out.file->f_mode & FMODE_WRITE) 1742 error = do_tee(in.file, out.file, 1743 len, flags); 1744 fdput(out); 1745 } 1746 } 1747 fdput(in); 1748 } 1749 1750 return error; 1751 } 1752