1 /* 2 * "splice": joining two ropes together by interweaving their strands. 3 * 4 * This is the "extended pipe" functionality, where a pipe is used as 5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 6 * buffer that you can use to transfer data from one end to the other. 7 * 8 * The traditional unix read/write is extended with a "splice()" operation 9 * that transfers data buffers to or from a pipe buffer. 10 * 11 * Named by Larry McVoy, original implementation from Linus, extended by 12 * Jens to support splicing to files, network, direct splicing, etc and 13 * fixing lots of bugs. 14 * 15 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 18 * 19 */ 20 #include <linux/fs.h> 21 #include <linux/file.h> 22 #include <linux/pagemap.h> 23 #include <linux/splice.h> 24 #include <linux/memcontrol.h> 25 #include <linux/mm_inline.h> 26 #include <linux/swap.h> 27 #include <linux/writeback.h> 28 #include <linux/export.h> 29 #include <linux/syscalls.h> 30 #include <linux/uio.h> 31 #include <linux/security.h> 32 #include <linux/gfp.h> 33 #include <linux/socket.h> 34 #include <linux/compat.h> 35 #include "internal.h" 36 37 /* 38 * Attempt to steal a page from a pipe buffer. This should perhaps go into 39 * a vm helper function, it's already simplified quite a bit by the 40 * addition of remove_mapping(). If success is returned, the caller may 41 * attempt to reuse this page for another destination. 42 */ 43 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, 44 struct pipe_buffer *buf) 45 { 46 struct page *page = buf->page; 47 struct address_space *mapping; 48 49 lock_page(page); 50 51 mapping = page_mapping(page); 52 if (mapping) { 53 WARN_ON(!PageUptodate(page)); 54 55 /* 56 * At least for ext2 with nobh option, we need to wait on 57 * writeback completing on this page, since we'll remove it 58 * from the pagecache. Otherwise truncate wont wait on the 59 * page, allowing the disk blocks to be reused by someone else 60 * before we actually wrote our data to them. fs corruption 61 * ensues. 62 */ 63 wait_on_page_writeback(page); 64 65 if (page_has_private(page) && 66 !try_to_release_page(page, GFP_KERNEL)) 67 goto out_unlock; 68 69 /* 70 * If we succeeded in removing the mapping, set LRU flag 71 * and return good. 72 */ 73 if (remove_mapping(mapping, page)) { 74 buf->flags |= PIPE_BUF_FLAG_LRU; 75 return 0; 76 } 77 } 78 79 /* 80 * Raced with truncate or failed to remove page from current 81 * address space, unlock and return failure. 82 */ 83 out_unlock: 84 unlock_page(page); 85 return 1; 86 } 87 88 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, 89 struct pipe_buffer *buf) 90 { 91 put_page(buf->page); 92 buf->flags &= ~PIPE_BUF_FLAG_LRU; 93 } 94 95 /* 96 * Check whether the contents of buf is OK to access. Since the content 97 * is a page cache page, IO may be in flight. 98 */ 99 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 100 struct pipe_buffer *buf) 101 { 102 struct page *page = buf->page; 103 int err; 104 105 if (!PageUptodate(page)) { 106 lock_page(page); 107 108 /* 109 * Page got truncated/unhashed. This will cause a 0-byte 110 * splice, if this is the first page. 111 */ 112 if (!page->mapping) { 113 err = -ENODATA; 114 goto error; 115 } 116 117 /* 118 * Uh oh, read-error from disk. 119 */ 120 if (!PageUptodate(page)) { 121 err = -EIO; 122 goto error; 123 } 124 125 /* 126 * Page is ok afterall, we are done. 127 */ 128 unlock_page(page); 129 } 130 131 return 0; 132 error: 133 unlock_page(page); 134 return err; 135 } 136 137 const struct pipe_buf_operations page_cache_pipe_buf_ops = { 138 .can_merge = 0, 139 .confirm = page_cache_pipe_buf_confirm, 140 .release = page_cache_pipe_buf_release, 141 .steal = page_cache_pipe_buf_steal, 142 .get = generic_pipe_buf_get, 143 }; 144 145 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, 146 struct pipe_buffer *buf) 147 { 148 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) 149 return 1; 150 151 buf->flags |= PIPE_BUF_FLAG_LRU; 152 return generic_pipe_buf_steal(pipe, buf); 153 } 154 155 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 156 .can_merge = 0, 157 .confirm = generic_pipe_buf_confirm, 158 .release = page_cache_pipe_buf_release, 159 .steal = user_page_pipe_buf_steal, 160 .get = generic_pipe_buf_get, 161 }; 162 163 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) 164 { 165 smp_mb(); 166 if (waitqueue_active(&pipe->wait)) 167 wake_up_interruptible(&pipe->wait); 168 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 169 } 170 171 /** 172 * splice_to_pipe - fill passed data into a pipe 173 * @pipe: pipe to fill 174 * @spd: data to fill 175 * 176 * Description: 177 * @spd contains a map of pages and len/offset tuples, along with 178 * the struct pipe_buf_operations associated with these pages. This 179 * function will link that data to the pipe. 180 * 181 */ 182 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 183 struct splice_pipe_desc *spd) 184 { 185 unsigned int spd_pages = spd->nr_pages; 186 int ret = 0, page_nr = 0; 187 188 if (!spd_pages) 189 return 0; 190 191 if (unlikely(!pipe->readers)) { 192 send_sig(SIGPIPE, current, 0); 193 ret = -EPIPE; 194 goto out; 195 } 196 197 while (pipe->nrbufs < pipe->buffers) { 198 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 199 struct pipe_buffer *buf = pipe->bufs + newbuf; 200 201 buf->page = spd->pages[page_nr]; 202 buf->offset = spd->partial[page_nr].offset; 203 buf->len = spd->partial[page_nr].len; 204 buf->private = spd->partial[page_nr].private; 205 buf->ops = spd->ops; 206 207 pipe->nrbufs++; 208 page_nr++; 209 ret += buf->len; 210 211 if (!--spd->nr_pages) 212 break; 213 } 214 215 if (!ret) 216 ret = -EAGAIN; 217 218 out: 219 while (page_nr < spd_pages) 220 spd->spd_release(spd, page_nr++); 221 222 return ret; 223 } 224 EXPORT_SYMBOL_GPL(splice_to_pipe); 225 226 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 227 { 228 int ret; 229 230 if (unlikely(!pipe->readers)) { 231 send_sig(SIGPIPE, current, 0); 232 ret = -EPIPE; 233 } else if (pipe->nrbufs == pipe->buffers) { 234 ret = -EAGAIN; 235 } else { 236 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 237 pipe->bufs[newbuf] = *buf; 238 pipe->nrbufs++; 239 return buf->len; 240 } 241 pipe_buf_release(pipe, buf); 242 return ret; 243 } 244 EXPORT_SYMBOL(add_to_pipe); 245 246 void spd_release_page(struct splice_pipe_desc *spd, unsigned int i) 247 { 248 put_page(spd->pages[i]); 249 } 250 251 /* 252 * Check if we need to grow the arrays holding pages and partial page 253 * descriptions. 254 */ 255 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) 256 { 257 unsigned int buffers = ACCESS_ONCE(pipe->buffers); 258 259 spd->nr_pages_max = buffers; 260 if (buffers <= PIPE_DEF_BUFFERS) 261 return 0; 262 263 spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL); 264 spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL); 265 266 if (spd->pages && spd->partial) 267 return 0; 268 269 kfree(spd->pages); 270 kfree(spd->partial); 271 return -ENOMEM; 272 } 273 274 void splice_shrink_spd(struct splice_pipe_desc *spd) 275 { 276 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) 277 return; 278 279 kfree(spd->pages); 280 kfree(spd->partial); 281 } 282 283 /** 284 * generic_file_splice_read - splice data from file to a pipe 285 * @in: file to splice from 286 * @ppos: position in @in 287 * @pipe: pipe to splice to 288 * @len: number of bytes to splice 289 * @flags: splice modifier flags 290 * 291 * Description: 292 * Will read pages from given file and fill them into a pipe. Can be 293 * used as long as it has more or less sane ->read_iter(). 294 * 295 */ 296 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, 297 struct pipe_inode_info *pipe, size_t len, 298 unsigned int flags) 299 { 300 struct iov_iter to; 301 struct kiocb kiocb; 302 loff_t isize; 303 int idx, ret; 304 305 isize = i_size_read(in->f_mapping->host); 306 if (unlikely(*ppos >= isize)) 307 return 0; 308 309 iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len); 310 idx = to.idx; 311 init_sync_kiocb(&kiocb, in); 312 kiocb.ki_pos = *ppos; 313 ret = in->f_op->read_iter(&kiocb, &to); 314 if (ret > 0) { 315 *ppos = kiocb.ki_pos; 316 file_accessed(in); 317 } else if (ret < 0) { 318 to.idx = idx; 319 to.iov_offset = 0; 320 iov_iter_advance(&to, 0); /* to free what was emitted */ 321 /* 322 * callers of ->splice_read() expect -EAGAIN on 323 * "can't put anything in there", rather than -EFAULT. 324 */ 325 if (ret == -EFAULT) 326 ret = -EAGAIN; 327 } 328 329 return ret; 330 } 331 EXPORT_SYMBOL(generic_file_splice_read); 332 333 const struct pipe_buf_operations default_pipe_buf_ops = { 334 .can_merge = 0, 335 .confirm = generic_pipe_buf_confirm, 336 .release = generic_pipe_buf_release, 337 .steal = generic_pipe_buf_steal, 338 .get = generic_pipe_buf_get, 339 }; 340 341 static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe, 342 struct pipe_buffer *buf) 343 { 344 return 1; 345 } 346 347 /* Pipe buffer operations for a socket and similar. */ 348 const struct pipe_buf_operations nosteal_pipe_buf_ops = { 349 .can_merge = 0, 350 .confirm = generic_pipe_buf_confirm, 351 .release = generic_pipe_buf_release, 352 .steal = generic_pipe_buf_nosteal, 353 .get = generic_pipe_buf_get, 354 }; 355 EXPORT_SYMBOL(nosteal_pipe_buf_ops); 356 357 static ssize_t kernel_readv(struct file *file, const struct kvec *vec, 358 unsigned long vlen, loff_t offset) 359 { 360 mm_segment_t old_fs; 361 loff_t pos = offset; 362 ssize_t res; 363 364 old_fs = get_fs(); 365 set_fs(get_ds()); 366 /* The cast to a user pointer is valid due to the set_fs() */ 367 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0); 368 set_fs(old_fs); 369 370 return res; 371 } 372 373 ssize_t kernel_write(struct file *file, const char *buf, size_t count, 374 loff_t pos) 375 { 376 mm_segment_t old_fs; 377 ssize_t res; 378 379 old_fs = get_fs(); 380 set_fs(get_ds()); 381 /* The cast to a user pointer is valid due to the set_fs() */ 382 res = vfs_write(file, (__force const char __user *)buf, count, &pos); 383 set_fs(old_fs); 384 385 return res; 386 } 387 EXPORT_SYMBOL(kernel_write); 388 389 static ssize_t default_file_splice_read(struct file *in, loff_t *ppos, 390 struct pipe_inode_info *pipe, size_t len, 391 unsigned int flags) 392 { 393 struct kvec *vec, __vec[PIPE_DEF_BUFFERS]; 394 struct iov_iter to; 395 struct page **pages; 396 unsigned int nr_pages; 397 size_t offset, dummy, copied = 0; 398 ssize_t res; 399 int i; 400 401 if (pipe->nrbufs == pipe->buffers) 402 return -EAGAIN; 403 404 /* 405 * Try to keep page boundaries matching to source pagecache ones - 406 * it probably won't be much help, but... 407 */ 408 offset = *ppos & ~PAGE_MASK; 409 410 iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len + offset); 411 412 res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &dummy); 413 if (res <= 0) 414 return -ENOMEM; 415 416 nr_pages = res / PAGE_SIZE; 417 418 vec = __vec; 419 if (nr_pages > PIPE_DEF_BUFFERS) { 420 vec = kmalloc(nr_pages * sizeof(struct kvec), GFP_KERNEL); 421 if (unlikely(!vec)) { 422 res = -ENOMEM; 423 goto out; 424 } 425 } 426 427 pipe->bufs[to.idx].offset = offset; 428 pipe->bufs[to.idx].len -= offset; 429 430 for (i = 0; i < nr_pages; i++) { 431 size_t this_len = min_t(size_t, len, PAGE_SIZE - offset); 432 vec[i].iov_base = page_address(pages[i]) + offset; 433 vec[i].iov_len = this_len; 434 len -= this_len; 435 offset = 0; 436 } 437 438 res = kernel_readv(in, vec, nr_pages, *ppos); 439 if (res > 0) { 440 copied = res; 441 *ppos += res; 442 } 443 444 if (vec != __vec) 445 kfree(vec); 446 out: 447 for (i = 0; i < nr_pages; i++) 448 put_page(pages[i]); 449 kvfree(pages); 450 iov_iter_advance(&to, copied); /* truncates and discards */ 451 return res; 452 } 453 454 /* 455 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 456 * using sendpage(). Return the number of bytes sent. 457 */ 458 static int pipe_to_sendpage(struct pipe_inode_info *pipe, 459 struct pipe_buffer *buf, struct splice_desc *sd) 460 { 461 struct file *file = sd->u.file; 462 loff_t pos = sd->pos; 463 int more; 464 465 if (!likely(file->f_op->sendpage)) 466 return -EINVAL; 467 468 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; 469 470 if (sd->len < sd->total_len && pipe->nrbufs > 1) 471 more |= MSG_SENDPAGE_NOTLAST; 472 473 return file->f_op->sendpage(file, buf->page, buf->offset, 474 sd->len, &pos, more); 475 } 476 477 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) 478 { 479 smp_mb(); 480 if (waitqueue_active(&pipe->wait)) 481 wake_up_interruptible(&pipe->wait); 482 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 483 } 484 485 /** 486 * splice_from_pipe_feed - feed available data from a pipe to a file 487 * @pipe: pipe to splice from 488 * @sd: information to @actor 489 * @actor: handler that splices the data 490 * 491 * Description: 492 * This function loops over the pipe and calls @actor to do the 493 * actual moving of a single struct pipe_buffer to the desired 494 * destination. It returns when there's no more buffers left in 495 * the pipe or if the requested number of bytes (@sd->total_len) 496 * have been copied. It returns a positive number (one) if the 497 * pipe needs to be filled with more data, zero if the required 498 * number of bytes have been copied and -errno on error. 499 * 500 * This, together with splice_from_pipe_{begin,end,next}, may be 501 * used to implement the functionality of __splice_from_pipe() when 502 * locking is required around copying the pipe buffers to the 503 * destination. 504 */ 505 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, 506 splice_actor *actor) 507 { 508 int ret; 509 510 while (pipe->nrbufs) { 511 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 512 513 sd->len = buf->len; 514 if (sd->len > sd->total_len) 515 sd->len = sd->total_len; 516 517 ret = pipe_buf_confirm(pipe, buf); 518 if (unlikely(ret)) { 519 if (ret == -ENODATA) 520 ret = 0; 521 return ret; 522 } 523 524 ret = actor(pipe, buf, sd); 525 if (ret <= 0) 526 return ret; 527 528 buf->offset += ret; 529 buf->len -= ret; 530 531 sd->num_spliced += ret; 532 sd->len -= ret; 533 sd->pos += ret; 534 sd->total_len -= ret; 535 536 if (!buf->len) { 537 pipe_buf_release(pipe, buf); 538 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 539 pipe->nrbufs--; 540 if (pipe->files) 541 sd->need_wakeup = true; 542 } 543 544 if (!sd->total_len) 545 return 0; 546 } 547 548 return 1; 549 } 550 551 /** 552 * splice_from_pipe_next - wait for some data to splice from 553 * @pipe: pipe to splice from 554 * @sd: information about the splice operation 555 * 556 * Description: 557 * This function will wait for some data and return a positive 558 * value (one) if pipe buffers are available. It will return zero 559 * or -errno if no more data needs to be spliced. 560 */ 561 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) 562 { 563 /* 564 * Check for signal early to make process killable when there are 565 * always buffers available 566 */ 567 if (signal_pending(current)) 568 return -ERESTARTSYS; 569 570 while (!pipe->nrbufs) { 571 if (!pipe->writers) 572 return 0; 573 574 if (!pipe->waiting_writers && sd->num_spliced) 575 return 0; 576 577 if (sd->flags & SPLICE_F_NONBLOCK) 578 return -EAGAIN; 579 580 if (signal_pending(current)) 581 return -ERESTARTSYS; 582 583 if (sd->need_wakeup) { 584 wakeup_pipe_writers(pipe); 585 sd->need_wakeup = false; 586 } 587 588 pipe_wait(pipe); 589 } 590 591 return 1; 592 } 593 594 /** 595 * splice_from_pipe_begin - start splicing from pipe 596 * @sd: information about the splice operation 597 * 598 * Description: 599 * This function should be called before a loop containing 600 * splice_from_pipe_next() and splice_from_pipe_feed() to 601 * initialize the necessary fields of @sd. 602 */ 603 static void splice_from_pipe_begin(struct splice_desc *sd) 604 { 605 sd->num_spliced = 0; 606 sd->need_wakeup = false; 607 } 608 609 /** 610 * splice_from_pipe_end - finish splicing from pipe 611 * @pipe: pipe to splice from 612 * @sd: information about the splice operation 613 * 614 * Description: 615 * This function will wake up pipe writers if necessary. It should 616 * be called after a loop containing splice_from_pipe_next() and 617 * splice_from_pipe_feed(). 618 */ 619 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) 620 { 621 if (sd->need_wakeup) 622 wakeup_pipe_writers(pipe); 623 } 624 625 /** 626 * __splice_from_pipe - splice data from a pipe to given actor 627 * @pipe: pipe to splice from 628 * @sd: information to @actor 629 * @actor: handler that splices the data 630 * 631 * Description: 632 * This function does little more than loop over the pipe and call 633 * @actor to do the actual moving of a single struct pipe_buffer to 634 * the desired destination. See pipe_to_file, pipe_to_sendpage, or 635 * pipe_to_user. 636 * 637 */ 638 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 639 splice_actor *actor) 640 { 641 int ret; 642 643 splice_from_pipe_begin(sd); 644 do { 645 cond_resched(); 646 ret = splice_from_pipe_next(pipe, sd); 647 if (ret > 0) 648 ret = splice_from_pipe_feed(pipe, sd, actor); 649 } while (ret > 0); 650 splice_from_pipe_end(pipe, sd); 651 652 return sd->num_spliced ? sd->num_spliced : ret; 653 } 654 EXPORT_SYMBOL(__splice_from_pipe); 655 656 /** 657 * splice_from_pipe - splice data from a pipe to a file 658 * @pipe: pipe to splice from 659 * @out: file to splice to 660 * @ppos: position in @out 661 * @len: how many bytes to splice 662 * @flags: splice modifier flags 663 * @actor: handler that splices the data 664 * 665 * Description: 666 * See __splice_from_pipe. This function locks the pipe inode, 667 * otherwise it's identical to __splice_from_pipe(). 668 * 669 */ 670 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 671 loff_t *ppos, size_t len, unsigned int flags, 672 splice_actor *actor) 673 { 674 ssize_t ret; 675 struct splice_desc sd = { 676 .total_len = len, 677 .flags = flags, 678 .pos = *ppos, 679 .u.file = out, 680 }; 681 682 pipe_lock(pipe); 683 ret = __splice_from_pipe(pipe, &sd, actor); 684 pipe_unlock(pipe); 685 686 return ret; 687 } 688 689 /** 690 * iter_file_splice_write - splice data from a pipe to a file 691 * @pipe: pipe info 692 * @out: file to write to 693 * @ppos: position in @out 694 * @len: number of bytes to splice 695 * @flags: splice modifier flags 696 * 697 * Description: 698 * Will either move or copy pages (determined by @flags options) from 699 * the given pipe inode to the given file. 700 * This one is ->write_iter-based. 701 * 702 */ 703 ssize_t 704 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 705 loff_t *ppos, size_t len, unsigned int flags) 706 { 707 struct splice_desc sd = { 708 .total_len = len, 709 .flags = flags, 710 .pos = *ppos, 711 .u.file = out, 712 }; 713 int nbufs = pipe->buffers; 714 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec), 715 GFP_KERNEL); 716 ssize_t ret; 717 718 if (unlikely(!array)) 719 return -ENOMEM; 720 721 pipe_lock(pipe); 722 723 splice_from_pipe_begin(&sd); 724 while (sd.total_len) { 725 struct iov_iter from; 726 size_t left; 727 int n, idx; 728 729 ret = splice_from_pipe_next(pipe, &sd); 730 if (ret <= 0) 731 break; 732 733 if (unlikely(nbufs < pipe->buffers)) { 734 kfree(array); 735 nbufs = pipe->buffers; 736 array = kcalloc(nbufs, sizeof(struct bio_vec), 737 GFP_KERNEL); 738 if (!array) { 739 ret = -ENOMEM; 740 break; 741 } 742 } 743 744 /* build the vector */ 745 left = sd.total_len; 746 for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) { 747 struct pipe_buffer *buf = pipe->bufs + idx; 748 size_t this_len = buf->len; 749 750 if (this_len > left) 751 this_len = left; 752 753 if (idx == pipe->buffers - 1) 754 idx = -1; 755 756 ret = pipe_buf_confirm(pipe, buf); 757 if (unlikely(ret)) { 758 if (ret == -ENODATA) 759 ret = 0; 760 goto done; 761 } 762 763 array[n].bv_page = buf->page; 764 array[n].bv_len = this_len; 765 array[n].bv_offset = buf->offset; 766 left -= this_len; 767 } 768 769 iov_iter_bvec(&from, ITER_BVEC | WRITE, array, n, 770 sd.total_len - left); 771 ret = vfs_iter_write(out, &from, &sd.pos); 772 if (ret <= 0) 773 break; 774 775 sd.num_spliced += ret; 776 sd.total_len -= ret; 777 *ppos = sd.pos; 778 779 /* dismiss the fully eaten buffers, adjust the partial one */ 780 while (ret) { 781 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 782 if (ret >= buf->len) { 783 ret -= buf->len; 784 buf->len = 0; 785 pipe_buf_release(pipe, buf); 786 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 787 pipe->nrbufs--; 788 if (pipe->files) 789 sd.need_wakeup = true; 790 } else { 791 buf->offset += ret; 792 buf->len -= ret; 793 ret = 0; 794 } 795 } 796 } 797 done: 798 kfree(array); 799 splice_from_pipe_end(pipe, &sd); 800 801 pipe_unlock(pipe); 802 803 if (sd.num_spliced) 804 ret = sd.num_spliced; 805 806 return ret; 807 } 808 809 EXPORT_SYMBOL(iter_file_splice_write); 810 811 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 812 struct splice_desc *sd) 813 { 814 int ret; 815 void *data; 816 loff_t tmp = sd->pos; 817 818 data = kmap(buf->page); 819 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp); 820 kunmap(buf->page); 821 822 return ret; 823 } 824 825 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, 826 struct file *out, loff_t *ppos, 827 size_t len, unsigned int flags) 828 { 829 ssize_t ret; 830 831 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); 832 if (ret > 0) 833 *ppos += ret; 834 835 return ret; 836 } 837 838 /** 839 * generic_splice_sendpage - splice data from a pipe to a socket 840 * @pipe: pipe to splice from 841 * @out: socket to write to 842 * @ppos: position in @out 843 * @len: number of bytes to splice 844 * @flags: splice modifier flags 845 * 846 * Description: 847 * Will send @len bytes from the pipe to a network socket. No data copying 848 * is involved. 849 * 850 */ 851 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 852 loff_t *ppos, size_t len, unsigned int flags) 853 { 854 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); 855 } 856 857 EXPORT_SYMBOL(generic_splice_sendpage); 858 859 /* 860 * Attempt to initiate a splice from pipe to file. 861 */ 862 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 863 loff_t *ppos, size_t len, unsigned int flags) 864 { 865 ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, 866 loff_t *, size_t, unsigned int); 867 868 if (out->f_op->splice_write) 869 splice_write = out->f_op->splice_write; 870 else 871 splice_write = default_file_splice_write; 872 873 return splice_write(pipe, out, ppos, len, flags); 874 } 875 876 /* 877 * Attempt to initiate a splice from a file to a pipe. 878 */ 879 static long do_splice_to(struct file *in, loff_t *ppos, 880 struct pipe_inode_info *pipe, size_t len, 881 unsigned int flags) 882 { 883 ssize_t (*splice_read)(struct file *, loff_t *, 884 struct pipe_inode_info *, size_t, unsigned int); 885 int ret; 886 887 if (unlikely(!(in->f_mode & FMODE_READ))) 888 return -EBADF; 889 890 ret = rw_verify_area(READ, in, ppos, len); 891 if (unlikely(ret < 0)) 892 return ret; 893 894 if (unlikely(len > MAX_RW_COUNT)) 895 len = MAX_RW_COUNT; 896 897 if (in->f_op->splice_read) 898 splice_read = in->f_op->splice_read; 899 else 900 splice_read = default_file_splice_read; 901 902 return splice_read(in, ppos, pipe, len, flags); 903 } 904 905 /** 906 * splice_direct_to_actor - splices data directly between two non-pipes 907 * @in: file to splice from 908 * @sd: actor information on where to splice to 909 * @actor: handles the data splicing 910 * 911 * Description: 912 * This is a special case helper to splice directly between two 913 * points, without requiring an explicit pipe. Internally an allocated 914 * pipe is cached in the process, and reused during the lifetime of 915 * that process. 916 * 917 */ 918 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 919 splice_direct_actor *actor) 920 { 921 struct pipe_inode_info *pipe; 922 long ret, bytes; 923 umode_t i_mode; 924 size_t len; 925 int i, flags, more; 926 927 /* 928 * We require the input being a regular file, as we don't want to 929 * randomly drop data for eg socket -> socket splicing. Use the 930 * piped splicing for that! 931 */ 932 i_mode = file_inode(in)->i_mode; 933 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) 934 return -EINVAL; 935 936 /* 937 * neither in nor out is a pipe, setup an internal pipe attached to 938 * 'out' and transfer the wanted data from 'in' to 'out' through that 939 */ 940 pipe = current->splice_pipe; 941 if (unlikely(!pipe)) { 942 pipe = alloc_pipe_info(); 943 if (!pipe) 944 return -ENOMEM; 945 946 /* 947 * We don't have an immediate reader, but we'll read the stuff 948 * out of the pipe right after the splice_to_pipe(). So set 949 * PIPE_READERS appropriately. 950 */ 951 pipe->readers = 1; 952 953 current->splice_pipe = pipe; 954 } 955 956 /* 957 * Do the splice. 958 */ 959 ret = 0; 960 bytes = 0; 961 len = sd->total_len; 962 flags = sd->flags; 963 964 /* 965 * Don't block on output, we have to drain the direct pipe. 966 */ 967 sd->flags &= ~SPLICE_F_NONBLOCK; 968 more = sd->flags & SPLICE_F_MORE; 969 970 while (len) { 971 size_t read_len; 972 loff_t pos = sd->pos, prev_pos = pos; 973 974 ret = do_splice_to(in, &pos, pipe, len, flags); 975 if (unlikely(ret <= 0)) 976 goto out_release; 977 978 read_len = ret; 979 sd->total_len = read_len; 980 981 /* 982 * If more data is pending, set SPLICE_F_MORE 983 * If this is the last data and SPLICE_F_MORE was not set 984 * initially, clears it. 985 */ 986 if (read_len < len) 987 sd->flags |= SPLICE_F_MORE; 988 else if (!more) 989 sd->flags &= ~SPLICE_F_MORE; 990 /* 991 * NOTE: nonblocking mode only applies to the input. We 992 * must not do the output in nonblocking mode as then we 993 * could get stuck data in the internal pipe: 994 */ 995 ret = actor(pipe, sd); 996 if (unlikely(ret <= 0)) { 997 sd->pos = prev_pos; 998 goto out_release; 999 } 1000 1001 bytes += ret; 1002 len -= ret; 1003 sd->pos = pos; 1004 1005 if (ret < read_len) { 1006 sd->pos = prev_pos + ret; 1007 goto out_release; 1008 } 1009 } 1010 1011 done: 1012 pipe->nrbufs = pipe->curbuf = 0; 1013 file_accessed(in); 1014 return bytes; 1015 1016 out_release: 1017 /* 1018 * If we did an incomplete transfer we must release 1019 * the pipe buffers in question: 1020 */ 1021 for (i = 0; i < pipe->buffers; i++) { 1022 struct pipe_buffer *buf = pipe->bufs + i; 1023 1024 if (buf->ops) 1025 pipe_buf_release(pipe, buf); 1026 } 1027 1028 if (!bytes) 1029 bytes = ret; 1030 1031 goto done; 1032 } 1033 EXPORT_SYMBOL(splice_direct_to_actor); 1034 1035 static int direct_splice_actor(struct pipe_inode_info *pipe, 1036 struct splice_desc *sd) 1037 { 1038 struct file *file = sd->u.file; 1039 1040 return do_splice_from(pipe, file, sd->opos, sd->total_len, 1041 sd->flags); 1042 } 1043 1044 /** 1045 * do_splice_direct - splices data directly between two files 1046 * @in: file to splice from 1047 * @ppos: input file offset 1048 * @out: file to splice to 1049 * @opos: output file offset 1050 * @len: number of bytes to splice 1051 * @flags: splice modifier flags 1052 * 1053 * Description: 1054 * For use by do_sendfile(). splice can easily emulate sendfile, but 1055 * doing it in the application would incur an extra system call 1056 * (splice in + splice out, as compared to just sendfile()). So this helper 1057 * can splice directly through a process-private pipe. 1058 * 1059 */ 1060 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 1061 loff_t *opos, size_t len, unsigned int flags) 1062 { 1063 struct splice_desc sd = { 1064 .len = len, 1065 .total_len = len, 1066 .flags = flags, 1067 .pos = *ppos, 1068 .u.file = out, 1069 .opos = opos, 1070 }; 1071 long ret; 1072 1073 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1074 return -EBADF; 1075 1076 if (unlikely(out->f_flags & O_APPEND)) 1077 return -EINVAL; 1078 1079 ret = rw_verify_area(WRITE, out, opos, len); 1080 if (unlikely(ret < 0)) 1081 return ret; 1082 1083 ret = splice_direct_to_actor(in, &sd, direct_splice_actor); 1084 if (ret > 0) 1085 *ppos = sd.pos; 1086 1087 return ret; 1088 } 1089 EXPORT_SYMBOL(do_splice_direct); 1090 1091 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) 1092 { 1093 while (pipe->nrbufs == pipe->buffers) { 1094 if (flags & SPLICE_F_NONBLOCK) 1095 return -EAGAIN; 1096 if (signal_pending(current)) 1097 return -ERESTARTSYS; 1098 pipe->waiting_writers++; 1099 pipe_wait(pipe); 1100 pipe->waiting_writers--; 1101 } 1102 return 0; 1103 } 1104 1105 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1106 struct pipe_inode_info *opipe, 1107 size_t len, unsigned int flags); 1108 1109 /* 1110 * Determine where to splice to/from. 1111 */ 1112 static long do_splice(struct file *in, loff_t __user *off_in, 1113 struct file *out, loff_t __user *off_out, 1114 size_t len, unsigned int flags) 1115 { 1116 struct pipe_inode_info *ipipe; 1117 struct pipe_inode_info *opipe; 1118 loff_t offset; 1119 long ret; 1120 1121 ipipe = get_pipe_info(in); 1122 opipe = get_pipe_info(out); 1123 1124 if (ipipe && opipe) { 1125 if (off_in || off_out) 1126 return -ESPIPE; 1127 1128 if (!(in->f_mode & FMODE_READ)) 1129 return -EBADF; 1130 1131 if (!(out->f_mode & FMODE_WRITE)) 1132 return -EBADF; 1133 1134 /* Splicing to self would be fun, but... */ 1135 if (ipipe == opipe) 1136 return -EINVAL; 1137 1138 return splice_pipe_to_pipe(ipipe, opipe, len, flags); 1139 } 1140 1141 if (ipipe) { 1142 if (off_in) 1143 return -ESPIPE; 1144 if (off_out) { 1145 if (!(out->f_mode & FMODE_PWRITE)) 1146 return -EINVAL; 1147 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1148 return -EFAULT; 1149 } else { 1150 offset = out->f_pos; 1151 } 1152 1153 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1154 return -EBADF; 1155 1156 if (unlikely(out->f_flags & O_APPEND)) 1157 return -EINVAL; 1158 1159 ret = rw_verify_area(WRITE, out, &offset, len); 1160 if (unlikely(ret < 0)) 1161 return ret; 1162 1163 file_start_write(out); 1164 ret = do_splice_from(ipipe, out, &offset, len, flags); 1165 file_end_write(out); 1166 1167 if (!off_out) 1168 out->f_pos = offset; 1169 else if (copy_to_user(off_out, &offset, sizeof(loff_t))) 1170 ret = -EFAULT; 1171 1172 return ret; 1173 } 1174 1175 if (opipe) { 1176 if (off_out) 1177 return -ESPIPE; 1178 if (off_in) { 1179 if (!(in->f_mode & FMODE_PREAD)) 1180 return -EINVAL; 1181 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1182 return -EFAULT; 1183 } else { 1184 offset = in->f_pos; 1185 } 1186 1187 pipe_lock(opipe); 1188 ret = wait_for_space(opipe, flags); 1189 if (!ret) 1190 ret = do_splice_to(in, &offset, opipe, len, flags); 1191 pipe_unlock(opipe); 1192 if (ret > 0) 1193 wakeup_pipe_readers(opipe); 1194 if (!off_in) 1195 in->f_pos = offset; 1196 else if (copy_to_user(off_in, &offset, sizeof(loff_t))) 1197 ret = -EFAULT; 1198 1199 return ret; 1200 } 1201 1202 return -EINVAL; 1203 } 1204 1205 static int iter_to_pipe(struct iov_iter *from, 1206 struct pipe_inode_info *pipe, 1207 unsigned flags) 1208 { 1209 struct pipe_buffer buf = { 1210 .ops = &user_page_pipe_buf_ops, 1211 .flags = flags 1212 }; 1213 size_t total = 0; 1214 int ret = 0; 1215 bool failed = false; 1216 1217 while (iov_iter_count(from) && !failed) { 1218 struct page *pages[16]; 1219 ssize_t copied; 1220 size_t start; 1221 int n; 1222 1223 copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start); 1224 if (copied <= 0) { 1225 ret = copied; 1226 break; 1227 } 1228 1229 for (n = 0; copied; n++, start = 0) { 1230 int size = min_t(int, copied, PAGE_SIZE - start); 1231 if (!failed) { 1232 buf.page = pages[n]; 1233 buf.offset = start; 1234 buf.len = size; 1235 ret = add_to_pipe(pipe, &buf); 1236 if (unlikely(ret < 0)) { 1237 failed = true; 1238 } else { 1239 iov_iter_advance(from, ret); 1240 total += ret; 1241 } 1242 } else { 1243 put_page(pages[n]); 1244 } 1245 copied -= size; 1246 } 1247 } 1248 return total ? total : ret; 1249 } 1250 1251 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1252 struct splice_desc *sd) 1253 { 1254 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); 1255 return n == sd->len ? n : -EFAULT; 1256 } 1257 1258 /* 1259 * For lack of a better implementation, implement vmsplice() to userspace 1260 * as a simple copy of the pipes pages to the user iov. 1261 */ 1262 static long vmsplice_to_user(struct file *file, const struct iovec __user *uiov, 1263 unsigned long nr_segs, unsigned int flags) 1264 { 1265 struct pipe_inode_info *pipe; 1266 struct splice_desc sd; 1267 long ret; 1268 struct iovec iovstack[UIO_FASTIOV]; 1269 struct iovec *iov = iovstack; 1270 struct iov_iter iter; 1271 1272 pipe = get_pipe_info(file); 1273 if (!pipe) 1274 return -EBADF; 1275 1276 ret = import_iovec(READ, uiov, nr_segs, 1277 ARRAY_SIZE(iovstack), &iov, &iter); 1278 if (ret < 0) 1279 return ret; 1280 1281 sd.total_len = iov_iter_count(&iter); 1282 sd.len = 0; 1283 sd.flags = flags; 1284 sd.u.data = &iter; 1285 sd.pos = 0; 1286 1287 if (sd.total_len) { 1288 pipe_lock(pipe); 1289 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); 1290 pipe_unlock(pipe); 1291 } 1292 1293 kfree(iov); 1294 return ret; 1295 } 1296 1297 /* 1298 * vmsplice splices a user address range into a pipe. It can be thought of 1299 * as splice-from-memory, where the regular splice is splice-from-file (or 1300 * to file). In both cases the output is a pipe, naturally. 1301 */ 1302 static long vmsplice_to_pipe(struct file *file, const struct iovec __user *uiov, 1303 unsigned long nr_segs, unsigned int flags) 1304 { 1305 struct pipe_inode_info *pipe; 1306 struct iovec iovstack[UIO_FASTIOV]; 1307 struct iovec *iov = iovstack; 1308 struct iov_iter from; 1309 long ret; 1310 unsigned buf_flag = 0; 1311 1312 if (flags & SPLICE_F_GIFT) 1313 buf_flag = PIPE_BUF_FLAG_GIFT; 1314 1315 pipe = get_pipe_info(file); 1316 if (!pipe) 1317 return -EBADF; 1318 1319 ret = import_iovec(WRITE, uiov, nr_segs, 1320 ARRAY_SIZE(iovstack), &iov, &from); 1321 if (ret < 0) 1322 return ret; 1323 1324 pipe_lock(pipe); 1325 ret = wait_for_space(pipe, flags); 1326 if (!ret) 1327 ret = iter_to_pipe(&from, pipe, buf_flag); 1328 pipe_unlock(pipe); 1329 if (ret > 0) 1330 wakeup_pipe_readers(pipe); 1331 kfree(iov); 1332 return ret; 1333 } 1334 1335 /* 1336 * Note that vmsplice only really supports true splicing _from_ user memory 1337 * to a pipe, not the other way around. Splicing from user memory is a simple 1338 * operation that can be supported without any funky alignment restrictions 1339 * or nasty vm tricks. We simply map in the user memory and fill them into 1340 * a pipe. The reverse isn't quite as easy, though. There are two possible 1341 * solutions for that: 1342 * 1343 * - memcpy() the data internally, at which point we might as well just 1344 * do a regular read() on the buffer anyway. 1345 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1346 * has restriction limitations on both ends of the pipe). 1347 * 1348 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1349 * 1350 */ 1351 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov, 1352 unsigned long, nr_segs, unsigned int, flags) 1353 { 1354 struct fd f; 1355 long error; 1356 1357 if (unlikely(nr_segs > UIO_MAXIOV)) 1358 return -EINVAL; 1359 else if (unlikely(!nr_segs)) 1360 return 0; 1361 1362 error = -EBADF; 1363 f = fdget(fd); 1364 if (f.file) { 1365 if (f.file->f_mode & FMODE_WRITE) 1366 error = vmsplice_to_pipe(f.file, iov, nr_segs, flags); 1367 else if (f.file->f_mode & FMODE_READ) 1368 error = vmsplice_to_user(f.file, iov, nr_segs, flags); 1369 1370 fdput(f); 1371 } 1372 1373 return error; 1374 } 1375 1376 #ifdef CONFIG_COMPAT 1377 COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32, 1378 unsigned int, nr_segs, unsigned int, flags) 1379 { 1380 unsigned i; 1381 struct iovec __user *iov; 1382 if (nr_segs > UIO_MAXIOV) 1383 return -EINVAL; 1384 iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec)); 1385 for (i = 0; i < nr_segs; i++) { 1386 struct compat_iovec v; 1387 if (get_user(v.iov_base, &iov32[i].iov_base) || 1388 get_user(v.iov_len, &iov32[i].iov_len) || 1389 put_user(compat_ptr(v.iov_base), &iov[i].iov_base) || 1390 put_user(v.iov_len, &iov[i].iov_len)) 1391 return -EFAULT; 1392 } 1393 return sys_vmsplice(fd, iov, nr_segs, flags); 1394 } 1395 #endif 1396 1397 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, 1398 int, fd_out, loff_t __user *, off_out, 1399 size_t, len, unsigned int, flags) 1400 { 1401 struct fd in, out; 1402 long error; 1403 1404 if (unlikely(!len)) 1405 return 0; 1406 1407 error = -EBADF; 1408 in = fdget(fd_in); 1409 if (in.file) { 1410 if (in.file->f_mode & FMODE_READ) { 1411 out = fdget(fd_out); 1412 if (out.file) { 1413 if (out.file->f_mode & FMODE_WRITE) 1414 error = do_splice(in.file, off_in, 1415 out.file, off_out, 1416 len, flags); 1417 fdput(out); 1418 } 1419 } 1420 fdput(in); 1421 } 1422 return error; 1423 } 1424 1425 /* 1426 * Make sure there's data to read. Wait for input if we can, otherwise 1427 * return an appropriate error. 1428 */ 1429 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1430 { 1431 int ret; 1432 1433 /* 1434 * Check ->nrbufs without the inode lock first. This function 1435 * is speculative anyways, so missing one is ok. 1436 */ 1437 if (pipe->nrbufs) 1438 return 0; 1439 1440 ret = 0; 1441 pipe_lock(pipe); 1442 1443 while (!pipe->nrbufs) { 1444 if (signal_pending(current)) { 1445 ret = -ERESTARTSYS; 1446 break; 1447 } 1448 if (!pipe->writers) 1449 break; 1450 if (!pipe->waiting_writers) { 1451 if (flags & SPLICE_F_NONBLOCK) { 1452 ret = -EAGAIN; 1453 break; 1454 } 1455 } 1456 pipe_wait(pipe); 1457 } 1458 1459 pipe_unlock(pipe); 1460 return ret; 1461 } 1462 1463 /* 1464 * Make sure there's writeable room. Wait for room if we can, otherwise 1465 * return an appropriate error. 1466 */ 1467 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1468 { 1469 int ret; 1470 1471 /* 1472 * Check ->nrbufs without the inode lock first. This function 1473 * is speculative anyways, so missing one is ok. 1474 */ 1475 if (pipe->nrbufs < pipe->buffers) 1476 return 0; 1477 1478 ret = 0; 1479 pipe_lock(pipe); 1480 1481 while (pipe->nrbufs >= pipe->buffers) { 1482 if (!pipe->readers) { 1483 send_sig(SIGPIPE, current, 0); 1484 ret = -EPIPE; 1485 break; 1486 } 1487 if (flags & SPLICE_F_NONBLOCK) { 1488 ret = -EAGAIN; 1489 break; 1490 } 1491 if (signal_pending(current)) { 1492 ret = -ERESTARTSYS; 1493 break; 1494 } 1495 pipe->waiting_writers++; 1496 pipe_wait(pipe); 1497 pipe->waiting_writers--; 1498 } 1499 1500 pipe_unlock(pipe); 1501 return ret; 1502 } 1503 1504 /* 1505 * Splice contents of ipipe to opipe. 1506 */ 1507 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1508 struct pipe_inode_info *opipe, 1509 size_t len, unsigned int flags) 1510 { 1511 struct pipe_buffer *ibuf, *obuf; 1512 int ret = 0, nbuf; 1513 bool input_wakeup = false; 1514 1515 1516 retry: 1517 ret = ipipe_prep(ipipe, flags); 1518 if (ret) 1519 return ret; 1520 1521 ret = opipe_prep(opipe, flags); 1522 if (ret) 1523 return ret; 1524 1525 /* 1526 * Potential ABBA deadlock, work around it by ordering lock 1527 * grabbing by pipe info address. Otherwise two different processes 1528 * could deadlock (one doing tee from A -> B, the other from B -> A). 1529 */ 1530 pipe_double_lock(ipipe, opipe); 1531 1532 do { 1533 if (!opipe->readers) { 1534 send_sig(SIGPIPE, current, 0); 1535 if (!ret) 1536 ret = -EPIPE; 1537 break; 1538 } 1539 1540 if (!ipipe->nrbufs && !ipipe->writers) 1541 break; 1542 1543 /* 1544 * Cannot make any progress, because either the input 1545 * pipe is empty or the output pipe is full. 1546 */ 1547 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) { 1548 /* Already processed some buffers, break */ 1549 if (ret) 1550 break; 1551 1552 if (flags & SPLICE_F_NONBLOCK) { 1553 ret = -EAGAIN; 1554 break; 1555 } 1556 1557 /* 1558 * We raced with another reader/writer and haven't 1559 * managed to process any buffers. A zero return 1560 * value means EOF, so retry instead. 1561 */ 1562 pipe_unlock(ipipe); 1563 pipe_unlock(opipe); 1564 goto retry; 1565 } 1566 1567 ibuf = ipipe->bufs + ipipe->curbuf; 1568 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1569 obuf = opipe->bufs + nbuf; 1570 1571 if (len >= ibuf->len) { 1572 /* 1573 * Simply move the whole buffer from ipipe to opipe 1574 */ 1575 *obuf = *ibuf; 1576 ibuf->ops = NULL; 1577 opipe->nrbufs++; 1578 ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1); 1579 ipipe->nrbufs--; 1580 input_wakeup = true; 1581 } else { 1582 /* 1583 * Get a reference to this pipe buffer, 1584 * so we can copy the contents over. 1585 */ 1586 pipe_buf_get(ipipe, ibuf); 1587 *obuf = *ibuf; 1588 1589 /* 1590 * Don't inherit the gift flag, we need to 1591 * prevent multiple steals of this page. 1592 */ 1593 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1594 1595 obuf->len = len; 1596 opipe->nrbufs++; 1597 ibuf->offset += obuf->len; 1598 ibuf->len -= obuf->len; 1599 } 1600 ret += obuf->len; 1601 len -= obuf->len; 1602 } while (len); 1603 1604 pipe_unlock(ipipe); 1605 pipe_unlock(opipe); 1606 1607 /* 1608 * If we put data in the output pipe, wakeup any potential readers. 1609 */ 1610 if (ret > 0) 1611 wakeup_pipe_readers(opipe); 1612 1613 if (input_wakeup) 1614 wakeup_pipe_writers(ipipe); 1615 1616 return ret; 1617 } 1618 1619 /* 1620 * Link contents of ipipe to opipe. 1621 */ 1622 static int link_pipe(struct pipe_inode_info *ipipe, 1623 struct pipe_inode_info *opipe, 1624 size_t len, unsigned int flags) 1625 { 1626 struct pipe_buffer *ibuf, *obuf; 1627 int ret = 0, i = 0, nbuf; 1628 1629 /* 1630 * Potential ABBA deadlock, work around it by ordering lock 1631 * grabbing by pipe info address. Otherwise two different processes 1632 * could deadlock (one doing tee from A -> B, the other from B -> A). 1633 */ 1634 pipe_double_lock(ipipe, opipe); 1635 1636 do { 1637 if (!opipe->readers) { 1638 send_sig(SIGPIPE, current, 0); 1639 if (!ret) 1640 ret = -EPIPE; 1641 break; 1642 } 1643 1644 /* 1645 * If we have iterated all input buffers or ran out of 1646 * output room, break. 1647 */ 1648 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) 1649 break; 1650 1651 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1)); 1652 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1653 1654 /* 1655 * Get a reference to this pipe buffer, 1656 * so we can copy the contents over. 1657 */ 1658 pipe_buf_get(ipipe, ibuf); 1659 1660 obuf = opipe->bufs + nbuf; 1661 *obuf = *ibuf; 1662 1663 /* 1664 * Don't inherit the gift flag, we need to 1665 * prevent multiple steals of this page. 1666 */ 1667 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1668 1669 if (obuf->len > len) 1670 obuf->len = len; 1671 1672 opipe->nrbufs++; 1673 ret += obuf->len; 1674 len -= obuf->len; 1675 i++; 1676 } while (len); 1677 1678 /* 1679 * return EAGAIN if we have the potential of some data in the 1680 * future, otherwise just return 0 1681 */ 1682 if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK)) 1683 ret = -EAGAIN; 1684 1685 pipe_unlock(ipipe); 1686 pipe_unlock(opipe); 1687 1688 /* 1689 * If we put data in the output pipe, wakeup any potential readers. 1690 */ 1691 if (ret > 0) 1692 wakeup_pipe_readers(opipe); 1693 1694 return ret; 1695 } 1696 1697 /* 1698 * This is a tee(1) implementation that works on pipes. It doesn't copy 1699 * any data, it simply references the 'in' pages on the 'out' pipe. 1700 * The 'flags' used are the SPLICE_F_* variants, currently the only 1701 * applicable one is SPLICE_F_NONBLOCK. 1702 */ 1703 static long do_tee(struct file *in, struct file *out, size_t len, 1704 unsigned int flags) 1705 { 1706 struct pipe_inode_info *ipipe = get_pipe_info(in); 1707 struct pipe_inode_info *opipe = get_pipe_info(out); 1708 int ret = -EINVAL; 1709 1710 /* 1711 * Duplicate the contents of ipipe to opipe without actually 1712 * copying the data. 1713 */ 1714 if (ipipe && opipe && ipipe != opipe) { 1715 /* 1716 * Keep going, unless we encounter an error. The ipipe/opipe 1717 * ordering doesn't really matter. 1718 */ 1719 ret = ipipe_prep(ipipe, flags); 1720 if (!ret) { 1721 ret = opipe_prep(opipe, flags); 1722 if (!ret) 1723 ret = link_pipe(ipipe, opipe, len, flags); 1724 } 1725 } 1726 1727 return ret; 1728 } 1729 1730 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) 1731 { 1732 struct fd in; 1733 int error; 1734 1735 if (unlikely(!len)) 1736 return 0; 1737 1738 error = -EBADF; 1739 in = fdget(fdin); 1740 if (in.file) { 1741 if (in.file->f_mode & FMODE_READ) { 1742 struct fd out = fdget(fdout); 1743 if (out.file) { 1744 if (out.file->f_mode & FMODE_WRITE) 1745 error = do_tee(in.file, out.file, 1746 len, flags); 1747 fdput(out); 1748 } 1749 } 1750 fdput(in); 1751 } 1752 1753 return error; 1754 } 1755