1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * "splice": joining two ropes together by interweaving their strands. 4 * 5 * This is the "extended pipe" functionality, where a pipe is used as 6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 7 * buffer that you can use to transfer data from one end to the other. 8 * 9 * The traditional unix read/write is extended with a "splice()" operation 10 * that transfers data buffers to or from a pipe buffer. 11 * 12 * Named by Larry McVoy, original implementation from Linus, extended by 13 * Jens to support splicing to files, network, direct splicing, etc and 14 * fixing lots of bugs. 15 * 16 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 17 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 18 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 19 * 20 */ 21 #include <linux/bvec.h> 22 #include <linux/fs.h> 23 #include <linux/file.h> 24 #include <linux/pagemap.h> 25 #include <linux/splice.h> 26 #include <linux/memcontrol.h> 27 #include <linux/mm_inline.h> 28 #include <linux/swap.h> 29 #include <linux/writeback.h> 30 #include <linux/export.h> 31 #include <linux/syscalls.h> 32 #include <linux/uio.h> 33 #include <linux/security.h> 34 #include <linux/gfp.h> 35 #include <linux/socket.h> 36 #include <linux/compat.h> 37 #include <linux/sched/signal.h> 38 39 #include "internal.h" 40 41 /* 42 * Attempt to steal a page from a pipe buffer. This should perhaps go into 43 * a vm helper function, it's already simplified quite a bit by the 44 * addition of remove_mapping(). If success is returned, the caller may 45 * attempt to reuse this page for another destination. 46 */ 47 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe, 48 struct pipe_buffer *buf) 49 { 50 struct page *page = buf->page; 51 struct address_space *mapping; 52 53 lock_page(page); 54 55 mapping = page_mapping(page); 56 if (mapping) { 57 WARN_ON(!PageUptodate(page)); 58 59 /* 60 * At least for ext2 with nobh option, we need to wait on 61 * writeback completing on this page, since we'll remove it 62 * from the pagecache. Otherwise truncate wont wait on the 63 * page, allowing the disk blocks to be reused by someone else 64 * before we actually wrote our data to them. fs corruption 65 * ensues. 66 */ 67 wait_on_page_writeback(page); 68 69 if (page_has_private(page) && 70 !try_to_release_page(page, GFP_KERNEL)) 71 goto out_unlock; 72 73 /* 74 * If we succeeded in removing the mapping, set LRU flag 75 * and return good. 76 */ 77 if (remove_mapping(mapping, page)) { 78 buf->flags |= PIPE_BUF_FLAG_LRU; 79 return true; 80 } 81 } 82 83 /* 84 * Raced with truncate or failed to remove page from current 85 * address space, unlock and return failure. 86 */ 87 out_unlock: 88 unlock_page(page); 89 return false; 90 } 91 92 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, 93 struct pipe_buffer *buf) 94 { 95 put_page(buf->page); 96 buf->flags &= ~PIPE_BUF_FLAG_LRU; 97 } 98 99 /* 100 * Check whether the contents of buf is OK to access. Since the content 101 * is a page cache page, IO may be in flight. 102 */ 103 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 104 struct pipe_buffer *buf) 105 { 106 struct page *page = buf->page; 107 int err; 108 109 if (!PageUptodate(page)) { 110 lock_page(page); 111 112 /* 113 * Page got truncated/unhashed. This will cause a 0-byte 114 * splice, if this is the first page. 115 */ 116 if (!page->mapping) { 117 err = -ENODATA; 118 goto error; 119 } 120 121 /* 122 * Uh oh, read-error from disk. 123 */ 124 if (!PageUptodate(page)) { 125 err = -EIO; 126 goto error; 127 } 128 129 /* 130 * Page is ok afterall, we are done. 131 */ 132 unlock_page(page); 133 } 134 135 return 0; 136 error: 137 unlock_page(page); 138 return err; 139 } 140 141 const struct pipe_buf_operations page_cache_pipe_buf_ops = { 142 .confirm = page_cache_pipe_buf_confirm, 143 .release = page_cache_pipe_buf_release, 144 .try_steal = page_cache_pipe_buf_try_steal, 145 .get = generic_pipe_buf_get, 146 }; 147 148 static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe, 149 struct pipe_buffer *buf) 150 { 151 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) 152 return false; 153 154 buf->flags |= PIPE_BUF_FLAG_LRU; 155 return generic_pipe_buf_try_steal(pipe, buf); 156 } 157 158 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 159 .release = page_cache_pipe_buf_release, 160 .try_steal = user_page_pipe_buf_try_steal, 161 .get = generic_pipe_buf_get, 162 }; 163 164 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) 165 { 166 smp_mb(); 167 if (waitqueue_active(&pipe->rd_wait)) 168 wake_up_interruptible(&pipe->rd_wait); 169 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 170 } 171 172 /** 173 * splice_to_pipe - fill passed data into a pipe 174 * @pipe: pipe to fill 175 * @spd: data to fill 176 * 177 * Description: 178 * @spd contains a map of pages and len/offset tuples, along with 179 * the struct pipe_buf_operations associated with these pages. This 180 * function will link that data to the pipe. 181 * 182 */ 183 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 184 struct splice_pipe_desc *spd) 185 { 186 unsigned int spd_pages = spd->nr_pages; 187 unsigned int tail = pipe->tail; 188 unsigned int head = pipe->head; 189 unsigned int mask = pipe->ring_size - 1; 190 int ret = 0, page_nr = 0; 191 192 if (!spd_pages) 193 return 0; 194 195 if (unlikely(!pipe->readers)) { 196 send_sig(SIGPIPE, current, 0); 197 ret = -EPIPE; 198 goto out; 199 } 200 201 while (!pipe_full(head, tail, pipe->max_usage)) { 202 struct pipe_buffer *buf = &pipe->bufs[head & mask]; 203 204 buf->page = spd->pages[page_nr]; 205 buf->offset = spd->partial[page_nr].offset; 206 buf->len = spd->partial[page_nr].len; 207 buf->private = spd->partial[page_nr].private; 208 buf->ops = spd->ops; 209 buf->flags = 0; 210 211 head++; 212 pipe->head = head; 213 page_nr++; 214 ret += buf->len; 215 216 if (!--spd->nr_pages) 217 break; 218 } 219 220 if (!ret) 221 ret = -EAGAIN; 222 223 out: 224 while (page_nr < spd_pages) 225 spd->spd_release(spd, page_nr++); 226 227 return ret; 228 } 229 EXPORT_SYMBOL_GPL(splice_to_pipe); 230 231 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 232 { 233 unsigned int head = pipe->head; 234 unsigned int tail = pipe->tail; 235 unsigned int mask = pipe->ring_size - 1; 236 int ret; 237 238 if (unlikely(!pipe->readers)) { 239 send_sig(SIGPIPE, current, 0); 240 ret = -EPIPE; 241 } else if (pipe_full(head, tail, pipe->max_usage)) { 242 ret = -EAGAIN; 243 } else { 244 pipe->bufs[head & mask] = *buf; 245 pipe->head = head + 1; 246 return buf->len; 247 } 248 pipe_buf_release(pipe, buf); 249 return ret; 250 } 251 EXPORT_SYMBOL(add_to_pipe); 252 253 /* 254 * Check if we need to grow the arrays holding pages and partial page 255 * descriptions. 256 */ 257 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) 258 { 259 unsigned int max_usage = READ_ONCE(pipe->max_usage); 260 261 spd->nr_pages_max = max_usage; 262 if (max_usage <= PIPE_DEF_BUFFERS) 263 return 0; 264 265 spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL); 266 spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page), 267 GFP_KERNEL); 268 269 if (spd->pages && spd->partial) 270 return 0; 271 272 kfree(spd->pages); 273 kfree(spd->partial); 274 return -ENOMEM; 275 } 276 277 void splice_shrink_spd(struct splice_pipe_desc *spd) 278 { 279 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) 280 return; 281 282 kfree(spd->pages); 283 kfree(spd->partial); 284 } 285 286 /** 287 * generic_file_splice_read - splice data from file to a pipe 288 * @in: file to splice from 289 * @ppos: position in @in 290 * @pipe: pipe to splice to 291 * @len: number of bytes to splice 292 * @flags: splice modifier flags 293 * 294 * Description: 295 * Will read pages from given file and fill them into a pipe. Can be 296 * used as long as it has more or less sane ->read_iter(). 297 * 298 */ 299 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, 300 struct pipe_inode_info *pipe, size_t len, 301 unsigned int flags) 302 { 303 struct iov_iter to; 304 struct kiocb kiocb; 305 unsigned int i_head; 306 int ret; 307 308 iov_iter_pipe(&to, READ, pipe, len); 309 i_head = to.head; 310 init_sync_kiocb(&kiocb, in); 311 kiocb.ki_pos = *ppos; 312 ret = call_read_iter(in, &kiocb, &to); 313 if (ret > 0) { 314 *ppos = kiocb.ki_pos; 315 file_accessed(in); 316 } else if (ret < 0) { 317 to.head = i_head; 318 to.iov_offset = 0; 319 iov_iter_advance(&to, 0); /* to free what was emitted */ 320 /* 321 * callers of ->splice_read() expect -EAGAIN on 322 * "can't put anything in there", rather than -EFAULT. 323 */ 324 if (ret == -EFAULT) 325 ret = -EAGAIN; 326 } 327 328 return ret; 329 } 330 EXPORT_SYMBOL(generic_file_splice_read); 331 332 const struct pipe_buf_operations default_pipe_buf_ops = { 333 .release = generic_pipe_buf_release, 334 .try_steal = generic_pipe_buf_try_steal, 335 .get = generic_pipe_buf_get, 336 }; 337 338 /* Pipe buffer operations for a socket and similar. */ 339 const struct pipe_buf_operations nosteal_pipe_buf_ops = { 340 .release = generic_pipe_buf_release, 341 .get = generic_pipe_buf_get, 342 }; 343 EXPORT_SYMBOL(nosteal_pipe_buf_ops); 344 345 static ssize_t kernel_readv(struct file *file, const struct kvec *vec, 346 unsigned long vlen, loff_t offset) 347 { 348 mm_segment_t old_fs; 349 loff_t pos = offset; 350 ssize_t res; 351 352 old_fs = get_fs(); 353 set_fs(KERNEL_DS); 354 /* The cast to a user pointer is valid due to the set_fs() */ 355 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0); 356 set_fs(old_fs); 357 358 return res; 359 } 360 361 static ssize_t default_file_splice_read(struct file *in, loff_t *ppos, 362 struct pipe_inode_info *pipe, size_t len, 363 unsigned int flags) 364 { 365 struct kvec *vec, __vec[PIPE_DEF_BUFFERS]; 366 struct iov_iter to; 367 struct page **pages; 368 unsigned int nr_pages; 369 unsigned int mask; 370 size_t offset, base, copied = 0; 371 ssize_t res; 372 int i; 373 374 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 375 return -EAGAIN; 376 377 /* 378 * Try to keep page boundaries matching to source pagecache ones - 379 * it probably won't be much help, but... 380 */ 381 offset = *ppos & ~PAGE_MASK; 382 383 iov_iter_pipe(&to, READ, pipe, len + offset); 384 385 res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &base); 386 if (res <= 0) 387 return -ENOMEM; 388 389 nr_pages = DIV_ROUND_UP(res + base, PAGE_SIZE); 390 391 vec = __vec; 392 if (nr_pages > PIPE_DEF_BUFFERS) { 393 vec = kmalloc_array(nr_pages, sizeof(struct kvec), GFP_KERNEL); 394 if (unlikely(!vec)) { 395 res = -ENOMEM; 396 goto out; 397 } 398 } 399 400 mask = pipe->ring_size - 1; 401 pipe->bufs[to.head & mask].offset = offset; 402 pipe->bufs[to.head & mask].len -= offset; 403 404 for (i = 0; i < nr_pages; i++) { 405 size_t this_len = min_t(size_t, len, PAGE_SIZE - offset); 406 vec[i].iov_base = page_address(pages[i]) + offset; 407 vec[i].iov_len = this_len; 408 len -= this_len; 409 offset = 0; 410 } 411 412 res = kernel_readv(in, vec, nr_pages, *ppos); 413 if (res > 0) { 414 copied = res; 415 *ppos += res; 416 } 417 418 if (vec != __vec) 419 kfree(vec); 420 out: 421 for (i = 0; i < nr_pages; i++) 422 put_page(pages[i]); 423 kvfree(pages); 424 iov_iter_advance(&to, copied); /* truncates and discards */ 425 return res; 426 } 427 428 /* 429 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 430 * using sendpage(). Return the number of bytes sent. 431 */ 432 static int pipe_to_sendpage(struct pipe_inode_info *pipe, 433 struct pipe_buffer *buf, struct splice_desc *sd) 434 { 435 struct file *file = sd->u.file; 436 loff_t pos = sd->pos; 437 int more; 438 439 if (!likely(file->f_op->sendpage)) 440 return -EINVAL; 441 442 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; 443 444 if (sd->len < sd->total_len && 445 pipe_occupancy(pipe->head, pipe->tail) > 1) 446 more |= MSG_SENDPAGE_NOTLAST; 447 448 return file->f_op->sendpage(file, buf->page, buf->offset, 449 sd->len, &pos, more); 450 } 451 452 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) 453 { 454 smp_mb(); 455 if (waitqueue_active(&pipe->wr_wait)) 456 wake_up_interruptible(&pipe->wr_wait); 457 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 458 } 459 460 /** 461 * splice_from_pipe_feed - feed available data from a pipe to a file 462 * @pipe: pipe to splice from 463 * @sd: information to @actor 464 * @actor: handler that splices the data 465 * 466 * Description: 467 * This function loops over the pipe and calls @actor to do the 468 * actual moving of a single struct pipe_buffer to the desired 469 * destination. It returns when there's no more buffers left in 470 * the pipe or if the requested number of bytes (@sd->total_len) 471 * have been copied. It returns a positive number (one) if the 472 * pipe needs to be filled with more data, zero if the required 473 * number of bytes have been copied and -errno on error. 474 * 475 * This, together with splice_from_pipe_{begin,end,next}, may be 476 * used to implement the functionality of __splice_from_pipe() when 477 * locking is required around copying the pipe buffers to the 478 * destination. 479 */ 480 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, 481 splice_actor *actor) 482 { 483 unsigned int head = pipe->head; 484 unsigned int tail = pipe->tail; 485 unsigned int mask = pipe->ring_size - 1; 486 int ret; 487 488 while (!pipe_empty(head, tail)) { 489 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; 490 491 sd->len = buf->len; 492 if (sd->len > sd->total_len) 493 sd->len = sd->total_len; 494 495 ret = pipe_buf_confirm(pipe, buf); 496 if (unlikely(ret)) { 497 if (ret == -ENODATA) 498 ret = 0; 499 return ret; 500 } 501 502 ret = actor(pipe, buf, sd); 503 if (ret <= 0) 504 return ret; 505 506 buf->offset += ret; 507 buf->len -= ret; 508 509 sd->num_spliced += ret; 510 sd->len -= ret; 511 sd->pos += ret; 512 sd->total_len -= ret; 513 514 if (!buf->len) { 515 pipe_buf_release(pipe, buf); 516 tail++; 517 pipe->tail = tail; 518 if (pipe->files) 519 sd->need_wakeup = true; 520 } 521 522 if (!sd->total_len) 523 return 0; 524 } 525 526 return 1; 527 } 528 529 /** 530 * splice_from_pipe_next - wait for some data to splice from 531 * @pipe: pipe to splice from 532 * @sd: information about the splice operation 533 * 534 * Description: 535 * This function will wait for some data and return a positive 536 * value (one) if pipe buffers are available. It will return zero 537 * or -errno if no more data needs to be spliced. 538 */ 539 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) 540 { 541 /* 542 * Check for signal early to make process killable when there are 543 * always buffers available 544 */ 545 if (signal_pending(current)) 546 return -ERESTARTSYS; 547 548 while (pipe_empty(pipe->head, pipe->tail)) { 549 if (!pipe->writers) 550 return 0; 551 552 if (sd->num_spliced) 553 return 0; 554 555 if (sd->flags & SPLICE_F_NONBLOCK) 556 return -EAGAIN; 557 558 if (signal_pending(current)) 559 return -ERESTARTSYS; 560 561 if (sd->need_wakeup) { 562 wakeup_pipe_writers(pipe); 563 sd->need_wakeup = false; 564 } 565 566 pipe_wait(pipe); 567 } 568 569 return 1; 570 } 571 572 /** 573 * splice_from_pipe_begin - start splicing from pipe 574 * @sd: information about the splice operation 575 * 576 * Description: 577 * This function should be called before a loop containing 578 * splice_from_pipe_next() and splice_from_pipe_feed() to 579 * initialize the necessary fields of @sd. 580 */ 581 static void splice_from_pipe_begin(struct splice_desc *sd) 582 { 583 sd->num_spliced = 0; 584 sd->need_wakeup = false; 585 } 586 587 /** 588 * splice_from_pipe_end - finish splicing from pipe 589 * @pipe: pipe to splice from 590 * @sd: information about the splice operation 591 * 592 * Description: 593 * This function will wake up pipe writers if necessary. It should 594 * be called after a loop containing splice_from_pipe_next() and 595 * splice_from_pipe_feed(). 596 */ 597 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) 598 { 599 if (sd->need_wakeup) 600 wakeup_pipe_writers(pipe); 601 } 602 603 /** 604 * __splice_from_pipe - splice data from a pipe to given actor 605 * @pipe: pipe to splice from 606 * @sd: information to @actor 607 * @actor: handler that splices the data 608 * 609 * Description: 610 * This function does little more than loop over the pipe and call 611 * @actor to do the actual moving of a single struct pipe_buffer to 612 * the desired destination. See pipe_to_file, pipe_to_sendpage, or 613 * pipe_to_user. 614 * 615 */ 616 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 617 splice_actor *actor) 618 { 619 int ret; 620 621 splice_from_pipe_begin(sd); 622 do { 623 cond_resched(); 624 ret = splice_from_pipe_next(pipe, sd); 625 if (ret > 0) 626 ret = splice_from_pipe_feed(pipe, sd, actor); 627 } while (ret > 0); 628 splice_from_pipe_end(pipe, sd); 629 630 return sd->num_spliced ? sd->num_spliced : ret; 631 } 632 EXPORT_SYMBOL(__splice_from_pipe); 633 634 /** 635 * splice_from_pipe - splice data from a pipe to a file 636 * @pipe: pipe to splice from 637 * @out: file to splice to 638 * @ppos: position in @out 639 * @len: how many bytes to splice 640 * @flags: splice modifier flags 641 * @actor: handler that splices the data 642 * 643 * Description: 644 * See __splice_from_pipe. This function locks the pipe inode, 645 * otherwise it's identical to __splice_from_pipe(). 646 * 647 */ 648 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 649 loff_t *ppos, size_t len, unsigned int flags, 650 splice_actor *actor) 651 { 652 ssize_t ret; 653 struct splice_desc sd = { 654 .total_len = len, 655 .flags = flags, 656 .pos = *ppos, 657 .u.file = out, 658 }; 659 660 pipe_lock(pipe); 661 ret = __splice_from_pipe(pipe, &sd, actor); 662 pipe_unlock(pipe); 663 664 return ret; 665 } 666 667 /** 668 * iter_file_splice_write - splice data from a pipe to a file 669 * @pipe: pipe info 670 * @out: file to write to 671 * @ppos: position in @out 672 * @len: number of bytes to splice 673 * @flags: splice modifier flags 674 * 675 * Description: 676 * Will either move or copy pages (determined by @flags options) from 677 * the given pipe inode to the given file. 678 * This one is ->write_iter-based. 679 * 680 */ 681 ssize_t 682 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 683 loff_t *ppos, size_t len, unsigned int flags) 684 { 685 struct splice_desc sd = { 686 .total_len = len, 687 .flags = flags, 688 .pos = *ppos, 689 .u.file = out, 690 }; 691 int nbufs = pipe->max_usage; 692 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec), 693 GFP_KERNEL); 694 ssize_t ret; 695 696 if (unlikely(!array)) 697 return -ENOMEM; 698 699 pipe_lock(pipe); 700 701 splice_from_pipe_begin(&sd); 702 while (sd.total_len) { 703 struct iov_iter from; 704 unsigned int head, tail, mask; 705 size_t left; 706 int n; 707 708 ret = splice_from_pipe_next(pipe, &sd); 709 if (ret <= 0) 710 break; 711 712 if (unlikely(nbufs < pipe->max_usage)) { 713 kfree(array); 714 nbufs = pipe->max_usage; 715 array = kcalloc(nbufs, sizeof(struct bio_vec), 716 GFP_KERNEL); 717 if (!array) { 718 ret = -ENOMEM; 719 break; 720 } 721 } 722 723 head = pipe->head; 724 tail = pipe->tail; 725 mask = pipe->ring_size - 1; 726 727 /* build the vector */ 728 left = sd.total_len; 729 for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) { 730 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; 731 size_t this_len = buf->len; 732 733 if (this_len > left) 734 this_len = left; 735 736 ret = pipe_buf_confirm(pipe, buf); 737 if (unlikely(ret)) { 738 if (ret == -ENODATA) 739 ret = 0; 740 goto done; 741 } 742 743 array[n].bv_page = buf->page; 744 array[n].bv_len = this_len; 745 array[n].bv_offset = buf->offset; 746 left -= this_len; 747 } 748 749 iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left); 750 ret = vfs_iter_write(out, &from, &sd.pos, 0); 751 if (ret <= 0) 752 break; 753 754 sd.num_spliced += ret; 755 sd.total_len -= ret; 756 *ppos = sd.pos; 757 758 /* dismiss the fully eaten buffers, adjust the partial one */ 759 tail = pipe->tail; 760 while (ret) { 761 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; 762 if (ret >= buf->len) { 763 ret -= buf->len; 764 buf->len = 0; 765 pipe_buf_release(pipe, buf); 766 tail++; 767 pipe->tail = tail; 768 if (pipe->files) 769 sd.need_wakeup = true; 770 } else { 771 buf->offset += ret; 772 buf->len -= ret; 773 ret = 0; 774 } 775 } 776 } 777 done: 778 kfree(array); 779 splice_from_pipe_end(pipe, &sd); 780 781 pipe_unlock(pipe); 782 783 if (sd.num_spliced) 784 ret = sd.num_spliced; 785 786 return ret; 787 } 788 789 EXPORT_SYMBOL(iter_file_splice_write); 790 791 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 792 struct splice_desc *sd) 793 { 794 int ret; 795 void *data; 796 loff_t tmp = sd->pos; 797 798 data = kmap(buf->page); 799 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp); 800 kunmap(buf->page); 801 802 return ret; 803 } 804 805 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, 806 struct file *out, loff_t *ppos, 807 size_t len, unsigned int flags) 808 { 809 ssize_t ret; 810 811 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); 812 if (ret > 0) 813 *ppos += ret; 814 815 return ret; 816 } 817 818 /** 819 * generic_splice_sendpage - splice data from a pipe to a socket 820 * @pipe: pipe to splice from 821 * @out: socket to write to 822 * @ppos: position in @out 823 * @len: number of bytes to splice 824 * @flags: splice modifier flags 825 * 826 * Description: 827 * Will send @len bytes from the pipe to a network socket. No data copying 828 * is involved. 829 * 830 */ 831 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 832 loff_t *ppos, size_t len, unsigned int flags) 833 { 834 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); 835 } 836 837 EXPORT_SYMBOL(generic_splice_sendpage); 838 839 /* 840 * Attempt to initiate a splice from pipe to file. 841 */ 842 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 843 loff_t *ppos, size_t len, unsigned int flags) 844 { 845 if (out->f_op->splice_write) 846 return out->f_op->splice_write(pipe, out, ppos, len, flags); 847 return default_file_splice_write(pipe, out, ppos, len, flags); 848 } 849 850 /* 851 * Attempt to initiate a splice from a file to a pipe. 852 */ 853 static long do_splice_to(struct file *in, loff_t *ppos, 854 struct pipe_inode_info *pipe, size_t len, 855 unsigned int flags) 856 { 857 int ret; 858 859 if (unlikely(!(in->f_mode & FMODE_READ))) 860 return -EBADF; 861 862 ret = rw_verify_area(READ, in, ppos, len); 863 if (unlikely(ret < 0)) 864 return ret; 865 866 if (unlikely(len > MAX_RW_COUNT)) 867 len = MAX_RW_COUNT; 868 869 if (in->f_op->splice_read) 870 return in->f_op->splice_read(in, ppos, pipe, len, flags); 871 return default_file_splice_read(in, ppos, pipe, len, flags); 872 } 873 874 /** 875 * splice_direct_to_actor - splices data directly between two non-pipes 876 * @in: file to splice from 877 * @sd: actor information on where to splice to 878 * @actor: handles the data splicing 879 * 880 * Description: 881 * This is a special case helper to splice directly between two 882 * points, without requiring an explicit pipe. Internally an allocated 883 * pipe is cached in the process, and reused during the lifetime of 884 * that process. 885 * 886 */ 887 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 888 splice_direct_actor *actor) 889 { 890 struct pipe_inode_info *pipe; 891 long ret, bytes; 892 umode_t i_mode; 893 size_t len; 894 int i, flags, more; 895 896 /* 897 * We require the input being a regular file, as we don't want to 898 * randomly drop data for eg socket -> socket splicing. Use the 899 * piped splicing for that! 900 */ 901 i_mode = file_inode(in)->i_mode; 902 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) 903 return -EINVAL; 904 905 /* 906 * neither in nor out is a pipe, setup an internal pipe attached to 907 * 'out' and transfer the wanted data from 'in' to 'out' through that 908 */ 909 pipe = current->splice_pipe; 910 if (unlikely(!pipe)) { 911 pipe = alloc_pipe_info(); 912 if (!pipe) 913 return -ENOMEM; 914 915 /* 916 * We don't have an immediate reader, but we'll read the stuff 917 * out of the pipe right after the splice_to_pipe(). So set 918 * PIPE_READERS appropriately. 919 */ 920 pipe->readers = 1; 921 922 current->splice_pipe = pipe; 923 } 924 925 /* 926 * Do the splice. 927 */ 928 ret = 0; 929 bytes = 0; 930 len = sd->total_len; 931 flags = sd->flags; 932 933 /* 934 * Don't block on output, we have to drain the direct pipe. 935 */ 936 sd->flags &= ~SPLICE_F_NONBLOCK; 937 more = sd->flags & SPLICE_F_MORE; 938 939 WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail)); 940 941 while (len) { 942 unsigned int p_space; 943 size_t read_len; 944 loff_t pos = sd->pos, prev_pos = pos; 945 946 /* Don't try to read more the pipe has space for. */ 947 p_space = pipe->max_usage - 948 pipe_occupancy(pipe->head, pipe->tail); 949 read_len = min_t(size_t, len, p_space << PAGE_SHIFT); 950 ret = do_splice_to(in, &pos, pipe, read_len, flags); 951 if (unlikely(ret <= 0)) 952 goto out_release; 953 954 read_len = ret; 955 sd->total_len = read_len; 956 957 /* 958 * If more data is pending, set SPLICE_F_MORE 959 * If this is the last data and SPLICE_F_MORE was not set 960 * initially, clears it. 961 */ 962 if (read_len < len) 963 sd->flags |= SPLICE_F_MORE; 964 else if (!more) 965 sd->flags &= ~SPLICE_F_MORE; 966 /* 967 * NOTE: nonblocking mode only applies to the input. We 968 * must not do the output in nonblocking mode as then we 969 * could get stuck data in the internal pipe: 970 */ 971 ret = actor(pipe, sd); 972 if (unlikely(ret <= 0)) { 973 sd->pos = prev_pos; 974 goto out_release; 975 } 976 977 bytes += ret; 978 len -= ret; 979 sd->pos = pos; 980 981 if (ret < read_len) { 982 sd->pos = prev_pos + ret; 983 goto out_release; 984 } 985 } 986 987 done: 988 pipe->tail = pipe->head = 0; 989 file_accessed(in); 990 return bytes; 991 992 out_release: 993 /* 994 * If we did an incomplete transfer we must release 995 * the pipe buffers in question: 996 */ 997 for (i = 0; i < pipe->ring_size; i++) { 998 struct pipe_buffer *buf = &pipe->bufs[i]; 999 1000 if (buf->ops) 1001 pipe_buf_release(pipe, buf); 1002 } 1003 1004 if (!bytes) 1005 bytes = ret; 1006 1007 goto done; 1008 } 1009 EXPORT_SYMBOL(splice_direct_to_actor); 1010 1011 static int direct_splice_actor(struct pipe_inode_info *pipe, 1012 struct splice_desc *sd) 1013 { 1014 struct file *file = sd->u.file; 1015 1016 return do_splice_from(pipe, file, sd->opos, sd->total_len, 1017 sd->flags); 1018 } 1019 1020 /** 1021 * do_splice_direct - splices data directly between two files 1022 * @in: file to splice from 1023 * @ppos: input file offset 1024 * @out: file to splice to 1025 * @opos: output file offset 1026 * @len: number of bytes to splice 1027 * @flags: splice modifier flags 1028 * 1029 * Description: 1030 * For use by do_sendfile(). splice can easily emulate sendfile, but 1031 * doing it in the application would incur an extra system call 1032 * (splice in + splice out, as compared to just sendfile()). So this helper 1033 * can splice directly through a process-private pipe. 1034 * 1035 */ 1036 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 1037 loff_t *opos, size_t len, unsigned int flags) 1038 { 1039 struct splice_desc sd = { 1040 .len = len, 1041 .total_len = len, 1042 .flags = flags, 1043 .pos = *ppos, 1044 .u.file = out, 1045 .opos = opos, 1046 }; 1047 long ret; 1048 1049 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1050 return -EBADF; 1051 1052 if (unlikely(out->f_flags & O_APPEND)) 1053 return -EINVAL; 1054 1055 ret = rw_verify_area(WRITE, out, opos, len); 1056 if (unlikely(ret < 0)) 1057 return ret; 1058 1059 ret = splice_direct_to_actor(in, &sd, direct_splice_actor); 1060 if (ret > 0) 1061 *ppos = sd.pos; 1062 1063 return ret; 1064 } 1065 EXPORT_SYMBOL(do_splice_direct); 1066 1067 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) 1068 { 1069 for (;;) { 1070 if (unlikely(!pipe->readers)) { 1071 send_sig(SIGPIPE, current, 0); 1072 return -EPIPE; 1073 } 1074 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 1075 return 0; 1076 if (flags & SPLICE_F_NONBLOCK) 1077 return -EAGAIN; 1078 if (signal_pending(current)) 1079 return -ERESTARTSYS; 1080 pipe_wait(pipe); 1081 } 1082 } 1083 1084 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1085 struct pipe_inode_info *opipe, 1086 size_t len, unsigned int flags); 1087 1088 /* 1089 * Determine where to splice to/from. 1090 */ 1091 long do_splice(struct file *in, loff_t __user *off_in, 1092 struct file *out, loff_t __user *off_out, 1093 size_t len, unsigned int flags) 1094 { 1095 struct pipe_inode_info *ipipe; 1096 struct pipe_inode_info *opipe; 1097 loff_t offset; 1098 long ret; 1099 1100 if (unlikely(!(in->f_mode & FMODE_READ) || 1101 !(out->f_mode & FMODE_WRITE))) 1102 return -EBADF; 1103 1104 ipipe = get_pipe_info(in, true); 1105 opipe = get_pipe_info(out, true); 1106 1107 if (ipipe && opipe) { 1108 if (off_in || off_out) 1109 return -ESPIPE; 1110 1111 /* Splicing to self would be fun, but... */ 1112 if (ipipe == opipe) 1113 return -EINVAL; 1114 1115 if ((in->f_flags | out->f_flags) & O_NONBLOCK) 1116 flags |= SPLICE_F_NONBLOCK; 1117 1118 return splice_pipe_to_pipe(ipipe, opipe, len, flags); 1119 } 1120 1121 if (ipipe) { 1122 if (off_in) 1123 return -ESPIPE; 1124 if (off_out) { 1125 if (!(out->f_mode & FMODE_PWRITE)) 1126 return -EINVAL; 1127 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1128 return -EFAULT; 1129 } else { 1130 offset = out->f_pos; 1131 } 1132 1133 if (unlikely(out->f_flags & O_APPEND)) 1134 return -EINVAL; 1135 1136 ret = rw_verify_area(WRITE, out, &offset, len); 1137 if (unlikely(ret < 0)) 1138 return ret; 1139 1140 if (in->f_flags & O_NONBLOCK) 1141 flags |= SPLICE_F_NONBLOCK; 1142 1143 file_start_write(out); 1144 ret = do_splice_from(ipipe, out, &offset, len, flags); 1145 file_end_write(out); 1146 1147 if (!off_out) 1148 out->f_pos = offset; 1149 else if (copy_to_user(off_out, &offset, sizeof(loff_t))) 1150 ret = -EFAULT; 1151 1152 return ret; 1153 } 1154 1155 if (opipe) { 1156 if (off_out) 1157 return -ESPIPE; 1158 if (off_in) { 1159 if (!(in->f_mode & FMODE_PREAD)) 1160 return -EINVAL; 1161 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1162 return -EFAULT; 1163 } else { 1164 offset = in->f_pos; 1165 } 1166 1167 if (out->f_flags & O_NONBLOCK) 1168 flags |= SPLICE_F_NONBLOCK; 1169 1170 pipe_lock(opipe); 1171 ret = wait_for_space(opipe, flags); 1172 if (!ret) { 1173 unsigned int p_space; 1174 1175 /* Don't try to read more the pipe has space for. */ 1176 p_space = opipe->max_usage - pipe_occupancy(opipe->head, opipe->tail); 1177 len = min_t(size_t, len, p_space << PAGE_SHIFT); 1178 1179 ret = do_splice_to(in, &offset, opipe, len, flags); 1180 } 1181 pipe_unlock(opipe); 1182 if (ret > 0) 1183 wakeup_pipe_readers(opipe); 1184 if (!off_in) 1185 in->f_pos = offset; 1186 else if (copy_to_user(off_in, &offset, sizeof(loff_t))) 1187 ret = -EFAULT; 1188 1189 return ret; 1190 } 1191 1192 return -EINVAL; 1193 } 1194 1195 static int iter_to_pipe(struct iov_iter *from, 1196 struct pipe_inode_info *pipe, 1197 unsigned flags) 1198 { 1199 struct pipe_buffer buf = { 1200 .ops = &user_page_pipe_buf_ops, 1201 .flags = flags 1202 }; 1203 size_t total = 0; 1204 int ret = 0; 1205 bool failed = false; 1206 1207 while (iov_iter_count(from) && !failed) { 1208 struct page *pages[16]; 1209 ssize_t copied; 1210 size_t start; 1211 int n; 1212 1213 copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start); 1214 if (copied <= 0) { 1215 ret = copied; 1216 break; 1217 } 1218 1219 for (n = 0; copied; n++, start = 0) { 1220 int size = min_t(int, copied, PAGE_SIZE - start); 1221 if (!failed) { 1222 buf.page = pages[n]; 1223 buf.offset = start; 1224 buf.len = size; 1225 ret = add_to_pipe(pipe, &buf); 1226 if (unlikely(ret < 0)) { 1227 failed = true; 1228 } else { 1229 iov_iter_advance(from, ret); 1230 total += ret; 1231 } 1232 } else { 1233 put_page(pages[n]); 1234 } 1235 copied -= size; 1236 } 1237 } 1238 return total ? total : ret; 1239 } 1240 1241 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1242 struct splice_desc *sd) 1243 { 1244 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); 1245 return n == sd->len ? n : -EFAULT; 1246 } 1247 1248 /* 1249 * For lack of a better implementation, implement vmsplice() to userspace 1250 * as a simple copy of the pipes pages to the user iov. 1251 */ 1252 static long vmsplice_to_user(struct file *file, struct iov_iter *iter, 1253 unsigned int flags) 1254 { 1255 struct pipe_inode_info *pipe = get_pipe_info(file, true); 1256 struct splice_desc sd = { 1257 .total_len = iov_iter_count(iter), 1258 .flags = flags, 1259 .u.data = iter 1260 }; 1261 long ret = 0; 1262 1263 if (!pipe) 1264 return -EBADF; 1265 1266 if (sd.total_len) { 1267 pipe_lock(pipe); 1268 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); 1269 pipe_unlock(pipe); 1270 } 1271 1272 return ret; 1273 } 1274 1275 /* 1276 * vmsplice splices a user address range into a pipe. It can be thought of 1277 * as splice-from-memory, where the regular splice is splice-from-file (or 1278 * to file). In both cases the output is a pipe, naturally. 1279 */ 1280 static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter, 1281 unsigned int flags) 1282 { 1283 struct pipe_inode_info *pipe; 1284 long ret = 0; 1285 unsigned buf_flag = 0; 1286 1287 if (flags & SPLICE_F_GIFT) 1288 buf_flag = PIPE_BUF_FLAG_GIFT; 1289 1290 pipe = get_pipe_info(file, true); 1291 if (!pipe) 1292 return -EBADF; 1293 1294 pipe_lock(pipe); 1295 ret = wait_for_space(pipe, flags); 1296 if (!ret) 1297 ret = iter_to_pipe(iter, pipe, buf_flag); 1298 pipe_unlock(pipe); 1299 if (ret > 0) 1300 wakeup_pipe_readers(pipe); 1301 return ret; 1302 } 1303 1304 static int vmsplice_type(struct fd f, int *type) 1305 { 1306 if (!f.file) 1307 return -EBADF; 1308 if (f.file->f_mode & FMODE_WRITE) { 1309 *type = WRITE; 1310 } else if (f.file->f_mode & FMODE_READ) { 1311 *type = READ; 1312 } else { 1313 fdput(f); 1314 return -EBADF; 1315 } 1316 return 0; 1317 } 1318 1319 /* 1320 * Note that vmsplice only really supports true splicing _from_ user memory 1321 * to a pipe, not the other way around. Splicing from user memory is a simple 1322 * operation that can be supported without any funky alignment restrictions 1323 * or nasty vm tricks. We simply map in the user memory and fill them into 1324 * a pipe. The reverse isn't quite as easy, though. There are two possible 1325 * solutions for that: 1326 * 1327 * - memcpy() the data internally, at which point we might as well just 1328 * do a regular read() on the buffer anyway. 1329 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1330 * has restriction limitations on both ends of the pipe). 1331 * 1332 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1333 * 1334 */ 1335 static long do_vmsplice(struct file *f, struct iov_iter *iter, unsigned int flags) 1336 { 1337 if (unlikely(flags & ~SPLICE_F_ALL)) 1338 return -EINVAL; 1339 1340 if (!iov_iter_count(iter)) 1341 return 0; 1342 1343 if (iov_iter_rw(iter) == WRITE) 1344 return vmsplice_to_pipe(f, iter, flags); 1345 else 1346 return vmsplice_to_user(f, iter, flags); 1347 } 1348 1349 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov, 1350 unsigned long, nr_segs, unsigned int, flags) 1351 { 1352 struct iovec iovstack[UIO_FASTIOV]; 1353 struct iovec *iov = iovstack; 1354 struct iov_iter iter; 1355 ssize_t error; 1356 struct fd f; 1357 int type; 1358 1359 f = fdget(fd); 1360 error = vmsplice_type(f, &type); 1361 if (error) 1362 return error; 1363 1364 error = import_iovec(type, uiov, nr_segs, 1365 ARRAY_SIZE(iovstack), &iov, &iter); 1366 if (error >= 0) { 1367 error = do_vmsplice(f.file, &iter, flags); 1368 kfree(iov); 1369 } 1370 fdput(f); 1371 return error; 1372 } 1373 1374 #ifdef CONFIG_COMPAT 1375 COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32, 1376 unsigned int, nr_segs, unsigned int, flags) 1377 { 1378 struct iovec iovstack[UIO_FASTIOV]; 1379 struct iovec *iov = iovstack; 1380 struct iov_iter iter; 1381 ssize_t error; 1382 struct fd f; 1383 int type; 1384 1385 f = fdget(fd); 1386 error = vmsplice_type(f, &type); 1387 if (error) 1388 return error; 1389 1390 error = compat_import_iovec(type, iov32, nr_segs, 1391 ARRAY_SIZE(iovstack), &iov, &iter); 1392 if (error >= 0) { 1393 error = do_vmsplice(f.file, &iter, flags); 1394 kfree(iov); 1395 } 1396 fdput(f); 1397 return error; 1398 } 1399 #endif 1400 1401 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, 1402 int, fd_out, loff_t __user *, off_out, 1403 size_t, len, unsigned int, flags) 1404 { 1405 struct fd in, out; 1406 long error; 1407 1408 if (unlikely(!len)) 1409 return 0; 1410 1411 if (unlikely(flags & ~SPLICE_F_ALL)) 1412 return -EINVAL; 1413 1414 error = -EBADF; 1415 in = fdget(fd_in); 1416 if (in.file) { 1417 out = fdget(fd_out); 1418 if (out.file) { 1419 error = do_splice(in.file, off_in, out.file, off_out, 1420 len, flags); 1421 fdput(out); 1422 } 1423 fdput(in); 1424 } 1425 return error; 1426 } 1427 1428 /* 1429 * Make sure there's data to read. Wait for input if we can, otherwise 1430 * return an appropriate error. 1431 */ 1432 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1433 { 1434 int ret; 1435 1436 /* 1437 * Check the pipe occupancy without the inode lock first. This function 1438 * is speculative anyways, so missing one is ok. 1439 */ 1440 if (!pipe_empty(pipe->head, pipe->tail)) 1441 return 0; 1442 1443 ret = 0; 1444 pipe_lock(pipe); 1445 1446 while (pipe_empty(pipe->head, pipe->tail)) { 1447 if (signal_pending(current)) { 1448 ret = -ERESTARTSYS; 1449 break; 1450 } 1451 if (!pipe->writers) 1452 break; 1453 if (flags & SPLICE_F_NONBLOCK) { 1454 ret = -EAGAIN; 1455 break; 1456 } 1457 pipe_wait(pipe); 1458 } 1459 1460 pipe_unlock(pipe); 1461 return ret; 1462 } 1463 1464 /* 1465 * Make sure there's writeable room. Wait for room if we can, otherwise 1466 * return an appropriate error. 1467 */ 1468 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1469 { 1470 int ret; 1471 1472 /* 1473 * Check pipe occupancy without the inode lock first. This function 1474 * is speculative anyways, so missing one is ok. 1475 */ 1476 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 1477 return 0; 1478 1479 ret = 0; 1480 pipe_lock(pipe); 1481 1482 while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { 1483 if (!pipe->readers) { 1484 send_sig(SIGPIPE, current, 0); 1485 ret = -EPIPE; 1486 break; 1487 } 1488 if (flags & SPLICE_F_NONBLOCK) { 1489 ret = -EAGAIN; 1490 break; 1491 } 1492 if (signal_pending(current)) { 1493 ret = -ERESTARTSYS; 1494 break; 1495 } 1496 pipe_wait(pipe); 1497 } 1498 1499 pipe_unlock(pipe); 1500 return ret; 1501 } 1502 1503 /* 1504 * Splice contents of ipipe to opipe. 1505 */ 1506 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1507 struct pipe_inode_info *opipe, 1508 size_t len, unsigned int flags) 1509 { 1510 struct pipe_buffer *ibuf, *obuf; 1511 unsigned int i_head, o_head; 1512 unsigned int i_tail, o_tail; 1513 unsigned int i_mask, o_mask; 1514 int ret = 0; 1515 bool input_wakeup = false; 1516 1517 1518 retry: 1519 ret = ipipe_prep(ipipe, flags); 1520 if (ret) 1521 return ret; 1522 1523 ret = opipe_prep(opipe, flags); 1524 if (ret) 1525 return ret; 1526 1527 /* 1528 * Potential ABBA deadlock, work around it by ordering lock 1529 * grabbing by pipe info address. Otherwise two different processes 1530 * could deadlock (one doing tee from A -> B, the other from B -> A). 1531 */ 1532 pipe_double_lock(ipipe, opipe); 1533 1534 i_tail = ipipe->tail; 1535 i_mask = ipipe->ring_size - 1; 1536 o_head = opipe->head; 1537 o_mask = opipe->ring_size - 1; 1538 1539 do { 1540 size_t o_len; 1541 1542 if (!opipe->readers) { 1543 send_sig(SIGPIPE, current, 0); 1544 if (!ret) 1545 ret = -EPIPE; 1546 break; 1547 } 1548 1549 i_head = ipipe->head; 1550 o_tail = opipe->tail; 1551 1552 if (pipe_empty(i_head, i_tail) && !ipipe->writers) 1553 break; 1554 1555 /* 1556 * Cannot make any progress, because either the input 1557 * pipe is empty or the output pipe is full. 1558 */ 1559 if (pipe_empty(i_head, i_tail) || 1560 pipe_full(o_head, o_tail, opipe->max_usage)) { 1561 /* Already processed some buffers, break */ 1562 if (ret) 1563 break; 1564 1565 if (flags & SPLICE_F_NONBLOCK) { 1566 ret = -EAGAIN; 1567 break; 1568 } 1569 1570 /* 1571 * We raced with another reader/writer and haven't 1572 * managed to process any buffers. A zero return 1573 * value means EOF, so retry instead. 1574 */ 1575 pipe_unlock(ipipe); 1576 pipe_unlock(opipe); 1577 goto retry; 1578 } 1579 1580 ibuf = &ipipe->bufs[i_tail & i_mask]; 1581 obuf = &opipe->bufs[o_head & o_mask]; 1582 1583 if (len >= ibuf->len) { 1584 /* 1585 * Simply move the whole buffer from ipipe to opipe 1586 */ 1587 *obuf = *ibuf; 1588 ibuf->ops = NULL; 1589 i_tail++; 1590 ipipe->tail = i_tail; 1591 input_wakeup = true; 1592 o_len = obuf->len; 1593 o_head++; 1594 opipe->head = o_head; 1595 } else { 1596 /* 1597 * Get a reference to this pipe buffer, 1598 * so we can copy the contents over. 1599 */ 1600 if (!pipe_buf_get(ipipe, ibuf)) { 1601 if (ret == 0) 1602 ret = -EFAULT; 1603 break; 1604 } 1605 *obuf = *ibuf; 1606 1607 /* 1608 * Don't inherit the gift and merge flags, we need to 1609 * prevent multiple steals of this page. 1610 */ 1611 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1612 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; 1613 1614 obuf->len = len; 1615 ibuf->offset += len; 1616 ibuf->len -= len; 1617 o_len = len; 1618 o_head++; 1619 opipe->head = o_head; 1620 } 1621 ret += o_len; 1622 len -= o_len; 1623 } while (len); 1624 1625 pipe_unlock(ipipe); 1626 pipe_unlock(opipe); 1627 1628 /* 1629 * If we put data in the output pipe, wakeup any potential readers. 1630 */ 1631 if (ret > 0) 1632 wakeup_pipe_readers(opipe); 1633 1634 if (input_wakeup) 1635 wakeup_pipe_writers(ipipe); 1636 1637 return ret; 1638 } 1639 1640 /* 1641 * Link contents of ipipe to opipe. 1642 */ 1643 static int link_pipe(struct pipe_inode_info *ipipe, 1644 struct pipe_inode_info *opipe, 1645 size_t len, unsigned int flags) 1646 { 1647 struct pipe_buffer *ibuf, *obuf; 1648 unsigned int i_head, o_head; 1649 unsigned int i_tail, o_tail; 1650 unsigned int i_mask, o_mask; 1651 int ret = 0; 1652 1653 /* 1654 * Potential ABBA deadlock, work around it by ordering lock 1655 * grabbing by pipe info address. Otherwise two different processes 1656 * could deadlock (one doing tee from A -> B, the other from B -> A). 1657 */ 1658 pipe_double_lock(ipipe, opipe); 1659 1660 i_tail = ipipe->tail; 1661 i_mask = ipipe->ring_size - 1; 1662 o_head = opipe->head; 1663 o_mask = opipe->ring_size - 1; 1664 1665 do { 1666 if (!opipe->readers) { 1667 send_sig(SIGPIPE, current, 0); 1668 if (!ret) 1669 ret = -EPIPE; 1670 break; 1671 } 1672 1673 i_head = ipipe->head; 1674 o_tail = opipe->tail; 1675 1676 /* 1677 * If we have iterated all input buffers or run out of 1678 * output room, break. 1679 */ 1680 if (pipe_empty(i_head, i_tail) || 1681 pipe_full(o_head, o_tail, opipe->max_usage)) 1682 break; 1683 1684 ibuf = &ipipe->bufs[i_tail & i_mask]; 1685 obuf = &opipe->bufs[o_head & o_mask]; 1686 1687 /* 1688 * Get a reference to this pipe buffer, 1689 * so we can copy the contents over. 1690 */ 1691 if (!pipe_buf_get(ipipe, ibuf)) { 1692 if (ret == 0) 1693 ret = -EFAULT; 1694 break; 1695 } 1696 1697 *obuf = *ibuf; 1698 1699 /* 1700 * Don't inherit the gift and merge flag, we need to prevent 1701 * multiple steals of this page. 1702 */ 1703 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1704 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; 1705 1706 if (obuf->len > len) 1707 obuf->len = len; 1708 ret += obuf->len; 1709 len -= obuf->len; 1710 1711 o_head++; 1712 opipe->head = o_head; 1713 i_tail++; 1714 } while (len); 1715 1716 pipe_unlock(ipipe); 1717 pipe_unlock(opipe); 1718 1719 /* 1720 * If we put data in the output pipe, wakeup any potential readers. 1721 */ 1722 if (ret > 0) 1723 wakeup_pipe_readers(opipe); 1724 1725 return ret; 1726 } 1727 1728 /* 1729 * This is a tee(1) implementation that works on pipes. It doesn't copy 1730 * any data, it simply references the 'in' pages on the 'out' pipe. 1731 * The 'flags' used are the SPLICE_F_* variants, currently the only 1732 * applicable one is SPLICE_F_NONBLOCK. 1733 */ 1734 long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags) 1735 { 1736 struct pipe_inode_info *ipipe = get_pipe_info(in, true); 1737 struct pipe_inode_info *opipe = get_pipe_info(out, true); 1738 int ret = -EINVAL; 1739 1740 if (unlikely(!(in->f_mode & FMODE_READ) || 1741 !(out->f_mode & FMODE_WRITE))) 1742 return -EBADF; 1743 1744 /* 1745 * Duplicate the contents of ipipe to opipe without actually 1746 * copying the data. 1747 */ 1748 if (ipipe && opipe && ipipe != opipe) { 1749 if ((in->f_flags | out->f_flags) & O_NONBLOCK) 1750 flags |= SPLICE_F_NONBLOCK; 1751 1752 /* 1753 * Keep going, unless we encounter an error. The ipipe/opipe 1754 * ordering doesn't really matter. 1755 */ 1756 ret = ipipe_prep(ipipe, flags); 1757 if (!ret) { 1758 ret = opipe_prep(opipe, flags); 1759 if (!ret) 1760 ret = link_pipe(ipipe, opipe, len, flags); 1761 } 1762 } 1763 1764 return ret; 1765 } 1766 1767 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) 1768 { 1769 struct fd in, out; 1770 int error; 1771 1772 if (unlikely(flags & ~SPLICE_F_ALL)) 1773 return -EINVAL; 1774 1775 if (unlikely(!len)) 1776 return 0; 1777 1778 error = -EBADF; 1779 in = fdget(fdin); 1780 if (in.file) { 1781 out = fdget(fdout); 1782 if (out.file) { 1783 error = do_tee(in.file, out.file, len, flags); 1784 fdput(out); 1785 } 1786 fdput(in); 1787 } 1788 1789 return error; 1790 } 1791