1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * "splice": joining two ropes together by interweaving their strands. 4 * 5 * This is the "extended pipe" functionality, where a pipe is used as 6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 7 * buffer that you can use to transfer data from one end to the other. 8 * 9 * The traditional unix read/write is extended with a "splice()" operation 10 * that transfers data buffers to or from a pipe buffer. 11 * 12 * Named by Larry McVoy, original implementation from Linus, extended by 13 * Jens to support splicing to files, network, direct splicing, etc and 14 * fixing lots of bugs. 15 * 16 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 17 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 18 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 19 * 20 */ 21 #include <linux/bvec.h> 22 #include <linux/fs.h> 23 #include <linux/file.h> 24 #include <linux/pagemap.h> 25 #include <linux/splice.h> 26 #include <linux/memcontrol.h> 27 #include <linux/mm_inline.h> 28 #include <linux/swap.h> 29 #include <linux/writeback.h> 30 #include <linux/export.h> 31 #include <linux/syscalls.h> 32 #include <linux/uio.h> 33 #include <linux/security.h> 34 #include <linux/gfp.h> 35 #include <linux/socket.h> 36 #include <linux/sched/signal.h> 37 38 #include "internal.h" 39 40 /* 41 * Attempt to steal a page from a pipe buffer. This should perhaps go into 42 * a vm helper function, it's already simplified quite a bit by the 43 * addition of remove_mapping(). If success is returned, the caller may 44 * attempt to reuse this page for another destination. 45 */ 46 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe, 47 struct pipe_buffer *buf) 48 { 49 struct folio *folio = page_folio(buf->page); 50 struct address_space *mapping; 51 52 folio_lock(folio); 53 54 mapping = folio_mapping(folio); 55 if (mapping) { 56 WARN_ON(!folio_test_uptodate(folio)); 57 58 /* 59 * At least for ext2 with nobh option, we need to wait on 60 * writeback completing on this folio, since we'll remove it 61 * from the pagecache. Otherwise truncate wont wait on the 62 * folio, allowing the disk blocks to be reused by someone else 63 * before we actually wrote our data to them. fs corruption 64 * ensues. 65 */ 66 folio_wait_writeback(folio); 67 68 if (folio_has_private(folio) && 69 !filemap_release_folio(folio, GFP_KERNEL)) 70 goto out_unlock; 71 72 /* 73 * If we succeeded in removing the mapping, set LRU flag 74 * and return good. 75 */ 76 if (remove_mapping(mapping, folio)) { 77 buf->flags |= PIPE_BUF_FLAG_LRU; 78 return true; 79 } 80 } 81 82 /* 83 * Raced with truncate or failed to remove folio from current 84 * address space, unlock and return failure. 85 */ 86 out_unlock: 87 folio_unlock(folio); 88 return false; 89 } 90 91 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, 92 struct pipe_buffer *buf) 93 { 94 put_page(buf->page); 95 buf->flags &= ~PIPE_BUF_FLAG_LRU; 96 } 97 98 /* 99 * Check whether the contents of buf is OK to access. Since the content 100 * is a page cache page, IO may be in flight. 101 */ 102 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 103 struct pipe_buffer *buf) 104 { 105 struct page *page = buf->page; 106 int err; 107 108 if (!PageUptodate(page)) { 109 lock_page(page); 110 111 /* 112 * Page got truncated/unhashed. This will cause a 0-byte 113 * splice, if this is the first page. 114 */ 115 if (!page->mapping) { 116 err = -ENODATA; 117 goto error; 118 } 119 120 /* 121 * Uh oh, read-error from disk. 122 */ 123 if (!PageUptodate(page)) { 124 err = -EIO; 125 goto error; 126 } 127 128 /* 129 * Page is ok afterall, we are done. 130 */ 131 unlock_page(page); 132 } 133 134 return 0; 135 error: 136 unlock_page(page); 137 return err; 138 } 139 140 const struct pipe_buf_operations page_cache_pipe_buf_ops = { 141 .confirm = page_cache_pipe_buf_confirm, 142 .release = page_cache_pipe_buf_release, 143 .try_steal = page_cache_pipe_buf_try_steal, 144 .get = generic_pipe_buf_get, 145 }; 146 147 static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe, 148 struct pipe_buffer *buf) 149 { 150 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) 151 return false; 152 153 buf->flags |= PIPE_BUF_FLAG_LRU; 154 return generic_pipe_buf_try_steal(pipe, buf); 155 } 156 157 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 158 .release = page_cache_pipe_buf_release, 159 .try_steal = user_page_pipe_buf_try_steal, 160 .get = generic_pipe_buf_get, 161 }; 162 163 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) 164 { 165 smp_mb(); 166 if (waitqueue_active(&pipe->rd_wait)) 167 wake_up_interruptible(&pipe->rd_wait); 168 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 169 } 170 171 /** 172 * splice_to_pipe - fill passed data into a pipe 173 * @pipe: pipe to fill 174 * @spd: data to fill 175 * 176 * Description: 177 * @spd contains a map of pages and len/offset tuples, along with 178 * the struct pipe_buf_operations associated with these pages. This 179 * function will link that data to the pipe. 180 * 181 */ 182 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 183 struct splice_pipe_desc *spd) 184 { 185 unsigned int spd_pages = spd->nr_pages; 186 unsigned int tail = pipe->tail; 187 unsigned int head = pipe->head; 188 unsigned int mask = pipe->ring_size - 1; 189 int ret = 0, page_nr = 0; 190 191 if (!spd_pages) 192 return 0; 193 194 if (unlikely(!pipe->readers)) { 195 send_sig(SIGPIPE, current, 0); 196 ret = -EPIPE; 197 goto out; 198 } 199 200 while (!pipe_full(head, tail, pipe->max_usage)) { 201 struct pipe_buffer *buf = &pipe->bufs[head & mask]; 202 203 buf->page = spd->pages[page_nr]; 204 buf->offset = spd->partial[page_nr].offset; 205 buf->len = spd->partial[page_nr].len; 206 buf->private = spd->partial[page_nr].private; 207 buf->ops = spd->ops; 208 buf->flags = 0; 209 210 head++; 211 pipe->head = head; 212 page_nr++; 213 ret += buf->len; 214 215 if (!--spd->nr_pages) 216 break; 217 } 218 219 if (!ret) 220 ret = -EAGAIN; 221 222 out: 223 while (page_nr < spd_pages) 224 spd->spd_release(spd, page_nr++); 225 226 return ret; 227 } 228 EXPORT_SYMBOL_GPL(splice_to_pipe); 229 230 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) 231 { 232 unsigned int head = pipe->head; 233 unsigned int tail = pipe->tail; 234 unsigned int mask = pipe->ring_size - 1; 235 int ret; 236 237 if (unlikely(!pipe->readers)) { 238 send_sig(SIGPIPE, current, 0); 239 ret = -EPIPE; 240 } else if (pipe_full(head, tail, pipe->max_usage)) { 241 ret = -EAGAIN; 242 } else { 243 pipe->bufs[head & mask] = *buf; 244 pipe->head = head + 1; 245 return buf->len; 246 } 247 pipe_buf_release(pipe, buf); 248 return ret; 249 } 250 EXPORT_SYMBOL(add_to_pipe); 251 252 /* 253 * Check if we need to grow the arrays holding pages and partial page 254 * descriptions. 255 */ 256 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) 257 { 258 unsigned int max_usage = READ_ONCE(pipe->max_usage); 259 260 spd->nr_pages_max = max_usage; 261 if (max_usage <= PIPE_DEF_BUFFERS) 262 return 0; 263 264 spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL); 265 spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page), 266 GFP_KERNEL); 267 268 if (spd->pages && spd->partial) 269 return 0; 270 271 kfree(spd->pages); 272 kfree(spd->partial); 273 return -ENOMEM; 274 } 275 276 void splice_shrink_spd(struct splice_pipe_desc *spd) 277 { 278 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) 279 return; 280 281 kfree(spd->pages); 282 kfree(spd->partial); 283 } 284 285 /** 286 * generic_file_splice_read - splice data from file to a pipe 287 * @in: file to splice from 288 * @ppos: position in @in 289 * @pipe: pipe to splice to 290 * @len: number of bytes to splice 291 * @flags: splice modifier flags 292 * 293 * Description: 294 * Will read pages from given file and fill them into a pipe. Can be 295 * used as long as it has more or less sane ->read_iter(). 296 * 297 */ 298 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, 299 struct pipe_inode_info *pipe, size_t len, 300 unsigned int flags) 301 { 302 struct iov_iter to; 303 struct kiocb kiocb; 304 int ret; 305 306 iov_iter_pipe(&to, ITER_DEST, pipe, len); 307 init_sync_kiocb(&kiocb, in); 308 kiocb.ki_pos = *ppos; 309 ret = call_read_iter(in, &kiocb, &to); 310 if (ret > 0) { 311 *ppos = kiocb.ki_pos; 312 file_accessed(in); 313 } else if (ret < 0) { 314 /* free what was emitted */ 315 pipe_discard_from(pipe, to.start_head); 316 /* 317 * callers of ->splice_read() expect -EAGAIN on 318 * "can't put anything in there", rather than -EFAULT. 319 */ 320 if (ret == -EFAULT) 321 ret = -EAGAIN; 322 } 323 324 return ret; 325 } 326 EXPORT_SYMBOL(generic_file_splice_read); 327 328 const struct pipe_buf_operations default_pipe_buf_ops = { 329 .release = generic_pipe_buf_release, 330 .try_steal = generic_pipe_buf_try_steal, 331 .get = generic_pipe_buf_get, 332 }; 333 334 /* Pipe buffer operations for a socket and similar. */ 335 const struct pipe_buf_operations nosteal_pipe_buf_ops = { 336 .release = generic_pipe_buf_release, 337 .get = generic_pipe_buf_get, 338 }; 339 EXPORT_SYMBOL(nosteal_pipe_buf_ops); 340 341 /* 342 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 343 * using sendpage(). Return the number of bytes sent. 344 */ 345 static int pipe_to_sendpage(struct pipe_inode_info *pipe, 346 struct pipe_buffer *buf, struct splice_desc *sd) 347 { 348 struct file *file = sd->u.file; 349 loff_t pos = sd->pos; 350 int more; 351 352 if (!likely(file->f_op->sendpage)) 353 return -EINVAL; 354 355 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; 356 357 if (sd->len < sd->total_len && 358 pipe_occupancy(pipe->head, pipe->tail) > 1) 359 more |= MSG_SENDPAGE_NOTLAST; 360 361 return file->f_op->sendpage(file, buf->page, buf->offset, 362 sd->len, &pos, more); 363 } 364 365 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) 366 { 367 smp_mb(); 368 if (waitqueue_active(&pipe->wr_wait)) 369 wake_up_interruptible(&pipe->wr_wait); 370 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 371 } 372 373 /** 374 * splice_from_pipe_feed - feed available data from a pipe to a file 375 * @pipe: pipe to splice from 376 * @sd: information to @actor 377 * @actor: handler that splices the data 378 * 379 * Description: 380 * This function loops over the pipe and calls @actor to do the 381 * actual moving of a single struct pipe_buffer to the desired 382 * destination. It returns when there's no more buffers left in 383 * the pipe or if the requested number of bytes (@sd->total_len) 384 * have been copied. It returns a positive number (one) if the 385 * pipe needs to be filled with more data, zero if the required 386 * number of bytes have been copied and -errno on error. 387 * 388 * This, together with splice_from_pipe_{begin,end,next}, may be 389 * used to implement the functionality of __splice_from_pipe() when 390 * locking is required around copying the pipe buffers to the 391 * destination. 392 */ 393 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, 394 splice_actor *actor) 395 { 396 unsigned int head = pipe->head; 397 unsigned int tail = pipe->tail; 398 unsigned int mask = pipe->ring_size - 1; 399 int ret; 400 401 while (!pipe_empty(head, tail)) { 402 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; 403 404 sd->len = buf->len; 405 if (sd->len > sd->total_len) 406 sd->len = sd->total_len; 407 408 ret = pipe_buf_confirm(pipe, buf); 409 if (unlikely(ret)) { 410 if (ret == -ENODATA) 411 ret = 0; 412 return ret; 413 } 414 415 ret = actor(pipe, buf, sd); 416 if (ret <= 0) 417 return ret; 418 419 buf->offset += ret; 420 buf->len -= ret; 421 422 sd->num_spliced += ret; 423 sd->len -= ret; 424 sd->pos += ret; 425 sd->total_len -= ret; 426 427 if (!buf->len) { 428 pipe_buf_release(pipe, buf); 429 tail++; 430 pipe->tail = tail; 431 if (pipe->files) 432 sd->need_wakeup = true; 433 } 434 435 if (!sd->total_len) 436 return 0; 437 } 438 439 return 1; 440 } 441 442 /* We know we have a pipe buffer, but maybe it's empty? */ 443 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe) 444 { 445 unsigned int tail = pipe->tail; 446 unsigned int mask = pipe->ring_size - 1; 447 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; 448 449 if (unlikely(!buf->len)) { 450 pipe_buf_release(pipe, buf); 451 pipe->tail = tail+1; 452 return true; 453 } 454 455 return false; 456 } 457 458 /** 459 * splice_from_pipe_next - wait for some data to splice from 460 * @pipe: pipe to splice from 461 * @sd: information about the splice operation 462 * 463 * Description: 464 * This function will wait for some data and return a positive 465 * value (one) if pipe buffers are available. It will return zero 466 * or -errno if no more data needs to be spliced. 467 */ 468 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) 469 { 470 /* 471 * Check for signal early to make process killable when there are 472 * always buffers available 473 */ 474 if (signal_pending(current)) 475 return -ERESTARTSYS; 476 477 repeat: 478 while (pipe_empty(pipe->head, pipe->tail)) { 479 if (!pipe->writers) 480 return 0; 481 482 if (sd->num_spliced) 483 return 0; 484 485 if (sd->flags & SPLICE_F_NONBLOCK) 486 return -EAGAIN; 487 488 if (signal_pending(current)) 489 return -ERESTARTSYS; 490 491 if (sd->need_wakeup) { 492 wakeup_pipe_writers(pipe); 493 sd->need_wakeup = false; 494 } 495 496 pipe_wait_readable(pipe); 497 } 498 499 if (eat_empty_buffer(pipe)) 500 goto repeat; 501 502 return 1; 503 } 504 505 /** 506 * splice_from_pipe_begin - start splicing from pipe 507 * @sd: information about the splice operation 508 * 509 * Description: 510 * This function should be called before a loop containing 511 * splice_from_pipe_next() and splice_from_pipe_feed() to 512 * initialize the necessary fields of @sd. 513 */ 514 static void splice_from_pipe_begin(struct splice_desc *sd) 515 { 516 sd->num_spliced = 0; 517 sd->need_wakeup = false; 518 } 519 520 /** 521 * splice_from_pipe_end - finish splicing from pipe 522 * @pipe: pipe to splice from 523 * @sd: information about the splice operation 524 * 525 * Description: 526 * This function will wake up pipe writers if necessary. It should 527 * be called after a loop containing splice_from_pipe_next() and 528 * splice_from_pipe_feed(). 529 */ 530 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) 531 { 532 if (sd->need_wakeup) 533 wakeup_pipe_writers(pipe); 534 } 535 536 /** 537 * __splice_from_pipe - splice data from a pipe to given actor 538 * @pipe: pipe to splice from 539 * @sd: information to @actor 540 * @actor: handler that splices the data 541 * 542 * Description: 543 * This function does little more than loop over the pipe and call 544 * @actor to do the actual moving of a single struct pipe_buffer to 545 * the desired destination. See pipe_to_file, pipe_to_sendpage, or 546 * pipe_to_user. 547 * 548 */ 549 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 550 splice_actor *actor) 551 { 552 int ret; 553 554 splice_from_pipe_begin(sd); 555 do { 556 cond_resched(); 557 ret = splice_from_pipe_next(pipe, sd); 558 if (ret > 0) 559 ret = splice_from_pipe_feed(pipe, sd, actor); 560 } while (ret > 0); 561 splice_from_pipe_end(pipe, sd); 562 563 return sd->num_spliced ? sd->num_spliced : ret; 564 } 565 EXPORT_SYMBOL(__splice_from_pipe); 566 567 /** 568 * splice_from_pipe - splice data from a pipe to a file 569 * @pipe: pipe to splice from 570 * @out: file to splice to 571 * @ppos: position in @out 572 * @len: how many bytes to splice 573 * @flags: splice modifier flags 574 * @actor: handler that splices the data 575 * 576 * Description: 577 * See __splice_from_pipe. This function locks the pipe inode, 578 * otherwise it's identical to __splice_from_pipe(). 579 * 580 */ 581 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 582 loff_t *ppos, size_t len, unsigned int flags, 583 splice_actor *actor) 584 { 585 ssize_t ret; 586 struct splice_desc sd = { 587 .total_len = len, 588 .flags = flags, 589 .pos = *ppos, 590 .u.file = out, 591 }; 592 593 pipe_lock(pipe); 594 ret = __splice_from_pipe(pipe, &sd, actor); 595 pipe_unlock(pipe); 596 597 return ret; 598 } 599 600 /** 601 * iter_file_splice_write - splice data from a pipe to a file 602 * @pipe: pipe info 603 * @out: file to write to 604 * @ppos: position in @out 605 * @len: number of bytes to splice 606 * @flags: splice modifier flags 607 * 608 * Description: 609 * Will either move or copy pages (determined by @flags options) from 610 * the given pipe inode to the given file. 611 * This one is ->write_iter-based. 612 * 613 */ 614 ssize_t 615 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 616 loff_t *ppos, size_t len, unsigned int flags) 617 { 618 struct splice_desc sd = { 619 .total_len = len, 620 .flags = flags, 621 .pos = *ppos, 622 .u.file = out, 623 }; 624 int nbufs = pipe->max_usage; 625 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec), 626 GFP_KERNEL); 627 ssize_t ret; 628 629 if (unlikely(!array)) 630 return -ENOMEM; 631 632 pipe_lock(pipe); 633 634 splice_from_pipe_begin(&sd); 635 while (sd.total_len) { 636 struct iov_iter from; 637 unsigned int head, tail, mask; 638 size_t left; 639 int n; 640 641 ret = splice_from_pipe_next(pipe, &sd); 642 if (ret <= 0) 643 break; 644 645 if (unlikely(nbufs < pipe->max_usage)) { 646 kfree(array); 647 nbufs = pipe->max_usage; 648 array = kcalloc(nbufs, sizeof(struct bio_vec), 649 GFP_KERNEL); 650 if (!array) { 651 ret = -ENOMEM; 652 break; 653 } 654 } 655 656 head = pipe->head; 657 tail = pipe->tail; 658 mask = pipe->ring_size - 1; 659 660 /* build the vector */ 661 left = sd.total_len; 662 for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) { 663 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; 664 size_t this_len = buf->len; 665 666 /* zero-length bvecs are not supported, skip them */ 667 if (!this_len) 668 continue; 669 this_len = min(this_len, left); 670 671 ret = pipe_buf_confirm(pipe, buf); 672 if (unlikely(ret)) { 673 if (ret == -ENODATA) 674 ret = 0; 675 goto done; 676 } 677 678 bvec_set_page(&array[n], buf->page, this_len, 679 buf->offset); 680 left -= this_len; 681 n++; 682 } 683 684 iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left); 685 ret = vfs_iter_write(out, &from, &sd.pos, 0); 686 if (ret <= 0) 687 break; 688 689 sd.num_spliced += ret; 690 sd.total_len -= ret; 691 *ppos = sd.pos; 692 693 /* dismiss the fully eaten buffers, adjust the partial one */ 694 tail = pipe->tail; 695 while (ret) { 696 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; 697 if (ret >= buf->len) { 698 ret -= buf->len; 699 buf->len = 0; 700 pipe_buf_release(pipe, buf); 701 tail++; 702 pipe->tail = tail; 703 if (pipe->files) 704 sd.need_wakeup = true; 705 } else { 706 buf->offset += ret; 707 buf->len -= ret; 708 ret = 0; 709 } 710 } 711 } 712 done: 713 kfree(array); 714 splice_from_pipe_end(pipe, &sd); 715 716 pipe_unlock(pipe); 717 718 if (sd.num_spliced) 719 ret = sd.num_spliced; 720 721 return ret; 722 } 723 724 EXPORT_SYMBOL(iter_file_splice_write); 725 726 /** 727 * generic_splice_sendpage - splice data from a pipe to a socket 728 * @pipe: pipe to splice from 729 * @out: socket to write to 730 * @ppos: position in @out 731 * @len: number of bytes to splice 732 * @flags: splice modifier flags 733 * 734 * Description: 735 * Will send @len bytes from the pipe to a network socket. No data copying 736 * is involved. 737 * 738 */ 739 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 740 loff_t *ppos, size_t len, unsigned int flags) 741 { 742 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); 743 } 744 745 EXPORT_SYMBOL(generic_splice_sendpage); 746 747 static int warn_unsupported(struct file *file, const char *op) 748 { 749 pr_debug_ratelimited( 750 "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n", 751 op, file, current->pid, current->comm); 752 return -EINVAL; 753 } 754 755 /* 756 * Attempt to initiate a splice from pipe to file. 757 */ 758 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 759 loff_t *ppos, size_t len, unsigned int flags) 760 { 761 if (unlikely(!out->f_op->splice_write)) 762 return warn_unsupported(out, "write"); 763 return out->f_op->splice_write(pipe, out, ppos, len, flags); 764 } 765 766 /* 767 * Attempt to initiate a splice from a file to a pipe. 768 */ 769 static long do_splice_to(struct file *in, loff_t *ppos, 770 struct pipe_inode_info *pipe, size_t len, 771 unsigned int flags) 772 { 773 unsigned int p_space; 774 int ret; 775 776 if (unlikely(!(in->f_mode & FMODE_READ))) 777 return -EBADF; 778 779 /* Don't try to read more the pipe has space for. */ 780 p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail); 781 len = min_t(size_t, len, p_space << PAGE_SHIFT); 782 783 ret = rw_verify_area(READ, in, ppos, len); 784 if (unlikely(ret < 0)) 785 return ret; 786 787 if (unlikely(len > MAX_RW_COUNT)) 788 len = MAX_RW_COUNT; 789 790 if (unlikely(!in->f_op->splice_read)) 791 return warn_unsupported(in, "read"); 792 return in->f_op->splice_read(in, ppos, pipe, len, flags); 793 } 794 795 /** 796 * splice_direct_to_actor - splices data directly between two non-pipes 797 * @in: file to splice from 798 * @sd: actor information on where to splice to 799 * @actor: handles the data splicing 800 * 801 * Description: 802 * This is a special case helper to splice directly between two 803 * points, without requiring an explicit pipe. Internally an allocated 804 * pipe is cached in the process, and reused during the lifetime of 805 * that process. 806 * 807 */ 808 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 809 splice_direct_actor *actor) 810 { 811 struct pipe_inode_info *pipe; 812 long ret, bytes; 813 size_t len; 814 int i, flags, more; 815 816 /* 817 * We require the input to be seekable, as we don't want to randomly 818 * drop data for eg socket -> socket splicing. Use the piped splicing 819 * for that! 820 */ 821 if (unlikely(!(in->f_mode & FMODE_LSEEK))) 822 return -EINVAL; 823 824 /* 825 * neither in nor out is a pipe, setup an internal pipe attached to 826 * 'out' and transfer the wanted data from 'in' to 'out' through that 827 */ 828 pipe = current->splice_pipe; 829 if (unlikely(!pipe)) { 830 pipe = alloc_pipe_info(); 831 if (!pipe) 832 return -ENOMEM; 833 834 /* 835 * We don't have an immediate reader, but we'll read the stuff 836 * out of the pipe right after the splice_to_pipe(). So set 837 * PIPE_READERS appropriately. 838 */ 839 pipe->readers = 1; 840 841 current->splice_pipe = pipe; 842 } 843 844 /* 845 * Do the splice. 846 */ 847 ret = 0; 848 bytes = 0; 849 len = sd->total_len; 850 flags = sd->flags; 851 852 /* 853 * Don't block on output, we have to drain the direct pipe. 854 */ 855 sd->flags &= ~SPLICE_F_NONBLOCK; 856 more = sd->flags & SPLICE_F_MORE; 857 858 WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail)); 859 860 while (len) { 861 size_t read_len; 862 loff_t pos = sd->pos, prev_pos = pos; 863 864 ret = do_splice_to(in, &pos, pipe, len, flags); 865 if (unlikely(ret <= 0)) 866 goto out_release; 867 868 read_len = ret; 869 sd->total_len = read_len; 870 871 /* 872 * If more data is pending, set SPLICE_F_MORE 873 * If this is the last data and SPLICE_F_MORE was not set 874 * initially, clears it. 875 */ 876 if (read_len < len) 877 sd->flags |= SPLICE_F_MORE; 878 else if (!more) 879 sd->flags &= ~SPLICE_F_MORE; 880 /* 881 * NOTE: nonblocking mode only applies to the input. We 882 * must not do the output in nonblocking mode as then we 883 * could get stuck data in the internal pipe: 884 */ 885 ret = actor(pipe, sd); 886 if (unlikely(ret <= 0)) { 887 sd->pos = prev_pos; 888 goto out_release; 889 } 890 891 bytes += ret; 892 len -= ret; 893 sd->pos = pos; 894 895 if (ret < read_len) { 896 sd->pos = prev_pos + ret; 897 goto out_release; 898 } 899 } 900 901 done: 902 pipe->tail = pipe->head = 0; 903 file_accessed(in); 904 return bytes; 905 906 out_release: 907 /* 908 * If we did an incomplete transfer we must release 909 * the pipe buffers in question: 910 */ 911 for (i = 0; i < pipe->ring_size; i++) { 912 struct pipe_buffer *buf = &pipe->bufs[i]; 913 914 if (buf->ops) 915 pipe_buf_release(pipe, buf); 916 } 917 918 if (!bytes) 919 bytes = ret; 920 921 goto done; 922 } 923 EXPORT_SYMBOL(splice_direct_to_actor); 924 925 static int direct_splice_actor(struct pipe_inode_info *pipe, 926 struct splice_desc *sd) 927 { 928 struct file *file = sd->u.file; 929 930 return do_splice_from(pipe, file, sd->opos, sd->total_len, 931 sd->flags); 932 } 933 934 /** 935 * do_splice_direct - splices data directly between two files 936 * @in: file to splice from 937 * @ppos: input file offset 938 * @out: file to splice to 939 * @opos: output file offset 940 * @len: number of bytes to splice 941 * @flags: splice modifier flags 942 * 943 * Description: 944 * For use by do_sendfile(). splice can easily emulate sendfile, but 945 * doing it in the application would incur an extra system call 946 * (splice in + splice out, as compared to just sendfile()). So this helper 947 * can splice directly through a process-private pipe. 948 * 949 */ 950 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 951 loff_t *opos, size_t len, unsigned int flags) 952 { 953 struct splice_desc sd = { 954 .len = len, 955 .total_len = len, 956 .flags = flags, 957 .pos = *ppos, 958 .u.file = out, 959 .opos = opos, 960 }; 961 long ret; 962 963 if (unlikely(!(out->f_mode & FMODE_WRITE))) 964 return -EBADF; 965 966 if (unlikely(out->f_flags & O_APPEND)) 967 return -EINVAL; 968 969 ret = rw_verify_area(WRITE, out, opos, len); 970 if (unlikely(ret < 0)) 971 return ret; 972 973 ret = splice_direct_to_actor(in, &sd, direct_splice_actor); 974 if (ret > 0) 975 *ppos = sd.pos; 976 977 return ret; 978 } 979 EXPORT_SYMBOL(do_splice_direct); 980 981 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) 982 { 983 for (;;) { 984 if (unlikely(!pipe->readers)) { 985 send_sig(SIGPIPE, current, 0); 986 return -EPIPE; 987 } 988 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 989 return 0; 990 if (flags & SPLICE_F_NONBLOCK) 991 return -EAGAIN; 992 if (signal_pending(current)) 993 return -ERESTARTSYS; 994 pipe_wait_writable(pipe); 995 } 996 } 997 998 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 999 struct pipe_inode_info *opipe, 1000 size_t len, unsigned int flags); 1001 1002 long splice_file_to_pipe(struct file *in, 1003 struct pipe_inode_info *opipe, 1004 loff_t *offset, 1005 size_t len, unsigned int flags) 1006 { 1007 long ret; 1008 1009 pipe_lock(opipe); 1010 ret = wait_for_space(opipe, flags); 1011 if (!ret) 1012 ret = do_splice_to(in, offset, opipe, len, flags); 1013 pipe_unlock(opipe); 1014 if (ret > 0) 1015 wakeup_pipe_readers(opipe); 1016 return ret; 1017 } 1018 1019 /* 1020 * Determine where to splice to/from. 1021 */ 1022 long do_splice(struct file *in, loff_t *off_in, struct file *out, 1023 loff_t *off_out, size_t len, unsigned int flags) 1024 { 1025 struct pipe_inode_info *ipipe; 1026 struct pipe_inode_info *opipe; 1027 loff_t offset; 1028 long ret; 1029 1030 if (unlikely(!(in->f_mode & FMODE_READ) || 1031 !(out->f_mode & FMODE_WRITE))) 1032 return -EBADF; 1033 1034 ipipe = get_pipe_info(in, true); 1035 opipe = get_pipe_info(out, true); 1036 1037 if (ipipe && opipe) { 1038 if (off_in || off_out) 1039 return -ESPIPE; 1040 1041 /* Splicing to self would be fun, but... */ 1042 if (ipipe == opipe) 1043 return -EINVAL; 1044 1045 if ((in->f_flags | out->f_flags) & O_NONBLOCK) 1046 flags |= SPLICE_F_NONBLOCK; 1047 1048 return splice_pipe_to_pipe(ipipe, opipe, len, flags); 1049 } 1050 1051 if (ipipe) { 1052 if (off_in) 1053 return -ESPIPE; 1054 if (off_out) { 1055 if (!(out->f_mode & FMODE_PWRITE)) 1056 return -EINVAL; 1057 offset = *off_out; 1058 } else { 1059 offset = out->f_pos; 1060 } 1061 1062 if (unlikely(out->f_flags & O_APPEND)) 1063 return -EINVAL; 1064 1065 ret = rw_verify_area(WRITE, out, &offset, len); 1066 if (unlikely(ret < 0)) 1067 return ret; 1068 1069 if (in->f_flags & O_NONBLOCK) 1070 flags |= SPLICE_F_NONBLOCK; 1071 1072 file_start_write(out); 1073 ret = do_splice_from(ipipe, out, &offset, len, flags); 1074 file_end_write(out); 1075 1076 if (!off_out) 1077 out->f_pos = offset; 1078 else 1079 *off_out = offset; 1080 1081 return ret; 1082 } 1083 1084 if (opipe) { 1085 if (off_out) 1086 return -ESPIPE; 1087 if (off_in) { 1088 if (!(in->f_mode & FMODE_PREAD)) 1089 return -EINVAL; 1090 offset = *off_in; 1091 } else { 1092 offset = in->f_pos; 1093 } 1094 1095 if (out->f_flags & O_NONBLOCK) 1096 flags |= SPLICE_F_NONBLOCK; 1097 1098 ret = splice_file_to_pipe(in, opipe, &offset, len, flags); 1099 if (!off_in) 1100 in->f_pos = offset; 1101 else 1102 *off_in = offset; 1103 1104 return ret; 1105 } 1106 1107 return -EINVAL; 1108 } 1109 1110 static long __do_splice(struct file *in, loff_t __user *off_in, 1111 struct file *out, loff_t __user *off_out, 1112 size_t len, unsigned int flags) 1113 { 1114 struct pipe_inode_info *ipipe; 1115 struct pipe_inode_info *opipe; 1116 loff_t offset, *__off_in = NULL, *__off_out = NULL; 1117 long ret; 1118 1119 ipipe = get_pipe_info(in, true); 1120 opipe = get_pipe_info(out, true); 1121 1122 if (ipipe && off_in) 1123 return -ESPIPE; 1124 if (opipe && off_out) 1125 return -ESPIPE; 1126 1127 if (off_out) { 1128 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1129 return -EFAULT; 1130 __off_out = &offset; 1131 } 1132 if (off_in) { 1133 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1134 return -EFAULT; 1135 __off_in = &offset; 1136 } 1137 1138 ret = do_splice(in, __off_in, out, __off_out, len, flags); 1139 if (ret < 0) 1140 return ret; 1141 1142 if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t))) 1143 return -EFAULT; 1144 if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t))) 1145 return -EFAULT; 1146 1147 return ret; 1148 } 1149 1150 static int iter_to_pipe(struct iov_iter *from, 1151 struct pipe_inode_info *pipe, 1152 unsigned flags) 1153 { 1154 struct pipe_buffer buf = { 1155 .ops = &user_page_pipe_buf_ops, 1156 .flags = flags 1157 }; 1158 size_t total = 0; 1159 int ret = 0; 1160 1161 while (iov_iter_count(from)) { 1162 struct page *pages[16]; 1163 ssize_t left; 1164 size_t start; 1165 int i, n; 1166 1167 left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start); 1168 if (left <= 0) { 1169 ret = left; 1170 break; 1171 } 1172 1173 n = DIV_ROUND_UP(left + start, PAGE_SIZE); 1174 for (i = 0; i < n; i++) { 1175 int size = min_t(int, left, PAGE_SIZE - start); 1176 1177 buf.page = pages[i]; 1178 buf.offset = start; 1179 buf.len = size; 1180 ret = add_to_pipe(pipe, &buf); 1181 if (unlikely(ret < 0)) { 1182 iov_iter_revert(from, left); 1183 // this one got dropped by add_to_pipe() 1184 while (++i < n) 1185 put_page(pages[i]); 1186 goto out; 1187 } 1188 total += ret; 1189 left -= size; 1190 start = 0; 1191 } 1192 } 1193 out: 1194 return total ? total : ret; 1195 } 1196 1197 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1198 struct splice_desc *sd) 1199 { 1200 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); 1201 return n == sd->len ? n : -EFAULT; 1202 } 1203 1204 /* 1205 * For lack of a better implementation, implement vmsplice() to userspace 1206 * as a simple copy of the pipes pages to the user iov. 1207 */ 1208 static long vmsplice_to_user(struct file *file, struct iov_iter *iter, 1209 unsigned int flags) 1210 { 1211 struct pipe_inode_info *pipe = get_pipe_info(file, true); 1212 struct splice_desc sd = { 1213 .total_len = iov_iter_count(iter), 1214 .flags = flags, 1215 .u.data = iter 1216 }; 1217 long ret = 0; 1218 1219 if (!pipe) 1220 return -EBADF; 1221 1222 if (sd.total_len) { 1223 pipe_lock(pipe); 1224 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); 1225 pipe_unlock(pipe); 1226 } 1227 1228 return ret; 1229 } 1230 1231 /* 1232 * vmsplice splices a user address range into a pipe. It can be thought of 1233 * as splice-from-memory, where the regular splice is splice-from-file (or 1234 * to file). In both cases the output is a pipe, naturally. 1235 */ 1236 static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter, 1237 unsigned int flags) 1238 { 1239 struct pipe_inode_info *pipe; 1240 long ret = 0; 1241 unsigned buf_flag = 0; 1242 1243 if (flags & SPLICE_F_GIFT) 1244 buf_flag = PIPE_BUF_FLAG_GIFT; 1245 1246 pipe = get_pipe_info(file, true); 1247 if (!pipe) 1248 return -EBADF; 1249 1250 pipe_lock(pipe); 1251 ret = wait_for_space(pipe, flags); 1252 if (!ret) 1253 ret = iter_to_pipe(iter, pipe, buf_flag); 1254 pipe_unlock(pipe); 1255 if (ret > 0) 1256 wakeup_pipe_readers(pipe); 1257 return ret; 1258 } 1259 1260 static int vmsplice_type(struct fd f, int *type) 1261 { 1262 if (!f.file) 1263 return -EBADF; 1264 if (f.file->f_mode & FMODE_WRITE) { 1265 *type = ITER_SOURCE; 1266 } else if (f.file->f_mode & FMODE_READ) { 1267 *type = ITER_DEST; 1268 } else { 1269 fdput(f); 1270 return -EBADF; 1271 } 1272 return 0; 1273 } 1274 1275 /* 1276 * Note that vmsplice only really supports true splicing _from_ user memory 1277 * to a pipe, not the other way around. Splicing from user memory is a simple 1278 * operation that can be supported without any funky alignment restrictions 1279 * or nasty vm tricks. We simply map in the user memory and fill them into 1280 * a pipe. The reverse isn't quite as easy, though. There are two possible 1281 * solutions for that: 1282 * 1283 * - memcpy() the data internally, at which point we might as well just 1284 * do a regular read() on the buffer anyway. 1285 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1286 * has restriction limitations on both ends of the pipe). 1287 * 1288 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1289 * 1290 */ 1291 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov, 1292 unsigned long, nr_segs, unsigned int, flags) 1293 { 1294 struct iovec iovstack[UIO_FASTIOV]; 1295 struct iovec *iov = iovstack; 1296 struct iov_iter iter; 1297 ssize_t error; 1298 struct fd f; 1299 int type; 1300 1301 if (unlikely(flags & ~SPLICE_F_ALL)) 1302 return -EINVAL; 1303 1304 f = fdget(fd); 1305 error = vmsplice_type(f, &type); 1306 if (error) 1307 return error; 1308 1309 error = import_iovec(type, uiov, nr_segs, 1310 ARRAY_SIZE(iovstack), &iov, &iter); 1311 if (error < 0) 1312 goto out_fdput; 1313 1314 if (!iov_iter_count(&iter)) 1315 error = 0; 1316 else if (type == ITER_SOURCE) 1317 error = vmsplice_to_pipe(f.file, &iter, flags); 1318 else 1319 error = vmsplice_to_user(f.file, &iter, flags); 1320 1321 kfree(iov); 1322 out_fdput: 1323 fdput(f); 1324 return error; 1325 } 1326 1327 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, 1328 int, fd_out, loff_t __user *, off_out, 1329 size_t, len, unsigned int, flags) 1330 { 1331 struct fd in, out; 1332 long error; 1333 1334 if (unlikely(!len)) 1335 return 0; 1336 1337 if (unlikely(flags & ~SPLICE_F_ALL)) 1338 return -EINVAL; 1339 1340 error = -EBADF; 1341 in = fdget(fd_in); 1342 if (in.file) { 1343 out = fdget(fd_out); 1344 if (out.file) { 1345 error = __do_splice(in.file, off_in, out.file, off_out, 1346 len, flags); 1347 fdput(out); 1348 } 1349 fdput(in); 1350 } 1351 return error; 1352 } 1353 1354 /* 1355 * Make sure there's data to read. Wait for input if we can, otherwise 1356 * return an appropriate error. 1357 */ 1358 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1359 { 1360 int ret; 1361 1362 /* 1363 * Check the pipe occupancy without the inode lock first. This function 1364 * is speculative anyways, so missing one is ok. 1365 */ 1366 if (!pipe_empty(pipe->head, pipe->tail)) 1367 return 0; 1368 1369 ret = 0; 1370 pipe_lock(pipe); 1371 1372 while (pipe_empty(pipe->head, pipe->tail)) { 1373 if (signal_pending(current)) { 1374 ret = -ERESTARTSYS; 1375 break; 1376 } 1377 if (!pipe->writers) 1378 break; 1379 if (flags & SPLICE_F_NONBLOCK) { 1380 ret = -EAGAIN; 1381 break; 1382 } 1383 pipe_wait_readable(pipe); 1384 } 1385 1386 pipe_unlock(pipe); 1387 return ret; 1388 } 1389 1390 /* 1391 * Make sure there's writeable room. Wait for room if we can, otherwise 1392 * return an appropriate error. 1393 */ 1394 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1395 { 1396 int ret; 1397 1398 /* 1399 * Check pipe occupancy without the inode lock first. This function 1400 * is speculative anyways, so missing one is ok. 1401 */ 1402 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 1403 return 0; 1404 1405 ret = 0; 1406 pipe_lock(pipe); 1407 1408 while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { 1409 if (!pipe->readers) { 1410 send_sig(SIGPIPE, current, 0); 1411 ret = -EPIPE; 1412 break; 1413 } 1414 if (flags & SPLICE_F_NONBLOCK) { 1415 ret = -EAGAIN; 1416 break; 1417 } 1418 if (signal_pending(current)) { 1419 ret = -ERESTARTSYS; 1420 break; 1421 } 1422 pipe_wait_writable(pipe); 1423 } 1424 1425 pipe_unlock(pipe); 1426 return ret; 1427 } 1428 1429 /* 1430 * Splice contents of ipipe to opipe. 1431 */ 1432 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1433 struct pipe_inode_info *opipe, 1434 size_t len, unsigned int flags) 1435 { 1436 struct pipe_buffer *ibuf, *obuf; 1437 unsigned int i_head, o_head; 1438 unsigned int i_tail, o_tail; 1439 unsigned int i_mask, o_mask; 1440 int ret = 0; 1441 bool input_wakeup = false; 1442 1443 1444 retry: 1445 ret = ipipe_prep(ipipe, flags); 1446 if (ret) 1447 return ret; 1448 1449 ret = opipe_prep(opipe, flags); 1450 if (ret) 1451 return ret; 1452 1453 /* 1454 * Potential ABBA deadlock, work around it by ordering lock 1455 * grabbing by pipe info address. Otherwise two different processes 1456 * could deadlock (one doing tee from A -> B, the other from B -> A). 1457 */ 1458 pipe_double_lock(ipipe, opipe); 1459 1460 i_tail = ipipe->tail; 1461 i_mask = ipipe->ring_size - 1; 1462 o_head = opipe->head; 1463 o_mask = opipe->ring_size - 1; 1464 1465 do { 1466 size_t o_len; 1467 1468 if (!opipe->readers) { 1469 send_sig(SIGPIPE, current, 0); 1470 if (!ret) 1471 ret = -EPIPE; 1472 break; 1473 } 1474 1475 i_head = ipipe->head; 1476 o_tail = opipe->tail; 1477 1478 if (pipe_empty(i_head, i_tail) && !ipipe->writers) 1479 break; 1480 1481 /* 1482 * Cannot make any progress, because either the input 1483 * pipe is empty or the output pipe is full. 1484 */ 1485 if (pipe_empty(i_head, i_tail) || 1486 pipe_full(o_head, o_tail, opipe->max_usage)) { 1487 /* Already processed some buffers, break */ 1488 if (ret) 1489 break; 1490 1491 if (flags & SPLICE_F_NONBLOCK) { 1492 ret = -EAGAIN; 1493 break; 1494 } 1495 1496 /* 1497 * We raced with another reader/writer and haven't 1498 * managed to process any buffers. A zero return 1499 * value means EOF, so retry instead. 1500 */ 1501 pipe_unlock(ipipe); 1502 pipe_unlock(opipe); 1503 goto retry; 1504 } 1505 1506 ibuf = &ipipe->bufs[i_tail & i_mask]; 1507 obuf = &opipe->bufs[o_head & o_mask]; 1508 1509 if (len >= ibuf->len) { 1510 /* 1511 * Simply move the whole buffer from ipipe to opipe 1512 */ 1513 *obuf = *ibuf; 1514 ibuf->ops = NULL; 1515 i_tail++; 1516 ipipe->tail = i_tail; 1517 input_wakeup = true; 1518 o_len = obuf->len; 1519 o_head++; 1520 opipe->head = o_head; 1521 } else { 1522 /* 1523 * Get a reference to this pipe buffer, 1524 * so we can copy the contents over. 1525 */ 1526 if (!pipe_buf_get(ipipe, ibuf)) { 1527 if (ret == 0) 1528 ret = -EFAULT; 1529 break; 1530 } 1531 *obuf = *ibuf; 1532 1533 /* 1534 * Don't inherit the gift and merge flags, we need to 1535 * prevent multiple steals of this page. 1536 */ 1537 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1538 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; 1539 1540 obuf->len = len; 1541 ibuf->offset += len; 1542 ibuf->len -= len; 1543 o_len = len; 1544 o_head++; 1545 opipe->head = o_head; 1546 } 1547 ret += o_len; 1548 len -= o_len; 1549 } while (len); 1550 1551 pipe_unlock(ipipe); 1552 pipe_unlock(opipe); 1553 1554 /* 1555 * If we put data in the output pipe, wakeup any potential readers. 1556 */ 1557 if (ret > 0) 1558 wakeup_pipe_readers(opipe); 1559 1560 if (input_wakeup) 1561 wakeup_pipe_writers(ipipe); 1562 1563 return ret; 1564 } 1565 1566 /* 1567 * Link contents of ipipe to opipe. 1568 */ 1569 static int link_pipe(struct pipe_inode_info *ipipe, 1570 struct pipe_inode_info *opipe, 1571 size_t len, unsigned int flags) 1572 { 1573 struct pipe_buffer *ibuf, *obuf; 1574 unsigned int i_head, o_head; 1575 unsigned int i_tail, o_tail; 1576 unsigned int i_mask, o_mask; 1577 int ret = 0; 1578 1579 /* 1580 * Potential ABBA deadlock, work around it by ordering lock 1581 * grabbing by pipe info address. Otherwise two different processes 1582 * could deadlock (one doing tee from A -> B, the other from B -> A). 1583 */ 1584 pipe_double_lock(ipipe, opipe); 1585 1586 i_tail = ipipe->tail; 1587 i_mask = ipipe->ring_size - 1; 1588 o_head = opipe->head; 1589 o_mask = opipe->ring_size - 1; 1590 1591 do { 1592 if (!opipe->readers) { 1593 send_sig(SIGPIPE, current, 0); 1594 if (!ret) 1595 ret = -EPIPE; 1596 break; 1597 } 1598 1599 i_head = ipipe->head; 1600 o_tail = opipe->tail; 1601 1602 /* 1603 * If we have iterated all input buffers or run out of 1604 * output room, break. 1605 */ 1606 if (pipe_empty(i_head, i_tail) || 1607 pipe_full(o_head, o_tail, opipe->max_usage)) 1608 break; 1609 1610 ibuf = &ipipe->bufs[i_tail & i_mask]; 1611 obuf = &opipe->bufs[o_head & o_mask]; 1612 1613 /* 1614 * Get a reference to this pipe buffer, 1615 * so we can copy the contents over. 1616 */ 1617 if (!pipe_buf_get(ipipe, ibuf)) { 1618 if (ret == 0) 1619 ret = -EFAULT; 1620 break; 1621 } 1622 1623 *obuf = *ibuf; 1624 1625 /* 1626 * Don't inherit the gift and merge flag, we need to prevent 1627 * multiple steals of this page. 1628 */ 1629 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1630 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE; 1631 1632 if (obuf->len > len) 1633 obuf->len = len; 1634 ret += obuf->len; 1635 len -= obuf->len; 1636 1637 o_head++; 1638 opipe->head = o_head; 1639 i_tail++; 1640 } while (len); 1641 1642 pipe_unlock(ipipe); 1643 pipe_unlock(opipe); 1644 1645 /* 1646 * If we put data in the output pipe, wakeup any potential readers. 1647 */ 1648 if (ret > 0) 1649 wakeup_pipe_readers(opipe); 1650 1651 return ret; 1652 } 1653 1654 /* 1655 * This is a tee(1) implementation that works on pipes. It doesn't copy 1656 * any data, it simply references the 'in' pages on the 'out' pipe. 1657 * The 'flags' used are the SPLICE_F_* variants, currently the only 1658 * applicable one is SPLICE_F_NONBLOCK. 1659 */ 1660 long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags) 1661 { 1662 struct pipe_inode_info *ipipe = get_pipe_info(in, true); 1663 struct pipe_inode_info *opipe = get_pipe_info(out, true); 1664 int ret = -EINVAL; 1665 1666 if (unlikely(!(in->f_mode & FMODE_READ) || 1667 !(out->f_mode & FMODE_WRITE))) 1668 return -EBADF; 1669 1670 /* 1671 * Duplicate the contents of ipipe to opipe without actually 1672 * copying the data. 1673 */ 1674 if (ipipe && opipe && ipipe != opipe) { 1675 if ((in->f_flags | out->f_flags) & O_NONBLOCK) 1676 flags |= SPLICE_F_NONBLOCK; 1677 1678 /* 1679 * Keep going, unless we encounter an error. The ipipe/opipe 1680 * ordering doesn't really matter. 1681 */ 1682 ret = ipipe_prep(ipipe, flags); 1683 if (!ret) { 1684 ret = opipe_prep(opipe, flags); 1685 if (!ret) 1686 ret = link_pipe(ipipe, opipe, len, flags); 1687 } 1688 } 1689 1690 return ret; 1691 } 1692 1693 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) 1694 { 1695 struct fd in, out; 1696 int error; 1697 1698 if (unlikely(flags & ~SPLICE_F_ALL)) 1699 return -EINVAL; 1700 1701 if (unlikely(!len)) 1702 return 0; 1703 1704 error = -EBADF; 1705 in = fdget(fdin); 1706 if (in.file) { 1707 out = fdget(fdout); 1708 if (out.file) { 1709 error = do_tee(in.file, out.file, len, flags); 1710 fdput(out); 1711 } 1712 fdput(in); 1713 } 1714 1715 return error; 1716 } 1717