1 /* 2 * "splice": joining two ropes together by interweaving their strands. 3 * 4 * This is the "extended pipe" functionality, where a pipe is used as 5 * an arbitrary in-memory buffer. Think of a pipe as a small kernel 6 * buffer that you can use to transfer data from one end to the other. 7 * 8 * The traditional unix read/write is extended with a "splice()" operation 9 * that transfers data buffers to or from a pipe buffer. 10 * 11 * Named by Larry McVoy, original implementation from Linus, extended by 12 * Jens to support splicing to files, network, direct splicing, etc and 13 * fixing lots of bugs. 14 * 15 * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk> 16 * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org> 17 * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu> 18 * 19 */ 20 #include <linux/fs.h> 21 #include <linux/file.h> 22 #include <linux/pagemap.h> 23 #include <linux/splice.h> 24 #include <linux/memcontrol.h> 25 #include <linux/mm_inline.h> 26 #include <linux/swap.h> 27 #include <linux/writeback.h> 28 #include <linux/export.h> 29 #include <linux/syscalls.h> 30 #include <linux/uio.h> 31 #include <linux/security.h> 32 #include <linux/gfp.h> 33 #include <linux/socket.h> 34 #include <linux/compat.h> 35 #include "internal.h" 36 37 /* 38 * Attempt to steal a page from a pipe buffer. This should perhaps go into 39 * a vm helper function, it's already simplified quite a bit by the 40 * addition of remove_mapping(). If success is returned, the caller may 41 * attempt to reuse this page for another destination. 42 */ 43 static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe, 44 struct pipe_buffer *buf) 45 { 46 struct page *page = buf->page; 47 struct address_space *mapping; 48 49 lock_page(page); 50 51 mapping = page_mapping(page); 52 if (mapping) { 53 WARN_ON(!PageUptodate(page)); 54 55 /* 56 * At least for ext2 with nobh option, we need to wait on 57 * writeback completing on this page, since we'll remove it 58 * from the pagecache. Otherwise truncate wont wait on the 59 * page, allowing the disk blocks to be reused by someone else 60 * before we actually wrote our data to them. fs corruption 61 * ensues. 62 */ 63 wait_on_page_writeback(page); 64 65 if (page_has_private(page) && 66 !try_to_release_page(page, GFP_KERNEL)) 67 goto out_unlock; 68 69 /* 70 * If we succeeded in removing the mapping, set LRU flag 71 * and return good. 72 */ 73 if (remove_mapping(mapping, page)) { 74 buf->flags |= PIPE_BUF_FLAG_LRU; 75 return 0; 76 } 77 } 78 79 /* 80 * Raced with truncate or failed to remove page from current 81 * address space, unlock and return failure. 82 */ 83 out_unlock: 84 unlock_page(page); 85 return 1; 86 } 87 88 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, 89 struct pipe_buffer *buf) 90 { 91 page_cache_release(buf->page); 92 buf->flags &= ~PIPE_BUF_FLAG_LRU; 93 } 94 95 /* 96 * Check whether the contents of buf is OK to access. Since the content 97 * is a page cache page, IO may be in flight. 98 */ 99 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, 100 struct pipe_buffer *buf) 101 { 102 struct page *page = buf->page; 103 int err; 104 105 if (!PageUptodate(page)) { 106 lock_page(page); 107 108 /* 109 * Page got truncated/unhashed. This will cause a 0-byte 110 * splice, if this is the first page. 111 */ 112 if (!page->mapping) { 113 err = -ENODATA; 114 goto error; 115 } 116 117 /* 118 * Uh oh, read-error from disk. 119 */ 120 if (!PageUptodate(page)) { 121 err = -EIO; 122 goto error; 123 } 124 125 /* 126 * Page is ok afterall, we are done. 127 */ 128 unlock_page(page); 129 } 130 131 return 0; 132 error: 133 unlock_page(page); 134 return err; 135 } 136 137 const struct pipe_buf_operations page_cache_pipe_buf_ops = { 138 .can_merge = 0, 139 .confirm = page_cache_pipe_buf_confirm, 140 .release = page_cache_pipe_buf_release, 141 .steal = page_cache_pipe_buf_steal, 142 .get = generic_pipe_buf_get, 143 }; 144 145 static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe, 146 struct pipe_buffer *buf) 147 { 148 if (!(buf->flags & PIPE_BUF_FLAG_GIFT)) 149 return 1; 150 151 buf->flags |= PIPE_BUF_FLAG_LRU; 152 return generic_pipe_buf_steal(pipe, buf); 153 } 154 155 static const struct pipe_buf_operations user_page_pipe_buf_ops = { 156 .can_merge = 0, 157 .confirm = generic_pipe_buf_confirm, 158 .release = page_cache_pipe_buf_release, 159 .steal = user_page_pipe_buf_steal, 160 .get = generic_pipe_buf_get, 161 }; 162 163 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) 164 { 165 smp_mb(); 166 if (waitqueue_active(&pipe->wait)) 167 wake_up_interruptible(&pipe->wait); 168 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 169 } 170 171 /** 172 * splice_to_pipe - fill passed data into a pipe 173 * @pipe: pipe to fill 174 * @spd: data to fill 175 * 176 * Description: 177 * @spd contains a map of pages and len/offset tuples, along with 178 * the struct pipe_buf_operations associated with these pages. This 179 * function will link that data to the pipe. 180 * 181 */ 182 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, 183 struct splice_pipe_desc *spd) 184 { 185 unsigned int spd_pages = spd->nr_pages; 186 int ret, do_wakeup, page_nr; 187 188 ret = 0; 189 do_wakeup = 0; 190 page_nr = 0; 191 192 pipe_lock(pipe); 193 194 for (;;) { 195 if (!pipe->readers) { 196 send_sig(SIGPIPE, current, 0); 197 if (!ret) 198 ret = -EPIPE; 199 break; 200 } 201 202 if (pipe->nrbufs < pipe->buffers) { 203 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 204 struct pipe_buffer *buf = pipe->bufs + newbuf; 205 206 buf->page = spd->pages[page_nr]; 207 buf->offset = spd->partial[page_nr].offset; 208 buf->len = spd->partial[page_nr].len; 209 buf->private = spd->partial[page_nr].private; 210 buf->ops = spd->ops; 211 if (spd->flags & SPLICE_F_GIFT) 212 buf->flags |= PIPE_BUF_FLAG_GIFT; 213 214 pipe->nrbufs++; 215 page_nr++; 216 ret += buf->len; 217 218 if (pipe->files) 219 do_wakeup = 1; 220 221 if (!--spd->nr_pages) 222 break; 223 if (pipe->nrbufs < pipe->buffers) 224 continue; 225 226 break; 227 } 228 229 if (spd->flags & SPLICE_F_NONBLOCK) { 230 if (!ret) 231 ret = -EAGAIN; 232 break; 233 } 234 235 if (signal_pending(current)) { 236 if (!ret) 237 ret = -ERESTARTSYS; 238 break; 239 } 240 241 if (do_wakeup) { 242 smp_mb(); 243 if (waitqueue_active(&pipe->wait)) 244 wake_up_interruptible_sync(&pipe->wait); 245 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 246 do_wakeup = 0; 247 } 248 249 pipe->waiting_writers++; 250 pipe_wait(pipe); 251 pipe->waiting_writers--; 252 } 253 254 pipe_unlock(pipe); 255 256 if (do_wakeup) 257 wakeup_pipe_readers(pipe); 258 259 while (page_nr < spd_pages) 260 spd->spd_release(spd, page_nr++); 261 262 return ret; 263 } 264 265 void spd_release_page(struct splice_pipe_desc *spd, unsigned int i) 266 { 267 page_cache_release(spd->pages[i]); 268 } 269 270 /* 271 * Check if we need to grow the arrays holding pages and partial page 272 * descriptions. 273 */ 274 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) 275 { 276 unsigned int buffers = ACCESS_ONCE(pipe->buffers); 277 278 spd->nr_pages_max = buffers; 279 if (buffers <= PIPE_DEF_BUFFERS) 280 return 0; 281 282 spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL); 283 spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL); 284 285 if (spd->pages && spd->partial) 286 return 0; 287 288 kfree(spd->pages); 289 kfree(spd->partial); 290 return -ENOMEM; 291 } 292 293 void splice_shrink_spd(struct splice_pipe_desc *spd) 294 { 295 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS) 296 return; 297 298 kfree(spd->pages); 299 kfree(spd->partial); 300 } 301 302 static int 303 __generic_file_splice_read(struct file *in, loff_t *ppos, 304 struct pipe_inode_info *pipe, size_t len, 305 unsigned int flags) 306 { 307 struct address_space *mapping = in->f_mapping; 308 unsigned int loff, nr_pages, req_pages; 309 struct page *pages[PIPE_DEF_BUFFERS]; 310 struct partial_page partial[PIPE_DEF_BUFFERS]; 311 struct page *page; 312 pgoff_t index, end_index; 313 loff_t isize; 314 int error, page_nr; 315 struct splice_pipe_desc spd = { 316 .pages = pages, 317 .partial = partial, 318 .nr_pages_max = PIPE_DEF_BUFFERS, 319 .flags = flags, 320 .ops = &page_cache_pipe_buf_ops, 321 .spd_release = spd_release_page, 322 }; 323 324 if (splice_grow_spd(pipe, &spd)) 325 return -ENOMEM; 326 327 index = *ppos >> PAGE_CACHE_SHIFT; 328 loff = *ppos & ~PAGE_CACHE_MASK; 329 req_pages = (len + loff + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 330 nr_pages = min(req_pages, spd.nr_pages_max); 331 332 /* 333 * Lookup the (hopefully) full range of pages we need. 334 */ 335 spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, spd.pages); 336 index += spd.nr_pages; 337 338 /* 339 * If find_get_pages_contig() returned fewer pages than we needed, 340 * readahead/allocate the rest and fill in the holes. 341 */ 342 if (spd.nr_pages < nr_pages) 343 page_cache_sync_readahead(mapping, &in->f_ra, in, 344 index, req_pages - spd.nr_pages); 345 346 error = 0; 347 while (spd.nr_pages < nr_pages) { 348 /* 349 * Page could be there, find_get_pages_contig() breaks on 350 * the first hole. 351 */ 352 page = find_get_page(mapping, index); 353 if (!page) { 354 /* 355 * page didn't exist, allocate one. 356 */ 357 page = page_cache_alloc_cold(mapping); 358 if (!page) 359 break; 360 361 error = add_to_page_cache_lru(page, mapping, index, 362 GFP_KERNEL); 363 if (unlikely(error)) { 364 page_cache_release(page); 365 if (error == -EEXIST) 366 continue; 367 break; 368 } 369 /* 370 * add_to_page_cache() locks the page, unlock it 371 * to avoid convoluting the logic below even more. 372 */ 373 unlock_page(page); 374 } 375 376 spd.pages[spd.nr_pages++] = page; 377 index++; 378 } 379 380 /* 381 * Now loop over the map and see if we need to start IO on any 382 * pages, fill in the partial map, etc. 383 */ 384 index = *ppos >> PAGE_CACHE_SHIFT; 385 nr_pages = spd.nr_pages; 386 spd.nr_pages = 0; 387 for (page_nr = 0; page_nr < nr_pages; page_nr++) { 388 unsigned int this_len; 389 390 if (!len) 391 break; 392 393 /* 394 * this_len is the max we'll use from this page 395 */ 396 this_len = min_t(unsigned long, len, PAGE_CACHE_SIZE - loff); 397 page = spd.pages[page_nr]; 398 399 if (PageReadahead(page)) 400 page_cache_async_readahead(mapping, &in->f_ra, in, 401 page, index, req_pages - page_nr); 402 403 /* 404 * If the page isn't uptodate, we may need to start io on it 405 */ 406 if (!PageUptodate(page)) { 407 lock_page(page); 408 409 /* 410 * Page was truncated, or invalidated by the 411 * filesystem. Redo the find/create, but this time the 412 * page is kept locked, so there's no chance of another 413 * race with truncate/invalidate. 414 */ 415 if (!page->mapping) { 416 unlock_page(page); 417 page = find_or_create_page(mapping, index, 418 mapping_gfp_mask(mapping)); 419 420 if (!page) { 421 error = -ENOMEM; 422 break; 423 } 424 page_cache_release(spd.pages[page_nr]); 425 spd.pages[page_nr] = page; 426 } 427 /* 428 * page was already under io and is now done, great 429 */ 430 if (PageUptodate(page)) { 431 unlock_page(page); 432 goto fill_it; 433 } 434 435 /* 436 * need to read in the page 437 */ 438 error = mapping->a_ops->readpage(in, page); 439 if (unlikely(error)) { 440 /* 441 * We really should re-lookup the page here, 442 * but it complicates things a lot. Instead 443 * lets just do what we already stored, and 444 * we'll get it the next time we are called. 445 */ 446 if (error == AOP_TRUNCATED_PAGE) 447 error = 0; 448 449 break; 450 } 451 } 452 fill_it: 453 /* 454 * i_size must be checked after PageUptodate. 455 */ 456 isize = i_size_read(mapping->host); 457 end_index = (isize - 1) >> PAGE_CACHE_SHIFT; 458 if (unlikely(!isize || index > end_index)) 459 break; 460 461 /* 462 * if this is the last page, see if we need to shrink 463 * the length and stop 464 */ 465 if (end_index == index) { 466 unsigned int plen; 467 468 /* 469 * max good bytes in this page 470 */ 471 plen = ((isize - 1) & ~PAGE_CACHE_MASK) + 1; 472 if (plen <= loff) 473 break; 474 475 /* 476 * force quit after adding this page 477 */ 478 this_len = min(this_len, plen - loff); 479 len = this_len; 480 } 481 482 spd.partial[page_nr].offset = loff; 483 spd.partial[page_nr].len = this_len; 484 len -= this_len; 485 loff = 0; 486 spd.nr_pages++; 487 index++; 488 } 489 490 /* 491 * Release any pages at the end, if we quit early. 'page_nr' is how far 492 * we got, 'nr_pages' is how many pages are in the map. 493 */ 494 while (page_nr < nr_pages) 495 page_cache_release(spd.pages[page_nr++]); 496 in->f_ra.prev_pos = (loff_t)index << PAGE_CACHE_SHIFT; 497 498 if (spd.nr_pages) 499 error = splice_to_pipe(pipe, &spd); 500 501 splice_shrink_spd(&spd); 502 return error; 503 } 504 505 /** 506 * generic_file_splice_read - splice data from file to a pipe 507 * @in: file to splice from 508 * @ppos: position in @in 509 * @pipe: pipe to splice to 510 * @len: number of bytes to splice 511 * @flags: splice modifier flags 512 * 513 * Description: 514 * Will read pages from given file and fill them into a pipe. Can be 515 * used as long as the address_space operations for the source implements 516 * a readpage() hook. 517 * 518 */ 519 ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, 520 struct pipe_inode_info *pipe, size_t len, 521 unsigned int flags) 522 { 523 loff_t isize, left; 524 int ret; 525 526 if (IS_DAX(in->f_mapping->host)) 527 return default_file_splice_read(in, ppos, pipe, len, flags); 528 529 isize = i_size_read(in->f_mapping->host); 530 if (unlikely(*ppos >= isize)) 531 return 0; 532 533 left = isize - *ppos; 534 if (unlikely(left < len)) 535 len = left; 536 537 ret = __generic_file_splice_read(in, ppos, pipe, len, flags); 538 if (ret > 0) { 539 *ppos += ret; 540 file_accessed(in); 541 } 542 543 return ret; 544 } 545 EXPORT_SYMBOL(generic_file_splice_read); 546 547 static const struct pipe_buf_operations default_pipe_buf_ops = { 548 .can_merge = 0, 549 .confirm = generic_pipe_buf_confirm, 550 .release = generic_pipe_buf_release, 551 .steal = generic_pipe_buf_steal, 552 .get = generic_pipe_buf_get, 553 }; 554 555 static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe, 556 struct pipe_buffer *buf) 557 { 558 return 1; 559 } 560 561 /* Pipe buffer operations for a socket and similar. */ 562 const struct pipe_buf_operations nosteal_pipe_buf_ops = { 563 .can_merge = 0, 564 .confirm = generic_pipe_buf_confirm, 565 .release = generic_pipe_buf_release, 566 .steal = generic_pipe_buf_nosteal, 567 .get = generic_pipe_buf_get, 568 }; 569 EXPORT_SYMBOL(nosteal_pipe_buf_ops); 570 571 static ssize_t kernel_readv(struct file *file, const struct iovec *vec, 572 unsigned long vlen, loff_t offset) 573 { 574 mm_segment_t old_fs; 575 loff_t pos = offset; 576 ssize_t res; 577 578 old_fs = get_fs(); 579 set_fs(get_ds()); 580 /* The cast to a user pointer is valid due to the set_fs() */ 581 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos); 582 set_fs(old_fs); 583 584 return res; 585 } 586 587 ssize_t kernel_write(struct file *file, const char *buf, size_t count, 588 loff_t pos) 589 { 590 mm_segment_t old_fs; 591 ssize_t res; 592 593 old_fs = get_fs(); 594 set_fs(get_ds()); 595 /* The cast to a user pointer is valid due to the set_fs() */ 596 res = vfs_write(file, (__force const char __user *)buf, count, &pos); 597 set_fs(old_fs); 598 599 return res; 600 } 601 EXPORT_SYMBOL(kernel_write); 602 603 ssize_t default_file_splice_read(struct file *in, loff_t *ppos, 604 struct pipe_inode_info *pipe, size_t len, 605 unsigned int flags) 606 { 607 unsigned int nr_pages; 608 unsigned int nr_freed; 609 size_t offset; 610 struct page *pages[PIPE_DEF_BUFFERS]; 611 struct partial_page partial[PIPE_DEF_BUFFERS]; 612 struct iovec *vec, __vec[PIPE_DEF_BUFFERS]; 613 ssize_t res; 614 size_t this_len; 615 int error; 616 int i; 617 struct splice_pipe_desc spd = { 618 .pages = pages, 619 .partial = partial, 620 .nr_pages_max = PIPE_DEF_BUFFERS, 621 .flags = flags, 622 .ops = &default_pipe_buf_ops, 623 .spd_release = spd_release_page, 624 }; 625 626 if (splice_grow_spd(pipe, &spd)) 627 return -ENOMEM; 628 629 res = -ENOMEM; 630 vec = __vec; 631 if (spd.nr_pages_max > PIPE_DEF_BUFFERS) { 632 vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL); 633 if (!vec) 634 goto shrink_ret; 635 } 636 637 offset = *ppos & ~PAGE_CACHE_MASK; 638 nr_pages = (len + offset + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 639 640 for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) { 641 struct page *page; 642 643 page = alloc_page(GFP_USER); 644 error = -ENOMEM; 645 if (!page) 646 goto err; 647 648 this_len = min_t(size_t, len, PAGE_CACHE_SIZE - offset); 649 vec[i].iov_base = (void __user *) page_address(page); 650 vec[i].iov_len = this_len; 651 spd.pages[i] = page; 652 spd.nr_pages++; 653 len -= this_len; 654 offset = 0; 655 } 656 657 res = kernel_readv(in, vec, spd.nr_pages, *ppos); 658 if (res < 0) { 659 error = res; 660 goto err; 661 } 662 663 error = 0; 664 if (!res) 665 goto err; 666 667 nr_freed = 0; 668 for (i = 0; i < spd.nr_pages; i++) { 669 this_len = min_t(size_t, vec[i].iov_len, res); 670 spd.partial[i].offset = 0; 671 spd.partial[i].len = this_len; 672 if (!this_len) { 673 __free_page(spd.pages[i]); 674 spd.pages[i] = NULL; 675 nr_freed++; 676 } 677 res -= this_len; 678 } 679 spd.nr_pages -= nr_freed; 680 681 res = splice_to_pipe(pipe, &spd); 682 if (res > 0) 683 *ppos += res; 684 685 shrink_ret: 686 if (vec != __vec) 687 kfree(vec); 688 splice_shrink_spd(&spd); 689 return res; 690 691 err: 692 for (i = 0; i < spd.nr_pages; i++) 693 __free_page(spd.pages[i]); 694 695 res = error; 696 goto shrink_ret; 697 } 698 EXPORT_SYMBOL(default_file_splice_read); 699 700 /* 701 * Send 'sd->len' bytes to socket from 'sd->file' at position 'sd->pos' 702 * using sendpage(). Return the number of bytes sent. 703 */ 704 static int pipe_to_sendpage(struct pipe_inode_info *pipe, 705 struct pipe_buffer *buf, struct splice_desc *sd) 706 { 707 struct file *file = sd->u.file; 708 loff_t pos = sd->pos; 709 int more; 710 711 if (!likely(file->f_op->sendpage)) 712 return -EINVAL; 713 714 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0; 715 716 if (sd->len < sd->total_len && pipe->nrbufs > 1) 717 more |= MSG_SENDPAGE_NOTLAST; 718 719 return file->f_op->sendpage(file, buf->page, buf->offset, 720 sd->len, &pos, more); 721 } 722 723 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) 724 { 725 smp_mb(); 726 if (waitqueue_active(&pipe->wait)) 727 wake_up_interruptible(&pipe->wait); 728 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); 729 } 730 731 /** 732 * splice_from_pipe_feed - feed available data from a pipe to a file 733 * @pipe: pipe to splice from 734 * @sd: information to @actor 735 * @actor: handler that splices the data 736 * 737 * Description: 738 * This function loops over the pipe and calls @actor to do the 739 * actual moving of a single struct pipe_buffer to the desired 740 * destination. It returns when there's no more buffers left in 741 * the pipe or if the requested number of bytes (@sd->total_len) 742 * have been copied. It returns a positive number (one) if the 743 * pipe needs to be filled with more data, zero if the required 744 * number of bytes have been copied and -errno on error. 745 * 746 * This, together with splice_from_pipe_{begin,end,next}, may be 747 * used to implement the functionality of __splice_from_pipe() when 748 * locking is required around copying the pipe buffers to the 749 * destination. 750 */ 751 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, 752 splice_actor *actor) 753 { 754 int ret; 755 756 while (pipe->nrbufs) { 757 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 758 const struct pipe_buf_operations *ops = buf->ops; 759 760 sd->len = buf->len; 761 if (sd->len > sd->total_len) 762 sd->len = sd->total_len; 763 764 ret = buf->ops->confirm(pipe, buf); 765 if (unlikely(ret)) { 766 if (ret == -ENODATA) 767 ret = 0; 768 return ret; 769 } 770 771 ret = actor(pipe, buf, sd); 772 if (ret <= 0) 773 return ret; 774 775 buf->offset += ret; 776 buf->len -= ret; 777 778 sd->num_spliced += ret; 779 sd->len -= ret; 780 sd->pos += ret; 781 sd->total_len -= ret; 782 783 if (!buf->len) { 784 buf->ops = NULL; 785 ops->release(pipe, buf); 786 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 787 pipe->nrbufs--; 788 if (pipe->files) 789 sd->need_wakeup = true; 790 } 791 792 if (!sd->total_len) 793 return 0; 794 } 795 796 return 1; 797 } 798 799 /** 800 * splice_from_pipe_next - wait for some data to splice from 801 * @pipe: pipe to splice from 802 * @sd: information about the splice operation 803 * 804 * Description: 805 * This function will wait for some data and return a positive 806 * value (one) if pipe buffers are available. It will return zero 807 * or -errno if no more data needs to be spliced. 808 */ 809 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) 810 { 811 while (!pipe->nrbufs) { 812 if (!pipe->writers) 813 return 0; 814 815 if (!pipe->waiting_writers && sd->num_spliced) 816 return 0; 817 818 if (sd->flags & SPLICE_F_NONBLOCK) 819 return -EAGAIN; 820 821 if (signal_pending(current)) 822 return -ERESTARTSYS; 823 824 if (sd->need_wakeup) { 825 wakeup_pipe_writers(pipe); 826 sd->need_wakeup = false; 827 } 828 829 pipe_wait(pipe); 830 } 831 832 return 1; 833 } 834 835 /** 836 * splice_from_pipe_begin - start splicing from pipe 837 * @sd: information about the splice operation 838 * 839 * Description: 840 * This function should be called before a loop containing 841 * splice_from_pipe_next() and splice_from_pipe_feed() to 842 * initialize the necessary fields of @sd. 843 */ 844 static void splice_from_pipe_begin(struct splice_desc *sd) 845 { 846 sd->num_spliced = 0; 847 sd->need_wakeup = false; 848 } 849 850 /** 851 * splice_from_pipe_end - finish splicing from pipe 852 * @pipe: pipe to splice from 853 * @sd: information about the splice operation 854 * 855 * Description: 856 * This function will wake up pipe writers if necessary. It should 857 * be called after a loop containing splice_from_pipe_next() and 858 * splice_from_pipe_feed(). 859 */ 860 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) 861 { 862 if (sd->need_wakeup) 863 wakeup_pipe_writers(pipe); 864 } 865 866 /** 867 * __splice_from_pipe - splice data from a pipe to given actor 868 * @pipe: pipe to splice from 869 * @sd: information to @actor 870 * @actor: handler that splices the data 871 * 872 * Description: 873 * This function does little more than loop over the pipe and call 874 * @actor to do the actual moving of a single struct pipe_buffer to 875 * the desired destination. See pipe_to_file, pipe_to_sendpage, or 876 * pipe_to_user. 877 * 878 */ 879 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, 880 splice_actor *actor) 881 { 882 int ret; 883 884 splice_from_pipe_begin(sd); 885 do { 886 ret = splice_from_pipe_next(pipe, sd); 887 if (ret > 0) 888 ret = splice_from_pipe_feed(pipe, sd, actor); 889 } while (ret > 0); 890 splice_from_pipe_end(pipe, sd); 891 892 return sd->num_spliced ? sd->num_spliced : ret; 893 } 894 EXPORT_SYMBOL(__splice_from_pipe); 895 896 /** 897 * splice_from_pipe - splice data from a pipe to a file 898 * @pipe: pipe to splice from 899 * @out: file to splice to 900 * @ppos: position in @out 901 * @len: how many bytes to splice 902 * @flags: splice modifier flags 903 * @actor: handler that splices the data 904 * 905 * Description: 906 * See __splice_from_pipe. This function locks the pipe inode, 907 * otherwise it's identical to __splice_from_pipe(). 908 * 909 */ 910 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, 911 loff_t *ppos, size_t len, unsigned int flags, 912 splice_actor *actor) 913 { 914 ssize_t ret; 915 struct splice_desc sd = { 916 .total_len = len, 917 .flags = flags, 918 .pos = *ppos, 919 .u.file = out, 920 }; 921 922 pipe_lock(pipe); 923 ret = __splice_from_pipe(pipe, &sd, actor); 924 pipe_unlock(pipe); 925 926 return ret; 927 } 928 929 /** 930 * iter_file_splice_write - splice data from a pipe to a file 931 * @pipe: pipe info 932 * @out: file to write to 933 * @ppos: position in @out 934 * @len: number of bytes to splice 935 * @flags: splice modifier flags 936 * 937 * Description: 938 * Will either move or copy pages (determined by @flags options) from 939 * the given pipe inode to the given file. 940 * This one is ->write_iter-based. 941 * 942 */ 943 ssize_t 944 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, 945 loff_t *ppos, size_t len, unsigned int flags) 946 { 947 struct splice_desc sd = { 948 .total_len = len, 949 .flags = flags, 950 .pos = *ppos, 951 .u.file = out, 952 }; 953 int nbufs = pipe->buffers; 954 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec), 955 GFP_KERNEL); 956 ssize_t ret; 957 958 if (unlikely(!array)) 959 return -ENOMEM; 960 961 pipe_lock(pipe); 962 963 splice_from_pipe_begin(&sd); 964 while (sd.total_len) { 965 struct iov_iter from; 966 size_t left; 967 int n, idx; 968 969 ret = splice_from_pipe_next(pipe, &sd); 970 if (ret <= 0) 971 break; 972 973 if (unlikely(nbufs < pipe->buffers)) { 974 kfree(array); 975 nbufs = pipe->buffers; 976 array = kcalloc(nbufs, sizeof(struct bio_vec), 977 GFP_KERNEL); 978 if (!array) { 979 ret = -ENOMEM; 980 break; 981 } 982 } 983 984 /* build the vector */ 985 left = sd.total_len; 986 for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) { 987 struct pipe_buffer *buf = pipe->bufs + idx; 988 size_t this_len = buf->len; 989 990 if (this_len > left) 991 this_len = left; 992 993 if (idx == pipe->buffers - 1) 994 idx = -1; 995 996 ret = buf->ops->confirm(pipe, buf); 997 if (unlikely(ret)) { 998 if (ret == -ENODATA) 999 ret = 0; 1000 goto done; 1001 } 1002 1003 array[n].bv_page = buf->page; 1004 array[n].bv_len = this_len; 1005 array[n].bv_offset = buf->offset; 1006 left -= this_len; 1007 } 1008 1009 iov_iter_bvec(&from, ITER_BVEC | WRITE, array, n, 1010 sd.total_len - left); 1011 ret = vfs_iter_write(out, &from, &sd.pos); 1012 if (ret <= 0) 1013 break; 1014 1015 sd.num_spliced += ret; 1016 sd.total_len -= ret; 1017 *ppos = sd.pos; 1018 1019 /* dismiss the fully eaten buffers, adjust the partial one */ 1020 while (ret) { 1021 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; 1022 if (ret >= buf->len) { 1023 const struct pipe_buf_operations *ops = buf->ops; 1024 ret -= buf->len; 1025 buf->len = 0; 1026 buf->ops = NULL; 1027 ops->release(pipe, buf); 1028 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); 1029 pipe->nrbufs--; 1030 if (pipe->files) 1031 sd.need_wakeup = true; 1032 } else { 1033 buf->offset += ret; 1034 buf->len -= ret; 1035 ret = 0; 1036 } 1037 } 1038 } 1039 done: 1040 kfree(array); 1041 splice_from_pipe_end(pipe, &sd); 1042 1043 pipe_unlock(pipe); 1044 1045 if (sd.num_spliced) 1046 ret = sd.num_spliced; 1047 1048 return ret; 1049 } 1050 1051 EXPORT_SYMBOL(iter_file_splice_write); 1052 1053 static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1054 struct splice_desc *sd) 1055 { 1056 int ret; 1057 void *data; 1058 loff_t tmp = sd->pos; 1059 1060 data = kmap(buf->page); 1061 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp); 1062 kunmap(buf->page); 1063 1064 return ret; 1065 } 1066 1067 static ssize_t default_file_splice_write(struct pipe_inode_info *pipe, 1068 struct file *out, loff_t *ppos, 1069 size_t len, unsigned int flags) 1070 { 1071 ssize_t ret; 1072 1073 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf); 1074 if (ret > 0) 1075 *ppos += ret; 1076 1077 return ret; 1078 } 1079 1080 /** 1081 * generic_splice_sendpage - splice data from a pipe to a socket 1082 * @pipe: pipe to splice from 1083 * @out: socket to write to 1084 * @ppos: position in @out 1085 * @len: number of bytes to splice 1086 * @flags: splice modifier flags 1087 * 1088 * Description: 1089 * Will send @len bytes from the pipe to a network socket. No data copying 1090 * is involved. 1091 * 1092 */ 1093 ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, 1094 loff_t *ppos, size_t len, unsigned int flags) 1095 { 1096 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage); 1097 } 1098 1099 EXPORT_SYMBOL(generic_splice_sendpage); 1100 1101 /* 1102 * Attempt to initiate a splice from pipe to file. 1103 */ 1104 static long do_splice_from(struct pipe_inode_info *pipe, struct file *out, 1105 loff_t *ppos, size_t len, unsigned int flags) 1106 { 1107 ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, 1108 loff_t *, size_t, unsigned int); 1109 1110 if (out->f_op->splice_write) 1111 splice_write = out->f_op->splice_write; 1112 else 1113 splice_write = default_file_splice_write; 1114 1115 return splice_write(pipe, out, ppos, len, flags); 1116 } 1117 1118 /* 1119 * Attempt to initiate a splice from a file to a pipe. 1120 */ 1121 static long do_splice_to(struct file *in, loff_t *ppos, 1122 struct pipe_inode_info *pipe, size_t len, 1123 unsigned int flags) 1124 { 1125 ssize_t (*splice_read)(struct file *, loff_t *, 1126 struct pipe_inode_info *, size_t, unsigned int); 1127 int ret; 1128 1129 if (unlikely(!(in->f_mode & FMODE_READ))) 1130 return -EBADF; 1131 1132 ret = rw_verify_area(READ, in, ppos, len); 1133 if (unlikely(ret < 0)) 1134 return ret; 1135 1136 if (in->f_op->splice_read) 1137 splice_read = in->f_op->splice_read; 1138 else 1139 splice_read = default_file_splice_read; 1140 1141 return splice_read(in, ppos, pipe, len, flags); 1142 } 1143 1144 /** 1145 * splice_direct_to_actor - splices data directly between two non-pipes 1146 * @in: file to splice from 1147 * @sd: actor information on where to splice to 1148 * @actor: handles the data splicing 1149 * 1150 * Description: 1151 * This is a special case helper to splice directly between two 1152 * points, without requiring an explicit pipe. Internally an allocated 1153 * pipe is cached in the process, and reused during the lifetime of 1154 * that process. 1155 * 1156 */ 1157 ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd, 1158 splice_direct_actor *actor) 1159 { 1160 struct pipe_inode_info *pipe; 1161 long ret, bytes; 1162 umode_t i_mode; 1163 size_t len; 1164 int i, flags, more; 1165 1166 /* 1167 * We require the input being a regular file, as we don't want to 1168 * randomly drop data for eg socket -> socket splicing. Use the 1169 * piped splicing for that! 1170 */ 1171 i_mode = file_inode(in)->i_mode; 1172 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode))) 1173 return -EINVAL; 1174 1175 /* 1176 * neither in nor out is a pipe, setup an internal pipe attached to 1177 * 'out' and transfer the wanted data from 'in' to 'out' through that 1178 */ 1179 pipe = current->splice_pipe; 1180 if (unlikely(!pipe)) { 1181 pipe = alloc_pipe_info(); 1182 if (!pipe) 1183 return -ENOMEM; 1184 1185 /* 1186 * We don't have an immediate reader, but we'll read the stuff 1187 * out of the pipe right after the splice_to_pipe(). So set 1188 * PIPE_READERS appropriately. 1189 */ 1190 pipe->readers = 1; 1191 1192 current->splice_pipe = pipe; 1193 } 1194 1195 /* 1196 * Do the splice. 1197 */ 1198 ret = 0; 1199 bytes = 0; 1200 len = sd->total_len; 1201 flags = sd->flags; 1202 1203 /* 1204 * Don't block on output, we have to drain the direct pipe. 1205 */ 1206 sd->flags &= ~SPLICE_F_NONBLOCK; 1207 more = sd->flags & SPLICE_F_MORE; 1208 1209 while (len) { 1210 size_t read_len; 1211 loff_t pos = sd->pos, prev_pos = pos; 1212 1213 ret = do_splice_to(in, &pos, pipe, len, flags); 1214 if (unlikely(ret <= 0)) 1215 goto out_release; 1216 1217 read_len = ret; 1218 sd->total_len = read_len; 1219 1220 /* 1221 * If more data is pending, set SPLICE_F_MORE 1222 * If this is the last data and SPLICE_F_MORE was not set 1223 * initially, clears it. 1224 */ 1225 if (read_len < len) 1226 sd->flags |= SPLICE_F_MORE; 1227 else if (!more) 1228 sd->flags &= ~SPLICE_F_MORE; 1229 /* 1230 * NOTE: nonblocking mode only applies to the input. We 1231 * must not do the output in nonblocking mode as then we 1232 * could get stuck data in the internal pipe: 1233 */ 1234 ret = actor(pipe, sd); 1235 if (unlikely(ret <= 0)) { 1236 sd->pos = prev_pos; 1237 goto out_release; 1238 } 1239 1240 bytes += ret; 1241 len -= ret; 1242 sd->pos = pos; 1243 1244 if (ret < read_len) { 1245 sd->pos = prev_pos + ret; 1246 goto out_release; 1247 } 1248 } 1249 1250 done: 1251 pipe->nrbufs = pipe->curbuf = 0; 1252 file_accessed(in); 1253 return bytes; 1254 1255 out_release: 1256 /* 1257 * If we did an incomplete transfer we must release 1258 * the pipe buffers in question: 1259 */ 1260 for (i = 0; i < pipe->buffers; i++) { 1261 struct pipe_buffer *buf = pipe->bufs + i; 1262 1263 if (buf->ops) { 1264 buf->ops->release(pipe, buf); 1265 buf->ops = NULL; 1266 } 1267 } 1268 1269 if (!bytes) 1270 bytes = ret; 1271 1272 goto done; 1273 } 1274 EXPORT_SYMBOL(splice_direct_to_actor); 1275 1276 static int direct_splice_actor(struct pipe_inode_info *pipe, 1277 struct splice_desc *sd) 1278 { 1279 struct file *file = sd->u.file; 1280 1281 return do_splice_from(pipe, file, sd->opos, sd->total_len, 1282 sd->flags); 1283 } 1284 1285 /** 1286 * do_splice_direct - splices data directly between two files 1287 * @in: file to splice from 1288 * @ppos: input file offset 1289 * @out: file to splice to 1290 * @opos: output file offset 1291 * @len: number of bytes to splice 1292 * @flags: splice modifier flags 1293 * 1294 * Description: 1295 * For use by do_sendfile(). splice can easily emulate sendfile, but 1296 * doing it in the application would incur an extra system call 1297 * (splice in + splice out, as compared to just sendfile()). So this helper 1298 * can splice directly through a process-private pipe. 1299 * 1300 */ 1301 long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, 1302 loff_t *opos, size_t len, unsigned int flags) 1303 { 1304 struct splice_desc sd = { 1305 .len = len, 1306 .total_len = len, 1307 .flags = flags, 1308 .pos = *ppos, 1309 .u.file = out, 1310 .opos = opos, 1311 }; 1312 long ret; 1313 1314 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1315 return -EBADF; 1316 1317 if (unlikely(out->f_flags & O_APPEND)) 1318 return -EINVAL; 1319 1320 ret = rw_verify_area(WRITE, out, opos, len); 1321 if (unlikely(ret < 0)) 1322 return ret; 1323 1324 ret = splice_direct_to_actor(in, &sd, direct_splice_actor); 1325 if (ret > 0) 1326 *ppos = sd.pos; 1327 1328 return ret; 1329 } 1330 EXPORT_SYMBOL(do_splice_direct); 1331 1332 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1333 struct pipe_inode_info *opipe, 1334 size_t len, unsigned int flags); 1335 1336 /* 1337 * Determine where to splice to/from. 1338 */ 1339 static long do_splice(struct file *in, loff_t __user *off_in, 1340 struct file *out, loff_t __user *off_out, 1341 size_t len, unsigned int flags) 1342 { 1343 struct pipe_inode_info *ipipe; 1344 struct pipe_inode_info *opipe; 1345 loff_t offset; 1346 long ret; 1347 1348 ipipe = get_pipe_info(in); 1349 opipe = get_pipe_info(out); 1350 1351 if (ipipe && opipe) { 1352 if (off_in || off_out) 1353 return -ESPIPE; 1354 1355 if (!(in->f_mode & FMODE_READ)) 1356 return -EBADF; 1357 1358 if (!(out->f_mode & FMODE_WRITE)) 1359 return -EBADF; 1360 1361 /* Splicing to self would be fun, but... */ 1362 if (ipipe == opipe) 1363 return -EINVAL; 1364 1365 return splice_pipe_to_pipe(ipipe, opipe, len, flags); 1366 } 1367 1368 if (ipipe) { 1369 if (off_in) 1370 return -ESPIPE; 1371 if (off_out) { 1372 if (!(out->f_mode & FMODE_PWRITE)) 1373 return -EINVAL; 1374 if (copy_from_user(&offset, off_out, sizeof(loff_t))) 1375 return -EFAULT; 1376 } else { 1377 offset = out->f_pos; 1378 } 1379 1380 if (unlikely(!(out->f_mode & FMODE_WRITE))) 1381 return -EBADF; 1382 1383 if (unlikely(out->f_flags & O_APPEND)) 1384 return -EINVAL; 1385 1386 ret = rw_verify_area(WRITE, out, &offset, len); 1387 if (unlikely(ret < 0)) 1388 return ret; 1389 1390 file_start_write(out); 1391 ret = do_splice_from(ipipe, out, &offset, len, flags); 1392 file_end_write(out); 1393 1394 if (!off_out) 1395 out->f_pos = offset; 1396 else if (copy_to_user(off_out, &offset, sizeof(loff_t))) 1397 ret = -EFAULT; 1398 1399 return ret; 1400 } 1401 1402 if (opipe) { 1403 if (off_out) 1404 return -ESPIPE; 1405 if (off_in) { 1406 if (!(in->f_mode & FMODE_PREAD)) 1407 return -EINVAL; 1408 if (copy_from_user(&offset, off_in, sizeof(loff_t))) 1409 return -EFAULT; 1410 } else { 1411 offset = in->f_pos; 1412 } 1413 1414 ret = do_splice_to(in, &offset, opipe, len, flags); 1415 1416 if (!off_in) 1417 in->f_pos = offset; 1418 else if (copy_to_user(off_in, &offset, sizeof(loff_t))) 1419 ret = -EFAULT; 1420 1421 return ret; 1422 } 1423 1424 return -EINVAL; 1425 } 1426 1427 /* 1428 * Map an iov into an array of pages and offset/length tupples. With the 1429 * partial_page structure, we can map several non-contiguous ranges into 1430 * our ones pages[] map instead of splitting that operation into pieces. 1431 * Could easily be exported as a generic helper for other users, in which 1432 * case one would probably want to add a 'max_nr_pages' parameter as well. 1433 */ 1434 static int get_iovec_page_array(const struct iovec __user *iov, 1435 unsigned int nr_vecs, struct page **pages, 1436 struct partial_page *partial, bool aligned, 1437 unsigned int pipe_buffers) 1438 { 1439 int buffers = 0, error = 0; 1440 1441 while (nr_vecs) { 1442 unsigned long off, npages; 1443 struct iovec entry; 1444 void __user *base; 1445 size_t len; 1446 int i; 1447 1448 error = -EFAULT; 1449 if (copy_from_user(&entry, iov, sizeof(entry))) 1450 break; 1451 1452 base = entry.iov_base; 1453 len = entry.iov_len; 1454 1455 /* 1456 * Sanity check this iovec. 0 read succeeds. 1457 */ 1458 error = 0; 1459 if (unlikely(!len)) 1460 break; 1461 error = -EFAULT; 1462 if (!access_ok(VERIFY_READ, base, len)) 1463 break; 1464 1465 /* 1466 * Get this base offset and number of pages, then map 1467 * in the user pages. 1468 */ 1469 off = (unsigned long) base & ~PAGE_MASK; 1470 1471 /* 1472 * If asked for alignment, the offset must be zero and the 1473 * length a multiple of the PAGE_SIZE. 1474 */ 1475 error = -EINVAL; 1476 if (aligned && (off || len & ~PAGE_MASK)) 1477 break; 1478 1479 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1480 if (npages > pipe_buffers - buffers) 1481 npages = pipe_buffers - buffers; 1482 1483 error = get_user_pages_fast((unsigned long)base, npages, 1484 0, &pages[buffers]); 1485 1486 if (unlikely(error <= 0)) 1487 break; 1488 1489 /* 1490 * Fill this contiguous range into the partial page map. 1491 */ 1492 for (i = 0; i < error; i++) { 1493 const int plen = min_t(size_t, len, PAGE_SIZE - off); 1494 1495 partial[buffers].offset = off; 1496 partial[buffers].len = plen; 1497 1498 off = 0; 1499 len -= plen; 1500 buffers++; 1501 } 1502 1503 /* 1504 * We didn't complete this iov, stop here since it probably 1505 * means we have to move some of this into a pipe to 1506 * be able to continue. 1507 */ 1508 if (len) 1509 break; 1510 1511 /* 1512 * Don't continue if we mapped fewer pages than we asked for, 1513 * or if we mapped the max number of pages that we have 1514 * room for. 1515 */ 1516 if (error < npages || buffers == pipe_buffers) 1517 break; 1518 1519 nr_vecs--; 1520 iov++; 1521 } 1522 1523 if (buffers) 1524 return buffers; 1525 1526 return error; 1527 } 1528 1529 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, 1530 struct splice_desc *sd) 1531 { 1532 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data); 1533 return n == sd->len ? n : -EFAULT; 1534 } 1535 1536 /* 1537 * For lack of a better implementation, implement vmsplice() to userspace 1538 * as a simple copy of the pipes pages to the user iov. 1539 */ 1540 static long vmsplice_to_user(struct file *file, const struct iovec __user *uiov, 1541 unsigned long nr_segs, unsigned int flags) 1542 { 1543 struct pipe_inode_info *pipe; 1544 struct splice_desc sd; 1545 long ret; 1546 struct iovec iovstack[UIO_FASTIOV]; 1547 struct iovec *iov = iovstack; 1548 struct iov_iter iter; 1549 1550 pipe = get_pipe_info(file); 1551 if (!pipe) 1552 return -EBADF; 1553 1554 ret = import_iovec(READ, uiov, nr_segs, 1555 ARRAY_SIZE(iovstack), &iov, &iter); 1556 if (ret < 0) 1557 return ret; 1558 1559 sd.total_len = iov_iter_count(&iter); 1560 sd.len = 0; 1561 sd.flags = flags; 1562 sd.u.data = &iter; 1563 sd.pos = 0; 1564 1565 if (sd.total_len) { 1566 pipe_lock(pipe); 1567 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); 1568 pipe_unlock(pipe); 1569 } 1570 1571 kfree(iov); 1572 return ret; 1573 } 1574 1575 /* 1576 * vmsplice splices a user address range into a pipe. It can be thought of 1577 * as splice-from-memory, where the regular splice is splice-from-file (or 1578 * to file). In both cases the output is a pipe, naturally. 1579 */ 1580 static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov, 1581 unsigned long nr_segs, unsigned int flags) 1582 { 1583 struct pipe_inode_info *pipe; 1584 struct page *pages[PIPE_DEF_BUFFERS]; 1585 struct partial_page partial[PIPE_DEF_BUFFERS]; 1586 struct splice_pipe_desc spd = { 1587 .pages = pages, 1588 .partial = partial, 1589 .nr_pages_max = PIPE_DEF_BUFFERS, 1590 .flags = flags, 1591 .ops = &user_page_pipe_buf_ops, 1592 .spd_release = spd_release_page, 1593 }; 1594 long ret; 1595 1596 pipe = get_pipe_info(file); 1597 if (!pipe) 1598 return -EBADF; 1599 1600 if (splice_grow_spd(pipe, &spd)) 1601 return -ENOMEM; 1602 1603 spd.nr_pages = get_iovec_page_array(iov, nr_segs, spd.pages, 1604 spd.partial, false, 1605 spd.nr_pages_max); 1606 if (spd.nr_pages <= 0) 1607 ret = spd.nr_pages; 1608 else 1609 ret = splice_to_pipe(pipe, &spd); 1610 1611 splice_shrink_spd(&spd); 1612 return ret; 1613 } 1614 1615 /* 1616 * Note that vmsplice only really supports true splicing _from_ user memory 1617 * to a pipe, not the other way around. Splicing from user memory is a simple 1618 * operation that can be supported without any funky alignment restrictions 1619 * or nasty vm tricks. We simply map in the user memory and fill them into 1620 * a pipe. The reverse isn't quite as easy, though. There are two possible 1621 * solutions for that: 1622 * 1623 * - memcpy() the data internally, at which point we might as well just 1624 * do a regular read() on the buffer anyway. 1625 * - Lots of nasty vm tricks, that are neither fast nor flexible (it 1626 * has restriction limitations on both ends of the pipe). 1627 * 1628 * Currently we punt and implement it as a normal copy, see pipe_to_user(). 1629 * 1630 */ 1631 SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov, 1632 unsigned long, nr_segs, unsigned int, flags) 1633 { 1634 struct fd f; 1635 long error; 1636 1637 if (unlikely(nr_segs > UIO_MAXIOV)) 1638 return -EINVAL; 1639 else if (unlikely(!nr_segs)) 1640 return 0; 1641 1642 error = -EBADF; 1643 f = fdget(fd); 1644 if (f.file) { 1645 if (f.file->f_mode & FMODE_WRITE) 1646 error = vmsplice_to_pipe(f.file, iov, nr_segs, flags); 1647 else if (f.file->f_mode & FMODE_READ) 1648 error = vmsplice_to_user(f.file, iov, nr_segs, flags); 1649 1650 fdput(f); 1651 } 1652 1653 return error; 1654 } 1655 1656 #ifdef CONFIG_COMPAT 1657 COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32, 1658 unsigned int, nr_segs, unsigned int, flags) 1659 { 1660 unsigned i; 1661 struct iovec __user *iov; 1662 if (nr_segs > UIO_MAXIOV) 1663 return -EINVAL; 1664 iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec)); 1665 for (i = 0; i < nr_segs; i++) { 1666 struct compat_iovec v; 1667 if (get_user(v.iov_base, &iov32[i].iov_base) || 1668 get_user(v.iov_len, &iov32[i].iov_len) || 1669 put_user(compat_ptr(v.iov_base), &iov[i].iov_base) || 1670 put_user(v.iov_len, &iov[i].iov_len)) 1671 return -EFAULT; 1672 } 1673 return sys_vmsplice(fd, iov, nr_segs, flags); 1674 } 1675 #endif 1676 1677 SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in, 1678 int, fd_out, loff_t __user *, off_out, 1679 size_t, len, unsigned int, flags) 1680 { 1681 struct fd in, out; 1682 long error; 1683 1684 if (unlikely(!len)) 1685 return 0; 1686 1687 error = -EBADF; 1688 in = fdget(fd_in); 1689 if (in.file) { 1690 if (in.file->f_mode & FMODE_READ) { 1691 out = fdget(fd_out); 1692 if (out.file) { 1693 if (out.file->f_mode & FMODE_WRITE) 1694 error = do_splice(in.file, off_in, 1695 out.file, off_out, 1696 len, flags); 1697 fdput(out); 1698 } 1699 } 1700 fdput(in); 1701 } 1702 return error; 1703 } 1704 1705 /* 1706 * Make sure there's data to read. Wait for input if we can, otherwise 1707 * return an appropriate error. 1708 */ 1709 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1710 { 1711 int ret; 1712 1713 /* 1714 * Check ->nrbufs without the inode lock first. This function 1715 * is speculative anyways, so missing one is ok. 1716 */ 1717 if (pipe->nrbufs) 1718 return 0; 1719 1720 ret = 0; 1721 pipe_lock(pipe); 1722 1723 while (!pipe->nrbufs) { 1724 if (signal_pending(current)) { 1725 ret = -ERESTARTSYS; 1726 break; 1727 } 1728 if (!pipe->writers) 1729 break; 1730 if (!pipe->waiting_writers) { 1731 if (flags & SPLICE_F_NONBLOCK) { 1732 ret = -EAGAIN; 1733 break; 1734 } 1735 } 1736 pipe_wait(pipe); 1737 } 1738 1739 pipe_unlock(pipe); 1740 return ret; 1741 } 1742 1743 /* 1744 * Make sure there's writeable room. Wait for room if we can, otherwise 1745 * return an appropriate error. 1746 */ 1747 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) 1748 { 1749 int ret; 1750 1751 /* 1752 * Check ->nrbufs without the inode lock first. This function 1753 * is speculative anyways, so missing one is ok. 1754 */ 1755 if (pipe->nrbufs < pipe->buffers) 1756 return 0; 1757 1758 ret = 0; 1759 pipe_lock(pipe); 1760 1761 while (pipe->nrbufs >= pipe->buffers) { 1762 if (!pipe->readers) { 1763 send_sig(SIGPIPE, current, 0); 1764 ret = -EPIPE; 1765 break; 1766 } 1767 if (flags & SPLICE_F_NONBLOCK) { 1768 ret = -EAGAIN; 1769 break; 1770 } 1771 if (signal_pending(current)) { 1772 ret = -ERESTARTSYS; 1773 break; 1774 } 1775 pipe->waiting_writers++; 1776 pipe_wait(pipe); 1777 pipe->waiting_writers--; 1778 } 1779 1780 pipe_unlock(pipe); 1781 return ret; 1782 } 1783 1784 /* 1785 * Splice contents of ipipe to opipe. 1786 */ 1787 static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe, 1788 struct pipe_inode_info *opipe, 1789 size_t len, unsigned int flags) 1790 { 1791 struct pipe_buffer *ibuf, *obuf; 1792 int ret = 0, nbuf; 1793 bool input_wakeup = false; 1794 1795 1796 retry: 1797 ret = ipipe_prep(ipipe, flags); 1798 if (ret) 1799 return ret; 1800 1801 ret = opipe_prep(opipe, flags); 1802 if (ret) 1803 return ret; 1804 1805 /* 1806 * Potential ABBA deadlock, work around it by ordering lock 1807 * grabbing by pipe info address. Otherwise two different processes 1808 * could deadlock (one doing tee from A -> B, the other from B -> A). 1809 */ 1810 pipe_double_lock(ipipe, opipe); 1811 1812 do { 1813 if (!opipe->readers) { 1814 send_sig(SIGPIPE, current, 0); 1815 if (!ret) 1816 ret = -EPIPE; 1817 break; 1818 } 1819 1820 if (!ipipe->nrbufs && !ipipe->writers) 1821 break; 1822 1823 /* 1824 * Cannot make any progress, because either the input 1825 * pipe is empty or the output pipe is full. 1826 */ 1827 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) { 1828 /* Already processed some buffers, break */ 1829 if (ret) 1830 break; 1831 1832 if (flags & SPLICE_F_NONBLOCK) { 1833 ret = -EAGAIN; 1834 break; 1835 } 1836 1837 /* 1838 * We raced with another reader/writer and haven't 1839 * managed to process any buffers. A zero return 1840 * value means EOF, so retry instead. 1841 */ 1842 pipe_unlock(ipipe); 1843 pipe_unlock(opipe); 1844 goto retry; 1845 } 1846 1847 ibuf = ipipe->bufs + ipipe->curbuf; 1848 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1849 obuf = opipe->bufs + nbuf; 1850 1851 if (len >= ibuf->len) { 1852 /* 1853 * Simply move the whole buffer from ipipe to opipe 1854 */ 1855 *obuf = *ibuf; 1856 ibuf->ops = NULL; 1857 opipe->nrbufs++; 1858 ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1); 1859 ipipe->nrbufs--; 1860 input_wakeup = true; 1861 } else { 1862 /* 1863 * Get a reference to this pipe buffer, 1864 * so we can copy the contents over. 1865 */ 1866 ibuf->ops->get(ipipe, ibuf); 1867 *obuf = *ibuf; 1868 1869 /* 1870 * Don't inherit the gift flag, we need to 1871 * prevent multiple steals of this page. 1872 */ 1873 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1874 1875 obuf->len = len; 1876 opipe->nrbufs++; 1877 ibuf->offset += obuf->len; 1878 ibuf->len -= obuf->len; 1879 } 1880 ret += obuf->len; 1881 len -= obuf->len; 1882 } while (len); 1883 1884 pipe_unlock(ipipe); 1885 pipe_unlock(opipe); 1886 1887 /* 1888 * If we put data in the output pipe, wakeup any potential readers. 1889 */ 1890 if (ret > 0) 1891 wakeup_pipe_readers(opipe); 1892 1893 if (input_wakeup) 1894 wakeup_pipe_writers(ipipe); 1895 1896 return ret; 1897 } 1898 1899 /* 1900 * Link contents of ipipe to opipe. 1901 */ 1902 static int link_pipe(struct pipe_inode_info *ipipe, 1903 struct pipe_inode_info *opipe, 1904 size_t len, unsigned int flags) 1905 { 1906 struct pipe_buffer *ibuf, *obuf; 1907 int ret = 0, i = 0, nbuf; 1908 1909 /* 1910 * Potential ABBA deadlock, work around it by ordering lock 1911 * grabbing by pipe info address. Otherwise two different processes 1912 * could deadlock (one doing tee from A -> B, the other from B -> A). 1913 */ 1914 pipe_double_lock(ipipe, opipe); 1915 1916 do { 1917 if (!opipe->readers) { 1918 send_sig(SIGPIPE, current, 0); 1919 if (!ret) 1920 ret = -EPIPE; 1921 break; 1922 } 1923 1924 /* 1925 * If we have iterated all input buffers or ran out of 1926 * output room, break. 1927 */ 1928 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) 1929 break; 1930 1931 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1)); 1932 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1); 1933 1934 /* 1935 * Get a reference to this pipe buffer, 1936 * so we can copy the contents over. 1937 */ 1938 ibuf->ops->get(ipipe, ibuf); 1939 1940 obuf = opipe->bufs + nbuf; 1941 *obuf = *ibuf; 1942 1943 /* 1944 * Don't inherit the gift flag, we need to 1945 * prevent multiple steals of this page. 1946 */ 1947 obuf->flags &= ~PIPE_BUF_FLAG_GIFT; 1948 1949 if (obuf->len > len) 1950 obuf->len = len; 1951 1952 opipe->nrbufs++; 1953 ret += obuf->len; 1954 len -= obuf->len; 1955 i++; 1956 } while (len); 1957 1958 /* 1959 * return EAGAIN if we have the potential of some data in the 1960 * future, otherwise just return 0 1961 */ 1962 if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK)) 1963 ret = -EAGAIN; 1964 1965 pipe_unlock(ipipe); 1966 pipe_unlock(opipe); 1967 1968 /* 1969 * If we put data in the output pipe, wakeup any potential readers. 1970 */ 1971 if (ret > 0) 1972 wakeup_pipe_readers(opipe); 1973 1974 return ret; 1975 } 1976 1977 /* 1978 * This is a tee(1) implementation that works on pipes. It doesn't copy 1979 * any data, it simply references the 'in' pages on the 'out' pipe. 1980 * The 'flags' used are the SPLICE_F_* variants, currently the only 1981 * applicable one is SPLICE_F_NONBLOCK. 1982 */ 1983 static long do_tee(struct file *in, struct file *out, size_t len, 1984 unsigned int flags) 1985 { 1986 struct pipe_inode_info *ipipe = get_pipe_info(in); 1987 struct pipe_inode_info *opipe = get_pipe_info(out); 1988 int ret = -EINVAL; 1989 1990 /* 1991 * Duplicate the contents of ipipe to opipe without actually 1992 * copying the data. 1993 */ 1994 if (ipipe && opipe && ipipe != opipe) { 1995 /* 1996 * Keep going, unless we encounter an error. The ipipe/opipe 1997 * ordering doesn't really matter. 1998 */ 1999 ret = ipipe_prep(ipipe, flags); 2000 if (!ret) { 2001 ret = opipe_prep(opipe, flags); 2002 if (!ret) 2003 ret = link_pipe(ipipe, opipe, len, flags); 2004 } 2005 } 2006 2007 return ret; 2008 } 2009 2010 SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags) 2011 { 2012 struct fd in; 2013 int error; 2014 2015 if (unlikely(!len)) 2016 return 0; 2017 2018 error = -EBADF; 2019 in = fdget(fdin); 2020 if (in.file) { 2021 if (in.file->f_mode & FMODE_READ) { 2022 struct fd out = fdget(fdout); 2023 if (out.file) { 2024 if (out.file->f_mode & FMODE_WRITE) 2025 error = do_tee(in.file, out.file, 2026 len, flags); 2027 fdput(out); 2028 } 2029 } 2030 fdput(in); 2031 } 2032 2033 return error; 2034 } 2035