1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
25274f052SJens Axboe /*
35274f052SJens Axboe * "splice": joining two ropes together by interweaving their strands.
45274f052SJens Axboe *
55274f052SJens Axboe * This is the "extended pipe" functionality, where a pipe is used as
65274f052SJens Axboe * an arbitrary in-memory buffer. Think of a pipe as a small kernel
75274f052SJens Axboe * buffer that you can use to transfer data from one end to the other.
85274f052SJens Axboe *
95274f052SJens Axboe * The traditional unix read/write is extended with a "splice()" operation
105274f052SJens Axboe * that transfers data buffers to or from a pipe buffer.
115274f052SJens Axboe *
125274f052SJens Axboe * Named by Larry McVoy, original implementation from Linus, extended by
13c2058e06SJens Axboe * Jens to support splicing to files, network, direct splicing, etc and
14c2058e06SJens Axboe * fixing lots of bugs.
155274f052SJens Axboe *
160fe23479SJens Axboe * Copyright (C) 2005-2006 Jens Axboe <axboe@kernel.dk>
17c2058e06SJens Axboe * Copyright (C) 2005-2006 Linus Torvalds <torvalds@osdl.org>
18c2058e06SJens Axboe * Copyright (C) 2006 Ingo Molnar <mingo@elte.hu>
195274f052SJens Axboe *
205274f052SJens Axboe */
21be297968SChristoph Hellwig #include <linux/bvec.h>
225274f052SJens Axboe #include <linux/fs.h>
235274f052SJens Axboe #include <linux/file.h>
245274f052SJens Axboe #include <linux/pagemap.h>
25d6b29d7cSJens Axboe #include <linux/splice.h>
2608e552c6SKAMEZAWA Hiroyuki #include <linux/memcontrol.h>
275274f052SJens Axboe #include <linux/mm_inline.h>
285abc97aaSJens Axboe #include <linux/swap.h>
294f6f0bd2SJens Axboe #include <linux/writeback.h>
30630d9c47SPaul Gortmaker #include <linux/export.h>
314f6f0bd2SJens Axboe #include <linux/syscalls.h>
32912d35f8SJens Axboe #include <linux/uio.h>
33983652c6SChung-Chiang Cheng #include <linux/fsnotify.h>
3429ce2058SJames Morris #include <linux/security.h>
355a0e3ad6STejun Heo #include <linux/gfp.h>
362dc334f1SDavid Howells #include <linux/net.h>
3735f9c09fSEric Dumazet #include <linux/socket.h>
38174cd4b1SIngo Molnar #include <linux/sched/signal.h>
39174cd4b1SIngo Molnar
4006ae43f3SAl Viro #include "internal.h"
415274f052SJens Axboe
4283f9135bSJens Axboe /*
430f99fc51SJens Axboe * Splice doesn't support FMODE_NOWAIT. Since pipes may set this flag to
440f99fc51SJens Axboe * indicate they support non-blocking reads or writes, we must clear it
450f99fc51SJens Axboe * here if set to avoid blocking other users of this pipe if splice is
460f99fc51SJens Axboe * being done on it.
470f99fc51SJens Axboe */
pipe_clear_nowait(struct file * file)480f99fc51SJens Axboe static noinline void noinline pipe_clear_nowait(struct file *file)
490f99fc51SJens Axboe {
500f99fc51SJens Axboe fmode_t fmode = READ_ONCE(file->f_mode);
510f99fc51SJens Axboe
520f99fc51SJens Axboe do {
530f99fc51SJens Axboe if (!(fmode & FMODE_NOWAIT))
540f99fc51SJens Axboe break;
550f99fc51SJens Axboe } while (!try_cmpxchg(&file->f_mode, &fmode, fmode & ~FMODE_NOWAIT));
560f99fc51SJens Axboe }
570f99fc51SJens Axboe
580f99fc51SJens Axboe /*
5983f9135bSJens Axboe * Attempt to steal a page from a pipe buffer. This should perhaps go into
6083f9135bSJens Axboe * a vm helper function, it's already simplified quite a bit by the
6183f9135bSJens Axboe * addition of remove_mapping(). If success is returned, the caller may
6283f9135bSJens Axboe * attempt to reuse this page for another destination.
6383f9135bSJens Axboe */
page_cache_pipe_buf_try_steal(struct pipe_inode_info * pipe,struct pipe_buffer * buf)64c928f642SChristoph Hellwig static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
655abc97aaSJens Axboe struct pipe_buffer *buf)
665abc97aaSJens Axboe {
675100da38SMatthew Wilcox (Oracle) struct folio *folio = page_folio(buf->page);
689e94cd4fSJens Axboe struct address_space *mapping;
695abc97aaSJens Axboe
70b9ccad2eSMatthew Wilcox (Oracle) folio_lock(folio);
719e0267c2SJens Axboe
72b9ccad2eSMatthew Wilcox (Oracle) mapping = folio_mapping(folio);
739e94cd4fSJens Axboe if (mapping) {
74b9ccad2eSMatthew Wilcox (Oracle) WARN_ON(!folio_test_uptodate(folio));
755abc97aaSJens Axboe
76ad8d6f0aSJens Axboe /*
779e94cd4fSJens Axboe * At least for ext2 with nobh option, we need to wait on
78b9ccad2eSMatthew Wilcox (Oracle) * writeback completing on this folio, since we'll remove it
799e94cd4fSJens Axboe * from the pagecache. Otherwise truncate wont wait on the
80b9ccad2eSMatthew Wilcox (Oracle) * folio, allowing the disk blocks to be reused by someone else
819e94cd4fSJens Axboe * before we actually wrote our data to them. fs corruption
829e94cd4fSJens Axboe * ensues.
83ad8d6f0aSJens Axboe */
84b9ccad2eSMatthew Wilcox (Oracle) folio_wait_writeback(folio);
85ad8d6f0aSJens Axboe
86b9ccad2eSMatthew Wilcox (Oracle) if (!filemap_release_folio(folio, GFP_KERNEL))
87b9ccad2eSMatthew Wilcox (Oracle) goto out_unlock;
88ca39d651SJens Axboe
894f6f0bd2SJens Axboe /*
909e94cd4fSJens Axboe * If we succeeded in removing the mapping, set LRU flag
919e94cd4fSJens Axboe * and return good.
929e94cd4fSJens Axboe */
939e94cd4fSJens Axboe if (remove_mapping(mapping, folio)) {
945100da38SMatthew Wilcox (Oracle) buf->flags |= PIPE_BUF_FLAG_LRU;
951432873aSJens Axboe return true;
96c928f642SChristoph Hellwig }
975abc97aaSJens Axboe }
989e94cd4fSJens Axboe
999e94cd4fSJens Axboe /*
1009e94cd4fSJens Axboe * Raced with truncate or failed to remove folio from current
101b9ccad2eSMatthew Wilcox (Oracle) * address space, unlock and return failure.
1029e94cd4fSJens Axboe */
1039e94cd4fSJens Axboe out_unlock:
104ca39d651SJens Axboe folio_unlock(folio);
105b9ccad2eSMatthew Wilcox (Oracle) return false;
106c928f642SChristoph Hellwig }
1079e94cd4fSJens Axboe
page_cache_pipe_buf_release(struct pipe_inode_info * pipe,struct pipe_buffer * buf)1085abc97aaSJens Axboe static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
10976ad4d11SJens Axboe struct pipe_buffer *buf)
1105274f052SJens Axboe {
1115274f052SJens Axboe put_page(buf->page);
11209cbfeafSKirill A. Shutemov buf->flags &= ~PIPE_BUF_FLAG_LRU;
1131432873aSJens Axboe }
1145274f052SJens Axboe
1155274f052SJens Axboe /*
1160845718dSJens Axboe * Check whether the contents of buf is OK to access. Since the content
1170845718dSJens Axboe * is a page cache page, IO may be in flight.
1180845718dSJens Axboe */
page_cache_pipe_buf_confirm(struct pipe_inode_info * pipe,struct pipe_buffer * buf)1190845718dSJens Axboe static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
120cac36bb0SJens Axboe struct pipe_buffer *buf)
1215274f052SJens Axboe {
1225274f052SJens Axboe struct folio *folio = page_folio(buf->page);
123*781ca602SMatthew Wilcox (Oracle) int err;
12449d0b21bSJens Axboe
1255274f052SJens Axboe if (!folio_test_uptodate(folio)) {
126*781ca602SMatthew Wilcox (Oracle) folio_lock(folio);
127*781ca602SMatthew Wilcox (Oracle)
1285274f052SJens Axboe /*
12949d0b21bSJens Axboe * Folio got truncated/unhashed. This will cause a 0-byte
130*781ca602SMatthew Wilcox (Oracle) * splice, if this is the first page.
13173d62d83SIngo Molnar */
13249d0b21bSJens Axboe if (!folio->mapping) {
133*781ca602SMatthew Wilcox (Oracle) err = -ENODATA;
13449d0b21bSJens Axboe goto error;
13549d0b21bSJens Axboe }
1365274f052SJens Axboe
1375274f052SJens Axboe /*
13849d0b21bSJens Axboe * Uh oh, read-error from disk.
13973d62d83SIngo Molnar */
14049d0b21bSJens Axboe if (!folio_test_uptodate(folio)) {
141*781ca602SMatthew Wilcox (Oracle) err = -EIO;
14249d0b21bSJens Axboe goto error;
14349d0b21bSJens Axboe }
14449d0b21bSJens Axboe
14549d0b21bSJens Axboe /* Folio is ok after all, we are done */
146*781ca602SMatthew Wilcox (Oracle) folio_unlock(folio);
147*781ca602SMatthew Wilcox (Oracle) }
14849d0b21bSJens Axboe
14949d0b21bSJens Axboe return 0;
150f84d7519SJens Axboe error:
15149d0b21bSJens Axboe folio_unlock(folio);
152*781ca602SMatthew Wilcox (Oracle) return err;
153f84d7519SJens Axboe }
15470524490SJens Axboe
15570524490SJens Axboe const struct pipe_buf_operations page_cache_pipe_buf_ops = {
156708e3508SHugh Dickins .confirm = page_cache_pipe_buf_confirm,
157cac36bb0SJens Axboe .release = page_cache_pipe_buf_release,
1585274f052SJens Axboe .try_steal = page_cache_pipe_buf_try_steal,
159c928f642SChristoph Hellwig .get = generic_pipe_buf_get,
160f84d7519SJens Axboe };
1615274f052SJens Axboe
user_page_pipe_buf_try_steal(struct pipe_inode_info * pipe,struct pipe_buffer * buf)1625274f052SJens Axboe static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
163c928f642SChristoph Hellwig struct pipe_buffer *buf)
164912d35f8SJens Axboe {
165912d35f8SJens Axboe if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
1667afa6fd0SJens Axboe return false;
167c928f642SChristoph Hellwig
1687afa6fd0SJens Axboe buf->flags |= PIPE_BUF_FLAG_LRU;
1691432873aSJens Axboe return generic_pipe_buf_try_steal(pipe, buf);
170c928f642SChristoph Hellwig }
171912d35f8SJens Axboe
172912d35f8SJens Axboe static const struct pipe_buf_operations user_page_pipe_buf_ops = {
173d4c3cca9SEric Dumazet .release = page_cache_pipe_buf_release,
174912d35f8SJens Axboe .try_steal = user_page_pipe_buf_try_steal,
175c928f642SChristoph Hellwig .get = generic_pipe_buf_get,
176f84d7519SJens Axboe };
177912d35f8SJens Axboe
wakeup_pipe_readers(struct pipe_inode_info * pipe)178912d35f8SJens Axboe static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
179825cdcb1SNamhyung Kim {
180825cdcb1SNamhyung Kim smp_mb();
181825cdcb1SNamhyung Kim if (waitqueue_active(&pipe->rd_wait))
1820ddad21dSLinus Torvalds wake_up_interruptible(&pipe->rd_wait);
1830ddad21dSLinus Torvalds kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
184825cdcb1SNamhyung Kim }
185825cdcb1SNamhyung Kim
186825cdcb1SNamhyung Kim /**
187932cc6d4SJens Axboe * splice_to_pipe - fill passed data into a pipe
188932cc6d4SJens Axboe * @pipe: pipe to fill
189932cc6d4SJens Axboe * @spd: data to fill
190932cc6d4SJens Axboe *
191932cc6d4SJens Axboe * Description:
192932cc6d4SJens Axboe * @spd contains a map of pages and len/offset tuples, along with
19379685b8dSRandy Dunlap * the struct pipe_buf_operations associated with these pages. This
194932cc6d4SJens Axboe * function will link that data to the pipe.
195932cc6d4SJens Axboe *
196932cc6d4SJens Axboe */
splice_to_pipe(struct pipe_inode_info * pipe,struct splice_pipe_desc * spd)19783f9135bSJens Axboe ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
198d6b29d7cSJens Axboe struct splice_pipe_desc *spd)
199912d35f8SJens Axboe {
2005274f052SJens Axboe unsigned int spd_pages = spd->nr_pages;
20100de00bdSJens Axboe unsigned int tail = pipe->tail;
2028cefc107SDavid Howells unsigned int head = pipe->head;
2038cefc107SDavid Howells unsigned int mask = pipe->ring_size - 1;
2048cefc107SDavid Howells int ret = 0, page_nr = 0;
2058924feffSAl Viro
2065274f052SJens Axboe if (!spd_pages)
207d6785d91SRabin Vincent return 0;
208d6785d91SRabin Vincent
209d6785d91SRabin Vincent if (unlikely(!pipe->readers)) {
2108924feffSAl Viro send_sig(SIGPIPE, current, 0);
2115274f052SJens Axboe ret = -EPIPE;
2125274f052SJens Axboe goto out;
2138924feffSAl Viro }
2145274f052SJens Axboe
2155274f052SJens Axboe while (!pipe_full(head, tail, pipe->max_usage)) {
2166718b6f8SDavid Howells struct pipe_buffer *buf = &pipe->bufs[head & mask];
2178cefc107SDavid Howells
2185274f052SJens Axboe buf->page = spd->pages[page_nr];
219912d35f8SJens Axboe buf->offset = spd->partial[page_nr].offset;
220912d35f8SJens Axboe buf->len = spd->partial[page_nr].len;
221912d35f8SJens Axboe buf->private = spd->partial[page_nr].private;
222497f9625SJens Axboe buf->ops = spd->ops;
223912d35f8SJens Axboe buf->flags = 0;
2245a81e6a1SMiklos Szeredi
2257afa6fd0SJens Axboe head++;
2268cefc107SDavid Howells pipe->head = head;
2278cefc107SDavid Howells page_nr++;
228912d35f8SJens Axboe ret += buf->len;
229912d35f8SJens Axboe
230912d35f8SJens Axboe if (!--spd->nr_pages)
231912d35f8SJens Axboe break;
2325274f052SJens Axboe }
2335274f052SJens Axboe
2345274f052SJens Axboe if (!ret)
23529e35094SLinus Torvalds ret = -EAGAIN;
23629e35094SLinus Torvalds
23729e35094SLinus Torvalds out:
2388924feffSAl Viro while (page_nr < spd_pages)
23900de00bdSJens Axboe spd->spd_release(spd, page_nr++);
240bbdfc2f7SJens Axboe
2415274f052SJens Axboe return ret;
2425274f052SJens Axboe }
2435274f052SJens Axboe EXPORT_SYMBOL_GPL(splice_to_pipe);
2442b514574SHannes Frederic Sowa
add_to_pipe(struct pipe_inode_info * pipe,struct pipe_buffer * buf)2455274f052SJens Axboe ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
24679fddc4eSAl Viro {
24779fddc4eSAl Viro unsigned int head = pipe->head;
2488cefc107SDavid Howells unsigned int tail = pipe->tail;
2498cefc107SDavid Howells unsigned int mask = pipe->ring_size - 1;
2508cefc107SDavid Howells int ret;
25179fddc4eSAl Viro
25279fddc4eSAl Viro if (unlikely(!pipe->readers)) {
25379fddc4eSAl Viro send_sig(SIGPIPE, current, 0);
25479fddc4eSAl Viro ret = -EPIPE;
25579fddc4eSAl Viro } else if (pipe_full(head, tail, pipe->max_usage)) {
2566718b6f8SDavid Howells ret = -EAGAIN;
25779fddc4eSAl Viro } else {
25879fddc4eSAl Viro pipe->bufs[head & mask] = *buf;
2598cefc107SDavid Howells pipe->head = head + 1;
2608cefc107SDavid Howells return buf->len;
26179fddc4eSAl Viro }
26279fddc4eSAl Viro pipe_buf_release(pipe, buf);
263a779638cSMiklos Szeredi return ret;
26479fddc4eSAl Viro }
26579fddc4eSAl Viro EXPORT_SYMBOL(add_to_pipe);
26679fddc4eSAl Viro
26779fddc4eSAl Viro /*
26835f3d14dSJens Axboe * Check if we need to grow the arrays holding pages and partial page
26935f3d14dSJens Axboe * descriptions.
27035f3d14dSJens Axboe */
splice_grow_spd(const struct pipe_inode_info * pipe,struct splice_pipe_desc * spd)27135f3d14dSJens Axboe int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
272047fe360SEric Dumazet {
27335f3d14dSJens Axboe unsigned int max_usage = READ_ONCE(pipe->max_usage);
2746718b6f8SDavid Howells
275047fe360SEric Dumazet spd->nr_pages_max = max_usage;
2768cefc107SDavid Howells if (max_usage <= PIPE_DEF_BUFFERS)
2778cefc107SDavid Howells return 0;
27835f3d14dSJens Axboe
27935f3d14dSJens Axboe spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
2808cefc107SDavid Howells spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
2818cefc107SDavid Howells GFP_KERNEL);
2826da2ec56SKees Cook
28335f3d14dSJens Axboe if (spd->pages && spd->partial)
28435f3d14dSJens Axboe return 0;
28535f3d14dSJens Axboe
28635f3d14dSJens Axboe kfree(spd->pages);
28735f3d14dSJens Axboe kfree(spd->partial);
28835f3d14dSJens Axboe return -ENOMEM;
28935f3d14dSJens Axboe }
29035f3d14dSJens Axboe
splice_shrink_spd(struct splice_pipe_desc * spd)29135f3d14dSJens Axboe void splice_shrink_spd(struct splice_pipe_desc *spd)
292047fe360SEric Dumazet {
29335f3d14dSJens Axboe if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
294047fe360SEric Dumazet return;
29535f3d14dSJens Axboe
29635f3d14dSJens Axboe kfree(spd->pages);
29735f3d14dSJens Axboe kfree(spd->partial);
29835f3d14dSJens Axboe }
29935f3d14dSJens Axboe
30035f3d14dSJens Axboe /**
3019eee8bd8SDavid Howells * copy_splice_read - Copy data from a file and splice the copy into a pipe
3029eee8bd8SDavid Howells * @in: The file to read from
3039eee8bd8SDavid Howells * @ppos: Pointer to the file position to read from
3049eee8bd8SDavid Howells * @pipe: The pipe to splice into
3059eee8bd8SDavid Howells * @len: The amount to splice
3069eee8bd8SDavid Howells * @flags: The SPLICE_F_* flags
3079eee8bd8SDavid Howells *
3089eee8bd8SDavid Howells * This function allocates a bunch of pages sufficient to hold the requested
3099eee8bd8SDavid Howells * amount of data (but limited by the remaining pipe capacity), passes it to
3109eee8bd8SDavid Howells * the file's ->read_iter() to read into and then splices the used pages into
3119eee8bd8SDavid Howells * the pipe.
3129eee8bd8SDavid Howells *
3139eee8bd8SDavid Howells * Return: On success, the number of bytes read will be returned and *@ppos
3149eee8bd8SDavid Howells * will be updated if appropriate; 0 will be returned if there is no more data
3159eee8bd8SDavid Howells * to be read; -EAGAIN will be returned if the pipe had no space, and some
3169eee8bd8SDavid Howells * other negative error code will be returned on error. A short read may occur
3179eee8bd8SDavid Howells * if the pipe has insufficient space, we reach the end of the data or we hit a
3189eee8bd8SDavid Howells * hole.
3199eee8bd8SDavid Howells */
copy_splice_read(struct file * in,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)32033b3b041SDavid Howells ssize_t copy_splice_read(struct file *in, loff_t *ppos,
32169df79a4SDavid Howells struct pipe_inode_info *pipe,
32233b3b041SDavid Howells size_t len, unsigned int flags)
32333b3b041SDavid Howells {
32433b3b041SDavid Howells struct iov_iter to;
32533b3b041SDavid Howells struct bio_vec *bv;
32633b3b041SDavid Howells struct kiocb kiocb;
32733b3b041SDavid Howells struct page **pages;
32833b3b041SDavid Howells ssize_t ret;
32933b3b041SDavid Howells size_t used, npages, chunk, remain, keep = 0;
330e69f37bcSDavid Howells int i;
33133b3b041SDavid Howells
33233b3b041SDavid Howells /* Work out how much data we can actually add into the pipe */
33333b3b041SDavid Howells used = pipe_occupancy(pipe->head, pipe->tail);
33433b3b041SDavid Howells npages = max_t(ssize_t, pipe->max_usage - used, 0);
33533b3b041SDavid Howells len = min_t(size_t, len, npages * PAGE_SIZE);
33633b3b041SDavid Howells npages = DIV_ROUND_UP(len, PAGE_SIZE);
33733b3b041SDavid Howells
33833b3b041SDavid Howells bv = kzalloc(array_size(npages, sizeof(bv[0])) +
33933b3b041SDavid Howells array_size(npages, sizeof(struct page *)), GFP_KERNEL);
34033b3b041SDavid Howells if (!bv)
34133b3b041SDavid Howells return -ENOMEM;
34233b3b041SDavid Howells
34333b3b041SDavid Howells pages = (struct page **)(bv + npages);
344e69f37bcSDavid Howells npages = alloc_pages_bulk_array(GFP_USER, npages, pages);
34533b3b041SDavid Howells if (!npages) {
34633b3b041SDavid Howells kfree(bv);
34733b3b041SDavid Howells return -ENOMEM;
34833b3b041SDavid Howells }
34933b3b041SDavid Howells
35033b3b041SDavid Howells remain = len = min_t(size_t, len, npages * PAGE_SIZE);
35133b3b041SDavid Howells
35233b3b041SDavid Howells for (i = 0; i < npages; i++) {
35333b3b041SDavid Howells chunk = min_t(size_t, PAGE_SIZE, remain);
35433b3b041SDavid Howells bv[i].bv_page = pages[i];
35533b3b041SDavid Howells bv[i].bv_offset = 0;
35633b3b041SDavid Howells bv[i].bv_len = chunk;
35733b3b041SDavid Howells remain -= chunk;
35833b3b041SDavid Howells }
35933b3b041SDavid Howells
36033b3b041SDavid Howells /* Do the I/O */
36133b3b041SDavid Howells iov_iter_bvec(&to, ITER_DEST, bv, npages, len);
36233b3b041SDavid Howells init_sync_kiocb(&kiocb, in);
36333b3b041SDavid Howells kiocb.ki_pos = *ppos;
36433b3b041SDavid Howells ret = call_read_iter(in, &kiocb, &to);
36533b3b041SDavid Howells
36633b3b041SDavid Howells if (ret > 0) {
36733b3b041SDavid Howells keep = DIV_ROUND_UP(ret, PAGE_SIZE);
368e69f37bcSDavid Howells *ppos = kiocb.ki_pos;
36933b3b041SDavid Howells }
3702e82f6c3SChristoph Hellwig
3712e82f6c3SChristoph Hellwig /*
37233b3b041SDavid Howells * Callers of ->splice_read() expect -EAGAIN on "can't put anything in
3732e82f6c3SChristoph Hellwig * there", rather than -EFAULT.
3742e82f6c3SChristoph Hellwig */
37533b3b041SDavid Howells if (ret == -EFAULT)
37633b3b041SDavid Howells ret = -EAGAIN;
37733b3b041SDavid Howells
37833b3b041SDavid Howells /* Free any pages that didn't get touched at all. */
37933b3b041SDavid Howells if (keep < npages)
380e69f37bcSDavid Howells release_pages(pages + keep, npages - keep);
381e69f37bcSDavid Howells
38233b3b041SDavid Howells /* Push the remaining pages into the pipe. */
38333b3b041SDavid Howells remain = ret;
384e69f37bcSDavid Howells for (i = 0; i < keep; i++) {
385e69f37bcSDavid Howells struct pipe_buffer *buf = pipe_head_buf(pipe);
38633b3b041SDavid Howells
38733b3b041SDavid Howells chunk = min_t(size_t, remain, PAGE_SIZE);
38833b3b041SDavid Howells *buf = (struct pipe_buffer) {
38933b3b041SDavid Howells .ops = &default_pipe_buf_ops,
39033b3b041SDavid Howells .page = bv[i].bv_page,
39133b3b041SDavid Howells .offset = 0,
39233b3b041SDavid Howells .len = chunk,
39333b3b041SDavid Howells };
39433b3b041SDavid Howells pipe->head++;
39533b3b041SDavid Howells remain -= chunk;
39633b3b041SDavid Howells }
39733b3b041SDavid Howells
39833b3b041SDavid Howells kfree(bv);
39933b3b041SDavid Howells return ret;
40033b3b041SDavid Howells }
40133b3b041SDavid Howells EXPORT_SYMBOL(copy_splice_read);
40269df79a4SDavid Howells
40333b3b041SDavid Howells const struct pipe_buf_operations default_pipe_buf_ops = {
404241699cdSAl Viro .release = generic_pipe_buf_release,
4056818173bSMiklos Szeredi .try_steal = generic_pipe_buf_try_steal,
406c928f642SChristoph Hellwig .get = generic_pipe_buf_get,
4076818173bSMiklos Szeredi };
4086818173bSMiklos Szeredi
4096818173bSMiklos Szeredi /* Pipe buffer operations for a socket and similar. */
41028a625cbSMiklos Szeredi const struct pipe_buf_operations nosteal_pipe_buf_ops = {
41128a625cbSMiklos Szeredi .release = generic_pipe_buf_release,
41228a625cbSMiklos Szeredi .get = generic_pipe_buf_get,
41328a625cbSMiklos Szeredi };
41428a625cbSMiklos Szeredi EXPORT_SYMBOL(nosteal_pipe_buf_ops);
41528a625cbSMiklos Szeredi
wakeup_pipe_writers(struct pipe_inode_info * pipe)41628a625cbSMiklos Szeredi static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
417b3c2d2ddSMiklos Szeredi {
418b3c2d2ddSMiklos Szeredi smp_mb();
419b3c2d2ddSMiklos Szeredi if (waitqueue_active(&pipe->wr_wait))
4200ddad21dSLinus Torvalds wake_up_interruptible(&pipe->wr_wait);
4210ddad21dSLinus Torvalds kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
422b3c2d2ddSMiklos Szeredi }
423b3c2d2ddSMiklos Szeredi
424b3c2d2ddSMiklos Szeredi /**
425b3c2d2ddSMiklos Szeredi * splice_from_pipe_feed - feed available data from a pipe to a file
426b3c2d2ddSMiklos Szeredi * @pipe: pipe to splice from
427b3c2d2ddSMiklos Szeredi * @sd: information to @actor
428b3c2d2ddSMiklos Szeredi * @actor: handler that splices the data
429b3c2d2ddSMiklos Szeredi *
430b3c2d2ddSMiklos Szeredi * Description:
431b3c2d2ddSMiklos Szeredi * This function loops over the pipe and calls @actor to do the
432b3c2d2ddSMiklos Szeredi * actual moving of a single struct pipe_buffer to the desired
433b3c2d2ddSMiklos Szeredi * destination. It returns when there's no more buffers left in
434b3c2d2ddSMiklos Szeredi * the pipe or if the requested number of bytes (@sd->total_len)
435b3c2d2ddSMiklos Szeredi * have been copied. It returns a positive number (one) if the
436b3c2d2ddSMiklos Szeredi * pipe needs to be filled with more data, zero if the required
437b3c2d2ddSMiklos Szeredi * number of bytes have been copied and -errno on error.
438b3c2d2ddSMiklos Szeredi *
439b3c2d2ddSMiklos Szeredi * This, together with splice_from_pipe_{begin,end,next}, may be
440b3c2d2ddSMiklos Szeredi * used to implement the functionality of __splice_from_pipe() when
441b3c2d2ddSMiklos Szeredi * locking is required around copying the pipe buffers to the
442b3c2d2ddSMiklos Szeredi * destination.
443b3c2d2ddSMiklos Szeredi */
splice_from_pipe_feed(struct pipe_inode_info * pipe,struct splice_desc * sd,splice_actor * actor)444b3c2d2ddSMiklos Szeredi static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
44596f9bc8fSAl Viro splice_actor *actor)
446b3c2d2ddSMiklos Szeredi {
447b3c2d2ddSMiklos Szeredi unsigned int head = pipe->head;
4488cefc107SDavid Howells unsigned int tail = pipe->tail;
4498cefc107SDavid Howells unsigned int mask = pipe->ring_size - 1;
4508cefc107SDavid Howells int ret;
451b3c2d2ddSMiklos Szeredi
452b3c2d2ddSMiklos Szeredi while (!pipe_empty(head, tail)) {
453ec057595SLinus Torvalds struct pipe_buffer *buf = &pipe->bufs[tail & mask];
4548cefc107SDavid Howells
455b3c2d2ddSMiklos Szeredi sd->len = buf->len;
456b3c2d2ddSMiklos Szeredi if (sd->len > sd->total_len)
457b3c2d2ddSMiklos Szeredi sd->len = sd->total_len;
458b3c2d2ddSMiklos Szeredi
459b3c2d2ddSMiklos Szeredi ret = pipe_buf_confirm(pipe, buf);
460fba597dbSMiklos Szeredi if (unlikely(ret)) {
461a8adbe37SMichał Mirosław if (ret == -ENODATA)
462b3c2d2ddSMiklos Szeredi ret = 0;
463b3c2d2ddSMiklos Szeredi return ret;
464b3c2d2ddSMiklos Szeredi }
465b3c2d2ddSMiklos Szeredi
466a8adbe37SMichał Mirosław ret = actor(pipe, buf, sd);
467a8adbe37SMichał Mirosław if (ret <= 0)
468a8adbe37SMichał Mirosław return ret;
469a8adbe37SMichał Mirosław
470a8adbe37SMichał Mirosław buf->offset += ret;
471b3c2d2ddSMiklos Szeredi buf->len -= ret;
472b3c2d2ddSMiklos Szeredi
473b3c2d2ddSMiklos Szeredi sd->num_spliced += ret;
474b3c2d2ddSMiklos Szeredi sd->len -= ret;
475b3c2d2ddSMiklos Szeredi sd->pos += ret;
476b3c2d2ddSMiklos Szeredi sd->total_len -= ret;
477b3c2d2ddSMiklos Szeredi
478b3c2d2ddSMiklos Szeredi if (!buf->len) {
479b3c2d2ddSMiklos Szeredi pipe_buf_release(pipe, buf);
480a779638cSMiklos Szeredi tail++;
4818cefc107SDavid Howells pipe->tail = tail;
4828cefc107SDavid Howells if (pipe->files)
4836447a3cfSAl Viro sd->need_wakeup = true;
484b3c2d2ddSMiklos Szeredi }
485b3c2d2ddSMiklos Szeredi
486b3c2d2ddSMiklos Szeredi if (!sd->total_len)
487b3c2d2ddSMiklos Szeredi return 0;
488b3c2d2ddSMiklos Szeredi }
489b3c2d2ddSMiklos Szeredi
490b3c2d2ddSMiklos Szeredi return 1;
491b3c2d2ddSMiklos Szeredi }
492b3c2d2ddSMiklos Szeredi
493b3c2d2ddSMiklos Szeredi /* We know we have a pipe buffer, but maybe it's empty? */
eat_empty_buffer(struct pipe_inode_info * pipe)494d1a819a2SLinus Torvalds static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
495d1a819a2SLinus Torvalds {
496d1a819a2SLinus Torvalds unsigned int tail = pipe->tail;
497d1a819a2SLinus Torvalds unsigned int mask = pipe->ring_size - 1;
498d1a819a2SLinus Torvalds struct pipe_buffer *buf = &pipe->bufs[tail & mask];
499d1a819a2SLinus Torvalds
500d1a819a2SLinus Torvalds if (unlikely(!buf->len)) {
501d1a819a2SLinus Torvalds pipe_buf_release(pipe, buf);
502d1a819a2SLinus Torvalds pipe->tail = tail+1;
503d1a819a2SLinus Torvalds return true;
504d1a819a2SLinus Torvalds }
505d1a819a2SLinus Torvalds
506d1a819a2SLinus Torvalds return false;
507d1a819a2SLinus Torvalds }
508d1a819a2SLinus Torvalds
509d1a819a2SLinus Torvalds /**
510b3c2d2ddSMiklos Szeredi * splice_from_pipe_next - wait for some data to splice from
511b3c2d2ddSMiklos Szeredi * @pipe: pipe to splice from
512b3c2d2ddSMiklos Szeredi * @sd: information about the splice operation
513b3c2d2ddSMiklos Szeredi *
514b3c2d2ddSMiklos Szeredi * Description:
515b3c2d2ddSMiklos Szeredi * This function will wait for some data and return a positive
516b3c2d2ddSMiklos Szeredi * value (one) if pipe buffers are available. It will return zero
517b3c2d2ddSMiklos Szeredi * or -errno if no more data needs to be spliced.
518b3c2d2ddSMiklos Szeredi */
splice_from_pipe_next(struct pipe_inode_info * pipe,struct splice_desc * sd)519b3c2d2ddSMiklos Szeredi static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
52096f9bc8fSAl Viro {
521b3c2d2ddSMiklos Szeredi /*
522c725bfceSJan Kara * Check for signal early to make process killable when there are
523c725bfceSJan Kara * always buffers available
524c725bfceSJan Kara */
525c725bfceSJan Kara if (signal_pending(current))
526c725bfceSJan Kara return -ERESTARTSYS;
527c725bfceSJan Kara
528c725bfceSJan Kara repeat:
529d1a819a2SLinus Torvalds while (pipe_empty(pipe->head, pipe->tail)) {
5308cefc107SDavid Howells if (!pipe->writers)
531b3c2d2ddSMiklos Szeredi return 0;
532b3c2d2ddSMiklos Szeredi
533b3c2d2ddSMiklos Szeredi if (sd->num_spliced)
534a28c8b9dSLinus Torvalds return 0;
535b3c2d2ddSMiklos Szeredi
536b3c2d2ddSMiklos Szeredi if (sd->flags & SPLICE_F_NONBLOCK)
537b3c2d2ddSMiklos Szeredi return -EAGAIN;
538b3c2d2ddSMiklos Szeredi
539b3c2d2ddSMiklos Szeredi if (signal_pending(current))
540b3c2d2ddSMiklos Szeredi return -ERESTARTSYS;
541b3c2d2ddSMiklos Szeredi
542b3c2d2ddSMiklos Szeredi if (sd->need_wakeup) {
543b3c2d2ddSMiklos Szeredi wakeup_pipe_writers(pipe);
544b3c2d2ddSMiklos Szeredi sd->need_wakeup = false;
545b3c2d2ddSMiklos Szeredi }
546b3c2d2ddSMiklos Szeredi
547b3c2d2ddSMiklos Szeredi pipe_wait_readable(pipe);
548472e5b05SLinus Torvalds }
549b3c2d2ddSMiklos Szeredi
550b3c2d2ddSMiklos Szeredi if (eat_empty_buffer(pipe))
551d1a819a2SLinus Torvalds goto repeat;
552d1a819a2SLinus Torvalds
553d1a819a2SLinus Torvalds return 1;
554b3c2d2ddSMiklos Szeredi }
555b3c2d2ddSMiklos Szeredi
556b3c2d2ddSMiklos Szeredi /**
557b3c2d2ddSMiklos Szeredi * splice_from_pipe_begin - start splicing from pipe
558b3c2d2ddSMiklos Szeredi * @sd: information about the splice operation
559b80901bbSRandy Dunlap *
560b3c2d2ddSMiklos Szeredi * Description:
561b3c2d2ddSMiklos Szeredi * This function should be called before a loop containing
562b3c2d2ddSMiklos Szeredi * splice_from_pipe_next() and splice_from_pipe_feed() to
563b3c2d2ddSMiklos Szeredi * initialize the necessary fields of @sd.
564b3c2d2ddSMiklos Szeredi */
splice_from_pipe_begin(struct splice_desc * sd)565b3c2d2ddSMiklos Szeredi static void splice_from_pipe_begin(struct splice_desc *sd)
56696f9bc8fSAl Viro {
567b3c2d2ddSMiklos Szeredi sd->num_spliced = 0;
568b3c2d2ddSMiklos Szeredi sd->need_wakeup = false;
569b3c2d2ddSMiklos Szeredi }
570b3c2d2ddSMiklos Szeredi
571b3c2d2ddSMiklos Szeredi /**
572b3c2d2ddSMiklos Szeredi * splice_from_pipe_end - finish splicing from pipe
573b3c2d2ddSMiklos Szeredi * @pipe: pipe to splice from
574b3c2d2ddSMiklos Szeredi * @sd: information about the splice operation
575b3c2d2ddSMiklos Szeredi *
576b3c2d2ddSMiklos Szeredi * Description:
577b3c2d2ddSMiklos Szeredi * This function will wake up pipe writers if necessary. It should
578b3c2d2ddSMiklos Szeredi * be called after a loop containing splice_from_pipe_next() and
579b3c2d2ddSMiklos Szeredi * splice_from_pipe_feed().
580b3c2d2ddSMiklos Szeredi */
splice_from_pipe_end(struct pipe_inode_info * pipe,struct splice_desc * sd)581b3c2d2ddSMiklos Szeredi static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
58296f9bc8fSAl Viro {
583b3c2d2ddSMiklos Szeredi if (sd->need_wakeup)
584b3c2d2ddSMiklos Szeredi wakeup_pipe_writers(pipe);
585b3c2d2ddSMiklos Szeredi }
586b3c2d2ddSMiklos Szeredi
587b3c2d2ddSMiklos Szeredi /**
588932cc6d4SJens Axboe * __splice_from_pipe - splice data from a pipe to given actor
589932cc6d4SJens Axboe * @pipe: pipe to splice from
590932cc6d4SJens Axboe * @sd: information to @actor
591932cc6d4SJens Axboe * @actor: handler that splices the data
592932cc6d4SJens Axboe *
593932cc6d4SJens Axboe * Description:
594932cc6d4SJens Axboe * This function does little more than loop over the pipe and call
595932cc6d4SJens Axboe * @actor to do the actual moving of a single struct pipe_buffer to
596932cc6d4SJens Axboe * the desired destination. See pipe_to_file, pipe_to_sendmsg, or
5972dc334f1SDavid Howells * pipe_to_user.
598932cc6d4SJens Axboe *
599932cc6d4SJens Axboe */
__splice_from_pipe(struct pipe_inode_info * pipe,struct splice_desc * sd,splice_actor * actor)60083f9135bSJens Axboe ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
601c66ab6faSJens Axboe splice_actor *actor)
602c66ab6faSJens Axboe {
6035274f052SJens Axboe int ret;
604b3c2d2ddSMiklos Szeredi
6055274f052SJens Axboe splice_from_pipe_begin(sd);
606b3c2d2ddSMiklos Szeredi do {
607b3c2d2ddSMiklos Szeredi cond_resched();
608c2489e07SJan Kara ret = splice_from_pipe_next(pipe, sd);
609b3c2d2ddSMiklos Szeredi if (ret > 0)
610b3c2d2ddSMiklos Szeredi ret = splice_from_pipe_feed(pipe, sd, actor);
611b3c2d2ddSMiklos Szeredi } while (ret > 0);
612b3c2d2ddSMiklos Szeredi splice_from_pipe_end(pipe, sd);
613b3c2d2ddSMiklos Szeredi
6145274f052SJens Axboe return sd->num_spliced ? sd->num_spliced : ret;
615b3c2d2ddSMiklos Szeredi }
6165274f052SJens Axboe EXPORT_SYMBOL(__splice_from_pipe);
61740bee44eSMark Fasheh
6185274f052SJens Axboe /**
619932cc6d4SJens Axboe * splice_from_pipe - splice data from a pipe to a file
620932cc6d4SJens Axboe * @pipe: pipe to splice from
621932cc6d4SJens Axboe * @out: file to splice to
622932cc6d4SJens Axboe * @ppos: position in @out
623932cc6d4SJens Axboe * @len: how many bytes to splice
624932cc6d4SJens Axboe * @flags: splice modifier flags
625932cc6d4SJens Axboe * @actor: handler that splices the data
626932cc6d4SJens Axboe *
627932cc6d4SJens Axboe * Description:
628932cc6d4SJens Axboe * See __splice_from_pipe. This function locks the pipe inode,
6292933970bSMiklos Szeredi * otherwise it's identical to __splice_from_pipe().
630932cc6d4SJens Axboe *
631932cc6d4SJens Axboe */
splice_from_pipe(struct pipe_inode_info * pipe,struct file * out,loff_t * ppos,size_t len,unsigned int flags,splice_actor * actor)632932cc6d4SJens Axboe ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
6336da61809SMark Fasheh loff_t *ppos, size_t len, unsigned int flags,
6346da61809SMark Fasheh splice_actor *actor)
6356da61809SMark Fasheh {
6366da61809SMark Fasheh ssize_t ret;
6376da61809SMark Fasheh struct splice_desc sd = {
638c66ab6faSJens Axboe .total_len = len,
639c66ab6faSJens Axboe .flags = flags,
640c66ab6faSJens Axboe .pos = *ppos,
641c66ab6faSJens Axboe .u.file = out,
6426a14b90bSJens Axboe };
643c66ab6faSJens Axboe
6446da61809SMark Fasheh pipe_lock(pipe);
64561e0d47cSMiklos Szeredi ret = __splice_from_pipe(pipe, &sd, actor);
646c66ab6faSJens Axboe pipe_unlock(pipe);
64761e0d47cSMiklos Szeredi
6486da61809SMark Fasheh return ret;
6496da61809SMark Fasheh }
6506da61809SMark Fasheh
6516da61809SMark Fasheh /**
6526da61809SMark Fasheh * iter_file_splice_write - splice data from a pipe to a file
6538d020765SAl Viro * @pipe: pipe info
6548d020765SAl Viro * @out: file to write to
6558d020765SAl Viro * @ppos: position in @out
6568d020765SAl Viro * @len: number of bytes to splice
6578d020765SAl Viro * @flags: splice modifier flags
6588d020765SAl Viro *
6598d020765SAl Viro * Description:
6608d020765SAl Viro * Will either move or copy pages (determined by @flags options) from
6618d020765SAl Viro * the given pipe inode to the given file.
6628d020765SAl Viro * This one is ->write_iter-based.
6638d020765SAl Viro *
6648d020765SAl Viro */
6658d020765SAl Viro ssize_t
iter_file_splice_write(struct pipe_inode_info * pipe,struct file * out,loff_t * ppos,size_t len,unsigned int flags)6668d020765SAl Viro iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
6678d020765SAl Viro loff_t *ppos, size_t len, unsigned int flags)
6688d020765SAl Viro {
6698d020765SAl Viro struct splice_desc sd = {
6708d020765SAl Viro .total_len = len,
6718d020765SAl Viro .flags = flags,
6728d020765SAl Viro .pos = *ppos,
6738d020765SAl Viro .u.file = out,
6748d020765SAl Viro };
6758d020765SAl Viro int nbufs = pipe->max_usage;
6766718b6f8SDavid Howells struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
6778d020765SAl Viro GFP_KERNEL);
6788d020765SAl Viro ssize_t ret;
6798d020765SAl Viro
6808d020765SAl Viro if (unlikely(!array))
6818d020765SAl Viro return -ENOMEM;
6828d020765SAl Viro
6838d020765SAl Viro pipe_lock(pipe);
6848d020765SAl Viro
6858d020765SAl Viro splice_from_pipe_begin(&sd);
6868d020765SAl Viro while (sd.total_len) {
6878d020765SAl Viro struct iov_iter from;
6888d020765SAl Viro unsigned int head, tail, mask;
689ec057595SLinus Torvalds size_t left;
6908d020765SAl Viro int n;
6918cefc107SDavid Howells
6928d020765SAl Viro ret = splice_from_pipe_next(pipe, &sd);
6938d020765SAl Viro if (ret <= 0)
6948d020765SAl Viro break;
6958d020765SAl Viro
6968d020765SAl Viro if (unlikely(nbufs < pipe->max_usage)) {
6976718b6f8SDavid Howells kfree(array);
6988d020765SAl Viro nbufs = pipe->max_usage;
6996718b6f8SDavid Howells array = kcalloc(nbufs, sizeof(struct bio_vec),
7008d020765SAl Viro GFP_KERNEL);
7018d020765SAl Viro if (!array) {
7028d020765SAl Viro ret = -ENOMEM;
7038d020765SAl Viro break;
7048d020765SAl Viro }
7058d020765SAl Viro }
7068d020765SAl Viro
7078d020765SAl Viro head = pipe->head;
708ec057595SLinus Torvalds tail = pipe->tail;
709ec057595SLinus Torvalds mask = pipe->ring_size - 1;
710ec057595SLinus Torvalds
711ec057595SLinus Torvalds /* build the vector */
7128d020765SAl Viro left = sd.total_len;
7138d020765SAl Viro for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++) {
7140f1d344fSPavel Begunkov struct pipe_buffer *buf = &pipe->bufs[tail & mask];
7158cefc107SDavid Howells size_t this_len = buf->len;
7168d020765SAl Viro
7178d020765SAl Viro /* zero-length bvecs are not supported, skip them */
7180f1d344fSPavel Begunkov if (!this_len)
7190f1d344fSPavel Begunkov continue;
7200f1d344fSPavel Begunkov this_len = min(this_len, left);
7210f1d344fSPavel Begunkov
7228d020765SAl Viro ret = pipe_buf_confirm(pipe, buf);
723fba597dbSMiklos Szeredi if (unlikely(ret)) {
7248d020765SAl Viro if (ret == -ENODATA)
7258d020765SAl Viro ret = 0;
7268d020765SAl Viro goto done;
7278d020765SAl Viro }
7288d020765SAl Viro
7298d020765SAl Viro bvec_set_page(&array[n], buf->page, this_len,
730664e4078SChristoph Hellwig buf->offset);
731664e4078SChristoph Hellwig left -= this_len;
7328d020765SAl Viro n++;
7330f1d344fSPavel Begunkov }
7348d020765SAl Viro
7358d020765SAl Viro iov_iter_bvec(&from, ITER_SOURCE, array, n, sd.total_len - left);
736de4eda9dSAl Viro ret = vfs_iter_write(out, &from, &sd.pos, 0);
737abbb6589SChristoph Hellwig if (ret <= 0)
7388d020765SAl Viro break;
7398d020765SAl Viro
7408d020765SAl Viro sd.num_spliced += ret;
7418d020765SAl Viro sd.total_len -= ret;
7428d020765SAl Viro *ppos = sd.pos;
743dbe4e192SChristoph Hellwig
7448d020765SAl Viro /* dismiss the fully eaten buffers, adjust the partial one */
7458d020765SAl Viro tail = pipe->tail;
7468cefc107SDavid Howells while (ret) {
7478d020765SAl Viro struct pipe_buffer *buf = &pipe->bufs[tail & mask];
7488cefc107SDavid Howells if (ret >= buf->len) {
7498d020765SAl Viro ret -= buf->len;
7508d020765SAl Viro buf->len = 0;
7518d020765SAl Viro pipe_buf_release(pipe, buf);
752a779638cSMiklos Szeredi tail++;
7538cefc107SDavid Howells pipe->tail = tail;
7548cefc107SDavid Howells if (pipe->files)
7558d020765SAl Viro sd.need_wakeup = true;
7568d020765SAl Viro } else {
7578d020765SAl Viro buf->offset += ret;
7588d020765SAl Viro buf->len -= ret;
7598d020765SAl Viro ret = 0;
7608d020765SAl Viro }
7618d020765SAl Viro }
7628d020765SAl Viro }
7638d020765SAl Viro done:
7648d020765SAl Viro kfree(array);
7658d020765SAl Viro splice_from_pipe_end(pipe, &sd);
7668d020765SAl Viro
7678d020765SAl Viro pipe_unlock(pipe);
7688d020765SAl Viro
7698d020765SAl Viro if (sd.num_spliced)
7708d020765SAl Viro ret = sd.num_spliced;
7718d020765SAl Viro
7728d020765SAl Viro return ret;
7738d020765SAl Viro }
7748d020765SAl Viro
7758d020765SAl Viro EXPORT_SYMBOL(iter_file_splice_write);
7768d020765SAl Viro
7778d020765SAl Viro #ifdef CONFIG_NET
7782dc334f1SDavid Howells /**
77983f9135bSJens Axboe * splice_to_socket - splice data from a pipe to a socket
7802dc334f1SDavid Howells * @pipe: pipe to splice from
781932cc6d4SJens Axboe * @out: socket to write to
78283f9135bSJens Axboe * @ppos: position in @out
783932cc6d4SJens Axboe * @len: number of bytes to splice
78483f9135bSJens Axboe * @flags: splice modifier flags
78583f9135bSJens Axboe *
78683f9135bSJens Axboe * Description:
787932cc6d4SJens Axboe * Will send @len bytes from the pipe to a network socket. No data copying
78883f9135bSJens Axboe * is involved.
78983f9135bSJens Axboe *
79083f9135bSJens Axboe */
splice_to_socket(struct pipe_inode_info * pipe,struct file * out,loff_t * ppos,size_t len,unsigned int flags)79183f9135bSJens Axboe ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out,
7922dc334f1SDavid Howells loff_t *ppos, size_t len, unsigned int flags)
793cbb7e577SJens Axboe {
7945274f052SJens Axboe struct socket *sock = sock_from_file(out);
7952dc334f1SDavid Howells struct bio_vec bvec[16];
7962dc334f1SDavid Howells struct msghdr msg = {};
7972dc334f1SDavid Howells ssize_t ret = 0;
7982dc334f1SDavid Howells size_t spliced = 0;
7992dc334f1SDavid Howells bool need_wakeup = false;
8002dc334f1SDavid Howells
8012dc334f1SDavid Howells pipe_lock(pipe);
8022dc334f1SDavid Howells
8032dc334f1SDavid Howells while (len > 0) {
8042dc334f1SDavid Howells unsigned int head, tail, mask, bc = 0;
8052dc334f1SDavid Howells size_t remain = len;
8062dc334f1SDavid Howells
8072dc334f1SDavid Howells /*
8082dc334f1SDavid Howells * Check for signal early to make process killable when there
8092dc334f1SDavid Howells * are always buffers available
8102dc334f1SDavid Howells */
8112dc334f1SDavid Howells ret = -ERESTARTSYS;
8122dc334f1SDavid Howells if (signal_pending(current))
8132dc334f1SDavid Howells break;
8142dc334f1SDavid Howells
8152dc334f1SDavid Howells while (pipe_empty(pipe->head, pipe->tail)) {
8162dc334f1SDavid Howells ret = 0;
8172dc334f1SDavid Howells if (!pipe->writers)
8182dc334f1SDavid Howells goto out;
8192dc334f1SDavid Howells
8202dc334f1SDavid Howells if (spliced)
8212dc334f1SDavid Howells goto out;
8222dc334f1SDavid Howells
8232dc334f1SDavid Howells ret = -EAGAIN;
8242dc334f1SDavid Howells if (flags & SPLICE_F_NONBLOCK)
8252dc334f1SDavid Howells goto out;
8262dc334f1SDavid Howells
8272dc334f1SDavid Howells ret = -ERESTARTSYS;
8282dc334f1SDavid Howells if (signal_pending(current))
8292dc334f1SDavid Howells goto out;
8302dc334f1SDavid Howells
8312dc334f1SDavid Howells if (need_wakeup) {
8322dc334f1SDavid Howells wakeup_pipe_writers(pipe);
8332dc334f1SDavid Howells need_wakeup = false;
8342dc334f1SDavid Howells }
8355274f052SJens Axboe
8365274f052SJens Axboe pipe_wait_readable(pipe);
8372dc334f1SDavid Howells }
8382dc334f1SDavid Howells
8392dc334f1SDavid Howells head = pipe->head;
8402dc334f1SDavid Howells tail = pipe->tail;
8412dc334f1SDavid Howells mask = pipe->ring_size - 1;
8422dc334f1SDavid Howells
8432dc334f1SDavid Howells while (!pipe_empty(head, tail)) {
8442dc334f1SDavid Howells struct pipe_buffer *buf = &pipe->bufs[tail & mask];
8452dc334f1SDavid Howells size_t seg;
8462dc334f1SDavid Howells
8472dc334f1SDavid Howells if (!buf->len) {
8482dc334f1SDavid Howells tail++;
8492dc334f1SDavid Howells continue;
8502dc334f1SDavid Howells }
8512dc334f1SDavid Howells
8522dc334f1SDavid Howells seg = min_t(size_t, remain, buf->len);
8532dc334f1SDavid Howells
8542dc334f1SDavid Howells ret = pipe_buf_confirm(pipe, buf);
8552dc334f1SDavid Howells if (unlikely(ret)) {
8562dc334f1SDavid Howells if (ret == -ENODATA)
8572dc334f1SDavid Howells ret = 0;
8582dc334f1SDavid Howells break;
8592dc334f1SDavid Howells }
8602dc334f1SDavid Howells
8612dc334f1SDavid Howells bvec_set_page(&bvec[bc++], buf->page, seg, buf->offset);
8622dc334f1SDavid Howells remain -= seg;
8632dc334f1SDavid Howells if (remain == 0 || bc >= ARRAY_SIZE(bvec))
864ca2d49f7SDavid Howells break;
8652dc334f1SDavid Howells tail++;
866ca2d49f7SDavid Howells }
8672dc334f1SDavid Howells
8682dc334f1SDavid Howells if (!bc)
8692dc334f1SDavid Howells break;
8702dc334f1SDavid Howells
8712dc334f1SDavid Howells msg.msg_flags = MSG_SPLICE_PAGES;
8722dc334f1SDavid Howells if (flags & SPLICE_F_MORE)
8732dc334f1SDavid Howells msg.msg_flags |= MSG_MORE;
8742dc334f1SDavid Howells if (remain && pipe_occupancy(pipe->head, tail) > 0)
8752dc334f1SDavid Howells msg.msg_flags |= MSG_MORE;
8762dc334f1SDavid Howells if (out->f_flags & O_NONBLOCK)
8772dc334f1SDavid Howells msg.msg_flags |= MSG_DONTWAIT;
8782dc334f1SDavid Howells
8792dc334f1SDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, bvec, bc,
8802dc334f1SDavid Howells len - remain);
8812dc334f1SDavid Howells ret = sock_sendmsg(sock, &msg);
8822dc334f1SDavid Howells if (ret <= 0)
8832dc334f1SDavid Howells break;
8842dc334f1SDavid Howells
8852dc334f1SDavid Howells spliced += ret;
8862dc334f1SDavid Howells len -= ret;
8872dc334f1SDavid Howells tail = pipe->tail;
8882dc334f1SDavid Howells while (ret > 0) {
8892dc334f1SDavid Howells struct pipe_buffer *buf = &pipe->bufs[tail & mask];
8902dc334f1SDavid Howells size_t seg = min_t(size_t, ret, buf->len);
8912dc334f1SDavid Howells
8922dc334f1SDavid Howells buf->offset += seg;
8932dc334f1SDavid Howells buf->len -= seg;
8942dc334f1SDavid Howells ret -= seg;
8952dc334f1SDavid Howells
8962dc334f1SDavid Howells if (!buf->len) {
8972dc334f1SDavid Howells pipe_buf_release(pipe, buf);
8982dc334f1SDavid Howells tail++;
8992dc334f1SDavid Howells }
9002dc334f1SDavid Howells }
9012dc334f1SDavid Howells
9022dc334f1SDavid Howells if (tail != pipe->tail) {
9032dc334f1SDavid Howells pipe->tail = tail;
9042dc334f1SDavid Howells if (pipe->files)
9052dc334f1SDavid Howells need_wakeup = true;
9062dc334f1SDavid Howells }
9072dc334f1SDavid Howells }
9082dc334f1SDavid Howells
9092dc334f1SDavid Howells out:
9102dc334f1SDavid Howells pipe_unlock(pipe);
9112dc334f1SDavid Howells if (need_wakeup)
9122dc334f1SDavid Howells wakeup_pipe_writers(pipe);
9132dc334f1SDavid Howells return spliced ?: ret;
9142dc334f1SDavid Howells }
915a0f06780SJeff Garzik #endif
91636e2c742SChristoph Hellwig
warn_unsupported(struct file * file,const char * op)91736e2c742SChristoph Hellwig static int warn_unsupported(struct file *file, const char *op)
91836e2c742SChristoph Hellwig {
91936e2c742SChristoph Hellwig pr_debug_ratelimited(
92036e2c742SChristoph Hellwig "splice %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
92136e2c742SChristoph Hellwig op, file, current->pid, current->comm);
92236e2c742SChristoph Hellwig return -EINVAL;
92336e2c742SChristoph Hellwig }
92483f9135bSJens Axboe
92583f9135bSJens Axboe /*
92683f9135bSJens Axboe * Attempt to initiate a splice from pipe to file.
9273a326a2cSIngo Molnar */
do_splice_from(struct pipe_inode_info * pipe,struct file * out,loff_t * ppos,size_t len,unsigned int flags)928cbb7e577SJens Axboe static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
9295274f052SJens Axboe loff_t *ppos, size_t len, unsigned int flags)
93036e2c742SChristoph Hellwig {
93136e2c742SChristoph Hellwig if (unlikely(!out->f_op->splice_write))
93200c285d0SChristoph Hellwig return warn_unsupported(out, "write");
9335274f052SJens Axboe return out->f_op->splice_write(pipe, out, ppos, len, flags);
9345274f052SJens Axboe }
93583f9135bSJens Axboe
9362bfc6685SDavid Howells /*
9372bfc6685SDavid Howells * Indicate to the caller that there was a premature EOF when reading from the
9382bfc6685SDavid Howells * source and the caller didn't indicate they would be sending more data after
9392bfc6685SDavid Howells * this.
9402bfc6685SDavid Howells */
do_splice_eof(struct splice_desc * sd)9412bfc6685SDavid Howells static void do_splice_eof(struct splice_desc *sd)
9422bfc6685SDavid Howells {
9432bfc6685SDavid Howells if (sd->splice_eof)
9442bfc6685SDavid Howells sd->splice_eof(sd);
9452bfc6685SDavid Howells }
9466a3f30b8SDavid Howells
9476a3f30b8SDavid Howells /**
9486a3f30b8SDavid Howells * vfs_splice_read - Read data from a file and splice it into a pipe
9496a3f30b8SDavid Howells * @in: File to splice from
9506a3f30b8SDavid Howells * @ppos: Input file offset
9516a3f30b8SDavid Howells * @pipe: Pipe to splice to
9526a3f30b8SDavid Howells * @len: Number of bytes to splice
9536a3f30b8SDavid Howells * @flags: Splice modifier flags (SPLICE_F_*)
9546a3f30b8SDavid Howells *
9556a3f30b8SDavid Howells * Splice the requested amount of data from the input file to the pipe. This
9566a3f30b8SDavid Howells * is synchronous as the caller must hold the pipe lock across the entire
9576a3f30b8SDavid Howells * operation.
9586a3f30b8SDavid Howells *
9596a3f30b8SDavid Howells * If successful, it returns the amount of data spliced, 0 if it hit the EOF or
96083f9135bSJens Axboe * a hole and a negative error code otherwise.
9616a3f30b8SDavid Howells */
vfs_splice_read(struct file * in,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)962cbb7e577SJens Axboe long vfs_splice_read(struct file *in, loff_t *ppos,
963cbb7e577SJens Axboe struct pipe_inode_info *pipe, size_t len,
9645274f052SJens Axboe unsigned int flags)
965313d64a3SAl Viro {
9665274f052SJens Axboe unsigned int p_space;
9675274f052SJens Axboe int ret;
96849570e9bSJens Axboe
9695274f052SJens Axboe if (unlikely(!(in->f_mode & FMODE_READ)))
970123856f0SDavid Howells return -EBADF;
971123856f0SDavid Howells if (!len)
9725274f052SJens Axboe return 0;
973313d64a3SAl Viro
974313d64a3SAl Viro /* Don't try to read more the pipe has space for. */
975313d64a3SAl Viro p_space = pipe->max_usage - pipe_occupancy(pipe->head, pipe->tail);
976313d64a3SAl Viro len = min_t(size_t, len, p_space << PAGE_SHIFT);
977cbb7e577SJens Axboe
9785274f052SJens Axboe ret = rw_verify_area(READ, in, ppos, len);
9795274f052SJens Axboe if (unlikely(ret < 0))
9805274f052SJens Axboe return ret;
98103cc0789SAl Viro
98203cc0789SAl Viro if (unlikely(len > MAX_RW_COUNT))
98303cc0789SAl Viro len = MAX_RW_COUNT;
98436e2c742SChristoph Hellwig
98536e2c742SChristoph Hellwig if (unlikely(!in->f_op->splice_read))
986aa3dbde8SDavid Howells return warn_unsupported(in, "read");
987b85930a0SDavid Howells /*
988b85930a0SDavid Howells * O_DIRECT and DAX don't deal with the pagecache, so we allocate a
989aa3dbde8SDavid Howells * buffer, copy into it and splice that into the pipe.
990b85930a0SDavid Howells */
991aa3dbde8SDavid Howells if ((in->f_flags & O_DIRECT) || IS_DAX(in->f_mapping->host))
9922bc01060SChristoph Hellwig return copy_splice_read(in, ppos, pipe, len, flags);
9935274f052SJens Axboe return in->f_op->splice_read(in, ppos, pipe, len, flags);
9946a3f30b8SDavid Howells }
9955274f052SJens Axboe EXPORT_SYMBOL_GPL(vfs_splice_read);
996932cc6d4SJens Axboe
997932cc6d4SJens Axboe /**
998932cc6d4SJens Axboe * splice_direct_to_actor - splices data directly between two non-pipes
999932cc6d4SJens Axboe * @in: file to splice from
1000932cc6d4SJens Axboe * @sd: actor information on where to splice to
1001932cc6d4SJens Axboe * @actor: handles the data splicing
1002932cc6d4SJens Axboe *
1003932cc6d4SJens Axboe * Description:
1004932cc6d4SJens Axboe * This is a special case helper to splice directly between two
1005932cc6d4SJens Axboe * points, without requiring an explicit pipe. Internally an allocated
1006932cc6d4SJens Axboe * pipe is cached in the process, and reused during the lifetime of
1007932cc6d4SJens Axboe * that process.
1008c66ab6faSJens Axboe *
1009c66ab6faSJens Axboe */
splice_direct_to_actor(struct file * in,struct splice_desc * sd,splice_direct_actor * actor)1010c66ab6faSJens Axboe ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1011b92ce558SJens Axboe splice_direct_actor *actor)
1012b92ce558SJens Axboe {
1013b92ce558SJens Axboe struct pipe_inode_info *pipe;
1014c66ab6faSJens Axboe long ret, bytes;
10150ff28d9fSChristophe Leroy size_t len;
1016b92ce558SJens Axboe int i, flags, more;
1017b92ce558SJens Axboe
101897ef77c5SJason A. Donenfeld /*
101997ef77c5SJason A. Donenfeld * We require the input to be seekable, as we don't want to randomly
102097ef77c5SJason A. Donenfeld * drop data for eg socket -> socket splicing. Use the piped splicing
1021b92ce558SJens Axboe * for that!
102297ef77c5SJason A. Donenfeld */
1023b92ce558SJens Axboe if (unlikely(!(in->f_mode & FMODE_LSEEK)))
1024b92ce558SJens Axboe return -EINVAL;
1025b92ce558SJens Axboe
1026b92ce558SJens Axboe /*
1027b92ce558SJens Axboe * neither in nor out is a pipe, setup an internal pipe attached to
1028b92ce558SJens Axboe * 'out' and transfer the wanted data from 'in' to 'out' through that
1029b92ce558SJens Axboe */
103049570e9bSJens Axboe pipe = current->splice_pipe;
10317bee130eSAl Viro if (unlikely(!pipe)) {
1032b92ce558SJens Axboe pipe = alloc_pipe_info();
1033b92ce558SJens Axboe if (!pipe)
1034b92ce558SJens Axboe return -ENOMEM;
1035b92ce558SJens Axboe
1036b92ce558SJens Axboe /*
103700522fb4SJens Axboe * We don't have an immediate reader, but we'll read the stuff
1038b92ce558SJens Axboe * out of the pipe right after the splice_to_pipe(). So set
1039b92ce558SJens Axboe * PIPE_READERS appropriately.
1040b92ce558SJens Axboe */
1041b92ce558SJens Axboe pipe->readers = 1;
1042b92ce558SJens Axboe
1043b92ce558SJens Axboe current->splice_pipe = pipe;
1044b92ce558SJens Axboe }
1045b92ce558SJens Axboe
104673d62d83SIngo Molnar /*
1047b92ce558SJens Axboe * Do the splice.
1048b92ce558SJens Axboe */
1049c66ab6faSJens Axboe bytes = 0;
1050219d9205SDavid Howells len = sd->total_len;
1051219d9205SDavid Howells
1052c66ab6faSJens Axboe /* Don't block on output, we have to drain the direct pipe. */
1053219d9205SDavid Howells flags = sd->flags;
1054c66ab6faSJens Axboe sd->flags &= ~SPLICE_F_NONBLOCK;
1055c66ab6faSJens Axboe
1056219d9205SDavid Howells /*
1057219d9205SDavid Howells * We signal MORE until we've read sufficient data to fulfill the
1058c66ab6faSJens Axboe * request and we keep signalling it if the caller set it.
10590ff28d9fSChristophe Leroy */
1060219d9205SDavid Howells more = sd->flags & SPLICE_F_MORE;
1061b92ce558SJens Axboe sd->flags |= SPLICE_F_MORE;
10628cefc107SDavid Howells
106317614445SDarrick J. Wong WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));
1064b92ce558SJens Axboe
106551a92c0fSJens Axboe while (len) {
1066a82c53a0STom Zanussi size_t read_len;
1067b92ce558SJens Axboe loff_t pos = sd->pos, prev_pos = pos;
10686a3f30b8SDavid Howells
106951a92c0fSJens Axboe ret = vfs_splice_read(in, &pos, pipe, len, flags);
10702bfc6685SDavid Howells if (unlikely(ret <= 0))
1071b92ce558SJens Axboe goto read_failure;
1072b92ce558SJens Axboe
1073c66ab6faSJens Axboe read_len = ret;
1074b92ce558SJens Axboe sd->total_len = read_len;
1075b92ce558SJens Axboe
1076219d9205SDavid Howells /*
1077219d9205SDavid Howells * If we now have sufficient data to fulfill the request then
10780ff28d9fSChristophe Leroy * we clear SPLICE_F_MORE if it was not set initially.
1079219d9205SDavid Howells */
10800ff28d9fSChristophe Leroy if (read_len >= len && !more)
1081219d9205SDavid Howells sd->flags &= ~SPLICE_F_MORE;
10820ff28d9fSChristophe Leroy
1083b92ce558SJens Axboe /*
1084b92ce558SJens Axboe * NOTE: nonblocking mode only applies to the input. We
1085b92ce558SJens Axboe * must not do the output in nonblocking mode as then we
1086b92ce558SJens Axboe * could get stuck data in the internal pipe:
1087c66ab6faSJens Axboe */
1088a82c53a0STom Zanussi ret = actor(pipe, sd);
1089a82c53a0STom Zanussi if (unlikely(ret <= 0)) {
1090b92ce558SJens Axboe sd->pos = prev_pos;
1091a82c53a0STom Zanussi goto out_release;
1092b92ce558SJens Axboe }
1093b92ce558SJens Axboe
1094b92ce558SJens Axboe bytes += ret;
1095bcd4f3acSJens Axboe len -= ret;
1096b92ce558SJens Axboe sd->pos = pos;
1097a82c53a0STom Zanussi
1098a82c53a0STom Zanussi if (ret < read_len) {
109951a92c0fSJens Axboe sd->pos = prev_pos + ret;
1100b92ce558SJens Axboe goto out_release;
1101a82c53a0STom Zanussi }
1102b92ce558SJens Axboe }
11039e97198dSJens Axboe
11048cefc107SDavid Howells done:
11059e97198dSJens Axboe pipe->tail = pipe->head = 0;
1106b92ce558SJens Axboe file_accessed(in);
1107b92ce558SJens Axboe return bytes;
11082bfc6685SDavid Howells
11092bfc6685SDavid Howells read_failure:
11102bfc6685SDavid Howells /*
11112bfc6685SDavid Howells * If the user did *not* set SPLICE_F_MORE *and* we didn't hit that
11122bfc6685SDavid Howells * "use all of len" case that cleared SPLICE_F_MORE, *and* we did a
11132bfc6685SDavid Howells * "->splice_in()" that returned EOF (ie zero) *and* we have sent at
11142bfc6685SDavid Howells * least 1 byte *then* we will also do the ->splice_eof() call.
11152bfc6685SDavid Howells */
11162bfc6685SDavid Howells if (ret == 0 && !more && len > 0 && bytes)
1117b92ce558SJens Axboe do_splice_eof(sd);
1118b92ce558SJens Axboe out_release:
1119b92ce558SJens Axboe /*
1120b92ce558SJens Axboe * If we did an incomplete transfer we must release
1121b92ce558SJens Axboe * the pipe buffers in question:
11228cefc107SDavid Howells */
11238cefc107SDavid Howells for (i = 0; i < pipe->ring_size; i++) {
1124b92ce558SJens Axboe struct pipe_buffer *buf = &pipe->bufs[i];
1125a779638cSMiklos Szeredi
1126a779638cSMiklos Szeredi if (buf->ops)
1127b92ce558SJens Axboe pipe_buf_release(pipe, buf);
1128b92ce558SJens Axboe }
11299e97198dSJens Axboe
11309e97198dSJens Axboe if (!bytes)
1131b92ce558SJens Axboe bytes = ret;
11329e97198dSJens Axboe
1133c66ab6faSJens Axboe goto done;
1134c66ab6faSJens Axboe }
1135c66ab6faSJens Axboe EXPORT_SYMBOL(splice_direct_to_actor);
1136c66ab6faSJens Axboe
direct_splice_actor(struct pipe_inode_info * pipe,struct splice_desc * sd)1137c66ab6faSJens Axboe static int direct_splice_actor(struct pipe_inode_info *pipe,
1138c66ab6faSJens Axboe struct splice_desc *sd)
11396a14b90bSJens Axboe {
1140c66ab6faSJens Axboe struct file *file = sd->u.file;
11417995bd28SAl Viro
11422cb4b05eSChangli Gao return do_splice_from(pipe, file, sd->opos, sd->total_len,
1143c66ab6faSJens Axboe sd->flags);
1144c66ab6faSJens Axboe }
11452bfc6685SDavid Howells
direct_file_splice_eof(struct splice_desc * sd)11462bfc6685SDavid Howells static void direct_file_splice_eof(struct splice_desc *sd)
11472bfc6685SDavid Howells {
11482bfc6685SDavid Howells struct file *file = sd->u.file;
11492bfc6685SDavid Howells
11502bfc6685SDavid Howells if (file->f_op->splice_eof)
11512bfc6685SDavid Howells file->f_op->splice_eof(file);
11522bfc6685SDavid Howells }
1153932cc6d4SJens Axboe
1154932cc6d4SJens Axboe /**
1155932cc6d4SJens Axboe * do_splice_direct - splices data directly between two files
1156932cc6d4SJens Axboe * @in: file to splice from
1157932cc6d4SJens Axboe * @ppos: input file offset
1158acdb37c3SRandy Dunlap * @out: file to splice to
1159932cc6d4SJens Axboe * @opos: output file offset
1160932cc6d4SJens Axboe * @len: number of bytes to splice
1161932cc6d4SJens Axboe * @flags: splice modifier flags
1162932cc6d4SJens Axboe *
1163932cc6d4SJens Axboe * Description:
1164932cc6d4SJens Axboe * For use by do_sendfile(). splice can easily emulate sendfile, but
1165932cc6d4SJens Axboe * doing it in the application would incur an extra system call
1166932cc6d4SJens Axboe * (splice in + splice out, as compared to just sendfile()). So this helper
1167932cc6d4SJens Axboe * can splice directly through a process-private pipe.
1168932cc6d4SJens Axboe *
1169c66ab6faSJens Axboe */
do_splice_direct(struct file * in,loff_t * ppos,struct file * out,loff_t * opos,size_t len,unsigned int flags)11707995bd28SAl Viro long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1171c66ab6faSJens Axboe loff_t *opos, size_t len, unsigned int flags)
1172c66ab6faSJens Axboe {
1173c66ab6faSJens Axboe struct splice_desc sd = {
1174c66ab6faSJens Axboe .len = len,
1175c66ab6faSJens Axboe .total_len = len,
1176c66ab6faSJens Axboe .flags = flags,
11776a14b90bSJens Axboe .pos = *ppos,
11782bfc6685SDavid Howells .u.file = out,
11797995bd28SAl Viro .splice_eof = direct_file_splice_eof,
1180c66ab6faSJens Axboe .opos = opos,
118151a92c0fSJens Axboe };
1182c66ab6faSJens Axboe long ret;
118318c67cb9SAl Viro
118418c67cb9SAl Viro if (unlikely(!(out->f_mode & FMODE_WRITE)))
118518c67cb9SAl Viro return -EBADF;
118618c67cb9SAl Viro
118718c67cb9SAl Viro if (unlikely(out->f_flags & O_APPEND))
118818c67cb9SAl Viro return -EINVAL;
118918c67cb9SAl Viro
119018c67cb9SAl Viro ret = rw_verify_area(WRITE, out, opos, len);
119118c67cb9SAl Viro if (unlikely(ret < 0))
119218c67cb9SAl Viro return ret;
1193c66ab6faSJens Axboe
119451a92c0fSJens Axboe ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1195a82c53a0STom Zanussi if (ret > 0)
119651a92c0fSJens Axboe *ppos = sd.pos;
1197c66ab6faSJens Axboe
1198b92ce558SJens Axboe return ret;
11991c118596SMiklos Szeredi }
1200b92ce558SJens Axboe EXPORT_SYMBOL(do_splice_direct);
12018924feffSAl Viro
wait_for_space(struct pipe_inode_info * pipe,unsigned flags)12028924feffSAl Viro static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
120352bce911SLinus Torvalds {
120452bce911SLinus Torvalds for (;;) {
120552bce911SLinus Torvalds if (unlikely(!pipe->readers)) {
120652bce911SLinus Torvalds send_sig(SIGPIPE, current, 0);
120752bce911SLinus Torvalds return -EPIPE;
12086718b6f8SDavid Howells }
120952bce911SLinus Torvalds if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
12108924feffSAl Viro return 0;
12118924feffSAl Viro if (flags & SPLICE_F_NONBLOCK)
12128924feffSAl Viro return -EAGAIN;
12138924feffSAl Viro if (signal_pending(current))
1214472e5b05SLinus Torvalds return -ERESTARTSYS;
12158924feffSAl Viro pipe_wait_writable(pipe);
12168924feffSAl Viro }
12178924feffSAl Viro }
12187c77f0b3SMiklos Szeredi
12197c77f0b3SMiklos Szeredi static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
12207c77f0b3SMiklos Szeredi struct pipe_inode_info *opipe,
1221ddac0d39SJens Axboe size_t len, unsigned int flags);
1222b964bf53SAl Viro
splice_file_to_pipe(struct file * in,struct pipe_inode_info * opipe,loff_t * offset,size_t len,unsigned int flags)1223faa97c48SAl Viro long splice_file_to_pipe(struct file *in,
1224faa97c48SAl Viro struct pipe_inode_info *opipe,
1225faa97c48SAl Viro loff_t *offset,
1226faa97c48SAl Viro size_t len, unsigned int flags)
1227faa97c48SAl Viro {
1228faa97c48SAl Viro long ret;
1229faa97c48SAl Viro
1230faa97c48SAl Viro pipe_lock(opipe);
1231faa97c48SAl Viro ret = wait_for_space(opipe, flags);
12326a3f30b8SDavid Howells if (!ret)
1233faa97c48SAl Viro ret = vfs_splice_read(in, offset, opipe, len, flags);
1234faa97c48SAl Viro pipe_unlock(opipe);
1235faa97c48SAl Viro if (ret > 0)
1236faa97c48SAl Viro wakeup_pipe_readers(opipe);
1237faa97c48SAl Viro return ret;
1238faa97c48SAl Viro }
1239ddac0d39SJens Axboe
124083f9135bSJens Axboe /*
124183f9135bSJens Axboe * Determine where to splice to/from.
1242ee6e00c8SJens Axboe */
do_splice(struct file * in,loff_t * off_in,struct file * out,loff_t * off_out,size_t len,unsigned int flags)1243ee6e00c8SJens Axboe long do_splice(struct file *in, loff_t *off_in, struct file *out,
12445274f052SJens Axboe loff_t *off_out, size_t len, unsigned int flags)
12457c77f0b3SMiklos Szeredi {
12467c77f0b3SMiklos Szeredi struct pipe_inode_info *ipipe;
12477995bd28SAl Viro struct pipe_inode_info *opipe;
1248a4514ebdSJens Axboe loff_t offset;
12495274f052SJens Axboe long ret;
125090da2e3fSPavel Begunkov
125190da2e3fSPavel Begunkov if (unlikely(!(in->f_mode & FMODE_READ) ||
125290da2e3fSPavel Begunkov !(out->f_mode & FMODE_WRITE)))
125390da2e3fSPavel Begunkov return -EBADF;
1254c73be61cSDavid Howells
1255c73be61cSDavid Howells ipipe = get_pipe_info(in, true);
12567c77f0b3SMiklos Szeredi opipe = get_pipe_info(out, true);
12577c77f0b3SMiklos Szeredi
12587c77f0b3SMiklos Szeredi if (ipipe && opipe) {
12597c77f0b3SMiklos Szeredi if (off_in || off_out)
12607c77f0b3SMiklos Szeredi return -ESPIPE;
12617c77f0b3SMiklos Szeredi
12627c77f0b3SMiklos Szeredi /* Splicing to self would be fun, but... */
12637c77f0b3SMiklos Szeredi if (ipipe == opipe)
12647c77f0b3SMiklos Szeredi return -EINVAL;
1265ee5e0011SSlavomir Kaslev
1266ee5e0011SSlavomir Kaslev if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1267ee5e0011SSlavomir Kaslev flags |= SPLICE_F_NONBLOCK;
126812ee4b66SAhelenia Ziemiańska
126912ee4b66SAhelenia Ziemiańska ret = splice_pipe_to_pipe(ipipe, opipe, len, flags);
1270529565dcSIngo Molnar } else if (ipipe) {
1271529565dcSIngo Molnar if (off_in)
1272b92ce558SJens Axboe return -ESPIPE;
127319c9a49bSChangli Gao if (off_out) {
1274b92ce558SJens Axboe if (!(out->f_mode & FMODE_PWRITE))
1275ee6e00c8SJens Axboe return -EINVAL;
12767995bd28SAl Viro offset = *off_out;
12777995bd28SAl Viro } else {
12787995bd28SAl Viro offset = out->f_pos;
1279529565dcSIngo Molnar }
128018c67cb9SAl Viro
128118c67cb9SAl Viro if (unlikely(out->f_flags & O_APPEND))
128218c67cb9SAl Viro return -EINVAL;
128318c67cb9SAl Viro
128418c67cb9SAl Viro ret = rw_verify_area(WRITE, out, &offset, len);
128518c67cb9SAl Viro if (unlikely(ret < 0))
128618c67cb9SAl Viro return ret;
1287ee5e0011SSlavomir Kaslev
1288ee5e0011SSlavomir Kaslev if (in->f_flags & O_NONBLOCK)
1289ee5e0011SSlavomir Kaslev flags |= SPLICE_F_NONBLOCK;
1290500368f7SAl Viro
12917995bd28SAl Viro file_start_write(out);
1292500368f7SAl Viro ret = do_splice_from(ipipe, out, &offset, len, flags);
1293a4514ebdSJens Axboe file_end_write(out);
12947995bd28SAl Viro
12957995bd28SAl Viro if (!off_out)
1296ee6e00c8SJens Axboe out->f_pos = offset;
1297ee6e00c8SJens Axboe else
129812ee4b66SAhelenia Ziemiańska *off_out = offset;
1299529565dcSIngo Molnar } else if (opipe) {
1300529565dcSIngo Molnar if (off_out)
1301b92ce558SJens Axboe return -ESPIPE;
130219c9a49bSChangli Gao if (off_in) {
1303b92ce558SJens Axboe if (!(in->f_mode & FMODE_PREAD))
1304ee6e00c8SJens Axboe return -EINVAL;
13057995bd28SAl Viro offset = *off_in;
13067995bd28SAl Viro } else {
13077995bd28SAl Viro offset = in->f_pos;
1308529565dcSIngo Molnar }
1309ee5e0011SSlavomir Kaslev
1310ee5e0011SSlavomir Kaslev if (out->f_flags & O_NONBLOCK)
1311ee5e0011SSlavomir Kaslev flags |= SPLICE_F_NONBLOCK;
1312faa97c48SAl Viro
1313983652c6SChung-Chiang Cheng ret = splice_file_to_pipe(in, opipe, &offset, len, flags);
13147995bd28SAl Viro
13157995bd28SAl Viro if (!off_in)
1316ee6e00c8SJens Axboe in->f_pos = offset;
1317ee6e00c8SJens Axboe else
131812ee4b66SAhelenia Ziemiańska *off_in = offset;
131912ee4b66SAhelenia Ziemiańska } else {
1320529565dcSIngo Molnar ret = -EINVAL;
13215274f052SJens Axboe }
132212ee4b66SAhelenia Ziemiańska
132312ee4b66SAhelenia Ziemiańska if (ret > 0) {
132412ee4b66SAhelenia Ziemiańska /*
132512ee4b66SAhelenia Ziemiańska * Generate modify out before access in:
132612ee4b66SAhelenia Ziemiańska * do_splice_from() may've already sent modify out,
132712ee4b66SAhelenia Ziemiańska * and this ensures the events get merged.
132812ee4b66SAhelenia Ziemiańska */
132912ee4b66SAhelenia Ziemiańska fsnotify_modify(out);
133012ee4b66SAhelenia Ziemiańska fsnotify_access(in);
133112ee4b66SAhelenia Ziemiańska }
133212ee4b66SAhelenia Ziemiańska
13335274f052SJens Axboe return ret;
13345274f052SJens Axboe }
1335ee6e00c8SJens Axboe
__do_splice(struct file * in,loff_t __user * off_in,struct file * out,loff_t __user * off_out,size_t len,unsigned int flags)1336ee6e00c8SJens Axboe static long __do_splice(struct file *in, loff_t __user *off_in,
1337ee6e00c8SJens Axboe struct file *out, loff_t __user *off_out,
1338ee6e00c8SJens Axboe size_t len, unsigned int flags)
1339ee6e00c8SJens Axboe {
1340ee6e00c8SJens Axboe struct pipe_inode_info *ipipe;
1341ee6e00c8SJens Axboe struct pipe_inode_info *opipe;
1342ee6e00c8SJens Axboe loff_t offset, *__off_in = NULL, *__off_out = NULL;
1343ee6e00c8SJens Axboe long ret;
1344ee6e00c8SJens Axboe
1345ee6e00c8SJens Axboe ipipe = get_pipe_info(in, true);
1346ee6e00c8SJens Axboe opipe = get_pipe_info(out, true);
13470f99fc51SJens Axboe
13480f99fc51SJens Axboe if (ipipe) {
1349ee6e00c8SJens Axboe if (off_in)
13500f99fc51SJens Axboe return -ESPIPE;
13510f99fc51SJens Axboe pipe_clear_nowait(in);
13520f99fc51SJens Axboe }
13530f99fc51SJens Axboe if (opipe) {
1354ee6e00c8SJens Axboe if (off_out)
13550f99fc51SJens Axboe return -ESPIPE;
13560f99fc51SJens Axboe pipe_clear_nowait(out);
1357ee6e00c8SJens Axboe }
1358ee6e00c8SJens Axboe
1359ee6e00c8SJens Axboe if (off_out) {
1360ee6e00c8SJens Axboe if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1361ee6e00c8SJens Axboe return -EFAULT;
1362ee6e00c8SJens Axboe __off_out = &offset;
1363ee6e00c8SJens Axboe }
1364ee6e00c8SJens Axboe if (off_in) {
1365ee6e00c8SJens Axboe if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1366ee6e00c8SJens Axboe return -EFAULT;
1367ee6e00c8SJens Axboe __off_in = &offset;
1368ee6e00c8SJens Axboe }
1369ee6e00c8SJens Axboe
1370ee6e00c8SJens Axboe ret = do_splice(in, __off_in, out, __off_out, len, flags);
1371ee6e00c8SJens Axboe if (ret < 0)
1372ee6e00c8SJens Axboe return ret;
1373ee6e00c8SJens Axboe
1374ee6e00c8SJens Axboe if (__off_out && copy_to_user(off_out, __off_out, sizeof(loff_t)))
1375ee6e00c8SJens Axboe return -EFAULT;
1376ee6e00c8SJens Axboe if (__off_in && copy_to_user(off_in, __off_in, sizeof(loff_t)))
1377ee6e00c8SJens Axboe return -EFAULT;
1378ee6e00c8SJens Axboe
1379ee6e00c8SJens Axboe return ret;
1380ee6e00c8SJens Axboe }
138179fddc4eSAl Viro
iter_to_pipe(struct iov_iter * from,struct pipe_inode_info * pipe,unsigned flags)138279fddc4eSAl Viro static int iter_to_pipe(struct iov_iter *from,
138379fddc4eSAl Viro struct pipe_inode_info *pipe,
1384912d35f8SJens Axboe unsigned flags)
138579fddc4eSAl Viro {
138679fddc4eSAl Viro struct pipe_buffer buf = {
138779fddc4eSAl Viro .ops = &user_page_pipe_buf_ops,
138879fddc4eSAl Viro .flags = flags
138979fddc4eSAl Viro };
139079fddc4eSAl Viro size_t total = 0;
139179fddc4eSAl Viro int ret = 0;
13927d690c15SAl Viro
139379fddc4eSAl Viro while (iov_iter_count(from)) {
13947d690c15SAl Viro struct page *pages[16];
1395db85a9ebSAl Viro ssize_t left;
13967d690c15SAl Viro size_t start;
1397912d35f8SJens Axboe int i, n;
13987d690c15SAl Viro
13997d690c15SAl Viro left = iov_iter_get_pages2(from, pages, ~0UL, 16, &start);
14007d690c15SAl Viro if (left <= 0) {
140179fddc4eSAl Viro ret = left;
140279fddc4eSAl Viro break;
1403912d35f8SJens Axboe }
14047d690c15SAl Viro
14057d690c15SAl Viro n = DIV_ROUND_UP(left + start, PAGE_SIZE);
14067d690c15SAl Viro for (i = 0; i < n; i++) {
14077d690c15SAl Viro int size = min_t(int, left, PAGE_SIZE - start);
14087d690c15SAl Viro
140979fddc4eSAl Viro buf.page = pages[i];
141079fddc4eSAl Viro buf.offset = start;
141179fddc4eSAl Viro buf.len = size;
141279fddc4eSAl Viro ret = add_to_pipe(pipe, &buf);
14137d690c15SAl Viro if (unlikely(ret < 0)) {
14147d690c15SAl Viro iov_iter_revert(from, left);
14157d690c15SAl Viro // this one got dropped by add_to_pipe()
14167d690c15SAl Viro while (++i < n)
14177d690c15SAl Viro put_page(pages[i]);
14187d690c15SAl Viro goto out;
141979fddc4eSAl Viro }
14207d690c15SAl Viro total += ret;
14217d690c15SAl Viro left -= size;
1422912d35f8SJens Axboe start = 0;
1423912d35f8SJens Axboe }
14247d690c15SAl Viro }
142579fddc4eSAl Viro out:
1426912d35f8SJens Axboe return total ? total : ret;
1427912d35f8SJens Axboe }
14286a14b90bSJens Axboe
pipe_to_user(struct pipe_inode_info * pipe,struct pipe_buffer * buf,struct splice_desc * sd)14296a14b90bSJens Axboe static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
14306a14b90bSJens Axboe struct splice_desc *sd)
14316130f531SAl Viro {
14326130f531SAl Viro int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
14336a14b90bSJens Axboe return n == sd->len ? n : -EFAULT;
14346a14b90bSJens Axboe }
14356a14b90bSJens Axboe
14366a14b90bSJens Axboe /*
14376a14b90bSJens Axboe * For lack of a better implementation, implement vmsplice() to userspace
14386a14b90bSJens Axboe * as a simple copy of the pipes pages to the user iov.
143987a3002aSAl Viro */
vmsplice_to_user(struct file * file,struct iov_iter * iter,unsigned int flags)144087a3002aSAl Viro static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
14416a14b90bSJens Axboe unsigned int flags)
1442c73be61cSDavid Howells {
144387a3002aSAl Viro struct pipe_inode_info *pipe = get_pipe_info(file, true);
144487a3002aSAl Viro struct splice_desc sd = {
144587a3002aSAl Viro .total_len = iov_iter_count(iter),
144687a3002aSAl Viro .flags = flags,
144787a3002aSAl Viro .u.data = iter
144887a3002aSAl Viro };
14496a14b90bSJens Axboe long ret = 0;
14506a14b90bSJens Axboe
14516a14b90bSJens Axboe if (!pipe)
14526a14b90bSJens Axboe return -EBADF;
14530f99fc51SJens Axboe
14540f99fc51SJens Axboe pipe_clear_nowait(file);
1455345995faSAl Viro
14566130f531SAl Viro if (sd.total_len) {
14576130f531SAl Viro pipe_lock(pipe);
145861e0d47cSMiklos Szeredi ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1459345995faSAl Viro pipe_unlock(pipe);
14606a14b90bSJens Axboe }
14617f0f1ea0SAhelenia Ziemiańska
14627f0f1ea0SAhelenia Ziemiańska if (ret > 0)
14637f0f1ea0SAhelenia Ziemiańska fsnotify_access(file);
14646a14b90bSJens Axboe
14656a14b90bSJens Axboe return ret;
14666a14b90bSJens Axboe }
1467912d35f8SJens Axboe
1468912d35f8SJens Axboe /*
1469912d35f8SJens Axboe * vmsplice splices a user address range into a pipe. It can be thought of
1470912d35f8SJens Axboe * as splice-from-memory, where the regular splice is splice-from-file (or
1471912d35f8SJens Axboe * to file). In both cases the output is a pipe, naturally.
147287a3002aSAl Viro */
vmsplice_to_pipe(struct file * file,struct iov_iter * iter,unsigned int flags)147387a3002aSAl Viro static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
1474912d35f8SJens Axboe unsigned int flags)
1475ddac0d39SJens Axboe {
147687a3002aSAl Viro struct pipe_inode_info *pipe;
147779fddc4eSAl Viro long ret = 0;
147879fddc4eSAl Viro unsigned buf_flag = 0;
147979fddc4eSAl Viro
148079fddc4eSAl Viro if (flags & SPLICE_F_GIFT)
1481912d35f8SJens Axboe buf_flag = PIPE_BUF_FLAG_GIFT;
1482c73be61cSDavid Howells
1483ddac0d39SJens Axboe pipe = get_pipe_info(file, true);
1484912d35f8SJens Axboe if (!pipe)
1485912d35f8SJens Axboe return -EBADF;
14860f99fc51SJens Axboe
14870f99fc51SJens Axboe pipe_clear_nowait(file);
14888924feffSAl Viro
14898924feffSAl Viro pipe_lock(pipe);
149079fddc4eSAl Viro ret = wait_for_space(pipe, flags);
149187a3002aSAl Viro if (!ret)
14928924feffSAl Viro ret = iter_to_pipe(iter, pipe, buf_flag);
14937f0f1ea0SAhelenia Ziemiańska pipe_unlock(pipe);
14948924feffSAl Viro if (ret > 0) {
14957f0f1ea0SAhelenia Ziemiańska wakeup_pipe_readers(pipe);
14967f0f1ea0SAhelenia Ziemiańska fsnotify_modify(file);
149735f3d14dSJens Axboe }
1498912d35f8SJens Axboe return ret;
1499912d35f8SJens Axboe }
150087a3002aSAl Viro
vmsplice_type(struct fd f,int * type)150187a3002aSAl Viro static int vmsplice_type(struct fd f, int *type)
150287a3002aSAl Viro {
150387a3002aSAl Viro if (!f.file)
150487a3002aSAl Viro return -EBADF;
1505de4eda9dSAl Viro if (f.file->f_mode & FMODE_WRITE) {
150687a3002aSAl Viro *type = ITER_SOURCE;
1507de4eda9dSAl Viro } else if (f.file->f_mode & FMODE_READ) {
150887a3002aSAl Viro *type = ITER_DEST;
150987a3002aSAl Viro } else {
151087a3002aSAl Viro fdput(f);
151187a3002aSAl Viro return -EBADF;
151287a3002aSAl Viro }
151387a3002aSAl Viro return 0;
151487a3002aSAl Viro }
15156a14b90bSJens Axboe
15166a14b90bSJens Axboe /*
15176a14b90bSJens Axboe * Note that vmsplice only really supports true splicing _from_ user memory
15186a14b90bSJens Axboe * to a pipe, not the other way around. Splicing from user memory is a simple
15196a14b90bSJens Axboe * operation that can be supported without any funky alignment restrictions
15206a14b90bSJens Axboe * or nasty vm tricks. We simply map in the user memory and fill them into
15216a14b90bSJens Axboe * a pipe. The reverse isn't quite as easy, though. There are two possible
15226a14b90bSJens Axboe * solutions for that:
15236a14b90bSJens Axboe *
15246a14b90bSJens Axboe * - memcpy() the data internally, at which point we might as well just
15256a14b90bSJens Axboe * do a regular read() on the buffer anyway.
15266a14b90bSJens Axboe * - Lots of nasty vm tricks, that are neither fast nor flexible (it
15276a14b90bSJens Axboe * has restriction limitations on both ends of the pipe).
15286a14b90bSJens Axboe *
15296a14b90bSJens Axboe * Currently we punt and implement it as a normal copy, see pipe_to_user().
15306a14b90bSJens Axboe *
153187a3002aSAl Viro */
SYSCALL_DEFINE4(vmsplice,int,fd,const struct iovec __user *,uiov,unsigned long,nr_segs,unsigned int,flags)153230cfe4efSDominik Brodowski SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
153330cfe4efSDominik Brodowski unsigned long, nr_segs, unsigned int, flags)
153487a3002aSAl Viro {
153587a3002aSAl Viro struct iovec iovstack[UIO_FASTIOV];
153687a3002aSAl Viro struct iovec *iov = iovstack;
153787e5e6daSJens Axboe struct iov_iter iter;
153887a3002aSAl Viro ssize_t error;
153987a3002aSAl Viro struct fd f;
154087a3002aSAl Viro int type;
1541598b3cecSChristoph Hellwig
1542598b3cecSChristoph Hellwig if (unlikely(flags & ~SPLICE_F_ALL))
1543598b3cecSChristoph Hellwig return -EINVAL;
154487a3002aSAl Viro
154587a3002aSAl Viro f = fdget(fd);
154687a3002aSAl Viro error = vmsplice_type(f, &type);
154787a3002aSAl Viro if (error)
154887a3002aSAl Viro return error;
154987a3002aSAl Viro
155087a3002aSAl Viro error = import_iovec(type, uiov, nr_segs,
1551598b3cecSChristoph Hellwig ARRAY_SIZE(iovstack), &iov, &iter);
1552598b3cecSChristoph Hellwig if (error < 0)
1553598b3cecSChristoph Hellwig goto out_fdput;
1554598b3cecSChristoph Hellwig
1555598b3cecSChristoph Hellwig if (!iov_iter_count(&iter))
1556de4eda9dSAl Viro error = 0;
1557598b3cecSChristoph Hellwig else if (type == ITER_SOURCE)
1558598b3cecSChristoph Hellwig error = vmsplice_to_pipe(f.file, &iter, flags);
1559598b3cecSChristoph Hellwig else
1560598b3cecSChristoph Hellwig error = vmsplice_to_user(f.file, &iter, flags);
156187a3002aSAl Viro
1562598b3cecSChristoph Hellwig kfree(iov);
156387a3002aSAl Viro out_fdput:
156487a3002aSAl Viro fdput(f);
156530cfe4efSDominik Brodowski return error;
156630cfe4efSDominik Brodowski }
1567836f92adSHeiko Carstens
SYSCALL_DEFINE6(splice,int,fd_in,loff_t __user *,off_in,int,fd_out,loff_t __user *,off_out,size_t,len,unsigned int,flags)1568836f92adSHeiko Carstens SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1569836f92adSHeiko Carstens int, fd_out, loff_t __user *, off_out,
15705274f052SJens Axboe size_t, len, unsigned int, flags)
15712903ff01SAl Viro {
15725274f052SJens Axboe struct fd in, out;
15735274f052SJens Axboe long error;
15745274f052SJens Axboe
15755274f052SJens Axboe if (unlikely(!len))
15765274f052SJens Axboe return 0;
15773d6ea290SAl Viro
15783d6ea290SAl Viro if (unlikely(flags & ~SPLICE_F_ALL))
15793d6ea290SAl Viro return -EINVAL;
15805274f052SJens Axboe
15812903ff01SAl Viro error = -EBADF;
15822903ff01SAl Viro in = fdget(fd_in);
15832903ff01SAl Viro if (in.file) {
15842903ff01SAl Viro out = fdget(fd_out);
1585ee6e00c8SJens Axboe if (out.file) {
1586529565dcSIngo Molnar error = __do_splice(in.file, off_in, out.file, off_out,
15872903ff01SAl Viro len, flags);
15885274f052SJens Axboe fdput(out);
15892903ff01SAl Viro }
15905274f052SJens Axboe fdput(in);
15915274f052SJens Axboe }
15925274f052SJens Axboe return error;
159370524490SJens Axboe }
159470524490SJens Axboe
1595aadd06e5SJens Axboe /*
1596aadd06e5SJens Axboe * Make sure there's data to read. Wait for input if we can, otherwise
1597aadd06e5SJens Axboe * return an appropriate error.
15987c77f0b3SMiklos Szeredi */
ipipe_prep(struct pipe_inode_info * pipe,unsigned int flags)1599aadd06e5SJens Axboe static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1600aadd06e5SJens Axboe {
1601aadd06e5SJens Axboe int ret;
1602aadd06e5SJens Axboe
16038cefc107SDavid Howells /*
1604aadd06e5SJens Axboe * Check the pipe occupancy without the inode lock first. This function
1605aadd06e5SJens Axboe * is speculative anyways, so missing one is ok.
16068cefc107SDavid Howells */
1607aadd06e5SJens Axboe if (!pipe_empty(pipe->head, pipe->tail))
1608aadd06e5SJens Axboe return 0;
1609aadd06e5SJens Axboe
161061e0d47cSMiklos Szeredi ret = 0;
1611aadd06e5SJens Axboe pipe_lock(pipe);
16128cefc107SDavid Howells
1613aadd06e5SJens Axboe while (pipe_empty(pipe->head, pipe->tail)) {
1614aadd06e5SJens Axboe if (signal_pending(current)) {
1615aadd06e5SJens Axboe ret = -ERESTARTSYS;
1616aadd06e5SJens Axboe break;
1617aadd06e5SJens Axboe }
1618aadd06e5SJens Axboe if (!pipe->writers)
1619aadd06e5SJens Axboe break;
1620aadd06e5SJens Axboe if (flags & SPLICE_F_NONBLOCK) {
1621aadd06e5SJens Axboe ret = -EAGAIN;
1622aadd06e5SJens Axboe break;
1623472e5b05SLinus Torvalds }
1624aadd06e5SJens Axboe pipe_wait_readable(pipe);
1625aadd06e5SJens Axboe }
162661e0d47cSMiklos Szeredi
1627aadd06e5SJens Axboe pipe_unlock(pipe);
1628aadd06e5SJens Axboe return ret;
1629aadd06e5SJens Axboe }
1630aadd06e5SJens Axboe
1631aadd06e5SJens Axboe /*
1632aadd06e5SJens Axboe * Make sure there's writeable room. Wait for room if we can, otherwise
1633aadd06e5SJens Axboe * return an appropriate error.
16347c77f0b3SMiklos Szeredi */
opipe_prep(struct pipe_inode_info * pipe,unsigned int flags)1635aadd06e5SJens Axboe static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1636aadd06e5SJens Axboe {
1637aadd06e5SJens Axboe int ret;
1638aadd06e5SJens Axboe
16398cefc107SDavid Howells /*
1640aadd06e5SJens Axboe * Check pipe occupancy without the inode lock first. This function
1641aadd06e5SJens Axboe * is speculative anyways, so missing one is ok.
1642566d1362STetsuo Handa */
1643aadd06e5SJens Axboe if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1644aadd06e5SJens Axboe return 0;
1645aadd06e5SJens Axboe
164661e0d47cSMiklos Szeredi ret = 0;
1647aadd06e5SJens Axboe pipe_lock(pipe);
16486718b6f8SDavid Howells
1649aadd06e5SJens Axboe while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
1650aadd06e5SJens Axboe if (!pipe->readers) {
1651aadd06e5SJens Axboe send_sig(SIGPIPE, current, 0);
1652aadd06e5SJens Axboe ret = -EPIPE;
1653aadd06e5SJens Axboe break;
1654aadd06e5SJens Axboe }
1655aadd06e5SJens Axboe if (flags & SPLICE_F_NONBLOCK) {
1656aadd06e5SJens Axboe ret = -EAGAIN;
1657aadd06e5SJens Axboe break;
1658aadd06e5SJens Axboe }
1659aadd06e5SJens Axboe if (signal_pending(current)) {
1660aadd06e5SJens Axboe ret = -ERESTARTSYS;
1661aadd06e5SJens Axboe break;
1662472e5b05SLinus Torvalds }
1663aadd06e5SJens Axboe pipe_wait_writable(pipe);
1664aadd06e5SJens Axboe }
166561e0d47cSMiklos Szeredi
1666aadd06e5SJens Axboe pipe_unlock(pipe);
1667aadd06e5SJens Axboe return ret;
1668aadd06e5SJens Axboe }
1669aadd06e5SJens Axboe
16707c77f0b3SMiklos Szeredi /*
16717c77f0b3SMiklos Szeredi * Splice contents of ipipe to opipe.
16727c77f0b3SMiklos Szeredi */
splice_pipe_to_pipe(struct pipe_inode_info * ipipe,struct pipe_inode_info * opipe,size_t len,unsigned int flags)16737c77f0b3SMiklos Szeredi static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
16747c77f0b3SMiklos Szeredi struct pipe_inode_info *opipe,
16757c77f0b3SMiklos Szeredi size_t len, unsigned int flags)
16767c77f0b3SMiklos Szeredi {
16778cefc107SDavid Howells struct pipe_buffer *ibuf, *obuf;
16788cefc107SDavid Howells unsigned int i_head, o_head;
16798cefc107SDavid Howells unsigned int i_tail, o_tail;
16808cefc107SDavid Howells unsigned int i_mask, o_mask;
16817c77f0b3SMiklos Szeredi int ret = 0;
16827c77f0b3SMiklos Szeredi bool input_wakeup = false;
16837c77f0b3SMiklos Szeredi
16847c77f0b3SMiklos Szeredi
16857c77f0b3SMiklos Szeredi retry:
16867c77f0b3SMiklos Szeredi ret = ipipe_prep(ipipe, flags);
16877c77f0b3SMiklos Szeredi if (ret)
16887c77f0b3SMiklos Szeredi return ret;
16897c77f0b3SMiklos Szeredi
16907c77f0b3SMiklos Szeredi ret = opipe_prep(opipe, flags);
16917c77f0b3SMiklos Szeredi if (ret)
16927c77f0b3SMiklos Szeredi return ret;
16937c77f0b3SMiklos Szeredi
16947c77f0b3SMiklos Szeredi /*
16957c77f0b3SMiklos Szeredi * Potential ABBA deadlock, work around it by ordering lock
16967c77f0b3SMiklos Szeredi * grabbing by pipe info address. Otherwise two different processes
16977c77f0b3SMiklos Szeredi * could deadlock (one doing tee from A -> B, the other from B -> A).
16987c77f0b3SMiklos Szeredi */
16997c77f0b3SMiklos Szeredi pipe_double_lock(ipipe, opipe);
17008cefc107SDavid Howells
17018cefc107SDavid Howells i_tail = ipipe->tail;
17028cefc107SDavid Howells i_mask = ipipe->ring_size - 1;
17038cefc107SDavid Howells o_head = opipe->head;
17048cefc107SDavid Howells o_mask = opipe->ring_size - 1;
17057c77f0b3SMiklos Szeredi
17068cefc107SDavid Howells do {
17078cefc107SDavid Howells size_t o_len;
17087c77f0b3SMiklos Szeredi
17097c77f0b3SMiklos Szeredi if (!opipe->readers) {
17107c77f0b3SMiklos Szeredi send_sig(SIGPIPE, current, 0);
17117c77f0b3SMiklos Szeredi if (!ret)
17127c77f0b3SMiklos Szeredi ret = -EPIPE;
17137c77f0b3SMiklos Szeredi break;
17147c77f0b3SMiklos Szeredi }
17158cefc107SDavid Howells
17168cefc107SDavid Howells i_head = ipipe->head;
17178cefc107SDavid Howells o_tail = opipe->tail;
17188cefc107SDavid Howells
17197c77f0b3SMiklos Szeredi if (pipe_empty(i_head, i_tail) && !ipipe->writers)
17207c77f0b3SMiklos Szeredi break;
17217c77f0b3SMiklos Szeredi
17227c77f0b3SMiklos Szeredi /*
17237c77f0b3SMiklos Szeredi * Cannot make any progress, because either the input
17247c77f0b3SMiklos Szeredi * pipe is empty or the output pipe is full.
17258cefc107SDavid Howells */
17266718b6f8SDavid Howells if (pipe_empty(i_head, i_tail) ||
17277c77f0b3SMiklos Szeredi pipe_full(o_head, o_tail, opipe->max_usage)) {
17287c77f0b3SMiklos Szeredi /* Already processed some buffers, break */
17297c77f0b3SMiklos Szeredi if (ret)
17307c77f0b3SMiklos Szeredi break;
17317c77f0b3SMiklos Szeredi
17327c77f0b3SMiklos Szeredi if (flags & SPLICE_F_NONBLOCK) {
17337c77f0b3SMiklos Szeredi ret = -EAGAIN;
17347c77f0b3SMiklos Szeredi break;
17357c77f0b3SMiklos Szeredi }
17367c77f0b3SMiklos Szeredi
17377c77f0b3SMiklos Szeredi /*
17387c77f0b3SMiklos Szeredi * We raced with another reader/writer and haven't
17397c77f0b3SMiklos Szeredi * managed to process any buffers. A zero return
17407c77f0b3SMiklos Szeredi * value means EOF, so retry instead.
17417c77f0b3SMiklos Szeredi */
17427c77f0b3SMiklos Szeredi pipe_unlock(ipipe);
17437c77f0b3SMiklos Szeredi pipe_unlock(opipe);
17447c77f0b3SMiklos Szeredi goto retry;
17457c77f0b3SMiklos Szeredi }
17468cefc107SDavid Howells
17478cefc107SDavid Howells ibuf = &ipipe->bufs[i_tail & i_mask];
17487c77f0b3SMiklos Szeredi obuf = &opipe->bufs[o_head & o_mask];
17497c77f0b3SMiklos Szeredi
17507c77f0b3SMiklos Szeredi if (len >= ibuf->len) {
17517c77f0b3SMiklos Szeredi /*
17527c77f0b3SMiklos Szeredi * Simply move the whole buffer from ipipe to opipe
17537c77f0b3SMiklos Szeredi */
17547c77f0b3SMiklos Szeredi *obuf = *ibuf;
17558cefc107SDavid Howells ibuf->ops = NULL;
17568cefc107SDavid Howells i_tail++;
17577c77f0b3SMiklos Szeredi ipipe->tail = i_tail;
17588cefc107SDavid Howells input_wakeup = true;
17598cefc107SDavid Howells o_len = obuf->len;
17608cefc107SDavid Howells o_head++;
17617c77f0b3SMiklos Szeredi opipe->head = o_head;
17627c77f0b3SMiklos Szeredi } else {
17637c77f0b3SMiklos Szeredi /*
17647c77f0b3SMiklos Szeredi * Get a reference to this pipe buffer,
17657c77f0b3SMiklos Szeredi * so we can copy the contents over.
176615fab63eSMatthew Wilcox */
176715fab63eSMatthew Wilcox if (!pipe_buf_get(ipipe, ibuf)) {
176815fab63eSMatthew Wilcox if (ret == 0)
176915fab63eSMatthew Wilcox ret = -EFAULT;
177015fab63eSMatthew Wilcox break;
17717c77f0b3SMiklos Szeredi }
17727c77f0b3SMiklos Szeredi *obuf = *ibuf;
17737c77f0b3SMiklos Szeredi
1774f6dd9755SChristoph Hellwig /*
17757c77f0b3SMiklos Szeredi * Don't inherit the gift and merge flags, we need to
17767c77f0b3SMiklos Szeredi * prevent multiple steals of this page.
17777c77f0b3SMiklos Szeredi */
1778f6dd9755SChristoph Hellwig obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1779a0ce2f0aSJann Horn obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
17807c77f0b3SMiklos Szeredi
17818cefc107SDavid Howells obuf->len = len;
17828cefc107SDavid Howells ibuf->offset += len;
17838cefc107SDavid Howells ibuf->len -= len;
17848cefc107SDavid Howells o_len = len;
17858cefc107SDavid Howells o_head++;
17867c77f0b3SMiklos Szeredi opipe->head = o_head;
17878cefc107SDavid Howells }
17888cefc107SDavid Howells ret += o_len;
17897c77f0b3SMiklos Szeredi len -= o_len;
17907c77f0b3SMiklos Szeredi } while (len);
17917c77f0b3SMiklos Szeredi
17927c77f0b3SMiklos Szeredi pipe_unlock(ipipe);
17937c77f0b3SMiklos Szeredi pipe_unlock(opipe);
17947c77f0b3SMiklos Szeredi
17957c77f0b3SMiklos Szeredi /*
17967c77f0b3SMiklos Szeredi * If we put data in the output pipe, wakeup any potential readers.
1797825cdcb1SNamhyung Kim */
1798825cdcb1SNamhyung Kim if (ret > 0)
1799825cdcb1SNamhyung Kim wakeup_pipe_readers(opipe);
18007c77f0b3SMiklos Szeredi
18017c77f0b3SMiklos Szeredi if (input_wakeup)
18027c77f0b3SMiklos Szeredi wakeup_pipe_writers(ipipe);
18037c77f0b3SMiklos Szeredi
18047c77f0b3SMiklos Szeredi return ret;
18057c77f0b3SMiklos Szeredi }
18067c77f0b3SMiklos Szeredi
180770524490SJens Axboe /*
180870524490SJens Axboe * Link contents of ipipe to opipe.
180970524490SJens Axboe */
link_pipe(struct pipe_inode_info * ipipe,struct pipe_inode_info * opipe,size_t len,unsigned int flags)181070524490SJens Axboe static int link_pipe(struct pipe_inode_info *ipipe,
181170524490SJens Axboe struct pipe_inode_info *opipe,
181270524490SJens Axboe size_t len, unsigned int flags)
181370524490SJens Axboe {
18148cefc107SDavid Howells struct pipe_buffer *ibuf, *obuf;
18158cefc107SDavid Howells unsigned int i_head, o_head;
18168cefc107SDavid Howells unsigned int i_tail, o_tail;
18178cefc107SDavid Howells unsigned int i_mask, o_mask;
181870524490SJens Axboe int ret = 0;
181970524490SJens Axboe
182070524490SJens Axboe /*
182161e0d47cSMiklos Szeredi * Potential ABBA deadlock, work around it by ordering lock
182270524490SJens Axboe * grabbing by pipe info address. Otherwise two different processes
182370524490SJens Axboe * could deadlock (one doing tee from A -> B, the other from B -> A).
182461e0d47cSMiklos Szeredi */
182570524490SJens Axboe pipe_double_lock(ipipe, opipe);
18268cefc107SDavid Howells
18278cefc107SDavid Howells i_tail = ipipe->tail;
18288cefc107SDavid Howells i_mask = ipipe->ring_size - 1;
18298cefc107SDavid Howells o_head = opipe->head;
18308cefc107SDavid Howells o_mask = opipe->ring_size - 1;
1831aadd06e5SJens Axboe
183270524490SJens Axboe do {
183370524490SJens Axboe if (!opipe->readers) {
183470524490SJens Axboe send_sig(SIGPIPE, current, 0);
183570524490SJens Axboe if (!ret)
183670524490SJens Axboe ret = -EPIPE;
183770524490SJens Axboe break;
183870524490SJens Axboe }
18398cefc107SDavid Howells
18408cefc107SDavid Howells i_head = ipipe->head;
18418cefc107SDavid Howells o_tail = opipe->tail;
184270524490SJens Axboe
18438cefc107SDavid Howells /*
1844aadd06e5SJens Axboe * If we have iterated all input buffers or run out of
184570524490SJens Axboe * output room, break.
18468cefc107SDavid Howells */
18476718b6f8SDavid Howells if (pipe_empty(i_head, i_tail) ||
1848aadd06e5SJens Axboe pipe_full(o_head, o_tail, opipe->max_usage))
1849aadd06e5SJens Axboe break;
18508cefc107SDavid Howells
18518cefc107SDavid Howells ibuf = &ipipe->bufs[i_tail & i_mask];
185270524490SJens Axboe obuf = &opipe->bufs[o_head & o_mask];
185370524490SJens Axboe
185470524490SJens Axboe /*
185570524490SJens Axboe * Get a reference to this pipe buffer,
185670524490SJens Axboe * so we can copy the contents over.
185715fab63eSMatthew Wilcox */
185815fab63eSMatthew Wilcox if (!pipe_buf_get(ipipe, ibuf)) {
185915fab63eSMatthew Wilcox if (ret == 0)
186015fab63eSMatthew Wilcox ret = -EFAULT;
186115fab63eSMatthew Wilcox break;
186270524490SJens Axboe }
186370524490SJens Axboe
186470524490SJens Axboe *obuf = *ibuf;
18657afa6fd0SJens Axboe
1866f6dd9755SChristoph Hellwig /*
1867f6dd9755SChristoph Hellwig * Don't inherit the gift and merge flag, we need to prevent
18687afa6fd0SJens Axboe * multiple steals of this page.
18697afa6fd0SJens Axboe */
1870f6dd9755SChristoph Hellwig obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1871a0ce2f0aSJann Horn obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
187270524490SJens Axboe
187370524490SJens Axboe if (obuf->len > len)
187470524490SJens Axboe obuf->len = len;
187570524490SJens Axboe ret += obuf->len;
18768cefc107SDavid Howells len -= obuf->len;
18778cefc107SDavid Howells
18788cefc107SDavid Howells o_head++;
18798cefc107SDavid Howells opipe->head = o_head;
1880aadd06e5SJens Axboe i_tail++;
188170524490SJens Axboe } while (len);
188261e0d47cSMiklos Szeredi
188361e0d47cSMiklos Szeredi pipe_unlock(ipipe);
188470524490SJens Axboe pipe_unlock(opipe);
1885aadd06e5SJens Axboe
1886aadd06e5SJens Axboe /*
1887aadd06e5SJens Axboe * If we put data in the output pipe, wakeup any potential readers.
1888825cdcb1SNamhyung Kim */
1889825cdcb1SNamhyung Kim if (ret > 0)
189070524490SJens Axboe wakeup_pipe_readers(opipe);
189170524490SJens Axboe
189270524490SJens Axboe return ret;
189370524490SJens Axboe }
189470524490SJens Axboe
189570524490SJens Axboe /*
189670524490SJens Axboe * This is a tee(1) implementation that works on pipes. It doesn't copy
189770524490SJens Axboe * any data, it simply references the 'in' pages on the 'out' pipe.
189870524490SJens Axboe * The 'flags' used are the SPLICE_F_* variants, currently the only
189970524490SJens Axboe * applicable one is SPLICE_F_NONBLOCK.
19009dafdfc2SPavel Begunkov */
do_tee(struct file * in,struct file * out,size_t len,unsigned int flags)190170524490SJens Axboe long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags)
1902c73be61cSDavid Howells {
1903c73be61cSDavid Howells struct pipe_inode_info *ipipe = get_pipe_info(in, true);
1904aadd06e5SJens Axboe struct pipe_inode_info *opipe = get_pipe_info(out, true);
190570524490SJens Axboe int ret = -EINVAL;
190690da2e3fSPavel Begunkov
190790da2e3fSPavel Begunkov if (unlikely(!(in->f_mode & FMODE_READ) ||
190890da2e3fSPavel Begunkov !(out->f_mode & FMODE_WRITE)))
190990da2e3fSPavel Begunkov return -EBADF;
191070524490SJens Axboe
1911aadd06e5SJens Axboe /*
1912aadd06e5SJens Axboe * Duplicate the contents of ipipe to opipe without actually
191370524490SJens Axboe * copying the data.
1914aadd06e5SJens Axboe */
1915ee5e0011SSlavomir Kaslev if (ipipe && opipe && ipipe != opipe) {
1916ee5e0011SSlavomir Kaslev if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1917ee5e0011SSlavomir Kaslev flags |= SPLICE_F_NONBLOCK;
1918aadd06e5SJens Axboe
1919aadd06e5SJens Axboe /*
1920aadd06e5SJens Axboe * Keep going, unless we encounter an error. The ipipe/opipe
1921aadd06e5SJens Axboe * ordering doesn't really matter.
19227c77f0b3SMiklos Szeredi */
1923aadd06e5SJens Axboe ret = ipipe_prep(ipipe, flags);
19247c77f0b3SMiklos Szeredi if (!ret) {
192502cf01aeSJens Axboe ret = opipe_prep(opipe, flags);
1926aadd06e5SJens Axboe if (!ret)
1927aadd06e5SJens Axboe ret = link_pipe(ipipe, opipe, len, flags);
1928aadd06e5SJens Axboe }
192970524490SJens Axboe }
1930576d498eSAhelenia Ziemiańska
1931576d498eSAhelenia Ziemiańska if (ret > 0) {
1932576d498eSAhelenia Ziemiańska fsnotify_access(in);
1933576d498eSAhelenia Ziemiańska fsnotify_modify(out);
1934576d498eSAhelenia Ziemiańska }
1935aadd06e5SJens Axboe
193670524490SJens Axboe return ret;
193770524490SJens Axboe }
1938836f92adSHeiko Carstens
SYSCALL_DEFINE4(tee,int,fdin,int,fdout,size_t,len,unsigned int,flags)193970524490SJens Axboe SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
194090da2e3fSPavel Begunkov {
19412903ff01SAl Viro struct fd in, out;
194270524490SJens Axboe int error;
19433d6ea290SAl Viro
19443d6ea290SAl Viro if (unlikely(flags & ~SPLICE_F_ALL))
19453d6ea290SAl Viro return -EINVAL;
194670524490SJens Axboe
194770524490SJens Axboe if (unlikely(!len))
194870524490SJens Axboe return 0;
194970524490SJens Axboe
19502903ff01SAl Viro error = -EBADF;
19512903ff01SAl Viro in = fdget(fdin);
195290da2e3fSPavel Begunkov if (in.file) {
19532903ff01SAl Viro out = fdget(fdout);
195490da2e3fSPavel Begunkov if (out.file) {
19552903ff01SAl Viro error = do_tee(in.file, out.file, len, flags);
195670524490SJens Axboe fdput(out);
19572903ff01SAl Viro }
195870524490SJens Axboe fdput(in);
195970524490SJens Axboe }
196070524490SJens Axboe
196170524490SJens Axboe return error;
1962 }
1963