1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * linux/fs/read_write.c
41da177e4SLinus Torvalds *
51da177e4SLinus Torvalds * Copyright (C) 1991, 1992 Linus Torvalds
61da177e4SLinus Torvalds */
71da177e4SLinus Torvalds
81da177e4SLinus Torvalds #include <linux/slab.h>
91da177e4SLinus Torvalds #include <linux/stat.h>
10b12fb7f4SIngo Molnar #include <linux/sched/xacct.h>
111da177e4SLinus Torvalds #include <linux/fcntl.h>
121da177e4SLinus Torvalds #include <linux/file.h>
131da177e4SLinus Torvalds #include <linux/uio.h>
140eeca283SRobert Love #include <linux/fsnotify.h>
151da177e4SLinus Torvalds #include <linux/security.h>
16630d9c47SPaul Gortmaker #include <linux/export.h>
171da177e4SLinus Torvalds #include <linux/syscalls.h>
18e28cc715SLinus Torvalds #include <linux/pagemap.h>
19d6b29d7cSJens Axboe #include <linux/splice.h>
20561c6731SAl Viro #include <linux/compat.h>
2129732938SZach Brown #include <linux/mount.h>
222feb55f8SWouter van Kesteren #include <linux/fs.h>
2306ae43f3SAl Viro #include "internal.h"
241da177e4SLinus Torvalds
257c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
261da177e4SLinus Torvalds #include <asm/unistd.h>
271da177e4SLinus Torvalds
284b6f5d20SArjan van de Ven const struct file_operations generic_ro_fops = {
291da177e4SLinus Torvalds .llseek = generic_file_llseek,
30aad4f8bbSAl Viro .read_iter = generic_file_read_iter,
311da177e4SLinus Torvalds .mmap = generic_file_readonly_mmap,
322cb1e089SDavid Howells .splice_read = filemap_splice_read,
331da177e4SLinus Torvalds };
341da177e4SLinus Torvalds
351da177e4SLinus Torvalds EXPORT_SYMBOL(generic_ro_fops);
361da177e4SLinus Torvalds
unsigned_offsets(struct file * file)37ddef7ed2SChristoph Hellwig static inline bool unsigned_offsets(struct file *file)
384a3956c7SKAMEZAWA Hiroyuki {
39cccb5a1eSAl Viro return file->f_mode & FMODE_UNSIGNED_OFFSET;
404a3956c7SKAMEZAWA Hiroyuki }
414a3956c7SKAMEZAWA Hiroyuki
4246a1c2c7SJie Liu /**
4346a1c2c7SJie Liu * vfs_setpos - update the file offset for lseek
4446a1c2c7SJie Liu * @file: file structure in question
4546a1c2c7SJie Liu * @offset: file offset to seek to
4646a1c2c7SJie Liu * @maxsize: maximum file size
4746a1c2c7SJie Liu *
4846a1c2c7SJie Liu * This is a low-level filesystem helper for updating the file offset to
4946a1c2c7SJie Liu * the value specified by @offset if the given offset is valid and it is
5046a1c2c7SJie Liu * not equal to the current file offset.
5146a1c2c7SJie Liu *
5246a1c2c7SJie Liu * Return the specified offset on success and -EINVAL on invalid offset.
5346a1c2c7SJie Liu */
vfs_setpos(struct file * file,loff_t offset,loff_t maxsize)5446a1c2c7SJie Liu loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize)
55ef3d0fd2SAndi Kleen {
56ef3d0fd2SAndi Kleen if (offset < 0 && !unsigned_offsets(file))
57ef3d0fd2SAndi Kleen return -EINVAL;
58ef3d0fd2SAndi Kleen if (offset > maxsize)
59ef3d0fd2SAndi Kleen return -EINVAL;
60ef3d0fd2SAndi Kleen
61ef3d0fd2SAndi Kleen if (offset != file->f_pos) {
62ef3d0fd2SAndi Kleen file->f_pos = offset;
63ef3d0fd2SAndi Kleen file->f_version = 0;
64ef3d0fd2SAndi Kleen }
65ef3d0fd2SAndi Kleen return offset;
66ef3d0fd2SAndi Kleen }
6746a1c2c7SJie Liu EXPORT_SYMBOL(vfs_setpos);
68ef3d0fd2SAndi Kleen
693a8cff4fSChristoph Hellwig /**
705760495aSAndi Kleen * generic_file_llseek_size - generic llseek implementation for regular files
713a8cff4fSChristoph Hellwig * @file: file structure to seek on
723a8cff4fSChristoph Hellwig * @offset: file offset to seek to
73965c8e59SAndrew Morton * @whence: type of seek
74*89cbd4c0SYang Li * @maxsize: max size of this file in file system
75e8b96eb5SEric Sandeen * @eof: offset used for SEEK_END position
763a8cff4fSChristoph Hellwig *
775760495aSAndi Kleen * This is a variant of generic_file_llseek that allows passing in a custom
78e8b96eb5SEric Sandeen * maximum file size and a custom EOF position, for e.g. hashed directories
79ef3d0fd2SAndi Kleen *
80ef3d0fd2SAndi Kleen * Synchronization:
815760495aSAndi Kleen * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms)
82ef3d0fd2SAndi Kleen * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes.
83ef3d0fd2SAndi Kleen * read/writes behave like SEEK_SET against seeks.
843a8cff4fSChristoph Hellwig */
859465efc9SAndi Kleen loff_t
generic_file_llseek_size(struct file * file,loff_t offset,int whence,loff_t maxsize,loff_t eof)86965c8e59SAndrew Morton generic_file_llseek_size(struct file *file, loff_t offset, int whence,
87e8b96eb5SEric Sandeen loff_t maxsize, loff_t eof)
881da177e4SLinus Torvalds {
89965c8e59SAndrew Morton switch (whence) {
907b8e8924SChris Snook case SEEK_END:
91e8b96eb5SEric Sandeen offset += eof;
921da177e4SLinus Torvalds break;
937b8e8924SChris Snook case SEEK_CUR:
945b6f1eb9SAlain Knaff /*
955b6f1eb9SAlain Knaff * Here we special-case the lseek(fd, 0, SEEK_CUR)
965b6f1eb9SAlain Knaff * position-querying operation. Avoid rewriting the "same"
975b6f1eb9SAlain Knaff * f_pos value back to the file because a concurrent read(),
985b6f1eb9SAlain Knaff * write() or lseek() might have altered it
995b6f1eb9SAlain Knaff */
1005b6f1eb9SAlain Knaff if (offset == 0)
1015b6f1eb9SAlain Knaff return file->f_pos;
102ef3d0fd2SAndi Kleen /*
103ef3d0fd2SAndi Kleen * f_lock protects against read/modify/write race with other
104ef3d0fd2SAndi Kleen * SEEK_CURs. Note that parallel writes and reads behave
105ef3d0fd2SAndi Kleen * like SEEK_SET.
106ef3d0fd2SAndi Kleen */
107ef3d0fd2SAndi Kleen spin_lock(&file->f_lock);
10846a1c2c7SJie Liu offset = vfs_setpos(file, file->f_pos + offset, maxsize);
109ef3d0fd2SAndi Kleen spin_unlock(&file->f_lock);
110ef3d0fd2SAndi Kleen return offset;
111982d8165SJosef Bacik case SEEK_DATA:
112982d8165SJosef Bacik /*
113982d8165SJosef Bacik * In the generic case the entire file is data, so as long as
114982d8165SJosef Bacik * offset isn't at the end of the file then the offset is data.
115982d8165SJosef Bacik */
116fc46820bSAndreas Gruenbacher if ((unsigned long long)offset >= eof)
117982d8165SJosef Bacik return -ENXIO;
118982d8165SJosef Bacik break;
119982d8165SJosef Bacik case SEEK_HOLE:
120982d8165SJosef Bacik /*
121982d8165SJosef Bacik * There is a virtual hole at the end of the file, so as long as
122982d8165SJosef Bacik * offset isn't i_size or larger, return i_size.
123982d8165SJosef Bacik */
124fc46820bSAndreas Gruenbacher if ((unsigned long long)offset >= eof)
125982d8165SJosef Bacik return -ENXIO;
126e8b96eb5SEric Sandeen offset = eof;
127982d8165SJosef Bacik break;
1281da177e4SLinus Torvalds }
1293a8cff4fSChristoph Hellwig
13046a1c2c7SJie Liu return vfs_setpos(file, offset, maxsize);
1315760495aSAndi Kleen }
1325760495aSAndi Kleen EXPORT_SYMBOL(generic_file_llseek_size);
1335760495aSAndi Kleen
1345760495aSAndi Kleen /**
1355760495aSAndi Kleen * generic_file_llseek - generic llseek implementation for regular files
1365760495aSAndi Kleen * @file: file structure to seek on
1375760495aSAndi Kleen * @offset: file offset to seek to
138965c8e59SAndrew Morton * @whence: type of seek
1395760495aSAndi Kleen *
1405760495aSAndi Kleen * This is a generic implemenation of ->llseek useable for all normal local
1415760495aSAndi Kleen * filesystems. It just updates the file offset to the value specified by
142546ae2d2SMing Lei * @offset and @whence.
1435760495aSAndi Kleen */
generic_file_llseek(struct file * file,loff_t offset,int whence)144965c8e59SAndrew Morton loff_t generic_file_llseek(struct file *file, loff_t offset, int whence)
1455760495aSAndi Kleen {
1465760495aSAndi Kleen struct inode *inode = file->f_mapping->host;
1475760495aSAndi Kleen
148965c8e59SAndrew Morton return generic_file_llseek_size(file, offset, whence,
149e8b96eb5SEric Sandeen inode->i_sb->s_maxbytes,
150e8b96eb5SEric Sandeen i_size_read(inode));
1511da177e4SLinus Torvalds }
1529465efc9SAndi Kleen EXPORT_SYMBOL(generic_file_llseek);
1531da177e4SLinus Torvalds
154ae6afc3fSjan Blunck /**
1551bf9d14dSAl Viro * fixed_size_llseek - llseek implementation for fixed-sized devices
1561bf9d14dSAl Viro * @file: file structure to seek on
1571bf9d14dSAl Viro * @offset: file offset to seek to
1581bf9d14dSAl Viro * @whence: type of seek
1591bf9d14dSAl Viro * @size: size of the file
1601bf9d14dSAl Viro *
1611bf9d14dSAl Viro */
fixed_size_llseek(struct file * file,loff_t offset,int whence,loff_t size)1621bf9d14dSAl Viro loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size)
1631bf9d14dSAl Viro {
1641bf9d14dSAl Viro switch (whence) {
1651bf9d14dSAl Viro case SEEK_SET: case SEEK_CUR: case SEEK_END:
1661bf9d14dSAl Viro return generic_file_llseek_size(file, offset, whence,
1671bf9d14dSAl Viro size, size);
1681bf9d14dSAl Viro default:
1691bf9d14dSAl Viro return -EINVAL;
1701bf9d14dSAl Viro }
1711bf9d14dSAl Viro }
1721bf9d14dSAl Viro EXPORT_SYMBOL(fixed_size_llseek);
1731bf9d14dSAl Viro
1741bf9d14dSAl Viro /**
175b25472f9SAl Viro * no_seek_end_llseek - llseek implementation for fixed-sized devices
176b25472f9SAl Viro * @file: file structure to seek on
177b25472f9SAl Viro * @offset: file offset to seek to
178b25472f9SAl Viro * @whence: type of seek
179b25472f9SAl Viro *
180b25472f9SAl Viro */
no_seek_end_llseek(struct file * file,loff_t offset,int whence)181b25472f9SAl Viro loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence)
182b25472f9SAl Viro {
183b25472f9SAl Viro switch (whence) {
184b25472f9SAl Viro case SEEK_SET: case SEEK_CUR:
185b25472f9SAl Viro return generic_file_llseek_size(file, offset, whence,
1862feb55f8SWouter van Kesteren OFFSET_MAX, 0);
187b25472f9SAl Viro default:
188b25472f9SAl Viro return -EINVAL;
189b25472f9SAl Viro }
190b25472f9SAl Viro }
191b25472f9SAl Viro EXPORT_SYMBOL(no_seek_end_llseek);
192b25472f9SAl Viro
193b25472f9SAl Viro /**
194b25472f9SAl Viro * no_seek_end_llseek_size - llseek implementation for fixed-sized devices
195b25472f9SAl Viro * @file: file structure to seek on
196b25472f9SAl Viro * @offset: file offset to seek to
197b25472f9SAl Viro * @whence: type of seek
198b25472f9SAl Viro * @size: maximal offset allowed
199b25472f9SAl Viro *
200b25472f9SAl Viro */
no_seek_end_llseek_size(struct file * file,loff_t offset,int whence,loff_t size)201b25472f9SAl Viro loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size)
202b25472f9SAl Viro {
203b25472f9SAl Viro switch (whence) {
204b25472f9SAl Viro case SEEK_SET: case SEEK_CUR:
205b25472f9SAl Viro return generic_file_llseek_size(file, offset, whence,
206b25472f9SAl Viro size, 0);
207b25472f9SAl Viro default:
208b25472f9SAl Viro return -EINVAL;
209b25472f9SAl Viro }
210b25472f9SAl Viro }
211b25472f9SAl Viro EXPORT_SYMBOL(no_seek_end_llseek_size);
212b25472f9SAl Viro
213b25472f9SAl Viro /**
214ae6afc3fSjan Blunck * noop_llseek - No Operation Performed llseek implementation
215ae6afc3fSjan Blunck * @file: file structure to seek on
216ae6afc3fSjan Blunck * @offset: file offset to seek to
217965c8e59SAndrew Morton * @whence: type of seek
218ae6afc3fSjan Blunck *
219ae6afc3fSjan Blunck * This is an implementation of ->llseek useable for the rare special case when
220ae6afc3fSjan Blunck * userspace expects the seek to succeed but the (device) file is actually not
221ae6afc3fSjan Blunck * able to perform the seek. In this case you use noop_llseek() instead of
222ae6afc3fSjan Blunck * falling back to the default implementation of ->llseek.
223ae6afc3fSjan Blunck */
noop_llseek(struct file * file,loff_t offset,int whence)224965c8e59SAndrew Morton loff_t noop_llseek(struct file *file, loff_t offset, int whence)
225ae6afc3fSjan Blunck {
226ae6afc3fSjan Blunck return file->f_pos;
227ae6afc3fSjan Blunck }
228ae6afc3fSjan Blunck EXPORT_SYMBOL(noop_llseek);
229ae6afc3fSjan Blunck
default_llseek(struct file * file,loff_t offset,int whence)230965c8e59SAndrew Morton loff_t default_llseek(struct file *file, loff_t offset, int whence)
2311da177e4SLinus Torvalds {
232496ad9aaSAl Viro struct inode *inode = file_inode(file);
23316abef0eSDavid Sterba loff_t retval;
2341da177e4SLinus Torvalds
2355955102cSAl Viro inode_lock(inode);
236965c8e59SAndrew Morton switch (whence) {
2377b8e8924SChris Snook case SEEK_END:
238982d8165SJosef Bacik offset += i_size_read(inode);
2391da177e4SLinus Torvalds break;
2407b8e8924SChris Snook case SEEK_CUR:
2415b6f1eb9SAlain Knaff if (offset == 0) {
2425b6f1eb9SAlain Knaff retval = file->f_pos;
2435b6f1eb9SAlain Knaff goto out;
2445b6f1eb9SAlain Knaff }
2451da177e4SLinus Torvalds offset += file->f_pos;
246982d8165SJosef Bacik break;
247982d8165SJosef Bacik case SEEK_DATA:
248982d8165SJosef Bacik /*
249982d8165SJosef Bacik * In the generic case the entire file is data, so as
250982d8165SJosef Bacik * long as offset isn't at the end of the file then the
251982d8165SJosef Bacik * offset is data.
252982d8165SJosef Bacik */
253bacb2d81SDan Carpenter if (offset >= inode->i_size) {
254bacb2d81SDan Carpenter retval = -ENXIO;
255bacb2d81SDan Carpenter goto out;
256bacb2d81SDan Carpenter }
257982d8165SJosef Bacik break;
258982d8165SJosef Bacik case SEEK_HOLE:
259982d8165SJosef Bacik /*
260982d8165SJosef Bacik * There is a virtual hole at the end of the file, so
261982d8165SJosef Bacik * as long as offset isn't i_size or larger, return
262982d8165SJosef Bacik * i_size.
263982d8165SJosef Bacik */
264bacb2d81SDan Carpenter if (offset >= inode->i_size) {
265bacb2d81SDan Carpenter retval = -ENXIO;
266bacb2d81SDan Carpenter goto out;
267bacb2d81SDan Carpenter }
268982d8165SJosef Bacik offset = inode->i_size;
269982d8165SJosef Bacik break;
2701da177e4SLinus Torvalds }
2711da177e4SLinus Torvalds retval = -EINVAL;
272cccb5a1eSAl Viro if (offset >= 0 || unsigned_offsets(file)) {
2731da177e4SLinus Torvalds if (offset != file->f_pos) {
2741da177e4SLinus Torvalds file->f_pos = offset;
2751da177e4SLinus Torvalds file->f_version = 0;
2761da177e4SLinus Torvalds }
2771da177e4SLinus Torvalds retval = offset;
2781da177e4SLinus Torvalds }
2795b6f1eb9SAlain Knaff out:
2805955102cSAl Viro inode_unlock(inode);
2811da177e4SLinus Torvalds return retval;
2821da177e4SLinus Torvalds }
2831da177e4SLinus Torvalds EXPORT_SYMBOL(default_llseek);
2841da177e4SLinus Torvalds
vfs_llseek(struct file * file,loff_t offset,int whence)285965c8e59SAndrew Morton loff_t vfs_llseek(struct file *file, loff_t offset, int whence)
2861da177e4SLinus Torvalds {
2874e3299eaSJason A. Donenfeld if (!(file->f_mode & FMODE_LSEEK))
2884e3299eaSJason A. Donenfeld return -ESPIPE;
2894e3299eaSJason A. Donenfeld return file->f_op->llseek(file, offset, whence);
2901da177e4SLinus Torvalds }
2911da177e4SLinus Torvalds EXPORT_SYMBOL(vfs_llseek);
2921da177e4SLinus Torvalds
ksys_lseek(unsigned int fd,off_t offset,unsigned int whence)293bef17329SChristoph Hellwig static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence)
2941da177e4SLinus Torvalds {
2951da177e4SLinus Torvalds off_t retval;
2969c225f26SLinus Torvalds struct fd f = fdget_pos(fd);
2972903ff01SAl Viro if (!f.file)
2982903ff01SAl Viro return -EBADF;
2991da177e4SLinus Torvalds
3001da177e4SLinus Torvalds retval = -EINVAL;
301965c8e59SAndrew Morton if (whence <= SEEK_MAX) {
302965c8e59SAndrew Morton loff_t res = vfs_llseek(f.file, offset, whence);
3031da177e4SLinus Torvalds retval = res;
3041da177e4SLinus Torvalds if (res != (loff_t)retval)
3051da177e4SLinus Torvalds retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */
3061da177e4SLinus Torvalds }
3079c225f26SLinus Torvalds fdput_pos(f);
3081da177e4SLinus Torvalds return retval;
3091da177e4SLinus Torvalds }
3101da177e4SLinus Torvalds
SYSCALL_DEFINE3(lseek,unsigned int,fd,off_t,offset,unsigned int,whence)31176847e43SDominik Brodowski SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence)
31276847e43SDominik Brodowski {
31376847e43SDominik Brodowski return ksys_lseek(fd, offset, whence);
31476847e43SDominik Brodowski }
31576847e43SDominik Brodowski
316561c6731SAl Viro #ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE3(lseek,unsigned int,fd,compat_off_t,offset,unsigned int,whence)317561c6731SAl Viro COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence)
318561c6731SAl Viro {
31976847e43SDominik Brodowski return ksys_lseek(fd, offset, whence);
320561c6731SAl Viro }
321561c6731SAl Viro #endif
322561c6731SAl Viro
3239e62ccecSMichal Suchanek #if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \
3249e62ccecSMichal Suchanek defined(__ARCH_WANT_SYS_LLSEEK)
SYSCALL_DEFINE5(llseek,unsigned int,fd,unsigned long,offset_high,unsigned long,offset_low,loff_t __user *,result,unsigned int,whence)325003d7ab4SHeiko Carstens SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high,
326003d7ab4SHeiko Carstens unsigned long, offset_low, loff_t __user *, result,
327965c8e59SAndrew Morton unsigned int, whence)
3281da177e4SLinus Torvalds {
3291da177e4SLinus Torvalds int retval;
330d7a15f8dSEric Biggers struct fd f = fdget_pos(fd);
3311da177e4SLinus Torvalds loff_t offset;
3321da177e4SLinus Torvalds
3332903ff01SAl Viro if (!f.file)
3342903ff01SAl Viro return -EBADF;
3351da177e4SLinus Torvalds
3361da177e4SLinus Torvalds retval = -EINVAL;
337965c8e59SAndrew Morton if (whence > SEEK_MAX)
3381da177e4SLinus Torvalds goto out_putf;
3391da177e4SLinus Torvalds
3402903ff01SAl Viro offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low,
341965c8e59SAndrew Morton whence);
3421da177e4SLinus Torvalds
3431da177e4SLinus Torvalds retval = (int)offset;
3441da177e4SLinus Torvalds if (offset >= 0) {
3451da177e4SLinus Torvalds retval = -EFAULT;
3461da177e4SLinus Torvalds if (!copy_to_user(result, &offset, sizeof(offset)))
3471da177e4SLinus Torvalds retval = 0;
3481da177e4SLinus Torvalds }
3491da177e4SLinus Torvalds out_putf:
350d7a15f8dSEric Biggers fdput_pos(f);
3511da177e4SLinus Torvalds return retval;
3521da177e4SLinus Torvalds }
3531da177e4SLinus Torvalds #endif
3541da177e4SLinus Torvalds
rw_verify_area(int read_write,struct file * file,const loff_t * ppos,size_t count)35568d70d03SAl Viro int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count)
3561da177e4SLinus Torvalds {
357e28cc715SLinus Torvalds if (unlikely((ssize_t) count < 0))
3582949e842SLukas Bulwahn return -EINVAL;
359438ab720SKirill Smelkov
360438ab720SKirill Smelkov if (ppos) {
361438ab720SKirill Smelkov loff_t pos = *ppos;
362438ab720SKirill Smelkov
363cccb5a1eSAl Viro if (unlikely(pos < 0)) {
364cccb5a1eSAl Viro if (!unsigned_offsets(file))
3652949e842SLukas Bulwahn return -EINVAL;
366cccb5a1eSAl Viro if (count >= -pos) /* both values are in 0..LLONG_MAX */
367cccb5a1eSAl Viro return -EOVERFLOW;
368cccb5a1eSAl Viro } else if (unlikely((loff_t) (pos + count) < 0)) {
369cccb5a1eSAl Viro if (!unsigned_offsets(file))
3702949e842SLukas Bulwahn return -EINVAL;
3714a3956c7SKAMEZAWA Hiroyuki }
372438ab720SKirill Smelkov }
373438ab720SKirill Smelkov
374bc61384dSAl Viro return security_file_permission(file,
375c43e259cSJames Morris read_write == READ ? MAY_READ : MAY_WRITE);
3761da177e4SLinus Torvalds }
37787112933SOmar Sandoval EXPORT_SYMBOL(rw_verify_area);
3781da177e4SLinus Torvalds
new_sync_read(struct file * filp,char __user * buf,size_t len,loff_t * ppos)3795d5d5689SAl Viro static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
380293bc982SAl Viro {
381293bc982SAl Viro struct kiocb kiocb;
382293bc982SAl Viro struct iov_iter iter;
383293bc982SAl Viro ssize_t ret;
384293bc982SAl Viro
385293bc982SAl Viro init_sync_kiocb(&kiocb, filp);
386438ab720SKirill Smelkov kiocb.ki_pos = (ppos ? *ppos : 0);
387de4eda9dSAl Viro iov_iter_ubuf(&iter, ITER_DEST, buf, len);
388293bc982SAl Viro
389bb7462b6SMiklos Szeredi ret = call_read_iter(filp, &kiocb, &iter);
390599bd19bSChristoph Hellwig BUG_ON(ret == -EIOCBQUEUED);
391438ab720SKirill Smelkov if (ppos)
392293bc982SAl Viro *ppos = kiocb.ki_pos;
393293bc982SAl Viro return ret;
394293bc982SAl Viro }
395293bc982SAl Viro
warn_unsupported(struct file * file,const char * op)3964d03e3ccSChristoph Hellwig static int warn_unsupported(struct file *file, const char *op)
3974d03e3ccSChristoph Hellwig {
3984d03e3ccSChristoph Hellwig pr_warn_ratelimited(
3994d03e3ccSChristoph Hellwig "kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n",
4004d03e3ccSChristoph Hellwig op, file, current->pid, current->comm);
4014d03e3ccSChristoph Hellwig return -EINVAL;
4024d03e3ccSChristoph Hellwig }
4034d03e3ccSChristoph Hellwig
__kernel_read(struct file * file,void * buf,size_t count,loff_t * pos)40461a707c5SChristoph Hellwig ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
40561a707c5SChristoph Hellwig {
4064d03e3ccSChristoph Hellwig struct kvec iov = {
4074d03e3ccSChristoph Hellwig .iov_base = buf,
4084d03e3ccSChristoph Hellwig .iov_len = min_t(size_t, count, MAX_RW_COUNT),
4094d03e3ccSChristoph Hellwig };
4104d03e3ccSChristoph Hellwig struct kiocb kiocb;
4114d03e3ccSChristoph Hellwig struct iov_iter iter;
41261a707c5SChristoph Hellwig ssize_t ret;
41361a707c5SChristoph Hellwig
41461a707c5SChristoph Hellwig if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ)))
41561a707c5SChristoph Hellwig return -EINVAL;
41661a707c5SChristoph Hellwig if (!(file->f_mode & FMODE_CAN_READ))
41761a707c5SChristoph Hellwig return -EINVAL;
4184d03e3ccSChristoph Hellwig /*
4194d03e3ccSChristoph Hellwig * Also fail if ->read_iter and ->read are both wired up as that
4204d03e3ccSChristoph Hellwig * implies very convoluted semantics.
4214d03e3ccSChristoph Hellwig */
4224d03e3ccSChristoph Hellwig if (unlikely(!file->f_op->read_iter || file->f_op->read))
4234d03e3ccSChristoph Hellwig return warn_unsupported(file, "read");
42461a707c5SChristoph Hellwig
4254d03e3ccSChristoph Hellwig init_sync_kiocb(&kiocb, file);
4267b84b665SMatthew Wilcox (Oracle) kiocb.ki_pos = pos ? *pos : 0;
427de4eda9dSAl Viro iov_iter_kvec(&iter, ITER_DEST, &iov, 1, iov.iov_len);
4284d03e3ccSChristoph Hellwig ret = file->f_op->read_iter(&kiocb, &iter);
42961a707c5SChristoph Hellwig if (ret > 0) {
4307b84b665SMatthew Wilcox (Oracle) if (pos)
4314d03e3ccSChristoph Hellwig *pos = kiocb.ki_pos;
43261a707c5SChristoph Hellwig fsnotify_access(file);
43361a707c5SChristoph Hellwig add_rchar(current, ret);
43461a707c5SChristoph Hellwig }
43561a707c5SChristoph Hellwig inc_syscr(current);
43661a707c5SChristoph Hellwig return ret;
43761a707c5SChristoph Hellwig }
43861a707c5SChristoph Hellwig
kernel_read(struct file * file,void * buf,size_t count,loff_t * pos)439bdd1d2d3SChristoph Hellwig ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos)
440c41fbad0SChristoph Hellwig {
4416209dd91SChristoph Hellwig ssize_t ret;
442c41fbad0SChristoph Hellwig
4436209dd91SChristoph Hellwig ret = rw_verify_area(READ, file, pos, count);
4446209dd91SChristoph Hellwig if (ret)
4456209dd91SChristoph Hellwig return ret;
4466209dd91SChristoph Hellwig return __kernel_read(file, buf, count, pos);
447c41fbad0SChristoph Hellwig }
448c41fbad0SChristoph Hellwig EXPORT_SYMBOL(kernel_read);
4496fb5032eSDmitry Kasatkin
vfs_read(struct file * file,char __user * buf,size_t count,loff_t * pos)4501da177e4SLinus Torvalds ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
4511da177e4SLinus Torvalds {
4521da177e4SLinus Torvalds ssize_t ret;
4531da177e4SLinus Torvalds
4541da177e4SLinus Torvalds if (!(file->f_mode & FMODE_READ))
4551da177e4SLinus Torvalds return -EBADF;
4567f7f25e8SAl Viro if (!(file->f_mode & FMODE_CAN_READ))
4571da177e4SLinus Torvalds return -EINVAL;
45896d4f267SLinus Torvalds if (unlikely(!access_ok(buf, count)))
4591da177e4SLinus Torvalds return -EFAULT;
4601da177e4SLinus Torvalds
4611da177e4SLinus Torvalds ret = rw_verify_area(READ, file, pos, count);
462775802c0SChristoph Hellwig if (ret)
463775802c0SChristoph Hellwig return ret;
464bc61384dSAl Viro if (count > MAX_RW_COUNT)
465bc61384dSAl Viro count = MAX_RW_COUNT;
466775802c0SChristoph Hellwig
467775802c0SChristoph Hellwig if (file->f_op->read)
468775802c0SChristoph Hellwig ret = file->f_op->read(file, buf, count, pos);
469775802c0SChristoph Hellwig else if (file->f_op->read_iter)
470775802c0SChristoph Hellwig ret = new_sync_read(file, buf, count, pos);
471775802c0SChristoph Hellwig else
472775802c0SChristoph Hellwig ret = -EINVAL;
4731da177e4SLinus Torvalds if (ret > 0) {
4742a12a9d7SEric Paris fsnotify_access(file);
4754b98d11bSAlexey Dobriyan add_rchar(current, ret);
4761da177e4SLinus Torvalds }
4774b98d11bSAlexey Dobriyan inc_syscr(current);
4781da177e4SLinus Torvalds return ret;
4791da177e4SLinus Torvalds }
4801da177e4SLinus Torvalds
new_sync_write(struct file * filp,const char __user * buf,size_t len,loff_t * ppos)4815d5d5689SAl Viro static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
482293bc982SAl Viro {
483293bc982SAl Viro struct kiocb kiocb;
484293bc982SAl Viro struct iov_iter iter;
485293bc982SAl Viro ssize_t ret;
486293bc982SAl Viro
487293bc982SAl Viro init_sync_kiocb(&kiocb, filp);
488438ab720SKirill Smelkov kiocb.ki_pos = (ppos ? *ppos : 0);
489de4eda9dSAl Viro iov_iter_ubuf(&iter, ITER_SOURCE, (void __user *)buf, len);
490293bc982SAl Viro
491bb7462b6SMiklos Szeredi ret = call_write_iter(filp, &kiocb, &iter);
492599bd19bSChristoph Hellwig BUG_ON(ret == -EIOCBQUEUED);
493438ab720SKirill Smelkov if (ret > 0 && ppos)
494293bc982SAl Viro *ppos = kiocb.ki_pos;
495293bc982SAl Viro return ret;
496293bc982SAl Viro }
497293bc982SAl Viro
49881238b2cSChristoph Hellwig /* caller is responsible for file_start_write/file_end_write */
__kernel_write_iter(struct file * file,struct iov_iter * from,loff_t * pos)49906bbaa6dSAl Viro ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos)
50006ae43f3SAl Viro {
5014d03e3ccSChristoph Hellwig struct kiocb kiocb;
50206ae43f3SAl Viro ssize_t ret;
50306ae43f3SAl Viro
504a01ac27bSChristoph Hellwig if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE)))
505a01ac27bSChristoph Hellwig return -EBADF;
5067f7f25e8SAl Viro if (!(file->f_mode & FMODE_CAN_WRITE))
5073e84f48eSAl Viro return -EINVAL;
5084d03e3ccSChristoph Hellwig /*
5094d03e3ccSChristoph Hellwig * Also fail if ->write_iter and ->write are both wired up as that
5104d03e3ccSChristoph Hellwig * implies very convoluted semantics.
5114d03e3ccSChristoph Hellwig */
5124d03e3ccSChristoph Hellwig if (unlikely(!file->f_op->write_iter || file->f_op->write))
5134d03e3ccSChristoph Hellwig return warn_unsupported(file, "write");
5143e84f48eSAl Viro
5154d03e3ccSChristoph Hellwig init_sync_kiocb(&kiocb, file);
5164c207ef4SMatthew Wilcox (Oracle) kiocb.ki_pos = pos ? *pos : 0;
51706bbaa6dSAl Viro ret = file->f_op->write_iter(&kiocb, from);
51806ae43f3SAl Viro if (ret > 0) {
5194c207ef4SMatthew Wilcox (Oracle) if (pos)
5204d03e3ccSChristoph Hellwig *pos = kiocb.ki_pos;
52106ae43f3SAl Viro fsnotify_modify(file);
52206ae43f3SAl Viro add_wchar(current, ret);
52306ae43f3SAl Viro }
52406ae43f3SAl Viro inc_syscw(current);
52506ae43f3SAl Viro return ret;
52606ae43f3SAl Viro }
52706bbaa6dSAl Viro
52806bbaa6dSAl Viro /* caller is responsible for file_start_write/file_end_write */
__kernel_write(struct file * file,const void * buf,size_t count,loff_t * pos)52906bbaa6dSAl Viro ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos)
53006bbaa6dSAl Viro {
53106bbaa6dSAl Viro struct kvec iov = {
53206bbaa6dSAl Viro .iov_base = (void *)buf,
53306bbaa6dSAl Viro .iov_len = min_t(size_t, count, MAX_RW_COUNT),
53406bbaa6dSAl Viro };
53506bbaa6dSAl Viro struct iov_iter iter;
536de4eda9dSAl Viro iov_iter_kvec(&iter, ITER_SOURCE, &iov, 1, iov.iov_len);
53706bbaa6dSAl Viro return __kernel_write_iter(file, &iter, pos);
53806bbaa6dSAl Viro }
53990fb7027SLinus Torvalds /*
54090fb7027SLinus Torvalds * This "EXPORT_SYMBOL_GPL()" is more of a "EXPORT_SYMBOL_DONTUSE()",
54190fb7027SLinus Torvalds * but autofs is one of the few internal kernel users that actually
54290fb7027SLinus Torvalds * wants this _and_ can be built as a module. So we need to export
54390fb7027SLinus Torvalds * this symbol for autofs, even though it really isn't appropriate
54490fb7027SLinus Torvalds * for any other kernel modules.
54590fb7027SLinus Torvalds */
54690fb7027SLinus Torvalds EXPORT_SYMBOL_GPL(__kernel_write);
5472ec3a12aSAl Viro
kernel_write(struct file * file,const void * buf,size_t count,loff_t * pos)548e13ec939SChristoph Hellwig ssize_t kernel_write(struct file *file, const void *buf, size_t count,
549e13ec939SChristoph Hellwig loff_t *pos)
550ac452acaSChristoph Hellwig {
55181238b2cSChristoph Hellwig ssize_t ret;
552ac452acaSChristoph Hellwig
55381238b2cSChristoph Hellwig ret = rw_verify_area(WRITE, file, pos, count);
55481238b2cSChristoph Hellwig if (ret)
55581238b2cSChristoph Hellwig return ret;
556ac452acaSChristoph Hellwig
55781238b2cSChristoph Hellwig file_start_write(file);
55881238b2cSChristoph Hellwig ret = __kernel_write(file, buf, count, pos);
55981238b2cSChristoph Hellwig file_end_write(file);
56081238b2cSChristoph Hellwig return ret;
561ac452acaSChristoph Hellwig }
562ac452acaSChristoph Hellwig EXPORT_SYMBOL(kernel_write);
563ac452acaSChristoph Hellwig
vfs_write(struct file * file,const char __user * buf,size_t count,loff_t * pos)5641da177e4SLinus Torvalds ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
5651da177e4SLinus Torvalds {
5661da177e4SLinus Torvalds ssize_t ret;
5671da177e4SLinus Torvalds
5681da177e4SLinus Torvalds if (!(file->f_mode & FMODE_WRITE))
5691da177e4SLinus Torvalds return -EBADF;
5707f7f25e8SAl Viro if (!(file->f_mode & FMODE_CAN_WRITE))
5711da177e4SLinus Torvalds return -EINVAL;
57296d4f267SLinus Torvalds if (unlikely(!access_ok(buf, count)))
5731da177e4SLinus Torvalds return -EFAULT;
5741da177e4SLinus Torvalds
5751da177e4SLinus Torvalds ret = rw_verify_area(WRITE, file, pos, count);
57653ad8626SChristoph Hellwig if (ret)
57753ad8626SChristoph Hellwig return ret;
578bc61384dSAl Viro if (count > MAX_RW_COUNT)
579bc61384dSAl Viro count = MAX_RW_COUNT;
58003d95eb2SAl Viro file_start_write(file);
58153ad8626SChristoph Hellwig if (file->f_op->write)
58253ad8626SChristoph Hellwig ret = file->f_op->write(file, buf, count, pos);
58353ad8626SChristoph Hellwig else if (file->f_op->write_iter)
58453ad8626SChristoph Hellwig ret = new_sync_write(file, buf, count, pos);
58553ad8626SChristoph Hellwig else
58653ad8626SChristoph Hellwig ret = -EINVAL;
5871da177e4SLinus Torvalds if (ret > 0) {
5882a12a9d7SEric Paris fsnotify_modify(file);
5894b98d11bSAlexey Dobriyan add_wchar(current, ret);
5901da177e4SLinus Torvalds }
5914b98d11bSAlexey Dobriyan inc_syscw(current);
59203d95eb2SAl Viro file_end_write(file);
5931da177e4SLinus Torvalds return ret;
5941da177e4SLinus Torvalds }
5951da177e4SLinus Torvalds
596438ab720SKirill Smelkov /* file_ppos returns &file->f_pos or NULL if file is stream */
file_ppos(struct file * file)597438ab720SKirill Smelkov static inline loff_t *file_ppos(struct file *file)
5981da177e4SLinus Torvalds {
599438ab720SKirill Smelkov return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos;
6001da177e4SLinus Torvalds }
6011da177e4SLinus Torvalds
ksys_read(unsigned int fd,char __user * buf,size_t count)6023ce4a7bfSDominik Brodowski ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count)
6031da177e4SLinus Torvalds {
6049c225f26SLinus Torvalds struct fd f = fdget_pos(fd);
6051da177e4SLinus Torvalds ssize_t ret = -EBADF;
6061da177e4SLinus Torvalds
6072903ff01SAl Viro if (f.file) {
608438ab720SKirill Smelkov loff_t pos, *ppos = file_ppos(f.file);
609438ab720SKirill Smelkov if (ppos) {
610438ab720SKirill Smelkov pos = *ppos;
611438ab720SKirill Smelkov ppos = &pos;
612438ab720SKirill Smelkov }
613438ab720SKirill Smelkov ret = vfs_read(f.file, buf, count, ppos);
614438ab720SKirill Smelkov if (ret >= 0 && ppos)
615438ab720SKirill Smelkov f.file->f_pos = pos;
6169c225f26SLinus Torvalds fdput_pos(f);
6171da177e4SLinus Torvalds }
6181da177e4SLinus Torvalds return ret;
6191da177e4SLinus Torvalds }
6201da177e4SLinus Torvalds
SYSCALL_DEFINE3(read,unsigned int,fd,char __user *,buf,size_t,count)6213ce4a7bfSDominik Brodowski SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count)
6223ce4a7bfSDominik Brodowski {
6233ce4a7bfSDominik Brodowski return ksys_read(fd, buf, count);
6243ce4a7bfSDominik Brodowski }
6253ce4a7bfSDominik Brodowski
ksys_write(unsigned int fd,const char __user * buf,size_t count)626e7a3e8b2SDominik Brodowski ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count)
6271da177e4SLinus Torvalds {
6289c225f26SLinus Torvalds struct fd f = fdget_pos(fd);
6291da177e4SLinus Torvalds ssize_t ret = -EBADF;
6301da177e4SLinus Torvalds
6312903ff01SAl Viro if (f.file) {
632438ab720SKirill Smelkov loff_t pos, *ppos = file_ppos(f.file);
633438ab720SKirill Smelkov if (ppos) {
634438ab720SKirill Smelkov pos = *ppos;
635438ab720SKirill Smelkov ppos = &pos;
636438ab720SKirill Smelkov }
637438ab720SKirill Smelkov ret = vfs_write(f.file, buf, count, ppos);
638438ab720SKirill Smelkov if (ret >= 0 && ppos)
639438ab720SKirill Smelkov f.file->f_pos = pos;
6409c225f26SLinus Torvalds fdput_pos(f);
6411da177e4SLinus Torvalds }
6421da177e4SLinus Torvalds
6431da177e4SLinus Torvalds return ret;
6441da177e4SLinus Torvalds }
6451da177e4SLinus Torvalds
SYSCALL_DEFINE3(write,unsigned int,fd,const char __user *,buf,size_t,count)646e7a3e8b2SDominik Brodowski SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf,
647e7a3e8b2SDominik Brodowski size_t, count)
648e7a3e8b2SDominik Brodowski {
649e7a3e8b2SDominik Brodowski return ksys_write(fd, buf, count);
650e7a3e8b2SDominik Brodowski }
651e7a3e8b2SDominik Brodowski
ksys_pread64(unsigned int fd,char __user * buf,size_t count,loff_t pos)65236028d5dSDominik Brodowski ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count,
65336028d5dSDominik Brodowski loff_t pos)
6541da177e4SLinus Torvalds {
6552903ff01SAl Viro struct fd f;
6561da177e4SLinus Torvalds ssize_t ret = -EBADF;
6571da177e4SLinus Torvalds
6581da177e4SLinus Torvalds if (pos < 0)
6591da177e4SLinus Torvalds return -EINVAL;
6601da177e4SLinus Torvalds
6612903ff01SAl Viro f = fdget(fd);
6622903ff01SAl Viro if (f.file) {
6631da177e4SLinus Torvalds ret = -ESPIPE;
6642903ff01SAl Viro if (f.file->f_mode & FMODE_PREAD)
6652903ff01SAl Viro ret = vfs_read(f.file, buf, count, &pos);
6662903ff01SAl Viro fdput(f);
6671da177e4SLinus Torvalds }
6681da177e4SLinus Torvalds
6691da177e4SLinus Torvalds return ret;
6701da177e4SLinus Torvalds }
6711da177e4SLinus Torvalds
SYSCALL_DEFINE4(pread64,unsigned int,fd,char __user *,buf,size_t,count,loff_t,pos)67236028d5dSDominik Brodowski SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf,
6734a0fd5bfSAl Viro size_t, count, loff_t, pos)
6741da177e4SLinus Torvalds {
67536028d5dSDominik Brodowski return ksys_pread64(fd, buf, count, pos);
67636028d5dSDominik Brodowski }
67736028d5dSDominik Brodowski
67859c10c52SGuo Ren #if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PREAD64)
COMPAT_SYSCALL_DEFINE5(pread64,unsigned int,fd,char __user *,buf,size_t,count,compat_arg_u64_dual (pos))67959c10c52SGuo Ren COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, buf,
68059c10c52SGuo Ren size_t, count, compat_arg_u64_dual(pos))
68159c10c52SGuo Ren {
68259c10c52SGuo Ren return ksys_pread64(fd, buf, count, compat_arg_u64_glue(pos));
68359c10c52SGuo Ren }
68459c10c52SGuo Ren #endif
68559c10c52SGuo Ren
ksys_pwrite64(unsigned int fd,const char __user * buf,size_t count,loff_t pos)68636028d5dSDominik Brodowski ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf,
68736028d5dSDominik Brodowski size_t count, loff_t pos)
68836028d5dSDominik Brodowski {
6892903ff01SAl Viro struct fd f;
6901da177e4SLinus Torvalds ssize_t ret = -EBADF;
6911da177e4SLinus Torvalds
6921da177e4SLinus Torvalds if (pos < 0)
6931da177e4SLinus Torvalds return -EINVAL;
6941da177e4SLinus Torvalds
6952903ff01SAl Viro f = fdget(fd);
6962903ff01SAl Viro if (f.file) {
6971da177e4SLinus Torvalds ret = -ESPIPE;
6982903ff01SAl Viro if (f.file->f_mode & FMODE_PWRITE)
6992903ff01SAl Viro ret = vfs_write(f.file, buf, count, &pos);
7002903ff01SAl Viro fdput(f);
7011da177e4SLinus Torvalds }
7021da177e4SLinus Torvalds
7031da177e4SLinus Torvalds return ret;
7041da177e4SLinus Torvalds }
7051da177e4SLinus Torvalds
SYSCALL_DEFINE4(pwrite64,unsigned int,fd,const char __user *,buf,size_t,count,loff_t,pos)70636028d5dSDominik Brodowski SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf,
70736028d5dSDominik Brodowski size_t, count, loff_t, pos)
70836028d5dSDominik Brodowski {
70936028d5dSDominik Brodowski return ksys_pwrite64(fd, buf, count, pos);
71036028d5dSDominik Brodowski }
71136028d5dSDominik Brodowski
71259c10c52SGuo Ren #if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PWRITE64)
COMPAT_SYSCALL_DEFINE5(pwrite64,unsigned int,fd,const char __user *,buf,size_t,count,compat_arg_u64_dual (pos))71359c10c52SGuo Ren COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, const char __user *, buf,
71459c10c52SGuo Ren size_t, count, compat_arg_u64_dual(pos))
71559c10c52SGuo Ren {
71659c10c52SGuo Ren return ksys_pwrite64(fd, buf, count, compat_arg_u64_glue(pos));
71759c10c52SGuo Ren }
71859c10c52SGuo Ren #endif
71959c10c52SGuo Ren
do_iter_readv_writev(struct file * filp,struct iov_iter * iter,loff_t * ppos,int type,rwf_t flags)720ac15ac06SAl Viro static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter,
721ddef7ed2SChristoph Hellwig loff_t *ppos, int type, rwf_t flags)
722293bc982SAl Viro {
723293bc982SAl Viro struct kiocb kiocb;
724293bc982SAl Viro ssize_t ret;
725293bc982SAl Viro
726293bc982SAl Viro init_sync_kiocb(&kiocb, filp);
727fdd2f5b7SGoldwyn Rodrigues ret = kiocb_set_rw_flags(&kiocb, flags);
728fdd2f5b7SGoldwyn Rodrigues if (ret)
729fdd2f5b7SGoldwyn Rodrigues return ret;
730438ab720SKirill Smelkov kiocb.ki_pos = (ppos ? *ppos : 0);
731293bc982SAl Viro
7320f78d06aSMiklos Szeredi if (type == READ)
733bb7462b6SMiklos Szeredi ret = call_read_iter(filp, &kiocb, iter);
7340f78d06aSMiklos Szeredi else
735bb7462b6SMiklos Szeredi ret = call_write_iter(filp, &kiocb, iter);
736599bd19bSChristoph Hellwig BUG_ON(ret == -EIOCBQUEUED);
737438ab720SKirill Smelkov if (ppos)
738293bc982SAl Viro *ppos = kiocb.ki_pos;
739293bc982SAl Viro return ret;
740293bc982SAl Viro }
741293bc982SAl Viro
742ee0b3e67SBadari Pulavarty /* Do it by hand, with file-ops */
do_loop_readv_writev(struct file * filp,struct iov_iter * iter,loff_t * ppos,int type,rwf_t flags)743ac15ac06SAl Viro static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter,
744ddef7ed2SChristoph Hellwig loff_t *ppos, int type, rwf_t flags)
745ee0b3e67SBadari Pulavarty {
746ee0b3e67SBadari Pulavarty ssize_t ret = 0;
747ee0b3e67SBadari Pulavarty
74897be7ebeSChristoph Hellwig if (flags & ~RWF_HIPRI)
749793b80efSChristoph Hellwig return -EOPNOTSUPP;
750793b80efSChristoph Hellwig
751ac15ac06SAl Viro while (iov_iter_count(iter)) {
752ee0b3e67SBadari Pulavarty ssize_t nr;
753ee0b3e67SBadari Pulavarty
7540f78d06aSMiklos Szeredi if (type == READ) {
75595e49cf8SJens Axboe nr = filp->f_op->read(filp, iter_iov_addr(iter),
75695e49cf8SJens Axboe iter_iov_len(iter), ppos);
7570f78d06aSMiklos Szeredi } else {
75895e49cf8SJens Axboe nr = filp->f_op->write(filp, iter_iov_addr(iter),
75995e49cf8SJens Axboe iter_iov_len(iter), ppos);
7600f78d06aSMiklos Szeredi }
761ee0b3e67SBadari Pulavarty
762ee0b3e67SBadari Pulavarty if (nr < 0) {
763ee0b3e67SBadari Pulavarty if (!ret)
764ee0b3e67SBadari Pulavarty ret = nr;
765ee0b3e67SBadari Pulavarty break;
766ee0b3e67SBadari Pulavarty }
767ee0b3e67SBadari Pulavarty ret += nr;
76895e49cf8SJens Axboe if (nr != iter_iov_len(iter))
769ee0b3e67SBadari Pulavarty break;
770ac15ac06SAl Viro iov_iter_advance(iter, nr);
771ee0b3e67SBadari Pulavarty }
772ee0b3e67SBadari Pulavarty
773ee0b3e67SBadari Pulavarty return ret;
774ee0b3e67SBadari Pulavarty }
775ee0b3e67SBadari Pulavarty
do_iter_read(struct file * file,struct iov_iter * iter,loff_t * pos,rwf_t flags)77619c73586SChristoph Hellwig static ssize_t do_iter_read(struct file *file, struct iov_iter *iter,
777ddef7ed2SChristoph Hellwig loff_t *pos, rwf_t flags)
7781da177e4SLinus Torvalds {
7791da177e4SLinus Torvalds size_t tot_len;
7807687a7a4SMiklos Szeredi ssize_t ret = 0;
7811da177e4SLinus Torvalds
782edab5fe3SChristoph Hellwig if (!(file->f_mode & FMODE_READ))
783edab5fe3SChristoph Hellwig return -EBADF;
784edab5fe3SChristoph Hellwig if (!(file->f_mode & FMODE_CAN_READ))
785edab5fe3SChristoph Hellwig return -EINVAL;
786edab5fe3SChristoph Hellwig
7877687a7a4SMiklos Szeredi tot_len = iov_iter_count(iter);
7880504c074SAl Viro if (!tot_len)
7890504c074SAl Viro goto out;
79019c73586SChristoph Hellwig ret = rw_verify_area(READ, file, pos, tot_len);
791e28cc715SLinus Torvalds if (ret < 0)
79219c73586SChristoph Hellwig return ret;
7931da177e4SLinus Torvalds
79419c73586SChristoph Hellwig if (file->f_op->read_iter)
79519c73586SChristoph Hellwig ret = do_iter_readv_writev(file, iter, pos, READ, flags);
796ee0b3e67SBadari Pulavarty else
79719c73586SChristoph Hellwig ret = do_loop_readv_writev(file, iter, pos, READ, flags);
7981da177e4SLinus Torvalds out:
79919c73586SChristoph Hellwig if (ret >= 0)
8002a12a9d7SEric Paris fsnotify_access(file);
8011da177e4SLinus Torvalds return ret;
8021da177e4SLinus Torvalds }
8031da177e4SLinus Torvalds
vfs_iocb_iter_read(struct file * file,struct kiocb * iocb,struct iov_iter * iter)8045dcdc43eSJiufei Xue ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb,
8055dcdc43eSJiufei Xue struct iov_iter *iter)
8065dcdc43eSJiufei Xue {
8075dcdc43eSJiufei Xue size_t tot_len;
8085dcdc43eSJiufei Xue ssize_t ret = 0;
8095dcdc43eSJiufei Xue
8105dcdc43eSJiufei Xue if (!file->f_op->read_iter)
8115dcdc43eSJiufei Xue return -EINVAL;
8125dcdc43eSJiufei Xue if (!(file->f_mode & FMODE_READ))
8135dcdc43eSJiufei Xue return -EBADF;
8145dcdc43eSJiufei Xue if (!(file->f_mode & FMODE_CAN_READ))
8155dcdc43eSJiufei Xue return -EINVAL;
8165dcdc43eSJiufei Xue
8175dcdc43eSJiufei Xue tot_len = iov_iter_count(iter);
8185dcdc43eSJiufei Xue if (!tot_len)
8195dcdc43eSJiufei Xue goto out;
8205dcdc43eSJiufei Xue ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len);
8215dcdc43eSJiufei Xue if (ret < 0)
8225dcdc43eSJiufei Xue return ret;
8235dcdc43eSJiufei Xue
8245dcdc43eSJiufei Xue ret = call_read_iter(file, iocb, iter);
8255dcdc43eSJiufei Xue out:
8265dcdc43eSJiufei Xue if (ret >= 0)
8275dcdc43eSJiufei Xue fsnotify_access(file);
8285dcdc43eSJiufei Xue return ret;
8295dcdc43eSJiufei Xue }
8305dcdc43eSJiufei Xue EXPORT_SYMBOL(vfs_iocb_iter_read);
8315dcdc43eSJiufei Xue
vfs_iter_read(struct file * file,struct iov_iter * iter,loff_t * ppos,rwf_t flags)83218e9710eSChristoph Hellwig ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos,
833ddef7ed2SChristoph Hellwig rwf_t flags)
8347687a7a4SMiklos Szeredi {
83518e9710eSChristoph Hellwig if (!file->f_op->read_iter)
83618e9710eSChristoph Hellwig return -EINVAL;
83718e9710eSChristoph Hellwig return do_iter_read(file, iter, ppos, flags);
83818e9710eSChristoph Hellwig }
83918e9710eSChristoph Hellwig EXPORT_SYMBOL(vfs_iter_read);
84018e9710eSChristoph Hellwig
do_iter_write(struct file * file,struct iov_iter * iter,loff_t * pos,rwf_t flags)84119c73586SChristoph Hellwig static ssize_t do_iter_write(struct file *file, struct iov_iter *iter,
842ddef7ed2SChristoph Hellwig loff_t *pos, rwf_t flags)
84319c73586SChristoph Hellwig {
84419c73586SChristoph Hellwig size_t tot_len;
84519c73586SChristoph Hellwig ssize_t ret = 0;
84619c73586SChristoph Hellwig
847edab5fe3SChristoph Hellwig if (!(file->f_mode & FMODE_WRITE))
848edab5fe3SChristoph Hellwig return -EBADF;
849edab5fe3SChristoph Hellwig if (!(file->f_mode & FMODE_CAN_WRITE))
850edab5fe3SChristoph Hellwig return -EINVAL;
851edab5fe3SChristoph Hellwig
85219c73586SChristoph Hellwig tot_len = iov_iter_count(iter);
85319c73586SChristoph Hellwig if (!tot_len)
85419c73586SChristoph Hellwig return 0;
85519c73586SChristoph Hellwig ret = rw_verify_area(WRITE, file, pos, tot_len);
85619c73586SChristoph Hellwig if (ret < 0)
85719c73586SChristoph Hellwig return ret;
85819c73586SChristoph Hellwig
85919c73586SChristoph Hellwig if (file->f_op->write_iter)
86019c73586SChristoph Hellwig ret = do_iter_readv_writev(file, iter, pos, WRITE, flags);
86119c73586SChristoph Hellwig else
86219c73586SChristoph Hellwig ret = do_loop_readv_writev(file, iter, pos, WRITE, flags);
86319c73586SChristoph Hellwig if (ret > 0)
86419c73586SChristoph Hellwig fsnotify_modify(file);
865ee0b3e67SBadari Pulavarty return ret;
8661da177e4SLinus Torvalds }
8671da177e4SLinus Torvalds
vfs_iocb_iter_write(struct file * file,struct kiocb * iocb,struct iov_iter * iter)8685dcdc43eSJiufei Xue ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb,
8695dcdc43eSJiufei Xue struct iov_iter *iter)
8705dcdc43eSJiufei Xue {
8715dcdc43eSJiufei Xue size_t tot_len;
8725dcdc43eSJiufei Xue ssize_t ret = 0;
8735dcdc43eSJiufei Xue
8745dcdc43eSJiufei Xue if (!file->f_op->write_iter)
8755dcdc43eSJiufei Xue return -EINVAL;
8765dcdc43eSJiufei Xue if (!(file->f_mode & FMODE_WRITE))
8775dcdc43eSJiufei Xue return -EBADF;
8785dcdc43eSJiufei Xue if (!(file->f_mode & FMODE_CAN_WRITE))
8795dcdc43eSJiufei Xue return -EINVAL;
8805dcdc43eSJiufei Xue
8815dcdc43eSJiufei Xue tot_len = iov_iter_count(iter);
8825dcdc43eSJiufei Xue if (!tot_len)
8835dcdc43eSJiufei Xue return 0;
8845dcdc43eSJiufei Xue ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len);
8855dcdc43eSJiufei Xue if (ret < 0)
8865dcdc43eSJiufei Xue return ret;
8875dcdc43eSJiufei Xue
8885dcdc43eSJiufei Xue ret = call_write_iter(file, iocb, iter);
8895dcdc43eSJiufei Xue if (ret > 0)
8905dcdc43eSJiufei Xue fsnotify_modify(file);
8915dcdc43eSJiufei Xue
8925dcdc43eSJiufei Xue return ret;
8935dcdc43eSJiufei Xue }
8945dcdc43eSJiufei Xue EXPORT_SYMBOL(vfs_iocb_iter_write);
8955dcdc43eSJiufei Xue
vfs_iter_write(struct file * file,struct iov_iter * iter,loff_t * ppos,rwf_t flags)896abbb6589SChristoph Hellwig ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos,
897ddef7ed2SChristoph Hellwig rwf_t flags)
898abbb6589SChristoph Hellwig {
899abbb6589SChristoph Hellwig if (!file->f_op->write_iter)
900abbb6589SChristoph Hellwig return -EINVAL;
901abbb6589SChristoph Hellwig return do_iter_write(file, iter, ppos, flags);
902abbb6589SChristoph Hellwig }
903abbb6589SChristoph Hellwig EXPORT_SYMBOL(vfs_iter_write);
904abbb6589SChristoph Hellwig
vfs_readv(struct file * file,const struct iovec __user * vec,unsigned long vlen,loff_t * pos,rwf_t flags)90536e2c742SChristoph Hellwig static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
906ddef7ed2SChristoph Hellwig unsigned long vlen, loff_t *pos, rwf_t flags)
9077687a7a4SMiklos Szeredi {
9087687a7a4SMiklos Szeredi struct iovec iovstack[UIO_FASTIOV];
9097687a7a4SMiklos Szeredi struct iovec *iov = iovstack;
9107687a7a4SMiklos Szeredi struct iov_iter iter;
9117687a7a4SMiklos Szeredi ssize_t ret;
9127687a7a4SMiklos Szeredi
913de4eda9dSAl Viro ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
914edab5fe3SChristoph Hellwig if (ret >= 0) {
91519c73586SChristoph Hellwig ret = do_iter_read(file, &iter, pos, flags);
9167687a7a4SMiklos Szeredi kfree(iov);
917edab5fe3SChristoph Hellwig }
9187687a7a4SMiklos Szeredi
9197687a7a4SMiklos Szeredi return ret;
9207687a7a4SMiklos Szeredi }
9211da177e4SLinus Torvalds
vfs_writev(struct file * file,const struct iovec __user * vec,unsigned long vlen,loff_t * pos,rwf_t flags)9229725d4ceSChristoph Hellwig static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
923ddef7ed2SChristoph Hellwig unsigned long vlen, loff_t *pos, rwf_t flags)
9241da177e4SLinus Torvalds {
925251b42a1SChristoph Hellwig struct iovec iovstack[UIO_FASTIOV];
926251b42a1SChristoph Hellwig struct iovec *iov = iovstack;
927251b42a1SChristoph Hellwig struct iov_iter iter;
928251b42a1SChristoph Hellwig ssize_t ret;
9291da177e4SLinus Torvalds
930de4eda9dSAl Viro ret = import_iovec(ITER_SOURCE, vec, vlen, ARRAY_SIZE(iovstack), &iov, &iter);
931edab5fe3SChristoph Hellwig if (ret >= 0) {
93262473a2dSAl Viro file_start_write(file);
93319c73586SChristoph Hellwig ret = do_iter_write(file, &iter, pos, flags);
93462473a2dSAl Viro file_end_write(file);
935251b42a1SChristoph Hellwig kfree(iov);
9361da177e4SLinus Torvalds }
937251b42a1SChristoph Hellwig return ret;
938251b42a1SChristoph Hellwig }
9391da177e4SLinus Torvalds
do_readv(unsigned long fd,const struct iovec __user * vec,unsigned long vlen,rwf_t flags)940f17d8b35SMilosz Tanski static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec,
941ddef7ed2SChristoph Hellwig unsigned long vlen, rwf_t flags)
9421da177e4SLinus Torvalds {
9439c225f26SLinus Torvalds struct fd f = fdget_pos(fd);
9441da177e4SLinus Torvalds ssize_t ret = -EBADF;
9451da177e4SLinus Torvalds
9462903ff01SAl Viro if (f.file) {
947438ab720SKirill Smelkov loff_t pos, *ppos = file_ppos(f.file);
948438ab720SKirill Smelkov if (ppos) {
949438ab720SKirill Smelkov pos = *ppos;
950438ab720SKirill Smelkov ppos = &pos;
951438ab720SKirill Smelkov }
952438ab720SKirill Smelkov ret = vfs_readv(f.file, vec, vlen, ppos, flags);
953438ab720SKirill Smelkov if (ret >= 0 && ppos)
954438ab720SKirill Smelkov f.file->f_pos = pos;
9559c225f26SLinus Torvalds fdput_pos(f);
9561da177e4SLinus Torvalds }
9571da177e4SLinus Torvalds
9581da177e4SLinus Torvalds if (ret > 0)
9594b98d11bSAlexey Dobriyan add_rchar(current, ret);
9604b98d11bSAlexey Dobriyan inc_syscr(current);
9611da177e4SLinus Torvalds return ret;
9621da177e4SLinus Torvalds }
9631da177e4SLinus Torvalds
do_writev(unsigned long fd,const struct iovec __user * vec,unsigned long vlen,rwf_t flags)964f17d8b35SMilosz Tanski static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec,
965ddef7ed2SChristoph Hellwig unsigned long vlen, rwf_t flags)
9661da177e4SLinus Torvalds {
9679c225f26SLinus Torvalds struct fd f = fdget_pos(fd);
9681da177e4SLinus Torvalds ssize_t ret = -EBADF;
9691da177e4SLinus Torvalds
9702903ff01SAl Viro if (f.file) {
971438ab720SKirill Smelkov loff_t pos, *ppos = file_ppos(f.file);
972438ab720SKirill Smelkov if (ppos) {
973438ab720SKirill Smelkov pos = *ppos;
974438ab720SKirill Smelkov ppos = &pos;
975438ab720SKirill Smelkov }
976438ab720SKirill Smelkov ret = vfs_writev(f.file, vec, vlen, ppos, flags);
977438ab720SKirill Smelkov if (ret >= 0 && ppos)
978438ab720SKirill Smelkov f.file->f_pos = pos;
9799c225f26SLinus Torvalds fdput_pos(f);
9801da177e4SLinus Torvalds }
9811da177e4SLinus Torvalds
9821da177e4SLinus Torvalds if (ret > 0)
9834b98d11bSAlexey Dobriyan add_wchar(current, ret);
9844b98d11bSAlexey Dobriyan inc_syscw(current);
9851da177e4SLinus Torvalds return ret;
9861da177e4SLinus Torvalds }
9871da177e4SLinus Torvalds
pos_from_hilo(unsigned long high,unsigned long low)988601cc11dSLinus Torvalds static inline loff_t pos_from_hilo(unsigned long high, unsigned long low)
989f3554f4bSGerd Hoffmann {
990601cc11dSLinus Torvalds #define HALF_LONG_BITS (BITS_PER_LONG / 2)
991601cc11dSLinus Torvalds return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low;
992601cc11dSLinus Torvalds }
993601cc11dSLinus Torvalds
do_preadv(unsigned long fd,const struct iovec __user * vec,unsigned long vlen,loff_t pos,rwf_t flags)994f17d8b35SMilosz Tanski static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec,
995ddef7ed2SChristoph Hellwig unsigned long vlen, loff_t pos, rwf_t flags)
996601cc11dSLinus Torvalds {
9972903ff01SAl Viro struct fd f;
998f3554f4bSGerd Hoffmann ssize_t ret = -EBADF;
999f3554f4bSGerd Hoffmann
1000f3554f4bSGerd Hoffmann if (pos < 0)
1001f3554f4bSGerd Hoffmann return -EINVAL;
1002f3554f4bSGerd Hoffmann
10032903ff01SAl Viro f = fdget(fd);
10042903ff01SAl Viro if (f.file) {
1005f3554f4bSGerd Hoffmann ret = -ESPIPE;
10062903ff01SAl Viro if (f.file->f_mode & FMODE_PREAD)
1007f17d8b35SMilosz Tanski ret = vfs_readv(f.file, vec, vlen, &pos, flags);
10082903ff01SAl Viro fdput(f);
1009f3554f4bSGerd Hoffmann }
1010f3554f4bSGerd Hoffmann
1011f3554f4bSGerd Hoffmann if (ret > 0)
1012f3554f4bSGerd Hoffmann add_rchar(current, ret);
1013f3554f4bSGerd Hoffmann inc_syscr(current);
1014f3554f4bSGerd Hoffmann return ret;
1015f3554f4bSGerd Hoffmann }
1016f3554f4bSGerd Hoffmann
do_pwritev(unsigned long fd,const struct iovec __user * vec,unsigned long vlen,loff_t pos,rwf_t flags)1017f17d8b35SMilosz Tanski static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec,
1018ddef7ed2SChristoph Hellwig unsigned long vlen, loff_t pos, rwf_t flags)
1019f3554f4bSGerd Hoffmann {
10202903ff01SAl Viro struct fd f;
1021f3554f4bSGerd Hoffmann ssize_t ret = -EBADF;
1022f3554f4bSGerd Hoffmann
1023f3554f4bSGerd Hoffmann if (pos < 0)
1024f3554f4bSGerd Hoffmann return -EINVAL;
1025f3554f4bSGerd Hoffmann
10262903ff01SAl Viro f = fdget(fd);
10272903ff01SAl Viro if (f.file) {
1028f3554f4bSGerd Hoffmann ret = -ESPIPE;
10292903ff01SAl Viro if (f.file->f_mode & FMODE_PWRITE)
1030f17d8b35SMilosz Tanski ret = vfs_writev(f.file, vec, vlen, &pos, flags);
10312903ff01SAl Viro fdput(f);
1032f3554f4bSGerd Hoffmann }
1033f3554f4bSGerd Hoffmann
1034f3554f4bSGerd Hoffmann if (ret > 0)
1035f3554f4bSGerd Hoffmann add_wchar(current, ret);
1036f3554f4bSGerd Hoffmann inc_syscw(current);
1037f3554f4bSGerd Hoffmann return ret;
1038f3554f4bSGerd Hoffmann }
1039f3554f4bSGerd Hoffmann
SYSCALL_DEFINE3(readv,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen)1040f17d8b35SMilosz Tanski SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec,
1041f17d8b35SMilosz Tanski unsigned long, vlen)
1042f17d8b35SMilosz Tanski {
1043f17d8b35SMilosz Tanski return do_readv(fd, vec, vlen, 0);
1044f17d8b35SMilosz Tanski }
1045f17d8b35SMilosz Tanski
SYSCALL_DEFINE3(writev,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen)1046f17d8b35SMilosz Tanski SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec,
1047f17d8b35SMilosz Tanski unsigned long, vlen)
1048f17d8b35SMilosz Tanski {
1049f17d8b35SMilosz Tanski return do_writev(fd, vec, vlen, 0);
1050f17d8b35SMilosz Tanski }
1051f17d8b35SMilosz Tanski
SYSCALL_DEFINE5(preadv,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,unsigned long,pos_l,unsigned long,pos_h)1052f17d8b35SMilosz Tanski SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec,
1053f17d8b35SMilosz Tanski unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1054f17d8b35SMilosz Tanski {
1055f17d8b35SMilosz Tanski loff_t pos = pos_from_hilo(pos_h, pos_l);
1056f17d8b35SMilosz Tanski
1057f17d8b35SMilosz Tanski return do_preadv(fd, vec, vlen, pos, 0);
1058f17d8b35SMilosz Tanski }
1059f17d8b35SMilosz Tanski
SYSCALL_DEFINE6(preadv2,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,unsigned long,pos_l,unsigned long,pos_h,rwf_t,flags)1060f17d8b35SMilosz Tanski SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec,
1061f17d8b35SMilosz Tanski unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1062ddef7ed2SChristoph Hellwig rwf_t, flags)
1063f17d8b35SMilosz Tanski {
1064f17d8b35SMilosz Tanski loff_t pos = pos_from_hilo(pos_h, pos_l);
1065f17d8b35SMilosz Tanski
1066f17d8b35SMilosz Tanski if (pos == -1)
1067f17d8b35SMilosz Tanski return do_readv(fd, vec, vlen, flags);
1068f17d8b35SMilosz Tanski
1069f17d8b35SMilosz Tanski return do_preadv(fd, vec, vlen, pos, flags);
1070f17d8b35SMilosz Tanski }
1071f17d8b35SMilosz Tanski
SYSCALL_DEFINE5(pwritev,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,unsigned long,pos_l,unsigned long,pos_h)1072f17d8b35SMilosz Tanski SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec,
1073f17d8b35SMilosz Tanski unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h)
1074f17d8b35SMilosz Tanski {
1075f17d8b35SMilosz Tanski loff_t pos = pos_from_hilo(pos_h, pos_l);
1076f17d8b35SMilosz Tanski
1077f17d8b35SMilosz Tanski return do_pwritev(fd, vec, vlen, pos, 0);
1078f17d8b35SMilosz Tanski }
1079f17d8b35SMilosz Tanski
SYSCALL_DEFINE6(pwritev2,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,unsigned long,pos_l,unsigned long,pos_h,rwf_t,flags)1080f17d8b35SMilosz Tanski SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec,
1081f17d8b35SMilosz Tanski unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h,
1082ddef7ed2SChristoph Hellwig rwf_t, flags)
1083f17d8b35SMilosz Tanski {
1084f17d8b35SMilosz Tanski loff_t pos = pos_from_hilo(pos_h, pos_l);
1085f17d8b35SMilosz Tanski
1086f17d8b35SMilosz Tanski if (pos == -1)
1087f17d8b35SMilosz Tanski return do_writev(fd, vec, vlen, flags);
1088f17d8b35SMilosz Tanski
1089f17d8b35SMilosz Tanski return do_pwritev(fd, vec, vlen, pos, flags);
1090f17d8b35SMilosz Tanski }
1091f17d8b35SMilosz Tanski
10923523a9d4SChristoph Hellwig /*
10933523a9d4SChristoph Hellwig * Various compat syscalls. Note that they all pretend to take a native
10943523a9d4SChristoph Hellwig * iovec - import_iovec will properly treat those as compat_iovecs based on
10953523a9d4SChristoph Hellwig * in_compat_syscall().
10963523a9d4SChristoph Hellwig */
109772ec3516SAl Viro #ifdef CONFIG_COMPAT
1098378a10f3SHeiko Carstens #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64
COMPAT_SYSCALL_DEFINE4(preadv64,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,loff_t,pos)1099378a10f3SHeiko Carstens COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd,
11003523a9d4SChristoph Hellwig const struct iovec __user *, vec,
1101378a10f3SHeiko Carstens unsigned long, vlen, loff_t, pos)
1102378a10f3SHeiko Carstens {
11033523a9d4SChristoph Hellwig return do_preadv(fd, vec, vlen, pos, 0);
1104378a10f3SHeiko Carstens }
1105378a10f3SHeiko Carstens #endif
1106378a10f3SHeiko Carstens
COMPAT_SYSCALL_DEFINE5(preadv,compat_ulong_t,fd,const struct iovec __user *,vec,compat_ulong_t,vlen,u32,pos_low,u32,pos_high)1107dfd948e3SHeiko Carstens COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd,
11083523a9d4SChristoph Hellwig const struct iovec __user *, vec,
1109dfd948e3SHeiko Carstens compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
111072ec3516SAl Viro {
111172ec3516SAl Viro loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1112378a10f3SHeiko Carstens
11133523a9d4SChristoph Hellwig return do_preadv(fd, vec, vlen, pos, 0);
1114f17d8b35SMilosz Tanski }
1115f17d8b35SMilosz Tanski
11163ebfd81fSH.J. Lu #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2
COMPAT_SYSCALL_DEFINE5(preadv64v2,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,loff_t,pos,rwf_t,flags)11173ebfd81fSH.J. Lu COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd,
11183523a9d4SChristoph Hellwig const struct iovec __user *, vec,
1119ddef7ed2SChristoph Hellwig unsigned long, vlen, loff_t, pos, rwf_t, flags)
11203ebfd81fSH.J. Lu {
1121cc4b1242SAurelien Jarno if (pos == -1)
11223523a9d4SChristoph Hellwig return do_readv(fd, vec, vlen, flags);
11233523a9d4SChristoph Hellwig return do_preadv(fd, vec, vlen, pos, flags);
11243ebfd81fSH.J. Lu }
11253ebfd81fSH.J. Lu #endif
11263ebfd81fSH.J. Lu
COMPAT_SYSCALL_DEFINE6(preadv2,compat_ulong_t,fd,const struct iovec __user *,vec,compat_ulong_t,vlen,u32,pos_low,u32,pos_high,rwf_t,flags)1127f17d8b35SMilosz Tanski COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
11283523a9d4SChristoph Hellwig const struct iovec __user *, vec,
1129f17d8b35SMilosz Tanski compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
1130ddef7ed2SChristoph Hellwig rwf_t, flags)
1131f17d8b35SMilosz Tanski {
1132f17d8b35SMilosz Tanski loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1133f17d8b35SMilosz Tanski
1134f17d8b35SMilosz Tanski if (pos == -1)
11353523a9d4SChristoph Hellwig return do_readv(fd, vec, vlen, flags);
11363523a9d4SChristoph Hellwig return do_preadv(fd, vec, vlen, pos, flags);
113772ec3516SAl Viro }
113872ec3516SAl Viro
1139378a10f3SHeiko Carstens #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64
COMPAT_SYSCALL_DEFINE4(pwritev64,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,loff_t,pos)1140378a10f3SHeiko Carstens COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd,
11413523a9d4SChristoph Hellwig const struct iovec __user *, vec,
1142378a10f3SHeiko Carstens unsigned long, vlen, loff_t, pos)
1143378a10f3SHeiko Carstens {
11443523a9d4SChristoph Hellwig return do_pwritev(fd, vec, vlen, pos, 0);
1145378a10f3SHeiko Carstens }
1146378a10f3SHeiko Carstens #endif
1147378a10f3SHeiko Carstens
COMPAT_SYSCALL_DEFINE5(pwritev,compat_ulong_t,fd,const struct iovec __user *,vec,compat_ulong_t,vlen,u32,pos_low,u32,pos_high)1148dfd948e3SHeiko Carstens COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd,
11493523a9d4SChristoph Hellwig const struct iovec __user *,vec,
1150dfd948e3SHeiko Carstens compat_ulong_t, vlen, u32, pos_low, u32, pos_high)
115172ec3516SAl Viro {
115272ec3516SAl Viro loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1153378a10f3SHeiko Carstens
11543523a9d4SChristoph Hellwig return do_pwritev(fd, vec, vlen, pos, 0);
115572ec3516SAl Viro }
1156f17d8b35SMilosz Tanski
11573ebfd81fSH.J. Lu #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2
COMPAT_SYSCALL_DEFINE5(pwritev64v2,unsigned long,fd,const struct iovec __user *,vec,unsigned long,vlen,loff_t,pos,rwf_t,flags)11583ebfd81fSH.J. Lu COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd,
11593523a9d4SChristoph Hellwig const struct iovec __user *, vec,
1160ddef7ed2SChristoph Hellwig unsigned long, vlen, loff_t, pos, rwf_t, flags)
11613ebfd81fSH.J. Lu {
1162cc4b1242SAurelien Jarno if (pos == -1)
11633523a9d4SChristoph Hellwig return do_writev(fd, vec, vlen, flags);
11643523a9d4SChristoph Hellwig return do_pwritev(fd, vec, vlen, pos, flags);
11653ebfd81fSH.J. Lu }
11663ebfd81fSH.J. Lu #endif
11673ebfd81fSH.J. Lu
COMPAT_SYSCALL_DEFINE6(pwritev2,compat_ulong_t,fd,const struct iovec __user *,vec,compat_ulong_t,vlen,u32,pos_low,u32,pos_high,rwf_t,flags)1168f17d8b35SMilosz Tanski COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd,
11693523a9d4SChristoph Hellwig const struct iovec __user *,vec,
1170ddef7ed2SChristoph Hellwig compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags)
1171f17d8b35SMilosz Tanski {
1172f17d8b35SMilosz Tanski loff_t pos = ((loff_t)pos_high << 32) | pos_low;
1173f17d8b35SMilosz Tanski
1174f17d8b35SMilosz Tanski if (pos == -1)
11753523a9d4SChristoph Hellwig return do_writev(fd, vec, vlen, flags);
11763523a9d4SChristoph Hellwig return do_pwritev(fd, vec, vlen, pos, flags);
1177f17d8b35SMilosz Tanski }
11783523a9d4SChristoph Hellwig #endif /* CONFIG_COMPAT */
117972ec3516SAl Viro
do_sendfile(int out_fd,int in_fd,loff_t * ppos,size_t count,loff_t max)118019f4fc3aSAl Viro static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
118119f4fc3aSAl Viro size_t count, loff_t max)
11821da177e4SLinus Torvalds {
11832903ff01SAl Viro struct fd in, out;
11841da177e4SLinus Torvalds struct inode *in_inode, *out_inode;
1185b964bf53SAl Viro struct pipe_inode_info *opipe;
11861da177e4SLinus Torvalds loff_t pos;
11877995bd28SAl Viro loff_t out_pos;
11881da177e4SLinus Torvalds ssize_t retval;
11892903ff01SAl Viro int fl;
11901da177e4SLinus Torvalds
11911da177e4SLinus Torvalds /*
11921da177e4SLinus Torvalds * Get input file, and verify that it is ok..
11931da177e4SLinus Torvalds */
11941da177e4SLinus Torvalds retval = -EBADF;
11952903ff01SAl Viro in = fdget(in_fd);
11962903ff01SAl Viro if (!in.file)
11971da177e4SLinus Torvalds goto out;
11982903ff01SAl Viro if (!(in.file->f_mode & FMODE_READ))
11991da177e4SLinus Torvalds goto fput_in;
12001da177e4SLinus Torvalds retval = -ESPIPE;
12017995bd28SAl Viro if (!ppos) {
12027995bd28SAl Viro pos = in.file->f_pos;
12037995bd28SAl Viro } else {
12047995bd28SAl Viro pos = *ppos;
12052903ff01SAl Viro if (!(in.file->f_mode & FMODE_PREAD))
12061da177e4SLinus Torvalds goto fput_in;
12077995bd28SAl Viro }
12087995bd28SAl Viro retval = rw_verify_area(READ, in.file, &pos, count);
1209e28cc715SLinus Torvalds if (retval < 0)
12101da177e4SLinus Torvalds goto fput_in;
1211bc61384dSAl Viro if (count > MAX_RW_COUNT)
1212bc61384dSAl Viro count = MAX_RW_COUNT;
12131da177e4SLinus Torvalds
12141da177e4SLinus Torvalds /*
12151da177e4SLinus Torvalds * Get output file, and verify that it is ok..
12161da177e4SLinus Torvalds */
12171da177e4SLinus Torvalds retval = -EBADF;
12182903ff01SAl Viro out = fdget(out_fd);
12192903ff01SAl Viro if (!out.file)
12201da177e4SLinus Torvalds goto fput_in;
12212903ff01SAl Viro if (!(out.file->f_mode & FMODE_WRITE))
12221da177e4SLinus Torvalds goto fput_out;
1223496ad9aaSAl Viro in_inode = file_inode(in.file);
1224496ad9aaSAl Viro out_inode = file_inode(out.file);
12257995bd28SAl Viro out_pos = out.file->f_pos;
12261da177e4SLinus Torvalds
12271da177e4SLinus Torvalds if (!max)
12281da177e4SLinus Torvalds max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
12291da177e4SLinus Torvalds
12301da177e4SLinus Torvalds if (unlikely(pos + count > max)) {
12311da177e4SLinus Torvalds retval = -EOVERFLOW;
12321da177e4SLinus Torvalds if (pos >= max)
12331da177e4SLinus Torvalds goto fput_out;
12341da177e4SLinus Torvalds count = max - pos;
12351da177e4SLinus Torvalds }
12361da177e4SLinus Torvalds
1237534f2aaaSJens Axboe fl = 0;
1238534f2aaaSJens Axboe #if 0
1239534f2aaaSJens Axboe /*
1240534f2aaaSJens Axboe * We need to debate whether we can enable this or not. The
1241534f2aaaSJens Axboe * man page documents EAGAIN return for the output at least,
1242534f2aaaSJens Axboe * and the application is arguably buggy if it doesn't expect
1243534f2aaaSJens Axboe * EAGAIN on a non-blocking file descriptor.
1244534f2aaaSJens Axboe */
12452903ff01SAl Viro if (in.file->f_flags & O_NONBLOCK)
1246534f2aaaSJens Axboe fl = SPLICE_F_NONBLOCK;
1247534f2aaaSJens Axboe #endif
1248b964bf53SAl Viro opipe = get_pipe_info(out.file, true);
1249b964bf53SAl Viro if (!opipe) {
1250b964bf53SAl Viro retval = rw_verify_area(WRITE, out.file, &out_pos, count);
1251b964bf53SAl Viro if (retval < 0)
1252b964bf53SAl Viro goto fput_out;
125350cd2c57SAl Viro file_start_write(out.file);
1254b964bf53SAl Viro retval = do_splice_direct(in.file, &pos, out.file, &out_pos,
1255b964bf53SAl Viro count, fl);
125650cd2c57SAl Viro file_end_write(out.file);
1257b964bf53SAl Viro } else {
1258bdeb77bcSAndrei Vagin if (out.file->f_flags & O_NONBLOCK)
1259bdeb77bcSAndrei Vagin fl |= SPLICE_F_NONBLOCK;
1260bdeb77bcSAndrei Vagin
1261b964bf53SAl Viro retval = splice_file_to_pipe(in.file, opipe, &pos, count, fl);
1262b964bf53SAl Viro }
12631da177e4SLinus Torvalds
12641da177e4SLinus Torvalds if (retval > 0) {
12654b98d11bSAlexey Dobriyan add_rchar(current, retval);
12664b98d11bSAlexey Dobriyan add_wchar(current, retval);
1267a68c2f12SScott Wolchok fsnotify_access(in.file);
1268a68c2f12SScott Wolchok fsnotify_modify(out.file);
12697995bd28SAl Viro out.file->f_pos = out_pos;
12707995bd28SAl Viro if (ppos)
12717995bd28SAl Viro *ppos = pos;
12727995bd28SAl Viro else
12737995bd28SAl Viro in.file->f_pos = pos;
12741da177e4SLinus Torvalds }
12751da177e4SLinus Torvalds
12764b98d11bSAlexey Dobriyan inc_syscr(current);
12774b98d11bSAlexey Dobriyan inc_syscw(current);
12787995bd28SAl Viro if (pos > max)
12791da177e4SLinus Torvalds retval = -EOVERFLOW;
12801da177e4SLinus Torvalds
12811da177e4SLinus Torvalds fput_out:
12822903ff01SAl Viro fdput(out);
12831da177e4SLinus Torvalds fput_in:
12842903ff01SAl Viro fdput(in);
12851da177e4SLinus Torvalds out:
12861da177e4SLinus Torvalds return retval;
12871da177e4SLinus Torvalds }
12881da177e4SLinus Torvalds
SYSCALL_DEFINE4(sendfile,int,out_fd,int,in_fd,off_t __user *,offset,size_t,count)1289002c8976SHeiko Carstens SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count)
12901da177e4SLinus Torvalds {
12911da177e4SLinus Torvalds loff_t pos;
12921da177e4SLinus Torvalds off_t off;
12931da177e4SLinus Torvalds ssize_t ret;
12941da177e4SLinus Torvalds
12951da177e4SLinus Torvalds if (offset) {
12961da177e4SLinus Torvalds if (unlikely(get_user(off, offset)))
12971da177e4SLinus Torvalds return -EFAULT;
12981da177e4SLinus Torvalds pos = off;
12991da177e4SLinus Torvalds ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
13001da177e4SLinus Torvalds if (unlikely(put_user(pos, offset)))
13011da177e4SLinus Torvalds return -EFAULT;
13021da177e4SLinus Torvalds return ret;
13031da177e4SLinus Torvalds }
13041da177e4SLinus Torvalds
13051da177e4SLinus Torvalds return do_sendfile(out_fd, in_fd, NULL, count, 0);
13061da177e4SLinus Torvalds }
13071da177e4SLinus Torvalds
SYSCALL_DEFINE4(sendfile64,int,out_fd,int,in_fd,loff_t __user *,offset,size_t,count)1308002c8976SHeiko Carstens SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count)
13091da177e4SLinus Torvalds {
13101da177e4SLinus Torvalds loff_t pos;
13111da177e4SLinus Torvalds ssize_t ret;
13121da177e4SLinus Torvalds
13131da177e4SLinus Torvalds if (offset) {
13141da177e4SLinus Torvalds if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
13151da177e4SLinus Torvalds return -EFAULT;
13161da177e4SLinus Torvalds ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
13171da177e4SLinus Torvalds if (unlikely(put_user(pos, offset)))
13181da177e4SLinus Torvalds return -EFAULT;
13191da177e4SLinus Torvalds return ret;
13201da177e4SLinus Torvalds }
13211da177e4SLinus Torvalds
13221da177e4SLinus Torvalds return do_sendfile(out_fd, in_fd, NULL, count, 0);
13231da177e4SLinus Torvalds }
132419f4fc3aSAl Viro
132519f4fc3aSAl Viro #ifdef CONFIG_COMPAT
COMPAT_SYSCALL_DEFINE4(sendfile,int,out_fd,int,in_fd,compat_off_t __user *,offset,compat_size_t,count)132619f4fc3aSAl Viro COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd,
132719f4fc3aSAl Viro compat_off_t __user *, offset, compat_size_t, count)
132819f4fc3aSAl Viro {
132919f4fc3aSAl Viro loff_t pos;
133019f4fc3aSAl Viro off_t off;
133119f4fc3aSAl Viro ssize_t ret;
133219f4fc3aSAl Viro
133319f4fc3aSAl Viro if (offset) {
133419f4fc3aSAl Viro if (unlikely(get_user(off, offset)))
133519f4fc3aSAl Viro return -EFAULT;
133619f4fc3aSAl Viro pos = off;
133719f4fc3aSAl Viro ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
133819f4fc3aSAl Viro if (unlikely(put_user(pos, offset)))
133919f4fc3aSAl Viro return -EFAULT;
134019f4fc3aSAl Viro return ret;
134119f4fc3aSAl Viro }
134219f4fc3aSAl Viro
134319f4fc3aSAl Viro return do_sendfile(out_fd, in_fd, NULL, count, 0);
134419f4fc3aSAl Viro }
134519f4fc3aSAl Viro
COMPAT_SYSCALL_DEFINE4(sendfile64,int,out_fd,int,in_fd,compat_loff_t __user *,offset,compat_size_t,count)134619f4fc3aSAl Viro COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd,
134719f4fc3aSAl Viro compat_loff_t __user *, offset, compat_size_t, count)
134819f4fc3aSAl Viro {
134919f4fc3aSAl Viro loff_t pos;
135019f4fc3aSAl Viro ssize_t ret;
135119f4fc3aSAl Viro
135219f4fc3aSAl Viro if (offset) {
135319f4fc3aSAl Viro if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
135419f4fc3aSAl Viro return -EFAULT;
135519f4fc3aSAl Viro ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
135619f4fc3aSAl Viro if (unlikely(put_user(pos, offset)))
135719f4fc3aSAl Viro return -EFAULT;
135819f4fc3aSAl Viro return ret;
135919f4fc3aSAl Viro }
136019f4fc3aSAl Viro
136119f4fc3aSAl Viro return do_sendfile(out_fd, in_fd, NULL, count, 0);
136219f4fc3aSAl Viro }
136319f4fc3aSAl Viro #endif
136429732938SZach Brown
1365f16acc9dSDave Chinner /**
1366f16acc9dSDave Chinner * generic_copy_file_range - copy data between two files
1367f16acc9dSDave Chinner * @file_in: file structure to read from
1368f16acc9dSDave Chinner * @pos_in: file offset to read from
1369f16acc9dSDave Chinner * @file_out: file structure to write data to
1370f16acc9dSDave Chinner * @pos_out: file offset to write data to
1371f16acc9dSDave Chinner * @len: amount of data to copy
1372f16acc9dSDave Chinner * @flags: copy flags
1373f16acc9dSDave Chinner *
1374f16acc9dSDave Chinner * This is a generic filesystem helper to copy data from one file to another.
1375f16acc9dSDave Chinner * It has no constraints on the source or destination file owners - the files
1376f16acc9dSDave Chinner * can belong to different superblocks and different filesystem types. Short
1377f16acc9dSDave Chinner * copies are allowed.
1378f16acc9dSDave Chinner *
1379f16acc9dSDave Chinner * This should be called from the @file_out filesystem, as per the
1380f16acc9dSDave Chinner * ->copy_file_range() method.
1381f16acc9dSDave Chinner *
1382f16acc9dSDave Chinner * Returns the number of bytes copied or a negative error indicating the
1383f16acc9dSDave Chinner * failure.
1384f16acc9dSDave Chinner */
1385f16acc9dSDave Chinner
generic_copy_file_range(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,size_t len,unsigned int flags)1386f16acc9dSDave Chinner ssize_t generic_copy_file_range(struct file *file_in, loff_t pos_in,
1387f16acc9dSDave Chinner struct file *file_out, loff_t pos_out,
1388f16acc9dSDave Chinner size_t len, unsigned int flags)
1389f16acc9dSDave Chinner {
139010bc8e4aSAmir Goldstein lockdep_assert(sb_write_started(file_inode(file_out)->i_sb));
139110bc8e4aSAmir Goldstein
1392f16acc9dSDave Chinner return do_splice_direct(file_in, &pos_in, file_out, &pos_out,
1393f16acc9dSDave Chinner len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0);
1394f16acc9dSDave Chinner }
1395f16acc9dSDave Chinner EXPORT_SYMBOL(generic_copy_file_range);
1396f16acc9dSDave Chinner
139729732938SZach Brown /*
1398407e9c63SDarrick J. Wong * Performs necessary checks before doing a file copy
1399407e9c63SDarrick J. Wong *
1400407e9c63SDarrick J. Wong * Can adjust amount of bytes to copy via @req_count argument.
1401407e9c63SDarrick J. Wong * Returns appropriate error code that caller should return or
1402407e9c63SDarrick J. Wong * zero in case the copy should be allowed.
1403407e9c63SDarrick J. Wong */
generic_copy_file_checks(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,size_t * req_count,unsigned int flags)1404407e9c63SDarrick J. Wong static int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
1405407e9c63SDarrick J. Wong struct file *file_out, loff_t pos_out,
1406407e9c63SDarrick J. Wong size_t *req_count, unsigned int flags)
1407407e9c63SDarrick J. Wong {
1408407e9c63SDarrick J. Wong struct inode *inode_in = file_inode(file_in);
1409407e9c63SDarrick J. Wong struct inode *inode_out = file_inode(file_out);
1410407e9c63SDarrick J. Wong uint64_t count = *req_count;
1411407e9c63SDarrick J. Wong loff_t size_in;
1412407e9c63SDarrick J. Wong int ret;
1413407e9c63SDarrick J. Wong
1414407e9c63SDarrick J. Wong ret = generic_file_rw_checks(file_in, file_out);
1415407e9c63SDarrick J. Wong if (ret)
1416407e9c63SDarrick J. Wong return ret;
1417407e9c63SDarrick J. Wong
1418868f9f2fSAmir Goldstein /*
1419868f9f2fSAmir Goldstein * We allow some filesystems to handle cross sb copy, but passing
1420868f9f2fSAmir Goldstein * a file of the wrong filesystem type to filesystem driver can result
1421868f9f2fSAmir Goldstein * in an attempt to dereference the wrong type of ->private_data, so
1422868f9f2fSAmir Goldstein * avoid doing that until we really have a good reason.
1423868f9f2fSAmir Goldstein *
1424868f9f2fSAmir Goldstein * nfs and cifs define several different file_system_type structures
1425868f9f2fSAmir Goldstein * and several different sets of file_operations, but they all end up
1426868f9f2fSAmir Goldstein * using the same ->copy_file_range() function pointer.
1427868f9f2fSAmir Goldstein */
142810bc8e4aSAmir Goldstein if (flags & COPY_FILE_SPLICE) {
142910bc8e4aSAmir Goldstein /* cross sb splice is allowed */
143010bc8e4aSAmir Goldstein } else if (file_out->f_op->copy_file_range) {
1431868f9f2fSAmir Goldstein if (file_in->f_op->copy_file_range !=
1432868f9f2fSAmir Goldstein file_out->f_op->copy_file_range)
1433868f9f2fSAmir Goldstein return -EXDEV;
1434868f9f2fSAmir Goldstein } else if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) {
1435868f9f2fSAmir Goldstein return -EXDEV;
1436868f9f2fSAmir Goldstein }
1437868f9f2fSAmir Goldstein
1438407e9c63SDarrick J. Wong /* Don't touch certain kinds of inodes */
1439407e9c63SDarrick J. Wong if (IS_IMMUTABLE(inode_out))
1440407e9c63SDarrick J. Wong return -EPERM;
1441407e9c63SDarrick J. Wong
1442407e9c63SDarrick J. Wong if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out))
1443407e9c63SDarrick J. Wong return -ETXTBSY;
1444407e9c63SDarrick J. Wong
1445407e9c63SDarrick J. Wong /* Ensure offsets don't wrap. */
1446407e9c63SDarrick J. Wong if (pos_in + count < pos_in || pos_out + count < pos_out)
1447407e9c63SDarrick J. Wong return -EOVERFLOW;
1448407e9c63SDarrick J. Wong
1449407e9c63SDarrick J. Wong /* Shorten the copy to EOF */
1450407e9c63SDarrick J. Wong size_in = i_size_read(inode_in);
1451407e9c63SDarrick J. Wong if (pos_in >= size_in)
1452407e9c63SDarrick J. Wong count = 0;
1453407e9c63SDarrick J. Wong else
1454407e9c63SDarrick J. Wong count = min(count, size_in - (uint64_t)pos_in);
1455407e9c63SDarrick J. Wong
1456407e9c63SDarrick J. Wong ret = generic_write_check_limits(file_out, pos_out, &count);
1457407e9c63SDarrick J. Wong if (ret)
1458407e9c63SDarrick J. Wong return ret;
1459407e9c63SDarrick J. Wong
1460407e9c63SDarrick J. Wong /* Don't allow overlapped copying within the same file. */
1461407e9c63SDarrick J. Wong if (inode_in == inode_out &&
1462407e9c63SDarrick J. Wong pos_out + count > pos_in &&
1463407e9c63SDarrick J. Wong pos_out < pos_in + count)
1464407e9c63SDarrick J. Wong return -EINVAL;
1465407e9c63SDarrick J. Wong
1466407e9c63SDarrick J. Wong *req_count = count;
1467407e9c63SDarrick J. Wong return 0;
1468407e9c63SDarrick J. Wong }
1469407e9c63SDarrick J. Wong
1470407e9c63SDarrick J. Wong /*
147129732938SZach Brown * copy_file_range() differs from regular file read and write in that it
147229732938SZach Brown * specifically allows return partial success. When it does so is up to
147329732938SZach Brown * the copy_file_range method.
147429732938SZach Brown */
vfs_copy_file_range(struct file * file_in,loff_t pos_in,struct file * file_out,loff_t pos_out,size_t len,unsigned int flags)147529732938SZach Brown ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in,
147629732938SZach Brown struct file *file_out, loff_t pos_out,
147729732938SZach Brown size_t len, unsigned int flags)
147829732938SZach Brown {
147929732938SZach Brown ssize_t ret;
148010bc8e4aSAmir Goldstein bool splice = flags & COPY_FILE_SPLICE;
148129732938SZach Brown
148210bc8e4aSAmir Goldstein if (flags & ~COPY_FILE_SPLICE)
148329732938SZach Brown return -EINVAL;
148429732938SZach Brown
148596e6e8f4SAmir Goldstein ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len,
148696e6e8f4SAmir Goldstein flags);
1487a3171351SAmir Goldstein if (unlikely(ret))
1488a3171351SAmir Goldstein return ret;
148911cbfb10SAmir Goldstein
149029732938SZach Brown ret = rw_verify_area(READ, file_in, &pos_in, len);
1491bc61384dSAl Viro if (unlikely(ret))
1492bc61384dSAl Viro return ret;
1493bc61384dSAl Viro
149429732938SZach Brown ret = rw_verify_area(WRITE, file_out, &pos_out, len);
1495bc61384dSAl Viro if (unlikely(ret))
149629732938SZach Brown return ret;
149729732938SZach Brown
149829732938SZach Brown if (len == 0)
149929732938SZach Brown return 0;
150029732938SZach Brown
1501bfe219d3SAmir Goldstein file_start_write(file_out);
150229732938SZach Brown
1503a76b5b04SChristoph Hellwig /*
1504868f9f2fSAmir Goldstein * Cloning is supported by more file systems, so we implement copy on
1505868f9f2fSAmir Goldstein * same sb using clone, but for filesystems where both clone and copy
1506868f9f2fSAmir Goldstein * are supported (e.g. nfs,cifs), we only call the copy method.
1507a76b5b04SChristoph Hellwig */
150810bc8e4aSAmir Goldstein if (!splice && file_out->f_op->copy_file_range) {
1509868f9f2fSAmir Goldstein ret = file_out->f_op->copy_file_range(file_in, pos_in,
1510868f9f2fSAmir Goldstein file_out, pos_out,
1511868f9f2fSAmir Goldstein len, flags);
1512868f9f2fSAmir Goldstein goto done;
1513868f9f2fSAmir Goldstein }
1514868f9f2fSAmir Goldstein
151510bc8e4aSAmir Goldstein if (!splice && file_in->f_op->remap_file_range &&
15165dae222aSAmir Goldstein file_inode(file_in)->i_sb == file_inode(file_out)->i_sb) {
1517868f9f2fSAmir Goldstein ret = file_in->f_op->remap_file_range(file_in, pos_in,
151842ec3d4cSDarrick J. Wong file_out, pos_out,
1519eca3654eSDarrick J. Wong min_t(loff_t, MAX_RW_COUNT, len),
1520eca3654eSDarrick J. Wong REMAP_FILE_CAN_SHORTEN);
1521868f9f2fSAmir Goldstein if (ret > 0)
1522a76b5b04SChristoph Hellwig goto done;
1523a76b5b04SChristoph Hellwig }
1524a76b5b04SChristoph Hellwig
1525868f9f2fSAmir Goldstein /*
1526868f9f2fSAmir Goldstein * We can get here for same sb copy of filesystems that do not implement
1527868f9f2fSAmir Goldstein * ->copy_file_range() in case filesystem does not support clone or in
1528868f9f2fSAmir Goldstein * case filesystem supports clone but rejected the clone request (e.g.
1529868f9f2fSAmir Goldstein * because it was not block aligned).
1530868f9f2fSAmir Goldstein *
1531868f9f2fSAmir Goldstein * In both cases, fall back to kernel copy so we are able to maintain a
1532868f9f2fSAmir Goldstein * consistent story about which filesystems support copy_file_range()
1533868f9f2fSAmir Goldstein * and which filesystems do not, that will allow userspace tools to
1534868f9f2fSAmir Goldstein * make consistent desicions w.r.t using copy_file_range().
153510bc8e4aSAmir Goldstein *
153610bc8e4aSAmir Goldstein * We also get here if caller (e.g. nfsd) requested COPY_FILE_SPLICE.
1537868f9f2fSAmir Goldstein */
1538868f9f2fSAmir Goldstein ret = generic_copy_file_range(file_in, pos_in, file_out, pos_out, len,
1539f16acc9dSDave Chinner flags);
1540868f9f2fSAmir Goldstein
1541a76b5b04SChristoph Hellwig done:
154229732938SZach Brown if (ret > 0) {
154329732938SZach Brown fsnotify_access(file_in);
154429732938SZach Brown add_rchar(current, ret);
154529732938SZach Brown fsnotify_modify(file_out);
154629732938SZach Brown add_wchar(current, ret);
154729732938SZach Brown }
1548a76b5b04SChristoph Hellwig
154929732938SZach Brown inc_syscr(current);
155029732938SZach Brown inc_syscw(current);
155129732938SZach Brown
1552bfe219d3SAmir Goldstein file_end_write(file_out);
155329732938SZach Brown
155429732938SZach Brown return ret;
155529732938SZach Brown }
155629732938SZach Brown EXPORT_SYMBOL(vfs_copy_file_range);
155729732938SZach Brown
SYSCALL_DEFINE6(copy_file_range,int,fd_in,loff_t __user *,off_in,int,fd_out,loff_t __user *,off_out,size_t,len,unsigned int,flags)155829732938SZach Brown SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in,
155929732938SZach Brown int, fd_out, loff_t __user *, off_out,
156029732938SZach Brown size_t, len, unsigned int, flags)
156129732938SZach Brown {
156229732938SZach Brown loff_t pos_in;
156329732938SZach Brown loff_t pos_out;
156429732938SZach Brown struct fd f_in;
156529732938SZach Brown struct fd f_out;
156629732938SZach Brown ssize_t ret = -EBADF;
156729732938SZach Brown
156829732938SZach Brown f_in = fdget(fd_in);
156929732938SZach Brown if (!f_in.file)
157029732938SZach Brown goto out2;
157129732938SZach Brown
157229732938SZach Brown f_out = fdget(fd_out);
157329732938SZach Brown if (!f_out.file)
157429732938SZach Brown goto out1;
157529732938SZach Brown
157629732938SZach Brown ret = -EFAULT;
157729732938SZach Brown if (off_in) {
157829732938SZach Brown if (copy_from_user(&pos_in, off_in, sizeof(loff_t)))
157929732938SZach Brown goto out;
158029732938SZach Brown } else {
158129732938SZach Brown pos_in = f_in.file->f_pos;
158229732938SZach Brown }
158329732938SZach Brown
158429732938SZach Brown if (off_out) {
158529732938SZach Brown if (copy_from_user(&pos_out, off_out, sizeof(loff_t)))
158629732938SZach Brown goto out;
158729732938SZach Brown } else {
158829732938SZach Brown pos_out = f_out.file->f_pos;
158929732938SZach Brown }
159029732938SZach Brown
159110bc8e4aSAmir Goldstein ret = -EINVAL;
159210bc8e4aSAmir Goldstein if (flags != 0)
159310bc8e4aSAmir Goldstein goto out;
159410bc8e4aSAmir Goldstein
159529732938SZach Brown ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len,
159629732938SZach Brown flags);
159729732938SZach Brown if (ret > 0) {
159829732938SZach Brown pos_in += ret;
159929732938SZach Brown pos_out += ret;
160029732938SZach Brown
160129732938SZach Brown if (off_in) {
160229732938SZach Brown if (copy_to_user(off_in, &pos_in, sizeof(loff_t)))
160329732938SZach Brown ret = -EFAULT;
160429732938SZach Brown } else {
160529732938SZach Brown f_in.file->f_pos = pos_in;
160629732938SZach Brown }
160729732938SZach Brown
160829732938SZach Brown if (off_out) {
160929732938SZach Brown if (copy_to_user(off_out, &pos_out, sizeof(loff_t)))
161029732938SZach Brown ret = -EFAULT;
161129732938SZach Brown } else {
161229732938SZach Brown f_out.file->f_pos = pos_out;
161329732938SZach Brown }
161429732938SZach Brown }
161529732938SZach Brown
161629732938SZach Brown out:
161729732938SZach Brown fdput(f_out);
161829732938SZach Brown out1:
161929732938SZach Brown fdput(f_in);
162029732938SZach Brown out2:
162129732938SZach Brown return ret;
162229732938SZach Brown }
162304b38d60SChristoph Hellwig
1624407e9c63SDarrick J. Wong /*
1625407e9c63SDarrick J. Wong * Don't operate on ranges the page cache doesn't support, and don't exceed the
1626407e9c63SDarrick J. Wong * LFS limits. If pos is under the limit it becomes a short access. If it
1627407e9c63SDarrick J. Wong * exceeds the limit we return -EFBIG.
1628407e9c63SDarrick J. Wong */
generic_write_check_limits(struct file * file,loff_t pos,loff_t * count)1629407e9c63SDarrick J. Wong int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count)
163004b38d60SChristoph Hellwig {
1631407e9c63SDarrick J. Wong struct inode *inode = file->f_mapping->host;
1632407e9c63SDarrick J. Wong loff_t max_size = inode->i_sb->s_maxbytes;
1633407e9c63SDarrick J. Wong loff_t limit = rlimit(RLIMIT_FSIZE);
163404b38d60SChristoph Hellwig
1635407e9c63SDarrick J. Wong if (limit != RLIM_INFINITY) {
1636407e9c63SDarrick J. Wong if (pos >= limit) {
1637407e9c63SDarrick J. Wong send_sig(SIGXFSZ, current, 0);
1638407e9c63SDarrick J. Wong return -EFBIG;
1639407e9c63SDarrick J. Wong }
1640407e9c63SDarrick J. Wong *count = min(*count, limit - pos);
1641407e9c63SDarrick J. Wong }
1642407e9c63SDarrick J. Wong
1643407e9c63SDarrick J. Wong if (!(file->f_flags & O_LARGEFILE))
1644407e9c63SDarrick J. Wong max_size = MAX_NON_LFS;
1645407e9c63SDarrick J. Wong
1646407e9c63SDarrick J. Wong if (unlikely(pos >= max_size))
1647407e9c63SDarrick J. Wong return -EFBIG;
1648407e9c63SDarrick J. Wong
1649407e9c63SDarrick J. Wong *count = min(*count, max_size - pos);
1650407e9c63SDarrick J. Wong
1651407e9c63SDarrick J. Wong return 0;
1652407e9c63SDarrick J. Wong }
1653407e9c63SDarrick J. Wong
1654f6f7a25aSOmar Sandoval /* Like generic_write_checks(), but takes size of write instead of iter. */
generic_write_checks_count(struct kiocb * iocb,loff_t * count)1655f6f7a25aSOmar Sandoval int generic_write_checks_count(struct kiocb *iocb, loff_t *count)
1656407e9c63SDarrick J. Wong {
1657407e9c63SDarrick J. Wong struct file *file = iocb->ki_filp;
1658407e9c63SDarrick J. Wong struct inode *inode = file->f_mapping->host;
1659407e9c63SDarrick J. Wong
1660407e9c63SDarrick J. Wong if (IS_SWAPFILE(inode))
1661407e9c63SDarrick J. Wong return -ETXTBSY;
1662407e9c63SDarrick J. Wong
1663f6f7a25aSOmar Sandoval if (!*count)
1664407e9c63SDarrick J. Wong return 0;
1665407e9c63SDarrick J. Wong
1666407e9c63SDarrick J. Wong if (iocb->ki_flags & IOCB_APPEND)
1667407e9c63SDarrick J. Wong iocb->ki_pos = i_size_read(inode);
1668407e9c63SDarrick J. Wong
166980175539SStefan Roesch if ((iocb->ki_flags & IOCB_NOWAIT) &&
167080175539SStefan Roesch !((iocb->ki_flags & IOCB_DIRECT) ||
167180175539SStefan Roesch (file->f_mode & FMODE_BUF_WASYNC)))
167204b38d60SChristoph Hellwig return -EINVAL;
167304b38d60SChristoph Hellwig
1674f6f7a25aSOmar Sandoval return generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, count);
1675f6f7a25aSOmar Sandoval }
1676f6f7a25aSOmar Sandoval EXPORT_SYMBOL(generic_write_checks_count);
1677f6f7a25aSOmar Sandoval
1678f6f7a25aSOmar Sandoval /*
1679f6f7a25aSOmar Sandoval * Performs necessary checks before doing a write
1680f6f7a25aSOmar Sandoval *
1681f6f7a25aSOmar Sandoval * Can adjust writing position or amount of bytes to write.
1682f6f7a25aSOmar Sandoval * Returns appropriate error code that caller should return or
1683f6f7a25aSOmar Sandoval * zero in case that write should be allowed.
1684f6f7a25aSOmar Sandoval */
generic_write_checks(struct kiocb * iocb,struct iov_iter * from)1685f6f7a25aSOmar Sandoval ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from)
1686f6f7a25aSOmar Sandoval {
1687f6f7a25aSOmar Sandoval loff_t count = iov_iter_count(from);
1688f6f7a25aSOmar Sandoval int ret;
1689f6f7a25aSOmar Sandoval
1690f6f7a25aSOmar Sandoval ret = generic_write_checks_count(iocb, &count);
1691407e9c63SDarrick J. Wong if (ret)
1692407e9c63SDarrick J. Wong return ret;
169304b38d60SChristoph Hellwig
1694407e9c63SDarrick J. Wong iov_iter_truncate(from, count);
1695407e9c63SDarrick J. Wong return iov_iter_count(from);
169604b38d60SChristoph Hellwig }
1697407e9c63SDarrick J. Wong EXPORT_SYMBOL(generic_write_checks);
1698876bec6fSDarrick J. Wong
1699876bec6fSDarrick J. Wong /*
1700407e9c63SDarrick J. Wong * Performs common checks before doing a file copy/clone
1701407e9c63SDarrick J. Wong * from @file_in to @file_out.
1702edc58dd0SDarrick J. Wong */
generic_file_rw_checks(struct file * file_in,struct file * file_out)1703407e9c63SDarrick J. Wong int generic_file_rw_checks(struct file *file_in, struct file *file_out)
17041da177e4SLinus Torvalds {
17051383a7edSDarrick J. Wong struct inode *inode_in = file_inode(file_in);
17061383a7edSDarrick J. Wong struct inode *inode_out = file_inode(file_out);
17071da177e4SLinus Torvalds
1708407e9c63SDarrick J. Wong /* Don't copy dirs, pipes, sockets... */
17091da177e4SLinus Torvalds if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
17101da177e4SLinus Torvalds return -EISDIR;
17111da177e4SLinus Torvalds if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
17121da177e4SLinus Torvalds return -EINVAL;
17131da177e4SLinus Torvalds
1714407e9c63SDarrick J. Wong if (!(file_in->f_mode & FMODE_READ) ||
1715407e9c63SDarrick J. Wong !(file_out->f_mode & FMODE_WRITE) ||
1716407e9c63SDarrick J. Wong (file_out->f_flags & O_APPEND))
1717407e9c63SDarrick J. Wong return -EBADF;
17181383a7edSDarrick J. Wong
17192c5773f1SDarrick J. Wong return 0;
17201da177e4SLinus Torvalds }
1721