xref: /openbmc/linux/fs/read_write.c (revision 22246614)
1 /*
2  *  linux/fs/read_write.c
3  *
4  *  Copyright (C) 1991, 1992  Linus Torvalds
5  */
6 
7 #include <linux/slab.h>
8 #include <linux/stat.h>
9 #include <linux/fcntl.h>
10 #include <linux/file.h>
11 #include <linux/uio.h>
12 #include <linux/smp_lock.h>
13 #include <linux/fsnotify.h>
14 #include <linux/security.h>
15 #include <linux/module.h>
16 #include <linux/syscalls.h>
17 #include <linux/pagemap.h>
18 #include <linux/splice.h>
19 #include "read_write.h"
20 
21 #include <asm/uaccess.h>
22 #include <asm/unistd.h>
23 
24 const struct file_operations generic_ro_fops = {
25 	.llseek		= generic_file_llseek,
26 	.read		= do_sync_read,
27 	.aio_read	= generic_file_aio_read,
28 	.mmap		= generic_file_readonly_mmap,
29 	.splice_read	= generic_file_splice_read,
30 };
31 
32 EXPORT_SYMBOL(generic_ro_fops);
33 
34 loff_t generic_file_llseek(struct file *file, loff_t offset, int origin)
35 {
36 	loff_t retval;
37 	struct inode *inode = file->f_mapping->host;
38 
39 	mutex_lock(&inode->i_mutex);
40 	switch (origin) {
41 		case SEEK_END:
42 			offset += inode->i_size;
43 			break;
44 		case SEEK_CUR:
45 			offset += file->f_pos;
46 	}
47 	retval = -EINVAL;
48 	if (offset>=0 && offset<=inode->i_sb->s_maxbytes) {
49 		if (offset != file->f_pos) {
50 			file->f_pos = offset;
51 			file->f_version = 0;
52 		}
53 		retval = offset;
54 	}
55 	mutex_unlock(&inode->i_mutex);
56 	return retval;
57 }
58 
59 EXPORT_SYMBOL(generic_file_llseek);
60 
61 loff_t remote_llseek(struct file *file, loff_t offset, int origin)
62 {
63 	loff_t retval;
64 
65 	lock_kernel();
66 	switch (origin) {
67 		case SEEK_END:
68 			offset += i_size_read(file->f_path.dentry->d_inode);
69 			break;
70 		case SEEK_CUR:
71 			offset += file->f_pos;
72 	}
73 	retval = -EINVAL;
74 	if (offset>=0 && offset<=file->f_path.dentry->d_inode->i_sb->s_maxbytes) {
75 		if (offset != file->f_pos) {
76 			file->f_pos = offset;
77 			file->f_version = 0;
78 		}
79 		retval = offset;
80 	}
81 	unlock_kernel();
82 	return retval;
83 }
84 EXPORT_SYMBOL(remote_llseek);
85 
86 loff_t no_llseek(struct file *file, loff_t offset, int origin)
87 {
88 	return -ESPIPE;
89 }
90 EXPORT_SYMBOL(no_llseek);
91 
92 loff_t default_llseek(struct file *file, loff_t offset, int origin)
93 {
94 	loff_t retval;
95 
96 	lock_kernel();
97 	switch (origin) {
98 		case SEEK_END:
99 			offset += i_size_read(file->f_path.dentry->d_inode);
100 			break;
101 		case SEEK_CUR:
102 			offset += file->f_pos;
103 	}
104 	retval = -EINVAL;
105 	if (offset >= 0) {
106 		if (offset != file->f_pos) {
107 			file->f_pos = offset;
108 			file->f_version = 0;
109 		}
110 		retval = offset;
111 	}
112 	unlock_kernel();
113 	return retval;
114 }
115 EXPORT_SYMBOL(default_llseek);
116 
117 loff_t vfs_llseek(struct file *file, loff_t offset, int origin)
118 {
119 	loff_t (*fn)(struct file *, loff_t, int);
120 
121 	fn = no_llseek;
122 	if (file->f_mode & FMODE_LSEEK) {
123 		fn = default_llseek;
124 		if (file->f_op && file->f_op->llseek)
125 			fn = file->f_op->llseek;
126 	}
127 	return fn(file, offset, origin);
128 }
129 EXPORT_SYMBOL(vfs_llseek);
130 
131 asmlinkage off_t sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
132 {
133 	off_t retval;
134 	struct file * file;
135 	int fput_needed;
136 
137 	retval = -EBADF;
138 	file = fget_light(fd, &fput_needed);
139 	if (!file)
140 		goto bad;
141 
142 	retval = -EINVAL;
143 	if (origin <= SEEK_MAX) {
144 		loff_t res = vfs_llseek(file, offset, origin);
145 		retval = res;
146 		if (res != (loff_t)retval)
147 			retval = -EOVERFLOW;	/* LFS: should only happen on 32 bit platforms */
148 	}
149 	fput_light(file, fput_needed);
150 bad:
151 	return retval;
152 }
153 
154 #ifdef __ARCH_WANT_SYS_LLSEEK
155 asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
156 			   unsigned long offset_low, loff_t __user * result,
157 			   unsigned int origin)
158 {
159 	int retval;
160 	struct file * file;
161 	loff_t offset;
162 	int fput_needed;
163 
164 	retval = -EBADF;
165 	file = fget_light(fd, &fput_needed);
166 	if (!file)
167 		goto bad;
168 
169 	retval = -EINVAL;
170 	if (origin > SEEK_MAX)
171 		goto out_putf;
172 
173 	offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low,
174 			origin);
175 
176 	retval = (int)offset;
177 	if (offset >= 0) {
178 		retval = -EFAULT;
179 		if (!copy_to_user(result, &offset, sizeof(offset)))
180 			retval = 0;
181 	}
182 out_putf:
183 	fput_light(file, fput_needed);
184 bad:
185 	return retval;
186 }
187 #endif
188 
189 /*
190  * rw_verify_area doesn't like huge counts. We limit
191  * them to something that fits in "int" so that others
192  * won't have to do range checks all the time.
193  */
194 #define MAX_RW_COUNT (INT_MAX & PAGE_CACHE_MASK)
195 
196 int rw_verify_area(int read_write, struct file *file, loff_t *ppos, size_t count)
197 {
198 	struct inode *inode;
199 	loff_t pos;
200 	int retval = -EINVAL;
201 
202 	inode = file->f_path.dentry->d_inode;
203 	if (unlikely((ssize_t) count < 0))
204 		return retval;
205 	pos = *ppos;
206 	if (unlikely((pos < 0) || (loff_t) (pos + count) < 0))
207 		return retval;
208 
209 	if (unlikely(inode->i_flock && mandatory_lock(inode))) {
210 		retval = locks_mandatory_area(
211 			read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE,
212 			inode, file, pos, count);
213 		if (retval < 0)
214 			return retval;
215 	}
216 	retval = security_file_permission(file,
217 				read_write == READ ? MAY_READ : MAY_WRITE);
218 	if (retval)
219 		return retval;
220 	return count > MAX_RW_COUNT ? MAX_RW_COUNT : count;
221 }
222 
223 static void wait_on_retry_sync_kiocb(struct kiocb *iocb)
224 {
225 	set_current_state(TASK_UNINTERRUPTIBLE);
226 	if (!kiocbIsKicked(iocb))
227 		schedule();
228 	else
229 		kiocbClearKicked(iocb);
230 	__set_current_state(TASK_RUNNING);
231 }
232 
233 ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos)
234 {
235 	struct iovec iov = { .iov_base = buf, .iov_len = len };
236 	struct kiocb kiocb;
237 	ssize_t ret;
238 
239 	init_sync_kiocb(&kiocb, filp);
240 	kiocb.ki_pos = *ppos;
241 	kiocb.ki_left = len;
242 
243 	for (;;) {
244 		ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos);
245 		if (ret != -EIOCBRETRY)
246 			break;
247 		wait_on_retry_sync_kiocb(&kiocb);
248 	}
249 
250 	if (-EIOCBQUEUED == ret)
251 		ret = wait_on_sync_kiocb(&kiocb);
252 	*ppos = kiocb.ki_pos;
253 	return ret;
254 }
255 
256 EXPORT_SYMBOL(do_sync_read);
257 
258 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos)
259 {
260 	ssize_t ret;
261 
262 	if (!(file->f_mode & FMODE_READ))
263 		return -EBADF;
264 	if (!file->f_op || (!file->f_op->read && !file->f_op->aio_read))
265 		return -EINVAL;
266 	if (unlikely(!access_ok(VERIFY_WRITE, buf, count)))
267 		return -EFAULT;
268 
269 	ret = rw_verify_area(READ, file, pos, count);
270 	if (ret >= 0) {
271 		count = ret;
272 		if (file->f_op->read)
273 			ret = file->f_op->read(file, buf, count, pos);
274 		else
275 			ret = do_sync_read(file, buf, count, pos);
276 		if (ret > 0) {
277 			fsnotify_access(file->f_path.dentry);
278 			add_rchar(current, ret);
279 		}
280 		inc_syscr(current);
281 	}
282 
283 	return ret;
284 }
285 
286 EXPORT_SYMBOL(vfs_read);
287 
288 ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos)
289 {
290 	struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len };
291 	struct kiocb kiocb;
292 	ssize_t ret;
293 
294 	init_sync_kiocb(&kiocb, filp);
295 	kiocb.ki_pos = *ppos;
296 	kiocb.ki_left = len;
297 
298 	for (;;) {
299 		ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos);
300 		if (ret != -EIOCBRETRY)
301 			break;
302 		wait_on_retry_sync_kiocb(&kiocb);
303 	}
304 
305 	if (-EIOCBQUEUED == ret)
306 		ret = wait_on_sync_kiocb(&kiocb);
307 	*ppos = kiocb.ki_pos;
308 	return ret;
309 }
310 
311 EXPORT_SYMBOL(do_sync_write);
312 
313 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos)
314 {
315 	ssize_t ret;
316 
317 	if (!(file->f_mode & FMODE_WRITE))
318 		return -EBADF;
319 	if (!file->f_op || (!file->f_op->write && !file->f_op->aio_write))
320 		return -EINVAL;
321 	if (unlikely(!access_ok(VERIFY_READ, buf, count)))
322 		return -EFAULT;
323 
324 	ret = rw_verify_area(WRITE, file, pos, count);
325 	if (ret >= 0) {
326 		count = ret;
327 		if (file->f_op->write)
328 			ret = file->f_op->write(file, buf, count, pos);
329 		else
330 			ret = do_sync_write(file, buf, count, pos);
331 		if (ret > 0) {
332 			fsnotify_modify(file->f_path.dentry);
333 			add_wchar(current, ret);
334 		}
335 		inc_syscw(current);
336 	}
337 
338 	return ret;
339 }
340 
341 EXPORT_SYMBOL(vfs_write);
342 
343 static inline loff_t file_pos_read(struct file *file)
344 {
345 	return file->f_pos;
346 }
347 
348 static inline void file_pos_write(struct file *file, loff_t pos)
349 {
350 	file->f_pos = pos;
351 }
352 
353 asmlinkage ssize_t sys_read(unsigned int fd, char __user * buf, size_t count)
354 {
355 	struct file *file;
356 	ssize_t ret = -EBADF;
357 	int fput_needed;
358 
359 	file = fget_light(fd, &fput_needed);
360 	if (file) {
361 		loff_t pos = file_pos_read(file);
362 		ret = vfs_read(file, buf, count, &pos);
363 		file_pos_write(file, pos);
364 		fput_light(file, fput_needed);
365 	}
366 
367 	return ret;
368 }
369 
370 asmlinkage ssize_t sys_write(unsigned int fd, const char __user * buf, size_t count)
371 {
372 	struct file *file;
373 	ssize_t ret = -EBADF;
374 	int fput_needed;
375 
376 	file = fget_light(fd, &fput_needed);
377 	if (file) {
378 		loff_t pos = file_pos_read(file);
379 		ret = vfs_write(file, buf, count, &pos);
380 		file_pos_write(file, pos);
381 		fput_light(file, fput_needed);
382 	}
383 
384 	return ret;
385 }
386 
387 asmlinkage ssize_t sys_pread64(unsigned int fd, char __user *buf,
388 			     size_t count, loff_t pos)
389 {
390 	struct file *file;
391 	ssize_t ret = -EBADF;
392 	int fput_needed;
393 
394 	if (pos < 0)
395 		return -EINVAL;
396 
397 	file = fget_light(fd, &fput_needed);
398 	if (file) {
399 		ret = -ESPIPE;
400 		if (file->f_mode & FMODE_PREAD)
401 			ret = vfs_read(file, buf, count, &pos);
402 		fput_light(file, fput_needed);
403 	}
404 
405 	return ret;
406 }
407 
408 asmlinkage ssize_t sys_pwrite64(unsigned int fd, const char __user *buf,
409 			      size_t count, loff_t pos)
410 {
411 	struct file *file;
412 	ssize_t ret = -EBADF;
413 	int fput_needed;
414 
415 	if (pos < 0)
416 		return -EINVAL;
417 
418 	file = fget_light(fd, &fput_needed);
419 	if (file) {
420 		ret = -ESPIPE;
421 		if (file->f_mode & FMODE_PWRITE)
422 			ret = vfs_write(file, buf, count, &pos);
423 		fput_light(file, fput_needed);
424 	}
425 
426 	return ret;
427 }
428 
429 /*
430  * Reduce an iovec's length in-place.  Return the resulting number of segments
431  */
432 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to)
433 {
434 	unsigned long seg = 0;
435 	size_t len = 0;
436 
437 	while (seg < nr_segs) {
438 		seg++;
439 		if (len + iov->iov_len >= to) {
440 			iov->iov_len = to - len;
441 			break;
442 		}
443 		len += iov->iov_len;
444 		iov++;
445 	}
446 	return seg;
447 }
448 EXPORT_SYMBOL(iov_shorten);
449 
450 ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov,
451 		unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn)
452 {
453 	struct kiocb kiocb;
454 	ssize_t ret;
455 
456 	init_sync_kiocb(&kiocb, filp);
457 	kiocb.ki_pos = *ppos;
458 	kiocb.ki_left = len;
459 	kiocb.ki_nbytes = len;
460 
461 	for (;;) {
462 		ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos);
463 		if (ret != -EIOCBRETRY)
464 			break;
465 		wait_on_retry_sync_kiocb(&kiocb);
466 	}
467 
468 	if (ret == -EIOCBQUEUED)
469 		ret = wait_on_sync_kiocb(&kiocb);
470 	*ppos = kiocb.ki_pos;
471 	return ret;
472 }
473 
474 /* Do it by hand, with file-ops */
475 ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov,
476 		unsigned long nr_segs, loff_t *ppos, io_fn_t fn)
477 {
478 	struct iovec *vector = iov;
479 	ssize_t ret = 0;
480 
481 	while (nr_segs > 0) {
482 		void __user *base;
483 		size_t len;
484 		ssize_t nr;
485 
486 		base = vector->iov_base;
487 		len = vector->iov_len;
488 		vector++;
489 		nr_segs--;
490 
491 		nr = fn(filp, base, len, ppos);
492 
493 		if (nr < 0) {
494 			if (!ret)
495 				ret = nr;
496 			break;
497 		}
498 		ret += nr;
499 		if (nr != len)
500 			break;
501 	}
502 
503 	return ret;
504 }
505 
506 /* A write operation does a read from user space and vice versa */
507 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ)
508 
509 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector,
510 			      unsigned long nr_segs, unsigned long fast_segs,
511 			      struct iovec *fast_pointer,
512 			      struct iovec **ret_pointer)
513   {
514 	unsigned long seg;
515   	ssize_t ret;
516 	struct iovec *iov = fast_pointer;
517 
518   	/*
519   	 * SuS says "The readv() function *may* fail if the iovcnt argument
520   	 * was less than or equal to 0, or greater than {IOV_MAX}.  Linux has
521   	 * traditionally returned zero for zero segments, so...
522   	 */
523 	if (nr_segs == 0) {
524 		ret = 0;
525   		goto out;
526 	}
527 
528   	/*
529   	 * First get the "struct iovec" from user memory and
530   	 * verify all the pointers
531   	 */
532 	if (nr_segs > UIO_MAXIOV) {
533 		ret = -EINVAL;
534   		goto out;
535 	}
536 	if (nr_segs > fast_segs) {
537   		iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL);
538 		if (iov == NULL) {
539 			ret = -ENOMEM;
540   			goto out;
541 		}
542   	}
543 	if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) {
544 		ret = -EFAULT;
545   		goto out;
546 	}
547 
548   	/*
549 	 * According to the Single Unix Specification we should return EINVAL
550 	 * if an element length is < 0 when cast to ssize_t or if the
551 	 * total length would overflow the ssize_t return value of the
552 	 * system call.
553   	 */
554 	ret = 0;
555   	for (seg = 0; seg < nr_segs; seg++) {
556   		void __user *buf = iov[seg].iov_base;
557   		ssize_t len = (ssize_t)iov[seg].iov_len;
558 
559 		/* see if we we're about to use an invalid len or if
560 		 * it's about to overflow ssize_t */
561 		if (len < 0 || (ret + len < ret)) {
562 			ret = -EINVAL;
563   			goto out;
564 		}
565 		if (unlikely(!access_ok(vrfy_dir(type), buf, len))) {
566 			ret = -EFAULT;
567   			goto out;
568 		}
569 
570 		ret += len;
571   	}
572 out:
573 	*ret_pointer = iov;
574 	return ret;
575 }
576 
577 static ssize_t do_readv_writev(int type, struct file *file,
578 			       const struct iovec __user * uvector,
579 			       unsigned long nr_segs, loff_t *pos)
580 {
581 	size_t tot_len;
582 	struct iovec iovstack[UIO_FASTIOV];
583 	struct iovec *iov = iovstack;
584 	ssize_t ret;
585 	io_fn_t fn;
586 	iov_fn_t fnv;
587 
588 	if (!file->f_op) {
589 		ret = -EINVAL;
590 		goto out;
591 	}
592 
593 	ret = rw_copy_check_uvector(type, uvector, nr_segs,
594 			ARRAY_SIZE(iovstack), iovstack, &iov);
595 	if (ret <= 0)
596 		goto out;
597 
598 	tot_len = ret;
599 	ret = rw_verify_area(type, file, pos, tot_len);
600 	if (ret < 0)
601 		goto out;
602 
603 	fnv = NULL;
604 	if (type == READ) {
605 		fn = file->f_op->read;
606 		fnv = file->f_op->aio_read;
607 	} else {
608 		fn = (io_fn_t)file->f_op->write;
609 		fnv = file->f_op->aio_write;
610 	}
611 
612 	if (fnv)
613 		ret = do_sync_readv_writev(file, iov, nr_segs, tot_len,
614 						pos, fnv);
615 	else
616 		ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn);
617 
618 out:
619 	if (iov != iovstack)
620 		kfree(iov);
621 	if ((ret + (type == READ)) > 0) {
622 		if (type == READ)
623 			fsnotify_access(file->f_path.dentry);
624 		else
625 			fsnotify_modify(file->f_path.dentry);
626 	}
627 	return ret;
628 }
629 
630 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec,
631 		  unsigned long vlen, loff_t *pos)
632 {
633 	if (!(file->f_mode & FMODE_READ))
634 		return -EBADF;
635 	if (!file->f_op || (!file->f_op->aio_read && !file->f_op->read))
636 		return -EINVAL;
637 
638 	return do_readv_writev(READ, file, vec, vlen, pos);
639 }
640 
641 EXPORT_SYMBOL(vfs_readv);
642 
643 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec,
644 		   unsigned long vlen, loff_t *pos)
645 {
646 	if (!(file->f_mode & FMODE_WRITE))
647 		return -EBADF;
648 	if (!file->f_op || (!file->f_op->aio_write && !file->f_op->write))
649 		return -EINVAL;
650 
651 	return do_readv_writev(WRITE, file, vec, vlen, pos);
652 }
653 
654 EXPORT_SYMBOL(vfs_writev);
655 
656 asmlinkage ssize_t
657 sys_readv(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
658 {
659 	struct file *file;
660 	ssize_t ret = -EBADF;
661 	int fput_needed;
662 
663 	file = fget_light(fd, &fput_needed);
664 	if (file) {
665 		loff_t pos = file_pos_read(file);
666 		ret = vfs_readv(file, vec, vlen, &pos);
667 		file_pos_write(file, pos);
668 		fput_light(file, fput_needed);
669 	}
670 
671 	if (ret > 0)
672 		add_rchar(current, ret);
673 	inc_syscr(current);
674 	return ret;
675 }
676 
677 asmlinkage ssize_t
678 sys_writev(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
679 {
680 	struct file *file;
681 	ssize_t ret = -EBADF;
682 	int fput_needed;
683 
684 	file = fget_light(fd, &fput_needed);
685 	if (file) {
686 		loff_t pos = file_pos_read(file);
687 		ret = vfs_writev(file, vec, vlen, &pos);
688 		file_pos_write(file, pos);
689 		fput_light(file, fput_needed);
690 	}
691 
692 	if (ret > 0)
693 		add_wchar(current, ret);
694 	inc_syscw(current);
695 	return ret;
696 }
697 
698 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
699 			   size_t count, loff_t max)
700 {
701 	struct file * in_file, * out_file;
702 	struct inode * in_inode, * out_inode;
703 	loff_t pos;
704 	ssize_t retval;
705 	int fput_needed_in, fput_needed_out, fl;
706 
707 	/*
708 	 * Get input file, and verify that it is ok..
709 	 */
710 	retval = -EBADF;
711 	in_file = fget_light(in_fd, &fput_needed_in);
712 	if (!in_file)
713 		goto out;
714 	if (!(in_file->f_mode & FMODE_READ))
715 		goto fput_in;
716 	retval = -EINVAL;
717 	in_inode = in_file->f_path.dentry->d_inode;
718 	if (!in_inode)
719 		goto fput_in;
720 	if (!in_file->f_op || !in_file->f_op->splice_read)
721 		goto fput_in;
722 	retval = -ESPIPE;
723 	if (!ppos)
724 		ppos = &in_file->f_pos;
725 	else
726 		if (!(in_file->f_mode & FMODE_PREAD))
727 			goto fput_in;
728 	retval = rw_verify_area(READ, in_file, ppos, count);
729 	if (retval < 0)
730 		goto fput_in;
731 	count = retval;
732 
733 	/*
734 	 * Get output file, and verify that it is ok..
735 	 */
736 	retval = -EBADF;
737 	out_file = fget_light(out_fd, &fput_needed_out);
738 	if (!out_file)
739 		goto fput_in;
740 	if (!(out_file->f_mode & FMODE_WRITE))
741 		goto fput_out;
742 	retval = -EINVAL;
743 	if (!out_file->f_op || !out_file->f_op->sendpage)
744 		goto fput_out;
745 	out_inode = out_file->f_path.dentry->d_inode;
746 	retval = rw_verify_area(WRITE, out_file, &out_file->f_pos, count);
747 	if (retval < 0)
748 		goto fput_out;
749 	count = retval;
750 
751 	if (!max)
752 		max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes);
753 
754 	pos = *ppos;
755 	retval = -EINVAL;
756 	if (unlikely(pos < 0))
757 		goto fput_out;
758 	if (unlikely(pos + count > max)) {
759 		retval = -EOVERFLOW;
760 		if (pos >= max)
761 			goto fput_out;
762 		count = max - pos;
763 	}
764 
765 	fl = 0;
766 #if 0
767 	/*
768 	 * We need to debate whether we can enable this or not. The
769 	 * man page documents EAGAIN return for the output at least,
770 	 * and the application is arguably buggy if it doesn't expect
771 	 * EAGAIN on a non-blocking file descriptor.
772 	 */
773 	if (in_file->f_flags & O_NONBLOCK)
774 		fl = SPLICE_F_NONBLOCK;
775 #endif
776 	retval = do_splice_direct(in_file, ppos, out_file, count, fl);
777 
778 	if (retval > 0) {
779 		add_rchar(current, retval);
780 		add_wchar(current, retval);
781 	}
782 
783 	inc_syscr(current);
784 	inc_syscw(current);
785 	if (*ppos > max)
786 		retval = -EOVERFLOW;
787 
788 fput_out:
789 	fput_light(out_file, fput_needed_out);
790 fput_in:
791 	fput_light(in_file, fput_needed_in);
792 out:
793 	return retval;
794 }
795 
796 asmlinkage ssize_t sys_sendfile(int out_fd, int in_fd, off_t __user *offset, size_t count)
797 {
798 	loff_t pos;
799 	off_t off;
800 	ssize_t ret;
801 
802 	if (offset) {
803 		if (unlikely(get_user(off, offset)))
804 			return -EFAULT;
805 		pos = off;
806 		ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS);
807 		if (unlikely(put_user(pos, offset)))
808 			return -EFAULT;
809 		return ret;
810 	}
811 
812 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
813 }
814 
815 asmlinkage ssize_t sys_sendfile64(int out_fd, int in_fd, loff_t __user *offset, size_t count)
816 {
817 	loff_t pos;
818 	ssize_t ret;
819 
820 	if (offset) {
821 		if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t))))
822 			return -EFAULT;
823 		ret = do_sendfile(out_fd, in_fd, &pos, count, 0);
824 		if (unlikely(put_user(pos, offset)))
825 			return -EFAULT;
826 		return ret;
827 	}
828 
829 	return do_sendfile(out_fd, in_fd, NULL, count, 0);
830 }
831