1 /* 2 * linux/fs/read_write.c 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 */ 6 7 #include <linux/slab.h> 8 #include <linux/stat.h> 9 #include <linux/fcntl.h> 10 #include <linux/file.h> 11 #include <linux/uio.h> 12 #include <linux/aio.h> 13 #include <linux/fsnotify.h> 14 #include <linux/security.h> 15 #include <linux/export.h> 16 #include <linux/syscalls.h> 17 #include <linux/pagemap.h> 18 #include <linux/splice.h> 19 #include <linux/compat.h> 20 #include "internal.h" 21 22 #include <asm/uaccess.h> 23 #include <asm/unistd.h> 24 25 typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *); 26 typedef ssize_t (*iov_fn_t)(struct kiocb *, const struct iovec *, 27 unsigned long, loff_t); 28 typedef ssize_t (*iter_fn_t)(struct kiocb *, struct iov_iter *); 29 30 const struct file_operations generic_ro_fops = { 31 .llseek = generic_file_llseek, 32 .read = new_sync_read, 33 .read_iter = generic_file_read_iter, 34 .mmap = generic_file_readonly_mmap, 35 .splice_read = generic_file_splice_read, 36 }; 37 38 EXPORT_SYMBOL(generic_ro_fops); 39 40 static inline int unsigned_offsets(struct file *file) 41 { 42 return file->f_mode & FMODE_UNSIGNED_OFFSET; 43 } 44 45 /** 46 * vfs_setpos - update the file offset for lseek 47 * @file: file structure in question 48 * @offset: file offset to seek to 49 * @maxsize: maximum file size 50 * 51 * This is a low-level filesystem helper for updating the file offset to 52 * the value specified by @offset if the given offset is valid and it is 53 * not equal to the current file offset. 54 * 55 * Return the specified offset on success and -EINVAL on invalid offset. 56 */ 57 loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize) 58 { 59 if (offset < 0 && !unsigned_offsets(file)) 60 return -EINVAL; 61 if (offset > maxsize) 62 return -EINVAL; 63 64 if (offset != file->f_pos) { 65 file->f_pos = offset; 66 file->f_version = 0; 67 } 68 return offset; 69 } 70 EXPORT_SYMBOL(vfs_setpos); 71 72 /** 73 * generic_file_llseek_size - generic llseek implementation for regular files 74 * @file: file structure to seek on 75 * @offset: file offset to seek to 76 * @whence: type of seek 77 * @size: max size of this file in file system 78 * @eof: offset used for SEEK_END position 79 * 80 * This is a variant of generic_file_llseek that allows passing in a custom 81 * maximum file size and a custom EOF position, for e.g. hashed directories 82 * 83 * Synchronization: 84 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms) 85 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes. 86 * read/writes behave like SEEK_SET against seeks. 87 */ 88 loff_t 89 generic_file_llseek_size(struct file *file, loff_t offset, int whence, 90 loff_t maxsize, loff_t eof) 91 { 92 switch (whence) { 93 case SEEK_END: 94 offset += eof; 95 break; 96 case SEEK_CUR: 97 /* 98 * Here we special-case the lseek(fd, 0, SEEK_CUR) 99 * position-querying operation. Avoid rewriting the "same" 100 * f_pos value back to the file because a concurrent read(), 101 * write() or lseek() might have altered it 102 */ 103 if (offset == 0) 104 return file->f_pos; 105 /* 106 * f_lock protects against read/modify/write race with other 107 * SEEK_CURs. Note that parallel writes and reads behave 108 * like SEEK_SET. 109 */ 110 spin_lock(&file->f_lock); 111 offset = vfs_setpos(file, file->f_pos + offset, maxsize); 112 spin_unlock(&file->f_lock); 113 return offset; 114 case SEEK_DATA: 115 /* 116 * In the generic case the entire file is data, so as long as 117 * offset isn't at the end of the file then the offset is data. 118 */ 119 if (offset >= eof) 120 return -ENXIO; 121 break; 122 case SEEK_HOLE: 123 /* 124 * There is a virtual hole at the end of the file, so as long as 125 * offset isn't i_size or larger, return i_size. 126 */ 127 if (offset >= eof) 128 return -ENXIO; 129 offset = eof; 130 break; 131 } 132 133 return vfs_setpos(file, offset, maxsize); 134 } 135 EXPORT_SYMBOL(generic_file_llseek_size); 136 137 /** 138 * generic_file_llseek - generic llseek implementation for regular files 139 * @file: file structure to seek on 140 * @offset: file offset to seek to 141 * @whence: type of seek 142 * 143 * This is a generic implemenation of ->llseek useable for all normal local 144 * filesystems. It just updates the file offset to the value specified by 145 * @offset and @whence. 146 */ 147 loff_t generic_file_llseek(struct file *file, loff_t offset, int whence) 148 { 149 struct inode *inode = file->f_mapping->host; 150 151 return generic_file_llseek_size(file, offset, whence, 152 inode->i_sb->s_maxbytes, 153 i_size_read(inode)); 154 } 155 EXPORT_SYMBOL(generic_file_llseek); 156 157 /** 158 * fixed_size_llseek - llseek implementation for fixed-sized devices 159 * @file: file structure to seek on 160 * @offset: file offset to seek to 161 * @whence: type of seek 162 * @size: size of the file 163 * 164 */ 165 loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size) 166 { 167 switch (whence) { 168 case SEEK_SET: case SEEK_CUR: case SEEK_END: 169 return generic_file_llseek_size(file, offset, whence, 170 size, size); 171 default: 172 return -EINVAL; 173 } 174 } 175 EXPORT_SYMBOL(fixed_size_llseek); 176 177 /** 178 * noop_llseek - No Operation Performed llseek implementation 179 * @file: file structure to seek on 180 * @offset: file offset to seek to 181 * @whence: type of seek 182 * 183 * This is an implementation of ->llseek useable for the rare special case when 184 * userspace expects the seek to succeed but the (device) file is actually not 185 * able to perform the seek. In this case you use noop_llseek() instead of 186 * falling back to the default implementation of ->llseek. 187 */ 188 loff_t noop_llseek(struct file *file, loff_t offset, int whence) 189 { 190 return file->f_pos; 191 } 192 EXPORT_SYMBOL(noop_llseek); 193 194 loff_t no_llseek(struct file *file, loff_t offset, int whence) 195 { 196 return -ESPIPE; 197 } 198 EXPORT_SYMBOL(no_llseek); 199 200 loff_t default_llseek(struct file *file, loff_t offset, int whence) 201 { 202 struct inode *inode = file_inode(file); 203 loff_t retval; 204 205 mutex_lock(&inode->i_mutex); 206 switch (whence) { 207 case SEEK_END: 208 offset += i_size_read(inode); 209 break; 210 case SEEK_CUR: 211 if (offset == 0) { 212 retval = file->f_pos; 213 goto out; 214 } 215 offset += file->f_pos; 216 break; 217 case SEEK_DATA: 218 /* 219 * In the generic case the entire file is data, so as 220 * long as offset isn't at the end of the file then the 221 * offset is data. 222 */ 223 if (offset >= inode->i_size) { 224 retval = -ENXIO; 225 goto out; 226 } 227 break; 228 case SEEK_HOLE: 229 /* 230 * There is a virtual hole at the end of the file, so 231 * as long as offset isn't i_size or larger, return 232 * i_size. 233 */ 234 if (offset >= inode->i_size) { 235 retval = -ENXIO; 236 goto out; 237 } 238 offset = inode->i_size; 239 break; 240 } 241 retval = -EINVAL; 242 if (offset >= 0 || unsigned_offsets(file)) { 243 if (offset != file->f_pos) { 244 file->f_pos = offset; 245 file->f_version = 0; 246 } 247 retval = offset; 248 } 249 out: 250 mutex_unlock(&inode->i_mutex); 251 return retval; 252 } 253 EXPORT_SYMBOL(default_llseek); 254 255 loff_t vfs_llseek(struct file *file, loff_t offset, int whence) 256 { 257 loff_t (*fn)(struct file *, loff_t, int); 258 259 fn = no_llseek; 260 if (file->f_mode & FMODE_LSEEK) { 261 if (file->f_op->llseek) 262 fn = file->f_op->llseek; 263 } 264 return fn(file, offset, whence); 265 } 266 EXPORT_SYMBOL(vfs_llseek); 267 268 static inline struct fd fdget_pos(int fd) 269 { 270 return __to_fd(__fdget_pos(fd)); 271 } 272 273 static inline void fdput_pos(struct fd f) 274 { 275 if (f.flags & FDPUT_POS_UNLOCK) 276 mutex_unlock(&f.file->f_pos_lock); 277 fdput(f); 278 } 279 280 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence) 281 { 282 off_t retval; 283 struct fd f = fdget_pos(fd); 284 if (!f.file) 285 return -EBADF; 286 287 retval = -EINVAL; 288 if (whence <= SEEK_MAX) { 289 loff_t res = vfs_llseek(f.file, offset, whence); 290 retval = res; 291 if (res != (loff_t)retval) 292 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */ 293 } 294 fdput_pos(f); 295 return retval; 296 } 297 298 #ifdef CONFIG_COMPAT 299 COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence) 300 { 301 return sys_lseek(fd, offset, whence); 302 } 303 #endif 304 305 #ifdef __ARCH_WANT_SYS_LLSEEK 306 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high, 307 unsigned long, offset_low, loff_t __user *, result, 308 unsigned int, whence) 309 { 310 int retval; 311 struct fd f = fdget_pos(fd); 312 loff_t offset; 313 314 if (!f.file) 315 return -EBADF; 316 317 retval = -EINVAL; 318 if (whence > SEEK_MAX) 319 goto out_putf; 320 321 offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low, 322 whence); 323 324 retval = (int)offset; 325 if (offset >= 0) { 326 retval = -EFAULT; 327 if (!copy_to_user(result, &offset, sizeof(offset))) 328 retval = 0; 329 } 330 out_putf: 331 fdput_pos(f); 332 return retval; 333 } 334 #endif 335 336 /* 337 * rw_verify_area doesn't like huge counts. We limit 338 * them to something that fits in "int" so that others 339 * won't have to do range checks all the time. 340 */ 341 int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count) 342 { 343 struct inode *inode; 344 loff_t pos; 345 int retval = -EINVAL; 346 347 inode = file_inode(file); 348 if (unlikely((ssize_t) count < 0)) 349 return retval; 350 pos = *ppos; 351 if (unlikely(pos < 0)) { 352 if (!unsigned_offsets(file)) 353 return retval; 354 if (count >= -pos) /* both values are in 0..LLONG_MAX */ 355 return -EOVERFLOW; 356 } else if (unlikely((loff_t) (pos + count) < 0)) { 357 if (!unsigned_offsets(file)) 358 return retval; 359 } 360 361 if (unlikely(inode->i_flock && mandatory_lock(inode))) { 362 retval = locks_mandatory_area( 363 read_write == READ ? FLOCK_VERIFY_READ : FLOCK_VERIFY_WRITE, 364 inode, file, pos, count); 365 if (retval < 0) 366 return retval; 367 } 368 retval = security_file_permission(file, 369 read_write == READ ? MAY_READ : MAY_WRITE); 370 if (retval) 371 return retval; 372 return count > MAX_RW_COUNT ? MAX_RW_COUNT : count; 373 } 374 375 ssize_t do_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos) 376 { 377 struct iovec iov = { .iov_base = buf, .iov_len = len }; 378 struct kiocb kiocb; 379 ssize_t ret; 380 381 init_sync_kiocb(&kiocb, filp); 382 kiocb.ki_pos = *ppos; 383 kiocb.ki_nbytes = len; 384 385 ret = filp->f_op->aio_read(&kiocb, &iov, 1, kiocb.ki_pos); 386 if (-EIOCBQUEUED == ret) 387 ret = wait_on_sync_kiocb(&kiocb); 388 *ppos = kiocb.ki_pos; 389 return ret; 390 } 391 392 EXPORT_SYMBOL(do_sync_read); 393 394 ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos) 395 { 396 struct iovec iov = { .iov_base = buf, .iov_len = len }; 397 struct kiocb kiocb; 398 struct iov_iter iter; 399 ssize_t ret; 400 401 init_sync_kiocb(&kiocb, filp); 402 kiocb.ki_pos = *ppos; 403 kiocb.ki_nbytes = len; 404 iov_iter_init(&iter, READ, &iov, 1, len); 405 406 ret = filp->f_op->read_iter(&kiocb, &iter); 407 if (-EIOCBQUEUED == ret) 408 ret = wait_on_sync_kiocb(&kiocb); 409 *ppos = kiocb.ki_pos; 410 return ret; 411 } 412 413 EXPORT_SYMBOL(new_sync_read); 414 415 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) 416 { 417 ssize_t ret; 418 419 if (!(file->f_mode & FMODE_READ)) 420 return -EBADF; 421 if (!(file->f_mode & FMODE_CAN_READ)) 422 return -EINVAL; 423 if (unlikely(!access_ok(VERIFY_WRITE, buf, count))) 424 return -EFAULT; 425 426 ret = rw_verify_area(READ, file, pos, count); 427 if (ret >= 0) { 428 count = ret; 429 if (file->f_op->read) 430 ret = file->f_op->read(file, buf, count, pos); 431 else if (file->f_op->aio_read) 432 ret = do_sync_read(file, buf, count, pos); 433 else 434 ret = new_sync_read(file, buf, count, pos); 435 if (ret > 0) { 436 fsnotify_access(file); 437 add_rchar(current, ret); 438 } 439 inc_syscr(current); 440 } 441 442 return ret; 443 } 444 445 EXPORT_SYMBOL(vfs_read); 446 447 ssize_t do_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos) 448 { 449 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len }; 450 struct kiocb kiocb; 451 ssize_t ret; 452 453 init_sync_kiocb(&kiocb, filp); 454 kiocb.ki_pos = *ppos; 455 kiocb.ki_nbytes = len; 456 457 ret = filp->f_op->aio_write(&kiocb, &iov, 1, kiocb.ki_pos); 458 if (-EIOCBQUEUED == ret) 459 ret = wait_on_sync_kiocb(&kiocb); 460 *ppos = kiocb.ki_pos; 461 return ret; 462 } 463 464 EXPORT_SYMBOL(do_sync_write); 465 466 ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos) 467 { 468 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len }; 469 struct kiocb kiocb; 470 struct iov_iter iter; 471 ssize_t ret; 472 473 init_sync_kiocb(&kiocb, filp); 474 kiocb.ki_pos = *ppos; 475 kiocb.ki_nbytes = len; 476 iov_iter_init(&iter, WRITE, &iov, 1, len); 477 478 ret = filp->f_op->write_iter(&kiocb, &iter); 479 if (-EIOCBQUEUED == ret) 480 ret = wait_on_sync_kiocb(&kiocb); 481 *ppos = kiocb.ki_pos; 482 return ret; 483 } 484 485 EXPORT_SYMBOL(new_sync_write); 486 487 ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos) 488 { 489 mm_segment_t old_fs; 490 const char __user *p; 491 ssize_t ret; 492 493 if (!(file->f_mode & FMODE_CAN_WRITE)) 494 return -EINVAL; 495 496 old_fs = get_fs(); 497 set_fs(get_ds()); 498 p = (__force const char __user *)buf; 499 if (count > MAX_RW_COUNT) 500 count = MAX_RW_COUNT; 501 if (file->f_op->write) 502 ret = file->f_op->write(file, p, count, pos); 503 else if (file->f_op->aio_write) 504 ret = do_sync_write(file, p, count, pos); 505 else 506 ret = new_sync_write(file, p, count, pos); 507 set_fs(old_fs); 508 if (ret > 0) { 509 fsnotify_modify(file); 510 add_wchar(current, ret); 511 } 512 inc_syscw(current); 513 return ret; 514 } 515 516 EXPORT_SYMBOL(__kernel_write); 517 518 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) 519 { 520 ssize_t ret; 521 522 if (!(file->f_mode & FMODE_WRITE)) 523 return -EBADF; 524 if (!(file->f_mode & FMODE_CAN_WRITE)) 525 return -EINVAL; 526 if (unlikely(!access_ok(VERIFY_READ, buf, count))) 527 return -EFAULT; 528 529 ret = rw_verify_area(WRITE, file, pos, count); 530 if (ret >= 0) { 531 count = ret; 532 file_start_write(file); 533 if (file->f_op->write) 534 ret = file->f_op->write(file, buf, count, pos); 535 else if (file->f_op->aio_write) 536 ret = do_sync_write(file, buf, count, pos); 537 else 538 ret = new_sync_write(file, buf, count, pos); 539 if (ret > 0) { 540 fsnotify_modify(file); 541 add_wchar(current, ret); 542 } 543 inc_syscw(current); 544 file_end_write(file); 545 } 546 547 return ret; 548 } 549 550 EXPORT_SYMBOL(vfs_write); 551 552 static inline loff_t file_pos_read(struct file *file) 553 { 554 return file->f_pos; 555 } 556 557 static inline void file_pos_write(struct file *file, loff_t pos) 558 { 559 file->f_pos = pos; 560 } 561 562 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count) 563 { 564 struct fd f = fdget_pos(fd); 565 ssize_t ret = -EBADF; 566 567 if (f.file) { 568 loff_t pos = file_pos_read(f.file); 569 ret = vfs_read(f.file, buf, count, &pos); 570 if (ret >= 0) 571 file_pos_write(f.file, pos); 572 fdput_pos(f); 573 } 574 return ret; 575 } 576 577 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf, 578 size_t, count) 579 { 580 struct fd f = fdget_pos(fd); 581 ssize_t ret = -EBADF; 582 583 if (f.file) { 584 loff_t pos = file_pos_read(f.file); 585 ret = vfs_write(f.file, buf, count, &pos); 586 if (ret >= 0) 587 file_pos_write(f.file, pos); 588 fdput_pos(f); 589 } 590 591 return ret; 592 } 593 594 SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf, 595 size_t, count, loff_t, pos) 596 { 597 struct fd f; 598 ssize_t ret = -EBADF; 599 600 if (pos < 0) 601 return -EINVAL; 602 603 f = fdget(fd); 604 if (f.file) { 605 ret = -ESPIPE; 606 if (f.file->f_mode & FMODE_PREAD) 607 ret = vfs_read(f.file, buf, count, &pos); 608 fdput(f); 609 } 610 611 return ret; 612 } 613 614 SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf, 615 size_t, count, loff_t, pos) 616 { 617 struct fd f; 618 ssize_t ret = -EBADF; 619 620 if (pos < 0) 621 return -EINVAL; 622 623 f = fdget(fd); 624 if (f.file) { 625 ret = -ESPIPE; 626 if (f.file->f_mode & FMODE_PWRITE) 627 ret = vfs_write(f.file, buf, count, &pos); 628 fdput(f); 629 } 630 631 return ret; 632 } 633 634 /* 635 * Reduce an iovec's length in-place. Return the resulting number of segments 636 */ 637 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to) 638 { 639 unsigned long seg = 0; 640 size_t len = 0; 641 642 while (seg < nr_segs) { 643 seg++; 644 if (len + iov->iov_len >= to) { 645 iov->iov_len = to - len; 646 break; 647 } 648 len += iov->iov_len; 649 iov++; 650 } 651 return seg; 652 } 653 EXPORT_SYMBOL(iov_shorten); 654 655 static ssize_t do_iter_readv_writev(struct file *filp, int rw, const struct iovec *iov, 656 unsigned long nr_segs, size_t len, loff_t *ppos, iter_fn_t fn) 657 { 658 struct kiocb kiocb; 659 struct iov_iter iter; 660 ssize_t ret; 661 662 init_sync_kiocb(&kiocb, filp); 663 kiocb.ki_pos = *ppos; 664 kiocb.ki_nbytes = len; 665 666 iov_iter_init(&iter, rw, iov, nr_segs, len); 667 ret = fn(&kiocb, &iter); 668 if (ret == -EIOCBQUEUED) 669 ret = wait_on_sync_kiocb(&kiocb); 670 *ppos = kiocb.ki_pos; 671 return ret; 672 } 673 674 static ssize_t do_sync_readv_writev(struct file *filp, const struct iovec *iov, 675 unsigned long nr_segs, size_t len, loff_t *ppos, iov_fn_t fn) 676 { 677 struct kiocb kiocb; 678 ssize_t ret; 679 680 init_sync_kiocb(&kiocb, filp); 681 kiocb.ki_pos = *ppos; 682 kiocb.ki_nbytes = len; 683 684 ret = fn(&kiocb, iov, nr_segs, kiocb.ki_pos); 685 if (ret == -EIOCBQUEUED) 686 ret = wait_on_sync_kiocb(&kiocb); 687 *ppos = kiocb.ki_pos; 688 return ret; 689 } 690 691 /* Do it by hand, with file-ops */ 692 static ssize_t do_loop_readv_writev(struct file *filp, struct iovec *iov, 693 unsigned long nr_segs, loff_t *ppos, io_fn_t fn) 694 { 695 struct iovec *vector = iov; 696 ssize_t ret = 0; 697 698 while (nr_segs > 0) { 699 void __user *base; 700 size_t len; 701 ssize_t nr; 702 703 base = vector->iov_base; 704 len = vector->iov_len; 705 vector++; 706 nr_segs--; 707 708 nr = fn(filp, base, len, ppos); 709 710 if (nr < 0) { 711 if (!ret) 712 ret = nr; 713 break; 714 } 715 ret += nr; 716 if (nr != len) 717 break; 718 } 719 720 return ret; 721 } 722 723 /* A write operation does a read from user space and vice versa */ 724 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ) 725 726 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, 727 unsigned long nr_segs, unsigned long fast_segs, 728 struct iovec *fast_pointer, 729 struct iovec **ret_pointer) 730 { 731 unsigned long seg; 732 ssize_t ret; 733 struct iovec *iov = fast_pointer; 734 735 /* 736 * SuS says "The readv() function *may* fail if the iovcnt argument 737 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has 738 * traditionally returned zero for zero segments, so... 739 */ 740 if (nr_segs == 0) { 741 ret = 0; 742 goto out; 743 } 744 745 /* 746 * First get the "struct iovec" from user memory and 747 * verify all the pointers 748 */ 749 if (nr_segs > UIO_MAXIOV) { 750 ret = -EINVAL; 751 goto out; 752 } 753 if (nr_segs > fast_segs) { 754 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL); 755 if (iov == NULL) { 756 ret = -ENOMEM; 757 goto out; 758 } 759 } 760 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) { 761 ret = -EFAULT; 762 goto out; 763 } 764 765 /* 766 * According to the Single Unix Specification we should return EINVAL 767 * if an element length is < 0 when cast to ssize_t or if the 768 * total length would overflow the ssize_t return value of the 769 * system call. 770 * 771 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the 772 * overflow case. 773 */ 774 ret = 0; 775 for (seg = 0; seg < nr_segs; seg++) { 776 void __user *buf = iov[seg].iov_base; 777 ssize_t len = (ssize_t)iov[seg].iov_len; 778 779 /* see if we we're about to use an invalid len or if 780 * it's about to overflow ssize_t */ 781 if (len < 0) { 782 ret = -EINVAL; 783 goto out; 784 } 785 if (type >= 0 786 && unlikely(!access_ok(vrfy_dir(type), buf, len))) { 787 ret = -EFAULT; 788 goto out; 789 } 790 if (len > MAX_RW_COUNT - ret) { 791 len = MAX_RW_COUNT - ret; 792 iov[seg].iov_len = len; 793 } 794 ret += len; 795 } 796 out: 797 *ret_pointer = iov; 798 return ret; 799 } 800 801 static ssize_t do_readv_writev(int type, struct file *file, 802 const struct iovec __user * uvector, 803 unsigned long nr_segs, loff_t *pos) 804 { 805 size_t tot_len; 806 struct iovec iovstack[UIO_FASTIOV]; 807 struct iovec *iov = iovstack; 808 ssize_t ret; 809 io_fn_t fn; 810 iov_fn_t fnv; 811 iter_fn_t iter_fn; 812 813 ret = rw_copy_check_uvector(type, uvector, nr_segs, 814 ARRAY_SIZE(iovstack), iovstack, &iov); 815 if (ret <= 0) 816 goto out; 817 818 tot_len = ret; 819 ret = rw_verify_area(type, file, pos, tot_len); 820 if (ret < 0) 821 goto out; 822 823 fnv = NULL; 824 if (type == READ) { 825 fn = file->f_op->read; 826 fnv = file->f_op->aio_read; 827 iter_fn = file->f_op->read_iter; 828 } else { 829 fn = (io_fn_t)file->f_op->write; 830 fnv = file->f_op->aio_write; 831 iter_fn = file->f_op->write_iter; 832 file_start_write(file); 833 } 834 835 if (iter_fn) 836 ret = do_iter_readv_writev(file, type, iov, nr_segs, tot_len, 837 pos, iter_fn); 838 else if (fnv) 839 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len, 840 pos, fnv); 841 else 842 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn); 843 844 if (type != READ) 845 file_end_write(file); 846 847 out: 848 if (iov != iovstack) 849 kfree(iov); 850 if ((ret + (type == READ)) > 0) { 851 if (type == READ) 852 fsnotify_access(file); 853 else 854 fsnotify_modify(file); 855 } 856 return ret; 857 } 858 859 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, 860 unsigned long vlen, loff_t *pos) 861 { 862 if (!(file->f_mode & FMODE_READ)) 863 return -EBADF; 864 if (!(file->f_mode & FMODE_CAN_READ)) 865 return -EINVAL; 866 867 return do_readv_writev(READ, file, vec, vlen, pos); 868 } 869 870 EXPORT_SYMBOL(vfs_readv); 871 872 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec, 873 unsigned long vlen, loff_t *pos) 874 { 875 if (!(file->f_mode & FMODE_WRITE)) 876 return -EBADF; 877 if (!(file->f_mode & FMODE_CAN_WRITE)) 878 return -EINVAL; 879 880 return do_readv_writev(WRITE, file, vec, vlen, pos); 881 } 882 883 EXPORT_SYMBOL(vfs_writev); 884 885 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec, 886 unsigned long, vlen) 887 { 888 struct fd f = fdget_pos(fd); 889 ssize_t ret = -EBADF; 890 891 if (f.file) { 892 loff_t pos = file_pos_read(f.file); 893 ret = vfs_readv(f.file, vec, vlen, &pos); 894 if (ret >= 0) 895 file_pos_write(f.file, pos); 896 fdput_pos(f); 897 } 898 899 if (ret > 0) 900 add_rchar(current, ret); 901 inc_syscr(current); 902 return ret; 903 } 904 905 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec, 906 unsigned long, vlen) 907 { 908 struct fd f = fdget_pos(fd); 909 ssize_t ret = -EBADF; 910 911 if (f.file) { 912 loff_t pos = file_pos_read(f.file); 913 ret = vfs_writev(f.file, vec, vlen, &pos); 914 if (ret >= 0) 915 file_pos_write(f.file, pos); 916 fdput_pos(f); 917 } 918 919 if (ret > 0) 920 add_wchar(current, ret); 921 inc_syscw(current); 922 return ret; 923 } 924 925 static inline loff_t pos_from_hilo(unsigned long high, unsigned long low) 926 { 927 #define HALF_LONG_BITS (BITS_PER_LONG / 2) 928 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low; 929 } 930 931 SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec, 932 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h) 933 { 934 loff_t pos = pos_from_hilo(pos_h, pos_l); 935 struct fd f; 936 ssize_t ret = -EBADF; 937 938 if (pos < 0) 939 return -EINVAL; 940 941 f = fdget(fd); 942 if (f.file) { 943 ret = -ESPIPE; 944 if (f.file->f_mode & FMODE_PREAD) 945 ret = vfs_readv(f.file, vec, vlen, &pos); 946 fdput(f); 947 } 948 949 if (ret > 0) 950 add_rchar(current, ret); 951 inc_syscr(current); 952 return ret; 953 } 954 955 SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec, 956 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h) 957 { 958 loff_t pos = pos_from_hilo(pos_h, pos_l); 959 struct fd f; 960 ssize_t ret = -EBADF; 961 962 if (pos < 0) 963 return -EINVAL; 964 965 f = fdget(fd); 966 if (f.file) { 967 ret = -ESPIPE; 968 if (f.file->f_mode & FMODE_PWRITE) 969 ret = vfs_writev(f.file, vec, vlen, &pos); 970 fdput(f); 971 } 972 973 if (ret > 0) 974 add_wchar(current, ret); 975 inc_syscw(current); 976 return ret; 977 } 978 979 #ifdef CONFIG_COMPAT 980 981 static ssize_t compat_do_readv_writev(int type, struct file *file, 982 const struct compat_iovec __user *uvector, 983 unsigned long nr_segs, loff_t *pos) 984 { 985 compat_ssize_t tot_len; 986 struct iovec iovstack[UIO_FASTIOV]; 987 struct iovec *iov = iovstack; 988 ssize_t ret; 989 io_fn_t fn; 990 iov_fn_t fnv; 991 iter_fn_t iter_fn; 992 993 ret = compat_rw_copy_check_uvector(type, uvector, nr_segs, 994 UIO_FASTIOV, iovstack, &iov); 995 if (ret <= 0) 996 goto out; 997 998 tot_len = ret; 999 ret = rw_verify_area(type, file, pos, tot_len); 1000 if (ret < 0) 1001 goto out; 1002 1003 fnv = NULL; 1004 if (type == READ) { 1005 fn = file->f_op->read; 1006 fnv = file->f_op->aio_read; 1007 iter_fn = file->f_op->read_iter; 1008 } else { 1009 fn = (io_fn_t)file->f_op->write; 1010 fnv = file->f_op->aio_write; 1011 iter_fn = file->f_op->write_iter; 1012 file_start_write(file); 1013 } 1014 1015 if (iter_fn) 1016 ret = do_iter_readv_writev(file, type, iov, nr_segs, tot_len, 1017 pos, iter_fn); 1018 else if (fnv) 1019 ret = do_sync_readv_writev(file, iov, nr_segs, tot_len, 1020 pos, fnv); 1021 else 1022 ret = do_loop_readv_writev(file, iov, nr_segs, pos, fn); 1023 1024 if (type != READ) 1025 file_end_write(file); 1026 1027 out: 1028 if (iov != iovstack) 1029 kfree(iov); 1030 if ((ret + (type == READ)) > 0) { 1031 if (type == READ) 1032 fsnotify_access(file); 1033 else 1034 fsnotify_modify(file); 1035 } 1036 return ret; 1037 } 1038 1039 static size_t compat_readv(struct file *file, 1040 const struct compat_iovec __user *vec, 1041 unsigned long vlen, loff_t *pos) 1042 { 1043 ssize_t ret = -EBADF; 1044 1045 if (!(file->f_mode & FMODE_READ)) 1046 goto out; 1047 1048 ret = -EINVAL; 1049 if (!(file->f_mode & FMODE_CAN_READ)) 1050 goto out; 1051 1052 ret = compat_do_readv_writev(READ, file, vec, vlen, pos); 1053 1054 out: 1055 if (ret > 0) 1056 add_rchar(current, ret); 1057 inc_syscr(current); 1058 return ret; 1059 } 1060 1061 COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd, 1062 const struct compat_iovec __user *,vec, 1063 compat_ulong_t, vlen) 1064 { 1065 struct fd f = fdget_pos(fd); 1066 ssize_t ret; 1067 loff_t pos; 1068 1069 if (!f.file) 1070 return -EBADF; 1071 pos = f.file->f_pos; 1072 ret = compat_readv(f.file, vec, vlen, &pos); 1073 if (ret >= 0) 1074 f.file->f_pos = pos; 1075 fdput_pos(f); 1076 return ret; 1077 } 1078 1079 static long __compat_sys_preadv64(unsigned long fd, 1080 const struct compat_iovec __user *vec, 1081 unsigned long vlen, loff_t pos) 1082 { 1083 struct fd f; 1084 ssize_t ret; 1085 1086 if (pos < 0) 1087 return -EINVAL; 1088 f = fdget(fd); 1089 if (!f.file) 1090 return -EBADF; 1091 ret = -ESPIPE; 1092 if (f.file->f_mode & FMODE_PREAD) 1093 ret = compat_readv(f.file, vec, vlen, &pos); 1094 fdput(f); 1095 return ret; 1096 } 1097 1098 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64 1099 COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd, 1100 const struct compat_iovec __user *,vec, 1101 unsigned long, vlen, loff_t, pos) 1102 { 1103 return __compat_sys_preadv64(fd, vec, vlen, pos); 1104 } 1105 #endif 1106 1107 COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd, 1108 const struct compat_iovec __user *,vec, 1109 compat_ulong_t, vlen, u32, pos_low, u32, pos_high) 1110 { 1111 loff_t pos = ((loff_t)pos_high << 32) | pos_low; 1112 1113 return __compat_sys_preadv64(fd, vec, vlen, pos); 1114 } 1115 1116 static size_t compat_writev(struct file *file, 1117 const struct compat_iovec __user *vec, 1118 unsigned long vlen, loff_t *pos) 1119 { 1120 ssize_t ret = -EBADF; 1121 1122 if (!(file->f_mode & FMODE_WRITE)) 1123 goto out; 1124 1125 ret = -EINVAL; 1126 if (!(file->f_mode & FMODE_CAN_WRITE)) 1127 goto out; 1128 1129 ret = compat_do_readv_writev(WRITE, file, vec, vlen, pos); 1130 1131 out: 1132 if (ret > 0) 1133 add_wchar(current, ret); 1134 inc_syscw(current); 1135 return ret; 1136 } 1137 1138 COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd, 1139 const struct compat_iovec __user *, vec, 1140 compat_ulong_t, vlen) 1141 { 1142 struct fd f = fdget_pos(fd); 1143 ssize_t ret; 1144 loff_t pos; 1145 1146 if (!f.file) 1147 return -EBADF; 1148 pos = f.file->f_pos; 1149 ret = compat_writev(f.file, vec, vlen, &pos); 1150 if (ret >= 0) 1151 f.file->f_pos = pos; 1152 fdput_pos(f); 1153 return ret; 1154 } 1155 1156 static long __compat_sys_pwritev64(unsigned long fd, 1157 const struct compat_iovec __user *vec, 1158 unsigned long vlen, loff_t pos) 1159 { 1160 struct fd f; 1161 ssize_t ret; 1162 1163 if (pos < 0) 1164 return -EINVAL; 1165 f = fdget(fd); 1166 if (!f.file) 1167 return -EBADF; 1168 ret = -ESPIPE; 1169 if (f.file->f_mode & FMODE_PWRITE) 1170 ret = compat_writev(f.file, vec, vlen, &pos); 1171 fdput(f); 1172 return ret; 1173 } 1174 1175 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64 1176 COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd, 1177 const struct compat_iovec __user *,vec, 1178 unsigned long, vlen, loff_t, pos) 1179 { 1180 return __compat_sys_pwritev64(fd, vec, vlen, pos); 1181 } 1182 #endif 1183 1184 COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd, 1185 const struct compat_iovec __user *,vec, 1186 compat_ulong_t, vlen, u32, pos_low, u32, pos_high) 1187 { 1188 loff_t pos = ((loff_t)pos_high << 32) | pos_low; 1189 1190 return __compat_sys_pwritev64(fd, vec, vlen, pos); 1191 } 1192 #endif 1193 1194 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, 1195 size_t count, loff_t max) 1196 { 1197 struct fd in, out; 1198 struct inode *in_inode, *out_inode; 1199 loff_t pos; 1200 loff_t out_pos; 1201 ssize_t retval; 1202 int fl; 1203 1204 /* 1205 * Get input file, and verify that it is ok.. 1206 */ 1207 retval = -EBADF; 1208 in = fdget(in_fd); 1209 if (!in.file) 1210 goto out; 1211 if (!(in.file->f_mode & FMODE_READ)) 1212 goto fput_in; 1213 retval = -ESPIPE; 1214 if (!ppos) { 1215 pos = in.file->f_pos; 1216 } else { 1217 pos = *ppos; 1218 if (!(in.file->f_mode & FMODE_PREAD)) 1219 goto fput_in; 1220 } 1221 retval = rw_verify_area(READ, in.file, &pos, count); 1222 if (retval < 0) 1223 goto fput_in; 1224 count = retval; 1225 1226 /* 1227 * Get output file, and verify that it is ok.. 1228 */ 1229 retval = -EBADF; 1230 out = fdget(out_fd); 1231 if (!out.file) 1232 goto fput_in; 1233 if (!(out.file->f_mode & FMODE_WRITE)) 1234 goto fput_out; 1235 retval = -EINVAL; 1236 in_inode = file_inode(in.file); 1237 out_inode = file_inode(out.file); 1238 out_pos = out.file->f_pos; 1239 retval = rw_verify_area(WRITE, out.file, &out_pos, count); 1240 if (retval < 0) 1241 goto fput_out; 1242 count = retval; 1243 1244 if (!max) 1245 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes); 1246 1247 if (unlikely(pos + count > max)) { 1248 retval = -EOVERFLOW; 1249 if (pos >= max) 1250 goto fput_out; 1251 count = max - pos; 1252 } 1253 1254 fl = 0; 1255 #if 0 1256 /* 1257 * We need to debate whether we can enable this or not. The 1258 * man page documents EAGAIN return for the output at least, 1259 * and the application is arguably buggy if it doesn't expect 1260 * EAGAIN on a non-blocking file descriptor. 1261 */ 1262 if (in.file->f_flags & O_NONBLOCK) 1263 fl = SPLICE_F_NONBLOCK; 1264 #endif 1265 file_start_write(out.file); 1266 retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl); 1267 file_end_write(out.file); 1268 1269 if (retval > 0) { 1270 add_rchar(current, retval); 1271 add_wchar(current, retval); 1272 fsnotify_access(in.file); 1273 fsnotify_modify(out.file); 1274 out.file->f_pos = out_pos; 1275 if (ppos) 1276 *ppos = pos; 1277 else 1278 in.file->f_pos = pos; 1279 } 1280 1281 inc_syscr(current); 1282 inc_syscw(current); 1283 if (pos > max) 1284 retval = -EOVERFLOW; 1285 1286 fput_out: 1287 fdput(out); 1288 fput_in: 1289 fdput(in); 1290 out: 1291 return retval; 1292 } 1293 1294 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count) 1295 { 1296 loff_t pos; 1297 off_t off; 1298 ssize_t ret; 1299 1300 if (offset) { 1301 if (unlikely(get_user(off, offset))) 1302 return -EFAULT; 1303 pos = off; 1304 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS); 1305 if (unlikely(put_user(pos, offset))) 1306 return -EFAULT; 1307 return ret; 1308 } 1309 1310 return do_sendfile(out_fd, in_fd, NULL, count, 0); 1311 } 1312 1313 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count) 1314 { 1315 loff_t pos; 1316 ssize_t ret; 1317 1318 if (offset) { 1319 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t)))) 1320 return -EFAULT; 1321 ret = do_sendfile(out_fd, in_fd, &pos, count, 0); 1322 if (unlikely(put_user(pos, offset))) 1323 return -EFAULT; 1324 return ret; 1325 } 1326 1327 return do_sendfile(out_fd, in_fd, NULL, count, 0); 1328 } 1329 1330 #ifdef CONFIG_COMPAT 1331 COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, 1332 compat_off_t __user *, offset, compat_size_t, count) 1333 { 1334 loff_t pos; 1335 off_t off; 1336 ssize_t ret; 1337 1338 if (offset) { 1339 if (unlikely(get_user(off, offset))) 1340 return -EFAULT; 1341 pos = off; 1342 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS); 1343 if (unlikely(put_user(pos, offset))) 1344 return -EFAULT; 1345 return ret; 1346 } 1347 1348 return do_sendfile(out_fd, in_fd, NULL, count, 0); 1349 } 1350 1351 COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, 1352 compat_loff_t __user *, offset, compat_size_t, count) 1353 { 1354 loff_t pos; 1355 ssize_t ret; 1356 1357 if (offset) { 1358 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t)))) 1359 return -EFAULT; 1360 ret = do_sendfile(out_fd, in_fd, &pos, count, 0); 1361 if (unlikely(put_user(pos, offset))) 1362 return -EFAULT; 1363 return ret; 1364 } 1365 1366 return do_sendfile(out_fd, in_fd, NULL, count, 0); 1367 } 1368 #endif 1369