1 /* 2 * linux/fs/read_write.c 3 * 4 * Copyright (C) 1991, 1992 Linus Torvalds 5 */ 6 7 #include <linux/slab.h> 8 #include <linux/stat.h> 9 #include <linux/fcntl.h> 10 #include <linux/file.h> 11 #include <linux/uio.h> 12 #include <linux/fsnotify.h> 13 #include <linux/security.h> 14 #include <linux/export.h> 15 #include <linux/syscalls.h> 16 #include <linux/pagemap.h> 17 #include <linux/splice.h> 18 #include <linux/compat.h> 19 #include <linux/mount.h> 20 #include <linux/fs.h> 21 #include "internal.h" 22 23 #include <linux/uaccess.h> 24 #include <asm/unistd.h> 25 26 typedef ssize_t (*io_fn_t)(struct file *, char __user *, size_t, loff_t *); 27 typedef ssize_t (*iter_fn_t)(struct kiocb *, struct iov_iter *); 28 29 const struct file_operations generic_ro_fops = { 30 .llseek = generic_file_llseek, 31 .read_iter = generic_file_read_iter, 32 .mmap = generic_file_readonly_mmap, 33 .splice_read = generic_file_splice_read, 34 }; 35 36 EXPORT_SYMBOL(generic_ro_fops); 37 38 static inline int unsigned_offsets(struct file *file) 39 { 40 return file->f_mode & FMODE_UNSIGNED_OFFSET; 41 } 42 43 /** 44 * vfs_setpos - update the file offset for lseek 45 * @file: file structure in question 46 * @offset: file offset to seek to 47 * @maxsize: maximum file size 48 * 49 * This is a low-level filesystem helper for updating the file offset to 50 * the value specified by @offset if the given offset is valid and it is 51 * not equal to the current file offset. 52 * 53 * Return the specified offset on success and -EINVAL on invalid offset. 54 */ 55 loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize) 56 { 57 if (offset < 0 && !unsigned_offsets(file)) 58 return -EINVAL; 59 if (offset > maxsize) 60 return -EINVAL; 61 62 if (offset != file->f_pos) { 63 file->f_pos = offset; 64 file->f_version = 0; 65 } 66 return offset; 67 } 68 EXPORT_SYMBOL(vfs_setpos); 69 70 /** 71 * generic_file_llseek_size - generic llseek implementation for regular files 72 * @file: file structure to seek on 73 * @offset: file offset to seek to 74 * @whence: type of seek 75 * @size: max size of this file in file system 76 * @eof: offset used for SEEK_END position 77 * 78 * This is a variant of generic_file_llseek that allows passing in a custom 79 * maximum file size and a custom EOF position, for e.g. hashed directories 80 * 81 * Synchronization: 82 * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms) 83 * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes. 84 * read/writes behave like SEEK_SET against seeks. 85 */ 86 loff_t 87 generic_file_llseek_size(struct file *file, loff_t offset, int whence, 88 loff_t maxsize, loff_t eof) 89 { 90 switch (whence) { 91 case SEEK_END: 92 offset += eof; 93 break; 94 case SEEK_CUR: 95 /* 96 * Here we special-case the lseek(fd, 0, SEEK_CUR) 97 * position-querying operation. Avoid rewriting the "same" 98 * f_pos value back to the file because a concurrent read(), 99 * write() or lseek() might have altered it 100 */ 101 if (offset == 0) 102 return file->f_pos; 103 /* 104 * f_lock protects against read/modify/write race with other 105 * SEEK_CURs. Note that parallel writes and reads behave 106 * like SEEK_SET. 107 */ 108 spin_lock(&file->f_lock); 109 offset = vfs_setpos(file, file->f_pos + offset, maxsize); 110 spin_unlock(&file->f_lock); 111 return offset; 112 case SEEK_DATA: 113 /* 114 * In the generic case the entire file is data, so as long as 115 * offset isn't at the end of the file then the offset is data. 116 */ 117 if (offset >= eof) 118 return -ENXIO; 119 break; 120 case SEEK_HOLE: 121 /* 122 * There is a virtual hole at the end of the file, so as long as 123 * offset isn't i_size or larger, return i_size. 124 */ 125 if (offset >= eof) 126 return -ENXIO; 127 offset = eof; 128 break; 129 } 130 131 return vfs_setpos(file, offset, maxsize); 132 } 133 EXPORT_SYMBOL(generic_file_llseek_size); 134 135 /** 136 * generic_file_llseek - generic llseek implementation for regular files 137 * @file: file structure to seek on 138 * @offset: file offset to seek to 139 * @whence: type of seek 140 * 141 * This is a generic implemenation of ->llseek useable for all normal local 142 * filesystems. It just updates the file offset to the value specified by 143 * @offset and @whence. 144 */ 145 loff_t generic_file_llseek(struct file *file, loff_t offset, int whence) 146 { 147 struct inode *inode = file->f_mapping->host; 148 149 return generic_file_llseek_size(file, offset, whence, 150 inode->i_sb->s_maxbytes, 151 i_size_read(inode)); 152 } 153 EXPORT_SYMBOL(generic_file_llseek); 154 155 /** 156 * fixed_size_llseek - llseek implementation for fixed-sized devices 157 * @file: file structure to seek on 158 * @offset: file offset to seek to 159 * @whence: type of seek 160 * @size: size of the file 161 * 162 */ 163 loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size) 164 { 165 switch (whence) { 166 case SEEK_SET: case SEEK_CUR: case SEEK_END: 167 return generic_file_llseek_size(file, offset, whence, 168 size, size); 169 default: 170 return -EINVAL; 171 } 172 } 173 EXPORT_SYMBOL(fixed_size_llseek); 174 175 /** 176 * no_seek_end_llseek - llseek implementation for fixed-sized devices 177 * @file: file structure to seek on 178 * @offset: file offset to seek to 179 * @whence: type of seek 180 * 181 */ 182 loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence) 183 { 184 switch (whence) { 185 case SEEK_SET: case SEEK_CUR: 186 return generic_file_llseek_size(file, offset, whence, 187 OFFSET_MAX, 0); 188 default: 189 return -EINVAL; 190 } 191 } 192 EXPORT_SYMBOL(no_seek_end_llseek); 193 194 /** 195 * no_seek_end_llseek_size - llseek implementation for fixed-sized devices 196 * @file: file structure to seek on 197 * @offset: file offset to seek to 198 * @whence: type of seek 199 * @size: maximal offset allowed 200 * 201 */ 202 loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size) 203 { 204 switch (whence) { 205 case SEEK_SET: case SEEK_CUR: 206 return generic_file_llseek_size(file, offset, whence, 207 size, 0); 208 default: 209 return -EINVAL; 210 } 211 } 212 EXPORT_SYMBOL(no_seek_end_llseek_size); 213 214 /** 215 * noop_llseek - No Operation Performed llseek implementation 216 * @file: file structure to seek on 217 * @offset: file offset to seek to 218 * @whence: type of seek 219 * 220 * This is an implementation of ->llseek useable for the rare special case when 221 * userspace expects the seek to succeed but the (device) file is actually not 222 * able to perform the seek. In this case you use noop_llseek() instead of 223 * falling back to the default implementation of ->llseek. 224 */ 225 loff_t noop_llseek(struct file *file, loff_t offset, int whence) 226 { 227 return file->f_pos; 228 } 229 EXPORT_SYMBOL(noop_llseek); 230 231 loff_t no_llseek(struct file *file, loff_t offset, int whence) 232 { 233 return -ESPIPE; 234 } 235 EXPORT_SYMBOL(no_llseek); 236 237 loff_t default_llseek(struct file *file, loff_t offset, int whence) 238 { 239 struct inode *inode = file_inode(file); 240 loff_t retval; 241 242 inode_lock(inode); 243 switch (whence) { 244 case SEEK_END: 245 offset += i_size_read(inode); 246 break; 247 case SEEK_CUR: 248 if (offset == 0) { 249 retval = file->f_pos; 250 goto out; 251 } 252 offset += file->f_pos; 253 break; 254 case SEEK_DATA: 255 /* 256 * In the generic case the entire file is data, so as 257 * long as offset isn't at the end of the file then the 258 * offset is data. 259 */ 260 if (offset >= inode->i_size) { 261 retval = -ENXIO; 262 goto out; 263 } 264 break; 265 case SEEK_HOLE: 266 /* 267 * There is a virtual hole at the end of the file, so 268 * as long as offset isn't i_size or larger, return 269 * i_size. 270 */ 271 if (offset >= inode->i_size) { 272 retval = -ENXIO; 273 goto out; 274 } 275 offset = inode->i_size; 276 break; 277 } 278 retval = -EINVAL; 279 if (offset >= 0 || unsigned_offsets(file)) { 280 if (offset != file->f_pos) { 281 file->f_pos = offset; 282 file->f_version = 0; 283 } 284 retval = offset; 285 } 286 out: 287 inode_unlock(inode); 288 return retval; 289 } 290 EXPORT_SYMBOL(default_llseek); 291 292 loff_t vfs_llseek(struct file *file, loff_t offset, int whence) 293 { 294 loff_t (*fn)(struct file *, loff_t, int); 295 296 fn = no_llseek; 297 if (file->f_mode & FMODE_LSEEK) { 298 if (file->f_op->llseek) 299 fn = file->f_op->llseek; 300 } 301 return fn(file, offset, whence); 302 } 303 EXPORT_SYMBOL(vfs_llseek); 304 305 SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence) 306 { 307 off_t retval; 308 struct fd f = fdget_pos(fd); 309 if (!f.file) 310 return -EBADF; 311 312 retval = -EINVAL; 313 if (whence <= SEEK_MAX) { 314 loff_t res = vfs_llseek(f.file, offset, whence); 315 retval = res; 316 if (res != (loff_t)retval) 317 retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */ 318 } 319 fdput_pos(f); 320 return retval; 321 } 322 323 #ifdef CONFIG_COMPAT 324 COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence) 325 { 326 return sys_lseek(fd, offset, whence); 327 } 328 #endif 329 330 #ifdef __ARCH_WANT_SYS_LLSEEK 331 SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high, 332 unsigned long, offset_low, loff_t __user *, result, 333 unsigned int, whence) 334 { 335 int retval; 336 struct fd f = fdget_pos(fd); 337 loff_t offset; 338 339 if (!f.file) 340 return -EBADF; 341 342 retval = -EINVAL; 343 if (whence > SEEK_MAX) 344 goto out_putf; 345 346 offset = vfs_llseek(f.file, ((loff_t) offset_high << 32) | offset_low, 347 whence); 348 349 retval = (int)offset; 350 if (offset >= 0) { 351 retval = -EFAULT; 352 if (!copy_to_user(result, &offset, sizeof(offset))) 353 retval = 0; 354 } 355 out_putf: 356 fdput_pos(f); 357 return retval; 358 } 359 #endif 360 361 ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos) 362 { 363 struct kiocb kiocb; 364 ssize_t ret; 365 366 if (!file->f_op->read_iter) 367 return -EINVAL; 368 369 init_sync_kiocb(&kiocb, file); 370 kiocb.ki_pos = *ppos; 371 372 iter->type |= READ; 373 ret = file->f_op->read_iter(&kiocb, iter); 374 BUG_ON(ret == -EIOCBQUEUED); 375 if (ret > 0) 376 *ppos = kiocb.ki_pos; 377 return ret; 378 } 379 EXPORT_SYMBOL(vfs_iter_read); 380 381 ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos) 382 { 383 struct kiocb kiocb; 384 ssize_t ret; 385 386 if (!file->f_op->write_iter) 387 return -EINVAL; 388 389 init_sync_kiocb(&kiocb, file); 390 kiocb.ki_pos = *ppos; 391 392 iter->type |= WRITE; 393 ret = file->f_op->write_iter(&kiocb, iter); 394 BUG_ON(ret == -EIOCBQUEUED); 395 if (ret > 0) 396 *ppos = kiocb.ki_pos; 397 return ret; 398 } 399 EXPORT_SYMBOL(vfs_iter_write); 400 401 int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count) 402 { 403 struct inode *inode; 404 loff_t pos; 405 int retval = -EINVAL; 406 407 inode = file_inode(file); 408 if (unlikely((ssize_t) count < 0)) 409 return retval; 410 pos = *ppos; 411 if (unlikely(pos < 0)) { 412 if (!unsigned_offsets(file)) 413 return retval; 414 if (count >= -pos) /* both values are in 0..LLONG_MAX */ 415 return -EOVERFLOW; 416 } else if (unlikely((loff_t) (pos + count) < 0)) { 417 if (!unsigned_offsets(file)) 418 return retval; 419 } 420 421 if (unlikely(inode->i_flctx && mandatory_lock(inode))) { 422 retval = locks_mandatory_area(inode, file, pos, pos + count - 1, 423 read_write == READ ? F_RDLCK : F_WRLCK); 424 if (retval < 0) 425 return retval; 426 } 427 return security_file_permission(file, 428 read_write == READ ? MAY_READ : MAY_WRITE); 429 } 430 431 static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos) 432 { 433 struct iovec iov = { .iov_base = buf, .iov_len = len }; 434 struct kiocb kiocb; 435 struct iov_iter iter; 436 ssize_t ret; 437 438 init_sync_kiocb(&kiocb, filp); 439 kiocb.ki_pos = *ppos; 440 iov_iter_init(&iter, READ, &iov, 1, len); 441 442 ret = filp->f_op->read_iter(&kiocb, &iter); 443 BUG_ON(ret == -EIOCBQUEUED); 444 *ppos = kiocb.ki_pos; 445 return ret; 446 } 447 448 ssize_t __vfs_read(struct file *file, char __user *buf, size_t count, 449 loff_t *pos) 450 { 451 if (file->f_op->read) 452 return file->f_op->read(file, buf, count, pos); 453 else if (file->f_op->read_iter) 454 return new_sync_read(file, buf, count, pos); 455 else 456 return -EINVAL; 457 } 458 EXPORT_SYMBOL(__vfs_read); 459 460 ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) 461 { 462 ssize_t ret; 463 464 if (!(file->f_mode & FMODE_READ)) 465 return -EBADF; 466 if (!(file->f_mode & FMODE_CAN_READ)) 467 return -EINVAL; 468 if (unlikely(!access_ok(VERIFY_WRITE, buf, count))) 469 return -EFAULT; 470 471 ret = rw_verify_area(READ, file, pos, count); 472 if (!ret) { 473 if (count > MAX_RW_COUNT) 474 count = MAX_RW_COUNT; 475 ret = __vfs_read(file, buf, count, pos); 476 if (ret > 0) { 477 fsnotify_access(file); 478 add_rchar(current, ret); 479 } 480 inc_syscr(current); 481 } 482 483 return ret; 484 } 485 486 EXPORT_SYMBOL(vfs_read); 487 488 static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos) 489 { 490 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = len }; 491 struct kiocb kiocb; 492 struct iov_iter iter; 493 ssize_t ret; 494 495 init_sync_kiocb(&kiocb, filp); 496 kiocb.ki_pos = *ppos; 497 iov_iter_init(&iter, WRITE, &iov, 1, len); 498 499 ret = filp->f_op->write_iter(&kiocb, &iter); 500 BUG_ON(ret == -EIOCBQUEUED); 501 if (ret > 0) 502 *ppos = kiocb.ki_pos; 503 return ret; 504 } 505 506 ssize_t __vfs_write(struct file *file, const char __user *p, size_t count, 507 loff_t *pos) 508 { 509 if (file->f_op->write) 510 return file->f_op->write(file, p, count, pos); 511 else if (file->f_op->write_iter) 512 return new_sync_write(file, p, count, pos); 513 else 514 return -EINVAL; 515 } 516 EXPORT_SYMBOL(__vfs_write); 517 518 ssize_t __kernel_write(struct file *file, const char *buf, size_t count, loff_t *pos) 519 { 520 mm_segment_t old_fs; 521 const char __user *p; 522 ssize_t ret; 523 524 if (!(file->f_mode & FMODE_CAN_WRITE)) 525 return -EINVAL; 526 527 old_fs = get_fs(); 528 set_fs(get_ds()); 529 p = (__force const char __user *)buf; 530 if (count > MAX_RW_COUNT) 531 count = MAX_RW_COUNT; 532 ret = __vfs_write(file, p, count, pos); 533 set_fs(old_fs); 534 if (ret > 0) { 535 fsnotify_modify(file); 536 add_wchar(current, ret); 537 } 538 inc_syscw(current); 539 return ret; 540 } 541 542 EXPORT_SYMBOL(__kernel_write); 543 544 ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) 545 { 546 ssize_t ret; 547 548 if (!(file->f_mode & FMODE_WRITE)) 549 return -EBADF; 550 if (!(file->f_mode & FMODE_CAN_WRITE)) 551 return -EINVAL; 552 if (unlikely(!access_ok(VERIFY_READ, buf, count))) 553 return -EFAULT; 554 555 ret = rw_verify_area(WRITE, file, pos, count); 556 if (!ret) { 557 if (count > MAX_RW_COUNT) 558 count = MAX_RW_COUNT; 559 file_start_write(file); 560 ret = __vfs_write(file, buf, count, pos); 561 if (ret > 0) { 562 fsnotify_modify(file); 563 add_wchar(current, ret); 564 } 565 inc_syscw(current); 566 file_end_write(file); 567 } 568 569 return ret; 570 } 571 572 EXPORT_SYMBOL(vfs_write); 573 574 static inline loff_t file_pos_read(struct file *file) 575 { 576 return file->f_pos; 577 } 578 579 static inline void file_pos_write(struct file *file, loff_t pos) 580 { 581 file->f_pos = pos; 582 } 583 584 SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count) 585 { 586 struct fd f = fdget_pos(fd); 587 ssize_t ret = -EBADF; 588 589 if (f.file) { 590 loff_t pos = file_pos_read(f.file); 591 ret = vfs_read(f.file, buf, count, &pos); 592 if (ret >= 0) 593 file_pos_write(f.file, pos); 594 fdput_pos(f); 595 } 596 return ret; 597 } 598 599 SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf, 600 size_t, count) 601 { 602 struct fd f = fdget_pos(fd); 603 ssize_t ret = -EBADF; 604 605 if (f.file) { 606 loff_t pos = file_pos_read(f.file); 607 ret = vfs_write(f.file, buf, count, &pos); 608 if (ret >= 0) 609 file_pos_write(f.file, pos); 610 fdput_pos(f); 611 } 612 613 return ret; 614 } 615 616 SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf, 617 size_t, count, loff_t, pos) 618 { 619 struct fd f; 620 ssize_t ret = -EBADF; 621 622 if (pos < 0) 623 return -EINVAL; 624 625 f = fdget(fd); 626 if (f.file) { 627 ret = -ESPIPE; 628 if (f.file->f_mode & FMODE_PREAD) 629 ret = vfs_read(f.file, buf, count, &pos); 630 fdput(f); 631 } 632 633 return ret; 634 } 635 636 SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf, 637 size_t, count, loff_t, pos) 638 { 639 struct fd f; 640 ssize_t ret = -EBADF; 641 642 if (pos < 0) 643 return -EINVAL; 644 645 f = fdget(fd); 646 if (f.file) { 647 ret = -ESPIPE; 648 if (f.file->f_mode & FMODE_PWRITE) 649 ret = vfs_write(f.file, buf, count, &pos); 650 fdput(f); 651 } 652 653 return ret; 654 } 655 656 /* 657 * Reduce an iovec's length in-place. Return the resulting number of segments 658 */ 659 unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to) 660 { 661 unsigned long seg = 0; 662 size_t len = 0; 663 664 while (seg < nr_segs) { 665 seg++; 666 if (len + iov->iov_len >= to) { 667 iov->iov_len = to - len; 668 break; 669 } 670 len += iov->iov_len; 671 iov++; 672 } 673 return seg; 674 } 675 EXPORT_SYMBOL(iov_shorten); 676 677 static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter, 678 loff_t *ppos, iter_fn_t fn, int flags) 679 { 680 struct kiocb kiocb; 681 ssize_t ret; 682 683 if (flags & ~(RWF_HIPRI | RWF_DSYNC | RWF_SYNC)) 684 return -EOPNOTSUPP; 685 686 init_sync_kiocb(&kiocb, filp); 687 if (flags & RWF_HIPRI) 688 kiocb.ki_flags |= IOCB_HIPRI; 689 if (flags & RWF_DSYNC) 690 kiocb.ki_flags |= IOCB_DSYNC; 691 if (flags & RWF_SYNC) 692 kiocb.ki_flags |= (IOCB_DSYNC | IOCB_SYNC); 693 kiocb.ki_pos = *ppos; 694 695 ret = fn(&kiocb, iter); 696 BUG_ON(ret == -EIOCBQUEUED); 697 *ppos = kiocb.ki_pos; 698 return ret; 699 } 700 701 /* Do it by hand, with file-ops */ 702 static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter, 703 loff_t *ppos, io_fn_t fn, int flags) 704 { 705 ssize_t ret = 0; 706 707 if (flags & ~RWF_HIPRI) 708 return -EOPNOTSUPP; 709 710 while (iov_iter_count(iter)) { 711 struct iovec iovec = iov_iter_iovec(iter); 712 ssize_t nr; 713 714 nr = fn(filp, iovec.iov_base, iovec.iov_len, ppos); 715 716 if (nr < 0) { 717 if (!ret) 718 ret = nr; 719 break; 720 } 721 ret += nr; 722 if (nr != iovec.iov_len) 723 break; 724 iov_iter_advance(iter, nr); 725 } 726 727 return ret; 728 } 729 730 /* A write operation does a read from user space and vice versa */ 731 #define vrfy_dir(type) ((type) == READ ? VERIFY_WRITE : VERIFY_READ) 732 733 /** 734 * rw_copy_check_uvector() - Copy an array of &struct iovec from userspace 735 * into the kernel and check that it is valid. 736 * 737 * @type: One of %CHECK_IOVEC_ONLY, %READ, or %WRITE. 738 * @uvector: Pointer to the userspace array. 739 * @nr_segs: Number of elements in userspace array. 740 * @fast_segs: Number of elements in @fast_pointer. 741 * @fast_pointer: Pointer to (usually small on-stack) kernel array. 742 * @ret_pointer: (output parameter) Pointer to a variable that will point to 743 * either @fast_pointer, a newly allocated kernel array, or NULL, 744 * depending on which array was used. 745 * 746 * This function copies an array of &struct iovec of @nr_segs from 747 * userspace into the kernel and checks that each element is valid (e.g. 748 * it does not point to a kernel address or cause overflow by being too 749 * large, etc.). 750 * 751 * As an optimization, the caller may provide a pointer to a small 752 * on-stack array in @fast_pointer, typically %UIO_FASTIOV elements long 753 * (the size of this array, or 0 if unused, should be given in @fast_segs). 754 * 755 * @ret_pointer will always point to the array that was used, so the 756 * caller must take care not to call kfree() on it e.g. in case the 757 * @fast_pointer array was used and it was allocated on the stack. 758 * 759 * Return: The total number of bytes covered by the iovec array on success 760 * or a negative error code on error. 761 */ 762 ssize_t rw_copy_check_uvector(int type, const struct iovec __user * uvector, 763 unsigned long nr_segs, unsigned long fast_segs, 764 struct iovec *fast_pointer, 765 struct iovec **ret_pointer) 766 { 767 unsigned long seg; 768 ssize_t ret; 769 struct iovec *iov = fast_pointer; 770 771 /* 772 * SuS says "The readv() function *may* fail if the iovcnt argument 773 * was less than or equal to 0, or greater than {IOV_MAX}. Linux has 774 * traditionally returned zero for zero segments, so... 775 */ 776 if (nr_segs == 0) { 777 ret = 0; 778 goto out; 779 } 780 781 /* 782 * First get the "struct iovec" from user memory and 783 * verify all the pointers 784 */ 785 if (nr_segs > UIO_MAXIOV) { 786 ret = -EINVAL; 787 goto out; 788 } 789 if (nr_segs > fast_segs) { 790 iov = kmalloc(nr_segs*sizeof(struct iovec), GFP_KERNEL); 791 if (iov == NULL) { 792 ret = -ENOMEM; 793 goto out; 794 } 795 } 796 if (copy_from_user(iov, uvector, nr_segs*sizeof(*uvector))) { 797 ret = -EFAULT; 798 goto out; 799 } 800 801 /* 802 * According to the Single Unix Specification we should return EINVAL 803 * if an element length is < 0 when cast to ssize_t or if the 804 * total length would overflow the ssize_t return value of the 805 * system call. 806 * 807 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the 808 * overflow case. 809 */ 810 ret = 0; 811 for (seg = 0; seg < nr_segs; seg++) { 812 void __user *buf = iov[seg].iov_base; 813 ssize_t len = (ssize_t)iov[seg].iov_len; 814 815 /* see if we we're about to use an invalid len or if 816 * it's about to overflow ssize_t */ 817 if (len < 0) { 818 ret = -EINVAL; 819 goto out; 820 } 821 if (type >= 0 822 && unlikely(!access_ok(vrfy_dir(type), buf, len))) { 823 ret = -EFAULT; 824 goto out; 825 } 826 if (len > MAX_RW_COUNT - ret) { 827 len = MAX_RW_COUNT - ret; 828 iov[seg].iov_len = len; 829 } 830 ret += len; 831 } 832 out: 833 *ret_pointer = iov; 834 return ret; 835 } 836 837 static ssize_t do_readv_writev(int type, struct file *file, 838 const struct iovec __user * uvector, 839 unsigned long nr_segs, loff_t *pos, 840 int flags) 841 { 842 size_t tot_len; 843 struct iovec iovstack[UIO_FASTIOV]; 844 struct iovec *iov = iovstack; 845 struct iov_iter iter; 846 ssize_t ret; 847 io_fn_t fn; 848 iter_fn_t iter_fn; 849 850 ret = import_iovec(type, uvector, nr_segs, 851 ARRAY_SIZE(iovstack), &iov, &iter); 852 if (ret < 0) 853 return ret; 854 855 tot_len = iov_iter_count(&iter); 856 if (!tot_len) 857 goto out; 858 ret = rw_verify_area(type, file, pos, tot_len); 859 if (ret < 0) 860 goto out; 861 862 if (type == READ) { 863 fn = file->f_op->read; 864 iter_fn = file->f_op->read_iter; 865 } else { 866 fn = (io_fn_t)file->f_op->write; 867 iter_fn = file->f_op->write_iter; 868 file_start_write(file); 869 } 870 871 if (iter_fn) 872 ret = do_iter_readv_writev(file, &iter, pos, iter_fn, flags); 873 else 874 ret = do_loop_readv_writev(file, &iter, pos, fn, flags); 875 876 if (type != READ) 877 file_end_write(file); 878 879 out: 880 kfree(iov); 881 if ((ret + (type == READ)) > 0) { 882 if (type == READ) 883 fsnotify_access(file); 884 else 885 fsnotify_modify(file); 886 } 887 return ret; 888 } 889 890 ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, 891 unsigned long vlen, loff_t *pos, int flags) 892 { 893 if (!(file->f_mode & FMODE_READ)) 894 return -EBADF; 895 if (!(file->f_mode & FMODE_CAN_READ)) 896 return -EINVAL; 897 898 return do_readv_writev(READ, file, vec, vlen, pos, flags); 899 } 900 901 EXPORT_SYMBOL(vfs_readv); 902 903 ssize_t vfs_writev(struct file *file, const struct iovec __user *vec, 904 unsigned long vlen, loff_t *pos, int flags) 905 { 906 if (!(file->f_mode & FMODE_WRITE)) 907 return -EBADF; 908 if (!(file->f_mode & FMODE_CAN_WRITE)) 909 return -EINVAL; 910 911 return do_readv_writev(WRITE, file, vec, vlen, pos, flags); 912 } 913 914 EXPORT_SYMBOL(vfs_writev); 915 916 static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec, 917 unsigned long vlen, int flags) 918 { 919 struct fd f = fdget_pos(fd); 920 ssize_t ret = -EBADF; 921 922 if (f.file) { 923 loff_t pos = file_pos_read(f.file); 924 ret = vfs_readv(f.file, vec, vlen, &pos, flags); 925 if (ret >= 0) 926 file_pos_write(f.file, pos); 927 fdput_pos(f); 928 } 929 930 if (ret > 0) 931 add_rchar(current, ret); 932 inc_syscr(current); 933 return ret; 934 } 935 936 static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec, 937 unsigned long vlen, int flags) 938 { 939 struct fd f = fdget_pos(fd); 940 ssize_t ret = -EBADF; 941 942 if (f.file) { 943 loff_t pos = file_pos_read(f.file); 944 ret = vfs_writev(f.file, vec, vlen, &pos, flags); 945 if (ret >= 0) 946 file_pos_write(f.file, pos); 947 fdput_pos(f); 948 } 949 950 if (ret > 0) 951 add_wchar(current, ret); 952 inc_syscw(current); 953 return ret; 954 } 955 956 static inline loff_t pos_from_hilo(unsigned long high, unsigned long low) 957 { 958 #define HALF_LONG_BITS (BITS_PER_LONG / 2) 959 return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low; 960 } 961 962 static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec, 963 unsigned long vlen, loff_t pos, int flags) 964 { 965 struct fd f; 966 ssize_t ret = -EBADF; 967 968 if (pos < 0) 969 return -EINVAL; 970 971 f = fdget(fd); 972 if (f.file) { 973 ret = -ESPIPE; 974 if (f.file->f_mode & FMODE_PREAD) 975 ret = vfs_readv(f.file, vec, vlen, &pos, flags); 976 fdput(f); 977 } 978 979 if (ret > 0) 980 add_rchar(current, ret); 981 inc_syscr(current); 982 return ret; 983 } 984 985 static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec, 986 unsigned long vlen, loff_t pos, int flags) 987 { 988 struct fd f; 989 ssize_t ret = -EBADF; 990 991 if (pos < 0) 992 return -EINVAL; 993 994 f = fdget(fd); 995 if (f.file) { 996 ret = -ESPIPE; 997 if (f.file->f_mode & FMODE_PWRITE) 998 ret = vfs_writev(f.file, vec, vlen, &pos, flags); 999 fdput(f); 1000 } 1001 1002 if (ret > 0) 1003 add_wchar(current, ret); 1004 inc_syscw(current); 1005 return ret; 1006 } 1007 1008 SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec, 1009 unsigned long, vlen) 1010 { 1011 return do_readv(fd, vec, vlen, 0); 1012 } 1013 1014 SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec, 1015 unsigned long, vlen) 1016 { 1017 return do_writev(fd, vec, vlen, 0); 1018 } 1019 1020 SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec, 1021 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h) 1022 { 1023 loff_t pos = pos_from_hilo(pos_h, pos_l); 1024 1025 return do_preadv(fd, vec, vlen, pos, 0); 1026 } 1027 1028 SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec, 1029 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h, 1030 int, flags) 1031 { 1032 loff_t pos = pos_from_hilo(pos_h, pos_l); 1033 1034 if (pos == -1) 1035 return do_readv(fd, vec, vlen, flags); 1036 1037 return do_preadv(fd, vec, vlen, pos, flags); 1038 } 1039 1040 SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec, 1041 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h) 1042 { 1043 loff_t pos = pos_from_hilo(pos_h, pos_l); 1044 1045 return do_pwritev(fd, vec, vlen, pos, 0); 1046 } 1047 1048 SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec, 1049 unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h, 1050 int, flags) 1051 { 1052 loff_t pos = pos_from_hilo(pos_h, pos_l); 1053 1054 if (pos == -1) 1055 return do_writev(fd, vec, vlen, flags); 1056 1057 return do_pwritev(fd, vec, vlen, pos, flags); 1058 } 1059 1060 #ifdef CONFIG_COMPAT 1061 1062 static ssize_t compat_do_readv_writev(int type, struct file *file, 1063 const struct compat_iovec __user *uvector, 1064 unsigned long nr_segs, loff_t *pos, 1065 int flags) 1066 { 1067 compat_ssize_t tot_len; 1068 struct iovec iovstack[UIO_FASTIOV]; 1069 struct iovec *iov = iovstack; 1070 struct iov_iter iter; 1071 ssize_t ret; 1072 io_fn_t fn; 1073 iter_fn_t iter_fn; 1074 1075 ret = compat_import_iovec(type, uvector, nr_segs, 1076 UIO_FASTIOV, &iov, &iter); 1077 if (ret < 0) 1078 return ret; 1079 1080 tot_len = iov_iter_count(&iter); 1081 if (!tot_len) 1082 goto out; 1083 ret = rw_verify_area(type, file, pos, tot_len); 1084 if (ret < 0) 1085 goto out; 1086 1087 if (type == READ) { 1088 fn = file->f_op->read; 1089 iter_fn = file->f_op->read_iter; 1090 } else { 1091 fn = (io_fn_t)file->f_op->write; 1092 iter_fn = file->f_op->write_iter; 1093 file_start_write(file); 1094 } 1095 1096 if (iter_fn) 1097 ret = do_iter_readv_writev(file, &iter, pos, iter_fn, flags); 1098 else 1099 ret = do_loop_readv_writev(file, &iter, pos, fn, flags); 1100 1101 if (type != READ) 1102 file_end_write(file); 1103 1104 out: 1105 kfree(iov); 1106 if ((ret + (type == READ)) > 0) { 1107 if (type == READ) 1108 fsnotify_access(file); 1109 else 1110 fsnotify_modify(file); 1111 } 1112 return ret; 1113 } 1114 1115 static size_t compat_readv(struct file *file, 1116 const struct compat_iovec __user *vec, 1117 unsigned long vlen, loff_t *pos, int flags) 1118 { 1119 ssize_t ret = -EBADF; 1120 1121 if (!(file->f_mode & FMODE_READ)) 1122 goto out; 1123 1124 ret = -EINVAL; 1125 if (!(file->f_mode & FMODE_CAN_READ)) 1126 goto out; 1127 1128 ret = compat_do_readv_writev(READ, file, vec, vlen, pos, flags); 1129 1130 out: 1131 if (ret > 0) 1132 add_rchar(current, ret); 1133 inc_syscr(current); 1134 return ret; 1135 } 1136 1137 static size_t do_compat_readv(compat_ulong_t fd, 1138 const struct compat_iovec __user *vec, 1139 compat_ulong_t vlen, int flags) 1140 { 1141 struct fd f = fdget_pos(fd); 1142 ssize_t ret; 1143 loff_t pos; 1144 1145 if (!f.file) 1146 return -EBADF; 1147 pos = f.file->f_pos; 1148 ret = compat_readv(f.file, vec, vlen, &pos, flags); 1149 if (ret >= 0) 1150 f.file->f_pos = pos; 1151 fdput_pos(f); 1152 return ret; 1153 1154 } 1155 1156 COMPAT_SYSCALL_DEFINE3(readv, compat_ulong_t, fd, 1157 const struct compat_iovec __user *,vec, 1158 compat_ulong_t, vlen) 1159 { 1160 return do_compat_readv(fd, vec, vlen, 0); 1161 } 1162 1163 static long do_compat_preadv64(unsigned long fd, 1164 const struct compat_iovec __user *vec, 1165 unsigned long vlen, loff_t pos, int flags) 1166 { 1167 struct fd f; 1168 ssize_t ret; 1169 1170 if (pos < 0) 1171 return -EINVAL; 1172 f = fdget(fd); 1173 if (!f.file) 1174 return -EBADF; 1175 ret = -ESPIPE; 1176 if (f.file->f_mode & FMODE_PREAD) 1177 ret = compat_readv(f.file, vec, vlen, &pos, flags); 1178 fdput(f); 1179 return ret; 1180 } 1181 1182 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64 1183 COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd, 1184 const struct compat_iovec __user *,vec, 1185 unsigned long, vlen, loff_t, pos) 1186 { 1187 return do_compat_preadv64(fd, vec, vlen, pos, 0); 1188 } 1189 #endif 1190 1191 COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd, 1192 const struct compat_iovec __user *,vec, 1193 compat_ulong_t, vlen, u32, pos_low, u32, pos_high) 1194 { 1195 loff_t pos = ((loff_t)pos_high << 32) | pos_low; 1196 1197 return do_compat_preadv64(fd, vec, vlen, pos, 0); 1198 } 1199 1200 #ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2 1201 COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd, 1202 const struct compat_iovec __user *,vec, 1203 unsigned long, vlen, loff_t, pos, int, flags) 1204 { 1205 return do_compat_preadv64(fd, vec, vlen, pos, flags); 1206 } 1207 #endif 1208 1209 COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd, 1210 const struct compat_iovec __user *,vec, 1211 compat_ulong_t, vlen, u32, pos_low, u32, pos_high, 1212 int, flags) 1213 { 1214 loff_t pos = ((loff_t)pos_high << 32) | pos_low; 1215 1216 if (pos == -1) 1217 return do_compat_readv(fd, vec, vlen, flags); 1218 1219 return do_compat_preadv64(fd, vec, vlen, pos, flags); 1220 } 1221 1222 static size_t compat_writev(struct file *file, 1223 const struct compat_iovec __user *vec, 1224 unsigned long vlen, loff_t *pos, int flags) 1225 { 1226 ssize_t ret = -EBADF; 1227 1228 if (!(file->f_mode & FMODE_WRITE)) 1229 goto out; 1230 1231 ret = -EINVAL; 1232 if (!(file->f_mode & FMODE_CAN_WRITE)) 1233 goto out; 1234 1235 ret = compat_do_readv_writev(WRITE, file, vec, vlen, pos, 0); 1236 1237 out: 1238 if (ret > 0) 1239 add_wchar(current, ret); 1240 inc_syscw(current); 1241 return ret; 1242 } 1243 1244 static size_t do_compat_writev(compat_ulong_t fd, 1245 const struct compat_iovec __user* vec, 1246 compat_ulong_t vlen, int flags) 1247 { 1248 struct fd f = fdget_pos(fd); 1249 ssize_t ret; 1250 loff_t pos; 1251 1252 if (!f.file) 1253 return -EBADF; 1254 pos = f.file->f_pos; 1255 ret = compat_writev(f.file, vec, vlen, &pos, flags); 1256 if (ret >= 0) 1257 f.file->f_pos = pos; 1258 fdput_pos(f); 1259 return ret; 1260 } 1261 1262 COMPAT_SYSCALL_DEFINE3(writev, compat_ulong_t, fd, 1263 const struct compat_iovec __user *, vec, 1264 compat_ulong_t, vlen) 1265 { 1266 return do_compat_writev(fd, vec, vlen, 0); 1267 } 1268 1269 static long do_compat_pwritev64(unsigned long fd, 1270 const struct compat_iovec __user *vec, 1271 unsigned long vlen, loff_t pos, int flags) 1272 { 1273 struct fd f; 1274 ssize_t ret; 1275 1276 if (pos < 0) 1277 return -EINVAL; 1278 f = fdget(fd); 1279 if (!f.file) 1280 return -EBADF; 1281 ret = -ESPIPE; 1282 if (f.file->f_mode & FMODE_PWRITE) 1283 ret = compat_writev(f.file, vec, vlen, &pos, flags); 1284 fdput(f); 1285 return ret; 1286 } 1287 1288 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64 1289 COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd, 1290 const struct compat_iovec __user *,vec, 1291 unsigned long, vlen, loff_t, pos) 1292 { 1293 return do_compat_pwritev64(fd, vec, vlen, pos, 0); 1294 } 1295 #endif 1296 1297 COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd, 1298 const struct compat_iovec __user *,vec, 1299 compat_ulong_t, vlen, u32, pos_low, u32, pos_high) 1300 { 1301 loff_t pos = ((loff_t)pos_high << 32) | pos_low; 1302 1303 return do_compat_pwritev64(fd, vec, vlen, pos, 0); 1304 } 1305 1306 #ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2 1307 COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd, 1308 const struct compat_iovec __user *,vec, 1309 unsigned long, vlen, loff_t, pos, int, flags) 1310 { 1311 return do_compat_pwritev64(fd, vec, vlen, pos, flags); 1312 } 1313 #endif 1314 1315 COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd, 1316 const struct compat_iovec __user *,vec, 1317 compat_ulong_t, vlen, u32, pos_low, u32, pos_high, int, flags) 1318 { 1319 loff_t pos = ((loff_t)pos_high << 32) | pos_low; 1320 1321 if (pos == -1) 1322 return do_compat_writev(fd, vec, vlen, flags); 1323 1324 return do_compat_pwritev64(fd, vec, vlen, pos, flags); 1325 } 1326 1327 #endif 1328 1329 static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, 1330 size_t count, loff_t max) 1331 { 1332 struct fd in, out; 1333 struct inode *in_inode, *out_inode; 1334 loff_t pos; 1335 loff_t out_pos; 1336 ssize_t retval; 1337 int fl; 1338 1339 /* 1340 * Get input file, and verify that it is ok.. 1341 */ 1342 retval = -EBADF; 1343 in = fdget(in_fd); 1344 if (!in.file) 1345 goto out; 1346 if (!(in.file->f_mode & FMODE_READ)) 1347 goto fput_in; 1348 retval = -ESPIPE; 1349 if (!ppos) { 1350 pos = in.file->f_pos; 1351 } else { 1352 pos = *ppos; 1353 if (!(in.file->f_mode & FMODE_PREAD)) 1354 goto fput_in; 1355 } 1356 retval = rw_verify_area(READ, in.file, &pos, count); 1357 if (retval < 0) 1358 goto fput_in; 1359 if (count > MAX_RW_COUNT) 1360 count = MAX_RW_COUNT; 1361 1362 /* 1363 * Get output file, and verify that it is ok.. 1364 */ 1365 retval = -EBADF; 1366 out = fdget(out_fd); 1367 if (!out.file) 1368 goto fput_in; 1369 if (!(out.file->f_mode & FMODE_WRITE)) 1370 goto fput_out; 1371 retval = -EINVAL; 1372 in_inode = file_inode(in.file); 1373 out_inode = file_inode(out.file); 1374 out_pos = out.file->f_pos; 1375 retval = rw_verify_area(WRITE, out.file, &out_pos, count); 1376 if (retval < 0) 1377 goto fput_out; 1378 1379 if (!max) 1380 max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes); 1381 1382 if (unlikely(pos + count > max)) { 1383 retval = -EOVERFLOW; 1384 if (pos >= max) 1385 goto fput_out; 1386 count = max - pos; 1387 } 1388 1389 fl = 0; 1390 #if 0 1391 /* 1392 * We need to debate whether we can enable this or not. The 1393 * man page documents EAGAIN return for the output at least, 1394 * and the application is arguably buggy if it doesn't expect 1395 * EAGAIN on a non-blocking file descriptor. 1396 */ 1397 if (in.file->f_flags & O_NONBLOCK) 1398 fl = SPLICE_F_NONBLOCK; 1399 #endif 1400 file_start_write(out.file); 1401 retval = do_splice_direct(in.file, &pos, out.file, &out_pos, count, fl); 1402 file_end_write(out.file); 1403 1404 if (retval > 0) { 1405 add_rchar(current, retval); 1406 add_wchar(current, retval); 1407 fsnotify_access(in.file); 1408 fsnotify_modify(out.file); 1409 out.file->f_pos = out_pos; 1410 if (ppos) 1411 *ppos = pos; 1412 else 1413 in.file->f_pos = pos; 1414 } 1415 1416 inc_syscr(current); 1417 inc_syscw(current); 1418 if (pos > max) 1419 retval = -EOVERFLOW; 1420 1421 fput_out: 1422 fdput(out); 1423 fput_in: 1424 fdput(in); 1425 out: 1426 return retval; 1427 } 1428 1429 SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count) 1430 { 1431 loff_t pos; 1432 off_t off; 1433 ssize_t ret; 1434 1435 if (offset) { 1436 if (unlikely(get_user(off, offset))) 1437 return -EFAULT; 1438 pos = off; 1439 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS); 1440 if (unlikely(put_user(pos, offset))) 1441 return -EFAULT; 1442 return ret; 1443 } 1444 1445 return do_sendfile(out_fd, in_fd, NULL, count, 0); 1446 } 1447 1448 SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count) 1449 { 1450 loff_t pos; 1451 ssize_t ret; 1452 1453 if (offset) { 1454 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t)))) 1455 return -EFAULT; 1456 ret = do_sendfile(out_fd, in_fd, &pos, count, 0); 1457 if (unlikely(put_user(pos, offset))) 1458 return -EFAULT; 1459 return ret; 1460 } 1461 1462 return do_sendfile(out_fd, in_fd, NULL, count, 0); 1463 } 1464 1465 #ifdef CONFIG_COMPAT 1466 COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, 1467 compat_off_t __user *, offset, compat_size_t, count) 1468 { 1469 loff_t pos; 1470 off_t off; 1471 ssize_t ret; 1472 1473 if (offset) { 1474 if (unlikely(get_user(off, offset))) 1475 return -EFAULT; 1476 pos = off; 1477 ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS); 1478 if (unlikely(put_user(pos, offset))) 1479 return -EFAULT; 1480 return ret; 1481 } 1482 1483 return do_sendfile(out_fd, in_fd, NULL, count, 0); 1484 } 1485 1486 COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, 1487 compat_loff_t __user *, offset, compat_size_t, count) 1488 { 1489 loff_t pos; 1490 ssize_t ret; 1491 1492 if (offset) { 1493 if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t)))) 1494 return -EFAULT; 1495 ret = do_sendfile(out_fd, in_fd, &pos, count, 0); 1496 if (unlikely(put_user(pos, offset))) 1497 return -EFAULT; 1498 return ret; 1499 } 1500 1501 return do_sendfile(out_fd, in_fd, NULL, count, 0); 1502 } 1503 #endif 1504 1505 /* 1506 * copy_file_range() differs from regular file read and write in that it 1507 * specifically allows return partial success. When it does so is up to 1508 * the copy_file_range method. 1509 */ 1510 ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in, 1511 struct file *file_out, loff_t pos_out, 1512 size_t len, unsigned int flags) 1513 { 1514 struct inode *inode_in = file_inode(file_in); 1515 struct inode *inode_out = file_inode(file_out); 1516 ssize_t ret; 1517 1518 if (flags != 0) 1519 return -EINVAL; 1520 1521 ret = rw_verify_area(READ, file_in, &pos_in, len); 1522 if (unlikely(ret)) 1523 return ret; 1524 1525 ret = rw_verify_area(WRITE, file_out, &pos_out, len); 1526 if (unlikely(ret)) 1527 return ret; 1528 1529 if (!(file_in->f_mode & FMODE_READ) || 1530 !(file_out->f_mode & FMODE_WRITE) || 1531 (file_out->f_flags & O_APPEND)) 1532 return -EBADF; 1533 1534 /* this could be relaxed once a method supports cross-fs copies */ 1535 if (inode_in->i_sb != inode_out->i_sb) 1536 return -EXDEV; 1537 1538 if (len == 0) 1539 return 0; 1540 1541 sb_start_write(inode_out->i_sb); 1542 1543 /* 1544 * Try cloning first, this is supported by more file systems, and 1545 * more efficient if both clone and copy are supported (e.g. NFS). 1546 */ 1547 if (file_in->f_op->clone_file_range) { 1548 ret = file_in->f_op->clone_file_range(file_in, pos_in, 1549 file_out, pos_out, len); 1550 if (ret == 0) { 1551 ret = len; 1552 goto done; 1553 } 1554 } 1555 1556 if (file_out->f_op->copy_file_range) { 1557 ret = file_out->f_op->copy_file_range(file_in, pos_in, file_out, 1558 pos_out, len, flags); 1559 if (ret != -EOPNOTSUPP) 1560 goto done; 1561 } 1562 1563 ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out, 1564 len > MAX_RW_COUNT ? MAX_RW_COUNT : len, 0); 1565 1566 done: 1567 if (ret > 0) { 1568 fsnotify_access(file_in); 1569 add_rchar(current, ret); 1570 fsnotify_modify(file_out); 1571 add_wchar(current, ret); 1572 } 1573 1574 inc_syscr(current); 1575 inc_syscw(current); 1576 1577 sb_end_write(inode_out->i_sb); 1578 1579 return ret; 1580 } 1581 EXPORT_SYMBOL(vfs_copy_file_range); 1582 1583 SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in, 1584 int, fd_out, loff_t __user *, off_out, 1585 size_t, len, unsigned int, flags) 1586 { 1587 loff_t pos_in; 1588 loff_t pos_out; 1589 struct fd f_in; 1590 struct fd f_out; 1591 ssize_t ret = -EBADF; 1592 1593 f_in = fdget(fd_in); 1594 if (!f_in.file) 1595 goto out2; 1596 1597 f_out = fdget(fd_out); 1598 if (!f_out.file) 1599 goto out1; 1600 1601 ret = -EFAULT; 1602 if (off_in) { 1603 if (copy_from_user(&pos_in, off_in, sizeof(loff_t))) 1604 goto out; 1605 } else { 1606 pos_in = f_in.file->f_pos; 1607 } 1608 1609 if (off_out) { 1610 if (copy_from_user(&pos_out, off_out, sizeof(loff_t))) 1611 goto out; 1612 } else { 1613 pos_out = f_out.file->f_pos; 1614 } 1615 1616 ret = vfs_copy_file_range(f_in.file, pos_in, f_out.file, pos_out, len, 1617 flags); 1618 if (ret > 0) { 1619 pos_in += ret; 1620 pos_out += ret; 1621 1622 if (off_in) { 1623 if (copy_to_user(off_in, &pos_in, sizeof(loff_t))) 1624 ret = -EFAULT; 1625 } else { 1626 f_in.file->f_pos = pos_in; 1627 } 1628 1629 if (off_out) { 1630 if (copy_to_user(off_out, &pos_out, sizeof(loff_t))) 1631 ret = -EFAULT; 1632 } else { 1633 f_out.file->f_pos = pos_out; 1634 } 1635 } 1636 1637 out: 1638 fdput(f_out); 1639 out1: 1640 fdput(f_in); 1641 out2: 1642 return ret; 1643 } 1644 1645 static int clone_verify_area(struct file *file, loff_t pos, u64 len, bool write) 1646 { 1647 struct inode *inode = file_inode(file); 1648 1649 if (unlikely(pos < 0)) 1650 return -EINVAL; 1651 1652 if (unlikely((loff_t) (pos + len) < 0)) 1653 return -EINVAL; 1654 1655 if (unlikely(inode->i_flctx && mandatory_lock(inode))) { 1656 loff_t end = len ? pos + len - 1 : OFFSET_MAX; 1657 int retval; 1658 1659 retval = locks_mandatory_area(inode, file, pos, end, 1660 write ? F_WRLCK : F_RDLCK); 1661 if (retval < 0) 1662 return retval; 1663 } 1664 1665 return security_file_permission(file, write ? MAY_WRITE : MAY_READ); 1666 } 1667 1668 /* 1669 * Check that the two inodes are eligible for cloning, the ranges make 1670 * sense, and then flush all dirty data. Caller must ensure that the 1671 * inodes have been locked against any other modifications. 1672 * 1673 * Returns: 0 for "nothing to clone", 1 for "something to clone", or 1674 * the usual negative error code. 1675 */ 1676 int vfs_clone_file_prep_inodes(struct inode *inode_in, loff_t pos_in, 1677 struct inode *inode_out, loff_t pos_out, 1678 u64 *len, bool is_dedupe) 1679 { 1680 loff_t bs = inode_out->i_sb->s_blocksize; 1681 loff_t blen; 1682 loff_t isize; 1683 bool same_inode = (inode_in == inode_out); 1684 int ret; 1685 1686 /* Don't touch certain kinds of inodes */ 1687 if (IS_IMMUTABLE(inode_out)) 1688 return -EPERM; 1689 1690 if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out)) 1691 return -ETXTBSY; 1692 1693 /* Don't reflink dirs, pipes, sockets... */ 1694 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) 1695 return -EISDIR; 1696 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) 1697 return -EINVAL; 1698 1699 /* Are we going all the way to the end? */ 1700 isize = i_size_read(inode_in); 1701 if (isize == 0) 1702 return 0; 1703 1704 /* Zero length dedupe exits immediately; reflink goes to EOF. */ 1705 if (*len == 0) { 1706 if (is_dedupe || pos_in == isize) 1707 return 0; 1708 if (pos_in > isize) 1709 return -EINVAL; 1710 *len = isize - pos_in; 1711 } 1712 1713 /* Ensure offsets don't wrap and the input is inside i_size */ 1714 if (pos_in + *len < pos_in || pos_out + *len < pos_out || 1715 pos_in + *len > isize) 1716 return -EINVAL; 1717 1718 /* Don't allow dedupe past EOF in the dest file */ 1719 if (is_dedupe) { 1720 loff_t disize; 1721 1722 disize = i_size_read(inode_out); 1723 if (pos_out >= disize || pos_out + *len > disize) 1724 return -EINVAL; 1725 } 1726 1727 /* If we're linking to EOF, continue to the block boundary. */ 1728 if (pos_in + *len == isize) 1729 blen = ALIGN(isize, bs) - pos_in; 1730 else 1731 blen = *len; 1732 1733 /* Only reflink if we're aligned to block boundaries */ 1734 if (!IS_ALIGNED(pos_in, bs) || !IS_ALIGNED(pos_in + blen, bs) || 1735 !IS_ALIGNED(pos_out, bs) || !IS_ALIGNED(pos_out + blen, bs)) 1736 return -EINVAL; 1737 1738 /* Don't allow overlapped reflink within the same file */ 1739 if (same_inode) { 1740 if (pos_out + blen > pos_in && pos_out < pos_in + blen) 1741 return -EINVAL; 1742 } 1743 1744 /* Wait for the completion of any pending IOs on both files */ 1745 inode_dio_wait(inode_in); 1746 if (!same_inode) 1747 inode_dio_wait(inode_out); 1748 1749 ret = filemap_write_and_wait_range(inode_in->i_mapping, 1750 pos_in, pos_in + *len - 1); 1751 if (ret) 1752 return ret; 1753 1754 ret = filemap_write_and_wait_range(inode_out->i_mapping, 1755 pos_out, pos_out + *len - 1); 1756 if (ret) 1757 return ret; 1758 1759 /* 1760 * Check that the extents are the same. 1761 */ 1762 if (is_dedupe) { 1763 bool is_same = false; 1764 1765 ret = vfs_dedupe_file_range_compare(inode_in, pos_in, 1766 inode_out, pos_out, *len, &is_same); 1767 if (ret) 1768 return ret; 1769 if (!is_same) 1770 return -EBADE; 1771 } 1772 1773 return 1; 1774 } 1775 EXPORT_SYMBOL(vfs_clone_file_prep_inodes); 1776 1777 int vfs_clone_file_range(struct file *file_in, loff_t pos_in, 1778 struct file *file_out, loff_t pos_out, u64 len) 1779 { 1780 struct inode *inode_in = file_inode(file_in); 1781 struct inode *inode_out = file_inode(file_out); 1782 int ret; 1783 1784 if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) 1785 return -EISDIR; 1786 if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) 1787 return -EINVAL; 1788 1789 /* 1790 * FICLONE/FICLONERANGE ioctls enforce that src and dest files are on 1791 * the same mount. Practically, they only need to be on the same file 1792 * system. 1793 */ 1794 if (inode_in->i_sb != inode_out->i_sb) 1795 return -EXDEV; 1796 1797 if (!(file_in->f_mode & FMODE_READ) || 1798 !(file_out->f_mode & FMODE_WRITE) || 1799 (file_out->f_flags & O_APPEND)) 1800 return -EBADF; 1801 1802 if (!file_in->f_op->clone_file_range) 1803 return -EOPNOTSUPP; 1804 1805 ret = clone_verify_area(file_in, pos_in, len, false); 1806 if (ret) 1807 return ret; 1808 1809 ret = clone_verify_area(file_out, pos_out, len, true); 1810 if (ret) 1811 return ret; 1812 1813 if (pos_in + len > i_size_read(inode_in)) 1814 return -EINVAL; 1815 1816 ret = file_in->f_op->clone_file_range(file_in, pos_in, 1817 file_out, pos_out, len); 1818 if (!ret) { 1819 fsnotify_access(file_in); 1820 fsnotify_modify(file_out); 1821 } 1822 1823 return ret; 1824 } 1825 EXPORT_SYMBOL(vfs_clone_file_range); 1826 1827 /* 1828 * Read a page's worth of file data into the page cache. Return the page 1829 * locked. 1830 */ 1831 static struct page *vfs_dedupe_get_page(struct inode *inode, loff_t offset) 1832 { 1833 struct address_space *mapping; 1834 struct page *page; 1835 pgoff_t n; 1836 1837 n = offset >> PAGE_SHIFT; 1838 mapping = inode->i_mapping; 1839 page = read_mapping_page(mapping, n, NULL); 1840 if (IS_ERR(page)) 1841 return page; 1842 if (!PageUptodate(page)) { 1843 put_page(page); 1844 return ERR_PTR(-EIO); 1845 } 1846 lock_page(page); 1847 return page; 1848 } 1849 1850 /* 1851 * Compare extents of two files to see if they are the same. 1852 * Caller must have locked both inodes to prevent write races. 1853 */ 1854 int vfs_dedupe_file_range_compare(struct inode *src, loff_t srcoff, 1855 struct inode *dest, loff_t destoff, 1856 loff_t len, bool *is_same) 1857 { 1858 loff_t src_poff; 1859 loff_t dest_poff; 1860 void *src_addr; 1861 void *dest_addr; 1862 struct page *src_page; 1863 struct page *dest_page; 1864 loff_t cmp_len; 1865 bool same; 1866 int error; 1867 1868 error = -EINVAL; 1869 same = true; 1870 while (len) { 1871 src_poff = srcoff & (PAGE_SIZE - 1); 1872 dest_poff = destoff & (PAGE_SIZE - 1); 1873 cmp_len = min(PAGE_SIZE - src_poff, 1874 PAGE_SIZE - dest_poff); 1875 cmp_len = min(cmp_len, len); 1876 if (cmp_len <= 0) 1877 goto out_error; 1878 1879 src_page = vfs_dedupe_get_page(src, srcoff); 1880 if (IS_ERR(src_page)) { 1881 error = PTR_ERR(src_page); 1882 goto out_error; 1883 } 1884 dest_page = vfs_dedupe_get_page(dest, destoff); 1885 if (IS_ERR(dest_page)) { 1886 error = PTR_ERR(dest_page); 1887 unlock_page(src_page); 1888 put_page(src_page); 1889 goto out_error; 1890 } 1891 src_addr = kmap_atomic(src_page); 1892 dest_addr = kmap_atomic(dest_page); 1893 1894 flush_dcache_page(src_page); 1895 flush_dcache_page(dest_page); 1896 1897 if (memcmp(src_addr + src_poff, dest_addr + dest_poff, cmp_len)) 1898 same = false; 1899 1900 kunmap_atomic(dest_addr); 1901 kunmap_atomic(src_addr); 1902 unlock_page(dest_page); 1903 unlock_page(src_page); 1904 put_page(dest_page); 1905 put_page(src_page); 1906 1907 if (!same) 1908 break; 1909 1910 srcoff += cmp_len; 1911 destoff += cmp_len; 1912 len -= cmp_len; 1913 } 1914 1915 *is_same = same; 1916 return 0; 1917 1918 out_error: 1919 return error; 1920 } 1921 EXPORT_SYMBOL(vfs_dedupe_file_range_compare); 1922 1923 int vfs_dedupe_file_range(struct file *file, struct file_dedupe_range *same) 1924 { 1925 struct file_dedupe_range_info *info; 1926 struct inode *src = file_inode(file); 1927 u64 off; 1928 u64 len; 1929 int i; 1930 int ret; 1931 bool is_admin = capable(CAP_SYS_ADMIN); 1932 u16 count = same->dest_count; 1933 struct file *dst_file; 1934 loff_t dst_off; 1935 ssize_t deduped; 1936 1937 if (!(file->f_mode & FMODE_READ)) 1938 return -EINVAL; 1939 1940 if (same->reserved1 || same->reserved2) 1941 return -EINVAL; 1942 1943 off = same->src_offset; 1944 len = same->src_length; 1945 1946 ret = -EISDIR; 1947 if (S_ISDIR(src->i_mode)) 1948 goto out; 1949 1950 ret = -EINVAL; 1951 if (!S_ISREG(src->i_mode)) 1952 goto out; 1953 1954 ret = clone_verify_area(file, off, len, false); 1955 if (ret < 0) 1956 goto out; 1957 ret = 0; 1958 1959 if (off + len > i_size_read(src)) 1960 return -EINVAL; 1961 1962 /* pre-format output fields to sane values */ 1963 for (i = 0; i < count; i++) { 1964 same->info[i].bytes_deduped = 0ULL; 1965 same->info[i].status = FILE_DEDUPE_RANGE_SAME; 1966 } 1967 1968 for (i = 0, info = same->info; i < count; i++, info++) { 1969 struct inode *dst; 1970 struct fd dst_fd = fdget(info->dest_fd); 1971 1972 dst_file = dst_fd.file; 1973 if (!dst_file) { 1974 info->status = -EBADF; 1975 goto next_loop; 1976 } 1977 dst = file_inode(dst_file); 1978 1979 ret = mnt_want_write_file(dst_file); 1980 if (ret) { 1981 info->status = ret; 1982 goto next_loop; 1983 } 1984 1985 dst_off = info->dest_offset; 1986 ret = clone_verify_area(dst_file, dst_off, len, true); 1987 if (ret < 0) { 1988 info->status = ret; 1989 goto next_file; 1990 } 1991 ret = 0; 1992 1993 if (info->reserved) { 1994 info->status = -EINVAL; 1995 } else if (!(is_admin || (dst_file->f_mode & FMODE_WRITE))) { 1996 info->status = -EINVAL; 1997 } else if (file->f_path.mnt != dst_file->f_path.mnt) { 1998 info->status = -EXDEV; 1999 } else if (S_ISDIR(dst->i_mode)) { 2000 info->status = -EISDIR; 2001 } else if (dst_file->f_op->dedupe_file_range == NULL) { 2002 info->status = -EINVAL; 2003 } else { 2004 deduped = dst_file->f_op->dedupe_file_range(file, off, 2005 len, dst_file, 2006 info->dest_offset); 2007 if (deduped == -EBADE) 2008 info->status = FILE_DEDUPE_RANGE_DIFFERS; 2009 else if (deduped < 0) 2010 info->status = deduped; 2011 else 2012 info->bytes_deduped += deduped; 2013 } 2014 2015 next_file: 2016 mnt_drop_write_file(dst_file); 2017 next_loop: 2018 fdput(dst_fd); 2019 2020 if (fatal_signal_pending(current)) 2021 goto out; 2022 } 2023 2024 out: 2025 return ret; 2026 } 2027 EXPORT_SYMBOL(vfs_dedupe_file_range); 2028