1 // SPDX-License-Identifier: GPL-2.0-only 2 #include <crypto/hash.h> 3 #include <linux/export.h> 4 #include <linux/bvec.h> 5 #include <linux/fault-inject-usercopy.h> 6 #include <linux/uio.h> 7 #include <linux/pagemap.h> 8 #include <linux/highmem.h> 9 #include <linux/slab.h> 10 #include <linux/vmalloc.h> 11 #include <linux/splice.h> 12 #include <linux/compat.h> 13 #include <net/checksum.h> 14 #include <linux/scatterlist.h> 15 #include <linux/instrumented.h> 16 17 #define PIPE_PARANOIA /* for now */ 18 19 /* covers ubuf and kbuf alike */ 20 #define iterate_buf(i, n, base, len, off, __p, STEP) { \ 21 size_t __maybe_unused off = 0; \ 22 len = n; \ 23 base = __p + i->iov_offset; \ 24 len -= (STEP); \ 25 i->iov_offset += len; \ 26 n = len; \ 27 } 28 29 /* covers iovec and kvec alike */ 30 #define iterate_iovec(i, n, base, len, off, __p, STEP) { \ 31 size_t off = 0; \ 32 size_t skip = i->iov_offset; \ 33 do { \ 34 len = min(n, __p->iov_len - skip); \ 35 if (likely(len)) { \ 36 base = __p->iov_base + skip; \ 37 len -= (STEP); \ 38 off += len; \ 39 skip += len; \ 40 n -= len; \ 41 if (skip < __p->iov_len) \ 42 break; \ 43 } \ 44 __p++; \ 45 skip = 0; \ 46 } while (n); \ 47 i->iov_offset = skip; \ 48 n = off; \ 49 } 50 51 #define iterate_bvec(i, n, base, len, off, p, STEP) { \ 52 size_t off = 0; \ 53 unsigned skip = i->iov_offset; \ 54 while (n) { \ 55 unsigned offset = p->bv_offset + skip; \ 56 unsigned left; \ 57 void *kaddr = kmap_local_page(p->bv_page + \ 58 offset / PAGE_SIZE); \ 59 base = kaddr + offset % PAGE_SIZE; \ 60 len = min(min(n, (size_t)(p->bv_len - skip)), \ 61 (size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \ 62 left = (STEP); \ 63 kunmap_local(kaddr); \ 64 len -= left; \ 65 off += len; \ 66 skip += len; \ 67 if (skip == p->bv_len) { \ 68 skip = 0; \ 69 p++; \ 70 } \ 71 n -= len; \ 72 if (left) \ 73 break; \ 74 } \ 75 i->iov_offset = skip; \ 76 n = off; \ 77 } 78 79 #define iterate_xarray(i, n, base, len, __off, STEP) { \ 80 __label__ __out; \ 81 size_t __off = 0; \ 82 struct folio *folio; \ 83 loff_t start = i->xarray_start + i->iov_offset; \ 84 pgoff_t index = start / PAGE_SIZE; \ 85 XA_STATE(xas, i->xarray, index); \ 86 \ 87 len = PAGE_SIZE - offset_in_page(start); \ 88 rcu_read_lock(); \ 89 xas_for_each(&xas, folio, ULONG_MAX) { \ 90 unsigned left; \ 91 size_t offset; \ 92 if (xas_retry(&xas, folio)) \ 93 continue; \ 94 if (WARN_ON(xa_is_value(folio))) \ 95 break; \ 96 if (WARN_ON(folio_test_hugetlb(folio))) \ 97 break; \ 98 offset = offset_in_folio(folio, start + __off); \ 99 while (offset < folio_size(folio)) { \ 100 base = kmap_local_folio(folio, offset); \ 101 len = min(n, len); \ 102 left = (STEP); \ 103 kunmap_local(base); \ 104 len -= left; \ 105 __off += len; \ 106 n -= len; \ 107 if (left || n == 0) \ 108 goto __out; \ 109 offset += len; \ 110 len = PAGE_SIZE; \ 111 } \ 112 } \ 113 __out: \ 114 rcu_read_unlock(); \ 115 i->iov_offset += __off; \ 116 n = __off; \ 117 } 118 119 #define __iterate_and_advance(i, n, base, len, off, I, K) { \ 120 if (unlikely(i->count < n)) \ 121 n = i->count; \ 122 if (likely(n)) { \ 123 if (likely(iter_is_ubuf(i))) { \ 124 void __user *base; \ 125 size_t len; \ 126 iterate_buf(i, n, base, len, off, \ 127 i->ubuf, (I)) \ 128 } else if (likely(iter_is_iovec(i))) { \ 129 const struct iovec *iov = i->iov; \ 130 void __user *base; \ 131 size_t len; \ 132 iterate_iovec(i, n, base, len, off, \ 133 iov, (I)) \ 134 i->nr_segs -= iov - i->iov; \ 135 i->iov = iov; \ 136 } else if (iov_iter_is_bvec(i)) { \ 137 const struct bio_vec *bvec = i->bvec; \ 138 void *base; \ 139 size_t len; \ 140 iterate_bvec(i, n, base, len, off, \ 141 bvec, (K)) \ 142 i->nr_segs -= bvec - i->bvec; \ 143 i->bvec = bvec; \ 144 } else if (iov_iter_is_kvec(i)) { \ 145 const struct kvec *kvec = i->kvec; \ 146 void *base; \ 147 size_t len; \ 148 iterate_iovec(i, n, base, len, off, \ 149 kvec, (K)) \ 150 i->nr_segs -= kvec - i->kvec; \ 151 i->kvec = kvec; \ 152 } else if (iov_iter_is_xarray(i)) { \ 153 void *base; \ 154 size_t len; \ 155 iterate_xarray(i, n, base, len, off, \ 156 (K)) \ 157 } \ 158 i->count -= n; \ 159 } \ 160 } 161 #define iterate_and_advance(i, n, base, len, off, I, K) \ 162 __iterate_and_advance(i, n, base, len, off, I, ((void)(K),0)) 163 164 static int copyout(void __user *to, const void *from, size_t n) 165 { 166 if (should_fail_usercopy()) 167 return n; 168 if (access_ok(to, n)) { 169 instrument_copy_to_user(to, from, n); 170 n = raw_copy_to_user(to, from, n); 171 } 172 return n; 173 } 174 175 static int copyin(void *to, const void __user *from, size_t n) 176 { 177 if (should_fail_usercopy()) 178 return n; 179 if (access_ok(from, n)) { 180 instrument_copy_from_user(to, from, n); 181 n = raw_copy_from_user(to, from, n); 182 } 183 return n; 184 } 185 186 static inline struct pipe_buffer *pipe_buf(const struct pipe_inode_info *pipe, 187 unsigned int slot) 188 { 189 return &pipe->bufs[slot & (pipe->ring_size - 1)]; 190 } 191 192 #ifdef PIPE_PARANOIA 193 static bool sanity(const struct iov_iter *i) 194 { 195 struct pipe_inode_info *pipe = i->pipe; 196 unsigned int p_head = pipe->head; 197 unsigned int p_tail = pipe->tail; 198 unsigned int p_occupancy = pipe_occupancy(p_head, p_tail); 199 unsigned int i_head = i->head; 200 unsigned int idx; 201 202 if (i->iov_offset) { 203 struct pipe_buffer *p; 204 if (unlikely(p_occupancy == 0)) 205 goto Bad; // pipe must be non-empty 206 if (unlikely(i_head != p_head - 1)) 207 goto Bad; // must be at the last buffer... 208 209 p = pipe_buf(pipe, i_head); 210 if (unlikely(p->offset + p->len != i->iov_offset)) 211 goto Bad; // ... at the end of segment 212 } else { 213 if (i_head != p_head) 214 goto Bad; // must be right after the last buffer 215 } 216 return true; 217 Bad: 218 printk(KERN_ERR "idx = %d, offset = %zd\n", i_head, i->iov_offset); 219 printk(KERN_ERR "head = %d, tail = %d, buffers = %d\n", 220 p_head, p_tail, pipe->ring_size); 221 for (idx = 0; idx < pipe->ring_size; idx++) 222 printk(KERN_ERR "[%p %p %d %d]\n", 223 pipe->bufs[idx].ops, 224 pipe->bufs[idx].page, 225 pipe->bufs[idx].offset, 226 pipe->bufs[idx].len); 227 WARN_ON(1); 228 return false; 229 } 230 #else 231 #define sanity(i) true 232 #endif 233 234 static struct page *push_anon(struct pipe_inode_info *pipe, unsigned size) 235 { 236 struct page *page = alloc_page(GFP_USER); 237 if (page) { 238 struct pipe_buffer *buf = pipe_buf(pipe, pipe->head++); 239 *buf = (struct pipe_buffer) { 240 .ops = &default_pipe_buf_ops, 241 .page = page, 242 .offset = 0, 243 .len = size 244 }; 245 } 246 return page; 247 } 248 249 static void push_page(struct pipe_inode_info *pipe, struct page *page, 250 unsigned int offset, unsigned int size) 251 { 252 struct pipe_buffer *buf = pipe_buf(pipe, pipe->head++); 253 *buf = (struct pipe_buffer) { 254 .ops = &page_cache_pipe_buf_ops, 255 .page = page, 256 .offset = offset, 257 .len = size 258 }; 259 get_page(page); 260 } 261 262 static inline bool allocated(struct pipe_buffer *buf) 263 { 264 return buf->ops == &default_pipe_buf_ops; 265 } 266 267 static struct page *append_pipe(struct iov_iter *i, size_t size, 268 unsigned int *off) 269 { 270 struct pipe_inode_info *pipe = i->pipe; 271 size_t offset = i->iov_offset; 272 struct pipe_buffer *buf; 273 struct page *page; 274 275 if (offset && offset < PAGE_SIZE) { 276 // some space in the last buffer; can we add to it? 277 buf = pipe_buf(pipe, pipe->head - 1); 278 if (allocated(buf)) { 279 size = min_t(size_t, size, PAGE_SIZE - offset); 280 buf->len += size; 281 i->iov_offset += size; 282 i->count -= size; 283 *off = offset; 284 return buf->page; 285 } 286 } 287 // OK, we need a new buffer 288 *off = 0; 289 size = min_t(size_t, size, PAGE_SIZE); 290 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 291 return NULL; 292 page = push_anon(pipe, size); 293 if (!page) 294 return NULL; 295 i->head = pipe->head - 1; 296 i->iov_offset = size; 297 i->count -= size; 298 return page; 299 } 300 301 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes, 302 struct iov_iter *i) 303 { 304 struct pipe_inode_info *pipe = i->pipe; 305 unsigned int head = pipe->head; 306 307 if (unlikely(bytes > i->count)) 308 bytes = i->count; 309 310 if (unlikely(!bytes)) 311 return 0; 312 313 if (!sanity(i)) 314 return 0; 315 316 if (offset && i->iov_offset == offset) { // could we merge it? 317 struct pipe_buffer *buf = pipe_buf(pipe, head - 1); 318 if (buf->page == page) { 319 buf->len += bytes; 320 i->iov_offset += bytes; 321 i->count -= bytes; 322 return bytes; 323 } 324 } 325 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 326 return 0; 327 328 push_page(pipe, page, offset, bytes); 329 i->iov_offset = offset + bytes; 330 i->head = head; 331 i->count -= bytes; 332 return bytes; 333 } 334 335 /* 336 * fault_in_iov_iter_readable - fault in iov iterator for reading 337 * @i: iterator 338 * @size: maximum length 339 * 340 * Fault in one or more iovecs of the given iov_iter, to a maximum length of 341 * @size. For each iovec, fault in each page that constitutes the iovec. 342 * 343 * Returns the number of bytes not faulted in (like copy_to_user() and 344 * copy_from_user()). 345 * 346 * Always returns 0 for non-userspace iterators. 347 */ 348 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size) 349 { 350 if (iter_is_ubuf(i)) { 351 size_t n = min(size, iov_iter_count(i)); 352 n -= fault_in_readable(i->ubuf + i->iov_offset, n); 353 return size - n; 354 } else if (iter_is_iovec(i)) { 355 size_t count = min(size, iov_iter_count(i)); 356 const struct iovec *p; 357 size_t skip; 358 359 size -= count; 360 for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { 361 size_t len = min(count, p->iov_len - skip); 362 size_t ret; 363 364 if (unlikely(!len)) 365 continue; 366 ret = fault_in_readable(p->iov_base + skip, len); 367 count -= len - ret; 368 if (ret) 369 break; 370 } 371 return count + size; 372 } 373 return 0; 374 } 375 EXPORT_SYMBOL(fault_in_iov_iter_readable); 376 377 /* 378 * fault_in_iov_iter_writeable - fault in iov iterator for writing 379 * @i: iterator 380 * @size: maximum length 381 * 382 * Faults in the iterator using get_user_pages(), i.e., without triggering 383 * hardware page faults. This is primarily useful when we already know that 384 * some or all of the pages in @i aren't in memory. 385 * 386 * Returns the number of bytes not faulted in, like copy_to_user() and 387 * copy_from_user(). 388 * 389 * Always returns 0 for non-user-space iterators. 390 */ 391 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size) 392 { 393 if (iter_is_ubuf(i)) { 394 size_t n = min(size, iov_iter_count(i)); 395 n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n); 396 return size - n; 397 } else if (iter_is_iovec(i)) { 398 size_t count = min(size, iov_iter_count(i)); 399 const struct iovec *p; 400 size_t skip; 401 402 size -= count; 403 for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) { 404 size_t len = min(count, p->iov_len - skip); 405 size_t ret; 406 407 if (unlikely(!len)) 408 continue; 409 ret = fault_in_safe_writeable(p->iov_base + skip, len); 410 count -= len - ret; 411 if (ret) 412 break; 413 } 414 return count + size; 415 } 416 return 0; 417 } 418 EXPORT_SYMBOL(fault_in_iov_iter_writeable); 419 420 void iov_iter_init(struct iov_iter *i, unsigned int direction, 421 const struct iovec *iov, unsigned long nr_segs, 422 size_t count) 423 { 424 WARN_ON(direction & ~(READ | WRITE)); 425 *i = (struct iov_iter) { 426 .iter_type = ITER_IOVEC, 427 .nofault = false, 428 .user_backed = true, 429 .data_source = direction, 430 .iov = iov, 431 .nr_segs = nr_segs, 432 .iov_offset = 0, 433 .count = count 434 }; 435 } 436 EXPORT_SYMBOL(iov_iter_init); 437 438 static inline void data_start(const struct iov_iter *i, 439 unsigned int *iter_headp, size_t *offp) 440 { 441 unsigned int iter_head = i->head; 442 size_t off = i->iov_offset; 443 444 if (off && (!allocated(pipe_buf(i->pipe, iter_head)) || 445 off == PAGE_SIZE)) { 446 iter_head++; 447 off = 0; 448 } 449 *iter_headp = iter_head; 450 *offp = off; 451 } 452 453 static size_t push_pipe(struct iov_iter *i, size_t size, 454 int *iter_headp, size_t *offp) 455 { 456 struct pipe_inode_info *pipe = i->pipe; 457 unsigned int iter_head; 458 size_t off; 459 ssize_t left; 460 461 if (unlikely(size > i->count)) 462 size = i->count; 463 if (unlikely(!size)) 464 return 0; 465 466 left = size; 467 data_start(i, &iter_head, &off); 468 *iter_headp = iter_head; 469 *offp = off; 470 if (off) { 471 struct pipe_buffer *buf = pipe_buf(pipe, iter_head); 472 473 left -= PAGE_SIZE - off; 474 if (left <= 0) { 475 buf->len += size; 476 return size; 477 } 478 buf->len = PAGE_SIZE; 479 } 480 while (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { 481 struct page *page = push_anon(pipe, 482 min_t(ssize_t, left, PAGE_SIZE)); 483 if (!page) 484 break; 485 486 left -= PAGE_SIZE; 487 if (left <= 0) 488 return size; 489 } 490 return size - left; 491 } 492 493 static size_t copy_pipe_to_iter(const void *addr, size_t bytes, 494 struct iov_iter *i) 495 { 496 unsigned int off, chunk; 497 498 if (unlikely(bytes > i->count)) 499 bytes = i->count; 500 if (unlikely(!bytes)) 501 return 0; 502 503 if (!sanity(i)) 504 return 0; 505 506 for (size_t n = bytes; n; n -= chunk) { 507 struct page *page = append_pipe(i, n, &off); 508 chunk = min_t(size_t, n, PAGE_SIZE - off); 509 if (!page) 510 return bytes - n; 511 memcpy_to_page(page, off, addr, chunk); 512 addr += chunk; 513 } 514 return bytes; 515 } 516 517 static __wsum csum_and_memcpy(void *to, const void *from, size_t len, 518 __wsum sum, size_t off) 519 { 520 __wsum next = csum_partial_copy_nocheck(from, to, len); 521 return csum_block_add(sum, next, off); 522 } 523 524 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes, 525 struct iov_iter *i, __wsum *sump) 526 { 527 __wsum sum = *sump; 528 size_t off = 0; 529 unsigned int chunk, r; 530 531 if (unlikely(bytes > i->count)) 532 bytes = i->count; 533 if (unlikely(!bytes)) 534 return 0; 535 536 if (!sanity(i)) 537 return 0; 538 539 while (bytes) { 540 struct page *page = append_pipe(i, bytes, &r); 541 char *p; 542 543 if (!page) 544 break; 545 chunk = min_t(size_t, bytes, PAGE_SIZE - r); 546 p = kmap_local_page(page); 547 sum = csum_and_memcpy(p + r, addr + off, chunk, sum, off); 548 kunmap_local(p); 549 off += chunk; 550 bytes -= chunk; 551 } 552 *sump = sum; 553 return off; 554 } 555 556 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) 557 { 558 if (unlikely(iov_iter_is_pipe(i))) 559 return copy_pipe_to_iter(addr, bytes, i); 560 if (user_backed_iter(i)) 561 might_fault(); 562 iterate_and_advance(i, bytes, base, len, off, 563 copyout(base, addr + off, len), 564 memcpy(base, addr + off, len) 565 ) 566 567 return bytes; 568 } 569 EXPORT_SYMBOL(_copy_to_iter); 570 571 #ifdef CONFIG_ARCH_HAS_COPY_MC 572 static int copyout_mc(void __user *to, const void *from, size_t n) 573 { 574 if (access_ok(to, n)) { 575 instrument_copy_to_user(to, from, n); 576 n = copy_mc_to_user((__force void *) to, from, n); 577 } 578 return n; 579 } 580 581 static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes, 582 struct iov_iter *i) 583 { 584 size_t xfer = 0; 585 unsigned int off, chunk; 586 587 if (unlikely(bytes > i->count)) 588 bytes = i->count; 589 if (unlikely(!bytes)) 590 return 0; 591 592 if (!sanity(i)) 593 return 0; 594 595 while (bytes) { 596 struct page *page = append_pipe(i, bytes, &off); 597 unsigned long rem; 598 char *p; 599 600 if (!page) 601 break; 602 chunk = min_t(size_t, bytes, PAGE_SIZE - off); 603 p = kmap_local_page(page); 604 rem = copy_mc_to_kernel(p + off, addr + xfer, chunk); 605 chunk -= rem; 606 kunmap_local(p); 607 xfer += chunk; 608 bytes -= chunk; 609 if (rem) { 610 iov_iter_revert(i, rem); 611 break; 612 } 613 } 614 return xfer; 615 } 616 617 /** 618 * _copy_mc_to_iter - copy to iter with source memory error exception handling 619 * @addr: source kernel address 620 * @bytes: total transfer length 621 * @i: destination iterator 622 * 623 * The pmem driver deploys this for the dax operation 624 * (dax_copy_to_iter()) for dax reads (bypass page-cache and the 625 * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes 626 * successfully copied. 627 * 628 * The main differences between this and typical _copy_to_iter(). 629 * 630 * * Typical tail/residue handling after a fault retries the copy 631 * byte-by-byte until the fault happens again. Re-triggering machine 632 * checks is potentially fatal so the implementation uses source 633 * alignment and poison alignment assumptions to avoid re-triggering 634 * hardware exceptions. 635 * 636 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies. 637 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return 638 * a short copy. 639 * 640 * Return: number of bytes copied (may be %0) 641 */ 642 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i) 643 { 644 if (unlikely(iov_iter_is_pipe(i))) 645 return copy_mc_pipe_to_iter(addr, bytes, i); 646 if (user_backed_iter(i)) 647 might_fault(); 648 __iterate_and_advance(i, bytes, base, len, off, 649 copyout_mc(base, addr + off, len), 650 copy_mc_to_kernel(base, addr + off, len) 651 ) 652 653 return bytes; 654 } 655 EXPORT_SYMBOL_GPL(_copy_mc_to_iter); 656 #endif /* CONFIG_ARCH_HAS_COPY_MC */ 657 658 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 659 { 660 if (unlikely(iov_iter_is_pipe(i))) { 661 WARN_ON(1); 662 return 0; 663 } 664 if (user_backed_iter(i)) 665 might_fault(); 666 iterate_and_advance(i, bytes, base, len, off, 667 copyin(addr + off, base, len), 668 memcpy(addr + off, base, len) 669 ) 670 671 return bytes; 672 } 673 EXPORT_SYMBOL(_copy_from_iter); 674 675 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 676 { 677 if (unlikely(iov_iter_is_pipe(i))) { 678 WARN_ON(1); 679 return 0; 680 } 681 iterate_and_advance(i, bytes, base, len, off, 682 __copy_from_user_inatomic_nocache(addr + off, base, len), 683 memcpy(addr + off, base, len) 684 ) 685 686 return bytes; 687 } 688 EXPORT_SYMBOL(_copy_from_iter_nocache); 689 690 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE 691 /** 692 * _copy_from_iter_flushcache - write destination through cpu cache 693 * @addr: destination kernel address 694 * @bytes: total transfer length 695 * @i: source iterator 696 * 697 * The pmem driver arranges for filesystem-dax to use this facility via 698 * dax_copy_from_iter() for ensuring that writes to persistent memory 699 * are flushed through the CPU cache. It is differentiated from 700 * _copy_from_iter_nocache() in that guarantees all data is flushed for 701 * all iterator types. The _copy_from_iter_nocache() only attempts to 702 * bypass the cache for the ITER_IOVEC case, and on some archs may use 703 * instructions that strand dirty-data in the cache. 704 * 705 * Return: number of bytes copied (may be %0) 706 */ 707 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) 708 { 709 if (unlikely(iov_iter_is_pipe(i))) { 710 WARN_ON(1); 711 return 0; 712 } 713 iterate_and_advance(i, bytes, base, len, off, 714 __copy_from_user_flushcache(addr + off, base, len), 715 memcpy_flushcache(addr + off, base, len) 716 ) 717 718 return bytes; 719 } 720 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache); 721 #endif 722 723 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) 724 { 725 struct page *head; 726 size_t v = n + offset; 727 728 /* 729 * The general case needs to access the page order in order 730 * to compute the page size. 731 * However, we mostly deal with order-0 pages and thus can 732 * avoid a possible cache line miss for requests that fit all 733 * page orders. 734 */ 735 if (n <= v && v <= PAGE_SIZE) 736 return true; 737 738 head = compound_head(page); 739 v += (page - head) << PAGE_SHIFT; 740 741 if (likely(n <= v && v <= (page_size(head)))) 742 return true; 743 WARN_ON(1); 744 return false; 745 } 746 747 static size_t __copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 748 struct iov_iter *i) 749 { 750 if (unlikely(iov_iter_is_pipe(i))) { 751 return copy_page_to_iter_pipe(page, offset, bytes, i); 752 } else { 753 void *kaddr = kmap_local_page(page); 754 size_t wanted = _copy_to_iter(kaddr + offset, bytes, i); 755 kunmap_local(kaddr); 756 return wanted; 757 } 758 } 759 760 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 761 struct iov_iter *i) 762 { 763 size_t res = 0; 764 if (unlikely(!page_copy_sane(page, offset, bytes))) 765 return 0; 766 page += offset / PAGE_SIZE; // first subpage 767 offset %= PAGE_SIZE; 768 while (1) { 769 size_t n = __copy_page_to_iter(page, offset, 770 min(bytes, (size_t)PAGE_SIZE - offset), i); 771 res += n; 772 bytes -= n; 773 if (!bytes || !n) 774 break; 775 offset += n; 776 if (offset == PAGE_SIZE) { 777 page++; 778 offset = 0; 779 } 780 } 781 return res; 782 } 783 EXPORT_SYMBOL(copy_page_to_iter); 784 785 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 786 struct iov_iter *i) 787 { 788 if (page_copy_sane(page, offset, bytes)) { 789 void *kaddr = kmap_local_page(page); 790 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); 791 kunmap_local(kaddr); 792 return wanted; 793 } 794 return 0; 795 } 796 EXPORT_SYMBOL(copy_page_from_iter); 797 798 static size_t pipe_zero(size_t bytes, struct iov_iter *i) 799 { 800 unsigned int chunk, off; 801 802 if (unlikely(bytes > i->count)) 803 bytes = i->count; 804 if (unlikely(!bytes)) 805 return 0; 806 807 if (!sanity(i)) 808 return 0; 809 810 for (size_t n = bytes; n; n -= chunk) { 811 struct page *page = append_pipe(i, n, &off); 812 char *p; 813 814 if (!page) 815 return bytes - n; 816 chunk = min_t(size_t, n, PAGE_SIZE - off); 817 p = kmap_local_page(page); 818 memset(p + off, 0, chunk); 819 kunmap_local(p); 820 } 821 return bytes; 822 } 823 824 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) 825 { 826 if (unlikely(iov_iter_is_pipe(i))) 827 return pipe_zero(bytes, i); 828 iterate_and_advance(i, bytes, base, len, count, 829 clear_user(base, len), 830 memset(base, 0, len) 831 ) 832 833 return bytes; 834 } 835 EXPORT_SYMBOL(iov_iter_zero); 836 837 size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes, 838 struct iov_iter *i) 839 { 840 char *kaddr = kmap_atomic(page), *p = kaddr + offset; 841 if (unlikely(!page_copy_sane(page, offset, bytes))) { 842 kunmap_atomic(kaddr); 843 return 0; 844 } 845 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { 846 kunmap_atomic(kaddr); 847 WARN_ON(1); 848 return 0; 849 } 850 iterate_and_advance(i, bytes, base, len, off, 851 copyin(p + off, base, len), 852 memcpy(p + off, base, len) 853 ) 854 kunmap_atomic(kaddr); 855 return bytes; 856 } 857 EXPORT_SYMBOL(copy_page_from_iter_atomic); 858 859 static inline void pipe_truncate(struct iov_iter *i) 860 { 861 struct pipe_inode_info *pipe = i->pipe; 862 unsigned int p_tail = pipe->tail; 863 unsigned int p_head = pipe->head; 864 unsigned int p_mask = pipe->ring_size - 1; 865 866 if (!pipe_empty(p_head, p_tail)) { 867 struct pipe_buffer *buf; 868 unsigned int i_head = i->head; 869 size_t off = i->iov_offset; 870 871 if (off) { 872 buf = &pipe->bufs[i_head & p_mask]; 873 buf->len = off - buf->offset; 874 i_head++; 875 } 876 while (p_head != i_head) { 877 p_head--; 878 pipe_buf_release(pipe, &pipe->bufs[p_head & p_mask]); 879 } 880 881 pipe->head = p_head; 882 } 883 } 884 885 static void pipe_advance(struct iov_iter *i, size_t size) 886 { 887 struct pipe_inode_info *pipe = i->pipe; 888 if (size) { 889 struct pipe_buffer *buf; 890 unsigned int p_mask = pipe->ring_size - 1; 891 unsigned int i_head = i->head; 892 size_t off = i->iov_offset, left = size; 893 894 if (off) /* make it relative to the beginning of buffer */ 895 left += off - pipe->bufs[i_head & p_mask].offset; 896 while (1) { 897 buf = &pipe->bufs[i_head & p_mask]; 898 if (left <= buf->len) 899 break; 900 left -= buf->len; 901 i_head++; 902 } 903 i->head = i_head; 904 i->iov_offset = buf->offset + left; 905 } 906 i->count -= size; 907 /* ... and discard everything past that point */ 908 pipe_truncate(i); 909 } 910 911 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size) 912 { 913 const struct bio_vec *bvec, *end; 914 915 if (!i->count) 916 return; 917 i->count -= size; 918 919 size += i->iov_offset; 920 921 for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) { 922 if (likely(size < bvec->bv_len)) 923 break; 924 size -= bvec->bv_len; 925 } 926 i->iov_offset = size; 927 i->nr_segs -= bvec - i->bvec; 928 i->bvec = bvec; 929 } 930 931 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size) 932 { 933 const struct iovec *iov, *end; 934 935 if (!i->count) 936 return; 937 i->count -= size; 938 939 size += i->iov_offset; // from beginning of current segment 940 for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) { 941 if (likely(size < iov->iov_len)) 942 break; 943 size -= iov->iov_len; 944 } 945 i->iov_offset = size; 946 i->nr_segs -= iov - i->iov; 947 i->iov = iov; 948 } 949 950 void iov_iter_advance(struct iov_iter *i, size_t size) 951 { 952 if (unlikely(i->count < size)) 953 size = i->count; 954 if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) { 955 i->iov_offset += size; 956 i->count -= size; 957 } else if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) { 958 /* iovec and kvec have identical layouts */ 959 iov_iter_iovec_advance(i, size); 960 } else if (iov_iter_is_bvec(i)) { 961 iov_iter_bvec_advance(i, size); 962 } else if (iov_iter_is_pipe(i)) { 963 pipe_advance(i, size); 964 } else if (iov_iter_is_discard(i)) { 965 i->count -= size; 966 } 967 } 968 EXPORT_SYMBOL(iov_iter_advance); 969 970 void iov_iter_revert(struct iov_iter *i, size_t unroll) 971 { 972 if (!unroll) 973 return; 974 if (WARN_ON(unroll > MAX_RW_COUNT)) 975 return; 976 i->count += unroll; 977 if (unlikely(iov_iter_is_pipe(i))) { 978 struct pipe_inode_info *pipe = i->pipe; 979 unsigned int p_mask = pipe->ring_size - 1; 980 unsigned int i_head = i->head; 981 size_t off = i->iov_offset; 982 while (1) { 983 struct pipe_buffer *b = &pipe->bufs[i_head & p_mask]; 984 size_t n = off - b->offset; 985 if (unroll < n) { 986 off -= unroll; 987 break; 988 } 989 unroll -= n; 990 if (!unroll && i_head == i->start_head) { 991 off = 0; 992 break; 993 } 994 i_head--; 995 b = &pipe->bufs[i_head & p_mask]; 996 off = b->offset + b->len; 997 } 998 i->iov_offset = off; 999 i->head = i_head; 1000 pipe_truncate(i); 1001 return; 1002 } 1003 if (unlikely(iov_iter_is_discard(i))) 1004 return; 1005 if (unroll <= i->iov_offset) { 1006 i->iov_offset -= unroll; 1007 return; 1008 } 1009 unroll -= i->iov_offset; 1010 if (iov_iter_is_xarray(i) || iter_is_ubuf(i)) { 1011 BUG(); /* We should never go beyond the start of the specified 1012 * range since we might then be straying into pages that 1013 * aren't pinned. 1014 */ 1015 } else if (iov_iter_is_bvec(i)) { 1016 const struct bio_vec *bvec = i->bvec; 1017 while (1) { 1018 size_t n = (--bvec)->bv_len; 1019 i->nr_segs++; 1020 if (unroll <= n) { 1021 i->bvec = bvec; 1022 i->iov_offset = n - unroll; 1023 return; 1024 } 1025 unroll -= n; 1026 } 1027 } else { /* same logics for iovec and kvec */ 1028 const struct iovec *iov = i->iov; 1029 while (1) { 1030 size_t n = (--iov)->iov_len; 1031 i->nr_segs++; 1032 if (unroll <= n) { 1033 i->iov = iov; 1034 i->iov_offset = n - unroll; 1035 return; 1036 } 1037 unroll -= n; 1038 } 1039 } 1040 } 1041 EXPORT_SYMBOL(iov_iter_revert); 1042 1043 /* 1044 * Return the count of just the current iov_iter segment. 1045 */ 1046 size_t iov_iter_single_seg_count(const struct iov_iter *i) 1047 { 1048 if (i->nr_segs > 1) { 1049 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) 1050 return min(i->count, i->iov->iov_len - i->iov_offset); 1051 if (iov_iter_is_bvec(i)) 1052 return min(i->count, i->bvec->bv_len - i->iov_offset); 1053 } 1054 return i->count; 1055 } 1056 EXPORT_SYMBOL(iov_iter_single_seg_count); 1057 1058 void iov_iter_kvec(struct iov_iter *i, unsigned int direction, 1059 const struct kvec *kvec, unsigned long nr_segs, 1060 size_t count) 1061 { 1062 WARN_ON(direction & ~(READ | WRITE)); 1063 *i = (struct iov_iter){ 1064 .iter_type = ITER_KVEC, 1065 .data_source = direction, 1066 .kvec = kvec, 1067 .nr_segs = nr_segs, 1068 .iov_offset = 0, 1069 .count = count 1070 }; 1071 } 1072 EXPORT_SYMBOL(iov_iter_kvec); 1073 1074 void iov_iter_bvec(struct iov_iter *i, unsigned int direction, 1075 const struct bio_vec *bvec, unsigned long nr_segs, 1076 size_t count) 1077 { 1078 WARN_ON(direction & ~(READ | WRITE)); 1079 *i = (struct iov_iter){ 1080 .iter_type = ITER_BVEC, 1081 .data_source = direction, 1082 .bvec = bvec, 1083 .nr_segs = nr_segs, 1084 .iov_offset = 0, 1085 .count = count 1086 }; 1087 } 1088 EXPORT_SYMBOL(iov_iter_bvec); 1089 1090 void iov_iter_pipe(struct iov_iter *i, unsigned int direction, 1091 struct pipe_inode_info *pipe, 1092 size_t count) 1093 { 1094 BUG_ON(direction != READ); 1095 WARN_ON(pipe_full(pipe->head, pipe->tail, pipe->ring_size)); 1096 *i = (struct iov_iter){ 1097 .iter_type = ITER_PIPE, 1098 .data_source = false, 1099 .pipe = pipe, 1100 .head = pipe->head, 1101 .start_head = pipe->head, 1102 .iov_offset = 0, 1103 .count = count 1104 }; 1105 } 1106 EXPORT_SYMBOL(iov_iter_pipe); 1107 1108 /** 1109 * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray 1110 * @i: The iterator to initialise. 1111 * @direction: The direction of the transfer. 1112 * @xarray: The xarray to access. 1113 * @start: The start file position. 1114 * @count: The size of the I/O buffer in bytes. 1115 * 1116 * Set up an I/O iterator to either draw data out of the pages attached to an 1117 * inode or to inject data into those pages. The pages *must* be prevented 1118 * from evaporation, either by taking a ref on them or locking them by the 1119 * caller. 1120 */ 1121 void iov_iter_xarray(struct iov_iter *i, unsigned int direction, 1122 struct xarray *xarray, loff_t start, size_t count) 1123 { 1124 BUG_ON(direction & ~1); 1125 *i = (struct iov_iter) { 1126 .iter_type = ITER_XARRAY, 1127 .data_source = direction, 1128 .xarray = xarray, 1129 .xarray_start = start, 1130 .count = count, 1131 .iov_offset = 0 1132 }; 1133 } 1134 EXPORT_SYMBOL(iov_iter_xarray); 1135 1136 /** 1137 * iov_iter_discard - Initialise an I/O iterator that discards data 1138 * @i: The iterator to initialise. 1139 * @direction: The direction of the transfer. 1140 * @count: The size of the I/O buffer in bytes. 1141 * 1142 * Set up an I/O iterator that just discards everything that's written to it. 1143 * It's only available as a READ iterator. 1144 */ 1145 void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count) 1146 { 1147 BUG_ON(direction != READ); 1148 *i = (struct iov_iter){ 1149 .iter_type = ITER_DISCARD, 1150 .data_source = false, 1151 .count = count, 1152 .iov_offset = 0 1153 }; 1154 } 1155 EXPORT_SYMBOL(iov_iter_discard); 1156 1157 static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask, 1158 unsigned len_mask) 1159 { 1160 size_t size = i->count; 1161 size_t skip = i->iov_offset; 1162 unsigned k; 1163 1164 for (k = 0; k < i->nr_segs; k++, skip = 0) { 1165 size_t len = i->iov[k].iov_len - skip; 1166 1167 if (len > size) 1168 len = size; 1169 if (len & len_mask) 1170 return false; 1171 if ((unsigned long)(i->iov[k].iov_base + skip) & addr_mask) 1172 return false; 1173 1174 size -= len; 1175 if (!size) 1176 break; 1177 } 1178 return true; 1179 } 1180 1181 static bool iov_iter_aligned_bvec(const struct iov_iter *i, unsigned addr_mask, 1182 unsigned len_mask) 1183 { 1184 size_t size = i->count; 1185 unsigned skip = i->iov_offset; 1186 unsigned k; 1187 1188 for (k = 0; k < i->nr_segs; k++, skip = 0) { 1189 size_t len = i->bvec[k].bv_len - skip; 1190 1191 if (len > size) 1192 len = size; 1193 if (len & len_mask) 1194 return false; 1195 if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask) 1196 return false; 1197 1198 size -= len; 1199 if (!size) 1200 break; 1201 } 1202 return true; 1203 } 1204 1205 /** 1206 * iov_iter_is_aligned() - Check if the addresses and lengths of each segments 1207 * are aligned to the parameters. 1208 * 1209 * @i: &struct iov_iter to restore 1210 * @addr_mask: bit mask to check against the iov element's addresses 1211 * @len_mask: bit mask to check against the iov element's lengths 1212 * 1213 * Return: false if any addresses or lengths intersect with the provided masks 1214 */ 1215 bool iov_iter_is_aligned(const struct iov_iter *i, unsigned addr_mask, 1216 unsigned len_mask) 1217 { 1218 if (likely(iter_is_ubuf(i))) { 1219 if (i->count & len_mask) 1220 return false; 1221 if ((unsigned long)(i->ubuf + i->iov_offset) & addr_mask) 1222 return false; 1223 return true; 1224 } 1225 1226 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) 1227 return iov_iter_aligned_iovec(i, addr_mask, len_mask); 1228 1229 if (iov_iter_is_bvec(i)) 1230 return iov_iter_aligned_bvec(i, addr_mask, len_mask); 1231 1232 if (iov_iter_is_pipe(i)) { 1233 unsigned int p_mask = i->pipe->ring_size - 1; 1234 size_t size = i->count; 1235 1236 if (size & len_mask) 1237 return false; 1238 if (size && allocated(&i->pipe->bufs[i->head & p_mask])) { 1239 if (i->iov_offset & addr_mask) 1240 return false; 1241 } 1242 1243 return true; 1244 } 1245 1246 if (iov_iter_is_xarray(i)) { 1247 if (i->count & len_mask) 1248 return false; 1249 if ((i->xarray_start + i->iov_offset) & addr_mask) 1250 return false; 1251 } 1252 1253 return true; 1254 } 1255 EXPORT_SYMBOL_GPL(iov_iter_is_aligned); 1256 1257 static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i) 1258 { 1259 unsigned long res = 0; 1260 size_t size = i->count; 1261 size_t skip = i->iov_offset; 1262 unsigned k; 1263 1264 for (k = 0; k < i->nr_segs; k++, skip = 0) { 1265 size_t len = i->iov[k].iov_len - skip; 1266 if (len) { 1267 res |= (unsigned long)i->iov[k].iov_base + skip; 1268 if (len > size) 1269 len = size; 1270 res |= len; 1271 size -= len; 1272 if (!size) 1273 break; 1274 } 1275 } 1276 return res; 1277 } 1278 1279 static unsigned long iov_iter_alignment_bvec(const struct iov_iter *i) 1280 { 1281 unsigned res = 0; 1282 size_t size = i->count; 1283 unsigned skip = i->iov_offset; 1284 unsigned k; 1285 1286 for (k = 0; k < i->nr_segs; k++, skip = 0) { 1287 size_t len = i->bvec[k].bv_len - skip; 1288 res |= (unsigned long)i->bvec[k].bv_offset + skip; 1289 if (len > size) 1290 len = size; 1291 res |= len; 1292 size -= len; 1293 if (!size) 1294 break; 1295 } 1296 return res; 1297 } 1298 1299 unsigned long iov_iter_alignment(const struct iov_iter *i) 1300 { 1301 if (likely(iter_is_ubuf(i))) { 1302 size_t size = i->count; 1303 if (size) 1304 return ((unsigned long)i->ubuf + i->iov_offset) | size; 1305 return 0; 1306 } 1307 1308 /* iovec and kvec have identical layouts */ 1309 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) 1310 return iov_iter_alignment_iovec(i); 1311 1312 if (iov_iter_is_bvec(i)) 1313 return iov_iter_alignment_bvec(i); 1314 1315 if (iov_iter_is_pipe(i)) { 1316 size_t size = i->count; 1317 1318 if (size && i->iov_offset && allocated(pipe_buf(i->pipe, i->head))) 1319 return size | i->iov_offset; 1320 return size; 1321 } 1322 1323 if (iov_iter_is_xarray(i)) 1324 return (i->xarray_start + i->iov_offset) | i->count; 1325 1326 return 0; 1327 } 1328 EXPORT_SYMBOL(iov_iter_alignment); 1329 1330 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) 1331 { 1332 unsigned long res = 0; 1333 unsigned long v = 0; 1334 size_t size = i->count; 1335 unsigned k; 1336 1337 if (iter_is_ubuf(i)) 1338 return 0; 1339 1340 if (WARN_ON(!iter_is_iovec(i))) 1341 return ~0U; 1342 1343 for (k = 0; k < i->nr_segs; k++) { 1344 if (i->iov[k].iov_len) { 1345 unsigned long base = (unsigned long)i->iov[k].iov_base; 1346 if (v) // if not the first one 1347 res |= base | v; // this start | previous end 1348 v = base + i->iov[k].iov_len; 1349 if (size <= i->iov[k].iov_len) 1350 break; 1351 size -= i->iov[k].iov_len; 1352 } 1353 } 1354 return res; 1355 } 1356 EXPORT_SYMBOL(iov_iter_gap_alignment); 1357 1358 static inline ssize_t __pipe_get_pages(struct iov_iter *i, 1359 size_t maxsize, 1360 struct page **pages, 1361 int iter_head, 1362 size_t *start) 1363 { 1364 struct pipe_inode_info *pipe = i->pipe; 1365 unsigned int p_mask = pipe->ring_size - 1; 1366 ssize_t n = push_pipe(i, maxsize, &iter_head, start); 1367 if (!n) 1368 return -EFAULT; 1369 1370 maxsize = n; 1371 n += *start; 1372 while (n > 0) { 1373 get_page(*pages++ = pipe->bufs[iter_head & p_mask].page); 1374 iter_head++; 1375 n -= PAGE_SIZE; 1376 } 1377 1378 return maxsize; 1379 } 1380 1381 static ssize_t pipe_get_pages(struct iov_iter *i, 1382 struct page **pages, size_t maxsize, unsigned maxpages, 1383 size_t *start) 1384 { 1385 unsigned int iter_head, npages; 1386 size_t capacity; 1387 1388 if (!sanity(i)) 1389 return -EFAULT; 1390 1391 data_start(i, &iter_head, start); 1392 /* Amount of free space: some of this one + all after this one */ 1393 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); 1394 capacity = min(npages, maxpages) * PAGE_SIZE - *start; 1395 1396 return __pipe_get_pages(i, min(maxsize, capacity), pages, iter_head, start); 1397 } 1398 1399 static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa, 1400 pgoff_t index, unsigned int nr_pages) 1401 { 1402 XA_STATE(xas, xa, index); 1403 struct page *page; 1404 unsigned int ret = 0; 1405 1406 rcu_read_lock(); 1407 for (page = xas_load(&xas); page; page = xas_next(&xas)) { 1408 if (xas_retry(&xas, page)) 1409 continue; 1410 1411 /* Has the page moved or been split? */ 1412 if (unlikely(page != xas_reload(&xas))) { 1413 xas_reset(&xas); 1414 continue; 1415 } 1416 1417 pages[ret] = find_subpage(page, xas.xa_index); 1418 get_page(pages[ret]); 1419 if (++ret == nr_pages) 1420 break; 1421 } 1422 rcu_read_unlock(); 1423 return ret; 1424 } 1425 1426 static ssize_t iter_xarray_get_pages(struct iov_iter *i, 1427 struct page **pages, size_t maxsize, 1428 unsigned maxpages, size_t *_start_offset) 1429 { 1430 unsigned nr, offset; 1431 pgoff_t index, count; 1432 size_t size = maxsize; 1433 loff_t pos; 1434 1435 if (!size || !maxpages) 1436 return 0; 1437 1438 pos = i->xarray_start + i->iov_offset; 1439 index = pos >> PAGE_SHIFT; 1440 offset = pos & ~PAGE_MASK; 1441 *_start_offset = offset; 1442 1443 count = 1; 1444 if (size > PAGE_SIZE - offset) { 1445 size -= PAGE_SIZE - offset; 1446 count += size >> PAGE_SHIFT; 1447 size &= ~PAGE_MASK; 1448 if (size) 1449 count++; 1450 } 1451 1452 if (count > maxpages) 1453 count = maxpages; 1454 1455 nr = iter_xarray_populate_pages(pages, i->xarray, index, count); 1456 if (nr == 0) 1457 return 0; 1458 1459 return min_t(size_t, nr * PAGE_SIZE - offset, maxsize); 1460 } 1461 1462 /* must be done on non-empty ITER_UBUF or ITER_IOVEC one */ 1463 static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size) 1464 { 1465 size_t skip; 1466 long k; 1467 1468 if (iter_is_ubuf(i)) 1469 return (unsigned long)i->ubuf + i->iov_offset; 1470 1471 for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) { 1472 size_t len = i->iov[k].iov_len - skip; 1473 1474 if (unlikely(!len)) 1475 continue; 1476 if (*size > len) 1477 *size = len; 1478 return (unsigned long)i->iov[k].iov_base + skip; 1479 } 1480 BUG(); // if it had been empty, we wouldn't get called 1481 } 1482 1483 /* must be done on non-empty ITER_BVEC one */ 1484 static struct page *first_bvec_segment(const struct iov_iter *i, 1485 size_t *size, size_t *start) 1486 { 1487 struct page *page; 1488 size_t skip = i->iov_offset, len; 1489 1490 len = i->bvec->bv_len - skip; 1491 if (*size > len) 1492 *size = len; 1493 skip += i->bvec->bv_offset; 1494 page = i->bvec->bv_page + skip / PAGE_SIZE; 1495 *start = skip % PAGE_SIZE; 1496 return page; 1497 } 1498 1499 ssize_t iov_iter_get_pages(struct iov_iter *i, 1500 struct page **pages, size_t maxsize, unsigned maxpages, 1501 size_t *start) 1502 { 1503 int n, res; 1504 1505 if (maxsize > i->count) 1506 maxsize = i->count; 1507 if (!maxsize) 1508 return 0; 1509 if (maxsize > MAX_RW_COUNT) 1510 maxsize = MAX_RW_COUNT; 1511 1512 if (likely(user_backed_iter(i))) { 1513 unsigned int gup_flags = 0; 1514 unsigned long addr; 1515 1516 if (iov_iter_rw(i) != WRITE) 1517 gup_flags |= FOLL_WRITE; 1518 if (i->nofault) 1519 gup_flags |= FOLL_NOFAULT; 1520 1521 addr = first_iovec_segment(i, &maxsize); 1522 *start = addr % PAGE_SIZE; 1523 addr &= PAGE_MASK; 1524 n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); 1525 if (n > maxpages) 1526 n = maxpages; 1527 res = get_user_pages_fast(addr, n, gup_flags, pages); 1528 if (unlikely(res <= 0)) 1529 return res; 1530 return min_t(size_t, maxsize, res * PAGE_SIZE - *start); 1531 } 1532 if (iov_iter_is_bvec(i)) { 1533 struct page *page; 1534 1535 page = first_bvec_segment(i, &maxsize, start); 1536 n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); 1537 if (n > maxpages) 1538 n = maxpages; 1539 for (int k = 0; k < n; k++) 1540 get_page(*pages++ = page++); 1541 return min_t(size_t, maxsize, n * PAGE_SIZE - *start); 1542 } 1543 if (iov_iter_is_pipe(i)) 1544 return pipe_get_pages(i, pages, maxsize, maxpages, start); 1545 if (iov_iter_is_xarray(i)) 1546 return iter_xarray_get_pages(i, pages, maxsize, maxpages, start); 1547 return -EFAULT; 1548 } 1549 EXPORT_SYMBOL(iov_iter_get_pages); 1550 1551 static struct page **get_pages_array(size_t n) 1552 { 1553 return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL); 1554 } 1555 1556 static ssize_t pipe_get_pages_alloc(struct iov_iter *i, 1557 struct page ***pages, size_t maxsize, 1558 size_t *start) 1559 { 1560 struct page **p; 1561 unsigned int iter_head, npages; 1562 ssize_t n; 1563 1564 if (!sanity(i)) 1565 return -EFAULT; 1566 1567 data_start(i, &iter_head, start); 1568 /* Amount of free space: some of this one + all after this one */ 1569 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); 1570 n = npages * PAGE_SIZE - *start; 1571 if (maxsize > n) 1572 maxsize = n; 1573 else 1574 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); 1575 p = get_pages_array(npages); 1576 if (!p) 1577 return -ENOMEM; 1578 n = __pipe_get_pages(i, maxsize, p, iter_head, start); 1579 if (n > 0) 1580 *pages = p; 1581 else 1582 kvfree(p); 1583 return n; 1584 } 1585 1586 static ssize_t iter_xarray_get_pages_alloc(struct iov_iter *i, 1587 struct page ***pages, size_t maxsize, 1588 size_t *_start_offset) 1589 { 1590 struct page **p; 1591 unsigned nr, offset; 1592 pgoff_t index, count; 1593 size_t size = maxsize; 1594 loff_t pos; 1595 1596 if (!size) 1597 return 0; 1598 1599 pos = i->xarray_start + i->iov_offset; 1600 index = pos >> PAGE_SHIFT; 1601 offset = pos & ~PAGE_MASK; 1602 *_start_offset = offset; 1603 1604 count = 1; 1605 if (size > PAGE_SIZE - offset) { 1606 size -= PAGE_SIZE - offset; 1607 count += size >> PAGE_SHIFT; 1608 size &= ~PAGE_MASK; 1609 if (size) 1610 count++; 1611 } 1612 1613 p = get_pages_array(count); 1614 if (!p) 1615 return -ENOMEM; 1616 *pages = p; 1617 1618 nr = iter_xarray_populate_pages(p, i->xarray, index, count); 1619 if (nr == 0) 1620 return 0; 1621 1622 return min_t(size_t, nr * PAGE_SIZE - offset, maxsize); 1623 } 1624 1625 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, 1626 struct page ***pages, size_t maxsize, 1627 size_t *start) 1628 { 1629 struct page **p; 1630 int n, res; 1631 1632 if (maxsize > i->count) 1633 maxsize = i->count; 1634 if (!maxsize) 1635 return 0; 1636 if (maxsize > MAX_RW_COUNT) 1637 maxsize = MAX_RW_COUNT; 1638 1639 if (likely(user_backed_iter(i))) { 1640 unsigned int gup_flags = 0; 1641 unsigned long addr; 1642 1643 if (iov_iter_rw(i) != WRITE) 1644 gup_flags |= FOLL_WRITE; 1645 if (i->nofault) 1646 gup_flags |= FOLL_NOFAULT; 1647 1648 addr = first_iovec_segment(i, &maxsize); 1649 *start = addr % PAGE_SIZE; 1650 addr &= PAGE_MASK; 1651 n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); 1652 p = get_pages_array(n); 1653 if (!p) 1654 return -ENOMEM; 1655 res = get_user_pages_fast(addr, n, gup_flags, p); 1656 if (unlikely(res <= 0)) { 1657 kvfree(p); 1658 *pages = NULL; 1659 return res; 1660 } 1661 *pages = p; 1662 return min_t(size_t, maxsize, res * PAGE_SIZE - *start); 1663 } 1664 if (iov_iter_is_bvec(i)) { 1665 struct page *page; 1666 1667 page = first_bvec_segment(i, &maxsize, start); 1668 n = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); 1669 *pages = p = get_pages_array(n); 1670 if (!p) 1671 return -ENOMEM; 1672 for (int k = 0; k < n; k++) 1673 get_page(*p++ = page++); 1674 return min_t(size_t, maxsize, n * PAGE_SIZE - *start); 1675 } 1676 if (iov_iter_is_pipe(i)) 1677 return pipe_get_pages_alloc(i, pages, maxsize, start); 1678 if (iov_iter_is_xarray(i)) 1679 return iter_xarray_get_pages_alloc(i, pages, maxsize, start); 1680 return -EFAULT; 1681 } 1682 EXPORT_SYMBOL(iov_iter_get_pages_alloc); 1683 1684 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, 1685 struct iov_iter *i) 1686 { 1687 __wsum sum, next; 1688 sum = *csum; 1689 if (unlikely(iov_iter_is_pipe(i) || iov_iter_is_discard(i))) { 1690 WARN_ON(1); 1691 return 0; 1692 } 1693 iterate_and_advance(i, bytes, base, len, off, ({ 1694 next = csum_and_copy_from_user(base, addr + off, len); 1695 sum = csum_block_add(sum, next, off); 1696 next ? 0 : len; 1697 }), ({ 1698 sum = csum_and_memcpy(addr + off, base, len, sum, off); 1699 }) 1700 ) 1701 *csum = sum; 1702 return bytes; 1703 } 1704 EXPORT_SYMBOL(csum_and_copy_from_iter); 1705 1706 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *_csstate, 1707 struct iov_iter *i) 1708 { 1709 struct csum_state *csstate = _csstate; 1710 __wsum sum, next; 1711 1712 if (unlikely(iov_iter_is_discard(i))) { 1713 WARN_ON(1); /* for now */ 1714 return 0; 1715 } 1716 1717 sum = csum_shift(csstate->csum, csstate->off); 1718 if (unlikely(iov_iter_is_pipe(i))) 1719 bytes = csum_and_copy_to_pipe_iter(addr, bytes, i, &sum); 1720 else iterate_and_advance(i, bytes, base, len, off, ({ 1721 next = csum_and_copy_to_user(addr + off, base, len); 1722 sum = csum_block_add(sum, next, off); 1723 next ? 0 : len; 1724 }), ({ 1725 sum = csum_and_memcpy(base, addr + off, len, sum, off); 1726 }) 1727 ) 1728 csstate->csum = csum_shift(sum, csstate->off); 1729 csstate->off += bytes; 1730 return bytes; 1731 } 1732 EXPORT_SYMBOL(csum_and_copy_to_iter); 1733 1734 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp, 1735 struct iov_iter *i) 1736 { 1737 #ifdef CONFIG_CRYPTO_HASH 1738 struct ahash_request *hash = hashp; 1739 struct scatterlist sg; 1740 size_t copied; 1741 1742 copied = copy_to_iter(addr, bytes, i); 1743 sg_init_one(&sg, addr, copied); 1744 ahash_request_set_crypt(hash, &sg, NULL, copied); 1745 crypto_ahash_update(hash); 1746 return copied; 1747 #else 1748 return 0; 1749 #endif 1750 } 1751 EXPORT_SYMBOL(hash_and_copy_to_iter); 1752 1753 static int iov_npages(const struct iov_iter *i, int maxpages) 1754 { 1755 size_t skip = i->iov_offset, size = i->count; 1756 const struct iovec *p; 1757 int npages = 0; 1758 1759 for (p = i->iov; size; skip = 0, p++) { 1760 unsigned offs = offset_in_page(p->iov_base + skip); 1761 size_t len = min(p->iov_len - skip, size); 1762 1763 if (len) { 1764 size -= len; 1765 npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); 1766 if (unlikely(npages > maxpages)) 1767 return maxpages; 1768 } 1769 } 1770 return npages; 1771 } 1772 1773 static int bvec_npages(const struct iov_iter *i, int maxpages) 1774 { 1775 size_t skip = i->iov_offset, size = i->count; 1776 const struct bio_vec *p; 1777 int npages = 0; 1778 1779 for (p = i->bvec; size; skip = 0, p++) { 1780 unsigned offs = (p->bv_offset + skip) % PAGE_SIZE; 1781 size_t len = min(p->bv_len - skip, size); 1782 1783 size -= len; 1784 npages += DIV_ROUND_UP(offs + len, PAGE_SIZE); 1785 if (unlikely(npages > maxpages)) 1786 return maxpages; 1787 } 1788 return npages; 1789 } 1790 1791 int iov_iter_npages(const struct iov_iter *i, int maxpages) 1792 { 1793 if (unlikely(!i->count)) 1794 return 0; 1795 if (likely(iter_is_ubuf(i))) { 1796 unsigned offs = offset_in_page(i->ubuf + i->iov_offset); 1797 int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE); 1798 return min(npages, maxpages); 1799 } 1800 /* iovec and kvec have identical layouts */ 1801 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i))) 1802 return iov_npages(i, maxpages); 1803 if (iov_iter_is_bvec(i)) 1804 return bvec_npages(i, maxpages); 1805 if (iov_iter_is_pipe(i)) { 1806 unsigned int iter_head; 1807 int npages; 1808 size_t off; 1809 1810 if (!sanity(i)) 1811 return 0; 1812 1813 data_start(i, &iter_head, &off); 1814 /* some of this one + all after this one */ 1815 npages = pipe_space_for_user(iter_head, i->pipe->tail, i->pipe); 1816 return min(npages, maxpages); 1817 } 1818 if (iov_iter_is_xarray(i)) { 1819 unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE; 1820 int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE); 1821 return min(npages, maxpages); 1822 } 1823 return 0; 1824 } 1825 EXPORT_SYMBOL(iov_iter_npages); 1826 1827 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) 1828 { 1829 *new = *old; 1830 if (unlikely(iov_iter_is_pipe(new))) { 1831 WARN_ON(1); 1832 return NULL; 1833 } 1834 if (iov_iter_is_bvec(new)) 1835 return new->bvec = kmemdup(new->bvec, 1836 new->nr_segs * sizeof(struct bio_vec), 1837 flags); 1838 else if (iov_iter_is_kvec(new) || iter_is_iovec(new)) 1839 /* iovec and kvec have identical layout */ 1840 return new->iov = kmemdup(new->iov, 1841 new->nr_segs * sizeof(struct iovec), 1842 flags); 1843 return NULL; 1844 } 1845 EXPORT_SYMBOL(dup_iter); 1846 1847 static int copy_compat_iovec_from_user(struct iovec *iov, 1848 const struct iovec __user *uvec, unsigned long nr_segs) 1849 { 1850 const struct compat_iovec __user *uiov = 1851 (const struct compat_iovec __user *)uvec; 1852 int ret = -EFAULT, i; 1853 1854 if (!user_access_begin(uiov, nr_segs * sizeof(*uiov))) 1855 return -EFAULT; 1856 1857 for (i = 0; i < nr_segs; i++) { 1858 compat_uptr_t buf; 1859 compat_ssize_t len; 1860 1861 unsafe_get_user(len, &uiov[i].iov_len, uaccess_end); 1862 unsafe_get_user(buf, &uiov[i].iov_base, uaccess_end); 1863 1864 /* check for compat_size_t not fitting in compat_ssize_t .. */ 1865 if (len < 0) { 1866 ret = -EINVAL; 1867 goto uaccess_end; 1868 } 1869 iov[i].iov_base = compat_ptr(buf); 1870 iov[i].iov_len = len; 1871 } 1872 1873 ret = 0; 1874 uaccess_end: 1875 user_access_end(); 1876 return ret; 1877 } 1878 1879 static int copy_iovec_from_user(struct iovec *iov, 1880 const struct iovec __user *uvec, unsigned long nr_segs) 1881 { 1882 unsigned long seg; 1883 1884 if (copy_from_user(iov, uvec, nr_segs * sizeof(*uvec))) 1885 return -EFAULT; 1886 for (seg = 0; seg < nr_segs; seg++) { 1887 if ((ssize_t)iov[seg].iov_len < 0) 1888 return -EINVAL; 1889 } 1890 1891 return 0; 1892 } 1893 1894 struct iovec *iovec_from_user(const struct iovec __user *uvec, 1895 unsigned long nr_segs, unsigned long fast_segs, 1896 struct iovec *fast_iov, bool compat) 1897 { 1898 struct iovec *iov = fast_iov; 1899 int ret; 1900 1901 /* 1902 * SuS says "The readv() function *may* fail if the iovcnt argument was 1903 * less than or equal to 0, or greater than {IOV_MAX}. Linux has 1904 * traditionally returned zero for zero segments, so... 1905 */ 1906 if (nr_segs == 0) 1907 return iov; 1908 if (nr_segs > UIO_MAXIOV) 1909 return ERR_PTR(-EINVAL); 1910 if (nr_segs > fast_segs) { 1911 iov = kmalloc_array(nr_segs, sizeof(struct iovec), GFP_KERNEL); 1912 if (!iov) 1913 return ERR_PTR(-ENOMEM); 1914 } 1915 1916 if (compat) 1917 ret = copy_compat_iovec_from_user(iov, uvec, nr_segs); 1918 else 1919 ret = copy_iovec_from_user(iov, uvec, nr_segs); 1920 if (ret) { 1921 if (iov != fast_iov) 1922 kfree(iov); 1923 return ERR_PTR(ret); 1924 } 1925 1926 return iov; 1927 } 1928 1929 ssize_t __import_iovec(int type, const struct iovec __user *uvec, 1930 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, 1931 struct iov_iter *i, bool compat) 1932 { 1933 ssize_t total_len = 0; 1934 unsigned long seg; 1935 struct iovec *iov; 1936 1937 iov = iovec_from_user(uvec, nr_segs, fast_segs, *iovp, compat); 1938 if (IS_ERR(iov)) { 1939 *iovp = NULL; 1940 return PTR_ERR(iov); 1941 } 1942 1943 /* 1944 * According to the Single Unix Specification we should return EINVAL if 1945 * an element length is < 0 when cast to ssize_t or if the total length 1946 * would overflow the ssize_t return value of the system call. 1947 * 1948 * Linux caps all read/write calls to MAX_RW_COUNT, and avoids the 1949 * overflow case. 1950 */ 1951 for (seg = 0; seg < nr_segs; seg++) { 1952 ssize_t len = (ssize_t)iov[seg].iov_len; 1953 1954 if (!access_ok(iov[seg].iov_base, len)) { 1955 if (iov != *iovp) 1956 kfree(iov); 1957 *iovp = NULL; 1958 return -EFAULT; 1959 } 1960 1961 if (len > MAX_RW_COUNT - total_len) { 1962 len = MAX_RW_COUNT - total_len; 1963 iov[seg].iov_len = len; 1964 } 1965 total_len += len; 1966 } 1967 1968 iov_iter_init(i, type, iov, nr_segs, total_len); 1969 if (iov == *iovp) 1970 *iovp = NULL; 1971 else 1972 *iovp = iov; 1973 return total_len; 1974 } 1975 1976 /** 1977 * import_iovec() - Copy an array of &struct iovec from userspace 1978 * into the kernel, check that it is valid, and initialize a new 1979 * &struct iov_iter iterator to access it. 1980 * 1981 * @type: One of %READ or %WRITE. 1982 * @uvec: Pointer to the userspace array. 1983 * @nr_segs: Number of elements in userspace array. 1984 * @fast_segs: Number of elements in @iov. 1985 * @iovp: (input and output parameter) Pointer to pointer to (usually small 1986 * on-stack) kernel array. 1987 * @i: Pointer to iterator that will be initialized on success. 1988 * 1989 * If the array pointed to by *@iov is large enough to hold all @nr_segs, 1990 * then this function places %NULL in *@iov on return. Otherwise, a new 1991 * array will be allocated and the result placed in *@iov. This means that 1992 * the caller may call kfree() on *@iov regardless of whether the small 1993 * on-stack array was used or not (and regardless of whether this function 1994 * returns an error or not). 1995 * 1996 * Return: Negative error code on error, bytes imported on success 1997 */ 1998 ssize_t import_iovec(int type, const struct iovec __user *uvec, 1999 unsigned nr_segs, unsigned fast_segs, 2000 struct iovec **iovp, struct iov_iter *i) 2001 { 2002 return __import_iovec(type, uvec, nr_segs, fast_segs, iovp, i, 2003 in_compat_syscall()); 2004 } 2005 EXPORT_SYMBOL(import_iovec); 2006 2007 int import_single_range(int rw, void __user *buf, size_t len, 2008 struct iovec *iov, struct iov_iter *i) 2009 { 2010 if (len > MAX_RW_COUNT) 2011 len = MAX_RW_COUNT; 2012 if (unlikely(!access_ok(buf, len))) 2013 return -EFAULT; 2014 2015 iov->iov_base = buf; 2016 iov->iov_len = len; 2017 iov_iter_init(i, rw, iov, 1, len); 2018 return 0; 2019 } 2020 EXPORT_SYMBOL(import_single_range); 2021 2022 /** 2023 * iov_iter_restore() - Restore a &struct iov_iter to the same state as when 2024 * iov_iter_save_state() was called. 2025 * 2026 * @i: &struct iov_iter to restore 2027 * @state: state to restore from 2028 * 2029 * Used after iov_iter_save_state() to bring restore @i, if operations may 2030 * have advanced it. 2031 * 2032 * Note: only works on ITER_IOVEC, ITER_BVEC, and ITER_KVEC 2033 */ 2034 void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state) 2035 { 2036 if (WARN_ON_ONCE(!iov_iter_is_bvec(i) && !iter_is_iovec(i)) && 2037 !iov_iter_is_kvec(i) && !iter_is_ubuf(i)) 2038 return; 2039 i->iov_offset = state->iov_offset; 2040 i->count = state->count; 2041 if (iter_is_ubuf(i)) 2042 return; 2043 /* 2044 * For the *vec iters, nr_segs + iov is constant - if we increment 2045 * the vec, then we also decrement the nr_segs count. Hence we don't 2046 * need to track both of these, just one is enough and we can deduct 2047 * the other from that. ITER_KVEC and ITER_IOVEC are the same struct 2048 * size, so we can just increment the iov pointer as they are unionzed. 2049 * ITER_BVEC _may_ be the same size on some archs, but on others it is 2050 * not. Be safe and handle it separately. 2051 */ 2052 BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec)); 2053 if (iov_iter_is_bvec(i)) 2054 i->bvec -= state->nr_segs - i->nr_segs; 2055 else 2056 i->iov -= state->nr_segs - i->nr_segs; 2057 i->nr_segs = state->nr_segs; 2058 } 2059