1 #include <linux/export.h> 2 #include <linux/bvec.h> 3 #include <linux/uio.h> 4 #include <linux/pagemap.h> 5 #include <linux/slab.h> 6 #include <linux/vmalloc.h> 7 #include <linux/splice.h> 8 #include <net/checksum.h> 9 10 #define PIPE_PARANOIA /* for now */ 11 12 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ 13 size_t left; \ 14 size_t wanted = n; \ 15 __p = i->iov; \ 16 __v.iov_len = min(n, __p->iov_len - skip); \ 17 if (likely(__v.iov_len)) { \ 18 __v.iov_base = __p->iov_base + skip; \ 19 left = (STEP); \ 20 __v.iov_len -= left; \ 21 skip += __v.iov_len; \ 22 n -= __v.iov_len; \ 23 } else { \ 24 left = 0; \ 25 } \ 26 while (unlikely(!left && n)) { \ 27 __p++; \ 28 __v.iov_len = min(n, __p->iov_len); \ 29 if (unlikely(!__v.iov_len)) \ 30 continue; \ 31 __v.iov_base = __p->iov_base; \ 32 left = (STEP); \ 33 __v.iov_len -= left; \ 34 skip = __v.iov_len; \ 35 n -= __v.iov_len; \ 36 } \ 37 n = wanted - n; \ 38 } 39 40 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ 41 size_t wanted = n; \ 42 __p = i->kvec; \ 43 __v.iov_len = min(n, __p->iov_len - skip); \ 44 if (likely(__v.iov_len)) { \ 45 __v.iov_base = __p->iov_base + skip; \ 46 (void)(STEP); \ 47 skip += __v.iov_len; \ 48 n -= __v.iov_len; \ 49 } \ 50 while (unlikely(n)) { \ 51 __p++; \ 52 __v.iov_len = min(n, __p->iov_len); \ 53 if (unlikely(!__v.iov_len)) \ 54 continue; \ 55 __v.iov_base = __p->iov_base; \ 56 (void)(STEP); \ 57 skip = __v.iov_len; \ 58 n -= __v.iov_len; \ 59 } \ 60 n = wanted; \ 61 } 62 63 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ 64 struct bvec_iter __start; \ 65 __start.bi_size = n; \ 66 __start.bi_bvec_done = skip; \ 67 __start.bi_idx = 0; \ 68 for_each_bvec(__v, i->bvec, __bi, __start) { \ 69 if (!__v.bv_len) \ 70 continue; \ 71 (void)(STEP); \ 72 } \ 73 } 74 75 #define iterate_all_kinds(i, n, v, I, B, K) { \ 76 if (likely(n)) { \ 77 size_t skip = i->iov_offset; \ 78 if (unlikely(i->type & ITER_BVEC)) { \ 79 struct bio_vec v; \ 80 struct bvec_iter __bi; \ 81 iterate_bvec(i, n, v, __bi, skip, (B)) \ 82 } else if (unlikely(i->type & ITER_KVEC)) { \ 83 const struct kvec *kvec; \ 84 struct kvec v; \ 85 iterate_kvec(i, n, v, kvec, skip, (K)) \ 86 } else { \ 87 const struct iovec *iov; \ 88 struct iovec v; \ 89 iterate_iovec(i, n, v, iov, skip, (I)) \ 90 } \ 91 } \ 92 } 93 94 #define iterate_and_advance(i, n, v, I, B, K) { \ 95 if (unlikely(i->count < n)) \ 96 n = i->count; \ 97 if (i->count) { \ 98 size_t skip = i->iov_offset; \ 99 if (unlikely(i->type & ITER_BVEC)) { \ 100 const struct bio_vec *bvec = i->bvec; \ 101 struct bio_vec v; \ 102 struct bvec_iter __bi; \ 103 iterate_bvec(i, n, v, __bi, skip, (B)) \ 104 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \ 105 i->nr_segs -= i->bvec - bvec; \ 106 skip = __bi.bi_bvec_done; \ 107 } else if (unlikely(i->type & ITER_KVEC)) { \ 108 const struct kvec *kvec; \ 109 struct kvec v; \ 110 iterate_kvec(i, n, v, kvec, skip, (K)) \ 111 if (skip == kvec->iov_len) { \ 112 kvec++; \ 113 skip = 0; \ 114 } \ 115 i->nr_segs -= kvec - i->kvec; \ 116 i->kvec = kvec; \ 117 } else { \ 118 const struct iovec *iov; \ 119 struct iovec v; \ 120 iterate_iovec(i, n, v, iov, skip, (I)) \ 121 if (skip == iov->iov_len) { \ 122 iov++; \ 123 skip = 0; \ 124 } \ 125 i->nr_segs -= iov - i->iov; \ 126 i->iov = iov; \ 127 } \ 128 i->count -= n; \ 129 i->iov_offset = skip; \ 130 } \ 131 } 132 133 static int copyout(void __user *to, const void *from, size_t n) 134 { 135 if (access_ok(VERIFY_WRITE, to, n)) { 136 kasan_check_read(from, n); 137 n = raw_copy_to_user(to, from, n); 138 } 139 return n; 140 } 141 142 static int copyin(void *to, const void __user *from, size_t n) 143 { 144 if (access_ok(VERIFY_READ, from, n)) { 145 kasan_check_write(to, n); 146 n = raw_copy_from_user(to, from, n); 147 } 148 return n; 149 } 150 151 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, 152 struct iov_iter *i) 153 { 154 size_t skip, copy, left, wanted; 155 const struct iovec *iov; 156 char __user *buf; 157 void *kaddr, *from; 158 159 if (unlikely(bytes > i->count)) 160 bytes = i->count; 161 162 if (unlikely(!bytes)) 163 return 0; 164 165 might_fault(); 166 wanted = bytes; 167 iov = i->iov; 168 skip = i->iov_offset; 169 buf = iov->iov_base + skip; 170 copy = min(bytes, iov->iov_len - skip); 171 172 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) { 173 kaddr = kmap_atomic(page); 174 from = kaddr + offset; 175 176 /* first chunk, usually the only one */ 177 left = copyout(buf, from, copy); 178 copy -= left; 179 skip += copy; 180 from += copy; 181 bytes -= copy; 182 183 while (unlikely(!left && bytes)) { 184 iov++; 185 buf = iov->iov_base; 186 copy = min(bytes, iov->iov_len); 187 left = copyout(buf, from, copy); 188 copy -= left; 189 skip = copy; 190 from += copy; 191 bytes -= copy; 192 } 193 if (likely(!bytes)) { 194 kunmap_atomic(kaddr); 195 goto done; 196 } 197 offset = from - kaddr; 198 buf += copy; 199 kunmap_atomic(kaddr); 200 copy = min(bytes, iov->iov_len - skip); 201 } 202 /* Too bad - revert to non-atomic kmap */ 203 204 kaddr = kmap(page); 205 from = kaddr + offset; 206 left = copyout(buf, from, copy); 207 copy -= left; 208 skip += copy; 209 from += copy; 210 bytes -= copy; 211 while (unlikely(!left && bytes)) { 212 iov++; 213 buf = iov->iov_base; 214 copy = min(bytes, iov->iov_len); 215 left = copyout(buf, from, copy); 216 copy -= left; 217 skip = copy; 218 from += copy; 219 bytes -= copy; 220 } 221 kunmap(page); 222 223 done: 224 if (skip == iov->iov_len) { 225 iov++; 226 skip = 0; 227 } 228 i->count -= wanted - bytes; 229 i->nr_segs -= iov - i->iov; 230 i->iov = iov; 231 i->iov_offset = skip; 232 return wanted - bytes; 233 } 234 235 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, 236 struct iov_iter *i) 237 { 238 size_t skip, copy, left, wanted; 239 const struct iovec *iov; 240 char __user *buf; 241 void *kaddr, *to; 242 243 if (unlikely(bytes > i->count)) 244 bytes = i->count; 245 246 if (unlikely(!bytes)) 247 return 0; 248 249 might_fault(); 250 wanted = bytes; 251 iov = i->iov; 252 skip = i->iov_offset; 253 buf = iov->iov_base + skip; 254 copy = min(bytes, iov->iov_len - skip); 255 256 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) { 257 kaddr = kmap_atomic(page); 258 to = kaddr + offset; 259 260 /* first chunk, usually the only one */ 261 left = copyin(to, buf, copy); 262 copy -= left; 263 skip += copy; 264 to += copy; 265 bytes -= copy; 266 267 while (unlikely(!left && bytes)) { 268 iov++; 269 buf = iov->iov_base; 270 copy = min(bytes, iov->iov_len); 271 left = copyin(to, buf, copy); 272 copy -= left; 273 skip = copy; 274 to += copy; 275 bytes -= copy; 276 } 277 if (likely(!bytes)) { 278 kunmap_atomic(kaddr); 279 goto done; 280 } 281 offset = to - kaddr; 282 buf += copy; 283 kunmap_atomic(kaddr); 284 copy = min(bytes, iov->iov_len - skip); 285 } 286 /* Too bad - revert to non-atomic kmap */ 287 288 kaddr = kmap(page); 289 to = kaddr + offset; 290 left = copyin(to, buf, copy); 291 copy -= left; 292 skip += copy; 293 to += copy; 294 bytes -= copy; 295 while (unlikely(!left && bytes)) { 296 iov++; 297 buf = iov->iov_base; 298 copy = min(bytes, iov->iov_len); 299 left = copyin(to, buf, copy); 300 copy -= left; 301 skip = copy; 302 to += copy; 303 bytes -= copy; 304 } 305 kunmap(page); 306 307 done: 308 if (skip == iov->iov_len) { 309 iov++; 310 skip = 0; 311 } 312 i->count -= wanted - bytes; 313 i->nr_segs -= iov - i->iov; 314 i->iov = iov; 315 i->iov_offset = skip; 316 return wanted - bytes; 317 } 318 319 #ifdef PIPE_PARANOIA 320 static bool sanity(const struct iov_iter *i) 321 { 322 struct pipe_inode_info *pipe = i->pipe; 323 int idx = i->idx; 324 int next = pipe->curbuf + pipe->nrbufs; 325 if (i->iov_offset) { 326 struct pipe_buffer *p; 327 if (unlikely(!pipe->nrbufs)) 328 goto Bad; // pipe must be non-empty 329 if (unlikely(idx != ((next - 1) & (pipe->buffers - 1)))) 330 goto Bad; // must be at the last buffer... 331 332 p = &pipe->bufs[idx]; 333 if (unlikely(p->offset + p->len != i->iov_offset)) 334 goto Bad; // ... at the end of segment 335 } else { 336 if (idx != (next & (pipe->buffers - 1))) 337 goto Bad; // must be right after the last buffer 338 } 339 return true; 340 Bad: 341 printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset); 342 printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n", 343 pipe->curbuf, pipe->nrbufs, pipe->buffers); 344 for (idx = 0; idx < pipe->buffers; idx++) 345 printk(KERN_ERR "[%p %p %d %d]\n", 346 pipe->bufs[idx].ops, 347 pipe->bufs[idx].page, 348 pipe->bufs[idx].offset, 349 pipe->bufs[idx].len); 350 WARN_ON(1); 351 return false; 352 } 353 #else 354 #define sanity(i) true 355 #endif 356 357 static inline int next_idx(int idx, struct pipe_inode_info *pipe) 358 { 359 return (idx + 1) & (pipe->buffers - 1); 360 } 361 362 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes, 363 struct iov_iter *i) 364 { 365 struct pipe_inode_info *pipe = i->pipe; 366 struct pipe_buffer *buf; 367 size_t off; 368 int idx; 369 370 if (unlikely(bytes > i->count)) 371 bytes = i->count; 372 373 if (unlikely(!bytes)) 374 return 0; 375 376 if (!sanity(i)) 377 return 0; 378 379 off = i->iov_offset; 380 idx = i->idx; 381 buf = &pipe->bufs[idx]; 382 if (off) { 383 if (offset == off && buf->page == page) { 384 /* merge with the last one */ 385 buf->len += bytes; 386 i->iov_offset += bytes; 387 goto out; 388 } 389 idx = next_idx(idx, pipe); 390 buf = &pipe->bufs[idx]; 391 } 392 if (idx == pipe->curbuf && pipe->nrbufs) 393 return 0; 394 pipe->nrbufs++; 395 buf->ops = &page_cache_pipe_buf_ops; 396 get_page(buf->page = page); 397 buf->offset = offset; 398 buf->len = bytes; 399 i->iov_offset = offset + bytes; 400 i->idx = idx; 401 out: 402 i->count -= bytes; 403 return bytes; 404 } 405 406 /* 407 * Fault in one or more iovecs of the given iov_iter, to a maximum length of 408 * bytes. For each iovec, fault in each page that constitutes the iovec. 409 * 410 * Return 0 on success, or non-zero if the memory could not be accessed (i.e. 411 * because it is an invalid address). 412 */ 413 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 414 { 415 size_t skip = i->iov_offset; 416 const struct iovec *iov; 417 int err; 418 struct iovec v; 419 420 if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 421 iterate_iovec(i, bytes, v, iov, skip, ({ 422 err = fault_in_pages_readable(v.iov_base, v.iov_len); 423 if (unlikely(err)) 424 return err; 425 0;})) 426 } 427 return 0; 428 } 429 EXPORT_SYMBOL(iov_iter_fault_in_readable); 430 431 void iov_iter_init(struct iov_iter *i, int direction, 432 const struct iovec *iov, unsigned long nr_segs, 433 size_t count) 434 { 435 /* It will get better. Eventually... */ 436 if (uaccess_kernel()) { 437 direction |= ITER_KVEC; 438 i->type = direction; 439 i->kvec = (struct kvec *)iov; 440 } else { 441 i->type = direction; 442 i->iov = iov; 443 } 444 i->nr_segs = nr_segs; 445 i->iov_offset = 0; 446 i->count = count; 447 } 448 EXPORT_SYMBOL(iov_iter_init); 449 450 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) 451 { 452 char *from = kmap_atomic(page); 453 memcpy(to, from + offset, len); 454 kunmap_atomic(from); 455 } 456 457 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len) 458 { 459 char *to = kmap_atomic(page); 460 memcpy(to + offset, from, len); 461 kunmap_atomic(to); 462 } 463 464 static void memzero_page(struct page *page, size_t offset, size_t len) 465 { 466 char *addr = kmap_atomic(page); 467 memset(addr + offset, 0, len); 468 kunmap_atomic(addr); 469 } 470 471 static inline bool allocated(struct pipe_buffer *buf) 472 { 473 return buf->ops == &default_pipe_buf_ops; 474 } 475 476 static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp) 477 { 478 size_t off = i->iov_offset; 479 int idx = i->idx; 480 if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) { 481 idx = next_idx(idx, i->pipe); 482 off = 0; 483 } 484 *idxp = idx; 485 *offp = off; 486 } 487 488 static size_t push_pipe(struct iov_iter *i, size_t size, 489 int *idxp, size_t *offp) 490 { 491 struct pipe_inode_info *pipe = i->pipe; 492 size_t off; 493 int idx; 494 ssize_t left; 495 496 if (unlikely(size > i->count)) 497 size = i->count; 498 if (unlikely(!size)) 499 return 0; 500 501 left = size; 502 data_start(i, &idx, &off); 503 *idxp = idx; 504 *offp = off; 505 if (off) { 506 left -= PAGE_SIZE - off; 507 if (left <= 0) { 508 pipe->bufs[idx].len += size; 509 return size; 510 } 511 pipe->bufs[idx].len = PAGE_SIZE; 512 idx = next_idx(idx, pipe); 513 } 514 while (idx != pipe->curbuf || !pipe->nrbufs) { 515 struct page *page = alloc_page(GFP_USER); 516 if (!page) 517 break; 518 pipe->nrbufs++; 519 pipe->bufs[idx].ops = &default_pipe_buf_ops; 520 pipe->bufs[idx].page = page; 521 pipe->bufs[idx].offset = 0; 522 if (left <= PAGE_SIZE) { 523 pipe->bufs[idx].len = left; 524 return size; 525 } 526 pipe->bufs[idx].len = PAGE_SIZE; 527 left -= PAGE_SIZE; 528 idx = next_idx(idx, pipe); 529 } 530 return size - left; 531 } 532 533 static size_t copy_pipe_to_iter(const void *addr, size_t bytes, 534 struct iov_iter *i) 535 { 536 struct pipe_inode_info *pipe = i->pipe; 537 size_t n, off; 538 int idx; 539 540 if (!sanity(i)) 541 return 0; 542 543 bytes = n = push_pipe(i, bytes, &idx, &off); 544 if (unlikely(!n)) 545 return 0; 546 for ( ; n; idx = next_idx(idx, pipe), off = 0) { 547 size_t chunk = min_t(size_t, n, PAGE_SIZE - off); 548 memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk); 549 i->idx = idx; 550 i->iov_offset = off + chunk; 551 n -= chunk; 552 addr += chunk; 553 } 554 i->count -= bytes; 555 return bytes; 556 } 557 558 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) 559 { 560 const char *from = addr; 561 if (unlikely(i->type & ITER_PIPE)) 562 return copy_pipe_to_iter(addr, bytes, i); 563 if (iter_is_iovec(i)) 564 might_fault(); 565 iterate_and_advance(i, bytes, v, 566 copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), 567 memcpy_to_page(v.bv_page, v.bv_offset, 568 (from += v.bv_len) - v.bv_len, v.bv_len), 569 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) 570 ) 571 572 return bytes; 573 } 574 EXPORT_SYMBOL(_copy_to_iter); 575 576 #ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE 577 static int copyout_mcsafe(void __user *to, const void *from, size_t n) 578 { 579 if (access_ok(VERIFY_WRITE, to, n)) { 580 kasan_check_read(from, n); 581 n = copy_to_user_mcsafe((__force void *) to, from, n); 582 } 583 return n; 584 } 585 586 static unsigned long memcpy_mcsafe_to_page(struct page *page, size_t offset, 587 const char *from, size_t len) 588 { 589 unsigned long ret; 590 char *to; 591 592 to = kmap_atomic(page); 593 ret = memcpy_mcsafe(to + offset, from, len); 594 kunmap_atomic(to); 595 596 return ret; 597 } 598 599 /** 600 * _copy_to_iter_mcsafe - copy to user with source-read error exception handling 601 * @addr: source kernel address 602 * @bytes: total transfer length 603 * @iter: destination iterator 604 * 605 * The pmem driver arranges for filesystem-dax to use this facility via 606 * dax_copy_to_iter() for protecting read/write to persistent memory. 607 * Unless / until an architecture can guarantee identical performance 608 * between _copy_to_iter_mcsafe() and _copy_to_iter() it would be a 609 * performance regression to switch more users to the mcsafe version. 610 * 611 * Otherwise, the main differences between this and typical _copy_to_iter(). 612 * 613 * * Typical tail/residue handling after a fault retries the copy 614 * byte-by-byte until the fault happens again. Re-triggering machine 615 * checks is potentially fatal so the implementation uses source 616 * alignment and poison alignment assumptions to avoid re-triggering 617 * hardware exceptions. 618 * 619 * * ITER_KVEC, ITER_PIPE, and ITER_BVEC can return short copies. 620 * Compare to copy_to_iter() where only ITER_IOVEC attempts might return 621 * a short copy. 622 * 623 * See MCSAFE_TEST for self-test. 624 */ 625 size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i) 626 { 627 const char *from = addr; 628 unsigned long rem, curr_addr, s_addr = (unsigned long) addr; 629 630 if (unlikely(i->type & ITER_PIPE)) { 631 WARN_ON(1); 632 return 0; 633 } 634 if (iter_is_iovec(i)) 635 might_fault(); 636 iterate_and_advance(i, bytes, v, 637 copyout_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), 638 ({ 639 rem = memcpy_mcsafe_to_page(v.bv_page, v.bv_offset, 640 (from += v.bv_len) - v.bv_len, v.bv_len); 641 if (rem) { 642 curr_addr = (unsigned long) from; 643 bytes = curr_addr - s_addr - rem; 644 return bytes; 645 } 646 }), 647 ({ 648 rem = memcpy_mcsafe(v.iov_base, (from += v.iov_len) - v.iov_len, 649 v.iov_len); 650 if (rem) { 651 curr_addr = (unsigned long) from; 652 bytes = curr_addr - s_addr - rem; 653 return bytes; 654 } 655 }) 656 ) 657 658 return bytes; 659 } 660 EXPORT_SYMBOL_GPL(_copy_to_iter_mcsafe); 661 #endif /* CONFIG_ARCH_HAS_UACCESS_MCSAFE */ 662 663 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 664 { 665 char *to = addr; 666 if (unlikely(i->type & ITER_PIPE)) { 667 WARN_ON(1); 668 return 0; 669 } 670 if (iter_is_iovec(i)) 671 might_fault(); 672 iterate_and_advance(i, bytes, v, 673 copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), 674 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 675 v.bv_offset, v.bv_len), 676 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 677 ) 678 679 return bytes; 680 } 681 EXPORT_SYMBOL(_copy_from_iter); 682 683 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) 684 { 685 char *to = addr; 686 if (unlikely(i->type & ITER_PIPE)) { 687 WARN_ON(1); 688 return false; 689 } 690 if (unlikely(i->count < bytes)) 691 return false; 692 693 if (iter_is_iovec(i)) 694 might_fault(); 695 iterate_all_kinds(i, bytes, v, ({ 696 if (copyin((to += v.iov_len) - v.iov_len, 697 v.iov_base, v.iov_len)) 698 return false; 699 0;}), 700 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 701 v.bv_offset, v.bv_len), 702 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 703 ) 704 705 iov_iter_advance(i, bytes); 706 return true; 707 } 708 EXPORT_SYMBOL(_copy_from_iter_full); 709 710 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 711 { 712 char *to = addr; 713 if (unlikely(i->type & ITER_PIPE)) { 714 WARN_ON(1); 715 return 0; 716 } 717 iterate_and_advance(i, bytes, v, 718 __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, 719 v.iov_base, v.iov_len), 720 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 721 v.bv_offset, v.bv_len), 722 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 723 ) 724 725 return bytes; 726 } 727 EXPORT_SYMBOL(_copy_from_iter_nocache); 728 729 #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE 730 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i) 731 { 732 char *to = addr; 733 if (unlikely(i->type & ITER_PIPE)) { 734 WARN_ON(1); 735 return 0; 736 } 737 iterate_and_advance(i, bytes, v, 738 __copy_from_user_flushcache((to += v.iov_len) - v.iov_len, 739 v.iov_base, v.iov_len), 740 memcpy_page_flushcache((to += v.bv_len) - v.bv_len, v.bv_page, 741 v.bv_offset, v.bv_len), 742 memcpy_flushcache((to += v.iov_len) - v.iov_len, v.iov_base, 743 v.iov_len) 744 ) 745 746 return bytes; 747 } 748 EXPORT_SYMBOL_GPL(_copy_from_iter_flushcache); 749 #endif 750 751 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) 752 { 753 char *to = addr; 754 if (unlikely(i->type & ITER_PIPE)) { 755 WARN_ON(1); 756 return false; 757 } 758 if (unlikely(i->count < bytes)) 759 return false; 760 iterate_all_kinds(i, bytes, v, ({ 761 if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, 762 v.iov_base, v.iov_len)) 763 return false; 764 0;}), 765 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 766 v.bv_offset, v.bv_len), 767 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 768 ) 769 770 iov_iter_advance(i, bytes); 771 return true; 772 } 773 EXPORT_SYMBOL(_copy_from_iter_full_nocache); 774 775 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) 776 { 777 struct page *head = compound_head(page); 778 size_t v = n + offset + page_address(page) - page_address(head); 779 780 if (likely(n <= v && v <= (PAGE_SIZE << compound_order(head)))) 781 return true; 782 WARN_ON(1); 783 return false; 784 } 785 786 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 787 struct iov_iter *i) 788 { 789 if (unlikely(!page_copy_sane(page, offset, bytes))) 790 return 0; 791 if (i->type & (ITER_BVEC|ITER_KVEC)) { 792 void *kaddr = kmap_atomic(page); 793 size_t wanted = copy_to_iter(kaddr + offset, bytes, i); 794 kunmap_atomic(kaddr); 795 return wanted; 796 } else if (likely(!(i->type & ITER_PIPE))) 797 return copy_page_to_iter_iovec(page, offset, bytes, i); 798 else 799 return copy_page_to_iter_pipe(page, offset, bytes, i); 800 } 801 EXPORT_SYMBOL(copy_page_to_iter); 802 803 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 804 struct iov_iter *i) 805 { 806 if (unlikely(!page_copy_sane(page, offset, bytes))) 807 return 0; 808 if (unlikely(i->type & ITER_PIPE)) { 809 WARN_ON(1); 810 return 0; 811 } 812 if (i->type & (ITER_BVEC|ITER_KVEC)) { 813 void *kaddr = kmap_atomic(page); 814 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); 815 kunmap_atomic(kaddr); 816 return wanted; 817 } else 818 return copy_page_from_iter_iovec(page, offset, bytes, i); 819 } 820 EXPORT_SYMBOL(copy_page_from_iter); 821 822 static size_t pipe_zero(size_t bytes, struct iov_iter *i) 823 { 824 struct pipe_inode_info *pipe = i->pipe; 825 size_t n, off; 826 int idx; 827 828 if (!sanity(i)) 829 return 0; 830 831 bytes = n = push_pipe(i, bytes, &idx, &off); 832 if (unlikely(!n)) 833 return 0; 834 835 for ( ; n; idx = next_idx(idx, pipe), off = 0) { 836 size_t chunk = min_t(size_t, n, PAGE_SIZE - off); 837 memzero_page(pipe->bufs[idx].page, off, chunk); 838 i->idx = idx; 839 i->iov_offset = off + chunk; 840 n -= chunk; 841 } 842 i->count -= bytes; 843 return bytes; 844 } 845 846 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) 847 { 848 if (unlikely(i->type & ITER_PIPE)) 849 return pipe_zero(bytes, i); 850 iterate_and_advance(i, bytes, v, 851 clear_user(v.iov_base, v.iov_len), 852 memzero_page(v.bv_page, v.bv_offset, v.bv_len), 853 memset(v.iov_base, 0, v.iov_len) 854 ) 855 856 return bytes; 857 } 858 EXPORT_SYMBOL(iov_iter_zero); 859 860 size_t iov_iter_copy_from_user_atomic(struct page *page, 861 struct iov_iter *i, unsigned long offset, size_t bytes) 862 { 863 char *kaddr = kmap_atomic(page), *p = kaddr + offset; 864 if (unlikely(!page_copy_sane(page, offset, bytes))) { 865 kunmap_atomic(kaddr); 866 return 0; 867 } 868 if (unlikely(i->type & ITER_PIPE)) { 869 kunmap_atomic(kaddr); 870 WARN_ON(1); 871 return 0; 872 } 873 iterate_all_kinds(i, bytes, v, 874 copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), 875 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, 876 v.bv_offset, v.bv_len), 877 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 878 ) 879 kunmap_atomic(kaddr); 880 return bytes; 881 } 882 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 883 884 static inline void pipe_truncate(struct iov_iter *i) 885 { 886 struct pipe_inode_info *pipe = i->pipe; 887 if (pipe->nrbufs) { 888 size_t off = i->iov_offset; 889 int idx = i->idx; 890 int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1); 891 if (off) { 892 pipe->bufs[idx].len = off - pipe->bufs[idx].offset; 893 idx = next_idx(idx, pipe); 894 nrbufs++; 895 } 896 while (pipe->nrbufs > nrbufs) { 897 pipe_buf_release(pipe, &pipe->bufs[idx]); 898 idx = next_idx(idx, pipe); 899 pipe->nrbufs--; 900 } 901 } 902 } 903 904 static void pipe_advance(struct iov_iter *i, size_t size) 905 { 906 struct pipe_inode_info *pipe = i->pipe; 907 if (unlikely(i->count < size)) 908 size = i->count; 909 if (size) { 910 struct pipe_buffer *buf; 911 size_t off = i->iov_offset, left = size; 912 int idx = i->idx; 913 if (off) /* make it relative to the beginning of buffer */ 914 left += off - pipe->bufs[idx].offset; 915 while (1) { 916 buf = &pipe->bufs[idx]; 917 if (left <= buf->len) 918 break; 919 left -= buf->len; 920 idx = next_idx(idx, pipe); 921 } 922 i->idx = idx; 923 i->iov_offset = buf->offset + left; 924 } 925 i->count -= size; 926 /* ... and discard everything past that point */ 927 pipe_truncate(i); 928 } 929 930 void iov_iter_advance(struct iov_iter *i, size_t size) 931 { 932 if (unlikely(i->type & ITER_PIPE)) { 933 pipe_advance(i, size); 934 return; 935 } 936 iterate_and_advance(i, size, v, 0, 0, 0) 937 } 938 EXPORT_SYMBOL(iov_iter_advance); 939 940 void iov_iter_revert(struct iov_iter *i, size_t unroll) 941 { 942 if (!unroll) 943 return; 944 if (WARN_ON(unroll > MAX_RW_COUNT)) 945 return; 946 i->count += unroll; 947 if (unlikely(i->type & ITER_PIPE)) { 948 struct pipe_inode_info *pipe = i->pipe; 949 int idx = i->idx; 950 size_t off = i->iov_offset; 951 while (1) { 952 size_t n = off - pipe->bufs[idx].offset; 953 if (unroll < n) { 954 off -= unroll; 955 break; 956 } 957 unroll -= n; 958 if (!unroll && idx == i->start_idx) { 959 off = 0; 960 break; 961 } 962 if (!idx--) 963 idx = pipe->buffers - 1; 964 off = pipe->bufs[idx].offset + pipe->bufs[idx].len; 965 } 966 i->iov_offset = off; 967 i->idx = idx; 968 pipe_truncate(i); 969 return; 970 } 971 if (unroll <= i->iov_offset) { 972 i->iov_offset -= unroll; 973 return; 974 } 975 unroll -= i->iov_offset; 976 if (i->type & ITER_BVEC) { 977 const struct bio_vec *bvec = i->bvec; 978 while (1) { 979 size_t n = (--bvec)->bv_len; 980 i->nr_segs++; 981 if (unroll <= n) { 982 i->bvec = bvec; 983 i->iov_offset = n - unroll; 984 return; 985 } 986 unroll -= n; 987 } 988 } else { /* same logics for iovec and kvec */ 989 const struct iovec *iov = i->iov; 990 while (1) { 991 size_t n = (--iov)->iov_len; 992 i->nr_segs++; 993 if (unroll <= n) { 994 i->iov = iov; 995 i->iov_offset = n - unroll; 996 return; 997 } 998 unroll -= n; 999 } 1000 } 1001 } 1002 EXPORT_SYMBOL(iov_iter_revert); 1003 1004 /* 1005 * Return the count of just the current iov_iter segment. 1006 */ 1007 size_t iov_iter_single_seg_count(const struct iov_iter *i) 1008 { 1009 if (unlikely(i->type & ITER_PIPE)) 1010 return i->count; // it is a silly place, anyway 1011 if (i->nr_segs == 1) 1012 return i->count; 1013 else if (i->type & ITER_BVEC) 1014 return min(i->count, i->bvec->bv_len - i->iov_offset); 1015 else 1016 return min(i->count, i->iov->iov_len - i->iov_offset); 1017 } 1018 EXPORT_SYMBOL(iov_iter_single_seg_count); 1019 1020 void iov_iter_kvec(struct iov_iter *i, int direction, 1021 const struct kvec *kvec, unsigned long nr_segs, 1022 size_t count) 1023 { 1024 BUG_ON(!(direction & ITER_KVEC)); 1025 i->type = direction; 1026 i->kvec = kvec; 1027 i->nr_segs = nr_segs; 1028 i->iov_offset = 0; 1029 i->count = count; 1030 } 1031 EXPORT_SYMBOL(iov_iter_kvec); 1032 1033 void iov_iter_bvec(struct iov_iter *i, int direction, 1034 const struct bio_vec *bvec, unsigned long nr_segs, 1035 size_t count) 1036 { 1037 BUG_ON(!(direction & ITER_BVEC)); 1038 i->type = direction; 1039 i->bvec = bvec; 1040 i->nr_segs = nr_segs; 1041 i->iov_offset = 0; 1042 i->count = count; 1043 } 1044 EXPORT_SYMBOL(iov_iter_bvec); 1045 1046 void iov_iter_pipe(struct iov_iter *i, int direction, 1047 struct pipe_inode_info *pipe, 1048 size_t count) 1049 { 1050 BUG_ON(direction != ITER_PIPE); 1051 WARN_ON(pipe->nrbufs == pipe->buffers); 1052 i->type = direction; 1053 i->pipe = pipe; 1054 i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 1055 i->iov_offset = 0; 1056 i->count = count; 1057 i->start_idx = i->idx; 1058 } 1059 EXPORT_SYMBOL(iov_iter_pipe); 1060 1061 unsigned long iov_iter_alignment(const struct iov_iter *i) 1062 { 1063 unsigned long res = 0; 1064 size_t size = i->count; 1065 1066 if (unlikely(i->type & ITER_PIPE)) { 1067 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx])) 1068 return size | i->iov_offset; 1069 return size; 1070 } 1071 iterate_all_kinds(i, size, v, 1072 (res |= (unsigned long)v.iov_base | v.iov_len, 0), 1073 res |= v.bv_offset | v.bv_len, 1074 res |= (unsigned long)v.iov_base | v.iov_len 1075 ) 1076 return res; 1077 } 1078 EXPORT_SYMBOL(iov_iter_alignment); 1079 1080 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) 1081 { 1082 unsigned long res = 0; 1083 size_t size = i->count; 1084 1085 if (unlikely(i->type & ITER_PIPE)) { 1086 WARN_ON(1); 1087 return ~0U; 1088 } 1089 1090 iterate_all_kinds(i, size, v, 1091 (res |= (!res ? 0 : (unsigned long)v.iov_base) | 1092 (size != v.iov_len ? size : 0), 0), 1093 (res |= (!res ? 0 : (unsigned long)v.bv_offset) | 1094 (size != v.bv_len ? size : 0)), 1095 (res |= (!res ? 0 : (unsigned long)v.iov_base) | 1096 (size != v.iov_len ? size : 0)) 1097 ); 1098 return res; 1099 } 1100 EXPORT_SYMBOL(iov_iter_gap_alignment); 1101 1102 static inline ssize_t __pipe_get_pages(struct iov_iter *i, 1103 size_t maxsize, 1104 struct page **pages, 1105 int idx, 1106 size_t *start) 1107 { 1108 struct pipe_inode_info *pipe = i->pipe; 1109 ssize_t n = push_pipe(i, maxsize, &idx, start); 1110 if (!n) 1111 return -EFAULT; 1112 1113 maxsize = n; 1114 n += *start; 1115 while (n > 0) { 1116 get_page(*pages++ = pipe->bufs[idx].page); 1117 idx = next_idx(idx, pipe); 1118 n -= PAGE_SIZE; 1119 } 1120 1121 return maxsize; 1122 } 1123 1124 static ssize_t pipe_get_pages(struct iov_iter *i, 1125 struct page **pages, size_t maxsize, unsigned maxpages, 1126 size_t *start) 1127 { 1128 unsigned npages; 1129 size_t capacity; 1130 int idx; 1131 1132 if (!maxsize) 1133 return 0; 1134 1135 if (!sanity(i)) 1136 return -EFAULT; 1137 1138 data_start(i, &idx, start); 1139 /* some of this one + all after this one */ 1140 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; 1141 capacity = min(npages,maxpages) * PAGE_SIZE - *start; 1142 1143 return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start); 1144 } 1145 1146 ssize_t iov_iter_get_pages(struct iov_iter *i, 1147 struct page **pages, size_t maxsize, unsigned maxpages, 1148 size_t *start) 1149 { 1150 if (maxsize > i->count) 1151 maxsize = i->count; 1152 1153 if (unlikely(i->type & ITER_PIPE)) 1154 return pipe_get_pages(i, pages, maxsize, maxpages, start); 1155 iterate_all_kinds(i, maxsize, v, ({ 1156 unsigned long addr = (unsigned long)v.iov_base; 1157 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 1158 int n; 1159 int res; 1160 1161 if (len > maxpages * PAGE_SIZE) 1162 len = maxpages * PAGE_SIZE; 1163 addr &= ~(PAGE_SIZE - 1); 1164 n = DIV_ROUND_UP(len, PAGE_SIZE); 1165 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); 1166 if (unlikely(res < 0)) 1167 return res; 1168 return (res == n ? len : res * PAGE_SIZE) - *start; 1169 0;}),({ 1170 /* can't be more than PAGE_SIZE */ 1171 *start = v.bv_offset; 1172 get_page(*pages = v.bv_page); 1173 return v.bv_len; 1174 }),({ 1175 return -EFAULT; 1176 }) 1177 ) 1178 return 0; 1179 } 1180 EXPORT_SYMBOL(iov_iter_get_pages); 1181 1182 static struct page **get_pages_array(size_t n) 1183 { 1184 return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL); 1185 } 1186 1187 static ssize_t pipe_get_pages_alloc(struct iov_iter *i, 1188 struct page ***pages, size_t maxsize, 1189 size_t *start) 1190 { 1191 struct page **p; 1192 ssize_t n; 1193 int idx; 1194 int npages; 1195 1196 if (!maxsize) 1197 return 0; 1198 1199 if (!sanity(i)) 1200 return -EFAULT; 1201 1202 data_start(i, &idx, start); 1203 /* some of this one + all after this one */ 1204 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; 1205 n = npages * PAGE_SIZE - *start; 1206 if (maxsize > n) 1207 maxsize = n; 1208 else 1209 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); 1210 p = get_pages_array(npages); 1211 if (!p) 1212 return -ENOMEM; 1213 n = __pipe_get_pages(i, maxsize, p, idx, start); 1214 if (n > 0) 1215 *pages = p; 1216 else 1217 kvfree(p); 1218 return n; 1219 } 1220 1221 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, 1222 struct page ***pages, size_t maxsize, 1223 size_t *start) 1224 { 1225 struct page **p; 1226 1227 if (maxsize > i->count) 1228 maxsize = i->count; 1229 1230 if (unlikely(i->type & ITER_PIPE)) 1231 return pipe_get_pages_alloc(i, pages, maxsize, start); 1232 iterate_all_kinds(i, maxsize, v, ({ 1233 unsigned long addr = (unsigned long)v.iov_base; 1234 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 1235 int n; 1236 int res; 1237 1238 addr &= ~(PAGE_SIZE - 1); 1239 n = DIV_ROUND_UP(len, PAGE_SIZE); 1240 p = get_pages_array(n); 1241 if (!p) 1242 return -ENOMEM; 1243 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); 1244 if (unlikely(res < 0)) { 1245 kvfree(p); 1246 return res; 1247 } 1248 *pages = p; 1249 return (res == n ? len : res * PAGE_SIZE) - *start; 1250 0;}),({ 1251 /* can't be more than PAGE_SIZE */ 1252 *start = v.bv_offset; 1253 *pages = p = get_pages_array(1); 1254 if (!p) 1255 return -ENOMEM; 1256 get_page(*p = v.bv_page); 1257 return v.bv_len; 1258 }),({ 1259 return -EFAULT; 1260 }) 1261 ) 1262 return 0; 1263 } 1264 EXPORT_SYMBOL(iov_iter_get_pages_alloc); 1265 1266 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, 1267 struct iov_iter *i) 1268 { 1269 char *to = addr; 1270 __wsum sum, next; 1271 size_t off = 0; 1272 sum = *csum; 1273 if (unlikely(i->type & ITER_PIPE)) { 1274 WARN_ON(1); 1275 return 0; 1276 } 1277 iterate_and_advance(i, bytes, v, ({ 1278 int err = 0; 1279 next = csum_and_copy_from_user(v.iov_base, 1280 (to += v.iov_len) - v.iov_len, 1281 v.iov_len, 0, &err); 1282 if (!err) { 1283 sum = csum_block_add(sum, next, off); 1284 off += v.iov_len; 1285 } 1286 err ? v.iov_len : 0; 1287 }), ({ 1288 char *p = kmap_atomic(v.bv_page); 1289 next = csum_partial_copy_nocheck(p + v.bv_offset, 1290 (to += v.bv_len) - v.bv_len, 1291 v.bv_len, 0); 1292 kunmap_atomic(p); 1293 sum = csum_block_add(sum, next, off); 1294 off += v.bv_len; 1295 }),({ 1296 next = csum_partial_copy_nocheck(v.iov_base, 1297 (to += v.iov_len) - v.iov_len, 1298 v.iov_len, 0); 1299 sum = csum_block_add(sum, next, off); 1300 off += v.iov_len; 1301 }) 1302 ) 1303 *csum = sum; 1304 return bytes; 1305 } 1306 EXPORT_SYMBOL(csum_and_copy_from_iter); 1307 1308 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, 1309 struct iov_iter *i) 1310 { 1311 char *to = addr; 1312 __wsum sum, next; 1313 size_t off = 0; 1314 sum = *csum; 1315 if (unlikely(i->type & ITER_PIPE)) { 1316 WARN_ON(1); 1317 return false; 1318 } 1319 if (unlikely(i->count < bytes)) 1320 return false; 1321 iterate_all_kinds(i, bytes, v, ({ 1322 int err = 0; 1323 next = csum_and_copy_from_user(v.iov_base, 1324 (to += v.iov_len) - v.iov_len, 1325 v.iov_len, 0, &err); 1326 if (err) 1327 return false; 1328 sum = csum_block_add(sum, next, off); 1329 off += v.iov_len; 1330 0; 1331 }), ({ 1332 char *p = kmap_atomic(v.bv_page); 1333 next = csum_partial_copy_nocheck(p + v.bv_offset, 1334 (to += v.bv_len) - v.bv_len, 1335 v.bv_len, 0); 1336 kunmap_atomic(p); 1337 sum = csum_block_add(sum, next, off); 1338 off += v.bv_len; 1339 }),({ 1340 next = csum_partial_copy_nocheck(v.iov_base, 1341 (to += v.iov_len) - v.iov_len, 1342 v.iov_len, 0); 1343 sum = csum_block_add(sum, next, off); 1344 off += v.iov_len; 1345 }) 1346 ) 1347 *csum = sum; 1348 iov_iter_advance(i, bytes); 1349 return true; 1350 } 1351 EXPORT_SYMBOL(csum_and_copy_from_iter_full); 1352 1353 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, 1354 struct iov_iter *i) 1355 { 1356 const char *from = addr; 1357 __wsum sum, next; 1358 size_t off = 0; 1359 sum = *csum; 1360 if (unlikely(i->type & ITER_PIPE)) { 1361 WARN_ON(1); /* for now */ 1362 return 0; 1363 } 1364 iterate_and_advance(i, bytes, v, ({ 1365 int err = 0; 1366 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, 1367 v.iov_base, 1368 v.iov_len, 0, &err); 1369 if (!err) { 1370 sum = csum_block_add(sum, next, off); 1371 off += v.iov_len; 1372 } 1373 err ? v.iov_len : 0; 1374 }), ({ 1375 char *p = kmap_atomic(v.bv_page); 1376 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len, 1377 p + v.bv_offset, 1378 v.bv_len, 0); 1379 kunmap_atomic(p); 1380 sum = csum_block_add(sum, next, off); 1381 off += v.bv_len; 1382 }),({ 1383 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len, 1384 v.iov_base, 1385 v.iov_len, 0); 1386 sum = csum_block_add(sum, next, off); 1387 off += v.iov_len; 1388 }) 1389 ) 1390 *csum = sum; 1391 return bytes; 1392 } 1393 EXPORT_SYMBOL(csum_and_copy_to_iter); 1394 1395 int iov_iter_npages(const struct iov_iter *i, int maxpages) 1396 { 1397 size_t size = i->count; 1398 int npages = 0; 1399 1400 if (!size) 1401 return 0; 1402 1403 if (unlikely(i->type & ITER_PIPE)) { 1404 struct pipe_inode_info *pipe = i->pipe; 1405 size_t off; 1406 int idx; 1407 1408 if (!sanity(i)) 1409 return 0; 1410 1411 data_start(i, &idx, &off); 1412 /* some of this one + all after this one */ 1413 npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1; 1414 if (npages >= maxpages) 1415 return maxpages; 1416 } else iterate_all_kinds(i, size, v, ({ 1417 unsigned long p = (unsigned long)v.iov_base; 1418 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 1419 - p / PAGE_SIZE; 1420 if (npages >= maxpages) 1421 return maxpages; 1422 0;}),({ 1423 npages++; 1424 if (npages >= maxpages) 1425 return maxpages; 1426 }),({ 1427 unsigned long p = (unsigned long)v.iov_base; 1428 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 1429 - p / PAGE_SIZE; 1430 if (npages >= maxpages) 1431 return maxpages; 1432 }) 1433 ) 1434 return npages; 1435 } 1436 EXPORT_SYMBOL(iov_iter_npages); 1437 1438 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) 1439 { 1440 *new = *old; 1441 if (unlikely(new->type & ITER_PIPE)) { 1442 WARN_ON(1); 1443 return NULL; 1444 } 1445 if (new->type & ITER_BVEC) 1446 return new->bvec = kmemdup(new->bvec, 1447 new->nr_segs * sizeof(struct bio_vec), 1448 flags); 1449 else 1450 /* iovec and kvec have identical layout */ 1451 return new->iov = kmemdup(new->iov, 1452 new->nr_segs * sizeof(struct iovec), 1453 flags); 1454 } 1455 EXPORT_SYMBOL(dup_iter); 1456 1457 /** 1458 * import_iovec() - Copy an array of &struct iovec from userspace 1459 * into the kernel, check that it is valid, and initialize a new 1460 * &struct iov_iter iterator to access it. 1461 * 1462 * @type: One of %READ or %WRITE. 1463 * @uvector: Pointer to the userspace array. 1464 * @nr_segs: Number of elements in userspace array. 1465 * @fast_segs: Number of elements in @iov. 1466 * @iov: (input and output parameter) Pointer to pointer to (usually small 1467 * on-stack) kernel array. 1468 * @i: Pointer to iterator that will be initialized on success. 1469 * 1470 * If the array pointed to by *@iov is large enough to hold all @nr_segs, 1471 * then this function places %NULL in *@iov on return. Otherwise, a new 1472 * array will be allocated and the result placed in *@iov. This means that 1473 * the caller may call kfree() on *@iov regardless of whether the small 1474 * on-stack array was used or not (and regardless of whether this function 1475 * returns an error or not). 1476 * 1477 * Return: 0 on success or negative error code on error. 1478 */ 1479 int import_iovec(int type, const struct iovec __user * uvector, 1480 unsigned nr_segs, unsigned fast_segs, 1481 struct iovec **iov, struct iov_iter *i) 1482 { 1483 ssize_t n; 1484 struct iovec *p; 1485 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 1486 *iov, &p); 1487 if (n < 0) { 1488 if (p != *iov) 1489 kfree(p); 1490 *iov = NULL; 1491 return n; 1492 } 1493 iov_iter_init(i, type, p, nr_segs, n); 1494 *iov = p == *iov ? NULL : p; 1495 return 0; 1496 } 1497 EXPORT_SYMBOL(import_iovec); 1498 1499 #ifdef CONFIG_COMPAT 1500 #include <linux/compat.h> 1501 1502 int compat_import_iovec(int type, const struct compat_iovec __user * uvector, 1503 unsigned nr_segs, unsigned fast_segs, 1504 struct iovec **iov, struct iov_iter *i) 1505 { 1506 ssize_t n; 1507 struct iovec *p; 1508 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 1509 *iov, &p); 1510 if (n < 0) { 1511 if (p != *iov) 1512 kfree(p); 1513 *iov = NULL; 1514 return n; 1515 } 1516 iov_iter_init(i, type, p, nr_segs, n); 1517 *iov = p == *iov ? NULL : p; 1518 return 0; 1519 } 1520 #endif 1521 1522 int import_single_range(int rw, void __user *buf, size_t len, 1523 struct iovec *iov, struct iov_iter *i) 1524 { 1525 if (len > MAX_RW_COUNT) 1526 len = MAX_RW_COUNT; 1527 if (unlikely(!access_ok(!rw, buf, len))) 1528 return -EFAULT; 1529 1530 iov->iov_base = buf; 1531 iov->iov_len = len; 1532 iov_iter_init(i, rw, iov, 1, len); 1533 return 0; 1534 } 1535 EXPORT_SYMBOL(import_single_range); 1536 1537 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes, 1538 int (*f)(struct kvec *vec, void *context), 1539 void *context) 1540 { 1541 struct kvec w; 1542 int err = -EINVAL; 1543 if (!bytes) 1544 return 0; 1545 1546 iterate_all_kinds(i, bytes, v, -EINVAL, ({ 1547 w.iov_base = kmap(v.bv_page) + v.bv_offset; 1548 w.iov_len = v.bv_len; 1549 err = f(&w, context); 1550 kunmap(v.bv_page); 1551 err;}), ({ 1552 w = v; 1553 err = f(&w, context);}) 1554 ) 1555 return err; 1556 } 1557 EXPORT_SYMBOL(iov_iter_for_each_range); 1558