1 #include <linux/export.h> 2 #include <linux/bvec.h> 3 #include <linux/uio.h> 4 #include <linux/pagemap.h> 5 #include <linux/slab.h> 6 #include <linux/vmalloc.h> 7 #include <linux/splice.h> 8 #include <net/checksum.h> 9 10 #define PIPE_PARANOIA /* for now */ 11 12 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ 13 size_t left; \ 14 size_t wanted = n; \ 15 __p = i->iov; \ 16 __v.iov_len = min(n, __p->iov_len - skip); \ 17 if (likely(__v.iov_len)) { \ 18 __v.iov_base = __p->iov_base + skip; \ 19 left = (STEP); \ 20 __v.iov_len -= left; \ 21 skip += __v.iov_len; \ 22 n -= __v.iov_len; \ 23 } else { \ 24 left = 0; \ 25 } \ 26 while (unlikely(!left && n)) { \ 27 __p++; \ 28 __v.iov_len = min(n, __p->iov_len); \ 29 if (unlikely(!__v.iov_len)) \ 30 continue; \ 31 __v.iov_base = __p->iov_base; \ 32 left = (STEP); \ 33 __v.iov_len -= left; \ 34 skip = __v.iov_len; \ 35 n -= __v.iov_len; \ 36 } \ 37 n = wanted - n; \ 38 } 39 40 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ 41 size_t wanted = n; \ 42 __p = i->kvec; \ 43 __v.iov_len = min(n, __p->iov_len - skip); \ 44 if (likely(__v.iov_len)) { \ 45 __v.iov_base = __p->iov_base + skip; \ 46 (void)(STEP); \ 47 skip += __v.iov_len; \ 48 n -= __v.iov_len; \ 49 } \ 50 while (unlikely(n)) { \ 51 __p++; \ 52 __v.iov_len = min(n, __p->iov_len); \ 53 if (unlikely(!__v.iov_len)) \ 54 continue; \ 55 __v.iov_base = __p->iov_base; \ 56 (void)(STEP); \ 57 skip = __v.iov_len; \ 58 n -= __v.iov_len; \ 59 } \ 60 n = wanted; \ 61 } 62 63 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ 64 struct bvec_iter __start; \ 65 __start.bi_size = n; \ 66 __start.bi_bvec_done = skip; \ 67 __start.bi_idx = 0; \ 68 for_each_bvec(__v, i->bvec, __bi, __start) { \ 69 if (!__v.bv_len) \ 70 continue; \ 71 (void)(STEP); \ 72 } \ 73 } 74 75 #define iterate_all_kinds(i, n, v, I, B, K) { \ 76 if (likely(n)) { \ 77 size_t skip = i->iov_offset; \ 78 if (unlikely(i->type & ITER_BVEC)) { \ 79 struct bio_vec v; \ 80 struct bvec_iter __bi; \ 81 iterate_bvec(i, n, v, __bi, skip, (B)) \ 82 } else if (unlikely(i->type & ITER_KVEC)) { \ 83 const struct kvec *kvec; \ 84 struct kvec v; \ 85 iterate_kvec(i, n, v, kvec, skip, (K)) \ 86 } else { \ 87 const struct iovec *iov; \ 88 struct iovec v; \ 89 iterate_iovec(i, n, v, iov, skip, (I)) \ 90 } \ 91 } \ 92 } 93 94 #define iterate_and_advance(i, n, v, I, B, K) { \ 95 if (unlikely(i->count < n)) \ 96 n = i->count; \ 97 if (i->count) { \ 98 size_t skip = i->iov_offset; \ 99 if (unlikely(i->type & ITER_BVEC)) { \ 100 const struct bio_vec *bvec = i->bvec; \ 101 struct bio_vec v; \ 102 struct bvec_iter __bi; \ 103 iterate_bvec(i, n, v, __bi, skip, (B)) \ 104 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \ 105 i->nr_segs -= i->bvec - bvec; \ 106 skip = __bi.bi_bvec_done; \ 107 } else if (unlikely(i->type & ITER_KVEC)) { \ 108 const struct kvec *kvec; \ 109 struct kvec v; \ 110 iterate_kvec(i, n, v, kvec, skip, (K)) \ 111 if (skip == kvec->iov_len) { \ 112 kvec++; \ 113 skip = 0; \ 114 } \ 115 i->nr_segs -= kvec - i->kvec; \ 116 i->kvec = kvec; \ 117 } else { \ 118 const struct iovec *iov; \ 119 struct iovec v; \ 120 iterate_iovec(i, n, v, iov, skip, (I)) \ 121 if (skip == iov->iov_len) { \ 122 iov++; \ 123 skip = 0; \ 124 } \ 125 i->nr_segs -= iov - i->iov; \ 126 i->iov = iov; \ 127 } \ 128 i->count -= n; \ 129 i->iov_offset = skip; \ 130 } \ 131 } 132 133 static int copyout(void __user *to, const void *from, size_t n) 134 { 135 if (access_ok(VERIFY_WRITE, to, n)) { 136 kasan_check_read(from, n); 137 n = raw_copy_to_user(to, from, n); 138 } 139 return n; 140 } 141 142 static int copyin(void *to, const void __user *from, size_t n) 143 { 144 if (access_ok(VERIFY_READ, from, n)) { 145 kasan_check_write(to, n); 146 n = raw_copy_from_user(to, from, n); 147 } 148 return n; 149 } 150 151 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, 152 struct iov_iter *i) 153 { 154 size_t skip, copy, left, wanted; 155 const struct iovec *iov; 156 char __user *buf; 157 void *kaddr, *from; 158 159 if (unlikely(bytes > i->count)) 160 bytes = i->count; 161 162 if (unlikely(!bytes)) 163 return 0; 164 165 might_fault(); 166 wanted = bytes; 167 iov = i->iov; 168 skip = i->iov_offset; 169 buf = iov->iov_base + skip; 170 copy = min(bytes, iov->iov_len - skip); 171 172 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) { 173 kaddr = kmap_atomic(page); 174 from = kaddr + offset; 175 176 /* first chunk, usually the only one */ 177 left = copyout(buf, from, copy); 178 copy -= left; 179 skip += copy; 180 from += copy; 181 bytes -= copy; 182 183 while (unlikely(!left && bytes)) { 184 iov++; 185 buf = iov->iov_base; 186 copy = min(bytes, iov->iov_len); 187 left = copyout(buf, from, copy); 188 copy -= left; 189 skip = copy; 190 from += copy; 191 bytes -= copy; 192 } 193 if (likely(!bytes)) { 194 kunmap_atomic(kaddr); 195 goto done; 196 } 197 offset = from - kaddr; 198 buf += copy; 199 kunmap_atomic(kaddr); 200 copy = min(bytes, iov->iov_len - skip); 201 } 202 /* Too bad - revert to non-atomic kmap */ 203 204 kaddr = kmap(page); 205 from = kaddr + offset; 206 left = copyout(buf, from, copy); 207 copy -= left; 208 skip += copy; 209 from += copy; 210 bytes -= copy; 211 while (unlikely(!left && bytes)) { 212 iov++; 213 buf = iov->iov_base; 214 copy = min(bytes, iov->iov_len); 215 left = copyout(buf, from, copy); 216 copy -= left; 217 skip = copy; 218 from += copy; 219 bytes -= copy; 220 } 221 kunmap(page); 222 223 done: 224 if (skip == iov->iov_len) { 225 iov++; 226 skip = 0; 227 } 228 i->count -= wanted - bytes; 229 i->nr_segs -= iov - i->iov; 230 i->iov = iov; 231 i->iov_offset = skip; 232 return wanted - bytes; 233 } 234 235 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, 236 struct iov_iter *i) 237 { 238 size_t skip, copy, left, wanted; 239 const struct iovec *iov; 240 char __user *buf; 241 void *kaddr, *to; 242 243 if (unlikely(bytes > i->count)) 244 bytes = i->count; 245 246 if (unlikely(!bytes)) 247 return 0; 248 249 might_fault(); 250 wanted = bytes; 251 iov = i->iov; 252 skip = i->iov_offset; 253 buf = iov->iov_base + skip; 254 copy = min(bytes, iov->iov_len - skip); 255 256 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) { 257 kaddr = kmap_atomic(page); 258 to = kaddr + offset; 259 260 /* first chunk, usually the only one */ 261 left = copyin(to, buf, copy); 262 copy -= left; 263 skip += copy; 264 to += copy; 265 bytes -= copy; 266 267 while (unlikely(!left && bytes)) { 268 iov++; 269 buf = iov->iov_base; 270 copy = min(bytes, iov->iov_len); 271 left = copyin(to, buf, copy); 272 copy -= left; 273 skip = copy; 274 to += copy; 275 bytes -= copy; 276 } 277 if (likely(!bytes)) { 278 kunmap_atomic(kaddr); 279 goto done; 280 } 281 offset = to - kaddr; 282 buf += copy; 283 kunmap_atomic(kaddr); 284 copy = min(bytes, iov->iov_len - skip); 285 } 286 /* Too bad - revert to non-atomic kmap */ 287 288 kaddr = kmap(page); 289 to = kaddr + offset; 290 left = copyin(to, buf, copy); 291 copy -= left; 292 skip += copy; 293 to += copy; 294 bytes -= copy; 295 while (unlikely(!left && bytes)) { 296 iov++; 297 buf = iov->iov_base; 298 copy = min(bytes, iov->iov_len); 299 left = copyin(to, buf, copy); 300 copy -= left; 301 skip = copy; 302 to += copy; 303 bytes -= copy; 304 } 305 kunmap(page); 306 307 done: 308 if (skip == iov->iov_len) { 309 iov++; 310 skip = 0; 311 } 312 i->count -= wanted - bytes; 313 i->nr_segs -= iov - i->iov; 314 i->iov = iov; 315 i->iov_offset = skip; 316 return wanted - bytes; 317 } 318 319 #ifdef PIPE_PARANOIA 320 static bool sanity(const struct iov_iter *i) 321 { 322 struct pipe_inode_info *pipe = i->pipe; 323 int idx = i->idx; 324 int next = pipe->curbuf + pipe->nrbufs; 325 if (i->iov_offset) { 326 struct pipe_buffer *p; 327 if (unlikely(!pipe->nrbufs)) 328 goto Bad; // pipe must be non-empty 329 if (unlikely(idx != ((next - 1) & (pipe->buffers - 1)))) 330 goto Bad; // must be at the last buffer... 331 332 p = &pipe->bufs[idx]; 333 if (unlikely(p->offset + p->len != i->iov_offset)) 334 goto Bad; // ... at the end of segment 335 } else { 336 if (idx != (next & (pipe->buffers - 1))) 337 goto Bad; // must be right after the last buffer 338 } 339 return true; 340 Bad: 341 printk(KERN_ERR "idx = %d, offset = %zd\n", i->idx, i->iov_offset); 342 printk(KERN_ERR "curbuf = %d, nrbufs = %d, buffers = %d\n", 343 pipe->curbuf, pipe->nrbufs, pipe->buffers); 344 for (idx = 0; idx < pipe->buffers; idx++) 345 printk(KERN_ERR "[%p %p %d %d]\n", 346 pipe->bufs[idx].ops, 347 pipe->bufs[idx].page, 348 pipe->bufs[idx].offset, 349 pipe->bufs[idx].len); 350 WARN_ON(1); 351 return false; 352 } 353 #else 354 #define sanity(i) true 355 #endif 356 357 static inline int next_idx(int idx, struct pipe_inode_info *pipe) 358 { 359 return (idx + 1) & (pipe->buffers - 1); 360 } 361 362 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes, 363 struct iov_iter *i) 364 { 365 struct pipe_inode_info *pipe = i->pipe; 366 struct pipe_buffer *buf; 367 size_t off; 368 int idx; 369 370 if (unlikely(bytes > i->count)) 371 bytes = i->count; 372 373 if (unlikely(!bytes)) 374 return 0; 375 376 if (!sanity(i)) 377 return 0; 378 379 off = i->iov_offset; 380 idx = i->idx; 381 buf = &pipe->bufs[idx]; 382 if (off) { 383 if (offset == off && buf->page == page) { 384 /* merge with the last one */ 385 buf->len += bytes; 386 i->iov_offset += bytes; 387 goto out; 388 } 389 idx = next_idx(idx, pipe); 390 buf = &pipe->bufs[idx]; 391 } 392 if (idx == pipe->curbuf && pipe->nrbufs) 393 return 0; 394 pipe->nrbufs++; 395 buf->ops = &page_cache_pipe_buf_ops; 396 get_page(buf->page = page); 397 buf->offset = offset; 398 buf->len = bytes; 399 i->iov_offset = offset + bytes; 400 i->idx = idx; 401 out: 402 i->count -= bytes; 403 return bytes; 404 } 405 406 /* 407 * Fault in one or more iovecs of the given iov_iter, to a maximum length of 408 * bytes. For each iovec, fault in each page that constitutes the iovec. 409 * 410 * Return 0 on success, or non-zero if the memory could not be accessed (i.e. 411 * because it is an invalid address). 412 */ 413 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 414 { 415 size_t skip = i->iov_offset; 416 const struct iovec *iov; 417 int err; 418 struct iovec v; 419 420 if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 421 iterate_iovec(i, bytes, v, iov, skip, ({ 422 err = fault_in_pages_readable(v.iov_base, v.iov_len); 423 if (unlikely(err)) 424 return err; 425 0;})) 426 } 427 return 0; 428 } 429 EXPORT_SYMBOL(iov_iter_fault_in_readable); 430 431 void iov_iter_init(struct iov_iter *i, int direction, 432 const struct iovec *iov, unsigned long nr_segs, 433 size_t count) 434 { 435 /* It will get better. Eventually... */ 436 if (uaccess_kernel()) { 437 direction |= ITER_KVEC; 438 i->type = direction; 439 i->kvec = (struct kvec *)iov; 440 } else { 441 i->type = direction; 442 i->iov = iov; 443 } 444 i->nr_segs = nr_segs; 445 i->iov_offset = 0; 446 i->count = count; 447 } 448 EXPORT_SYMBOL(iov_iter_init); 449 450 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) 451 { 452 char *from = kmap_atomic(page); 453 memcpy(to, from + offset, len); 454 kunmap_atomic(from); 455 } 456 457 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len) 458 { 459 char *to = kmap_atomic(page); 460 memcpy(to + offset, from, len); 461 kunmap_atomic(to); 462 } 463 464 static void memzero_page(struct page *page, size_t offset, size_t len) 465 { 466 char *addr = kmap_atomic(page); 467 memset(addr + offset, 0, len); 468 kunmap_atomic(addr); 469 } 470 471 static inline bool allocated(struct pipe_buffer *buf) 472 { 473 return buf->ops == &default_pipe_buf_ops; 474 } 475 476 static inline void data_start(const struct iov_iter *i, int *idxp, size_t *offp) 477 { 478 size_t off = i->iov_offset; 479 int idx = i->idx; 480 if (off && (!allocated(&i->pipe->bufs[idx]) || off == PAGE_SIZE)) { 481 idx = next_idx(idx, i->pipe); 482 off = 0; 483 } 484 *idxp = idx; 485 *offp = off; 486 } 487 488 static size_t push_pipe(struct iov_iter *i, size_t size, 489 int *idxp, size_t *offp) 490 { 491 struct pipe_inode_info *pipe = i->pipe; 492 size_t off; 493 int idx; 494 ssize_t left; 495 496 if (unlikely(size > i->count)) 497 size = i->count; 498 if (unlikely(!size)) 499 return 0; 500 501 left = size; 502 data_start(i, &idx, &off); 503 *idxp = idx; 504 *offp = off; 505 if (off) { 506 left -= PAGE_SIZE - off; 507 if (left <= 0) { 508 pipe->bufs[idx].len += size; 509 return size; 510 } 511 pipe->bufs[idx].len = PAGE_SIZE; 512 idx = next_idx(idx, pipe); 513 } 514 while (idx != pipe->curbuf || !pipe->nrbufs) { 515 struct page *page = alloc_page(GFP_USER); 516 if (!page) 517 break; 518 pipe->nrbufs++; 519 pipe->bufs[idx].ops = &default_pipe_buf_ops; 520 pipe->bufs[idx].page = page; 521 pipe->bufs[idx].offset = 0; 522 if (left <= PAGE_SIZE) { 523 pipe->bufs[idx].len = left; 524 return size; 525 } 526 pipe->bufs[idx].len = PAGE_SIZE; 527 left -= PAGE_SIZE; 528 idx = next_idx(idx, pipe); 529 } 530 return size - left; 531 } 532 533 static size_t copy_pipe_to_iter(const void *addr, size_t bytes, 534 struct iov_iter *i) 535 { 536 struct pipe_inode_info *pipe = i->pipe; 537 size_t n, off; 538 int idx; 539 540 if (!sanity(i)) 541 return 0; 542 543 bytes = n = push_pipe(i, bytes, &idx, &off); 544 if (unlikely(!n)) 545 return 0; 546 for ( ; n; idx = next_idx(idx, pipe), off = 0) { 547 size_t chunk = min_t(size_t, n, PAGE_SIZE - off); 548 memcpy_to_page(pipe->bufs[idx].page, off, addr, chunk); 549 i->idx = idx; 550 i->iov_offset = off + chunk; 551 n -= chunk; 552 addr += chunk; 553 } 554 i->count -= bytes; 555 return bytes; 556 } 557 558 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) 559 { 560 const char *from = addr; 561 if (unlikely(i->type & ITER_PIPE)) 562 return copy_pipe_to_iter(addr, bytes, i); 563 if (iter_is_iovec(i)) 564 might_fault(); 565 iterate_and_advance(i, bytes, v, 566 copyout(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len), 567 memcpy_to_page(v.bv_page, v.bv_offset, 568 (from += v.bv_len) - v.bv_len, v.bv_len), 569 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) 570 ) 571 572 return bytes; 573 } 574 EXPORT_SYMBOL(_copy_to_iter); 575 576 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 577 { 578 char *to = addr; 579 if (unlikely(i->type & ITER_PIPE)) { 580 WARN_ON(1); 581 return 0; 582 } 583 if (iter_is_iovec(i)) 584 might_fault(); 585 iterate_and_advance(i, bytes, v, 586 copyin((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), 587 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 588 v.bv_offset, v.bv_len), 589 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 590 ) 591 592 return bytes; 593 } 594 EXPORT_SYMBOL(_copy_from_iter); 595 596 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i) 597 { 598 char *to = addr; 599 if (unlikely(i->type & ITER_PIPE)) { 600 WARN_ON(1); 601 return false; 602 } 603 if (unlikely(i->count < bytes)) 604 return false; 605 606 if (iter_is_iovec(i)) 607 might_fault(); 608 iterate_all_kinds(i, bytes, v, ({ 609 if (copyin((to += v.iov_len) - v.iov_len, 610 v.iov_base, v.iov_len)) 611 return false; 612 0;}), 613 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 614 v.bv_offset, v.bv_len), 615 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 616 ) 617 618 iov_iter_advance(i, bytes); 619 return true; 620 } 621 EXPORT_SYMBOL(_copy_from_iter_full); 622 623 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 624 { 625 char *to = addr; 626 if (unlikely(i->type & ITER_PIPE)) { 627 WARN_ON(1); 628 return 0; 629 } 630 iterate_and_advance(i, bytes, v, 631 __copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, 632 v.iov_base, v.iov_len), 633 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 634 v.bv_offset, v.bv_len), 635 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 636 ) 637 638 return bytes; 639 } 640 EXPORT_SYMBOL(_copy_from_iter_nocache); 641 642 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i) 643 { 644 char *to = addr; 645 if (unlikely(i->type & ITER_PIPE)) { 646 WARN_ON(1); 647 return false; 648 } 649 if (unlikely(i->count < bytes)) 650 return false; 651 iterate_all_kinds(i, bytes, v, ({ 652 if (__copy_from_user_inatomic_nocache((to += v.iov_len) - v.iov_len, 653 v.iov_base, v.iov_len)) 654 return false; 655 0;}), 656 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 657 v.bv_offset, v.bv_len), 658 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 659 ) 660 661 iov_iter_advance(i, bytes); 662 return true; 663 } 664 EXPORT_SYMBOL(_copy_from_iter_full_nocache); 665 666 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n) 667 { 668 size_t v = n + offset; 669 if (likely(n <= v && v <= (PAGE_SIZE << compound_order(page)))) 670 return true; 671 WARN_ON(1); 672 return false; 673 } 674 675 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 676 struct iov_iter *i) 677 { 678 if (unlikely(!page_copy_sane(page, offset, bytes))) 679 return 0; 680 if (i->type & (ITER_BVEC|ITER_KVEC)) { 681 void *kaddr = kmap_atomic(page); 682 size_t wanted = copy_to_iter(kaddr + offset, bytes, i); 683 kunmap_atomic(kaddr); 684 return wanted; 685 } else if (likely(!(i->type & ITER_PIPE))) 686 return copy_page_to_iter_iovec(page, offset, bytes, i); 687 else 688 return copy_page_to_iter_pipe(page, offset, bytes, i); 689 } 690 EXPORT_SYMBOL(copy_page_to_iter); 691 692 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 693 struct iov_iter *i) 694 { 695 if (unlikely(!page_copy_sane(page, offset, bytes))) 696 return 0; 697 if (unlikely(i->type & ITER_PIPE)) { 698 WARN_ON(1); 699 return 0; 700 } 701 if (i->type & (ITER_BVEC|ITER_KVEC)) { 702 void *kaddr = kmap_atomic(page); 703 size_t wanted = _copy_from_iter(kaddr + offset, bytes, i); 704 kunmap_atomic(kaddr); 705 return wanted; 706 } else 707 return copy_page_from_iter_iovec(page, offset, bytes, i); 708 } 709 EXPORT_SYMBOL(copy_page_from_iter); 710 711 static size_t pipe_zero(size_t bytes, struct iov_iter *i) 712 { 713 struct pipe_inode_info *pipe = i->pipe; 714 size_t n, off; 715 int idx; 716 717 if (!sanity(i)) 718 return 0; 719 720 bytes = n = push_pipe(i, bytes, &idx, &off); 721 if (unlikely(!n)) 722 return 0; 723 724 for ( ; n; idx = next_idx(idx, pipe), off = 0) { 725 size_t chunk = min_t(size_t, n, PAGE_SIZE - off); 726 memzero_page(pipe->bufs[idx].page, off, chunk); 727 i->idx = idx; 728 i->iov_offset = off + chunk; 729 n -= chunk; 730 } 731 i->count -= bytes; 732 return bytes; 733 } 734 735 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) 736 { 737 if (unlikely(i->type & ITER_PIPE)) 738 return pipe_zero(bytes, i); 739 iterate_and_advance(i, bytes, v, 740 clear_user(v.iov_base, v.iov_len), 741 memzero_page(v.bv_page, v.bv_offset, v.bv_len), 742 memset(v.iov_base, 0, v.iov_len) 743 ) 744 745 return bytes; 746 } 747 EXPORT_SYMBOL(iov_iter_zero); 748 749 size_t iov_iter_copy_from_user_atomic(struct page *page, 750 struct iov_iter *i, unsigned long offset, size_t bytes) 751 { 752 char *kaddr = kmap_atomic(page), *p = kaddr + offset; 753 if (unlikely(!page_copy_sane(page, offset, bytes))) { 754 kunmap_atomic(kaddr); 755 return 0; 756 } 757 if (unlikely(i->type & ITER_PIPE)) { 758 kunmap_atomic(kaddr); 759 WARN_ON(1); 760 return 0; 761 } 762 iterate_all_kinds(i, bytes, v, 763 copyin((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len), 764 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, 765 v.bv_offset, v.bv_len), 766 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 767 ) 768 kunmap_atomic(kaddr); 769 return bytes; 770 } 771 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 772 773 static inline void pipe_truncate(struct iov_iter *i) 774 { 775 struct pipe_inode_info *pipe = i->pipe; 776 if (pipe->nrbufs) { 777 size_t off = i->iov_offset; 778 int idx = i->idx; 779 int nrbufs = (idx - pipe->curbuf) & (pipe->buffers - 1); 780 if (off) { 781 pipe->bufs[idx].len = off - pipe->bufs[idx].offset; 782 idx = next_idx(idx, pipe); 783 nrbufs++; 784 } 785 while (pipe->nrbufs > nrbufs) { 786 pipe_buf_release(pipe, &pipe->bufs[idx]); 787 idx = next_idx(idx, pipe); 788 pipe->nrbufs--; 789 } 790 } 791 } 792 793 static void pipe_advance(struct iov_iter *i, size_t size) 794 { 795 struct pipe_inode_info *pipe = i->pipe; 796 if (unlikely(i->count < size)) 797 size = i->count; 798 if (size) { 799 struct pipe_buffer *buf; 800 size_t off = i->iov_offset, left = size; 801 int idx = i->idx; 802 if (off) /* make it relative to the beginning of buffer */ 803 left += off - pipe->bufs[idx].offset; 804 while (1) { 805 buf = &pipe->bufs[idx]; 806 if (left <= buf->len) 807 break; 808 left -= buf->len; 809 idx = next_idx(idx, pipe); 810 } 811 i->idx = idx; 812 i->iov_offset = buf->offset + left; 813 } 814 i->count -= size; 815 /* ... and discard everything past that point */ 816 pipe_truncate(i); 817 } 818 819 void iov_iter_advance(struct iov_iter *i, size_t size) 820 { 821 if (unlikely(i->type & ITER_PIPE)) { 822 pipe_advance(i, size); 823 return; 824 } 825 iterate_and_advance(i, size, v, 0, 0, 0) 826 } 827 EXPORT_SYMBOL(iov_iter_advance); 828 829 void iov_iter_revert(struct iov_iter *i, size_t unroll) 830 { 831 if (!unroll) 832 return; 833 if (WARN_ON(unroll > MAX_RW_COUNT)) 834 return; 835 i->count += unroll; 836 if (unlikely(i->type & ITER_PIPE)) { 837 struct pipe_inode_info *pipe = i->pipe; 838 int idx = i->idx; 839 size_t off = i->iov_offset; 840 while (1) { 841 size_t n = off - pipe->bufs[idx].offset; 842 if (unroll < n) { 843 off -= unroll; 844 break; 845 } 846 unroll -= n; 847 if (!unroll && idx == i->start_idx) { 848 off = 0; 849 break; 850 } 851 if (!idx--) 852 idx = pipe->buffers - 1; 853 off = pipe->bufs[idx].offset + pipe->bufs[idx].len; 854 } 855 i->iov_offset = off; 856 i->idx = idx; 857 pipe_truncate(i); 858 return; 859 } 860 if (unroll <= i->iov_offset) { 861 i->iov_offset -= unroll; 862 return; 863 } 864 unroll -= i->iov_offset; 865 if (i->type & ITER_BVEC) { 866 const struct bio_vec *bvec = i->bvec; 867 while (1) { 868 size_t n = (--bvec)->bv_len; 869 i->nr_segs++; 870 if (unroll <= n) { 871 i->bvec = bvec; 872 i->iov_offset = n - unroll; 873 return; 874 } 875 unroll -= n; 876 } 877 } else { /* same logics for iovec and kvec */ 878 const struct iovec *iov = i->iov; 879 while (1) { 880 size_t n = (--iov)->iov_len; 881 i->nr_segs++; 882 if (unroll <= n) { 883 i->iov = iov; 884 i->iov_offset = n - unroll; 885 return; 886 } 887 unroll -= n; 888 } 889 } 890 } 891 EXPORT_SYMBOL(iov_iter_revert); 892 893 /* 894 * Return the count of just the current iov_iter segment. 895 */ 896 size_t iov_iter_single_seg_count(const struct iov_iter *i) 897 { 898 if (unlikely(i->type & ITER_PIPE)) 899 return i->count; // it is a silly place, anyway 900 if (i->nr_segs == 1) 901 return i->count; 902 else if (i->type & ITER_BVEC) 903 return min(i->count, i->bvec->bv_len - i->iov_offset); 904 else 905 return min(i->count, i->iov->iov_len - i->iov_offset); 906 } 907 EXPORT_SYMBOL(iov_iter_single_seg_count); 908 909 void iov_iter_kvec(struct iov_iter *i, int direction, 910 const struct kvec *kvec, unsigned long nr_segs, 911 size_t count) 912 { 913 BUG_ON(!(direction & ITER_KVEC)); 914 i->type = direction; 915 i->kvec = kvec; 916 i->nr_segs = nr_segs; 917 i->iov_offset = 0; 918 i->count = count; 919 } 920 EXPORT_SYMBOL(iov_iter_kvec); 921 922 void iov_iter_bvec(struct iov_iter *i, int direction, 923 const struct bio_vec *bvec, unsigned long nr_segs, 924 size_t count) 925 { 926 BUG_ON(!(direction & ITER_BVEC)); 927 i->type = direction; 928 i->bvec = bvec; 929 i->nr_segs = nr_segs; 930 i->iov_offset = 0; 931 i->count = count; 932 } 933 EXPORT_SYMBOL(iov_iter_bvec); 934 935 void iov_iter_pipe(struct iov_iter *i, int direction, 936 struct pipe_inode_info *pipe, 937 size_t count) 938 { 939 BUG_ON(direction != ITER_PIPE); 940 WARN_ON(pipe->nrbufs == pipe->buffers); 941 i->type = direction; 942 i->pipe = pipe; 943 i->idx = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1); 944 i->iov_offset = 0; 945 i->count = count; 946 i->start_idx = i->idx; 947 } 948 EXPORT_SYMBOL(iov_iter_pipe); 949 950 unsigned long iov_iter_alignment(const struct iov_iter *i) 951 { 952 unsigned long res = 0; 953 size_t size = i->count; 954 955 if (unlikely(i->type & ITER_PIPE)) { 956 if (size && i->iov_offset && allocated(&i->pipe->bufs[i->idx])) 957 return size | i->iov_offset; 958 return size; 959 } 960 iterate_all_kinds(i, size, v, 961 (res |= (unsigned long)v.iov_base | v.iov_len, 0), 962 res |= v.bv_offset | v.bv_len, 963 res |= (unsigned long)v.iov_base | v.iov_len 964 ) 965 return res; 966 } 967 EXPORT_SYMBOL(iov_iter_alignment); 968 969 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) 970 { 971 unsigned long res = 0; 972 size_t size = i->count; 973 974 if (unlikely(i->type & ITER_PIPE)) { 975 WARN_ON(1); 976 return ~0U; 977 } 978 979 iterate_all_kinds(i, size, v, 980 (res |= (!res ? 0 : (unsigned long)v.iov_base) | 981 (size != v.iov_len ? size : 0), 0), 982 (res |= (!res ? 0 : (unsigned long)v.bv_offset) | 983 (size != v.bv_len ? size : 0)), 984 (res |= (!res ? 0 : (unsigned long)v.iov_base) | 985 (size != v.iov_len ? size : 0)) 986 ); 987 return res; 988 } 989 EXPORT_SYMBOL(iov_iter_gap_alignment); 990 991 static inline size_t __pipe_get_pages(struct iov_iter *i, 992 size_t maxsize, 993 struct page **pages, 994 int idx, 995 size_t *start) 996 { 997 struct pipe_inode_info *pipe = i->pipe; 998 ssize_t n = push_pipe(i, maxsize, &idx, start); 999 if (!n) 1000 return -EFAULT; 1001 1002 maxsize = n; 1003 n += *start; 1004 while (n > 0) { 1005 get_page(*pages++ = pipe->bufs[idx].page); 1006 idx = next_idx(idx, pipe); 1007 n -= PAGE_SIZE; 1008 } 1009 1010 return maxsize; 1011 } 1012 1013 static ssize_t pipe_get_pages(struct iov_iter *i, 1014 struct page **pages, size_t maxsize, unsigned maxpages, 1015 size_t *start) 1016 { 1017 unsigned npages; 1018 size_t capacity; 1019 int idx; 1020 1021 if (!maxsize) 1022 return 0; 1023 1024 if (!sanity(i)) 1025 return -EFAULT; 1026 1027 data_start(i, &idx, start); 1028 /* some of this one + all after this one */ 1029 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; 1030 capacity = min(npages,maxpages) * PAGE_SIZE - *start; 1031 1032 return __pipe_get_pages(i, min(maxsize, capacity), pages, idx, start); 1033 } 1034 1035 ssize_t iov_iter_get_pages(struct iov_iter *i, 1036 struct page **pages, size_t maxsize, unsigned maxpages, 1037 size_t *start) 1038 { 1039 if (maxsize > i->count) 1040 maxsize = i->count; 1041 1042 if (unlikely(i->type & ITER_PIPE)) 1043 return pipe_get_pages(i, pages, maxsize, maxpages, start); 1044 iterate_all_kinds(i, maxsize, v, ({ 1045 unsigned long addr = (unsigned long)v.iov_base; 1046 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 1047 int n; 1048 int res; 1049 1050 if (len > maxpages * PAGE_SIZE) 1051 len = maxpages * PAGE_SIZE; 1052 addr &= ~(PAGE_SIZE - 1); 1053 n = DIV_ROUND_UP(len, PAGE_SIZE); 1054 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); 1055 if (unlikely(res < 0)) 1056 return res; 1057 return (res == n ? len : res * PAGE_SIZE) - *start; 1058 0;}),({ 1059 /* can't be more than PAGE_SIZE */ 1060 *start = v.bv_offset; 1061 get_page(*pages = v.bv_page); 1062 return v.bv_len; 1063 }),({ 1064 return -EFAULT; 1065 }) 1066 ) 1067 return 0; 1068 } 1069 EXPORT_SYMBOL(iov_iter_get_pages); 1070 1071 static struct page **get_pages_array(size_t n) 1072 { 1073 return kvmalloc_array(n, sizeof(struct page *), GFP_KERNEL); 1074 } 1075 1076 static ssize_t pipe_get_pages_alloc(struct iov_iter *i, 1077 struct page ***pages, size_t maxsize, 1078 size_t *start) 1079 { 1080 struct page **p; 1081 size_t n; 1082 int idx; 1083 int npages; 1084 1085 if (!maxsize) 1086 return 0; 1087 1088 if (!sanity(i)) 1089 return -EFAULT; 1090 1091 data_start(i, &idx, start); 1092 /* some of this one + all after this one */ 1093 npages = ((i->pipe->curbuf - idx - 1) & (i->pipe->buffers - 1)) + 1; 1094 n = npages * PAGE_SIZE - *start; 1095 if (maxsize > n) 1096 maxsize = n; 1097 else 1098 npages = DIV_ROUND_UP(maxsize + *start, PAGE_SIZE); 1099 p = get_pages_array(npages); 1100 if (!p) 1101 return -ENOMEM; 1102 n = __pipe_get_pages(i, maxsize, p, idx, start); 1103 if (n > 0) 1104 *pages = p; 1105 else 1106 kvfree(p); 1107 return n; 1108 } 1109 1110 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, 1111 struct page ***pages, size_t maxsize, 1112 size_t *start) 1113 { 1114 struct page **p; 1115 1116 if (maxsize > i->count) 1117 maxsize = i->count; 1118 1119 if (unlikely(i->type & ITER_PIPE)) 1120 return pipe_get_pages_alloc(i, pages, maxsize, start); 1121 iterate_all_kinds(i, maxsize, v, ({ 1122 unsigned long addr = (unsigned long)v.iov_base; 1123 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 1124 int n; 1125 int res; 1126 1127 addr &= ~(PAGE_SIZE - 1); 1128 n = DIV_ROUND_UP(len, PAGE_SIZE); 1129 p = get_pages_array(n); 1130 if (!p) 1131 return -ENOMEM; 1132 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); 1133 if (unlikely(res < 0)) { 1134 kvfree(p); 1135 return res; 1136 } 1137 *pages = p; 1138 return (res == n ? len : res * PAGE_SIZE) - *start; 1139 0;}),({ 1140 /* can't be more than PAGE_SIZE */ 1141 *start = v.bv_offset; 1142 *pages = p = get_pages_array(1); 1143 if (!p) 1144 return -ENOMEM; 1145 get_page(*p = v.bv_page); 1146 return v.bv_len; 1147 }),({ 1148 return -EFAULT; 1149 }) 1150 ) 1151 return 0; 1152 } 1153 EXPORT_SYMBOL(iov_iter_get_pages_alloc); 1154 1155 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, 1156 struct iov_iter *i) 1157 { 1158 char *to = addr; 1159 __wsum sum, next; 1160 size_t off = 0; 1161 sum = *csum; 1162 if (unlikely(i->type & ITER_PIPE)) { 1163 WARN_ON(1); 1164 return 0; 1165 } 1166 iterate_and_advance(i, bytes, v, ({ 1167 int err = 0; 1168 next = csum_and_copy_from_user(v.iov_base, 1169 (to += v.iov_len) - v.iov_len, 1170 v.iov_len, 0, &err); 1171 if (!err) { 1172 sum = csum_block_add(sum, next, off); 1173 off += v.iov_len; 1174 } 1175 err ? v.iov_len : 0; 1176 }), ({ 1177 char *p = kmap_atomic(v.bv_page); 1178 next = csum_partial_copy_nocheck(p + v.bv_offset, 1179 (to += v.bv_len) - v.bv_len, 1180 v.bv_len, 0); 1181 kunmap_atomic(p); 1182 sum = csum_block_add(sum, next, off); 1183 off += v.bv_len; 1184 }),({ 1185 next = csum_partial_copy_nocheck(v.iov_base, 1186 (to += v.iov_len) - v.iov_len, 1187 v.iov_len, 0); 1188 sum = csum_block_add(sum, next, off); 1189 off += v.iov_len; 1190 }) 1191 ) 1192 *csum = sum; 1193 return bytes; 1194 } 1195 EXPORT_SYMBOL(csum_and_copy_from_iter); 1196 1197 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, 1198 struct iov_iter *i) 1199 { 1200 char *to = addr; 1201 __wsum sum, next; 1202 size_t off = 0; 1203 sum = *csum; 1204 if (unlikely(i->type & ITER_PIPE)) { 1205 WARN_ON(1); 1206 return false; 1207 } 1208 if (unlikely(i->count < bytes)) 1209 return false; 1210 iterate_all_kinds(i, bytes, v, ({ 1211 int err = 0; 1212 next = csum_and_copy_from_user(v.iov_base, 1213 (to += v.iov_len) - v.iov_len, 1214 v.iov_len, 0, &err); 1215 if (err) 1216 return false; 1217 sum = csum_block_add(sum, next, off); 1218 off += v.iov_len; 1219 0; 1220 }), ({ 1221 char *p = kmap_atomic(v.bv_page); 1222 next = csum_partial_copy_nocheck(p + v.bv_offset, 1223 (to += v.bv_len) - v.bv_len, 1224 v.bv_len, 0); 1225 kunmap_atomic(p); 1226 sum = csum_block_add(sum, next, off); 1227 off += v.bv_len; 1228 }),({ 1229 next = csum_partial_copy_nocheck(v.iov_base, 1230 (to += v.iov_len) - v.iov_len, 1231 v.iov_len, 0); 1232 sum = csum_block_add(sum, next, off); 1233 off += v.iov_len; 1234 }) 1235 ) 1236 *csum = sum; 1237 iov_iter_advance(i, bytes); 1238 return true; 1239 } 1240 EXPORT_SYMBOL(csum_and_copy_from_iter_full); 1241 1242 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, 1243 struct iov_iter *i) 1244 { 1245 const char *from = addr; 1246 __wsum sum, next; 1247 size_t off = 0; 1248 sum = *csum; 1249 if (unlikely(i->type & ITER_PIPE)) { 1250 WARN_ON(1); /* for now */ 1251 return 0; 1252 } 1253 iterate_and_advance(i, bytes, v, ({ 1254 int err = 0; 1255 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, 1256 v.iov_base, 1257 v.iov_len, 0, &err); 1258 if (!err) { 1259 sum = csum_block_add(sum, next, off); 1260 off += v.iov_len; 1261 } 1262 err ? v.iov_len : 0; 1263 }), ({ 1264 char *p = kmap_atomic(v.bv_page); 1265 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len, 1266 p + v.bv_offset, 1267 v.bv_len, 0); 1268 kunmap_atomic(p); 1269 sum = csum_block_add(sum, next, off); 1270 off += v.bv_len; 1271 }),({ 1272 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len, 1273 v.iov_base, 1274 v.iov_len, 0); 1275 sum = csum_block_add(sum, next, off); 1276 off += v.iov_len; 1277 }) 1278 ) 1279 *csum = sum; 1280 return bytes; 1281 } 1282 EXPORT_SYMBOL(csum_and_copy_to_iter); 1283 1284 int iov_iter_npages(const struct iov_iter *i, int maxpages) 1285 { 1286 size_t size = i->count; 1287 int npages = 0; 1288 1289 if (!size) 1290 return 0; 1291 1292 if (unlikely(i->type & ITER_PIPE)) { 1293 struct pipe_inode_info *pipe = i->pipe; 1294 size_t off; 1295 int idx; 1296 1297 if (!sanity(i)) 1298 return 0; 1299 1300 data_start(i, &idx, &off); 1301 /* some of this one + all after this one */ 1302 npages = ((pipe->curbuf - idx - 1) & (pipe->buffers - 1)) + 1; 1303 if (npages >= maxpages) 1304 return maxpages; 1305 } else iterate_all_kinds(i, size, v, ({ 1306 unsigned long p = (unsigned long)v.iov_base; 1307 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 1308 - p / PAGE_SIZE; 1309 if (npages >= maxpages) 1310 return maxpages; 1311 0;}),({ 1312 npages++; 1313 if (npages >= maxpages) 1314 return maxpages; 1315 }),({ 1316 unsigned long p = (unsigned long)v.iov_base; 1317 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 1318 - p / PAGE_SIZE; 1319 if (npages >= maxpages) 1320 return maxpages; 1321 }) 1322 ) 1323 return npages; 1324 } 1325 EXPORT_SYMBOL(iov_iter_npages); 1326 1327 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) 1328 { 1329 *new = *old; 1330 if (unlikely(new->type & ITER_PIPE)) { 1331 WARN_ON(1); 1332 return NULL; 1333 } 1334 if (new->type & ITER_BVEC) 1335 return new->bvec = kmemdup(new->bvec, 1336 new->nr_segs * sizeof(struct bio_vec), 1337 flags); 1338 else 1339 /* iovec and kvec have identical layout */ 1340 return new->iov = kmemdup(new->iov, 1341 new->nr_segs * sizeof(struct iovec), 1342 flags); 1343 } 1344 EXPORT_SYMBOL(dup_iter); 1345 1346 /** 1347 * import_iovec() - Copy an array of &struct iovec from userspace 1348 * into the kernel, check that it is valid, and initialize a new 1349 * &struct iov_iter iterator to access it. 1350 * 1351 * @type: One of %READ or %WRITE. 1352 * @uvector: Pointer to the userspace array. 1353 * @nr_segs: Number of elements in userspace array. 1354 * @fast_segs: Number of elements in @iov. 1355 * @iov: (input and output parameter) Pointer to pointer to (usually small 1356 * on-stack) kernel array. 1357 * @i: Pointer to iterator that will be initialized on success. 1358 * 1359 * If the array pointed to by *@iov is large enough to hold all @nr_segs, 1360 * then this function places %NULL in *@iov on return. Otherwise, a new 1361 * array will be allocated and the result placed in *@iov. This means that 1362 * the caller may call kfree() on *@iov regardless of whether the small 1363 * on-stack array was used or not (and regardless of whether this function 1364 * returns an error or not). 1365 * 1366 * Return: 0 on success or negative error code on error. 1367 */ 1368 int import_iovec(int type, const struct iovec __user * uvector, 1369 unsigned nr_segs, unsigned fast_segs, 1370 struct iovec **iov, struct iov_iter *i) 1371 { 1372 ssize_t n; 1373 struct iovec *p; 1374 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 1375 *iov, &p); 1376 if (n < 0) { 1377 if (p != *iov) 1378 kfree(p); 1379 *iov = NULL; 1380 return n; 1381 } 1382 iov_iter_init(i, type, p, nr_segs, n); 1383 *iov = p == *iov ? NULL : p; 1384 return 0; 1385 } 1386 EXPORT_SYMBOL(import_iovec); 1387 1388 #ifdef CONFIG_COMPAT 1389 #include <linux/compat.h> 1390 1391 int compat_import_iovec(int type, const struct compat_iovec __user * uvector, 1392 unsigned nr_segs, unsigned fast_segs, 1393 struct iovec **iov, struct iov_iter *i) 1394 { 1395 ssize_t n; 1396 struct iovec *p; 1397 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 1398 *iov, &p); 1399 if (n < 0) { 1400 if (p != *iov) 1401 kfree(p); 1402 *iov = NULL; 1403 return n; 1404 } 1405 iov_iter_init(i, type, p, nr_segs, n); 1406 *iov = p == *iov ? NULL : p; 1407 return 0; 1408 } 1409 #endif 1410 1411 int import_single_range(int rw, void __user *buf, size_t len, 1412 struct iovec *iov, struct iov_iter *i) 1413 { 1414 if (len > MAX_RW_COUNT) 1415 len = MAX_RW_COUNT; 1416 if (unlikely(!access_ok(!rw, buf, len))) 1417 return -EFAULT; 1418 1419 iov->iov_base = buf; 1420 iov->iov_len = len; 1421 iov_iter_init(i, rw, iov, 1, len); 1422 return 0; 1423 } 1424 EXPORT_SYMBOL(import_single_range); 1425