1 #include <linux/export.h> 2 #include <linux/uio.h> 3 #include <linux/pagemap.h> 4 #include <linux/slab.h> 5 #include <linux/vmalloc.h> 6 #include <net/checksum.h> 7 8 #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ 9 size_t left; \ 10 size_t wanted = n; \ 11 __p = i->iov; \ 12 __v.iov_len = min(n, __p->iov_len - skip); \ 13 if (likely(__v.iov_len)) { \ 14 __v.iov_base = __p->iov_base + skip; \ 15 left = (STEP); \ 16 __v.iov_len -= left; \ 17 skip += __v.iov_len; \ 18 n -= __v.iov_len; \ 19 } else { \ 20 left = 0; \ 21 } \ 22 while (unlikely(!left && n)) { \ 23 __p++; \ 24 __v.iov_len = min(n, __p->iov_len); \ 25 if (unlikely(!__v.iov_len)) \ 26 continue; \ 27 __v.iov_base = __p->iov_base; \ 28 left = (STEP); \ 29 __v.iov_len -= left; \ 30 skip = __v.iov_len; \ 31 n -= __v.iov_len; \ 32 } \ 33 n = wanted - n; \ 34 } 35 36 #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ 37 size_t wanted = n; \ 38 __p = i->kvec; \ 39 __v.iov_len = min(n, __p->iov_len - skip); \ 40 if (likely(__v.iov_len)) { \ 41 __v.iov_base = __p->iov_base + skip; \ 42 (void)(STEP); \ 43 skip += __v.iov_len; \ 44 n -= __v.iov_len; \ 45 } \ 46 while (unlikely(n)) { \ 47 __p++; \ 48 __v.iov_len = min(n, __p->iov_len); \ 49 if (unlikely(!__v.iov_len)) \ 50 continue; \ 51 __v.iov_base = __p->iov_base; \ 52 (void)(STEP); \ 53 skip = __v.iov_len; \ 54 n -= __v.iov_len; \ 55 } \ 56 n = wanted; \ 57 } 58 59 #define iterate_bvec(i, n, __v, __bi, skip, STEP) { \ 60 struct bvec_iter __start; \ 61 __start.bi_size = n; \ 62 __start.bi_bvec_done = skip; \ 63 __start.bi_idx = 0; \ 64 for_each_bvec(__v, i->bvec, __bi, __start) { \ 65 if (!__v.bv_len) \ 66 continue; \ 67 (void)(STEP); \ 68 } \ 69 } 70 71 #define iterate_all_kinds(i, n, v, I, B, K) { \ 72 size_t skip = i->iov_offset; \ 73 if (unlikely(i->type & ITER_BVEC)) { \ 74 struct bio_vec v; \ 75 struct bvec_iter __bi; \ 76 iterate_bvec(i, n, v, __bi, skip, (B)) \ 77 } else if (unlikely(i->type & ITER_KVEC)) { \ 78 const struct kvec *kvec; \ 79 struct kvec v; \ 80 iterate_kvec(i, n, v, kvec, skip, (K)) \ 81 } else { \ 82 const struct iovec *iov; \ 83 struct iovec v; \ 84 iterate_iovec(i, n, v, iov, skip, (I)) \ 85 } \ 86 } 87 88 #define iterate_and_advance(i, n, v, I, B, K) { \ 89 if (unlikely(i->count < n)) \ 90 n = i->count; \ 91 if (i->count) { \ 92 size_t skip = i->iov_offset; \ 93 if (unlikely(i->type & ITER_BVEC)) { \ 94 const struct bio_vec *bvec = i->bvec; \ 95 struct bio_vec v; \ 96 struct bvec_iter __bi; \ 97 iterate_bvec(i, n, v, __bi, skip, (B)) \ 98 i->bvec = __bvec_iter_bvec(i->bvec, __bi); \ 99 i->nr_segs -= i->bvec - bvec; \ 100 skip = __bi.bi_bvec_done; \ 101 } else if (unlikely(i->type & ITER_KVEC)) { \ 102 const struct kvec *kvec; \ 103 struct kvec v; \ 104 iterate_kvec(i, n, v, kvec, skip, (K)) \ 105 if (skip == kvec->iov_len) { \ 106 kvec++; \ 107 skip = 0; \ 108 } \ 109 i->nr_segs -= kvec - i->kvec; \ 110 i->kvec = kvec; \ 111 } else { \ 112 const struct iovec *iov; \ 113 struct iovec v; \ 114 iterate_iovec(i, n, v, iov, skip, (I)) \ 115 if (skip == iov->iov_len) { \ 116 iov++; \ 117 skip = 0; \ 118 } \ 119 i->nr_segs -= iov - i->iov; \ 120 i->iov = iov; \ 121 } \ 122 i->count -= n; \ 123 i->iov_offset = skip; \ 124 } \ 125 } 126 127 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, 128 struct iov_iter *i) 129 { 130 size_t skip, copy, left, wanted; 131 const struct iovec *iov; 132 char __user *buf; 133 void *kaddr, *from; 134 135 if (unlikely(bytes > i->count)) 136 bytes = i->count; 137 138 if (unlikely(!bytes)) 139 return 0; 140 141 wanted = bytes; 142 iov = i->iov; 143 skip = i->iov_offset; 144 buf = iov->iov_base + skip; 145 copy = min(bytes, iov->iov_len - skip); 146 147 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_writeable(buf, copy)) { 148 kaddr = kmap_atomic(page); 149 from = kaddr + offset; 150 151 /* first chunk, usually the only one */ 152 left = __copy_to_user_inatomic(buf, from, copy); 153 copy -= left; 154 skip += copy; 155 from += copy; 156 bytes -= copy; 157 158 while (unlikely(!left && bytes)) { 159 iov++; 160 buf = iov->iov_base; 161 copy = min(bytes, iov->iov_len); 162 left = __copy_to_user_inatomic(buf, from, copy); 163 copy -= left; 164 skip = copy; 165 from += copy; 166 bytes -= copy; 167 } 168 if (likely(!bytes)) { 169 kunmap_atomic(kaddr); 170 goto done; 171 } 172 offset = from - kaddr; 173 buf += copy; 174 kunmap_atomic(kaddr); 175 copy = min(bytes, iov->iov_len - skip); 176 } 177 /* Too bad - revert to non-atomic kmap */ 178 179 kaddr = kmap(page); 180 from = kaddr + offset; 181 left = __copy_to_user(buf, from, copy); 182 copy -= left; 183 skip += copy; 184 from += copy; 185 bytes -= copy; 186 while (unlikely(!left && bytes)) { 187 iov++; 188 buf = iov->iov_base; 189 copy = min(bytes, iov->iov_len); 190 left = __copy_to_user(buf, from, copy); 191 copy -= left; 192 skip = copy; 193 from += copy; 194 bytes -= copy; 195 } 196 kunmap(page); 197 198 done: 199 if (skip == iov->iov_len) { 200 iov++; 201 skip = 0; 202 } 203 i->count -= wanted - bytes; 204 i->nr_segs -= iov - i->iov; 205 i->iov = iov; 206 i->iov_offset = skip; 207 return wanted - bytes; 208 } 209 210 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, 211 struct iov_iter *i) 212 { 213 size_t skip, copy, left, wanted; 214 const struct iovec *iov; 215 char __user *buf; 216 void *kaddr, *to; 217 218 if (unlikely(bytes > i->count)) 219 bytes = i->count; 220 221 if (unlikely(!bytes)) 222 return 0; 223 224 wanted = bytes; 225 iov = i->iov; 226 skip = i->iov_offset; 227 buf = iov->iov_base + skip; 228 copy = min(bytes, iov->iov_len - skip); 229 230 if (IS_ENABLED(CONFIG_HIGHMEM) && !fault_in_pages_readable(buf, copy)) { 231 kaddr = kmap_atomic(page); 232 to = kaddr + offset; 233 234 /* first chunk, usually the only one */ 235 left = __copy_from_user_inatomic(to, buf, copy); 236 copy -= left; 237 skip += copy; 238 to += copy; 239 bytes -= copy; 240 241 while (unlikely(!left && bytes)) { 242 iov++; 243 buf = iov->iov_base; 244 copy = min(bytes, iov->iov_len); 245 left = __copy_from_user_inatomic(to, buf, copy); 246 copy -= left; 247 skip = copy; 248 to += copy; 249 bytes -= copy; 250 } 251 if (likely(!bytes)) { 252 kunmap_atomic(kaddr); 253 goto done; 254 } 255 offset = to - kaddr; 256 buf += copy; 257 kunmap_atomic(kaddr); 258 copy = min(bytes, iov->iov_len - skip); 259 } 260 /* Too bad - revert to non-atomic kmap */ 261 262 kaddr = kmap(page); 263 to = kaddr + offset; 264 left = __copy_from_user(to, buf, copy); 265 copy -= left; 266 skip += copy; 267 to += copy; 268 bytes -= copy; 269 while (unlikely(!left && bytes)) { 270 iov++; 271 buf = iov->iov_base; 272 copy = min(bytes, iov->iov_len); 273 left = __copy_from_user(to, buf, copy); 274 copy -= left; 275 skip = copy; 276 to += copy; 277 bytes -= copy; 278 } 279 kunmap(page); 280 281 done: 282 if (skip == iov->iov_len) { 283 iov++; 284 skip = 0; 285 } 286 i->count -= wanted - bytes; 287 i->nr_segs -= iov - i->iov; 288 i->iov = iov; 289 i->iov_offset = skip; 290 return wanted - bytes; 291 } 292 293 /* 294 * Fault in the first iovec of the given iov_iter, to a maximum length 295 * of bytes. Returns 0 on success, or non-zero if the memory could not be 296 * accessed (ie. because it is an invalid address). 297 * 298 * writev-intensive code may want this to prefault several iovecs -- that 299 * would be possible (callers must not rely on the fact that _only_ the 300 * first iovec will be faulted with the current implementation). 301 */ 302 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 303 { 304 if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 305 char __user *buf = i->iov->iov_base + i->iov_offset; 306 bytes = min(bytes, i->iov->iov_len - i->iov_offset); 307 return fault_in_pages_readable(buf, bytes); 308 } 309 return 0; 310 } 311 EXPORT_SYMBOL(iov_iter_fault_in_readable); 312 313 /* 314 * Fault in one or more iovecs of the given iov_iter, to a maximum length of 315 * bytes. For each iovec, fault in each page that constitutes the iovec. 316 * 317 * Return 0 on success, or non-zero if the memory could not be accessed (i.e. 318 * because it is an invalid address). 319 */ 320 int iov_iter_fault_in_multipages_readable(struct iov_iter *i, size_t bytes) 321 { 322 size_t skip = i->iov_offset; 323 const struct iovec *iov; 324 int err; 325 struct iovec v; 326 327 if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 328 iterate_iovec(i, bytes, v, iov, skip, ({ 329 err = fault_in_multipages_readable(v.iov_base, 330 v.iov_len); 331 if (unlikely(err)) 332 return err; 333 0;})) 334 } 335 return 0; 336 } 337 EXPORT_SYMBOL(iov_iter_fault_in_multipages_readable); 338 339 void iov_iter_init(struct iov_iter *i, int direction, 340 const struct iovec *iov, unsigned long nr_segs, 341 size_t count) 342 { 343 /* It will get better. Eventually... */ 344 if (segment_eq(get_fs(), KERNEL_DS)) { 345 direction |= ITER_KVEC; 346 i->type = direction; 347 i->kvec = (struct kvec *)iov; 348 } else { 349 i->type = direction; 350 i->iov = iov; 351 } 352 i->nr_segs = nr_segs; 353 i->iov_offset = 0; 354 i->count = count; 355 } 356 EXPORT_SYMBOL(iov_iter_init); 357 358 static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) 359 { 360 char *from = kmap_atomic(page); 361 memcpy(to, from + offset, len); 362 kunmap_atomic(from); 363 } 364 365 static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len) 366 { 367 char *to = kmap_atomic(page); 368 memcpy(to + offset, from, len); 369 kunmap_atomic(to); 370 } 371 372 static void memzero_page(struct page *page, size_t offset, size_t len) 373 { 374 char *addr = kmap_atomic(page); 375 memset(addr + offset, 0, len); 376 kunmap_atomic(addr); 377 } 378 379 size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) 380 { 381 const char *from = addr; 382 iterate_and_advance(i, bytes, v, 383 __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len, 384 v.iov_len), 385 memcpy_to_page(v.bv_page, v.bv_offset, 386 (from += v.bv_len) - v.bv_len, v.bv_len), 387 memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) 388 ) 389 390 return bytes; 391 } 392 EXPORT_SYMBOL(copy_to_iter); 393 394 size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 395 { 396 char *to = addr; 397 iterate_and_advance(i, bytes, v, 398 __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base, 399 v.iov_len), 400 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 401 v.bv_offset, v.bv_len), 402 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 403 ) 404 405 return bytes; 406 } 407 EXPORT_SYMBOL(copy_from_iter); 408 409 size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 410 { 411 char *to = addr; 412 iterate_and_advance(i, bytes, v, 413 __copy_from_user_nocache((to += v.iov_len) - v.iov_len, 414 v.iov_base, v.iov_len), 415 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 416 v.bv_offset, v.bv_len), 417 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 418 ) 419 420 return bytes; 421 } 422 EXPORT_SYMBOL(copy_from_iter_nocache); 423 424 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 425 struct iov_iter *i) 426 { 427 if (i->type & (ITER_BVEC|ITER_KVEC)) { 428 void *kaddr = kmap_atomic(page); 429 size_t wanted = copy_to_iter(kaddr + offset, bytes, i); 430 kunmap_atomic(kaddr); 431 return wanted; 432 } else 433 return copy_page_to_iter_iovec(page, offset, bytes, i); 434 } 435 EXPORT_SYMBOL(copy_page_to_iter); 436 437 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 438 struct iov_iter *i) 439 { 440 if (i->type & (ITER_BVEC|ITER_KVEC)) { 441 void *kaddr = kmap_atomic(page); 442 size_t wanted = copy_from_iter(kaddr + offset, bytes, i); 443 kunmap_atomic(kaddr); 444 return wanted; 445 } else 446 return copy_page_from_iter_iovec(page, offset, bytes, i); 447 } 448 EXPORT_SYMBOL(copy_page_from_iter); 449 450 size_t iov_iter_zero(size_t bytes, struct iov_iter *i) 451 { 452 iterate_and_advance(i, bytes, v, 453 __clear_user(v.iov_base, v.iov_len), 454 memzero_page(v.bv_page, v.bv_offset, v.bv_len), 455 memset(v.iov_base, 0, v.iov_len) 456 ) 457 458 return bytes; 459 } 460 EXPORT_SYMBOL(iov_iter_zero); 461 462 size_t iov_iter_copy_from_user_atomic(struct page *page, 463 struct iov_iter *i, unsigned long offset, size_t bytes) 464 { 465 char *kaddr = kmap_atomic(page), *p = kaddr + offset; 466 iterate_all_kinds(i, bytes, v, 467 __copy_from_user_inatomic((p += v.iov_len) - v.iov_len, 468 v.iov_base, v.iov_len), 469 memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, 470 v.bv_offset, v.bv_len), 471 memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 472 ) 473 kunmap_atomic(kaddr); 474 return bytes; 475 } 476 EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 477 478 void iov_iter_advance(struct iov_iter *i, size_t size) 479 { 480 iterate_and_advance(i, size, v, 0, 0, 0) 481 } 482 EXPORT_SYMBOL(iov_iter_advance); 483 484 /* 485 * Return the count of just the current iov_iter segment. 486 */ 487 size_t iov_iter_single_seg_count(const struct iov_iter *i) 488 { 489 if (i->nr_segs == 1) 490 return i->count; 491 else if (i->type & ITER_BVEC) 492 return min(i->count, i->bvec->bv_len - i->iov_offset); 493 else 494 return min(i->count, i->iov->iov_len - i->iov_offset); 495 } 496 EXPORT_SYMBOL(iov_iter_single_seg_count); 497 498 void iov_iter_kvec(struct iov_iter *i, int direction, 499 const struct kvec *kvec, unsigned long nr_segs, 500 size_t count) 501 { 502 BUG_ON(!(direction & ITER_KVEC)); 503 i->type = direction; 504 i->kvec = kvec; 505 i->nr_segs = nr_segs; 506 i->iov_offset = 0; 507 i->count = count; 508 } 509 EXPORT_SYMBOL(iov_iter_kvec); 510 511 void iov_iter_bvec(struct iov_iter *i, int direction, 512 const struct bio_vec *bvec, unsigned long nr_segs, 513 size_t count) 514 { 515 BUG_ON(!(direction & ITER_BVEC)); 516 i->type = direction; 517 i->bvec = bvec; 518 i->nr_segs = nr_segs; 519 i->iov_offset = 0; 520 i->count = count; 521 } 522 EXPORT_SYMBOL(iov_iter_bvec); 523 524 unsigned long iov_iter_alignment(const struct iov_iter *i) 525 { 526 unsigned long res = 0; 527 size_t size = i->count; 528 529 if (!size) 530 return 0; 531 532 iterate_all_kinds(i, size, v, 533 (res |= (unsigned long)v.iov_base | v.iov_len, 0), 534 res |= v.bv_offset | v.bv_len, 535 res |= (unsigned long)v.iov_base | v.iov_len 536 ) 537 return res; 538 } 539 EXPORT_SYMBOL(iov_iter_alignment); 540 541 unsigned long iov_iter_gap_alignment(const struct iov_iter *i) 542 { 543 unsigned long res = 0; 544 size_t size = i->count; 545 if (!size) 546 return 0; 547 548 iterate_all_kinds(i, size, v, 549 (res |= (!res ? 0 : (unsigned long)v.iov_base) | 550 (size != v.iov_len ? size : 0), 0), 551 (res |= (!res ? 0 : (unsigned long)v.bv_offset) | 552 (size != v.bv_len ? size : 0)), 553 (res |= (!res ? 0 : (unsigned long)v.iov_base) | 554 (size != v.iov_len ? size : 0)) 555 ); 556 return res; 557 } 558 EXPORT_SYMBOL(iov_iter_gap_alignment); 559 560 ssize_t iov_iter_get_pages(struct iov_iter *i, 561 struct page **pages, size_t maxsize, unsigned maxpages, 562 size_t *start) 563 { 564 if (maxsize > i->count) 565 maxsize = i->count; 566 567 if (!maxsize) 568 return 0; 569 570 iterate_all_kinds(i, maxsize, v, ({ 571 unsigned long addr = (unsigned long)v.iov_base; 572 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 573 int n; 574 int res; 575 576 if (len > maxpages * PAGE_SIZE) 577 len = maxpages * PAGE_SIZE; 578 addr &= ~(PAGE_SIZE - 1); 579 n = DIV_ROUND_UP(len, PAGE_SIZE); 580 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); 581 if (unlikely(res < 0)) 582 return res; 583 return (res == n ? len : res * PAGE_SIZE) - *start; 584 0;}),({ 585 /* can't be more than PAGE_SIZE */ 586 *start = v.bv_offset; 587 get_page(*pages = v.bv_page); 588 return v.bv_len; 589 }),({ 590 return -EFAULT; 591 }) 592 ) 593 return 0; 594 } 595 EXPORT_SYMBOL(iov_iter_get_pages); 596 597 static struct page **get_pages_array(size_t n) 598 { 599 struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL); 600 if (!p) 601 p = vmalloc(n * sizeof(struct page *)); 602 return p; 603 } 604 605 ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, 606 struct page ***pages, size_t maxsize, 607 size_t *start) 608 { 609 struct page **p; 610 611 if (maxsize > i->count) 612 maxsize = i->count; 613 614 if (!maxsize) 615 return 0; 616 617 iterate_all_kinds(i, maxsize, v, ({ 618 unsigned long addr = (unsigned long)v.iov_base; 619 size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 620 int n; 621 int res; 622 623 addr &= ~(PAGE_SIZE - 1); 624 n = DIV_ROUND_UP(len, PAGE_SIZE); 625 p = get_pages_array(n); 626 if (!p) 627 return -ENOMEM; 628 res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); 629 if (unlikely(res < 0)) { 630 kvfree(p); 631 return res; 632 } 633 *pages = p; 634 return (res == n ? len : res * PAGE_SIZE) - *start; 635 0;}),({ 636 /* can't be more than PAGE_SIZE */ 637 *start = v.bv_offset; 638 *pages = p = get_pages_array(1); 639 if (!p) 640 return -ENOMEM; 641 get_page(*p = v.bv_page); 642 return v.bv_len; 643 }),({ 644 return -EFAULT; 645 }) 646 ) 647 return 0; 648 } 649 EXPORT_SYMBOL(iov_iter_get_pages_alloc); 650 651 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, 652 struct iov_iter *i) 653 { 654 char *to = addr; 655 __wsum sum, next; 656 size_t off = 0; 657 sum = *csum; 658 iterate_and_advance(i, bytes, v, ({ 659 int err = 0; 660 next = csum_and_copy_from_user(v.iov_base, 661 (to += v.iov_len) - v.iov_len, 662 v.iov_len, 0, &err); 663 if (!err) { 664 sum = csum_block_add(sum, next, off); 665 off += v.iov_len; 666 } 667 err ? v.iov_len : 0; 668 }), ({ 669 char *p = kmap_atomic(v.bv_page); 670 next = csum_partial_copy_nocheck(p + v.bv_offset, 671 (to += v.bv_len) - v.bv_len, 672 v.bv_len, 0); 673 kunmap_atomic(p); 674 sum = csum_block_add(sum, next, off); 675 off += v.bv_len; 676 }),({ 677 next = csum_partial_copy_nocheck(v.iov_base, 678 (to += v.iov_len) - v.iov_len, 679 v.iov_len, 0); 680 sum = csum_block_add(sum, next, off); 681 off += v.iov_len; 682 }) 683 ) 684 *csum = sum; 685 return bytes; 686 } 687 EXPORT_SYMBOL(csum_and_copy_from_iter); 688 689 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, 690 struct iov_iter *i) 691 { 692 const char *from = addr; 693 __wsum sum, next; 694 size_t off = 0; 695 sum = *csum; 696 iterate_and_advance(i, bytes, v, ({ 697 int err = 0; 698 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, 699 v.iov_base, 700 v.iov_len, 0, &err); 701 if (!err) { 702 sum = csum_block_add(sum, next, off); 703 off += v.iov_len; 704 } 705 err ? v.iov_len : 0; 706 }), ({ 707 char *p = kmap_atomic(v.bv_page); 708 next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len, 709 p + v.bv_offset, 710 v.bv_len, 0); 711 kunmap_atomic(p); 712 sum = csum_block_add(sum, next, off); 713 off += v.bv_len; 714 }),({ 715 next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len, 716 v.iov_base, 717 v.iov_len, 0); 718 sum = csum_block_add(sum, next, off); 719 off += v.iov_len; 720 }) 721 ) 722 *csum = sum; 723 return bytes; 724 } 725 EXPORT_SYMBOL(csum_and_copy_to_iter); 726 727 int iov_iter_npages(const struct iov_iter *i, int maxpages) 728 { 729 size_t size = i->count; 730 int npages = 0; 731 732 if (!size) 733 return 0; 734 735 iterate_all_kinds(i, size, v, ({ 736 unsigned long p = (unsigned long)v.iov_base; 737 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 738 - p / PAGE_SIZE; 739 if (npages >= maxpages) 740 return maxpages; 741 0;}),({ 742 npages++; 743 if (npages >= maxpages) 744 return maxpages; 745 }),({ 746 unsigned long p = (unsigned long)v.iov_base; 747 npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 748 - p / PAGE_SIZE; 749 if (npages >= maxpages) 750 return maxpages; 751 }) 752 ) 753 return npages; 754 } 755 EXPORT_SYMBOL(iov_iter_npages); 756 757 const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) 758 { 759 *new = *old; 760 if (new->type & ITER_BVEC) 761 return new->bvec = kmemdup(new->bvec, 762 new->nr_segs * sizeof(struct bio_vec), 763 flags); 764 else 765 /* iovec and kvec have identical layout */ 766 return new->iov = kmemdup(new->iov, 767 new->nr_segs * sizeof(struct iovec), 768 flags); 769 } 770 EXPORT_SYMBOL(dup_iter); 771 772 int import_iovec(int type, const struct iovec __user * uvector, 773 unsigned nr_segs, unsigned fast_segs, 774 struct iovec **iov, struct iov_iter *i) 775 { 776 ssize_t n; 777 struct iovec *p; 778 n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 779 *iov, &p); 780 if (n < 0) { 781 if (p != *iov) 782 kfree(p); 783 *iov = NULL; 784 return n; 785 } 786 iov_iter_init(i, type, p, nr_segs, n); 787 *iov = p == *iov ? NULL : p; 788 return 0; 789 } 790 EXPORT_SYMBOL(import_iovec); 791 792 #ifdef CONFIG_COMPAT 793 #include <linux/compat.h> 794 795 int compat_import_iovec(int type, const struct compat_iovec __user * uvector, 796 unsigned nr_segs, unsigned fast_segs, 797 struct iovec **iov, struct iov_iter *i) 798 { 799 ssize_t n; 800 struct iovec *p; 801 n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 802 *iov, &p); 803 if (n < 0) { 804 if (p != *iov) 805 kfree(p); 806 *iov = NULL; 807 return n; 808 } 809 iov_iter_init(i, type, p, nr_segs, n); 810 *iov = p == *iov ? NULL : p; 811 return 0; 812 } 813 #endif 814 815 int import_single_range(int rw, void __user *buf, size_t len, 816 struct iovec *iov, struct iov_iter *i) 817 { 818 if (len > MAX_RW_COUNT) 819 len = MAX_RW_COUNT; 820 if (unlikely(!access_ok(!rw, buf, len))) 821 return -EFAULT; 822 823 iov->iov_base = buf; 824 iov->iov_len = len; 825 iov_iter_init(i, rw, iov, 1, len); 826 return 0; 827 } 828 EXPORT_SYMBOL(import_single_range); 829