iov_iter.c (93cd6fa6806cb3455e8231578840afb031606352) iov_iter.c (cbbd26b8b1a6af9c02e2b6523e12bd50cc765059)
1#include <linux/export.h>
2#include <linux/uio.h>
3#include <linux/pagemap.h>
4#include <linux/slab.h>
5#include <linux/vmalloc.h>
6#include <linux/splice.h>
7#include <net/checksum.h>
8

--- 554 unchanged lines hidden (view full) ---

563 v.bv_offset, v.bv_len),
564 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
565 )
566
567 return bytes;
568}
569EXPORT_SYMBOL(copy_from_iter);
570
1#include <linux/export.h>
2#include <linux/uio.h>
3#include <linux/pagemap.h>
4#include <linux/slab.h>
5#include <linux/vmalloc.h>
6#include <linux/splice.h>
7#include <net/checksum.h>
8

--- 554 unchanged lines hidden (view full) ---

563 v.bv_offset, v.bv_len),
564 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
565 )
566
567 return bytes;
568}
569EXPORT_SYMBOL(copy_from_iter);
570
571bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
572{
573 char *to = addr;
574 if (unlikely(i->type & ITER_PIPE)) {
575 WARN_ON(1);
576 return false;
577 }
578 if (unlikely(i->count < bytes)) \
579 return false;
580
581 iterate_all_kinds(i, bytes, v, ({
582 if (__copy_from_user((to += v.iov_len) - v.iov_len,
583 v.iov_base, v.iov_len))
584 return false;
585 0;}),
586 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
587 v.bv_offset, v.bv_len),
588 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
589 )
590
591 iov_iter_advance(i, bytes);
592 return true;
593}
594EXPORT_SYMBOL(copy_from_iter_full);
595
571size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
572{
573 char *to = addr;
574 if (unlikely(i->type & ITER_PIPE)) {
575 WARN_ON(1);
576 return 0;
577 }
578 iterate_and_advance(i, bytes, v,
579 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
580 v.iov_base, v.iov_len),
581 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
582 v.bv_offset, v.bv_len),
583 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
584 )
585
586 return bytes;
587}
588EXPORT_SYMBOL(copy_from_iter_nocache);
589
596size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
597{
598 char *to = addr;
599 if (unlikely(i->type & ITER_PIPE)) {
600 WARN_ON(1);
601 return 0;
602 }
603 iterate_and_advance(i, bytes, v,
604 __copy_from_user_nocache((to += v.iov_len) - v.iov_len,
605 v.iov_base, v.iov_len),
606 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
607 v.bv_offset, v.bv_len),
608 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
609 )
610
611 return bytes;
612}
613EXPORT_SYMBOL(copy_from_iter_nocache);
614
615bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
616{
617 char *to = addr;
618 if (unlikely(i->type & ITER_PIPE)) {
619 WARN_ON(1);
620 return false;
621 }
622 if (unlikely(i->count < bytes)) \
623 return false;
624 iterate_all_kinds(i, bytes, v, ({
625 if (__copy_from_user_nocache((to += v.iov_len) - v.iov_len,
626 v.iov_base, v.iov_len))
627 return false;
628 0;}),
629 memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page,
630 v.bv_offset, v.bv_len),
631 memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len)
632 )
633
634 iov_iter_advance(i, bytes);
635 return true;
636}
637EXPORT_SYMBOL(copy_from_iter_full_nocache);
638
590size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
591 struct iov_iter *i)
592{
593 if (i->type & (ITER_BVEC|ITER_KVEC)) {
594 void *kaddr = kmap_atomic(page);
595 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
596 kunmap_atomic(kaddr);
597 return wanted;

--- 405 unchanged lines hidden (view full) ---

1003 size_t off = 0;
1004 sum = *csum;
1005 if (unlikely(i->type & ITER_PIPE)) {
1006 WARN_ON(1);
1007 return 0;
1008 }
1009 iterate_and_advance(i, bytes, v, ({
1010 int err = 0;
639size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
640 struct iov_iter *i)
641{
642 if (i->type & (ITER_BVEC|ITER_KVEC)) {
643 void *kaddr = kmap_atomic(page);
644 size_t wanted = copy_to_iter(kaddr + offset, bytes, i);
645 kunmap_atomic(kaddr);
646 return wanted;

--- 405 unchanged lines hidden (view full) ---

1052 size_t off = 0;
1053 sum = *csum;
1054 if (unlikely(i->type & ITER_PIPE)) {
1055 WARN_ON(1);
1056 return 0;
1057 }
1058 iterate_and_advance(i, bytes, v, ({
1059 int err = 0;
1011 next = csum_and_copy_from_user(v.iov_base,
1060 next = csum_and_copy_from_user(v.iov_base,
1012 (to += v.iov_len) - v.iov_len,
1013 v.iov_len, 0, &err);
1014 if (!err) {
1015 sum = csum_block_add(sum, next, off);
1016 off += v.iov_len;
1017 }
1018 err ? v.iov_len : 0;
1019 }), ({

--- 12 unchanged lines hidden (view full) ---

1032 off += v.iov_len;
1033 })
1034 )
1035 *csum = sum;
1036 return bytes;
1037}
1038EXPORT_SYMBOL(csum_and_copy_from_iter);
1039
1061 (to += v.iov_len) - v.iov_len,
1062 v.iov_len, 0, &err);
1063 if (!err) {
1064 sum = csum_block_add(sum, next, off);
1065 off += v.iov_len;
1066 }
1067 err ? v.iov_len : 0;
1068 }), ({

--- 12 unchanged lines hidden (view full) ---

1081 off += v.iov_len;
1082 })
1083 )
1084 *csum = sum;
1085 return bytes;
1086}
1087EXPORT_SYMBOL(csum_and_copy_from_iter);
1088
1089bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,
1090 struct iov_iter *i)
1091{
1092 char *to = addr;
1093 __wsum sum, next;
1094 size_t off = 0;
1095 sum = *csum;
1096 if (unlikely(i->type & ITER_PIPE)) {
1097 WARN_ON(1);
1098 return false;
1099 }
1100 if (unlikely(i->count < bytes))
1101 return false;
1102 iterate_all_kinds(i, bytes, v, ({
1103 int err = 0;
1104 next = csum_and_copy_from_user(v.iov_base,
1105 (to += v.iov_len) - v.iov_len,
1106 v.iov_len, 0, &err);
1107 if (err)
1108 return false;
1109 sum = csum_block_add(sum, next, off);
1110 off += v.iov_len;
1111 0;
1112 }), ({
1113 char *p = kmap_atomic(v.bv_page);
1114 next = csum_partial_copy_nocheck(p + v.bv_offset,
1115 (to += v.bv_len) - v.bv_len,
1116 v.bv_len, 0);
1117 kunmap_atomic(p);
1118 sum = csum_block_add(sum, next, off);
1119 off += v.bv_len;
1120 }),({
1121 next = csum_partial_copy_nocheck(v.iov_base,
1122 (to += v.iov_len) - v.iov_len,
1123 v.iov_len, 0);
1124 sum = csum_block_add(sum, next, off);
1125 off += v.iov_len;
1126 })
1127 )
1128 *csum = sum;
1129 iov_iter_advance(i, bytes);
1130 return true;
1131}
1132EXPORT_SYMBOL(csum_and_copy_from_iter_full);
1133
1040size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
1041 struct iov_iter *i)
1042{
1043 const char *from = addr;
1044 __wsum sum, next;
1045 size_t off = 0;
1046 sum = *csum;
1047 if (unlikely(i->type & ITER_PIPE)) {
1048 WARN_ON(1); /* for now */
1049 return 0;
1050 }
1051 iterate_and_advance(i, bytes, v, ({
1052 int err = 0;
1053 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1134size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum,
1135 struct iov_iter *i)
1136{
1137 const char *from = addr;
1138 __wsum sum, next;
1139 size_t off = 0;
1140 sum = *csum;
1141 if (unlikely(i->type & ITER_PIPE)) {
1142 WARN_ON(1); /* for now */
1143 return 0;
1144 }
1145 iterate_and_advance(i, bytes, v, ({
1146 int err = 0;
1147 next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len,
1054 v.iov_base,
1148 v.iov_base,
1055 v.iov_len, 0, &err);
1056 if (!err) {
1057 sum = csum_block_add(sum, next, off);
1058 off += v.iov_len;
1059 }
1060 err ? v.iov_len : 0;
1061 }), ({
1062 char *p = kmap_atomic(v.bv_page);

--- 160 unchanged lines hidden --- 		1149 v.iov_len, 0, &err);
1150 if (!err) {
1151 sum = csum_block_add(sum, next, off);
1152 off += v.iov_len;
1153 }
1154 err ? v.iov_len : 0;
1155 }), ({
1156 char *p = kmap_atomic(v.bv_page);

--- 160 unchanged lines hidden ---