iov_iter.c (cecdd52a3dd312564f81a39df08378b7b39a2654) iov_iter.c (de4f5fed3f231a8ff4790bf52975f847b95b85ea)
1// SPDX-License-Identifier: GPL-2.0-only
2#include <crypto/hash.h>
3#include <linux/export.h>
4#include <linux/bvec.h>
5#include <linux/fault-inject-usercopy.h>
6#include <linux/uio.h>
7#include <linux/pagemap.h>
8#include <linux/highmem.h>

--- 112 unchanged lines hidden (view full) ---

121 n = i->count; \
122 if (likely(n)) { \
123 if (likely(iter_is_ubuf(i))) { \
124 void __user *base; \
125 size_t len; \
126 iterate_buf(i, n, base, len, off, \
127 i->ubuf, (I)) \
128 } else if (likely(iter_is_iovec(i))) { \
1// SPDX-License-Identifier: GPL-2.0-only
2#include <crypto/hash.h>
3#include <linux/export.h>
4#include <linux/bvec.h>
5#include <linux/fault-inject-usercopy.h>
6#include <linux/uio.h>
7#include <linux/pagemap.h>
8#include <linux/highmem.h>

--- 112 unchanged lines hidden (view full) ---

121 n = i->count; \
122 if (likely(n)) { \
123 if (likely(iter_is_ubuf(i))) { \
124 void __user *base; \
125 size_t len; \
126 iterate_buf(i, n, base, len, off, \
127 i->ubuf, (I)) \
128 } else if (likely(iter_is_iovec(i))) { \
129 const struct iovec *iov = i->iov; \
129 const struct iovec *iov = iter_iov(i); \
130 void __user *base; \
131 size_t len; \
132 iterate_iovec(i, n, base, len, off, \
133 iov, (I)) \
130 void __user *base; \
131 size_t len; \
132 iterate_iovec(i, n, base, len, off, \
133 iov, (I)) \
134 i->nr_segs -= iov - i->iov; \
135 i->iov = iov; \
134 i->nr_segs -= iov - iter_iov(i); \
135 i->__iov = iov; \
136 } else if (iov_iter_is_bvec(i)) { \
137 const struct bio_vec *bvec = i->bvec; \
138 void *base; \
139 size_t len; \
140 iterate_bvec(i, n, base, len, off, \
141 bvec, (K)) \
142 i->nr_segs -= bvec - i->bvec; \
143 i->bvec = bvec; \

--- 206 unchanged lines hidden (view full) ---

350 n -= fault_in_readable(i->ubuf + i->iov_offset, n);
351 return size - n;
352 } else if (iter_is_iovec(i)) {
353 size_t count = min(size, iov_iter_count(i));
354 const struct iovec *p;
355 size_t skip;
356
357 size -= count;
136 } else if (iov_iter_is_bvec(i)) { \
137 const struct bio_vec *bvec = i->bvec; \
138 void *base; \
139 size_t len; \
140 iterate_bvec(i, n, base, len, off, \
141 bvec, (K)) \
142 i->nr_segs -= bvec - i->bvec; \
143 i->bvec = bvec; \

--- 206 unchanged lines hidden (view full) ---

350 n -= fault_in_readable(i->ubuf + i->iov_offset, n);
351 return size - n;
352 } else if (iter_is_iovec(i)) {
353 size_t count = min(size, iov_iter_count(i));
354 const struct iovec *p;
355 size_t skip;
356
357 size -= count;
358 for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
358 for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
359 size_t len = min(count, p->iov_len - skip);
360 size_t ret;
361
362 if (unlikely(!len))
363 continue;
364 ret = fault_in_readable(p->iov_base + skip, len);
365 count -= len - ret;
366 if (ret)

--- 26 unchanged lines hidden (view full) ---

393 n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n);
394 return size - n;
395 } else if (iter_is_iovec(i)) {
396 size_t count = min(size, iov_iter_count(i));
397 const struct iovec *p;
398 size_t skip;
399
400 size -= count;
359 size_t len = min(count, p->iov_len - skip);
360 size_t ret;
361
362 if (unlikely(!len))
363 continue;
364 ret = fault_in_readable(p->iov_base + skip, len);
365 count -= len - ret;
366 if (ret)

--- 26 unchanged lines hidden (view full) ---

393 n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n);
394 return size - n;
395 } else if (iter_is_iovec(i)) {
396 size_t count = min(size, iov_iter_count(i));
397 const struct iovec *p;
398 size_t skip;
399
400 size -= count;
401 for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
401 for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {
402 size_t len = min(count, p->iov_len - skip);
403 size_t ret;
404
405 if (unlikely(!len))
406 continue;
407 ret = fault_in_safe_writeable(p->iov_base + skip, len);
408 count -= len - ret;
409 if (ret)

--- 10 unchanged lines hidden (view full) ---

420 size_t count)
421{
422 WARN_ON(direction & ~(READ | WRITE));
423 *i = (struct iov_iter) {
424 .iter_type = ITER_IOVEC,
425 .nofault = false,
426 .user_backed = true,
427 .data_source = direction,
402 size_t len = min(count, p->iov_len - skip);
403 size_t ret;
404
405 if (unlikely(!len))
406 continue;
407 ret = fault_in_safe_writeable(p->iov_base + skip, len);
408 count -= len - ret;
409 if (ret)

--- 10 unchanged lines hidden (view full) ---

420 size_t count)
421{
422 WARN_ON(direction & ~(READ | WRITE));
423 *i = (struct iov_iter) {
424 .iter_type = ITER_IOVEC,
425 .nofault = false,
426 .user_backed = true,
427 .data_source = direction,
428 .iov = iov,
428 .__iov = iov,
429 .nr_segs = nr_segs,
430 .iov_offset = 0,
431 .count = count
432 };
433}
434EXPORT_SYMBOL(iov_iter_init);
435
436// returns the offset in partial buffer (if any)

--- 434 unchanged lines hidden (view full) ---

871{
872 const struct iovec *iov, *end;
873
874 if (!i->count)
875 return;
876 i->count -= size;
877
878 size += i->iov_offset; // from beginning of current segment
429 .nr_segs = nr_segs,
430 .iov_offset = 0,
431 .count = count
432 };
433}
434EXPORT_SYMBOL(iov_iter_init);
435
436// returns the offset in partial buffer (if any)

--- 434 unchanged lines hidden (view full) ---

871{
872 const struct iovec *iov, *end;
873
874 if (!i->count)
875 return;
876 i->count -= size;
877
878 size += i->iov_offset; // from beginning of current segment
879 for (iov = i->iov, end = iov + i->nr_segs; iov < end; iov++) {
879 for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) {
880 if (likely(size < iov->iov_len))
881 break;
882 size -= iov->iov_len;
883 }
884 i->iov_offset = size;
880 if (likely(size < iov->iov_len))
881 break;
882 size -= iov->iov_len;
883 }
884 i->iov_offset = size;
885 i->nr_segs -= iov - i->iov;
886 i->iov = iov;
885 i->nr_segs -= iov - iter_iov(i);
886 i->__iov = iov;
887}
888
889void iov_iter_advance(struct iov_iter *i, size_t size)
890{
891 if (unlikely(i->count < size))
892 size = i->count;
893 if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) {
894 i->iov_offset += size;

--- 58 unchanged lines hidden (view full) ---

953 if (unroll <= n) {
954 i->bvec = bvec;
955 i->iov_offset = n - unroll;
956 return;
957 }
958 unroll -= n;
959 }
960 } else { /* same logics for iovec and kvec */
887}
888
889void iov_iter_advance(struct iov_iter *i, size_t size)
890{
891 if (unlikely(i->count < size))
892 size = i->count;
893 if (likely(iter_is_ubuf(i)) || unlikely(iov_iter_is_xarray(i))) {
894 i->iov_offset += size;

--- 58 unchanged lines hidden (view full) ---

953 if (unroll <= n) {
954 i->bvec = bvec;
955 i->iov_offset = n - unroll;
956 return;
957 }
958 unroll -= n;
959 }
960 } else { /* same logics for iovec and kvec */
961 const struct iovec *iov = i->iov;
961 const struct iovec *iov = iter_iov(i);
962 while (1) {
963 size_t n = (--iov)->iov_len;
964 i->nr_segs++;
965 if (unroll <= n) {
962 while (1) {
963 size_t n = (--iov)->iov_len;
964 i->nr_segs++;
965 if (unroll <= n) {
966 i->iov = iov;
966 i->__iov = iov;
967 i->iov_offset = n - unroll;
968 return;
969 }
970 unroll -= n;
971 }
972 }
973}
974EXPORT_SYMBOL(iov_iter_revert);
975
976/*
977 * Return the count of just the current iov_iter segment.
978 */
979size_t iov_iter_single_seg_count(const struct iov_iter *i)
980{
981 if (i->nr_segs > 1) {
982 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
967 i->iov_offset = n - unroll;
968 return;
969 }
970 unroll -= n;
971 }
972 }
973}
974EXPORT_SYMBOL(iov_iter_revert);
975
976/*
977 * Return the count of just the current iov_iter segment.
978 */
979size_t iov_iter_single_seg_count(const struct iov_iter *i)
980{
981 if (i->nr_segs > 1) {
982 if (likely(iter_is_iovec(i) || iov_iter_is_kvec(i)))
983 return min(i->count, i->iov->iov_len - i->iov_offset);
983 return min(i->count, iter_iov(i)->iov_len - i->iov_offset);
984 if (iov_iter_is_bvec(i))
985 return min(i->count, i->bvec->bv_len - i->iov_offset);
986 }
987 return i->count;
988}
989EXPORT_SYMBOL(iov_iter_single_seg_count);
990
991void iov_iter_kvec(struct iov_iter *i, unsigned int direction,

--- 98 unchanged lines hidden (view full) ---

1090static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
1091 unsigned len_mask)
1092{
1093 size_t size = i->count;
1094 size_t skip = i->iov_offset;
1095 unsigned k;
1096
1097 for (k = 0; k < i->nr_segs; k++, skip = 0) {
984 if (iov_iter_is_bvec(i))
985 return min(i->count, i->bvec->bv_len - i->iov_offset);
986 }
987 return i->count;
988}
989EXPORT_SYMBOL(iov_iter_single_seg_count);
990
991void iov_iter_kvec(struct iov_iter *i, unsigned int direction,

--- 98 unchanged lines hidden (view full) ---

1090static bool iov_iter_aligned_iovec(const struct iov_iter *i, unsigned addr_mask,
1091 unsigned len_mask)
1092{
1093 size_t size = i->count;
1094 size_t skip = i->iov_offset;
1095 unsigned k;
1096
1097 for (k = 0; k < i->nr_segs; k++, skip = 0) {
1098 size_t len = i->iov[k].iov_len - skip;
1098 const struct iovec *iov = iter_iov(i) + k;
1099 size_t len = iov->iov_len - skip;
1099
1100 if (len > size)
1101 len = size;
1102 if (len & len_mask)
1103 return false;
1100
1101 if (len > size)
1102 len = size;
1103 if (len & len_mask)
1104 return false;
1104 if ((unsigned long)(i->iov[k].iov_base + skip) & addr_mask)
1105 if ((unsigned long)(iov->iov_base + skip) & addr_mask)
1105 return false;
1106
1107 size -= len;
1108 if (!size)
1109 break;
1110 }
1111 return true;
1112}

--- 76 unchanged lines hidden (view full) ---

1189static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
1190{
1191 unsigned long res = 0;
1192 size_t size = i->count;
1193 size_t skip = i->iov_offset;
1194 unsigned k;
1195
1196 for (k = 0; k < i->nr_segs; k++, skip = 0) {
1106 return false;
1107
1108 size -= len;
1109 if (!size)
1110 break;
1111 }
1112 return true;
1113}

--- 76 unchanged lines hidden (view full) ---

1190static unsigned long iov_iter_alignment_iovec(const struct iov_iter *i)
1191{
1192 unsigned long res = 0;
1193 size_t size = i->count;
1194 size_t skip = i->iov_offset;
1195 unsigned k;
1196
1197 for (k = 0; k < i->nr_segs; k++, skip = 0) {
1197 size_t len = i->iov[k].iov_len - skip;
1198 const struct iovec *iov = iter_iov(i) + k;
1199 size_t len = iov->iov_len - skip;
1198 if (len) {
1200 if (len) {
1199 res |= (unsigned long)i->iov[k].iov_base + skip;
1201 res |= (unsigned long)iov->iov_base + skip;
1200 if (len > size)
1201 len = size;
1202 res |= len;
1203 size -= len;
1204 if (!size)
1205 break;
1206 }
1207 }

--- 60 unchanged lines hidden (view full) ---

1268
1269 if (iter_is_ubuf(i))
1270 return 0;
1271
1272 if (WARN_ON(!iter_is_iovec(i)))
1273 return ~0U;
1274
1275 for (k = 0; k < i->nr_segs; k++) {
1202 if (len > size)
1203 len = size;
1204 res |= len;
1205 size -= len;
1206 if (!size)
1207 break;
1208 }
1209 }

--- 60 unchanged lines hidden (view full) ---

1270
1271 if (iter_is_ubuf(i))
1272 return 0;
1273
1274 if (WARN_ON(!iter_is_iovec(i)))
1275 return ~0U;
1276
1277 for (k = 0; k < i->nr_segs; k++) {
1276 if (i->iov[k].iov_len) {
1277 unsigned long base = (unsigned long)i->iov[k].iov_base;
1278 const struct iovec *iov = iter_iov(i) + k;
1279 if (iov->iov_len) {
1280 unsigned long base = (unsigned long)iov->iov_base;
1278 if (v) // if not the first one
1279 res |= base | v; // this start | previous end
1281 if (v) // if not the first one
1282 res |= base | v; // this start | previous end
1280 v = base + i->iov[k].iov_len;
1281 if (size <= i->iov[k].iov_len)
1283 v = base + iov->iov_len;
1284 if (size <= iov->iov_len)
1282 break;
1285 break;
1283 size -= i->iov[k].iov_len;
1286 size -= iov->iov_len;
1284 }
1285 }
1286 return res;
1287}
1288EXPORT_SYMBOL(iov_iter_gap_alignment);
1289
1290static int want_pages_array(struct page ***res, size_t size,
1291 size_t start, unsigned int maxpages)

--- 99 unchanged lines hidden (view full) ---

1391{
1392 size_t skip;
1393 long k;
1394
1395 if (iter_is_ubuf(i))
1396 return (unsigned long)i->ubuf + i->iov_offset;
1397
1398 for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {
1287 }
1288 }
1289 return res;
1290}
1291EXPORT_SYMBOL(iov_iter_gap_alignment);
1292
1293static int want_pages_array(struct page ***res, size_t size,
1294 size_t start, unsigned int maxpages)

--- 99 unchanged lines hidden (view full) ---

1394{
1395 size_t skip;
1396 long k;
1397
1398 if (iter_is_ubuf(i))
1399 return (unsigned long)i->ubuf + i->iov_offset;
1400
1401 for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {
1399 size_t len = i->iov[k].iov_len - skip;
1402 const struct iovec *iov = iter_iov(i) + k;
1403 size_t len = iov->iov_len - skip;
1400
1401 if (unlikely(!len))
1402 continue;
1403 if (*size > len)
1404 *size = len;
1404
1405 if (unlikely(!len))
1406 continue;
1407 if (*size > len)
1408 *size = len;
1405 return (unsigned long)i->iov[k].iov_base + skip;
1409 return (unsigned long)iov->iov_base + skip;
1406 }
1407 BUG(); // if it had been empty, we wouldn't get called
1408}
1409
1410/* must be done on non-empty ITER_BVEC one */
1411static struct page *first_bvec_segment(const struct iov_iter *i,
1412 size_t *size, size_t *start)
1413{

--- 195 unchanged lines hidden (view full) ---

1609EXPORT_SYMBOL(hash_and_copy_to_iter);
1610
1611static int iov_npages(const struct iov_iter *i, int maxpages)
1612{
1613 size_t skip = i->iov_offset, size = i->count;
1614 const struct iovec *p;
1615 int npages = 0;
1616
1410 }
1411 BUG(); // if it had been empty, we wouldn't get called
1412}
1413
1414/* must be done on non-empty ITER_BVEC one */
1415static struct page *first_bvec_segment(const struct iov_iter *i,
1416 size_t *size, size_t *start)
1417{

--- 195 unchanged lines hidden (view full) ---

1613EXPORT_SYMBOL(hash_and_copy_to_iter);
1614
1615static int iov_npages(const struct iov_iter *i, int maxpages)
1616{
1617 size_t skip = i->iov_offset, size = i->count;
1618 const struct iovec *p;
1619 int npages = 0;
1620
1617 for (p = i->iov; size; skip = 0, p++) {
1621 for (p = iter_iov(i); size; skip = 0, p++) {
1618 unsigned offs = offset_in_page(p->iov_base + skip);
1619 size_t len = min(p->iov_len - skip, size);
1620
1621 if (len) {
1622 size -= len;
1623 npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
1624 if (unlikely(npages > maxpages))
1625 return maxpages;

--- 60 unchanged lines hidden (view full) ---

1686 return NULL;
1687 }
1688 if (iov_iter_is_bvec(new))
1689 return new->bvec = kmemdup(new->bvec,
1690 new->nr_segs * sizeof(struct bio_vec),
1691 flags);
1692 else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
1693 /* iovec and kvec have identical layout */
1622 unsigned offs = offset_in_page(p->iov_base + skip);
1623 size_t len = min(p->iov_len - skip, size);
1624
1625 if (len) {
1626 size -= len;
1627 npages += DIV_ROUND_UP(offs + len, PAGE_SIZE);
1628 if (unlikely(npages > maxpages))
1629 return maxpages;

--- 60 unchanged lines hidden (view full) ---

1690 return NULL;
1691 }
1692 if (iov_iter_is_bvec(new))
1693 return new->bvec = kmemdup(new->bvec,
1694 new->nr_segs * sizeof(struct bio_vec),
1695 flags);
1696 else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
1697 /* iovec and kvec have identical layout */
1694 return new->iov = kmemdup(new->iov,
1698 return new->__iov = kmemdup(new->__iov,
1695 new->nr_segs * sizeof(struct iovec),
1696 flags);
1697 return NULL;
1698}
1699EXPORT_SYMBOL(dup_iter);
1700
1701static int copy_compat_iovec_from_user(struct iovec *iov,
1702 const struct iovec __user *uvec, unsigned long nr_segs)

--- 210 unchanged lines hidden (view full) ---

1913 * size, so we can just increment the iov pointer as they are unionzed.
1914 * ITER_BVEC _may_ be the same size on some archs, but on others it is
1915 * not. Be safe and handle it separately.
1916 */
1917 BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
1918 if (iov_iter_is_bvec(i))
1919 i->bvec -= state->nr_segs - i->nr_segs;
1920 else
1699 new->nr_segs * sizeof(struct iovec),
1700 flags);
1701 return NULL;
1702}
1703EXPORT_SYMBOL(dup_iter);
1704
1705static int copy_compat_iovec_from_user(struct iovec *iov,
1706 const struct iovec __user *uvec, unsigned long nr_segs)

--- 210 unchanged lines hidden (view full) ---

1917 * size, so we can just increment the iov pointer as they are unionzed.
1918 * ITER_BVEC _may_ be the same size on some archs, but on others it is
1919 * not. Be safe and handle it separately.
1920 */
1921 BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec));
1922 if (iov_iter_is_bvec(i))
1923 i->bvec -= state->nr_segs - i->nr_segs;
1924 else
1921 i->iov -= state->nr_segs - i->nr_segs;
1925 i->__iov -= state->nr_segs - i->nr_segs;
1922 i->nr_segs = state->nr_segs;
1923}
1924
1925/*
1926 * Extract a list of contiguous pages from an ITER_XARRAY iterator. This does not
1927 * get references on the pages, nor does it get a pin on them.
1928 */
1929static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i,

--- 258 unchanged lines hidden --- 		1926 i->nr_segs = state->nr_segs;
1927}
1928
1929/*
1930 * Extract a list of contiguous pages from an ITER_XARRAY iterator. This does not
1931 * get references on the pages, nor does it get a pin on them.
1932 */
1933static ssize_t iov_iter_extract_xarray_pages(struct iov_iter *i,

--- 258 unchanged lines hidden ---