1d879cb83SAl Viro #include <linux/export.h> 2d879cb83SAl Viro #include <linux/uio.h> 3d879cb83SAl Viro #include <linux/pagemap.h> 4d879cb83SAl Viro #include <linux/slab.h> 5d879cb83SAl Viro #include <linux/vmalloc.h> 6d879cb83SAl Viro #include <net/checksum.h> 7d879cb83SAl Viro 8d879cb83SAl Viro #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ 9d879cb83SAl Viro size_t left; \ 10d879cb83SAl Viro size_t wanted = n; \ 11d879cb83SAl Viro __p = i->iov; \ 12d879cb83SAl Viro __v.iov_len = min(n, __p->iov_len - skip); \ 13d879cb83SAl Viro if (likely(__v.iov_len)) { \ 14d879cb83SAl Viro __v.iov_base = __p->iov_base + skip; \ 15d879cb83SAl Viro left = (STEP); \ 16d879cb83SAl Viro __v.iov_len -= left; \ 17d879cb83SAl Viro skip += __v.iov_len; \ 18d879cb83SAl Viro n -= __v.iov_len; \ 19d879cb83SAl Viro } else { \ 20d879cb83SAl Viro left = 0; \ 21d879cb83SAl Viro } \ 22d879cb83SAl Viro while (unlikely(!left && n)) { \ 23d879cb83SAl Viro __p++; \ 24d879cb83SAl Viro __v.iov_len = min(n, __p->iov_len); \ 25d879cb83SAl Viro if (unlikely(!__v.iov_len)) \ 26d879cb83SAl Viro continue; \ 27d879cb83SAl Viro __v.iov_base = __p->iov_base; \ 28d879cb83SAl Viro left = (STEP); \ 29d879cb83SAl Viro __v.iov_len -= left; \ 30d879cb83SAl Viro skip = __v.iov_len; \ 31d879cb83SAl Viro n -= __v.iov_len; \ 32d879cb83SAl Viro } \ 33d879cb83SAl Viro n = wanted - n; \ 34d879cb83SAl Viro } 35d879cb83SAl Viro 36d879cb83SAl Viro #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ 37d879cb83SAl Viro size_t wanted = n; \ 38d879cb83SAl Viro __p = i->kvec; \ 39d879cb83SAl Viro __v.iov_len = min(n, __p->iov_len - skip); \ 40d879cb83SAl Viro if (likely(__v.iov_len)) { \ 41d879cb83SAl Viro __v.iov_base = __p->iov_base + skip; \ 42d879cb83SAl Viro (void)(STEP); \ 43d879cb83SAl Viro skip += __v.iov_len; \ 44d879cb83SAl Viro n -= __v.iov_len; \ 45d879cb83SAl Viro } \ 46d879cb83SAl Viro while (unlikely(n)) { \ 47d879cb83SAl Viro __p++; \ 48d879cb83SAl Viro __v.iov_len = min(n, __p->iov_len); \ 49d879cb83SAl Viro if (unlikely(!__v.iov_len)) \ 50d879cb83SAl Viro continue; \ 51d879cb83SAl Viro __v.iov_base = __p->iov_base; \ 52d879cb83SAl Viro (void)(STEP); \ 53d879cb83SAl Viro skip = __v.iov_len; \ 54d879cb83SAl Viro n -= __v.iov_len; \ 55d879cb83SAl Viro } \ 56d879cb83SAl Viro n = wanted; \ 57d879cb83SAl Viro } 58d879cb83SAl Viro 59d879cb83SAl Viro #define iterate_bvec(i, n, __v, __p, skip, STEP) { \ 60d879cb83SAl Viro size_t wanted = n; \ 61d879cb83SAl Viro __p = i->bvec; \ 62d879cb83SAl Viro __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \ 63d879cb83SAl Viro if (likely(__v.bv_len)) { \ 64d879cb83SAl Viro __v.bv_page = __p->bv_page; \ 65d879cb83SAl Viro __v.bv_offset = __p->bv_offset + skip; \ 66d879cb83SAl Viro (void)(STEP); \ 67d879cb83SAl Viro skip += __v.bv_len; \ 68d879cb83SAl Viro n -= __v.bv_len; \ 69d879cb83SAl Viro } \ 70d879cb83SAl Viro while (unlikely(n)) { \ 71d879cb83SAl Viro __p++; \ 72d879cb83SAl Viro __v.bv_len = min_t(size_t, n, __p->bv_len); \ 73d879cb83SAl Viro if (unlikely(!__v.bv_len)) \ 74d879cb83SAl Viro continue; \ 75d879cb83SAl Viro __v.bv_page = __p->bv_page; \ 76d879cb83SAl Viro __v.bv_offset = __p->bv_offset; \ 77d879cb83SAl Viro (void)(STEP); \ 78d879cb83SAl Viro skip = __v.bv_len; \ 79d879cb83SAl Viro n -= __v.bv_len; \ 80d879cb83SAl Viro } \ 81d879cb83SAl Viro n = wanted; \ 82d879cb83SAl Viro } 83d879cb83SAl Viro 84d879cb83SAl Viro #define iterate_all_kinds(i, n, v, I, B, K) { \ 85d879cb83SAl Viro size_t skip = i->iov_offset; \ 86d879cb83SAl Viro if (unlikely(i->type & ITER_BVEC)) { \ 87d879cb83SAl Viro const struct bio_vec *bvec; \ 88d879cb83SAl Viro struct bio_vec v; \ 89d879cb83SAl Viro iterate_bvec(i, n, v, bvec, skip, (B)) \ 90d879cb83SAl Viro } else if (unlikely(i->type & ITER_KVEC)) { \ 91d879cb83SAl Viro const struct kvec *kvec; \ 92d879cb83SAl Viro struct kvec v; \ 93d879cb83SAl Viro iterate_kvec(i, n, v, kvec, skip, (K)) \ 94d879cb83SAl Viro } else { \ 95d879cb83SAl Viro const struct iovec *iov; \ 96d879cb83SAl Viro struct iovec v; \ 97d879cb83SAl Viro iterate_iovec(i, n, v, iov, skip, (I)) \ 98d879cb83SAl Viro } \ 99d879cb83SAl Viro } 100d879cb83SAl Viro 101d879cb83SAl Viro #define iterate_and_advance(i, n, v, I, B, K) { \ 102dd254f5aSAl Viro if (unlikely(i->count < n)) \ 103dd254f5aSAl Viro n = i->count; \ 104dd254f5aSAl Viro if (n) { \ 105d879cb83SAl Viro size_t skip = i->iov_offset; \ 106d879cb83SAl Viro if (unlikely(i->type & ITER_BVEC)) { \ 107d879cb83SAl Viro const struct bio_vec *bvec; \ 108d879cb83SAl Viro struct bio_vec v; \ 109d879cb83SAl Viro iterate_bvec(i, n, v, bvec, skip, (B)) \ 110d879cb83SAl Viro if (skip == bvec->bv_len) { \ 111d879cb83SAl Viro bvec++; \ 112d879cb83SAl Viro skip = 0; \ 113d879cb83SAl Viro } \ 114d879cb83SAl Viro i->nr_segs -= bvec - i->bvec; \ 115d879cb83SAl Viro i->bvec = bvec; \ 116d879cb83SAl Viro } else if (unlikely(i->type & ITER_KVEC)) { \ 117d879cb83SAl Viro const struct kvec *kvec; \ 118d879cb83SAl Viro struct kvec v; \ 119d879cb83SAl Viro iterate_kvec(i, n, v, kvec, skip, (K)) \ 120d879cb83SAl Viro if (skip == kvec->iov_len) { \ 121d879cb83SAl Viro kvec++; \ 122d879cb83SAl Viro skip = 0; \ 123d879cb83SAl Viro } \ 124d879cb83SAl Viro i->nr_segs -= kvec - i->kvec; \ 125d879cb83SAl Viro i->kvec = kvec; \ 126d879cb83SAl Viro } else { \ 127d879cb83SAl Viro const struct iovec *iov; \ 128d879cb83SAl Viro struct iovec v; \ 129d879cb83SAl Viro iterate_iovec(i, n, v, iov, skip, (I)) \ 130d879cb83SAl Viro if (skip == iov->iov_len) { \ 131d879cb83SAl Viro iov++; \ 132d879cb83SAl Viro skip = 0; \ 133d879cb83SAl Viro } \ 134d879cb83SAl Viro i->nr_segs -= iov - i->iov; \ 135d879cb83SAl Viro i->iov = iov; \ 136d879cb83SAl Viro } \ 137d879cb83SAl Viro i->count -= n; \ 138d879cb83SAl Viro i->iov_offset = skip; \ 139dd254f5aSAl Viro } \ 140d879cb83SAl Viro } 141d879cb83SAl Viro 142d879cb83SAl Viro static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, 143d879cb83SAl Viro struct iov_iter *i) 144d879cb83SAl Viro { 145d879cb83SAl Viro size_t skip, copy, left, wanted; 146d879cb83SAl Viro const struct iovec *iov; 147d879cb83SAl Viro char __user *buf; 148d879cb83SAl Viro void *kaddr, *from; 149d879cb83SAl Viro 150d879cb83SAl Viro if (unlikely(bytes > i->count)) 151d879cb83SAl Viro bytes = i->count; 152d879cb83SAl Viro 153d879cb83SAl Viro if (unlikely(!bytes)) 154d879cb83SAl Viro return 0; 155d879cb83SAl Viro 156d879cb83SAl Viro wanted = bytes; 157d879cb83SAl Viro iov = i->iov; 158d879cb83SAl Viro skip = i->iov_offset; 159d879cb83SAl Viro buf = iov->iov_base + skip; 160d879cb83SAl Viro copy = min(bytes, iov->iov_len - skip); 161d879cb83SAl Viro 162d879cb83SAl Viro if (!fault_in_pages_writeable(buf, copy)) { 163d879cb83SAl Viro kaddr = kmap_atomic(page); 164d879cb83SAl Viro from = kaddr + offset; 165d879cb83SAl Viro 166d879cb83SAl Viro /* first chunk, usually the only one */ 167d879cb83SAl Viro left = __copy_to_user_inatomic(buf, from, copy); 168d879cb83SAl Viro copy -= left; 169d879cb83SAl Viro skip += copy; 170d879cb83SAl Viro from += copy; 171d879cb83SAl Viro bytes -= copy; 172d879cb83SAl Viro 173d879cb83SAl Viro while (unlikely(!left && bytes)) { 174d879cb83SAl Viro iov++; 175d879cb83SAl Viro buf = iov->iov_base; 176d879cb83SAl Viro copy = min(bytes, iov->iov_len); 177d879cb83SAl Viro left = __copy_to_user_inatomic(buf, from, copy); 178d879cb83SAl Viro copy -= left; 179d879cb83SAl Viro skip = copy; 180d879cb83SAl Viro from += copy; 181d879cb83SAl Viro bytes -= copy; 182d879cb83SAl Viro } 183d879cb83SAl Viro if (likely(!bytes)) { 184d879cb83SAl Viro kunmap_atomic(kaddr); 185d879cb83SAl Viro goto done; 186d879cb83SAl Viro } 187d879cb83SAl Viro offset = from - kaddr; 188d879cb83SAl Viro buf += copy; 189d879cb83SAl Viro kunmap_atomic(kaddr); 190d879cb83SAl Viro copy = min(bytes, iov->iov_len - skip); 191d879cb83SAl Viro } 192d879cb83SAl Viro /* Too bad - revert to non-atomic kmap */ 193d879cb83SAl Viro kaddr = kmap(page); 194d879cb83SAl Viro from = kaddr + offset; 195d879cb83SAl Viro left = __copy_to_user(buf, from, copy); 196d879cb83SAl Viro copy -= left; 197d879cb83SAl Viro skip += copy; 198d879cb83SAl Viro from += copy; 199d879cb83SAl Viro bytes -= copy; 200d879cb83SAl Viro while (unlikely(!left && bytes)) { 201d879cb83SAl Viro iov++; 202d879cb83SAl Viro buf = iov->iov_base; 203d879cb83SAl Viro copy = min(bytes, iov->iov_len); 204d879cb83SAl Viro left = __copy_to_user(buf, from, copy); 205d879cb83SAl Viro copy -= left; 206d879cb83SAl Viro skip = copy; 207d879cb83SAl Viro from += copy; 208d879cb83SAl Viro bytes -= copy; 209d879cb83SAl Viro } 210d879cb83SAl Viro kunmap(page); 211d879cb83SAl Viro done: 212d879cb83SAl Viro if (skip == iov->iov_len) { 213d879cb83SAl Viro iov++; 214d879cb83SAl Viro skip = 0; 215d879cb83SAl Viro } 216d879cb83SAl Viro i->count -= wanted - bytes; 217d879cb83SAl Viro i->nr_segs -= iov - i->iov; 218d879cb83SAl Viro i->iov = iov; 219d879cb83SAl Viro i->iov_offset = skip; 220d879cb83SAl Viro return wanted - bytes; 221d879cb83SAl Viro } 222d879cb83SAl Viro 223d879cb83SAl Viro static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, 224d879cb83SAl Viro struct iov_iter *i) 225d879cb83SAl Viro { 226d879cb83SAl Viro size_t skip, copy, left, wanted; 227d879cb83SAl Viro const struct iovec *iov; 228d879cb83SAl Viro char __user *buf; 229d879cb83SAl Viro void *kaddr, *to; 230d879cb83SAl Viro 231d879cb83SAl Viro if (unlikely(bytes > i->count)) 232d879cb83SAl Viro bytes = i->count; 233d879cb83SAl Viro 234d879cb83SAl Viro if (unlikely(!bytes)) 235d879cb83SAl Viro return 0; 236d879cb83SAl Viro 237d879cb83SAl Viro wanted = bytes; 238d879cb83SAl Viro iov = i->iov; 239d879cb83SAl Viro skip = i->iov_offset; 240d879cb83SAl Viro buf = iov->iov_base + skip; 241d879cb83SAl Viro copy = min(bytes, iov->iov_len - skip); 242d879cb83SAl Viro 243d879cb83SAl Viro if (!fault_in_pages_readable(buf, copy)) { 244d879cb83SAl Viro kaddr = kmap_atomic(page); 245d879cb83SAl Viro to = kaddr + offset; 246d879cb83SAl Viro 247d879cb83SAl Viro /* first chunk, usually the only one */ 248d879cb83SAl Viro left = __copy_from_user_inatomic(to, buf, copy); 249d879cb83SAl Viro copy -= left; 250d879cb83SAl Viro skip += copy; 251d879cb83SAl Viro to += copy; 252d879cb83SAl Viro bytes -= copy; 253d879cb83SAl Viro 254d879cb83SAl Viro while (unlikely(!left && bytes)) { 255d879cb83SAl Viro iov++; 256d879cb83SAl Viro buf = iov->iov_base; 257d879cb83SAl Viro copy = min(bytes, iov->iov_len); 258d879cb83SAl Viro left = __copy_from_user_inatomic(to, buf, copy); 259d879cb83SAl Viro copy -= left; 260d879cb83SAl Viro skip = copy; 261d879cb83SAl Viro to += copy; 262d879cb83SAl Viro bytes -= copy; 263d879cb83SAl Viro } 264d879cb83SAl Viro if (likely(!bytes)) { 265d879cb83SAl Viro kunmap_atomic(kaddr); 266d879cb83SAl Viro goto done; 267d879cb83SAl Viro } 268d879cb83SAl Viro offset = to - kaddr; 269d879cb83SAl Viro buf += copy; 270d879cb83SAl Viro kunmap_atomic(kaddr); 271d879cb83SAl Viro copy = min(bytes, iov->iov_len - skip); 272d879cb83SAl Viro } 273d879cb83SAl Viro /* Too bad - revert to non-atomic kmap */ 274d879cb83SAl Viro kaddr = kmap(page); 275d879cb83SAl Viro to = kaddr + offset; 276d879cb83SAl Viro left = __copy_from_user(to, buf, copy); 277d879cb83SAl Viro copy -= left; 278d879cb83SAl Viro skip += copy; 279d879cb83SAl Viro to += copy; 280d879cb83SAl Viro bytes -= copy; 281d879cb83SAl Viro while (unlikely(!left && bytes)) { 282d879cb83SAl Viro iov++; 283d879cb83SAl Viro buf = iov->iov_base; 284d879cb83SAl Viro copy = min(bytes, iov->iov_len); 285d879cb83SAl Viro left = __copy_from_user(to, buf, copy); 286d879cb83SAl Viro copy -= left; 287d879cb83SAl Viro skip = copy; 288d879cb83SAl Viro to += copy; 289d879cb83SAl Viro bytes -= copy; 290d879cb83SAl Viro } 291d879cb83SAl Viro kunmap(page); 292d879cb83SAl Viro done: 293d879cb83SAl Viro if (skip == iov->iov_len) { 294d879cb83SAl Viro iov++; 295d879cb83SAl Viro skip = 0; 296d879cb83SAl Viro } 297d879cb83SAl Viro i->count -= wanted - bytes; 298d879cb83SAl Viro i->nr_segs -= iov - i->iov; 299d879cb83SAl Viro i->iov = iov; 300d879cb83SAl Viro i->iov_offset = skip; 301d879cb83SAl Viro return wanted - bytes; 302d879cb83SAl Viro } 303d879cb83SAl Viro 304d879cb83SAl Viro /* 305d879cb83SAl Viro * Fault in the first iovec of the given iov_iter, to a maximum length 306d879cb83SAl Viro * of bytes. Returns 0 on success, or non-zero if the memory could not be 307d879cb83SAl Viro * accessed (ie. because it is an invalid address). 308d879cb83SAl Viro * 309d879cb83SAl Viro * writev-intensive code may want this to prefault several iovecs -- that 310d879cb83SAl Viro * would be possible (callers must not rely on the fact that _only_ the 311d879cb83SAl Viro * first iovec will be faulted with the current implementation). 312d879cb83SAl Viro */ 313d879cb83SAl Viro int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 314d879cb83SAl Viro { 315d879cb83SAl Viro if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 316d879cb83SAl Viro char __user *buf = i->iov->iov_base + i->iov_offset; 317d879cb83SAl Viro bytes = min(bytes, i->iov->iov_len - i->iov_offset); 318d879cb83SAl Viro return fault_in_pages_readable(buf, bytes); 319d879cb83SAl Viro } 320d879cb83SAl Viro return 0; 321d879cb83SAl Viro } 322d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_fault_in_readable); 323d879cb83SAl Viro 324171a0203SAnton Altaparmakov /* 325171a0203SAnton Altaparmakov * Fault in one or more iovecs of the given iov_iter, to a maximum length of 326171a0203SAnton Altaparmakov * bytes. For each iovec, fault in each page that constitutes the iovec. 327171a0203SAnton Altaparmakov * 328171a0203SAnton Altaparmakov * Return 0 on success, or non-zero if the memory could not be accessed (i.e. 329171a0203SAnton Altaparmakov * because it is an invalid address). 330171a0203SAnton Altaparmakov */ 331171a0203SAnton Altaparmakov int iov_iter_fault_in_multipages_readable(struct iov_iter *i, size_t bytes) 332171a0203SAnton Altaparmakov { 333171a0203SAnton Altaparmakov size_t skip = i->iov_offset; 334171a0203SAnton Altaparmakov const struct iovec *iov; 335171a0203SAnton Altaparmakov int err; 336171a0203SAnton Altaparmakov struct iovec v; 337171a0203SAnton Altaparmakov 338171a0203SAnton Altaparmakov if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 339171a0203SAnton Altaparmakov iterate_iovec(i, bytes, v, iov, skip, ({ 340171a0203SAnton Altaparmakov err = fault_in_multipages_readable(v.iov_base, 341171a0203SAnton Altaparmakov v.iov_len); 342171a0203SAnton Altaparmakov if (unlikely(err)) 343171a0203SAnton Altaparmakov return err; 344171a0203SAnton Altaparmakov 0;})) 345171a0203SAnton Altaparmakov } 346171a0203SAnton Altaparmakov return 0; 347171a0203SAnton Altaparmakov } 348171a0203SAnton Altaparmakov EXPORT_SYMBOL(iov_iter_fault_in_multipages_readable); 349171a0203SAnton Altaparmakov 350d879cb83SAl Viro void iov_iter_init(struct iov_iter *i, int direction, 351d879cb83SAl Viro const struct iovec *iov, unsigned long nr_segs, 352d879cb83SAl Viro size_t count) 353d879cb83SAl Viro { 354d879cb83SAl Viro /* It will get better. Eventually... */ 355d879cb83SAl Viro if (segment_eq(get_fs(), KERNEL_DS)) { 356d879cb83SAl Viro direction |= ITER_KVEC; 357d879cb83SAl Viro i->type = direction; 358d879cb83SAl Viro i->kvec = (struct kvec *)iov; 359d879cb83SAl Viro } else { 360d879cb83SAl Viro i->type = direction; 361d879cb83SAl Viro i->iov = iov; 362d879cb83SAl Viro } 363d879cb83SAl Viro i->nr_segs = nr_segs; 364d879cb83SAl Viro i->iov_offset = 0; 365d879cb83SAl Viro i->count = count; 366d879cb83SAl Viro } 367d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_init); 368d879cb83SAl Viro 369d879cb83SAl Viro static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) 370d879cb83SAl Viro { 371d879cb83SAl Viro char *from = kmap_atomic(page); 372d879cb83SAl Viro memcpy(to, from + offset, len); 373d879cb83SAl Viro kunmap_atomic(from); 374d879cb83SAl Viro } 375d879cb83SAl Viro 37636f7a8a4SAl Viro static void memcpy_to_page(struct page *page, size_t offset, const char *from, size_t len) 377d879cb83SAl Viro { 378d879cb83SAl Viro char *to = kmap_atomic(page); 379d879cb83SAl Viro memcpy(to + offset, from, len); 380d879cb83SAl Viro kunmap_atomic(to); 381d879cb83SAl Viro } 382d879cb83SAl Viro 383d879cb83SAl Viro static void memzero_page(struct page *page, size_t offset, size_t len) 384d879cb83SAl Viro { 385d879cb83SAl Viro char *addr = kmap_atomic(page); 386d879cb83SAl Viro memset(addr + offset, 0, len); 387d879cb83SAl Viro kunmap_atomic(addr); 388d879cb83SAl Viro } 389d879cb83SAl Viro 39036f7a8a4SAl Viro size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i) 391d879cb83SAl Viro { 39236f7a8a4SAl Viro const char *from = addr; 393d879cb83SAl Viro iterate_and_advance(i, bytes, v, 394d879cb83SAl Viro __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len, 395d879cb83SAl Viro v.iov_len), 396d879cb83SAl Viro memcpy_to_page(v.bv_page, v.bv_offset, 397d879cb83SAl Viro (from += v.bv_len) - v.bv_len, v.bv_len), 398d879cb83SAl Viro memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) 399d879cb83SAl Viro ) 400d879cb83SAl Viro 401d879cb83SAl Viro return bytes; 402d879cb83SAl Viro } 403d879cb83SAl Viro EXPORT_SYMBOL(copy_to_iter); 404d879cb83SAl Viro 405d879cb83SAl Viro size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 406d879cb83SAl Viro { 407d879cb83SAl Viro char *to = addr; 408d879cb83SAl Viro iterate_and_advance(i, bytes, v, 409d879cb83SAl Viro __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base, 410d879cb83SAl Viro v.iov_len), 411d879cb83SAl Viro memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 412d879cb83SAl Viro v.bv_offset, v.bv_len), 413d879cb83SAl Viro memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 414d879cb83SAl Viro ) 415d879cb83SAl Viro 416d879cb83SAl Viro return bytes; 417d879cb83SAl Viro } 418d879cb83SAl Viro EXPORT_SYMBOL(copy_from_iter); 419d879cb83SAl Viro 420d879cb83SAl Viro size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 421d879cb83SAl Viro { 422d879cb83SAl Viro char *to = addr; 423d879cb83SAl Viro iterate_and_advance(i, bytes, v, 424d879cb83SAl Viro __copy_from_user_nocache((to += v.iov_len) - v.iov_len, 425d879cb83SAl Viro v.iov_base, v.iov_len), 426d879cb83SAl Viro memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 427d879cb83SAl Viro v.bv_offset, v.bv_len), 428d879cb83SAl Viro memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 429d879cb83SAl Viro ) 430d879cb83SAl Viro 431d879cb83SAl Viro return bytes; 432d879cb83SAl Viro } 433d879cb83SAl Viro EXPORT_SYMBOL(copy_from_iter_nocache); 434d879cb83SAl Viro 435d879cb83SAl Viro size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 436d879cb83SAl Viro struct iov_iter *i) 437d879cb83SAl Viro { 438d879cb83SAl Viro if (i->type & (ITER_BVEC|ITER_KVEC)) { 439d879cb83SAl Viro void *kaddr = kmap_atomic(page); 440d879cb83SAl Viro size_t wanted = copy_to_iter(kaddr + offset, bytes, i); 441d879cb83SAl Viro kunmap_atomic(kaddr); 442d879cb83SAl Viro return wanted; 443d879cb83SAl Viro } else 444d879cb83SAl Viro return copy_page_to_iter_iovec(page, offset, bytes, i); 445d879cb83SAl Viro } 446d879cb83SAl Viro EXPORT_SYMBOL(copy_page_to_iter); 447d879cb83SAl Viro 448d879cb83SAl Viro size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 449d879cb83SAl Viro struct iov_iter *i) 450d879cb83SAl Viro { 451d879cb83SAl Viro if (i->type & (ITER_BVEC|ITER_KVEC)) { 452d879cb83SAl Viro void *kaddr = kmap_atomic(page); 453d879cb83SAl Viro size_t wanted = copy_from_iter(kaddr + offset, bytes, i); 454d879cb83SAl Viro kunmap_atomic(kaddr); 455d879cb83SAl Viro return wanted; 456d879cb83SAl Viro } else 457d879cb83SAl Viro return copy_page_from_iter_iovec(page, offset, bytes, i); 458d879cb83SAl Viro } 459d879cb83SAl Viro EXPORT_SYMBOL(copy_page_from_iter); 460d879cb83SAl Viro 461d879cb83SAl Viro size_t iov_iter_zero(size_t bytes, struct iov_iter *i) 462d879cb83SAl Viro { 463d879cb83SAl Viro iterate_and_advance(i, bytes, v, 464d879cb83SAl Viro __clear_user(v.iov_base, v.iov_len), 465d879cb83SAl Viro memzero_page(v.bv_page, v.bv_offset, v.bv_len), 466d879cb83SAl Viro memset(v.iov_base, 0, v.iov_len) 467d879cb83SAl Viro ) 468d879cb83SAl Viro 469d879cb83SAl Viro return bytes; 470d879cb83SAl Viro } 471d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_zero); 472d879cb83SAl Viro 473d879cb83SAl Viro size_t iov_iter_copy_from_user_atomic(struct page *page, 474d879cb83SAl Viro struct iov_iter *i, unsigned long offset, size_t bytes) 475d879cb83SAl Viro { 476d879cb83SAl Viro char *kaddr = kmap_atomic(page), *p = kaddr + offset; 477d879cb83SAl Viro iterate_all_kinds(i, bytes, v, 478d879cb83SAl Viro __copy_from_user_inatomic((p += v.iov_len) - v.iov_len, 479d879cb83SAl Viro v.iov_base, v.iov_len), 480d879cb83SAl Viro memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, 481d879cb83SAl Viro v.bv_offset, v.bv_len), 482d879cb83SAl Viro memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 483d879cb83SAl Viro ) 484d879cb83SAl Viro kunmap_atomic(kaddr); 485d879cb83SAl Viro return bytes; 486d879cb83SAl Viro } 487d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 488d879cb83SAl Viro 489d879cb83SAl Viro void iov_iter_advance(struct iov_iter *i, size_t size) 490d879cb83SAl Viro { 491d879cb83SAl Viro iterate_and_advance(i, size, v, 0, 0, 0) 492d879cb83SAl Viro } 493d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_advance); 494d879cb83SAl Viro 495d879cb83SAl Viro /* 496d879cb83SAl Viro * Return the count of just the current iov_iter segment. 497d879cb83SAl Viro */ 498d879cb83SAl Viro size_t iov_iter_single_seg_count(const struct iov_iter *i) 499d879cb83SAl Viro { 500d879cb83SAl Viro if (i->nr_segs == 1) 501d879cb83SAl Viro return i->count; 502d879cb83SAl Viro else if (i->type & ITER_BVEC) 503d879cb83SAl Viro return min(i->count, i->bvec->bv_len - i->iov_offset); 504d879cb83SAl Viro else 505d879cb83SAl Viro return min(i->count, i->iov->iov_len - i->iov_offset); 506d879cb83SAl Viro } 507d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_single_seg_count); 508d879cb83SAl Viro 509d879cb83SAl Viro void iov_iter_kvec(struct iov_iter *i, int direction, 510d879cb83SAl Viro const struct kvec *kvec, unsigned long nr_segs, 511d879cb83SAl Viro size_t count) 512d879cb83SAl Viro { 513d879cb83SAl Viro BUG_ON(!(direction & ITER_KVEC)); 514d879cb83SAl Viro i->type = direction; 515d879cb83SAl Viro i->kvec = kvec; 516d879cb83SAl Viro i->nr_segs = nr_segs; 517d879cb83SAl Viro i->iov_offset = 0; 518d879cb83SAl Viro i->count = count; 519d879cb83SAl Viro } 520d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_kvec); 521d879cb83SAl Viro 522d879cb83SAl Viro void iov_iter_bvec(struct iov_iter *i, int direction, 523d879cb83SAl Viro const struct bio_vec *bvec, unsigned long nr_segs, 524d879cb83SAl Viro size_t count) 525d879cb83SAl Viro { 526d879cb83SAl Viro BUG_ON(!(direction & ITER_BVEC)); 527d879cb83SAl Viro i->type = direction; 528d879cb83SAl Viro i->bvec = bvec; 529d879cb83SAl Viro i->nr_segs = nr_segs; 530d879cb83SAl Viro i->iov_offset = 0; 531d879cb83SAl Viro i->count = count; 532d879cb83SAl Viro } 533d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_bvec); 534d879cb83SAl Viro 535d879cb83SAl Viro unsigned long iov_iter_alignment(const struct iov_iter *i) 536d879cb83SAl Viro { 537d879cb83SAl Viro unsigned long res = 0; 538d879cb83SAl Viro size_t size = i->count; 539d879cb83SAl Viro 540d879cb83SAl Viro if (!size) 541d879cb83SAl Viro return 0; 542d879cb83SAl Viro 543d879cb83SAl Viro iterate_all_kinds(i, size, v, 544d879cb83SAl Viro (res |= (unsigned long)v.iov_base | v.iov_len, 0), 545d879cb83SAl Viro res |= v.bv_offset | v.bv_len, 546d879cb83SAl Viro res |= (unsigned long)v.iov_base | v.iov_len 547d879cb83SAl Viro ) 548d879cb83SAl Viro return res; 549d879cb83SAl Viro } 550d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_alignment); 551d879cb83SAl Viro 552d879cb83SAl Viro ssize_t iov_iter_get_pages(struct iov_iter *i, 553d879cb83SAl Viro struct page **pages, size_t maxsize, unsigned maxpages, 554d879cb83SAl Viro size_t *start) 555d879cb83SAl Viro { 556d879cb83SAl Viro if (maxsize > i->count) 557d879cb83SAl Viro maxsize = i->count; 558d879cb83SAl Viro 559d879cb83SAl Viro if (!maxsize) 560d879cb83SAl Viro return 0; 561d879cb83SAl Viro 562d879cb83SAl Viro iterate_all_kinds(i, maxsize, v, ({ 563d879cb83SAl Viro unsigned long addr = (unsigned long)v.iov_base; 564d879cb83SAl Viro size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 565d879cb83SAl Viro int n; 566d879cb83SAl Viro int res; 567d879cb83SAl Viro 568d879cb83SAl Viro if (len > maxpages * PAGE_SIZE) 569d879cb83SAl Viro len = maxpages * PAGE_SIZE; 570d879cb83SAl Viro addr &= ~(PAGE_SIZE - 1); 571d879cb83SAl Viro n = DIV_ROUND_UP(len, PAGE_SIZE); 572d879cb83SAl Viro res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); 573d879cb83SAl Viro if (unlikely(res < 0)) 574d879cb83SAl Viro return res; 575d879cb83SAl Viro return (res == n ? len : res * PAGE_SIZE) - *start; 576d879cb83SAl Viro 0;}),({ 577d879cb83SAl Viro /* can't be more than PAGE_SIZE */ 578d879cb83SAl Viro *start = v.bv_offset; 579d879cb83SAl Viro get_page(*pages = v.bv_page); 580d879cb83SAl Viro return v.bv_len; 581d879cb83SAl Viro }),({ 582d879cb83SAl Viro return -EFAULT; 583d879cb83SAl Viro }) 584d879cb83SAl Viro ) 585d879cb83SAl Viro return 0; 586d879cb83SAl Viro } 587d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_get_pages); 588d879cb83SAl Viro 589d879cb83SAl Viro static struct page **get_pages_array(size_t n) 590d879cb83SAl Viro { 591d879cb83SAl Viro struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL); 592d879cb83SAl Viro if (!p) 593d879cb83SAl Viro p = vmalloc(n * sizeof(struct page *)); 594d879cb83SAl Viro return p; 595d879cb83SAl Viro } 596d879cb83SAl Viro 597d879cb83SAl Viro ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, 598d879cb83SAl Viro struct page ***pages, size_t maxsize, 599d879cb83SAl Viro size_t *start) 600d879cb83SAl Viro { 601d879cb83SAl Viro struct page **p; 602d879cb83SAl Viro 603d879cb83SAl Viro if (maxsize > i->count) 604d879cb83SAl Viro maxsize = i->count; 605d879cb83SAl Viro 606d879cb83SAl Viro if (!maxsize) 607d879cb83SAl Viro return 0; 608d879cb83SAl Viro 609d879cb83SAl Viro iterate_all_kinds(i, maxsize, v, ({ 610d879cb83SAl Viro unsigned long addr = (unsigned long)v.iov_base; 611d879cb83SAl Viro size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 612d879cb83SAl Viro int n; 613d879cb83SAl Viro int res; 614d879cb83SAl Viro 615d879cb83SAl Viro addr &= ~(PAGE_SIZE - 1); 616d879cb83SAl Viro n = DIV_ROUND_UP(len, PAGE_SIZE); 617d879cb83SAl Viro p = get_pages_array(n); 618d879cb83SAl Viro if (!p) 619d879cb83SAl Viro return -ENOMEM; 620d879cb83SAl Viro res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); 621d879cb83SAl Viro if (unlikely(res < 0)) { 622d879cb83SAl Viro kvfree(p); 623d879cb83SAl Viro return res; 624d879cb83SAl Viro } 625d879cb83SAl Viro *pages = p; 626d879cb83SAl Viro return (res == n ? len : res * PAGE_SIZE) - *start; 627d879cb83SAl Viro 0;}),({ 628d879cb83SAl Viro /* can't be more than PAGE_SIZE */ 629d879cb83SAl Viro *start = v.bv_offset; 630d879cb83SAl Viro *pages = p = get_pages_array(1); 631d879cb83SAl Viro if (!p) 632d879cb83SAl Viro return -ENOMEM; 633d879cb83SAl Viro get_page(*p = v.bv_page); 634d879cb83SAl Viro return v.bv_len; 635d879cb83SAl Viro }),({ 636d879cb83SAl Viro return -EFAULT; 637d879cb83SAl Viro }) 638d879cb83SAl Viro ) 639d879cb83SAl Viro return 0; 640d879cb83SAl Viro } 641d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_get_pages_alloc); 642d879cb83SAl Viro 643d879cb83SAl Viro size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, 644d879cb83SAl Viro struct iov_iter *i) 645d879cb83SAl Viro { 646d879cb83SAl Viro char *to = addr; 647d879cb83SAl Viro __wsum sum, next; 648d879cb83SAl Viro size_t off = 0; 649d879cb83SAl Viro sum = *csum; 650d879cb83SAl Viro iterate_and_advance(i, bytes, v, ({ 651d879cb83SAl Viro int err = 0; 652d879cb83SAl Viro next = csum_and_copy_from_user(v.iov_base, 653d879cb83SAl Viro (to += v.iov_len) - v.iov_len, 654d879cb83SAl Viro v.iov_len, 0, &err); 655d879cb83SAl Viro if (!err) { 656d879cb83SAl Viro sum = csum_block_add(sum, next, off); 657d879cb83SAl Viro off += v.iov_len; 658d879cb83SAl Viro } 659d879cb83SAl Viro err ? v.iov_len : 0; 660d879cb83SAl Viro }), ({ 661d879cb83SAl Viro char *p = kmap_atomic(v.bv_page); 662d879cb83SAl Viro next = csum_partial_copy_nocheck(p + v.bv_offset, 663d879cb83SAl Viro (to += v.bv_len) - v.bv_len, 664d879cb83SAl Viro v.bv_len, 0); 665d879cb83SAl Viro kunmap_atomic(p); 666d879cb83SAl Viro sum = csum_block_add(sum, next, off); 667d879cb83SAl Viro off += v.bv_len; 668d879cb83SAl Viro }),({ 669d879cb83SAl Viro next = csum_partial_copy_nocheck(v.iov_base, 670d879cb83SAl Viro (to += v.iov_len) - v.iov_len, 671d879cb83SAl Viro v.iov_len, 0); 672d879cb83SAl Viro sum = csum_block_add(sum, next, off); 673d879cb83SAl Viro off += v.iov_len; 674d879cb83SAl Viro }) 675d879cb83SAl Viro ) 676d879cb83SAl Viro *csum = sum; 677d879cb83SAl Viro return bytes; 678d879cb83SAl Viro } 679d879cb83SAl Viro EXPORT_SYMBOL(csum_and_copy_from_iter); 680d879cb83SAl Viro 68136f7a8a4SAl Viro size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, 682d879cb83SAl Viro struct iov_iter *i) 683d879cb83SAl Viro { 68436f7a8a4SAl Viro const char *from = addr; 685d879cb83SAl Viro __wsum sum, next; 686d879cb83SAl Viro size_t off = 0; 687d879cb83SAl Viro sum = *csum; 688d879cb83SAl Viro iterate_and_advance(i, bytes, v, ({ 689d879cb83SAl Viro int err = 0; 690d879cb83SAl Viro next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, 691d879cb83SAl Viro v.iov_base, 692d879cb83SAl Viro v.iov_len, 0, &err); 693d879cb83SAl Viro if (!err) { 694d879cb83SAl Viro sum = csum_block_add(sum, next, off); 695d879cb83SAl Viro off += v.iov_len; 696d879cb83SAl Viro } 697d879cb83SAl Viro err ? v.iov_len : 0; 698d879cb83SAl Viro }), ({ 699d879cb83SAl Viro char *p = kmap_atomic(v.bv_page); 700d879cb83SAl Viro next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len, 701d879cb83SAl Viro p + v.bv_offset, 702d879cb83SAl Viro v.bv_len, 0); 703d879cb83SAl Viro kunmap_atomic(p); 704d879cb83SAl Viro sum = csum_block_add(sum, next, off); 705d879cb83SAl Viro off += v.bv_len; 706d879cb83SAl Viro }),({ 707d879cb83SAl Viro next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len, 708d879cb83SAl Viro v.iov_base, 709d879cb83SAl Viro v.iov_len, 0); 710d879cb83SAl Viro sum = csum_block_add(sum, next, off); 711d879cb83SAl Viro off += v.iov_len; 712d879cb83SAl Viro }) 713d879cb83SAl Viro ) 714d879cb83SAl Viro *csum = sum; 715d879cb83SAl Viro return bytes; 716d879cb83SAl Viro } 717d879cb83SAl Viro EXPORT_SYMBOL(csum_and_copy_to_iter); 718d879cb83SAl Viro 719d879cb83SAl Viro int iov_iter_npages(const struct iov_iter *i, int maxpages) 720d879cb83SAl Viro { 721d879cb83SAl Viro size_t size = i->count; 722d879cb83SAl Viro int npages = 0; 723d879cb83SAl Viro 724d879cb83SAl Viro if (!size) 725d879cb83SAl Viro return 0; 726d879cb83SAl Viro 727d879cb83SAl Viro iterate_all_kinds(i, size, v, ({ 728d879cb83SAl Viro unsigned long p = (unsigned long)v.iov_base; 729d879cb83SAl Viro npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 730d879cb83SAl Viro - p / PAGE_SIZE; 731d879cb83SAl Viro if (npages >= maxpages) 732d879cb83SAl Viro return maxpages; 733d879cb83SAl Viro 0;}),({ 734d879cb83SAl Viro npages++; 735d879cb83SAl Viro if (npages >= maxpages) 736d879cb83SAl Viro return maxpages; 737d879cb83SAl Viro }),({ 738d879cb83SAl Viro unsigned long p = (unsigned long)v.iov_base; 739d879cb83SAl Viro npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 740d879cb83SAl Viro - p / PAGE_SIZE; 741d879cb83SAl Viro if (npages >= maxpages) 742d879cb83SAl Viro return maxpages; 743d879cb83SAl Viro }) 744d879cb83SAl Viro ) 745d879cb83SAl Viro return npages; 746d879cb83SAl Viro } 747d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_npages); 748d879cb83SAl Viro 749d879cb83SAl Viro const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) 750d879cb83SAl Viro { 751d879cb83SAl Viro *new = *old; 752d879cb83SAl Viro if (new->type & ITER_BVEC) 753d879cb83SAl Viro return new->bvec = kmemdup(new->bvec, 754d879cb83SAl Viro new->nr_segs * sizeof(struct bio_vec), 755d879cb83SAl Viro flags); 756d879cb83SAl Viro else 757d879cb83SAl Viro /* iovec and kvec have identical layout */ 758d879cb83SAl Viro return new->iov = kmemdup(new->iov, 759d879cb83SAl Viro new->nr_segs * sizeof(struct iovec), 760d879cb83SAl Viro flags); 761d879cb83SAl Viro } 762d879cb83SAl Viro EXPORT_SYMBOL(dup_iter); 763bc917be8SAl Viro 764bc917be8SAl Viro int import_iovec(int type, const struct iovec __user * uvector, 765bc917be8SAl Viro unsigned nr_segs, unsigned fast_segs, 766bc917be8SAl Viro struct iovec **iov, struct iov_iter *i) 767bc917be8SAl Viro { 768bc917be8SAl Viro ssize_t n; 769bc917be8SAl Viro struct iovec *p; 770bc917be8SAl Viro n = rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 771bc917be8SAl Viro *iov, &p); 772bc917be8SAl Viro if (n < 0) { 773bc917be8SAl Viro if (p != *iov) 774bc917be8SAl Viro kfree(p); 775bc917be8SAl Viro *iov = NULL; 776bc917be8SAl Viro return n; 777bc917be8SAl Viro } 778bc917be8SAl Viro iov_iter_init(i, type, p, nr_segs, n); 779bc917be8SAl Viro *iov = p == *iov ? NULL : p; 780bc917be8SAl Viro return 0; 781bc917be8SAl Viro } 782bc917be8SAl Viro EXPORT_SYMBOL(import_iovec); 783bc917be8SAl Viro 784bc917be8SAl Viro #ifdef CONFIG_COMPAT 785bc917be8SAl Viro #include <linux/compat.h> 786bc917be8SAl Viro 787bc917be8SAl Viro int compat_import_iovec(int type, const struct compat_iovec __user * uvector, 788bc917be8SAl Viro unsigned nr_segs, unsigned fast_segs, 789bc917be8SAl Viro struct iovec **iov, struct iov_iter *i) 790bc917be8SAl Viro { 791bc917be8SAl Viro ssize_t n; 792bc917be8SAl Viro struct iovec *p; 793bc917be8SAl Viro n = compat_rw_copy_check_uvector(type, uvector, nr_segs, fast_segs, 794bc917be8SAl Viro *iov, &p); 795bc917be8SAl Viro if (n < 0) { 796bc917be8SAl Viro if (p != *iov) 797bc917be8SAl Viro kfree(p); 798bc917be8SAl Viro *iov = NULL; 799bc917be8SAl Viro return n; 800bc917be8SAl Viro } 801bc917be8SAl Viro iov_iter_init(i, type, p, nr_segs, n); 802bc917be8SAl Viro *iov = p == *iov ? NULL : p; 803bc917be8SAl Viro return 0; 804bc917be8SAl Viro } 805bc917be8SAl Viro #endif 806bc917be8SAl Viro 807bc917be8SAl Viro int import_single_range(int rw, void __user *buf, size_t len, 808bc917be8SAl Viro struct iovec *iov, struct iov_iter *i) 809bc917be8SAl Viro { 810bc917be8SAl Viro if (len > MAX_RW_COUNT) 811bc917be8SAl Viro len = MAX_RW_COUNT; 812bc917be8SAl Viro if (unlikely(!access_ok(!rw, buf, len))) 813bc917be8SAl Viro return -EFAULT; 814bc917be8SAl Viro 815bc917be8SAl Viro iov->iov_base = buf; 816bc917be8SAl Viro iov->iov_len = len; 817bc917be8SAl Viro iov_iter_init(i, rw, iov, 1, len); 818bc917be8SAl Viro return 0; 819bc917be8SAl Viro } 820e1267585SAl Viro EXPORT_SYMBOL(import_single_range); 821