1d879cb83SAl Viro #include <linux/export.h> 2d879cb83SAl Viro #include <linux/uio.h> 3d879cb83SAl Viro #include <linux/pagemap.h> 4d879cb83SAl Viro #include <linux/slab.h> 5d879cb83SAl Viro #include <linux/vmalloc.h> 6d879cb83SAl Viro #include <net/checksum.h> 7d879cb83SAl Viro 8d879cb83SAl Viro #define iterate_iovec(i, n, __v, __p, skip, STEP) { \ 9d879cb83SAl Viro size_t left; \ 10d879cb83SAl Viro size_t wanted = n; \ 11d879cb83SAl Viro __p = i->iov; \ 12d879cb83SAl Viro __v.iov_len = min(n, __p->iov_len - skip); \ 13d879cb83SAl Viro if (likely(__v.iov_len)) { \ 14d879cb83SAl Viro __v.iov_base = __p->iov_base + skip; \ 15d879cb83SAl Viro left = (STEP); \ 16d879cb83SAl Viro __v.iov_len -= left; \ 17d879cb83SAl Viro skip += __v.iov_len; \ 18d879cb83SAl Viro n -= __v.iov_len; \ 19d879cb83SAl Viro } else { \ 20d879cb83SAl Viro left = 0; \ 21d879cb83SAl Viro } \ 22d879cb83SAl Viro while (unlikely(!left && n)) { \ 23d879cb83SAl Viro __p++; \ 24d879cb83SAl Viro __v.iov_len = min(n, __p->iov_len); \ 25d879cb83SAl Viro if (unlikely(!__v.iov_len)) \ 26d879cb83SAl Viro continue; \ 27d879cb83SAl Viro __v.iov_base = __p->iov_base; \ 28d879cb83SAl Viro left = (STEP); \ 29d879cb83SAl Viro __v.iov_len -= left; \ 30d879cb83SAl Viro skip = __v.iov_len; \ 31d879cb83SAl Viro n -= __v.iov_len; \ 32d879cb83SAl Viro } \ 33d879cb83SAl Viro n = wanted - n; \ 34d879cb83SAl Viro } 35d879cb83SAl Viro 36d879cb83SAl Viro #define iterate_kvec(i, n, __v, __p, skip, STEP) { \ 37d879cb83SAl Viro size_t wanted = n; \ 38d879cb83SAl Viro __p = i->kvec; \ 39d879cb83SAl Viro __v.iov_len = min(n, __p->iov_len - skip); \ 40d879cb83SAl Viro if (likely(__v.iov_len)) { \ 41d879cb83SAl Viro __v.iov_base = __p->iov_base + skip; \ 42d879cb83SAl Viro (void)(STEP); \ 43d879cb83SAl Viro skip += __v.iov_len; \ 44d879cb83SAl Viro n -= __v.iov_len; \ 45d879cb83SAl Viro } \ 46d879cb83SAl Viro while (unlikely(n)) { \ 47d879cb83SAl Viro __p++; \ 48d879cb83SAl Viro __v.iov_len = min(n, __p->iov_len); \ 49d879cb83SAl Viro if (unlikely(!__v.iov_len)) \ 50d879cb83SAl Viro continue; \ 51d879cb83SAl Viro __v.iov_base = __p->iov_base; \ 52d879cb83SAl Viro (void)(STEP); \ 53d879cb83SAl Viro skip = __v.iov_len; \ 54d879cb83SAl Viro n -= __v.iov_len; \ 55d879cb83SAl Viro } \ 56d879cb83SAl Viro n = wanted; \ 57d879cb83SAl Viro } 58d879cb83SAl Viro 59d879cb83SAl Viro #define iterate_bvec(i, n, __v, __p, skip, STEP) { \ 60d879cb83SAl Viro size_t wanted = n; \ 61d879cb83SAl Viro __p = i->bvec; \ 62d879cb83SAl Viro __v.bv_len = min_t(size_t, n, __p->bv_len - skip); \ 63d879cb83SAl Viro if (likely(__v.bv_len)) { \ 64d879cb83SAl Viro __v.bv_page = __p->bv_page; \ 65d879cb83SAl Viro __v.bv_offset = __p->bv_offset + skip; \ 66d879cb83SAl Viro (void)(STEP); \ 67d879cb83SAl Viro skip += __v.bv_len; \ 68d879cb83SAl Viro n -= __v.bv_len; \ 69d879cb83SAl Viro } \ 70d879cb83SAl Viro while (unlikely(n)) { \ 71d879cb83SAl Viro __p++; \ 72d879cb83SAl Viro __v.bv_len = min_t(size_t, n, __p->bv_len); \ 73d879cb83SAl Viro if (unlikely(!__v.bv_len)) \ 74d879cb83SAl Viro continue; \ 75d879cb83SAl Viro __v.bv_page = __p->bv_page; \ 76d879cb83SAl Viro __v.bv_offset = __p->bv_offset; \ 77d879cb83SAl Viro (void)(STEP); \ 78d879cb83SAl Viro skip = __v.bv_len; \ 79d879cb83SAl Viro n -= __v.bv_len; \ 80d879cb83SAl Viro } \ 81d879cb83SAl Viro n = wanted; \ 82d879cb83SAl Viro } 83d879cb83SAl Viro 84d879cb83SAl Viro #define iterate_all_kinds(i, n, v, I, B, K) { \ 85d879cb83SAl Viro size_t skip = i->iov_offset; \ 86d879cb83SAl Viro if (unlikely(i->type & ITER_BVEC)) { \ 87d879cb83SAl Viro const struct bio_vec *bvec; \ 88d879cb83SAl Viro struct bio_vec v; \ 89d879cb83SAl Viro iterate_bvec(i, n, v, bvec, skip, (B)) \ 90d879cb83SAl Viro } else if (unlikely(i->type & ITER_KVEC)) { \ 91d879cb83SAl Viro const struct kvec *kvec; \ 92d879cb83SAl Viro struct kvec v; \ 93d879cb83SAl Viro iterate_kvec(i, n, v, kvec, skip, (K)) \ 94d879cb83SAl Viro } else { \ 95d879cb83SAl Viro const struct iovec *iov; \ 96d879cb83SAl Viro struct iovec v; \ 97d879cb83SAl Viro iterate_iovec(i, n, v, iov, skip, (I)) \ 98d879cb83SAl Viro } \ 99d879cb83SAl Viro } 100d879cb83SAl Viro 101d879cb83SAl Viro #define iterate_and_advance(i, n, v, I, B, K) { \ 102d879cb83SAl Viro size_t skip = i->iov_offset; \ 103d879cb83SAl Viro if (unlikely(i->type & ITER_BVEC)) { \ 104d879cb83SAl Viro const struct bio_vec *bvec; \ 105d879cb83SAl Viro struct bio_vec v; \ 106d879cb83SAl Viro iterate_bvec(i, n, v, bvec, skip, (B)) \ 107d879cb83SAl Viro if (skip == bvec->bv_len) { \ 108d879cb83SAl Viro bvec++; \ 109d879cb83SAl Viro skip = 0; \ 110d879cb83SAl Viro } \ 111d879cb83SAl Viro i->nr_segs -= bvec - i->bvec; \ 112d879cb83SAl Viro i->bvec = bvec; \ 113d879cb83SAl Viro } else if (unlikely(i->type & ITER_KVEC)) { \ 114d879cb83SAl Viro const struct kvec *kvec; \ 115d879cb83SAl Viro struct kvec v; \ 116d879cb83SAl Viro iterate_kvec(i, n, v, kvec, skip, (K)) \ 117d879cb83SAl Viro if (skip == kvec->iov_len) { \ 118d879cb83SAl Viro kvec++; \ 119d879cb83SAl Viro skip = 0; \ 120d879cb83SAl Viro } \ 121d879cb83SAl Viro i->nr_segs -= kvec - i->kvec; \ 122d879cb83SAl Viro i->kvec = kvec; \ 123d879cb83SAl Viro } else { \ 124d879cb83SAl Viro const struct iovec *iov; \ 125d879cb83SAl Viro struct iovec v; \ 126d879cb83SAl Viro iterate_iovec(i, n, v, iov, skip, (I)) \ 127d879cb83SAl Viro if (skip == iov->iov_len) { \ 128d879cb83SAl Viro iov++; \ 129d879cb83SAl Viro skip = 0; \ 130d879cb83SAl Viro } \ 131d879cb83SAl Viro i->nr_segs -= iov - i->iov; \ 132d879cb83SAl Viro i->iov = iov; \ 133d879cb83SAl Viro } \ 134d879cb83SAl Viro i->count -= n; \ 135d879cb83SAl Viro i->iov_offset = skip; \ 136d879cb83SAl Viro } 137d879cb83SAl Viro 138d879cb83SAl Viro static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, 139d879cb83SAl Viro struct iov_iter *i) 140d879cb83SAl Viro { 141d879cb83SAl Viro size_t skip, copy, left, wanted; 142d879cb83SAl Viro const struct iovec *iov; 143d879cb83SAl Viro char __user *buf; 144d879cb83SAl Viro void *kaddr, *from; 145d879cb83SAl Viro 146d879cb83SAl Viro if (unlikely(bytes > i->count)) 147d879cb83SAl Viro bytes = i->count; 148d879cb83SAl Viro 149d879cb83SAl Viro if (unlikely(!bytes)) 150d879cb83SAl Viro return 0; 151d879cb83SAl Viro 152d879cb83SAl Viro wanted = bytes; 153d879cb83SAl Viro iov = i->iov; 154d879cb83SAl Viro skip = i->iov_offset; 155d879cb83SAl Viro buf = iov->iov_base + skip; 156d879cb83SAl Viro copy = min(bytes, iov->iov_len - skip); 157d879cb83SAl Viro 158d879cb83SAl Viro if (!fault_in_pages_writeable(buf, copy)) { 159d879cb83SAl Viro kaddr = kmap_atomic(page); 160d879cb83SAl Viro from = kaddr + offset; 161d879cb83SAl Viro 162d879cb83SAl Viro /* first chunk, usually the only one */ 163d879cb83SAl Viro left = __copy_to_user_inatomic(buf, from, copy); 164d879cb83SAl Viro copy -= left; 165d879cb83SAl Viro skip += copy; 166d879cb83SAl Viro from += copy; 167d879cb83SAl Viro bytes -= copy; 168d879cb83SAl Viro 169d879cb83SAl Viro while (unlikely(!left && bytes)) { 170d879cb83SAl Viro iov++; 171d879cb83SAl Viro buf = iov->iov_base; 172d879cb83SAl Viro copy = min(bytes, iov->iov_len); 173d879cb83SAl Viro left = __copy_to_user_inatomic(buf, from, copy); 174d879cb83SAl Viro copy -= left; 175d879cb83SAl Viro skip = copy; 176d879cb83SAl Viro from += copy; 177d879cb83SAl Viro bytes -= copy; 178d879cb83SAl Viro } 179d879cb83SAl Viro if (likely(!bytes)) { 180d879cb83SAl Viro kunmap_atomic(kaddr); 181d879cb83SAl Viro goto done; 182d879cb83SAl Viro } 183d879cb83SAl Viro offset = from - kaddr; 184d879cb83SAl Viro buf += copy; 185d879cb83SAl Viro kunmap_atomic(kaddr); 186d879cb83SAl Viro copy = min(bytes, iov->iov_len - skip); 187d879cb83SAl Viro } 188d879cb83SAl Viro /* Too bad - revert to non-atomic kmap */ 189d879cb83SAl Viro kaddr = kmap(page); 190d879cb83SAl Viro from = kaddr + offset; 191d879cb83SAl Viro left = __copy_to_user(buf, from, copy); 192d879cb83SAl Viro copy -= left; 193d879cb83SAl Viro skip += copy; 194d879cb83SAl Viro from += copy; 195d879cb83SAl Viro bytes -= copy; 196d879cb83SAl Viro while (unlikely(!left && bytes)) { 197d879cb83SAl Viro iov++; 198d879cb83SAl Viro buf = iov->iov_base; 199d879cb83SAl Viro copy = min(bytes, iov->iov_len); 200d879cb83SAl Viro left = __copy_to_user(buf, from, copy); 201d879cb83SAl Viro copy -= left; 202d879cb83SAl Viro skip = copy; 203d879cb83SAl Viro from += copy; 204d879cb83SAl Viro bytes -= copy; 205d879cb83SAl Viro } 206d879cb83SAl Viro kunmap(page); 207d879cb83SAl Viro done: 208d879cb83SAl Viro if (skip == iov->iov_len) { 209d879cb83SAl Viro iov++; 210d879cb83SAl Viro skip = 0; 211d879cb83SAl Viro } 212d879cb83SAl Viro i->count -= wanted - bytes; 213d879cb83SAl Viro i->nr_segs -= iov - i->iov; 214d879cb83SAl Viro i->iov = iov; 215d879cb83SAl Viro i->iov_offset = skip; 216d879cb83SAl Viro return wanted - bytes; 217d879cb83SAl Viro } 218d879cb83SAl Viro 219d879cb83SAl Viro static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes, 220d879cb83SAl Viro struct iov_iter *i) 221d879cb83SAl Viro { 222d879cb83SAl Viro size_t skip, copy, left, wanted; 223d879cb83SAl Viro const struct iovec *iov; 224d879cb83SAl Viro char __user *buf; 225d879cb83SAl Viro void *kaddr, *to; 226d879cb83SAl Viro 227d879cb83SAl Viro if (unlikely(bytes > i->count)) 228d879cb83SAl Viro bytes = i->count; 229d879cb83SAl Viro 230d879cb83SAl Viro if (unlikely(!bytes)) 231d879cb83SAl Viro return 0; 232d879cb83SAl Viro 233d879cb83SAl Viro wanted = bytes; 234d879cb83SAl Viro iov = i->iov; 235d879cb83SAl Viro skip = i->iov_offset; 236d879cb83SAl Viro buf = iov->iov_base + skip; 237d879cb83SAl Viro copy = min(bytes, iov->iov_len - skip); 238d879cb83SAl Viro 239d879cb83SAl Viro if (!fault_in_pages_readable(buf, copy)) { 240d879cb83SAl Viro kaddr = kmap_atomic(page); 241d879cb83SAl Viro to = kaddr + offset; 242d879cb83SAl Viro 243d879cb83SAl Viro /* first chunk, usually the only one */ 244d879cb83SAl Viro left = __copy_from_user_inatomic(to, buf, copy); 245d879cb83SAl Viro copy -= left; 246d879cb83SAl Viro skip += copy; 247d879cb83SAl Viro to += copy; 248d879cb83SAl Viro bytes -= copy; 249d879cb83SAl Viro 250d879cb83SAl Viro while (unlikely(!left && bytes)) { 251d879cb83SAl Viro iov++; 252d879cb83SAl Viro buf = iov->iov_base; 253d879cb83SAl Viro copy = min(bytes, iov->iov_len); 254d879cb83SAl Viro left = __copy_from_user_inatomic(to, buf, copy); 255d879cb83SAl Viro copy -= left; 256d879cb83SAl Viro skip = copy; 257d879cb83SAl Viro to += copy; 258d879cb83SAl Viro bytes -= copy; 259d879cb83SAl Viro } 260d879cb83SAl Viro if (likely(!bytes)) { 261d879cb83SAl Viro kunmap_atomic(kaddr); 262d879cb83SAl Viro goto done; 263d879cb83SAl Viro } 264d879cb83SAl Viro offset = to - kaddr; 265d879cb83SAl Viro buf += copy; 266d879cb83SAl Viro kunmap_atomic(kaddr); 267d879cb83SAl Viro copy = min(bytes, iov->iov_len - skip); 268d879cb83SAl Viro } 269d879cb83SAl Viro /* Too bad - revert to non-atomic kmap */ 270d879cb83SAl Viro kaddr = kmap(page); 271d879cb83SAl Viro to = kaddr + offset; 272d879cb83SAl Viro left = __copy_from_user(to, buf, copy); 273d879cb83SAl Viro copy -= left; 274d879cb83SAl Viro skip += copy; 275d879cb83SAl Viro to += copy; 276d879cb83SAl Viro bytes -= copy; 277d879cb83SAl Viro while (unlikely(!left && bytes)) { 278d879cb83SAl Viro iov++; 279d879cb83SAl Viro buf = iov->iov_base; 280d879cb83SAl Viro copy = min(bytes, iov->iov_len); 281d879cb83SAl Viro left = __copy_from_user(to, buf, copy); 282d879cb83SAl Viro copy -= left; 283d879cb83SAl Viro skip = copy; 284d879cb83SAl Viro to += copy; 285d879cb83SAl Viro bytes -= copy; 286d879cb83SAl Viro } 287d879cb83SAl Viro kunmap(page); 288d879cb83SAl Viro done: 289d879cb83SAl Viro if (skip == iov->iov_len) { 290d879cb83SAl Viro iov++; 291d879cb83SAl Viro skip = 0; 292d879cb83SAl Viro } 293d879cb83SAl Viro i->count -= wanted - bytes; 294d879cb83SAl Viro i->nr_segs -= iov - i->iov; 295d879cb83SAl Viro i->iov = iov; 296d879cb83SAl Viro i->iov_offset = skip; 297d879cb83SAl Viro return wanted - bytes; 298d879cb83SAl Viro } 299d879cb83SAl Viro 300d879cb83SAl Viro /* 301d879cb83SAl Viro * Fault in the first iovec of the given iov_iter, to a maximum length 302d879cb83SAl Viro * of bytes. Returns 0 on success, or non-zero if the memory could not be 303d879cb83SAl Viro * accessed (ie. because it is an invalid address). 304d879cb83SAl Viro * 305d879cb83SAl Viro * writev-intensive code may want this to prefault several iovecs -- that 306d879cb83SAl Viro * would be possible (callers must not rely on the fact that _only_ the 307d879cb83SAl Viro * first iovec will be faulted with the current implementation). 308d879cb83SAl Viro */ 309d879cb83SAl Viro int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes) 310d879cb83SAl Viro { 311d879cb83SAl Viro if (!(i->type & (ITER_BVEC|ITER_KVEC))) { 312d879cb83SAl Viro char __user *buf = i->iov->iov_base + i->iov_offset; 313d879cb83SAl Viro bytes = min(bytes, i->iov->iov_len - i->iov_offset); 314d879cb83SAl Viro return fault_in_pages_readable(buf, bytes); 315d879cb83SAl Viro } 316d879cb83SAl Viro return 0; 317d879cb83SAl Viro } 318d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_fault_in_readable); 319d879cb83SAl Viro 320d879cb83SAl Viro void iov_iter_init(struct iov_iter *i, int direction, 321d879cb83SAl Viro const struct iovec *iov, unsigned long nr_segs, 322d879cb83SAl Viro size_t count) 323d879cb83SAl Viro { 324d879cb83SAl Viro /* It will get better. Eventually... */ 325d879cb83SAl Viro if (segment_eq(get_fs(), KERNEL_DS)) { 326d879cb83SAl Viro direction |= ITER_KVEC; 327d879cb83SAl Viro i->type = direction; 328d879cb83SAl Viro i->kvec = (struct kvec *)iov; 329d879cb83SAl Viro } else { 330d879cb83SAl Viro i->type = direction; 331d879cb83SAl Viro i->iov = iov; 332d879cb83SAl Viro } 333d879cb83SAl Viro i->nr_segs = nr_segs; 334d879cb83SAl Viro i->iov_offset = 0; 335d879cb83SAl Viro i->count = count; 336d879cb83SAl Viro } 337d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_init); 338d879cb83SAl Viro 339d879cb83SAl Viro static void memcpy_from_page(char *to, struct page *page, size_t offset, size_t len) 340d879cb83SAl Viro { 341d879cb83SAl Viro char *from = kmap_atomic(page); 342d879cb83SAl Viro memcpy(to, from + offset, len); 343d879cb83SAl Viro kunmap_atomic(from); 344d879cb83SAl Viro } 345d879cb83SAl Viro 346d879cb83SAl Viro static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t len) 347d879cb83SAl Viro { 348d879cb83SAl Viro char *to = kmap_atomic(page); 349d879cb83SAl Viro memcpy(to + offset, from, len); 350d879cb83SAl Viro kunmap_atomic(to); 351d879cb83SAl Viro } 352d879cb83SAl Viro 353d879cb83SAl Viro static void memzero_page(struct page *page, size_t offset, size_t len) 354d879cb83SAl Viro { 355d879cb83SAl Viro char *addr = kmap_atomic(page); 356d879cb83SAl Viro memset(addr + offset, 0, len); 357d879cb83SAl Viro kunmap_atomic(addr); 358d879cb83SAl Viro } 359d879cb83SAl Viro 360d879cb83SAl Viro size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i) 361d879cb83SAl Viro { 362d879cb83SAl Viro char *from = addr; 363d879cb83SAl Viro if (unlikely(bytes > i->count)) 364d879cb83SAl Viro bytes = i->count; 365d879cb83SAl Viro 366d879cb83SAl Viro if (unlikely(!bytes)) 367d879cb83SAl Viro return 0; 368d879cb83SAl Viro 369d879cb83SAl Viro iterate_and_advance(i, bytes, v, 370d879cb83SAl Viro __copy_to_user(v.iov_base, (from += v.iov_len) - v.iov_len, 371d879cb83SAl Viro v.iov_len), 372d879cb83SAl Viro memcpy_to_page(v.bv_page, v.bv_offset, 373d879cb83SAl Viro (from += v.bv_len) - v.bv_len, v.bv_len), 374d879cb83SAl Viro memcpy(v.iov_base, (from += v.iov_len) - v.iov_len, v.iov_len) 375d879cb83SAl Viro ) 376d879cb83SAl Viro 377d879cb83SAl Viro return bytes; 378d879cb83SAl Viro } 379d879cb83SAl Viro EXPORT_SYMBOL(copy_to_iter); 380d879cb83SAl Viro 381d879cb83SAl Viro size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) 382d879cb83SAl Viro { 383d879cb83SAl Viro char *to = addr; 384d879cb83SAl Viro if (unlikely(bytes > i->count)) 385d879cb83SAl Viro bytes = i->count; 386d879cb83SAl Viro 387d879cb83SAl Viro if (unlikely(!bytes)) 388d879cb83SAl Viro return 0; 389d879cb83SAl Viro 390d879cb83SAl Viro iterate_and_advance(i, bytes, v, 391d879cb83SAl Viro __copy_from_user((to += v.iov_len) - v.iov_len, v.iov_base, 392d879cb83SAl Viro v.iov_len), 393d879cb83SAl Viro memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 394d879cb83SAl Viro v.bv_offset, v.bv_len), 395d879cb83SAl Viro memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 396d879cb83SAl Viro ) 397d879cb83SAl Viro 398d879cb83SAl Viro return bytes; 399d879cb83SAl Viro } 400d879cb83SAl Viro EXPORT_SYMBOL(copy_from_iter); 401d879cb83SAl Viro 402d879cb83SAl Viro size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i) 403d879cb83SAl Viro { 404d879cb83SAl Viro char *to = addr; 405d879cb83SAl Viro if (unlikely(bytes > i->count)) 406d879cb83SAl Viro bytes = i->count; 407d879cb83SAl Viro 408d879cb83SAl Viro if (unlikely(!bytes)) 409d879cb83SAl Viro return 0; 410d879cb83SAl Viro 411d879cb83SAl Viro iterate_and_advance(i, bytes, v, 412d879cb83SAl Viro __copy_from_user_nocache((to += v.iov_len) - v.iov_len, 413d879cb83SAl Viro v.iov_base, v.iov_len), 414d879cb83SAl Viro memcpy_from_page((to += v.bv_len) - v.bv_len, v.bv_page, 415d879cb83SAl Viro v.bv_offset, v.bv_len), 416d879cb83SAl Viro memcpy((to += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 417d879cb83SAl Viro ) 418d879cb83SAl Viro 419d879cb83SAl Viro return bytes; 420d879cb83SAl Viro } 421d879cb83SAl Viro EXPORT_SYMBOL(copy_from_iter_nocache); 422d879cb83SAl Viro 423d879cb83SAl Viro size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, 424d879cb83SAl Viro struct iov_iter *i) 425d879cb83SAl Viro { 426d879cb83SAl Viro if (i->type & (ITER_BVEC|ITER_KVEC)) { 427d879cb83SAl Viro void *kaddr = kmap_atomic(page); 428d879cb83SAl Viro size_t wanted = copy_to_iter(kaddr + offset, bytes, i); 429d879cb83SAl Viro kunmap_atomic(kaddr); 430d879cb83SAl Viro return wanted; 431d879cb83SAl Viro } else 432d879cb83SAl Viro return copy_page_to_iter_iovec(page, offset, bytes, i); 433d879cb83SAl Viro } 434d879cb83SAl Viro EXPORT_SYMBOL(copy_page_to_iter); 435d879cb83SAl Viro 436d879cb83SAl Viro size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, 437d879cb83SAl Viro struct iov_iter *i) 438d879cb83SAl Viro { 439d879cb83SAl Viro if (i->type & (ITER_BVEC|ITER_KVEC)) { 440d879cb83SAl Viro void *kaddr = kmap_atomic(page); 441d879cb83SAl Viro size_t wanted = copy_from_iter(kaddr + offset, bytes, i); 442d879cb83SAl Viro kunmap_atomic(kaddr); 443d879cb83SAl Viro return wanted; 444d879cb83SAl Viro } else 445d879cb83SAl Viro return copy_page_from_iter_iovec(page, offset, bytes, i); 446d879cb83SAl Viro } 447d879cb83SAl Viro EXPORT_SYMBOL(copy_page_from_iter); 448d879cb83SAl Viro 449d879cb83SAl Viro size_t iov_iter_zero(size_t bytes, struct iov_iter *i) 450d879cb83SAl Viro { 451d879cb83SAl Viro if (unlikely(bytes > i->count)) 452d879cb83SAl Viro bytes = i->count; 453d879cb83SAl Viro 454d879cb83SAl Viro if (unlikely(!bytes)) 455d879cb83SAl Viro return 0; 456d879cb83SAl Viro 457d879cb83SAl Viro iterate_and_advance(i, bytes, v, 458d879cb83SAl Viro __clear_user(v.iov_base, v.iov_len), 459d879cb83SAl Viro memzero_page(v.bv_page, v.bv_offset, v.bv_len), 460d879cb83SAl Viro memset(v.iov_base, 0, v.iov_len) 461d879cb83SAl Viro ) 462d879cb83SAl Viro 463d879cb83SAl Viro return bytes; 464d879cb83SAl Viro } 465d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_zero); 466d879cb83SAl Viro 467d879cb83SAl Viro size_t iov_iter_copy_from_user_atomic(struct page *page, 468d879cb83SAl Viro struct iov_iter *i, unsigned long offset, size_t bytes) 469d879cb83SAl Viro { 470d879cb83SAl Viro char *kaddr = kmap_atomic(page), *p = kaddr + offset; 471d879cb83SAl Viro iterate_all_kinds(i, bytes, v, 472d879cb83SAl Viro __copy_from_user_inatomic((p += v.iov_len) - v.iov_len, 473d879cb83SAl Viro v.iov_base, v.iov_len), 474d879cb83SAl Viro memcpy_from_page((p += v.bv_len) - v.bv_len, v.bv_page, 475d879cb83SAl Viro v.bv_offset, v.bv_len), 476d879cb83SAl Viro memcpy((p += v.iov_len) - v.iov_len, v.iov_base, v.iov_len) 477d879cb83SAl Viro ) 478d879cb83SAl Viro kunmap_atomic(kaddr); 479d879cb83SAl Viro return bytes; 480d879cb83SAl Viro } 481d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_copy_from_user_atomic); 482d879cb83SAl Viro 483d879cb83SAl Viro void iov_iter_advance(struct iov_iter *i, size_t size) 484d879cb83SAl Viro { 485d879cb83SAl Viro iterate_and_advance(i, size, v, 0, 0, 0) 486d879cb83SAl Viro } 487d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_advance); 488d879cb83SAl Viro 489d879cb83SAl Viro /* 490d879cb83SAl Viro * Return the count of just the current iov_iter segment. 491d879cb83SAl Viro */ 492d879cb83SAl Viro size_t iov_iter_single_seg_count(const struct iov_iter *i) 493d879cb83SAl Viro { 494d879cb83SAl Viro if (i->nr_segs == 1) 495d879cb83SAl Viro return i->count; 496d879cb83SAl Viro else if (i->type & ITER_BVEC) 497d879cb83SAl Viro return min(i->count, i->bvec->bv_len - i->iov_offset); 498d879cb83SAl Viro else 499d879cb83SAl Viro return min(i->count, i->iov->iov_len - i->iov_offset); 500d879cb83SAl Viro } 501d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_single_seg_count); 502d879cb83SAl Viro 503d879cb83SAl Viro void iov_iter_kvec(struct iov_iter *i, int direction, 504d879cb83SAl Viro const struct kvec *kvec, unsigned long nr_segs, 505d879cb83SAl Viro size_t count) 506d879cb83SAl Viro { 507d879cb83SAl Viro BUG_ON(!(direction & ITER_KVEC)); 508d879cb83SAl Viro i->type = direction; 509d879cb83SAl Viro i->kvec = kvec; 510d879cb83SAl Viro i->nr_segs = nr_segs; 511d879cb83SAl Viro i->iov_offset = 0; 512d879cb83SAl Viro i->count = count; 513d879cb83SAl Viro } 514d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_kvec); 515d879cb83SAl Viro 516d879cb83SAl Viro void iov_iter_bvec(struct iov_iter *i, int direction, 517d879cb83SAl Viro const struct bio_vec *bvec, unsigned long nr_segs, 518d879cb83SAl Viro size_t count) 519d879cb83SAl Viro { 520d879cb83SAl Viro BUG_ON(!(direction & ITER_BVEC)); 521d879cb83SAl Viro i->type = direction; 522d879cb83SAl Viro i->bvec = bvec; 523d879cb83SAl Viro i->nr_segs = nr_segs; 524d879cb83SAl Viro i->iov_offset = 0; 525d879cb83SAl Viro i->count = count; 526d879cb83SAl Viro } 527d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_bvec); 528d879cb83SAl Viro 529d879cb83SAl Viro unsigned long iov_iter_alignment(const struct iov_iter *i) 530d879cb83SAl Viro { 531d879cb83SAl Viro unsigned long res = 0; 532d879cb83SAl Viro size_t size = i->count; 533d879cb83SAl Viro 534d879cb83SAl Viro if (!size) 535d879cb83SAl Viro return 0; 536d879cb83SAl Viro 537d879cb83SAl Viro iterate_all_kinds(i, size, v, 538d879cb83SAl Viro (res |= (unsigned long)v.iov_base | v.iov_len, 0), 539d879cb83SAl Viro res |= v.bv_offset | v.bv_len, 540d879cb83SAl Viro res |= (unsigned long)v.iov_base | v.iov_len 541d879cb83SAl Viro ) 542d879cb83SAl Viro return res; 543d879cb83SAl Viro } 544d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_alignment); 545d879cb83SAl Viro 546d879cb83SAl Viro ssize_t iov_iter_get_pages(struct iov_iter *i, 547d879cb83SAl Viro struct page **pages, size_t maxsize, unsigned maxpages, 548d879cb83SAl Viro size_t *start) 549d879cb83SAl Viro { 550d879cb83SAl Viro if (maxsize > i->count) 551d879cb83SAl Viro maxsize = i->count; 552d879cb83SAl Viro 553d879cb83SAl Viro if (!maxsize) 554d879cb83SAl Viro return 0; 555d879cb83SAl Viro 556d879cb83SAl Viro iterate_all_kinds(i, maxsize, v, ({ 557d879cb83SAl Viro unsigned long addr = (unsigned long)v.iov_base; 558d879cb83SAl Viro size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 559d879cb83SAl Viro int n; 560d879cb83SAl Viro int res; 561d879cb83SAl Viro 562d879cb83SAl Viro if (len > maxpages * PAGE_SIZE) 563d879cb83SAl Viro len = maxpages * PAGE_SIZE; 564d879cb83SAl Viro addr &= ~(PAGE_SIZE - 1); 565d879cb83SAl Viro n = DIV_ROUND_UP(len, PAGE_SIZE); 566d879cb83SAl Viro res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, pages); 567d879cb83SAl Viro if (unlikely(res < 0)) 568d879cb83SAl Viro return res; 569d879cb83SAl Viro return (res == n ? len : res * PAGE_SIZE) - *start; 570d879cb83SAl Viro 0;}),({ 571d879cb83SAl Viro /* can't be more than PAGE_SIZE */ 572d879cb83SAl Viro *start = v.bv_offset; 573d879cb83SAl Viro get_page(*pages = v.bv_page); 574d879cb83SAl Viro return v.bv_len; 575d879cb83SAl Viro }),({ 576d879cb83SAl Viro return -EFAULT; 577d879cb83SAl Viro }) 578d879cb83SAl Viro ) 579d879cb83SAl Viro return 0; 580d879cb83SAl Viro } 581d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_get_pages); 582d879cb83SAl Viro 583d879cb83SAl Viro static struct page **get_pages_array(size_t n) 584d879cb83SAl Viro { 585d879cb83SAl Viro struct page **p = kmalloc(n * sizeof(struct page *), GFP_KERNEL); 586d879cb83SAl Viro if (!p) 587d879cb83SAl Viro p = vmalloc(n * sizeof(struct page *)); 588d879cb83SAl Viro return p; 589d879cb83SAl Viro } 590d879cb83SAl Viro 591d879cb83SAl Viro ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, 592d879cb83SAl Viro struct page ***pages, size_t maxsize, 593d879cb83SAl Viro size_t *start) 594d879cb83SAl Viro { 595d879cb83SAl Viro struct page **p; 596d879cb83SAl Viro 597d879cb83SAl Viro if (maxsize > i->count) 598d879cb83SAl Viro maxsize = i->count; 599d879cb83SAl Viro 600d879cb83SAl Viro if (!maxsize) 601d879cb83SAl Viro return 0; 602d879cb83SAl Viro 603d879cb83SAl Viro iterate_all_kinds(i, maxsize, v, ({ 604d879cb83SAl Viro unsigned long addr = (unsigned long)v.iov_base; 605d879cb83SAl Viro size_t len = v.iov_len + (*start = addr & (PAGE_SIZE - 1)); 606d879cb83SAl Viro int n; 607d879cb83SAl Viro int res; 608d879cb83SAl Viro 609d879cb83SAl Viro addr &= ~(PAGE_SIZE - 1); 610d879cb83SAl Viro n = DIV_ROUND_UP(len, PAGE_SIZE); 611d879cb83SAl Viro p = get_pages_array(n); 612d879cb83SAl Viro if (!p) 613d879cb83SAl Viro return -ENOMEM; 614d879cb83SAl Viro res = get_user_pages_fast(addr, n, (i->type & WRITE) != WRITE, p); 615d879cb83SAl Viro if (unlikely(res < 0)) { 616d879cb83SAl Viro kvfree(p); 617d879cb83SAl Viro return res; 618d879cb83SAl Viro } 619d879cb83SAl Viro *pages = p; 620d879cb83SAl Viro return (res == n ? len : res * PAGE_SIZE) - *start; 621d879cb83SAl Viro 0;}),({ 622d879cb83SAl Viro /* can't be more than PAGE_SIZE */ 623d879cb83SAl Viro *start = v.bv_offset; 624d879cb83SAl Viro *pages = p = get_pages_array(1); 625d879cb83SAl Viro if (!p) 626d879cb83SAl Viro return -ENOMEM; 627d879cb83SAl Viro get_page(*p = v.bv_page); 628d879cb83SAl Viro return v.bv_len; 629d879cb83SAl Viro }),({ 630d879cb83SAl Viro return -EFAULT; 631d879cb83SAl Viro }) 632d879cb83SAl Viro ) 633d879cb83SAl Viro return 0; 634d879cb83SAl Viro } 635d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_get_pages_alloc); 636d879cb83SAl Viro 637d879cb83SAl Viro size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, 638d879cb83SAl Viro struct iov_iter *i) 639d879cb83SAl Viro { 640d879cb83SAl Viro char *to = addr; 641d879cb83SAl Viro __wsum sum, next; 642d879cb83SAl Viro size_t off = 0; 643d879cb83SAl Viro if (unlikely(bytes > i->count)) 644d879cb83SAl Viro bytes = i->count; 645d879cb83SAl Viro 646d879cb83SAl Viro if (unlikely(!bytes)) 647d879cb83SAl Viro return 0; 648d879cb83SAl Viro 649d879cb83SAl Viro sum = *csum; 650d879cb83SAl Viro iterate_and_advance(i, bytes, v, ({ 651d879cb83SAl Viro int err = 0; 652d879cb83SAl Viro next = csum_and_copy_from_user(v.iov_base, 653d879cb83SAl Viro (to += v.iov_len) - v.iov_len, 654d879cb83SAl Viro v.iov_len, 0, &err); 655d879cb83SAl Viro if (!err) { 656d879cb83SAl Viro sum = csum_block_add(sum, next, off); 657d879cb83SAl Viro off += v.iov_len; 658d879cb83SAl Viro } 659d879cb83SAl Viro err ? v.iov_len : 0; 660d879cb83SAl Viro }), ({ 661d879cb83SAl Viro char *p = kmap_atomic(v.bv_page); 662d879cb83SAl Viro next = csum_partial_copy_nocheck(p + v.bv_offset, 663d879cb83SAl Viro (to += v.bv_len) - v.bv_len, 664d879cb83SAl Viro v.bv_len, 0); 665d879cb83SAl Viro kunmap_atomic(p); 666d879cb83SAl Viro sum = csum_block_add(sum, next, off); 667d879cb83SAl Viro off += v.bv_len; 668d879cb83SAl Viro }),({ 669d879cb83SAl Viro next = csum_partial_copy_nocheck(v.iov_base, 670d879cb83SAl Viro (to += v.iov_len) - v.iov_len, 671d879cb83SAl Viro v.iov_len, 0); 672d879cb83SAl Viro sum = csum_block_add(sum, next, off); 673d879cb83SAl Viro off += v.iov_len; 674d879cb83SAl Viro }) 675d879cb83SAl Viro ) 676d879cb83SAl Viro *csum = sum; 677d879cb83SAl Viro return bytes; 678d879cb83SAl Viro } 679d879cb83SAl Viro EXPORT_SYMBOL(csum_and_copy_from_iter); 680d879cb83SAl Viro 681d879cb83SAl Viro size_t csum_and_copy_to_iter(void *addr, size_t bytes, __wsum *csum, 682d879cb83SAl Viro struct iov_iter *i) 683d879cb83SAl Viro { 684d879cb83SAl Viro char *from = addr; 685d879cb83SAl Viro __wsum sum, next; 686d879cb83SAl Viro size_t off = 0; 687d879cb83SAl Viro if (unlikely(bytes > i->count)) 688d879cb83SAl Viro bytes = i->count; 689d879cb83SAl Viro 690d879cb83SAl Viro if (unlikely(!bytes)) 691d879cb83SAl Viro return 0; 692d879cb83SAl Viro 693d879cb83SAl Viro sum = *csum; 694d879cb83SAl Viro iterate_and_advance(i, bytes, v, ({ 695d879cb83SAl Viro int err = 0; 696d879cb83SAl Viro next = csum_and_copy_to_user((from += v.iov_len) - v.iov_len, 697d879cb83SAl Viro v.iov_base, 698d879cb83SAl Viro v.iov_len, 0, &err); 699d879cb83SAl Viro if (!err) { 700d879cb83SAl Viro sum = csum_block_add(sum, next, off); 701d879cb83SAl Viro off += v.iov_len; 702d879cb83SAl Viro } 703d879cb83SAl Viro err ? v.iov_len : 0; 704d879cb83SAl Viro }), ({ 705d879cb83SAl Viro char *p = kmap_atomic(v.bv_page); 706d879cb83SAl Viro next = csum_partial_copy_nocheck((from += v.bv_len) - v.bv_len, 707d879cb83SAl Viro p + v.bv_offset, 708d879cb83SAl Viro v.bv_len, 0); 709d879cb83SAl Viro kunmap_atomic(p); 710d879cb83SAl Viro sum = csum_block_add(sum, next, off); 711d879cb83SAl Viro off += v.bv_len; 712d879cb83SAl Viro }),({ 713d879cb83SAl Viro next = csum_partial_copy_nocheck((from += v.iov_len) - v.iov_len, 714d879cb83SAl Viro v.iov_base, 715d879cb83SAl Viro v.iov_len, 0); 716d879cb83SAl Viro sum = csum_block_add(sum, next, off); 717d879cb83SAl Viro off += v.iov_len; 718d879cb83SAl Viro }) 719d879cb83SAl Viro ) 720d879cb83SAl Viro *csum = sum; 721d879cb83SAl Viro return bytes; 722d879cb83SAl Viro } 723d879cb83SAl Viro EXPORT_SYMBOL(csum_and_copy_to_iter); 724d879cb83SAl Viro 725d879cb83SAl Viro int iov_iter_npages(const struct iov_iter *i, int maxpages) 726d879cb83SAl Viro { 727d879cb83SAl Viro size_t size = i->count; 728d879cb83SAl Viro int npages = 0; 729d879cb83SAl Viro 730d879cb83SAl Viro if (!size) 731d879cb83SAl Viro return 0; 732d879cb83SAl Viro 733d879cb83SAl Viro iterate_all_kinds(i, size, v, ({ 734d879cb83SAl Viro unsigned long p = (unsigned long)v.iov_base; 735d879cb83SAl Viro npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 736d879cb83SAl Viro - p / PAGE_SIZE; 737d879cb83SAl Viro if (npages >= maxpages) 738d879cb83SAl Viro return maxpages; 739d879cb83SAl Viro 0;}),({ 740d879cb83SAl Viro npages++; 741d879cb83SAl Viro if (npages >= maxpages) 742d879cb83SAl Viro return maxpages; 743d879cb83SAl Viro }),({ 744d879cb83SAl Viro unsigned long p = (unsigned long)v.iov_base; 745d879cb83SAl Viro npages += DIV_ROUND_UP(p + v.iov_len, PAGE_SIZE) 746d879cb83SAl Viro - p / PAGE_SIZE; 747d879cb83SAl Viro if (npages >= maxpages) 748d879cb83SAl Viro return maxpages; 749d879cb83SAl Viro }) 750d879cb83SAl Viro ) 751d879cb83SAl Viro return npages; 752d879cb83SAl Viro } 753d879cb83SAl Viro EXPORT_SYMBOL(iov_iter_npages); 754d879cb83SAl Viro 755d879cb83SAl Viro const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags) 756d879cb83SAl Viro { 757d879cb83SAl Viro *new = *old; 758d879cb83SAl Viro if (new->type & ITER_BVEC) 759d879cb83SAl Viro return new->bvec = kmemdup(new->bvec, 760d879cb83SAl Viro new->nr_segs * sizeof(struct bio_vec), 761d879cb83SAl Viro flags); 762d879cb83SAl Viro else 763d879cb83SAl Viro /* iovec and kvec have identical layout */ 764d879cb83SAl Viro return new->iov = kmemdup(new->iov, 765d879cb83SAl Viro new->nr_segs * sizeof(struct iovec), 766d879cb83SAl Viro flags); 767d879cb83SAl Viro } 768d879cb83SAl Viro EXPORT_SYMBOL(dup_iter); 769