1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 286db1e29SJens Axboe /* 386db1e29SJens Axboe * Functions related to mapping data to requests 486db1e29SJens Axboe */ 586db1e29SJens Axboe #include <linux/kernel.h> 668db0cf1SIngo Molnar #include <linux/sched/task_stack.h> 786db1e29SJens Axboe #include <linux/module.h> 886db1e29SJens Axboe #include <linux/bio.h> 986db1e29SJens Axboe #include <linux/blkdev.h> 1026e49cfcSKent Overstreet #include <linux/uio.h> 1186db1e29SJens Axboe 1286db1e29SJens Axboe #include "blk.h" 1386db1e29SJens Axboe 14130879f1SChristoph Hellwig struct bio_map_data { 15f3256075SChristoph Hellwig bool is_our_pages : 1; 16f3256075SChristoph Hellwig bool is_null_mapped : 1; 17130879f1SChristoph Hellwig struct iov_iter iter; 18130879f1SChristoph Hellwig struct iovec iov[]; 19130879f1SChristoph Hellwig }; 20130879f1SChristoph Hellwig 21130879f1SChristoph Hellwig static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, 22130879f1SChristoph Hellwig gfp_t gfp_mask) 23130879f1SChristoph Hellwig { 24130879f1SChristoph Hellwig struct bio_map_data *bmd; 25130879f1SChristoph Hellwig 26130879f1SChristoph Hellwig if (data->nr_segs > UIO_MAXIOV) 27130879f1SChristoph Hellwig return NULL; 28130879f1SChristoph Hellwig 29130879f1SChristoph Hellwig bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); 30130879f1SChristoph Hellwig if (!bmd) 31130879f1SChristoph Hellwig return NULL; 32130879f1SChristoph Hellwig memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); 33130879f1SChristoph Hellwig bmd->iter = *data; 34130879f1SChristoph Hellwig bmd->iter.iov = bmd->iov; 35130879f1SChristoph Hellwig return bmd; 36130879f1SChristoph Hellwig } 37130879f1SChristoph Hellwig 38130879f1SChristoph Hellwig /** 39130879f1SChristoph Hellwig * bio_copy_from_iter - copy all pages from iov_iter to bio 40130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as destination 41130879f1SChristoph Hellwig * @iter: iov_iter as source 42130879f1SChristoph Hellwig * 43130879f1SChristoph Hellwig * Copy all pages from iov_iter to bio. 44130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 45130879f1SChristoph Hellwig */ 46130879f1SChristoph Hellwig static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) 47130879f1SChristoph Hellwig { 48130879f1SChristoph Hellwig struct bio_vec *bvec; 49130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 50130879f1SChristoph Hellwig 51130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 52130879f1SChristoph Hellwig ssize_t ret; 53130879f1SChristoph Hellwig 54130879f1SChristoph Hellwig ret = copy_page_from_iter(bvec->bv_page, 55130879f1SChristoph Hellwig bvec->bv_offset, 56130879f1SChristoph Hellwig bvec->bv_len, 57130879f1SChristoph Hellwig iter); 58130879f1SChristoph Hellwig 59130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 60130879f1SChristoph Hellwig break; 61130879f1SChristoph Hellwig 62130879f1SChristoph Hellwig if (ret < bvec->bv_len) 63130879f1SChristoph Hellwig return -EFAULT; 64130879f1SChristoph Hellwig } 65130879f1SChristoph Hellwig 66130879f1SChristoph Hellwig return 0; 67130879f1SChristoph Hellwig } 68130879f1SChristoph Hellwig 69130879f1SChristoph Hellwig /** 70130879f1SChristoph Hellwig * bio_copy_to_iter - copy all pages from bio to iov_iter 71130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as source 72130879f1SChristoph Hellwig * @iter: iov_iter as destination 73130879f1SChristoph Hellwig * 74130879f1SChristoph Hellwig * Copy all pages from bio to iov_iter. 75130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 76130879f1SChristoph Hellwig */ 77130879f1SChristoph Hellwig static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) 78130879f1SChristoph Hellwig { 79130879f1SChristoph Hellwig struct bio_vec *bvec; 80130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 81130879f1SChristoph Hellwig 82130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 83130879f1SChristoph Hellwig ssize_t ret; 84130879f1SChristoph Hellwig 85130879f1SChristoph Hellwig ret = copy_page_to_iter(bvec->bv_page, 86130879f1SChristoph Hellwig bvec->bv_offset, 87130879f1SChristoph Hellwig bvec->bv_len, 88130879f1SChristoph Hellwig &iter); 89130879f1SChristoph Hellwig 90130879f1SChristoph Hellwig if (!iov_iter_count(&iter)) 91130879f1SChristoph Hellwig break; 92130879f1SChristoph Hellwig 93130879f1SChristoph Hellwig if (ret < bvec->bv_len) 94130879f1SChristoph Hellwig return -EFAULT; 95130879f1SChristoph Hellwig } 96130879f1SChristoph Hellwig 97130879f1SChristoph Hellwig return 0; 98130879f1SChristoph Hellwig } 99130879f1SChristoph Hellwig 100130879f1SChristoph Hellwig /** 101130879f1SChristoph Hellwig * bio_uncopy_user - finish previously mapped bio 102130879f1SChristoph Hellwig * @bio: bio being terminated 103130879f1SChristoph Hellwig * 104130879f1SChristoph Hellwig * Free pages allocated from bio_copy_user_iov() and write back data 105130879f1SChristoph Hellwig * to user space in case of a read. 106130879f1SChristoph Hellwig */ 107130879f1SChristoph Hellwig static int bio_uncopy_user(struct bio *bio) 108130879f1SChristoph Hellwig { 109130879f1SChristoph Hellwig struct bio_map_data *bmd = bio->bi_private; 110130879f1SChristoph Hellwig int ret = 0; 111130879f1SChristoph Hellwig 1123310eebaSChristoph Hellwig if (!bmd->is_null_mapped) { 113130879f1SChristoph Hellwig /* 114130879f1SChristoph Hellwig * if we're in a workqueue, the request is orphaned, so 115130879f1SChristoph Hellwig * don't copy into a random user address space, just free 116130879f1SChristoph Hellwig * and return -EINTR so user space doesn't expect any data. 117130879f1SChristoph Hellwig */ 118130879f1SChristoph Hellwig if (!current->mm) 119130879f1SChristoph Hellwig ret = -EINTR; 120130879f1SChristoph Hellwig else if (bio_data_dir(bio) == READ) 121130879f1SChristoph Hellwig ret = bio_copy_to_iter(bio, bmd->iter); 122130879f1SChristoph Hellwig if (bmd->is_our_pages) 123130879f1SChristoph Hellwig bio_free_pages(bio); 124130879f1SChristoph Hellwig } 125130879f1SChristoph Hellwig kfree(bmd); 126130879f1SChristoph Hellwig return ret; 127130879f1SChristoph Hellwig } 128130879f1SChristoph Hellwig 1297589ad67SChristoph Hellwig static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, 1307589ad67SChristoph Hellwig struct iov_iter *iter, gfp_t gfp_mask) 131130879f1SChristoph Hellwig { 132130879f1SChristoph Hellwig struct bio_map_data *bmd; 133130879f1SChristoph Hellwig struct page *page; 134393bb12eSChristoph Hellwig struct bio *bio; 135130879f1SChristoph Hellwig int i = 0, ret; 136130879f1SChristoph Hellwig int nr_pages; 137130879f1SChristoph Hellwig unsigned int len = iter->count; 138130879f1SChristoph Hellwig unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; 139130879f1SChristoph Hellwig 140130879f1SChristoph Hellwig bmd = bio_alloc_map_data(iter, gfp_mask); 141130879f1SChristoph Hellwig if (!bmd) 1427589ad67SChristoph Hellwig return -ENOMEM; 143130879f1SChristoph Hellwig 144130879f1SChristoph Hellwig /* 145130879f1SChristoph Hellwig * We need to do a deep copy of the iov_iter including the iovecs. 146130879f1SChristoph Hellwig * The caller provided iov might point to an on-stack or otherwise 147130879f1SChristoph Hellwig * shortlived one. 148130879f1SChristoph Hellwig */ 149f3256075SChristoph Hellwig bmd->is_our_pages = !map_data; 15003859717SChristoph Hellwig bmd->is_null_mapped = (map_data && map_data->null_mapped); 151130879f1SChristoph Hellwig 1525f7136dbSMatthew Wilcox (Oracle) nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); 153130879f1SChristoph Hellwig 154130879f1SChristoph Hellwig ret = -ENOMEM; 155066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 156130879f1SChristoph Hellwig if (!bio) 157130879f1SChristoph Hellwig goto out_bmd; 158066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); 159130879f1SChristoph Hellwig 160130879f1SChristoph Hellwig if (map_data) { 161f5d632d1SJens Axboe nr_pages = 1U << map_data->page_order; 162130879f1SChristoph Hellwig i = map_data->offset / PAGE_SIZE; 163130879f1SChristoph Hellwig } 164130879f1SChristoph Hellwig while (len) { 165130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 166130879f1SChristoph Hellwig 167130879f1SChristoph Hellwig bytes -= offset; 168130879f1SChristoph Hellwig 169130879f1SChristoph Hellwig if (bytes > len) 170130879f1SChristoph Hellwig bytes = len; 171130879f1SChristoph Hellwig 172130879f1SChristoph Hellwig if (map_data) { 173130879f1SChristoph Hellwig if (i == map_data->nr_entries * nr_pages) { 174130879f1SChristoph Hellwig ret = -ENOMEM; 1757589ad67SChristoph Hellwig goto cleanup; 176130879f1SChristoph Hellwig } 177130879f1SChristoph Hellwig 178130879f1SChristoph Hellwig page = map_data->pages[i / nr_pages]; 179130879f1SChristoph Hellwig page += (i % nr_pages); 180130879f1SChristoph Hellwig 181130879f1SChristoph Hellwig i++; 182130879f1SChristoph Hellwig } else { 183ce288e05SChristoph Hellwig page = alloc_page(GFP_NOIO | gfp_mask); 184130879f1SChristoph Hellwig if (!page) { 185130879f1SChristoph Hellwig ret = -ENOMEM; 1867589ad67SChristoph Hellwig goto cleanup; 187130879f1SChristoph Hellwig } 188130879f1SChristoph Hellwig } 189130879f1SChristoph Hellwig 1907589ad67SChristoph Hellwig if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) { 191130879f1SChristoph Hellwig if (!map_data) 192130879f1SChristoph Hellwig __free_page(page); 193130879f1SChristoph Hellwig break; 194130879f1SChristoph Hellwig } 195130879f1SChristoph Hellwig 196130879f1SChristoph Hellwig len -= bytes; 197130879f1SChristoph Hellwig offset = 0; 198130879f1SChristoph Hellwig } 199130879f1SChristoph Hellwig 200130879f1SChristoph Hellwig if (map_data) 201130879f1SChristoph Hellwig map_data->offset += bio->bi_iter.bi_size; 202130879f1SChristoph Hellwig 203130879f1SChristoph Hellwig /* 204130879f1SChristoph Hellwig * success 205130879f1SChristoph Hellwig */ 206130879f1SChristoph Hellwig if ((iov_iter_rw(iter) == WRITE && 207130879f1SChristoph Hellwig (!map_data || !map_data->null_mapped)) || 208130879f1SChristoph Hellwig (map_data && map_data->from_user)) { 209130879f1SChristoph Hellwig ret = bio_copy_from_iter(bio, iter); 210130879f1SChristoph Hellwig if (ret) 211130879f1SChristoph Hellwig goto cleanup; 212130879f1SChristoph Hellwig } else { 213130879f1SChristoph Hellwig if (bmd->is_our_pages) 214130879f1SChristoph Hellwig zero_fill_bio(bio); 215130879f1SChristoph Hellwig iov_iter_advance(iter, bio->bi_iter.bi_size); 216130879f1SChristoph Hellwig } 217130879f1SChristoph Hellwig 218130879f1SChristoph Hellwig bio->bi_private = bmd; 2197589ad67SChristoph Hellwig 220393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 2217589ad67SChristoph Hellwig if (ret) 2227589ad67SChristoph Hellwig goto cleanup; 2237589ad67SChristoph Hellwig return 0; 224130879f1SChristoph Hellwig cleanup: 225130879f1SChristoph Hellwig if (!map_data) 226130879f1SChristoph Hellwig bio_free_pages(bio); 227066ff571SChristoph Hellwig bio_uninit(bio); 228066ff571SChristoph Hellwig kfree(bio); 229130879f1SChristoph Hellwig out_bmd: 230130879f1SChristoph Hellwig kfree(bmd); 2317589ad67SChristoph Hellwig return ret; 232130879f1SChristoph Hellwig } 233130879f1SChristoph Hellwig 23432f1c71bSAnuj Gupta static void blk_mq_map_bio_put(struct bio *bio) 2358af870aaSJens Axboe { 2368af870aaSJens Axboe if (bio->bi_opf & REQ_ALLOC_CACHE) { 2378af870aaSJens Axboe bio_put(bio); 2388af870aaSJens Axboe } else { 2398af870aaSJens Axboe bio_uninit(bio); 2408af870aaSJens Axboe kfree(bio); 2418af870aaSJens Axboe } 2428af870aaSJens Axboe } 2438af870aaSJens Axboe 244*ab89e8e7SKanchan Joshi static struct bio *blk_rq_map_bio_alloc(struct request *rq, 245*ab89e8e7SKanchan Joshi unsigned int nr_vecs, gfp_t gfp_mask) 246*ab89e8e7SKanchan Joshi { 247*ab89e8e7SKanchan Joshi struct bio *bio; 248*ab89e8e7SKanchan Joshi 249*ab89e8e7SKanchan Joshi if (rq->cmd_flags & REQ_POLLED) { 250*ab89e8e7SKanchan Joshi blk_opf_t opf = rq->cmd_flags | REQ_ALLOC_CACHE; 251*ab89e8e7SKanchan Joshi 252*ab89e8e7SKanchan Joshi bio = bio_alloc_bioset(NULL, nr_vecs, opf, gfp_mask, 253*ab89e8e7SKanchan Joshi &fs_bio_set); 254*ab89e8e7SKanchan Joshi if (!bio) 255*ab89e8e7SKanchan Joshi return NULL; 256*ab89e8e7SKanchan Joshi } else { 257*ab89e8e7SKanchan Joshi bio = bio_kmalloc(nr_vecs, gfp_mask); 258*ab89e8e7SKanchan Joshi if (!bio) 259*ab89e8e7SKanchan Joshi return NULL; 260*ab89e8e7SKanchan Joshi bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); 261*ab89e8e7SKanchan Joshi } 262*ab89e8e7SKanchan Joshi return bio; 263*ab89e8e7SKanchan Joshi } 264*ab89e8e7SKanchan Joshi 2657589ad67SChristoph Hellwig static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, 2667589ad67SChristoph Hellwig gfp_t gfp_mask) 267130879f1SChristoph Hellwig { 2687589ad67SChristoph Hellwig unsigned int max_sectors = queue_max_hw_sectors(rq->q); 269066ff571SChristoph Hellwig unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); 270393bb12eSChristoph Hellwig struct bio *bio; 271130879f1SChristoph Hellwig int ret; 2727589ad67SChristoph Hellwig int j; 273130879f1SChristoph Hellwig 274130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 2757589ad67SChristoph Hellwig return -EINVAL; 276130879f1SChristoph Hellwig 277*ab89e8e7SKanchan Joshi bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); 278*ab89e8e7SKanchan Joshi if (bio == NULL) 2798af870aaSJens Axboe return -ENOMEM; 280130879f1SChristoph Hellwig 281130879f1SChristoph Hellwig while (iov_iter_count(iter)) { 282e88811bcSJens Axboe struct page **pages, *stack_pages[UIO_FASTIOV]; 283130879f1SChristoph Hellwig ssize_t bytes; 28491e5addaSJiapeng Chong size_t offs; 285130879f1SChristoph Hellwig int npages; 286130879f1SChristoph Hellwig 287e88811bcSJens Axboe if (nr_vecs <= ARRAY_SIZE(stack_pages)) { 288e88811bcSJens Axboe pages = stack_pages; 289e88811bcSJens Axboe bytes = iov_iter_get_pages2(iter, pages, LONG_MAX, 290e88811bcSJens Axboe nr_vecs, &offs); 291e88811bcSJens Axboe } else { 292e88811bcSJens Axboe bytes = iov_iter_get_pages_alloc2(iter, &pages, 293e88811bcSJens Axboe LONG_MAX, &offs); 294e88811bcSJens Axboe } 295130879f1SChristoph Hellwig if (unlikely(bytes <= 0)) { 296130879f1SChristoph Hellwig ret = bytes ? bytes : -EFAULT; 297130879f1SChristoph Hellwig goto out_unmap; 298130879f1SChristoph Hellwig } 299130879f1SChristoph Hellwig 300130879f1SChristoph Hellwig npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); 301130879f1SChristoph Hellwig 3027ab89db9SMichal Orzel if (unlikely(offs & queue_dma_alignment(rq->q))) 303130879f1SChristoph Hellwig j = 0; 3047ab89db9SMichal Orzel else { 305130879f1SChristoph Hellwig for (j = 0; j < npages; j++) { 306130879f1SChristoph Hellwig struct page *page = pages[j]; 307130879f1SChristoph Hellwig unsigned int n = PAGE_SIZE - offs; 308130879f1SChristoph Hellwig bool same_page = false; 309130879f1SChristoph Hellwig 310130879f1SChristoph Hellwig if (n > bytes) 311130879f1SChristoph Hellwig n = bytes; 312130879f1SChristoph Hellwig 3137589ad67SChristoph Hellwig if (!bio_add_hw_page(rq->q, bio, page, n, offs, 314e4581105SChristoph Hellwig max_sectors, &same_page)) { 315130879f1SChristoph Hellwig if (same_page) 316130879f1SChristoph Hellwig put_page(page); 317130879f1SChristoph Hellwig break; 318130879f1SChristoph Hellwig } 319130879f1SChristoph Hellwig 320130879f1SChristoph Hellwig bytes -= n; 321130879f1SChristoph Hellwig offs = 0; 322130879f1SChristoph Hellwig } 323130879f1SChristoph Hellwig } 324130879f1SChristoph Hellwig /* 325130879f1SChristoph Hellwig * release the pages we didn't map into the bio, if any 326130879f1SChristoph Hellwig */ 327130879f1SChristoph Hellwig while (j < npages) 328130879f1SChristoph Hellwig put_page(pages[j++]); 329e88811bcSJens Axboe if (pages != stack_pages) 330130879f1SChristoph Hellwig kvfree(pages); 331130879f1SChristoph Hellwig /* couldn't stuff something into bio? */ 332480cb846SAl Viro if (bytes) { 333480cb846SAl Viro iov_iter_revert(iter, bytes); 334130879f1SChristoph Hellwig break; 335130879f1SChristoph Hellwig } 336480cb846SAl Viro } 337130879f1SChristoph Hellwig 338393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 3397589ad67SChristoph Hellwig if (ret) 340393bb12eSChristoph Hellwig goto out_unmap; 3417589ad67SChristoph Hellwig return 0; 3427589ad67SChristoph Hellwig 343130879f1SChristoph Hellwig out_unmap: 344130879f1SChristoph Hellwig bio_release_pages(bio, false); 34532f1c71bSAnuj Gupta blk_mq_map_bio_put(bio); 3467589ad67SChristoph Hellwig return ret; 347130879f1SChristoph Hellwig } 348130879f1SChristoph Hellwig 349130879f1SChristoph Hellwig static void bio_invalidate_vmalloc_pages(struct bio *bio) 350130879f1SChristoph Hellwig { 351f358afc5SChristoph Hellwig #ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 352130879f1SChristoph Hellwig if (bio->bi_private && !op_is_write(bio_op(bio))) { 353130879f1SChristoph Hellwig unsigned long i, len = 0; 354130879f1SChristoph Hellwig 355130879f1SChristoph Hellwig for (i = 0; i < bio->bi_vcnt; i++) 356130879f1SChristoph Hellwig len += bio->bi_io_vec[i].bv_len; 357130879f1SChristoph Hellwig invalidate_kernel_vmap_range(bio->bi_private, len); 358130879f1SChristoph Hellwig } 359130879f1SChristoph Hellwig #endif 360130879f1SChristoph Hellwig } 361130879f1SChristoph Hellwig 362130879f1SChristoph Hellwig static void bio_map_kern_endio(struct bio *bio) 363130879f1SChristoph Hellwig { 364130879f1SChristoph Hellwig bio_invalidate_vmalloc_pages(bio); 365066ff571SChristoph Hellwig bio_uninit(bio); 366066ff571SChristoph Hellwig kfree(bio); 367130879f1SChristoph Hellwig } 368130879f1SChristoph Hellwig 369130879f1SChristoph Hellwig /** 370130879f1SChristoph Hellwig * bio_map_kern - map kernel address into bio 371130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 372130879f1SChristoph Hellwig * @data: pointer to buffer to map 373130879f1SChristoph Hellwig * @len: length in bytes 374130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio allocation 375130879f1SChristoph Hellwig * 376130879f1SChristoph Hellwig * Map the kernel address into a bio suitable for io to a block 377130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 378130879f1SChristoph Hellwig */ 379130879f1SChristoph Hellwig static struct bio *bio_map_kern(struct request_queue *q, void *data, 380130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask) 381130879f1SChristoph Hellwig { 382130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 383130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 384130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 385130879f1SChristoph Hellwig const int nr_pages = end - start; 386130879f1SChristoph Hellwig bool is_vmalloc = is_vmalloc_addr(data); 387130879f1SChristoph Hellwig struct page *page; 388130879f1SChristoph Hellwig int offset, i; 389130879f1SChristoph Hellwig struct bio *bio; 390130879f1SChristoph Hellwig 391066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 392130879f1SChristoph Hellwig if (!bio) 393130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 394066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 395130879f1SChristoph Hellwig 396130879f1SChristoph Hellwig if (is_vmalloc) { 397130879f1SChristoph Hellwig flush_kernel_vmap_range(data, len); 398130879f1SChristoph Hellwig bio->bi_private = data; 399130879f1SChristoph Hellwig } 400130879f1SChristoph Hellwig 401130879f1SChristoph Hellwig offset = offset_in_page(kaddr); 402130879f1SChristoph Hellwig for (i = 0; i < nr_pages; i++) { 403130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE - offset; 404130879f1SChristoph Hellwig 405130879f1SChristoph Hellwig if (len <= 0) 406130879f1SChristoph Hellwig break; 407130879f1SChristoph Hellwig 408130879f1SChristoph Hellwig if (bytes > len) 409130879f1SChristoph Hellwig bytes = len; 410130879f1SChristoph Hellwig 411130879f1SChristoph Hellwig if (!is_vmalloc) 412130879f1SChristoph Hellwig page = virt_to_page(data); 413130879f1SChristoph Hellwig else 414130879f1SChristoph Hellwig page = vmalloc_to_page(data); 415130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 416130879f1SChristoph Hellwig offset) < bytes) { 417130879f1SChristoph Hellwig /* we don't support partial mappings */ 418066ff571SChristoph Hellwig bio_uninit(bio); 419066ff571SChristoph Hellwig kfree(bio); 420130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 421130879f1SChristoph Hellwig } 422130879f1SChristoph Hellwig 423130879f1SChristoph Hellwig data += bytes; 424130879f1SChristoph Hellwig len -= bytes; 425130879f1SChristoph Hellwig offset = 0; 426130879f1SChristoph Hellwig } 427130879f1SChristoph Hellwig 428130879f1SChristoph Hellwig bio->bi_end_io = bio_map_kern_endio; 429130879f1SChristoph Hellwig return bio; 430130879f1SChristoph Hellwig } 431130879f1SChristoph Hellwig 432130879f1SChristoph Hellwig static void bio_copy_kern_endio(struct bio *bio) 433130879f1SChristoph Hellwig { 434130879f1SChristoph Hellwig bio_free_pages(bio); 435066ff571SChristoph Hellwig bio_uninit(bio); 436066ff571SChristoph Hellwig kfree(bio); 437130879f1SChristoph Hellwig } 438130879f1SChristoph Hellwig 439130879f1SChristoph Hellwig static void bio_copy_kern_endio_read(struct bio *bio) 440130879f1SChristoph Hellwig { 441130879f1SChristoph Hellwig char *p = bio->bi_private; 442130879f1SChristoph Hellwig struct bio_vec *bvec; 443130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 444130879f1SChristoph Hellwig 445130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 446d24920e2SChristoph Hellwig memcpy_from_bvec(p, bvec); 447130879f1SChristoph Hellwig p += bvec->bv_len; 448130879f1SChristoph Hellwig } 449130879f1SChristoph Hellwig 450130879f1SChristoph Hellwig bio_copy_kern_endio(bio); 451130879f1SChristoph Hellwig } 452130879f1SChristoph Hellwig 453130879f1SChristoph Hellwig /** 454130879f1SChristoph Hellwig * bio_copy_kern - copy kernel address into bio 455130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 456130879f1SChristoph Hellwig * @data: pointer to buffer to copy 457130879f1SChristoph Hellwig * @len: length in bytes 458130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio and page allocation 459130879f1SChristoph Hellwig * @reading: data direction is READ 460130879f1SChristoph Hellwig * 461130879f1SChristoph Hellwig * copy the kernel address into a bio suitable for io to a block 462130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 463130879f1SChristoph Hellwig */ 464130879f1SChristoph Hellwig static struct bio *bio_copy_kern(struct request_queue *q, void *data, 465130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask, int reading) 466130879f1SChristoph Hellwig { 467130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 468130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 469130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 470130879f1SChristoph Hellwig struct bio *bio; 471130879f1SChristoph Hellwig void *p = data; 472130879f1SChristoph Hellwig int nr_pages = 0; 473130879f1SChristoph Hellwig 474130879f1SChristoph Hellwig /* 475130879f1SChristoph Hellwig * Overflow, abort 476130879f1SChristoph Hellwig */ 477130879f1SChristoph Hellwig if (end < start) 478130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 479130879f1SChristoph Hellwig 480130879f1SChristoph Hellwig nr_pages = end - start; 481066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 482130879f1SChristoph Hellwig if (!bio) 483130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 484066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 485130879f1SChristoph Hellwig 486130879f1SChristoph Hellwig while (len) { 487130879f1SChristoph Hellwig struct page *page; 488130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 489130879f1SChristoph Hellwig 490130879f1SChristoph Hellwig if (bytes > len) 491130879f1SChristoph Hellwig bytes = len; 492130879f1SChristoph Hellwig 493cc8f7fe1SHaimin Zhang page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask); 494130879f1SChristoph Hellwig if (!page) 495130879f1SChristoph Hellwig goto cleanup; 496130879f1SChristoph Hellwig 497130879f1SChristoph Hellwig if (!reading) 498130879f1SChristoph Hellwig memcpy(page_address(page), p, bytes); 499130879f1SChristoph Hellwig 500130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) 501130879f1SChristoph Hellwig break; 502130879f1SChristoph Hellwig 503130879f1SChristoph Hellwig len -= bytes; 504130879f1SChristoph Hellwig p += bytes; 505130879f1SChristoph Hellwig } 506130879f1SChristoph Hellwig 507130879f1SChristoph Hellwig if (reading) { 508130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio_read; 509130879f1SChristoph Hellwig bio->bi_private = data; 510130879f1SChristoph Hellwig } else { 511130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio; 512130879f1SChristoph Hellwig } 513130879f1SChristoph Hellwig 514130879f1SChristoph Hellwig return bio; 515130879f1SChristoph Hellwig 516130879f1SChristoph Hellwig cleanup: 517130879f1SChristoph Hellwig bio_free_pages(bio); 518066ff571SChristoph Hellwig bio_uninit(bio); 519066ff571SChristoph Hellwig kfree(bio); 520130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 521130879f1SChristoph Hellwig } 522130879f1SChristoph Hellwig 52398d61d5bSChristoph Hellwig /* 5240abc2a10SJens Axboe * Append a bio to a passthrough request. Only works if the bio can be merged 5250abc2a10SJens Axboe * into the request based on the driver constraints. 52698d61d5bSChristoph Hellwig */ 527393bb12eSChristoph Hellwig int blk_rq_append_bio(struct request *rq, struct bio *bio) 52886db1e29SJens Axboe { 52914ccb66bSChristoph Hellwig struct bvec_iter iter; 53014ccb66bSChristoph Hellwig struct bio_vec bv; 53114ccb66bSChristoph Hellwig unsigned int nr_segs = 0; 5320abc2a10SJens Axboe 533393bb12eSChristoph Hellwig bio_for_each_bvec(bv, bio, iter) 53414ccb66bSChristoph Hellwig nr_segs++; 53514ccb66bSChristoph Hellwig 53698d61d5bSChristoph Hellwig if (!rq->bio) { 537393bb12eSChristoph Hellwig blk_rq_bio_prep(rq, bio, nr_segs); 53898d61d5bSChristoph Hellwig } else { 539393bb12eSChristoph Hellwig if (!ll_back_merge_fn(rq, bio, nr_segs)) 54086db1e29SJens Axboe return -EINVAL; 541393bb12eSChristoph Hellwig rq->biotail->bi_next = bio; 542393bb12eSChristoph Hellwig rq->biotail = bio; 543393bb12eSChristoph Hellwig rq->__data_len += (bio)->bi_iter.bi_size; 544393bb12eSChristoph Hellwig bio_crypt_free_ctx(bio); 54586db1e29SJens Axboe } 54698d61d5bSChristoph Hellwig 54786db1e29SJens Axboe return 0; 54886db1e29SJens Axboe } 54998d61d5bSChristoph Hellwig EXPORT_SYMBOL(blk_rq_append_bio); 55086db1e29SJens Axboe 55186db1e29SJens Axboe /** 552aebf526bSChristoph Hellwig * blk_rq_map_user_iov - map user data to a request, for passthrough requests 55386db1e29SJens Axboe * @q: request queue where request should be inserted 55486db1e29SJens Axboe * @rq: request to map data to 555152e283fSFUJITA Tomonori * @map_data: pointer to the rq_map_data holding pages (if necessary) 55626e49cfcSKent Overstreet * @iter: iovec iterator 557a3bce90eSFUJITA Tomonori * @gfp_mask: memory allocation flags 55886db1e29SJens Axboe * 55986db1e29SJens Axboe * Description: 560710027a4SRandy Dunlap * Data will be mapped directly for zero copy I/O, if possible. Otherwise 56186db1e29SJens Axboe * a kernel bounce buffer is used. 56286db1e29SJens Axboe * 563710027a4SRandy Dunlap * A matching blk_rq_unmap_user() must be issued at the end of I/O, while 56486db1e29SJens Axboe * still in process context. 56586db1e29SJens Axboe */ 56686db1e29SJens Axboe int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, 56726e49cfcSKent Overstreet struct rq_map_data *map_data, 56826e49cfcSKent Overstreet const struct iov_iter *iter, gfp_t gfp_mask) 56986db1e29SJens Axboe { 570357f435dSAl Viro bool copy = false; 571357f435dSAl Viro unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); 5724d6af73dSChristoph Hellwig struct bio *bio = NULL; 5734d6af73dSChristoph Hellwig struct iov_iter i; 57469e0927bSDouglas Gilbert int ret = -EINVAL; 57586db1e29SJens Axboe 576a0ac402cSLinus Torvalds if (!iter_is_iovec(iter)) 577a0ac402cSLinus Torvalds goto fail; 578a0ac402cSLinus Torvalds 579357f435dSAl Viro if (map_data) 5804d6af73dSChristoph Hellwig copy = true; 581393bb12eSChristoph Hellwig else if (blk_queue_may_bounce(q)) 582393bb12eSChristoph Hellwig copy = true; 583357f435dSAl Viro else if (iov_iter_alignment(iter) & align) 584357f435dSAl Viro copy = true; 585357f435dSAl Viro else if (queue_virt_boundary(q)) 586357f435dSAl Viro copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); 587afdc1a78SFUJITA Tomonori 5884d6af73dSChristoph Hellwig i = *iter; 5894d6af73dSChristoph Hellwig do { 5907589ad67SChristoph Hellwig if (copy) 5917589ad67SChristoph Hellwig ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask); 5927589ad67SChristoph Hellwig else 5937589ad67SChristoph Hellwig ret = bio_map_user_iov(rq, &i, gfp_mask); 5944d6af73dSChristoph Hellwig if (ret) 5954d6af73dSChristoph Hellwig goto unmap_rq; 5964d6af73dSChristoph Hellwig if (!bio) 5974d6af73dSChristoph Hellwig bio = rq->bio; 5984d6af73dSChristoph Hellwig } while (iov_iter_count(&i)); 59986db1e29SJens Axboe 60086db1e29SJens Axboe return 0; 6014d6af73dSChristoph Hellwig 6024d6af73dSChristoph Hellwig unmap_rq: 6033b7995a9SYang Yingliang blk_rq_unmap_user(bio); 604a0ac402cSLinus Torvalds fail: 6054d6af73dSChristoph Hellwig rq->bio = NULL; 60669e0927bSDouglas Gilbert return ret; 60786db1e29SJens Axboe } 608152e283fSFUJITA Tomonori EXPORT_SYMBOL(blk_rq_map_user_iov); 60986db1e29SJens Axboe 610ddad8dd0SChristoph Hellwig int blk_rq_map_user(struct request_queue *q, struct request *rq, 611ddad8dd0SChristoph Hellwig struct rq_map_data *map_data, void __user *ubuf, 612ddad8dd0SChristoph Hellwig unsigned long len, gfp_t gfp_mask) 613ddad8dd0SChristoph Hellwig { 61426e49cfcSKent Overstreet struct iovec iov; 61526e49cfcSKent Overstreet struct iov_iter i; 6168f7e885aSAl Viro int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i); 617ddad8dd0SChristoph Hellwig 6188f7e885aSAl Viro if (unlikely(ret < 0)) 6198f7e885aSAl Viro return ret; 620ddad8dd0SChristoph Hellwig 62126e49cfcSKent Overstreet return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); 622ddad8dd0SChristoph Hellwig } 623ddad8dd0SChristoph Hellwig EXPORT_SYMBOL(blk_rq_map_user); 624ddad8dd0SChristoph Hellwig 62555765402SAnuj Gupta int blk_rq_map_user_io(struct request *req, struct rq_map_data *map_data, 62655765402SAnuj Gupta void __user *ubuf, unsigned long buf_len, gfp_t gfp_mask, 62755765402SAnuj Gupta bool vec, int iov_count, bool check_iter_count, int rw) 62855765402SAnuj Gupta { 62955765402SAnuj Gupta int ret = 0; 63055765402SAnuj Gupta 63155765402SAnuj Gupta if (vec) { 63255765402SAnuj Gupta struct iovec fast_iov[UIO_FASTIOV]; 63355765402SAnuj Gupta struct iovec *iov = fast_iov; 63455765402SAnuj Gupta struct iov_iter iter; 63555765402SAnuj Gupta 63655765402SAnuj Gupta ret = import_iovec(rw, ubuf, iov_count ? iov_count : buf_len, 63755765402SAnuj Gupta UIO_FASTIOV, &iov, &iter); 63855765402SAnuj Gupta if (ret < 0) 63955765402SAnuj Gupta return ret; 64055765402SAnuj Gupta 64155765402SAnuj Gupta if (iov_count) { 64255765402SAnuj Gupta /* SG_IO howto says that the shorter of the two wins */ 64355765402SAnuj Gupta iov_iter_truncate(&iter, buf_len); 64455765402SAnuj Gupta if (check_iter_count && !iov_iter_count(&iter)) { 64555765402SAnuj Gupta kfree(iov); 64655765402SAnuj Gupta return -EINVAL; 64755765402SAnuj Gupta } 64855765402SAnuj Gupta } 64955765402SAnuj Gupta 65055765402SAnuj Gupta ret = blk_rq_map_user_iov(req->q, req, map_data, &iter, 65155765402SAnuj Gupta gfp_mask); 65255765402SAnuj Gupta kfree(iov); 65355765402SAnuj Gupta } else if (buf_len) { 65455765402SAnuj Gupta ret = blk_rq_map_user(req->q, req, map_data, ubuf, buf_len, 65555765402SAnuj Gupta gfp_mask); 65655765402SAnuj Gupta } 65755765402SAnuj Gupta return ret; 65855765402SAnuj Gupta } 65955765402SAnuj Gupta EXPORT_SYMBOL(blk_rq_map_user_io); 66055765402SAnuj Gupta 66186db1e29SJens Axboe /** 66286db1e29SJens Axboe * blk_rq_unmap_user - unmap a request with user data 66386db1e29SJens Axboe * @bio: start of bio list 66486db1e29SJens Axboe * 66586db1e29SJens Axboe * Description: 66686db1e29SJens Axboe * Unmap a rq previously mapped by blk_rq_map_user(). The caller must 66786db1e29SJens Axboe * supply the original rq->bio from the blk_rq_map_user() return, since 668710027a4SRandy Dunlap * the I/O completion may have changed rq->bio. 66986db1e29SJens Axboe */ 67086db1e29SJens Axboe int blk_rq_unmap_user(struct bio *bio) 67186db1e29SJens Axboe { 672393bb12eSChristoph Hellwig struct bio *next_bio; 67386db1e29SJens Axboe int ret = 0, ret2; 67486db1e29SJens Axboe 67586db1e29SJens Axboe while (bio) { 6763310eebaSChristoph Hellwig if (bio->bi_private) { 677393bb12eSChristoph Hellwig ret2 = bio_uncopy_user(bio); 67886db1e29SJens Axboe if (ret2 && !ret) 67986db1e29SJens Axboe ret = ret2; 6803310eebaSChristoph Hellwig } else { 681393bb12eSChristoph Hellwig bio_release_pages(bio, bio_data_dir(bio) == READ); 6827b63c052SChristoph Hellwig } 68386db1e29SJens Axboe 684393bb12eSChristoph Hellwig next_bio = bio; 68586db1e29SJens Axboe bio = bio->bi_next; 68632f1c71bSAnuj Gupta blk_mq_map_bio_put(next_bio); 68786db1e29SJens Axboe } 68886db1e29SJens Axboe 68986db1e29SJens Axboe return ret; 69086db1e29SJens Axboe } 69186db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_unmap_user); 69286db1e29SJens Axboe 69386db1e29SJens Axboe /** 694aebf526bSChristoph Hellwig * blk_rq_map_kern - map kernel data to a request, for passthrough requests 69586db1e29SJens Axboe * @q: request queue where request should be inserted 69686db1e29SJens Axboe * @rq: request to fill 69786db1e29SJens Axboe * @kbuf: the kernel buffer 69886db1e29SJens Axboe * @len: length of user data 69986db1e29SJens Axboe * @gfp_mask: memory allocation flags 70068154e90SFUJITA Tomonori * 70168154e90SFUJITA Tomonori * Description: 70268154e90SFUJITA Tomonori * Data will be mapped directly if possible. Otherwise a bounce 703e227867fSMasanari Iida * buffer is used. Can be called multiple times to append multiple 7043a5a3927SJames Bottomley * buffers. 70586db1e29SJens Axboe */ 70686db1e29SJens Axboe int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, 70786db1e29SJens Axboe unsigned int len, gfp_t gfp_mask) 70886db1e29SJens Axboe { 70968154e90SFUJITA Tomonori int reading = rq_data_dir(rq) == READ; 71014417799SNamhyung Kim unsigned long addr = (unsigned long) kbuf; 711393bb12eSChristoph Hellwig struct bio *bio; 7123a5a3927SJames Bottomley int ret; 71386db1e29SJens Axboe 714ae03bf63SMartin K. Petersen if (len > (queue_max_hw_sectors(q) << 9)) 71586db1e29SJens Axboe return -EINVAL; 71686db1e29SJens Axboe if (!len || !kbuf) 71786db1e29SJens Axboe return -EINVAL; 71886db1e29SJens Axboe 719393bb12eSChristoph Hellwig if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || 720393bb12eSChristoph Hellwig blk_queue_may_bounce(q)) 72168154e90SFUJITA Tomonori bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); 72268154e90SFUJITA Tomonori else 72386db1e29SJens Axboe bio = bio_map_kern(q, kbuf, len, gfp_mask); 72468154e90SFUJITA Tomonori 72586db1e29SJens Axboe if (IS_ERR(bio)) 72686db1e29SJens Axboe return PTR_ERR(bio); 72786db1e29SJens Axboe 728aebf526bSChristoph Hellwig bio->bi_opf &= ~REQ_OP_MASK; 729aebf526bSChristoph Hellwig bio->bi_opf |= req_op(rq); 73086db1e29SJens Axboe 731393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 732066ff571SChristoph Hellwig if (unlikely(ret)) { 733066ff571SChristoph Hellwig bio_uninit(bio); 734066ff571SChristoph Hellwig kfree(bio); 735066ff571SChristoph Hellwig } 7363a5a3927SJames Bottomley return ret; 7373a5a3927SJames Bottomley } 73886db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_map_kern); 739