1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 286db1e29SJens Axboe /* 386db1e29SJens Axboe * Functions related to mapping data to requests 486db1e29SJens Axboe */ 586db1e29SJens Axboe #include <linux/kernel.h> 668db0cf1SIngo Molnar #include <linux/sched/task_stack.h> 786db1e29SJens Axboe #include <linux/module.h> 886db1e29SJens Axboe #include <linux/bio.h> 986db1e29SJens Axboe #include <linux/blkdev.h> 1026e49cfcSKent Overstreet #include <linux/uio.h> 1186db1e29SJens Axboe 1286db1e29SJens Axboe #include "blk.h" 1386db1e29SJens Axboe 14130879f1SChristoph Hellwig struct bio_map_data { 15f3256075SChristoph Hellwig bool is_our_pages : 1; 16f3256075SChristoph Hellwig bool is_null_mapped : 1; 17130879f1SChristoph Hellwig struct iov_iter iter; 18130879f1SChristoph Hellwig struct iovec iov[]; 19130879f1SChristoph Hellwig }; 20130879f1SChristoph Hellwig 21130879f1SChristoph Hellwig static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, 22130879f1SChristoph Hellwig gfp_t gfp_mask) 23130879f1SChristoph Hellwig { 24130879f1SChristoph Hellwig struct bio_map_data *bmd; 25130879f1SChristoph Hellwig 26130879f1SChristoph Hellwig if (data->nr_segs > UIO_MAXIOV) 27130879f1SChristoph Hellwig return NULL; 28130879f1SChristoph Hellwig 29130879f1SChristoph Hellwig bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); 30130879f1SChristoph Hellwig if (!bmd) 31130879f1SChristoph Hellwig return NULL; 32130879f1SChristoph Hellwig memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); 33130879f1SChristoph Hellwig bmd->iter = *data; 34130879f1SChristoph Hellwig bmd->iter.iov = bmd->iov; 35130879f1SChristoph Hellwig return bmd; 36130879f1SChristoph Hellwig } 37130879f1SChristoph Hellwig 38130879f1SChristoph Hellwig /** 39130879f1SChristoph Hellwig * bio_copy_from_iter - copy all pages from iov_iter to bio 40130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as destination 41130879f1SChristoph Hellwig * @iter: iov_iter as source 42130879f1SChristoph Hellwig * 43130879f1SChristoph Hellwig * Copy all pages from iov_iter to bio. 44130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 45130879f1SChristoph Hellwig */ 46130879f1SChristoph Hellwig static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) 47130879f1SChristoph Hellwig { 48130879f1SChristoph Hellwig struct bio_vec *bvec; 49130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 50130879f1SChristoph Hellwig 51130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 52130879f1SChristoph Hellwig ssize_t ret; 53130879f1SChristoph Hellwig 54130879f1SChristoph Hellwig ret = copy_page_from_iter(bvec->bv_page, 55130879f1SChristoph Hellwig bvec->bv_offset, 56130879f1SChristoph Hellwig bvec->bv_len, 57130879f1SChristoph Hellwig iter); 58130879f1SChristoph Hellwig 59130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 60130879f1SChristoph Hellwig break; 61130879f1SChristoph Hellwig 62130879f1SChristoph Hellwig if (ret < bvec->bv_len) 63130879f1SChristoph Hellwig return -EFAULT; 64130879f1SChristoph Hellwig } 65130879f1SChristoph Hellwig 66130879f1SChristoph Hellwig return 0; 67130879f1SChristoph Hellwig } 68130879f1SChristoph Hellwig 69130879f1SChristoph Hellwig /** 70130879f1SChristoph Hellwig * bio_copy_to_iter - copy all pages from bio to iov_iter 71130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as source 72130879f1SChristoph Hellwig * @iter: iov_iter as destination 73130879f1SChristoph Hellwig * 74130879f1SChristoph Hellwig * Copy all pages from bio to iov_iter. 75130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 76130879f1SChristoph Hellwig */ 77130879f1SChristoph Hellwig static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) 78130879f1SChristoph Hellwig { 79130879f1SChristoph Hellwig struct bio_vec *bvec; 80130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 81130879f1SChristoph Hellwig 82130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 83130879f1SChristoph Hellwig ssize_t ret; 84130879f1SChristoph Hellwig 85130879f1SChristoph Hellwig ret = copy_page_to_iter(bvec->bv_page, 86130879f1SChristoph Hellwig bvec->bv_offset, 87130879f1SChristoph Hellwig bvec->bv_len, 88130879f1SChristoph Hellwig &iter); 89130879f1SChristoph Hellwig 90130879f1SChristoph Hellwig if (!iov_iter_count(&iter)) 91130879f1SChristoph Hellwig break; 92130879f1SChristoph Hellwig 93130879f1SChristoph Hellwig if (ret < bvec->bv_len) 94130879f1SChristoph Hellwig return -EFAULT; 95130879f1SChristoph Hellwig } 96130879f1SChristoph Hellwig 97130879f1SChristoph Hellwig return 0; 98130879f1SChristoph Hellwig } 99130879f1SChristoph Hellwig 100130879f1SChristoph Hellwig /** 101130879f1SChristoph Hellwig * bio_uncopy_user - finish previously mapped bio 102130879f1SChristoph Hellwig * @bio: bio being terminated 103130879f1SChristoph Hellwig * 104130879f1SChristoph Hellwig * Free pages allocated from bio_copy_user_iov() and write back data 105130879f1SChristoph Hellwig * to user space in case of a read. 106130879f1SChristoph Hellwig */ 107130879f1SChristoph Hellwig static int bio_uncopy_user(struct bio *bio) 108130879f1SChristoph Hellwig { 109130879f1SChristoph Hellwig struct bio_map_data *bmd = bio->bi_private; 110130879f1SChristoph Hellwig int ret = 0; 111130879f1SChristoph Hellwig 1123310eebaSChristoph Hellwig if (!bmd->is_null_mapped) { 113130879f1SChristoph Hellwig /* 114130879f1SChristoph Hellwig * if we're in a workqueue, the request is orphaned, so 115130879f1SChristoph Hellwig * don't copy into a random user address space, just free 116130879f1SChristoph Hellwig * and return -EINTR so user space doesn't expect any data. 117130879f1SChristoph Hellwig */ 118130879f1SChristoph Hellwig if (!current->mm) 119130879f1SChristoph Hellwig ret = -EINTR; 120130879f1SChristoph Hellwig else if (bio_data_dir(bio) == READ) 121130879f1SChristoph Hellwig ret = bio_copy_to_iter(bio, bmd->iter); 122130879f1SChristoph Hellwig if (bmd->is_our_pages) 123130879f1SChristoph Hellwig bio_free_pages(bio); 124130879f1SChristoph Hellwig } 125130879f1SChristoph Hellwig kfree(bmd); 126130879f1SChristoph Hellwig return ret; 127130879f1SChristoph Hellwig } 128130879f1SChristoph Hellwig 1297589ad67SChristoph Hellwig static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, 1307589ad67SChristoph Hellwig struct iov_iter *iter, gfp_t gfp_mask) 131130879f1SChristoph Hellwig { 132130879f1SChristoph Hellwig struct bio_map_data *bmd; 133130879f1SChristoph Hellwig struct page *page; 134393bb12eSChristoph Hellwig struct bio *bio; 135130879f1SChristoph Hellwig int i = 0, ret; 136130879f1SChristoph Hellwig int nr_pages; 137130879f1SChristoph Hellwig unsigned int len = iter->count; 138130879f1SChristoph Hellwig unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; 139130879f1SChristoph Hellwig 140130879f1SChristoph Hellwig bmd = bio_alloc_map_data(iter, gfp_mask); 141130879f1SChristoph Hellwig if (!bmd) 1427589ad67SChristoph Hellwig return -ENOMEM; 143130879f1SChristoph Hellwig 144130879f1SChristoph Hellwig /* 145130879f1SChristoph Hellwig * We need to do a deep copy of the iov_iter including the iovecs. 146130879f1SChristoph Hellwig * The caller provided iov might point to an on-stack or otherwise 147130879f1SChristoph Hellwig * shortlived one. 148130879f1SChristoph Hellwig */ 149f3256075SChristoph Hellwig bmd->is_our_pages = !map_data; 15003859717SChristoph Hellwig bmd->is_null_mapped = (map_data && map_data->null_mapped); 151130879f1SChristoph Hellwig 1525f7136dbSMatthew Wilcox (Oracle) nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); 153130879f1SChristoph Hellwig 154130879f1SChristoph Hellwig ret = -ENOMEM; 155066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 156130879f1SChristoph Hellwig if (!bio) 157130879f1SChristoph Hellwig goto out_bmd; 158066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); 159130879f1SChristoph Hellwig 160130879f1SChristoph Hellwig if (map_data) { 161f5d632d1SJens Axboe nr_pages = 1U << map_data->page_order; 162130879f1SChristoph Hellwig i = map_data->offset / PAGE_SIZE; 163130879f1SChristoph Hellwig } 164130879f1SChristoph Hellwig while (len) { 165130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 166130879f1SChristoph Hellwig 167130879f1SChristoph Hellwig bytes -= offset; 168130879f1SChristoph Hellwig 169130879f1SChristoph Hellwig if (bytes > len) 170130879f1SChristoph Hellwig bytes = len; 171130879f1SChristoph Hellwig 172130879f1SChristoph Hellwig if (map_data) { 173130879f1SChristoph Hellwig if (i == map_data->nr_entries * nr_pages) { 174130879f1SChristoph Hellwig ret = -ENOMEM; 1757589ad67SChristoph Hellwig goto cleanup; 176130879f1SChristoph Hellwig } 177130879f1SChristoph Hellwig 178130879f1SChristoph Hellwig page = map_data->pages[i / nr_pages]; 179130879f1SChristoph Hellwig page += (i % nr_pages); 180130879f1SChristoph Hellwig 181130879f1SChristoph Hellwig i++; 182130879f1SChristoph Hellwig } else { 183ce288e05SChristoph Hellwig page = alloc_page(GFP_NOIO | gfp_mask); 184130879f1SChristoph Hellwig if (!page) { 185130879f1SChristoph Hellwig ret = -ENOMEM; 1867589ad67SChristoph Hellwig goto cleanup; 187130879f1SChristoph Hellwig } 188130879f1SChristoph Hellwig } 189130879f1SChristoph Hellwig 1907589ad67SChristoph Hellwig if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) { 191130879f1SChristoph Hellwig if (!map_data) 192130879f1SChristoph Hellwig __free_page(page); 193130879f1SChristoph Hellwig break; 194130879f1SChristoph Hellwig } 195130879f1SChristoph Hellwig 196130879f1SChristoph Hellwig len -= bytes; 197130879f1SChristoph Hellwig offset = 0; 198130879f1SChristoph Hellwig } 199130879f1SChristoph Hellwig 200130879f1SChristoph Hellwig if (map_data) 201130879f1SChristoph Hellwig map_data->offset += bio->bi_iter.bi_size; 202130879f1SChristoph Hellwig 203130879f1SChristoph Hellwig /* 204130879f1SChristoph Hellwig * success 205130879f1SChristoph Hellwig */ 206130879f1SChristoph Hellwig if ((iov_iter_rw(iter) == WRITE && 207130879f1SChristoph Hellwig (!map_data || !map_data->null_mapped)) || 208130879f1SChristoph Hellwig (map_data && map_data->from_user)) { 209130879f1SChristoph Hellwig ret = bio_copy_from_iter(bio, iter); 210130879f1SChristoph Hellwig if (ret) 211130879f1SChristoph Hellwig goto cleanup; 212130879f1SChristoph Hellwig } else { 213130879f1SChristoph Hellwig if (bmd->is_our_pages) 214130879f1SChristoph Hellwig zero_fill_bio(bio); 215130879f1SChristoph Hellwig iov_iter_advance(iter, bio->bi_iter.bi_size); 216130879f1SChristoph Hellwig } 217130879f1SChristoph Hellwig 218130879f1SChristoph Hellwig bio->bi_private = bmd; 2197589ad67SChristoph Hellwig 220393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 2217589ad67SChristoph Hellwig if (ret) 2227589ad67SChristoph Hellwig goto cleanup; 2237589ad67SChristoph Hellwig return 0; 224130879f1SChristoph Hellwig cleanup: 225130879f1SChristoph Hellwig if (!map_data) 226130879f1SChristoph Hellwig bio_free_pages(bio); 227066ff571SChristoph Hellwig bio_uninit(bio); 228066ff571SChristoph Hellwig kfree(bio); 229130879f1SChristoph Hellwig out_bmd: 230130879f1SChristoph Hellwig kfree(bmd); 2317589ad67SChristoph Hellwig return ret; 232130879f1SChristoph Hellwig } 233130879f1SChristoph Hellwig 2348af870aaSJens Axboe static void bio_map_put(struct bio *bio) 2358af870aaSJens Axboe { 2368af870aaSJens Axboe if (bio->bi_opf & REQ_ALLOC_CACHE) { 2378af870aaSJens Axboe bio_put(bio); 2388af870aaSJens Axboe } else { 2398af870aaSJens Axboe bio_uninit(bio); 2408af870aaSJens Axboe kfree(bio); 2418af870aaSJens Axboe } 2428af870aaSJens Axboe } 2438af870aaSJens Axboe 2447589ad67SChristoph Hellwig static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, 2457589ad67SChristoph Hellwig gfp_t gfp_mask) 246130879f1SChristoph Hellwig { 2477589ad67SChristoph Hellwig unsigned int max_sectors = queue_max_hw_sectors(rq->q); 248066ff571SChristoph Hellwig unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); 249393bb12eSChristoph Hellwig struct bio *bio; 250130879f1SChristoph Hellwig int ret; 2517589ad67SChristoph Hellwig int j; 252130879f1SChristoph Hellwig 253130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 2547589ad67SChristoph Hellwig return -EINVAL; 255130879f1SChristoph Hellwig 2568af870aaSJens Axboe if (rq->cmd_flags & REQ_POLLED) { 2578af870aaSJens Axboe blk_opf_t opf = rq->cmd_flags | REQ_ALLOC_CACHE; 2588af870aaSJens Axboe 2598af870aaSJens Axboe bio = bio_alloc_bioset(NULL, nr_vecs, opf, gfp_mask, 2608af870aaSJens Axboe &fs_bio_set); 2618af870aaSJens Axboe if (!bio) 2628af870aaSJens Axboe return -ENOMEM; 2638af870aaSJens Axboe } else { 264066ff571SChristoph Hellwig bio = bio_kmalloc(nr_vecs, gfp_mask); 265130879f1SChristoph Hellwig if (!bio) 2667589ad67SChristoph Hellwig return -ENOMEM; 267066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); 2688af870aaSJens Axboe } 269130879f1SChristoph Hellwig 270130879f1SChristoph Hellwig while (iov_iter_count(iter)) { 271e88811bcSJens Axboe struct page **pages, *stack_pages[UIO_FASTIOV]; 272130879f1SChristoph Hellwig ssize_t bytes; 27391e5addaSJiapeng Chong size_t offs; 274130879f1SChristoph Hellwig int npages; 275130879f1SChristoph Hellwig 276e88811bcSJens Axboe if (nr_vecs <= ARRAY_SIZE(stack_pages)) { 277e88811bcSJens Axboe pages = stack_pages; 278e88811bcSJens Axboe bytes = iov_iter_get_pages2(iter, pages, LONG_MAX, 279e88811bcSJens Axboe nr_vecs, &offs); 280e88811bcSJens Axboe } else { 281e88811bcSJens Axboe bytes = iov_iter_get_pages_alloc2(iter, &pages, 282e88811bcSJens Axboe LONG_MAX, &offs); 283e88811bcSJens Axboe } 284130879f1SChristoph Hellwig if (unlikely(bytes <= 0)) { 285130879f1SChristoph Hellwig ret = bytes ? bytes : -EFAULT; 286130879f1SChristoph Hellwig goto out_unmap; 287130879f1SChristoph Hellwig } 288130879f1SChristoph Hellwig 289130879f1SChristoph Hellwig npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); 290130879f1SChristoph Hellwig 2917ab89db9SMichal Orzel if (unlikely(offs & queue_dma_alignment(rq->q))) 292130879f1SChristoph Hellwig j = 0; 2937ab89db9SMichal Orzel else { 294130879f1SChristoph Hellwig for (j = 0; j < npages; j++) { 295130879f1SChristoph Hellwig struct page *page = pages[j]; 296130879f1SChristoph Hellwig unsigned int n = PAGE_SIZE - offs; 297130879f1SChristoph Hellwig bool same_page = false; 298130879f1SChristoph Hellwig 299130879f1SChristoph Hellwig if (n > bytes) 300130879f1SChristoph Hellwig n = bytes; 301130879f1SChristoph Hellwig 3027589ad67SChristoph Hellwig if (!bio_add_hw_page(rq->q, bio, page, n, offs, 303e4581105SChristoph Hellwig max_sectors, &same_page)) { 304130879f1SChristoph Hellwig if (same_page) 305130879f1SChristoph Hellwig put_page(page); 306130879f1SChristoph Hellwig break; 307130879f1SChristoph Hellwig } 308130879f1SChristoph Hellwig 309130879f1SChristoph Hellwig bytes -= n; 310130879f1SChristoph Hellwig offs = 0; 311130879f1SChristoph Hellwig } 312130879f1SChristoph Hellwig } 313130879f1SChristoph Hellwig /* 314130879f1SChristoph Hellwig * release the pages we didn't map into the bio, if any 315130879f1SChristoph Hellwig */ 316130879f1SChristoph Hellwig while (j < npages) 317130879f1SChristoph Hellwig put_page(pages[j++]); 318e88811bcSJens Axboe if (pages != stack_pages) 319130879f1SChristoph Hellwig kvfree(pages); 320130879f1SChristoph Hellwig /* couldn't stuff something into bio? */ 321480cb846SAl Viro if (bytes) { 322480cb846SAl Viro iov_iter_revert(iter, bytes); 323130879f1SChristoph Hellwig break; 324130879f1SChristoph Hellwig } 325480cb846SAl Viro } 326130879f1SChristoph Hellwig 327393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 3287589ad67SChristoph Hellwig if (ret) 329393bb12eSChristoph Hellwig goto out_unmap; 3307589ad67SChristoph Hellwig return 0; 3317589ad67SChristoph Hellwig 332130879f1SChristoph Hellwig out_unmap: 333130879f1SChristoph Hellwig bio_release_pages(bio, false); 3348af870aaSJens Axboe bio_map_put(bio); 3357589ad67SChristoph Hellwig return ret; 336130879f1SChristoph Hellwig } 337130879f1SChristoph Hellwig 338130879f1SChristoph Hellwig static void bio_invalidate_vmalloc_pages(struct bio *bio) 339130879f1SChristoph Hellwig { 340f358afc5SChristoph Hellwig #ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 341130879f1SChristoph Hellwig if (bio->bi_private && !op_is_write(bio_op(bio))) { 342130879f1SChristoph Hellwig unsigned long i, len = 0; 343130879f1SChristoph Hellwig 344130879f1SChristoph Hellwig for (i = 0; i < bio->bi_vcnt; i++) 345130879f1SChristoph Hellwig len += bio->bi_io_vec[i].bv_len; 346130879f1SChristoph Hellwig invalidate_kernel_vmap_range(bio->bi_private, len); 347130879f1SChristoph Hellwig } 348130879f1SChristoph Hellwig #endif 349130879f1SChristoph Hellwig } 350130879f1SChristoph Hellwig 351130879f1SChristoph Hellwig static void bio_map_kern_endio(struct bio *bio) 352130879f1SChristoph Hellwig { 353130879f1SChristoph Hellwig bio_invalidate_vmalloc_pages(bio); 354066ff571SChristoph Hellwig bio_uninit(bio); 355066ff571SChristoph Hellwig kfree(bio); 356130879f1SChristoph Hellwig } 357130879f1SChristoph Hellwig 358130879f1SChristoph Hellwig /** 359130879f1SChristoph Hellwig * bio_map_kern - map kernel address into bio 360130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 361130879f1SChristoph Hellwig * @data: pointer to buffer to map 362130879f1SChristoph Hellwig * @len: length in bytes 363130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio allocation 364130879f1SChristoph Hellwig * 365130879f1SChristoph Hellwig * Map the kernel address into a bio suitable for io to a block 366130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 367130879f1SChristoph Hellwig */ 368130879f1SChristoph Hellwig static struct bio *bio_map_kern(struct request_queue *q, void *data, 369130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask) 370130879f1SChristoph Hellwig { 371130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 372130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 373130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 374130879f1SChristoph Hellwig const int nr_pages = end - start; 375130879f1SChristoph Hellwig bool is_vmalloc = is_vmalloc_addr(data); 376130879f1SChristoph Hellwig struct page *page; 377130879f1SChristoph Hellwig int offset, i; 378130879f1SChristoph Hellwig struct bio *bio; 379130879f1SChristoph Hellwig 380066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 381130879f1SChristoph Hellwig if (!bio) 382130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 383066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 384130879f1SChristoph Hellwig 385130879f1SChristoph Hellwig if (is_vmalloc) { 386130879f1SChristoph Hellwig flush_kernel_vmap_range(data, len); 387130879f1SChristoph Hellwig bio->bi_private = data; 388130879f1SChristoph Hellwig } 389130879f1SChristoph Hellwig 390130879f1SChristoph Hellwig offset = offset_in_page(kaddr); 391130879f1SChristoph Hellwig for (i = 0; i < nr_pages; i++) { 392130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE - offset; 393130879f1SChristoph Hellwig 394130879f1SChristoph Hellwig if (len <= 0) 395130879f1SChristoph Hellwig break; 396130879f1SChristoph Hellwig 397130879f1SChristoph Hellwig if (bytes > len) 398130879f1SChristoph Hellwig bytes = len; 399130879f1SChristoph Hellwig 400130879f1SChristoph Hellwig if (!is_vmalloc) 401130879f1SChristoph Hellwig page = virt_to_page(data); 402130879f1SChristoph Hellwig else 403130879f1SChristoph Hellwig page = vmalloc_to_page(data); 404130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 405130879f1SChristoph Hellwig offset) < bytes) { 406130879f1SChristoph Hellwig /* we don't support partial mappings */ 407066ff571SChristoph Hellwig bio_uninit(bio); 408066ff571SChristoph Hellwig kfree(bio); 409130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 410130879f1SChristoph Hellwig } 411130879f1SChristoph Hellwig 412130879f1SChristoph Hellwig data += bytes; 413130879f1SChristoph Hellwig len -= bytes; 414130879f1SChristoph Hellwig offset = 0; 415130879f1SChristoph Hellwig } 416130879f1SChristoph Hellwig 417130879f1SChristoph Hellwig bio->bi_end_io = bio_map_kern_endio; 418130879f1SChristoph Hellwig return bio; 419130879f1SChristoph Hellwig } 420130879f1SChristoph Hellwig 421130879f1SChristoph Hellwig static void bio_copy_kern_endio(struct bio *bio) 422130879f1SChristoph Hellwig { 423130879f1SChristoph Hellwig bio_free_pages(bio); 424066ff571SChristoph Hellwig bio_uninit(bio); 425066ff571SChristoph Hellwig kfree(bio); 426130879f1SChristoph Hellwig } 427130879f1SChristoph Hellwig 428130879f1SChristoph Hellwig static void bio_copy_kern_endio_read(struct bio *bio) 429130879f1SChristoph Hellwig { 430130879f1SChristoph Hellwig char *p = bio->bi_private; 431130879f1SChristoph Hellwig struct bio_vec *bvec; 432130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 433130879f1SChristoph Hellwig 434130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 435d24920e2SChristoph Hellwig memcpy_from_bvec(p, bvec); 436130879f1SChristoph Hellwig p += bvec->bv_len; 437130879f1SChristoph Hellwig } 438130879f1SChristoph Hellwig 439130879f1SChristoph Hellwig bio_copy_kern_endio(bio); 440130879f1SChristoph Hellwig } 441130879f1SChristoph Hellwig 442130879f1SChristoph Hellwig /** 443130879f1SChristoph Hellwig * bio_copy_kern - copy kernel address into bio 444130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 445130879f1SChristoph Hellwig * @data: pointer to buffer to copy 446130879f1SChristoph Hellwig * @len: length in bytes 447130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio and page allocation 448130879f1SChristoph Hellwig * @reading: data direction is READ 449130879f1SChristoph Hellwig * 450130879f1SChristoph Hellwig * copy the kernel address into a bio suitable for io to a block 451130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 452130879f1SChristoph Hellwig */ 453130879f1SChristoph Hellwig static struct bio *bio_copy_kern(struct request_queue *q, void *data, 454130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask, int reading) 455130879f1SChristoph Hellwig { 456130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 457130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 458130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 459130879f1SChristoph Hellwig struct bio *bio; 460130879f1SChristoph Hellwig void *p = data; 461130879f1SChristoph Hellwig int nr_pages = 0; 462130879f1SChristoph Hellwig 463130879f1SChristoph Hellwig /* 464130879f1SChristoph Hellwig * Overflow, abort 465130879f1SChristoph Hellwig */ 466130879f1SChristoph Hellwig if (end < start) 467130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 468130879f1SChristoph Hellwig 469130879f1SChristoph Hellwig nr_pages = end - start; 470066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 471130879f1SChristoph Hellwig if (!bio) 472130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 473066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 474130879f1SChristoph Hellwig 475130879f1SChristoph Hellwig while (len) { 476130879f1SChristoph Hellwig struct page *page; 477130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 478130879f1SChristoph Hellwig 479130879f1SChristoph Hellwig if (bytes > len) 480130879f1SChristoph Hellwig bytes = len; 481130879f1SChristoph Hellwig 482cc8f7fe1SHaimin Zhang page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask); 483130879f1SChristoph Hellwig if (!page) 484130879f1SChristoph Hellwig goto cleanup; 485130879f1SChristoph Hellwig 486130879f1SChristoph Hellwig if (!reading) 487130879f1SChristoph Hellwig memcpy(page_address(page), p, bytes); 488130879f1SChristoph Hellwig 489130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) 490130879f1SChristoph Hellwig break; 491130879f1SChristoph Hellwig 492130879f1SChristoph Hellwig len -= bytes; 493130879f1SChristoph Hellwig p += bytes; 494130879f1SChristoph Hellwig } 495130879f1SChristoph Hellwig 496130879f1SChristoph Hellwig if (reading) { 497130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio_read; 498130879f1SChristoph Hellwig bio->bi_private = data; 499130879f1SChristoph Hellwig } else { 500130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio; 501130879f1SChristoph Hellwig } 502130879f1SChristoph Hellwig 503130879f1SChristoph Hellwig return bio; 504130879f1SChristoph Hellwig 505130879f1SChristoph Hellwig cleanup: 506130879f1SChristoph Hellwig bio_free_pages(bio); 507066ff571SChristoph Hellwig bio_uninit(bio); 508066ff571SChristoph Hellwig kfree(bio); 509130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 510130879f1SChristoph Hellwig } 511130879f1SChristoph Hellwig 51298d61d5bSChristoph Hellwig /* 5130abc2a10SJens Axboe * Append a bio to a passthrough request. Only works if the bio can be merged 5140abc2a10SJens Axboe * into the request based on the driver constraints. 51598d61d5bSChristoph Hellwig */ 516393bb12eSChristoph Hellwig int blk_rq_append_bio(struct request *rq, struct bio *bio) 51786db1e29SJens Axboe { 51814ccb66bSChristoph Hellwig struct bvec_iter iter; 51914ccb66bSChristoph Hellwig struct bio_vec bv; 52014ccb66bSChristoph Hellwig unsigned int nr_segs = 0; 5210abc2a10SJens Axboe 522393bb12eSChristoph Hellwig bio_for_each_bvec(bv, bio, iter) 52314ccb66bSChristoph Hellwig nr_segs++; 52414ccb66bSChristoph Hellwig 52598d61d5bSChristoph Hellwig if (!rq->bio) { 526393bb12eSChristoph Hellwig blk_rq_bio_prep(rq, bio, nr_segs); 52798d61d5bSChristoph Hellwig } else { 528393bb12eSChristoph Hellwig if (!ll_back_merge_fn(rq, bio, nr_segs)) 52986db1e29SJens Axboe return -EINVAL; 530393bb12eSChristoph Hellwig rq->biotail->bi_next = bio; 531393bb12eSChristoph Hellwig rq->biotail = bio; 532393bb12eSChristoph Hellwig rq->__data_len += (bio)->bi_iter.bi_size; 533393bb12eSChristoph Hellwig bio_crypt_free_ctx(bio); 53486db1e29SJens Axboe } 53598d61d5bSChristoph Hellwig 53686db1e29SJens Axboe return 0; 53786db1e29SJens Axboe } 53898d61d5bSChristoph Hellwig EXPORT_SYMBOL(blk_rq_append_bio); 53986db1e29SJens Axboe 54086db1e29SJens Axboe /** 541aebf526bSChristoph Hellwig * blk_rq_map_user_iov - map user data to a request, for passthrough requests 54286db1e29SJens Axboe * @q: request queue where request should be inserted 54386db1e29SJens Axboe * @rq: request to map data to 544152e283fSFUJITA Tomonori * @map_data: pointer to the rq_map_data holding pages (if necessary) 54526e49cfcSKent Overstreet * @iter: iovec iterator 546a3bce90eSFUJITA Tomonori * @gfp_mask: memory allocation flags 54786db1e29SJens Axboe * 54886db1e29SJens Axboe * Description: 549710027a4SRandy Dunlap * Data will be mapped directly for zero copy I/O, if possible. Otherwise 55086db1e29SJens Axboe * a kernel bounce buffer is used. 55186db1e29SJens Axboe * 552710027a4SRandy Dunlap * A matching blk_rq_unmap_user() must be issued at the end of I/O, while 55386db1e29SJens Axboe * still in process context. 55486db1e29SJens Axboe */ 55586db1e29SJens Axboe int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, 55626e49cfcSKent Overstreet struct rq_map_data *map_data, 55726e49cfcSKent Overstreet const struct iov_iter *iter, gfp_t gfp_mask) 55886db1e29SJens Axboe { 559357f435dSAl Viro bool copy = false; 560357f435dSAl Viro unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); 5614d6af73dSChristoph Hellwig struct bio *bio = NULL; 5624d6af73dSChristoph Hellwig struct iov_iter i; 56369e0927bSDouglas Gilbert int ret = -EINVAL; 56486db1e29SJens Axboe 565a0ac402cSLinus Torvalds if (!iter_is_iovec(iter)) 566a0ac402cSLinus Torvalds goto fail; 567a0ac402cSLinus Torvalds 568357f435dSAl Viro if (map_data) 5694d6af73dSChristoph Hellwig copy = true; 570393bb12eSChristoph Hellwig else if (blk_queue_may_bounce(q)) 571393bb12eSChristoph Hellwig copy = true; 572357f435dSAl Viro else if (iov_iter_alignment(iter) & align) 573357f435dSAl Viro copy = true; 574357f435dSAl Viro else if (queue_virt_boundary(q)) 575357f435dSAl Viro copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); 576afdc1a78SFUJITA Tomonori 5774d6af73dSChristoph Hellwig i = *iter; 5784d6af73dSChristoph Hellwig do { 5797589ad67SChristoph Hellwig if (copy) 5807589ad67SChristoph Hellwig ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask); 5817589ad67SChristoph Hellwig else 5827589ad67SChristoph Hellwig ret = bio_map_user_iov(rq, &i, gfp_mask); 5834d6af73dSChristoph Hellwig if (ret) 5844d6af73dSChristoph Hellwig goto unmap_rq; 5854d6af73dSChristoph Hellwig if (!bio) 5864d6af73dSChristoph Hellwig bio = rq->bio; 5874d6af73dSChristoph Hellwig } while (iov_iter_count(&i)); 58886db1e29SJens Axboe 58986db1e29SJens Axboe return 0; 5904d6af73dSChristoph Hellwig 5914d6af73dSChristoph Hellwig unmap_rq: 5923b7995a9SYang Yingliang blk_rq_unmap_user(bio); 593a0ac402cSLinus Torvalds fail: 5944d6af73dSChristoph Hellwig rq->bio = NULL; 59569e0927bSDouglas Gilbert return ret; 59686db1e29SJens Axboe } 597152e283fSFUJITA Tomonori EXPORT_SYMBOL(blk_rq_map_user_iov); 59886db1e29SJens Axboe 599ddad8dd0SChristoph Hellwig int blk_rq_map_user(struct request_queue *q, struct request *rq, 600ddad8dd0SChristoph Hellwig struct rq_map_data *map_data, void __user *ubuf, 601ddad8dd0SChristoph Hellwig unsigned long len, gfp_t gfp_mask) 602ddad8dd0SChristoph Hellwig { 60326e49cfcSKent Overstreet struct iovec iov; 60426e49cfcSKent Overstreet struct iov_iter i; 6058f7e885aSAl Viro int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i); 606ddad8dd0SChristoph Hellwig 6078f7e885aSAl Viro if (unlikely(ret < 0)) 6088f7e885aSAl Viro return ret; 609ddad8dd0SChristoph Hellwig 61026e49cfcSKent Overstreet return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); 611ddad8dd0SChristoph Hellwig } 612ddad8dd0SChristoph Hellwig EXPORT_SYMBOL(blk_rq_map_user); 613ddad8dd0SChristoph Hellwig 614*55765402SAnuj Gupta int blk_rq_map_user_io(struct request *req, struct rq_map_data *map_data, 615*55765402SAnuj Gupta void __user *ubuf, unsigned long buf_len, gfp_t gfp_mask, 616*55765402SAnuj Gupta bool vec, int iov_count, bool check_iter_count, int rw) 617*55765402SAnuj Gupta { 618*55765402SAnuj Gupta int ret = 0; 619*55765402SAnuj Gupta 620*55765402SAnuj Gupta if (vec) { 621*55765402SAnuj Gupta struct iovec fast_iov[UIO_FASTIOV]; 622*55765402SAnuj Gupta struct iovec *iov = fast_iov; 623*55765402SAnuj Gupta struct iov_iter iter; 624*55765402SAnuj Gupta 625*55765402SAnuj Gupta ret = import_iovec(rw, ubuf, iov_count ? iov_count : buf_len, 626*55765402SAnuj Gupta UIO_FASTIOV, &iov, &iter); 627*55765402SAnuj Gupta if (ret < 0) 628*55765402SAnuj Gupta return ret; 629*55765402SAnuj Gupta 630*55765402SAnuj Gupta if (iov_count) { 631*55765402SAnuj Gupta /* SG_IO howto says that the shorter of the two wins */ 632*55765402SAnuj Gupta iov_iter_truncate(&iter, buf_len); 633*55765402SAnuj Gupta if (check_iter_count && !iov_iter_count(&iter)) { 634*55765402SAnuj Gupta kfree(iov); 635*55765402SAnuj Gupta return -EINVAL; 636*55765402SAnuj Gupta } 637*55765402SAnuj Gupta } 638*55765402SAnuj Gupta 639*55765402SAnuj Gupta ret = blk_rq_map_user_iov(req->q, req, map_data, &iter, 640*55765402SAnuj Gupta gfp_mask); 641*55765402SAnuj Gupta kfree(iov); 642*55765402SAnuj Gupta } else if (buf_len) { 643*55765402SAnuj Gupta ret = blk_rq_map_user(req->q, req, map_data, ubuf, buf_len, 644*55765402SAnuj Gupta gfp_mask); 645*55765402SAnuj Gupta } 646*55765402SAnuj Gupta return ret; 647*55765402SAnuj Gupta } 648*55765402SAnuj Gupta EXPORT_SYMBOL(blk_rq_map_user_io); 649*55765402SAnuj Gupta 65086db1e29SJens Axboe /** 65186db1e29SJens Axboe * blk_rq_unmap_user - unmap a request with user data 65286db1e29SJens Axboe * @bio: start of bio list 65386db1e29SJens Axboe * 65486db1e29SJens Axboe * Description: 65586db1e29SJens Axboe * Unmap a rq previously mapped by blk_rq_map_user(). The caller must 65686db1e29SJens Axboe * supply the original rq->bio from the blk_rq_map_user() return, since 657710027a4SRandy Dunlap * the I/O completion may have changed rq->bio. 65886db1e29SJens Axboe */ 65986db1e29SJens Axboe int blk_rq_unmap_user(struct bio *bio) 66086db1e29SJens Axboe { 661393bb12eSChristoph Hellwig struct bio *next_bio; 66286db1e29SJens Axboe int ret = 0, ret2; 66386db1e29SJens Axboe 66486db1e29SJens Axboe while (bio) { 6653310eebaSChristoph Hellwig if (bio->bi_private) { 666393bb12eSChristoph Hellwig ret2 = bio_uncopy_user(bio); 66786db1e29SJens Axboe if (ret2 && !ret) 66886db1e29SJens Axboe ret = ret2; 6693310eebaSChristoph Hellwig } else { 670393bb12eSChristoph Hellwig bio_release_pages(bio, bio_data_dir(bio) == READ); 6717b63c052SChristoph Hellwig } 67286db1e29SJens Axboe 673393bb12eSChristoph Hellwig next_bio = bio; 67486db1e29SJens Axboe bio = bio->bi_next; 6758af870aaSJens Axboe bio_map_put(next_bio); 67686db1e29SJens Axboe } 67786db1e29SJens Axboe 67886db1e29SJens Axboe return ret; 67986db1e29SJens Axboe } 68086db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_unmap_user); 68186db1e29SJens Axboe 68286db1e29SJens Axboe /** 683aebf526bSChristoph Hellwig * blk_rq_map_kern - map kernel data to a request, for passthrough requests 68486db1e29SJens Axboe * @q: request queue where request should be inserted 68586db1e29SJens Axboe * @rq: request to fill 68686db1e29SJens Axboe * @kbuf: the kernel buffer 68786db1e29SJens Axboe * @len: length of user data 68886db1e29SJens Axboe * @gfp_mask: memory allocation flags 68968154e90SFUJITA Tomonori * 69068154e90SFUJITA Tomonori * Description: 69168154e90SFUJITA Tomonori * Data will be mapped directly if possible. Otherwise a bounce 692e227867fSMasanari Iida * buffer is used. Can be called multiple times to append multiple 6933a5a3927SJames Bottomley * buffers. 69486db1e29SJens Axboe */ 69586db1e29SJens Axboe int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, 69686db1e29SJens Axboe unsigned int len, gfp_t gfp_mask) 69786db1e29SJens Axboe { 69868154e90SFUJITA Tomonori int reading = rq_data_dir(rq) == READ; 69914417799SNamhyung Kim unsigned long addr = (unsigned long) kbuf; 700393bb12eSChristoph Hellwig struct bio *bio; 7013a5a3927SJames Bottomley int ret; 70286db1e29SJens Axboe 703ae03bf63SMartin K. Petersen if (len > (queue_max_hw_sectors(q) << 9)) 70486db1e29SJens Axboe return -EINVAL; 70586db1e29SJens Axboe if (!len || !kbuf) 70686db1e29SJens Axboe return -EINVAL; 70786db1e29SJens Axboe 708393bb12eSChristoph Hellwig if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || 709393bb12eSChristoph Hellwig blk_queue_may_bounce(q)) 71068154e90SFUJITA Tomonori bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); 71168154e90SFUJITA Tomonori else 71286db1e29SJens Axboe bio = bio_map_kern(q, kbuf, len, gfp_mask); 71368154e90SFUJITA Tomonori 71486db1e29SJens Axboe if (IS_ERR(bio)) 71586db1e29SJens Axboe return PTR_ERR(bio); 71686db1e29SJens Axboe 717aebf526bSChristoph Hellwig bio->bi_opf &= ~REQ_OP_MASK; 718aebf526bSChristoph Hellwig bio->bi_opf |= req_op(rq); 71986db1e29SJens Axboe 720393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 721066ff571SChristoph Hellwig if (unlikely(ret)) { 722066ff571SChristoph Hellwig bio_uninit(bio); 723066ff571SChristoph Hellwig kfree(bio); 724066ff571SChristoph Hellwig } 7253a5a3927SJames Bottomley return ret; 7263a5a3927SJames Bottomley } 72786db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_map_kern); 728