1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 286db1e29SJens Axboe /* 386db1e29SJens Axboe * Functions related to mapping data to requests 486db1e29SJens Axboe */ 586db1e29SJens Axboe #include <linux/kernel.h> 668db0cf1SIngo Molnar #include <linux/sched/task_stack.h> 786db1e29SJens Axboe #include <linux/module.h> 886db1e29SJens Axboe #include <linux/bio.h> 986db1e29SJens Axboe #include <linux/blkdev.h> 1026e49cfcSKent Overstreet #include <linux/uio.h> 1186db1e29SJens Axboe 1286db1e29SJens Axboe #include "blk.h" 1386db1e29SJens Axboe 14130879f1SChristoph Hellwig struct bio_map_data { 15f3256075SChristoph Hellwig bool is_our_pages : 1; 16f3256075SChristoph Hellwig bool is_null_mapped : 1; 17130879f1SChristoph Hellwig struct iov_iter iter; 18130879f1SChristoph Hellwig struct iovec iov[]; 19130879f1SChristoph Hellwig }; 20130879f1SChristoph Hellwig 21130879f1SChristoph Hellwig static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, 22130879f1SChristoph Hellwig gfp_t gfp_mask) 23130879f1SChristoph Hellwig { 24130879f1SChristoph Hellwig struct bio_map_data *bmd; 25130879f1SChristoph Hellwig 26130879f1SChristoph Hellwig if (data->nr_segs > UIO_MAXIOV) 27130879f1SChristoph Hellwig return NULL; 28130879f1SChristoph Hellwig 29130879f1SChristoph Hellwig bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); 30130879f1SChristoph Hellwig if (!bmd) 31130879f1SChristoph Hellwig return NULL; 32130879f1SChristoph Hellwig memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); 33130879f1SChristoph Hellwig bmd->iter = *data; 34*d46aa786SKeith Busch if (iter_is_iovec(data)) 35130879f1SChristoph Hellwig bmd->iter.iov = bmd->iov; 36130879f1SChristoph Hellwig return bmd; 37130879f1SChristoph Hellwig } 38130879f1SChristoph Hellwig 39130879f1SChristoph Hellwig /** 40130879f1SChristoph Hellwig * bio_copy_from_iter - copy all pages from iov_iter to bio 41130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as destination 42130879f1SChristoph Hellwig * @iter: iov_iter as source 43130879f1SChristoph Hellwig * 44130879f1SChristoph Hellwig * Copy all pages from iov_iter to bio. 45130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 46130879f1SChristoph Hellwig */ 47130879f1SChristoph Hellwig static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) 48130879f1SChristoph Hellwig { 49130879f1SChristoph Hellwig struct bio_vec *bvec; 50130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 51130879f1SChristoph Hellwig 52130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 53130879f1SChristoph Hellwig ssize_t ret; 54130879f1SChristoph Hellwig 55130879f1SChristoph Hellwig ret = copy_page_from_iter(bvec->bv_page, 56130879f1SChristoph Hellwig bvec->bv_offset, 57130879f1SChristoph Hellwig bvec->bv_len, 58130879f1SChristoph Hellwig iter); 59130879f1SChristoph Hellwig 60130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 61130879f1SChristoph Hellwig break; 62130879f1SChristoph Hellwig 63130879f1SChristoph Hellwig if (ret < bvec->bv_len) 64130879f1SChristoph Hellwig return -EFAULT; 65130879f1SChristoph Hellwig } 66130879f1SChristoph Hellwig 67130879f1SChristoph Hellwig return 0; 68130879f1SChristoph Hellwig } 69130879f1SChristoph Hellwig 70130879f1SChristoph Hellwig /** 71130879f1SChristoph Hellwig * bio_copy_to_iter - copy all pages from bio to iov_iter 72130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as source 73130879f1SChristoph Hellwig * @iter: iov_iter as destination 74130879f1SChristoph Hellwig * 75130879f1SChristoph Hellwig * Copy all pages from bio to iov_iter. 76130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 77130879f1SChristoph Hellwig */ 78130879f1SChristoph Hellwig static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) 79130879f1SChristoph Hellwig { 80130879f1SChristoph Hellwig struct bio_vec *bvec; 81130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 82130879f1SChristoph Hellwig 83130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 84130879f1SChristoph Hellwig ssize_t ret; 85130879f1SChristoph Hellwig 86130879f1SChristoph Hellwig ret = copy_page_to_iter(bvec->bv_page, 87130879f1SChristoph Hellwig bvec->bv_offset, 88130879f1SChristoph Hellwig bvec->bv_len, 89130879f1SChristoph Hellwig &iter); 90130879f1SChristoph Hellwig 91130879f1SChristoph Hellwig if (!iov_iter_count(&iter)) 92130879f1SChristoph Hellwig break; 93130879f1SChristoph Hellwig 94130879f1SChristoph Hellwig if (ret < bvec->bv_len) 95130879f1SChristoph Hellwig return -EFAULT; 96130879f1SChristoph Hellwig } 97130879f1SChristoph Hellwig 98130879f1SChristoph Hellwig return 0; 99130879f1SChristoph Hellwig } 100130879f1SChristoph Hellwig 101130879f1SChristoph Hellwig /** 102130879f1SChristoph Hellwig * bio_uncopy_user - finish previously mapped bio 103130879f1SChristoph Hellwig * @bio: bio being terminated 104130879f1SChristoph Hellwig * 105130879f1SChristoph Hellwig * Free pages allocated from bio_copy_user_iov() and write back data 106130879f1SChristoph Hellwig * to user space in case of a read. 107130879f1SChristoph Hellwig */ 108130879f1SChristoph Hellwig static int bio_uncopy_user(struct bio *bio) 109130879f1SChristoph Hellwig { 110130879f1SChristoph Hellwig struct bio_map_data *bmd = bio->bi_private; 111130879f1SChristoph Hellwig int ret = 0; 112130879f1SChristoph Hellwig 1133310eebaSChristoph Hellwig if (!bmd->is_null_mapped) { 114130879f1SChristoph Hellwig /* 115130879f1SChristoph Hellwig * if we're in a workqueue, the request is orphaned, so 116130879f1SChristoph Hellwig * don't copy into a random user address space, just free 117130879f1SChristoph Hellwig * and return -EINTR so user space doesn't expect any data. 118130879f1SChristoph Hellwig */ 119130879f1SChristoph Hellwig if (!current->mm) 120130879f1SChristoph Hellwig ret = -EINTR; 121130879f1SChristoph Hellwig else if (bio_data_dir(bio) == READ) 122130879f1SChristoph Hellwig ret = bio_copy_to_iter(bio, bmd->iter); 123130879f1SChristoph Hellwig if (bmd->is_our_pages) 124130879f1SChristoph Hellwig bio_free_pages(bio); 125130879f1SChristoph Hellwig } 126130879f1SChristoph Hellwig kfree(bmd); 127130879f1SChristoph Hellwig return ret; 128130879f1SChristoph Hellwig } 129130879f1SChristoph Hellwig 1307589ad67SChristoph Hellwig static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, 1317589ad67SChristoph Hellwig struct iov_iter *iter, gfp_t gfp_mask) 132130879f1SChristoph Hellwig { 133130879f1SChristoph Hellwig struct bio_map_data *bmd; 134130879f1SChristoph Hellwig struct page *page; 135393bb12eSChristoph Hellwig struct bio *bio; 136130879f1SChristoph Hellwig int i = 0, ret; 137130879f1SChristoph Hellwig int nr_pages; 138130879f1SChristoph Hellwig unsigned int len = iter->count; 139130879f1SChristoph Hellwig unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; 140130879f1SChristoph Hellwig 141130879f1SChristoph Hellwig bmd = bio_alloc_map_data(iter, gfp_mask); 142130879f1SChristoph Hellwig if (!bmd) 1437589ad67SChristoph Hellwig return -ENOMEM; 144130879f1SChristoph Hellwig 145130879f1SChristoph Hellwig /* 146130879f1SChristoph Hellwig * We need to do a deep copy of the iov_iter including the iovecs. 147130879f1SChristoph Hellwig * The caller provided iov might point to an on-stack or otherwise 148130879f1SChristoph Hellwig * shortlived one. 149130879f1SChristoph Hellwig */ 150f3256075SChristoph Hellwig bmd->is_our_pages = !map_data; 15103859717SChristoph Hellwig bmd->is_null_mapped = (map_data && map_data->null_mapped); 152130879f1SChristoph Hellwig 1535f7136dbSMatthew Wilcox (Oracle) nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); 154130879f1SChristoph Hellwig 155130879f1SChristoph Hellwig ret = -ENOMEM; 156066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 157130879f1SChristoph Hellwig if (!bio) 158130879f1SChristoph Hellwig goto out_bmd; 159066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); 160130879f1SChristoph Hellwig 161130879f1SChristoph Hellwig if (map_data) { 162f5d632d1SJens Axboe nr_pages = 1U << map_data->page_order; 163130879f1SChristoph Hellwig i = map_data->offset / PAGE_SIZE; 164130879f1SChristoph Hellwig } 165130879f1SChristoph Hellwig while (len) { 166130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 167130879f1SChristoph Hellwig 168130879f1SChristoph Hellwig bytes -= offset; 169130879f1SChristoph Hellwig 170130879f1SChristoph Hellwig if (bytes > len) 171130879f1SChristoph Hellwig bytes = len; 172130879f1SChristoph Hellwig 173130879f1SChristoph Hellwig if (map_data) { 174130879f1SChristoph Hellwig if (i == map_data->nr_entries * nr_pages) { 175130879f1SChristoph Hellwig ret = -ENOMEM; 1767589ad67SChristoph Hellwig goto cleanup; 177130879f1SChristoph Hellwig } 178130879f1SChristoph Hellwig 179130879f1SChristoph Hellwig page = map_data->pages[i / nr_pages]; 180130879f1SChristoph Hellwig page += (i % nr_pages); 181130879f1SChristoph Hellwig 182130879f1SChristoph Hellwig i++; 183130879f1SChristoph Hellwig } else { 184ce288e05SChristoph Hellwig page = alloc_page(GFP_NOIO | gfp_mask); 185130879f1SChristoph Hellwig if (!page) { 186130879f1SChristoph Hellwig ret = -ENOMEM; 1877589ad67SChristoph Hellwig goto cleanup; 188130879f1SChristoph Hellwig } 189130879f1SChristoph Hellwig } 190130879f1SChristoph Hellwig 1917589ad67SChristoph Hellwig if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) { 192130879f1SChristoph Hellwig if (!map_data) 193130879f1SChristoph Hellwig __free_page(page); 194130879f1SChristoph Hellwig break; 195130879f1SChristoph Hellwig } 196130879f1SChristoph Hellwig 197130879f1SChristoph Hellwig len -= bytes; 198130879f1SChristoph Hellwig offset = 0; 199130879f1SChristoph Hellwig } 200130879f1SChristoph Hellwig 201130879f1SChristoph Hellwig if (map_data) 202130879f1SChristoph Hellwig map_data->offset += bio->bi_iter.bi_size; 203130879f1SChristoph Hellwig 204130879f1SChristoph Hellwig /* 205130879f1SChristoph Hellwig * success 206130879f1SChristoph Hellwig */ 207130879f1SChristoph Hellwig if ((iov_iter_rw(iter) == WRITE && 208130879f1SChristoph Hellwig (!map_data || !map_data->null_mapped)) || 209130879f1SChristoph Hellwig (map_data && map_data->from_user)) { 210130879f1SChristoph Hellwig ret = bio_copy_from_iter(bio, iter); 211130879f1SChristoph Hellwig if (ret) 212130879f1SChristoph Hellwig goto cleanup; 213130879f1SChristoph Hellwig } else { 214130879f1SChristoph Hellwig if (bmd->is_our_pages) 215130879f1SChristoph Hellwig zero_fill_bio(bio); 216130879f1SChristoph Hellwig iov_iter_advance(iter, bio->bi_iter.bi_size); 217130879f1SChristoph Hellwig } 218130879f1SChristoph Hellwig 219130879f1SChristoph Hellwig bio->bi_private = bmd; 2207589ad67SChristoph Hellwig 221393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 2227589ad67SChristoph Hellwig if (ret) 2237589ad67SChristoph Hellwig goto cleanup; 2247589ad67SChristoph Hellwig return 0; 225130879f1SChristoph Hellwig cleanup: 226130879f1SChristoph Hellwig if (!map_data) 227130879f1SChristoph Hellwig bio_free_pages(bio); 228066ff571SChristoph Hellwig bio_uninit(bio); 229066ff571SChristoph Hellwig kfree(bio); 230130879f1SChristoph Hellwig out_bmd: 231130879f1SChristoph Hellwig kfree(bmd); 2327589ad67SChristoph Hellwig return ret; 233130879f1SChristoph Hellwig } 234130879f1SChristoph Hellwig 23532f1c71bSAnuj Gupta static void blk_mq_map_bio_put(struct bio *bio) 2368af870aaSJens Axboe { 2378af870aaSJens Axboe if (bio->bi_opf & REQ_ALLOC_CACHE) { 2388af870aaSJens Axboe bio_put(bio); 2398af870aaSJens Axboe } else { 2408af870aaSJens Axboe bio_uninit(bio); 2418af870aaSJens Axboe kfree(bio); 2428af870aaSJens Axboe } 2438af870aaSJens Axboe } 2448af870aaSJens Axboe 245ab89e8e7SKanchan Joshi static struct bio *blk_rq_map_bio_alloc(struct request *rq, 246ab89e8e7SKanchan Joshi unsigned int nr_vecs, gfp_t gfp_mask) 247ab89e8e7SKanchan Joshi { 248ab89e8e7SKanchan Joshi struct bio *bio; 249ab89e8e7SKanchan Joshi 250ab89e8e7SKanchan Joshi if (rq->cmd_flags & REQ_POLLED) { 251ab89e8e7SKanchan Joshi blk_opf_t opf = rq->cmd_flags | REQ_ALLOC_CACHE; 252ab89e8e7SKanchan Joshi 253ab89e8e7SKanchan Joshi bio = bio_alloc_bioset(NULL, nr_vecs, opf, gfp_mask, 254ab89e8e7SKanchan Joshi &fs_bio_set); 255ab89e8e7SKanchan Joshi if (!bio) 256ab89e8e7SKanchan Joshi return NULL; 257ab89e8e7SKanchan Joshi } else { 258ab89e8e7SKanchan Joshi bio = bio_kmalloc(nr_vecs, gfp_mask); 259ab89e8e7SKanchan Joshi if (!bio) 260ab89e8e7SKanchan Joshi return NULL; 261ab89e8e7SKanchan Joshi bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); 262ab89e8e7SKanchan Joshi } 263ab89e8e7SKanchan Joshi return bio; 264ab89e8e7SKanchan Joshi } 265ab89e8e7SKanchan Joshi 2667589ad67SChristoph Hellwig static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, 2677589ad67SChristoph Hellwig gfp_t gfp_mask) 268130879f1SChristoph Hellwig { 2697589ad67SChristoph Hellwig unsigned int max_sectors = queue_max_hw_sectors(rq->q); 270066ff571SChristoph Hellwig unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); 2717ee4ccf5SLogan Gunthorpe unsigned int gup_flags = 0; 272393bb12eSChristoph Hellwig struct bio *bio; 273130879f1SChristoph Hellwig int ret; 2747589ad67SChristoph Hellwig int j; 275130879f1SChristoph Hellwig 276130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 2777589ad67SChristoph Hellwig return -EINVAL; 278130879f1SChristoph Hellwig 279ab89e8e7SKanchan Joshi bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); 280ab89e8e7SKanchan Joshi if (bio == NULL) 2818af870aaSJens Axboe return -ENOMEM; 282130879f1SChristoph Hellwig 2837ee4ccf5SLogan Gunthorpe if (blk_queue_pci_p2pdma(rq->q)) 2847ee4ccf5SLogan Gunthorpe gup_flags |= FOLL_PCI_P2PDMA; 2857ee4ccf5SLogan Gunthorpe 286130879f1SChristoph Hellwig while (iov_iter_count(iter)) { 287e88811bcSJens Axboe struct page **pages, *stack_pages[UIO_FASTIOV]; 288130879f1SChristoph Hellwig ssize_t bytes; 28991e5addaSJiapeng Chong size_t offs; 290130879f1SChristoph Hellwig int npages; 291130879f1SChristoph Hellwig 292e88811bcSJens Axboe if (nr_vecs <= ARRAY_SIZE(stack_pages)) { 293e88811bcSJens Axboe pages = stack_pages; 2947ee4ccf5SLogan Gunthorpe bytes = iov_iter_get_pages(iter, pages, LONG_MAX, 2957ee4ccf5SLogan Gunthorpe nr_vecs, &offs, gup_flags); 296e88811bcSJens Axboe } else { 2977ee4ccf5SLogan Gunthorpe bytes = iov_iter_get_pages_alloc(iter, &pages, 2987ee4ccf5SLogan Gunthorpe LONG_MAX, &offs, gup_flags); 299e88811bcSJens Axboe } 300130879f1SChristoph Hellwig if (unlikely(bytes <= 0)) { 301130879f1SChristoph Hellwig ret = bytes ? bytes : -EFAULT; 302130879f1SChristoph Hellwig goto out_unmap; 303130879f1SChristoph Hellwig } 304130879f1SChristoph Hellwig 305130879f1SChristoph Hellwig npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); 306130879f1SChristoph Hellwig 3077ab89db9SMichal Orzel if (unlikely(offs & queue_dma_alignment(rq->q))) 308130879f1SChristoph Hellwig j = 0; 3097ab89db9SMichal Orzel else { 310130879f1SChristoph Hellwig for (j = 0; j < npages; j++) { 311130879f1SChristoph Hellwig struct page *page = pages[j]; 312130879f1SChristoph Hellwig unsigned int n = PAGE_SIZE - offs; 313130879f1SChristoph Hellwig bool same_page = false; 314130879f1SChristoph Hellwig 315130879f1SChristoph Hellwig if (n > bytes) 316130879f1SChristoph Hellwig n = bytes; 317130879f1SChristoph Hellwig 3187589ad67SChristoph Hellwig if (!bio_add_hw_page(rq->q, bio, page, n, offs, 319e4581105SChristoph Hellwig max_sectors, &same_page)) { 320130879f1SChristoph Hellwig if (same_page) 321130879f1SChristoph Hellwig put_page(page); 322130879f1SChristoph Hellwig break; 323130879f1SChristoph Hellwig } 324130879f1SChristoph Hellwig 325130879f1SChristoph Hellwig bytes -= n; 326130879f1SChristoph Hellwig offs = 0; 327130879f1SChristoph Hellwig } 328130879f1SChristoph Hellwig } 329130879f1SChristoph Hellwig /* 330130879f1SChristoph Hellwig * release the pages we didn't map into the bio, if any 331130879f1SChristoph Hellwig */ 332130879f1SChristoph Hellwig while (j < npages) 333130879f1SChristoph Hellwig put_page(pages[j++]); 334e88811bcSJens Axboe if (pages != stack_pages) 335130879f1SChristoph Hellwig kvfree(pages); 336130879f1SChristoph Hellwig /* couldn't stuff something into bio? */ 337480cb846SAl Viro if (bytes) { 338480cb846SAl Viro iov_iter_revert(iter, bytes); 339130879f1SChristoph Hellwig break; 340130879f1SChristoph Hellwig } 341480cb846SAl Viro } 342130879f1SChristoph Hellwig 343393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 3447589ad67SChristoph Hellwig if (ret) 345393bb12eSChristoph Hellwig goto out_unmap; 3467589ad67SChristoph Hellwig return 0; 3477589ad67SChristoph Hellwig 348130879f1SChristoph Hellwig out_unmap: 349130879f1SChristoph Hellwig bio_release_pages(bio, false); 35032f1c71bSAnuj Gupta blk_mq_map_bio_put(bio); 3517589ad67SChristoph Hellwig return ret; 352130879f1SChristoph Hellwig } 353130879f1SChristoph Hellwig 354130879f1SChristoph Hellwig static void bio_invalidate_vmalloc_pages(struct bio *bio) 355130879f1SChristoph Hellwig { 356f358afc5SChristoph Hellwig #ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 357130879f1SChristoph Hellwig if (bio->bi_private && !op_is_write(bio_op(bio))) { 358130879f1SChristoph Hellwig unsigned long i, len = 0; 359130879f1SChristoph Hellwig 360130879f1SChristoph Hellwig for (i = 0; i < bio->bi_vcnt; i++) 361130879f1SChristoph Hellwig len += bio->bi_io_vec[i].bv_len; 362130879f1SChristoph Hellwig invalidate_kernel_vmap_range(bio->bi_private, len); 363130879f1SChristoph Hellwig } 364130879f1SChristoph Hellwig #endif 365130879f1SChristoph Hellwig } 366130879f1SChristoph Hellwig 367130879f1SChristoph Hellwig static void bio_map_kern_endio(struct bio *bio) 368130879f1SChristoph Hellwig { 369130879f1SChristoph Hellwig bio_invalidate_vmalloc_pages(bio); 370066ff571SChristoph Hellwig bio_uninit(bio); 371066ff571SChristoph Hellwig kfree(bio); 372130879f1SChristoph Hellwig } 373130879f1SChristoph Hellwig 374130879f1SChristoph Hellwig /** 375130879f1SChristoph Hellwig * bio_map_kern - map kernel address into bio 376130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 377130879f1SChristoph Hellwig * @data: pointer to buffer to map 378130879f1SChristoph Hellwig * @len: length in bytes 379130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio allocation 380130879f1SChristoph Hellwig * 381130879f1SChristoph Hellwig * Map the kernel address into a bio suitable for io to a block 382130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 383130879f1SChristoph Hellwig */ 384130879f1SChristoph Hellwig static struct bio *bio_map_kern(struct request_queue *q, void *data, 385130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask) 386130879f1SChristoph Hellwig { 387130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 388130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 389130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 390130879f1SChristoph Hellwig const int nr_pages = end - start; 391130879f1SChristoph Hellwig bool is_vmalloc = is_vmalloc_addr(data); 392130879f1SChristoph Hellwig struct page *page; 393130879f1SChristoph Hellwig int offset, i; 394130879f1SChristoph Hellwig struct bio *bio; 395130879f1SChristoph Hellwig 396066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 397130879f1SChristoph Hellwig if (!bio) 398130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 399066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 400130879f1SChristoph Hellwig 401130879f1SChristoph Hellwig if (is_vmalloc) { 402130879f1SChristoph Hellwig flush_kernel_vmap_range(data, len); 403130879f1SChristoph Hellwig bio->bi_private = data; 404130879f1SChristoph Hellwig } 405130879f1SChristoph Hellwig 406130879f1SChristoph Hellwig offset = offset_in_page(kaddr); 407130879f1SChristoph Hellwig for (i = 0; i < nr_pages; i++) { 408130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE - offset; 409130879f1SChristoph Hellwig 410130879f1SChristoph Hellwig if (len <= 0) 411130879f1SChristoph Hellwig break; 412130879f1SChristoph Hellwig 413130879f1SChristoph Hellwig if (bytes > len) 414130879f1SChristoph Hellwig bytes = len; 415130879f1SChristoph Hellwig 416130879f1SChristoph Hellwig if (!is_vmalloc) 417130879f1SChristoph Hellwig page = virt_to_page(data); 418130879f1SChristoph Hellwig else 419130879f1SChristoph Hellwig page = vmalloc_to_page(data); 420130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 421130879f1SChristoph Hellwig offset) < bytes) { 422130879f1SChristoph Hellwig /* we don't support partial mappings */ 423066ff571SChristoph Hellwig bio_uninit(bio); 424066ff571SChristoph Hellwig kfree(bio); 425130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 426130879f1SChristoph Hellwig } 427130879f1SChristoph Hellwig 428130879f1SChristoph Hellwig data += bytes; 429130879f1SChristoph Hellwig len -= bytes; 430130879f1SChristoph Hellwig offset = 0; 431130879f1SChristoph Hellwig } 432130879f1SChristoph Hellwig 433130879f1SChristoph Hellwig bio->bi_end_io = bio_map_kern_endio; 434130879f1SChristoph Hellwig return bio; 435130879f1SChristoph Hellwig } 436130879f1SChristoph Hellwig 437130879f1SChristoph Hellwig static void bio_copy_kern_endio(struct bio *bio) 438130879f1SChristoph Hellwig { 439130879f1SChristoph Hellwig bio_free_pages(bio); 440066ff571SChristoph Hellwig bio_uninit(bio); 441066ff571SChristoph Hellwig kfree(bio); 442130879f1SChristoph Hellwig } 443130879f1SChristoph Hellwig 444130879f1SChristoph Hellwig static void bio_copy_kern_endio_read(struct bio *bio) 445130879f1SChristoph Hellwig { 446130879f1SChristoph Hellwig char *p = bio->bi_private; 447130879f1SChristoph Hellwig struct bio_vec *bvec; 448130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 449130879f1SChristoph Hellwig 450130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 451d24920e2SChristoph Hellwig memcpy_from_bvec(p, bvec); 452130879f1SChristoph Hellwig p += bvec->bv_len; 453130879f1SChristoph Hellwig } 454130879f1SChristoph Hellwig 455130879f1SChristoph Hellwig bio_copy_kern_endio(bio); 456130879f1SChristoph Hellwig } 457130879f1SChristoph Hellwig 458130879f1SChristoph Hellwig /** 459130879f1SChristoph Hellwig * bio_copy_kern - copy kernel address into bio 460130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 461130879f1SChristoph Hellwig * @data: pointer to buffer to copy 462130879f1SChristoph Hellwig * @len: length in bytes 463130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio and page allocation 464130879f1SChristoph Hellwig * @reading: data direction is READ 465130879f1SChristoph Hellwig * 466130879f1SChristoph Hellwig * copy the kernel address into a bio suitable for io to a block 467130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 468130879f1SChristoph Hellwig */ 469130879f1SChristoph Hellwig static struct bio *bio_copy_kern(struct request_queue *q, void *data, 470130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask, int reading) 471130879f1SChristoph Hellwig { 472130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 473130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 474130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 475130879f1SChristoph Hellwig struct bio *bio; 476130879f1SChristoph Hellwig void *p = data; 477130879f1SChristoph Hellwig int nr_pages = 0; 478130879f1SChristoph Hellwig 479130879f1SChristoph Hellwig /* 480130879f1SChristoph Hellwig * Overflow, abort 481130879f1SChristoph Hellwig */ 482130879f1SChristoph Hellwig if (end < start) 483130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 484130879f1SChristoph Hellwig 485130879f1SChristoph Hellwig nr_pages = end - start; 486066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 487130879f1SChristoph Hellwig if (!bio) 488130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 489066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 490130879f1SChristoph Hellwig 491130879f1SChristoph Hellwig while (len) { 492130879f1SChristoph Hellwig struct page *page; 493130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 494130879f1SChristoph Hellwig 495130879f1SChristoph Hellwig if (bytes > len) 496130879f1SChristoph Hellwig bytes = len; 497130879f1SChristoph Hellwig 498cc8f7fe1SHaimin Zhang page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask); 499130879f1SChristoph Hellwig if (!page) 500130879f1SChristoph Hellwig goto cleanup; 501130879f1SChristoph Hellwig 502130879f1SChristoph Hellwig if (!reading) 503130879f1SChristoph Hellwig memcpy(page_address(page), p, bytes); 504130879f1SChristoph Hellwig 505130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) 506130879f1SChristoph Hellwig break; 507130879f1SChristoph Hellwig 508130879f1SChristoph Hellwig len -= bytes; 509130879f1SChristoph Hellwig p += bytes; 510130879f1SChristoph Hellwig } 511130879f1SChristoph Hellwig 512130879f1SChristoph Hellwig if (reading) { 513130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio_read; 514130879f1SChristoph Hellwig bio->bi_private = data; 515130879f1SChristoph Hellwig } else { 516130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio; 517130879f1SChristoph Hellwig } 518130879f1SChristoph Hellwig 519130879f1SChristoph Hellwig return bio; 520130879f1SChristoph Hellwig 521130879f1SChristoph Hellwig cleanup: 522130879f1SChristoph Hellwig bio_free_pages(bio); 523066ff571SChristoph Hellwig bio_uninit(bio); 524066ff571SChristoph Hellwig kfree(bio); 525130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 526130879f1SChristoph Hellwig } 527130879f1SChristoph Hellwig 52898d61d5bSChristoph Hellwig /* 5290abc2a10SJens Axboe * Append a bio to a passthrough request. Only works if the bio can be merged 5300abc2a10SJens Axboe * into the request based on the driver constraints. 53198d61d5bSChristoph Hellwig */ 532393bb12eSChristoph Hellwig int blk_rq_append_bio(struct request *rq, struct bio *bio) 53386db1e29SJens Axboe { 53414ccb66bSChristoph Hellwig struct bvec_iter iter; 53514ccb66bSChristoph Hellwig struct bio_vec bv; 53614ccb66bSChristoph Hellwig unsigned int nr_segs = 0; 5370abc2a10SJens Axboe 538393bb12eSChristoph Hellwig bio_for_each_bvec(bv, bio, iter) 53914ccb66bSChristoph Hellwig nr_segs++; 54014ccb66bSChristoph Hellwig 54198d61d5bSChristoph Hellwig if (!rq->bio) { 542393bb12eSChristoph Hellwig blk_rq_bio_prep(rq, bio, nr_segs); 54398d61d5bSChristoph Hellwig } else { 544393bb12eSChristoph Hellwig if (!ll_back_merge_fn(rq, bio, nr_segs)) 54586db1e29SJens Axboe return -EINVAL; 546393bb12eSChristoph Hellwig rq->biotail->bi_next = bio; 547393bb12eSChristoph Hellwig rq->biotail = bio; 548393bb12eSChristoph Hellwig rq->__data_len += (bio)->bi_iter.bi_size; 549393bb12eSChristoph Hellwig bio_crypt_free_ctx(bio); 55086db1e29SJens Axboe } 55198d61d5bSChristoph Hellwig 55286db1e29SJens Axboe return 0; 55386db1e29SJens Axboe } 55498d61d5bSChristoph Hellwig EXPORT_SYMBOL(blk_rq_append_bio); 55586db1e29SJens Axboe 55637987547SKanchan Joshi /* Prepare bio for passthrough IO given ITER_BVEC iter */ 55737987547SKanchan Joshi static int blk_rq_map_user_bvec(struct request *rq, const struct iov_iter *iter) 55837987547SKanchan Joshi { 55937987547SKanchan Joshi struct request_queue *q = rq->q; 56037987547SKanchan Joshi size_t nr_iter = iov_iter_count(iter); 56137987547SKanchan Joshi size_t nr_segs = iter->nr_segs; 56237987547SKanchan Joshi struct bio_vec *bvecs, *bvprvp = NULL; 563aa261f20SBart Van Assche const struct queue_limits *lim = &q->limits; 56437987547SKanchan Joshi unsigned int nsegs = 0, bytes = 0; 56537987547SKanchan Joshi struct bio *bio; 56637987547SKanchan Joshi size_t i; 56737987547SKanchan Joshi 56837987547SKanchan Joshi if (!nr_iter || (nr_iter >> SECTOR_SHIFT) > queue_max_hw_sectors(q)) 56937987547SKanchan Joshi return -EINVAL; 57037987547SKanchan Joshi if (nr_segs > queue_max_segments(q)) 57137987547SKanchan Joshi return -EINVAL; 57237987547SKanchan Joshi 57337987547SKanchan Joshi /* no iovecs to alloc, as we already have a BVEC iterator */ 57437987547SKanchan Joshi bio = blk_rq_map_bio_alloc(rq, 0, GFP_KERNEL); 57537987547SKanchan Joshi if (bio == NULL) 57637987547SKanchan Joshi return -ENOMEM; 57737987547SKanchan Joshi 57837987547SKanchan Joshi bio_iov_bvec_set(bio, (struct iov_iter *)iter); 57937987547SKanchan Joshi blk_rq_bio_prep(rq, bio, nr_segs); 58037987547SKanchan Joshi 58137987547SKanchan Joshi /* loop to perform a bunch of sanity checks */ 58237987547SKanchan Joshi bvecs = (struct bio_vec *)iter->bvec; 58337987547SKanchan Joshi for (i = 0; i < nr_segs; i++) { 58437987547SKanchan Joshi struct bio_vec *bv = &bvecs[i]; 58537987547SKanchan Joshi 58637987547SKanchan Joshi /* 58737987547SKanchan Joshi * If the queue doesn't support SG gaps and adding this 58837987547SKanchan Joshi * offset would create a gap, fallback to copy. 58937987547SKanchan Joshi */ 59037987547SKanchan Joshi if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv->bv_offset)) { 59137987547SKanchan Joshi blk_mq_map_bio_put(bio); 59237987547SKanchan Joshi return -EREMOTEIO; 59337987547SKanchan Joshi } 59437987547SKanchan Joshi /* check full condition */ 59537987547SKanchan Joshi if (nsegs >= nr_segs || bytes > UINT_MAX - bv->bv_len) 59637987547SKanchan Joshi goto put_bio; 59737987547SKanchan Joshi if (bytes + bv->bv_len > nr_iter) 59837987547SKanchan Joshi goto put_bio; 59937987547SKanchan Joshi if (bv->bv_offset + bv->bv_len > PAGE_SIZE) 60037987547SKanchan Joshi goto put_bio; 60137987547SKanchan Joshi 60237987547SKanchan Joshi nsegs++; 60337987547SKanchan Joshi bytes += bv->bv_len; 60437987547SKanchan Joshi bvprvp = bv; 60537987547SKanchan Joshi } 60637987547SKanchan Joshi return 0; 60737987547SKanchan Joshi put_bio: 60837987547SKanchan Joshi blk_mq_map_bio_put(bio); 60937987547SKanchan Joshi return -EINVAL; 61037987547SKanchan Joshi } 61137987547SKanchan Joshi 61286db1e29SJens Axboe /** 613aebf526bSChristoph Hellwig * blk_rq_map_user_iov - map user data to a request, for passthrough requests 61486db1e29SJens Axboe * @q: request queue where request should be inserted 61586db1e29SJens Axboe * @rq: request to map data to 616152e283fSFUJITA Tomonori * @map_data: pointer to the rq_map_data holding pages (if necessary) 61726e49cfcSKent Overstreet * @iter: iovec iterator 618a3bce90eSFUJITA Tomonori * @gfp_mask: memory allocation flags 61986db1e29SJens Axboe * 62086db1e29SJens Axboe * Description: 621710027a4SRandy Dunlap * Data will be mapped directly for zero copy I/O, if possible. Otherwise 62286db1e29SJens Axboe * a kernel bounce buffer is used. 62386db1e29SJens Axboe * 624710027a4SRandy Dunlap * A matching blk_rq_unmap_user() must be issued at the end of I/O, while 62586db1e29SJens Axboe * still in process context. 62686db1e29SJens Axboe */ 62786db1e29SJens Axboe int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, 62826e49cfcSKent Overstreet struct rq_map_data *map_data, 62926e49cfcSKent Overstreet const struct iov_iter *iter, gfp_t gfp_mask) 63086db1e29SJens Axboe { 63137987547SKanchan Joshi bool copy = false, map_bvec = false; 632357f435dSAl Viro unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); 6334d6af73dSChristoph Hellwig struct bio *bio = NULL; 6344d6af73dSChristoph Hellwig struct iov_iter i; 63569e0927bSDouglas Gilbert int ret = -EINVAL; 63686db1e29SJens Axboe 637357f435dSAl Viro if (map_data) 6384d6af73dSChristoph Hellwig copy = true; 639393bb12eSChristoph Hellwig else if (blk_queue_may_bounce(q)) 640393bb12eSChristoph Hellwig copy = true; 641357f435dSAl Viro else if (iov_iter_alignment(iter) & align) 642357f435dSAl Viro copy = true; 64337987547SKanchan Joshi else if (iov_iter_is_bvec(iter)) 64437987547SKanchan Joshi map_bvec = true; 645*d46aa786SKeith Busch else if (!user_backed_iter(iter)) 64637987547SKanchan Joshi copy = true; 647357f435dSAl Viro else if (queue_virt_boundary(q)) 648357f435dSAl Viro copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); 649afdc1a78SFUJITA Tomonori 65037987547SKanchan Joshi if (map_bvec) { 65137987547SKanchan Joshi ret = blk_rq_map_user_bvec(rq, iter); 65237987547SKanchan Joshi if (!ret) 65337987547SKanchan Joshi return 0; 65437987547SKanchan Joshi if (ret != -EREMOTEIO) 65537987547SKanchan Joshi goto fail; 65637987547SKanchan Joshi /* fall back to copying the data on limits mismatches */ 65737987547SKanchan Joshi copy = true; 65837987547SKanchan Joshi } 65937987547SKanchan Joshi 6604d6af73dSChristoph Hellwig i = *iter; 6614d6af73dSChristoph Hellwig do { 6627589ad67SChristoph Hellwig if (copy) 6637589ad67SChristoph Hellwig ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask); 6647589ad67SChristoph Hellwig else 6657589ad67SChristoph Hellwig ret = bio_map_user_iov(rq, &i, gfp_mask); 6664d6af73dSChristoph Hellwig if (ret) 6674d6af73dSChristoph Hellwig goto unmap_rq; 6684d6af73dSChristoph Hellwig if (!bio) 6694d6af73dSChristoph Hellwig bio = rq->bio; 6704d6af73dSChristoph Hellwig } while (iov_iter_count(&i)); 67186db1e29SJens Axboe 67286db1e29SJens Axboe return 0; 6734d6af73dSChristoph Hellwig 6744d6af73dSChristoph Hellwig unmap_rq: 6753b7995a9SYang Yingliang blk_rq_unmap_user(bio); 676a0ac402cSLinus Torvalds fail: 6774d6af73dSChristoph Hellwig rq->bio = NULL; 67869e0927bSDouglas Gilbert return ret; 67986db1e29SJens Axboe } 680152e283fSFUJITA Tomonori EXPORT_SYMBOL(blk_rq_map_user_iov); 68186db1e29SJens Axboe 682ddad8dd0SChristoph Hellwig int blk_rq_map_user(struct request_queue *q, struct request *rq, 683ddad8dd0SChristoph Hellwig struct rq_map_data *map_data, void __user *ubuf, 684ddad8dd0SChristoph Hellwig unsigned long len, gfp_t gfp_mask) 685ddad8dd0SChristoph Hellwig { 68626e49cfcSKent Overstreet struct iov_iter i; 687*d46aa786SKeith Busch int ret = import_ubuf(rq_data_dir(rq), ubuf, len, &i); 688ddad8dd0SChristoph Hellwig 6898f7e885aSAl Viro if (unlikely(ret < 0)) 6908f7e885aSAl Viro return ret; 691ddad8dd0SChristoph Hellwig 69226e49cfcSKent Overstreet return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); 693ddad8dd0SChristoph Hellwig } 694ddad8dd0SChristoph Hellwig EXPORT_SYMBOL(blk_rq_map_user); 695ddad8dd0SChristoph Hellwig 69655765402SAnuj Gupta int blk_rq_map_user_io(struct request *req, struct rq_map_data *map_data, 69755765402SAnuj Gupta void __user *ubuf, unsigned long buf_len, gfp_t gfp_mask, 69855765402SAnuj Gupta bool vec, int iov_count, bool check_iter_count, int rw) 69955765402SAnuj Gupta { 70055765402SAnuj Gupta int ret = 0; 70155765402SAnuj Gupta 70255765402SAnuj Gupta if (vec) { 70355765402SAnuj Gupta struct iovec fast_iov[UIO_FASTIOV]; 70455765402SAnuj Gupta struct iovec *iov = fast_iov; 70555765402SAnuj Gupta struct iov_iter iter; 70655765402SAnuj Gupta 70755765402SAnuj Gupta ret = import_iovec(rw, ubuf, iov_count ? iov_count : buf_len, 70855765402SAnuj Gupta UIO_FASTIOV, &iov, &iter); 70955765402SAnuj Gupta if (ret < 0) 71055765402SAnuj Gupta return ret; 71155765402SAnuj Gupta 71255765402SAnuj Gupta if (iov_count) { 71355765402SAnuj Gupta /* SG_IO howto says that the shorter of the two wins */ 71455765402SAnuj Gupta iov_iter_truncate(&iter, buf_len); 71555765402SAnuj Gupta if (check_iter_count && !iov_iter_count(&iter)) { 71655765402SAnuj Gupta kfree(iov); 71755765402SAnuj Gupta return -EINVAL; 71855765402SAnuj Gupta } 71955765402SAnuj Gupta } 72055765402SAnuj Gupta 72155765402SAnuj Gupta ret = blk_rq_map_user_iov(req->q, req, map_data, &iter, 72255765402SAnuj Gupta gfp_mask); 72355765402SAnuj Gupta kfree(iov); 72455765402SAnuj Gupta } else if (buf_len) { 72555765402SAnuj Gupta ret = blk_rq_map_user(req->q, req, map_data, ubuf, buf_len, 72655765402SAnuj Gupta gfp_mask); 72755765402SAnuj Gupta } 72855765402SAnuj Gupta return ret; 72955765402SAnuj Gupta } 73055765402SAnuj Gupta EXPORT_SYMBOL(blk_rq_map_user_io); 73155765402SAnuj Gupta 73286db1e29SJens Axboe /** 73386db1e29SJens Axboe * blk_rq_unmap_user - unmap a request with user data 73486db1e29SJens Axboe * @bio: start of bio list 73586db1e29SJens Axboe * 73686db1e29SJens Axboe * Description: 73786db1e29SJens Axboe * Unmap a rq previously mapped by blk_rq_map_user(). The caller must 73886db1e29SJens Axboe * supply the original rq->bio from the blk_rq_map_user() return, since 739710027a4SRandy Dunlap * the I/O completion may have changed rq->bio. 74086db1e29SJens Axboe */ 74186db1e29SJens Axboe int blk_rq_unmap_user(struct bio *bio) 74286db1e29SJens Axboe { 743393bb12eSChristoph Hellwig struct bio *next_bio; 74486db1e29SJens Axboe int ret = 0, ret2; 74586db1e29SJens Axboe 74686db1e29SJens Axboe while (bio) { 7473310eebaSChristoph Hellwig if (bio->bi_private) { 748393bb12eSChristoph Hellwig ret2 = bio_uncopy_user(bio); 74986db1e29SJens Axboe if (ret2 && !ret) 75086db1e29SJens Axboe ret = ret2; 7513310eebaSChristoph Hellwig } else { 752393bb12eSChristoph Hellwig bio_release_pages(bio, bio_data_dir(bio) == READ); 7537b63c052SChristoph Hellwig } 75486db1e29SJens Axboe 755393bb12eSChristoph Hellwig next_bio = bio; 75686db1e29SJens Axboe bio = bio->bi_next; 75732f1c71bSAnuj Gupta blk_mq_map_bio_put(next_bio); 75886db1e29SJens Axboe } 75986db1e29SJens Axboe 76086db1e29SJens Axboe return ret; 76186db1e29SJens Axboe } 76286db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_unmap_user); 76386db1e29SJens Axboe 76486db1e29SJens Axboe /** 765aebf526bSChristoph Hellwig * blk_rq_map_kern - map kernel data to a request, for passthrough requests 76686db1e29SJens Axboe * @q: request queue where request should be inserted 76786db1e29SJens Axboe * @rq: request to fill 76886db1e29SJens Axboe * @kbuf: the kernel buffer 76986db1e29SJens Axboe * @len: length of user data 77086db1e29SJens Axboe * @gfp_mask: memory allocation flags 77168154e90SFUJITA Tomonori * 77268154e90SFUJITA Tomonori * Description: 77368154e90SFUJITA Tomonori * Data will be mapped directly if possible. Otherwise a bounce 774e227867fSMasanari Iida * buffer is used. Can be called multiple times to append multiple 7753a5a3927SJames Bottomley * buffers. 77686db1e29SJens Axboe */ 77786db1e29SJens Axboe int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, 77886db1e29SJens Axboe unsigned int len, gfp_t gfp_mask) 77986db1e29SJens Axboe { 78068154e90SFUJITA Tomonori int reading = rq_data_dir(rq) == READ; 78114417799SNamhyung Kim unsigned long addr = (unsigned long) kbuf; 782393bb12eSChristoph Hellwig struct bio *bio; 7833a5a3927SJames Bottomley int ret; 78486db1e29SJens Axboe 785ae03bf63SMartin K. Petersen if (len > (queue_max_hw_sectors(q) << 9)) 78686db1e29SJens Axboe return -EINVAL; 78786db1e29SJens Axboe if (!len || !kbuf) 78886db1e29SJens Axboe return -EINVAL; 78986db1e29SJens Axboe 790393bb12eSChristoph Hellwig if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || 791393bb12eSChristoph Hellwig blk_queue_may_bounce(q)) 79268154e90SFUJITA Tomonori bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); 79368154e90SFUJITA Tomonori else 79486db1e29SJens Axboe bio = bio_map_kern(q, kbuf, len, gfp_mask); 79568154e90SFUJITA Tomonori 79686db1e29SJens Axboe if (IS_ERR(bio)) 79786db1e29SJens Axboe return PTR_ERR(bio); 79886db1e29SJens Axboe 799aebf526bSChristoph Hellwig bio->bi_opf &= ~REQ_OP_MASK; 800aebf526bSChristoph Hellwig bio->bi_opf |= req_op(rq); 80186db1e29SJens Axboe 802393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 803066ff571SChristoph Hellwig if (unlikely(ret)) { 804066ff571SChristoph Hellwig bio_uninit(bio); 805066ff571SChristoph Hellwig kfree(bio); 806066ff571SChristoph Hellwig } 8073a5a3927SJames Bottomley return ret; 8083a5a3927SJames Bottomley } 80986db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_map_kern); 810