1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 286db1e29SJens Axboe /* 386db1e29SJens Axboe * Functions related to mapping data to requests 486db1e29SJens Axboe */ 586db1e29SJens Axboe #include <linux/kernel.h> 668db0cf1SIngo Molnar #include <linux/sched/task_stack.h> 786db1e29SJens Axboe #include <linux/module.h> 886db1e29SJens Axboe #include <linux/bio.h> 986db1e29SJens Axboe #include <linux/blkdev.h> 1026e49cfcSKent Overstreet #include <linux/uio.h> 1186db1e29SJens Axboe 1286db1e29SJens Axboe #include "blk.h" 1386db1e29SJens Axboe 14130879f1SChristoph Hellwig struct bio_map_data { 15f3256075SChristoph Hellwig bool is_our_pages : 1; 16f3256075SChristoph Hellwig bool is_null_mapped : 1; 17130879f1SChristoph Hellwig struct iov_iter iter; 18130879f1SChristoph Hellwig struct iovec iov[]; 19130879f1SChristoph Hellwig }; 20130879f1SChristoph Hellwig 21130879f1SChristoph Hellwig static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, 22130879f1SChristoph Hellwig gfp_t gfp_mask) 23130879f1SChristoph Hellwig { 24130879f1SChristoph Hellwig struct bio_map_data *bmd; 25130879f1SChristoph Hellwig 26130879f1SChristoph Hellwig if (data->nr_segs > UIO_MAXIOV) 27130879f1SChristoph Hellwig return NULL; 28130879f1SChristoph Hellwig 29130879f1SChristoph Hellwig bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); 30130879f1SChristoph Hellwig if (!bmd) 31130879f1SChristoph Hellwig return NULL; 32130879f1SChristoph Hellwig memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); 33130879f1SChristoph Hellwig bmd->iter = *data; 34130879f1SChristoph Hellwig bmd->iter.iov = bmd->iov; 35130879f1SChristoph Hellwig return bmd; 36130879f1SChristoph Hellwig } 37130879f1SChristoph Hellwig 38130879f1SChristoph Hellwig /** 39130879f1SChristoph Hellwig * bio_copy_from_iter - copy all pages from iov_iter to bio 40130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as destination 41130879f1SChristoph Hellwig * @iter: iov_iter as source 42130879f1SChristoph Hellwig * 43130879f1SChristoph Hellwig * Copy all pages from iov_iter to bio. 44130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 45130879f1SChristoph Hellwig */ 46130879f1SChristoph Hellwig static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) 47130879f1SChristoph Hellwig { 48130879f1SChristoph Hellwig struct bio_vec *bvec; 49130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 50130879f1SChristoph Hellwig 51130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 52130879f1SChristoph Hellwig ssize_t ret; 53130879f1SChristoph Hellwig 54130879f1SChristoph Hellwig ret = copy_page_from_iter(bvec->bv_page, 55130879f1SChristoph Hellwig bvec->bv_offset, 56130879f1SChristoph Hellwig bvec->bv_len, 57130879f1SChristoph Hellwig iter); 58130879f1SChristoph Hellwig 59130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 60130879f1SChristoph Hellwig break; 61130879f1SChristoph Hellwig 62130879f1SChristoph Hellwig if (ret < bvec->bv_len) 63130879f1SChristoph Hellwig return -EFAULT; 64130879f1SChristoph Hellwig } 65130879f1SChristoph Hellwig 66130879f1SChristoph Hellwig return 0; 67130879f1SChristoph Hellwig } 68130879f1SChristoph Hellwig 69130879f1SChristoph Hellwig /** 70130879f1SChristoph Hellwig * bio_copy_to_iter - copy all pages from bio to iov_iter 71130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as source 72130879f1SChristoph Hellwig * @iter: iov_iter as destination 73130879f1SChristoph Hellwig * 74130879f1SChristoph Hellwig * Copy all pages from bio to iov_iter. 75130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 76130879f1SChristoph Hellwig */ 77130879f1SChristoph Hellwig static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) 78130879f1SChristoph Hellwig { 79130879f1SChristoph Hellwig struct bio_vec *bvec; 80130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 81130879f1SChristoph Hellwig 82130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 83130879f1SChristoph Hellwig ssize_t ret; 84130879f1SChristoph Hellwig 85130879f1SChristoph Hellwig ret = copy_page_to_iter(bvec->bv_page, 86130879f1SChristoph Hellwig bvec->bv_offset, 87130879f1SChristoph Hellwig bvec->bv_len, 88130879f1SChristoph Hellwig &iter); 89130879f1SChristoph Hellwig 90130879f1SChristoph Hellwig if (!iov_iter_count(&iter)) 91130879f1SChristoph Hellwig break; 92130879f1SChristoph Hellwig 93130879f1SChristoph Hellwig if (ret < bvec->bv_len) 94130879f1SChristoph Hellwig return -EFAULT; 95130879f1SChristoph Hellwig } 96130879f1SChristoph Hellwig 97130879f1SChristoph Hellwig return 0; 98130879f1SChristoph Hellwig } 99130879f1SChristoph Hellwig 100130879f1SChristoph Hellwig /** 101130879f1SChristoph Hellwig * bio_uncopy_user - finish previously mapped bio 102130879f1SChristoph Hellwig * @bio: bio being terminated 103130879f1SChristoph Hellwig * 104130879f1SChristoph Hellwig * Free pages allocated from bio_copy_user_iov() and write back data 105130879f1SChristoph Hellwig * to user space in case of a read. 106130879f1SChristoph Hellwig */ 107130879f1SChristoph Hellwig static int bio_uncopy_user(struct bio *bio) 108130879f1SChristoph Hellwig { 109130879f1SChristoph Hellwig struct bio_map_data *bmd = bio->bi_private; 110130879f1SChristoph Hellwig int ret = 0; 111130879f1SChristoph Hellwig 1123310eebaSChristoph Hellwig if (!bmd->is_null_mapped) { 113130879f1SChristoph Hellwig /* 114130879f1SChristoph Hellwig * if we're in a workqueue, the request is orphaned, so 115130879f1SChristoph Hellwig * don't copy into a random user address space, just free 116130879f1SChristoph Hellwig * and return -EINTR so user space doesn't expect any data. 117130879f1SChristoph Hellwig */ 118130879f1SChristoph Hellwig if (!current->mm) 119130879f1SChristoph Hellwig ret = -EINTR; 120130879f1SChristoph Hellwig else if (bio_data_dir(bio) == READ) 121130879f1SChristoph Hellwig ret = bio_copy_to_iter(bio, bmd->iter); 122130879f1SChristoph Hellwig if (bmd->is_our_pages) 123130879f1SChristoph Hellwig bio_free_pages(bio); 124130879f1SChristoph Hellwig } 125130879f1SChristoph Hellwig kfree(bmd); 126130879f1SChristoph Hellwig return ret; 127130879f1SChristoph Hellwig } 128130879f1SChristoph Hellwig 1297589ad67SChristoph Hellwig static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, 1307589ad67SChristoph Hellwig struct iov_iter *iter, gfp_t gfp_mask) 131130879f1SChristoph Hellwig { 132130879f1SChristoph Hellwig struct bio_map_data *bmd; 133130879f1SChristoph Hellwig struct page *page; 134393bb12eSChristoph Hellwig struct bio *bio; 135130879f1SChristoph Hellwig int i = 0, ret; 136130879f1SChristoph Hellwig int nr_pages; 137130879f1SChristoph Hellwig unsigned int len = iter->count; 138130879f1SChristoph Hellwig unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; 139130879f1SChristoph Hellwig 140130879f1SChristoph Hellwig bmd = bio_alloc_map_data(iter, gfp_mask); 141130879f1SChristoph Hellwig if (!bmd) 1427589ad67SChristoph Hellwig return -ENOMEM; 143130879f1SChristoph Hellwig 144130879f1SChristoph Hellwig /* 145130879f1SChristoph Hellwig * We need to do a deep copy of the iov_iter including the iovecs. 146130879f1SChristoph Hellwig * The caller provided iov might point to an on-stack or otherwise 147130879f1SChristoph Hellwig * shortlived one. 148130879f1SChristoph Hellwig */ 149f3256075SChristoph Hellwig bmd->is_our_pages = !map_data; 15003859717SChristoph Hellwig bmd->is_null_mapped = (map_data && map_data->null_mapped); 151130879f1SChristoph Hellwig 1525f7136dbSMatthew Wilcox (Oracle) nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); 153130879f1SChristoph Hellwig 154130879f1SChristoph Hellwig ret = -ENOMEM; 155066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 156130879f1SChristoph Hellwig if (!bio) 157130879f1SChristoph Hellwig goto out_bmd; 158066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); 159130879f1SChristoph Hellwig 160130879f1SChristoph Hellwig if (map_data) { 161f5d632d1SJens Axboe nr_pages = 1U << map_data->page_order; 162130879f1SChristoph Hellwig i = map_data->offset / PAGE_SIZE; 163130879f1SChristoph Hellwig } 164130879f1SChristoph Hellwig while (len) { 165130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 166130879f1SChristoph Hellwig 167130879f1SChristoph Hellwig bytes -= offset; 168130879f1SChristoph Hellwig 169130879f1SChristoph Hellwig if (bytes > len) 170130879f1SChristoph Hellwig bytes = len; 171130879f1SChristoph Hellwig 172130879f1SChristoph Hellwig if (map_data) { 173130879f1SChristoph Hellwig if (i == map_data->nr_entries * nr_pages) { 174130879f1SChristoph Hellwig ret = -ENOMEM; 1757589ad67SChristoph Hellwig goto cleanup; 176130879f1SChristoph Hellwig } 177130879f1SChristoph Hellwig 178130879f1SChristoph Hellwig page = map_data->pages[i / nr_pages]; 179130879f1SChristoph Hellwig page += (i % nr_pages); 180130879f1SChristoph Hellwig 181130879f1SChristoph Hellwig i++; 182130879f1SChristoph Hellwig } else { 183ce288e05SChristoph Hellwig page = alloc_page(GFP_NOIO | gfp_mask); 184130879f1SChristoph Hellwig if (!page) { 185130879f1SChristoph Hellwig ret = -ENOMEM; 1867589ad67SChristoph Hellwig goto cleanup; 187130879f1SChristoph Hellwig } 188130879f1SChristoph Hellwig } 189130879f1SChristoph Hellwig 1907589ad67SChristoph Hellwig if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) { 191130879f1SChristoph Hellwig if (!map_data) 192130879f1SChristoph Hellwig __free_page(page); 193130879f1SChristoph Hellwig break; 194130879f1SChristoph Hellwig } 195130879f1SChristoph Hellwig 196130879f1SChristoph Hellwig len -= bytes; 197130879f1SChristoph Hellwig offset = 0; 198130879f1SChristoph Hellwig } 199130879f1SChristoph Hellwig 200130879f1SChristoph Hellwig if (map_data) 201130879f1SChristoph Hellwig map_data->offset += bio->bi_iter.bi_size; 202130879f1SChristoph Hellwig 203130879f1SChristoph Hellwig /* 204130879f1SChristoph Hellwig * success 205130879f1SChristoph Hellwig */ 206130879f1SChristoph Hellwig if ((iov_iter_rw(iter) == WRITE && 207130879f1SChristoph Hellwig (!map_data || !map_data->null_mapped)) || 208130879f1SChristoph Hellwig (map_data && map_data->from_user)) { 209130879f1SChristoph Hellwig ret = bio_copy_from_iter(bio, iter); 210130879f1SChristoph Hellwig if (ret) 211130879f1SChristoph Hellwig goto cleanup; 212130879f1SChristoph Hellwig } else { 213130879f1SChristoph Hellwig if (bmd->is_our_pages) 214130879f1SChristoph Hellwig zero_fill_bio(bio); 215130879f1SChristoph Hellwig iov_iter_advance(iter, bio->bi_iter.bi_size); 216130879f1SChristoph Hellwig } 217130879f1SChristoph Hellwig 218130879f1SChristoph Hellwig bio->bi_private = bmd; 2197589ad67SChristoph Hellwig 220393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 2217589ad67SChristoph Hellwig if (ret) 2227589ad67SChristoph Hellwig goto cleanup; 2237589ad67SChristoph Hellwig return 0; 224130879f1SChristoph Hellwig cleanup: 225130879f1SChristoph Hellwig if (!map_data) 226130879f1SChristoph Hellwig bio_free_pages(bio); 227066ff571SChristoph Hellwig bio_uninit(bio); 228066ff571SChristoph Hellwig kfree(bio); 229130879f1SChristoph Hellwig out_bmd: 230130879f1SChristoph Hellwig kfree(bmd); 2317589ad67SChristoph Hellwig return ret; 232130879f1SChristoph Hellwig } 233130879f1SChristoph Hellwig 234*8af870aaSJens Axboe static void bio_map_put(struct bio *bio) 235*8af870aaSJens Axboe { 236*8af870aaSJens Axboe if (bio->bi_opf & REQ_ALLOC_CACHE) { 237*8af870aaSJens Axboe bio_put(bio); 238*8af870aaSJens Axboe } else { 239*8af870aaSJens Axboe bio_uninit(bio); 240*8af870aaSJens Axboe kfree(bio); 241*8af870aaSJens Axboe } 242*8af870aaSJens Axboe } 243*8af870aaSJens Axboe 2447589ad67SChristoph Hellwig static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, 2457589ad67SChristoph Hellwig gfp_t gfp_mask) 246130879f1SChristoph Hellwig { 2477589ad67SChristoph Hellwig unsigned int max_sectors = queue_max_hw_sectors(rq->q); 248066ff571SChristoph Hellwig unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); 249393bb12eSChristoph Hellwig struct bio *bio; 250130879f1SChristoph Hellwig int ret; 2517589ad67SChristoph Hellwig int j; 252130879f1SChristoph Hellwig 253130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 2547589ad67SChristoph Hellwig return -EINVAL; 255130879f1SChristoph Hellwig 256*8af870aaSJens Axboe if (rq->cmd_flags & REQ_POLLED) { 257*8af870aaSJens Axboe blk_opf_t opf = rq->cmd_flags | REQ_ALLOC_CACHE; 258*8af870aaSJens Axboe 259*8af870aaSJens Axboe bio = bio_alloc_bioset(NULL, nr_vecs, opf, gfp_mask, 260*8af870aaSJens Axboe &fs_bio_set); 261*8af870aaSJens Axboe if (!bio) 262*8af870aaSJens Axboe return -ENOMEM; 263*8af870aaSJens Axboe } else { 264066ff571SChristoph Hellwig bio = bio_kmalloc(nr_vecs, gfp_mask); 265130879f1SChristoph Hellwig if (!bio) 2667589ad67SChristoph Hellwig return -ENOMEM; 267066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); 268*8af870aaSJens Axboe } 269130879f1SChristoph Hellwig 270130879f1SChristoph Hellwig while (iov_iter_count(iter)) { 271130879f1SChristoph Hellwig struct page **pages; 272130879f1SChristoph Hellwig ssize_t bytes; 273130879f1SChristoph Hellwig size_t offs, added = 0; 274130879f1SChristoph Hellwig int npages; 275130879f1SChristoph Hellwig 276480cb846SAl Viro bytes = iov_iter_get_pages_alloc2(iter, &pages, LONG_MAX, &offs); 277130879f1SChristoph Hellwig if (unlikely(bytes <= 0)) { 278130879f1SChristoph Hellwig ret = bytes ? bytes : -EFAULT; 279130879f1SChristoph Hellwig goto out_unmap; 280130879f1SChristoph Hellwig } 281130879f1SChristoph Hellwig 282130879f1SChristoph Hellwig npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); 283130879f1SChristoph Hellwig 2847ab89db9SMichal Orzel if (unlikely(offs & queue_dma_alignment(rq->q))) 285130879f1SChristoph Hellwig j = 0; 2867ab89db9SMichal Orzel else { 287130879f1SChristoph Hellwig for (j = 0; j < npages; j++) { 288130879f1SChristoph Hellwig struct page *page = pages[j]; 289130879f1SChristoph Hellwig unsigned int n = PAGE_SIZE - offs; 290130879f1SChristoph Hellwig bool same_page = false; 291130879f1SChristoph Hellwig 292130879f1SChristoph Hellwig if (n > bytes) 293130879f1SChristoph Hellwig n = bytes; 294130879f1SChristoph Hellwig 2957589ad67SChristoph Hellwig if (!bio_add_hw_page(rq->q, bio, page, n, offs, 296e4581105SChristoph Hellwig max_sectors, &same_page)) { 297130879f1SChristoph Hellwig if (same_page) 298130879f1SChristoph Hellwig put_page(page); 299130879f1SChristoph Hellwig break; 300130879f1SChristoph Hellwig } 301130879f1SChristoph Hellwig 302130879f1SChristoph Hellwig added += n; 303130879f1SChristoph Hellwig bytes -= n; 304130879f1SChristoph Hellwig offs = 0; 305130879f1SChristoph Hellwig } 306130879f1SChristoph Hellwig } 307130879f1SChristoph Hellwig /* 308130879f1SChristoph Hellwig * release the pages we didn't map into the bio, if any 309130879f1SChristoph Hellwig */ 310130879f1SChristoph Hellwig while (j < npages) 311130879f1SChristoph Hellwig put_page(pages[j++]); 312130879f1SChristoph Hellwig kvfree(pages); 313130879f1SChristoph Hellwig /* couldn't stuff something into bio? */ 314480cb846SAl Viro if (bytes) { 315480cb846SAl Viro iov_iter_revert(iter, bytes); 316130879f1SChristoph Hellwig break; 317130879f1SChristoph Hellwig } 318480cb846SAl Viro } 319130879f1SChristoph Hellwig 320393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 3217589ad67SChristoph Hellwig if (ret) 322393bb12eSChristoph Hellwig goto out_unmap; 3237589ad67SChristoph Hellwig return 0; 3247589ad67SChristoph Hellwig 325130879f1SChristoph Hellwig out_unmap: 326130879f1SChristoph Hellwig bio_release_pages(bio, false); 327*8af870aaSJens Axboe bio_map_put(bio); 3287589ad67SChristoph Hellwig return ret; 329130879f1SChristoph Hellwig } 330130879f1SChristoph Hellwig 331130879f1SChristoph Hellwig static void bio_invalidate_vmalloc_pages(struct bio *bio) 332130879f1SChristoph Hellwig { 333f358afc5SChristoph Hellwig #ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 334130879f1SChristoph Hellwig if (bio->bi_private && !op_is_write(bio_op(bio))) { 335130879f1SChristoph Hellwig unsigned long i, len = 0; 336130879f1SChristoph Hellwig 337130879f1SChristoph Hellwig for (i = 0; i < bio->bi_vcnt; i++) 338130879f1SChristoph Hellwig len += bio->bi_io_vec[i].bv_len; 339130879f1SChristoph Hellwig invalidate_kernel_vmap_range(bio->bi_private, len); 340130879f1SChristoph Hellwig } 341130879f1SChristoph Hellwig #endif 342130879f1SChristoph Hellwig } 343130879f1SChristoph Hellwig 344130879f1SChristoph Hellwig static void bio_map_kern_endio(struct bio *bio) 345130879f1SChristoph Hellwig { 346130879f1SChristoph Hellwig bio_invalidate_vmalloc_pages(bio); 347066ff571SChristoph Hellwig bio_uninit(bio); 348066ff571SChristoph Hellwig kfree(bio); 349130879f1SChristoph Hellwig } 350130879f1SChristoph Hellwig 351130879f1SChristoph Hellwig /** 352130879f1SChristoph Hellwig * bio_map_kern - map kernel address into bio 353130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 354130879f1SChristoph Hellwig * @data: pointer to buffer to map 355130879f1SChristoph Hellwig * @len: length in bytes 356130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio allocation 357130879f1SChristoph Hellwig * 358130879f1SChristoph Hellwig * Map the kernel address into a bio suitable for io to a block 359130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 360130879f1SChristoph Hellwig */ 361130879f1SChristoph Hellwig static struct bio *bio_map_kern(struct request_queue *q, void *data, 362130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask) 363130879f1SChristoph Hellwig { 364130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 365130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 366130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 367130879f1SChristoph Hellwig const int nr_pages = end - start; 368130879f1SChristoph Hellwig bool is_vmalloc = is_vmalloc_addr(data); 369130879f1SChristoph Hellwig struct page *page; 370130879f1SChristoph Hellwig int offset, i; 371130879f1SChristoph Hellwig struct bio *bio; 372130879f1SChristoph Hellwig 373066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 374130879f1SChristoph Hellwig if (!bio) 375130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 376066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 377130879f1SChristoph Hellwig 378130879f1SChristoph Hellwig if (is_vmalloc) { 379130879f1SChristoph Hellwig flush_kernel_vmap_range(data, len); 380130879f1SChristoph Hellwig bio->bi_private = data; 381130879f1SChristoph Hellwig } 382130879f1SChristoph Hellwig 383130879f1SChristoph Hellwig offset = offset_in_page(kaddr); 384130879f1SChristoph Hellwig for (i = 0; i < nr_pages; i++) { 385130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE - offset; 386130879f1SChristoph Hellwig 387130879f1SChristoph Hellwig if (len <= 0) 388130879f1SChristoph Hellwig break; 389130879f1SChristoph Hellwig 390130879f1SChristoph Hellwig if (bytes > len) 391130879f1SChristoph Hellwig bytes = len; 392130879f1SChristoph Hellwig 393130879f1SChristoph Hellwig if (!is_vmalloc) 394130879f1SChristoph Hellwig page = virt_to_page(data); 395130879f1SChristoph Hellwig else 396130879f1SChristoph Hellwig page = vmalloc_to_page(data); 397130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 398130879f1SChristoph Hellwig offset) < bytes) { 399130879f1SChristoph Hellwig /* we don't support partial mappings */ 400066ff571SChristoph Hellwig bio_uninit(bio); 401066ff571SChristoph Hellwig kfree(bio); 402130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 403130879f1SChristoph Hellwig } 404130879f1SChristoph Hellwig 405130879f1SChristoph Hellwig data += bytes; 406130879f1SChristoph Hellwig len -= bytes; 407130879f1SChristoph Hellwig offset = 0; 408130879f1SChristoph Hellwig } 409130879f1SChristoph Hellwig 410130879f1SChristoph Hellwig bio->bi_end_io = bio_map_kern_endio; 411130879f1SChristoph Hellwig return bio; 412130879f1SChristoph Hellwig } 413130879f1SChristoph Hellwig 414130879f1SChristoph Hellwig static void bio_copy_kern_endio(struct bio *bio) 415130879f1SChristoph Hellwig { 416130879f1SChristoph Hellwig bio_free_pages(bio); 417066ff571SChristoph Hellwig bio_uninit(bio); 418066ff571SChristoph Hellwig kfree(bio); 419130879f1SChristoph Hellwig } 420130879f1SChristoph Hellwig 421130879f1SChristoph Hellwig static void bio_copy_kern_endio_read(struct bio *bio) 422130879f1SChristoph Hellwig { 423130879f1SChristoph Hellwig char *p = bio->bi_private; 424130879f1SChristoph Hellwig struct bio_vec *bvec; 425130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 426130879f1SChristoph Hellwig 427130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 428d24920e2SChristoph Hellwig memcpy_from_bvec(p, bvec); 429130879f1SChristoph Hellwig p += bvec->bv_len; 430130879f1SChristoph Hellwig } 431130879f1SChristoph Hellwig 432130879f1SChristoph Hellwig bio_copy_kern_endio(bio); 433130879f1SChristoph Hellwig } 434130879f1SChristoph Hellwig 435130879f1SChristoph Hellwig /** 436130879f1SChristoph Hellwig * bio_copy_kern - copy kernel address into bio 437130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 438130879f1SChristoph Hellwig * @data: pointer to buffer to copy 439130879f1SChristoph Hellwig * @len: length in bytes 440130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio and page allocation 441130879f1SChristoph Hellwig * @reading: data direction is READ 442130879f1SChristoph Hellwig * 443130879f1SChristoph Hellwig * copy the kernel address into a bio suitable for io to a block 444130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 445130879f1SChristoph Hellwig */ 446130879f1SChristoph Hellwig static struct bio *bio_copy_kern(struct request_queue *q, void *data, 447130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask, int reading) 448130879f1SChristoph Hellwig { 449130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 450130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 451130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 452130879f1SChristoph Hellwig struct bio *bio; 453130879f1SChristoph Hellwig void *p = data; 454130879f1SChristoph Hellwig int nr_pages = 0; 455130879f1SChristoph Hellwig 456130879f1SChristoph Hellwig /* 457130879f1SChristoph Hellwig * Overflow, abort 458130879f1SChristoph Hellwig */ 459130879f1SChristoph Hellwig if (end < start) 460130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 461130879f1SChristoph Hellwig 462130879f1SChristoph Hellwig nr_pages = end - start; 463066ff571SChristoph Hellwig bio = bio_kmalloc(nr_pages, gfp_mask); 464130879f1SChristoph Hellwig if (!bio) 465130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 466066ff571SChristoph Hellwig bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 467130879f1SChristoph Hellwig 468130879f1SChristoph Hellwig while (len) { 469130879f1SChristoph Hellwig struct page *page; 470130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 471130879f1SChristoph Hellwig 472130879f1SChristoph Hellwig if (bytes > len) 473130879f1SChristoph Hellwig bytes = len; 474130879f1SChristoph Hellwig 475cc8f7fe1SHaimin Zhang page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask); 476130879f1SChristoph Hellwig if (!page) 477130879f1SChristoph Hellwig goto cleanup; 478130879f1SChristoph Hellwig 479130879f1SChristoph Hellwig if (!reading) 480130879f1SChristoph Hellwig memcpy(page_address(page), p, bytes); 481130879f1SChristoph Hellwig 482130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) 483130879f1SChristoph Hellwig break; 484130879f1SChristoph Hellwig 485130879f1SChristoph Hellwig len -= bytes; 486130879f1SChristoph Hellwig p += bytes; 487130879f1SChristoph Hellwig } 488130879f1SChristoph Hellwig 489130879f1SChristoph Hellwig if (reading) { 490130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio_read; 491130879f1SChristoph Hellwig bio->bi_private = data; 492130879f1SChristoph Hellwig } else { 493130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio; 494130879f1SChristoph Hellwig } 495130879f1SChristoph Hellwig 496130879f1SChristoph Hellwig return bio; 497130879f1SChristoph Hellwig 498130879f1SChristoph Hellwig cleanup: 499130879f1SChristoph Hellwig bio_free_pages(bio); 500066ff571SChristoph Hellwig bio_uninit(bio); 501066ff571SChristoph Hellwig kfree(bio); 502130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 503130879f1SChristoph Hellwig } 504130879f1SChristoph Hellwig 50598d61d5bSChristoph Hellwig /* 5060abc2a10SJens Axboe * Append a bio to a passthrough request. Only works if the bio can be merged 5070abc2a10SJens Axboe * into the request based on the driver constraints. 50898d61d5bSChristoph Hellwig */ 509393bb12eSChristoph Hellwig int blk_rq_append_bio(struct request *rq, struct bio *bio) 51086db1e29SJens Axboe { 51114ccb66bSChristoph Hellwig struct bvec_iter iter; 51214ccb66bSChristoph Hellwig struct bio_vec bv; 51314ccb66bSChristoph Hellwig unsigned int nr_segs = 0; 5140abc2a10SJens Axboe 515393bb12eSChristoph Hellwig bio_for_each_bvec(bv, bio, iter) 51614ccb66bSChristoph Hellwig nr_segs++; 51714ccb66bSChristoph Hellwig 51898d61d5bSChristoph Hellwig if (!rq->bio) { 519393bb12eSChristoph Hellwig blk_rq_bio_prep(rq, bio, nr_segs); 52098d61d5bSChristoph Hellwig } else { 521393bb12eSChristoph Hellwig if (!ll_back_merge_fn(rq, bio, nr_segs)) 52286db1e29SJens Axboe return -EINVAL; 523393bb12eSChristoph Hellwig rq->biotail->bi_next = bio; 524393bb12eSChristoph Hellwig rq->biotail = bio; 525393bb12eSChristoph Hellwig rq->__data_len += (bio)->bi_iter.bi_size; 526393bb12eSChristoph Hellwig bio_crypt_free_ctx(bio); 52786db1e29SJens Axboe } 52898d61d5bSChristoph Hellwig 52986db1e29SJens Axboe return 0; 53086db1e29SJens Axboe } 53198d61d5bSChristoph Hellwig EXPORT_SYMBOL(blk_rq_append_bio); 53286db1e29SJens Axboe 53386db1e29SJens Axboe /** 534aebf526bSChristoph Hellwig * blk_rq_map_user_iov - map user data to a request, for passthrough requests 53586db1e29SJens Axboe * @q: request queue where request should be inserted 53686db1e29SJens Axboe * @rq: request to map data to 537152e283fSFUJITA Tomonori * @map_data: pointer to the rq_map_data holding pages (if necessary) 53826e49cfcSKent Overstreet * @iter: iovec iterator 539a3bce90eSFUJITA Tomonori * @gfp_mask: memory allocation flags 54086db1e29SJens Axboe * 54186db1e29SJens Axboe * Description: 542710027a4SRandy Dunlap * Data will be mapped directly for zero copy I/O, if possible. Otherwise 54386db1e29SJens Axboe * a kernel bounce buffer is used. 54486db1e29SJens Axboe * 545710027a4SRandy Dunlap * A matching blk_rq_unmap_user() must be issued at the end of I/O, while 54686db1e29SJens Axboe * still in process context. 54786db1e29SJens Axboe */ 54886db1e29SJens Axboe int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, 54926e49cfcSKent Overstreet struct rq_map_data *map_data, 55026e49cfcSKent Overstreet const struct iov_iter *iter, gfp_t gfp_mask) 55186db1e29SJens Axboe { 552357f435dSAl Viro bool copy = false; 553357f435dSAl Viro unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); 5544d6af73dSChristoph Hellwig struct bio *bio = NULL; 5554d6af73dSChristoph Hellwig struct iov_iter i; 55669e0927bSDouglas Gilbert int ret = -EINVAL; 55786db1e29SJens Axboe 558a0ac402cSLinus Torvalds if (!iter_is_iovec(iter)) 559a0ac402cSLinus Torvalds goto fail; 560a0ac402cSLinus Torvalds 561357f435dSAl Viro if (map_data) 5624d6af73dSChristoph Hellwig copy = true; 563393bb12eSChristoph Hellwig else if (blk_queue_may_bounce(q)) 564393bb12eSChristoph Hellwig copy = true; 565357f435dSAl Viro else if (iov_iter_alignment(iter) & align) 566357f435dSAl Viro copy = true; 567357f435dSAl Viro else if (queue_virt_boundary(q)) 568357f435dSAl Viro copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); 569afdc1a78SFUJITA Tomonori 5704d6af73dSChristoph Hellwig i = *iter; 5714d6af73dSChristoph Hellwig do { 5727589ad67SChristoph Hellwig if (copy) 5737589ad67SChristoph Hellwig ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask); 5747589ad67SChristoph Hellwig else 5757589ad67SChristoph Hellwig ret = bio_map_user_iov(rq, &i, gfp_mask); 5764d6af73dSChristoph Hellwig if (ret) 5774d6af73dSChristoph Hellwig goto unmap_rq; 5784d6af73dSChristoph Hellwig if (!bio) 5794d6af73dSChristoph Hellwig bio = rq->bio; 5804d6af73dSChristoph Hellwig } while (iov_iter_count(&i)); 58186db1e29SJens Axboe 58286db1e29SJens Axboe return 0; 5834d6af73dSChristoph Hellwig 5844d6af73dSChristoph Hellwig unmap_rq: 5853b7995a9SYang Yingliang blk_rq_unmap_user(bio); 586a0ac402cSLinus Torvalds fail: 5874d6af73dSChristoph Hellwig rq->bio = NULL; 58869e0927bSDouglas Gilbert return ret; 58986db1e29SJens Axboe } 590152e283fSFUJITA Tomonori EXPORT_SYMBOL(blk_rq_map_user_iov); 59186db1e29SJens Axboe 592ddad8dd0SChristoph Hellwig int blk_rq_map_user(struct request_queue *q, struct request *rq, 593ddad8dd0SChristoph Hellwig struct rq_map_data *map_data, void __user *ubuf, 594ddad8dd0SChristoph Hellwig unsigned long len, gfp_t gfp_mask) 595ddad8dd0SChristoph Hellwig { 59626e49cfcSKent Overstreet struct iovec iov; 59726e49cfcSKent Overstreet struct iov_iter i; 5988f7e885aSAl Viro int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i); 599ddad8dd0SChristoph Hellwig 6008f7e885aSAl Viro if (unlikely(ret < 0)) 6018f7e885aSAl Viro return ret; 602ddad8dd0SChristoph Hellwig 60326e49cfcSKent Overstreet return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); 604ddad8dd0SChristoph Hellwig } 605ddad8dd0SChristoph Hellwig EXPORT_SYMBOL(blk_rq_map_user); 606ddad8dd0SChristoph Hellwig 60786db1e29SJens Axboe /** 60886db1e29SJens Axboe * blk_rq_unmap_user - unmap a request with user data 60986db1e29SJens Axboe * @bio: start of bio list 61086db1e29SJens Axboe * 61186db1e29SJens Axboe * Description: 61286db1e29SJens Axboe * Unmap a rq previously mapped by blk_rq_map_user(). The caller must 61386db1e29SJens Axboe * supply the original rq->bio from the blk_rq_map_user() return, since 614710027a4SRandy Dunlap * the I/O completion may have changed rq->bio. 61586db1e29SJens Axboe */ 61686db1e29SJens Axboe int blk_rq_unmap_user(struct bio *bio) 61786db1e29SJens Axboe { 618393bb12eSChristoph Hellwig struct bio *next_bio; 61986db1e29SJens Axboe int ret = 0, ret2; 62086db1e29SJens Axboe 62186db1e29SJens Axboe while (bio) { 6223310eebaSChristoph Hellwig if (bio->bi_private) { 623393bb12eSChristoph Hellwig ret2 = bio_uncopy_user(bio); 62486db1e29SJens Axboe if (ret2 && !ret) 62586db1e29SJens Axboe ret = ret2; 6263310eebaSChristoph Hellwig } else { 627393bb12eSChristoph Hellwig bio_release_pages(bio, bio_data_dir(bio) == READ); 6287b63c052SChristoph Hellwig } 62986db1e29SJens Axboe 630393bb12eSChristoph Hellwig next_bio = bio; 63186db1e29SJens Axboe bio = bio->bi_next; 632*8af870aaSJens Axboe bio_map_put(next_bio); 63386db1e29SJens Axboe } 63486db1e29SJens Axboe 63586db1e29SJens Axboe return ret; 63686db1e29SJens Axboe } 63786db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_unmap_user); 63886db1e29SJens Axboe 63986db1e29SJens Axboe /** 640aebf526bSChristoph Hellwig * blk_rq_map_kern - map kernel data to a request, for passthrough requests 64186db1e29SJens Axboe * @q: request queue where request should be inserted 64286db1e29SJens Axboe * @rq: request to fill 64386db1e29SJens Axboe * @kbuf: the kernel buffer 64486db1e29SJens Axboe * @len: length of user data 64586db1e29SJens Axboe * @gfp_mask: memory allocation flags 64668154e90SFUJITA Tomonori * 64768154e90SFUJITA Tomonori * Description: 64868154e90SFUJITA Tomonori * Data will be mapped directly if possible. Otherwise a bounce 649e227867fSMasanari Iida * buffer is used. Can be called multiple times to append multiple 6503a5a3927SJames Bottomley * buffers. 65186db1e29SJens Axboe */ 65286db1e29SJens Axboe int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, 65386db1e29SJens Axboe unsigned int len, gfp_t gfp_mask) 65486db1e29SJens Axboe { 65568154e90SFUJITA Tomonori int reading = rq_data_dir(rq) == READ; 65614417799SNamhyung Kim unsigned long addr = (unsigned long) kbuf; 657393bb12eSChristoph Hellwig struct bio *bio; 6583a5a3927SJames Bottomley int ret; 65986db1e29SJens Axboe 660ae03bf63SMartin K. Petersen if (len > (queue_max_hw_sectors(q) << 9)) 66186db1e29SJens Axboe return -EINVAL; 66286db1e29SJens Axboe if (!len || !kbuf) 66386db1e29SJens Axboe return -EINVAL; 66486db1e29SJens Axboe 665393bb12eSChristoph Hellwig if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || 666393bb12eSChristoph Hellwig blk_queue_may_bounce(q)) 66768154e90SFUJITA Tomonori bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); 66868154e90SFUJITA Tomonori else 66986db1e29SJens Axboe bio = bio_map_kern(q, kbuf, len, gfp_mask); 67068154e90SFUJITA Tomonori 67186db1e29SJens Axboe if (IS_ERR(bio)) 67286db1e29SJens Axboe return PTR_ERR(bio); 67386db1e29SJens Axboe 674aebf526bSChristoph Hellwig bio->bi_opf &= ~REQ_OP_MASK; 675aebf526bSChristoph Hellwig bio->bi_opf |= req_op(rq); 67686db1e29SJens Axboe 677393bb12eSChristoph Hellwig ret = blk_rq_append_bio(rq, bio); 678066ff571SChristoph Hellwig if (unlikely(ret)) { 679066ff571SChristoph Hellwig bio_uninit(bio); 680066ff571SChristoph Hellwig kfree(bio); 681066ff571SChristoph Hellwig } 6823a5a3927SJames Bottomley return ret; 6833a5a3927SJames Bottomley } 68486db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_map_kern); 685