1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 286db1e29SJens Axboe /* 386db1e29SJens Axboe * Functions related to mapping data to requests 486db1e29SJens Axboe */ 586db1e29SJens Axboe #include <linux/kernel.h> 668db0cf1SIngo Molnar #include <linux/sched/task_stack.h> 786db1e29SJens Axboe #include <linux/module.h> 886db1e29SJens Axboe #include <linux/bio.h> 986db1e29SJens Axboe #include <linux/blkdev.h> 1026e49cfcSKent Overstreet #include <linux/uio.h> 1186db1e29SJens Axboe 1286db1e29SJens Axboe #include "blk.h" 1386db1e29SJens Axboe 14130879f1SChristoph Hellwig struct bio_map_data { 15130879f1SChristoph Hellwig int is_our_pages; 16130879f1SChristoph Hellwig struct iov_iter iter; 17130879f1SChristoph Hellwig struct iovec iov[]; 18130879f1SChristoph Hellwig }; 19130879f1SChristoph Hellwig 20130879f1SChristoph Hellwig static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, 21130879f1SChristoph Hellwig gfp_t gfp_mask) 22130879f1SChristoph Hellwig { 23130879f1SChristoph Hellwig struct bio_map_data *bmd; 24130879f1SChristoph Hellwig 25130879f1SChristoph Hellwig if (data->nr_segs > UIO_MAXIOV) 26130879f1SChristoph Hellwig return NULL; 27130879f1SChristoph Hellwig 28130879f1SChristoph Hellwig bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); 29130879f1SChristoph Hellwig if (!bmd) 30130879f1SChristoph Hellwig return NULL; 31130879f1SChristoph Hellwig memcpy(bmd->iov, data->iov, sizeof(struct iovec) * data->nr_segs); 32130879f1SChristoph Hellwig bmd->iter = *data; 33130879f1SChristoph Hellwig bmd->iter.iov = bmd->iov; 34130879f1SChristoph Hellwig return bmd; 35130879f1SChristoph Hellwig } 36130879f1SChristoph Hellwig 37130879f1SChristoph Hellwig /** 38130879f1SChristoph Hellwig * bio_copy_from_iter - copy all pages from iov_iter to bio 39130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as destination 40130879f1SChristoph Hellwig * @iter: iov_iter as source 41130879f1SChristoph Hellwig * 42130879f1SChristoph Hellwig * Copy all pages from iov_iter to bio. 43130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 44130879f1SChristoph Hellwig */ 45130879f1SChristoph Hellwig static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) 46130879f1SChristoph Hellwig { 47130879f1SChristoph Hellwig struct bio_vec *bvec; 48130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 49130879f1SChristoph Hellwig 50130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 51130879f1SChristoph Hellwig ssize_t ret; 52130879f1SChristoph Hellwig 53130879f1SChristoph Hellwig ret = copy_page_from_iter(bvec->bv_page, 54130879f1SChristoph Hellwig bvec->bv_offset, 55130879f1SChristoph Hellwig bvec->bv_len, 56130879f1SChristoph Hellwig iter); 57130879f1SChristoph Hellwig 58130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 59130879f1SChristoph Hellwig break; 60130879f1SChristoph Hellwig 61130879f1SChristoph Hellwig if (ret < bvec->bv_len) 62130879f1SChristoph Hellwig return -EFAULT; 63130879f1SChristoph Hellwig } 64130879f1SChristoph Hellwig 65130879f1SChristoph Hellwig return 0; 66130879f1SChristoph Hellwig } 67130879f1SChristoph Hellwig 68130879f1SChristoph Hellwig /** 69130879f1SChristoph Hellwig * bio_copy_to_iter - copy all pages from bio to iov_iter 70130879f1SChristoph Hellwig * @bio: The &struct bio which describes the I/O as source 71130879f1SChristoph Hellwig * @iter: iov_iter as destination 72130879f1SChristoph Hellwig * 73130879f1SChristoph Hellwig * Copy all pages from bio to iov_iter. 74130879f1SChristoph Hellwig * Returns 0 on success, or error on failure. 75130879f1SChristoph Hellwig */ 76130879f1SChristoph Hellwig static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) 77130879f1SChristoph Hellwig { 78130879f1SChristoph Hellwig struct bio_vec *bvec; 79130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 80130879f1SChristoph Hellwig 81130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 82130879f1SChristoph Hellwig ssize_t ret; 83130879f1SChristoph Hellwig 84130879f1SChristoph Hellwig ret = copy_page_to_iter(bvec->bv_page, 85130879f1SChristoph Hellwig bvec->bv_offset, 86130879f1SChristoph Hellwig bvec->bv_len, 87130879f1SChristoph Hellwig &iter); 88130879f1SChristoph Hellwig 89130879f1SChristoph Hellwig if (!iov_iter_count(&iter)) 90130879f1SChristoph Hellwig break; 91130879f1SChristoph Hellwig 92130879f1SChristoph Hellwig if (ret < bvec->bv_len) 93130879f1SChristoph Hellwig return -EFAULT; 94130879f1SChristoph Hellwig } 95130879f1SChristoph Hellwig 96130879f1SChristoph Hellwig return 0; 97130879f1SChristoph Hellwig } 98130879f1SChristoph Hellwig 99130879f1SChristoph Hellwig /** 100130879f1SChristoph Hellwig * bio_uncopy_user - finish previously mapped bio 101130879f1SChristoph Hellwig * @bio: bio being terminated 102130879f1SChristoph Hellwig * 103130879f1SChristoph Hellwig * Free pages allocated from bio_copy_user_iov() and write back data 104130879f1SChristoph Hellwig * to user space in case of a read. 105130879f1SChristoph Hellwig */ 106130879f1SChristoph Hellwig static int bio_uncopy_user(struct bio *bio) 107130879f1SChristoph Hellwig { 108130879f1SChristoph Hellwig struct bio_map_data *bmd = bio->bi_private; 109130879f1SChristoph Hellwig int ret = 0; 110130879f1SChristoph Hellwig 111130879f1SChristoph Hellwig if (!bio_flagged(bio, BIO_NULL_MAPPED)) { 112130879f1SChristoph Hellwig /* 113130879f1SChristoph Hellwig * if we're in a workqueue, the request is orphaned, so 114130879f1SChristoph Hellwig * don't copy into a random user address space, just free 115130879f1SChristoph Hellwig * and return -EINTR so user space doesn't expect any data. 116130879f1SChristoph Hellwig */ 117130879f1SChristoph Hellwig if (!current->mm) 118130879f1SChristoph Hellwig ret = -EINTR; 119130879f1SChristoph Hellwig else if (bio_data_dir(bio) == READ) 120130879f1SChristoph Hellwig ret = bio_copy_to_iter(bio, bmd->iter); 121130879f1SChristoph Hellwig if (bmd->is_our_pages) 122130879f1SChristoph Hellwig bio_free_pages(bio); 123130879f1SChristoph Hellwig } 124130879f1SChristoph Hellwig kfree(bmd); 125130879f1SChristoph Hellwig bio_put(bio); 126130879f1SChristoph Hellwig return ret; 127130879f1SChristoph Hellwig } 128130879f1SChristoph Hellwig 129130879f1SChristoph Hellwig /** 130130879f1SChristoph Hellwig * bio_copy_user_iov - copy user data to bio 131130879f1SChristoph Hellwig * @q: destination block queue 132130879f1SChristoph Hellwig * @map_data: pointer to the rq_map_data holding pages (if necessary) 133130879f1SChristoph Hellwig * @iter: iovec iterator 134130879f1SChristoph Hellwig * @gfp_mask: memory allocation flags 135130879f1SChristoph Hellwig * 136130879f1SChristoph Hellwig * Prepares and returns a bio for indirect user io, bouncing data 137130879f1SChristoph Hellwig * to/from kernel pages as necessary. Must be paired with 138130879f1SChristoph Hellwig * call bio_uncopy_user() on io completion. 139130879f1SChristoph Hellwig */ 140130879f1SChristoph Hellwig static struct bio *bio_copy_user_iov(struct request_queue *q, 141130879f1SChristoph Hellwig struct rq_map_data *map_data, struct iov_iter *iter, 142130879f1SChristoph Hellwig gfp_t gfp_mask) 143130879f1SChristoph Hellwig { 144130879f1SChristoph Hellwig struct bio_map_data *bmd; 145130879f1SChristoph Hellwig struct page *page; 146130879f1SChristoph Hellwig struct bio *bio; 147130879f1SChristoph Hellwig int i = 0, ret; 148130879f1SChristoph Hellwig int nr_pages; 149130879f1SChristoph Hellwig unsigned int len = iter->count; 150130879f1SChristoph Hellwig unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; 151130879f1SChristoph Hellwig 152130879f1SChristoph Hellwig bmd = bio_alloc_map_data(iter, gfp_mask); 153130879f1SChristoph Hellwig if (!bmd) 154130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 155130879f1SChristoph Hellwig 156130879f1SChristoph Hellwig /* 157130879f1SChristoph Hellwig * We need to do a deep copy of the iov_iter including the iovecs. 158130879f1SChristoph Hellwig * The caller provided iov might point to an on-stack or otherwise 159130879f1SChristoph Hellwig * shortlived one. 160130879f1SChristoph Hellwig */ 161130879f1SChristoph Hellwig bmd->is_our_pages = map_data ? 0 : 1; 162130879f1SChristoph Hellwig 163130879f1SChristoph Hellwig nr_pages = DIV_ROUND_UP(offset + len, PAGE_SIZE); 164130879f1SChristoph Hellwig if (nr_pages > BIO_MAX_PAGES) 165130879f1SChristoph Hellwig nr_pages = BIO_MAX_PAGES; 166130879f1SChristoph Hellwig 167130879f1SChristoph Hellwig ret = -ENOMEM; 168130879f1SChristoph Hellwig bio = bio_kmalloc(gfp_mask, nr_pages); 169130879f1SChristoph Hellwig if (!bio) 170130879f1SChristoph Hellwig goto out_bmd; 171130879f1SChristoph Hellwig 172130879f1SChristoph Hellwig ret = 0; 173130879f1SChristoph Hellwig 174130879f1SChristoph Hellwig if (map_data) { 175130879f1SChristoph Hellwig nr_pages = 1 << map_data->page_order; 176130879f1SChristoph Hellwig i = map_data->offset / PAGE_SIZE; 177130879f1SChristoph Hellwig } 178130879f1SChristoph Hellwig while (len) { 179130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 180130879f1SChristoph Hellwig 181130879f1SChristoph Hellwig bytes -= offset; 182130879f1SChristoph Hellwig 183130879f1SChristoph Hellwig if (bytes > len) 184130879f1SChristoph Hellwig bytes = len; 185130879f1SChristoph Hellwig 186130879f1SChristoph Hellwig if (map_data) { 187130879f1SChristoph Hellwig if (i == map_data->nr_entries * nr_pages) { 188130879f1SChristoph Hellwig ret = -ENOMEM; 189130879f1SChristoph Hellwig break; 190130879f1SChristoph Hellwig } 191130879f1SChristoph Hellwig 192130879f1SChristoph Hellwig page = map_data->pages[i / nr_pages]; 193130879f1SChristoph Hellwig page += (i % nr_pages); 194130879f1SChristoph Hellwig 195130879f1SChristoph Hellwig i++; 196130879f1SChristoph Hellwig } else { 197130879f1SChristoph Hellwig page = alloc_page(q->bounce_gfp | gfp_mask); 198130879f1SChristoph Hellwig if (!page) { 199130879f1SChristoph Hellwig ret = -ENOMEM; 200130879f1SChristoph Hellwig break; 201130879f1SChristoph Hellwig } 202130879f1SChristoph Hellwig } 203130879f1SChristoph Hellwig 204130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, offset) < bytes) { 205130879f1SChristoph Hellwig if (!map_data) 206130879f1SChristoph Hellwig __free_page(page); 207130879f1SChristoph Hellwig break; 208130879f1SChristoph Hellwig } 209130879f1SChristoph Hellwig 210130879f1SChristoph Hellwig len -= bytes; 211130879f1SChristoph Hellwig offset = 0; 212130879f1SChristoph Hellwig } 213130879f1SChristoph Hellwig 214130879f1SChristoph Hellwig if (ret) 215130879f1SChristoph Hellwig goto cleanup; 216130879f1SChristoph Hellwig 217130879f1SChristoph Hellwig if (map_data) 218130879f1SChristoph Hellwig map_data->offset += bio->bi_iter.bi_size; 219130879f1SChristoph Hellwig 220130879f1SChristoph Hellwig /* 221130879f1SChristoph Hellwig * success 222130879f1SChristoph Hellwig */ 223130879f1SChristoph Hellwig if ((iov_iter_rw(iter) == WRITE && 224130879f1SChristoph Hellwig (!map_data || !map_data->null_mapped)) || 225130879f1SChristoph Hellwig (map_data && map_data->from_user)) { 226130879f1SChristoph Hellwig ret = bio_copy_from_iter(bio, iter); 227130879f1SChristoph Hellwig if (ret) 228130879f1SChristoph Hellwig goto cleanup; 229130879f1SChristoph Hellwig } else { 230130879f1SChristoph Hellwig if (bmd->is_our_pages) 231130879f1SChristoph Hellwig zero_fill_bio(bio); 232130879f1SChristoph Hellwig iov_iter_advance(iter, bio->bi_iter.bi_size); 233130879f1SChristoph Hellwig } 234130879f1SChristoph Hellwig 235130879f1SChristoph Hellwig bio->bi_private = bmd; 236130879f1SChristoph Hellwig if (map_data && map_data->null_mapped) 237130879f1SChristoph Hellwig bio_set_flag(bio, BIO_NULL_MAPPED); 238130879f1SChristoph Hellwig return bio; 239130879f1SChristoph Hellwig cleanup: 240130879f1SChristoph Hellwig if (!map_data) 241130879f1SChristoph Hellwig bio_free_pages(bio); 242130879f1SChristoph Hellwig bio_put(bio); 243130879f1SChristoph Hellwig out_bmd: 244130879f1SChristoph Hellwig kfree(bmd); 245130879f1SChristoph Hellwig return ERR_PTR(ret); 246130879f1SChristoph Hellwig } 247130879f1SChristoph Hellwig 248130879f1SChristoph Hellwig /** 249130879f1SChristoph Hellwig * bio_map_user_iov - map user iovec into bio 250130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 251130879f1SChristoph Hellwig * @iter: iovec iterator 252130879f1SChristoph Hellwig * @gfp_mask: memory allocation flags 253130879f1SChristoph Hellwig * 254130879f1SChristoph Hellwig * Map the user space address into a bio suitable for io to a block 255130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 256130879f1SChristoph Hellwig */ 257130879f1SChristoph Hellwig static struct bio *bio_map_user_iov(struct request_queue *q, 258130879f1SChristoph Hellwig struct iov_iter *iter, gfp_t gfp_mask) 259130879f1SChristoph Hellwig { 260130879f1SChristoph Hellwig int j; 261130879f1SChristoph Hellwig struct bio *bio; 262130879f1SChristoph Hellwig int ret; 263130879f1SChristoph Hellwig 264130879f1SChristoph Hellwig if (!iov_iter_count(iter)) 265130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 266130879f1SChristoph Hellwig 267130879f1SChristoph Hellwig bio = bio_kmalloc(gfp_mask, iov_iter_npages(iter, BIO_MAX_PAGES)); 268130879f1SChristoph Hellwig if (!bio) 269130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 270130879f1SChristoph Hellwig 271130879f1SChristoph Hellwig while (iov_iter_count(iter)) { 272130879f1SChristoph Hellwig struct page **pages; 273130879f1SChristoph Hellwig ssize_t bytes; 274130879f1SChristoph Hellwig size_t offs, added = 0; 275130879f1SChristoph Hellwig int npages; 276130879f1SChristoph Hellwig 277130879f1SChristoph Hellwig bytes = iov_iter_get_pages_alloc(iter, &pages, LONG_MAX, &offs); 278130879f1SChristoph Hellwig if (unlikely(bytes <= 0)) { 279130879f1SChristoph Hellwig ret = bytes ? bytes : -EFAULT; 280130879f1SChristoph Hellwig goto out_unmap; 281130879f1SChristoph Hellwig } 282130879f1SChristoph Hellwig 283130879f1SChristoph Hellwig npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); 284130879f1SChristoph Hellwig 285130879f1SChristoph Hellwig if (unlikely(offs & queue_dma_alignment(q))) { 286130879f1SChristoph Hellwig ret = -EINVAL; 287130879f1SChristoph Hellwig j = 0; 288130879f1SChristoph Hellwig } else { 289130879f1SChristoph Hellwig for (j = 0; j < npages; j++) { 290130879f1SChristoph Hellwig struct page *page = pages[j]; 291130879f1SChristoph Hellwig unsigned int n = PAGE_SIZE - offs; 292130879f1SChristoph Hellwig bool same_page = false; 293130879f1SChristoph Hellwig 294130879f1SChristoph Hellwig if (n > bytes) 295130879f1SChristoph Hellwig n = bytes; 296130879f1SChristoph Hellwig 297130879f1SChristoph Hellwig if (!__bio_add_pc_page(q, bio, page, n, offs, 298130879f1SChristoph Hellwig &same_page)) { 299130879f1SChristoph Hellwig if (same_page) 300130879f1SChristoph Hellwig put_page(page); 301130879f1SChristoph Hellwig break; 302130879f1SChristoph Hellwig } 303130879f1SChristoph Hellwig 304130879f1SChristoph Hellwig added += n; 305130879f1SChristoph Hellwig bytes -= n; 306130879f1SChristoph Hellwig offs = 0; 307130879f1SChristoph Hellwig } 308130879f1SChristoph Hellwig iov_iter_advance(iter, added); 309130879f1SChristoph Hellwig } 310130879f1SChristoph Hellwig /* 311130879f1SChristoph Hellwig * release the pages we didn't map into the bio, if any 312130879f1SChristoph Hellwig */ 313130879f1SChristoph Hellwig while (j < npages) 314130879f1SChristoph Hellwig put_page(pages[j++]); 315130879f1SChristoph Hellwig kvfree(pages); 316130879f1SChristoph Hellwig /* couldn't stuff something into bio? */ 317130879f1SChristoph Hellwig if (bytes) 318130879f1SChristoph Hellwig break; 319130879f1SChristoph Hellwig } 320130879f1SChristoph Hellwig 321130879f1SChristoph Hellwig bio_set_flag(bio, BIO_USER_MAPPED); 322130879f1SChristoph Hellwig 323130879f1SChristoph Hellwig /* 324130879f1SChristoph Hellwig * subtle -- if bio_map_user_iov() ended up bouncing a bio, 325130879f1SChristoph Hellwig * it would normally disappear when its bi_end_io is run. 326130879f1SChristoph Hellwig * however, we need it for the unmap, so grab an extra 327130879f1SChristoph Hellwig * reference to it 328130879f1SChristoph Hellwig */ 329130879f1SChristoph Hellwig bio_get(bio); 330130879f1SChristoph Hellwig return bio; 331130879f1SChristoph Hellwig 332130879f1SChristoph Hellwig out_unmap: 333130879f1SChristoph Hellwig bio_release_pages(bio, false); 334130879f1SChristoph Hellwig bio_put(bio); 335130879f1SChristoph Hellwig return ERR_PTR(ret); 336130879f1SChristoph Hellwig } 337130879f1SChristoph Hellwig 338130879f1SChristoph Hellwig /** 339130879f1SChristoph Hellwig * bio_unmap_user - unmap a bio 340130879f1SChristoph Hellwig * @bio: the bio being unmapped 341130879f1SChristoph Hellwig * 342130879f1SChristoph Hellwig * Unmap a bio previously mapped by bio_map_user_iov(). Must be called from 343130879f1SChristoph Hellwig * process context. 344130879f1SChristoph Hellwig * 345130879f1SChristoph Hellwig * bio_unmap_user() may sleep. 346130879f1SChristoph Hellwig */ 347130879f1SChristoph Hellwig static void bio_unmap_user(struct bio *bio) 348130879f1SChristoph Hellwig { 349130879f1SChristoph Hellwig bio_release_pages(bio, bio_data_dir(bio) == READ); 350130879f1SChristoph Hellwig bio_put(bio); 351130879f1SChristoph Hellwig bio_put(bio); 352130879f1SChristoph Hellwig } 353130879f1SChristoph Hellwig 354130879f1SChristoph Hellwig static void bio_invalidate_vmalloc_pages(struct bio *bio) 355130879f1SChristoph Hellwig { 356130879f1SChristoph Hellwig #ifdef ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE 357130879f1SChristoph Hellwig if (bio->bi_private && !op_is_write(bio_op(bio))) { 358130879f1SChristoph Hellwig unsigned long i, len = 0; 359130879f1SChristoph Hellwig 360130879f1SChristoph Hellwig for (i = 0; i < bio->bi_vcnt; i++) 361130879f1SChristoph Hellwig len += bio->bi_io_vec[i].bv_len; 362130879f1SChristoph Hellwig invalidate_kernel_vmap_range(bio->bi_private, len); 363130879f1SChristoph Hellwig } 364130879f1SChristoph Hellwig #endif 365130879f1SChristoph Hellwig } 366130879f1SChristoph Hellwig 367130879f1SChristoph Hellwig static void bio_map_kern_endio(struct bio *bio) 368130879f1SChristoph Hellwig { 369130879f1SChristoph Hellwig bio_invalidate_vmalloc_pages(bio); 370130879f1SChristoph Hellwig bio_put(bio); 371130879f1SChristoph Hellwig } 372130879f1SChristoph Hellwig 373130879f1SChristoph Hellwig /** 374130879f1SChristoph Hellwig * bio_map_kern - map kernel address into bio 375130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 376130879f1SChristoph Hellwig * @data: pointer to buffer to map 377130879f1SChristoph Hellwig * @len: length in bytes 378130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio allocation 379130879f1SChristoph Hellwig * 380130879f1SChristoph Hellwig * Map the kernel address into a bio suitable for io to a block 381130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 382130879f1SChristoph Hellwig */ 383130879f1SChristoph Hellwig static struct bio *bio_map_kern(struct request_queue *q, void *data, 384130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask) 385130879f1SChristoph Hellwig { 386130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 387130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 388130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 389130879f1SChristoph Hellwig const int nr_pages = end - start; 390130879f1SChristoph Hellwig bool is_vmalloc = is_vmalloc_addr(data); 391130879f1SChristoph Hellwig struct page *page; 392130879f1SChristoph Hellwig int offset, i; 393130879f1SChristoph Hellwig struct bio *bio; 394130879f1SChristoph Hellwig 395130879f1SChristoph Hellwig bio = bio_kmalloc(gfp_mask, nr_pages); 396130879f1SChristoph Hellwig if (!bio) 397130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 398130879f1SChristoph Hellwig 399130879f1SChristoph Hellwig if (is_vmalloc) { 400130879f1SChristoph Hellwig flush_kernel_vmap_range(data, len); 401130879f1SChristoph Hellwig bio->bi_private = data; 402130879f1SChristoph Hellwig } 403130879f1SChristoph Hellwig 404130879f1SChristoph Hellwig offset = offset_in_page(kaddr); 405130879f1SChristoph Hellwig for (i = 0; i < nr_pages; i++) { 406130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE - offset; 407130879f1SChristoph Hellwig 408130879f1SChristoph Hellwig if (len <= 0) 409130879f1SChristoph Hellwig break; 410130879f1SChristoph Hellwig 411130879f1SChristoph Hellwig if (bytes > len) 412130879f1SChristoph Hellwig bytes = len; 413130879f1SChristoph Hellwig 414130879f1SChristoph Hellwig if (!is_vmalloc) 415130879f1SChristoph Hellwig page = virt_to_page(data); 416130879f1SChristoph Hellwig else 417130879f1SChristoph Hellwig page = vmalloc_to_page(data); 418130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 419130879f1SChristoph Hellwig offset) < bytes) { 420130879f1SChristoph Hellwig /* we don't support partial mappings */ 421130879f1SChristoph Hellwig bio_put(bio); 422130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 423130879f1SChristoph Hellwig } 424130879f1SChristoph Hellwig 425130879f1SChristoph Hellwig data += bytes; 426130879f1SChristoph Hellwig len -= bytes; 427130879f1SChristoph Hellwig offset = 0; 428130879f1SChristoph Hellwig } 429130879f1SChristoph Hellwig 430130879f1SChristoph Hellwig bio->bi_end_io = bio_map_kern_endio; 431130879f1SChristoph Hellwig return bio; 432130879f1SChristoph Hellwig } 433130879f1SChristoph Hellwig 434130879f1SChristoph Hellwig static void bio_copy_kern_endio(struct bio *bio) 435130879f1SChristoph Hellwig { 436130879f1SChristoph Hellwig bio_free_pages(bio); 437130879f1SChristoph Hellwig bio_put(bio); 438130879f1SChristoph Hellwig } 439130879f1SChristoph Hellwig 440130879f1SChristoph Hellwig static void bio_copy_kern_endio_read(struct bio *bio) 441130879f1SChristoph Hellwig { 442130879f1SChristoph Hellwig char *p = bio->bi_private; 443130879f1SChristoph Hellwig struct bio_vec *bvec; 444130879f1SChristoph Hellwig struct bvec_iter_all iter_all; 445130879f1SChristoph Hellwig 446130879f1SChristoph Hellwig bio_for_each_segment_all(bvec, bio, iter_all) { 447130879f1SChristoph Hellwig memcpy(p, page_address(bvec->bv_page), bvec->bv_len); 448130879f1SChristoph Hellwig p += bvec->bv_len; 449130879f1SChristoph Hellwig } 450130879f1SChristoph Hellwig 451130879f1SChristoph Hellwig bio_copy_kern_endio(bio); 452130879f1SChristoph Hellwig } 453130879f1SChristoph Hellwig 454130879f1SChristoph Hellwig /** 455130879f1SChristoph Hellwig * bio_copy_kern - copy kernel address into bio 456130879f1SChristoph Hellwig * @q: the struct request_queue for the bio 457130879f1SChristoph Hellwig * @data: pointer to buffer to copy 458130879f1SChristoph Hellwig * @len: length in bytes 459130879f1SChristoph Hellwig * @gfp_mask: allocation flags for bio and page allocation 460130879f1SChristoph Hellwig * @reading: data direction is READ 461130879f1SChristoph Hellwig * 462130879f1SChristoph Hellwig * copy the kernel address into a bio suitable for io to a block 463130879f1SChristoph Hellwig * device. Returns an error pointer in case of error. 464130879f1SChristoph Hellwig */ 465130879f1SChristoph Hellwig static struct bio *bio_copy_kern(struct request_queue *q, void *data, 466130879f1SChristoph Hellwig unsigned int len, gfp_t gfp_mask, int reading) 467130879f1SChristoph Hellwig { 468130879f1SChristoph Hellwig unsigned long kaddr = (unsigned long)data; 469130879f1SChristoph Hellwig unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 470130879f1SChristoph Hellwig unsigned long start = kaddr >> PAGE_SHIFT; 471130879f1SChristoph Hellwig struct bio *bio; 472130879f1SChristoph Hellwig void *p = data; 473130879f1SChristoph Hellwig int nr_pages = 0; 474130879f1SChristoph Hellwig 475130879f1SChristoph Hellwig /* 476130879f1SChristoph Hellwig * Overflow, abort 477130879f1SChristoph Hellwig */ 478130879f1SChristoph Hellwig if (end < start) 479130879f1SChristoph Hellwig return ERR_PTR(-EINVAL); 480130879f1SChristoph Hellwig 481130879f1SChristoph Hellwig nr_pages = end - start; 482130879f1SChristoph Hellwig bio = bio_kmalloc(gfp_mask, nr_pages); 483130879f1SChristoph Hellwig if (!bio) 484130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 485130879f1SChristoph Hellwig 486130879f1SChristoph Hellwig while (len) { 487130879f1SChristoph Hellwig struct page *page; 488130879f1SChristoph Hellwig unsigned int bytes = PAGE_SIZE; 489130879f1SChristoph Hellwig 490130879f1SChristoph Hellwig if (bytes > len) 491130879f1SChristoph Hellwig bytes = len; 492130879f1SChristoph Hellwig 493130879f1SChristoph Hellwig page = alloc_page(q->bounce_gfp | gfp_mask); 494130879f1SChristoph Hellwig if (!page) 495130879f1SChristoph Hellwig goto cleanup; 496130879f1SChristoph Hellwig 497130879f1SChristoph Hellwig if (!reading) 498130879f1SChristoph Hellwig memcpy(page_address(page), p, bytes); 499130879f1SChristoph Hellwig 500130879f1SChristoph Hellwig if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) 501130879f1SChristoph Hellwig break; 502130879f1SChristoph Hellwig 503130879f1SChristoph Hellwig len -= bytes; 504130879f1SChristoph Hellwig p += bytes; 505130879f1SChristoph Hellwig } 506130879f1SChristoph Hellwig 507130879f1SChristoph Hellwig if (reading) { 508130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio_read; 509130879f1SChristoph Hellwig bio->bi_private = data; 510130879f1SChristoph Hellwig } else { 511130879f1SChristoph Hellwig bio->bi_end_io = bio_copy_kern_endio; 512130879f1SChristoph Hellwig } 513130879f1SChristoph Hellwig 514130879f1SChristoph Hellwig return bio; 515130879f1SChristoph Hellwig 516130879f1SChristoph Hellwig cleanup: 517130879f1SChristoph Hellwig bio_free_pages(bio); 518130879f1SChristoph Hellwig bio_put(bio); 519130879f1SChristoph Hellwig return ERR_PTR(-ENOMEM); 520130879f1SChristoph Hellwig } 521130879f1SChristoph Hellwig 52298d61d5bSChristoph Hellwig /* 5230abc2a10SJens Axboe * Append a bio to a passthrough request. Only works if the bio can be merged 5240abc2a10SJens Axboe * into the request based on the driver constraints. 52598d61d5bSChristoph Hellwig */ 5260abc2a10SJens Axboe int blk_rq_append_bio(struct request *rq, struct bio **bio) 52786db1e29SJens Axboe { 5280abc2a10SJens Axboe struct bio *orig_bio = *bio; 52914ccb66bSChristoph Hellwig struct bvec_iter iter; 53014ccb66bSChristoph Hellwig struct bio_vec bv; 53114ccb66bSChristoph Hellwig unsigned int nr_segs = 0; 5320abc2a10SJens Axboe 5330abc2a10SJens Axboe blk_queue_bounce(rq->q, bio); 534caa4b024SChristoph Hellwig 53514ccb66bSChristoph Hellwig bio_for_each_bvec(bv, *bio, iter) 53614ccb66bSChristoph Hellwig nr_segs++; 53714ccb66bSChristoph Hellwig 53898d61d5bSChristoph Hellwig if (!rq->bio) { 53914ccb66bSChristoph Hellwig blk_rq_bio_prep(rq, *bio, nr_segs); 54098d61d5bSChristoph Hellwig } else { 54114ccb66bSChristoph Hellwig if (!ll_back_merge_fn(rq, *bio, nr_segs)) { 5420abc2a10SJens Axboe if (orig_bio != *bio) { 5430abc2a10SJens Axboe bio_put(*bio); 5440abc2a10SJens Axboe *bio = orig_bio; 5450abc2a10SJens Axboe } 54686db1e29SJens Axboe return -EINVAL; 5470abc2a10SJens Axboe } 54898d61d5bSChristoph Hellwig 5490abc2a10SJens Axboe rq->biotail->bi_next = *bio; 5500abc2a10SJens Axboe rq->biotail = *bio; 5510abc2a10SJens Axboe rq->__data_len += (*bio)->bi_iter.bi_size; 55286db1e29SJens Axboe } 55398d61d5bSChristoph Hellwig 55486db1e29SJens Axboe return 0; 55586db1e29SJens Axboe } 55698d61d5bSChristoph Hellwig EXPORT_SYMBOL(blk_rq_append_bio); 55786db1e29SJens Axboe 55886db1e29SJens Axboe static int __blk_rq_unmap_user(struct bio *bio) 55986db1e29SJens Axboe { 56086db1e29SJens Axboe int ret = 0; 56186db1e29SJens Axboe 56286db1e29SJens Axboe if (bio) { 56386db1e29SJens Axboe if (bio_flagged(bio, BIO_USER_MAPPED)) 56486db1e29SJens Axboe bio_unmap_user(bio); 56586db1e29SJens Axboe else 56686db1e29SJens Axboe ret = bio_uncopy_user(bio); 56786db1e29SJens Axboe } 56886db1e29SJens Axboe 56986db1e29SJens Axboe return ret; 57086db1e29SJens Axboe } 57186db1e29SJens Axboe 5724d6af73dSChristoph Hellwig static int __blk_rq_map_user_iov(struct request *rq, 5734d6af73dSChristoph Hellwig struct rq_map_data *map_data, struct iov_iter *iter, 5744d6af73dSChristoph Hellwig gfp_t gfp_mask, bool copy) 5754d6af73dSChristoph Hellwig { 5764d6af73dSChristoph Hellwig struct request_queue *q = rq->q; 5774d6af73dSChristoph Hellwig struct bio *bio, *orig_bio; 5784d6af73dSChristoph Hellwig int ret; 5794d6af73dSChristoph Hellwig 5804d6af73dSChristoph Hellwig if (copy) 5814d6af73dSChristoph Hellwig bio = bio_copy_user_iov(q, map_data, iter, gfp_mask); 5824d6af73dSChristoph Hellwig else 5834d6af73dSChristoph Hellwig bio = bio_map_user_iov(q, iter, gfp_mask); 5844d6af73dSChristoph Hellwig 5854d6af73dSChristoph Hellwig if (IS_ERR(bio)) 5864d6af73dSChristoph Hellwig return PTR_ERR(bio); 5874d6af73dSChristoph Hellwig 588aebf526bSChristoph Hellwig bio->bi_opf &= ~REQ_OP_MASK; 589aebf526bSChristoph Hellwig bio->bi_opf |= req_op(rq); 590aebf526bSChristoph Hellwig 5914d6af73dSChristoph Hellwig orig_bio = bio; 5924d6af73dSChristoph Hellwig 5934d6af73dSChristoph Hellwig /* 5944d6af73dSChristoph Hellwig * We link the bounce buffer in and could have to traverse it 5954d6af73dSChristoph Hellwig * later so we have to get a ref to prevent it from being freed 5964d6af73dSChristoph Hellwig */ 5970abc2a10SJens Axboe ret = blk_rq_append_bio(rq, &bio); 5984d6af73dSChristoph Hellwig if (ret) { 5994d6af73dSChristoph Hellwig __blk_rq_unmap_user(orig_bio); 6004d6af73dSChristoph Hellwig return ret; 6014d6af73dSChristoph Hellwig } 6020abc2a10SJens Axboe bio_get(bio); 6034d6af73dSChristoph Hellwig 6044d6af73dSChristoph Hellwig return 0; 6054d6af73dSChristoph Hellwig } 6064d6af73dSChristoph Hellwig 60786db1e29SJens Axboe /** 608aebf526bSChristoph Hellwig * blk_rq_map_user_iov - map user data to a request, for passthrough requests 60986db1e29SJens Axboe * @q: request queue where request should be inserted 61086db1e29SJens Axboe * @rq: request to map data to 611152e283fSFUJITA Tomonori * @map_data: pointer to the rq_map_data holding pages (if necessary) 61226e49cfcSKent Overstreet * @iter: iovec iterator 613a3bce90eSFUJITA Tomonori * @gfp_mask: memory allocation flags 61486db1e29SJens Axboe * 61586db1e29SJens Axboe * Description: 616710027a4SRandy Dunlap * Data will be mapped directly for zero copy I/O, if possible. Otherwise 61786db1e29SJens Axboe * a kernel bounce buffer is used. 61886db1e29SJens Axboe * 619710027a4SRandy Dunlap * A matching blk_rq_unmap_user() must be issued at the end of I/O, while 62086db1e29SJens Axboe * still in process context. 62186db1e29SJens Axboe * 62286db1e29SJens Axboe * Note: The mapped bio may need to be bounced through blk_queue_bounce() 62386db1e29SJens Axboe * before being submitted to the device, as pages mapped may be out of 62486db1e29SJens Axboe * reach. It's the callers responsibility to make sure this happens. The 62586db1e29SJens Axboe * original bio must be passed back in to blk_rq_unmap_user() for proper 62686db1e29SJens Axboe * unmapping. 62786db1e29SJens Axboe */ 62886db1e29SJens Axboe int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, 62926e49cfcSKent Overstreet struct rq_map_data *map_data, 63026e49cfcSKent Overstreet const struct iov_iter *iter, gfp_t gfp_mask) 63186db1e29SJens Axboe { 632357f435dSAl Viro bool copy = false; 633357f435dSAl Viro unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); 6344d6af73dSChristoph Hellwig struct bio *bio = NULL; 6354d6af73dSChristoph Hellwig struct iov_iter i; 63669e0927bSDouglas Gilbert int ret = -EINVAL; 63786db1e29SJens Axboe 638a0ac402cSLinus Torvalds if (!iter_is_iovec(iter)) 639a0ac402cSLinus Torvalds goto fail; 640a0ac402cSLinus Torvalds 641357f435dSAl Viro if (map_data) 6424d6af73dSChristoph Hellwig copy = true; 643357f435dSAl Viro else if (iov_iter_alignment(iter) & align) 644357f435dSAl Viro copy = true; 645357f435dSAl Viro else if (queue_virt_boundary(q)) 646357f435dSAl Viro copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); 647afdc1a78SFUJITA Tomonori 6484d6af73dSChristoph Hellwig i = *iter; 6494d6af73dSChristoph Hellwig do { 6504d6af73dSChristoph Hellwig ret =__blk_rq_map_user_iov(rq, map_data, &i, gfp_mask, copy); 6514d6af73dSChristoph Hellwig if (ret) 6524d6af73dSChristoph Hellwig goto unmap_rq; 6534d6af73dSChristoph Hellwig if (!bio) 6544d6af73dSChristoph Hellwig bio = rq->bio; 6554d6af73dSChristoph Hellwig } while (iov_iter_count(&i)); 65686db1e29SJens Axboe 657f18573abSFUJITA Tomonori if (!bio_flagged(bio, BIO_USER_MAPPED)) 658e8064021SChristoph Hellwig rq->rq_flags |= RQF_COPY_USER; 65986db1e29SJens Axboe return 0; 6604d6af73dSChristoph Hellwig 6614d6af73dSChristoph Hellwig unmap_rq: 6623b7995a9SYang Yingliang blk_rq_unmap_user(bio); 663a0ac402cSLinus Torvalds fail: 6644d6af73dSChristoph Hellwig rq->bio = NULL; 66569e0927bSDouglas Gilbert return ret; 66686db1e29SJens Axboe } 667152e283fSFUJITA Tomonori EXPORT_SYMBOL(blk_rq_map_user_iov); 66886db1e29SJens Axboe 669ddad8dd0SChristoph Hellwig int blk_rq_map_user(struct request_queue *q, struct request *rq, 670ddad8dd0SChristoph Hellwig struct rq_map_data *map_data, void __user *ubuf, 671ddad8dd0SChristoph Hellwig unsigned long len, gfp_t gfp_mask) 672ddad8dd0SChristoph Hellwig { 67326e49cfcSKent Overstreet struct iovec iov; 67426e49cfcSKent Overstreet struct iov_iter i; 6758f7e885aSAl Viro int ret = import_single_range(rq_data_dir(rq), ubuf, len, &iov, &i); 676ddad8dd0SChristoph Hellwig 6778f7e885aSAl Viro if (unlikely(ret < 0)) 6788f7e885aSAl Viro return ret; 679ddad8dd0SChristoph Hellwig 68026e49cfcSKent Overstreet return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); 681ddad8dd0SChristoph Hellwig } 682ddad8dd0SChristoph Hellwig EXPORT_SYMBOL(blk_rq_map_user); 683ddad8dd0SChristoph Hellwig 68486db1e29SJens Axboe /** 68586db1e29SJens Axboe * blk_rq_unmap_user - unmap a request with user data 68686db1e29SJens Axboe * @bio: start of bio list 68786db1e29SJens Axboe * 68886db1e29SJens Axboe * Description: 68986db1e29SJens Axboe * Unmap a rq previously mapped by blk_rq_map_user(). The caller must 69086db1e29SJens Axboe * supply the original rq->bio from the blk_rq_map_user() return, since 691710027a4SRandy Dunlap * the I/O completion may have changed rq->bio. 69286db1e29SJens Axboe */ 69386db1e29SJens Axboe int blk_rq_unmap_user(struct bio *bio) 69486db1e29SJens Axboe { 69586db1e29SJens Axboe struct bio *mapped_bio; 69686db1e29SJens Axboe int ret = 0, ret2; 69786db1e29SJens Axboe 69886db1e29SJens Axboe while (bio) { 69986db1e29SJens Axboe mapped_bio = bio; 70086db1e29SJens Axboe if (unlikely(bio_flagged(bio, BIO_BOUNCED))) 70186db1e29SJens Axboe mapped_bio = bio->bi_private; 70286db1e29SJens Axboe 70386db1e29SJens Axboe ret2 = __blk_rq_unmap_user(mapped_bio); 70486db1e29SJens Axboe if (ret2 && !ret) 70586db1e29SJens Axboe ret = ret2; 70686db1e29SJens Axboe 70786db1e29SJens Axboe mapped_bio = bio; 70886db1e29SJens Axboe bio = bio->bi_next; 70986db1e29SJens Axboe bio_put(mapped_bio); 71086db1e29SJens Axboe } 71186db1e29SJens Axboe 71286db1e29SJens Axboe return ret; 71386db1e29SJens Axboe } 71486db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_unmap_user); 71586db1e29SJens Axboe 71686db1e29SJens Axboe /** 717aebf526bSChristoph Hellwig * blk_rq_map_kern - map kernel data to a request, for passthrough requests 71886db1e29SJens Axboe * @q: request queue where request should be inserted 71986db1e29SJens Axboe * @rq: request to fill 72086db1e29SJens Axboe * @kbuf: the kernel buffer 72186db1e29SJens Axboe * @len: length of user data 72286db1e29SJens Axboe * @gfp_mask: memory allocation flags 72368154e90SFUJITA Tomonori * 72468154e90SFUJITA Tomonori * Description: 72568154e90SFUJITA Tomonori * Data will be mapped directly if possible. Otherwise a bounce 726e227867fSMasanari Iida * buffer is used. Can be called multiple times to append multiple 7273a5a3927SJames Bottomley * buffers. 72886db1e29SJens Axboe */ 72986db1e29SJens Axboe int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, 73086db1e29SJens Axboe unsigned int len, gfp_t gfp_mask) 73186db1e29SJens Axboe { 73268154e90SFUJITA Tomonori int reading = rq_data_dir(rq) == READ; 73314417799SNamhyung Kim unsigned long addr = (unsigned long) kbuf; 73468154e90SFUJITA Tomonori int do_copy = 0; 7350abc2a10SJens Axboe struct bio *bio, *orig_bio; 7363a5a3927SJames Bottomley int ret; 73786db1e29SJens Axboe 738ae03bf63SMartin K. Petersen if (len > (queue_max_hw_sectors(q) << 9)) 73986db1e29SJens Axboe return -EINVAL; 74086db1e29SJens Axboe if (!len || !kbuf) 74186db1e29SJens Axboe return -EINVAL; 74286db1e29SJens Axboe 74314417799SNamhyung Kim do_copy = !blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf); 74468154e90SFUJITA Tomonori if (do_copy) 74568154e90SFUJITA Tomonori bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); 74668154e90SFUJITA Tomonori else 74786db1e29SJens Axboe bio = bio_map_kern(q, kbuf, len, gfp_mask); 74868154e90SFUJITA Tomonori 74986db1e29SJens Axboe if (IS_ERR(bio)) 75086db1e29SJens Axboe return PTR_ERR(bio); 75186db1e29SJens Axboe 752aebf526bSChristoph Hellwig bio->bi_opf &= ~REQ_OP_MASK; 753aebf526bSChristoph Hellwig bio->bi_opf |= req_op(rq); 75486db1e29SJens Axboe 75568154e90SFUJITA Tomonori if (do_copy) 756e8064021SChristoph Hellwig rq->rq_flags |= RQF_COPY_USER; 75768154e90SFUJITA Tomonori 7580abc2a10SJens Axboe orig_bio = bio; 7590abc2a10SJens Axboe ret = blk_rq_append_bio(rq, &bio); 7603a5a3927SJames Bottomley if (unlikely(ret)) { 7613a5a3927SJames Bottomley /* request is too big */ 7620abc2a10SJens Axboe bio_put(orig_bio); 7633a5a3927SJames Bottomley return ret; 7643a5a3927SJames Bottomley } 7653a5a3927SJames Bottomley 76686db1e29SJens Axboe return 0; 76786db1e29SJens Axboe } 76886db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_map_kern); 769