1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to mapping data to requests 4 */ 5 #include <linux/kernel.h> 6 #include <linux/sched/task_stack.h> 7 #include <linux/module.h> 8 #include <linux/bio.h> 9 #include <linux/blkdev.h> 10 #include <linux/uio.h> 11 12 #include "blk.h" 13 14 struct bio_map_data { 15 bool is_our_pages : 1; 16 bool is_null_mapped : 1; 17 struct iov_iter iter; 18 struct iovec iov[]; 19 }; 20 21 static struct bio_map_data *bio_alloc_map_data(struct iov_iter *data, 22 gfp_t gfp_mask) 23 { 24 struct bio_map_data *bmd; 25 26 if (data->nr_segs > UIO_MAXIOV) 27 return NULL; 28 29 bmd = kmalloc(struct_size(bmd, iov, data->nr_segs), gfp_mask); 30 if (!bmd) 31 return NULL; 32 bmd->iter = *data; 33 if (iter_is_iovec(data)) { 34 memcpy(bmd->iov, iter_iov(data), sizeof(struct iovec) * data->nr_segs); 35 bmd->iter.__iov = bmd->iov; 36 } 37 return bmd; 38 } 39 40 /** 41 * bio_copy_from_iter - copy all pages from iov_iter to bio 42 * @bio: The &struct bio which describes the I/O as destination 43 * @iter: iov_iter as source 44 * 45 * Copy all pages from iov_iter to bio. 46 * Returns 0 on success, or error on failure. 47 */ 48 static int bio_copy_from_iter(struct bio *bio, struct iov_iter *iter) 49 { 50 struct bio_vec *bvec; 51 struct bvec_iter_all iter_all; 52 53 bio_for_each_segment_all(bvec, bio, iter_all) { 54 ssize_t ret; 55 56 ret = copy_page_from_iter(bvec->bv_page, 57 bvec->bv_offset, 58 bvec->bv_len, 59 iter); 60 61 if (!iov_iter_count(iter)) 62 break; 63 64 if (ret < bvec->bv_len) 65 return -EFAULT; 66 } 67 68 return 0; 69 } 70 71 /** 72 * bio_copy_to_iter - copy all pages from bio to iov_iter 73 * @bio: The &struct bio which describes the I/O as source 74 * @iter: iov_iter as destination 75 * 76 * Copy all pages from bio to iov_iter. 77 * Returns 0 on success, or error on failure. 78 */ 79 static int bio_copy_to_iter(struct bio *bio, struct iov_iter iter) 80 { 81 struct bio_vec *bvec; 82 struct bvec_iter_all iter_all; 83 84 bio_for_each_segment_all(bvec, bio, iter_all) { 85 ssize_t ret; 86 87 ret = copy_page_to_iter(bvec->bv_page, 88 bvec->bv_offset, 89 bvec->bv_len, 90 &iter); 91 92 if (!iov_iter_count(&iter)) 93 break; 94 95 if (ret < bvec->bv_len) 96 return -EFAULT; 97 } 98 99 return 0; 100 } 101 102 /** 103 * bio_uncopy_user - finish previously mapped bio 104 * @bio: bio being terminated 105 * 106 * Free pages allocated from bio_copy_user_iov() and write back data 107 * to user space in case of a read. 108 */ 109 static int bio_uncopy_user(struct bio *bio) 110 { 111 struct bio_map_data *bmd = bio->bi_private; 112 int ret = 0; 113 114 if (!bmd->is_null_mapped) { 115 /* 116 * if we're in a workqueue, the request is orphaned, so 117 * don't copy into a random user address space, just free 118 * and return -EINTR so user space doesn't expect any data. 119 */ 120 if (!current->mm) 121 ret = -EINTR; 122 else if (bio_data_dir(bio) == READ) 123 ret = bio_copy_to_iter(bio, bmd->iter); 124 if (bmd->is_our_pages) 125 bio_free_pages(bio); 126 } 127 kfree(bmd); 128 return ret; 129 } 130 131 static int bio_copy_user_iov(struct request *rq, struct rq_map_data *map_data, 132 struct iov_iter *iter, gfp_t gfp_mask) 133 { 134 struct bio_map_data *bmd; 135 struct page *page; 136 struct bio *bio; 137 int i = 0, ret; 138 int nr_pages; 139 unsigned int len = iter->count; 140 unsigned int offset = map_data ? offset_in_page(map_data->offset) : 0; 141 142 bmd = bio_alloc_map_data(iter, gfp_mask); 143 if (!bmd) 144 return -ENOMEM; 145 146 /* 147 * We need to do a deep copy of the iov_iter including the iovecs. 148 * The caller provided iov might point to an on-stack or otherwise 149 * shortlived one. 150 */ 151 bmd->is_our_pages = !map_data; 152 bmd->is_null_mapped = (map_data && map_data->null_mapped); 153 154 nr_pages = bio_max_segs(DIV_ROUND_UP(offset + len, PAGE_SIZE)); 155 156 ret = -ENOMEM; 157 bio = bio_kmalloc(nr_pages, gfp_mask); 158 if (!bio) 159 goto out_bmd; 160 bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, req_op(rq)); 161 162 if (map_data) { 163 nr_pages = 1U << map_data->page_order; 164 i = map_data->offset / PAGE_SIZE; 165 } 166 while (len) { 167 unsigned int bytes = PAGE_SIZE; 168 169 bytes -= offset; 170 171 if (bytes > len) 172 bytes = len; 173 174 if (map_data) { 175 if (i == map_data->nr_entries * nr_pages) { 176 ret = -ENOMEM; 177 goto cleanup; 178 } 179 180 page = map_data->pages[i / nr_pages]; 181 page += (i % nr_pages); 182 183 i++; 184 } else { 185 page = alloc_page(GFP_NOIO | gfp_mask); 186 if (!page) { 187 ret = -ENOMEM; 188 goto cleanup; 189 } 190 } 191 192 if (bio_add_pc_page(rq->q, bio, page, bytes, offset) < bytes) { 193 if (!map_data) 194 __free_page(page); 195 break; 196 } 197 198 len -= bytes; 199 offset = 0; 200 } 201 202 if (map_data) 203 map_data->offset += bio->bi_iter.bi_size; 204 205 /* 206 * success 207 */ 208 if ((iov_iter_rw(iter) == WRITE && 209 (!map_data || !map_data->null_mapped)) || 210 (map_data && map_data->from_user)) { 211 ret = bio_copy_from_iter(bio, iter); 212 if (ret) 213 goto cleanup; 214 } else { 215 if (bmd->is_our_pages) 216 zero_fill_bio(bio); 217 iov_iter_advance(iter, bio->bi_iter.bi_size); 218 } 219 220 bio->bi_private = bmd; 221 222 ret = blk_rq_append_bio(rq, bio); 223 if (ret) 224 goto cleanup; 225 return 0; 226 cleanup: 227 if (!map_data) 228 bio_free_pages(bio); 229 bio_uninit(bio); 230 kfree(bio); 231 out_bmd: 232 kfree(bmd); 233 return ret; 234 } 235 236 static void blk_mq_map_bio_put(struct bio *bio) 237 { 238 if (bio->bi_opf & REQ_ALLOC_CACHE) { 239 bio_put(bio); 240 } else { 241 bio_uninit(bio); 242 kfree(bio); 243 } 244 } 245 246 static struct bio *blk_rq_map_bio_alloc(struct request *rq, 247 unsigned int nr_vecs, gfp_t gfp_mask) 248 { 249 struct bio *bio; 250 251 if (rq->cmd_flags & REQ_ALLOC_CACHE && (nr_vecs <= BIO_INLINE_VECS)) { 252 bio = bio_alloc_bioset(NULL, nr_vecs, rq->cmd_flags, gfp_mask, 253 &fs_bio_set); 254 if (!bio) 255 return NULL; 256 } else { 257 bio = bio_kmalloc(nr_vecs, gfp_mask); 258 if (!bio) 259 return NULL; 260 bio_init(bio, NULL, bio->bi_inline_vecs, nr_vecs, req_op(rq)); 261 } 262 return bio; 263 } 264 265 static int bio_map_user_iov(struct request *rq, struct iov_iter *iter, 266 gfp_t gfp_mask) 267 { 268 iov_iter_extraction_t extraction_flags = 0; 269 unsigned int max_sectors = queue_max_hw_sectors(rq->q); 270 unsigned int nr_vecs = iov_iter_npages(iter, BIO_MAX_VECS); 271 struct bio *bio; 272 int ret; 273 int j; 274 275 if (!iov_iter_count(iter)) 276 return -EINVAL; 277 278 bio = blk_rq_map_bio_alloc(rq, nr_vecs, gfp_mask); 279 if (bio == NULL) 280 return -ENOMEM; 281 282 if (blk_queue_pci_p2pdma(rq->q)) 283 extraction_flags |= ITER_ALLOW_P2PDMA; 284 285 while (iov_iter_count(iter)) { 286 struct page **pages, *stack_pages[UIO_FASTIOV]; 287 ssize_t bytes; 288 size_t offs; 289 int npages; 290 291 if (nr_vecs <= ARRAY_SIZE(stack_pages)) { 292 pages = stack_pages; 293 bytes = iov_iter_get_pages(iter, pages, LONG_MAX, 294 nr_vecs, &offs, extraction_flags); 295 } else { 296 bytes = iov_iter_get_pages_alloc(iter, &pages, 297 LONG_MAX, &offs, extraction_flags); 298 } 299 if (unlikely(bytes <= 0)) { 300 ret = bytes ? bytes : -EFAULT; 301 goto out_unmap; 302 } 303 304 npages = DIV_ROUND_UP(offs + bytes, PAGE_SIZE); 305 306 if (unlikely(offs & queue_dma_alignment(rq->q))) 307 j = 0; 308 else { 309 for (j = 0; j < npages; j++) { 310 struct page *page = pages[j]; 311 unsigned int n = PAGE_SIZE - offs; 312 bool same_page = false; 313 314 if (n > bytes) 315 n = bytes; 316 317 if (!bio_add_hw_page(rq->q, bio, page, n, offs, 318 max_sectors, &same_page)) { 319 if (same_page) 320 put_page(page); 321 break; 322 } 323 324 bytes -= n; 325 offs = 0; 326 } 327 } 328 /* 329 * release the pages we didn't map into the bio, if any 330 */ 331 while (j < npages) 332 put_page(pages[j++]); 333 if (pages != stack_pages) 334 kvfree(pages); 335 /* couldn't stuff something into bio? */ 336 if (bytes) { 337 iov_iter_revert(iter, bytes); 338 break; 339 } 340 } 341 342 ret = blk_rq_append_bio(rq, bio); 343 if (ret) 344 goto out_unmap; 345 return 0; 346 347 out_unmap: 348 bio_release_pages(bio, false); 349 blk_mq_map_bio_put(bio); 350 return ret; 351 } 352 353 static void bio_invalidate_vmalloc_pages(struct bio *bio) 354 { 355 #ifdef ARCH_IMPLEMENTS_FLUSH_KERNEL_VMAP_RANGE 356 if (bio->bi_private && !op_is_write(bio_op(bio))) { 357 unsigned long i, len = 0; 358 359 for (i = 0; i < bio->bi_vcnt; i++) 360 len += bio->bi_io_vec[i].bv_len; 361 invalidate_kernel_vmap_range(bio->bi_private, len); 362 } 363 #endif 364 } 365 366 static void bio_map_kern_endio(struct bio *bio) 367 { 368 bio_invalidate_vmalloc_pages(bio); 369 bio_uninit(bio); 370 kfree(bio); 371 } 372 373 /** 374 * bio_map_kern - map kernel address into bio 375 * @q: the struct request_queue for the bio 376 * @data: pointer to buffer to map 377 * @len: length in bytes 378 * @gfp_mask: allocation flags for bio allocation 379 * 380 * Map the kernel address into a bio suitable for io to a block 381 * device. Returns an error pointer in case of error. 382 */ 383 static struct bio *bio_map_kern(struct request_queue *q, void *data, 384 unsigned int len, gfp_t gfp_mask) 385 { 386 unsigned long kaddr = (unsigned long)data; 387 unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 388 unsigned long start = kaddr >> PAGE_SHIFT; 389 const int nr_pages = end - start; 390 bool is_vmalloc = is_vmalloc_addr(data); 391 struct page *page; 392 int offset, i; 393 struct bio *bio; 394 395 bio = bio_kmalloc(nr_pages, gfp_mask); 396 if (!bio) 397 return ERR_PTR(-ENOMEM); 398 bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 399 400 if (is_vmalloc) { 401 flush_kernel_vmap_range(data, len); 402 bio->bi_private = data; 403 } 404 405 offset = offset_in_page(kaddr); 406 for (i = 0; i < nr_pages; i++) { 407 unsigned int bytes = PAGE_SIZE - offset; 408 409 if (len <= 0) 410 break; 411 412 if (bytes > len) 413 bytes = len; 414 415 if (!is_vmalloc) 416 page = virt_to_page(data); 417 else 418 page = vmalloc_to_page(data); 419 if (bio_add_pc_page(q, bio, page, bytes, 420 offset) < bytes) { 421 /* we don't support partial mappings */ 422 bio_uninit(bio); 423 kfree(bio); 424 return ERR_PTR(-EINVAL); 425 } 426 427 data += bytes; 428 len -= bytes; 429 offset = 0; 430 } 431 432 bio->bi_end_io = bio_map_kern_endio; 433 return bio; 434 } 435 436 static void bio_copy_kern_endio(struct bio *bio) 437 { 438 bio_free_pages(bio); 439 bio_uninit(bio); 440 kfree(bio); 441 } 442 443 static void bio_copy_kern_endio_read(struct bio *bio) 444 { 445 char *p = bio->bi_private; 446 struct bio_vec *bvec; 447 struct bvec_iter_all iter_all; 448 449 bio_for_each_segment_all(bvec, bio, iter_all) { 450 memcpy_from_bvec(p, bvec); 451 p += bvec->bv_len; 452 } 453 454 bio_copy_kern_endio(bio); 455 } 456 457 /** 458 * bio_copy_kern - copy kernel address into bio 459 * @q: the struct request_queue for the bio 460 * @data: pointer to buffer to copy 461 * @len: length in bytes 462 * @gfp_mask: allocation flags for bio and page allocation 463 * @reading: data direction is READ 464 * 465 * copy the kernel address into a bio suitable for io to a block 466 * device. Returns an error pointer in case of error. 467 */ 468 static struct bio *bio_copy_kern(struct request_queue *q, void *data, 469 unsigned int len, gfp_t gfp_mask, int reading) 470 { 471 unsigned long kaddr = (unsigned long)data; 472 unsigned long end = (kaddr + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 473 unsigned long start = kaddr >> PAGE_SHIFT; 474 struct bio *bio; 475 void *p = data; 476 int nr_pages = 0; 477 478 /* 479 * Overflow, abort 480 */ 481 if (end < start) 482 return ERR_PTR(-EINVAL); 483 484 nr_pages = end - start; 485 bio = bio_kmalloc(nr_pages, gfp_mask); 486 if (!bio) 487 return ERR_PTR(-ENOMEM); 488 bio_init(bio, NULL, bio->bi_inline_vecs, nr_pages, 0); 489 490 while (len) { 491 struct page *page; 492 unsigned int bytes = PAGE_SIZE; 493 494 if (bytes > len) 495 bytes = len; 496 497 page = alloc_page(GFP_NOIO | __GFP_ZERO | gfp_mask); 498 if (!page) 499 goto cleanup; 500 501 if (!reading) 502 memcpy(page_address(page), p, bytes); 503 504 if (bio_add_pc_page(q, bio, page, bytes, 0) < bytes) 505 break; 506 507 len -= bytes; 508 p += bytes; 509 } 510 511 if (reading) { 512 bio->bi_end_io = bio_copy_kern_endio_read; 513 bio->bi_private = data; 514 } else { 515 bio->bi_end_io = bio_copy_kern_endio; 516 } 517 518 return bio; 519 520 cleanup: 521 bio_free_pages(bio); 522 bio_uninit(bio); 523 kfree(bio); 524 return ERR_PTR(-ENOMEM); 525 } 526 527 /* 528 * Append a bio to a passthrough request. Only works if the bio can be merged 529 * into the request based on the driver constraints. 530 */ 531 int blk_rq_append_bio(struct request *rq, struct bio *bio) 532 { 533 struct bvec_iter iter; 534 struct bio_vec bv; 535 unsigned int nr_segs = 0; 536 537 bio_for_each_bvec(bv, bio, iter) 538 nr_segs++; 539 540 if (!rq->bio) { 541 blk_rq_bio_prep(rq, bio, nr_segs); 542 } else { 543 if (!ll_back_merge_fn(rq, bio, nr_segs)) 544 return -EINVAL; 545 rq->biotail->bi_next = bio; 546 rq->biotail = bio; 547 rq->__data_len += (bio)->bi_iter.bi_size; 548 bio_crypt_free_ctx(bio); 549 } 550 551 return 0; 552 } 553 EXPORT_SYMBOL(blk_rq_append_bio); 554 555 /* Prepare bio for passthrough IO given ITER_BVEC iter */ 556 static int blk_rq_map_user_bvec(struct request *rq, const struct iov_iter *iter) 557 { 558 struct request_queue *q = rq->q; 559 size_t nr_iter = iov_iter_count(iter); 560 size_t nr_segs = iter->nr_segs; 561 struct bio_vec *bvecs, *bvprvp = NULL; 562 const struct queue_limits *lim = &q->limits; 563 unsigned int nsegs = 0, bytes = 0; 564 struct bio *bio; 565 size_t i; 566 567 if (!nr_iter || (nr_iter >> SECTOR_SHIFT) > queue_max_hw_sectors(q)) 568 return -EINVAL; 569 if (nr_segs > queue_max_segments(q)) 570 return -EINVAL; 571 572 /* no iovecs to alloc, as we already have a BVEC iterator */ 573 bio = blk_rq_map_bio_alloc(rq, 0, GFP_KERNEL); 574 if (bio == NULL) 575 return -ENOMEM; 576 577 bio_iov_bvec_set(bio, (struct iov_iter *)iter); 578 blk_rq_bio_prep(rq, bio, nr_segs); 579 580 /* loop to perform a bunch of sanity checks */ 581 bvecs = (struct bio_vec *)iter->bvec; 582 for (i = 0; i < nr_segs; i++) { 583 struct bio_vec *bv = &bvecs[i]; 584 585 /* 586 * If the queue doesn't support SG gaps and adding this 587 * offset would create a gap, fallback to copy. 588 */ 589 if (bvprvp && bvec_gap_to_prev(lim, bvprvp, bv->bv_offset)) { 590 blk_mq_map_bio_put(bio); 591 return -EREMOTEIO; 592 } 593 /* check full condition */ 594 if (nsegs >= nr_segs || bytes > UINT_MAX - bv->bv_len) 595 goto put_bio; 596 if (bytes + bv->bv_len > nr_iter) 597 goto put_bio; 598 if (bv->bv_offset + bv->bv_len > PAGE_SIZE) 599 goto put_bio; 600 601 nsegs++; 602 bytes += bv->bv_len; 603 bvprvp = bv; 604 } 605 return 0; 606 put_bio: 607 blk_mq_map_bio_put(bio); 608 return -EINVAL; 609 } 610 611 /** 612 * blk_rq_map_user_iov - map user data to a request, for passthrough requests 613 * @q: request queue where request should be inserted 614 * @rq: request to map data to 615 * @map_data: pointer to the rq_map_data holding pages (if necessary) 616 * @iter: iovec iterator 617 * @gfp_mask: memory allocation flags 618 * 619 * Description: 620 * Data will be mapped directly for zero copy I/O, if possible. Otherwise 621 * a kernel bounce buffer is used. 622 * 623 * A matching blk_rq_unmap_user() must be issued at the end of I/O, while 624 * still in process context. 625 */ 626 int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, 627 struct rq_map_data *map_data, 628 const struct iov_iter *iter, gfp_t gfp_mask) 629 { 630 bool copy = false, map_bvec = false; 631 unsigned long align = q->dma_pad_mask | queue_dma_alignment(q); 632 struct bio *bio = NULL; 633 struct iov_iter i; 634 int ret = -EINVAL; 635 636 if (map_data) 637 copy = true; 638 else if (blk_queue_may_bounce(q)) 639 copy = true; 640 else if (iov_iter_alignment(iter) & align) 641 copy = true; 642 else if (iov_iter_is_bvec(iter)) 643 map_bvec = true; 644 else if (!user_backed_iter(iter)) 645 copy = true; 646 else if (queue_virt_boundary(q)) 647 copy = queue_virt_boundary(q) & iov_iter_gap_alignment(iter); 648 649 if (map_bvec) { 650 ret = blk_rq_map_user_bvec(rq, iter); 651 if (!ret) 652 return 0; 653 if (ret != -EREMOTEIO) 654 goto fail; 655 /* fall back to copying the data on limits mismatches */ 656 copy = true; 657 } 658 659 i = *iter; 660 do { 661 if (copy) 662 ret = bio_copy_user_iov(rq, map_data, &i, gfp_mask); 663 else 664 ret = bio_map_user_iov(rq, &i, gfp_mask); 665 if (ret) 666 goto unmap_rq; 667 if (!bio) 668 bio = rq->bio; 669 } while (iov_iter_count(&i)); 670 671 return 0; 672 673 unmap_rq: 674 blk_rq_unmap_user(bio); 675 fail: 676 rq->bio = NULL; 677 return ret; 678 } 679 EXPORT_SYMBOL(blk_rq_map_user_iov); 680 681 int blk_rq_map_user(struct request_queue *q, struct request *rq, 682 struct rq_map_data *map_data, void __user *ubuf, 683 unsigned long len, gfp_t gfp_mask) 684 { 685 struct iov_iter i; 686 int ret = import_ubuf(rq_data_dir(rq), ubuf, len, &i); 687 688 if (unlikely(ret < 0)) 689 return ret; 690 691 return blk_rq_map_user_iov(q, rq, map_data, &i, gfp_mask); 692 } 693 EXPORT_SYMBOL(blk_rq_map_user); 694 695 int blk_rq_map_user_io(struct request *req, struct rq_map_data *map_data, 696 void __user *ubuf, unsigned long buf_len, gfp_t gfp_mask, 697 bool vec, int iov_count, bool check_iter_count, int rw) 698 { 699 int ret = 0; 700 701 if (vec) { 702 struct iovec fast_iov[UIO_FASTIOV]; 703 struct iovec *iov = fast_iov; 704 struct iov_iter iter; 705 706 ret = import_iovec(rw, ubuf, iov_count ? iov_count : buf_len, 707 UIO_FASTIOV, &iov, &iter); 708 if (ret < 0) 709 return ret; 710 711 if (iov_count) { 712 /* SG_IO howto says that the shorter of the two wins */ 713 iov_iter_truncate(&iter, buf_len); 714 if (check_iter_count && !iov_iter_count(&iter)) { 715 kfree(iov); 716 return -EINVAL; 717 } 718 } 719 720 ret = blk_rq_map_user_iov(req->q, req, map_data, &iter, 721 gfp_mask); 722 kfree(iov); 723 } else if (buf_len) { 724 ret = blk_rq_map_user(req->q, req, map_data, ubuf, buf_len, 725 gfp_mask); 726 } 727 return ret; 728 } 729 EXPORT_SYMBOL(blk_rq_map_user_io); 730 731 /** 732 * blk_rq_unmap_user - unmap a request with user data 733 * @bio: start of bio list 734 * 735 * Description: 736 * Unmap a rq previously mapped by blk_rq_map_user(). The caller must 737 * supply the original rq->bio from the blk_rq_map_user() return, since 738 * the I/O completion may have changed rq->bio. 739 */ 740 int blk_rq_unmap_user(struct bio *bio) 741 { 742 struct bio *next_bio; 743 int ret = 0, ret2; 744 745 while (bio) { 746 if (bio->bi_private) { 747 ret2 = bio_uncopy_user(bio); 748 if (ret2 && !ret) 749 ret = ret2; 750 } else { 751 bio_release_pages(bio, bio_data_dir(bio) == READ); 752 } 753 754 next_bio = bio; 755 bio = bio->bi_next; 756 blk_mq_map_bio_put(next_bio); 757 } 758 759 return ret; 760 } 761 EXPORT_SYMBOL(blk_rq_unmap_user); 762 763 /** 764 * blk_rq_map_kern - map kernel data to a request, for passthrough requests 765 * @q: request queue where request should be inserted 766 * @rq: request to fill 767 * @kbuf: the kernel buffer 768 * @len: length of user data 769 * @gfp_mask: memory allocation flags 770 * 771 * Description: 772 * Data will be mapped directly if possible. Otherwise a bounce 773 * buffer is used. Can be called multiple times to append multiple 774 * buffers. 775 */ 776 int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, 777 unsigned int len, gfp_t gfp_mask) 778 { 779 int reading = rq_data_dir(rq) == READ; 780 unsigned long addr = (unsigned long) kbuf; 781 struct bio *bio; 782 int ret; 783 784 if (len > (queue_max_hw_sectors(q) << 9)) 785 return -EINVAL; 786 if (!len || !kbuf) 787 return -EINVAL; 788 789 if (!blk_rq_aligned(q, addr, len) || object_is_on_stack(kbuf) || 790 blk_queue_may_bounce(q)) 791 bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); 792 else 793 bio = bio_map_kern(q, kbuf, len, gfp_mask); 794 795 if (IS_ERR(bio)) 796 return PTR_ERR(bio); 797 798 bio->bi_opf &= ~REQ_OP_MASK; 799 bio->bi_opf |= req_op(rq); 800 801 ret = blk_rq_append_bio(rq, bio); 802 if (unlikely(ret)) { 803 bio_uninit(bio); 804 kfree(bio); 805 } 806 return ret; 807 } 808 EXPORT_SYMBOL(blk_rq_map_kern); 809