186db1e29SJens Axboe /* 286db1e29SJens Axboe * Functions related to mapping data to requests 386db1e29SJens Axboe */ 486db1e29SJens Axboe #include <linux/kernel.h> 586db1e29SJens Axboe #include <linux/module.h> 686db1e29SJens Axboe #include <linux/bio.h> 786db1e29SJens Axboe #include <linux/blkdev.h> 8afdc1a78SFUJITA Tomonori #include <scsi/sg.h> /* for struct sg_iovec */ 986db1e29SJens Axboe 1086db1e29SJens Axboe #include "blk.h" 1186db1e29SJens Axboe 1286db1e29SJens Axboe int blk_rq_append_bio(struct request_queue *q, struct request *rq, 1386db1e29SJens Axboe struct bio *bio) 1486db1e29SJens Axboe { 1586db1e29SJens Axboe if (!rq->bio) 1686db1e29SJens Axboe blk_rq_bio_prep(q, rq, bio); 1786db1e29SJens Axboe else if (!ll_back_merge_fn(q, rq, bio)) 1886db1e29SJens Axboe return -EINVAL; 1986db1e29SJens Axboe else { 2086db1e29SJens Axboe rq->biotail->bi_next = bio; 2186db1e29SJens Axboe rq->biotail = bio; 2286db1e29SJens Axboe 2386db1e29SJens Axboe rq->data_len += bio->bi_size; 2486db1e29SJens Axboe } 2586db1e29SJens Axboe return 0; 2686db1e29SJens Axboe } 2786db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_append_bio); 2886db1e29SJens Axboe 2986db1e29SJens Axboe static int __blk_rq_unmap_user(struct bio *bio) 3086db1e29SJens Axboe { 3186db1e29SJens Axboe int ret = 0; 3286db1e29SJens Axboe 3386db1e29SJens Axboe if (bio) { 3486db1e29SJens Axboe if (bio_flagged(bio, BIO_USER_MAPPED)) 3586db1e29SJens Axboe bio_unmap_user(bio); 3686db1e29SJens Axboe else 3786db1e29SJens Axboe ret = bio_uncopy_user(bio); 3886db1e29SJens Axboe } 3986db1e29SJens Axboe 4086db1e29SJens Axboe return ret; 4186db1e29SJens Axboe } 4286db1e29SJens Axboe 4386db1e29SJens Axboe static int __blk_rq_map_user(struct request_queue *q, struct request *rq, 44a3bce90eSFUJITA Tomonori void __user *ubuf, unsigned int len, 45a3bce90eSFUJITA Tomonori gfp_t gfp_mask) 4686db1e29SJens Axboe { 4786db1e29SJens Axboe unsigned long uaddr; 48e3790c7dSTejun Heo unsigned int alignment; 4986db1e29SJens Axboe struct bio *bio, *orig_bio; 5086db1e29SJens Axboe int reading, ret; 5186db1e29SJens Axboe 5286db1e29SJens Axboe reading = rq_data_dir(rq) == READ; 5386db1e29SJens Axboe 5486db1e29SJens Axboe /* 5586db1e29SJens Axboe * if alignment requirement is satisfied, map in user pages for 5686db1e29SJens Axboe * direct dma. else, set up kernel bounce buffers 5786db1e29SJens Axboe */ 5886db1e29SJens Axboe uaddr = (unsigned long) ubuf; 59e3790c7dSTejun Heo alignment = queue_dma_alignment(q) | q->dma_pad_mask; 60e3790c7dSTejun Heo if (!(uaddr & alignment) && !(len & alignment)) 61a3bce90eSFUJITA Tomonori bio = bio_map_user(q, NULL, uaddr, len, reading, gfp_mask); 6286db1e29SJens Axboe else 63a3bce90eSFUJITA Tomonori bio = bio_copy_user(q, uaddr, len, reading, gfp_mask); 6486db1e29SJens Axboe 6586db1e29SJens Axboe if (IS_ERR(bio)) 6686db1e29SJens Axboe return PTR_ERR(bio); 6786db1e29SJens Axboe 6886db1e29SJens Axboe orig_bio = bio; 6986db1e29SJens Axboe blk_queue_bounce(q, &bio); 7086db1e29SJens Axboe 7186db1e29SJens Axboe /* 7286db1e29SJens Axboe * We link the bounce buffer in and could have to traverse it 7386db1e29SJens Axboe * later so we have to get a ref to prevent it from being freed 7486db1e29SJens Axboe */ 7586db1e29SJens Axboe bio_get(bio); 7686db1e29SJens Axboe 7786db1e29SJens Axboe ret = blk_rq_append_bio(q, rq, bio); 7886db1e29SJens Axboe if (!ret) 7986db1e29SJens Axboe return bio->bi_size; 8086db1e29SJens Axboe 8186db1e29SJens Axboe /* if it was boucned we must call the end io function */ 8286db1e29SJens Axboe bio_endio(bio, 0); 8386db1e29SJens Axboe __blk_rq_unmap_user(orig_bio); 8486db1e29SJens Axboe bio_put(bio); 8586db1e29SJens Axboe return ret; 8686db1e29SJens Axboe } 8786db1e29SJens Axboe 8886db1e29SJens Axboe /** 89710027a4SRandy Dunlap * blk_rq_map_user - map user data to a request, for REQ_TYPE_BLOCK_PC usage 9086db1e29SJens Axboe * @q: request queue where request should be inserted 9186db1e29SJens Axboe * @rq: request structure to fill 9286db1e29SJens Axboe * @ubuf: the user buffer 9386db1e29SJens Axboe * @len: length of user data 94a3bce90eSFUJITA Tomonori * @gfp_mask: memory allocation flags 9586db1e29SJens Axboe * 9686db1e29SJens Axboe * Description: 97710027a4SRandy Dunlap * Data will be mapped directly for zero copy I/O, if possible. Otherwise 9886db1e29SJens Axboe * a kernel bounce buffer is used. 9986db1e29SJens Axboe * 100710027a4SRandy Dunlap * A matching blk_rq_unmap_user() must be issued at the end of I/O, while 10186db1e29SJens Axboe * still in process context. 10286db1e29SJens Axboe * 10386db1e29SJens Axboe * Note: The mapped bio may need to be bounced through blk_queue_bounce() 10486db1e29SJens Axboe * before being submitted to the device, as pages mapped may be out of 10586db1e29SJens Axboe * reach. It's the callers responsibility to make sure this happens. The 10686db1e29SJens Axboe * original bio must be passed back in to blk_rq_unmap_user() for proper 10786db1e29SJens Axboe * unmapping. 10886db1e29SJens Axboe */ 10986db1e29SJens Axboe int blk_rq_map_user(struct request_queue *q, struct request *rq, 110a3bce90eSFUJITA Tomonori void __user *ubuf, unsigned long len, gfp_t gfp_mask) 11186db1e29SJens Axboe { 11286db1e29SJens Axboe unsigned long bytes_read = 0; 11386db1e29SJens Axboe struct bio *bio = NULL; 11486db1e29SJens Axboe int ret; 11586db1e29SJens Axboe 11686db1e29SJens Axboe if (len > (q->max_hw_sectors << 9)) 11786db1e29SJens Axboe return -EINVAL; 11886db1e29SJens Axboe if (!len || !ubuf) 11986db1e29SJens Axboe return -EINVAL; 12086db1e29SJens Axboe 12186db1e29SJens Axboe while (bytes_read != len) { 12286db1e29SJens Axboe unsigned long map_len, end, start; 12386db1e29SJens Axboe 12486db1e29SJens Axboe map_len = min_t(unsigned long, len - bytes_read, BIO_MAX_SIZE); 12586db1e29SJens Axboe end = ((unsigned long)ubuf + map_len + PAGE_SIZE - 1) 12686db1e29SJens Axboe >> PAGE_SHIFT; 12786db1e29SJens Axboe start = (unsigned long)ubuf >> PAGE_SHIFT; 12886db1e29SJens Axboe 12986db1e29SJens Axboe /* 13086db1e29SJens Axboe * A bad offset could cause us to require BIO_MAX_PAGES + 1 13186db1e29SJens Axboe * pages. If this happens we just lower the requested 13286db1e29SJens Axboe * mapping len by a page so that we can fit 13386db1e29SJens Axboe */ 13486db1e29SJens Axboe if (end - start > BIO_MAX_PAGES) 13586db1e29SJens Axboe map_len -= PAGE_SIZE; 13686db1e29SJens Axboe 137a3bce90eSFUJITA Tomonori ret = __blk_rq_map_user(q, rq, ubuf, map_len, gfp_mask); 13886db1e29SJens Axboe if (ret < 0) 13986db1e29SJens Axboe goto unmap_rq; 14086db1e29SJens Axboe if (!bio) 14186db1e29SJens Axboe bio = rq->bio; 14286db1e29SJens Axboe bytes_read += ret; 14386db1e29SJens Axboe ubuf += ret; 14486db1e29SJens Axboe } 14586db1e29SJens Axboe 146f18573abSFUJITA Tomonori if (!bio_flagged(bio, BIO_USER_MAPPED)) 147f18573abSFUJITA Tomonori rq->cmd_flags |= REQ_COPY_USER; 14840b01b9bSTejun Heo 14986db1e29SJens Axboe rq->buffer = rq->data = NULL; 15086db1e29SJens Axboe return 0; 15186db1e29SJens Axboe unmap_rq: 15286db1e29SJens Axboe blk_rq_unmap_user(bio); 15384e9e03cSJens Axboe rq->bio = NULL; 15486db1e29SJens Axboe return ret; 15586db1e29SJens Axboe } 15686db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_map_user); 15786db1e29SJens Axboe 15886db1e29SJens Axboe /** 159710027a4SRandy Dunlap * blk_rq_map_user_iov - map user data to a request, for REQ_TYPE_BLOCK_PC usage 16086db1e29SJens Axboe * @q: request queue where request should be inserted 16186db1e29SJens Axboe * @rq: request to map data to 16286db1e29SJens Axboe * @iov: pointer to the iovec 16386db1e29SJens Axboe * @iov_count: number of elements in the iovec 16486db1e29SJens Axboe * @len: I/O byte count 165a3bce90eSFUJITA Tomonori * @gfp_mask: memory allocation flags 16686db1e29SJens Axboe * 16786db1e29SJens Axboe * Description: 168710027a4SRandy Dunlap * Data will be mapped directly for zero copy I/O, if possible. Otherwise 16986db1e29SJens Axboe * a kernel bounce buffer is used. 17086db1e29SJens Axboe * 171710027a4SRandy Dunlap * A matching blk_rq_unmap_user() must be issued at the end of I/O, while 17286db1e29SJens Axboe * still in process context. 17386db1e29SJens Axboe * 17486db1e29SJens Axboe * Note: The mapped bio may need to be bounced through blk_queue_bounce() 17586db1e29SJens Axboe * before being submitted to the device, as pages mapped may be out of 17686db1e29SJens Axboe * reach. It's the callers responsibility to make sure this happens. The 17786db1e29SJens Axboe * original bio must be passed back in to blk_rq_unmap_user() for proper 17886db1e29SJens Axboe * unmapping. 17986db1e29SJens Axboe */ 18086db1e29SJens Axboe int blk_rq_map_user_iov(struct request_queue *q, struct request *rq, 181a3bce90eSFUJITA Tomonori struct sg_iovec *iov, int iov_count, unsigned int len, 182a3bce90eSFUJITA Tomonori gfp_t gfp_mask) 18386db1e29SJens Axboe { 18486db1e29SJens Axboe struct bio *bio; 185afdc1a78SFUJITA Tomonori int i, read = rq_data_dir(rq) == READ; 186afdc1a78SFUJITA Tomonori int unaligned = 0; 18786db1e29SJens Axboe 18886db1e29SJens Axboe if (!iov || iov_count <= 0) 18986db1e29SJens Axboe return -EINVAL; 19086db1e29SJens Axboe 191afdc1a78SFUJITA Tomonori for (i = 0; i < iov_count; i++) { 192afdc1a78SFUJITA Tomonori unsigned long uaddr = (unsigned long)iov[i].iov_base; 193afdc1a78SFUJITA Tomonori 194afdc1a78SFUJITA Tomonori if (uaddr & queue_dma_alignment(q)) { 195afdc1a78SFUJITA Tomonori unaligned = 1; 196afdc1a78SFUJITA Tomonori break; 197afdc1a78SFUJITA Tomonori } 198afdc1a78SFUJITA Tomonori } 199afdc1a78SFUJITA Tomonori 200afdc1a78SFUJITA Tomonori if (unaligned || (q->dma_pad_mask & len)) 201a3bce90eSFUJITA Tomonori bio = bio_copy_user_iov(q, iov, iov_count, read, gfp_mask); 202afdc1a78SFUJITA Tomonori else 203a3bce90eSFUJITA Tomonori bio = bio_map_user_iov(q, NULL, iov, iov_count, read, gfp_mask); 204afdc1a78SFUJITA Tomonori 20586db1e29SJens Axboe if (IS_ERR(bio)) 20686db1e29SJens Axboe return PTR_ERR(bio); 20786db1e29SJens Axboe 20886db1e29SJens Axboe if (bio->bi_size != len) { 20986db1e29SJens Axboe bio_endio(bio, 0); 21086db1e29SJens Axboe bio_unmap_user(bio); 21186db1e29SJens Axboe return -EINVAL; 21286db1e29SJens Axboe } 21386db1e29SJens Axboe 214f18573abSFUJITA Tomonori if (!bio_flagged(bio, BIO_USER_MAPPED)) 215f18573abSFUJITA Tomonori rq->cmd_flags |= REQ_COPY_USER; 216f18573abSFUJITA Tomonori 21707359fc6SFUJITA Tomonori blk_queue_bounce(q, &bio); 21886db1e29SJens Axboe bio_get(bio); 21986db1e29SJens Axboe blk_rq_bio_prep(q, rq, bio); 22086db1e29SJens Axboe rq->buffer = rq->data = NULL; 22186db1e29SJens Axboe return 0; 22286db1e29SJens Axboe } 22386db1e29SJens Axboe 22486db1e29SJens Axboe /** 22586db1e29SJens Axboe * blk_rq_unmap_user - unmap a request with user data 22686db1e29SJens Axboe * @bio: start of bio list 22786db1e29SJens Axboe * 22886db1e29SJens Axboe * Description: 22986db1e29SJens Axboe * Unmap a rq previously mapped by blk_rq_map_user(). The caller must 23086db1e29SJens Axboe * supply the original rq->bio from the blk_rq_map_user() return, since 231710027a4SRandy Dunlap * the I/O completion may have changed rq->bio. 23286db1e29SJens Axboe */ 23386db1e29SJens Axboe int blk_rq_unmap_user(struct bio *bio) 23486db1e29SJens Axboe { 23586db1e29SJens Axboe struct bio *mapped_bio; 23686db1e29SJens Axboe int ret = 0, ret2; 23786db1e29SJens Axboe 23886db1e29SJens Axboe while (bio) { 23986db1e29SJens Axboe mapped_bio = bio; 24086db1e29SJens Axboe if (unlikely(bio_flagged(bio, BIO_BOUNCED))) 24186db1e29SJens Axboe mapped_bio = bio->bi_private; 24286db1e29SJens Axboe 24386db1e29SJens Axboe ret2 = __blk_rq_unmap_user(mapped_bio); 24486db1e29SJens Axboe if (ret2 && !ret) 24586db1e29SJens Axboe ret = ret2; 24686db1e29SJens Axboe 24786db1e29SJens Axboe mapped_bio = bio; 24886db1e29SJens Axboe bio = bio->bi_next; 24986db1e29SJens Axboe bio_put(mapped_bio); 25086db1e29SJens Axboe } 25186db1e29SJens Axboe 25286db1e29SJens Axboe return ret; 25386db1e29SJens Axboe } 25486db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_unmap_user); 25586db1e29SJens Axboe 25686db1e29SJens Axboe /** 257710027a4SRandy Dunlap * blk_rq_map_kern - map kernel data to a request, for REQ_TYPE_BLOCK_PC usage 25886db1e29SJens Axboe * @q: request queue where request should be inserted 25986db1e29SJens Axboe * @rq: request to fill 26086db1e29SJens Axboe * @kbuf: the kernel buffer 26186db1e29SJens Axboe * @len: length of user data 26286db1e29SJens Axboe * @gfp_mask: memory allocation flags 26368154e90SFUJITA Tomonori * 26468154e90SFUJITA Tomonori * Description: 26568154e90SFUJITA Tomonori * Data will be mapped directly if possible. Otherwise a bounce 26668154e90SFUJITA Tomonori * buffer is used. 26786db1e29SJens Axboe */ 26886db1e29SJens Axboe int blk_rq_map_kern(struct request_queue *q, struct request *rq, void *kbuf, 26986db1e29SJens Axboe unsigned int len, gfp_t gfp_mask) 27086db1e29SJens Axboe { 27168154e90SFUJITA Tomonori unsigned long kaddr; 27268154e90SFUJITA Tomonori unsigned int alignment; 27368154e90SFUJITA Tomonori int reading = rq_data_dir(rq) == READ; 27468154e90SFUJITA Tomonori int do_copy = 0; 27586db1e29SJens Axboe struct bio *bio; 27686db1e29SJens Axboe 27786db1e29SJens Axboe if (len > (q->max_hw_sectors << 9)) 27886db1e29SJens Axboe return -EINVAL; 27986db1e29SJens Axboe if (!len || !kbuf) 28086db1e29SJens Axboe return -EINVAL; 28186db1e29SJens Axboe 28268154e90SFUJITA Tomonori kaddr = (unsigned long)kbuf; 28368154e90SFUJITA Tomonori alignment = queue_dma_alignment(q) | q->dma_pad_mask; 284a76eef95SFUJITA Tomonori do_copy = ((kaddr & alignment) || (len & alignment) || 285a76eef95SFUJITA Tomonori object_is_on_stack(kbuf)); 28630c00edaSFUJITA Tomonori 28768154e90SFUJITA Tomonori if (do_copy) 28868154e90SFUJITA Tomonori bio = bio_copy_kern(q, kbuf, len, gfp_mask, reading); 28968154e90SFUJITA Tomonori else 29086db1e29SJens Axboe bio = bio_map_kern(q, kbuf, len, gfp_mask); 29168154e90SFUJITA Tomonori 29286db1e29SJens Axboe if (IS_ERR(bio)) 29386db1e29SJens Axboe return PTR_ERR(bio); 29486db1e29SJens Axboe 29586db1e29SJens Axboe if (rq_data_dir(rq) == WRITE) 29686db1e29SJens Axboe bio->bi_rw |= (1 << BIO_RW); 29786db1e29SJens Axboe 29868154e90SFUJITA Tomonori if (do_copy) 29968154e90SFUJITA Tomonori rq->cmd_flags |= REQ_COPY_USER; 30068154e90SFUJITA Tomonori 30186db1e29SJens Axboe blk_rq_bio_prep(q, rq, bio); 30286db1e29SJens Axboe blk_queue_bounce(q, &rq->bio); 30386db1e29SJens Axboe rq->buffer = rq->data = NULL; 30486db1e29SJens Axboe return 0; 30586db1e29SJens Axboe } 30686db1e29SJens Axboe EXPORT_SYMBOL(blk_rq_map_kern); 307