1 /* 2 * Functions related to generic helpers functions 3 */ 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/bio.h> 7 #include <linux/blkdev.h> 8 #include <linux/scatterlist.h> 9 10 #include "blk.h" 11 12 struct bio_batch { 13 atomic_t done; 14 int error; 15 struct completion *wait; 16 }; 17 18 static void bio_batch_end_io(struct bio *bio) 19 { 20 struct bio_batch *bb = bio->bi_private; 21 22 if (bio->bi_error && bio->bi_error != -EOPNOTSUPP) 23 bb->error = bio->bi_error; 24 if (atomic_dec_and_test(&bb->done)) 25 complete(bb->wait); 26 bio_put(bio); 27 } 28 29 /** 30 * blkdev_issue_discard - queue a discard 31 * @bdev: blockdev to issue discard for 32 * @sector: start sector 33 * @nr_sects: number of sectors to discard 34 * @gfp_mask: memory allocation flags (for bio_alloc) 35 * @flags: BLKDEV_IFL_* flags to control behaviour 36 * 37 * Description: 38 * Issue a discard request for the sectors in question. 39 */ 40 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 41 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags) 42 { 43 DECLARE_COMPLETION_ONSTACK(wait); 44 struct request_queue *q = bdev_get_queue(bdev); 45 int type = REQ_WRITE | REQ_DISCARD; 46 unsigned int granularity; 47 int alignment; 48 struct bio_batch bb; 49 struct bio *bio; 50 int ret = 0; 51 struct blk_plug plug; 52 53 if (!q) 54 return -ENXIO; 55 56 if (!blk_queue_discard(q)) 57 return -EOPNOTSUPP; 58 59 /* Zero-sector (unknown) and one-sector granularities are the same. */ 60 granularity = max(q->limits.discard_granularity >> 9, 1U); 61 alignment = (bdev_discard_alignment(bdev) >> 9) % granularity; 62 63 if (flags & BLKDEV_DISCARD_SECURE) { 64 if (!blk_queue_secdiscard(q)) 65 return -EOPNOTSUPP; 66 type |= REQ_SECURE; 67 } 68 69 atomic_set(&bb.done, 1); 70 bb.error = 0; 71 bb.wait = &wait; 72 73 blk_start_plug(&plug); 74 while (nr_sects) { 75 unsigned int req_sects; 76 sector_t end_sect, tmp; 77 78 bio = bio_alloc(gfp_mask, 1); 79 if (!bio) { 80 ret = -ENOMEM; 81 break; 82 } 83 84 /* Make sure bi_size doesn't overflow */ 85 req_sects = min_t(sector_t, nr_sects, UINT_MAX >> 9); 86 87 /* 88 * If splitting a request, and the next starting sector would be 89 * misaligned, stop the discard at the previous aligned sector. 90 */ 91 end_sect = sector + req_sects; 92 tmp = end_sect; 93 if (req_sects < nr_sects && 94 sector_div(tmp, granularity) != alignment) { 95 end_sect = end_sect - alignment; 96 sector_div(end_sect, granularity); 97 end_sect = end_sect * granularity + alignment; 98 req_sects = end_sect - sector; 99 } 100 101 bio->bi_iter.bi_sector = sector; 102 bio->bi_end_io = bio_batch_end_io; 103 bio->bi_bdev = bdev; 104 bio->bi_private = &bb; 105 106 bio->bi_iter.bi_size = req_sects << 9; 107 nr_sects -= req_sects; 108 sector = end_sect; 109 110 atomic_inc(&bb.done); 111 submit_bio(type, bio); 112 113 /* 114 * We can loop for a long time in here, if someone does 115 * full device discards (like mkfs). Be nice and allow 116 * us to schedule out to avoid softlocking if preempt 117 * is disabled. 118 */ 119 cond_resched(); 120 } 121 blk_finish_plug(&plug); 122 123 /* Wait for bios in-flight */ 124 if (!atomic_dec_and_test(&bb.done)) 125 wait_for_completion_io(&wait); 126 127 if (bb.error) 128 return bb.error; 129 return ret; 130 } 131 EXPORT_SYMBOL(blkdev_issue_discard); 132 133 /** 134 * blkdev_issue_write_same - queue a write same operation 135 * @bdev: target blockdev 136 * @sector: start sector 137 * @nr_sects: number of sectors to write 138 * @gfp_mask: memory allocation flags (for bio_alloc) 139 * @page: page containing data to write 140 * 141 * Description: 142 * Issue a write same request for the sectors in question. 143 */ 144 int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, 145 sector_t nr_sects, gfp_t gfp_mask, 146 struct page *page) 147 { 148 DECLARE_COMPLETION_ONSTACK(wait); 149 struct request_queue *q = bdev_get_queue(bdev); 150 unsigned int max_write_same_sectors; 151 struct bio_batch bb; 152 struct bio *bio; 153 int ret = 0; 154 155 if (!q) 156 return -ENXIO; 157 158 /* Ensure that max_write_same_sectors doesn't overflow bi_size */ 159 max_write_same_sectors = UINT_MAX >> 9; 160 161 atomic_set(&bb.done, 1); 162 bb.error = 0; 163 bb.wait = &wait; 164 165 while (nr_sects) { 166 bio = bio_alloc(gfp_mask, 1); 167 if (!bio) { 168 ret = -ENOMEM; 169 break; 170 } 171 172 bio->bi_iter.bi_sector = sector; 173 bio->bi_end_io = bio_batch_end_io; 174 bio->bi_bdev = bdev; 175 bio->bi_private = &bb; 176 bio->bi_vcnt = 1; 177 bio->bi_io_vec->bv_page = page; 178 bio->bi_io_vec->bv_offset = 0; 179 bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev); 180 181 if (nr_sects > max_write_same_sectors) { 182 bio->bi_iter.bi_size = max_write_same_sectors << 9; 183 nr_sects -= max_write_same_sectors; 184 sector += max_write_same_sectors; 185 } else { 186 bio->bi_iter.bi_size = nr_sects << 9; 187 nr_sects = 0; 188 } 189 190 atomic_inc(&bb.done); 191 submit_bio(REQ_WRITE | REQ_WRITE_SAME, bio); 192 } 193 194 /* Wait for bios in-flight */ 195 if (!atomic_dec_and_test(&bb.done)) 196 wait_for_completion_io(&wait); 197 198 if (bb.error) 199 return bb.error; 200 return ret; 201 } 202 EXPORT_SYMBOL(blkdev_issue_write_same); 203 204 /** 205 * blkdev_issue_zeroout - generate number of zero filed write bios 206 * @bdev: blockdev to issue 207 * @sector: start sector 208 * @nr_sects: number of sectors to write 209 * @gfp_mask: memory allocation flags (for bio_alloc) 210 * 211 * Description: 212 * Generate and issue number of bios with zerofiled pages. 213 */ 214 215 static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 216 sector_t nr_sects, gfp_t gfp_mask) 217 { 218 int ret; 219 struct bio *bio; 220 struct bio_batch bb; 221 unsigned int sz; 222 DECLARE_COMPLETION_ONSTACK(wait); 223 224 atomic_set(&bb.done, 1); 225 bb.error = 0; 226 bb.wait = &wait; 227 228 ret = 0; 229 while (nr_sects != 0) { 230 bio = bio_alloc(gfp_mask, 231 min(nr_sects, (sector_t)BIO_MAX_PAGES)); 232 if (!bio) { 233 ret = -ENOMEM; 234 break; 235 } 236 237 bio->bi_iter.bi_sector = sector; 238 bio->bi_bdev = bdev; 239 bio->bi_end_io = bio_batch_end_io; 240 bio->bi_private = &bb; 241 242 while (nr_sects != 0) { 243 sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects); 244 ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0); 245 nr_sects -= ret >> 9; 246 sector += ret >> 9; 247 if (ret < (sz << 9)) 248 break; 249 } 250 ret = 0; 251 atomic_inc(&bb.done); 252 submit_bio(WRITE, bio); 253 } 254 255 /* Wait for bios in-flight */ 256 if (!atomic_dec_and_test(&bb.done)) 257 wait_for_completion_io(&wait); 258 259 if (bb.error) 260 return bb.error; 261 return ret; 262 } 263 264 /** 265 * blkdev_issue_zeroout - zero-fill a block range 266 * @bdev: blockdev to write 267 * @sector: start sector 268 * @nr_sects: number of sectors to write 269 * @gfp_mask: memory allocation flags (for bio_alloc) 270 * @discard: whether to discard the block range 271 * 272 * Description: 273 * Zero-fill a block range. If the discard flag is set and the block 274 * device guarantees that subsequent READ operations to the block range 275 * in question will return zeroes, the blocks will be discarded. Should 276 * the discard request fail, if the discard flag is not set, or if 277 * discard_zeroes_data is not supported, this function will resort to 278 * zeroing the blocks manually, thus provisioning (allocating, 279 * anchoring) them. If the block device supports the WRITE SAME command 280 * blkdev_issue_zeroout() will use it to optimize the process of 281 * clearing the block range. Otherwise the zeroing will be performed 282 * using regular WRITE calls. 283 */ 284 285 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 286 sector_t nr_sects, gfp_t gfp_mask, bool discard) 287 { 288 struct request_queue *q = bdev_get_queue(bdev); 289 290 if (discard && blk_queue_discard(q) && q->limits.discard_zeroes_data && 291 blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, 0) == 0) 292 return 0; 293 294 if (bdev_write_same(bdev) && 295 blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, 296 ZERO_PAGE(0)) == 0) 297 return 0; 298 299 return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask); 300 } 301 EXPORT_SYMBOL(blkdev_issue_zeroout); 302