1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to generic helpers functions 4 */ 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/scatterlist.h> 10 11 #include "blk.h" 12 13 static struct bio *next_bio(struct bio *bio, unsigned int nr_pages, 14 gfp_t gfp) 15 { 16 struct bio *new = bio_alloc(gfp, nr_pages); 17 18 if (bio) { 19 bio_chain(bio, new); 20 submit_bio(bio); 21 } 22 23 return new; 24 } 25 26 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 27 sector_t nr_sects, gfp_t gfp_mask, int flags, 28 struct bio **biop) 29 { 30 struct request_queue *q = bdev_get_queue(bdev); 31 struct bio *bio = *biop; 32 unsigned int op; 33 sector_t bs_mask; 34 35 if (!q) 36 return -ENXIO; 37 38 if (bdev_read_only(bdev)) 39 return -EPERM; 40 41 if (flags & BLKDEV_DISCARD_SECURE) { 42 if (!blk_queue_secure_erase(q)) 43 return -EOPNOTSUPP; 44 op = REQ_OP_SECURE_ERASE; 45 } else { 46 if (!blk_queue_discard(q)) 47 return -EOPNOTSUPP; 48 op = REQ_OP_DISCARD; 49 } 50 51 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 52 if ((sector | nr_sects) & bs_mask) 53 return -EINVAL; 54 55 while (nr_sects) { 56 unsigned int req_sects = nr_sects; 57 sector_t end_sect; 58 59 if (!req_sects) 60 goto fail; 61 if (req_sects > UINT_MAX >> 9) 62 req_sects = UINT_MAX >> 9; 63 64 end_sect = sector + req_sects; 65 66 bio = next_bio(bio, 0, gfp_mask); 67 bio->bi_iter.bi_sector = sector; 68 bio_set_dev(bio, bdev); 69 bio_set_op_attrs(bio, op, 0); 70 71 bio->bi_iter.bi_size = req_sects << 9; 72 nr_sects -= req_sects; 73 sector = end_sect; 74 75 /* 76 * We can loop for a long time in here, if someone does 77 * full device discards (like mkfs). Be nice and allow 78 * us to schedule out to avoid softlocking if preempt 79 * is disabled. 80 */ 81 cond_resched(); 82 } 83 84 *biop = bio; 85 return 0; 86 87 fail: 88 if (bio) { 89 submit_bio_wait(bio); 90 bio_put(bio); 91 } 92 *biop = NULL; 93 return -EOPNOTSUPP; 94 } 95 EXPORT_SYMBOL(__blkdev_issue_discard); 96 97 /** 98 * blkdev_issue_discard - queue a discard 99 * @bdev: blockdev to issue discard for 100 * @sector: start sector 101 * @nr_sects: number of sectors to discard 102 * @gfp_mask: memory allocation flags (for bio_alloc) 103 * @flags: BLKDEV_DISCARD_* flags to control behaviour 104 * 105 * Description: 106 * Issue a discard request for the sectors in question. 107 */ 108 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 109 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags) 110 { 111 struct bio *bio = NULL; 112 struct blk_plug plug; 113 int ret; 114 115 blk_start_plug(&plug); 116 ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags, 117 &bio); 118 if (!ret && bio) { 119 ret = submit_bio_wait(bio); 120 if (ret == -EOPNOTSUPP) 121 ret = 0; 122 bio_put(bio); 123 } 124 blk_finish_plug(&plug); 125 126 return ret; 127 } 128 EXPORT_SYMBOL(blkdev_issue_discard); 129 130 /** 131 * __blkdev_issue_write_same - generate number of bios with same page 132 * @bdev: target blockdev 133 * @sector: start sector 134 * @nr_sects: number of sectors to write 135 * @gfp_mask: memory allocation flags (for bio_alloc) 136 * @page: page containing data to write 137 * @biop: pointer to anchor bio 138 * 139 * Description: 140 * Generate and issue number of bios(REQ_OP_WRITE_SAME) with same page. 141 */ 142 static int __blkdev_issue_write_same(struct block_device *bdev, sector_t sector, 143 sector_t nr_sects, gfp_t gfp_mask, struct page *page, 144 struct bio **biop) 145 { 146 struct request_queue *q = bdev_get_queue(bdev); 147 unsigned int max_write_same_sectors; 148 struct bio *bio = *biop; 149 sector_t bs_mask; 150 151 if (!q) 152 return -ENXIO; 153 154 if (bdev_read_only(bdev)) 155 return -EPERM; 156 157 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 158 if ((sector | nr_sects) & bs_mask) 159 return -EINVAL; 160 161 if (!bdev_write_same(bdev)) 162 return -EOPNOTSUPP; 163 164 /* Ensure that max_write_same_sectors doesn't overflow bi_size */ 165 max_write_same_sectors = UINT_MAX >> 9; 166 167 while (nr_sects) { 168 bio = next_bio(bio, 1, gfp_mask); 169 bio->bi_iter.bi_sector = sector; 170 bio_set_dev(bio, bdev); 171 bio->bi_vcnt = 1; 172 bio->bi_io_vec->bv_page = page; 173 bio->bi_io_vec->bv_offset = 0; 174 bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev); 175 bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0); 176 177 if (nr_sects > max_write_same_sectors) { 178 bio->bi_iter.bi_size = max_write_same_sectors << 9; 179 nr_sects -= max_write_same_sectors; 180 sector += max_write_same_sectors; 181 } else { 182 bio->bi_iter.bi_size = nr_sects << 9; 183 nr_sects = 0; 184 } 185 cond_resched(); 186 } 187 188 *biop = bio; 189 return 0; 190 } 191 192 /** 193 * blkdev_issue_write_same - queue a write same operation 194 * @bdev: target blockdev 195 * @sector: start sector 196 * @nr_sects: number of sectors to write 197 * @gfp_mask: memory allocation flags (for bio_alloc) 198 * @page: page containing data 199 * 200 * Description: 201 * Issue a write same request for the sectors in question. 202 */ 203 int blkdev_issue_write_same(struct block_device *bdev, sector_t sector, 204 sector_t nr_sects, gfp_t gfp_mask, 205 struct page *page) 206 { 207 struct bio *bio = NULL; 208 struct blk_plug plug; 209 int ret; 210 211 blk_start_plug(&plug); 212 ret = __blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask, page, 213 &bio); 214 if (ret == 0 && bio) { 215 ret = submit_bio_wait(bio); 216 bio_put(bio); 217 } 218 blk_finish_plug(&plug); 219 return ret; 220 } 221 EXPORT_SYMBOL(blkdev_issue_write_same); 222 223 static int __blkdev_issue_write_zeroes(struct block_device *bdev, 224 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 225 struct bio **biop, unsigned flags) 226 { 227 struct bio *bio = *biop; 228 unsigned int max_write_zeroes_sectors; 229 struct request_queue *q = bdev_get_queue(bdev); 230 231 if (!q) 232 return -ENXIO; 233 234 if (bdev_read_only(bdev)) 235 return -EPERM; 236 237 /* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */ 238 max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev); 239 240 if (max_write_zeroes_sectors == 0) 241 return -EOPNOTSUPP; 242 243 while (nr_sects) { 244 bio = next_bio(bio, 0, gfp_mask); 245 bio->bi_iter.bi_sector = sector; 246 bio_set_dev(bio, bdev); 247 bio->bi_opf = REQ_OP_WRITE_ZEROES; 248 if (flags & BLKDEV_ZERO_NOUNMAP) 249 bio->bi_opf |= REQ_NOUNMAP; 250 251 if (nr_sects > max_write_zeroes_sectors) { 252 bio->bi_iter.bi_size = max_write_zeroes_sectors << 9; 253 nr_sects -= max_write_zeroes_sectors; 254 sector += max_write_zeroes_sectors; 255 } else { 256 bio->bi_iter.bi_size = nr_sects << 9; 257 nr_sects = 0; 258 } 259 cond_resched(); 260 } 261 262 *biop = bio; 263 return 0; 264 } 265 266 /* 267 * Convert a number of 512B sectors to a number of pages. 268 * The result is limited to a number of pages that can fit into a BIO. 269 * Also make sure that the result is always at least 1 (page) for the cases 270 * where nr_sects is lower than the number of sectors in a page. 271 */ 272 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) 273 { 274 sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512); 275 276 return min(pages, (sector_t)BIO_MAX_PAGES); 277 } 278 279 static int __blkdev_issue_zero_pages(struct block_device *bdev, 280 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 281 struct bio **biop) 282 { 283 struct request_queue *q = bdev_get_queue(bdev); 284 struct bio *bio = *biop; 285 int bi_size = 0; 286 unsigned int sz; 287 288 if (!q) 289 return -ENXIO; 290 291 if (bdev_read_only(bdev)) 292 return -EPERM; 293 294 while (nr_sects != 0) { 295 bio = next_bio(bio, __blkdev_sectors_to_bio_pages(nr_sects), 296 gfp_mask); 297 bio->bi_iter.bi_sector = sector; 298 bio_set_dev(bio, bdev); 299 bio_set_op_attrs(bio, REQ_OP_WRITE, 0); 300 301 while (nr_sects != 0) { 302 sz = min((sector_t) PAGE_SIZE, nr_sects << 9); 303 bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0); 304 nr_sects -= bi_size >> 9; 305 sector += bi_size >> 9; 306 if (bi_size < sz) 307 break; 308 } 309 cond_resched(); 310 } 311 312 *biop = bio; 313 return 0; 314 } 315 316 /** 317 * __blkdev_issue_zeroout - generate number of zero filed write bios 318 * @bdev: blockdev to issue 319 * @sector: start sector 320 * @nr_sects: number of sectors to write 321 * @gfp_mask: memory allocation flags (for bio_alloc) 322 * @biop: pointer to anchor bio 323 * @flags: controls detailed behavior 324 * 325 * Description: 326 * Zero-fill a block range, either using hardware offload or by explicitly 327 * writing zeroes to the device. 328 * 329 * If a device is using logical block provisioning, the underlying space will 330 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP. 331 * 332 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return 333 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided. 334 */ 335 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 336 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 337 unsigned flags) 338 { 339 int ret; 340 sector_t bs_mask; 341 342 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 343 if ((sector | nr_sects) & bs_mask) 344 return -EINVAL; 345 346 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask, 347 biop, flags); 348 if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK)) 349 return ret; 350 351 return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, 352 biop); 353 } 354 EXPORT_SYMBOL(__blkdev_issue_zeroout); 355 356 /** 357 * blkdev_issue_zeroout - zero-fill a block range 358 * @bdev: blockdev to write 359 * @sector: start sector 360 * @nr_sects: number of sectors to write 361 * @gfp_mask: memory allocation flags (for bio_alloc) 362 * @flags: controls detailed behavior 363 * 364 * Description: 365 * Zero-fill a block range, either using hardware offload or by explicitly 366 * writing zeroes to the device. See __blkdev_issue_zeroout() for the 367 * valid values for %flags. 368 */ 369 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 370 sector_t nr_sects, gfp_t gfp_mask, unsigned flags) 371 { 372 int ret = 0; 373 sector_t bs_mask; 374 struct bio *bio; 375 struct blk_plug plug; 376 bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev); 377 378 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 379 if ((sector | nr_sects) & bs_mask) 380 return -EINVAL; 381 382 retry: 383 bio = NULL; 384 blk_start_plug(&plug); 385 if (try_write_zeroes) { 386 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, 387 gfp_mask, &bio, flags); 388 } else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) { 389 ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects, 390 gfp_mask, &bio); 391 } else { 392 /* No zeroing offload support */ 393 ret = -EOPNOTSUPP; 394 } 395 if (ret == 0 && bio) { 396 ret = submit_bio_wait(bio); 397 bio_put(bio); 398 } 399 blk_finish_plug(&plug); 400 if (ret && try_write_zeroes) { 401 if (!(flags & BLKDEV_ZERO_NOFALLBACK)) { 402 try_write_zeroes = false; 403 goto retry; 404 } 405 if (!bdev_write_zeroes_sectors(bdev)) { 406 /* 407 * Zeroing offload support was indicated, but the 408 * device reported ILLEGAL REQUEST (for some devices 409 * there is no non-destructive way to verify whether 410 * WRITE ZEROES is actually supported). 411 */ 412 ret = -EOPNOTSUPP; 413 } 414 } 415 416 return ret; 417 } 418 EXPORT_SYMBOL(blkdev_issue_zeroout); 419