1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Functions related to generic helpers functions 4 */ 5 #include <linux/kernel.h> 6 #include <linux/module.h> 7 #include <linux/bio.h> 8 #include <linux/blkdev.h> 9 #include <linux/scatterlist.h> 10 11 #include "blk.h" 12 13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector) 14 { 15 unsigned int discard_granularity = bdev_discard_granularity(bdev); 16 sector_t granularity_aligned_sector; 17 18 if (bdev_is_partition(bdev)) 19 sector += bdev->bd_start_sect; 20 21 granularity_aligned_sector = 22 round_up(sector, discard_granularity >> SECTOR_SHIFT); 23 24 /* 25 * Make sure subsequent bios start aligned to the discard granularity if 26 * it needs to be split. 27 */ 28 if (granularity_aligned_sector != sector) 29 return granularity_aligned_sector - sector; 30 31 /* 32 * Align the bio size to the discard granularity to make splitting the bio 33 * at discard granularity boundaries easier in the driver if needed. 34 */ 35 return round_down(UINT_MAX, discard_granularity) >> SECTOR_SHIFT; 36 } 37 38 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector, 39 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop) 40 { 41 struct bio *bio = *biop; 42 sector_t bs_mask; 43 44 if (bdev_read_only(bdev)) 45 return -EPERM; 46 if (!bdev_max_discard_sectors(bdev)) 47 return -EOPNOTSUPP; 48 49 /* In case the discard granularity isn't set by buggy device driver */ 50 if (WARN_ON_ONCE(!bdev_discard_granularity(bdev))) { 51 char dev_name[BDEVNAME_SIZE]; 52 53 bdevname(bdev, dev_name); 54 pr_err_ratelimited("%s: Error: discard_granularity is 0.\n", dev_name); 55 return -EOPNOTSUPP; 56 } 57 58 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 59 if ((sector | nr_sects) & bs_mask) 60 return -EINVAL; 61 62 if (!nr_sects) 63 return -EINVAL; 64 65 while (nr_sects) { 66 sector_t req_sects = 67 min(nr_sects, bio_discard_limit(bdev, sector)); 68 69 bio = blk_next_bio(bio, bdev, 0, REQ_OP_DISCARD, gfp_mask); 70 bio->bi_iter.bi_sector = sector; 71 bio->bi_iter.bi_size = req_sects << 9; 72 sector += req_sects; 73 nr_sects -= req_sects; 74 75 /* 76 * We can loop for a long time in here, if someone does 77 * full device discards (like mkfs). Be nice and allow 78 * us to schedule out to avoid softlocking if preempt 79 * is disabled. 80 */ 81 cond_resched(); 82 } 83 84 *biop = bio; 85 return 0; 86 } 87 EXPORT_SYMBOL(__blkdev_issue_discard); 88 89 /** 90 * blkdev_issue_discard - queue a discard 91 * @bdev: blockdev to issue discard for 92 * @sector: start sector 93 * @nr_sects: number of sectors to discard 94 * @gfp_mask: memory allocation flags (for bio_alloc) 95 * 96 * Description: 97 * Issue a discard request for the sectors in question. 98 */ 99 int blkdev_issue_discard(struct block_device *bdev, sector_t sector, 100 sector_t nr_sects, gfp_t gfp_mask) 101 { 102 struct bio *bio = NULL; 103 struct blk_plug plug; 104 int ret; 105 106 blk_start_plug(&plug); 107 ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio); 108 if (!ret && bio) { 109 ret = submit_bio_wait(bio); 110 if (ret == -EOPNOTSUPP) 111 ret = 0; 112 bio_put(bio); 113 } 114 blk_finish_plug(&plug); 115 116 return ret; 117 } 118 EXPORT_SYMBOL(blkdev_issue_discard); 119 120 static int __blkdev_issue_write_zeroes(struct block_device *bdev, 121 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 122 struct bio **biop, unsigned flags) 123 { 124 struct bio *bio = *biop; 125 unsigned int max_write_zeroes_sectors; 126 127 if (bdev_read_only(bdev)) 128 return -EPERM; 129 130 /* Ensure that max_write_zeroes_sectors doesn't overflow bi_size */ 131 max_write_zeroes_sectors = bdev_write_zeroes_sectors(bdev); 132 133 if (max_write_zeroes_sectors == 0) 134 return -EOPNOTSUPP; 135 136 while (nr_sects) { 137 bio = blk_next_bio(bio, bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask); 138 bio->bi_iter.bi_sector = sector; 139 if (flags & BLKDEV_ZERO_NOUNMAP) 140 bio->bi_opf |= REQ_NOUNMAP; 141 142 if (nr_sects > max_write_zeroes_sectors) { 143 bio->bi_iter.bi_size = max_write_zeroes_sectors << 9; 144 nr_sects -= max_write_zeroes_sectors; 145 sector += max_write_zeroes_sectors; 146 } else { 147 bio->bi_iter.bi_size = nr_sects << 9; 148 nr_sects = 0; 149 } 150 cond_resched(); 151 } 152 153 *biop = bio; 154 return 0; 155 } 156 157 /* 158 * Convert a number of 512B sectors to a number of pages. 159 * The result is limited to a number of pages that can fit into a BIO. 160 * Also make sure that the result is always at least 1 (page) for the cases 161 * where nr_sects is lower than the number of sectors in a page. 162 */ 163 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects) 164 { 165 sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512); 166 167 return min(pages, (sector_t)BIO_MAX_VECS); 168 } 169 170 static int __blkdev_issue_zero_pages(struct block_device *bdev, 171 sector_t sector, sector_t nr_sects, gfp_t gfp_mask, 172 struct bio **biop) 173 { 174 struct bio *bio = *biop; 175 int bi_size = 0; 176 unsigned int sz; 177 178 if (bdev_read_only(bdev)) 179 return -EPERM; 180 181 while (nr_sects != 0) { 182 bio = blk_next_bio(bio, bdev, __blkdev_sectors_to_bio_pages(nr_sects), 183 REQ_OP_WRITE, gfp_mask); 184 bio->bi_iter.bi_sector = sector; 185 186 while (nr_sects != 0) { 187 sz = min((sector_t) PAGE_SIZE, nr_sects << 9); 188 bi_size = bio_add_page(bio, ZERO_PAGE(0), sz, 0); 189 nr_sects -= bi_size >> 9; 190 sector += bi_size >> 9; 191 if (bi_size < sz) 192 break; 193 } 194 cond_resched(); 195 } 196 197 *biop = bio; 198 return 0; 199 } 200 201 /** 202 * __blkdev_issue_zeroout - generate number of zero filed write bios 203 * @bdev: blockdev to issue 204 * @sector: start sector 205 * @nr_sects: number of sectors to write 206 * @gfp_mask: memory allocation flags (for bio_alloc) 207 * @biop: pointer to anchor bio 208 * @flags: controls detailed behavior 209 * 210 * Description: 211 * Zero-fill a block range, either using hardware offload or by explicitly 212 * writing zeroes to the device. 213 * 214 * If a device is using logical block provisioning, the underlying space will 215 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP. 216 * 217 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return 218 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided. 219 */ 220 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 221 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop, 222 unsigned flags) 223 { 224 int ret; 225 sector_t bs_mask; 226 227 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 228 if ((sector | nr_sects) & bs_mask) 229 return -EINVAL; 230 231 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp_mask, 232 biop, flags); 233 if (ret != -EOPNOTSUPP || (flags & BLKDEV_ZERO_NOFALLBACK)) 234 return ret; 235 236 return __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, 237 biop); 238 } 239 EXPORT_SYMBOL(__blkdev_issue_zeroout); 240 241 /** 242 * blkdev_issue_zeroout - zero-fill a block range 243 * @bdev: blockdev to write 244 * @sector: start sector 245 * @nr_sects: number of sectors to write 246 * @gfp_mask: memory allocation flags (for bio_alloc) 247 * @flags: controls detailed behavior 248 * 249 * Description: 250 * Zero-fill a block range, either using hardware offload or by explicitly 251 * writing zeroes to the device. See __blkdev_issue_zeroout() for the 252 * valid values for %flags. 253 */ 254 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector, 255 sector_t nr_sects, gfp_t gfp_mask, unsigned flags) 256 { 257 int ret = 0; 258 sector_t bs_mask; 259 struct bio *bio; 260 struct blk_plug plug; 261 bool try_write_zeroes = !!bdev_write_zeroes_sectors(bdev); 262 263 bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 264 if ((sector | nr_sects) & bs_mask) 265 return -EINVAL; 266 267 retry: 268 bio = NULL; 269 blk_start_plug(&plug); 270 if (try_write_zeroes) { 271 ret = __blkdev_issue_write_zeroes(bdev, sector, nr_sects, 272 gfp_mask, &bio, flags); 273 } else if (!(flags & BLKDEV_ZERO_NOFALLBACK)) { 274 ret = __blkdev_issue_zero_pages(bdev, sector, nr_sects, 275 gfp_mask, &bio); 276 } else { 277 /* No zeroing offload support */ 278 ret = -EOPNOTSUPP; 279 } 280 if (ret == 0 && bio) { 281 ret = submit_bio_wait(bio); 282 bio_put(bio); 283 } 284 blk_finish_plug(&plug); 285 if (ret && try_write_zeroes) { 286 if (!(flags & BLKDEV_ZERO_NOFALLBACK)) { 287 try_write_zeroes = false; 288 goto retry; 289 } 290 if (!bdev_write_zeroes_sectors(bdev)) { 291 /* 292 * Zeroing offload support was indicated, but the 293 * device reported ILLEGAL REQUEST (for some devices 294 * there is no non-destructive way to verify whether 295 * WRITE ZEROES is actually supported). 296 */ 297 ret = -EOPNOTSUPP; 298 } 299 } 300 301 return ret; 302 } 303 EXPORT_SYMBOL(blkdev_issue_zeroout); 304 305 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector, 306 sector_t nr_sects, gfp_t gfp) 307 { 308 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1; 309 unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev); 310 struct bio *bio = NULL; 311 struct blk_plug plug; 312 int ret = 0; 313 314 if (max_sectors == 0) 315 return -EOPNOTSUPP; 316 if ((sector | nr_sects) & bs_mask) 317 return -EINVAL; 318 if (bdev_read_only(bdev)) 319 return -EPERM; 320 321 blk_start_plug(&plug); 322 for (;;) { 323 unsigned int len = min_t(sector_t, nr_sects, max_sectors); 324 325 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp); 326 bio->bi_iter.bi_sector = sector; 327 bio->bi_iter.bi_size = len; 328 329 sector += len << SECTOR_SHIFT; 330 nr_sects -= len << SECTOR_SHIFT; 331 if (!nr_sects) { 332 ret = submit_bio_wait(bio); 333 bio_put(bio); 334 break; 335 } 336 cond_resched(); 337 } 338 blk_finish_plug(&plug); 339 340 return ret; 341 } 342 EXPORT_SYMBOL(blkdev_issue_secure_erase); 343