1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Zoned block device handling 4 * 5 * Copyright (c) 2015, Hannes Reinecke 6 * Copyright (c) 2015, SUSE Linux GmbH 7 * 8 * Copyright (c) 2016, Damien Le Moal 9 * Copyright (c) 2016, Western Digital 10 */ 11 12 #include <linux/kernel.h> 13 #include <linux/module.h> 14 #include <linux/rbtree.h> 15 #include <linux/blkdev.h> 16 #include <linux/blk-mq.h> 17 #include <linux/mm.h> 18 #include <linux/vmalloc.h> 19 #include <linux/sched/mm.h> 20 21 #include "blk.h" 22 23 static inline sector_t blk_zone_start(struct request_queue *q, 24 sector_t sector) 25 { 26 sector_t zone_mask = blk_queue_zone_sectors(q) - 1; 27 28 return sector & ~zone_mask; 29 } 30 31 /* 32 * Return true if a request is a write requests that needs zone write locking. 33 */ 34 bool blk_req_needs_zone_write_lock(struct request *rq) 35 { 36 if (!rq->q->seq_zones_wlock) 37 return false; 38 39 if (blk_rq_is_passthrough(rq)) 40 return false; 41 42 switch (req_op(rq)) { 43 case REQ_OP_WRITE_ZEROES: 44 case REQ_OP_WRITE_SAME: 45 case REQ_OP_WRITE: 46 return blk_rq_zone_is_seq(rq); 47 default: 48 return false; 49 } 50 } 51 EXPORT_SYMBOL_GPL(blk_req_needs_zone_write_lock); 52 53 void __blk_req_zone_write_lock(struct request *rq) 54 { 55 if (WARN_ON_ONCE(test_and_set_bit(blk_rq_zone_no(rq), 56 rq->q->seq_zones_wlock))) 57 return; 58 59 WARN_ON_ONCE(rq->rq_flags & RQF_ZONE_WRITE_LOCKED); 60 rq->rq_flags |= RQF_ZONE_WRITE_LOCKED; 61 } 62 EXPORT_SYMBOL_GPL(__blk_req_zone_write_lock); 63 64 void __blk_req_zone_write_unlock(struct request *rq) 65 { 66 rq->rq_flags &= ~RQF_ZONE_WRITE_LOCKED; 67 if (rq->q->seq_zones_wlock) 68 WARN_ON_ONCE(!test_and_clear_bit(blk_rq_zone_no(rq), 69 rq->q->seq_zones_wlock)); 70 } 71 EXPORT_SYMBOL_GPL(__blk_req_zone_write_unlock); 72 73 /** 74 * blkdev_nr_zones - Get number of zones 75 * @disk: Target gendisk 76 * 77 * Return the total number of zones of a zoned block device. For a block 78 * device without zone capabilities, the number of zones is always 0. 79 */ 80 unsigned int blkdev_nr_zones(struct gendisk *disk) 81 { 82 sector_t zone_sectors = blk_queue_zone_sectors(disk->queue); 83 84 if (!blk_queue_is_zoned(disk->queue)) 85 return 0; 86 return (get_capacity(disk) + zone_sectors - 1) >> ilog2(zone_sectors); 87 } 88 EXPORT_SYMBOL_GPL(blkdev_nr_zones); 89 90 /** 91 * blkdev_report_zones - Get zones information 92 * @bdev: Target block device 93 * @sector: Sector from which to report zones 94 * @nr_zones: Maximum number of zones to report 95 * @cb: Callback function called for each reported zone 96 * @data: Private data for the callback 97 * 98 * Description: 99 * Get zone information starting from the zone containing @sector for at most 100 * @nr_zones, and call @cb for each zone reported by the device. 101 * To report all zones in a device starting from @sector, the BLK_ALL_ZONES 102 * constant can be passed to @nr_zones. 103 * Returns the number of zones reported by the device, or a negative errno 104 * value in case of failure. 105 * 106 * Note: The caller must use memalloc_noXX_save/restore() calls to control 107 * memory allocations done within this function. 108 */ 109 int blkdev_report_zones(struct block_device *bdev, sector_t sector, 110 unsigned int nr_zones, report_zones_cb cb, void *data) 111 { 112 struct gendisk *disk = bdev->bd_disk; 113 sector_t capacity = get_capacity(disk); 114 115 if (!blk_queue_is_zoned(bdev_get_queue(bdev)) || 116 WARN_ON_ONCE(!disk->fops->report_zones)) 117 return -EOPNOTSUPP; 118 119 if (!nr_zones || sector >= capacity) 120 return 0; 121 122 return disk->fops->report_zones(disk, sector, nr_zones, cb, data); 123 } 124 EXPORT_SYMBOL_GPL(blkdev_report_zones); 125 126 static inline bool blkdev_allow_reset_all_zones(struct block_device *bdev, 127 sector_t sector, 128 sector_t nr_sectors) 129 { 130 if (!blk_queue_zone_resetall(bdev_get_queue(bdev))) 131 return false; 132 133 /* 134 * REQ_OP_ZONE_RESET_ALL can be executed only if the number of sectors 135 * of the applicable zone range is the entire disk. 136 */ 137 return !sector && nr_sectors == get_capacity(bdev->bd_disk); 138 } 139 140 /** 141 * blkdev_zone_mgmt - Execute a zone management operation on a range of zones 142 * @bdev: Target block device 143 * @op: Operation to be performed on the zones 144 * @sector: Start sector of the first zone to operate on 145 * @nr_sectors: Number of sectors, should be at least the length of one zone and 146 * must be zone size aligned. 147 * @gfp_mask: Memory allocation flags (for bio_alloc) 148 * 149 * Description: 150 * Perform the specified operation on the range of zones specified by 151 * @sector..@sector+@nr_sectors. Specifying the entire disk sector range 152 * is valid, but the specified range should not contain conventional zones. 153 * The operation to execute on each zone can be a zone reset, open, close 154 * or finish request. 155 */ 156 int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op, 157 sector_t sector, sector_t nr_sectors, 158 gfp_t gfp_mask) 159 { 160 struct request_queue *q = bdev_get_queue(bdev); 161 sector_t zone_sectors = blk_queue_zone_sectors(q); 162 sector_t capacity = get_capacity(bdev->bd_disk); 163 sector_t end_sector = sector + nr_sectors; 164 struct bio *bio = NULL; 165 int ret; 166 167 if (!blk_queue_is_zoned(q)) 168 return -EOPNOTSUPP; 169 170 if (bdev_read_only(bdev)) 171 return -EPERM; 172 173 if (!op_is_zone_mgmt(op)) 174 return -EOPNOTSUPP; 175 176 if (!nr_sectors || end_sector > capacity) 177 /* Out of range */ 178 return -EINVAL; 179 180 /* Check alignment (handle eventual smaller last zone) */ 181 if (sector & (zone_sectors - 1)) 182 return -EINVAL; 183 184 if ((nr_sectors & (zone_sectors - 1)) && end_sector != capacity) 185 return -EINVAL; 186 187 while (sector < end_sector) { 188 bio = blk_next_bio(bio, 0, gfp_mask); 189 bio_set_dev(bio, bdev); 190 191 /* 192 * Special case for the zone reset operation that reset all 193 * zones, this is useful for applications like mkfs. 194 */ 195 if (op == REQ_OP_ZONE_RESET && 196 blkdev_allow_reset_all_zones(bdev, sector, nr_sectors)) { 197 bio->bi_opf = REQ_OP_ZONE_RESET_ALL; 198 break; 199 } 200 201 bio->bi_opf = op | REQ_SYNC; 202 bio->bi_iter.bi_sector = sector; 203 sector += zone_sectors; 204 205 /* This may take a while, so be nice to others */ 206 cond_resched(); 207 } 208 209 ret = submit_bio_wait(bio); 210 bio_put(bio); 211 212 return ret; 213 } 214 EXPORT_SYMBOL_GPL(blkdev_zone_mgmt); 215 216 struct zone_report_args { 217 struct blk_zone __user *zones; 218 }; 219 220 static int blkdev_copy_zone_to_user(struct blk_zone *zone, unsigned int idx, 221 void *data) 222 { 223 struct zone_report_args *args = data; 224 225 if (copy_to_user(&args->zones[idx], zone, sizeof(struct blk_zone))) 226 return -EFAULT; 227 return 0; 228 } 229 230 /* 231 * BLKREPORTZONE ioctl processing. 232 * Called from blkdev_ioctl. 233 */ 234 int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode, 235 unsigned int cmd, unsigned long arg) 236 { 237 void __user *argp = (void __user *)arg; 238 struct zone_report_args args; 239 struct request_queue *q; 240 struct blk_zone_report rep; 241 int ret; 242 243 if (!argp) 244 return -EINVAL; 245 246 q = bdev_get_queue(bdev); 247 if (!q) 248 return -ENXIO; 249 250 if (!blk_queue_is_zoned(q)) 251 return -ENOTTY; 252 253 if (!capable(CAP_SYS_ADMIN)) 254 return -EACCES; 255 256 if (copy_from_user(&rep, argp, sizeof(struct blk_zone_report))) 257 return -EFAULT; 258 259 if (!rep.nr_zones) 260 return -EINVAL; 261 262 args.zones = argp + sizeof(struct blk_zone_report); 263 ret = blkdev_report_zones(bdev, rep.sector, rep.nr_zones, 264 blkdev_copy_zone_to_user, &args); 265 if (ret < 0) 266 return ret; 267 268 rep.nr_zones = ret; 269 if (copy_to_user(argp, &rep, sizeof(struct blk_zone_report))) 270 return -EFAULT; 271 return 0; 272 } 273 274 /* 275 * BLKRESETZONE, BLKOPENZONE, BLKCLOSEZONE and BLKFINISHZONE ioctl processing. 276 * Called from blkdev_ioctl. 277 */ 278 int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode, 279 unsigned int cmd, unsigned long arg) 280 { 281 void __user *argp = (void __user *)arg; 282 struct request_queue *q; 283 struct blk_zone_range zrange; 284 enum req_opf op; 285 286 if (!argp) 287 return -EINVAL; 288 289 q = bdev_get_queue(bdev); 290 if (!q) 291 return -ENXIO; 292 293 if (!blk_queue_is_zoned(q)) 294 return -ENOTTY; 295 296 if (!capable(CAP_SYS_ADMIN)) 297 return -EACCES; 298 299 if (!(mode & FMODE_WRITE)) 300 return -EBADF; 301 302 if (copy_from_user(&zrange, argp, sizeof(struct blk_zone_range))) 303 return -EFAULT; 304 305 switch (cmd) { 306 case BLKRESETZONE: 307 op = REQ_OP_ZONE_RESET; 308 break; 309 case BLKOPENZONE: 310 op = REQ_OP_ZONE_OPEN; 311 break; 312 case BLKCLOSEZONE: 313 op = REQ_OP_ZONE_CLOSE; 314 break; 315 case BLKFINISHZONE: 316 op = REQ_OP_ZONE_FINISH; 317 break; 318 default: 319 return -ENOTTY; 320 } 321 322 return blkdev_zone_mgmt(bdev, op, zrange.sector, zrange.nr_sectors, 323 GFP_KERNEL); 324 } 325 326 static inline unsigned long *blk_alloc_zone_bitmap(int node, 327 unsigned int nr_zones) 328 { 329 return kcalloc_node(BITS_TO_LONGS(nr_zones), sizeof(unsigned long), 330 GFP_NOIO, node); 331 } 332 333 void blk_queue_free_zone_bitmaps(struct request_queue *q) 334 { 335 kfree(q->conv_zones_bitmap); 336 q->conv_zones_bitmap = NULL; 337 kfree(q->seq_zones_wlock); 338 q->seq_zones_wlock = NULL; 339 } 340 341 struct blk_revalidate_zone_args { 342 struct gendisk *disk; 343 unsigned long *conv_zones_bitmap; 344 unsigned long *seq_zones_wlock; 345 unsigned int nr_zones; 346 sector_t zone_sectors; 347 sector_t sector; 348 }; 349 350 /* 351 * Helper function to check the validity of zones of a zoned block device. 352 */ 353 static int blk_revalidate_zone_cb(struct blk_zone *zone, unsigned int idx, 354 void *data) 355 { 356 struct blk_revalidate_zone_args *args = data; 357 struct gendisk *disk = args->disk; 358 struct request_queue *q = disk->queue; 359 sector_t capacity = get_capacity(disk); 360 361 /* 362 * All zones must have the same size, with the exception on an eventual 363 * smaller last zone. 364 */ 365 if (zone->start == 0) { 366 if (zone->len == 0 || !is_power_of_2(zone->len)) { 367 pr_warn("%s: Invalid zoned device with non power of two zone size (%llu)\n", 368 disk->disk_name, zone->len); 369 return -ENODEV; 370 } 371 372 args->zone_sectors = zone->len; 373 args->nr_zones = (capacity + zone->len - 1) >> ilog2(zone->len); 374 } else if (zone->start + args->zone_sectors < capacity) { 375 if (zone->len != args->zone_sectors) { 376 pr_warn("%s: Invalid zoned device with non constant zone size\n", 377 disk->disk_name); 378 return -ENODEV; 379 } 380 } else { 381 if (zone->len > args->zone_sectors) { 382 pr_warn("%s: Invalid zoned device with larger last zone size\n", 383 disk->disk_name); 384 return -ENODEV; 385 } 386 } 387 388 /* Check for holes in the zone report */ 389 if (zone->start != args->sector) { 390 pr_warn("%s: Zone gap at sectors %llu..%llu\n", 391 disk->disk_name, args->sector, zone->start); 392 return -ENODEV; 393 } 394 395 /* Check zone type */ 396 switch (zone->type) { 397 case BLK_ZONE_TYPE_CONVENTIONAL: 398 if (!args->conv_zones_bitmap) { 399 args->conv_zones_bitmap = 400 blk_alloc_zone_bitmap(q->node, args->nr_zones); 401 if (!args->conv_zones_bitmap) 402 return -ENOMEM; 403 } 404 set_bit(idx, args->conv_zones_bitmap); 405 break; 406 case BLK_ZONE_TYPE_SEQWRITE_REQ: 407 case BLK_ZONE_TYPE_SEQWRITE_PREF: 408 if (!args->seq_zones_wlock) { 409 args->seq_zones_wlock = 410 blk_alloc_zone_bitmap(q->node, args->nr_zones); 411 if (!args->seq_zones_wlock) 412 return -ENOMEM; 413 } 414 break; 415 default: 416 pr_warn("%s: Invalid zone type 0x%x at sectors %llu\n", 417 disk->disk_name, (int)zone->type, zone->start); 418 return -ENODEV; 419 } 420 421 args->sector += zone->len; 422 return 0; 423 } 424 425 /** 426 * blk_revalidate_disk_zones - (re)allocate and initialize zone bitmaps 427 * @disk: Target disk 428 * 429 * Helper function for low-level device drivers to (re) allocate and initialize 430 * a disk request queue zone bitmaps. This functions should normally be called 431 * within the disk ->revalidate method for blk-mq based drivers. For BIO based 432 * drivers only q->nr_zones needs to be updated so that the sysfs exposed value 433 * is correct. 434 */ 435 int blk_revalidate_disk_zones(struct gendisk *disk) 436 { 437 struct request_queue *q = disk->queue; 438 struct blk_revalidate_zone_args args = { 439 .disk = disk, 440 }; 441 unsigned int noio_flag; 442 int ret; 443 444 if (WARN_ON_ONCE(!blk_queue_is_zoned(q))) 445 return -EIO; 446 if (WARN_ON_ONCE(!queue_is_mq(q))) 447 return -EIO; 448 449 /* 450 * Ensure that all memory allocations in this context are done as if 451 * GFP_NOIO was specified. 452 */ 453 noio_flag = memalloc_noio_save(); 454 ret = disk->fops->report_zones(disk, 0, UINT_MAX, 455 blk_revalidate_zone_cb, &args); 456 memalloc_noio_restore(noio_flag); 457 458 /* 459 * Install the new bitmaps and update nr_zones only once the queue is 460 * stopped and all I/Os are completed (i.e. a scheduler is not 461 * referencing the bitmaps). 462 */ 463 blk_mq_freeze_queue(q); 464 if (ret >= 0) { 465 blk_queue_chunk_sectors(q, args.zone_sectors); 466 q->nr_zones = args.nr_zones; 467 swap(q->seq_zones_wlock, args.seq_zones_wlock); 468 swap(q->conv_zones_bitmap, args.conv_zones_bitmap); 469 ret = 0; 470 } else { 471 pr_warn("%s: failed to revalidate zones\n", disk->disk_name); 472 blk_queue_free_zone_bitmaps(q); 473 } 474 blk_mq_unfreeze_queue(q); 475 476 kfree(args.seq_zones_wlock); 477 kfree(args.conv_zones_bitmap); 478 return ret; 479 } 480 EXPORT_SYMBOL_GPL(blk_revalidate_disk_zones); 481