1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and 4 * Shaohua Li <shli@fb.com> 5 */ 6 #include <linux/module.h> 7 8 #include <linux/moduleparam.h> 9 #include <linux/sched.h> 10 #include <linux/fs.h> 11 #include <linux/init.h> 12 #include "null_blk.h" 13 14 #define FREE_BATCH 16 15 16 #define TICKS_PER_SEC 50ULL 17 #define TIMER_INTERVAL (NSEC_PER_SEC / TICKS_PER_SEC) 18 19 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION 20 static DECLARE_FAULT_ATTR(null_timeout_attr); 21 static DECLARE_FAULT_ATTR(null_requeue_attr); 22 static DECLARE_FAULT_ATTR(null_init_hctx_attr); 23 #endif 24 25 static inline u64 mb_per_tick(int mbps) 26 { 27 return (1 << 20) / TICKS_PER_SEC * ((u64) mbps); 28 } 29 30 /* 31 * Status flags for nullb_device. 32 * 33 * CONFIGURED: Device has been configured and turned on. Cannot reconfigure. 34 * UP: Device is currently on and visible in userspace. 35 * THROTTLED: Device is being throttled. 36 * CACHE: Device is using a write-back cache. 37 */ 38 enum nullb_device_flags { 39 NULLB_DEV_FL_CONFIGURED = 0, 40 NULLB_DEV_FL_UP = 1, 41 NULLB_DEV_FL_THROTTLED = 2, 42 NULLB_DEV_FL_CACHE = 3, 43 }; 44 45 #define MAP_SZ ((PAGE_SIZE >> SECTOR_SHIFT) + 2) 46 /* 47 * nullb_page is a page in memory for nullb devices. 48 * 49 * @page: The page holding the data. 50 * @bitmap: The bitmap represents which sector in the page has data. 51 * Each bit represents one block size. For example, sector 8 52 * will use the 7th bit 53 * The highest 2 bits of bitmap are for special purpose. LOCK means the cache 54 * page is being flushing to storage. FREE means the cache page is freed and 55 * should be skipped from flushing to storage. Please see 56 * null_make_cache_space 57 */ 58 struct nullb_page { 59 struct page *page; 60 DECLARE_BITMAP(bitmap, MAP_SZ); 61 }; 62 #define NULLB_PAGE_LOCK (MAP_SZ - 1) 63 #define NULLB_PAGE_FREE (MAP_SZ - 2) 64 65 static LIST_HEAD(nullb_list); 66 static struct mutex lock; 67 static int null_major; 68 static DEFINE_IDA(nullb_indexes); 69 static struct blk_mq_tag_set tag_set; 70 71 enum { 72 NULL_IRQ_NONE = 0, 73 NULL_IRQ_SOFTIRQ = 1, 74 NULL_IRQ_TIMER = 2, 75 }; 76 77 enum { 78 NULL_Q_BIO = 0, 79 NULL_Q_RQ = 1, 80 NULL_Q_MQ = 2, 81 }; 82 83 static bool g_virt_boundary = false; 84 module_param_named(virt_boundary, g_virt_boundary, bool, 0444); 85 MODULE_PARM_DESC(virt_boundary, "Require a virtual boundary for the device. Default: False"); 86 87 static int g_no_sched; 88 module_param_named(no_sched, g_no_sched, int, 0444); 89 MODULE_PARM_DESC(no_sched, "No io scheduler"); 90 91 static int g_submit_queues = 1; 92 module_param_named(submit_queues, g_submit_queues, int, 0444); 93 MODULE_PARM_DESC(submit_queues, "Number of submission queues"); 94 95 static int g_home_node = NUMA_NO_NODE; 96 module_param_named(home_node, g_home_node, int, 0444); 97 MODULE_PARM_DESC(home_node, "Home node for the device"); 98 99 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION 100 /* 101 * For more details about fault injection, please refer to 102 * Documentation/fault-injection/fault-injection.rst. 103 */ 104 static char g_timeout_str[80]; 105 module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444); 106 MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>"); 107 108 static char g_requeue_str[80]; 109 module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444); 110 MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>"); 111 112 static char g_init_hctx_str[80]; 113 module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444); 114 MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>"); 115 #endif 116 117 static int g_queue_mode = NULL_Q_MQ; 118 119 static int null_param_store_val(const char *str, int *val, int min, int max) 120 { 121 int ret, new_val; 122 123 ret = kstrtoint(str, 10, &new_val); 124 if (ret) 125 return -EINVAL; 126 127 if (new_val < min || new_val > max) 128 return -EINVAL; 129 130 *val = new_val; 131 return 0; 132 } 133 134 static int null_set_queue_mode(const char *str, const struct kernel_param *kp) 135 { 136 return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ); 137 } 138 139 static const struct kernel_param_ops null_queue_mode_param_ops = { 140 .set = null_set_queue_mode, 141 .get = param_get_int, 142 }; 143 144 device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444); 145 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)"); 146 147 static int g_gb = 250; 148 module_param_named(gb, g_gb, int, 0444); 149 MODULE_PARM_DESC(gb, "Size in GB"); 150 151 static int g_bs = 512; 152 module_param_named(bs, g_bs, int, 0444); 153 MODULE_PARM_DESC(bs, "Block size (in bytes)"); 154 155 static int g_max_sectors; 156 module_param_named(max_sectors, g_max_sectors, int, 0444); 157 MODULE_PARM_DESC(max_sectors, "Maximum size of a command (in 512B sectors)"); 158 159 static unsigned int nr_devices = 1; 160 module_param(nr_devices, uint, 0444); 161 MODULE_PARM_DESC(nr_devices, "Number of devices to register"); 162 163 static bool g_blocking; 164 module_param_named(blocking, g_blocking, bool, 0444); 165 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device"); 166 167 static bool shared_tags; 168 module_param(shared_tags, bool, 0444); 169 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq"); 170 171 static bool g_shared_tag_bitmap; 172 module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444); 173 MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq"); 174 175 static int g_irqmode = NULL_IRQ_SOFTIRQ; 176 177 static int null_set_irqmode(const char *str, const struct kernel_param *kp) 178 { 179 return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE, 180 NULL_IRQ_TIMER); 181 } 182 183 static const struct kernel_param_ops null_irqmode_param_ops = { 184 .set = null_set_irqmode, 185 .get = param_get_int, 186 }; 187 188 device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444); 189 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer"); 190 191 static unsigned long g_completion_nsec = 10000; 192 module_param_named(completion_nsec, g_completion_nsec, ulong, 0444); 193 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns"); 194 195 static int g_hw_queue_depth = 64; 196 module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444); 197 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64"); 198 199 static bool g_use_per_node_hctx; 200 module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444); 201 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false"); 202 203 static bool g_zoned; 204 module_param_named(zoned, g_zoned, bool, S_IRUGO); 205 MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false"); 206 207 static unsigned long g_zone_size = 256; 208 module_param_named(zone_size, g_zone_size, ulong, S_IRUGO); 209 MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256"); 210 211 static unsigned long g_zone_capacity; 212 module_param_named(zone_capacity, g_zone_capacity, ulong, 0444); 213 MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size"); 214 215 static unsigned int g_zone_nr_conv; 216 module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444); 217 MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0"); 218 219 static unsigned int g_zone_max_open; 220 module_param_named(zone_max_open, g_zone_max_open, uint, 0444); 221 MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)"); 222 223 static unsigned int g_zone_max_active; 224 module_param_named(zone_max_active, g_zone_max_active, uint, 0444); 225 MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)"); 226 227 static struct nullb_device *null_alloc_dev(void); 228 static void null_free_dev(struct nullb_device *dev); 229 static void null_del_dev(struct nullb *nullb); 230 static int null_add_dev(struct nullb_device *dev); 231 static void null_free_device_storage(struct nullb_device *dev, bool is_cache); 232 233 static inline struct nullb_device *to_nullb_device(struct config_item *item) 234 { 235 return item ? container_of(item, struct nullb_device, item) : NULL; 236 } 237 238 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page) 239 { 240 return snprintf(page, PAGE_SIZE, "%u\n", val); 241 } 242 243 static inline ssize_t nullb_device_ulong_attr_show(unsigned long val, 244 char *page) 245 { 246 return snprintf(page, PAGE_SIZE, "%lu\n", val); 247 } 248 249 static inline ssize_t nullb_device_bool_attr_show(bool val, char *page) 250 { 251 return snprintf(page, PAGE_SIZE, "%u\n", val); 252 } 253 254 static ssize_t nullb_device_uint_attr_store(unsigned int *val, 255 const char *page, size_t count) 256 { 257 unsigned int tmp; 258 int result; 259 260 result = kstrtouint(page, 0, &tmp); 261 if (result < 0) 262 return result; 263 264 *val = tmp; 265 return count; 266 } 267 268 static ssize_t nullb_device_ulong_attr_store(unsigned long *val, 269 const char *page, size_t count) 270 { 271 int result; 272 unsigned long tmp; 273 274 result = kstrtoul(page, 0, &tmp); 275 if (result < 0) 276 return result; 277 278 *val = tmp; 279 return count; 280 } 281 282 static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, 283 size_t count) 284 { 285 bool tmp; 286 int result; 287 288 result = kstrtobool(page, &tmp); 289 if (result < 0) 290 return result; 291 292 *val = tmp; 293 return count; 294 } 295 296 /* The following macro should only be used with TYPE = {uint, ulong, bool}. */ 297 #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \ 298 static ssize_t \ 299 nullb_device_##NAME##_show(struct config_item *item, char *page) \ 300 { \ 301 return nullb_device_##TYPE##_attr_show( \ 302 to_nullb_device(item)->NAME, page); \ 303 } \ 304 static ssize_t \ 305 nullb_device_##NAME##_store(struct config_item *item, const char *page, \ 306 size_t count) \ 307 { \ 308 int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\ 309 struct nullb_device *dev = to_nullb_device(item); \ 310 TYPE new_value = 0; \ 311 int ret; \ 312 \ 313 ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\ 314 if (ret < 0) \ 315 return ret; \ 316 if (apply_fn) \ 317 ret = apply_fn(dev, new_value); \ 318 else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \ 319 ret = -EBUSY; \ 320 if (ret < 0) \ 321 return ret; \ 322 dev->NAME = new_value; \ 323 return count; \ 324 } \ 325 CONFIGFS_ATTR(nullb_device_, NAME); 326 327 static int nullb_apply_submit_queues(struct nullb_device *dev, 328 unsigned int submit_queues) 329 { 330 struct nullb *nullb = dev->nullb; 331 struct blk_mq_tag_set *set; 332 333 if (!nullb) 334 return 0; 335 336 /* 337 * Make sure that null_init_hctx() does not access nullb->queues[] past 338 * the end of that array. 339 */ 340 if (submit_queues > nr_cpu_ids) 341 return -EINVAL; 342 set = nullb->tag_set; 343 blk_mq_update_nr_hw_queues(set, submit_queues); 344 return set->nr_hw_queues == submit_queues ? 0 : -ENOMEM; 345 } 346 347 NULLB_DEVICE_ATTR(size, ulong, NULL); 348 NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL); 349 NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues); 350 NULLB_DEVICE_ATTR(home_node, uint, NULL); 351 NULLB_DEVICE_ATTR(queue_mode, uint, NULL); 352 NULLB_DEVICE_ATTR(blocksize, uint, NULL); 353 NULLB_DEVICE_ATTR(max_sectors, uint, NULL); 354 NULLB_DEVICE_ATTR(irqmode, uint, NULL); 355 NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL); 356 NULLB_DEVICE_ATTR(index, uint, NULL); 357 NULLB_DEVICE_ATTR(blocking, bool, NULL); 358 NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL); 359 NULLB_DEVICE_ATTR(memory_backed, bool, NULL); 360 NULLB_DEVICE_ATTR(discard, bool, NULL); 361 NULLB_DEVICE_ATTR(mbps, uint, NULL); 362 NULLB_DEVICE_ATTR(cache_size, ulong, NULL); 363 NULLB_DEVICE_ATTR(zoned, bool, NULL); 364 NULLB_DEVICE_ATTR(zone_size, ulong, NULL); 365 NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL); 366 NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL); 367 NULLB_DEVICE_ATTR(zone_max_open, uint, NULL); 368 NULLB_DEVICE_ATTR(zone_max_active, uint, NULL); 369 NULLB_DEVICE_ATTR(virt_boundary, bool, NULL); 370 371 static ssize_t nullb_device_power_show(struct config_item *item, char *page) 372 { 373 return nullb_device_bool_attr_show(to_nullb_device(item)->power, page); 374 } 375 376 static ssize_t nullb_device_power_store(struct config_item *item, 377 const char *page, size_t count) 378 { 379 struct nullb_device *dev = to_nullb_device(item); 380 bool newp = false; 381 ssize_t ret; 382 383 ret = nullb_device_bool_attr_store(&newp, page, count); 384 if (ret < 0) 385 return ret; 386 387 if (!dev->power && newp) { 388 if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags)) 389 return count; 390 if (null_add_dev(dev)) { 391 clear_bit(NULLB_DEV_FL_UP, &dev->flags); 392 return -ENOMEM; 393 } 394 395 set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags); 396 dev->power = newp; 397 } else if (dev->power && !newp) { 398 if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) { 399 mutex_lock(&lock); 400 dev->power = newp; 401 null_del_dev(dev->nullb); 402 mutex_unlock(&lock); 403 } 404 clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags); 405 } 406 407 return count; 408 } 409 410 CONFIGFS_ATTR(nullb_device_, power); 411 412 static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page) 413 { 414 struct nullb_device *t_dev = to_nullb_device(item); 415 416 return badblocks_show(&t_dev->badblocks, page, 0); 417 } 418 419 static ssize_t nullb_device_badblocks_store(struct config_item *item, 420 const char *page, size_t count) 421 { 422 struct nullb_device *t_dev = to_nullb_device(item); 423 char *orig, *buf, *tmp; 424 u64 start, end; 425 int ret; 426 427 orig = kstrndup(page, count, GFP_KERNEL); 428 if (!orig) 429 return -ENOMEM; 430 431 buf = strstrip(orig); 432 433 ret = -EINVAL; 434 if (buf[0] != '+' && buf[0] != '-') 435 goto out; 436 tmp = strchr(&buf[1], '-'); 437 if (!tmp) 438 goto out; 439 *tmp = '\0'; 440 ret = kstrtoull(buf + 1, 0, &start); 441 if (ret) 442 goto out; 443 ret = kstrtoull(tmp + 1, 0, &end); 444 if (ret) 445 goto out; 446 ret = -EINVAL; 447 if (start > end) 448 goto out; 449 /* enable badblocks */ 450 cmpxchg(&t_dev->badblocks.shift, -1, 0); 451 if (buf[0] == '+') 452 ret = badblocks_set(&t_dev->badblocks, start, 453 end - start + 1, 1); 454 else 455 ret = badblocks_clear(&t_dev->badblocks, start, 456 end - start + 1); 457 if (ret == 0) 458 ret = count; 459 out: 460 kfree(orig); 461 return ret; 462 } 463 CONFIGFS_ATTR(nullb_device_, badblocks); 464 465 static struct configfs_attribute *nullb_device_attrs[] = { 466 &nullb_device_attr_size, 467 &nullb_device_attr_completion_nsec, 468 &nullb_device_attr_submit_queues, 469 &nullb_device_attr_home_node, 470 &nullb_device_attr_queue_mode, 471 &nullb_device_attr_blocksize, 472 &nullb_device_attr_max_sectors, 473 &nullb_device_attr_irqmode, 474 &nullb_device_attr_hw_queue_depth, 475 &nullb_device_attr_index, 476 &nullb_device_attr_blocking, 477 &nullb_device_attr_use_per_node_hctx, 478 &nullb_device_attr_power, 479 &nullb_device_attr_memory_backed, 480 &nullb_device_attr_discard, 481 &nullb_device_attr_mbps, 482 &nullb_device_attr_cache_size, 483 &nullb_device_attr_badblocks, 484 &nullb_device_attr_zoned, 485 &nullb_device_attr_zone_size, 486 &nullb_device_attr_zone_capacity, 487 &nullb_device_attr_zone_nr_conv, 488 &nullb_device_attr_zone_max_open, 489 &nullb_device_attr_zone_max_active, 490 &nullb_device_attr_virt_boundary, 491 NULL, 492 }; 493 494 static void nullb_device_release(struct config_item *item) 495 { 496 struct nullb_device *dev = to_nullb_device(item); 497 498 null_free_device_storage(dev, false); 499 null_free_dev(dev); 500 } 501 502 static struct configfs_item_operations nullb_device_ops = { 503 .release = nullb_device_release, 504 }; 505 506 static const struct config_item_type nullb_device_type = { 507 .ct_item_ops = &nullb_device_ops, 508 .ct_attrs = nullb_device_attrs, 509 .ct_owner = THIS_MODULE, 510 }; 511 512 static struct 513 config_item *nullb_group_make_item(struct config_group *group, const char *name) 514 { 515 struct nullb_device *dev; 516 517 dev = null_alloc_dev(); 518 if (!dev) 519 return ERR_PTR(-ENOMEM); 520 521 config_item_init_type_name(&dev->item, name, &nullb_device_type); 522 523 return &dev->item; 524 } 525 526 static void 527 nullb_group_drop_item(struct config_group *group, struct config_item *item) 528 { 529 struct nullb_device *dev = to_nullb_device(item); 530 531 if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) { 532 mutex_lock(&lock); 533 dev->power = false; 534 null_del_dev(dev->nullb); 535 mutex_unlock(&lock); 536 } 537 538 config_item_put(item); 539 } 540 541 static ssize_t memb_group_features_show(struct config_item *item, char *page) 542 { 543 return snprintf(page, PAGE_SIZE, 544 "memory_backed,discard,bandwidth,cache,badblocks,zoned,zone_size,zone_capacity,zone_nr_conv,zone_max_open,zone_max_active,blocksize,max_sectors,virt_boundary\n"); 545 } 546 547 CONFIGFS_ATTR_RO(memb_group_, features); 548 549 static struct configfs_attribute *nullb_group_attrs[] = { 550 &memb_group_attr_features, 551 NULL, 552 }; 553 554 static struct configfs_group_operations nullb_group_ops = { 555 .make_item = nullb_group_make_item, 556 .drop_item = nullb_group_drop_item, 557 }; 558 559 static const struct config_item_type nullb_group_type = { 560 .ct_group_ops = &nullb_group_ops, 561 .ct_attrs = nullb_group_attrs, 562 .ct_owner = THIS_MODULE, 563 }; 564 565 static struct configfs_subsystem nullb_subsys = { 566 .su_group = { 567 .cg_item = { 568 .ci_namebuf = "nullb", 569 .ci_type = &nullb_group_type, 570 }, 571 }, 572 }; 573 574 static inline int null_cache_active(struct nullb *nullb) 575 { 576 return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags); 577 } 578 579 static struct nullb_device *null_alloc_dev(void) 580 { 581 struct nullb_device *dev; 582 583 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 584 if (!dev) 585 return NULL; 586 INIT_RADIX_TREE(&dev->data, GFP_ATOMIC); 587 INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC); 588 if (badblocks_init(&dev->badblocks, 0)) { 589 kfree(dev); 590 return NULL; 591 } 592 593 dev->size = g_gb * 1024; 594 dev->completion_nsec = g_completion_nsec; 595 dev->submit_queues = g_submit_queues; 596 dev->home_node = g_home_node; 597 dev->queue_mode = g_queue_mode; 598 dev->blocksize = g_bs; 599 dev->max_sectors = g_max_sectors; 600 dev->irqmode = g_irqmode; 601 dev->hw_queue_depth = g_hw_queue_depth; 602 dev->blocking = g_blocking; 603 dev->use_per_node_hctx = g_use_per_node_hctx; 604 dev->zoned = g_zoned; 605 dev->zone_size = g_zone_size; 606 dev->zone_capacity = g_zone_capacity; 607 dev->zone_nr_conv = g_zone_nr_conv; 608 dev->zone_max_open = g_zone_max_open; 609 dev->zone_max_active = g_zone_max_active; 610 dev->virt_boundary = g_virt_boundary; 611 return dev; 612 } 613 614 static void null_free_dev(struct nullb_device *dev) 615 { 616 if (!dev) 617 return; 618 619 null_free_zoned_dev(dev); 620 badblocks_exit(&dev->badblocks); 621 kfree(dev); 622 } 623 624 static void put_tag(struct nullb_queue *nq, unsigned int tag) 625 { 626 clear_bit_unlock(tag, nq->tag_map); 627 628 if (waitqueue_active(&nq->wait)) 629 wake_up(&nq->wait); 630 } 631 632 static unsigned int get_tag(struct nullb_queue *nq) 633 { 634 unsigned int tag; 635 636 do { 637 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth); 638 if (tag >= nq->queue_depth) 639 return -1U; 640 } while (test_and_set_bit_lock(tag, nq->tag_map)); 641 642 return tag; 643 } 644 645 static void free_cmd(struct nullb_cmd *cmd) 646 { 647 put_tag(cmd->nq, cmd->tag); 648 } 649 650 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer); 651 652 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq) 653 { 654 struct nullb_cmd *cmd; 655 unsigned int tag; 656 657 tag = get_tag(nq); 658 if (tag != -1U) { 659 cmd = &nq->cmds[tag]; 660 cmd->tag = tag; 661 cmd->error = BLK_STS_OK; 662 cmd->nq = nq; 663 if (nq->dev->irqmode == NULL_IRQ_TIMER) { 664 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, 665 HRTIMER_MODE_REL); 666 cmd->timer.function = null_cmd_timer_expired; 667 } 668 return cmd; 669 } 670 671 return NULL; 672 } 673 674 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait) 675 { 676 struct nullb_cmd *cmd; 677 DEFINE_WAIT(wait); 678 679 cmd = __alloc_cmd(nq); 680 if (cmd || !can_wait) 681 return cmd; 682 683 do { 684 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE); 685 cmd = __alloc_cmd(nq); 686 if (cmd) 687 break; 688 689 io_schedule(); 690 } while (1); 691 692 finish_wait(&nq->wait, &wait); 693 return cmd; 694 } 695 696 static void end_cmd(struct nullb_cmd *cmd) 697 { 698 int queue_mode = cmd->nq->dev->queue_mode; 699 700 switch (queue_mode) { 701 case NULL_Q_MQ: 702 blk_mq_end_request(cmd->rq, cmd->error); 703 return; 704 case NULL_Q_BIO: 705 cmd->bio->bi_status = cmd->error; 706 bio_endio(cmd->bio); 707 break; 708 } 709 710 free_cmd(cmd); 711 } 712 713 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer) 714 { 715 end_cmd(container_of(timer, struct nullb_cmd, timer)); 716 717 return HRTIMER_NORESTART; 718 } 719 720 static void null_cmd_end_timer(struct nullb_cmd *cmd) 721 { 722 ktime_t kt = cmd->nq->dev->completion_nsec; 723 724 hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL); 725 } 726 727 static void null_complete_rq(struct request *rq) 728 { 729 end_cmd(blk_mq_rq_to_pdu(rq)); 730 } 731 732 static struct nullb_page *null_alloc_page(gfp_t gfp_flags) 733 { 734 struct nullb_page *t_page; 735 736 t_page = kmalloc(sizeof(struct nullb_page), gfp_flags); 737 if (!t_page) 738 goto out; 739 740 t_page->page = alloc_pages(gfp_flags, 0); 741 if (!t_page->page) 742 goto out_freepage; 743 744 memset(t_page->bitmap, 0, sizeof(t_page->bitmap)); 745 return t_page; 746 out_freepage: 747 kfree(t_page); 748 out: 749 return NULL; 750 } 751 752 static void null_free_page(struct nullb_page *t_page) 753 { 754 __set_bit(NULLB_PAGE_FREE, t_page->bitmap); 755 if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap)) 756 return; 757 __free_page(t_page->page); 758 kfree(t_page); 759 } 760 761 static bool null_page_empty(struct nullb_page *page) 762 { 763 int size = MAP_SZ - 2; 764 765 return find_first_bit(page->bitmap, size) == size; 766 } 767 768 static void null_free_sector(struct nullb *nullb, sector_t sector, 769 bool is_cache) 770 { 771 unsigned int sector_bit; 772 u64 idx; 773 struct nullb_page *t_page, *ret; 774 struct radix_tree_root *root; 775 776 root = is_cache ? &nullb->dev->cache : &nullb->dev->data; 777 idx = sector >> PAGE_SECTORS_SHIFT; 778 sector_bit = (sector & SECTOR_MASK); 779 780 t_page = radix_tree_lookup(root, idx); 781 if (t_page) { 782 __clear_bit(sector_bit, t_page->bitmap); 783 784 if (null_page_empty(t_page)) { 785 ret = radix_tree_delete_item(root, idx, t_page); 786 WARN_ON(ret != t_page); 787 null_free_page(ret); 788 if (is_cache) 789 nullb->dev->curr_cache -= PAGE_SIZE; 790 } 791 } 792 } 793 794 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx, 795 struct nullb_page *t_page, bool is_cache) 796 { 797 struct radix_tree_root *root; 798 799 root = is_cache ? &nullb->dev->cache : &nullb->dev->data; 800 801 if (radix_tree_insert(root, idx, t_page)) { 802 null_free_page(t_page); 803 t_page = radix_tree_lookup(root, idx); 804 WARN_ON(!t_page || t_page->page->index != idx); 805 } else if (is_cache) 806 nullb->dev->curr_cache += PAGE_SIZE; 807 808 return t_page; 809 } 810 811 static void null_free_device_storage(struct nullb_device *dev, bool is_cache) 812 { 813 unsigned long pos = 0; 814 int nr_pages; 815 struct nullb_page *ret, *t_pages[FREE_BATCH]; 816 struct radix_tree_root *root; 817 818 root = is_cache ? &dev->cache : &dev->data; 819 820 do { 821 int i; 822 823 nr_pages = radix_tree_gang_lookup(root, 824 (void **)t_pages, pos, FREE_BATCH); 825 826 for (i = 0; i < nr_pages; i++) { 827 pos = t_pages[i]->page->index; 828 ret = radix_tree_delete_item(root, pos, t_pages[i]); 829 WARN_ON(ret != t_pages[i]); 830 null_free_page(ret); 831 } 832 833 pos++; 834 } while (nr_pages == FREE_BATCH); 835 836 if (is_cache) 837 dev->curr_cache = 0; 838 } 839 840 static struct nullb_page *__null_lookup_page(struct nullb *nullb, 841 sector_t sector, bool for_write, bool is_cache) 842 { 843 unsigned int sector_bit; 844 u64 idx; 845 struct nullb_page *t_page; 846 struct radix_tree_root *root; 847 848 idx = sector >> PAGE_SECTORS_SHIFT; 849 sector_bit = (sector & SECTOR_MASK); 850 851 root = is_cache ? &nullb->dev->cache : &nullb->dev->data; 852 t_page = radix_tree_lookup(root, idx); 853 WARN_ON(t_page && t_page->page->index != idx); 854 855 if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap))) 856 return t_page; 857 858 return NULL; 859 } 860 861 static struct nullb_page *null_lookup_page(struct nullb *nullb, 862 sector_t sector, bool for_write, bool ignore_cache) 863 { 864 struct nullb_page *page = NULL; 865 866 if (!ignore_cache) 867 page = __null_lookup_page(nullb, sector, for_write, true); 868 if (page) 869 return page; 870 return __null_lookup_page(nullb, sector, for_write, false); 871 } 872 873 static struct nullb_page *null_insert_page(struct nullb *nullb, 874 sector_t sector, bool ignore_cache) 875 __releases(&nullb->lock) 876 __acquires(&nullb->lock) 877 { 878 u64 idx; 879 struct nullb_page *t_page; 880 881 t_page = null_lookup_page(nullb, sector, true, ignore_cache); 882 if (t_page) 883 return t_page; 884 885 spin_unlock_irq(&nullb->lock); 886 887 t_page = null_alloc_page(GFP_NOIO); 888 if (!t_page) 889 goto out_lock; 890 891 if (radix_tree_preload(GFP_NOIO)) 892 goto out_freepage; 893 894 spin_lock_irq(&nullb->lock); 895 idx = sector >> PAGE_SECTORS_SHIFT; 896 t_page->page->index = idx; 897 t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache); 898 radix_tree_preload_end(); 899 900 return t_page; 901 out_freepage: 902 null_free_page(t_page); 903 out_lock: 904 spin_lock_irq(&nullb->lock); 905 return null_lookup_page(nullb, sector, true, ignore_cache); 906 } 907 908 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page) 909 { 910 int i; 911 unsigned int offset; 912 u64 idx; 913 struct nullb_page *t_page, *ret; 914 void *dst, *src; 915 916 idx = c_page->page->index; 917 918 t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true); 919 920 __clear_bit(NULLB_PAGE_LOCK, c_page->bitmap); 921 if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) { 922 null_free_page(c_page); 923 if (t_page && null_page_empty(t_page)) { 924 ret = radix_tree_delete_item(&nullb->dev->data, 925 idx, t_page); 926 null_free_page(t_page); 927 } 928 return 0; 929 } 930 931 if (!t_page) 932 return -ENOMEM; 933 934 src = kmap_atomic(c_page->page); 935 dst = kmap_atomic(t_page->page); 936 937 for (i = 0; i < PAGE_SECTORS; 938 i += (nullb->dev->blocksize >> SECTOR_SHIFT)) { 939 if (test_bit(i, c_page->bitmap)) { 940 offset = (i << SECTOR_SHIFT); 941 memcpy(dst + offset, src + offset, 942 nullb->dev->blocksize); 943 __set_bit(i, t_page->bitmap); 944 } 945 } 946 947 kunmap_atomic(dst); 948 kunmap_atomic(src); 949 950 ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page); 951 null_free_page(ret); 952 nullb->dev->curr_cache -= PAGE_SIZE; 953 954 return 0; 955 } 956 957 static int null_make_cache_space(struct nullb *nullb, unsigned long n) 958 { 959 int i, err, nr_pages; 960 struct nullb_page *c_pages[FREE_BATCH]; 961 unsigned long flushed = 0, one_round; 962 963 again: 964 if ((nullb->dev->cache_size * 1024 * 1024) > 965 nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0) 966 return 0; 967 968 nr_pages = radix_tree_gang_lookup(&nullb->dev->cache, 969 (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH); 970 /* 971 * nullb_flush_cache_page could unlock before using the c_pages. To 972 * avoid race, we don't allow page free 973 */ 974 for (i = 0; i < nr_pages; i++) { 975 nullb->cache_flush_pos = c_pages[i]->page->index; 976 /* 977 * We found the page which is being flushed to disk by other 978 * threads 979 */ 980 if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap)) 981 c_pages[i] = NULL; 982 else 983 __set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap); 984 } 985 986 one_round = 0; 987 for (i = 0; i < nr_pages; i++) { 988 if (c_pages[i] == NULL) 989 continue; 990 err = null_flush_cache_page(nullb, c_pages[i]); 991 if (err) 992 return err; 993 one_round++; 994 } 995 flushed += one_round << PAGE_SHIFT; 996 997 if (n > flushed) { 998 if (nr_pages == 0) 999 nullb->cache_flush_pos = 0; 1000 if (one_round == 0) { 1001 /* give other threads a chance */ 1002 spin_unlock_irq(&nullb->lock); 1003 spin_lock_irq(&nullb->lock); 1004 } 1005 goto again; 1006 } 1007 return 0; 1008 } 1009 1010 static int copy_to_nullb(struct nullb *nullb, struct page *source, 1011 unsigned int off, sector_t sector, size_t n, bool is_fua) 1012 { 1013 size_t temp, count = 0; 1014 unsigned int offset; 1015 struct nullb_page *t_page; 1016 void *dst, *src; 1017 1018 while (count < n) { 1019 temp = min_t(size_t, nullb->dev->blocksize, n - count); 1020 1021 if (null_cache_active(nullb) && !is_fua) 1022 null_make_cache_space(nullb, PAGE_SIZE); 1023 1024 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT; 1025 t_page = null_insert_page(nullb, sector, 1026 !null_cache_active(nullb) || is_fua); 1027 if (!t_page) 1028 return -ENOSPC; 1029 1030 src = kmap_atomic(source); 1031 dst = kmap_atomic(t_page->page); 1032 memcpy(dst + offset, src + off + count, temp); 1033 kunmap_atomic(dst); 1034 kunmap_atomic(src); 1035 1036 __set_bit(sector & SECTOR_MASK, t_page->bitmap); 1037 1038 if (is_fua) 1039 null_free_sector(nullb, sector, true); 1040 1041 count += temp; 1042 sector += temp >> SECTOR_SHIFT; 1043 } 1044 return 0; 1045 } 1046 1047 static int copy_from_nullb(struct nullb *nullb, struct page *dest, 1048 unsigned int off, sector_t sector, size_t n) 1049 { 1050 size_t temp, count = 0; 1051 unsigned int offset; 1052 struct nullb_page *t_page; 1053 void *dst, *src; 1054 1055 while (count < n) { 1056 temp = min_t(size_t, nullb->dev->blocksize, n - count); 1057 1058 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT; 1059 t_page = null_lookup_page(nullb, sector, false, 1060 !null_cache_active(nullb)); 1061 1062 dst = kmap_atomic(dest); 1063 if (!t_page) { 1064 memset(dst + off + count, 0, temp); 1065 goto next; 1066 } 1067 src = kmap_atomic(t_page->page); 1068 memcpy(dst + off + count, src + offset, temp); 1069 kunmap_atomic(src); 1070 next: 1071 kunmap_atomic(dst); 1072 1073 count += temp; 1074 sector += temp >> SECTOR_SHIFT; 1075 } 1076 return 0; 1077 } 1078 1079 static void nullb_fill_pattern(struct nullb *nullb, struct page *page, 1080 unsigned int len, unsigned int off) 1081 { 1082 void *dst; 1083 1084 dst = kmap_atomic(page); 1085 memset(dst + off, 0xFF, len); 1086 kunmap_atomic(dst); 1087 } 1088 1089 blk_status_t null_handle_discard(struct nullb_device *dev, 1090 sector_t sector, sector_t nr_sectors) 1091 { 1092 struct nullb *nullb = dev->nullb; 1093 size_t n = nr_sectors << SECTOR_SHIFT; 1094 size_t temp; 1095 1096 spin_lock_irq(&nullb->lock); 1097 while (n > 0) { 1098 temp = min_t(size_t, n, dev->blocksize); 1099 null_free_sector(nullb, sector, false); 1100 if (null_cache_active(nullb)) 1101 null_free_sector(nullb, sector, true); 1102 sector += temp >> SECTOR_SHIFT; 1103 n -= temp; 1104 } 1105 spin_unlock_irq(&nullb->lock); 1106 1107 return BLK_STS_OK; 1108 } 1109 1110 static int null_handle_flush(struct nullb *nullb) 1111 { 1112 int err; 1113 1114 if (!null_cache_active(nullb)) 1115 return 0; 1116 1117 spin_lock_irq(&nullb->lock); 1118 while (true) { 1119 err = null_make_cache_space(nullb, 1120 nullb->dev->cache_size * 1024 * 1024); 1121 if (err || nullb->dev->curr_cache == 0) 1122 break; 1123 } 1124 1125 WARN_ON(!radix_tree_empty(&nullb->dev->cache)); 1126 spin_unlock_irq(&nullb->lock); 1127 return err; 1128 } 1129 1130 static int null_transfer(struct nullb *nullb, struct page *page, 1131 unsigned int len, unsigned int off, bool is_write, sector_t sector, 1132 bool is_fua) 1133 { 1134 struct nullb_device *dev = nullb->dev; 1135 unsigned int valid_len = len; 1136 int err = 0; 1137 1138 if (!is_write) { 1139 if (dev->zoned) 1140 valid_len = null_zone_valid_read_len(nullb, 1141 sector, len); 1142 1143 if (valid_len) { 1144 err = copy_from_nullb(nullb, page, off, 1145 sector, valid_len); 1146 off += valid_len; 1147 len -= valid_len; 1148 } 1149 1150 if (len) 1151 nullb_fill_pattern(nullb, page, len, off); 1152 flush_dcache_page(page); 1153 } else { 1154 flush_dcache_page(page); 1155 err = copy_to_nullb(nullb, page, off, sector, len, is_fua); 1156 } 1157 1158 return err; 1159 } 1160 1161 static int null_handle_rq(struct nullb_cmd *cmd) 1162 { 1163 struct request *rq = cmd->rq; 1164 struct nullb *nullb = cmd->nq->dev->nullb; 1165 int err; 1166 unsigned int len; 1167 sector_t sector = blk_rq_pos(rq); 1168 struct req_iterator iter; 1169 struct bio_vec bvec; 1170 1171 spin_lock_irq(&nullb->lock); 1172 rq_for_each_segment(bvec, rq, iter) { 1173 len = bvec.bv_len; 1174 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset, 1175 op_is_write(req_op(rq)), sector, 1176 rq->cmd_flags & REQ_FUA); 1177 if (err) { 1178 spin_unlock_irq(&nullb->lock); 1179 return err; 1180 } 1181 sector += len >> SECTOR_SHIFT; 1182 } 1183 spin_unlock_irq(&nullb->lock); 1184 1185 return 0; 1186 } 1187 1188 static int null_handle_bio(struct nullb_cmd *cmd) 1189 { 1190 struct bio *bio = cmd->bio; 1191 struct nullb *nullb = cmd->nq->dev->nullb; 1192 int err; 1193 unsigned int len; 1194 sector_t sector = bio->bi_iter.bi_sector; 1195 struct bio_vec bvec; 1196 struct bvec_iter iter; 1197 1198 spin_lock_irq(&nullb->lock); 1199 bio_for_each_segment(bvec, bio, iter) { 1200 len = bvec.bv_len; 1201 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset, 1202 op_is_write(bio_op(bio)), sector, 1203 bio->bi_opf & REQ_FUA); 1204 if (err) { 1205 spin_unlock_irq(&nullb->lock); 1206 return err; 1207 } 1208 sector += len >> SECTOR_SHIFT; 1209 } 1210 spin_unlock_irq(&nullb->lock); 1211 return 0; 1212 } 1213 1214 static void null_stop_queue(struct nullb *nullb) 1215 { 1216 struct request_queue *q = nullb->q; 1217 1218 if (nullb->dev->queue_mode == NULL_Q_MQ) 1219 blk_mq_stop_hw_queues(q); 1220 } 1221 1222 static void null_restart_queue_async(struct nullb *nullb) 1223 { 1224 struct request_queue *q = nullb->q; 1225 1226 if (nullb->dev->queue_mode == NULL_Q_MQ) 1227 blk_mq_start_stopped_hw_queues(q, true); 1228 } 1229 1230 static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd) 1231 { 1232 struct nullb_device *dev = cmd->nq->dev; 1233 struct nullb *nullb = dev->nullb; 1234 blk_status_t sts = BLK_STS_OK; 1235 struct request *rq = cmd->rq; 1236 1237 if (!hrtimer_active(&nullb->bw_timer)) 1238 hrtimer_restart(&nullb->bw_timer); 1239 1240 if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) { 1241 null_stop_queue(nullb); 1242 /* race with timer */ 1243 if (atomic_long_read(&nullb->cur_bytes) > 0) 1244 null_restart_queue_async(nullb); 1245 /* requeue request */ 1246 sts = BLK_STS_DEV_RESOURCE; 1247 } 1248 return sts; 1249 } 1250 1251 static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd, 1252 sector_t sector, 1253 sector_t nr_sectors) 1254 { 1255 struct badblocks *bb = &cmd->nq->dev->badblocks; 1256 sector_t first_bad; 1257 int bad_sectors; 1258 1259 if (badblocks_check(bb, sector, nr_sectors, &first_bad, &bad_sectors)) 1260 return BLK_STS_IOERR; 1261 1262 return BLK_STS_OK; 1263 } 1264 1265 static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd, 1266 enum req_opf op, 1267 sector_t sector, 1268 sector_t nr_sectors) 1269 { 1270 struct nullb_device *dev = cmd->nq->dev; 1271 int err; 1272 1273 if (op == REQ_OP_DISCARD) 1274 return null_handle_discard(dev, sector, nr_sectors); 1275 1276 if (dev->queue_mode == NULL_Q_BIO) 1277 err = null_handle_bio(cmd); 1278 else 1279 err = null_handle_rq(cmd); 1280 1281 return errno_to_blk_status(err); 1282 } 1283 1284 static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd) 1285 { 1286 struct nullb_device *dev = cmd->nq->dev; 1287 struct bio *bio; 1288 1289 if (dev->memory_backed) 1290 return; 1291 1292 if (dev->queue_mode == NULL_Q_BIO && bio_op(cmd->bio) == REQ_OP_READ) { 1293 zero_fill_bio(cmd->bio); 1294 } else if (req_op(cmd->rq) == REQ_OP_READ) { 1295 __rq_for_each_bio(bio, cmd->rq) 1296 zero_fill_bio(bio); 1297 } 1298 } 1299 1300 static inline void nullb_complete_cmd(struct nullb_cmd *cmd) 1301 { 1302 /* 1303 * Since root privileges are required to configure the null_blk 1304 * driver, it is fine that this driver does not initialize the 1305 * data buffers of read commands. Zero-initialize these buffers 1306 * anyway if KMSAN is enabled to prevent that KMSAN complains 1307 * about null_blk not initializing read data buffers. 1308 */ 1309 if (IS_ENABLED(CONFIG_KMSAN)) 1310 nullb_zero_read_cmd_buffer(cmd); 1311 1312 /* Complete IO by inline, softirq or timer */ 1313 switch (cmd->nq->dev->irqmode) { 1314 case NULL_IRQ_SOFTIRQ: 1315 switch (cmd->nq->dev->queue_mode) { 1316 case NULL_Q_MQ: 1317 if (likely(!blk_should_fake_timeout(cmd->rq->q))) 1318 blk_mq_complete_request(cmd->rq); 1319 break; 1320 case NULL_Q_BIO: 1321 /* 1322 * XXX: no proper submitting cpu information available. 1323 */ 1324 end_cmd(cmd); 1325 break; 1326 } 1327 break; 1328 case NULL_IRQ_NONE: 1329 end_cmd(cmd); 1330 break; 1331 case NULL_IRQ_TIMER: 1332 null_cmd_end_timer(cmd); 1333 break; 1334 } 1335 } 1336 1337 blk_status_t null_process_cmd(struct nullb_cmd *cmd, 1338 enum req_opf op, sector_t sector, 1339 unsigned int nr_sectors) 1340 { 1341 struct nullb_device *dev = cmd->nq->dev; 1342 blk_status_t ret; 1343 1344 if (dev->badblocks.shift != -1) { 1345 ret = null_handle_badblocks(cmd, sector, nr_sectors); 1346 if (ret != BLK_STS_OK) 1347 return ret; 1348 } 1349 1350 if (dev->memory_backed) 1351 return null_handle_memory_backed(cmd, op, sector, nr_sectors); 1352 1353 return BLK_STS_OK; 1354 } 1355 1356 static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector, 1357 sector_t nr_sectors, enum req_opf op) 1358 { 1359 struct nullb_device *dev = cmd->nq->dev; 1360 struct nullb *nullb = dev->nullb; 1361 blk_status_t sts; 1362 1363 if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) { 1364 sts = null_handle_throttled(cmd); 1365 if (sts != BLK_STS_OK) 1366 return sts; 1367 } 1368 1369 if (op == REQ_OP_FLUSH) { 1370 cmd->error = errno_to_blk_status(null_handle_flush(nullb)); 1371 goto out; 1372 } 1373 1374 if (dev->zoned) 1375 sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors); 1376 else 1377 sts = null_process_cmd(cmd, op, sector, nr_sectors); 1378 1379 /* Do not overwrite errors (e.g. timeout errors) */ 1380 if (cmd->error == BLK_STS_OK) 1381 cmd->error = sts; 1382 1383 out: 1384 nullb_complete_cmd(cmd); 1385 return BLK_STS_OK; 1386 } 1387 1388 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer) 1389 { 1390 struct nullb *nullb = container_of(timer, struct nullb, bw_timer); 1391 ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL); 1392 unsigned int mbps = nullb->dev->mbps; 1393 1394 if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps)) 1395 return HRTIMER_NORESTART; 1396 1397 atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps)); 1398 null_restart_queue_async(nullb); 1399 1400 hrtimer_forward_now(&nullb->bw_timer, timer_interval); 1401 1402 return HRTIMER_RESTART; 1403 } 1404 1405 static void nullb_setup_bwtimer(struct nullb *nullb) 1406 { 1407 ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL); 1408 1409 hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1410 nullb->bw_timer.function = nullb_bwtimer_fn; 1411 atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps)); 1412 hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL); 1413 } 1414 1415 static struct nullb_queue *nullb_to_queue(struct nullb *nullb) 1416 { 1417 int index = 0; 1418 1419 if (nullb->nr_queues != 1) 1420 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues); 1421 1422 return &nullb->queues[index]; 1423 } 1424 1425 static blk_qc_t null_submit_bio(struct bio *bio) 1426 { 1427 sector_t sector = bio->bi_iter.bi_sector; 1428 sector_t nr_sectors = bio_sectors(bio); 1429 struct nullb *nullb = bio->bi_bdev->bd_disk->private_data; 1430 struct nullb_queue *nq = nullb_to_queue(nullb); 1431 struct nullb_cmd *cmd; 1432 1433 cmd = alloc_cmd(nq, 1); 1434 cmd->bio = bio; 1435 1436 null_handle_cmd(cmd, sector, nr_sectors, bio_op(bio)); 1437 return BLK_QC_T_NONE; 1438 } 1439 1440 static bool should_timeout_request(struct request *rq) 1441 { 1442 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION 1443 if (g_timeout_str[0]) 1444 return should_fail(&null_timeout_attr, 1); 1445 #endif 1446 return false; 1447 } 1448 1449 static bool should_requeue_request(struct request *rq) 1450 { 1451 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION 1452 if (g_requeue_str[0]) 1453 return should_fail(&null_requeue_attr, 1); 1454 #endif 1455 return false; 1456 } 1457 1458 static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res) 1459 { 1460 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq); 1461 1462 pr_info("rq %p timed out\n", rq); 1463 1464 /* 1465 * If the device is marked as blocking (i.e. memory backed or zoned 1466 * device), the submission path may be blocked waiting for resources 1467 * and cause real timeouts. For these real timeouts, the submission 1468 * path will complete the request using blk_mq_complete_request(). 1469 * Only fake timeouts need to execute blk_mq_complete_request() here. 1470 */ 1471 cmd->error = BLK_STS_TIMEOUT; 1472 if (cmd->fake_timeout) 1473 blk_mq_complete_request(rq); 1474 return BLK_EH_DONE; 1475 } 1476 1477 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx, 1478 const struct blk_mq_queue_data *bd) 1479 { 1480 struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); 1481 struct nullb_queue *nq = hctx->driver_data; 1482 sector_t nr_sectors = blk_rq_sectors(bd->rq); 1483 sector_t sector = blk_rq_pos(bd->rq); 1484 1485 might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING); 1486 1487 if (nq->dev->irqmode == NULL_IRQ_TIMER) { 1488 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1489 cmd->timer.function = null_cmd_timer_expired; 1490 } 1491 cmd->rq = bd->rq; 1492 cmd->error = BLK_STS_OK; 1493 cmd->nq = nq; 1494 cmd->fake_timeout = should_timeout_request(bd->rq); 1495 1496 blk_mq_start_request(bd->rq); 1497 1498 if (should_requeue_request(bd->rq)) { 1499 /* 1500 * Alternate between hitting the core BUSY path, and the 1501 * driver driven requeue path 1502 */ 1503 nq->requeue_selection++; 1504 if (nq->requeue_selection & 1) 1505 return BLK_STS_RESOURCE; 1506 else { 1507 blk_mq_requeue_request(bd->rq, true); 1508 return BLK_STS_OK; 1509 } 1510 } 1511 if (cmd->fake_timeout) 1512 return BLK_STS_OK; 1513 1514 return null_handle_cmd(cmd, sector, nr_sectors, req_op(bd->rq)); 1515 } 1516 1517 static void cleanup_queue(struct nullb_queue *nq) 1518 { 1519 kfree(nq->tag_map); 1520 kfree(nq->cmds); 1521 } 1522 1523 static void cleanup_queues(struct nullb *nullb) 1524 { 1525 int i; 1526 1527 for (i = 0; i < nullb->nr_queues; i++) 1528 cleanup_queue(&nullb->queues[i]); 1529 1530 kfree(nullb->queues); 1531 } 1532 1533 static void null_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx) 1534 { 1535 struct nullb_queue *nq = hctx->driver_data; 1536 struct nullb *nullb = nq->dev->nullb; 1537 1538 nullb->nr_queues--; 1539 } 1540 1541 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq) 1542 { 1543 init_waitqueue_head(&nq->wait); 1544 nq->queue_depth = nullb->queue_depth; 1545 nq->dev = nullb->dev; 1546 } 1547 1548 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data, 1549 unsigned int hctx_idx) 1550 { 1551 struct nullb *nullb = hctx->queue->queuedata; 1552 struct nullb_queue *nq; 1553 1554 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION 1555 if (g_init_hctx_str[0] && should_fail(&null_init_hctx_attr, 1)) 1556 return -EFAULT; 1557 #endif 1558 1559 nq = &nullb->queues[hctx_idx]; 1560 hctx->driver_data = nq; 1561 null_init_queue(nullb, nq); 1562 nullb->nr_queues++; 1563 1564 return 0; 1565 } 1566 1567 static const struct blk_mq_ops null_mq_ops = { 1568 .queue_rq = null_queue_rq, 1569 .complete = null_complete_rq, 1570 .timeout = null_timeout_rq, 1571 .init_hctx = null_init_hctx, 1572 .exit_hctx = null_exit_hctx, 1573 }; 1574 1575 static void null_del_dev(struct nullb *nullb) 1576 { 1577 struct nullb_device *dev; 1578 1579 if (!nullb) 1580 return; 1581 1582 dev = nullb->dev; 1583 1584 ida_simple_remove(&nullb_indexes, nullb->index); 1585 1586 list_del_init(&nullb->list); 1587 1588 del_gendisk(nullb->disk); 1589 1590 if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) { 1591 hrtimer_cancel(&nullb->bw_timer); 1592 atomic_long_set(&nullb->cur_bytes, LONG_MAX); 1593 null_restart_queue_async(nullb); 1594 } 1595 1596 blk_cleanup_disk(nullb->disk); 1597 if (dev->queue_mode == NULL_Q_MQ && 1598 nullb->tag_set == &nullb->__tag_set) 1599 blk_mq_free_tag_set(nullb->tag_set); 1600 cleanup_queues(nullb); 1601 if (null_cache_active(nullb)) 1602 null_free_device_storage(nullb->dev, true); 1603 kfree(nullb); 1604 dev->nullb = NULL; 1605 } 1606 1607 static void null_config_discard(struct nullb *nullb) 1608 { 1609 if (nullb->dev->discard == false) 1610 return; 1611 1612 if (!nullb->dev->memory_backed) { 1613 nullb->dev->discard = false; 1614 pr_info("discard option is ignored without memory backing\n"); 1615 return; 1616 } 1617 1618 if (nullb->dev->zoned) { 1619 nullb->dev->discard = false; 1620 pr_info("discard option is ignored in zoned mode\n"); 1621 return; 1622 } 1623 1624 nullb->q->limits.discard_granularity = nullb->dev->blocksize; 1625 nullb->q->limits.discard_alignment = nullb->dev->blocksize; 1626 blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9); 1627 blk_queue_flag_set(QUEUE_FLAG_DISCARD, nullb->q); 1628 } 1629 1630 static const struct block_device_operations null_bio_ops = { 1631 .owner = THIS_MODULE, 1632 .submit_bio = null_submit_bio, 1633 .report_zones = null_report_zones, 1634 }; 1635 1636 static const struct block_device_operations null_rq_ops = { 1637 .owner = THIS_MODULE, 1638 .report_zones = null_report_zones, 1639 }; 1640 1641 static int setup_commands(struct nullb_queue *nq) 1642 { 1643 struct nullb_cmd *cmd; 1644 int i, tag_size; 1645 1646 nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL); 1647 if (!nq->cmds) 1648 return -ENOMEM; 1649 1650 tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG; 1651 nq->tag_map = kcalloc(tag_size, sizeof(unsigned long), GFP_KERNEL); 1652 if (!nq->tag_map) { 1653 kfree(nq->cmds); 1654 return -ENOMEM; 1655 } 1656 1657 for (i = 0; i < nq->queue_depth; i++) { 1658 cmd = &nq->cmds[i]; 1659 cmd->tag = -1U; 1660 } 1661 1662 return 0; 1663 } 1664 1665 static int setup_queues(struct nullb *nullb) 1666 { 1667 nullb->queues = kcalloc(nr_cpu_ids, sizeof(struct nullb_queue), 1668 GFP_KERNEL); 1669 if (!nullb->queues) 1670 return -ENOMEM; 1671 1672 nullb->queue_depth = nullb->dev->hw_queue_depth; 1673 1674 return 0; 1675 } 1676 1677 static int init_driver_queues(struct nullb *nullb) 1678 { 1679 struct nullb_queue *nq; 1680 int i, ret = 0; 1681 1682 for (i = 0; i < nullb->dev->submit_queues; i++) { 1683 nq = &nullb->queues[i]; 1684 1685 null_init_queue(nullb, nq); 1686 1687 ret = setup_commands(nq); 1688 if (ret) 1689 return ret; 1690 nullb->nr_queues++; 1691 } 1692 return 0; 1693 } 1694 1695 static int null_gendisk_register(struct nullb *nullb) 1696 { 1697 sector_t size = ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT; 1698 struct gendisk *disk = nullb->disk; 1699 1700 set_capacity(disk, size); 1701 1702 disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO; 1703 disk->major = null_major; 1704 disk->first_minor = nullb->index; 1705 disk->minors = 1; 1706 if (queue_is_mq(nullb->q)) 1707 disk->fops = &null_rq_ops; 1708 else 1709 disk->fops = &null_bio_ops; 1710 disk->private_data = nullb; 1711 strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN); 1712 1713 if (nullb->dev->zoned) { 1714 int ret = null_register_zoned_dev(nullb); 1715 1716 if (ret) 1717 return ret; 1718 } 1719 1720 return add_disk(disk); 1721 } 1722 1723 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set) 1724 { 1725 set->ops = &null_mq_ops; 1726 set->nr_hw_queues = nullb ? nullb->dev->submit_queues : 1727 g_submit_queues; 1728 set->queue_depth = nullb ? nullb->dev->hw_queue_depth : 1729 g_hw_queue_depth; 1730 set->numa_node = nullb ? nullb->dev->home_node : g_home_node; 1731 set->cmd_size = sizeof(struct nullb_cmd); 1732 set->flags = BLK_MQ_F_SHOULD_MERGE; 1733 if (g_no_sched) 1734 set->flags |= BLK_MQ_F_NO_SCHED; 1735 if (g_shared_tag_bitmap) 1736 set->flags |= BLK_MQ_F_TAG_HCTX_SHARED; 1737 set->driver_data = NULL; 1738 1739 if ((nullb && nullb->dev->blocking) || g_blocking) 1740 set->flags |= BLK_MQ_F_BLOCKING; 1741 1742 return blk_mq_alloc_tag_set(set); 1743 } 1744 1745 static int null_validate_conf(struct nullb_device *dev) 1746 { 1747 dev->blocksize = round_down(dev->blocksize, 512); 1748 dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096); 1749 1750 if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) { 1751 if (dev->submit_queues != nr_online_nodes) 1752 dev->submit_queues = nr_online_nodes; 1753 } else if (dev->submit_queues > nr_cpu_ids) 1754 dev->submit_queues = nr_cpu_ids; 1755 else if (dev->submit_queues == 0) 1756 dev->submit_queues = 1; 1757 1758 dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ); 1759 dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER); 1760 1761 /* Do memory allocation, so set blocking */ 1762 if (dev->memory_backed) 1763 dev->blocking = true; 1764 else /* cache is meaningless */ 1765 dev->cache_size = 0; 1766 dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024, 1767 dev->cache_size); 1768 dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps); 1769 /* can not stop a queue */ 1770 if (dev->queue_mode == NULL_Q_BIO) 1771 dev->mbps = 0; 1772 1773 if (dev->zoned && 1774 (!dev->zone_size || !is_power_of_2(dev->zone_size))) { 1775 pr_err("zone_size must be power-of-two\n"); 1776 return -EINVAL; 1777 } 1778 1779 return 0; 1780 } 1781 1782 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION 1783 static bool __null_setup_fault(struct fault_attr *attr, char *str) 1784 { 1785 if (!str[0]) 1786 return true; 1787 1788 if (!setup_fault_attr(attr, str)) 1789 return false; 1790 1791 attr->verbose = 0; 1792 return true; 1793 } 1794 #endif 1795 1796 static bool null_setup_fault(void) 1797 { 1798 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION 1799 if (!__null_setup_fault(&null_timeout_attr, g_timeout_str)) 1800 return false; 1801 if (!__null_setup_fault(&null_requeue_attr, g_requeue_str)) 1802 return false; 1803 if (!__null_setup_fault(&null_init_hctx_attr, g_init_hctx_str)) 1804 return false; 1805 #endif 1806 return true; 1807 } 1808 1809 static int null_add_dev(struct nullb_device *dev) 1810 { 1811 struct nullb *nullb; 1812 int rv; 1813 1814 rv = null_validate_conf(dev); 1815 if (rv) 1816 return rv; 1817 1818 nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node); 1819 if (!nullb) { 1820 rv = -ENOMEM; 1821 goto out; 1822 } 1823 nullb->dev = dev; 1824 dev->nullb = nullb; 1825 1826 spin_lock_init(&nullb->lock); 1827 1828 rv = setup_queues(nullb); 1829 if (rv) 1830 goto out_free_nullb; 1831 1832 if (dev->queue_mode == NULL_Q_MQ) { 1833 if (shared_tags) { 1834 nullb->tag_set = &tag_set; 1835 rv = 0; 1836 } else { 1837 nullb->tag_set = &nullb->__tag_set; 1838 rv = null_init_tag_set(nullb, nullb->tag_set); 1839 } 1840 1841 if (rv) 1842 goto out_cleanup_queues; 1843 1844 if (!null_setup_fault()) 1845 goto out_cleanup_tags; 1846 1847 nullb->tag_set->timeout = 5 * HZ; 1848 nullb->disk = blk_mq_alloc_disk(nullb->tag_set, nullb); 1849 if (IS_ERR(nullb->disk)) { 1850 rv = PTR_ERR(nullb->disk); 1851 goto out_cleanup_tags; 1852 } 1853 nullb->q = nullb->disk->queue; 1854 } else if (dev->queue_mode == NULL_Q_BIO) { 1855 rv = -ENOMEM; 1856 nullb->disk = blk_alloc_disk(nullb->dev->home_node); 1857 if (!nullb->disk) 1858 goto out_cleanup_queues; 1859 1860 nullb->q = nullb->disk->queue; 1861 rv = init_driver_queues(nullb); 1862 if (rv) 1863 goto out_cleanup_disk; 1864 } 1865 1866 if (dev->mbps) { 1867 set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags); 1868 nullb_setup_bwtimer(nullb); 1869 } 1870 1871 if (dev->cache_size > 0) { 1872 set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags); 1873 blk_queue_write_cache(nullb->q, true, true); 1874 } 1875 1876 if (dev->zoned) { 1877 rv = null_init_zoned_dev(dev, nullb->q); 1878 if (rv) 1879 goto out_cleanup_disk; 1880 } 1881 1882 nullb->q->queuedata = nullb; 1883 blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q); 1884 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q); 1885 1886 mutex_lock(&lock); 1887 nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL); 1888 dev->index = nullb->index; 1889 mutex_unlock(&lock); 1890 1891 blk_queue_logical_block_size(nullb->q, dev->blocksize); 1892 blk_queue_physical_block_size(nullb->q, dev->blocksize); 1893 if (!dev->max_sectors) 1894 dev->max_sectors = queue_max_hw_sectors(nullb->q); 1895 dev->max_sectors = min_t(unsigned int, dev->max_sectors, 1896 BLK_DEF_MAX_SECTORS); 1897 blk_queue_max_hw_sectors(nullb->q, dev->max_sectors); 1898 1899 if (dev->virt_boundary) 1900 blk_queue_virt_boundary(nullb->q, PAGE_SIZE - 1); 1901 1902 null_config_discard(nullb); 1903 1904 sprintf(nullb->disk_name, "nullb%d", nullb->index); 1905 1906 rv = null_gendisk_register(nullb); 1907 if (rv) 1908 goto out_cleanup_zone; 1909 1910 mutex_lock(&lock); 1911 list_add_tail(&nullb->list, &nullb_list); 1912 mutex_unlock(&lock); 1913 1914 return 0; 1915 out_cleanup_zone: 1916 null_free_zoned_dev(dev); 1917 out_cleanup_disk: 1918 blk_cleanup_disk(nullb->disk); 1919 out_cleanup_tags: 1920 if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set) 1921 blk_mq_free_tag_set(nullb->tag_set); 1922 out_cleanup_queues: 1923 cleanup_queues(nullb); 1924 out_free_nullb: 1925 kfree(nullb); 1926 dev->nullb = NULL; 1927 out: 1928 return rv; 1929 } 1930 1931 static int __init null_init(void) 1932 { 1933 int ret = 0; 1934 unsigned int i; 1935 struct nullb *nullb; 1936 struct nullb_device *dev; 1937 1938 if (g_bs > PAGE_SIZE) { 1939 pr_warn("invalid block size\n"); 1940 pr_warn("defaults block size to %lu\n", PAGE_SIZE); 1941 g_bs = PAGE_SIZE; 1942 } 1943 1944 if (g_max_sectors > BLK_DEF_MAX_SECTORS) { 1945 pr_warn("invalid max sectors\n"); 1946 pr_warn("defaults max sectors to %u\n", BLK_DEF_MAX_SECTORS); 1947 g_max_sectors = BLK_DEF_MAX_SECTORS; 1948 } 1949 1950 if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) { 1951 pr_err("invalid home_node value\n"); 1952 g_home_node = NUMA_NO_NODE; 1953 } 1954 1955 if (g_queue_mode == NULL_Q_RQ) { 1956 pr_err("legacy IO path no longer available\n"); 1957 return -EINVAL; 1958 } 1959 if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) { 1960 if (g_submit_queues != nr_online_nodes) { 1961 pr_warn("submit_queues param is set to %u.\n", 1962 nr_online_nodes); 1963 g_submit_queues = nr_online_nodes; 1964 } 1965 } else if (g_submit_queues > nr_cpu_ids) 1966 g_submit_queues = nr_cpu_ids; 1967 else if (g_submit_queues <= 0) 1968 g_submit_queues = 1; 1969 1970 if (g_queue_mode == NULL_Q_MQ && shared_tags) { 1971 ret = null_init_tag_set(NULL, &tag_set); 1972 if (ret) 1973 return ret; 1974 } 1975 1976 config_group_init(&nullb_subsys.su_group); 1977 mutex_init(&nullb_subsys.su_mutex); 1978 1979 ret = configfs_register_subsystem(&nullb_subsys); 1980 if (ret) 1981 goto err_tagset; 1982 1983 mutex_init(&lock); 1984 1985 null_major = register_blkdev(0, "nullb"); 1986 if (null_major < 0) { 1987 ret = null_major; 1988 goto err_conf; 1989 } 1990 1991 for (i = 0; i < nr_devices; i++) { 1992 dev = null_alloc_dev(); 1993 if (!dev) { 1994 ret = -ENOMEM; 1995 goto err_dev; 1996 } 1997 ret = null_add_dev(dev); 1998 if (ret) { 1999 null_free_dev(dev); 2000 goto err_dev; 2001 } 2002 } 2003 2004 pr_info("module loaded\n"); 2005 return 0; 2006 2007 err_dev: 2008 while (!list_empty(&nullb_list)) { 2009 nullb = list_entry(nullb_list.next, struct nullb, list); 2010 dev = nullb->dev; 2011 null_del_dev(nullb); 2012 null_free_dev(dev); 2013 } 2014 unregister_blkdev(null_major, "nullb"); 2015 err_conf: 2016 configfs_unregister_subsystem(&nullb_subsys); 2017 err_tagset: 2018 if (g_queue_mode == NULL_Q_MQ && shared_tags) 2019 blk_mq_free_tag_set(&tag_set); 2020 return ret; 2021 } 2022 2023 static void __exit null_exit(void) 2024 { 2025 struct nullb *nullb; 2026 2027 configfs_unregister_subsystem(&nullb_subsys); 2028 2029 unregister_blkdev(null_major, "nullb"); 2030 2031 mutex_lock(&lock); 2032 while (!list_empty(&nullb_list)) { 2033 struct nullb_device *dev; 2034 2035 nullb = list_entry(nullb_list.next, struct nullb, list); 2036 dev = nullb->dev; 2037 null_del_dev(nullb); 2038 null_free_dev(dev); 2039 } 2040 mutex_unlock(&lock); 2041 2042 if (g_queue_mode == NULL_Q_MQ && shared_tags) 2043 blk_mq_free_tag_set(&tag_set); 2044 } 2045 2046 module_init(null_init); 2047 module_exit(null_exit); 2048 2049 MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>"); 2050 MODULE_LICENSE("GPL"); 2051