xref: /openbmc/linux/drivers/block/null_blk/main.c (revision 8ffdff6a)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
4  * Shaohua Li <shli@fb.com>
5  */
6 #include <linux/module.h>
7 
8 #include <linux/moduleparam.h>
9 #include <linux/sched.h>
10 #include <linux/fs.h>
11 #include <linux/init.h>
12 #include "null_blk.h"
13 
14 #define PAGE_SECTORS_SHIFT	(PAGE_SHIFT - SECTOR_SHIFT)
15 #define PAGE_SECTORS		(1 << PAGE_SECTORS_SHIFT)
16 #define SECTOR_MASK		(PAGE_SECTORS - 1)
17 
18 #define FREE_BATCH		16
19 
20 #define TICKS_PER_SEC		50ULL
21 #define TIMER_INTERVAL		(NSEC_PER_SEC / TICKS_PER_SEC)
22 
23 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
24 static DECLARE_FAULT_ATTR(null_timeout_attr);
25 static DECLARE_FAULT_ATTR(null_requeue_attr);
26 static DECLARE_FAULT_ATTR(null_init_hctx_attr);
27 #endif
28 
29 static inline u64 mb_per_tick(int mbps)
30 {
31 	return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
32 }
33 
34 /*
35  * Status flags for nullb_device.
36  *
37  * CONFIGURED:	Device has been configured and turned on. Cannot reconfigure.
38  * UP:		Device is currently on and visible in userspace.
39  * THROTTLED:	Device is being throttled.
40  * CACHE:	Device is using a write-back cache.
41  */
42 enum nullb_device_flags {
43 	NULLB_DEV_FL_CONFIGURED	= 0,
44 	NULLB_DEV_FL_UP		= 1,
45 	NULLB_DEV_FL_THROTTLED	= 2,
46 	NULLB_DEV_FL_CACHE	= 3,
47 };
48 
49 #define MAP_SZ		((PAGE_SIZE >> SECTOR_SHIFT) + 2)
50 /*
51  * nullb_page is a page in memory for nullb devices.
52  *
53  * @page:	The page holding the data.
54  * @bitmap:	The bitmap represents which sector in the page has data.
55  *		Each bit represents one block size. For example, sector 8
56  *		will use the 7th bit
57  * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
58  * page is being flushing to storage. FREE means the cache page is freed and
59  * should be skipped from flushing to storage. Please see
60  * null_make_cache_space
61  */
62 struct nullb_page {
63 	struct page *page;
64 	DECLARE_BITMAP(bitmap, MAP_SZ);
65 };
66 #define NULLB_PAGE_LOCK (MAP_SZ - 1)
67 #define NULLB_PAGE_FREE (MAP_SZ - 2)
68 
69 static LIST_HEAD(nullb_list);
70 static struct mutex lock;
71 static int null_major;
72 static DEFINE_IDA(nullb_indexes);
73 static struct blk_mq_tag_set tag_set;
74 
75 enum {
76 	NULL_IRQ_NONE		= 0,
77 	NULL_IRQ_SOFTIRQ	= 1,
78 	NULL_IRQ_TIMER		= 2,
79 };
80 
81 enum {
82 	NULL_Q_BIO		= 0,
83 	NULL_Q_RQ		= 1,
84 	NULL_Q_MQ		= 2,
85 };
86 
87 static int g_no_sched;
88 module_param_named(no_sched, g_no_sched, int, 0444);
89 MODULE_PARM_DESC(no_sched, "No io scheduler");
90 
91 static int g_submit_queues = 1;
92 module_param_named(submit_queues, g_submit_queues, int, 0444);
93 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
94 
95 static int g_home_node = NUMA_NO_NODE;
96 module_param_named(home_node, g_home_node, int, 0444);
97 MODULE_PARM_DESC(home_node, "Home node for the device");
98 
99 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
100 /*
101  * For more details about fault injection, please refer to
102  * Documentation/fault-injection/fault-injection.rst.
103  */
104 static char g_timeout_str[80];
105 module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
106 MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>");
107 
108 static char g_requeue_str[80];
109 module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
110 MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>");
111 
112 static char g_init_hctx_str[80];
113 module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444);
114 MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>");
115 #endif
116 
117 static int g_queue_mode = NULL_Q_MQ;
118 
119 static int null_param_store_val(const char *str, int *val, int min, int max)
120 {
121 	int ret, new_val;
122 
123 	ret = kstrtoint(str, 10, &new_val);
124 	if (ret)
125 		return -EINVAL;
126 
127 	if (new_val < min || new_val > max)
128 		return -EINVAL;
129 
130 	*val = new_val;
131 	return 0;
132 }
133 
134 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
135 {
136 	return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
137 }
138 
139 static const struct kernel_param_ops null_queue_mode_param_ops = {
140 	.set	= null_set_queue_mode,
141 	.get	= param_get_int,
142 };
143 
144 device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
145 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
146 
147 static int g_gb = 250;
148 module_param_named(gb, g_gb, int, 0444);
149 MODULE_PARM_DESC(gb, "Size in GB");
150 
151 static int g_bs = 512;
152 module_param_named(bs, g_bs, int, 0444);
153 MODULE_PARM_DESC(bs, "Block size (in bytes)");
154 
155 static int g_max_sectors;
156 module_param_named(max_sectors, g_max_sectors, int, 0444);
157 MODULE_PARM_DESC(max_sectors, "Maximum size of a command (in 512B sectors)");
158 
159 static unsigned int nr_devices = 1;
160 module_param(nr_devices, uint, 0444);
161 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
162 
163 static bool g_blocking;
164 module_param_named(blocking, g_blocking, bool, 0444);
165 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
166 
167 static bool shared_tags;
168 module_param(shared_tags, bool, 0444);
169 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
170 
171 static bool g_shared_tag_bitmap;
172 module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444);
173 MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq");
174 
175 static int g_irqmode = NULL_IRQ_SOFTIRQ;
176 
177 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
178 {
179 	return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
180 					NULL_IRQ_TIMER);
181 }
182 
183 static const struct kernel_param_ops null_irqmode_param_ops = {
184 	.set	= null_set_irqmode,
185 	.get	= param_get_int,
186 };
187 
188 device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
189 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
190 
191 static unsigned long g_completion_nsec = 10000;
192 module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
193 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
194 
195 static int g_hw_queue_depth = 64;
196 module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
197 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
198 
199 static bool g_use_per_node_hctx;
200 module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
201 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
202 
203 static bool g_zoned;
204 module_param_named(zoned, g_zoned, bool, S_IRUGO);
205 MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
206 
207 static unsigned long g_zone_size = 256;
208 module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
209 MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
210 
211 static unsigned long g_zone_capacity;
212 module_param_named(zone_capacity, g_zone_capacity, ulong, 0444);
213 MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size");
214 
215 static unsigned int g_zone_nr_conv;
216 module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
217 MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
218 
219 static unsigned int g_zone_max_open;
220 module_param_named(zone_max_open, g_zone_max_open, uint, 0444);
221 MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)");
222 
223 static unsigned int g_zone_max_active;
224 module_param_named(zone_max_active, g_zone_max_active, uint, 0444);
225 MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)");
226 
227 static struct nullb_device *null_alloc_dev(void);
228 static void null_free_dev(struct nullb_device *dev);
229 static void null_del_dev(struct nullb *nullb);
230 static int null_add_dev(struct nullb_device *dev);
231 static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
232 
233 static inline struct nullb_device *to_nullb_device(struct config_item *item)
234 {
235 	return item ? container_of(item, struct nullb_device, item) : NULL;
236 }
237 
238 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
239 {
240 	return snprintf(page, PAGE_SIZE, "%u\n", val);
241 }
242 
243 static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
244 	char *page)
245 {
246 	return snprintf(page, PAGE_SIZE, "%lu\n", val);
247 }
248 
249 static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
250 {
251 	return snprintf(page, PAGE_SIZE, "%u\n", val);
252 }
253 
254 static ssize_t nullb_device_uint_attr_store(unsigned int *val,
255 	const char *page, size_t count)
256 {
257 	unsigned int tmp;
258 	int result;
259 
260 	result = kstrtouint(page, 0, &tmp);
261 	if (result < 0)
262 		return result;
263 
264 	*val = tmp;
265 	return count;
266 }
267 
268 static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
269 	const char *page, size_t count)
270 {
271 	int result;
272 	unsigned long tmp;
273 
274 	result = kstrtoul(page, 0, &tmp);
275 	if (result < 0)
276 		return result;
277 
278 	*val = tmp;
279 	return count;
280 }
281 
282 static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
283 	size_t count)
284 {
285 	bool tmp;
286 	int result;
287 
288 	result = kstrtobool(page,  &tmp);
289 	if (result < 0)
290 		return result;
291 
292 	*val = tmp;
293 	return count;
294 }
295 
296 /* The following macro should only be used with TYPE = {uint, ulong, bool}. */
297 #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY)				\
298 static ssize_t								\
299 nullb_device_##NAME##_show(struct config_item *item, char *page)	\
300 {									\
301 	return nullb_device_##TYPE##_attr_show(				\
302 				to_nullb_device(item)->NAME, page);	\
303 }									\
304 static ssize_t								\
305 nullb_device_##NAME##_store(struct config_item *item, const char *page,	\
306 			    size_t count)				\
307 {									\
308 	int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\
309 	struct nullb_device *dev = to_nullb_device(item);		\
310 	TYPE new_value = 0;						\
311 	int ret;							\
312 									\
313 	ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\
314 	if (ret < 0)							\
315 		return ret;						\
316 	if (apply_fn)							\
317 		ret = apply_fn(dev, new_value);				\
318 	else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) 	\
319 		ret = -EBUSY;						\
320 	if (ret < 0)							\
321 		return ret;						\
322 	dev->NAME = new_value;						\
323 	return count;							\
324 }									\
325 CONFIGFS_ATTR(nullb_device_, NAME);
326 
327 static int nullb_apply_submit_queues(struct nullb_device *dev,
328 				     unsigned int submit_queues)
329 {
330 	struct nullb *nullb = dev->nullb;
331 	struct blk_mq_tag_set *set;
332 
333 	if (!nullb)
334 		return 0;
335 
336 	/*
337 	 * Make sure that null_init_hctx() does not access nullb->queues[] past
338 	 * the end of that array.
339 	 */
340 	if (submit_queues > nr_cpu_ids)
341 		return -EINVAL;
342 	set = nullb->tag_set;
343 	blk_mq_update_nr_hw_queues(set, submit_queues);
344 	return set->nr_hw_queues == submit_queues ? 0 : -ENOMEM;
345 }
346 
347 NULLB_DEVICE_ATTR(size, ulong, NULL);
348 NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
349 NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
350 NULLB_DEVICE_ATTR(home_node, uint, NULL);
351 NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
352 NULLB_DEVICE_ATTR(blocksize, uint, NULL);
353 NULLB_DEVICE_ATTR(max_sectors, uint, NULL);
354 NULLB_DEVICE_ATTR(irqmode, uint, NULL);
355 NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
356 NULLB_DEVICE_ATTR(index, uint, NULL);
357 NULLB_DEVICE_ATTR(blocking, bool, NULL);
358 NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
359 NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
360 NULLB_DEVICE_ATTR(discard, bool, NULL);
361 NULLB_DEVICE_ATTR(mbps, uint, NULL);
362 NULLB_DEVICE_ATTR(cache_size, ulong, NULL);
363 NULLB_DEVICE_ATTR(zoned, bool, NULL);
364 NULLB_DEVICE_ATTR(zone_size, ulong, NULL);
365 NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL);
366 NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL);
367 NULLB_DEVICE_ATTR(zone_max_open, uint, NULL);
368 NULLB_DEVICE_ATTR(zone_max_active, uint, NULL);
369 
370 static ssize_t nullb_device_power_show(struct config_item *item, char *page)
371 {
372 	return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
373 }
374 
375 static ssize_t nullb_device_power_store(struct config_item *item,
376 				     const char *page, size_t count)
377 {
378 	struct nullb_device *dev = to_nullb_device(item);
379 	bool newp = false;
380 	ssize_t ret;
381 
382 	ret = nullb_device_bool_attr_store(&newp, page, count);
383 	if (ret < 0)
384 		return ret;
385 
386 	if (!dev->power && newp) {
387 		if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
388 			return count;
389 		if (null_add_dev(dev)) {
390 			clear_bit(NULLB_DEV_FL_UP, &dev->flags);
391 			return -ENOMEM;
392 		}
393 
394 		set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
395 		dev->power = newp;
396 	} else if (dev->power && !newp) {
397 		if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
398 			mutex_lock(&lock);
399 			dev->power = newp;
400 			null_del_dev(dev->nullb);
401 			mutex_unlock(&lock);
402 		}
403 		clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
404 	}
405 
406 	return count;
407 }
408 
409 CONFIGFS_ATTR(nullb_device_, power);
410 
411 static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
412 {
413 	struct nullb_device *t_dev = to_nullb_device(item);
414 
415 	return badblocks_show(&t_dev->badblocks, page, 0);
416 }
417 
418 static ssize_t nullb_device_badblocks_store(struct config_item *item,
419 				     const char *page, size_t count)
420 {
421 	struct nullb_device *t_dev = to_nullb_device(item);
422 	char *orig, *buf, *tmp;
423 	u64 start, end;
424 	int ret;
425 
426 	orig = kstrndup(page, count, GFP_KERNEL);
427 	if (!orig)
428 		return -ENOMEM;
429 
430 	buf = strstrip(orig);
431 
432 	ret = -EINVAL;
433 	if (buf[0] != '+' && buf[0] != '-')
434 		goto out;
435 	tmp = strchr(&buf[1], '-');
436 	if (!tmp)
437 		goto out;
438 	*tmp = '\0';
439 	ret = kstrtoull(buf + 1, 0, &start);
440 	if (ret)
441 		goto out;
442 	ret = kstrtoull(tmp + 1, 0, &end);
443 	if (ret)
444 		goto out;
445 	ret = -EINVAL;
446 	if (start > end)
447 		goto out;
448 	/* enable badblocks */
449 	cmpxchg(&t_dev->badblocks.shift, -1, 0);
450 	if (buf[0] == '+')
451 		ret = badblocks_set(&t_dev->badblocks, start,
452 			end - start + 1, 1);
453 	else
454 		ret = badblocks_clear(&t_dev->badblocks, start,
455 			end - start + 1);
456 	if (ret == 0)
457 		ret = count;
458 out:
459 	kfree(orig);
460 	return ret;
461 }
462 CONFIGFS_ATTR(nullb_device_, badblocks);
463 
464 static struct configfs_attribute *nullb_device_attrs[] = {
465 	&nullb_device_attr_size,
466 	&nullb_device_attr_completion_nsec,
467 	&nullb_device_attr_submit_queues,
468 	&nullb_device_attr_home_node,
469 	&nullb_device_attr_queue_mode,
470 	&nullb_device_attr_blocksize,
471 	&nullb_device_attr_max_sectors,
472 	&nullb_device_attr_irqmode,
473 	&nullb_device_attr_hw_queue_depth,
474 	&nullb_device_attr_index,
475 	&nullb_device_attr_blocking,
476 	&nullb_device_attr_use_per_node_hctx,
477 	&nullb_device_attr_power,
478 	&nullb_device_attr_memory_backed,
479 	&nullb_device_attr_discard,
480 	&nullb_device_attr_mbps,
481 	&nullb_device_attr_cache_size,
482 	&nullb_device_attr_badblocks,
483 	&nullb_device_attr_zoned,
484 	&nullb_device_attr_zone_size,
485 	&nullb_device_attr_zone_capacity,
486 	&nullb_device_attr_zone_nr_conv,
487 	&nullb_device_attr_zone_max_open,
488 	&nullb_device_attr_zone_max_active,
489 	NULL,
490 };
491 
492 static void nullb_device_release(struct config_item *item)
493 {
494 	struct nullb_device *dev = to_nullb_device(item);
495 
496 	null_free_device_storage(dev, false);
497 	null_free_dev(dev);
498 }
499 
500 static struct configfs_item_operations nullb_device_ops = {
501 	.release	= nullb_device_release,
502 };
503 
504 static const struct config_item_type nullb_device_type = {
505 	.ct_item_ops	= &nullb_device_ops,
506 	.ct_attrs	= nullb_device_attrs,
507 	.ct_owner	= THIS_MODULE,
508 };
509 
510 static struct
511 config_item *nullb_group_make_item(struct config_group *group, const char *name)
512 {
513 	struct nullb_device *dev;
514 
515 	dev = null_alloc_dev();
516 	if (!dev)
517 		return ERR_PTR(-ENOMEM);
518 
519 	config_item_init_type_name(&dev->item, name, &nullb_device_type);
520 
521 	return &dev->item;
522 }
523 
524 static void
525 nullb_group_drop_item(struct config_group *group, struct config_item *item)
526 {
527 	struct nullb_device *dev = to_nullb_device(item);
528 
529 	if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
530 		mutex_lock(&lock);
531 		dev->power = false;
532 		null_del_dev(dev->nullb);
533 		mutex_unlock(&lock);
534 	}
535 
536 	config_item_put(item);
537 }
538 
539 static ssize_t memb_group_features_show(struct config_item *item, char *page)
540 {
541 	return snprintf(page, PAGE_SIZE,
542 			"memory_backed,discard,bandwidth,cache,badblocks,zoned,zone_size,zone_capacity,zone_nr_conv,zone_max_open,zone_max_active,blocksize,max_sectors\n");
543 }
544 
545 CONFIGFS_ATTR_RO(memb_group_, features);
546 
547 static struct configfs_attribute *nullb_group_attrs[] = {
548 	&memb_group_attr_features,
549 	NULL,
550 };
551 
552 static struct configfs_group_operations nullb_group_ops = {
553 	.make_item	= nullb_group_make_item,
554 	.drop_item	= nullb_group_drop_item,
555 };
556 
557 static const struct config_item_type nullb_group_type = {
558 	.ct_group_ops	= &nullb_group_ops,
559 	.ct_attrs	= nullb_group_attrs,
560 	.ct_owner	= THIS_MODULE,
561 };
562 
563 static struct configfs_subsystem nullb_subsys = {
564 	.su_group = {
565 		.cg_item = {
566 			.ci_namebuf = "nullb",
567 			.ci_type = &nullb_group_type,
568 		},
569 	},
570 };
571 
572 static inline int null_cache_active(struct nullb *nullb)
573 {
574 	return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
575 }
576 
577 static struct nullb_device *null_alloc_dev(void)
578 {
579 	struct nullb_device *dev;
580 
581 	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
582 	if (!dev)
583 		return NULL;
584 	INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
585 	INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
586 	if (badblocks_init(&dev->badblocks, 0)) {
587 		kfree(dev);
588 		return NULL;
589 	}
590 
591 	dev->size = g_gb * 1024;
592 	dev->completion_nsec = g_completion_nsec;
593 	dev->submit_queues = g_submit_queues;
594 	dev->home_node = g_home_node;
595 	dev->queue_mode = g_queue_mode;
596 	dev->blocksize = g_bs;
597 	dev->max_sectors = g_max_sectors;
598 	dev->irqmode = g_irqmode;
599 	dev->hw_queue_depth = g_hw_queue_depth;
600 	dev->blocking = g_blocking;
601 	dev->use_per_node_hctx = g_use_per_node_hctx;
602 	dev->zoned = g_zoned;
603 	dev->zone_size = g_zone_size;
604 	dev->zone_capacity = g_zone_capacity;
605 	dev->zone_nr_conv = g_zone_nr_conv;
606 	dev->zone_max_open = g_zone_max_open;
607 	dev->zone_max_active = g_zone_max_active;
608 	return dev;
609 }
610 
611 static void null_free_dev(struct nullb_device *dev)
612 {
613 	if (!dev)
614 		return;
615 
616 	null_free_zoned_dev(dev);
617 	badblocks_exit(&dev->badblocks);
618 	kfree(dev);
619 }
620 
621 static void put_tag(struct nullb_queue *nq, unsigned int tag)
622 {
623 	clear_bit_unlock(tag, nq->tag_map);
624 
625 	if (waitqueue_active(&nq->wait))
626 		wake_up(&nq->wait);
627 }
628 
629 static unsigned int get_tag(struct nullb_queue *nq)
630 {
631 	unsigned int tag;
632 
633 	do {
634 		tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
635 		if (tag >= nq->queue_depth)
636 			return -1U;
637 	} while (test_and_set_bit_lock(tag, nq->tag_map));
638 
639 	return tag;
640 }
641 
642 static void free_cmd(struct nullb_cmd *cmd)
643 {
644 	put_tag(cmd->nq, cmd->tag);
645 }
646 
647 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
648 
649 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
650 {
651 	struct nullb_cmd *cmd;
652 	unsigned int tag;
653 
654 	tag = get_tag(nq);
655 	if (tag != -1U) {
656 		cmd = &nq->cmds[tag];
657 		cmd->tag = tag;
658 		cmd->error = BLK_STS_OK;
659 		cmd->nq = nq;
660 		if (nq->dev->irqmode == NULL_IRQ_TIMER) {
661 			hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
662 				     HRTIMER_MODE_REL);
663 			cmd->timer.function = null_cmd_timer_expired;
664 		}
665 		return cmd;
666 	}
667 
668 	return NULL;
669 }
670 
671 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
672 {
673 	struct nullb_cmd *cmd;
674 	DEFINE_WAIT(wait);
675 
676 	cmd = __alloc_cmd(nq);
677 	if (cmd || !can_wait)
678 		return cmd;
679 
680 	do {
681 		prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
682 		cmd = __alloc_cmd(nq);
683 		if (cmd)
684 			break;
685 
686 		io_schedule();
687 	} while (1);
688 
689 	finish_wait(&nq->wait, &wait);
690 	return cmd;
691 }
692 
693 static void end_cmd(struct nullb_cmd *cmd)
694 {
695 	int queue_mode = cmd->nq->dev->queue_mode;
696 
697 	switch (queue_mode)  {
698 	case NULL_Q_MQ:
699 		blk_mq_end_request(cmd->rq, cmd->error);
700 		return;
701 	case NULL_Q_BIO:
702 		cmd->bio->bi_status = cmd->error;
703 		bio_endio(cmd->bio);
704 		break;
705 	}
706 
707 	free_cmd(cmd);
708 }
709 
710 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
711 {
712 	end_cmd(container_of(timer, struct nullb_cmd, timer));
713 
714 	return HRTIMER_NORESTART;
715 }
716 
717 static void null_cmd_end_timer(struct nullb_cmd *cmd)
718 {
719 	ktime_t kt = cmd->nq->dev->completion_nsec;
720 
721 	hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
722 }
723 
724 static void null_complete_rq(struct request *rq)
725 {
726 	end_cmd(blk_mq_rq_to_pdu(rq));
727 }
728 
729 static struct nullb_page *null_alloc_page(gfp_t gfp_flags)
730 {
731 	struct nullb_page *t_page;
732 
733 	t_page = kmalloc(sizeof(struct nullb_page), gfp_flags);
734 	if (!t_page)
735 		goto out;
736 
737 	t_page->page = alloc_pages(gfp_flags, 0);
738 	if (!t_page->page)
739 		goto out_freepage;
740 
741 	memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
742 	return t_page;
743 out_freepage:
744 	kfree(t_page);
745 out:
746 	return NULL;
747 }
748 
749 static void null_free_page(struct nullb_page *t_page)
750 {
751 	__set_bit(NULLB_PAGE_FREE, t_page->bitmap);
752 	if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
753 		return;
754 	__free_page(t_page->page);
755 	kfree(t_page);
756 }
757 
758 static bool null_page_empty(struct nullb_page *page)
759 {
760 	int size = MAP_SZ - 2;
761 
762 	return find_first_bit(page->bitmap, size) == size;
763 }
764 
765 static void null_free_sector(struct nullb *nullb, sector_t sector,
766 	bool is_cache)
767 {
768 	unsigned int sector_bit;
769 	u64 idx;
770 	struct nullb_page *t_page, *ret;
771 	struct radix_tree_root *root;
772 
773 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
774 	idx = sector >> PAGE_SECTORS_SHIFT;
775 	sector_bit = (sector & SECTOR_MASK);
776 
777 	t_page = radix_tree_lookup(root, idx);
778 	if (t_page) {
779 		__clear_bit(sector_bit, t_page->bitmap);
780 
781 		if (null_page_empty(t_page)) {
782 			ret = radix_tree_delete_item(root, idx, t_page);
783 			WARN_ON(ret != t_page);
784 			null_free_page(ret);
785 			if (is_cache)
786 				nullb->dev->curr_cache -= PAGE_SIZE;
787 		}
788 	}
789 }
790 
791 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
792 	struct nullb_page *t_page, bool is_cache)
793 {
794 	struct radix_tree_root *root;
795 
796 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
797 
798 	if (radix_tree_insert(root, idx, t_page)) {
799 		null_free_page(t_page);
800 		t_page = radix_tree_lookup(root, idx);
801 		WARN_ON(!t_page || t_page->page->index != idx);
802 	} else if (is_cache)
803 		nullb->dev->curr_cache += PAGE_SIZE;
804 
805 	return t_page;
806 }
807 
808 static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
809 {
810 	unsigned long pos = 0;
811 	int nr_pages;
812 	struct nullb_page *ret, *t_pages[FREE_BATCH];
813 	struct radix_tree_root *root;
814 
815 	root = is_cache ? &dev->cache : &dev->data;
816 
817 	do {
818 		int i;
819 
820 		nr_pages = radix_tree_gang_lookup(root,
821 				(void **)t_pages, pos, FREE_BATCH);
822 
823 		for (i = 0; i < nr_pages; i++) {
824 			pos = t_pages[i]->page->index;
825 			ret = radix_tree_delete_item(root, pos, t_pages[i]);
826 			WARN_ON(ret != t_pages[i]);
827 			null_free_page(ret);
828 		}
829 
830 		pos++;
831 	} while (nr_pages == FREE_BATCH);
832 
833 	if (is_cache)
834 		dev->curr_cache = 0;
835 }
836 
837 static struct nullb_page *__null_lookup_page(struct nullb *nullb,
838 	sector_t sector, bool for_write, bool is_cache)
839 {
840 	unsigned int sector_bit;
841 	u64 idx;
842 	struct nullb_page *t_page;
843 	struct radix_tree_root *root;
844 
845 	idx = sector >> PAGE_SECTORS_SHIFT;
846 	sector_bit = (sector & SECTOR_MASK);
847 
848 	root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
849 	t_page = radix_tree_lookup(root, idx);
850 	WARN_ON(t_page && t_page->page->index != idx);
851 
852 	if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
853 		return t_page;
854 
855 	return NULL;
856 }
857 
858 static struct nullb_page *null_lookup_page(struct nullb *nullb,
859 	sector_t sector, bool for_write, bool ignore_cache)
860 {
861 	struct nullb_page *page = NULL;
862 
863 	if (!ignore_cache)
864 		page = __null_lookup_page(nullb, sector, for_write, true);
865 	if (page)
866 		return page;
867 	return __null_lookup_page(nullb, sector, for_write, false);
868 }
869 
870 static struct nullb_page *null_insert_page(struct nullb *nullb,
871 					   sector_t sector, bool ignore_cache)
872 	__releases(&nullb->lock)
873 	__acquires(&nullb->lock)
874 {
875 	u64 idx;
876 	struct nullb_page *t_page;
877 
878 	t_page = null_lookup_page(nullb, sector, true, ignore_cache);
879 	if (t_page)
880 		return t_page;
881 
882 	spin_unlock_irq(&nullb->lock);
883 
884 	t_page = null_alloc_page(GFP_NOIO);
885 	if (!t_page)
886 		goto out_lock;
887 
888 	if (radix_tree_preload(GFP_NOIO))
889 		goto out_freepage;
890 
891 	spin_lock_irq(&nullb->lock);
892 	idx = sector >> PAGE_SECTORS_SHIFT;
893 	t_page->page->index = idx;
894 	t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
895 	radix_tree_preload_end();
896 
897 	return t_page;
898 out_freepage:
899 	null_free_page(t_page);
900 out_lock:
901 	spin_lock_irq(&nullb->lock);
902 	return null_lookup_page(nullb, sector, true, ignore_cache);
903 }
904 
905 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
906 {
907 	int i;
908 	unsigned int offset;
909 	u64 idx;
910 	struct nullb_page *t_page, *ret;
911 	void *dst, *src;
912 
913 	idx = c_page->page->index;
914 
915 	t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
916 
917 	__clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
918 	if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
919 		null_free_page(c_page);
920 		if (t_page && null_page_empty(t_page)) {
921 			ret = radix_tree_delete_item(&nullb->dev->data,
922 				idx, t_page);
923 			null_free_page(t_page);
924 		}
925 		return 0;
926 	}
927 
928 	if (!t_page)
929 		return -ENOMEM;
930 
931 	src = kmap_atomic(c_page->page);
932 	dst = kmap_atomic(t_page->page);
933 
934 	for (i = 0; i < PAGE_SECTORS;
935 			i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
936 		if (test_bit(i, c_page->bitmap)) {
937 			offset = (i << SECTOR_SHIFT);
938 			memcpy(dst + offset, src + offset,
939 				nullb->dev->blocksize);
940 			__set_bit(i, t_page->bitmap);
941 		}
942 	}
943 
944 	kunmap_atomic(dst);
945 	kunmap_atomic(src);
946 
947 	ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
948 	null_free_page(ret);
949 	nullb->dev->curr_cache -= PAGE_SIZE;
950 
951 	return 0;
952 }
953 
954 static int null_make_cache_space(struct nullb *nullb, unsigned long n)
955 {
956 	int i, err, nr_pages;
957 	struct nullb_page *c_pages[FREE_BATCH];
958 	unsigned long flushed = 0, one_round;
959 
960 again:
961 	if ((nullb->dev->cache_size * 1024 * 1024) >
962 	     nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
963 		return 0;
964 
965 	nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
966 			(void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
967 	/*
968 	 * nullb_flush_cache_page could unlock before using the c_pages. To
969 	 * avoid race, we don't allow page free
970 	 */
971 	for (i = 0; i < nr_pages; i++) {
972 		nullb->cache_flush_pos = c_pages[i]->page->index;
973 		/*
974 		 * We found the page which is being flushed to disk by other
975 		 * threads
976 		 */
977 		if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
978 			c_pages[i] = NULL;
979 		else
980 			__set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
981 	}
982 
983 	one_round = 0;
984 	for (i = 0; i < nr_pages; i++) {
985 		if (c_pages[i] == NULL)
986 			continue;
987 		err = null_flush_cache_page(nullb, c_pages[i]);
988 		if (err)
989 			return err;
990 		one_round++;
991 	}
992 	flushed += one_round << PAGE_SHIFT;
993 
994 	if (n > flushed) {
995 		if (nr_pages == 0)
996 			nullb->cache_flush_pos = 0;
997 		if (one_round == 0) {
998 			/* give other threads a chance */
999 			spin_unlock_irq(&nullb->lock);
1000 			spin_lock_irq(&nullb->lock);
1001 		}
1002 		goto again;
1003 	}
1004 	return 0;
1005 }
1006 
1007 static int copy_to_nullb(struct nullb *nullb, struct page *source,
1008 	unsigned int off, sector_t sector, size_t n, bool is_fua)
1009 {
1010 	size_t temp, count = 0;
1011 	unsigned int offset;
1012 	struct nullb_page *t_page;
1013 	void *dst, *src;
1014 
1015 	while (count < n) {
1016 		temp = min_t(size_t, nullb->dev->blocksize, n - count);
1017 
1018 		if (null_cache_active(nullb) && !is_fua)
1019 			null_make_cache_space(nullb, PAGE_SIZE);
1020 
1021 		offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1022 		t_page = null_insert_page(nullb, sector,
1023 			!null_cache_active(nullb) || is_fua);
1024 		if (!t_page)
1025 			return -ENOSPC;
1026 
1027 		src = kmap_atomic(source);
1028 		dst = kmap_atomic(t_page->page);
1029 		memcpy(dst + offset, src + off + count, temp);
1030 		kunmap_atomic(dst);
1031 		kunmap_atomic(src);
1032 
1033 		__set_bit(sector & SECTOR_MASK, t_page->bitmap);
1034 
1035 		if (is_fua)
1036 			null_free_sector(nullb, sector, true);
1037 
1038 		count += temp;
1039 		sector += temp >> SECTOR_SHIFT;
1040 	}
1041 	return 0;
1042 }
1043 
1044 static int copy_from_nullb(struct nullb *nullb, struct page *dest,
1045 	unsigned int off, sector_t sector, size_t n)
1046 {
1047 	size_t temp, count = 0;
1048 	unsigned int offset;
1049 	struct nullb_page *t_page;
1050 	void *dst, *src;
1051 
1052 	while (count < n) {
1053 		temp = min_t(size_t, nullb->dev->blocksize, n - count);
1054 
1055 		offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1056 		t_page = null_lookup_page(nullb, sector, false,
1057 			!null_cache_active(nullb));
1058 
1059 		dst = kmap_atomic(dest);
1060 		if (!t_page) {
1061 			memset(dst + off + count, 0, temp);
1062 			goto next;
1063 		}
1064 		src = kmap_atomic(t_page->page);
1065 		memcpy(dst + off + count, src + offset, temp);
1066 		kunmap_atomic(src);
1067 next:
1068 		kunmap_atomic(dst);
1069 
1070 		count += temp;
1071 		sector += temp >> SECTOR_SHIFT;
1072 	}
1073 	return 0;
1074 }
1075 
1076 static void nullb_fill_pattern(struct nullb *nullb, struct page *page,
1077 			       unsigned int len, unsigned int off)
1078 {
1079 	void *dst;
1080 
1081 	dst = kmap_atomic(page);
1082 	memset(dst + off, 0xFF, len);
1083 	kunmap_atomic(dst);
1084 }
1085 
1086 blk_status_t null_handle_discard(struct nullb_device *dev,
1087 				 sector_t sector, sector_t nr_sectors)
1088 {
1089 	struct nullb *nullb = dev->nullb;
1090 	size_t n = nr_sectors << SECTOR_SHIFT;
1091 	size_t temp;
1092 
1093 	spin_lock_irq(&nullb->lock);
1094 	while (n > 0) {
1095 		temp = min_t(size_t, n, dev->blocksize);
1096 		null_free_sector(nullb, sector, false);
1097 		if (null_cache_active(nullb))
1098 			null_free_sector(nullb, sector, true);
1099 		sector += temp >> SECTOR_SHIFT;
1100 		n -= temp;
1101 	}
1102 	spin_unlock_irq(&nullb->lock);
1103 
1104 	return BLK_STS_OK;
1105 }
1106 
1107 static int null_handle_flush(struct nullb *nullb)
1108 {
1109 	int err;
1110 
1111 	if (!null_cache_active(nullb))
1112 		return 0;
1113 
1114 	spin_lock_irq(&nullb->lock);
1115 	while (true) {
1116 		err = null_make_cache_space(nullb,
1117 			nullb->dev->cache_size * 1024 * 1024);
1118 		if (err || nullb->dev->curr_cache == 0)
1119 			break;
1120 	}
1121 
1122 	WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1123 	spin_unlock_irq(&nullb->lock);
1124 	return err;
1125 }
1126 
1127 static int null_transfer(struct nullb *nullb, struct page *page,
1128 	unsigned int len, unsigned int off, bool is_write, sector_t sector,
1129 	bool is_fua)
1130 {
1131 	struct nullb_device *dev = nullb->dev;
1132 	unsigned int valid_len = len;
1133 	int err = 0;
1134 
1135 	if (!is_write) {
1136 		if (dev->zoned)
1137 			valid_len = null_zone_valid_read_len(nullb,
1138 				sector, len);
1139 
1140 		if (valid_len) {
1141 			err = copy_from_nullb(nullb, page, off,
1142 				sector, valid_len);
1143 			off += valid_len;
1144 			len -= valid_len;
1145 		}
1146 
1147 		if (len)
1148 			nullb_fill_pattern(nullb, page, len, off);
1149 		flush_dcache_page(page);
1150 	} else {
1151 		flush_dcache_page(page);
1152 		err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
1153 	}
1154 
1155 	return err;
1156 }
1157 
1158 static int null_handle_rq(struct nullb_cmd *cmd)
1159 {
1160 	struct request *rq = cmd->rq;
1161 	struct nullb *nullb = cmd->nq->dev->nullb;
1162 	int err;
1163 	unsigned int len;
1164 	sector_t sector = blk_rq_pos(rq);
1165 	struct req_iterator iter;
1166 	struct bio_vec bvec;
1167 
1168 	spin_lock_irq(&nullb->lock);
1169 	rq_for_each_segment(bvec, rq, iter) {
1170 		len = bvec.bv_len;
1171 		err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1172 				     op_is_write(req_op(rq)), sector,
1173 				     rq->cmd_flags & REQ_FUA);
1174 		if (err) {
1175 			spin_unlock_irq(&nullb->lock);
1176 			return err;
1177 		}
1178 		sector += len >> SECTOR_SHIFT;
1179 	}
1180 	spin_unlock_irq(&nullb->lock);
1181 
1182 	return 0;
1183 }
1184 
1185 static int null_handle_bio(struct nullb_cmd *cmd)
1186 {
1187 	struct bio *bio = cmd->bio;
1188 	struct nullb *nullb = cmd->nq->dev->nullb;
1189 	int err;
1190 	unsigned int len;
1191 	sector_t sector = bio->bi_iter.bi_sector;
1192 	struct bio_vec bvec;
1193 	struct bvec_iter iter;
1194 
1195 	spin_lock_irq(&nullb->lock);
1196 	bio_for_each_segment(bvec, bio, iter) {
1197 		len = bvec.bv_len;
1198 		err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1199 				     op_is_write(bio_op(bio)), sector,
1200 				     bio->bi_opf & REQ_FUA);
1201 		if (err) {
1202 			spin_unlock_irq(&nullb->lock);
1203 			return err;
1204 		}
1205 		sector += len >> SECTOR_SHIFT;
1206 	}
1207 	spin_unlock_irq(&nullb->lock);
1208 	return 0;
1209 }
1210 
1211 static void null_stop_queue(struct nullb *nullb)
1212 {
1213 	struct request_queue *q = nullb->q;
1214 
1215 	if (nullb->dev->queue_mode == NULL_Q_MQ)
1216 		blk_mq_stop_hw_queues(q);
1217 }
1218 
1219 static void null_restart_queue_async(struct nullb *nullb)
1220 {
1221 	struct request_queue *q = nullb->q;
1222 
1223 	if (nullb->dev->queue_mode == NULL_Q_MQ)
1224 		blk_mq_start_stopped_hw_queues(q, true);
1225 }
1226 
1227 static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
1228 {
1229 	struct nullb_device *dev = cmd->nq->dev;
1230 	struct nullb *nullb = dev->nullb;
1231 	blk_status_t sts = BLK_STS_OK;
1232 	struct request *rq = cmd->rq;
1233 
1234 	if (!hrtimer_active(&nullb->bw_timer))
1235 		hrtimer_restart(&nullb->bw_timer);
1236 
1237 	if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {
1238 		null_stop_queue(nullb);
1239 		/* race with timer */
1240 		if (atomic_long_read(&nullb->cur_bytes) > 0)
1241 			null_restart_queue_async(nullb);
1242 		/* requeue request */
1243 		sts = BLK_STS_DEV_RESOURCE;
1244 	}
1245 	return sts;
1246 }
1247 
1248 static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd,
1249 						 sector_t sector,
1250 						 sector_t nr_sectors)
1251 {
1252 	struct badblocks *bb = &cmd->nq->dev->badblocks;
1253 	sector_t first_bad;
1254 	int bad_sectors;
1255 
1256 	if (badblocks_check(bb, sector, nr_sectors, &first_bad, &bad_sectors))
1257 		return BLK_STS_IOERR;
1258 
1259 	return BLK_STS_OK;
1260 }
1261 
1262 static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd,
1263 						     enum req_opf op,
1264 						     sector_t sector,
1265 						     sector_t nr_sectors)
1266 {
1267 	struct nullb_device *dev = cmd->nq->dev;
1268 	int err;
1269 
1270 	if (op == REQ_OP_DISCARD)
1271 		return null_handle_discard(dev, sector, nr_sectors);
1272 
1273 	if (dev->queue_mode == NULL_Q_BIO)
1274 		err = null_handle_bio(cmd);
1275 	else
1276 		err = null_handle_rq(cmd);
1277 
1278 	return errno_to_blk_status(err);
1279 }
1280 
1281 static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd)
1282 {
1283 	struct nullb_device *dev = cmd->nq->dev;
1284 	struct bio *bio;
1285 
1286 	if (dev->memory_backed)
1287 		return;
1288 
1289 	if (dev->queue_mode == NULL_Q_BIO && bio_op(cmd->bio) == REQ_OP_READ) {
1290 		zero_fill_bio(cmd->bio);
1291 	} else if (req_op(cmd->rq) == REQ_OP_READ) {
1292 		__rq_for_each_bio(bio, cmd->rq)
1293 			zero_fill_bio(bio);
1294 	}
1295 }
1296 
1297 static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
1298 {
1299 	/*
1300 	 * Since root privileges are required to configure the null_blk
1301 	 * driver, it is fine that this driver does not initialize the
1302 	 * data buffers of read commands. Zero-initialize these buffers
1303 	 * anyway if KMSAN is enabled to prevent that KMSAN complains
1304 	 * about null_blk not initializing read data buffers.
1305 	 */
1306 	if (IS_ENABLED(CONFIG_KMSAN))
1307 		nullb_zero_read_cmd_buffer(cmd);
1308 
1309 	/* Complete IO by inline, softirq or timer */
1310 	switch (cmd->nq->dev->irqmode) {
1311 	case NULL_IRQ_SOFTIRQ:
1312 		switch (cmd->nq->dev->queue_mode) {
1313 		case NULL_Q_MQ:
1314 			if (likely(!blk_should_fake_timeout(cmd->rq->q)))
1315 				blk_mq_complete_request(cmd->rq);
1316 			break;
1317 		case NULL_Q_BIO:
1318 			/*
1319 			 * XXX: no proper submitting cpu information available.
1320 			 */
1321 			end_cmd(cmd);
1322 			break;
1323 		}
1324 		break;
1325 	case NULL_IRQ_NONE:
1326 		end_cmd(cmd);
1327 		break;
1328 	case NULL_IRQ_TIMER:
1329 		null_cmd_end_timer(cmd);
1330 		break;
1331 	}
1332 }
1333 
1334 blk_status_t null_process_cmd(struct nullb_cmd *cmd,
1335 			      enum req_opf op, sector_t sector,
1336 			      unsigned int nr_sectors)
1337 {
1338 	struct nullb_device *dev = cmd->nq->dev;
1339 	blk_status_t ret;
1340 
1341 	if (dev->badblocks.shift != -1) {
1342 		ret = null_handle_badblocks(cmd, sector, nr_sectors);
1343 		if (ret != BLK_STS_OK)
1344 			return ret;
1345 	}
1346 
1347 	if (dev->memory_backed)
1348 		return null_handle_memory_backed(cmd, op, sector, nr_sectors);
1349 
1350 	return BLK_STS_OK;
1351 }
1352 
1353 static blk_status_t null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
1354 				    sector_t nr_sectors, enum req_opf op)
1355 {
1356 	struct nullb_device *dev = cmd->nq->dev;
1357 	struct nullb *nullb = dev->nullb;
1358 	blk_status_t sts;
1359 
1360 	if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) {
1361 		sts = null_handle_throttled(cmd);
1362 		if (sts != BLK_STS_OK)
1363 			return sts;
1364 	}
1365 
1366 	if (op == REQ_OP_FLUSH) {
1367 		cmd->error = errno_to_blk_status(null_handle_flush(nullb));
1368 		goto out;
1369 	}
1370 
1371 	if (dev->zoned)
1372 		sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors);
1373 	else
1374 		sts = null_process_cmd(cmd, op, sector, nr_sectors);
1375 
1376 	/* Do not overwrite errors (e.g. timeout errors) */
1377 	if (cmd->error == BLK_STS_OK)
1378 		cmd->error = sts;
1379 
1380 out:
1381 	nullb_complete_cmd(cmd);
1382 	return BLK_STS_OK;
1383 }
1384 
1385 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1386 {
1387 	struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1388 	ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1389 	unsigned int mbps = nullb->dev->mbps;
1390 
1391 	if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
1392 		return HRTIMER_NORESTART;
1393 
1394 	atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
1395 	null_restart_queue_async(nullb);
1396 
1397 	hrtimer_forward_now(&nullb->bw_timer, timer_interval);
1398 
1399 	return HRTIMER_RESTART;
1400 }
1401 
1402 static void nullb_setup_bwtimer(struct nullb *nullb)
1403 {
1404 	ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1405 
1406 	hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1407 	nullb->bw_timer.function = nullb_bwtimer_fn;
1408 	atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
1409 	hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
1410 }
1411 
1412 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
1413 {
1414 	int index = 0;
1415 
1416 	if (nullb->nr_queues != 1)
1417 		index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
1418 
1419 	return &nullb->queues[index];
1420 }
1421 
1422 static blk_qc_t null_submit_bio(struct bio *bio)
1423 {
1424 	sector_t sector = bio->bi_iter.bi_sector;
1425 	sector_t nr_sectors = bio_sectors(bio);
1426 	struct nullb *nullb = bio->bi_bdev->bd_disk->private_data;
1427 	struct nullb_queue *nq = nullb_to_queue(nullb);
1428 	struct nullb_cmd *cmd;
1429 
1430 	cmd = alloc_cmd(nq, 1);
1431 	cmd->bio = bio;
1432 
1433 	null_handle_cmd(cmd, sector, nr_sectors, bio_op(bio));
1434 	return BLK_QC_T_NONE;
1435 }
1436 
1437 static bool should_timeout_request(struct request *rq)
1438 {
1439 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1440 	if (g_timeout_str[0])
1441 		return should_fail(&null_timeout_attr, 1);
1442 #endif
1443 	return false;
1444 }
1445 
1446 static bool should_requeue_request(struct request *rq)
1447 {
1448 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1449 	if (g_requeue_str[0])
1450 		return should_fail(&null_requeue_attr, 1);
1451 #endif
1452 	return false;
1453 }
1454 
1455 static enum blk_eh_timer_return null_timeout_rq(struct request *rq, bool res)
1456 {
1457 	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1458 
1459 	pr_info("rq %p timed out\n", rq);
1460 
1461 	/*
1462 	 * If the device is marked as blocking (i.e. memory backed or zoned
1463 	 * device), the submission path may be blocked waiting for resources
1464 	 * and cause real timeouts. For these real timeouts, the submission
1465 	 * path will complete the request using blk_mq_complete_request().
1466 	 * Only fake timeouts need to execute blk_mq_complete_request() here.
1467 	 */
1468 	cmd->error = BLK_STS_TIMEOUT;
1469 	if (cmd->fake_timeout)
1470 		blk_mq_complete_request(rq);
1471 	return BLK_EH_DONE;
1472 }
1473 
1474 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1475 			 const struct blk_mq_queue_data *bd)
1476 {
1477 	struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
1478 	struct nullb_queue *nq = hctx->driver_data;
1479 	sector_t nr_sectors = blk_rq_sectors(bd->rq);
1480 	sector_t sector = blk_rq_pos(bd->rq);
1481 
1482 	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1483 
1484 	if (nq->dev->irqmode == NULL_IRQ_TIMER) {
1485 		hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1486 		cmd->timer.function = null_cmd_timer_expired;
1487 	}
1488 	cmd->rq = bd->rq;
1489 	cmd->error = BLK_STS_OK;
1490 	cmd->nq = nq;
1491 	cmd->fake_timeout = should_timeout_request(bd->rq);
1492 
1493 	blk_mq_start_request(bd->rq);
1494 
1495 	if (should_requeue_request(bd->rq)) {
1496 		/*
1497 		 * Alternate between hitting the core BUSY path, and the
1498 		 * driver driven requeue path
1499 		 */
1500 		nq->requeue_selection++;
1501 		if (nq->requeue_selection & 1)
1502 			return BLK_STS_RESOURCE;
1503 		else {
1504 			blk_mq_requeue_request(bd->rq, true);
1505 			return BLK_STS_OK;
1506 		}
1507 	}
1508 	if (cmd->fake_timeout)
1509 		return BLK_STS_OK;
1510 
1511 	return null_handle_cmd(cmd, sector, nr_sectors, req_op(bd->rq));
1512 }
1513 
1514 static void cleanup_queue(struct nullb_queue *nq)
1515 {
1516 	kfree(nq->tag_map);
1517 	kfree(nq->cmds);
1518 }
1519 
1520 static void cleanup_queues(struct nullb *nullb)
1521 {
1522 	int i;
1523 
1524 	for (i = 0; i < nullb->nr_queues; i++)
1525 		cleanup_queue(&nullb->queues[i]);
1526 
1527 	kfree(nullb->queues);
1528 }
1529 
1530 static void null_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1531 {
1532 	struct nullb_queue *nq = hctx->driver_data;
1533 	struct nullb *nullb = nq->dev->nullb;
1534 
1535 	nullb->nr_queues--;
1536 }
1537 
1538 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1539 {
1540 	init_waitqueue_head(&nq->wait);
1541 	nq->queue_depth = nullb->queue_depth;
1542 	nq->dev = nullb->dev;
1543 }
1544 
1545 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1546 			  unsigned int hctx_idx)
1547 {
1548 	struct nullb *nullb = hctx->queue->queuedata;
1549 	struct nullb_queue *nq;
1550 
1551 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1552 	if (g_init_hctx_str[0] && should_fail(&null_init_hctx_attr, 1))
1553 		return -EFAULT;
1554 #endif
1555 
1556 	nq = &nullb->queues[hctx_idx];
1557 	hctx->driver_data = nq;
1558 	null_init_queue(nullb, nq);
1559 	nullb->nr_queues++;
1560 
1561 	return 0;
1562 }
1563 
1564 static const struct blk_mq_ops null_mq_ops = {
1565 	.queue_rq       = null_queue_rq,
1566 	.complete	= null_complete_rq,
1567 	.timeout	= null_timeout_rq,
1568 	.init_hctx	= null_init_hctx,
1569 	.exit_hctx	= null_exit_hctx,
1570 };
1571 
1572 static void null_del_dev(struct nullb *nullb)
1573 {
1574 	struct nullb_device *dev;
1575 
1576 	if (!nullb)
1577 		return;
1578 
1579 	dev = nullb->dev;
1580 
1581 	ida_simple_remove(&nullb_indexes, nullb->index);
1582 
1583 	list_del_init(&nullb->list);
1584 
1585 	del_gendisk(nullb->disk);
1586 
1587 	if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1588 		hrtimer_cancel(&nullb->bw_timer);
1589 		atomic_long_set(&nullb->cur_bytes, LONG_MAX);
1590 		null_restart_queue_async(nullb);
1591 	}
1592 
1593 	blk_cleanup_queue(nullb->q);
1594 	if (dev->queue_mode == NULL_Q_MQ &&
1595 	    nullb->tag_set == &nullb->__tag_set)
1596 		blk_mq_free_tag_set(nullb->tag_set);
1597 	put_disk(nullb->disk);
1598 	cleanup_queues(nullb);
1599 	if (null_cache_active(nullb))
1600 		null_free_device_storage(nullb->dev, true);
1601 	kfree(nullb);
1602 	dev->nullb = NULL;
1603 }
1604 
1605 static void null_config_discard(struct nullb *nullb)
1606 {
1607 	if (nullb->dev->discard == false)
1608 		return;
1609 
1610 	if (!nullb->dev->memory_backed) {
1611 		nullb->dev->discard = false;
1612 		pr_info("discard option is ignored without memory backing\n");
1613 		return;
1614 	}
1615 
1616 	if (nullb->dev->zoned) {
1617 		nullb->dev->discard = false;
1618 		pr_info("discard option is ignored in zoned mode\n");
1619 		return;
1620 	}
1621 
1622 	nullb->q->limits.discard_granularity = nullb->dev->blocksize;
1623 	nullb->q->limits.discard_alignment = nullb->dev->blocksize;
1624 	blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
1625 	blk_queue_flag_set(QUEUE_FLAG_DISCARD, nullb->q);
1626 }
1627 
1628 static const struct block_device_operations null_bio_ops = {
1629 	.owner		= THIS_MODULE,
1630 	.submit_bio	= null_submit_bio,
1631 	.report_zones	= null_report_zones,
1632 };
1633 
1634 static const struct block_device_operations null_rq_ops = {
1635 	.owner		= THIS_MODULE,
1636 	.report_zones	= null_report_zones,
1637 };
1638 
1639 static int setup_commands(struct nullb_queue *nq)
1640 {
1641 	struct nullb_cmd *cmd;
1642 	int i, tag_size;
1643 
1644 	nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);
1645 	if (!nq->cmds)
1646 		return -ENOMEM;
1647 
1648 	tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
1649 	nq->tag_map = kcalloc(tag_size, sizeof(unsigned long), GFP_KERNEL);
1650 	if (!nq->tag_map) {
1651 		kfree(nq->cmds);
1652 		return -ENOMEM;
1653 	}
1654 
1655 	for (i = 0; i < nq->queue_depth; i++) {
1656 		cmd = &nq->cmds[i];
1657 		cmd->tag = -1U;
1658 	}
1659 
1660 	return 0;
1661 }
1662 
1663 static int setup_queues(struct nullb *nullb)
1664 {
1665 	nullb->queues = kcalloc(nr_cpu_ids, sizeof(struct nullb_queue),
1666 				GFP_KERNEL);
1667 	if (!nullb->queues)
1668 		return -ENOMEM;
1669 
1670 	nullb->queue_depth = nullb->dev->hw_queue_depth;
1671 
1672 	return 0;
1673 }
1674 
1675 static int init_driver_queues(struct nullb *nullb)
1676 {
1677 	struct nullb_queue *nq;
1678 	int i, ret = 0;
1679 
1680 	for (i = 0; i < nullb->dev->submit_queues; i++) {
1681 		nq = &nullb->queues[i];
1682 
1683 		null_init_queue(nullb, nq);
1684 
1685 		ret = setup_commands(nq);
1686 		if (ret)
1687 			return ret;
1688 		nullb->nr_queues++;
1689 	}
1690 	return 0;
1691 }
1692 
1693 static int null_gendisk_register(struct nullb *nullb)
1694 {
1695 	sector_t size = ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT;
1696 	struct gendisk *disk;
1697 
1698 	disk = nullb->disk = alloc_disk_node(1, nullb->dev->home_node);
1699 	if (!disk)
1700 		return -ENOMEM;
1701 	set_capacity(disk, size);
1702 
1703 	disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
1704 	disk->major		= null_major;
1705 	disk->first_minor	= nullb->index;
1706 	if (queue_is_mq(nullb->q))
1707 		disk->fops		= &null_rq_ops;
1708 	else
1709 		disk->fops		= &null_bio_ops;
1710 	disk->private_data	= nullb;
1711 	disk->queue		= nullb->q;
1712 	strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
1713 
1714 	if (nullb->dev->zoned) {
1715 		int ret = null_register_zoned_dev(nullb);
1716 
1717 		if (ret)
1718 			return ret;
1719 	}
1720 
1721 	add_disk(disk);
1722 	return 0;
1723 }
1724 
1725 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
1726 {
1727 	set->ops = &null_mq_ops;
1728 	set->nr_hw_queues = nullb ? nullb->dev->submit_queues :
1729 						g_submit_queues;
1730 	set->queue_depth = nullb ? nullb->dev->hw_queue_depth :
1731 						g_hw_queue_depth;
1732 	set->numa_node = nullb ? nullb->dev->home_node : g_home_node;
1733 	set->cmd_size	= sizeof(struct nullb_cmd);
1734 	set->flags = BLK_MQ_F_SHOULD_MERGE;
1735 	if (g_no_sched)
1736 		set->flags |= BLK_MQ_F_NO_SCHED;
1737 	if (g_shared_tag_bitmap)
1738 		set->flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1739 	set->driver_data = NULL;
1740 
1741 	if ((nullb && nullb->dev->blocking) || g_blocking)
1742 		set->flags |= BLK_MQ_F_BLOCKING;
1743 
1744 	return blk_mq_alloc_tag_set(set);
1745 }
1746 
1747 static int null_validate_conf(struct nullb_device *dev)
1748 {
1749 	dev->blocksize = round_down(dev->blocksize, 512);
1750 	dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
1751 
1752 	if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
1753 		if (dev->submit_queues != nr_online_nodes)
1754 			dev->submit_queues = nr_online_nodes;
1755 	} else if (dev->submit_queues > nr_cpu_ids)
1756 		dev->submit_queues = nr_cpu_ids;
1757 	else if (dev->submit_queues == 0)
1758 		dev->submit_queues = 1;
1759 
1760 	dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
1761 	dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
1762 
1763 	/* Do memory allocation, so set blocking */
1764 	if (dev->memory_backed)
1765 		dev->blocking = true;
1766 	else /* cache is meaningless */
1767 		dev->cache_size = 0;
1768 	dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
1769 						dev->cache_size);
1770 	dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
1771 	/* can not stop a queue */
1772 	if (dev->queue_mode == NULL_Q_BIO)
1773 		dev->mbps = 0;
1774 
1775 	if (dev->zoned &&
1776 	    (!dev->zone_size || !is_power_of_2(dev->zone_size))) {
1777 		pr_err("zone_size must be power-of-two\n");
1778 		return -EINVAL;
1779 	}
1780 
1781 	return 0;
1782 }
1783 
1784 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1785 static bool __null_setup_fault(struct fault_attr *attr, char *str)
1786 {
1787 	if (!str[0])
1788 		return true;
1789 
1790 	if (!setup_fault_attr(attr, str))
1791 		return false;
1792 
1793 	attr->verbose = 0;
1794 	return true;
1795 }
1796 #endif
1797 
1798 static bool null_setup_fault(void)
1799 {
1800 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1801 	if (!__null_setup_fault(&null_timeout_attr, g_timeout_str))
1802 		return false;
1803 	if (!__null_setup_fault(&null_requeue_attr, g_requeue_str))
1804 		return false;
1805 	if (!__null_setup_fault(&null_init_hctx_attr, g_init_hctx_str))
1806 		return false;
1807 #endif
1808 	return true;
1809 }
1810 
1811 static int null_add_dev(struct nullb_device *dev)
1812 {
1813 	struct nullb *nullb;
1814 	int rv;
1815 
1816 	rv = null_validate_conf(dev);
1817 	if (rv)
1818 		return rv;
1819 
1820 	nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
1821 	if (!nullb) {
1822 		rv = -ENOMEM;
1823 		goto out;
1824 	}
1825 	nullb->dev = dev;
1826 	dev->nullb = nullb;
1827 
1828 	spin_lock_init(&nullb->lock);
1829 
1830 	rv = setup_queues(nullb);
1831 	if (rv)
1832 		goto out_free_nullb;
1833 
1834 	if (dev->queue_mode == NULL_Q_MQ) {
1835 		if (shared_tags) {
1836 			nullb->tag_set = &tag_set;
1837 			rv = 0;
1838 		} else {
1839 			nullb->tag_set = &nullb->__tag_set;
1840 			rv = null_init_tag_set(nullb, nullb->tag_set);
1841 		}
1842 
1843 		if (rv)
1844 			goto out_cleanup_queues;
1845 
1846 		if (!null_setup_fault())
1847 			goto out_cleanup_queues;
1848 
1849 		nullb->tag_set->timeout = 5 * HZ;
1850 		nullb->q = blk_mq_init_queue_data(nullb->tag_set, nullb);
1851 		if (IS_ERR(nullb->q)) {
1852 			rv = -ENOMEM;
1853 			goto out_cleanup_tags;
1854 		}
1855 	} else if (dev->queue_mode == NULL_Q_BIO) {
1856 		nullb->q = blk_alloc_queue(dev->home_node);
1857 		if (!nullb->q) {
1858 			rv = -ENOMEM;
1859 			goto out_cleanup_queues;
1860 		}
1861 		rv = init_driver_queues(nullb);
1862 		if (rv)
1863 			goto out_cleanup_blk_queue;
1864 	}
1865 
1866 	if (dev->mbps) {
1867 		set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
1868 		nullb_setup_bwtimer(nullb);
1869 	}
1870 
1871 	if (dev->cache_size > 0) {
1872 		set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
1873 		blk_queue_write_cache(nullb->q, true, true);
1874 	}
1875 
1876 	if (dev->zoned) {
1877 		rv = null_init_zoned_dev(dev, nullb->q);
1878 		if (rv)
1879 			goto out_cleanup_blk_queue;
1880 	}
1881 
1882 	nullb->q->queuedata = nullb;
1883 	blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);
1884 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, nullb->q);
1885 
1886 	mutex_lock(&lock);
1887 	nullb->index = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
1888 	dev->index = nullb->index;
1889 	mutex_unlock(&lock);
1890 
1891 	blk_queue_logical_block_size(nullb->q, dev->blocksize);
1892 	blk_queue_physical_block_size(nullb->q, dev->blocksize);
1893 	if (!dev->max_sectors)
1894 		dev->max_sectors = queue_max_hw_sectors(nullb->q);
1895 	dev->max_sectors = min_t(unsigned int, dev->max_sectors,
1896 				 BLK_DEF_MAX_SECTORS);
1897 	blk_queue_max_hw_sectors(nullb->q, dev->max_sectors);
1898 
1899 	null_config_discard(nullb);
1900 
1901 	sprintf(nullb->disk_name, "nullb%d", nullb->index);
1902 
1903 	rv = null_gendisk_register(nullb);
1904 	if (rv)
1905 		goto out_cleanup_zone;
1906 
1907 	mutex_lock(&lock);
1908 	list_add_tail(&nullb->list, &nullb_list);
1909 	mutex_unlock(&lock);
1910 
1911 	return 0;
1912 out_cleanup_zone:
1913 	null_free_zoned_dev(dev);
1914 out_cleanup_blk_queue:
1915 	blk_cleanup_queue(nullb->q);
1916 out_cleanup_tags:
1917 	if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
1918 		blk_mq_free_tag_set(nullb->tag_set);
1919 out_cleanup_queues:
1920 	cleanup_queues(nullb);
1921 out_free_nullb:
1922 	kfree(nullb);
1923 	dev->nullb = NULL;
1924 out:
1925 	return rv;
1926 }
1927 
1928 static int __init null_init(void)
1929 {
1930 	int ret = 0;
1931 	unsigned int i;
1932 	struct nullb *nullb;
1933 	struct nullb_device *dev;
1934 
1935 	if (g_bs > PAGE_SIZE) {
1936 		pr_warn("invalid block size\n");
1937 		pr_warn("defaults block size to %lu\n", PAGE_SIZE);
1938 		g_bs = PAGE_SIZE;
1939 	}
1940 
1941 	if (g_max_sectors > BLK_DEF_MAX_SECTORS) {
1942 		pr_warn("invalid max sectors\n");
1943 		pr_warn("defaults max sectors to %u\n", BLK_DEF_MAX_SECTORS);
1944 		g_max_sectors = BLK_DEF_MAX_SECTORS;
1945 	}
1946 
1947 	if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
1948 		pr_err("invalid home_node value\n");
1949 		g_home_node = NUMA_NO_NODE;
1950 	}
1951 
1952 	if (g_queue_mode == NULL_Q_RQ) {
1953 		pr_err("legacy IO path no longer available\n");
1954 		return -EINVAL;
1955 	}
1956 	if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
1957 		if (g_submit_queues != nr_online_nodes) {
1958 			pr_warn("submit_queues param is set to %u.\n",
1959 							nr_online_nodes);
1960 			g_submit_queues = nr_online_nodes;
1961 		}
1962 	} else if (g_submit_queues > nr_cpu_ids)
1963 		g_submit_queues = nr_cpu_ids;
1964 	else if (g_submit_queues <= 0)
1965 		g_submit_queues = 1;
1966 
1967 	if (g_queue_mode == NULL_Q_MQ && shared_tags) {
1968 		ret = null_init_tag_set(NULL, &tag_set);
1969 		if (ret)
1970 			return ret;
1971 	}
1972 
1973 	config_group_init(&nullb_subsys.su_group);
1974 	mutex_init(&nullb_subsys.su_mutex);
1975 
1976 	ret = configfs_register_subsystem(&nullb_subsys);
1977 	if (ret)
1978 		goto err_tagset;
1979 
1980 	mutex_init(&lock);
1981 
1982 	null_major = register_blkdev(0, "nullb");
1983 	if (null_major < 0) {
1984 		ret = null_major;
1985 		goto err_conf;
1986 	}
1987 
1988 	for (i = 0; i < nr_devices; i++) {
1989 		dev = null_alloc_dev();
1990 		if (!dev) {
1991 			ret = -ENOMEM;
1992 			goto err_dev;
1993 		}
1994 		ret = null_add_dev(dev);
1995 		if (ret) {
1996 			null_free_dev(dev);
1997 			goto err_dev;
1998 		}
1999 	}
2000 
2001 	pr_info("module loaded\n");
2002 	return 0;
2003 
2004 err_dev:
2005 	while (!list_empty(&nullb_list)) {
2006 		nullb = list_entry(nullb_list.next, struct nullb, list);
2007 		dev = nullb->dev;
2008 		null_del_dev(nullb);
2009 		null_free_dev(dev);
2010 	}
2011 	unregister_blkdev(null_major, "nullb");
2012 err_conf:
2013 	configfs_unregister_subsystem(&nullb_subsys);
2014 err_tagset:
2015 	if (g_queue_mode == NULL_Q_MQ && shared_tags)
2016 		blk_mq_free_tag_set(&tag_set);
2017 	return ret;
2018 }
2019 
2020 static void __exit null_exit(void)
2021 {
2022 	struct nullb *nullb;
2023 
2024 	configfs_unregister_subsystem(&nullb_subsys);
2025 
2026 	unregister_blkdev(null_major, "nullb");
2027 
2028 	mutex_lock(&lock);
2029 	while (!list_empty(&nullb_list)) {
2030 		struct nullb_device *dev;
2031 
2032 		nullb = list_entry(nullb_list.next, struct nullb, list);
2033 		dev = nullb->dev;
2034 		null_del_dev(nullb);
2035 		null_free_dev(dev);
2036 	}
2037 	mutex_unlock(&lock);
2038 
2039 	if (g_queue_mode == NULL_Q_MQ && shared_tags)
2040 		blk_mq_free_tag_set(&tag_set);
2041 }
2042 
2043 module_init(null_init);
2044 module_exit(null_exit);
2045 
2046 MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
2047 MODULE_LICENSE("GPL");
2048