xref: /openbmc/linux/block/blk-sysfs.c (revision 4f205687)
1 /*
2  * Functions related to sysfs handling
3  */
4 #include <linux/kernel.h>
5 #include <linux/slab.h>
6 #include <linux/module.h>
7 #include <linux/bio.h>
8 #include <linux/blkdev.h>
9 #include <linux/backing-dev.h>
10 #include <linux/blktrace_api.h>
11 #include <linux/blk-mq.h>
12 #include <linux/blk-cgroup.h>
13 
14 #include "blk.h"
15 #include "blk-mq.h"
16 
17 struct queue_sysfs_entry {
18 	struct attribute attr;
19 	ssize_t (*show)(struct request_queue *, char *);
20 	ssize_t (*store)(struct request_queue *, const char *, size_t);
21 };
22 
23 static ssize_t
24 queue_var_show(unsigned long var, char *page)
25 {
26 	return sprintf(page, "%lu\n", var);
27 }
28 
29 static ssize_t
30 queue_var_store(unsigned long *var, const char *page, size_t count)
31 {
32 	int err;
33 	unsigned long v;
34 
35 	err = kstrtoul(page, 10, &v);
36 	if (err || v > UINT_MAX)
37 		return -EINVAL;
38 
39 	*var = v;
40 
41 	return count;
42 }
43 
44 static ssize_t queue_requests_show(struct request_queue *q, char *page)
45 {
46 	return queue_var_show(q->nr_requests, (page));
47 }
48 
49 static ssize_t
50 queue_requests_store(struct request_queue *q, const char *page, size_t count)
51 {
52 	unsigned long nr;
53 	int ret, err;
54 
55 	if (!q->request_fn && !q->mq_ops)
56 		return -EINVAL;
57 
58 	ret = queue_var_store(&nr, page, count);
59 	if (ret < 0)
60 		return ret;
61 
62 	if (nr < BLKDEV_MIN_RQ)
63 		nr = BLKDEV_MIN_RQ;
64 
65 	if (q->request_fn)
66 		err = blk_update_nr_requests(q, nr);
67 	else
68 		err = blk_mq_update_nr_requests(q, nr);
69 
70 	if (err)
71 		return err;
72 
73 	return ret;
74 }
75 
76 static ssize_t queue_ra_show(struct request_queue *q, char *page)
77 {
78 	unsigned long ra_kb = q->backing_dev_info.ra_pages <<
79 					(PAGE_SHIFT - 10);
80 
81 	return queue_var_show(ra_kb, (page));
82 }
83 
84 static ssize_t
85 queue_ra_store(struct request_queue *q, const char *page, size_t count)
86 {
87 	unsigned long ra_kb;
88 	ssize_t ret = queue_var_store(&ra_kb, page, count);
89 
90 	if (ret < 0)
91 		return ret;
92 
93 	q->backing_dev_info.ra_pages = ra_kb >> (PAGE_SHIFT - 10);
94 
95 	return ret;
96 }
97 
98 static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
99 {
100 	int max_sectors_kb = queue_max_sectors(q) >> 1;
101 
102 	return queue_var_show(max_sectors_kb, (page));
103 }
104 
105 static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
106 {
107 	return queue_var_show(queue_max_segments(q), (page));
108 }
109 
110 static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
111 {
112 	return queue_var_show(q->limits.max_integrity_segments, (page));
113 }
114 
115 static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)
116 {
117 	if (blk_queue_cluster(q))
118 		return queue_var_show(queue_max_segment_size(q), (page));
119 
120 	return queue_var_show(PAGE_SIZE, (page));
121 }
122 
123 static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)
124 {
125 	return queue_var_show(queue_logical_block_size(q), page);
126 }
127 
128 static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)
129 {
130 	return queue_var_show(queue_physical_block_size(q), page);
131 }
132 
133 static ssize_t queue_io_min_show(struct request_queue *q, char *page)
134 {
135 	return queue_var_show(queue_io_min(q), page);
136 }
137 
138 static ssize_t queue_io_opt_show(struct request_queue *q, char *page)
139 {
140 	return queue_var_show(queue_io_opt(q), page);
141 }
142 
143 static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)
144 {
145 	return queue_var_show(q->limits.discard_granularity, page);
146 }
147 
148 static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
149 {
150 
151 	return sprintf(page, "%llu\n",
152 		(unsigned long long)q->limits.max_hw_discard_sectors << 9);
153 }
154 
155 static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
156 {
157 	return sprintf(page, "%llu\n",
158 		       (unsigned long long)q->limits.max_discard_sectors << 9);
159 }
160 
161 static ssize_t queue_discard_max_store(struct request_queue *q,
162 				       const char *page, size_t count)
163 {
164 	unsigned long max_discard;
165 	ssize_t ret = queue_var_store(&max_discard, page, count);
166 
167 	if (ret < 0)
168 		return ret;
169 
170 	if (max_discard & (q->limits.discard_granularity - 1))
171 		return -EINVAL;
172 
173 	max_discard >>= 9;
174 	if (max_discard > UINT_MAX)
175 		return -EINVAL;
176 
177 	if (max_discard > q->limits.max_hw_discard_sectors)
178 		max_discard = q->limits.max_hw_discard_sectors;
179 
180 	q->limits.max_discard_sectors = max_discard;
181 	return ret;
182 }
183 
184 static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
185 {
186 	return queue_var_show(queue_discard_zeroes_data(q), page);
187 }
188 
189 static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
190 {
191 	return sprintf(page, "%llu\n",
192 		(unsigned long long)q->limits.max_write_same_sectors << 9);
193 }
194 
195 
196 static ssize_t
197 queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
198 {
199 	unsigned long max_sectors_kb,
200 		max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1,
201 			page_kb = 1 << (PAGE_SHIFT - 10);
202 	ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
203 
204 	if (ret < 0)
205 		return ret;
206 
207 	max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long)
208 					 q->limits.max_dev_sectors >> 1);
209 
210 	if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
211 		return -EINVAL;
212 
213 	spin_lock_irq(q->queue_lock);
214 	q->limits.max_sectors = max_sectors_kb << 1;
215 	spin_unlock_irq(q->queue_lock);
216 
217 	return ret;
218 }
219 
220 static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
221 {
222 	int max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1;
223 
224 	return queue_var_show(max_hw_sectors_kb, (page));
225 }
226 
227 #define QUEUE_SYSFS_BIT_FNS(name, flag, neg)				\
228 static ssize_t								\
229 queue_show_##name(struct request_queue *q, char *page)			\
230 {									\
231 	int bit;							\
232 	bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags);		\
233 	return queue_var_show(neg ? !bit : bit, page);			\
234 }									\
235 static ssize_t								\
236 queue_store_##name(struct request_queue *q, const char *page, size_t count) \
237 {									\
238 	unsigned long val;						\
239 	ssize_t ret;							\
240 	ret = queue_var_store(&val, page, count);			\
241 	if (ret < 0)							\
242 		 return ret;						\
243 	if (neg)							\
244 		val = !val;						\
245 									\
246 	spin_lock_irq(q->queue_lock);					\
247 	if (val)							\
248 		queue_flag_set(QUEUE_FLAG_##flag, q);			\
249 	else								\
250 		queue_flag_clear(QUEUE_FLAG_##flag, q);			\
251 	spin_unlock_irq(q->queue_lock);					\
252 	return ret;							\
253 }
254 
255 QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
256 QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
257 QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
258 #undef QUEUE_SYSFS_BIT_FNS
259 
260 static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
261 {
262 	return queue_var_show((blk_queue_nomerges(q) << 1) |
263 			       blk_queue_noxmerges(q), page);
264 }
265 
266 static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,
267 				    size_t count)
268 {
269 	unsigned long nm;
270 	ssize_t ret = queue_var_store(&nm, page, count);
271 
272 	if (ret < 0)
273 		return ret;
274 
275 	spin_lock_irq(q->queue_lock);
276 	queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
277 	queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
278 	if (nm == 2)
279 		queue_flag_set(QUEUE_FLAG_NOMERGES, q);
280 	else if (nm)
281 		queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
282 	spin_unlock_irq(q->queue_lock);
283 
284 	return ret;
285 }
286 
287 static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)
288 {
289 	bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);
290 	bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);
291 
292 	return queue_var_show(set << force, page);
293 }
294 
295 static ssize_t
296 queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
297 {
298 	ssize_t ret = -EINVAL;
299 #ifdef CONFIG_SMP
300 	unsigned long val;
301 
302 	ret = queue_var_store(&val, page, count);
303 	if (ret < 0)
304 		return ret;
305 
306 	spin_lock_irq(q->queue_lock);
307 	if (val == 2) {
308 		queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
309 		queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
310 	} else if (val == 1) {
311 		queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
312 		queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
313 	} else if (val == 0) {
314 		queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
315 		queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
316 	}
317 	spin_unlock_irq(q->queue_lock);
318 #endif
319 	return ret;
320 }
321 
322 static ssize_t queue_poll_show(struct request_queue *q, char *page)
323 {
324 	return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
325 }
326 
327 static ssize_t queue_poll_store(struct request_queue *q, const char *page,
328 				size_t count)
329 {
330 	unsigned long poll_on;
331 	ssize_t ret;
332 
333 	if (!q->mq_ops || !q->mq_ops->poll)
334 		return -EINVAL;
335 
336 	ret = queue_var_store(&poll_on, page, count);
337 	if (ret < 0)
338 		return ret;
339 
340 	spin_lock_irq(q->queue_lock);
341 	if (poll_on)
342 		queue_flag_set(QUEUE_FLAG_POLL, q);
343 	else
344 		queue_flag_clear(QUEUE_FLAG_POLL, q);
345 	spin_unlock_irq(q->queue_lock);
346 
347 	return ret;
348 }
349 
350 static ssize_t queue_wc_show(struct request_queue *q, char *page)
351 {
352 	if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
353 		return sprintf(page, "write back\n");
354 
355 	return sprintf(page, "write through\n");
356 }
357 
358 static ssize_t queue_wc_store(struct request_queue *q, const char *page,
359 			      size_t count)
360 {
361 	int set = -1;
362 
363 	if (!strncmp(page, "write back", 10))
364 		set = 1;
365 	else if (!strncmp(page, "write through", 13) ||
366 		 !strncmp(page, "none", 4))
367 		set = 0;
368 
369 	if (set == -1)
370 		return -EINVAL;
371 
372 	spin_lock_irq(q->queue_lock);
373 	if (set)
374 		queue_flag_set(QUEUE_FLAG_WC, q);
375 	else
376 		queue_flag_clear(QUEUE_FLAG_WC, q);
377 	spin_unlock_irq(q->queue_lock);
378 
379 	return count;
380 }
381 
382 static struct queue_sysfs_entry queue_requests_entry = {
383 	.attr = {.name = "nr_requests", .mode = S_IRUGO | S_IWUSR },
384 	.show = queue_requests_show,
385 	.store = queue_requests_store,
386 };
387 
388 static struct queue_sysfs_entry queue_ra_entry = {
389 	.attr = {.name = "read_ahead_kb", .mode = S_IRUGO | S_IWUSR },
390 	.show = queue_ra_show,
391 	.store = queue_ra_store,
392 };
393 
394 static struct queue_sysfs_entry queue_max_sectors_entry = {
395 	.attr = {.name = "max_sectors_kb", .mode = S_IRUGO | S_IWUSR },
396 	.show = queue_max_sectors_show,
397 	.store = queue_max_sectors_store,
398 };
399 
400 static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
401 	.attr = {.name = "max_hw_sectors_kb", .mode = S_IRUGO },
402 	.show = queue_max_hw_sectors_show,
403 };
404 
405 static struct queue_sysfs_entry queue_max_segments_entry = {
406 	.attr = {.name = "max_segments", .mode = S_IRUGO },
407 	.show = queue_max_segments_show,
408 };
409 
410 static struct queue_sysfs_entry queue_max_integrity_segments_entry = {
411 	.attr = {.name = "max_integrity_segments", .mode = S_IRUGO },
412 	.show = queue_max_integrity_segments_show,
413 };
414 
415 static struct queue_sysfs_entry queue_max_segment_size_entry = {
416 	.attr = {.name = "max_segment_size", .mode = S_IRUGO },
417 	.show = queue_max_segment_size_show,
418 };
419 
420 static struct queue_sysfs_entry queue_iosched_entry = {
421 	.attr = {.name = "scheduler", .mode = S_IRUGO | S_IWUSR },
422 	.show = elv_iosched_show,
423 	.store = elv_iosched_store,
424 };
425 
426 static struct queue_sysfs_entry queue_hw_sector_size_entry = {
427 	.attr = {.name = "hw_sector_size", .mode = S_IRUGO },
428 	.show = queue_logical_block_size_show,
429 };
430 
431 static struct queue_sysfs_entry queue_logical_block_size_entry = {
432 	.attr = {.name = "logical_block_size", .mode = S_IRUGO },
433 	.show = queue_logical_block_size_show,
434 };
435 
436 static struct queue_sysfs_entry queue_physical_block_size_entry = {
437 	.attr = {.name = "physical_block_size", .mode = S_IRUGO },
438 	.show = queue_physical_block_size_show,
439 };
440 
441 static struct queue_sysfs_entry queue_io_min_entry = {
442 	.attr = {.name = "minimum_io_size", .mode = S_IRUGO },
443 	.show = queue_io_min_show,
444 };
445 
446 static struct queue_sysfs_entry queue_io_opt_entry = {
447 	.attr = {.name = "optimal_io_size", .mode = S_IRUGO },
448 	.show = queue_io_opt_show,
449 };
450 
451 static struct queue_sysfs_entry queue_discard_granularity_entry = {
452 	.attr = {.name = "discard_granularity", .mode = S_IRUGO },
453 	.show = queue_discard_granularity_show,
454 };
455 
456 static struct queue_sysfs_entry queue_discard_max_hw_entry = {
457 	.attr = {.name = "discard_max_hw_bytes", .mode = S_IRUGO },
458 	.show = queue_discard_max_hw_show,
459 };
460 
461 static struct queue_sysfs_entry queue_discard_max_entry = {
462 	.attr = {.name = "discard_max_bytes", .mode = S_IRUGO | S_IWUSR },
463 	.show = queue_discard_max_show,
464 	.store = queue_discard_max_store,
465 };
466 
467 static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
468 	.attr = {.name = "discard_zeroes_data", .mode = S_IRUGO },
469 	.show = queue_discard_zeroes_data_show,
470 };
471 
472 static struct queue_sysfs_entry queue_write_same_max_entry = {
473 	.attr = {.name = "write_same_max_bytes", .mode = S_IRUGO },
474 	.show = queue_write_same_max_show,
475 };
476 
477 static struct queue_sysfs_entry queue_nonrot_entry = {
478 	.attr = {.name = "rotational", .mode = S_IRUGO | S_IWUSR },
479 	.show = queue_show_nonrot,
480 	.store = queue_store_nonrot,
481 };
482 
483 static struct queue_sysfs_entry queue_nomerges_entry = {
484 	.attr = {.name = "nomerges", .mode = S_IRUGO | S_IWUSR },
485 	.show = queue_nomerges_show,
486 	.store = queue_nomerges_store,
487 };
488 
489 static struct queue_sysfs_entry queue_rq_affinity_entry = {
490 	.attr = {.name = "rq_affinity", .mode = S_IRUGO | S_IWUSR },
491 	.show = queue_rq_affinity_show,
492 	.store = queue_rq_affinity_store,
493 };
494 
495 static struct queue_sysfs_entry queue_iostats_entry = {
496 	.attr = {.name = "iostats", .mode = S_IRUGO | S_IWUSR },
497 	.show = queue_show_iostats,
498 	.store = queue_store_iostats,
499 };
500 
501 static struct queue_sysfs_entry queue_random_entry = {
502 	.attr = {.name = "add_random", .mode = S_IRUGO | S_IWUSR },
503 	.show = queue_show_random,
504 	.store = queue_store_random,
505 };
506 
507 static struct queue_sysfs_entry queue_poll_entry = {
508 	.attr = {.name = "io_poll", .mode = S_IRUGO | S_IWUSR },
509 	.show = queue_poll_show,
510 	.store = queue_poll_store,
511 };
512 
513 static struct queue_sysfs_entry queue_wc_entry = {
514 	.attr = {.name = "write_cache", .mode = S_IRUGO | S_IWUSR },
515 	.show = queue_wc_show,
516 	.store = queue_wc_store,
517 };
518 
519 static struct attribute *default_attrs[] = {
520 	&queue_requests_entry.attr,
521 	&queue_ra_entry.attr,
522 	&queue_max_hw_sectors_entry.attr,
523 	&queue_max_sectors_entry.attr,
524 	&queue_max_segments_entry.attr,
525 	&queue_max_integrity_segments_entry.attr,
526 	&queue_max_segment_size_entry.attr,
527 	&queue_iosched_entry.attr,
528 	&queue_hw_sector_size_entry.attr,
529 	&queue_logical_block_size_entry.attr,
530 	&queue_physical_block_size_entry.attr,
531 	&queue_io_min_entry.attr,
532 	&queue_io_opt_entry.attr,
533 	&queue_discard_granularity_entry.attr,
534 	&queue_discard_max_entry.attr,
535 	&queue_discard_max_hw_entry.attr,
536 	&queue_discard_zeroes_data_entry.attr,
537 	&queue_write_same_max_entry.attr,
538 	&queue_nonrot_entry.attr,
539 	&queue_nomerges_entry.attr,
540 	&queue_rq_affinity_entry.attr,
541 	&queue_iostats_entry.attr,
542 	&queue_random_entry.attr,
543 	&queue_poll_entry.attr,
544 	&queue_wc_entry.attr,
545 	NULL,
546 };
547 
548 #define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
549 
550 static ssize_t
551 queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
552 {
553 	struct queue_sysfs_entry *entry = to_queue(attr);
554 	struct request_queue *q =
555 		container_of(kobj, struct request_queue, kobj);
556 	ssize_t res;
557 
558 	if (!entry->show)
559 		return -EIO;
560 	mutex_lock(&q->sysfs_lock);
561 	if (blk_queue_dying(q)) {
562 		mutex_unlock(&q->sysfs_lock);
563 		return -ENOENT;
564 	}
565 	res = entry->show(q, page);
566 	mutex_unlock(&q->sysfs_lock);
567 	return res;
568 }
569 
570 static ssize_t
571 queue_attr_store(struct kobject *kobj, struct attribute *attr,
572 		    const char *page, size_t length)
573 {
574 	struct queue_sysfs_entry *entry = to_queue(attr);
575 	struct request_queue *q;
576 	ssize_t res;
577 
578 	if (!entry->store)
579 		return -EIO;
580 
581 	q = container_of(kobj, struct request_queue, kobj);
582 	mutex_lock(&q->sysfs_lock);
583 	if (blk_queue_dying(q)) {
584 		mutex_unlock(&q->sysfs_lock);
585 		return -ENOENT;
586 	}
587 	res = entry->store(q, page, length);
588 	mutex_unlock(&q->sysfs_lock);
589 	return res;
590 }
591 
592 static void blk_free_queue_rcu(struct rcu_head *rcu_head)
593 {
594 	struct request_queue *q = container_of(rcu_head, struct request_queue,
595 					       rcu_head);
596 	kmem_cache_free(blk_requestq_cachep, q);
597 }
598 
599 /**
600  * blk_release_queue: - release a &struct request_queue when it is no longer needed
601  * @kobj:    the kobj belonging to the request queue to be released
602  *
603  * Description:
604  *     blk_release_queue is the pair to blk_init_queue() or
605  *     blk_queue_make_request().  It should be called when a request queue is
606  *     being released; typically when a block device is being de-registered.
607  *     Currently, its primary task it to free all the &struct request
608  *     structures that were allocated to the queue and the queue itself.
609  *
610  * Note:
611  *     The low level driver must have finished any outstanding requests first
612  *     via blk_cleanup_queue().
613  **/
614 static void blk_release_queue(struct kobject *kobj)
615 {
616 	struct request_queue *q =
617 		container_of(kobj, struct request_queue, kobj);
618 
619 	bdi_exit(&q->backing_dev_info);
620 	blkcg_exit_queue(q);
621 
622 	if (q->elevator) {
623 		spin_lock_irq(q->queue_lock);
624 		ioc_clear_queue(q);
625 		spin_unlock_irq(q->queue_lock);
626 		elevator_exit(q->elevator);
627 	}
628 
629 	blk_exit_rl(&q->root_rl);
630 
631 	if (q->queue_tags)
632 		__blk_queue_free_tags(q);
633 
634 	if (!q->mq_ops)
635 		blk_free_flush_queue(q->fq);
636 	else
637 		blk_mq_release(q);
638 
639 	blk_trace_shutdown(q);
640 
641 	if (q->bio_split)
642 		bioset_free(q->bio_split);
643 
644 	ida_simple_remove(&blk_queue_ida, q->id);
645 	call_rcu(&q->rcu_head, blk_free_queue_rcu);
646 }
647 
648 static const struct sysfs_ops queue_sysfs_ops = {
649 	.show	= queue_attr_show,
650 	.store	= queue_attr_store,
651 };
652 
653 struct kobj_type blk_queue_ktype = {
654 	.sysfs_ops	= &queue_sysfs_ops,
655 	.default_attrs	= default_attrs,
656 	.release	= blk_release_queue,
657 };
658 
659 int blk_register_queue(struct gendisk *disk)
660 {
661 	int ret;
662 	struct device *dev = disk_to_dev(disk);
663 	struct request_queue *q = disk->queue;
664 
665 	if (WARN_ON(!q))
666 		return -ENXIO;
667 
668 	/*
669 	 * SCSI probing may synchronously create and destroy a lot of
670 	 * request_queues for non-existent devices.  Shutting down a fully
671 	 * functional queue takes measureable wallclock time as RCU grace
672 	 * periods are involved.  To avoid excessive latency in these
673 	 * cases, a request_queue starts out in a degraded mode which is
674 	 * faster to shut down and is made fully functional here as
675 	 * request_queues for non-existent devices never get registered.
676 	 */
677 	if (!blk_queue_init_done(q)) {
678 		queue_flag_set_unlocked(QUEUE_FLAG_INIT_DONE, q);
679 		percpu_ref_switch_to_percpu(&q->q_usage_counter);
680 		blk_queue_bypass_end(q);
681 	}
682 
683 	ret = blk_trace_init_sysfs(dev);
684 	if (ret)
685 		return ret;
686 
687 	ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
688 	if (ret < 0) {
689 		blk_trace_remove_sysfs(dev);
690 		return ret;
691 	}
692 
693 	kobject_uevent(&q->kobj, KOBJ_ADD);
694 
695 	if (q->mq_ops)
696 		blk_mq_register_disk(disk);
697 
698 	if (!q->request_fn)
699 		return 0;
700 
701 	ret = elv_register_queue(q);
702 	if (ret) {
703 		kobject_uevent(&q->kobj, KOBJ_REMOVE);
704 		kobject_del(&q->kobj);
705 		blk_trace_remove_sysfs(dev);
706 		kobject_put(&dev->kobj);
707 		return ret;
708 	}
709 
710 	return 0;
711 }
712 
713 void blk_unregister_queue(struct gendisk *disk)
714 {
715 	struct request_queue *q = disk->queue;
716 
717 	if (WARN_ON(!q))
718 		return;
719 
720 	if (q->mq_ops)
721 		blk_mq_unregister_disk(disk);
722 
723 	if (q->request_fn)
724 		elv_unregister_queue(q);
725 
726 	kobject_uevent(&q->kobj, KOBJ_REMOVE);
727 	kobject_del(&q->kobj);
728 	blk_trace_remove_sysfs(disk_to_dev(disk));
729 	kobject_put(&disk_to_dev(disk)->kobj);
730 }
731