xref: /openbmc/linux/block/blk-mq-debugfs.c (revision 6aa7de05)
1 /*
2  * Copyright (C) 2017 Facebook
3  *
4  * This program is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU General Public
6  * License v2 as published by the Free Software Foundation.
7  *
8  * This program is distributed in the hope that it will be useful,
9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
11  * General Public License for more details.
12  *
13  * You should have received a copy of the GNU General Public License
14  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
15  */
16 
17 #include <linux/kernel.h>
18 #include <linux/blkdev.h>
19 #include <linux/debugfs.h>
20 
21 #include <linux/blk-mq.h>
22 #include "blk.h"
23 #include "blk-mq.h"
24 #include "blk-mq-debugfs.h"
25 #include "blk-mq-tag.h"
26 
27 static int blk_flags_show(struct seq_file *m, const unsigned long flags,
28 			  const char *const *flag_name, int flag_name_count)
29 {
30 	bool sep = false;
31 	int i;
32 
33 	for (i = 0; i < sizeof(flags) * BITS_PER_BYTE; i++) {
34 		if (!(flags & BIT(i)))
35 			continue;
36 		if (sep)
37 			seq_puts(m, "|");
38 		sep = true;
39 		if (i < flag_name_count && flag_name[i])
40 			seq_puts(m, flag_name[i]);
41 		else
42 			seq_printf(m, "%d", i);
43 	}
44 	return 0;
45 }
46 
47 #define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name
48 static const char *const blk_queue_flag_name[] = {
49 	QUEUE_FLAG_NAME(QUEUED),
50 	QUEUE_FLAG_NAME(STOPPED),
51 	QUEUE_FLAG_NAME(DYING),
52 	QUEUE_FLAG_NAME(BYPASS),
53 	QUEUE_FLAG_NAME(BIDI),
54 	QUEUE_FLAG_NAME(NOMERGES),
55 	QUEUE_FLAG_NAME(SAME_COMP),
56 	QUEUE_FLAG_NAME(FAIL_IO),
57 	QUEUE_FLAG_NAME(STACKABLE),
58 	QUEUE_FLAG_NAME(NONROT),
59 	QUEUE_FLAG_NAME(IO_STAT),
60 	QUEUE_FLAG_NAME(DISCARD),
61 	QUEUE_FLAG_NAME(NOXMERGES),
62 	QUEUE_FLAG_NAME(ADD_RANDOM),
63 	QUEUE_FLAG_NAME(SECERASE),
64 	QUEUE_FLAG_NAME(SAME_FORCE),
65 	QUEUE_FLAG_NAME(DEAD),
66 	QUEUE_FLAG_NAME(INIT_DONE),
67 	QUEUE_FLAG_NAME(NO_SG_MERGE),
68 	QUEUE_FLAG_NAME(POLL),
69 	QUEUE_FLAG_NAME(WC),
70 	QUEUE_FLAG_NAME(FUA),
71 	QUEUE_FLAG_NAME(FLUSH_NQ),
72 	QUEUE_FLAG_NAME(DAX),
73 	QUEUE_FLAG_NAME(STATS),
74 	QUEUE_FLAG_NAME(POLL_STATS),
75 	QUEUE_FLAG_NAME(REGISTERED),
76 	QUEUE_FLAG_NAME(SCSI_PASSTHROUGH),
77 	QUEUE_FLAG_NAME(QUIESCED),
78 };
79 #undef QUEUE_FLAG_NAME
80 
81 static int queue_state_show(void *data, struct seq_file *m)
82 {
83 	struct request_queue *q = data;
84 
85 	blk_flags_show(m, q->queue_flags, blk_queue_flag_name,
86 		       ARRAY_SIZE(blk_queue_flag_name));
87 	seq_puts(m, "\n");
88 	return 0;
89 }
90 
91 static ssize_t queue_state_write(void *data, const char __user *buf,
92 				 size_t count, loff_t *ppos)
93 {
94 	struct request_queue *q = data;
95 	char opbuf[16] = { }, *op;
96 
97 	/*
98 	 * The "state" attribute is removed after blk_cleanup_queue() has called
99 	 * blk_mq_free_queue(). Return if QUEUE_FLAG_DEAD has been set to avoid
100 	 * triggering a use-after-free.
101 	 */
102 	if (blk_queue_dead(q))
103 		return -ENOENT;
104 
105 	if (count >= sizeof(opbuf)) {
106 		pr_err("%s: operation too long\n", __func__);
107 		goto inval;
108 	}
109 
110 	if (copy_from_user(opbuf, buf, count))
111 		return -EFAULT;
112 	op = strstrip(opbuf);
113 	if (strcmp(op, "run") == 0) {
114 		blk_mq_run_hw_queues(q, true);
115 	} else if (strcmp(op, "start") == 0) {
116 		blk_mq_start_stopped_hw_queues(q, true);
117 	} else if (strcmp(op, "kick") == 0) {
118 		blk_mq_kick_requeue_list(q);
119 	} else {
120 		pr_err("%s: unsupported operation '%s'\n", __func__, op);
121 inval:
122 		pr_err("%s: use 'run', 'start' or 'kick'\n", __func__);
123 		return -EINVAL;
124 	}
125 	return count;
126 }
127 
128 static void print_stat(struct seq_file *m, struct blk_rq_stat *stat)
129 {
130 	if (stat->nr_samples) {
131 		seq_printf(m, "samples=%d, mean=%lld, min=%llu, max=%llu",
132 			   stat->nr_samples, stat->mean, stat->min, stat->max);
133 	} else {
134 		seq_puts(m, "samples=0");
135 	}
136 }
137 
138 static int queue_write_hint_show(void *data, struct seq_file *m)
139 {
140 	struct request_queue *q = data;
141 	int i;
142 
143 	for (i = 0; i < BLK_MAX_WRITE_HINTS; i++)
144 		seq_printf(m, "hint%d: %llu\n", i, q->write_hints[i]);
145 
146 	return 0;
147 }
148 
149 static ssize_t queue_write_hint_store(void *data, const char __user *buf,
150 				      size_t count, loff_t *ppos)
151 {
152 	struct request_queue *q = data;
153 	int i;
154 
155 	for (i = 0; i < BLK_MAX_WRITE_HINTS; i++)
156 		q->write_hints[i] = 0;
157 
158 	return count;
159 }
160 
161 static int queue_poll_stat_show(void *data, struct seq_file *m)
162 {
163 	struct request_queue *q = data;
164 	int bucket;
165 
166 	for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS/2; bucket++) {
167 		seq_printf(m, "read  (%d Bytes): ", 1 << (9+bucket));
168 		print_stat(m, &q->poll_stat[2*bucket]);
169 		seq_puts(m, "\n");
170 
171 		seq_printf(m, "write (%d Bytes): ",  1 << (9+bucket));
172 		print_stat(m, &q->poll_stat[2*bucket+1]);
173 		seq_puts(m, "\n");
174 	}
175 	return 0;
176 }
177 
178 #define HCTX_STATE_NAME(name) [BLK_MQ_S_##name] = #name
179 static const char *const hctx_state_name[] = {
180 	HCTX_STATE_NAME(STOPPED),
181 	HCTX_STATE_NAME(TAG_ACTIVE),
182 	HCTX_STATE_NAME(SCHED_RESTART),
183 	HCTX_STATE_NAME(TAG_WAITING),
184 	HCTX_STATE_NAME(START_ON_RUN),
185 };
186 #undef HCTX_STATE_NAME
187 
188 static int hctx_state_show(void *data, struct seq_file *m)
189 {
190 	struct blk_mq_hw_ctx *hctx = data;
191 
192 	blk_flags_show(m, hctx->state, hctx_state_name,
193 		       ARRAY_SIZE(hctx_state_name));
194 	seq_puts(m, "\n");
195 	return 0;
196 }
197 
198 #define BLK_TAG_ALLOC_NAME(name) [BLK_TAG_ALLOC_##name] = #name
199 static const char *const alloc_policy_name[] = {
200 	BLK_TAG_ALLOC_NAME(FIFO),
201 	BLK_TAG_ALLOC_NAME(RR),
202 };
203 #undef BLK_TAG_ALLOC_NAME
204 
205 #define HCTX_FLAG_NAME(name) [ilog2(BLK_MQ_F_##name)] = #name
206 static const char *const hctx_flag_name[] = {
207 	HCTX_FLAG_NAME(SHOULD_MERGE),
208 	HCTX_FLAG_NAME(TAG_SHARED),
209 	HCTX_FLAG_NAME(SG_MERGE),
210 	HCTX_FLAG_NAME(BLOCKING),
211 	HCTX_FLAG_NAME(NO_SCHED),
212 };
213 #undef HCTX_FLAG_NAME
214 
215 static int hctx_flags_show(void *data, struct seq_file *m)
216 {
217 	struct blk_mq_hw_ctx *hctx = data;
218 	const int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(hctx->flags);
219 
220 	seq_puts(m, "alloc_policy=");
221 	if (alloc_policy < ARRAY_SIZE(alloc_policy_name) &&
222 	    alloc_policy_name[alloc_policy])
223 		seq_puts(m, alloc_policy_name[alloc_policy]);
224 	else
225 		seq_printf(m, "%d", alloc_policy);
226 	seq_puts(m, " ");
227 	blk_flags_show(m,
228 		       hctx->flags ^ BLK_ALLOC_POLICY_TO_MQ_FLAG(alloc_policy),
229 		       hctx_flag_name, ARRAY_SIZE(hctx_flag_name));
230 	seq_puts(m, "\n");
231 	return 0;
232 }
233 
234 #define REQ_OP_NAME(name) [REQ_OP_##name] = #name
235 static const char *const op_name[] = {
236 	REQ_OP_NAME(READ),
237 	REQ_OP_NAME(WRITE),
238 	REQ_OP_NAME(FLUSH),
239 	REQ_OP_NAME(DISCARD),
240 	REQ_OP_NAME(ZONE_REPORT),
241 	REQ_OP_NAME(SECURE_ERASE),
242 	REQ_OP_NAME(ZONE_RESET),
243 	REQ_OP_NAME(WRITE_SAME),
244 	REQ_OP_NAME(WRITE_ZEROES),
245 	REQ_OP_NAME(SCSI_IN),
246 	REQ_OP_NAME(SCSI_OUT),
247 	REQ_OP_NAME(DRV_IN),
248 	REQ_OP_NAME(DRV_OUT),
249 };
250 #undef REQ_OP_NAME
251 
252 #define CMD_FLAG_NAME(name) [__REQ_##name] = #name
253 static const char *const cmd_flag_name[] = {
254 	CMD_FLAG_NAME(FAILFAST_DEV),
255 	CMD_FLAG_NAME(FAILFAST_TRANSPORT),
256 	CMD_FLAG_NAME(FAILFAST_DRIVER),
257 	CMD_FLAG_NAME(SYNC),
258 	CMD_FLAG_NAME(META),
259 	CMD_FLAG_NAME(PRIO),
260 	CMD_FLAG_NAME(NOMERGE),
261 	CMD_FLAG_NAME(IDLE),
262 	CMD_FLAG_NAME(INTEGRITY),
263 	CMD_FLAG_NAME(FUA),
264 	CMD_FLAG_NAME(PREFLUSH),
265 	CMD_FLAG_NAME(RAHEAD),
266 	CMD_FLAG_NAME(BACKGROUND),
267 	CMD_FLAG_NAME(NOUNMAP),
268 	CMD_FLAG_NAME(NOWAIT),
269 };
270 #undef CMD_FLAG_NAME
271 
272 #define RQF_NAME(name) [ilog2((__force u32)RQF_##name)] = #name
273 static const char *const rqf_name[] = {
274 	RQF_NAME(SORTED),
275 	RQF_NAME(STARTED),
276 	RQF_NAME(QUEUED),
277 	RQF_NAME(SOFTBARRIER),
278 	RQF_NAME(FLUSH_SEQ),
279 	RQF_NAME(MIXED_MERGE),
280 	RQF_NAME(MQ_INFLIGHT),
281 	RQF_NAME(DONTPREP),
282 	RQF_NAME(PREEMPT),
283 	RQF_NAME(COPY_USER),
284 	RQF_NAME(FAILED),
285 	RQF_NAME(QUIET),
286 	RQF_NAME(ELVPRIV),
287 	RQF_NAME(IO_STAT),
288 	RQF_NAME(ALLOCED),
289 	RQF_NAME(PM),
290 	RQF_NAME(HASHED),
291 	RQF_NAME(STATS),
292 	RQF_NAME(SPECIAL_PAYLOAD),
293 };
294 #undef RQF_NAME
295 
296 #define RQAF_NAME(name) [REQ_ATOM_##name] = #name
297 static const char *const rqaf_name[] = {
298 	RQAF_NAME(COMPLETE),
299 	RQAF_NAME(STARTED),
300 	RQAF_NAME(POLL_SLEPT),
301 };
302 #undef RQAF_NAME
303 
304 int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq)
305 {
306 	const struct blk_mq_ops *const mq_ops = rq->q->mq_ops;
307 	const unsigned int op = rq->cmd_flags & REQ_OP_MASK;
308 
309 	seq_printf(m, "%p {.op=", rq);
310 	if (op < ARRAY_SIZE(op_name) && op_name[op])
311 		seq_printf(m, "%s", op_name[op]);
312 	else
313 		seq_printf(m, "%d", op);
314 	seq_puts(m, ", .cmd_flags=");
315 	blk_flags_show(m, rq->cmd_flags & ~REQ_OP_MASK, cmd_flag_name,
316 		       ARRAY_SIZE(cmd_flag_name));
317 	seq_puts(m, ", .rq_flags=");
318 	blk_flags_show(m, (__force unsigned int)rq->rq_flags, rqf_name,
319 		       ARRAY_SIZE(rqf_name));
320 	seq_puts(m, ", .atomic_flags=");
321 	blk_flags_show(m, rq->atomic_flags, rqaf_name, ARRAY_SIZE(rqaf_name));
322 	seq_printf(m, ", .tag=%d, .internal_tag=%d", rq->tag,
323 		   rq->internal_tag);
324 	if (mq_ops->show_rq)
325 		mq_ops->show_rq(m, rq);
326 	seq_puts(m, "}\n");
327 	return 0;
328 }
329 EXPORT_SYMBOL_GPL(__blk_mq_debugfs_rq_show);
330 
331 int blk_mq_debugfs_rq_show(struct seq_file *m, void *v)
332 {
333 	return __blk_mq_debugfs_rq_show(m, list_entry_rq(v));
334 }
335 EXPORT_SYMBOL_GPL(blk_mq_debugfs_rq_show);
336 
337 static void *queue_requeue_list_start(struct seq_file *m, loff_t *pos)
338 	__acquires(&q->requeue_lock)
339 {
340 	struct request_queue *q = m->private;
341 
342 	spin_lock_irq(&q->requeue_lock);
343 	return seq_list_start(&q->requeue_list, *pos);
344 }
345 
346 static void *queue_requeue_list_next(struct seq_file *m, void *v, loff_t *pos)
347 {
348 	struct request_queue *q = m->private;
349 
350 	return seq_list_next(v, &q->requeue_list, pos);
351 }
352 
353 static void queue_requeue_list_stop(struct seq_file *m, void *v)
354 	__releases(&q->requeue_lock)
355 {
356 	struct request_queue *q = m->private;
357 
358 	spin_unlock_irq(&q->requeue_lock);
359 }
360 
361 static const struct seq_operations queue_requeue_list_seq_ops = {
362 	.start	= queue_requeue_list_start,
363 	.next	= queue_requeue_list_next,
364 	.stop	= queue_requeue_list_stop,
365 	.show	= blk_mq_debugfs_rq_show,
366 };
367 
368 static void *hctx_dispatch_start(struct seq_file *m, loff_t *pos)
369 	__acquires(&hctx->lock)
370 {
371 	struct blk_mq_hw_ctx *hctx = m->private;
372 
373 	spin_lock(&hctx->lock);
374 	return seq_list_start(&hctx->dispatch, *pos);
375 }
376 
377 static void *hctx_dispatch_next(struct seq_file *m, void *v, loff_t *pos)
378 {
379 	struct blk_mq_hw_ctx *hctx = m->private;
380 
381 	return seq_list_next(v, &hctx->dispatch, pos);
382 }
383 
384 static void hctx_dispatch_stop(struct seq_file *m, void *v)
385 	__releases(&hctx->lock)
386 {
387 	struct blk_mq_hw_ctx *hctx = m->private;
388 
389 	spin_unlock(&hctx->lock);
390 }
391 
392 static const struct seq_operations hctx_dispatch_seq_ops = {
393 	.start	= hctx_dispatch_start,
394 	.next	= hctx_dispatch_next,
395 	.stop	= hctx_dispatch_stop,
396 	.show	= blk_mq_debugfs_rq_show,
397 };
398 
399 struct show_busy_params {
400 	struct seq_file		*m;
401 	struct blk_mq_hw_ctx	*hctx;
402 };
403 
404 /*
405  * Note: the state of a request may change while this function is in progress,
406  * e.g. due to a concurrent blk_mq_finish_request() call.
407  */
408 static void hctx_show_busy_rq(struct request *rq, void *data, bool reserved)
409 {
410 	const struct show_busy_params *params = data;
411 
412 	if (blk_mq_map_queue(rq->q, rq->mq_ctx->cpu) == params->hctx &&
413 	    test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
414 		__blk_mq_debugfs_rq_show(params->m,
415 					 list_entry_rq(&rq->queuelist));
416 }
417 
418 static int hctx_busy_show(void *data, struct seq_file *m)
419 {
420 	struct blk_mq_hw_ctx *hctx = data;
421 	struct show_busy_params params = { .m = m, .hctx = hctx };
422 
423 	blk_mq_tagset_busy_iter(hctx->queue->tag_set, hctx_show_busy_rq,
424 				&params);
425 
426 	return 0;
427 }
428 
429 static int hctx_ctx_map_show(void *data, struct seq_file *m)
430 {
431 	struct blk_mq_hw_ctx *hctx = data;
432 
433 	sbitmap_bitmap_show(&hctx->ctx_map, m);
434 	return 0;
435 }
436 
437 static void blk_mq_debugfs_tags_show(struct seq_file *m,
438 				     struct blk_mq_tags *tags)
439 {
440 	seq_printf(m, "nr_tags=%u\n", tags->nr_tags);
441 	seq_printf(m, "nr_reserved_tags=%u\n", tags->nr_reserved_tags);
442 	seq_printf(m, "active_queues=%d\n",
443 		   atomic_read(&tags->active_queues));
444 
445 	seq_puts(m, "\nbitmap_tags:\n");
446 	sbitmap_queue_show(&tags->bitmap_tags, m);
447 
448 	if (tags->nr_reserved_tags) {
449 		seq_puts(m, "\nbreserved_tags:\n");
450 		sbitmap_queue_show(&tags->breserved_tags, m);
451 	}
452 }
453 
454 static int hctx_tags_show(void *data, struct seq_file *m)
455 {
456 	struct blk_mq_hw_ctx *hctx = data;
457 	struct request_queue *q = hctx->queue;
458 	int res;
459 
460 	res = mutex_lock_interruptible(&q->sysfs_lock);
461 	if (res)
462 		goto out;
463 	if (hctx->tags)
464 		blk_mq_debugfs_tags_show(m, hctx->tags);
465 	mutex_unlock(&q->sysfs_lock);
466 
467 out:
468 	return res;
469 }
470 
471 static int hctx_tags_bitmap_show(void *data, struct seq_file *m)
472 {
473 	struct blk_mq_hw_ctx *hctx = data;
474 	struct request_queue *q = hctx->queue;
475 	int res;
476 
477 	res = mutex_lock_interruptible(&q->sysfs_lock);
478 	if (res)
479 		goto out;
480 	if (hctx->tags)
481 		sbitmap_bitmap_show(&hctx->tags->bitmap_tags.sb, m);
482 	mutex_unlock(&q->sysfs_lock);
483 
484 out:
485 	return res;
486 }
487 
488 static int hctx_sched_tags_show(void *data, struct seq_file *m)
489 {
490 	struct blk_mq_hw_ctx *hctx = data;
491 	struct request_queue *q = hctx->queue;
492 	int res;
493 
494 	res = mutex_lock_interruptible(&q->sysfs_lock);
495 	if (res)
496 		goto out;
497 	if (hctx->sched_tags)
498 		blk_mq_debugfs_tags_show(m, hctx->sched_tags);
499 	mutex_unlock(&q->sysfs_lock);
500 
501 out:
502 	return res;
503 }
504 
505 static int hctx_sched_tags_bitmap_show(void *data, struct seq_file *m)
506 {
507 	struct blk_mq_hw_ctx *hctx = data;
508 	struct request_queue *q = hctx->queue;
509 	int res;
510 
511 	res = mutex_lock_interruptible(&q->sysfs_lock);
512 	if (res)
513 		goto out;
514 	if (hctx->sched_tags)
515 		sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags.sb, m);
516 	mutex_unlock(&q->sysfs_lock);
517 
518 out:
519 	return res;
520 }
521 
522 static int hctx_io_poll_show(void *data, struct seq_file *m)
523 {
524 	struct blk_mq_hw_ctx *hctx = data;
525 
526 	seq_printf(m, "considered=%lu\n", hctx->poll_considered);
527 	seq_printf(m, "invoked=%lu\n", hctx->poll_invoked);
528 	seq_printf(m, "success=%lu\n", hctx->poll_success);
529 	return 0;
530 }
531 
532 static ssize_t hctx_io_poll_write(void *data, const char __user *buf,
533 				  size_t count, loff_t *ppos)
534 {
535 	struct blk_mq_hw_ctx *hctx = data;
536 
537 	hctx->poll_considered = hctx->poll_invoked = hctx->poll_success = 0;
538 	return count;
539 }
540 
541 static int hctx_dispatched_show(void *data, struct seq_file *m)
542 {
543 	struct blk_mq_hw_ctx *hctx = data;
544 	int i;
545 
546 	seq_printf(m, "%8u\t%lu\n", 0U, hctx->dispatched[0]);
547 
548 	for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER - 1; i++) {
549 		unsigned int d = 1U << (i - 1);
550 
551 		seq_printf(m, "%8u\t%lu\n", d, hctx->dispatched[i]);
552 	}
553 
554 	seq_printf(m, "%8u+\t%lu\n", 1U << (i - 1), hctx->dispatched[i]);
555 	return 0;
556 }
557 
558 static ssize_t hctx_dispatched_write(void *data, const char __user *buf,
559 				     size_t count, loff_t *ppos)
560 {
561 	struct blk_mq_hw_ctx *hctx = data;
562 	int i;
563 
564 	for (i = 0; i < BLK_MQ_MAX_DISPATCH_ORDER; i++)
565 		hctx->dispatched[i] = 0;
566 	return count;
567 }
568 
569 static int hctx_queued_show(void *data, struct seq_file *m)
570 {
571 	struct blk_mq_hw_ctx *hctx = data;
572 
573 	seq_printf(m, "%lu\n", hctx->queued);
574 	return 0;
575 }
576 
577 static ssize_t hctx_queued_write(void *data, const char __user *buf,
578 				 size_t count, loff_t *ppos)
579 {
580 	struct blk_mq_hw_ctx *hctx = data;
581 
582 	hctx->queued = 0;
583 	return count;
584 }
585 
586 static int hctx_run_show(void *data, struct seq_file *m)
587 {
588 	struct blk_mq_hw_ctx *hctx = data;
589 
590 	seq_printf(m, "%lu\n", hctx->run);
591 	return 0;
592 }
593 
594 static ssize_t hctx_run_write(void *data, const char __user *buf, size_t count,
595 			      loff_t *ppos)
596 {
597 	struct blk_mq_hw_ctx *hctx = data;
598 
599 	hctx->run = 0;
600 	return count;
601 }
602 
603 static int hctx_active_show(void *data, struct seq_file *m)
604 {
605 	struct blk_mq_hw_ctx *hctx = data;
606 
607 	seq_printf(m, "%d\n", atomic_read(&hctx->nr_active));
608 	return 0;
609 }
610 
611 static void *ctx_rq_list_start(struct seq_file *m, loff_t *pos)
612 	__acquires(&ctx->lock)
613 {
614 	struct blk_mq_ctx *ctx = m->private;
615 
616 	spin_lock(&ctx->lock);
617 	return seq_list_start(&ctx->rq_list, *pos);
618 }
619 
620 static void *ctx_rq_list_next(struct seq_file *m, void *v, loff_t *pos)
621 {
622 	struct blk_mq_ctx *ctx = m->private;
623 
624 	return seq_list_next(v, &ctx->rq_list, pos);
625 }
626 
627 static void ctx_rq_list_stop(struct seq_file *m, void *v)
628 	__releases(&ctx->lock)
629 {
630 	struct blk_mq_ctx *ctx = m->private;
631 
632 	spin_unlock(&ctx->lock);
633 }
634 
635 static const struct seq_operations ctx_rq_list_seq_ops = {
636 	.start	= ctx_rq_list_start,
637 	.next	= ctx_rq_list_next,
638 	.stop	= ctx_rq_list_stop,
639 	.show	= blk_mq_debugfs_rq_show,
640 };
641 static int ctx_dispatched_show(void *data, struct seq_file *m)
642 {
643 	struct blk_mq_ctx *ctx = data;
644 
645 	seq_printf(m, "%lu %lu\n", ctx->rq_dispatched[1], ctx->rq_dispatched[0]);
646 	return 0;
647 }
648 
649 static ssize_t ctx_dispatched_write(void *data, const char __user *buf,
650 				    size_t count, loff_t *ppos)
651 {
652 	struct blk_mq_ctx *ctx = data;
653 
654 	ctx->rq_dispatched[0] = ctx->rq_dispatched[1] = 0;
655 	return count;
656 }
657 
658 static int ctx_merged_show(void *data, struct seq_file *m)
659 {
660 	struct blk_mq_ctx *ctx = data;
661 
662 	seq_printf(m, "%lu\n", ctx->rq_merged);
663 	return 0;
664 }
665 
666 static ssize_t ctx_merged_write(void *data, const char __user *buf,
667 				size_t count, loff_t *ppos)
668 {
669 	struct blk_mq_ctx *ctx = data;
670 
671 	ctx->rq_merged = 0;
672 	return count;
673 }
674 
675 static int ctx_completed_show(void *data, struct seq_file *m)
676 {
677 	struct blk_mq_ctx *ctx = data;
678 
679 	seq_printf(m, "%lu %lu\n", ctx->rq_completed[1], ctx->rq_completed[0]);
680 	return 0;
681 }
682 
683 static ssize_t ctx_completed_write(void *data, const char __user *buf,
684 				   size_t count, loff_t *ppos)
685 {
686 	struct blk_mq_ctx *ctx = data;
687 
688 	ctx->rq_completed[0] = ctx->rq_completed[1] = 0;
689 	return count;
690 }
691 
692 static int blk_mq_debugfs_show(struct seq_file *m, void *v)
693 {
694 	const struct blk_mq_debugfs_attr *attr = m->private;
695 	void *data = d_inode(m->file->f_path.dentry->d_parent)->i_private;
696 
697 	return attr->show(data, m);
698 }
699 
700 static ssize_t blk_mq_debugfs_write(struct file *file, const char __user *buf,
701 				    size_t count, loff_t *ppos)
702 {
703 	struct seq_file *m = file->private_data;
704 	const struct blk_mq_debugfs_attr *attr = m->private;
705 	void *data = d_inode(file->f_path.dentry->d_parent)->i_private;
706 
707 	if (!attr->write)
708 		return -EPERM;
709 
710 	return attr->write(data, buf, count, ppos);
711 }
712 
713 static int blk_mq_debugfs_open(struct inode *inode, struct file *file)
714 {
715 	const struct blk_mq_debugfs_attr *attr = inode->i_private;
716 	void *data = d_inode(file->f_path.dentry->d_parent)->i_private;
717 	struct seq_file *m;
718 	int ret;
719 
720 	if (attr->seq_ops) {
721 		ret = seq_open(file, attr->seq_ops);
722 		if (!ret) {
723 			m = file->private_data;
724 			m->private = data;
725 		}
726 		return ret;
727 	}
728 
729 	if (WARN_ON_ONCE(!attr->show))
730 		return -EPERM;
731 
732 	return single_open(file, blk_mq_debugfs_show, inode->i_private);
733 }
734 
735 static int blk_mq_debugfs_release(struct inode *inode, struct file *file)
736 {
737 	const struct blk_mq_debugfs_attr *attr = inode->i_private;
738 
739 	if (attr->show)
740 		return single_release(inode, file);
741 	else
742 		return seq_release(inode, file);
743 }
744 
745 static const struct file_operations blk_mq_debugfs_fops = {
746 	.open		= blk_mq_debugfs_open,
747 	.read		= seq_read,
748 	.write		= blk_mq_debugfs_write,
749 	.llseek		= seq_lseek,
750 	.release	= blk_mq_debugfs_release,
751 };
752 
753 static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = {
754 	{"poll_stat", 0400, queue_poll_stat_show},
755 	{"requeue_list", 0400, .seq_ops = &queue_requeue_list_seq_ops},
756 	{"state", 0600, queue_state_show, queue_state_write},
757 	{"write_hints", 0600, queue_write_hint_show, queue_write_hint_store},
758 	{},
759 };
760 
761 static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = {
762 	{"state", 0400, hctx_state_show},
763 	{"flags", 0400, hctx_flags_show},
764 	{"dispatch", 0400, .seq_ops = &hctx_dispatch_seq_ops},
765 	{"busy", 0400, hctx_busy_show},
766 	{"ctx_map", 0400, hctx_ctx_map_show},
767 	{"tags", 0400, hctx_tags_show},
768 	{"tags_bitmap", 0400, hctx_tags_bitmap_show},
769 	{"sched_tags", 0400, hctx_sched_tags_show},
770 	{"sched_tags_bitmap", 0400, hctx_sched_tags_bitmap_show},
771 	{"io_poll", 0600, hctx_io_poll_show, hctx_io_poll_write},
772 	{"dispatched", 0600, hctx_dispatched_show, hctx_dispatched_write},
773 	{"queued", 0600, hctx_queued_show, hctx_queued_write},
774 	{"run", 0600, hctx_run_show, hctx_run_write},
775 	{"active", 0400, hctx_active_show},
776 	{},
777 };
778 
779 static const struct blk_mq_debugfs_attr blk_mq_debugfs_ctx_attrs[] = {
780 	{"rq_list", 0400, .seq_ops = &ctx_rq_list_seq_ops},
781 	{"dispatched", 0600, ctx_dispatched_show, ctx_dispatched_write},
782 	{"merged", 0600, ctx_merged_show, ctx_merged_write},
783 	{"completed", 0600, ctx_completed_show, ctx_completed_write},
784 	{},
785 };
786 
787 static bool debugfs_create_files(struct dentry *parent, void *data,
788 				 const struct blk_mq_debugfs_attr *attr)
789 {
790 	d_inode(parent)->i_private = data;
791 
792 	for (; attr->name; attr++) {
793 		if (!debugfs_create_file(attr->name, attr->mode, parent,
794 					 (void *)attr, &blk_mq_debugfs_fops))
795 			return false;
796 	}
797 	return true;
798 }
799 
800 int blk_mq_debugfs_register(struct request_queue *q)
801 {
802 	struct blk_mq_hw_ctx *hctx;
803 	int i;
804 
805 	if (!blk_debugfs_root)
806 		return -ENOENT;
807 
808 	q->debugfs_dir = debugfs_create_dir(kobject_name(q->kobj.parent),
809 					    blk_debugfs_root);
810 	if (!q->debugfs_dir)
811 		return -ENOMEM;
812 
813 	if (!debugfs_create_files(q->debugfs_dir, q,
814 				  blk_mq_debugfs_queue_attrs))
815 		goto err;
816 
817 	/*
818 	 * blk_mq_init_sched() attempted to do this already, but q->debugfs_dir
819 	 * didn't exist yet (because we don't know what to name the directory
820 	 * until the queue is registered to a gendisk).
821 	 */
822 	if (q->elevator && !q->sched_debugfs_dir)
823 		blk_mq_debugfs_register_sched(q);
824 
825 	/* Similarly, blk_mq_init_hctx() couldn't do this previously. */
826 	queue_for_each_hw_ctx(q, hctx, i) {
827 		if (!hctx->debugfs_dir && blk_mq_debugfs_register_hctx(q, hctx))
828 			goto err;
829 		if (q->elevator && !hctx->sched_debugfs_dir &&
830 		    blk_mq_debugfs_register_sched_hctx(q, hctx))
831 			goto err;
832 	}
833 
834 	return 0;
835 
836 err:
837 	blk_mq_debugfs_unregister(q);
838 	return -ENOMEM;
839 }
840 
841 void blk_mq_debugfs_unregister(struct request_queue *q)
842 {
843 	debugfs_remove_recursive(q->debugfs_dir);
844 	q->sched_debugfs_dir = NULL;
845 	q->debugfs_dir = NULL;
846 }
847 
848 static int blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx,
849 				       struct blk_mq_ctx *ctx)
850 {
851 	struct dentry *ctx_dir;
852 	char name[20];
853 
854 	snprintf(name, sizeof(name), "cpu%u", ctx->cpu);
855 	ctx_dir = debugfs_create_dir(name, hctx->debugfs_dir);
856 	if (!ctx_dir)
857 		return -ENOMEM;
858 
859 	if (!debugfs_create_files(ctx_dir, ctx, blk_mq_debugfs_ctx_attrs))
860 		return -ENOMEM;
861 
862 	return 0;
863 }
864 
865 int blk_mq_debugfs_register_hctx(struct request_queue *q,
866 				 struct blk_mq_hw_ctx *hctx)
867 {
868 	struct blk_mq_ctx *ctx;
869 	char name[20];
870 	int i;
871 
872 	if (!q->debugfs_dir)
873 		return -ENOENT;
874 
875 	snprintf(name, sizeof(name), "hctx%u", hctx->queue_num);
876 	hctx->debugfs_dir = debugfs_create_dir(name, q->debugfs_dir);
877 	if (!hctx->debugfs_dir)
878 		return -ENOMEM;
879 
880 	if (!debugfs_create_files(hctx->debugfs_dir, hctx,
881 				  blk_mq_debugfs_hctx_attrs))
882 		goto err;
883 
884 	hctx_for_each_ctx(hctx, ctx, i) {
885 		if (blk_mq_debugfs_register_ctx(hctx, ctx))
886 			goto err;
887 	}
888 
889 	return 0;
890 
891 err:
892 	blk_mq_debugfs_unregister_hctx(hctx);
893 	return -ENOMEM;
894 }
895 
896 void blk_mq_debugfs_unregister_hctx(struct blk_mq_hw_ctx *hctx)
897 {
898 	debugfs_remove_recursive(hctx->debugfs_dir);
899 	hctx->sched_debugfs_dir = NULL;
900 	hctx->debugfs_dir = NULL;
901 }
902 
903 int blk_mq_debugfs_register_hctxs(struct request_queue *q)
904 {
905 	struct blk_mq_hw_ctx *hctx;
906 	int i;
907 
908 	queue_for_each_hw_ctx(q, hctx, i) {
909 		if (blk_mq_debugfs_register_hctx(q, hctx))
910 			return -ENOMEM;
911 	}
912 
913 	return 0;
914 }
915 
916 void blk_mq_debugfs_unregister_hctxs(struct request_queue *q)
917 {
918 	struct blk_mq_hw_ctx *hctx;
919 	int i;
920 
921 	queue_for_each_hw_ctx(q, hctx, i)
922 		blk_mq_debugfs_unregister_hctx(hctx);
923 }
924 
925 int blk_mq_debugfs_register_sched(struct request_queue *q)
926 {
927 	struct elevator_type *e = q->elevator->type;
928 
929 	if (!q->debugfs_dir)
930 		return -ENOENT;
931 
932 	if (!e->queue_debugfs_attrs)
933 		return 0;
934 
935 	q->sched_debugfs_dir = debugfs_create_dir("sched", q->debugfs_dir);
936 	if (!q->sched_debugfs_dir)
937 		return -ENOMEM;
938 
939 	if (!debugfs_create_files(q->sched_debugfs_dir, q,
940 				  e->queue_debugfs_attrs))
941 		goto err;
942 
943 	return 0;
944 
945 err:
946 	blk_mq_debugfs_unregister_sched(q);
947 	return -ENOMEM;
948 }
949 
950 void blk_mq_debugfs_unregister_sched(struct request_queue *q)
951 {
952 	debugfs_remove_recursive(q->sched_debugfs_dir);
953 	q->sched_debugfs_dir = NULL;
954 }
955 
956 int blk_mq_debugfs_register_sched_hctx(struct request_queue *q,
957 				       struct blk_mq_hw_ctx *hctx)
958 {
959 	struct elevator_type *e = q->elevator->type;
960 
961 	if (!hctx->debugfs_dir)
962 		return -ENOENT;
963 
964 	if (!e->hctx_debugfs_attrs)
965 		return 0;
966 
967 	hctx->sched_debugfs_dir = debugfs_create_dir("sched",
968 						     hctx->debugfs_dir);
969 	if (!hctx->sched_debugfs_dir)
970 		return -ENOMEM;
971 
972 	if (!debugfs_create_files(hctx->sched_debugfs_dir, hctx,
973 				  e->hctx_debugfs_attrs))
974 		return -ENOMEM;
975 
976 	return 0;
977 }
978 
979 void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx)
980 {
981 	debugfs_remove_recursive(hctx->sched_debugfs_dir);
982 	hctx->sched_debugfs_dir = NULL;
983 }
984