1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2017 Facebook 4 */ 5 6 #include <linux/kernel.h> 7 #include <linux/blkdev.h> 8 #include <linux/debugfs.h> 9 10 #include <linux/blk-mq.h> 11 #include "blk.h" 12 #include "blk-mq.h" 13 #include "blk-mq-debugfs.h" 14 #include "blk-mq-tag.h" 15 #include "blk-rq-qos.h" 16 17 static void print_stat(struct seq_file *m, struct blk_rq_stat *stat) 18 { 19 if (stat->nr_samples) { 20 seq_printf(m, "samples=%d, mean=%llu, min=%llu, max=%llu", 21 stat->nr_samples, stat->mean, stat->min, stat->max); 22 } else { 23 seq_puts(m, "samples=0"); 24 } 25 } 26 27 static int queue_poll_stat_show(void *data, struct seq_file *m) 28 { 29 struct request_queue *q = data; 30 int bucket; 31 32 for (bucket = 0; bucket < (BLK_MQ_POLL_STATS_BKTS / 2); bucket++) { 33 seq_printf(m, "read (%d Bytes): ", 1 << (9 + bucket)); 34 print_stat(m, &q->poll_stat[2 * bucket]); 35 seq_puts(m, "\n"); 36 37 seq_printf(m, "write (%d Bytes): ", 1 << (9 + bucket)); 38 print_stat(m, &q->poll_stat[2 * bucket + 1]); 39 seq_puts(m, "\n"); 40 } 41 return 0; 42 } 43 44 static void *queue_requeue_list_start(struct seq_file *m, loff_t *pos) 45 __acquires(&q->requeue_lock) 46 { 47 struct request_queue *q = m->private; 48 49 spin_lock_irq(&q->requeue_lock); 50 return seq_list_start(&q->requeue_list, *pos); 51 } 52 53 static void *queue_requeue_list_next(struct seq_file *m, void *v, loff_t *pos) 54 { 55 struct request_queue *q = m->private; 56 57 return seq_list_next(v, &q->requeue_list, pos); 58 } 59 60 static void queue_requeue_list_stop(struct seq_file *m, void *v) 61 __releases(&q->requeue_lock) 62 { 63 struct request_queue *q = m->private; 64 65 spin_unlock_irq(&q->requeue_lock); 66 } 67 68 static const struct seq_operations queue_requeue_list_seq_ops = { 69 .start = queue_requeue_list_start, 70 .next = queue_requeue_list_next, 71 .stop = queue_requeue_list_stop, 72 .show = blk_mq_debugfs_rq_show, 73 }; 74 75 static int blk_flags_show(struct seq_file *m, const unsigned long flags, 76 const char *const *flag_name, int flag_name_count) 77 { 78 bool sep = false; 79 int i; 80 81 for (i = 0; i < sizeof(flags) * BITS_PER_BYTE; i++) { 82 if (!(flags & BIT(i))) 83 continue; 84 if (sep) 85 seq_puts(m, "|"); 86 sep = true; 87 if (i < flag_name_count && flag_name[i]) 88 seq_puts(m, flag_name[i]); 89 else 90 seq_printf(m, "%d", i); 91 } 92 return 0; 93 } 94 95 static int queue_pm_only_show(void *data, struct seq_file *m) 96 { 97 struct request_queue *q = data; 98 99 seq_printf(m, "%d\n", atomic_read(&q->pm_only)); 100 return 0; 101 } 102 103 #define QUEUE_FLAG_NAME(name) [QUEUE_FLAG_##name] = #name 104 static const char *const blk_queue_flag_name[] = { 105 QUEUE_FLAG_NAME(STOPPED), 106 QUEUE_FLAG_NAME(DYING), 107 QUEUE_FLAG_NAME(NOMERGES), 108 QUEUE_FLAG_NAME(SAME_COMP), 109 QUEUE_FLAG_NAME(FAIL_IO), 110 QUEUE_FLAG_NAME(NONROT), 111 QUEUE_FLAG_NAME(IO_STAT), 112 QUEUE_FLAG_NAME(DISCARD), 113 QUEUE_FLAG_NAME(NOXMERGES), 114 QUEUE_FLAG_NAME(ADD_RANDOM), 115 QUEUE_FLAG_NAME(SECERASE), 116 QUEUE_FLAG_NAME(SAME_FORCE), 117 QUEUE_FLAG_NAME(DEAD), 118 QUEUE_FLAG_NAME(INIT_DONE), 119 QUEUE_FLAG_NAME(POLL), 120 QUEUE_FLAG_NAME(WC), 121 QUEUE_FLAG_NAME(FUA), 122 QUEUE_FLAG_NAME(DAX), 123 QUEUE_FLAG_NAME(STATS), 124 QUEUE_FLAG_NAME(POLL_STATS), 125 QUEUE_FLAG_NAME(REGISTERED), 126 QUEUE_FLAG_NAME(SCSI_PASSTHROUGH), 127 QUEUE_FLAG_NAME(QUIESCED), 128 QUEUE_FLAG_NAME(PCI_P2PDMA), 129 QUEUE_FLAG_NAME(ZONE_RESETALL), 130 QUEUE_FLAG_NAME(RQ_ALLOC_TIME), 131 }; 132 #undef QUEUE_FLAG_NAME 133 134 static int queue_state_show(void *data, struct seq_file *m) 135 { 136 struct request_queue *q = data; 137 138 blk_flags_show(m, q->queue_flags, blk_queue_flag_name, 139 ARRAY_SIZE(blk_queue_flag_name)); 140 seq_puts(m, "\n"); 141 return 0; 142 } 143 144 static ssize_t queue_state_write(void *data, const char __user *buf, 145 size_t count, loff_t *ppos) 146 { 147 struct request_queue *q = data; 148 char opbuf[16] = { }, *op; 149 150 /* 151 * The "state" attribute is removed after blk_cleanup_queue() has called 152 * blk_mq_free_queue(). Return if QUEUE_FLAG_DEAD has been set to avoid 153 * triggering a use-after-free. 154 */ 155 if (blk_queue_dead(q)) 156 return -ENOENT; 157 158 if (count >= sizeof(opbuf)) { 159 pr_err("%s: operation too long\n", __func__); 160 goto inval; 161 } 162 163 if (copy_from_user(opbuf, buf, count)) 164 return -EFAULT; 165 op = strstrip(opbuf); 166 if (strcmp(op, "run") == 0) { 167 blk_mq_run_hw_queues(q, true); 168 } else if (strcmp(op, "start") == 0) { 169 blk_mq_start_stopped_hw_queues(q, true); 170 } else if (strcmp(op, "kick") == 0) { 171 blk_mq_kick_requeue_list(q); 172 } else { 173 pr_err("%s: unsupported operation '%s'\n", __func__, op); 174 inval: 175 pr_err("%s: use 'run', 'start' or 'kick'\n", __func__); 176 return -EINVAL; 177 } 178 return count; 179 } 180 181 static int queue_write_hint_show(void *data, struct seq_file *m) 182 { 183 struct request_queue *q = data; 184 int i; 185 186 for (i = 0; i < BLK_MAX_WRITE_HINTS; i++) 187 seq_printf(m, "hint%d: %llu\n", i, q->write_hints[i]); 188 189 return 0; 190 } 191 192 static ssize_t queue_write_hint_store(void *data, const char __user *buf, 193 size_t count, loff_t *ppos) 194 { 195 struct request_queue *q = data; 196 int i; 197 198 for (i = 0; i < BLK_MAX_WRITE_HINTS; i++) 199 q->write_hints[i] = 0; 200 201 return count; 202 } 203 204 static const struct blk_mq_debugfs_attr blk_mq_debugfs_queue_attrs[] = { 205 { "poll_stat", 0400, queue_poll_stat_show }, 206 { "requeue_list", 0400, .seq_ops = &queue_requeue_list_seq_ops }, 207 { "pm_only", 0600, queue_pm_only_show, NULL }, 208 { "state", 0600, queue_state_show, queue_state_write }, 209 { "write_hints", 0600, queue_write_hint_show, queue_write_hint_store }, 210 { "zone_wlock", 0400, queue_zone_wlock_show, NULL }, 211 { }, 212 }; 213 214 #define HCTX_STATE_NAME(name) [BLK_MQ_S_##name] = #name 215 static const char *const hctx_state_name[] = { 216 HCTX_STATE_NAME(STOPPED), 217 HCTX_STATE_NAME(TAG_ACTIVE), 218 HCTX_STATE_NAME(SCHED_RESTART), 219 HCTX_STATE_NAME(INACTIVE), 220 }; 221 #undef HCTX_STATE_NAME 222 223 static int hctx_state_show(void *data, struct seq_file *m) 224 { 225 struct blk_mq_hw_ctx *hctx = data; 226 227 blk_flags_show(m, hctx->state, hctx_state_name, 228 ARRAY_SIZE(hctx_state_name)); 229 seq_puts(m, "\n"); 230 return 0; 231 } 232 233 #define BLK_TAG_ALLOC_NAME(name) [BLK_TAG_ALLOC_##name] = #name 234 static const char *const alloc_policy_name[] = { 235 BLK_TAG_ALLOC_NAME(FIFO), 236 BLK_TAG_ALLOC_NAME(RR), 237 }; 238 #undef BLK_TAG_ALLOC_NAME 239 240 #define HCTX_FLAG_NAME(name) [ilog2(BLK_MQ_F_##name)] = #name 241 static const char *const hctx_flag_name[] = { 242 HCTX_FLAG_NAME(SHOULD_MERGE), 243 HCTX_FLAG_NAME(TAG_SHARED), 244 HCTX_FLAG_NAME(BLOCKING), 245 HCTX_FLAG_NAME(NO_SCHED), 246 HCTX_FLAG_NAME(STACKING), 247 }; 248 #undef HCTX_FLAG_NAME 249 250 static int hctx_flags_show(void *data, struct seq_file *m) 251 { 252 struct blk_mq_hw_ctx *hctx = data; 253 const int alloc_policy = BLK_MQ_FLAG_TO_ALLOC_POLICY(hctx->flags); 254 255 seq_puts(m, "alloc_policy="); 256 if (alloc_policy < ARRAY_SIZE(alloc_policy_name) && 257 alloc_policy_name[alloc_policy]) 258 seq_puts(m, alloc_policy_name[alloc_policy]); 259 else 260 seq_printf(m, "%d", alloc_policy); 261 seq_puts(m, " "); 262 blk_flags_show(m, 263 hctx->flags ^ BLK_ALLOC_POLICY_TO_MQ_FLAG(alloc_policy), 264 hctx_flag_name, ARRAY_SIZE(hctx_flag_name)); 265 seq_puts(m, "\n"); 266 return 0; 267 } 268 269 #define CMD_FLAG_NAME(name) [__REQ_##name] = #name 270 static const char *const cmd_flag_name[] = { 271 CMD_FLAG_NAME(FAILFAST_DEV), 272 CMD_FLAG_NAME(FAILFAST_TRANSPORT), 273 CMD_FLAG_NAME(FAILFAST_DRIVER), 274 CMD_FLAG_NAME(SYNC), 275 CMD_FLAG_NAME(META), 276 CMD_FLAG_NAME(PRIO), 277 CMD_FLAG_NAME(NOMERGE), 278 CMD_FLAG_NAME(IDLE), 279 CMD_FLAG_NAME(INTEGRITY), 280 CMD_FLAG_NAME(FUA), 281 CMD_FLAG_NAME(PREFLUSH), 282 CMD_FLAG_NAME(RAHEAD), 283 CMD_FLAG_NAME(BACKGROUND), 284 CMD_FLAG_NAME(NOWAIT), 285 CMD_FLAG_NAME(NOUNMAP), 286 CMD_FLAG_NAME(HIPRI), 287 }; 288 #undef CMD_FLAG_NAME 289 290 #define RQF_NAME(name) [ilog2((__force u32)RQF_##name)] = #name 291 static const char *const rqf_name[] = { 292 RQF_NAME(SORTED), 293 RQF_NAME(STARTED), 294 RQF_NAME(SOFTBARRIER), 295 RQF_NAME(FLUSH_SEQ), 296 RQF_NAME(MIXED_MERGE), 297 RQF_NAME(MQ_INFLIGHT), 298 RQF_NAME(DONTPREP), 299 RQF_NAME(PREEMPT), 300 RQF_NAME(FAILED), 301 RQF_NAME(QUIET), 302 RQF_NAME(ELVPRIV), 303 RQF_NAME(IO_STAT), 304 RQF_NAME(ALLOCED), 305 RQF_NAME(PM), 306 RQF_NAME(HASHED), 307 RQF_NAME(STATS), 308 RQF_NAME(SPECIAL_PAYLOAD), 309 RQF_NAME(ZONE_WRITE_LOCKED), 310 RQF_NAME(MQ_POLL_SLEPT), 311 }; 312 #undef RQF_NAME 313 314 static const char *const blk_mq_rq_state_name_array[] = { 315 [MQ_RQ_IDLE] = "idle", 316 [MQ_RQ_IN_FLIGHT] = "in_flight", 317 [MQ_RQ_COMPLETE] = "complete", 318 }; 319 320 static const char *blk_mq_rq_state_name(enum mq_rq_state rq_state) 321 { 322 if (WARN_ON_ONCE((unsigned int)rq_state >= 323 ARRAY_SIZE(blk_mq_rq_state_name_array))) 324 return "(?)"; 325 return blk_mq_rq_state_name_array[rq_state]; 326 } 327 328 int __blk_mq_debugfs_rq_show(struct seq_file *m, struct request *rq) 329 { 330 const struct blk_mq_ops *const mq_ops = rq->q->mq_ops; 331 const unsigned int op = req_op(rq); 332 const char *op_str = blk_op_str(op); 333 334 seq_printf(m, "%p {.op=", rq); 335 if (strcmp(op_str, "UNKNOWN") == 0) 336 seq_printf(m, "%u", op); 337 else 338 seq_printf(m, "%s", op_str); 339 seq_puts(m, ", .cmd_flags="); 340 blk_flags_show(m, rq->cmd_flags & ~REQ_OP_MASK, cmd_flag_name, 341 ARRAY_SIZE(cmd_flag_name)); 342 seq_puts(m, ", .rq_flags="); 343 blk_flags_show(m, (__force unsigned int)rq->rq_flags, rqf_name, 344 ARRAY_SIZE(rqf_name)); 345 seq_printf(m, ", .state=%s", blk_mq_rq_state_name(blk_mq_rq_state(rq))); 346 seq_printf(m, ", .tag=%d, .internal_tag=%d", rq->tag, 347 rq->internal_tag); 348 if (mq_ops->show_rq) 349 mq_ops->show_rq(m, rq); 350 seq_puts(m, "}\n"); 351 return 0; 352 } 353 EXPORT_SYMBOL_GPL(__blk_mq_debugfs_rq_show); 354 355 int blk_mq_debugfs_rq_show(struct seq_file *m, void *v) 356 { 357 return __blk_mq_debugfs_rq_show(m, list_entry_rq(v)); 358 } 359 EXPORT_SYMBOL_GPL(blk_mq_debugfs_rq_show); 360 361 static void *hctx_dispatch_start(struct seq_file *m, loff_t *pos) 362 __acquires(&hctx->lock) 363 { 364 struct blk_mq_hw_ctx *hctx = m->private; 365 366 spin_lock(&hctx->lock); 367 return seq_list_start(&hctx->dispatch, *pos); 368 } 369 370 static void *hctx_dispatch_next(struct seq_file *m, void *v, loff_t *pos) 371 { 372 struct blk_mq_hw_ctx *hctx = m->private; 373 374 return seq_list_next(v, &hctx->dispatch, pos); 375 } 376 377 static void hctx_dispatch_stop(struct seq_file *m, void *v) 378 __releases(&hctx->lock) 379 { 380 struct blk_mq_hw_ctx *hctx = m->private; 381 382 spin_unlock(&hctx->lock); 383 } 384 385 static const struct seq_operations hctx_dispatch_seq_ops = { 386 .start = hctx_dispatch_start, 387 .next = hctx_dispatch_next, 388 .stop = hctx_dispatch_stop, 389 .show = blk_mq_debugfs_rq_show, 390 }; 391 392 struct show_busy_params { 393 struct seq_file *m; 394 struct blk_mq_hw_ctx *hctx; 395 }; 396 397 /* 398 * Note: the state of a request may change while this function is in progress, 399 * e.g. due to a concurrent blk_mq_finish_request() call. Returns true to 400 * keep iterating requests. 401 */ 402 static bool hctx_show_busy_rq(struct request *rq, void *data, bool reserved) 403 { 404 const struct show_busy_params *params = data; 405 406 if (rq->mq_hctx == params->hctx) 407 __blk_mq_debugfs_rq_show(params->m, 408 list_entry_rq(&rq->queuelist)); 409 410 return true; 411 } 412 413 static int hctx_busy_show(void *data, struct seq_file *m) 414 { 415 struct blk_mq_hw_ctx *hctx = data; 416 struct show_busy_params params = { .m = m, .hctx = hctx }; 417 418 blk_mq_tagset_busy_iter(hctx->queue->tag_set, hctx_show_busy_rq, 419 ¶ms); 420 421 return 0; 422 } 423 424 static const char *const hctx_types[] = { 425 [HCTX_TYPE_DEFAULT] = "default", 426 [HCTX_TYPE_READ] = "read", 427 [HCTX_TYPE_POLL] = "poll", 428 }; 429 430 static int hctx_type_show(void *data, struct seq_file *m) 431 { 432 struct blk_mq_hw_ctx *hctx = data; 433 434 BUILD_BUG_ON(ARRAY_SIZE(hctx_types) != HCTX_MAX_TYPES); 435 seq_printf(m, "%s\n", hctx_types[hctx->type]); 436 return 0; 437 } 438 439 static int hctx_ctx_map_show(void *data, struct seq_file *m) 440 { 441 struct blk_mq_hw_ctx *hctx = data; 442 443 sbitmap_bitmap_show(&hctx->ctx_map, m); 444 return 0; 445 } 446 447 static void blk_mq_debugfs_tags_show(struct seq_file *m, 448 struct blk_mq_tags *tags) 449 { 450 seq_printf(m, "nr_tags=%u\n", tags->nr_tags); 451 seq_printf(m, "nr_reserved_tags=%u\n", tags->nr_reserved_tags); 452 seq_printf(m, "active_queues=%d\n", 453 atomic_read(&tags->active_queues)); 454 455 seq_puts(m, "\nbitmap_tags:\n"); 456 sbitmap_queue_show(&tags->bitmap_tags, m); 457 458 if (tags->nr_reserved_tags) { 459 seq_puts(m, "\nbreserved_tags:\n"); 460 sbitmap_queue_show(&tags->breserved_tags, m); 461 } 462 } 463 464 static int hctx_tags_show(void *data, struct seq_file *m) 465 { 466 struct blk_mq_hw_ctx *hctx = data; 467 struct request_queue *q = hctx->queue; 468 int res; 469 470 res = mutex_lock_interruptible(&q->sysfs_lock); 471 if (res) 472 goto out; 473 if (hctx->tags) 474 blk_mq_debugfs_tags_show(m, hctx->tags); 475 mutex_unlock(&q->sysfs_lock); 476 477 out: 478 return res; 479 } 480 481 static int hctx_tags_bitmap_show(void *data, struct seq_file *m) 482 { 483 struct blk_mq_hw_ctx *hctx = data; 484 struct request_queue *q = hctx->queue; 485 int res; 486 487 res = mutex_lock_interruptible(&q->sysfs_lock); 488 if (res) 489 goto out; 490 if (hctx->tags) 491 sbitmap_bitmap_show(&hctx->tags->bitmap_tags.sb, m); 492 mutex_unlock(&q->sysfs_lock); 493 494 out: 495 return res; 496 } 497 498 static int hctx_sched_tags_show(void *data, struct seq_file *m) 499 { 500 struct blk_mq_hw_ctx *hctx = data; 501 struct request_queue *q = hctx->queue; 502 int res; 503 504 res = mutex_lock_interruptible(&q->sysfs_lock); 505 if (res) 506 goto out; 507 if (hctx->sched_tags) 508 blk_mq_debugfs_tags_show(m, hctx->sched_tags); 509 mutex_unlock(&q->sysfs_lock); 510 511 out: 512 return res; 513 } 514 515 static int hctx_sched_tags_bitmap_show(void *data, struct seq_file *m) 516 { 517 struct blk_mq_hw_ctx *hctx = data; 518 struct request_queue *q = hctx->queue; 519 int res; 520 521 res = mutex_lock_interruptible(&q->sysfs_lock); 522 if (res) 523 goto out; 524 if (hctx->sched_tags) 525 sbitmap_bitmap_show(&hctx->sched_tags->bitmap_tags.sb, m); 526 mutex_unlock(&q->sysfs_lock); 527 528 out: 529 return res; 530 } 531 532 static int hctx_io_poll_show(void *data, struct seq_file *m) 533 { 534 struct blk_mq_hw_ctx *hctx = data; 535 536 seq_printf(m, "considered=%lu\n", hctx->poll_considered); 537 seq_printf(m, "invoked=%lu\n", hctx->poll_invoked); 538 seq_printf(m, "success=%lu\n", hctx->poll_success); 539 return 0; 540 } 541 542 static ssize_t hctx_io_poll_write(void *data, const char __user *buf, 543 size_t count, loff_t *ppos) 544 { 545 struct blk_mq_hw_ctx *hctx = data; 546 547 hctx->poll_considered = hctx->poll_invoked = hctx->poll_success = 0; 548 return count; 549 } 550 551 static int hctx_dispatched_show(void *data, struct seq_file *m) 552 { 553 struct blk_mq_hw_ctx *hctx = data; 554 int i; 555 556 seq_printf(m, "%8u\t%lu\n", 0U, hctx->dispatched[0]); 557 558 for (i = 1; i < BLK_MQ_MAX_DISPATCH_ORDER - 1; i++) { 559 unsigned int d = 1U << (i - 1); 560 561 seq_printf(m, "%8u\t%lu\n", d, hctx->dispatched[i]); 562 } 563 564 seq_printf(m, "%8u+\t%lu\n", 1U << (i - 1), hctx->dispatched[i]); 565 return 0; 566 } 567 568 static ssize_t hctx_dispatched_write(void *data, const char __user *buf, 569 size_t count, loff_t *ppos) 570 { 571 struct blk_mq_hw_ctx *hctx = data; 572 int i; 573 574 for (i = 0; i < BLK_MQ_MAX_DISPATCH_ORDER; i++) 575 hctx->dispatched[i] = 0; 576 return count; 577 } 578 579 static int hctx_queued_show(void *data, struct seq_file *m) 580 { 581 struct blk_mq_hw_ctx *hctx = data; 582 583 seq_printf(m, "%lu\n", hctx->queued); 584 return 0; 585 } 586 587 static ssize_t hctx_queued_write(void *data, const char __user *buf, 588 size_t count, loff_t *ppos) 589 { 590 struct blk_mq_hw_ctx *hctx = data; 591 592 hctx->queued = 0; 593 return count; 594 } 595 596 static int hctx_run_show(void *data, struct seq_file *m) 597 { 598 struct blk_mq_hw_ctx *hctx = data; 599 600 seq_printf(m, "%lu\n", hctx->run); 601 return 0; 602 } 603 604 static ssize_t hctx_run_write(void *data, const char __user *buf, size_t count, 605 loff_t *ppos) 606 { 607 struct blk_mq_hw_ctx *hctx = data; 608 609 hctx->run = 0; 610 return count; 611 } 612 613 static int hctx_active_show(void *data, struct seq_file *m) 614 { 615 struct blk_mq_hw_ctx *hctx = data; 616 617 seq_printf(m, "%d\n", atomic_read(&hctx->nr_active)); 618 return 0; 619 } 620 621 static int hctx_dispatch_busy_show(void *data, struct seq_file *m) 622 { 623 struct blk_mq_hw_ctx *hctx = data; 624 625 seq_printf(m, "%u\n", hctx->dispatch_busy); 626 return 0; 627 } 628 629 #define CTX_RQ_SEQ_OPS(name, type) \ 630 static void *ctx_##name##_rq_list_start(struct seq_file *m, loff_t *pos) \ 631 __acquires(&ctx->lock) \ 632 { \ 633 struct blk_mq_ctx *ctx = m->private; \ 634 \ 635 spin_lock(&ctx->lock); \ 636 return seq_list_start(&ctx->rq_lists[type], *pos); \ 637 } \ 638 \ 639 static void *ctx_##name##_rq_list_next(struct seq_file *m, void *v, \ 640 loff_t *pos) \ 641 { \ 642 struct blk_mq_ctx *ctx = m->private; \ 643 \ 644 return seq_list_next(v, &ctx->rq_lists[type], pos); \ 645 } \ 646 \ 647 static void ctx_##name##_rq_list_stop(struct seq_file *m, void *v) \ 648 __releases(&ctx->lock) \ 649 { \ 650 struct blk_mq_ctx *ctx = m->private; \ 651 \ 652 spin_unlock(&ctx->lock); \ 653 } \ 654 \ 655 static const struct seq_operations ctx_##name##_rq_list_seq_ops = { \ 656 .start = ctx_##name##_rq_list_start, \ 657 .next = ctx_##name##_rq_list_next, \ 658 .stop = ctx_##name##_rq_list_stop, \ 659 .show = blk_mq_debugfs_rq_show, \ 660 } 661 662 CTX_RQ_SEQ_OPS(default, HCTX_TYPE_DEFAULT); 663 CTX_RQ_SEQ_OPS(read, HCTX_TYPE_READ); 664 CTX_RQ_SEQ_OPS(poll, HCTX_TYPE_POLL); 665 666 static int ctx_dispatched_show(void *data, struct seq_file *m) 667 { 668 struct blk_mq_ctx *ctx = data; 669 670 seq_printf(m, "%lu %lu\n", ctx->rq_dispatched[1], ctx->rq_dispatched[0]); 671 return 0; 672 } 673 674 static ssize_t ctx_dispatched_write(void *data, const char __user *buf, 675 size_t count, loff_t *ppos) 676 { 677 struct blk_mq_ctx *ctx = data; 678 679 ctx->rq_dispatched[0] = ctx->rq_dispatched[1] = 0; 680 return count; 681 } 682 683 static int ctx_merged_show(void *data, struct seq_file *m) 684 { 685 struct blk_mq_ctx *ctx = data; 686 687 seq_printf(m, "%lu\n", ctx->rq_merged); 688 return 0; 689 } 690 691 static ssize_t ctx_merged_write(void *data, const char __user *buf, 692 size_t count, loff_t *ppos) 693 { 694 struct blk_mq_ctx *ctx = data; 695 696 ctx->rq_merged = 0; 697 return count; 698 } 699 700 static int ctx_completed_show(void *data, struct seq_file *m) 701 { 702 struct blk_mq_ctx *ctx = data; 703 704 seq_printf(m, "%lu %lu\n", ctx->rq_completed[1], ctx->rq_completed[0]); 705 return 0; 706 } 707 708 static ssize_t ctx_completed_write(void *data, const char __user *buf, 709 size_t count, loff_t *ppos) 710 { 711 struct blk_mq_ctx *ctx = data; 712 713 ctx->rq_completed[0] = ctx->rq_completed[1] = 0; 714 return count; 715 } 716 717 static int blk_mq_debugfs_show(struct seq_file *m, void *v) 718 { 719 const struct blk_mq_debugfs_attr *attr = m->private; 720 void *data = d_inode(m->file->f_path.dentry->d_parent)->i_private; 721 722 return attr->show(data, m); 723 } 724 725 static ssize_t blk_mq_debugfs_write(struct file *file, const char __user *buf, 726 size_t count, loff_t *ppos) 727 { 728 struct seq_file *m = file->private_data; 729 const struct blk_mq_debugfs_attr *attr = m->private; 730 void *data = d_inode(file->f_path.dentry->d_parent)->i_private; 731 732 /* 733 * Attributes that only implement .seq_ops are read-only and 'attr' is 734 * the same with 'data' in this case. 735 */ 736 if (attr == data || !attr->write) 737 return -EPERM; 738 739 return attr->write(data, buf, count, ppos); 740 } 741 742 static int blk_mq_debugfs_open(struct inode *inode, struct file *file) 743 { 744 const struct blk_mq_debugfs_attr *attr = inode->i_private; 745 void *data = d_inode(file->f_path.dentry->d_parent)->i_private; 746 struct seq_file *m; 747 int ret; 748 749 if (attr->seq_ops) { 750 ret = seq_open(file, attr->seq_ops); 751 if (!ret) { 752 m = file->private_data; 753 m->private = data; 754 } 755 return ret; 756 } 757 758 if (WARN_ON_ONCE(!attr->show)) 759 return -EPERM; 760 761 return single_open(file, blk_mq_debugfs_show, inode->i_private); 762 } 763 764 static int blk_mq_debugfs_release(struct inode *inode, struct file *file) 765 { 766 const struct blk_mq_debugfs_attr *attr = inode->i_private; 767 768 if (attr->show) 769 return single_release(inode, file); 770 771 return seq_release(inode, file); 772 } 773 774 static const struct file_operations blk_mq_debugfs_fops = { 775 .open = blk_mq_debugfs_open, 776 .read = seq_read, 777 .write = blk_mq_debugfs_write, 778 .llseek = seq_lseek, 779 .release = blk_mq_debugfs_release, 780 }; 781 782 static const struct blk_mq_debugfs_attr blk_mq_debugfs_hctx_attrs[] = { 783 {"state", 0400, hctx_state_show}, 784 {"flags", 0400, hctx_flags_show}, 785 {"dispatch", 0400, .seq_ops = &hctx_dispatch_seq_ops}, 786 {"busy", 0400, hctx_busy_show}, 787 {"ctx_map", 0400, hctx_ctx_map_show}, 788 {"tags", 0400, hctx_tags_show}, 789 {"tags_bitmap", 0400, hctx_tags_bitmap_show}, 790 {"sched_tags", 0400, hctx_sched_tags_show}, 791 {"sched_tags_bitmap", 0400, hctx_sched_tags_bitmap_show}, 792 {"io_poll", 0600, hctx_io_poll_show, hctx_io_poll_write}, 793 {"dispatched", 0600, hctx_dispatched_show, hctx_dispatched_write}, 794 {"queued", 0600, hctx_queued_show, hctx_queued_write}, 795 {"run", 0600, hctx_run_show, hctx_run_write}, 796 {"active", 0400, hctx_active_show}, 797 {"dispatch_busy", 0400, hctx_dispatch_busy_show}, 798 {"type", 0400, hctx_type_show}, 799 {}, 800 }; 801 802 static const struct blk_mq_debugfs_attr blk_mq_debugfs_ctx_attrs[] = { 803 {"default_rq_list", 0400, .seq_ops = &ctx_default_rq_list_seq_ops}, 804 {"read_rq_list", 0400, .seq_ops = &ctx_read_rq_list_seq_ops}, 805 {"poll_rq_list", 0400, .seq_ops = &ctx_poll_rq_list_seq_ops}, 806 {"dispatched", 0600, ctx_dispatched_show, ctx_dispatched_write}, 807 {"merged", 0600, ctx_merged_show, ctx_merged_write}, 808 {"completed", 0600, ctx_completed_show, ctx_completed_write}, 809 {}, 810 }; 811 812 static void debugfs_create_files(struct dentry *parent, void *data, 813 const struct blk_mq_debugfs_attr *attr) 814 { 815 if (IS_ERR_OR_NULL(parent)) 816 return; 817 818 d_inode(parent)->i_private = data; 819 820 for (; attr->name; attr++) 821 debugfs_create_file(attr->name, attr->mode, parent, 822 (void *)attr, &blk_mq_debugfs_fops); 823 } 824 825 void blk_mq_debugfs_register(struct request_queue *q) 826 { 827 struct blk_mq_hw_ctx *hctx; 828 int i; 829 830 q->debugfs_dir = debugfs_create_dir(kobject_name(q->kobj.parent), 831 blk_debugfs_root); 832 833 debugfs_create_files(q->debugfs_dir, q, blk_mq_debugfs_queue_attrs); 834 835 /* 836 * blk_mq_init_sched() attempted to do this already, but q->debugfs_dir 837 * didn't exist yet (because we don't know what to name the directory 838 * until the queue is registered to a gendisk). 839 */ 840 if (q->elevator && !q->sched_debugfs_dir) 841 blk_mq_debugfs_register_sched(q); 842 843 /* Similarly, blk_mq_init_hctx() couldn't do this previously. */ 844 queue_for_each_hw_ctx(q, hctx, i) { 845 if (!hctx->debugfs_dir) 846 blk_mq_debugfs_register_hctx(q, hctx); 847 if (q->elevator && !hctx->sched_debugfs_dir) 848 blk_mq_debugfs_register_sched_hctx(q, hctx); 849 } 850 851 if (q->rq_qos) { 852 struct rq_qos *rqos = q->rq_qos; 853 854 while (rqos) { 855 blk_mq_debugfs_register_rqos(rqos); 856 rqos = rqos->next; 857 } 858 } 859 } 860 861 void blk_mq_debugfs_unregister(struct request_queue *q) 862 { 863 debugfs_remove_recursive(q->debugfs_dir); 864 q->sched_debugfs_dir = NULL; 865 q->debugfs_dir = NULL; 866 } 867 868 static void blk_mq_debugfs_register_ctx(struct blk_mq_hw_ctx *hctx, 869 struct blk_mq_ctx *ctx) 870 { 871 struct dentry *ctx_dir; 872 char name[20]; 873 874 snprintf(name, sizeof(name), "cpu%u", ctx->cpu); 875 ctx_dir = debugfs_create_dir(name, hctx->debugfs_dir); 876 877 debugfs_create_files(ctx_dir, ctx, blk_mq_debugfs_ctx_attrs); 878 } 879 880 void blk_mq_debugfs_register_hctx(struct request_queue *q, 881 struct blk_mq_hw_ctx *hctx) 882 { 883 struct blk_mq_ctx *ctx; 884 char name[20]; 885 int i; 886 887 snprintf(name, sizeof(name), "hctx%u", hctx->queue_num); 888 hctx->debugfs_dir = debugfs_create_dir(name, q->debugfs_dir); 889 890 debugfs_create_files(hctx->debugfs_dir, hctx, blk_mq_debugfs_hctx_attrs); 891 892 hctx_for_each_ctx(hctx, ctx, i) 893 blk_mq_debugfs_register_ctx(hctx, ctx); 894 } 895 896 void blk_mq_debugfs_unregister_hctx(struct blk_mq_hw_ctx *hctx) 897 { 898 debugfs_remove_recursive(hctx->debugfs_dir); 899 hctx->sched_debugfs_dir = NULL; 900 hctx->debugfs_dir = NULL; 901 } 902 903 void blk_mq_debugfs_register_hctxs(struct request_queue *q) 904 { 905 struct blk_mq_hw_ctx *hctx; 906 int i; 907 908 queue_for_each_hw_ctx(q, hctx, i) 909 blk_mq_debugfs_register_hctx(q, hctx); 910 } 911 912 void blk_mq_debugfs_unregister_hctxs(struct request_queue *q) 913 { 914 struct blk_mq_hw_ctx *hctx; 915 int i; 916 917 queue_for_each_hw_ctx(q, hctx, i) 918 blk_mq_debugfs_unregister_hctx(hctx); 919 } 920 921 void blk_mq_debugfs_register_sched(struct request_queue *q) 922 { 923 struct elevator_type *e = q->elevator->type; 924 925 /* 926 * If the parent directory has not been created yet, return, we will be 927 * called again later on and the directory/files will be created then. 928 */ 929 if (!q->debugfs_dir) 930 return; 931 932 if (!e->queue_debugfs_attrs) 933 return; 934 935 q->sched_debugfs_dir = debugfs_create_dir("sched", q->debugfs_dir); 936 937 debugfs_create_files(q->sched_debugfs_dir, q, e->queue_debugfs_attrs); 938 } 939 940 void blk_mq_debugfs_unregister_sched(struct request_queue *q) 941 { 942 debugfs_remove_recursive(q->sched_debugfs_dir); 943 q->sched_debugfs_dir = NULL; 944 } 945 946 void blk_mq_debugfs_unregister_rqos(struct rq_qos *rqos) 947 { 948 debugfs_remove_recursive(rqos->debugfs_dir); 949 rqos->debugfs_dir = NULL; 950 } 951 952 void blk_mq_debugfs_register_rqos(struct rq_qos *rqos) 953 { 954 struct request_queue *q = rqos->q; 955 const char *dir_name = rq_qos_id_to_name(rqos->id); 956 957 if (rqos->debugfs_dir || !rqos->ops->debugfs_attrs) 958 return; 959 960 if (!q->rqos_debugfs_dir) 961 q->rqos_debugfs_dir = debugfs_create_dir("rqos", 962 q->debugfs_dir); 963 964 rqos->debugfs_dir = debugfs_create_dir(dir_name, 965 rqos->q->rqos_debugfs_dir); 966 967 debugfs_create_files(rqos->debugfs_dir, rqos, rqos->ops->debugfs_attrs); 968 } 969 970 void blk_mq_debugfs_unregister_queue_rqos(struct request_queue *q) 971 { 972 debugfs_remove_recursive(q->rqos_debugfs_dir); 973 q->rqos_debugfs_dir = NULL; 974 } 975 976 void blk_mq_debugfs_register_sched_hctx(struct request_queue *q, 977 struct blk_mq_hw_ctx *hctx) 978 { 979 struct elevator_type *e = q->elevator->type; 980 981 if (!e->hctx_debugfs_attrs) 982 return; 983 984 hctx->sched_debugfs_dir = debugfs_create_dir("sched", 985 hctx->debugfs_dir); 986 debugfs_create_files(hctx->sched_debugfs_dir, hctx, 987 e->hctx_debugfs_attrs); 988 } 989 990 void blk_mq_debugfs_unregister_sched_hctx(struct blk_mq_hw_ctx *hctx) 991 { 992 debugfs_remove_recursive(hctx->sched_debugfs_dir); 993 hctx->sched_debugfs_dir = NULL; 994 } 995