1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk> 4 * 5 */ 6 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 9 #include <linux/kernel.h> 10 #include <linux/blkdev.h> 11 #include <linux/blktrace_api.h> 12 #include <linux/percpu.h> 13 #include <linux/init.h> 14 #include <linux/mutex.h> 15 #include <linux/slab.h> 16 #include <linux/debugfs.h> 17 #include <linux/export.h> 18 #include <linux/time.h> 19 #include <linux/uaccess.h> 20 #include <linux/list.h> 21 #include <linux/blk-cgroup.h> 22 23 #include "../../block/blk.h" 24 25 #include <trace/events/block.h> 26 27 #include "trace_output.h" 28 29 #ifdef CONFIG_BLK_DEV_IO_TRACE 30 31 static unsigned int blktrace_seq __read_mostly = 1; 32 33 static struct trace_array *blk_tr; 34 static bool blk_tracer_enabled __read_mostly; 35 36 static LIST_HEAD(running_trace_list); 37 static __cacheline_aligned_in_smp DEFINE_SPINLOCK(running_trace_lock); 38 39 /* Select an alternative, minimalistic output than the original one */ 40 #define TRACE_BLK_OPT_CLASSIC 0x1 41 #define TRACE_BLK_OPT_CGROUP 0x2 42 #define TRACE_BLK_OPT_CGNAME 0x4 43 44 static struct tracer_opt blk_tracer_opts[] = { 45 /* Default disable the minimalistic output */ 46 { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) }, 47 #ifdef CONFIG_BLK_CGROUP 48 { TRACER_OPT(blk_cgroup, TRACE_BLK_OPT_CGROUP) }, 49 { TRACER_OPT(blk_cgname, TRACE_BLK_OPT_CGNAME) }, 50 #endif 51 { } 52 }; 53 54 static struct tracer_flags blk_tracer_flags = { 55 .val = 0, 56 .opts = blk_tracer_opts, 57 }; 58 59 /* Global reference count of probes */ 60 static DEFINE_MUTEX(blk_probe_mutex); 61 static int blk_probes_ref; 62 63 static void blk_register_tracepoints(void); 64 static void blk_unregister_tracepoints(void); 65 66 /* 67 * Send out a notify message. 68 */ 69 static void trace_note(struct blk_trace *bt, pid_t pid, int action, 70 const void *data, size_t len, u64 cgid) 71 { 72 struct blk_io_trace *t; 73 struct ring_buffer_event *event = NULL; 74 struct trace_buffer *buffer = NULL; 75 int pc = 0; 76 int cpu = smp_processor_id(); 77 bool blk_tracer = blk_tracer_enabled; 78 ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 79 80 if (blk_tracer) { 81 buffer = blk_tr->array_buffer.buffer; 82 pc = preempt_count(); 83 event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 84 sizeof(*t) + len + cgid_len, 85 0, pc); 86 if (!event) 87 return; 88 t = ring_buffer_event_data(event); 89 goto record_it; 90 } 91 92 if (!bt->rchan) 93 return; 94 95 t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len); 96 if (t) { 97 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 98 t->time = ktime_to_ns(ktime_get()); 99 record_it: 100 t->device = bt->dev; 101 t->action = action | (cgid ? __BLK_TN_CGROUP : 0); 102 t->pid = pid; 103 t->cpu = cpu; 104 t->pdu_len = len + cgid_len; 105 if (cgid_len) 106 memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 107 memcpy((void *) t + sizeof(*t) + cgid_len, data, len); 108 109 if (blk_tracer) 110 trace_buffer_unlock_commit(blk_tr, buffer, event, 0, pc); 111 } 112 } 113 114 /* 115 * Send out a notify for this process, if we haven't done so since a trace 116 * started 117 */ 118 static void trace_note_tsk(struct task_struct *tsk) 119 { 120 unsigned long flags; 121 struct blk_trace *bt; 122 123 tsk->btrace_seq = blktrace_seq; 124 spin_lock_irqsave(&running_trace_lock, flags); 125 list_for_each_entry(bt, &running_trace_list, running_list) { 126 trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, 127 sizeof(tsk->comm), 0); 128 } 129 spin_unlock_irqrestore(&running_trace_lock, flags); 130 } 131 132 static void trace_note_time(struct blk_trace *bt) 133 { 134 struct timespec64 now; 135 unsigned long flags; 136 u32 words[2]; 137 138 /* need to check user space to see if this breaks in y2038 or y2106 */ 139 ktime_get_real_ts64(&now); 140 words[0] = (u32)now.tv_sec; 141 words[1] = now.tv_nsec; 142 143 local_irq_save(flags); 144 trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0); 145 local_irq_restore(flags); 146 } 147 148 void __trace_note_message(struct blk_trace *bt, struct blkcg *blkcg, 149 const char *fmt, ...) 150 { 151 int n; 152 va_list args; 153 unsigned long flags; 154 char *buf; 155 156 if (unlikely(bt->trace_state != Blktrace_running && 157 !blk_tracer_enabled)) 158 return; 159 160 /* 161 * If the BLK_TC_NOTIFY action mask isn't set, don't send any note 162 * message to the trace. 163 */ 164 if (!(bt->act_mask & BLK_TC_NOTIFY)) 165 return; 166 167 local_irq_save(flags); 168 buf = this_cpu_ptr(bt->msg_data); 169 va_start(args, fmt); 170 n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args); 171 va_end(args); 172 173 if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 174 blkcg = NULL; 175 #ifdef CONFIG_BLK_CGROUP 176 trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, 177 blkcg ? cgroup_id(blkcg->css.cgroup) : 1); 178 #else 179 trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, 0); 180 #endif 181 local_irq_restore(flags); 182 } 183 EXPORT_SYMBOL_GPL(__trace_note_message); 184 185 static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector, 186 pid_t pid) 187 { 188 if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0) 189 return 1; 190 if (sector && (sector < bt->start_lba || sector > bt->end_lba)) 191 return 1; 192 if (bt->pid && pid != bt->pid) 193 return 1; 194 195 return 0; 196 } 197 198 /* 199 * Data direction bit lookup 200 */ 201 static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), 202 BLK_TC_ACT(BLK_TC_WRITE) }; 203 204 #define BLK_TC_RAHEAD BLK_TC_AHEAD 205 #define BLK_TC_PREFLUSH BLK_TC_FLUSH 206 207 /* The ilog2() calls fall out because they're constant */ 208 #define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \ 209 (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name)) 210 211 /* 212 * The worker for the various blk_add_trace*() types. Fills out a 213 * blk_io_trace structure and places it in a per-cpu subbuffer. 214 */ 215 static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, 216 int op, int op_flags, u32 what, int error, int pdu_len, 217 void *pdu_data, u64 cgid) 218 { 219 struct task_struct *tsk = current; 220 struct ring_buffer_event *event = NULL; 221 struct trace_buffer *buffer = NULL; 222 struct blk_io_trace *t; 223 unsigned long flags = 0; 224 unsigned long *sequence; 225 pid_t pid; 226 int cpu, pc = 0; 227 bool blk_tracer = blk_tracer_enabled; 228 ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 229 230 if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer)) 231 return; 232 233 what |= ddir_act[op_is_write(op) ? WRITE : READ]; 234 what |= MASK_TC_BIT(op_flags, SYNC); 235 what |= MASK_TC_BIT(op_flags, RAHEAD); 236 what |= MASK_TC_BIT(op_flags, META); 237 what |= MASK_TC_BIT(op_flags, PREFLUSH); 238 what |= MASK_TC_BIT(op_flags, FUA); 239 if (op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE) 240 what |= BLK_TC_ACT(BLK_TC_DISCARD); 241 if (op == REQ_OP_FLUSH) 242 what |= BLK_TC_ACT(BLK_TC_FLUSH); 243 if (cgid) 244 what |= __BLK_TA_CGROUP; 245 246 pid = tsk->pid; 247 if (act_log_check(bt, what, sector, pid)) 248 return; 249 cpu = raw_smp_processor_id(); 250 251 if (blk_tracer) { 252 tracing_record_cmdline(current); 253 254 buffer = blk_tr->array_buffer.buffer; 255 pc = preempt_count(); 256 event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 257 sizeof(*t) + pdu_len + cgid_len, 258 0, pc); 259 if (!event) 260 return; 261 t = ring_buffer_event_data(event); 262 goto record_it; 263 } 264 265 if (unlikely(tsk->btrace_seq != blktrace_seq)) 266 trace_note_tsk(tsk); 267 268 /* 269 * A word about the locking here - we disable interrupts to reserve 270 * some space in the relay per-cpu buffer, to prevent an irq 271 * from coming in and stepping on our toes. 272 */ 273 local_irq_save(flags); 274 t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len); 275 if (t) { 276 sequence = per_cpu_ptr(bt->sequence, cpu); 277 278 t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 279 t->sequence = ++(*sequence); 280 t->time = ktime_to_ns(ktime_get()); 281 record_it: 282 /* 283 * These two are not needed in ftrace as they are in the 284 * generic trace_entry, filled by tracing_generic_entry_update, 285 * but for the trace_event->bin() synthesizer benefit we do it 286 * here too. 287 */ 288 t->cpu = cpu; 289 t->pid = pid; 290 291 t->sector = sector; 292 t->bytes = bytes; 293 t->action = what; 294 t->device = bt->dev; 295 t->error = error; 296 t->pdu_len = pdu_len + cgid_len; 297 298 if (cgid_len) 299 memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 300 if (pdu_len) 301 memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len); 302 303 if (blk_tracer) { 304 trace_buffer_unlock_commit(blk_tr, buffer, event, 0, pc); 305 return; 306 } 307 } 308 309 local_irq_restore(flags); 310 } 311 312 static void blk_trace_free(struct blk_trace *bt) 313 { 314 debugfs_remove(bt->msg_file); 315 debugfs_remove(bt->dropped_file); 316 relay_close(bt->rchan); 317 debugfs_remove(bt->dir); 318 free_percpu(bt->sequence); 319 free_percpu(bt->msg_data); 320 kfree(bt); 321 } 322 323 static void get_probe_ref(void) 324 { 325 mutex_lock(&blk_probe_mutex); 326 if (++blk_probes_ref == 1) 327 blk_register_tracepoints(); 328 mutex_unlock(&blk_probe_mutex); 329 } 330 331 static void put_probe_ref(void) 332 { 333 mutex_lock(&blk_probe_mutex); 334 if (!--blk_probes_ref) 335 blk_unregister_tracepoints(); 336 mutex_unlock(&blk_probe_mutex); 337 } 338 339 static void blk_trace_cleanup(struct blk_trace *bt) 340 { 341 synchronize_rcu(); 342 blk_trace_free(bt); 343 put_probe_ref(); 344 } 345 346 static int __blk_trace_remove(struct request_queue *q) 347 { 348 struct blk_trace *bt; 349 350 bt = rcu_replace_pointer(q->blk_trace, NULL, 351 lockdep_is_held(&q->debugfs_mutex)); 352 if (!bt) 353 return -EINVAL; 354 355 if (bt->trace_state != Blktrace_running) 356 blk_trace_cleanup(bt); 357 358 return 0; 359 } 360 361 int blk_trace_remove(struct request_queue *q) 362 { 363 int ret; 364 365 mutex_lock(&q->debugfs_mutex); 366 ret = __blk_trace_remove(q); 367 mutex_unlock(&q->debugfs_mutex); 368 369 return ret; 370 } 371 EXPORT_SYMBOL_GPL(blk_trace_remove); 372 373 static ssize_t blk_dropped_read(struct file *filp, char __user *buffer, 374 size_t count, loff_t *ppos) 375 { 376 struct blk_trace *bt = filp->private_data; 377 char buf[16]; 378 379 snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped)); 380 381 return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf)); 382 } 383 384 static const struct file_operations blk_dropped_fops = { 385 .owner = THIS_MODULE, 386 .open = simple_open, 387 .read = blk_dropped_read, 388 .llseek = default_llseek, 389 }; 390 391 static ssize_t blk_msg_write(struct file *filp, const char __user *buffer, 392 size_t count, loff_t *ppos) 393 { 394 char *msg; 395 struct blk_trace *bt; 396 397 if (count >= BLK_TN_MAX_MSG) 398 return -EINVAL; 399 400 msg = memdup_user_nul(buffer, count); 401 if (IS_ERR(msg)) 402 return PTR_ERR(msg); 403 404 bt = filp->private_data; 405 __trace_note_message(bt, NULL, "%s", msg); 406 kfree(msg); 407 408 return count; 409 } 410 411 static const struct file_operations blk_msg_fops = { 412 .owner = THIS_MODULE, 413 .open = simple_open, 414 .write = blk_msg_write, 415 .llseek = noop_llseek, 416 }; 417 418 /* 419 * Keep track of how many times we encountered a full subbuffer, to aid 420 * the user space app in telling how many lost events there were. 421 */ 422 static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf, 423 void *prev_subbuf, size_t prev_padding) 424 { 425 struct blk_trace *bt; 426 427 if (!relay_buf_full(buf)) 428 return 1; 429 430 bt = buf->chan->private_data; 431 atomic_inc(&bt->dropped); 432 return 0; 433 } 434 435 static int blk_remove_buf_file_callback(struct dentry *dentry) 436 { 437 debugfs_remove(dentry); 438 439 return 0; 440 } 441 442 static struct dentry *blk_create_buf_file_callback(const char *filename, 443 struct dentry *parent, 444 umode_t mode, 445 struct rchan_buf *buf, 446 int *is_global) 447 { 448 return debugfs_create_file(filename, mode, parent, buf, 449 &relay_file_operations); 450 } 451 452 static const struct rchan_callbacks blk_relay_callbacks = { 453 .subbuf_start = blk_subbuf_start_callback, 454 .create_buf_file = blk_create_buf_file_callback, 455 .remove_buf_file = blk_remove_buf_file_callback, 456 }; 457 458 static void blk_trace_setup_lba(struct blk_trace *bt, 459 struct block_device *bdev) 460 { 461 if (bdev) { 462 bt->start_lba = bdev->bd_start_sect; 463 bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev); 464 } else { 465 bt->start_lba = 0; 466 bt->end_lba = -1ULL; 467 } 468 } 469 470 /* 471 * Setup everything required to start tracing 472 */ 473 static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 474 struct block_device *bdev, 475 struct blk_user_trace_setup *buts) 476 { 477 struct blk_trace *bt = NULL; 478 struct dentry *dir = NULL; 479 int ret; 480 481 lockdep_assert_held(&q->debugfs_mutex); 482 483 if (!buts->buf_size || !buts->buf_nr) 484 return -EINVAL; 485 486 strncpy(buts->name, name, BLKTRACE_BDEV_SIZE); 487 buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0'; 488 489 /* 490 * some device names have larger paths - convert the slashes 491 * to underscores for this to work as expected 492 */ 493 strreplace(buts->name, '/', '_'); 494 495 /* 496 * bdev can be NULL, as with scsi-generic, this is a helpful as 497 * we can be. 498 */ 499 if (rcu_dereference_protected(q->blk_trace, 500 lockdep_is_held(&q->debugfs_mutex))) { 501 pr_warn("Concurrent blktraces are not allowed on %s\n", 502 buts->name); 503 return -EBUSY; 504 } 505 506 bt = kzalloc(sizeof(*bt), GFP_KERNEL); 507 if (!bt) 508 return -ENOMEM; 509 510 ret = -ENOMEM; 511 bt->sequence = alloc_percpu(unsigned long); 512 if (!bt->sequence) 513 goto err; 514 515 bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 516 if (!bt->msg_data) 517 goto err; 518 519 /* 520 * When tracing the whole disk reuse the existing debugfs directory 521 * created by the block layer on init. For partitions block devices, 522 * and scsi-generic block devices we create a temporary new debugfs 523 * directory that will be removed once the trace ends. 524 */ 525 if (bdev && !bdev_is_partition(bdev)) 526 dir = q->debugfs_dir; 527 else 528 bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root); 529 530 /* 531 * As blktrace relies on debugfs for its interface the debugfs directory 532 * is required, contrary to the usual mantra of not checking for debugfs 533 * files or directories. 534 */ 535 if (IS_ERR_OR_NULL(dir)) { 536 pr_warn("debugfs_dir not present for %s so skipping\n", 537 buts->name); 538 ret = -ENOENT; 539 goto err; 540 } 541 542 bt->dev = dev; 543 atomic_set(&bt->dropped, 0); 544 INIT_LIST_HEAD(&bt->running_list); 545 546 ret = -EIO; 547 bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt, 548 &blk_dropped_fops); 549 550 bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops); 551 552 bt->rchan = relay_open("trace", dir, buts->buf_size, 553 buts->buf_nr, &blk_relay_callbacks, bt); 554 if (!bt->rchan) 555 goto err; 556 557 bt->act_mask = buts->act_mask; 558 if (!bt->act_mask) 559 bt->act_mask = (u16) -1; 560 561 blk_trace_setup_lba(bt, bdev); 562 563 /* overwrite with user settings */ 564 if (buts->start_lba) 565 bt->start_lba = buts->start_lba; 566 if (buts->end_lba) 567 bt->end_lba = buts->end_lba; 568 569 bt->pid = buts->pid; 570 bt->trace_state = Blktrace_setup; 571 572 rcu_assign_pointer(q->blk_trace, bt); 573 get_probe_ref(); 574 575 ret = 0; 576 err: 577 if (ret) 578 blk_trace_free(bt); 579 return ret; 580 } 581 582 static int __blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 583 struct block_device *bdev, char __user *arg) 584 { 585 struct blk_user_trace_setup buts; 586 int ret; 587 588 ret = copy_from_user(&buts, arg, sizeof(buts)); 589 if (ret) 590 return -EFAULT; 591 592 ret = do_blk_trace_setup(q, name, dev, bdev, &buts); 593 if (ret) 594 return ret; 595 596 if (copy_to_user(arg, &buts, sizeof(buts))) { 597 __blk_trace_remove(q); 598 return -EFAULT; 599 } 600 return 0; 601 } 602 603 int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 604 struct block_device *bdev, 605 char __user *arg) 606 { 607 int ret; 608 609 mutex_lock(&q->debugfs_mutex); 610 ret = __blk_trace_setup(q, name, dev, bdev, arg); 611 mutex_unlock(&q->debugfs_mutex); 612 613 return ret; 614 } 615 EXPORT_SYMBOL_GPL(blk_trace_setup); 616 617 #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 618 static int compat_blk_trace_setup(struct request_queue *q, char *name, 619 dev_t dev, struct block_device *bdev, 620 char __user *arg) 621 { 622 struct blk_user_trace_setup buts; 623 struct compat_blk_user_trace_setup cbuts; 624 int ret; 625 626 if (copy_from_user(&cbuts, arg, sizeof(cbuts))) 627 return -EFAULT; 628 629 buts = (struct blk_user_trace_setup) { 630 .act_mask = cbuts.act_mask, 631 .buf_size = cbuts.buf_size, 632 .buf_nr = cbuts.buf_nr, 633 .start_lba = cbuts.start_lba, 634 .end_lba = cbuts.end_lba, 635 .pid = cbuts.pid, 636 }; 637 638 ret = do_blk_trace_setup(q, name, dev, bdev, &buts); 639 if (ret) 640 return ret; 641 642 if (copy_to_user(arg, &buts.name, ARRAY_SIZE(buts.name))) { 643 __blk_trace_remove(q); 644 return -EFAULT; 645 } 646 647 return 0; 648 } 649 #endif 650 651 static int __blk_trace_startstop(struct request_queue *q, int start) 652 { 653 int ret; 654 struct blk_trace *bt; 655 656 bt = rcu_dereference_protected(q->blk_trace, 657 lockdep_is_held(&q->debugfs_mutex)); 658 if (bt == NULL) 659 return -EINVAL; 660 661 /* 662 * For starting a trace, we can transition from a setup or stopped 663 * trace. For stopping a trace, the state must be running 664 */ 665 ret = -EINVAL; 666 if (start) { 667 if (bt->trace_state == Blktrace_setup || 668 bt->trace_state == Blktrace_stopped) { 669 blktrace_seq++; 670 smp_mb(); 671 bt->trace_state = Blktrace_running; 672 spin_lock_irq(&running_trace_lock); 673 list_add(&bt->running_list, &running_trace_list); 674 spin_unlock_irq(&running_trace_lock); 675 676 trace_note_time(bt); 677 ret = 0; 678 } 679 } else { 680 if (bt->trace_state == Blktrace_running) { 681 bt->trace_state = Blktrace_stopped; 682 spin_lock_irq(&running_trace_lock); 683 list_del_init(&bt->running_list); 684 spin_unlock_irq(&running_trace_lock); 685 relay_flush(bt->rchan); 686 ret = 0; 687 } 688 } 689 690 return ret; 691 } 692 693 int blk_trace_startstop(struct request_queue *q, int start) 694 { 695 int ret; 696 697 mutex_lock(&q->debugfs_mutex); 698 ret = __blk_trace_startstop(q, start); 699 mutex_unlock(&q->debugfs_mutex); 700 701 return ret; 702 } 703 EXPORT_SYMBOL_GPL(blk_trace_startstop); 704 705 /* 706 * When reading or writing the blktrace sysfs files, the references to the 707 * opened sysfs or device files should prevent the underlying block device 708 * from being removed. So no further delete protection is really needed. 709 */ 710 711 /** 712 * blk_trace_ioctl: - handle the ioctls associated with tracing 713 * @bdev: the block device 714 * @cmd: the ioctl cmd 715 * @arg: the argument data, if any 716 * 717 **/ 718 int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) 719 { 720 struct request_queue *q; 721 int ret, start = 0; 722 char b[BDEVNAME_SIZE]; 723 724 q = bdev_get_queue(bdev); 725 if (!q) 726 return -ENXIO; 727 728 mutex_lock(&q->debugfs_mutex); 729 730 switch (cmd) { 731 case BLKTRACESETUP: 732 bdevname(bdev, b); 733 ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 734 break; 735 #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 736 case BLKTRACESETUP32: 737 bdevname(bdev, b); 738 ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 739 break; 740 #endif 741 case BLKTRACESTART: 742 start = 1; 743 fallthrough; 744 case BLKTRACESTOP: 745 ret = __blk_trace_startstop(q, start); 746 break; 747 case BLKTRACETEARDOWN: 748 ret = __blk_trace_remove(q); 749 break; 750 default: 751 ret = -ENOTTY; 752 break; 753 } 754 755 mutex_unlock(&q->debugfs_mutex); 756 return ret; 757 } 758 759 /** 760 * blk_trace_shutdown: - stop and cleanup trace structures 761 * @q: the request queue associated with the device 762 * 763 **/ 764 void blk_trace_shutdown(struct request_queue *q) 765 { 766 mutex_lock(&q->debugfs_mutex); 767 if (rcu_dereference_protected(q->blk_trace, 768 lockdep_is_held(&q->debugfs_mutex))) { 769 __blk_trace_startstop(q, 0); 770 __blk_trace_remove(q); 771 } 772 773 mutex_unlock(&q->debugfs_mutex); 774 } 775 776 #ifdef CONFIG_BLK_CGROUP 777 static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 778 { 779 struct blk_trace *bt; 780 781 /* We don't use the 'bt' value here except as an optimization... */ 782 bt = rcu_dereference_protected(q->blk_trace, 1); 783 if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 784 return 0; 785 786 if (!bio->bi_blkg) 787 return 0; 788 return cgroup_id(bio_blkcg(bio)->css.cgroup); 789 } 790 #else 791 static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 792 { 793 return 0; 794 } 795 #endif 796 797 static u64 798 blk_trace_request_get_cgid(struct request *rq) 799 { 800 if (!rq->bio) 801 return 0; 802 /* Use the first bio */ 803 return blk_trace_bio_get_cgid(rq->q, rq->bio); 804 } 805 806 /* 807 * blktrace probes 808 */ 809 810 /** 811 * blk_add_trace_rq - Add a trace for a request oriented action 812 * @rq: the source request 813 * @error: return status to log 814 * @nr_bytes: number of completed bytes 815 * @what: the action 816 * @cgid: the cgroup info 817 * 818 * Description: 819 * Records an action against a request. Will log the bio offset + size. 820 * 821 **/ 822 static void blk_add_trace_rq(struct request *rq, int error, 823 unsigned int nr_bytes, u32 what, u64 cgid) 824 { 825 struct blk_trace *bt; 826 827 rcu_read_lock(); 828 bt = rcu_dereference(rq->q->blk_trace); 829 if (likely(!bt)) { 830 rcu_read_unlock(); 831 return; 832 } 833 834 if (blk_rq_is_passthrough(rq)) 835 what |= BLK_TC_ACT(BLK_TC_PC); 836 else 837 what |= BLK_TC_ACT(BLK_TC_FS); 838 839 __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq), 840 rq->cmd_flags, what, error, 0, NULL, cgid); 841 rcu_read_unlock(); 842 } 843 844 static void blk_add_trace_rq_insert(void *ignore, struct request *rq) 845 { 846 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT, 847 blk_trace_request_get_cgid(rq)); 848 } 849 850 static void blk_add_trace_rq_issue(void *ignore, struct request *rq) 851 { 852 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ISSUE, 853 blk_trace_request_get_cgid(rq)); 854 } 855 856 static void blk_add_trace_rq_merge(void *ignore, struct request *rq) 857 { 858 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_BACKMERGE, 859 blk_trace_request_get_cgid(rq)); 860 } 861 862 static void blk_add_trace_rq_requeue(void *ignore, struct request *rq) 863 { 864 blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_REQUEUE, 865 blk_trace_request_get_cgid(rq)); 866 } 867 868 static void blk_add_trace_rq_complete(void *ignore, struct request *rq, 869 int error, unsigned int nr_bytes) 870 { 871 blk_add_trace_rq(rq, error, nr_bytes, BLK_TA_COMPLETE, 872 blk_trace_request_get_cgid(rq)); 873 } 874 875 /** 876 * blk_add_trace_bio - Add a trace for a bio oriented action 877 * @q: queue the io is for 878 * @bio: the source bio 879 * @what: the action 880 * @error: error, if any 881 * 882 * Description: 883 * Records an action against a bio. Will log the bio offset + size. 884 * 885 **/ 886 static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, 887 u32 what, int error) 888 { 889 struct blk_trace *bt; 890 891 rcu_read_lock(); 892 bt = rcu_dereference(q->blk_trace); 893 if (likely(!bt)) { 894 rcu_read_unlock(); 895 return; 896 } 897 898 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 899 bio_op(bio), bio->bi_opf, what, error, 0, NULL, 900 blk_trace_bio_get_cgid(q, bio)); 901 rcu_read_unlock(); 902 } 903 904 static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio) 905 { 906 blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BOUNCE, 0); 907 } 908 909 static void blk_add_trace_bio_complete(void *ignore, 910 struct request_queue *q, struct bio *bio) 911 { 912 blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, 913 blk_status_to_errno(bio->bi_status)); 914 } 915 916 static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio) 917 { 918 blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BACKMERGE, 0); 919 } 920 921 static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio) 922 { 923 blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_FRONTMERGE, 0); 924 } 925 926 static void blk_add_trace_bio_queue(void *ignore, struct bio *bio) 927 { 928 blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_QUEUE, 0); 929 } 930 931 static void blk_add_trace_getrq(void *ignore, struct bio *bio) 932 { 933 blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_GETRQ, 0); 934 } 935 936 static void blk_add_trace_plug(void *ignore, struct request_queue *q) 937 { 938 struct blk_trace *bt; 939 940 rcu_read_lock(); 941 bt = rcu_dereference(q->blk_trace); 942 if (bt) 943 __blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); 944 rcu_read_unlock(); 945 } 946 947 static void blk_add_trace_unplug(void *ignore, struct request_queue *q, 948 unsigned int depth, bool explicit) 949 { 950 struct blk_trace *bt; 951 952 rcu_read_lock(); 953 bt = rcu_dereference(q->blk_trace); 954 if (bt) { 955 __be64 rpdu = cpu_to_be64(depth); 956 u32 what; 957 958 if (explicit) 959 what = BLK_TA_UNPLUG_IO; 960 else 961 what = BLK_TA_UNPLUG_TIMER; 962 963 __blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); 964 } 965 rcu_read_unlock(); 966 } 967 968 static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) 969 { 970 struct request_queue *q = bio->bi_disk->queue; 971 struct blk_trace *bt; 972 973 rcu_read_lock(); 974 bt = rcu_dereference(q->blk_trace); 975 if (bt) { 976 __be64 rpdu = cpu_to_be64(pdu); 977 978 __blk_add_trace(bt, bio->bi_iter.bi_sector, 979 bio->bi_iter.bi_size, bio_op(bio), bio->bi_opf, 980 BLK_TA_SPLIT, 981 blk_status_to_errno(bio->bi_status), 982 sizeof(rpdu), &rpdu, 983 blk_trace_bio_get_cgid(q, bio)); 984 } 985 rcu_read_unlock(); 986 } 987 988 /** 989 * blk_add_trace_bio_remap - Add a trace for a bio-remap operation 990 * @ignore: trace callback data parameter (not used) 991 * @bio: the source bio 992 * @dev: source device 993 * @from: source sector 994 * 995 * Called after a bio is remapped to a different device and/or sector. 996 **/ 997 static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, 998 sector_t from) 999 { 1000 struct request_queue *q = bio->bi_disk->queue; 1001 struct blk_trace *bt; 1002 struct blk_io_trace_remap r; 1003 1004 rcu_read_lock(); 1005 bt = rcu_dereference(q->blk_trace); 1006 if (likely(!bt)) { 1007 rcu_read_unlock(); 1008 return; 1009 } 1010 1011 r.device_from = cpu_to_be32(dev); 1012 r.device_to = cpu_to_be32(bio_dev(bio)); 1013 r.sector_from = cpu_to_be64(from); 1014 1015 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 1016 bio_op(bio), bio->bi_opf, BLK_TA_REMAP, 1017 blk_status_to_errno(bio->bi_status), 1018 sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); 1019 rcu_read_unlock(); 1020 } 1021 1022 /** 1023 * blk_add_trace_rq_remap - Add a trace for a request-remap operation 1024 * @ignore: trace callback data parameter (not used) 1025 * @rq: the source request 1026 * @dev: target device 1027 * @from: source sector 1028 * 1029 * Description: 1030 * Device mapper remaps request to other devices. 1031 * Add a trace for that action. 1032 * 1033 **/ 1034 static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev, 1035 sector_t from) 1036 { 1037 struct blk_trace *bt; 1038 struct blk_io_trace_remap r; 1039 1040 rcu_read_lock(); 1041 bt = rcu_dereference(rq->q->blk_trace); 1042 if (likely(!bt)) { 1043 rcu_read_unlock(); 1044 return; 1045 } 1046 1047 r.device_from = cpu_to_be32(dev); 1048 r.device_to = cpu_to_be32(disk_devt(rq->rq_disk)); 1049 r.sector_from = cpu_to_be64(from); 1050 1051 __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 1052 rq_data_dir(rq), 0, BLK_TA_REMAP, 0, 1053 sizeof(r), &r, blk_trace_request_get_cgid(rq)); 1054 rcu_read_unlock(); 1055 } 1056 1057 /** 1058 * blk_add_driver_data - Add binary message with driver-specific data 1059 * @rq: io request 1060 * @data: driver-specific data 1061 * @len: length of driver-specific data 1062 * 1063 * Description: 1064 * Some drivers might want to write driver-specific data per request. 1065 * 1066 **/ 1067 void blk_add_driver_data(struct request *rq, void *data, size_t len) 1068 { 1069 struct blk_trace *bt; 1070 1071 rcu_read_lock(); 1072 bt = rcu_dereference(rq->q->blk_trace); 1073 if (likely(!bt)) { 1074 rcu_read_unlock(); 1075 return; 1076 } 1077 1078 __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0, 1079 BLK_TA_DRV_DATA, 0, len, data, 1080 blk_trace_request_get_cgid(rq)); 1081 rcu_read_unlock(); 1082 } 1083 EXPORT_SYMBOL_GPL(blk_add_driver_data); 1084 1085 static void blk_register_tracepoints(void) 1086 { 1087 int ret; 1088 1089 ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 1090 WARN_ON(ret); 1091 ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 1092 WARN_ON(ret); 1093 ret = register_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 1094 WARN_ON(ret); 1095 ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 1096 WARN_ON(ret); 1097 ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 1098 WARN_ON(ret); 1099 ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); 1100 WARN_ON(ret); 1101 ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 1102 WARN_ON(ret); 1103 ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 1104 WARN_ON(ret); 1105 ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 1106 WARN_ON(ret); 1107 ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 1108 WARN_ON(ret); 1109 ret = register_trace_block_getrq(blk_add_trace_getrq, NULL); 1110 WARN_ON(ret); 1111 ret = register_trace_block_plug(blk_add_trace_plug, NULL); 1112 WARN_ON(ret); 1113 ret = register_trace_block_unplug(blk_add_trace_unplug, NULL); 1114 WARN_ON(ret); 1115 ret = register_trace_block_split(blk_add_trace_split, NULL); 1116 WARN_ON(ret); 1117 ret = register_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 1118 WARN_ON(ret); 1119 ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1120 WARN_ON(ret); 1121 } 1122 1123 static void blk_unregister_tracepoints(void) 1124 { 1125 unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1126 unregister_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 1127 unregister_trace_block_split(blk_add_trace_split, NULL); 1128 unregister_trace_block_unplug(blk_add_trace_unplug, NULL); 1129 unregister_trace_block_plug(blk_add_trace_plug, NULL); 1130 unregister_trace_block_getrq(blk_add_trace_getrq, NULL); 1131 unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 1132 unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 1133 unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 1134 unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 1135 unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); 1136 unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 1137 unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 1138 unregister_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 1139 unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 1140 unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 1141 1142 tracepoint_synchronize_unregister(); 1143 } 1144 1145 /* 1146 * struct blk_io_tracer formatting routines 1147 */ 1148 1149 static void fill_rwbs(char *rwbs, const struct blk_io_trace *t) 1150 { 1151 int i = 0; 1152 int tc = t->action >> BLK_TC_SHIFT; 1153 1154 if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 1155 rwbs[i++] = 'N'; 1156 goto out; 1157 } 1158 1159 if (tc & BLK_TC_FLUSH) 1160 rwbs[i++] = 'F'; 1161 1162 if (tc & BLK_TC_DISCARD) 1163 rwbs[i++] = 'D'; 1164 else if (tc & BLK_TC_WRITE) 1165 rwbs[i++] = 'W'; 1166 else if (t->bytes) 1167 rwbs[i++] = 'R'; 1168 else 1169 rwbs[i++] = 'N'; 1170 1171 if (tc & BLK_TC_FUA) 1172 rwbs[i++] = 'F'; 1173 if (tc & BLK_TC_AHEAD) 1174 rwbs[i++] = 'A'; 1175 if (tc & BLK_TC_SYNC) 1176 rwbs[i++] = 'S'; 1177 if (tc & BLK_TC_META) 1178 rwbs[i++] = 'M'; 1179 out: 1180 rwbs[i] = '\0'; 1181 } 1182 1183 static inline 1184 const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent) 1185 { 1186 return (const struct blk_io_trace *)ent; 1187 } 1188 1189 static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg) 1190 { 1191 return (void *)(te_blk_io_trace(ent) + 1) + (has_cg ? sizeof(u64) : 0); 1192 } 1193 1194 static inline u64 t_cgid(const struct trace_entry *ent) 1195 { 1196 return *(u64 *)(te_blk_io_trace(ent) + 1); 1197 } 1198 1199 static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg) 1200 { 1201 return te_blk_io_trace(ent)->pdu_len - (has_cg ? sizeof(u64) : 0); 1202 } 1203 1204 static inline u32 t_action(const struct trace_entry *ent) 1205 { 1206 return te_blk_io_trace(ent)->action; 1207 } 1208 1209 static inline u32 t_bytes(const struct trace_entry *ent) 1210 { 1211 return te_blk_io_trace(ent)->bytes; 1212 } 1213 1214 static inline u32 t_sec(const struct trace_entry *ent) 1215 { 1216 return te_blk_io_trace(ent)->bytes >> 9; 1217 } 1218 1219 static inline unsigned long long t_sector(const struct trace_entry *ent) 1220 { 1221 return te_blk_io_trace(ent)->sector; 1222 } 1223 1224 static inline __u16 t_error(const struct trace_entry *ent) 1225 { 1226 return te_blk_io_trace(ent)->error; 1227 } 1228 1229 static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg) 1230 { 1231 const __be64 *val = pdu_start(ent, has_cg); 1232 return be64_to_cpu(*val); 1233 } 1234 1235 typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act, 1236 bool has_cg); 1237 1238 static void blk_log_action_classic(struct trace_iterator *iter, const char *act, 1239 bool has_cg) 1240 { 1241 char rwbs[RWBS_LEN]; 1242 unsigned long long ts = iter->ts; 1243 unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC); 1244 unsigned secs = (unsigned long)ts; 1245 const struct blk_io_trace *t = te_blk_io_trace(iter->ent); 1246 1247 fill_rwbs(rwbs, t); 1248 1249 trace_seq_printf(&iter->seq, 1250 "%3d,%-3d %2d %5d.%09lu %5u %2s %3s ", 1251 MAJOR(t->device), MINOR(t->device), iter->cpu, 1252 secs, nsec_rem, iter->ent->pid, act, rwbs); 1253 } 1254 1255 static void blk_log_action(struct trace_iterator *iter, const char *act, 1256 bool has_cg) 1257 { 1258 char rwbs[RWBS_LEN]; 1259 const struct blk_io_trace *t = te_blk_io_trace(iter->ent); 1260 1261 fill_rwbs(rwbs, t); 1262 if (has_cg) { 1263 u64 id = t_cgid(iter->ent); 1264 1265 if (blk_tracer_flags.val & TRACE_BLK_OPT_CGNAME) { 1266 char blkcg_name_buf[NAME_MAX + 1] = "<...>"; 1267 1268 cgroup_path_from_kernfs_id(id, blkcg_name_buf, 1269 sizeof(blkcg_name_buf)); 1270 trace_seq_printf(&iter->seq, "%3d,%-3d %s %2s %3s ", 1271 MAJOR(t->device), MINOR(t->device), 1272 blkcg_name_buf, act, rwbs); 1273 } else { 1274 /* 1275 * The cgid portion used to be "INO,GEN". Userland 1276 * builds a FILEID_INO32_GEN fid out of them and 1277 * opens the cgroup using open_by_handle_at(2). 1278 * While 32bit ino setups are still the same, 64bit 1279 * ones now use the 64bit ino as the whole ID and 1280 * no longer use generation. 1281 * 1282 * Regardless of the content, always output 1283 * "LOW32,HIGH32" so that FILEID_INO32_GEN fid can 1284 * be mapped back to @id on both 64 and 32bit ino 1285 * setups. See __kernfs_fh_to_dentry(). 1286 */ 1287 trace_seq_printf(&iter->seq, 1288 "%3d,%-3d %llx,%-llx %2s %3s ", 1289 MAJOR(t->device), MINOR(t->device), 1290 id & U32_MAX, id >> 32, act, rwbs); 1291 } 1292 } else 1293 trace_seq_printf(&iter->seq, "%3d,%-3d %2s %3s ", 1294 MAJOR(t->device), MINOR(t->device), act, rwbs); 1295 } 1296 1297 static void blk_log_dump_pdu(struct trace_seq *s, 1298 const struct trace_entry *ent, bool has_cg) 1299 { 1300 const unsigned char *pdu_buf; 1301 int pdu_len; 1302 int i, end; 1303 1304 pdu_buf = pdu_start(ent, has_cg); 1305 pdu_len = pdu_real_len(ent, has_cg); 1306 1307 if (!pdu_len) 1308 return; 1309 1310 /* find the last zero that needs to be printed */ 1311 for (end = pdu_len - 1; end >= 0; end--) 1312 if (pdu_buf[end]) 1313 break; 1314 end++; 1315 1316 trace_seq_putc(s, '('); 1317 1318 for (i = 0; i < pdu_len; i++) { 1319 1320 trace_seq_printf(s, "%s%02x", 1321 i == 0 ? "" : " ", pdu_buf[i]); 1322 1323 /* 1324 * stop when the rest is just zeros and indicate so 1325 * with a ".." appended 1326 */ 1327 if (i == end && end != pdu_len - 1) { 1328 trace_seq_puts(s, " ..) "); 1329 return; 1330 } 1331 } 1332 1333 trace_seq_puts(s, ") "); 1334 } 1335 1336 static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1337 { 1338 char cmd[TASK_COMM_LEN]; 1339 1340 trace_find_cmdline(ent->pid, cmd); 1341 1342 if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1343 trace_seq_printf(s, "%u ", t_bytes(ent)); 1344 blk_log_dump_pdu(s, ent, has_cg); 1345 trace_seq_printf(s, "[%s]\n", cmd); 1346 } else { 1347 if (t_sec(ent)) 1348 trace_seq_printf(s, "%llu + %u [%s]\n", 1349 t_sector(ent), t_sec(ent), cmd); 1350 else 1351 trace_seq_printf(s, "[%s]\n", cmd); 1352 } 1353 } 1354 1355 static void blk_log_with_error(struct trace_seq *s, 1356 const struct trace_entry *ent, bool has_cg) 1357 { 1358 if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1359 blk_log_dump_pdu(s, ent, has_cg); 1360 trace_seq_printf(s, "[%d]\n", t_error(ent)); 1361 } else { 1362 if (t_sec(ent)) 1363 trace_seq_printf(s, "%llu + %u [%d]\n", 1364 t_sector(ent), 1365 t_sec(ent), t_error(ent)); 1366 else 1367 trace_seq_printf(s, "%llu [%d]\n", 1368 t_sector(ent), t_error(ent)); 1369 } 1370 } 1371 1372 static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1373 { 1374 const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg); 1375 1376 trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n", 1377 t_sector(ent), t_sec(ent), 1378 MAJOR(be32_to_cpu(__r->device_from)), 1379 MINOR(be32_to_cpu(__r->device_from)), 1380 be64_to_cpu(__r->sector_from)); 1381 } 1382 1383 static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1384 { 1385 char cmd[TASK_COMM_LEN]; 1386 1387 trace_find_cmdline(ent->pid, cmd); 1388 1389 trace_seq_printf(s, "[%s]\n", cmd); 1390 } 1391 1392 static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1393 { 1394 char cmd[TASK_COMM_LEN]; 1395 1396 trace_find_cmdline(ent->pid, cmd); 1397 1398 trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent, has_cg)); 1399 } 1400 1401 static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 1402 { 1403 char cmd[TASK_COMM_LEN]; 1404 1405 trace_find_cmdline(ent->pid, cmd); 1406 1407 trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent), 1408 get_pdu_int(ent, has_cg), cmd); 1409 } 1410 1411 static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent, 1412 bool has_cg) 1413 { 1414 1415 trace_seq_putmem(s, pdu_start(ent, has_cg), 1416 pdu_real_len(ent, has_cg)); 1417 trace_seq_putc(s, '\n'); 1418 } 1419 1420 /* 1421 * struct tracer operations 1422 */ 1423 1424 static void blk_tracer_print_header(struct seq_file *m) 1425 { 1426 if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 1427 return; 1428 seq_puts(m, "# DEV CPU TIMESTAMP PID ACT FLG\n" 1429 "# | | | | | |\n"); 1430 } 1431 1432 static void blk_tracer_start(struct trace_array *tr) 1433 { 1434 blk_tracer_enabled = true; 1435 } 1436 1437 static int blk_tracer_init(struct trace_array *tr) 1438 { 1439 blk_tr = tr; 1440 blk_tracer_start(tr); 1441 return 0; 1442 } 1443 1444 static void blk_tracer_stop(struct trace_array *tr) 1445 { 1446 blk_tracer_enabled = false; 1447 } 1448 1449 static void blk_tracer_reset(struct trace_array *tr) 1450 { 1451 blk_tracer_stop(tr); 1452 } 1453 1454 static const struct { 1455 const char *act[2]; 1456 void (*print)(struct trace_seq *s, const struct trace_entry *ent, 1457 bool has_cg); 1458 } what2act[] = { 1459 [__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic }, 1460 [__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic }, 1461 [__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic }, 1462 [__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic }, 1463 [__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic }, 1464 [__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error }, 1465 [__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic }, 1466 [__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error }, 1467 [__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug }, 1468 [__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug }, 1469 [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug }, 1470 [__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic }, 1471 [__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split }, 1472 [__BLK_TA_BOUNCE] = {{ "B", "bounce" }, blk_log_generic }, 1473 [__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap }, 1474 }; 1475 1476 static enum print_line_t print_one_line(struct trace_iterator *iter, 1477 bool classic) 1478 { 1479 struct trace_array *tr = iter->tr; 1480 struct trace_seq *s = &iter->seq; 1481 const struct blk_io_trace *t; 1482 u16 what; 1483 bool long_act; 1484 blk_log_action_t *log_action; 1485 bool has_cg; 1486 1487 t = te_blk_io_trace(iter->ent); 1488 what = (t->action & ((1 << BLK_TC_SHIFT) - 1)) & ~__BLK_TA_CGROUP; 1489 long_act = !!(tr->trace_flags & TRACE_ITER_VERBOSE); 1490 log_action = classic ? &blk_log_action_classic : &blk_log_action; 1491 has_cg = t->action & __BLK_TA_CGROUP; 1492 1493 if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 1494 log_action(iter, long_act ? "message" : "m", has_cg); 1495 blk_log_msg(s, iter->ent, has_cg); 1496 return trace_handle_return(s); 1497 } 1498 1499 if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act))) 1500 trace_seq_printf(s, "Unknown action %x\n", what); 1501 else { 1502 log_action(iter, what2act[what].act[long_act], has_cg); 1503 what2act[what].print(s, iter->ent, has_cg); 1504 } 1505 1506 return trace_handle_return(s); 1507 } 1508 1509 static enum print_line_t blk_trace_event_print(struct trace_iterator *iter, 1510 int flags, struct trace_event *event) 1511 { 1512 return print_one_line(iter, false); 1513 } 1514 1515 static void blk_trace_synthesize_old_trace(struct trace_iterator *iter) 1516 { 1517 struct trace_seq *s = &iter->seq; 1518 struct blk_io_trace *t = (struct blk_io_trace *)iter->ent; 1519 const int offset = offsetof(struct blk_io_trace, sector); 1520 struct blk_io_trace old = { 1521 .magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION, 1522 .time = iter->ts, 1523 }; 1524 1525 trace_seq_putmem(s, &old, offset); 1526 trace_seq_putmem(s, &t->sector, 1527 sizeof(old) - offset + t->pdu_len); 1528 } 1529 1530 static enum print_line_t 1531 blk_trace_event_print_binary(struct trace_iterator *iter, int flags, 1532 struct trace_event *event) 1533 { 1534 blk_trace_synthesize_old_trace(iter); 1535 1536 return trace_handle_return(&iter->seq); 1537 } 1538 1539 static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter) 1540 { 1541 if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 1542 return TRACE_TYPE_UNHANDLED; 1543 1544 return print_one_line(iter, true); 1545 } 1546 1547 static int 1548 blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1549 { 1550 /* don't output context-info for blk_classic output */ 1551 if (bit == TRACE_BLK_OPT_CLASSIC) { 1552 if (set) 1553 tr->trace_flags &= ~TRACE_ITER_CONTEXT_INFO; 1554 else 1555 tr->trace_flags |= TRACE_ITER_CONTEXT_INFO; 1556 } 1557 return 0; 1558 } 1559 1560 static struct tracer blk_tracer __read_mostly = { 1561 .name = "blk", 1562 .init = blk_tracer_init, 1563 .reset = blk_tracer_reset, 1564 .start = blk_tracer_start, 1565 .stop = blk_tracer_stop, 1566 .print_header = blk_tracer_print_header, 1567 .print_line = blk_tracer_print_line, 1568 .flags = &blk_tracer_flags, 1569 .set_flag = blk_tracer_set_flag, 1570 }; 1571 1572 static struct trace_event_functions trace_blk_event_funcs = { 1573 .trace = blk_trace_event_print, 1574 .binary = blk_trace_event_print_binary, 1575 }; 1576 1577 static struct trace_event trace_blk_event = { 1578 .type = TRACE_BLK, 1579 .funcs = &trace_blk_event_funcs, 1580 }; 1581 1582 static int __init init_blk_tracer(void) 1583 { 1584 if (!register_trace_event(&trace_blk_event)) { 1585 pr_warn("Warning: could not register block events\n"); 1586 return 1; 1587 } 1588 1589 if (register_tracer(&blk_tracer) != 0) { 1590 pr_warn("Warning: could not register the block tracer\n"); 1591 unregister_trace_event(&trace_blk_event); 1592 return 1; 1593 } 1594 1595 return 0; 1596 } 1597 1598 device_initcall(init_blk_tracer); 1599 1600 static int blk_trace_remove_queue(struct request_queue *q) 1601 { 1602 struct blk_trace *bt; 1603 1604 bt = rcu_replace_pointer(q->blk_trace, NULL, 1605 lockdep_is_held(&q->debugfs_mutex)); 1606 if (bt == NULL) 1607 return -EINVAL; 1608 1609 put_probe_ref(); 1610 synchronize_rcu(); 1611 blk_trace_free(bt); 1612 return 0; 1613 } 1614 1615 /* 1616 * Setup everything required to start tracing 1617 */ 1618 static int blk_trace_setup_queue(struct request_queue *q, 1619 struct block_device *bdev) 1620 { 1621 struct blk_trace *bt = NULL; 1622 int ret = -ENOMEM; 1623 1624 bt = kzalloc(sizeof(*bt), GFP_KERNEL); 1625 if (!bt) 1626 return -ENOMEM; 1627 1628 bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 1629 if (!bt->msg_data) 1630 goto free_bt; 1631 1632 bt->dev = bdev->bd_dev; 1633 bt->act_mask = (u16)-1; 1634 1635 blk_trace_setup_lba(bt, bdev); 1636 1637 rcu_assign_pointer(q->blk_trace, bt); 1638 get_probe_ref(); 1639 return 0; 1640 1641 free_bt: 1642 blk_trace_free(bt); 1643 return ret; 1644 } 1645 1646 /* 1647 * sysfs interface to enable and configure tracing 1648 */ 1649 1650 static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 1651 struct device_attribute *attr, 1652 char *buf); 1653 static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 1654 struct device_attribute *attr, 1655 const char *buf, size_t count); 1656 #define BLK_TRACE_DEVICE_ATTR(_name) \ 1657 DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \ 1658 sysfs_blk_trace_attr_show, \ 1659 sysfs_blk_trace_attr_store) 1660 1661 static BLK_TRACE_DEVICE_ATTR(enable); 1662 static BLK_TRACE_DEVICE_ATTR(act_mask); 1663 static BLK_TRACE_DEVICE_ATTR(pid); 1664 static BLK_TRACE_DEVICE_ATTR(start_lba); 1665 static BLK_TRACE_DEVICE_ATTR(end_lba); 1666 1667 static struct attribute *blk_trace_attrs[] = { 1668 &dev_attr_enable.attr, 1669 &dev_attr_act_mask.attr, 1670 &dev_attr_pid.attr, 1671 &dev_attr_start_lba.attr, 1672 &dev_attr_end_lba.attr, 1673 NULL 1674 }; 1675 1676 struct attribute_group blk_trace_attr_group = { 1677 .name = "trace", 1678 .attrs = blk_trace_attrs, 1679 }; 1680 1681 static const struct { 1682 int mask; 1683 const char *str; 1684 } mask_maps[] = { 1685 { BLK_TC_READ, "read" }, 1686 { BLK_TC_WRITE, "write" }, 1687 { BLK_TC_FLUSH, "flush" }, 1688 { BLK_TC_SYNC, "sync" }, 1689 { BLK_TC_QUEUE, "queue" }, 1690 { BLK_TC_REQUEUE, "requeue" }, 1691 { BLK_TC_ISSUE, "issue" }, 1692 { BLK_TC_COMPLETE, "complete" }, 1693 { BLK_TC_FS, "fs" }, 1694 { BLK_TC_PC, "pc" }, 1695 { BLK_TC_NOTIFY, "notify" }, 1696 { BLK_TC_AHEAD, "ahead" }, 1697 { BLK_TC_META, "meta" }, 1698 { BLK_TC_DISCARD, "discard" }, 1699 { BLK_TC_DRV_DATA, "drv_data" }, 1700 { BLK_TC_FUA, "fua" }, 1701 }; 1702 1703 static int blk_trace_str2mask(const char *str) 1704 { 1705 int i; 1706 int mask = 0; 1707 char *buf, *s, *token; 1708 1709 buf = kstrdup(str, GFP_KERNEL); 1710 if (buf == NULL) 1711 return -ENOMEM; 1712 s = strstrip(buf); 1713 1714 while (1) { 1715 token = strsep(&s, ","); 1716 if (token == NULL) 1717 break; 1718 1719 if (*token == '\0') 1720 continue; 1721 1722 for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 1723 if (strcasecmp(token, mask_maps[i].str) == 0) { 1724 mask |= mask_maps[i].mask; 1725 break; 1726 } 1727 } 1728 if (i == ARRAY_SIZE(mask_maps)) { 1729 mask = -EINVAL; 1730 break; 1731 } 1732 } 1733 kfree(buf); 1734 1735 return mask; 1736 } 1737 1738 static ssize_t blk_trace_mask2str(char *buf, int mask) 1739 { 1740 int i; 1741 char *p = buf; 1742 1743 for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 1744 if (mask & mask_maps[i].mask) { 1745 p += sprintf(p, "%s%s", 1746 (p == buf) ? "" : ",", mask_maps[i].str); 1747 } 1748 } 1749 *p++ = '\n'; 1750 1751 return p - buf; 1752 } 1753 1754 static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 1755 struct device_attribute *attr, 1756 char *buf) 1757 { 1758 struct block_device *bdev = dev_to_bdev(dev); 1759 struct request_queue *q = bdev_get_queue(bdev); 1760 struct blk_trace *bt; 1761 ssize_t ret = -ENXIO; 1762 1763 mutex_lock(&q->debugfs_mutex); 1764 1765 bt = rcu_dereference_protected(q->blk_trace, 1766 lockdep_is_held(&q->debugfs_mutex)); 1767 if (attr == &dev_attr_enable) { 1768 ret = sprintf(buf, "%u\n", !!bt); 1769 goto out_unlock_bdev; 1770 } 1771 1772 if (bt == NULL) 1773 ret = sprintf(buf, "disabled\n"); 1774 else if (attr == &dev_attr_act_mask) 1775 ret = blk_trace_mask2str(buf, bt->act_mask); 1776 else if (attr == &dev_attr_pid) 1777 ret = sprintf(buf, "%u\n", bt->pid); 1778 else if (attr == &dev_attr_start_lba) 1779 ret = sprintf(buf, "%llu\n", bt->start_lba); 1780 else if (attr == &dev_attr_end_lba) 1781 ret = sprintf(buf, "%llu\n", bt->end_lba); 1782 1783 out_unlock_bdev: 1784 mutex_unlock(&q->debugfs_mutex); 1785 return ret; 1786 } 1787 1788 static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 1789 struct device_attribute *attr, 1790 const char *buf, size_t count) 1791 { 1792 struct block_device *bdev = dev_to_bdev(dev); 1793 struct request_queue *q = bdev_get_queue(bdev); 1794 struct blk_trace *bt; 1795 u64 value; 1796 ssize_t ret = -EINVAL; 1797 1798 if (count == 0) 1799 goto out; 1800 1801 if (attr == &dev_attr_act_mask) { 1802 if (kstrtoull(buf, 0, &value)) { 1803 /* Assume it is a list of trace category names */ 1804 ret = blk_trace_str2mask(buf); 1805 if (ret < 0) 1806 goto out; 1807 value = ret; 1808 } 1809 } else { 1810 if (kstrtoull(buf, 0, &value)) 1811 goto out; 1812 } 1813 1814 mutex_lock(&q->debugfs_mutex); 1815 1816 bt = rcu_dereference_protected(q->blk_trace, 1817 lockdep_is_held(&q->debugfs_mutex)); 1818 if (attr == &dev_attr_enable) { 1819 if (!!value == !!bt) { 1820 ret = 0; 1821 goto out_unlock_bdev; 1822 } 1823 if (value) 1824 ret = blk_trace_setup_queue(q, bdev); 1825 else 1826 ret = blk_trace_remove_queue(q); 1827 goto out_unlock_bdev; 1828 } 1829 1830 ret = 0; 1831 if (bt == NULL) { 1832 ret = blk_trace_setup_queue(q, bdev); 1833 bt = rcu_dereference_protected(q->blk_trace, 1834 lockdep_is_held(&q->debugfs_mutex)); 1835 } 1836 1837 if (ret == 0) { 1838 if (attr == &dev_attr_act_mask) 1839 bt->act_mask = value; 1840 else if (attr == &dev_attr_pid) 1841 bt->pid = value; 1842 else if (attr == &dev_attr_start_lba) 1843 bt->start_lba = value; 1844 else if (attr == &dev_attr_end_lba) 1845 bt->end_lba = value; 1846 } 1847 1848 out_unlock_bdev: 1849 mutex_unlock(&q->debugfs_mutex); 1850 out: 1851 return ret ? ret : count; 1852 } 1853 1854 int blk_trace_init_sysfs(struct device *dev) 1855 { 1856 return sysfs_create_group(&dev->kobj, &blk_trace_attr_group); 1857 } 1858 1859 void blk_trace_remove_sysfs(struct device *dev) 1860 { 1861 sysfs_remove_group(&dev->kobj, &blk_trace_attr_group); 1862 } 1863 1864 #endif /* CONFIG_BLK_DEV_IO_TRACE */ 1865 1866 #ifdef CONFIG_EVENT_TRACING 1867 1868 void blk_fill_rwbs(char *rwbs, unsigned int op, int bytes) 1869 { 1870 int i = 0; 1871 1872 if (op & REQ_PREFLUSH) 1873 rwbs[i++] = 'F'; 1874 1875 switch (op & REQ_OP_MASK) { 1876 case REQ_OP_WRITE: 1877 case REQ_OP_WRITE_SAME: 1878 rwbs[i++] = 'W'; 1879 break; 1880 case REQ_OP_DISCARD: 1881 rwbs[i++] = 'D'; 1882 break; 1883 case REQ_OP_SECURE_ERASE: 1884 rwbs[i++] = 'D'; 1885 rwbs[i++] = 'E'; 1886 break; 1887 case REQ_OP_FLUSH: 1888 rwbs[i++] = 'F'; 1889 break; 1890 case REQ_OP_READ: 1891 rwbs[i++] = 'R'; 1892 break; 1893 default: 1894 rwbs[i++] = 'N'; 1895 } 1896 1897 if (op & REQ_FUA) 1898 rwbs[i++] = 'F'; 1899 if (op & REQ_RAHEAD) 1900 rwbs[i++] = 'A'; 1901 if (op & REQ_SYNC) 1902 rwbs[i++] = 'S'; 1903 if (op & REQ_META) 1904 rwbs[i++] = 'M'; 1905 1906 rwbs[i] = '\0'; 1907 } 1908 EXPORT_SYMBOL_GPL(blk_fill_rwbs); 1909 1910 #endif /* CONFIG_EVENT_TRACING */ 1911 1912