191c1e6baSSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0 22db270a8SFrederic Weisbecker /* 32db270a8SFrederic Weisbecker * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk> 42db270a8SFrederic Weisbecker * 52db270a8SFrederic Weisbecker */ 61b0b2836SLuis Chamberlain 71b0b2836SLuis Chamberlain #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 81b0b2836SLuis Chamberlain 92db270a8SFrederic Weisbecker #include <linux/kernel.h> 102db270a8SFrederic Weisbecker #include <linux/blkdev.h> 112db270a8SFrederic Weisbecker #include <linux/blktrace_api.h> 122db270a8SFrederic Weisbecker #include <linux/percpu.h> 132db270a8SFrederic Weisbecker #include <linux/init.h> 142db270a8SFrederic Weisbecker #include <linux/mutex.h> 155a0e3ad6STejun Heo #include <linux/slab.h> 162db270a8SFrederic Weisbecker #include <linux/debugfs.h> 176e5fdeedSPaul Gortmaker #include <linux/export.h> 182db270a8SFrederic Weisbecker #include <linux/time.h> 192db270a8SFrederic Weisbecker #include <linux/uaccess.h> 20a404d557SJan Kara #include <linux/list.h> 21ca1136c9SShaohua Li #include <linux/blk-cgroup.h> 2255782138SLi Zefan 2318fbda91SOmar Sandoval #include "../../block/blk.h" 2418fbda91SOmar Sandoval 2555782138SLi Zefan #include <trace/events/block.h> 2655782138SLi Zefan 272db270a8SFrederic Weisbecker #include "trace_output.h" 282db270a8SFrederic Weisbecker 2955782138SLi Zefan #ifdef CONFIG_BLK_DEV_IO_TRACE 3055782138SLi Zefan 312db270a8SFrederic Weisbecker static unsigned int blktrace_seq __read_mostly = 1; 322db270a8SFrederic Weisbecker 332db270a8SFrederic Weisbecker static struct trace_array *blk_tr; 345006ea73SLi Zefan static bool blk_tracer_enabled __read_mostly; 352db270a8SFrederic Weisbecker 36a404d557SJan Kara static LIST_HEAD(running_trace_list); 37a404d557SJan Kara static __cacheline_aligned_in_smp DEFINE_SPINLOCK(running_trace_lock); 38a404d557SJan Kara 392db270a8SFrederic Weisbecker /* Select an alternative, minimalistic output than the original one */ 402db270a8SFrederic Weisbecker #define TRACE_BLK_OPT_CLASSIC 0x1 41ca1136c9SShaohua Li #define TRACE_BLK_OPT_CGROUP 0x2 4269fd5c39SShaohua Li #define TRACE_BLK_OPT_CGNAME 0x4 432db270a8SFrederic Weisbecker 442db270a8SFrederic Weisbecker static struct tracer_opt blk_tracer_opts[] = { 452db270a8SFrederic Weisbecker /* Default disable the minimalistic output */ 462db270a8SFrederic Weisbecker { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) }, 47ca1136c9SShaohua Li #ifdef CONFIG_BLK_CGROUP 48ca1136c9SShaohua Li { TRACER_OPT(blk_cgroup, TRACE_BLK_OPT_CGROUP) }, 4969fd5c39SShaohua Li { TRACER_OPT(blk_cgname, TRACE_BLK_OPT_CGNAME) }, 50ca1136c9SShaohua Li #endif 512db270a8SFrederic Weisbecker { } 522db270a8SFrederic Weisbecker }; 532db270a8SFrederic Weisbecker 542db270a8SFrederic Weisbecker static struct tracer_flags blk_tracer_flags = { 552db270a8SFrederic Weisbecker .val = 0, 562db270a8SFrederic Weisbecker .opts = blk_tracer_opts, 572db270a8SFrederic Weisbecker }; 582db270a8SFrederic Weisbecker 592db270a8SFrederic Weisbecker /* Global reference count of probes */ 60a6da0024SJens Axboe static DEFINE_MUTEX(blk_probe_mutex); 61a6da0024SJens Axboe static int blk_probes_ref; 622db270a8SFrederic Weisbecker 633c289ba7SLi Zefan static void blk_register_tracepoints(void); 642db270a8SFrederic Weisbecker static void blk_unregister_tracepoints(void); 652db270a8SFrederic Weisbecker 662db270a8SFrederic Weisbecker /* 672db270a8SFrederic Weisbecker * Send out a notify message. 682db270a8SFrederic Weisbecker */ 692db270a8SFrederic Weisbecker static void trace_note(struct blk_trace *bt, pid_t pid, int action, 7067c0496eSTejun Heo const void *data, size_t len, u64 cgid) 712db270a8SFrederic Weisbecker { 722db270a8SFrederic Weisbecker struct blk_io_trace *t; 7318cea459SLi Zefan struct ring_buffer_event *event = NULL; 7413292494SSteven Rostedt (VMware) struct trace_buffer *buffer = NULL; 7518cea459SLi Zefan int pc = 0; 7618cea459SLi Zefan int cpu = smp_processor_id(); 7718cea459SLi Zefan bool blk_tracer = blk_tracer_enabled; 7867c0496eSTejun Heo ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 7918cea459SLi Zefan 8018cea459SLi Zefan if (blk_tracer) { 811c5eb448SSteven Rostedt (VMware) buffer = blk_tr->array_buffer.buffer; 8218cea459SLi Zefan pc = preempt_count(); 83e77405adSSteven Rostedt event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 84ca1136c9SShaohua Li sizeof(*t) + len + cgid_len, 8518cea459SLi Zefan 0, pc); 8618cea459SLi Zefan if (!event) 8718cea459SLi Zefan return; 8818cea459SLi Zefan t = ring_buffer_event_data(event); 8918cea459SLi Zefan goto record_it; 9018cea459SLi Zefan } 912db270a8SFrederic Weisbecker 922db270a8SFrederic Weisbecker if (!bt->rchan) 932db270a8SFrederic Weisbecker return; 942db270a8SFrederic Weisbecker 95ca1136c9SShaohua Li t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len); 962db270a8SFrederic Weisbecker if (t) { 972db270a8SFrederic Weisbecker t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 982db270a8SFrederic Weisbecker t->time = ktime_to_ns(ktime_get()); 9918cea459SLi Zefan record_it: 1002db270a8SFrederic Weisbecker t->device = bt->dev; 101ca1136c9SShaohua Li t->action = action | (cgid ? __BLK_TN_CGROUP : 0); 1022db270a8SFrederic Weisbecker t->pid = pid; 1032db270a8SFrederic Weisbecker t->cpu = cpu; 104ca1136c9SShaohua Li t->pdu_len = len + cgid_len; 10567c0496eSTejun Heo if (cgid_len) 10667c0496eSTejun Heo memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 107ca1136c9SShaohua Li memcpy((void *) t + sizeof(*t) + cgid_len, data, len); 10818cea459SLi Zefan 10918cea459SLi Zefan if (blk_tracer) 110b7f0c959SSteven Rostedt (Red Hat) trace_buffer_unlock_commit(blk_tr, buffer, event, 0, pc); 1112db270a8SFrederic Weisbecker } 1122db270a8SFrederic Weisbecker } 1132db270a8SFrederic Weisbecker 1142db270a8SFrederic Weisbecker /* 1152db270a8SFrederic Weisbecker * Send out a notify for this process, if we haven't done so since a trace 1162db270a8SFrederic Weisbecker * started 1172db270a8SFrederic Weisbecker */ 118a404d557SJan Kara static void trace_note_tsk(struct task_struct *tsk) 1192db270a8SFrederic Weisbecker { 120a404d557SJan Kara unsigned long flags; 121a404d557SJan Kara struct blk_trace *bt; 122a404d557SJan Kara 1232db270a8SFrederic Weisbecker tsk->btrace_seq = blktrace_seq; 124a404d557SJan Kara spin_lock_irqsave(&running_trace_lock, flags); 125a404d557SJan Kara list_for_each_entry(bt, &running_trace_list, running_list) { 126a404d557SJan Kara trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, 12767c0496eSTejun Heo sizeof(tsk->comm), 0); 128a404d557SJan Kara } 129a404d557SJan Kara spin_unlock_irqrestore(&running_trace_lock, flags); 1302db270a8SFrederic Weisbecker } 1312db270a8SFrederic Weisbecker 1322db270a8SFrederic Weisbecker static void trace_note_time(struct blk_trace *bt) 1332db270a8SFrederic Weisbecker { 13459a37f8bSArnd Bergmann struct timespec64 now; 1352db270a8SFrederic Weisbecker unsigned long flags; 1362db270a8SFrederic Weisbecker u32 words[2]; 1372db270a8SFrederic Weisbecker 13859a37f8bSArnd Bergmann /* need to check user space to see if this breaks in y2038 or y2106 */ 13959a37f8bSArnd Bergmann ktime_get_real_ts64(&now); 14059a37f8bSArnd Bergmann words[0] = (u32)now.tv_sec; 1412db270a8SFrederic Weisbecker words[1] = now.tv_nsec; 1422db270a8SFrederic Weisbecker 1432db270a8SFrederic Weisbecker local_irq_save(flags); 14467c0496eSTejun Heo trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0); 1452db270a8SFrederic Weisbecker local_irq_restore(flags); 1462db270a8SFrederic Weisbecker } 1472db270a8SFrederic Weisbecker 14835fe6d76SShaohua Li void __trace_note_message(struct blk_trace *bt, struct blkcg *blkcg, 14935fe6d76SShaohua Li const char *fmt, ...) 1502db270a8SFrederic Weisbecker { 1512db270a8SFrederic Weisbecker int n; 1522db270a8SFrederic Weisbecker va_list args; 1532db270a8SFrederic Weisbecker unsigned long flags; 1542db270a8SFrederic Weisbecker char *buf; 1552db270a8SFrederic Weisbecker 15618cea459SLi Zefan if (unlikely(bt->trace_state != Blktrace_running && 15718cea459SLi Zefan !blk_tracer_enabled)) 1582db270a8SFrederic Weisbecker return; 1592db270a8SFrederic Weisbecker 160490da40dSTao Ma /* 161490da40dSTao Ma * If the BLK_TC_NOTIFY action mask isn't set, don't send any note 162490da40dSTao Ma * message to the trace. 163490da40dSTao Ma */ 164490da40dSTao Ma if (!(bt->act_mask & BLK_TC_NOTIFY)) 165490da40dSTao Ma return; 166490da40dSTao Ma 1672db270a8SFrederic Weisbecker local_irq_save(flags); 168d8a0349cSShan Wei buf = this_cpu_ptr(bt->msg_data); 1692db270a8SFrederic Weisbecker va_start(args, fmt); 1702db270a8SFrederic Weisbecker n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args); 1712db270a8SFrederic Weisbecker va_end(args); 1722db270a8SFrederic Weisbecker 17335fe6d76SShaohua Li if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 17435fe6d76SShaohua Li blkcg = NULL; 17535fe6d76SShaohua Li #ifdef CONFIG_BLK_CGROUP 176870c153cSJan Kara trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, 17774321038STejun Heo blkcg ? cgroup_id(blkcg->css.cgroup) : 1); 17835fe6d76SShaohua Li #else 179870c153cSJan Kara trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, 0); 18035fe6d76SShaohua Li #endif 1812db270a8SFrederic Weisbecker local_irq_restore(flags); 1822db270a8SFrederic Weisbecker } 1832db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(__trace_note_message); 1842db270a8SFrederic Weisbecker 1852db270a8SFrederic Weisbecker static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector, 1862db270a8SFrederic Weisbecker pid_t pid) 1872db270a8SFrederic Weisbecker { 1882db270a8SFrederic Weisbecker if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0) 1892db270a8SFrederic Weisbecker return 1; 190d0deef5bSShawn Du if (sector && (sector < bt->start_lba || sector > bt->end_lba)) 1912db270a8SFrederic Weisbecker return 1; 1922db270a8SFrederic Weisbecker if (bt->pid && pid != bt->pid) 1932db270a8SFrederic Weisbecker return 1; 1942db270a8SFrederic Weisbecker 1952db270a8SFrederic Weisbecker return 0; 1962db270a8SFrederic Weisbecker } 1972db270a8SFrederic Weisbecker 1982db270a8SFrederic Weisbecker /* 1992db270a8SFrederic Weisbecker * Data direction bit lookup 2002db270a8SFrederic Weisbecker */ 201e4955c99SLi Zefan static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), 2022db270a8SFrederic Weisbecker BLK_TC_ACT(BLK_TC_WRITE) }; 2032db270a8SFrederic Weisbecker 2047b6d91daSChristoph Hellwig #define BLK_TC_RAHEAD BLK_TC_AHEAD 20528a8f0d3SMike Christie #define BLK_TC_PREFLUSH BLK_TC_FLUSH 2067b6d91daSChristoph Hellwig 2072db270a8SFrederic Weisbecker /* The ilog2() calls fall out because they're constant */ 2087b6d91daSChristoph Hellwig #define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \ 2097b6d91daSChristoph Hellwig (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name)) 2102db270a8SFrederic Weisbecker 2112db270a8SFrederic Weisbecker /* 2122db270a8SFrederic Weisbecker * The worker for the various blk_add_trace*() types. Fills out a 2132db270a8SFrederic Weisbecker * blk_io_trace structure and places it in a per-cpu subbuffer. 2142db270a8SFrederic Weisbecker */ 2152db270a8SFrederic Weisbecker static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, 2161b9a9ab7SMike Christie int op, int op_flags, u32 what, int error, int pdu_len, 21767c0496eSTejun Heo void *pdu_data, u64 cgid) 2182db270a8SFrederic Weisbecker { 2192db270a8SFrederic Weisbecker struct task_struct *tsk = current; 2202db270a8SFrederic Weisbecker struct ring_buffer_event *event = NULL; 22113292494SSteven Rostedt (VMware) struct trace_buffer *buffer = NULL; 2222db270a8SFrederic Weisbecker struct blk_io_trace *t; 2232db270a8SFrederic Weisbecker unsigned long flags = 0; 2242db270a8SFrederic Weisbecker unsigned long *sequence; 2252db270a8SFrederic Weisbecker pid_t pid; 2262db270a8SFrederic Weisbecker int cpu, pc = 0; 22718cea459SLi Zefan bool blk_tracer = blk_tracer_enabled; 22867c0496eSTejun Heo ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 2292db270a8SFrederic Weisbecker 23018cea459SLi Zefan if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer)) 2312db270a8SFrederic Weisbecker return; 2322db270a8SFrederic Weisbecker 2331b9a9ab7SMike Christie what |= ddir_act[op_is_write(op) ? WRITE : READ]; 2341b9a9ab7SMike Christie what |= MASK_TC_BIT(op_flags, SYNC); 2351b9a9ab7SMike Christie what |= MASK_TC_BIT(op_flags, RAHEAD); 2361b9a9ab7SMike Christie what |= MASK_TC_BIT(op_flags, META); 23728a8f0d3SMike Christie what |= MASK_TC_BIT(op_flags, PREFLUSH); 2381b9a9ab7SMike Christie what |= MASK_TC_BIT(op_flags, FUA); 2397afafc8aSAdrian Hunter if (op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE) 2401b9a9ab7SMike Christie what |= BLK_TC_ACT(BLK_TC_DISCARD); 2413a5e02ceSMike Christie if (op == REQ_OP_FLUSH) 2423a5e02ceSMike Christie what |= BLK_TC_ACT(BLK_TC_FLUSH); 243ca1136c9SShaohua Li if (cgid) 244ca1136c9SShaohua Li what |= __BLK_TA_CGROUP; 2452db270a8SFrederic Weisbecker 2462db270a8SFrederic Weisbecker pid = tsk->pid; 247d0deef5bSShawn Du if (act_log_check(bt, what, sector, pid)) 2482db270a8SFrederic Weisbecker return; 2492db270a8SFrederic Weisbecker cpu = raw_smp_processor_id(); 2502db270a8SFrederic Weisbecker 25118cea459SLi Zefan if (blk_tracer) { 2522db270a8SFrederic Weisbecker tracing_record_cmdline(current); 2532db270a8SFrederic Weisbecker 2541c5eb448SSteven Rostedt (VMware) buffer = blk_tr->array_buffer.buffer; 2552db270a8SFrederic Weisbecker pc = preempt_count(); 256e77405adSSteven Rostedt event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 257ca1136c9SShaohua Li sizeof(*t) + pdu_len + cgid_len, 2582db270a8SFrederic Weisbecker 0, pc); 2592db270a8SFrederic Weisbecker if (!event) 2602db270a8SFrederic Weisbecker return; 2612db270a8SFrederic Weisbecker t = ring_buffer_event_data(event); 2622db270a8SFrederic Weisbecker goto record_it; 2632db270a8SFrederic Weisbecker } 2642db270a8SFrederic Weisbecker 265a404d557SJan Kara if (unlikely(tsk->btrace_seq != blktrace_seq)) 266a404d557SJan Kara trace_note_tsk(tsk); 267a404d557SJan Kara 2682db270a8SFrederic Weisbecker /* 2692db270a8SFrederic Weisbecker * A word about the locking here - we disable interrupts to reserve 2702db270a8SFrederic Weisbecker * some space in the relay per-cpu buffer, to prevent an irq 2712db270a8SFrederic Weisbecker * from coming in and stepping on our toes. 2722db270a8SFrederic Weisbecker */ 2732db270a8SFrederic Weisbecker local_irq_save(flags); 274ca1136c9SShaohua Li t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len); 2752db270a8SFrederic Weisbecker if (t) { 2762db270a8SFrederic Weisbecker sequence = per_cpu_ptr(bt->sequence, cpu); 2772db270a8SFrederic Weisbecker 2782db270a8SFrederic Weisbecker t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 2792db270a8SFrederic Weisbecker t->sequence = ++(*sequence); 2802db270a8SFrederic Weisbecker t->time = ktime_to_ns(ktime_get()); 2812db270a8SFrederic Weisbecker record_it: 2822db270a8SFrederic Weisbecker /* 2832db270a8SFrederic Weisbecker * These two are not needed in ftrace as they are in the 2842db270a8SFrederic Weisbecker * generic trace_entry, filled by tracing_generic_entry_update, 2852db270a8SFrederic Weisbecker * but for the trace_event->bin() synthesizer benefit we do it 2862db270a8SFrederic Weisbecker * here too. 2872db270a8SFrederic Weisbecker */ 2882db270a8SFrederic Weisbecker t->cpu = cpu; 2892db270a8SFrederic Weisbecker t->pid = pid; 2902db270a8SFrederic Weisbecker 2912db270a8SFrederic Weisbecker t->sector = sector; 2922db270a8SFrederic Weisbecker t->bytes = bytes; 2932db270a8SFrederic Weisbecker t->action = what; 2942db270a8SFrederic Weisbecker t->device = bt->dev; 2952db270a8SFrederic Weisbecker t->error = error; 296ca1136c9SShaohua Li t->pdu_len = pdu_len + cgid_len; 2972db270a8SFrederic Weisbecker 298ca1136c9SShaohua Li if (cgid_len) 29967c0496eSTejun Heo memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 3002db270a8SFrederic Weisbecker if (pdu_len) 301ca1136c9SShaohua Li memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len); 3022db270a8SFrederic Weisbecker 30318cea459SLi Zefan if (blk_tracer) { 304b7f0c959SSteven Rostedt (Red Hat) trace_buffer_unlock_commit(blk_tr, buffer, event, 0, pc); 3052db270a8SFrederic Weisbecker return; 3062db270a8SFrederic Weisbecker } 3072db270a8SFrederic Weisbecker } 3082db270a8SFrederic Weisbecker 3092db270a8SFrederic Weisbecker local_irq_restore(flags); 3102db270a8SFrederic Weisbecker } 3112db270a8SFrederic Weisbecker 312ad5dd549SLi Zefan static void blk_trace_free(struct blk_trace *bt) 3132db270a8SFrederic Weisbecker { 3142db270a8SFrederic Weisbecker debugfs_remove(bt->msg_file); 3152db270a8SFrederic Weisbecker debugfs_remove(bt->dropped_file); 3162db270a8SFrederic Weisbecker relay_close(bt->rchan); 31739cbb602SAlan D. Brunelle debugfs_remove(bt->dir); 3182db270a8SFrederic Weisbecker free_percpu(bt->sequence); 3192db270a8SFrederic Weisbecker free_percpu(bt->msg_data); 3202db270a8SFrederic Weisbecker kfree(bt); 321ad5dd549SLi Zefan } 322ad5dd549SLi Zefan 323a6da0024SJens Axboe static void get_probe_ref(void) 324a6da0024SJens Axboe { 325a6da0024SJens Axboe mutex_lock(&blk_probe_mutex); 326a6da0024SJens Axboe if (++blk_probes_ref == 1) 327a6da0024SJens Axboe blk_register_tracepoints(); 328a6da0024SJens Axboe mutex_unlock(&blk_probe_mutex); 329a6da0024SJens Axboe } 330a6da0024SJens Axboe 331a6da0024SJens Axboe static void put_probe_ref(void) 332a6da0024SJens Axboe { 333a6da0024SJens Axboe mutex_lock(&blk_probe_mutex); 334a6da0024SJens Axboe if (!--blk_probes_ref) 335a6da0024SJens Axboe blk_unregister_tracepoints(); 336a6da0024SJens Axboe mutex_unlock(&blk_probe_mutex); 337a6da0024SJens Axboe } 338a6da0024SJens Axboe 339ad5dd549SLi Zefan static void blk_trace_cleanup(struct blk_trace *bt) 340ad5dd549SLi Zefan { 341c780e86dSJan Kara synchronize_rcu(); 342ad5dd549SLi Zefan blk_trace_free(bt); 343a6da0024SJens Axboe put_probe_ref(); 3442db270a8SFrederic Weisbecker } 3452db270a8SFrederic Weisbecker 3461f2cac10SJens Axboe static int __blk_trace_remove(struct request_queue *q) 3472db270a8SFrederic Weisbecker { 3482db270a8SFrederic Weisbecker struct blk_trace *bt; 3492db270a8SFrederic Weisbecker 350c3dbe541SJan Kara bt = rcu_replace_pointer(q->blk_trace, NULL, 35185e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 3522db270a8SFrederic Weisbecker if (!bt) 3532db270a8SFrederic Weisbecker return -EINVAL; 3542db270a8SFrederic Weisbecker 35555547204SLi Zefan if (bt->trace_state != Blktrace_running) 3562db270a8SFrederic Weisbecker blk_trace_cleanup(bt); 3572db270a8SFrederic Weisbecker 3582db270a8SFrederic Weisbecker return 0; 3592db270a8SFrederic Weisbecker } 3601f2cac10SJens Axboe 3611f2cac10SJens Axboe int blk_trace_remove(struct request_queue *q) 3621f2cac10SJens Axboe { 3631f2cac10SJens Axboe int ret; 3641f2cac10SJens Axboe 36585e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 3661f2cac10SJens Axboe ret = __blk_trace_remove(q); 36785e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 3681f2cac10SJens Axboe 3691f2cac10SJens Axboe return ret; 3701f2cac10SJens Axboe } 3712db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_remove); 3722db270a8SFrederic Weisbecker 3732db270a8SFrederic Weisbecker static ssize_t blk_dropped_read(struct file *filp, char __user *buffer, 3742db270a8SFrederic Weisbecker size_t count, loff_t *ppos) 3752db270a8SFrederic Weisbecker { 3762db270a8SFrederic Weisbecker struct blk_trace *bt = filp->private_data; 3772db270a8SFrederic Weisbecker char buf[16]; 3782db270a8SFrederic Weisbecker 3792db270a8SFrederic Weisbecker snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped)); 3802db270a8SFrederic Weisbecker 3812db270a8SFrederic Weisbecker return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf)); 3822db270a8SFrederic Weisbecker } 3832db270a8SFrederic Weisbecker 3842db270a8SFrederic Weisbecker static const struct file_operations blk_dropped_fops = { 3852db270a8SFrederic Weisbecker .owner = THIS_MODULE, 386234e3405SStephen Boyd .open = simple_open, 3872db270a8SFrederic Weisbecker .read = blk_dropped_read, 3886038f373SArnd Bergmann .llseek = default_llseek, 3892db270a8SFrederic Weisbecker }; 3902db270a8SFrederic Weisbecker 3912db270a8SFrederic Weisbecker static ssize_t blk_msg_write(struct file *filp, const char __user *buffer, 3922db270a8SFrederic Weisbecker size_t count, loff_t *ppos) 3932db270a8SFrederic Weisbecker { 3942db270a8SFrederic Weisbecker char *msg; 3952db270a8SFrederic Weisbecker struct blk_trace *bt; 3962db270a8SFrederic Weisbecker 3977635b03aSLi Zefan if (count >= BLK_TN_MAX_MSG) 3982db270a8SFrederic Weisbecker return -EINVAL; 3992db270a8SFrederic Weisbecker 40016e5c1fcSAl Viro msg = memdup_user_nul(buffer, count); 40116e5c1fcSAl Viro if (IS_ERR(msg)) 40216e5c1fcSAl Viro return PTR_ERR(msg); 4032db270a8SFrederic Weisbecker 4042db270a8SFrederic Weisbecker bt = filp->private_data; 40535fe6d76SShaohua Li __trace_note_message(bt, NULL, "%s", msg); 4062db270a8SFrederic Weisbecker kfree(msg); 4072db270a8SFrederic Weisbecker 4082db270a8SFrederic Weisbecker return count; 4092db270a8SFrederic Weisbecker } 4102db270a8SFrederic Weisbecker 4112db270a8SFrederic Weisbecker static const struct file_operations blk_msg_fops = { 4122db270a8SFrederic Weisbecker .owner = THIS_MODULE, 413234e3405SStephen Boyd .open = simple_open, 4142db270a8SFrederic Weisbecker .write = blk_msg_write, 4156038f373SArnd Bergmann .llseek = noop_llseek, 4162db270a8SFrederic Weisbecker }; 4172db270a8SFrederic Weisbecker 4182db270a8SFrederic Weisbecker /* 4192db270a8SFrederic Weisbecker * Keep track of how many times we encountered a full subbuffer, to aid 4202db270a8SFrederic Weisbecker * the user space app in telling how many lost events there were. 4212db270a8SFrederic Weisbecker */ 4222db270a8SFrederic Weisbecker static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf, 4232db270a8SFrederic Weisbecker void *prev_subbuf, size_t prev_padding) 4242db270a8SFrederic Weisbecker { 4252db270a8SFrederic Weisbecker struct blk_trace *bt; 4262db270a8SFrederic Weisbecker 4272db270a8SFrederic Weisbecker if (!relay_buf_full(buf)) 4282db270a8SFrederic Weisbecker return 1; 4292db270a8SFrederic Weisbecker 4302db270a8SFrederic Weisbecker bt = buf->chan->private_data; 4312db270a8SFrederic Weisbecker atomic_inc(&bt->dropped); 4322db270a8SFrederic Weisbecker return 0; 4332db270a8SFrederic Weisbecker } 4342db270a8SFrederic Weisbecker 4352db270a8SFrederic Weisbecker static int blk_remove_buf_file_callback(struct dentry *dentry) 4362db270a8SFrederic Weisbecker { 4372db270a8SFrederic Weisbecker debugfs_remove(dentry); 4382db270a8SFrederic Weisbecker 4392db270a8SFrederic Weisbecker return 0; 4402db270a8SFrederic Weisbecker } 4412db270a8SFrederic Weisbecker 4422db270a8SFrederic Weisbecker static struct dentry *blk_create_buf_file_callback(const char *filename, 4432db270a8SFrederic Weisbecker struct dentry *parent, 444f4ae40a6SAl Viro umode_t mode, 4452db270a8SFrederic Weisbecker struct rchan_buf *buf, 4462db270a8SFrederic Weisbecker int *is_global) 4472db270a8SFrederic Weisbecker { 4482db270a8SFrederic Weisbecker return debugfs_create_file(filename, mode, parent, buf, 4492db270a8SFrederic Weisbecker &relay_file_operations); 4502db270a8SFrederic Weisbecker } 4512db270a8SFrederic Weisbecker 4522db270a8SFrederic Weisbecker static struct rchan_callbacks blk_relay_callbacks = { 4532db270a8SFrederic Weisbecker .subbuf_start = blk_subbuf_start_callback, 4542db270a8SFrederic Weisbecker .create_buf_file = blk_create_buf_file_callback, 4552db270a8SFrederic Weisbecker .remove_buf_file = blk_remove_buf_file_callback, 4562db270a8SFrederic Weisbecker }; 4572db270a8SFrederic Weisbecker 4589908c309SLi Zefan static void blk_trace_setup_lba(struct blk_trace *bt, 4599908c309SLi Zefan struct block_device *bdev) 4609908c309SLi Zefan { 46129ff57c6SChristoph Hellwig if (bdev) { 46229ff57c6SChristoph Hellwig bt->start_lba = bdev->bd_start_sect; 46329ff57c6SChristoph Hellwig bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev); 4649908c309SLi Zefan } else { 4659908c309SLi Zefan bt->start_lba = 0; 4669908c309SLi Zefan bt->end_lba = -1ULL; 4679908c309SLi Zefan } 4689908c309SLi Zefan } 4699908c309SLi Zefan 4702db270a8SFrederic Weisbecker /* 4712db270a8SFrederic Weisbecker * Setup everything required to start tracing 4722db270a8SFrederic Weisbecker */ 473a428d314SOmar Sandoval static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 474d0deef5bSShawn Du struct block_device *bdev, 4752db270a8SFrederic Weisbecker struct blk_user_trace_setup *buts) 4762db270a8SFrederic Weisbecker { 477cdea01b2SDavidlohr Bueso struct blk_trace *bt = NULL; 4782db270a8SFrederic Weisbecker struct dentry *dir = NULL; 479ff14417cSRasmus Villemoes int ret; 4802db270a8SFrederic Weisbecker 48185e0cbbbSLuis Chamberlain lockdep_assert_held(&q->debugfs_mutex); 482a67549c8SLuis Chamberlain 4832db270a8SFrederic Weisbecker if (!buts->buf_size || !buts->buf_nr) 4842db270a8SFrederic Weisbecker return -EINVAL; 4852db270a8SFrederic Weisbecker 4862db270a8SFrederic Weisbecker strncpy(buts->name, name, BLKTRACE_BDEV_SIZE); 4872db270a8SFrederic Weisbecker buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0'; 4882db270a8SFrederic Weisbecker 4892db270a8SFrederic Weisbecker /* 4902db270a8SFrederic Weisbecker * some device names have larger paths - convert the slashes 4912db270a8SFrederic Weisbecker * to underscores for this to work as expected 4922db270a8SFrederic Weisbecker */ 493ff14417cSRasmus Villemoes strreplace(buts->name, '/', '_'); 4942db270a8SFrederic Weisbecker 4951b0b2836SLuis Chamberlain /* 4961b0b2836SLuis Chamberlain * bdev can be NULL, as with scsi-generic, this is a helpful as 4971b0b2836SLuis Chamberlain * we can be. 4981b0b2836SLuis Chamberlain */ 499c3dbe541SJan Kara if (rcu_dereference_protected(q->blk_trace, 50085e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex))) { 5011b0b2836SLuis Chamberlain pr_warn("Concurrent blktraces are not allowed on %s\n", 5021b0b2836SLuis Chamberlain buts->name); 5031b0b2836SLuis Chamberlain return -EBUSY; 5041b0b2836SLuis Chamberlain } 5051b0b2836SLuis Chamberlain 5062db270a8SFrederic Weisbecker bt = kzalloc(sizeof(*bt), GFP_KERNEL); 5072db270a8SFrederic Weisbecker if (!bt) 508ad5dd549SLi Zefan return -ENOMEM; 5092db270a8SFrederic Weisbecker 510ad5dd549SLi Zefan ret = -ENOMEM; 5112db270a8SFrederic Weisbecker bt->sequence = alloc_percpu(unsigned long); 5122db270a8SFrederic Weisbecker if (!bt->sequence) 5132db270a8SFrederic Weisbecker goto err; 5142db270a8SFrederic Weisbecker 515f0ef0398SIngo Molnar bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 5162db270a8SFrederic Weisbecker if (!bt->msg_data) 5172db270a8SFrederic Weisbecker goto err; 5182db270a8SFrederic Weisbecker 519bad8e64fSLuis Chamberlain /* 52085e0cbbbSLuis Chamberlain * When tracing the whole disk reuse the existing debugfs directory 52185e0cbbbSLuis Chamberlain * created by the block layer on init. For partitions block devices, 522bad8e64fSLuis Chamberlain * and scsi-generic block devices we create a temporary new debugfs 523bad8e64fSLuis Chamberlain * directory that will be removed once the trace ends. 524bad8e64fSLuis Chamberlain */ 525fa01b1e9SChristoph Hellwig if (bdev && !bdev_is_partition(bdev)) 526bad8e64fSLuis Chamberlain dir = q->debugfs_dir; 527bad8e64fSLuis Chamberlain else 5286ac93117SOmar Sandoval bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root); 5292db270a8SFrederic Weisbecker 530b431ef83SLuis Chamberlain /* 531b431ef83SLuis Chamberlain * As blktrace relies on debugfs for its interface the debugfs directory 532b431ef83SLuis Chamberlain * is required, contrary to the usual mantra of not checking for debugfs 533b431ef83SLuis Chamberlain * files or directories. 534b431ef83SLuis Chamberlain */ 535b431ef83SLuis Chamberlain if (IS_ERR_OR_NULL(dir)) { 536b431ef83SLuis Chamberlain pr_warn("debugfs_dir not present for %s so skipping\n", 537b431ef83SLuis Chamberlain buts->name); 538b431ef83SLuis Chamberlain ret = -ENOENT; 539b431ef83SLuis Chamberlain goto err; 540b431ef83SLuis Chamberlain } 541b431ef83SLuis Chamberlain 5422db270a8SFrederic Weisbecker bt->dev = dev; 5432db270a8SFrederic Weisbecker atomic_set(&bt->dropped, 0); 544a404d557SJan Kara INIT_LIST_HEAD(&bt->running_list); 5452db270a8SFrederic Weisbecker 5462db270a8SFrederic Weisbecker ret = -EIO; 5472db270a8SFrederic Weisbecker bt->dropped_file = debugfs_create_file("dropped", 0444, dir, bt, 5482db270a8SFrederic Weisbecker &blk_dropped_fops); 5492db270a8SFrederic Weisbecker 5502db270a8SFrederic Weisbecker bt->msg_file = debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops); 5512db270a8SFrederic Weisbecker 5522db270a8SFrederic Weisbecker bt->rchan = relay_open("trace", dir, buts->buf_size, 5532db270a8SFrederic Weisbecker buts->buf_nr, &blk_relay_callbacks, bt); 5542db270a8SFrederic Weisbecker if (!bt->rchan) 5552db270a8SFrederic Weisbecker goto err; 5562db270a8SFrederic Weisbecker 5572db270a8SFrederic Weisbecker bt->act_mask = buts->act_mask; 5582db270a8SFrederic Weisbecker if (!bt->act_mask) 5592db270a8SFrederic Weisbecker bt->act_mask = (u16) -1; 5602db270a8SFrederic Weisbecker 5619908c309SLi Zefan blk_trace_setup_lba(bt, bdev); 5622db270a8SFrederic Weisbecker 563d0deef5bSShawn Du /* overwrite with user settings */ 564d0deef5bSShawn Du if (buts->start_lba) 565d0deef5bSShawn Du bt->start_lba = buts->start_lba; 566d0deef5bSShawn Du if (buts->end_lba) 567d0deef5bSShawn Du bt->end_lba = buts->end_lba; 568d0deef5bSShawn Du 5692db270a8SFrederic Weisbecker bt->pid = buts->pid; 5702db270a8SFrederic Weisbecker bt->trace_state = Blktrace_setup; 5712db270a8SFrederic Weisbecker 572c3dbe541SJan Kara rcu_assign_pointer(q->blk_trace, bt); 573a6da0024SJens Axboe get_probe_ref(); 574cbe28296SLi Zefan 5756ac93117SOmar Sandoval ret = 0; 5762db270a8SFrederic Weisbecker err: 5776ac93117SOmar Sandoval if (ret) 578ad5dd549SLi Zefan blk_trace_free(bt); 5792db270a8SFrederic Weisbecker return ret; 5802db270a8SFrederic Weisbecker } 5812db270a8SFrederic Weisbecker 5821f2cac10SJens Axboe static int __blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 5831f2cac10SJens Axboe struct block_device *bdev, char __user *arg) 5842db270a8SFrederic Weisbecker { 5852db270a8SFrederic Weisbecker struct blk_user_trace_setup buts; 5862db270a8SFrederic Weisbecker int ret; 5872db270a8SFrederic Weisbecker 5882db270a8SFrederic Weisbecker ret = copy_from_user(&buts, arg, sizeof(buts)); 5892db270a8SFrederic Weisbecker if (ret) 5902db270a8SFrederic Weisbecker return -EFAULT; 5912db270a8SFrederic Weisbecker 592d0deef5bSShawn Du ret = do_blk_trace_setup(q, name, dev, bdev, &buts); 5932db270a8SFrederic Weisbecker if (ret) 5942db270a8SFrederic Weisbecker return ret; 5952db270a8SFrederic Weisbecker 5969a8c28c8SDmitry Monakhov if (copy_to_user(arg, &buts, sizeof(buts))) { 5972967acbbSJens Axboe __blk_trace_remove(q); 5982db270a8SFrederic Weisbecker return -EFAULT; 5999a8c28c8SDmitry Monakhov } 6002db270a8SFrederic Weisbecker return 0; 6012db270a8SFrederic Weisbecker } 6021f2cac10SJens Axboe 6031f2cac10SJens Axboe int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 6041f2cac10SJens Axboe struct block_device *bdev, 6051f2cac10SJens Axboe char __user *arg) 6061f2cac10SJens Axboe { 6071f2cac10SJens Axboe int ret; 6081f2cac10SJens Axboe 60985e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 6101f2cac10SJens Axboe ret = __blk_trace_setup(q, name, dev, bdev, arg); 61185e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 6121f2cac10SJens Axboe 6131f2cac10SJens Axboe return ret; 6141f2cac10SJens Axboe } 6152db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_setup); 6162db270a8SFrederic Weisbecker 61762c2a7d9SArnd Bergmann #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 61862c2a7d9SArnd Bergmann static int compat_blk_trace_setup(struct request_queue *q, char *name, 61962c2a7d9SArnd Bergmann dev_t dev, struct block_device *bdev, 62062c2a7d9SArnd Bergmann char __user *arg) 62162c2a7d9SArnd Bergmann { 62262c2a7d9SArnd Bergmann struct blk_user_trace_setup buts; 62362c2a7d9SArnd Bergmann struct compat_blk_user_trace_setup cbuts; 62462c2a7d9SArnd Bergmann int ret; 62562c2a7d9SArnd Bergmann 62662c2a7d9SArnd Bergmann if (copy_from_user(&cbuts, arg, sizeof(cbuts))) 62762c2a7d9SArnd Bergmann return -EFAULT; 62862c2a7d9SArnd Bergmann 62962c2a7d9SArnd Bergmann buts = (struct blk_user_trace_setup) { 63062c2a7d9SArnd Bergmann .act_mask = cbuts.act_mask, 63162c2a7d9SArnd Bergmann .buf_size = cbuts.buf_size, 63262c2a7d9SArnd Bergmann .buf_nr = cbuts.buf_nr, 63362c2a7d9SArnd Bergmann .start_lba = cbuts.start_lba, 63462c2a7d9SArnd Bergmann .end_lba = cbuts.end_lba, 63562c2a7d9SArnd Bergmann .pid = cbuts.pid, 63662c2a7d9SArnd Bergmann }; 63762c2a7d9SArnd Bergmann 63862c2a7d9SArnd Bergmann ret = do_blk_trace_setup(q, name, dev, bdev, &buts); 63962c2a7d9SArnd Bergmann if (ret) 64062c2a7d9SArnd Bergmann return ret; 64162c2a7d9SArnd Bergmann 642f8c5e944SChen Gang if (copy_to_user(arg, &buts.name, ARRAY_SIZE(buts.name))) { 6432967acbbSJens Axboe __blk_trace_remove(q); 64462c2a7d9SArnd Bergmann return -EFAULT; 64562c2a7d9SArnd Bergmann } 64662c2a7d9SArnd Bergmann 64762c2a7d9SArnd Bergmann return 0; 64862c2a7d9SArnd Bergmann } 64962c2a7d9SArnd Bergmann #endif 65062c2a7d9SArnd Bergmann 6511f2cac10SJens Axboe static int __blk_trace_startstop(struct request_queue *q, int start) 6522db270a8SFrederic Weisbecker { 6532db270a8SFrederic Weisbecker int ret; 654c780e86dSJan Kara struct blk_trace *bt; 6552db270a8SFrederic Weisbecker 656c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 65785e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 6582db270a8SFrederic Weisbecker if (bt == NULL) 6592db270a8SFrederic Weisbecker return -EINVAL; 6602db270a8SFrederic Weisbecker 6612db270a8SFrederic Weisbecker /* 6622db270a8SFrederic Weisbecker * For starting a trace, we can transition from a setup or stopped 6632db270a8SFrederic Weisbecker * trace. For stopping a trace, the state must be running 6642db270a8SFrederic Weisbecker */ 6652db270a8SFrederic Weisbecker ret = -EINVAL; 6662db270a8SFrederic Weisbecker if (start) { 6672db270a8SFrederic Weisbecker if (bt->trace_state == Blktrace_setup || 6682db270a8SFrederic Weisbecker bt->trace_state == Blktrace_stopped) { 6692db270a8SFrederic Weisbecker blktrace_seq++; 6702db270a8SFrederic Weisbecker smp_mb(); 6712db270a8SFrederic Weisbecker bt->trace_state = Blktrace_running; 672a404d557SJan Kara spin_lock_irq(&running_trace_lock); 673a404d557SJan Kara list_add(&bt->running_list, &running_trace_list); 674a404d557SJan Kara spin_unlock_irq(&running_trace_lock); 6752db270a8SFrederic Weisbecker 6762db270a8SFrederic Weisbecker trace_note_time(bt); 6772db270a8SFrederic Weisbecker ret = 0; 6782db270a8SFrederic Weisbecker } 6792db270a8SFrederic Weisbecker } else { 6802db270a8SFrederic Weisbecker if (bt->trace_state == Blktrace_running) { 6812db270a8SFrederic Weisbecker bt->trace_state = Blktrace_stopped; 682a404d557SJan Kara spin_lock_irq(&running_trace_lock); 683a404d557SJan Kara list_del_init(&bt->running_list); 684a404d557SJan Kara spin_unlock_irq(&running_trace_lock); 6852db270a8SFrederic Weisbecker relay_flush(bt->rchan); 6862db270a8SFrederic Weisbecker ret = 0; 6872db270a8SFrederic Weisbecker } 6882db270a8SFrederic Weisbecker } 6892db270a8SFrederic Weisbecker 6902db270a8SFrederic Weisbecker return ret; 6912db270a8SFrederic Weisbecker } 6921f2cac10SJens Axboe 6931f2cac10SJens Axboe int blk_trace_startstop(struct request_queue *q, int start) 6941f2cac10SJens Axboe { 6951f2cac10SJens Axboe int ret; 6961f2cac10SJens Axboe 69785e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 6981f2cac10SJens Axboe ret = __blk_trace_startstop(q, start); 69985e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 7001f2cac10SJens Axboe 7011f2cac10SJens Axboe return ret; 7021f2cac10SJens Axboe } 7032db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_startstop); 7042db270a8SFrederic Weisbecker 7055acb3cc2SWaiman Long /* 7065acb3cc2SWaiman Long * When reading or writing the blktrace sysfs files, the references to the 7075acb3cc2SWaiman Long * opened sysfs or device files should prevent the underlying block device 7085acb3cc2SWaiman Long * from being removed. So no further delete protection is really needed. 7095acb3cc2SWaiman Long */ 7105acb3cc2SWaiman Long 7112db270a8SFrederic Weisbecker /** 7122db270a8SFrederic Weisbecker * blk_trace_ioctl: - handle the ioctls associated with tracing 7132db270a8SFrederic Weisbecker * @bdev: the block device 7142db270a8SFrederic Weisbecker * @cmd: the ioctl cmd 7152db270a8SFrederic Weisbecker * @arg: the argument data, if any 7162db270a8SFrederic Weisbecker * 7172db270a8SFrederic Weisbecker **/ 7182db270a8SFrederic Weisbecker int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) 7192db270a8SFrederic Weisbecker { 7202db270a8SFrederic Weisbecker struct request_queue *q; 7212db270a8SFrederic Weisbecker int ret, start = 0; 7222db270a8SFrederic Weisbecker char b[BDEVNAME_SIZE]; 7232db270a8SFrederic Weisbecker 7242db270a8SFrederic Weisbecker q = bdev_get_queue(bdev); 7252db270a8SFrederic Weisbecker if (!q) 7262db270a8SFrederic Weisbecker return -ENXIO; 7272db270a8SFrederic Weisbecker 72885e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 7292db270a8SFrederic Weisbecker 7302db270a8SFrederic Weisbecker switch (cmd) { 7312db270a8SFrederic Weisbecker case BLKTRACESETUP: 7322db270a8SFrederic Weisbecker bdevname(bdev, b); 7331f2cac10SJens Axboe ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 7342db270a8SFrederic Weisbecker break; 73562c2a7d9SArnd Bergmann #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 73662c2a7d9SArnd Bergmann case BLKTRACESETUP32: 73762c2a7d9SArnd Bergmann bdevname(bdev, b); 73862c2a7d9SArnd Bergmann ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 73962c2a7d9SArnd Bergmann break; 74062c2a7d9SArnd Bergmann #endif 7412db270a8SFrederic Weisbecker case BLKTRACESTART: 7422db270a8SFrederic Weisbecker start = 1; 743df561f66SGustavo A. R. Silva fallthrough; 7442db270a8SFrederic Weisbecker case BLKTRACESTOP: 7451f2cac10SJens Axboe ret = __blk_trace_startstop(q, start); 7462db270a8SFrederic Weisbecker break; 7472db270a8SFrederic Weisbecker case BLKTRACETEARDOWN: 7481f2cac10SJens Axboe ret = __blk_trace_remove(q); 7492db270a8SFrederic Weisbecker break; 7502db270a8SFrederic Weisbecker default: 7512db270a8SFrederic Weisbecker ret = -ENOTTY; 7522db270a8SFrederic Weisbecker break; 7532db270a8SFrederic Weisbecker } 7542db270a8SFrederic Weisbecker 75585e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 7562db270a8SFrederic Weisbecker return ret; 7572db270a8SFrederic Weisbecker } 7582db270a8SFrederic Weisbecker 7592db270a8SFrederic Weisbecker /** 7602db270a8SFrederic Weisbecker * blk_trace_shutdown: - stop and cleanup trace structures 7612db270a8SFrederic Weisbecker * @q: the request queue associated with the device 7622db270a8SFrederic Weisbecker * 7632db270a8SFrederic Weisbecker **/ 7642db270a8SFrederic Weisbecker void blk_trace_shutdown(struct request_queue *q) 7652db270a8SFrederic Weisbecker { 76685e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 767c780e86dSJan Kara if (rcu_dereference_protected(q->blk_trace, 76885e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex))) { 7691f2cac10SJens Axboe __blk_trace_startstop(q, 0); 7701f2cac10SJens Axboe __blk_trace_remove(q); 7712db270a8SFrederic Weisbecker } 7721f2cac10SJens Axboe 77385e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 7742db270a8SFrederic Weisbecker } 7752db270a8SFrederic Weisbecker 776ca1136c9SShaohua Li #ifdef CONFIG_BLK_CGROUP 77767c0496eSTejun Heo static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 778ca1136c9SShaohua Li { 779c780e86dSJan Kara struct blk_trace *bt; 780ca1136c9SShaohua Li 781c780e86dSJan Kara /* We don't use the 'bt' value here except as an optimization... */ 782c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 1); 783ca1136c9SShaohua Li if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 78467c0496eSTejun Heo return 0; 785ca1136c9SShaohua Li 786db6638d7SDennis Zhou if (!bio->bi_blkg) 78767c0496eSTejun Heo return 0; 78874321038STejun Heo return cgroup_id(bio_blkcg(bio)->css.cgroup); 789ca1136c9SShaohua Li } 790ca1136c9SShaohua Li #else 791e75ad2ccSWang Hai static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 792ca1136c9SShaohua Li { 79367c0496eSTejun Heo return 0; 794ca1136c9SShaohua Li } 795ca1136c9SShaohua Li #endif 796ca1136c9SShaohua Li 79767c0496eSTejun Heo static u64 798ca1136c9SShaohua Li blk_trace_request_get_cgid(struct request_queue *q, struct request *rq) 799ca1136c9SShaohua Li { 800ca1136c9SShaohua Li if (!rq->bio) 80167c0496eSTejun Heo return 0; 802ca1136c9SShaohua Li /* Use the first bio */ 803ca1136c9SShaohua Li return blk_trace_bio_get_cgid(q, rq->bio); 804ca1136c9SShaohua Li } 805ca1136c9SShaohua Li 8062db270a8SFrederic Weisbecker /* 8072db270a8SFrederic Weisbecker * blktrace probes 8082db270a8SFrederic Weisbecker */ 8092db270a8SFrederic Weisbecker 8102db270a8SFrederic Weisbecker /** 8112db270a8SFrederic Weisbecker * blk_add_trace_rq - Add a trace for a request oriented action 8122db270a8SFrederic Weisbecker * @rq: the source request 813caf7df12SChristoph Hellwig * @error: return status to log 814af5040daSRoman Pen * @nr_bytes: number of completed bytes 8152db270a8SFrederic Weisbecker * @what: the action 816ca1136c9SShaohua Li * @cgid: the cgroup info 8172db270a8SFrederic Weisbecker * 8182db270a8SFrederic Weisbecker * Description: 8192db270a8SFrederic Weisbecker * Records an action against a request. Will log the bio offset + size. 8202db270a8SFrederic Weisbecker * 8212db270a8SFrederic Weisbecker **/ 822caf7df12SChristoph Hellwig static void blk_add_trace_rq(struct request *rq, int error, 82367c0496eSTejun Heo unsigned int nr_bytes, u32 what, u64 cgid) 8242db270a8SFrederic Weisbecker { 825c780e86dSJan Kara struct blk_trace *bt; 8262db270a8SFrederic Weisbecker 827c780e86dSJan Kara rcu_read_lock(); 828c780e86dSJan Kara bt = rcu_dereference(rq->q->blk_trace); 829c780e86dSJan Kara if (likely(!bt)) { 830c780e86dSJan Kara rcu_read_unlock(); 8312db270a8SFrederic Weisbecker return; 832c780e86dSJan Kara } 8332db270a8SFrederic Weisbecker 83457292b58SChristoph Hellwig if (blk_rq_is_passthrough(rq)) 8352db270a8SFrederic Weisbecker what |= BLK_TC_ACT(BLK_TC_PC); 83648b77ad6SChristoph Hellwig else 8372db270a8SFrederic Weisbecker what |= BLK_TC_ACT(BLK_TC_FS); 83848b77ad6SChristoph Hellwig 83948b77ad6SChristoph Hellwig __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq), 840ca1136c9SShaohua Li rq->cmd_flags, what, error, 0, NULL, cgid); 841c780e86dSJan Kara rcu_read_unlock(); 8422db270a8SFrederic Weisbecker } 8432db270a8SFrederic Weisbecker 84438516ab5SSteven Rostedt static void blk_add_trace_rq_insert(void *ignore, 84538516ab5SSteven Rostedt struct request_queue *q, struct request *rq) 8462db270a8SFrederic Weisbecker { 847ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT, 848ca1136c9SShaohua Li blk_trace_request_get_cgid(q, rq)); 8492db270a8SFrederic Weisbecker } 8502db270a8SFrederic Weisbecker 85138516ab5SSteven Rostedt static void blk_add_trace_rq_issue(void *ignore, 85238516ab5SSteven Rostedt struct request_queue *q, struct request *rq) 8532db270a8SFrederic Weisbecker { 854ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ISSUE, 855ca1136c9SShaohua Li blk_trace_request_get_cgid(q, rq)); 8562db270a8SFrederic Weisbecker } 8572db270a8SFrederic Weisbecker 858f3bdc62fSJan Kara static void blk_add_trace_rq_merge(void *ignore, 859f3bdc62fSJan Kara struct request_queue *q, struct request *rq) 860f3bdc62fSJan Kara { 861f3bdc62fSJan Kara blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_BACKMERGE, 862f3bdc62fSJan Kara blk_trace_request_get_cgid(q, rq)); 863f3bdc62fSJan Kara } 864f3bdc62fSJan Kara 86538516ab5SSteven Rostedt static void blk_add_trace_rq_requeue(void *ignore, 86638516ab5SSteven Rostedt struct request_queue *q, 8672db270a8SFrederic Weisbecker struct request *rq) 8682db270a8SFrederic Weisbecker { 869ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_REQUEUE, 870ca1136c9SShaohua Li blk_trace_request_get_cgid(q, rq)); 8712db270a8SFrederic Weisbecker } 8722db270a8SFrederic Weisbecker 873caf7df12SChristoph Hellwig static void blk_add_trace_rq_complete(void *ignore, struct request *rq, 874caf7df12SChristoph Hellwig int error, unsigned int nr_bytes) 8752db270a8SFrederic Weisbecker { 876ca1136c9SShaohua Li blk_add_trace_rq(rq, error, nr_bytes, BLK_TA_COMPLETE, 877ca1136c9SShaohua Li blk_trace_request_get_cgid(rq->q, rq)); 8782db270a8SFrederic Weisbecker } 8792db270a8SFrederic Weisbecker 8802db270a8SFrederic Weisbecker /** 8812db270a8SFrederic Weisbecker * blk_add_trace_bio - Add a trace for a bio oriented action 8822db270a8SFrederic Weisbecker * @q: queue the io is for 8832db270a8SFrederic Weisbecker * @bio: the source bio 8842db270a8SFrederic Weisbecker * @what: the action 885797a455dSJens Axboe * @error: error, if any 8862db270a8SFrederic Weisbecker * 8872db270a8SFrederic Weisbecker * Description: 8882db270a8SFrederic Weisbecker * Records an action against a bio. Will log the bio offset + size. 8892db270a8SFrederic Weisbecker * 8902db270a8SFrederic Weisbecker **/ 8912db270a8SFrederic Weisbecker static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, 8921690102dSMarcos Paulo de Souza u32 what, int error) 8932db270a8SFrederic Weisbecker { 894c780e86dSJan Kara struct blk_trace *bt; 8952db270a8SFrederic Weisbecker 896c780e86dSJan Kara rcu_read_lock(); 897c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 898c780e86dSJan Kara if (likely(!bt)) { 899c780e86dSJan Kara rcu_read_unlock(); 9002db270a8SFrederic Weisbecker return; 901c780e86dSJan Kara } 9022db270a8SFrederic Weisbecker 9034f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 9041690102dSMarcos Paulo de Souza bio_op(bio), bio->bi_opf, what, error, 0, NULL, 9051690102dSMarcos Paulo de Souza blk_trace_bio_get_cgid(q, bio)); 906c780e86dSJan Kara rcu_read_unlock(); 9072db270a8SFrederic Weisbecker } 9082db270a8SFrederic Weisbecker 909e8a676d6SChristoph Hellwig static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio) 9102db270a8SFrederic Weisbecker { 911e8a676d6SChristoph Hellwig blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BOUNCE, 0); 9122db270a8SFrederic Weisbecker } 9132db270a8SFrederic Weisbecker 9140a82a8d1SLinus Torvalds static void blk_add_trace_bio_complete(void *ignore, 915d24de76aSChristoph Hellwig struct request_queue *q, struct bio *bio) 9162db270a8SFrederic Weisbecker { 917d24de76aSChristoph Hellwig blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, 918d24de76aSChristoph Hellwig blk_status_to_errno(bio->bi_status)); 9192db270a8SFrederic Weisbecker } 9202db270a8SFrederic Weisbecker 921e8a676d6SChristoph Hellwig static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio) 9222db270a8SFrederic Weisbecker { 923e8a676d6SChristoph Hellwig blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_BACKMERGE, 0); 9242db270a8SFrederic Weisbecker } 9252db270a8SFrederic Weisbecker 926e8a676d6SChristoph Hellwig static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio) 9272db270a8SFrederic Weisbecker { 928e8a676d6SChristoph Hellwig blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_FRONTMERGE, 0); 9292db270a8SFrederic Weisbecker } 9302db270a8SFrederic Weisbecker 931e8a676d6SChristoph Hellwig static void blk_add_trace_bio_queue(void *ignore, struct bio *bio) 9322db270a8SFrederic Weisbecker { 933e8a676d6SChristoph Hellwig blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_QUEUE, 0); 9342db270a8SFrederic Weisbecker } 9352db270a8SFrederic Weisbecker 936e8a676d6SChristoph Hellwig static void blk_add_trace_getrq(void *ignore, struct bio *bio) 9372db270a8SFrederic Weisbecker { 938e8a676d6SChristoph Hellwig blk_add_trace_bio(bio->bi_disk->queue, bio, BLK_TA_GETRQ, 0); 9392db270a8SFrederic Weisbecker } 9402db270a8SFrederic Weisbecker 94138516ab5SSteven Rostedt static void blk_add_trace_plug(void *ignore, struct request_queue *q) 9422db270a8SFrederic Weisbecker { 943c780e86dSJan Kara struct blk_trace *bt; 9442db270a8SFrederic Weisbecker 945c780e86dSJan Kara rcu_read_lock(); 946c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9472db270a8SFrederic Weisbecker if (bt) 94867c0496eSTejun Heo __blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); 949c780e86dSJan Kara rcu_read_unlock(); 9502db270a8SFrederic Weisbecker } 9512db270a8SFrederic Weisbecker 95249cac01eSJens Axboe static void blk_add_trace_unplug(void *ignore, struct request_queue *q, 95349cac01eSJens Axboe unsigned int depth, bool explicit) 9542db270a8SFrederic Weisbecker { 955c780e86dSJan Kara struct blk_trace *bt; 9562db270a8SFrederic Weisbecker 957c780e86dSJan Kara rcu_read_lock(); 958c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9592db270a8SFrederic Weisbecker if (bt) { 96094b5eb28SJens Axboe __be64 rpdu = cpu_to_be64(depth); 96149cac01eSJens Axboe u32 what; 9622db270a8SFrederic Weisbecker 96349cac01eSJens Axboe if (explicit) 96449cac01eSJens Axboe what = BLK_TA_UNPLUG_IO; 96549cac01eSJens Axboe else 96649cac01eSJens Axboe what = BLK_TA_UNPLUG_TIMER; 96749cac01eSJens Axboe 96867c0496eSTejun Heo __blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); 9692db270a8SFrederic Weisbecker } 970c780e86dSJan Kara rcu_read_unlock(); 9712db270a8SFrederic Weisbecker } 9722db270a8SFrederic Weisbecker 973eb6f7f7cSChristoph Hellwig static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) 9742db270a8SFrederic Weisbecker { 975eb6f7f7cSChristoph Hellwig struct request_queue *q = bio->bi_disk->queue; 976c780e86dSJan Kara struct blk_trace *bt; 9772db270a8SFrederic Weisbecker 978c780e86dSJan Kara rcu_read_lock(); 979c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9802db270a8SFrederic Weisbecker if (bt) { 9812db270a8SFrederic Weisbecker __be64 rpdu = cpu_to_be64(pdu); 9822db270a8SFrederic Weisbecker 9834f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, 9841eff9d32SJens Axboe bio->bi_iter.bi_size, bio_op(bio), bio->bi_opf, 98548bc3cd3SChaitanya Kulkarni BLK_TA_SPLIT, 98648bc3cd3SChaitanya Kulkarni blk_status_to_errno(bio->bi_status), 98748bc3cd3SChaitanya Kulkarni sizeof(rpdu), &rpdu, 98848bc3cd3SChaitanya Kulkarni blk_trace_bio_get_cgid(q, bio)); 9892db270a8SFrederic Weisbecker } 990c780e86dSJan Kara rcu_read_unlock(); 9912db270a8SFrederic Weisbecker } 9922db270a8SFrederic Weisbecker 9932db270a8SFrederic Weisbecker /** 994d07335e5SMike Snitzer * blk_add_trace_bio_remap - Add a trace for a bio-remap operation 995546cf44aSRandy Dunlap * @ignore: trace callback data parameter (not used) 9962db270a8SFrederic Weisbecker * @bio: the source bio 997*1c02fca6SChristoph Hellwig * @dev: source device 998a42aaa3bSAlan D. Brunelle * @from: source sector 9992db270a8SFrederic Weisbecker * 1000*1c02fca6SChristoph Hellwig * Called after a bio is remapped to a different device and/or sector. 10012db270a8SFrederic Weisbecker **/ 1002*1c02fca6SChristoph Hellwig static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, 1003*1c02fca6SChristoph Hellwig sector_t from) 10042db270a8SFrederic Weisbecker { 1005*1c02fca6SChristoph Hellwig struct request_queue *q = bio->bi_disk->queue; 1006c780e86dSJan Kara struct blk_trace *bt; 10072db270a8SFrederic Weisbecker struct blk_io_trace_remap r; 10082db270a8SFrederic Weisbecker 1009c780e86dSJan Kara rcu_read_lock(); 1010c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 1011c780e86dSJan Kara if (likely(!bt)) { 1012c780e86dSJan Kara rcu_read_unlock(); 10132db270a8SFrederic Weisbecker return; 1014c780e86dSJan Kara } 10152db270a8SFrederic Weisbecker 1016a42aaa3bSAlan D. Brunelle r.device_from = cpu_to_be32(dev); 101774d46992SChristoph Hellwig r.device_to = cpu_to_be32(bio_dev(bio)); 1018a42aaa3bSAlan D. Brunelle r.sector_from = cpu_to_be64(from); 10192db270a8SFrederic Weisbecker 10204f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 102148bc3cd3SChaitanya Kulkarni bio_op(bio), bio->bi_opf, BLK_TA_REMAP, 102248bc3cd3SChaitanya Kulkarni blk_status_to_errno(bio->bi_status), 1023ca1136c9SShaohua Li sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); 1024c780e86dSJan Kara rcu_read_unlock(); 10252db270a8SFrederic Weisbecker } 10262db270a8SFrederic Weisbecker 10272db270a8SFrederic Weisbecker /** 1028b0da3f0dSJun'ichi Nomura * blk_add_trace_rq_remap - Add a trace for a request-remap operation 1029546cf44aSRandy Dunlap * @ignore: trace callback data parameter (not used) 1030b0da3f0dSJun'ichi Nomura * @q: queue the io is for 1031b0da3f0dSJun'ichi Nomura * @rq: the source request 1032b0da3f0dSJun'ichi Nomura * @dev: target device 1033b0da3f0dSJun'ichi Nomura * @from: source sector 1034b0da3f0dSJun'ichi Nomura * 1035b0da3f0dSJun'ichi Nomura * Description: 1036b0da3f0dSJun'ichi Nomura * Device mapper remaps request to other devices. 1037b0da3f0dSJun'ichi Nomura * Add a trace for that action. 1038b0da3f0dSJun'ichi Nomura * 1039b0da3f0dSJun'ichi Nomura **/ 104038516ab5SSteven Rostedt static void blk_add_trace_rq_remap(void *ignore, 104138516ab5SSteven Rostedt struct request_queue *q, 1042b0da3f0dSJun'ichi Nomura struct request *rq, dev_t dev, 1043b0da3f0dSJun'ichi Nomura sector_t from) 1044b0da3f0dSJun'ichi Nomura { 1045c780e86dSJan Kara struct blk_trace *bt; 1046b0da3f0dSJun'ichi Nomura struct blk_io_trace_remap r; 1047b0da3f0dSJun'ichi Nomura 1048c780e86dSJan Kara rcu_read_lock(); 1049c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 1050c780e86dSJan Kara if (likely(!bt)) { 1051c780e86dSJan Kara rcu_read_unlock(); 1052b0da3f0dSJun'ichi Nomura return; 1053c780e86dSJan Kara } 1054b0da3f0dSJun'ichi Nomura 1055b0da3f0dSJun'ichi Nomura r.device_from = cpu_to_be32(dev); 1056b0da3f0dSJun'ichi Nomura r.device_to = cpu_to_be32(disk_devt(rq->rq_disk)); 1057b0da3f0dSJun'ichi Nomura r.sector_from = cpu_to_be64(from); 1058b0da3f0dSJun'ichi Nomura 1059b0da3f0dSJun'ichi Nomura __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 1060caf7df12SChristoph Hellwig rq_data_dir(rq), 0, BLK_TA_REMAP, 0, 1061ca1136c9SShaohua Li sizeof(r), &r, blk_trace_request_get_cgid(q, rq)); 1062c780e86dSJan Kara rcu_read_unlock(); 1063b0da3f0dSJun'ichi Nomura } 1064b0da3f0dSJun'ichi Nomura 1065b0da3f0dSJun'ichi Nomura /** 10662db270a8SFrederic Weisbecker * blk_add_driver_data - Add binary message with driver-specific data 10672db270a8SFrederic Weisbecker * @q: queue the io is for 10682db270a8SFrederic Weisbecker * @rq: io request 10692db270a8SFrederic Weisbecker * @data: driver-specific data 10702db270a8SFrederic Weisbecker * @len: length of driver-specific data 10712db270a8SFrederic Weisbecker * 10722db270a8SFrederic Weisbecker * Description: 10732db270a8SFrederic Weisbecker * Some drivers might want to write driver-specific data per request. 10742db270a8SFrederic Weisbecker * 10752db270a8SFrederic Weisbecker **/ 10762db270a8SFrederic Weisbecker void blk_add_driver_data(struct request_queue *q, 10772db270a8SFrederic Weisbecker struct request *rq, 10782db270a8SFrederic Weisbecker void *data, size_t len) 10792db270a8SFrederic Weisbecker { 1080c780e86dSJan Kara struct blk_trace *bt; 10812db270a8SFrederic Weisbecker 1082c780e86dSJan Kara rcu_read_lock(); 1083c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 1084c780e86dSJan Kara if (likely(!bt)) { 1085c780e86dSJan Kara rcu_read_unlock(); 10862db270a8SFrederic Weisbecker return; 1087c780e86dSJan Kara } 10882db270a8SFrederic Weisbecker 108948b77ad6SChristoph Hellwig __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0, 1090ca1136c9SShaohua Li BLK_TA_DRV_DATA, 0, len, data, 1091ca1136c9SShaohua Li blk_trace_request_get_cgid(q, rq)); 1092c780e86dSJan Kara rcu_read_unlock(); 10932db270a8SFrederic Weisbecker } 10942db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_add_driver_data); 10952db270a8SFrederic Weisbecker 10963c289ba7SLi Zefan static void blk_register_tracepoints(void) 10972db270a8SFrederic Weisbecker { 10982db270a8SFrederic Weisbecker int ret; 10992db270a8SFrederic Weisbecker 110038516ab5SSteven Rostedt ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 11012db270a8SFrederic Weisbecker WARN_ON(ret); 110238516ab5SSteven Rostedt ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 11032db270a8SFrederic Weisbecker WARN_ON(ret); 1104f3bdc62fSJan Kara ret = register_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 1105f3bdc62fSJan Kara WARN_ON(ret); 110638516ab5SSteven Rostedt ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 11072db270a8SFrederic Weisbecker WARN_ON(ret); 110838516ab5SSteven Rostedt ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 11092db270a8SFrederic Weisbecker WARN_ON(ret); 111038516ab5SSteven Rostedt ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); 11112db270a8SFrederic Weisbecker WARN_ON(ret); 111238516ab5SSteven Rostedt ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 11132db270a8SFrederic Weisbecker WARN_ON(ret); 111438516ab5SSteven Rostedt ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 11152db270a8SFrederic Weisbecker WARN_ON(ret); 111638516ab5SSteven Rostedt ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 11172db270a8SFrederic Weisbecker WARN_ON(ret); 111838516ab5SSteven Rostedt ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 11192db270a8SFrederic Weisbecker WARN_ON(ret); 112038516ab5SSteven Rostedt ret = register_trace_block_getrq(blk_add_trace_getrq, NULL); 11212db270a8SFrederic Weisbecker WARN_ON(ret); 112238516ab5SSteven Rostedt ret = register_trace_block_plug(blk_add_trace_plug, NULL); 11232db270a8SFrederic Weisbecker WARN_ON(ret); 112449cac01eSJens Axboe ret = register_trace_block_unplug(blk_add_trace_unplug, NULL); 11252db270a8SFrederic Weisbecker WARN_ON(ret); 112638516ab5SSteven Rostedt ret = register_trace_block_split(blk_add_trace_split, NULL); 11272db270a8SFrederic Weisbecker WARN_ON(ret); 1128d07335e5SMike Snitzer ret = register_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 11292db270a8SFrederic Weisbecker WARN_ON(ret); 113038516ab5SSteven Rostedt ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1131b0da3f0dSJun'ichi Nomura WARN_ON(ret); 11322db270a8SFrederic Weisbecker } 11332db270a8SFrederic Weisbecker 11342db270a8SFrederic Weisbecker static void blk_unregister_tracepoints(void) 11352db270a8SFrederic Weisbecker { 113638516ab5SSteven Rostedt unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1137d07335e5SMike Snitzer unregister_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 113838516ab5SSteven Rostedt unregister_trace_block_split(blk_add_trace_split, NULL); 113949cac01eSJens Axboe unregister_trace_block_unplug(blk_add_trace_unplug, NULL); 114038516ab5SSteven Rostedt unregister_trace_block_plug(blk_add_trace_plug, NULL); 114138516ab5SSteven Rostedt unregister_trace_block_getrq(blk_add_trace_getrq, NULL); 114238516ab5SSteven Rostedt unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 114338516ab5SSteven Rostedt unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 114438516ab5SSteven Rostedt unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 114538516ab5SSteven Rostedt unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 114638516ab5SSteven Rostedt unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); 114738516ab5SSteven Rostedt unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 114838516ab5SSteven Rostedt unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 1149f3bdc62fSJan Kara unregister_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 115038516ab5SSteven Rostedt unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 115138516ab5SSteven Rostedt unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 11522db270a8SFrederic Weisbecker 11532db270a8SFrederic Weisbecker tracepoint_synchronize_unregister(); 11542db270a8SFrederic Weisbecker } 11552db270a8SFrederic Weisbecker 11562db270a8SFrederic Weisbecker /* 11572db270a8SFrederic Weisbecker * struct blk_io_tracer formatting routines 11582db270a8SFrederic Weisbecker */ 11592db270a8SFrederic Weisbecker 11602db270a8SFrederic Weisbecker static void fill_rwbs(char *rwbs, const struct blk_io_trace *t) 11612db270a8SFrederic Weisbecker { 11622db270a8SFrederic Weisbecker int i = 0; 116365796348SLi Zefan int tc = t->action >> BLK_TC_SHIFT; 11642db270a8SFrederic Weisbecker 1165ca1136c9SShaohua Li if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 116618cea459SLi Zefan rwbs[i++] = 'N'; 116718cea459SLi Zefan goto out; 116818cea459SLi Zefan } 116918cea459SLi Zefan 1170c09c47caSNamhyung Kim if (tc & BLK_TC_FLUSH) 1171c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1172c09c47caSNamhyung Kim 117365796348SLi Zefan if (tc & BLK_TC_DISCARD) 11742db270a8SFrederic Weisbecker rwbs[i++] = 'D'; 117565796348SLi Zefan else if (tc & BLK_TC_WRITE) 11762db270a8SFrederic Weisbecker rwbs[i++] = 'W'; 11772db270a8SFrederic Weisbecker else if (t->bytes) 11782db270a8SFrederic Weisbecker rwbs[i++] = 'R'; 11792db270a8SFrederic Weisbecker else 11802db270a8SFrederic Weisbecker rwbs[i++] = 'N'; 11812db270a8SFrederic Weisbecker 1182c09c47caSNamhyung Kim if (tc & BLK_TC_FUA) 1183c09c47caSNamhyung Kim rwbs[i++] = 'F'; 118465796348SLi Zefan if (tc & BLK_TC_AHEAD) 11852db270a8SFrederic Weisbecker rwbs[i++] = 'A'; 118665796348SLi Zefan if (tc & BLK_TC_SYNC) 11872db270a8SFrederic Weisbecker rwbs[i++] = 'S'; 118865796348SLi Zefan if (tc & BLK_TC_META) 11892db270a8SFrederic Weisbecker rwbs[i++] = 'M'; 119018cea459SLi Zefan out: 11912db270a8SFrederic Weisbecker rwbs[i] = '\0'; 11922db270a8SFrederic Weisbecker } 11932db270a8SFrederic Weisbecker 11942db270a8SFrederic Weisbecker static inline 11952db270a8SFrederic Weisbecker const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent) 11962db270a8SFrederic Weisbecker { 11972db270a8SFrederic Weisbecker return (const struct blk_io_trace *)ent; 11982db270a8SFrederic Weisbecker } 11992db270a8SFrederic Weisbecker 1200ca1136c9SShaohua Li static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg) 12012db270a8SFrederic Weisbecker { 120267c0496eSTejun Heo return (void *)(te_blk_io_trace(ent) + 1) + (has_cg ? sizeof(u64) : 0); 1203ca1136c9SShaohua Li } 1204ca1136c9SShaohua Li 120567c0496eSTejun Heo static inline u64 t_cgid(const struct trace_entry *ent) 1206ca1136c9SShaohua Li { 120767c0496eSTejun Heo return *(u64 *)(te_blk_io_trace(ent) + 1); 1208ca1136c9SShaohua Li } 1209ca1136c9SShaohua Li 1210ca1136c9SShaohua Li static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg) 1211ca1136c9SShaohua Li { 121267c0496eSTejun Heo return te_blk_io_trace(ent)->pdu_len - (has_cg ? sizeof(u64) : 0); 12132db270a8SFrederic Weisbecker } 12142db270a8SFrederic Weisbecker 121566de7792SLi Zefan static inline u32 t_action(const struct trace_entry *ent) 121666de7792SLi Zefan { 121766de7792SLi Zefan return te_blk_io_trace(ent)->action; 121866de7792SLi Zefan } 121966de7792SLi Zefan 122066de7792SLi Zefan static inline u32 t_bytes(const struct trace_entry *ent) 122166de7792SLi Zefan { 122266de7792SLi Zefan return te_blk_io_trace(ent)->bytes; 122366de7792SLi Zefan } 122466de7792SLi Zefan 12252db270a8SFrederic Weisbecker static inline u32 t_sec(const struct trace_entry *ent) 12262db270a8SFrederic Weisbecker { 12272db270a8SFrederic Weisbecker return te_blk_io_trace(ent)->bytes >> 9; 12282db270a8SFrederic Weisbecker } 12292db270a8SFrederic Weisbecker 12302db270a8SFrederic Weisbecker static inline unsigned long long t_sector(const struct trace_entry *ent) 12312db270a8SFrederic Weisbecker { 12322db270a8SFrederic Weisbecker return te_blk_io_trace(ent)->sector; 12332db270a8SFrederic Weisbecker } 12342db270a8SFrederic Weisbecker 12352db270a8SFrederic Weisbecker static inline __u16 t_error(const struct trace_entry *ent) 12362db270a8SFrederic Weisbecker { 1237e0dc81beSLi Zefan return te_blk_io_trace(ent)->error; 12382db270a8SFrederic Weisbecker } 12392db270a8SFrederic Weisbecker 1240ca1136c9SShaohua Li static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg) 12412db270a8SFrederic Weisbecker { 124271df3fd8SChaitanya Kulkarni const __be64 *val = pdu_start(ent, has_cg); 12432db270a8SFrederic Weisbecker return be64_to_cpu(*val); 12442db270a8SFrederic Weisbecker } 12452db270a8SFrederic Weisbecker 1246ca1136c9SShaohua Li typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act, 1247ca1136c9SShaohua Li bool has_cg); 1248b6a4b0c3SLi Zefan 1249ca1136c9SShaohua Li static void blk_log_action_classic(struct trace_iterator *iter, const char *act, 1250ca1136c9SShaohua Li bool has_cg) 12512db270a8SFrederic Weisbecker { 1252c09c47caSNamhyung Kim char rwbs[RWBS_LEN]; 125335ac51bfSLi Zefan unsigned long long ts = iter->ts; 125435ac51bfSLi Zefan unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC); 12552db270a8SFrederic Weisbecker unsigned secs = (unsigned long)ts; 1256b6a4b0c3SLi Zefan const struct blk_io_trace *t = te_blk_io_trace(iter->ent); 12572db270a8SFrederic Weisbecker 12582db270a8SFrederic Weisbecker fill_rwbs(rwbs, t); 12592db270a8SFrederic Weisbecker 1260f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(&iter->seq, 126135ac51bfSLi Zefan "%3d,%-3d %2d %5d.%09lu %5u %2s %3s ", 12622db270a8SFrederic Weisbecker MAJOR(t->device), MINOR(t->device), iter->cpu, 1263b6a4b0c3SLi Zefan secs, nsec_rem, iter->ent->pid, act, rwbs); 12642db270a8SFrederic Weisbecker } 12652db270a8SFrederic Weisbecker 1266ca1136c9SShaohua Li static void blk_log_action(struct trace_iterator *iter, const char *act, 1267ca1136c9SShaohua Li bool has_cg) 12682db270a8SFrederic Weisbecker { 1269c09c47caSNamhyung Kim char rwbs[RWBS_LEN]; 1270b6a4b0c3SLi Zefan const struct blk_io_trace *t = te_blk_io_trace(iter->ent); 1271b6a4b0c3SLi Zefan 12722db270a8SFrederic Weisbecker fill_rwbs(rwbs, t); 1273ca1136c9SShaohua Li if (has_cg) { 127467c0496eSTejun Heo u64 id = t_cgid(iter->ent); 1275ca1136c9SShaohua Li 127669fd5c39SShaohua Li if (blk_tracer_flags.val & TRACE_BLK_OPT_CGNAME) { 127769fd5c39SShaohua Li char blkcg_name_buf[NAME_MAX + 1] = "<...>"; 127869fd5c39SShaohua Li 127969fd5c39SShaohua Li cgroup_path_from_kernfs_id(id, blkcg_name_buf, 128069fd5c39SShaohua Li sizeof(blkcg_name_buf)); 128169fd5c39SShaohua Li trace_seq_printf(&iter->seq, "%3d,%-3d %s %2s %3s ", 128269fd5c39SShaohua Li MAJOR(t->device), MINOR(t->device), 128369fd5c39SShaohua Li blkcg_name_buf, act, rwbs); 128440430452STejun Heo } else { 128540430452STejun Heo /* 128640430452STejun Heo * The cgid portion used to be "INO,GEN". Userland 128740430452STejun Heo * builds a FILEID_INO32_GEN fid out of them and 128840430452STejun Heo * opens the cgroup using open_by_handle_at(2). 128940430452STejun Heo * While 32bit ino setups are still the same, 64bit 129040430452STejun Heo * ones now use the 64bit ino as the whole ID and 129140430452STejun Heo * no longer use generation. 129240430452STejun Heo * 129340430452STejun Heo * Regarldess of the content, always output 129440430452STejun Heo * "LOW32,HIGH32" so that FILEID_INO32_GEN fid can 129540430452STejun Heo * be mapped back to @id on both 64 and 32bit ino 129640430452STejun Heo * setups. See __kernfs_fh_to_dentry(). 129740430452STejun Heo */ 129869fd5c39SShaohua Li trace_seq_printf(&iter->seq, 129940430452STejun Heo "%3d,%-3d %llx,%-llx %2s %3s ", 1300ca1136c9SShaohua Li MAJOR(t->device), MINOR(t->device), 130140430452STejun Heo id & U32_MAX, id >> 32, act, rwbs); 130240430452STejun Heo } 1303ca1136c9SShaohua Li } else 1304f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(&iter->seq, "%3d,%-3d %2s %3s ", 13052db270a8SFrederic Weisbecker MAJOR(t->device), MINOR(t->device), act, rwbs); 13062db270a8SFrederic Weisbecker } 13072db270a8SFrederic Weisbecker 1308ca1136c9SShaohua Li static void blk_log_dump_pdu(struct trace_seq *s, 1309ca1136c9SShaohua Li const struct trace_entry *ent, bool has_cg) 131066de7792SLi Zefan { 131104986257SLi Zefan const unsigned char *pdu_buf; 131266de7792SLi Zefan int pdu_len; 1313f4a1d08cSSteven Rostedt (Red Hat) int i, end; 131466de7792SLi Zefan 1315ca1136c9SShaohua Li pdu_buf = pdu_start(ent, has_cg); 1316ca1136c9SShaohua Li pdu_len = pdu_real_len(ent, has_cg); 131766de7792SLi Zefan 131866de7792SLi Zefan if (!pdu_len) 1319f4a1d08cSSteven Rostedt (Red Hat) return; 132066de7792SLi Zefan 132166de7792SLi Zefan /* find the last zero that needs to be printed */ 132266de7792SLi Zefan for (end = pdu_len - 1; end >= 0; end--) 132366de7792SLi Zefan if (pdu_buf[end]) 132466de7792SLi Zefan break; 132566de7792SLi Zefan end++; 132666de7792SLi Zefan 1327f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putc(s, '('); 132866de7792SLi Zefan 132966de7792SLi Zefan for (i = 0; i < pdu_len; i++) { 133066de7792SLi Zefan 1331f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%s%02x", 133266de7792SLi Zefan i == 0 ? "" : " ", pdu_buf[i]); 133366de7792SLi Zefan 133466de7792SLi Zefan /* 133566de7792SLi Zefan * stop when the rest is just zeroes and indicate so 133666de7792SLi Zefan * with a ".." appended 133766de7792SLi Zefan */ 1338f4a1d08cSSteven Rostedt (Red Hat) if (i == end && end != pdu_len - 1) { 1339f4a1d08cSSteven Rostedt (Red Hat) trace_seq_puts(s, " ..) "); 1340f4a1d08cSSteven Rostedt (Red Hat) return; 1341f4a1d08cSSteven Rostedt (Red Hat) } 134266de7792SLi Zefan } 134366de7792SLi Zefan 1344f4a1d08cSSteven Rostedt (Red Hat) trace_seq_puts(s, ") "); 134566de7792SLi Zefan } 134666de7792SLi Zefan 1347ca1136c9SShaohua Li static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13482db270a8SFrederic Weisbecker { 13494ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 13504ca53085SSteven Rostedt 13514ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 13522db270a8SFrederic Weisbecker 135366de7792SLi Zefan if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1354f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%u ", t_bytes(ent)); 1355ca1136c9SShaohua Li blk_log_dump_pdu(s, ent, has_cg); 1356f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 135766de7792SLi Zefan } else { 13582db270a8SFrederic Weisbecker if (t_sec(ent)) 1359f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u [%s]\n", 13602db270a8SFrederic Weisbecker t_sector(ent), t_sec(ent), cmd); 1361f4a1d08cSSteven Rostedt (Red Hat) else 1362f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 13632db270a8SFrederic Weisbecker } 136466de7792SLi Zefan } 13652db270a8SFrederic Weisbecker 1366f4a1d08cSSteven Rostedt (Red Hat) static void blk_log_with_error(struct trace_seq *s, 1367ca1136c9SShaohua Li const struct trace_entry *ent, bool has_cg) 13682db270a8SFrederic Weisbecker { 136966de7792SLi Zefan if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1370ca1136c9SShaohua Li blk_log_dump_pdu(s, ent, has_cg); 1371f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%d]\n", t_error(ent)); 137266de7792SLi Zefan } else { 13732db270a8SFrederic Weisbecker if (t_sec(ent)) 1374f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u [%d]\n", 137566de7792SLi Zefan t_sector(ent), 13762db270a8SFrederic Weisbecker t_sec(ent), t_error(ent)); 1377f4a1d08cSSteven Rostedt (Red Hat) else 1378f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu [%d]\n", 137966de7792SLi Zefan t_sector(ent), t_error(ent)); 138066de7792SLi Zefan } 13812db270a8SFrederic Weisbecker } 13822db270a8SFrederic Weisbecker 1383ca1136c9SShaohua Li static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13842db270a8SFrederic Weisbecker { 13855aec598cSChaitanya Kulkarni const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg); 13862db270a8SFrederic Weisbecker 1387f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n", 1388a42aaa3bSAlan D. Brunelle t_sector(ent), t_sec(ent), 13895aec598cSChaitanya Kulkarni MAJOR(be32_to_cpu(__r->device_from)), 13905aec598cSChaitanya Kulkarni MINOR(be32_to_cpu(__r->device_from)), 13915aec598cSChaitanya Kulkarni be64_to_cpu(__r->sector_from)); 13922db270a8SFrederic Weisbecker } 13932db270a8SFrederic Weisbecker 1394ca1136c9SShaohua Li static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13952db270a8SFrederic Weisbecker { 13964ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 13974ca53085SSteven Rostedt 13984ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 13994ca53085SSteven Rostedt 1400f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 14012db270a8SFrederic Weisbecker } 14022db270a8SFrederic Weisbecker 1403ca1136c9SShaohua Li static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 14042db270a8SFrederic Weisbecker { 14054ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 14064ca53085SSteven Rostedt 14074ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 14084ca53085SSteven Rostedt 1409ca1136c9SShaohua Li trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent, has_cg)); 14102db270a8SFrederic Weisbecker } 14112db270a8SFrederic Weisbecker 1412ca1136c9SShaohua Li static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 14132db270a8SFrederic Weisbecker { 14144ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 14154ca53085SSteven Rostedt 14164ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 14174ca53085SSteven Rostedt 1418f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent), 1419ca1136c9SShaohua Li get_pdu_int(ent, has_cg), cmd); 14202db270a8SFrederic Weisbecker } 14212db270a8SFrederic Weisbecker 1422ca1136c9SShaohua Li static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent, 1423ca1136c9SShaohua Li bool has_cg) 142418cea459SLi Zefan { 142518cea459SLi Zefan 1426ca1136c9SShaohua Li trace_seq_putmem(s, pdu_start(ent, has_cg), 1427ca1136c9SShaohua Li pdu_real_len(ent, has_cg)); 1428f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putc(s, '\n'); 142918cea459SLi Zefan } 143018cea459SLi Zefan 14312db270a8SFrederic Weisbecker /* 14322db270a8SFrederic Weisbecker * struct tracer operations 14332db270a8SFrederic Weisbecker */ 14342db270a8SFrederic Weisbecker 14352db270a8SFrederic Weisbecker static void blk_tracer_print_header(struct seq_file *m) 14362db270a8SFrederic Weisbecker { 14372db270a8SFrederic Weisbecker if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 14382db270a8SFrederic Weisbecker return; 14392db270a8SFrederic Weisbecker seq_puts(m, "# DEV CPU TIMESTAMP PID ACT FLG\n" 14402db270a8SFrederic Weisbecker "# | | | | | |\n"); 14412db270a8SFrederic Weisbecker } 14422db270a8SFrederic Weisbecker 14432db270a8SFrederic Weisbecker static void blk_tracer_start(struct trace_array *tr) 14442db270a8SFrederic Weisbecker { 1445ad5dd549SLi Zefan blk_tracer_enabled = true; 14462db270a8SFrederic Weisbecker } 14472db270a8SFrederic Weisbecker 14482db270a8SFrederic Weisbecker static int blk_tracer_init(struct trace_array *tr) 14492db270a8SFrederic Weisbecker { 14502db270a8SFrederic Weisbecker blk_tr = tr; 14512db270a8SFrederic Weisbecker blk_tracer_start(tr); 14522db270a8SFrederic Weisbecker return 0; 14532db270a8SFrederic Weisbecker } 14542db270a8SFrederic Weisbecker 14552db270a8SFrederic Weisbecker static void blk_tracer_stop(struct trace_array *tr) 14562db270a8SFrederic Weisbecker { 1457ad5dd549SLi Zefan blk_tracer_enabled = false; 14582db270a8SFrederic Weisbecker } 14592db270a8SFrederic Weisbecker 14602db270a8SFrederic Weisbecker static void blk_tracer_reset(struct trace_array *tr) 14612db270a8SFrederic Weisbecker { 14622db270a8SFrederic Weisbecker blk_tracer_stop(tr); 14632db270a8SFrederic Weisbecker } 14642db270a8SFrederic Weisbecker 1465e4955c99SLi Zefan static const struct { 14662db270a8SFrederic Weisbecker const char *act[2]; 1467ca1136c9SShaohua Li void (*print)(struct trace_seq *s, const struct trace_entry *ent, 1468ca1136c9SShaohua Li bool has_cg); 1469e4955c99SLi Zefan } what2act[] = { 14702db270a8SFrederic Weisbecker [__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic }, 14712db270a8SFrederic Weisbecker [__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic }, 14722db270a8SFrederic Weisbecker [__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic }, 14732db270a8SFrederic Weisbecker [__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic }, 14742db270a8SFrederic Weisbecker [__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic }, 14752db270a8SFrederic Weisbecker [__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error }, 14762db270a8SFrederic Weisbecker [__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic }, 14772db270a8SFrederic Weisbecker [__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error }, 14782db270a8SFrederic Weisbecker [__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug }, 14792db270a8SFrederic Weisbecker [__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug }, 148049cac01eSJens Axboe [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug }, 14812db270a8SFrederic Weisbecker [__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic }, 14822db270a8SFrederic Weisbecker [__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split }, 14832db270a8SFrederic Weisbecker [__BLK_TA_BOUNCE] = {{ "B", "bounce" }, blk_log_generic }, 14842db270a8SFrederic Weisbecker [__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap }, 14852db270a8SFrederic Weisbecker }; 14862db270a8SFrederic Weisbecker 1487b6a4b0c3SLi Zefan static enum print_line_t print_one_line(struct trace_iterator *iter, 1488b6a4b0c3SLi Zefan bool classic) 14892db270a8SFrederic Weisbecker { 1490983f938aSSteven Rostedt (Red Hat) struct trace_array *tr = iter->tr; 14912db270a8SFrederic Weisbecker struct trace_seq *s = &iter->seq; 1492b6a4b0c3SLi Zefan const struct blk_io_trace *t; 1493b6a4b0c3SLi Zefan u16 what; 1494b6a4b0c3SLi Zefan bool long_act; 1495b6a4b0c3SLi Zefan blk_log_action_t *log_action; 1496ca1136c9SShaohua Li bool has_cg; 14972db270a8SFrederic Weisbecker 1498b6a4b0c3SLi Zefan t = te_blk_io_trace(iter->ent); 1499ca1136c9SShaohua Li what = (t->action & ((1 << BLK_TC_SHIFT) - 1)) & ~__BLK_TA_CGROUP; 1500983f938aSSteven Rostedt (Red Hat) long_act = !!(tr->trace_flags & TRACE_ITER_VERBOSE); 1501b6a4b0c3SLi Zefan log_action = classic ? &blk_log_action_classic : &blk_log_action; 1502ca1136c9SShaohua Li has_cg = t->action & __BLK_TA_CGROUP; 15032db270a8SFrederic Weisbecker 1504ca1136c9SShaohua Li if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 1505ca1136c9SShaohua Li log_action(iter, long_act ? "message" : "m", has_cg); 1506ca1136c9SShaohua Li blk_log_msg(s, iter->ent, has_cg); 1507b7d7641eSShaohua Li return trace_handle_return(s); 150818cea459SLi Zefan } 150918cea459SLi Zefan 1510eb08f8ebSLi Zefan if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act))) 1511f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "Unknown action %x\n", what); 15122db270a8SFrederic Weisbecker else { 1513ca1136c9SShaohua Li log_action(iter, what2act[what].act[long_act], has_cg); 1514ca1136c9SShaohua Li what2act[what].print(s, iter->ent, has_cg); 15152db270a8SFrederic Weisbecker } 1516f4a1d08cSSteven Rostedt (Red Hat) 1517f4a1d08cSSteven Rostedt (Red Hat) return trace_handle_return(s); 15182db270a8SFrederic Weisbecker } 15192db270a8SFrederic Weisbecker 1520b6a4b0c3SLi Zefan static enum print_line_t blk_trace_event_print(struct trace_iterator *iter, 1521a9a57763SSteven Rostedt int flags, struct trace_event *event) 1522b6a4b0c3SLi Zefan { 1523b6a4b0c3SLi Zefan return print_one_line(iter, false); 1524b6a4b0c3SLi Zefan } 1525b6a4b0c3SLi Zefan 1526f4a1d08cSSteven Rostedt (Red Hat) static void blk_trace_synthesize_old_trace(struct trace_iterator *iter) 15272db270a8SFrederic Weisbecker { 15282db270a8SFrederic Weisbecker struct trace_seq *s = &iter->seq; 15292db270a8SFrederic Weisbecker struct blk_io_trace *t = (struct blk_io_trace *)iter->ent; 15302db270a8SFrederic Weisbecker const int offset = offsetof(struct blk_io_trace, sector); 15312db270a8SFrederic Weisbecker struct blk_io_trace old = { 15322db270a8SFrederic Weisbecker .magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION, 15336c051ce0SLi Zefan .time = iter->ts, 15342db270a8SFrederic Weisbecker }; 15352db270a8SFrederic Weisbecker 1536f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putmem(s, &old, offset); 1537f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putmem(s, &t->sector, 15382db270a8SFrederic Weisbecker sizeof(old) - offset + t->pdu_len); 15392db270a8SFrederic Weisbecker } 15402db270a8SFrederic Weisbecker 15412db270a8SFrederic Weisbecker static enum print_line_t 1542a9a57763SSteven Rostedt blk_trace_event_print_binary(struct trace_iterator *iter, int flags, 1543a9a57763SSteven Rostedt struct trace_event *event) 15442db270a8SFrederic Weisbecker { 1545f4a1d08cSSteven Rostedt (Red Hat) blk_trace_synthesize_old_trace(iter); 1546f4a1d08cSSteven Rostedt (Red Hat) 1547f4a1d08cSSteven Rostedt (Red Hat) return trace_handle_return(&iter->seq); 15482db270a8SFrederic Weisbecker } 15492db270a8SFrederic Weisbecker 15502db270a8SFrederic Weisbecker static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter) 15512db270a8SFrederic Weisbecker { 15522db270a8SFrederic Weisbecker if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 15532db270a8SFrederic Weisbecker return TRACE_TYPE_UNHANDLED; 15542db270a8SFrederic Weisbecker 1555b6a4b0c3SLi Zefan return print_one_line(iter, true); 15562db270a8SFrederic Weisbecker } 15572db270a8SFrederic Weisbecker 15588c1a49aeSSteven Rostedt (Red Hat) static int 15598c1a49aeSSteven Rostedt (Red Hat) blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1560f3948f88SLi Zefan { 1561f3948f88SLi Zefan /* don't output context-info for blk_classic output */ 1562f3948f88SLi Zefan if (bit == TRACE_BLK_OPT_CLASSIC) { 1563f3948f88SLi Zefan if (set) 1564983f938aSSteven Rostedt (Red Hat) tr->trace_flags &= ~TRACE_ITER_CONTEXT_INFO; 1565f3948f88SLi Zefan else 1566983f938aSSteven Rostedt (Red Hat) tr->trace_flags |= TRACE_ITER_CONTEXT_INFO; 1567f3948f88SLi Zefan } 1568f3948f88SLi Zefan return 0; 1569f3948f88SLi Zefan } 1570f3948f88SLi Zefan 15712db270a8SFrederic Weisbecker static struct tracer blk_tracer __read_mostly = { 15722db270a8SFrederic Weisbecker .name = "blk", 15732db270a8SFrederic Weisbecker .init = blk_tracer_init, 15742db270a8SFrederic Weisbecker .reset = blk_tracer_reset, 15752db270a8SFrederic Weisbecker .start = blk_tracer_start, 15762db270a8SFrederic Weisbecker .stop = blk_tracer_stop, 15772db270a8SFrederic Weisbecker .print_header = blk_tracer_print_header, 15782db270a8SFrederic Weisbecker .print_line = blk_tracer_print_line, 15792db270a8SFrederic Weisbecker .flags = &blk_tracer_flags, 1580f3948f88SLi Zefan .set_flag = blk_tracer_set_flag, 15812db270a8SFrederic Weisbecker }; 15822db270a8SFrederic Weisbecker 1583a9a57763SSteven Rostedt static struct trace_event_functions trace_blk_event_funcs = { 15842db270a8SFrederic Weisbecker .trace = blk_trace_event_print, 15852db270a8SFrederic Weisbecker .binary = blk_trace_event_print_binary, 15862db270a8SFrederic Weisbecker }; 15872db270a8SFrederic Weisbecker 1588a9a57763SSteven Rostedt static struct trace_event trace_blk_event = { 1589a9a57763SSteven Rostedt .type = TRACE_BLK, 1590a9a57763SSteven Rostedt .funcs = &trace_blk_event_funcs, 1591a9a57763SSteven Rostedt }; 1592a9a57763SSteven Rostedt 15932db270a8SFrederic Weisbecker static int __init init_blk_tracer(void) 15942db270a8SFrederic Weisbecker { 15959023c930SSteven Rostedt (Red Hat) if (!register_trace_event(&trace_blk_event)) { 1596a395d6a7SJoe Perches pr_warn("Warning: could not register block events\n"); 15972db270a8SFrederic Weisbecker return 1; 15982db270a8SFrederic Weisbecker } 15992db270a8SFrederic Weisbecker 16002db270a8SFrederic Weisbecker if (register_tracer(&blk_tracer) != 0) { 1601a395d6a7SJoe Perches pr_warn("Warning: could not register the block tracer\n"); 16029023c930SSteven Rostedt (Red Hat) unregister_trace_event(&trace_blk_event); 16032db270a8SFrederic Weisbecker return 1; 16042db270a8SFrederic Weisbecker } 16052db270a8SFrederic Weisbecker 16062db270a8SFrederic Weisbecker return 0; 16072db270a8SFrederic Weisbecker } 16082db270a8SFrederic Weisbecker 16092db270a8SFrederic Weisbecker device_initcall(init_blk_tracer); 16102db270a8SFrederic Weisbecker 16112db270a8SFrederic Weisbecker static int blk_trace_remove_queue(struct request_queue *q) 16122db270a8SFrederic Weisbecker { 16132db270a8SFrederic Weisbecker struct blk_trace *bt; 16142db270a8SFrederic Weisbecker 1615c3dbe541SJan Kara bt = rcu_replace_pointer(q->blk_trace, NULL, 161685e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 16172db270a8SFrederic Weisbecker if (bt == NULL) 16182db270a8SFrederic Weisbecker return -EINVAL; 16192db270a8SFrederic Weisbecker 1620a6da0024SJens Axboe put_probe_ref(); 1621c780e86dSJan Kara synchronize_rcu(); 1622ad5dd549SLi Zefan blk_trace_free(bt); 16232db270a8SFrederic Weisbecker return 0; 16242db270a8SFrederic Weisbecker } 16252db270a8SFrederic Weisbecker 16262db270a8SFrederic Weisbecker /* 16272db270a8SFrederic Weisbecker * Setup everything required to start tracing 16282db270a8SFrederic Weisbecker */ 16299908c309SLi Zefan static int blk_trace_setup_queue(struct request_queue *q, 16309908c309SLi Zefan struct block_device *bdev) 16312db270a8SFrederic Weisbecker { 1632cdea01b2SDavidlohr Bueso struct blk_trace *bt = NULL; 163318cea459SLi Zefan int ret = -ENOMEM; 16342db270a8SFrederic Weisbecker 16352db270a8SFrederic Weisbecker bt = kzalloc(sizeof(*bt), GFP_KERNEL); 16362db270a8SFrederic Weisbecker if (!bt) 163715152e44SLi Zefan return -ENOMEM; 16382db270a8SFrederic Weisbecker 163918cea459SLi Zefan bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 164018cea459SLi Zefan if (!bt->msg_data) 164118cea459SLi Zefan goto free_bt; 164218cea459SLi Zefan 16439908c309SLi Zefan bt->dev = bdev->bd_dev; 16442db270a8SFrederic Weisbecker bt->act_mask = (u16)-1; 16459908c309SLi Zefan 16469908c309SLi Zefan blk_trace_setup_lba(bt, bdev); 16472db270a8SFrederic Weisbecker 1648c3dbe541SJan Kara rcu_assign_pointer(q->blk_trace, bt); 1649a6da0024SJens Axboe get_probe_ref(); 16502db270a8SFrederic Weisbecker return 0; 165118cea459SLi Zefan 165218cea459SLi Zefan free_bt: 165318cea459SLi Zefan blk_trace_free(bt); 165418cea459SLi Zefan return ret; 16552db270a8SFrederic Weisbecker } 16562db270a8SFrederic Weisbecker 16572db270a8SFrederic Weisbecker /* 16582db270a8SFrederic Weisbecker * sysfs interface to enable and configure tracing 16592db270a8SFrederic Weisbecker */ 16602db270a8SFrederic Weisbecker 16612db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 16622db270a8SFrederic Weisbecker struct device_attribute *attr, 16632db270a8SFrederic Weisbecker char *buf); 16642db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 16652db270a8SFrederic Weisbecker struct device_attribute *attr, 16662db270a8SFrederic Weisbecker const char *buf, size_t count); 16672db270a8SFrederic Weisbecker #define BLK_TRACE_DEVICE_ATTR(_name) \ 16682db270a8SFrederic Weisbecker DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \ 16692db270a8SFrederic Weisbecker sysfs_blk_trace_attr_show, \ 16702db270a8SFrederic Weisbecker sysfs_blk_trace_attr_store) 16712db270a8SFrederic Weisbecker 1672cd649b8bSLi Zefan static BLK_TRACE_DEVICE_ATTR(enable); 16732db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(act_mask); 16742db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(pid); 16752db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(start_lba); 16762db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(end_lba); 16772db270a8SFrederic Weisbecker 16782db270a8SFrederic Weisbecker static struct attribute *blk_trace_attrs[] = { 16792db270a8SFrederic Weisbecker &dev_attr_enable.attr, 16802db270a8SFrederic Weisbecker &dev_attr_act_mask.attr, 16812db270a8SFrederic Weisbecker &dev_attr_pid.attr, 16822db270a8SFrederic Weisbecker &dev_attr_start_lba.attr, 16832db270a8SFrederic Weisbecker &dev_attr_end_lba.attr, 16842db270a8SFrederic Weisbecker NULL 16852db270a8SFrederic Weisbecker }; 16862db270a8SFrederic Weisbecker 16872db270a8SFrederic Weisbecker struct attribute_group blk_trace_attr_group = { 16882db270a8SFrederic Weisbecker .name = "trace", 16892db270a8SFrederic Weisbecker .attrs = blk_trace_attrs, 16902db270a8SFrederic Weisbecker }; 16912db270a8SFrederic Weisbecker 169209341997SLi Zefan static const struct { 169309341997SLi Zefan int mask; 169409341997SLi Zefan const char *str; 169509341997SLi Zefan } mask_maps[] = { 169609341997SLi Zefan { BLK_TC_READ, "read" }, 169709341997SLi Zefan { BLK_TC_WRITE, "write" }, 1698c09c47caSNamhyung Kim { BLK_TC_FLUSH, "flush" }, 169909341997SLi Zefan { BLK_TC_SYNC, "sync" }, 170009341997SLi Zefan { BLK_TC_QUEUE, "queue" }, 170109341997SLi Zefan { BLK_TC_REQUEUE, "requeue" }, 170209341997SLi Zefan { BLK_TC_ISSUE, "issue" }, 170309341997SLi Zefan { BLK_TC_COMPLETE, "complete" }, 170409341997SLi Zefan { BLK_TC_FS, "fs" }, 170509341997SLi Zefan { BLK_TC_PC, "pc" }, 17068d1547e0SShaohua Li { BLK_TC_NOTIFY, "notify" }, 170709341997SLi Zefan { BLK_TC_AHEAD, "ahead" }, 170809341997SLi Zefan { BLK_TC_META, "meta" }, 170909341997SLi Zefan { BLK_TC_DISCARD, "discard" }, 171009341997SLi Zefan { BLK_TC_DRV_DATA, "drv_data" }, 1711c09c47caSNamhyung Kim { BLK_TC_FUA, "fua" }, 171209341997SLi Zefan }; 171309341997SLi Zefan 171409341997SLi Zefan static int blk_trace_str2mask(const char *str) 17152db270a8SFrederic Weisbecker { 171609341997SLi Zefan int i; 17172db270a8SFrederic Weisbecker int mask = 0; 17189eb85125SLi Zefan char *buf, *s, *token; 17192db270a8SFrederic Weisbecker 17209eb85125SLi Zefan buf = kstrdup(str, GFP_KERNEL); 17219eb85125SLi Zefan if (buf == NULL) 17222db270a8SFrederic Weisbecker return -ENOMEM; 17239eb85125SLi Zefan s = strstrip(buf); 17242db270a8SFrederic Weisbecker 17252db270a8SFrederic Weisbecker while (1) { 172609341997SLi Zefan token = strsep(&s, ","); 172709341997SLi Zefan if (token == NULL) 17282db270a8SFrederic Weisbecker break; 17292db270a8SFrederic Weisbecker 173009341997SLi Zefan if (*token == '\0') 173109341997SLi Zefan continue; 173209341997SLi Zefan 173309341997SLi Zefan for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 173409341997SLi Zefan if (strcasecmp(token, mask_maps[i].str) == 0) { 173509341997SLi Zefan mask |= mask_maps[i].mask; 173609341997SLi Zefan break; 17372db270a8SFrederic Weisbecker } 173809341997SLi Zefan } 173909341997SLi Zefan if (i == ARRAY_SIZE(mask_maps)) { 174009341997SLi Zefan mask = -EINVAL; 174109341997SLi Zefan break; 174209341997SLi Zefan } 174309341997SLi Zefan } 17449eb85125SLi Zefan kfree(buf); 17452db270a8SFrederic Weisbecker 17462db270a8SFrederic Weisbecker return mask; 17472db270a8SFrederic Weisbecker } 17482db270a8SFrederic Weisbecker 174909341997SLi Zefan static ssize_t blk_trace_mask2str(char *buf, int mask) 175009341997SLi Zefan { 175109341997SLi Zefan int i; 175209341997SLi Zefan char *p = buf; 175309341997SLi Zefan 175409341997SLi Zefan for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 175509341997SLi Zefan if (mask & mask_maps[i].mask) { 175609341997SLi Zefan p += sprintf(p, "%s%s", 175709341997SLi Zefan (p == buf) ? "" : ",", mask_maps[i].str); 175809341997SLi Zefan } 175909341997SLi Zefan } 176009341997SLi Zefan *p++ = '\n'; 176109341997SLi Zefan 176209341997SLi Zefan return p - buf; 176309341997SLi Zefan } 176409341997SLi Zefan 17652db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 17662db270a8SFrederic Weisbecker struct device_attribute *attr, 17672db270a8SFrederic Weisbecker char *buf) 17682db270a8SFrederic Weisbecker { 17690d02129eSChristoph Hellwig struct block_device *bdev = dev_to_bdev(dev); 17700d02129eSChristoph Hellwig struct request_queue *q = bdev_get_queue(bdev); 1771c780e86dSJan Kara struct blk_trace *bt; 17722db270a8SFrederic Weisbecker ssize_t ret = -ENXIO; 17732db270a8SFrederic Weisbecker 177485e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 1775cd649b8bSLi Zefan 1776c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 177785e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1778cd649b8bSLi Zefan if (attr == &dev_attr_enable) { 1779c780e86dSJan Kara ret = sprintf(buf, "%u\n", !!bt); 1780cd649b8bSLi Zefan goto out_unlock_bdev; 1781cd649b8bSLi Zefan } 1782cd649b8bSLi Zefan 1783c780e86dSJan Kara if (bt == NULL) 17842db270a8SFrederic Weisbecker ret = sprintf(buf, "disabled\n"); 17852db270a8SFrederic Weisbecker else if (attr == &dev_attr_act_mask) 1786c780e86dSJan Kara ret = blk_trace_mask2str(buf, bt->act_mask); 17872db270a8SFrederic Weisbecker else if (attr == &dev_attr_pid) 1788c780e86dSJan Kara ret = sprintf(buf, "%u\n", bt->pid); 17892db270a8SFrederic Weisbecker else if (attr == &dev_attr_start_lba) 1790c780e86dSJan Kara ret = sprintf(buf, "%llu\n", bt->start_lba); 17912db270a8SFrederic Weisbecker else if (attr == &dev_attr_end_lba) 1792c780e86dSJan Kara ret = sprintf(buf, "%llu\n", bt->end_lba); 1793cd649b8bSLi Zefan 1794cd649b8bSLi Zefan out_unlock_bdev: 179585e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 17962db270a8SFrederic Weisbecker return ret; 17972db270a8SFrederic Weisbecker } 17982db270a8SFrederic Weisbecker 17992db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 18002db270a8SFrederic Weisbecker struct device_attribute *attr, 18012db270a8SFrederic Weisbecker const char *buf, size_t count) 18022db270a8SFrederic Weisbecker { 18030d02129eSChristoph Hellwig struct block_device *bdev = dev_to_bdev(dev); 18040d02129eSChristoph Hellwig struct request_queue *q = bdev_get_queue(bdev); 1805c780e86dSJan Kara struct blk_trace *bt; 18062db270a8SFrederic Weisbecker u64 value; 180709341997SLi Zefan ssize_t ret = -EINVAL; 18082db270a8SFrederic Weisbecker 18092db270a8SFrederic Weisbecker if (count == 0) 18102db270a8SFrederic Weisbecker goto out; 18112db270a8SFrederic Weisbecker 18122db270a8SFrederic Weisbecker if (attr == &dev_attr_act_mask) { 18135f339453SShaohua Li if (kstrtoull(buf, 0, &value)) { 18142db270a8SFrederic Weisbecker /* Assume it is a list of trace category names */ 181509341997SLi Zefan ret = blk_trace_str2mask(buf); 181609341997SLi Zefan if (ret < 0) 18172db270a8SFrederic Weisbecker goto out; 181809341997SLi Zefan value = ret; 18192db270a8SFrederic Weisbecker } 18200d02129eSChristoph Hellwig } else { 18210d02129eSChristoph Hellwig if (kstrtoull(buf, 0, &value)) 18222db270a8SFrederic Weisbecker goto out; 18230d02129eSChristoph Hellwig } 18242db270a8SFrederic Weisbecker 182585e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 1826cd649b8bSLi Zefan 1827c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 182885e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1829cd649b8bSLi Zefan if (attr == &dev_attr_enable) { 1830c780e86dSJan Kara if (!!value == !!bt) { 1831757d9140SSteven Rostedt (VMware) ret = 0; 1832757d9140SSteven Rostedt (VMware) goto out_unlock_bdev; 1833757d9140SSteven Rostedt (VMware) } 1834cd649b8bSLi Zefan if (value) 18359908c309SLi Zefan ret = blk_trace_setup_queue(q, bdev); 1836cd649b8bSLi Zefan else 1837cd649b8bSLi Zefan ret = blk_trace_remove_queue(q); 1838cd649b8bSLi Zefan goto out_unlock_bdev; 1839cd649b8bSLi Zefan } 1840cd649b8bSLi Zefan 18412db270a8SFrederic Weisbecker ret = 0; 1842153031a3SCengiz Can if (bt == NULL) { 18439908c309SLi Zefan ret = blk_trace_setup_queue(q, bdev); 1844153031a3SCengiz Can bt = rcu_dereference_protected(q->blk_trace, 184585e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1846153031a3SCengiz Can } 18472db270a8SFrederic Weisbecker 18482db270a8SFrederic Weisbecker if (ret == 0) { 18492db270a8SFrederic Weisbecker if (attr == &dev_attr_act_mask) 1850c780e86dSJan Kara bt->act_mask = value; 18512db270a8SFrederic Weisbecker else if (attr == &dev_attr_pid) 1852c780e86dSJan Kara bt->pid = value; 18532db270a8SFrederic Weisbecker else if (attr == &dev_attr_start_lba) 1854c780e86dSJan Kara bt->start_lba = value; 18552db270a8SFrederic Weisbecker else if (attr == &dev_attr_end_lba) 1856c780e86dSJan Kara bt->end_lba = value; 18572db270a8SFrederic Weisbecker } 1858cd649b8bSLi Zefan 1859cd649b8bSLi Zefan out_unlock_bdev: 186085e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 18612db270a8SFrederic Weisbecker out: 1862cd649b8bSLi Zefan return ret ? ret : count; 18632db270a8SFrederic Weisbecker } 1864cd649b8bSLi Zefan 18651d54ad6dSLi Zefan int blk_trace_init_sysfs(struct device *dev) 18661d54ad6dSLi Zefan { 18671d54ad6dSLi Zefan return sysfs_create_group(&dev->kobj, &blk_trace_attr_group); 18681d54ad6dSLi Zefan } 18691d54ad6dSLi Zefan 187048c0d4d4SZdenek Kabelac void blk_trace_remove_sysfs(struct device *dev) 187148c0d4d4SZdenek Kabelac { 187248c0d4d4SZdenek Kabelac sysfs_remove_group(&dev->kobj, &blk_trace_attr_group); 187348c0d4d4SZdenek Kabelac } 187448c0d4d4SZdenek Kabelac 187555782138SLi Zefan #endif /* CONFIG_BLK_DEV_IO_TRACE */ 187655782138SLi Zefan 187755782138SLi Zefan #ifdef CONFIG_EVENT_TRACING 187855782138SLi Zefan 1879ef295ecfSChristoph Hellwig void blk_fill_rwbs(char *rwbs, unsigned int op, int bytes) 188055782138SLi Zefan { 188155782138SLi Zefan int i = 0; 188255782138SLi Zefan 1883ef295ecfSChristoph Hellwig if (op & REQ_PREFLUSH) 1884c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1885c09c47caSNamhyung Kim 1886ef295ecfSChristoph Hellwig switch (op & REQ_OP_MASK) { 18871b9a9ab7SMike Christie case REQ_OP_WRITE: 18881b9a9ab7SMike Christie case REQ_OP_WRITE_SAME: 188955782138SLi Zefan rwbs[i++] = 'W'; 18901b9a9ab7SMike Christie break; 18911b9a9ab7SMike Christie case REQ_OP_DISCARD: 189255782138SLi Zefan rwbs[i++] = 'D'; 18931b9a9ab7SMike Christie break; 1894288dab8aSChristoph Hellwig case REQ_OP_SECURE_ERASE: 1895288dab8aSChristoph Hellwig rwbs[i++] = 'D'; 1896288dab8aSChristoph Hellwig rwbs[i++] = 'E'; 1897288dab8aSChristoph Hellwig break; 18983a5e02ceSMike Christie case REQ_OP_FLUSH: 18993a5e02ceSMike Christie rwbs[i++] = 'F'; 19003a5e02ceSMike Christie break; 19011b9a9ab7SMike Christie case REQ_OP_READ: 190255782138SLi Zefan rwbs[i++] = 'R'; 19031b9a9ab7SMike Christie break; 19041b9a9ab7SMike Christie default: 190555782138SLi Zefan rwbs[i++] = 'N'; 19061b9a9ab7SMike Christie } 190755782138SLi Zefan 1908ef295ecfSChristoph Hellwig if (op & REQ_FUA) 1909c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1910ef295ecfSChristoph Hellwig if (op & REQ_RAHEAD) 191155782138SLi Zefan rwbs[i++] = 'A'; 1912ef295ecfSChristoph Hellwig if (op & REQ_SYNC) 191355782138SLi Zefan rwbs[i++] = 'S'; 1914ef295ecfSChristoph Hellwig if (op & REQ_META) 191555782138SLi Zefan rwbs[i++] = 'M'; 191655782138SLi Zefan 191755782138SLi Zefan rwbs[i] = '\0'; 191855782138SLi Zefan } 19199ca8f8e5SKent Overstreet EXPORT_SYMBOL_GPL(blk_fill_rwbs); 192055782138SLi Zefan 192155782138SLi Zefan #endif /* CONFIG_EVENT_TRACING */ 192255782138SLi Zefan 1923