191c1e6baSSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0 22db270a8SFrederic Weisbecker /* 32db270a8SFrederic Weisbecker * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk> 42db270a8SFrederic Weisbecker * 52db270a8SFrederic Weisbecker */ 61b0b2836SLuis Chamberlain 71b0b2836SLuis Chamberlain #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 81b0b2836SLuis Chamberlain 92db270a8SFrederic Weisbecker #include <linux/kernel.h> 102db270a8SFrederic Weisbecker #include <linux/blkdev.h> 112db270a8SFrederic Weisbecker #include <linux/blktrace_api.h> 122db270a8SFrederic Weisbecker #include <linux/percpu.h> 132db270a8SFrederic Weisbecker #include <linux/init.h> 142db270a8SFrederic Weisbecker #include <linux/mutex.h> 155a0e3ad6STejun Heo #include <linux/slab.h> 162db270a8SFrederic Weisbecker #include <linux/debugfs.h> 176e5fdeedSPaul Gortmaker #include <linux/export.h> 182db270a8SFrederic Weisbecker #include <linux/time.h> 192db270a8SFrederic Weisbecker #include <linux/uaccess.h> 20a404d557SJan Kara #include <linux/list.h> 21ca1136c9SShaohua Li #include <linux/blk-cgroup.h> 2255782138SLi Zefan 2318fbda91SOmar Sandoval #include "../../block/blk.h" 2418fbda91SOmar Sandoval 2555782138SLi Zefan #include <trace/events/block.h> 2655782138SLi Zefan 272db270a8SFrederic Weisbecker #include "trace_output.h" 282db270a8SFrederic Weisbecker 2955782138SLi Zefan #ifdef CONFIG_BLK_DEV_IO_TRACE 3055782138SLi Zefan 312db270a8SFrederic Weisbecker static unsigned int blktrace_seq __read_mostly = 1; 322db270a8SFrederic Weisbecker 332db270a8SFrederic Weisbecker static struct trace_array *blk_tr; 345006ea73SLi Zefan static bool blk_tracer_enabled __read_mostly; 352db270a8SFrederic Weisbecker 36a404d557SJan Kara static LIST_HEAD(running_trace_list); 37361c81dbSWander Lairson Costa static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(running_trace_lock); 38a404d557SJan Kara 392db270a8SFrederic Weisbecker /* Select an alternative, minimalistic output than the original one */ 402db270a8SFrederic Weisbecker #define TRACE_BLK_OPT_CLASSIC 0x1 41ca1136c9SShaohua Li #define TRACE_BLK_OPT_CGROUP 0x2 4269fd5c39SShaohua Li #define TRACE_BLK_OPT_CGNAME 0x4 432db270a8SFrederic Weisbecker 442db270a8SFrederic Weisbecker static struct tracer_opt blk_tracer_opts[] = { 452db270a8SFrederic Weisbecker /* Default disable the minimalistic output */ 462db270a8SFrederic Weisbecker { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) }, 47ca1136c9SShaohua Li #ifdef CONFIG_BLK_CGROUP 48ca1136c9SShaohua Li { TRACER_OPT(blk_cgroup, TRACE_BLK_OPT_CGROUP) }, 4969fd5c39SShaohua Li { TRACER_OPT(blk_cgname, TRACE_BLK_OPT_CGNAME) }, 50ca1136c9SShaohua Li #endif 512db270a8SFrederic Weisbecker { } 522db270a8SFrederic Weisbecker }; 532db270a8SFrederic Weisbecker 542db270a8SFrederic Weisbecker static struct tracer_flags blk_tracer_flags = { 552db270a8SFrederic Weisbecker .val = 0, 562db270a8SFrederic Weisbecker .opts = blk_tracer_opts, 572db270a8SFrederic Weisbecker }; 582db270a8SFrederic Weisbecker 592db270a8SFrederic Weisbecker /* Global reference count of probes */ 60a6da0024SJens Axboe static DEFINE_MUTEX(blk_probe_mutex); 61a6da0024SJens Axboe static int blk_probes_ref; 622db270a8SFrederic Weisbecker 633c289ba7SLi Zefan static void blk_register_tracepoints(void); 642db270a8SFrederic Weisbecker static void blk_unregister_tracepoints(void); 652db270a8SFrederic Weisbecker 662db270a8SFrederic Weisbecker /* 672db270a8SFrederic Weisbecker * Send out a notify message. 682db270a8SFrederic Weisbecker */ 692db270a8SFrederic Weisbecker static void trace_note(struct blk_trace *bt, pid_t pid, int action, 7067c0496eSTejun Heo const void *data, size_t len, u64 cgid) 712db270a8SFrederic Weisbecker { 722db270a8SFrederic Weisbecker struct blk_io_trace *t; 7318cea459SLi Zefan struct ring_buffer_event *event = NULL; 7413292494SSteven Rostedt (VMware) struct trace_buffer *buffer = NULL; 7536590c50SSebastian Andrzej Siewior unsigned int trace_ctx = 0; 7618cea459SLi Zefan int cpu = smp_processor_id(); 7718cea459SLi Zefan bool blk_tracer = blk_tracer_enabled; 7867c0496eSTejun Heo ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 7918cea459SLi Zefan 8018cea459SLi Zefan if (blk_tracer) { 811c5eb448SSteven Rostedt (VMware) buffer = blk_tr->array_buffer.buffer; 8236590c50SSebastian Andrzej Siewior trace_ctx = tracing_gen_ctx_flags(0); 83e77405adSSteven Rostedt event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 84ca1136c9SShaohua Li sizeof(*t) + len + cgid_len, 8536590c50SSebastian Andrzej Siewior trace_ctx); 8618cea459SLi Zefan if (!event) 8718cea459SLi Zefan return; 8818cea459SLi Zefan t = ring_buffer_event_data(event); 8918cea459SLi Zefan goto record_it; 9018cea459SLi Zefan } 912db270a8SFrederic Weisbecker 922db270a8SFrederic Weisbecker if (!bt->rchan) 932db270a8SFrederic Weisbecker return; 942db270a8SFrederic Weisbecker 95ca1136c9SShaohua Li t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len); 962db270a8SFrederic Weisbecker if (t) { 972db270a8SFrederic Weisbecker t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 982db270a8SFrederic Weisbecker t->time = ktime_to_ns(ktime_get()); 9918cea459SLi Zefan record_it: 1002db270a8SFrederic Weisbecker t->device = bt->dev; 101ca1136c9SShaohua Li t->action = action | (cgid ? __BLK_TN_CGROUP : 0); 1022db270a8SFrederic Weisbecker t->pid = pid; 1032db270a8SFrederic Weisbecker t->cpu = cpu; 104ca1136c9SShaohua Li t->pdu_len = len + cgid_len; 10567c0496eSTejun Heo if (cgid_len) 10667c0496eSTejun Heo memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 107ca1136c9SShaohua Li memcpy((void *) t + sizeof(*t) + cgid_len, data, len); 10818cea459SLi Zefan 10918cea459SLi Zefan if (blk_tracer) 11036590c50SSebastian Andrzej Siewior trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx); 1112db270a8SFrederic Weisbecker } 1122db270a8SFrederic Weisbecker } 1132db270a8SFrederic Weisbecker 1142db270a8SFrederic Weisbecker /* 1152db270a8SFrederic Weisbecker * Send out a notify for this process, if we haven't done so since a trace 1162db270a8SFrederic Weisbecker * started 1172db270a8SFrederic Weisbecker */ 118a404d557SJan Kara static void trace_note_tsk(struct task_struct *tsk) 1192db270a8SFrederic Weisbecker { 120a404d557SJan Kara unsigned long flags; 121a404d557SJan Kara struct blk_trace *bt; 122a404d557SJan Kara 1232db270a8SFrederic Weisbecker tsk->btrace_seq = blktrace_seq; 124361c81dbSWander Lairson Costa raw_spin_lock_irqsave(&running_trace_lock, flags); 125a404d557SJan Kara list_for_each_entry(bt, &running_trace_list, running_list) { 126a404d557SJan Kara trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, 12767c0496eSTejun Heo sizeof(tsk->comm), 0); 128a404d557SJan Kara } 129361c81dbSWander Lairson Costa raw_spin_unlock_irqrestore(&running_trace_lock, flags); 1302db270a8SFrederic Weisbecker } 1312db270a8SFrederic Weisbecker 1322db270a8SFrederic Weisbecker static void trace_note_time(struct blk_trace *bt) 1332db270a8SFrederic Weisbecker { 13459a37f8bSArnd Bergmann struct timespec64 now; 1352db270a8SFrederic Weisbecker unsigned long flags; 1362db270a8SFrederic Weisbecker u32 words[2]; 1372db270a8SFrederic Weisbecker 13859a37f8bSArnd Bergmann /* need to check user space to see if this breaks in y2038 or y2106 */ 13959a37f8bSArnd Bergmann ktime_get_real_ts64(&now); 14059a37f8bSArnd Bergmann words[0] = (u32)now.tv_sec; 1412db270a8SFrederic Weisbecker words[1] = now.tv_nsec; 1422db270a8SFrederic Weisbecker 1432db270a8SFrederic Weisbecker local_irq_save(flags); 14467c0496eSTejun Heo trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0); 1452db270a8SFrederic Weisbecker local_irq_restore(flags); 1462db270a8SFrederic Weisbecker } 1472db270a8SFrederic Weisbecker 14835fe6d76SShaohua Li void __trace_note_message(struct blk_trace *bt, struct blkcg *blkcg, 14935fe6d76SShaohua Li const char *fmt, ...) 1502db270a8SFrederic Weisbecker { 1512db270a8SFrederic Weisbecker int n; 1522db270a8SFrederic Weisbecker va_list args; 1532db270a8SFrederic Weisbecker unsigned long flags; 1542db270a8SFrederic Weisbecker char *buf; 1552db270a8SFrederic Weisbecker 15618cea459SLi Zefan if (unlikely(bt->trace_state != Blktrace_running && 15718cea459SLi Zefan !blk_tracer_enabled)) 1582db270a8SFrederic Weisbecker return; 1592db270a8SFrederic Weisbecker 160490da40dSTao Ma /* 161490da40dSTao Ma * If the BLK_TC_NOTIFY action mask isn't set, don't send any note 162490da40dSTao Ma * message to the trace. 163490da40dSTao Ma */ 164490da40dSTao Ma if (!(bt->act_mask & BLK_TC_NOTIFY)) 165490da40dSTao Ma return; 166490da40dSTao Ma 1672db270a8SFrederic Weisbecker local_irq_save(flags); 168d8a0349cSShan Wei buf = this_cpu_ptr(bt->msg_data); 1692db270a8SFrederic Weisbecker va_start(args, fmt); 1702db270a8SFrederic Weisbecker n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args); 1712db270a8SFrederic Weisbecker va_end(args); 1722db270a8SFrederic Weisbecker 17335fe6d76SShaohua Li if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 17435fe6d76SShaohua Li blkcg = NULL; 17535fe6d76SShaohua Li #ifdef CONFIG_BLK_CGROUP 176870c153cSJan Kara trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, 17774321038STejun Heo blkcg ? cgroup_id(blkcg->css.cgroup) : 1); 17835fe6d76SShaohua Li #else 179870c153cSJan Kara trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, 0); 18035fe6d76SShaohua Li #endif 1812db270a8SFrederic Weisbecker local_irq_restore(flags); 1822db270a8SFrederic Weisbecker } 1832db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(__trace_note_message); 1842db270a8SFrederic Weisbecker 1852db270a8SFrederic Weisbecker static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector, 1862db270a8SFrederic Weisbecker pid_t pid) 1872db270a8SFrederic Weisbecker { 1882db270a8SFrederic Weisbecker if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0) 1892db270a8SFrederic Weisbecker return 1; 190d0deef5bSShawn Du if (sector && (sector < bt->start_lba || sector > bt->end_lba)) 1912db270a8SFrederic Weisbecker return 1; 1922db270a8SFrederic Weisbecker if (bt->pid && pid != bt->pid) 1932db270a8SFrederic Weisbecker return 1; 1942db270a8SFrederic Weisbecker 1952db270a8SFrederic Weisbecker return 0; 1962db270a8SFrederic Weisbecker } 1972db270a8SFrederic Weisbecker 1982db270a8SFrederic Weisbecker /* 1992db270a8SFrederic Weisbecker * Data direction bit lookup 2002db270a8SFrederic Weisbecker */ 201e4955c99SLi Zefan static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), 2022db270a8SFrederic Weisbecker BLK_TC_ACT(BLK_TC_WRITE) }; 2032db270a8SFrederic Weisbecker 2047b6d91daSChristoph Hellwig #define BLK_TC_RAHEAD BLK_TC_AHEAD 20528a8f0d3SMike Christie #define BLK_TC_PREFLUSH BLK_TC_FLUSH 2067b6d91daSChristoph Hellwig 2072db270a8SFrederic Weisbecker /* The ilog2() calls fall out because they're constant */ 2087b6d91daSChristoph Hellwig #define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \ 2097b6d91daSChristoph Hellwig (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name)) 2102db270a8SFrederic Weisbecker 2112db270a8SFrederic Weisbecker /* 2122db270a8SFrederic Weisbecker * The worker for the various blk_add_trace*() types. Fills out a 2132db270a8SFrederic Weisbecker * blk_io_trace structure and places it in a per-cpu subbuffer. 2142db270a8SFrederic Weisbecker */ 2152db270a8SFrederic Weisbecker static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, 2161b9a9ab7SMike Christie int op, int op_flags, u32 what, int error, int pdu_len, 21767c0496eSTejun Heo void *pdu_data, u64 cgid) 2182db270a8SFrederic Weisbecker { 2192db270a8SFrederic Weisbecker struct task_struct *tsk = current; 2202db270a8SFrederic Weisbecker struct ring_buffer_event *event = NULL; 22113292494SSteven Rostedt (VMware) struct trace_buffer *buffer = NULL; 2222db270a8SFrederic Weisbecker struct blk_io_trace *t; 2232db270a8SFrederic Weisbecker unsigned long flags = 0; 2242db270a8SFrederic Weisbecker unsigned long *sequence; 22536590c50SSebastian Andrzej Siewior unsigned int trace_ctx = 0; 2262db270a8SFrederic Weisbecker pid_t pid; 22736590c50SSebastian Andrzej Siewior int cpu; 22818cea459SLi Zefan bool blk_tracer = blk_tracer_enabled; 22967c0496eSTejun Heo ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 2302db270a8SFrederic Weisbecker 23118cea459SLi Zefan if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer)) 2322db270a8SFrederic Weisbecker return; 2332db270a8SFrederic Weisbecker 2341b9a9ab7SMike Christie what |= ddir_act[op_is_write(op) ? WRITE : READ]; 2351b9a9ab7SMike Christie what |= MASK_TC_BIT(op_flags, SYNC); 2361b9a9ab7SMike Christie what |= MASK_TC_BIT(op_flags, RAHEAD); 2371b9a9ab7SMike Christie what |= MASK_TC_BIT(op_flags, META); 23828a8f0d3SMike Christie what |= MASK_TC_BIT(op_flags, PREFLUSH); 2391b9a9ab7SMike Christie what |= MASK_TC_BIT(op_flags, FUA); 2407afafc8aSAdrian Hunter if (op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE) 2411b9a9ab7SMike Christie what |= BLK_TC_ACT(BLK_TC_DISCARD); 2423a5e02ceSMike Christie if (op == REQ_OP_FLUSH) 2433a5e02ceSMike Christie what |= BLK_TC_ACT(BLK_TC_FLUSH); 244ca1136c9SShaohua Li if (cgid) 245ca1136c9SShaohua Li what |= __BLK_TA_CGROUP; 2462db270a8SFrederic Weisbecker 2472db270a8SFrederic Weisbecker pid = tsk->pid; 248d0deef5bSShawn Du if (act_log_check(bt, what, sector, pid)) 2492db270a8SFrederic Weisbecker return; 2502db270a8SFrederic Weisbecker cpu = raw_smp_processor_id(); 2512db270a8SFrederic Weisbecker 25218cea459SLi Zefan if (blk_tracer) { 2532db270a8SFrederic Weisbecker tracing_record_cmdline(current); 2542db270a8SFrederic Weisbecker 2551c5eb448SSteven Rostedt (VMware) buffer = blk_tr->array_buffer.buffer; 25636590c50SSebastian Andrzej Siewior trace_ctx = tracing_gen_ctx_flags(0); 257e77405adSSteven Rostedt event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 258ca1136c9SShaohua Li sizeof(*t) + pdu_len + cgid_len, 25936590c50SSebastian Andrzej Siewior trace_ctx); 2602db270a8SFrederic Weisbecker if (!event) 2612db270a8SFrederic Weisbecker return; 2622db270a8SFrederic Weisbecker t = ring_buffer_event_data(event); 2632db270a8SFrederic Weisbecker goto record_it; 2642db270a8SFrederic Weisbecker } 2652db270a8SFrederic Weisbecker 266a404d557SJan Kara if (unlikely(tsk->btrace_seq != blktrace_seq)) 267a404d557SJan Kara trace_note_tsk(tsk); 268a404d557SJan Kara 2692db270a8SFrederic Weisbecker /* 2702db270a8SFrederic Weisbecker * A word about the locking here - we disable interrupts to reserve 2712db270a8SFrederic Weisbecker * some space in the relay per-cpu buffer, to prevent an irq 2722db270a8SFrederic Weisbecker * from coming in and stepping on our toes. 2732db270a8SFrederic Weisbecker */ 2742db270a8SFrederic Weisbecker local_irq_save(flags); 275ca1136c9SShaohua Li t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len); 2762db270a8SFrederic Weisbecker if (t) { 2772db270a8SFrederic Weisbecker sequence = per_cpu_ptr(bt->sequence, cpu); 2782db270a8SFrederic Weisbecker 2792db270a8SFrederic Weisbecker t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 2802db270a8SFrederic Weisbecker t->sequence = ++(*sequence); 2812db270a8SFrederic Weisbecker t->time = ktime_to_ns(ktime_get()); 2822db270a8SFrederic Weisbecker record_it: 2832db270a8SFrederic Weisbecker /* 2842db270a8SFrederic Weisbecker * These two are not needed in ftrace as they are in the 2852db270a8SFrederic Weisbecker * generic trace_entry, filled by tracing_generic_entry_update, 2862db270a8SFrederic Weisbecker * but for the trace_event->bin() synthesizer benefit we do it 2872db270a8SFrederic Weisbecker * here too. 2882db270a8SFrederic Weisbecker */ 2892db270a8SFrederic Weisbecker t->cpu = cpu; 2902db270a8SFrederic Weisbecker t->pid = pid; 2912db270a8SFrederic Weisbecker 2922db270a8SFrederic Weisbecker t->sector = sector; 2932db270a8SFrederic Weisbecker t->bytes = bytes; 2942db270a8SFrederic Weisbecker t->action = what; 2952db270a8SFrederic Weisbecker t->device = bt->dev; 2962db270a8SFrederic Weisbecker t->error = error; 297ca1136c9SShaohua Li t->pdu_len = pdu_len + cgid_len; 2982db270a8SFrederic Weisbecker 299ca1136c9SShaohua Li if (cgid_len) 30067c0496eSTejun Heo memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 3012db270a8SFrederic Weisbecker if (pdu_len) 302ca1136c9SShaohua Li memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len); 3032db270a8SFrederic Weisbecker 30418cea459SLi Zefan if (blk_tracer) { 30536590c50SSebastian Andrzej Siewior trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx); 3062db270a8SFrederic Weisbecker return; 3072db270a8SFrederic Weisbecker } 3082db270a8SFrederic Weisbecker } 3092db270a8SFrederic Weisbecker 3102db270a8SFrederic Weisbecker local_irq_restore(flags); 3112db270a8SFrederic Weisbecker } 3122db270a8SFrederic Weisbecker 313*30939293SYu Kuai static void blk_trace_free(struct request_queue *q, struct blk_trace *bt) 3142db270a8SFrederic Weisbecker { 3152db270a8SFrederic Weisbecker relay_close(bt->rchan); 316*30939293SYu Kuai 317*30939293SYu Kuai /* 318*30939293SYu Kuai * If 'bt->dir' is not set, then both 'dropped' and 'msg' are created 319*30939293SYu Kuai * under 'q->debugfs_dir', thus lookup and remove them. 320*30939293SYu Kuai */ 321*30939293SYu Kuai if (!bt->dir) { 322*30939293SYu Kuai debugfs_remove(debugfs_lookup("dropped", q->debugfs_dir)); 323*30939293SYu Kuai debugfs_remove(debugfs_lookup("msg", q->debugfs_dir)); 324*30939293SYu Kuai } else { 32539cbb602SAlan D. Brunelle debugfs_remove(bt->dir); 326*30939293SYu Kuai } 3272db270a8SFrederic Weisbecker free_percpu(bt->sequence); 3282db270a8SFrederic Weisbecker free_percpu(bt->msg_data); 3292db270a8SFrederic Weisbecker kfree(bt); 330ad5dd549SLi Zefan } 331ad5dd549SLi Zefan 332a6da0024SJens Axboe static void get_probe_ref(void) 333a6da0024SJens Axboe { 334a6da0024SJens Axboe mutex_lock(&blk_probe_mutex); 335a6da0024SJens Axboe if (++blk_probes_ref == 1) 336a6da0024SJens Axboe blk_register_tracepoints(); 337a6da0024SJens Axboe mutex_unlock(&blk_probe_mutex); 338a6da0024SJens Axboe } 339a6da0024SJens Axboe 340a6da0024SJens Axboe static void put_probe_ref(void) 341a6da0024SJens Axboe { 342a6da0024SJens Axboe mutex_lock(&blk_probe_mutex); 343a6da0024SJens Axboe if (!--blk_probes_ref) 344a6da0024SJens Axboe blk_unregister_tracepoints(); 345a6da0024SJens Axboe mutex_unlock(&blk_probe_mutex); 346a6da0024SJens Axboe } 347a6da0024SJens Axboe 348*30939293SYu Kuai static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt) 349ad5dd549SLi Zefan { 350c780e86dSJan Kara synchronize_rcu(); 351*30939293SYu Kuai blk_trace_free(q, bt); 352a6da0024SJens Axboe put_probe_ref(); 3532db270a8SFrederic Weisbecker } 3542db270a8SFrederic Weisbecker 3551f2cac10SJens Axboe static int __blk_trace_remove(struct request_queue *q) 3562db270a8SFrederic Weisbecker { 3572db270a8SFrederic Weisbecker struct blk_trace *bt; 3582db270a8SFrederic Weisbecker 359c3dbe541SJan Kara bt = rcu_replace_pointer(q->blk_trace, NULL, 36085e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 3612db270a8SFrederic Weisbecker if (!bt) 3622db270a8SFrederic Weisbecker return -EINVAL; 3632db270a8SFrederic Weisbecker 36455547204SLi Zefan if (bt->trace_state != Blktrace_running) 365*30939293SYu Kuai blk_trace_cleanup(q, bt); 3662db270a8SFrederic Weisbecker 3672db270a8SFrederic Weisbecker return 0; 3682db270a8SFrederic Weisbecker } 3691f2cac10SJens Axboe 3701f2cac10SJens Axboe int blk_trace_remove(struct request_queue *q) 3711f2cac10SJens Axboe { 3721f2cac10SJens Axboe int ret; 3731f2cac10SJens Axboe 37485e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 3751f2cac10SJens Axboe ret = __blk_trace_remove(q); 37685e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 3771f2cac10SJens Axboe 3781f2cac10SJens Axboe return ret; 3791f2cac10SJens Axboe } 3802db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_remove); 3812db270a8SFrederic Weisbecker 3822db270a8SFrederic Weisbecker static ssize_t blk_dropped_read(struct file *filp, char __user *buffer, 3832db270a8SFrederic Weisbecker size_t count, loff_t *ppos) 3842db270a8SFrederic Weisbecker { 3852db270a8SFrederic Weisbecker struct blk_trace *bt = filp->private_data; 3862db270a8SFrederic Weisbecker char buf[16]; 3872db270a8SFrederic Weisbecker 3882db270a8SFrederic Weisbecker snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped)); 3892db270a8SFrederic Weisbecker 3902db270a8SFrederic Weisbecker return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf)); 3912db270a8SFrederic Weisbecker } 3922db270a8SFrederic Weisbecker 3932db270a8SFrederic Weisbecker static const struct file_operations blk_dropped_fops = { 3942db270a8SFrederic Weisbecker .owner = THIS_MODULE, 395234e3405SStephen Boyd .open = simple_open, 3962db270a8SFrederic Weisbecker .read = blk_dropped_read, 3976038f373SArnd Bergmann .llseek = default_llseek, 3982db270a8SFrederic Weisbecker }; 3992db270a8SFrederic Weisbecker 4002db270a8SFrederic Weisbecker static ssize_t blk_msg_write(struct file *filp, const char __user *buffer, 4012db270a8SFrederic Weisbecker size_t count, loff_t *ppos) 4022db270a8SFrederic Weisbecker { 4032db270a8SFrederic Weisbecker char *msg; 4042db270a8SFrederic Weisbecker struct blk_trace *bt; 4052db270a8SFrederic Weisbecker 4067635b03aSLi Zefan if (count >= BLK_TN_MAX_MSG) 4072db270a8SFrederic Weisbecker return -EINVAL; 4082db270a8SFrederic Weisbecker 40916e5c1fcSAl Viro msg = memdup_user_nul(buffer, count); 41016e5c1fcSAl Viro if (IS_ERR(msg)) 41116e5c1fcSAl Viro return PTR_ERR(msg); 4122db270a8SFrederic Weisbecker 4132db270a8SFrederic Weisbecker bt = filp->private_data; 41435fe6d76SShaohua Li __trace_note_message(bt, NULL, "%s", msg); 4152db270a8SFrederic Weisbecker kfree(msg); 4162db270a8SFrederic Weisbecker 4172db270a8SFrederic Weisbecker return count; 4182db270a8SFrederic Weisbecker } 4192db270a8SFrederic Weisbecker 4202db270a8SFrederic Weisbecker static const struct file_operations blk_msg_fops = { 4212db270a8SFrederic Weisbecker .owner = THIS_MODULE, 422234e3405SStephen Boyd .open = simple_open, 4232db270a8SFrederic Weisbecker .write = blk_msg_write, 4246038f373SArnd Bergmann .llseek = noop_llseek, 4252db270a8SFrederic Weisbecker }; 4262db270a8SFrederic Weisbecker 4272db270a8SFrederic Weisbecker /* 4282db270a8SFrederic Weisbecker * Keep track of how many times we encountered a full subbuffer, to aid 4292db270a8SFrederic Weisbecker * the user space app in telling how many lost events there were. 4302db270a8SFrederic Weisbecker */ 4312db270a8SFrederic Weisbecker static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf, 4322db270a8SFrederic Weisbecker void *prev_subbuf, size_t prev_padding) 4332db270a8SFrederic Weisbecker { 4342db270a8SFrederic Weisbecker struct blk_trace *bt; 4352db270a8SFrederic Weisbecker 4362db270a8SFrederic Weisbecker if (!relay_buf_full(buf)) 4372db270a8SFrederic Weisbecker return 1; 4382db270a8SFrederic Weisbecker 4392db270a8SFrederic Weisbecker bt = buf->chan->private_data; 4402db270a8SFrederic Weisbecker atomic_inc(&bt->dropped); 4412db270a8SFrederic Weisbecker return 0; 4422db270a8SFrederic Weisbecker } 4432db270a8SFrederic Weisbecker 4442db270a8SFrederic Weisbecker static int blk_remove_buf_file_callback(struct dentry *dentry) 4452db270a8SFrederic Weisbecker { 4462db270a8SFrederic Weisbecker debugfs_remove(dentry); 4472db270a8SFrederic Weisbecker 4482db270a8SFrederic Weisbecker return 0; 4492db270a8SFrederic Weisbecker } 4502db270a8SFrederic Weisbecker 4512db270a8SFrederic Weisbecker static struct dentry *blk_create_buf_file_callback(const char *filename, 4522db270a8SFrederic Weisbecker struct dentry *parent, 453f4ae40a6SAl Viro umode_t mode, 4542db270a8SFrederic Weisbecker struct rchan_buf *buf, 4552db270a8SFrederic Weisbecker int *is_global) 4562db270a8SFrederic Weisbecker { 4572db270a8SFrederic Weisbecker return debugfs_create_file(filename, mode, parent, buf, 4582db270a8SFrederic Weisbecker &relay_file_operations); 4592db270a8SFrederic Weisbecker } 4602db270a8SFrederic Weisbecker 461abf4e00cSJani Nikula static const struct rchan_callbacks blk_relay_callbacks = { 4622db270a8SFrederic Weisbecker .subbuf_start = blk_subbuf_start_callback, 4632db270a8SFrederic Weisbecker .create_buf_file = blk_create_buf_file_callback, 4642db270a8SFrederic Weisbecker .remove_buf_file = blk_remove_buf_file_callback, 4652db270a8SFrederic Weisbecker }; 4662db270a8SFrederic Weisbecker 4679908c309SLi Zefan static void blk_trace_setup_lba(struct blk_trace *bt, 4689908c309SLi Zefan struct block_device *bdev) 4699908c309SLi Zefan { 47029ff57c6SChristoph Hellwig if (bdev) { 47129ff57c6SChristoph Hellwig bt->start_lba = bdev->bd_start_sect; 47229ff57c6SChristoph Hellwig bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev); 4739908c309SLi Zefan } else { 4749908c309SLi Zefan bt->start_lba = 0; 4759908c309SLi Zefan bt->end_lba = -1ULL; 4769908c309SLi Zefan } 4779908c309SLi Zefan } 4789908c309SLi Zefan 4792db270a8SFrederic Weisbecker /* 4802db270a8SFrederic Weisbecker * Setup everything required to start tracing 4812db270a8SFrederic Weisbecker */ 482a428d314SOmar Sandoval static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 483d0deef5bSShawn Du struct block_device *bdev, 4842db270a8SFrederic Weisbecker struct blk_user_trace_setup *buts) 4852db270a8SFrederic Weisbecker { 486cdea01b2SDavidlohr Bueso struct blk_trace *bt = NULL; 4872db270a8SFrederic Weisbecker struct dentry *dir = NULL; 488ff14417cSRasmus Villemoes int ret; 4892db270a8SFrederic Weisbecker 49085e0cbbbSLuis Chamberlain lockdep_assert_held(&q->debugfs_mutex); 491a67549c8SLuis Chamberlain 4922db270a8SFrederic Weisbecker if (!buts->buf_size || !buts->buf_nr) 4932db270a8SFrederic Weisbecker return -EINVAL; 4942db270a8SFrederic Weisbecker 4952db270a8SFrederic Weisbecker strncpy(buts->name, name, BLKTRACE_BDEV_SIZE); 4962db270a8SFrederic Weisbecker buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0'; 4972db270a8SFrederic Weisbecker 4982db270a8SFrederic Weisbecker /* 4992db270a8SFrederic Weisbecker * some device names have larger paths - convert the slashes 5002db270a8SFrederic Weisbecker * to underscores for this to work as expected 5012db270a8SFrederic Weisbecker */ 502ff14417cSRasmus Villemoes strreplace(buts->name, '/', '_'); 5032db270a8SFrederic Weisbecker 5041b0b2836SLuis Chamberlain /* 5051b0b2836SLuis Chamberlain * bdev can be NULL, as with scsi-generic, this is a helpful as 5061b0b2836SLuis Chamberlain * we can be. 5071b0b2836SLuis Chamberlain */ 508c3dbe541SJan Kara if (rcu_dereference_protected(q->blk_trace, 50985e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex))) { 5101b0b2836SLuis Chamberlain pr_warn("Concurrent blktraces are not allowed on %s\n", 5111b0b2836SLuis Chamberlain buts->name); 5121b0b2836SLuis Chamberlain return -EBUSY; 5131b0b2836SLuis Chamberlain } 5141b0b2836SLuis Chamberlain 5152db270a8SFrederic Weisbecker bt = kzalloc(sizeof(*bt), GFP_KERNEL); 5162db270a8SFrederic Weisbecker if (!bt) 517ad5dd549SLi Zefan return -ENOMEM; 5182db270a8SFrederic Weisbecker 519ad5dd549SLi Zefan ret = -ENOMEM; 5202db270a8SFrederic Weisbecker bt->sequence = alloc_percpu(unsigned long); 5212db270a8SFrederic Weisbecker if (!bt->sequence) 5222db270a8SFrederic Weisbecker goto err; 5232db270a8SFrederic Weisbecker 524f0ef0398SIngo Molnar bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 5252db270a8SFrederic Weisbecker if (!bt->msg_data) 5262db270a8SFrederic Weisbecker goto err; 5272db270a8SFrederic Weisbecker 528bad8e64fSLuis Chamberlain /* 52985e0cbbbSLuis Chamberlain * When tracing the whole disk reuse the existing debugfs directory 53085e0cbbbSLuis Chamberlain * created by the block layer on init. For partitions block devices, 531bad8e64fSLuis Chamberlain * and scsi-generic block devices we create a temporary new debugfs 532bad8e64fSLuis Chamberlain * directory that will be removed once the trace ends. 533bad8e64fSLuis Chamberlain */ 534fa01b1e9SChristoph Hellwig if (bdev && !bdev_is_partition(bdev)) 535bad8e64fSLuis Chamberlain dir = q->debugfs_dir; 536bad8e64fSLuis Chamberlain else 5376ac93117SOmar Sandoval bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root); 5382db270a8SFrederic Weisbecker 539b431ef83SLuis Chamberlain /* 540b431ef83SLuis Chamberlain * As blktrace relies on debugfs for its interface the debugfs directory 541b431ef83SLuis Chamberlain * is required, contrary to the usual mantra of not checking for debugfs 542b431ef83SLuis Chamberlain * files or directories. 543b431ef83SLuis Chamberlain */ 544b431ef83SLuis Chamberlain if (IS_ERR_OR_NULL(dir)) { 545b431ef83SLuis Chamberlain pr_warn("debugfs_dir not present for %s so skipping\n", 546b431ef83SLuis Chamberlain buts->name); 547b431ef83SLuis Chamberlain ret = -ENOENT; 548b431ef83SLuis Chamberlain goto err; 549b431ef83SLuis Chamberlain } 550b431ef83SLuis Chamberlain 5512db270a8SFrederic Weisbecker bt->dev = dev; 5522db270a8SFrederic Weisbecker atomic_set(&bt->dropped, 0); 553a404d557SJan Kara INIT_LIST_HEAD(&bt->running_list); 5542db270a8SFrederic Weisbecker 5552db270a8SFrederic Weisbecker ret = -EIO; 556c0ea5760SGreg Kroah-Hartman debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops); 557c0ea5760SGreg Kroah-Hartman debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops); 5582db270a8SFrederic Weisbecker 5592db270a8SFrederic Weisbecker bt->rchan = relay_open("trace", dir, buts->buf_size, 5602db270a8SFrederic Weisbecker buts->buf_nr, &blk_relay_callbacks, bt); 5612db270a8SFrederic Weisbecker if (!bt->rchan) 5622db270a8SFrederic Weisbecker goto err; 5632db270a8SFrederic Weisbecker 5642db270a8SFrederic Weisbecker bt->act_mask = buts->act_mask; 5652db270a8SFrederic Weisbecker if (!bt->act_mask) 5662db270a8SFrederic Weisbecker bt->act_mask = (u16) -1; 5672db270a8SFrederic Weisbecker 5689908c309SLi Zefan blk_trace_setup_lba(bt, bdev); 5692db270a8SFrederic Weisbecker 570d0deef5bSShawn Du /* overwrite with user settings */ 571d0deef5bSShawn Du if (buts->start_lba) 572d0deef5bSShawn Du bt->start_lba = buts->start_lba; 573d0deef5bSShawn Du if (buts->end_lba) 574d0deef5bSShawn Du bt->end_lba = buts->end_lba; 575d0deef5bSShawn Du 5762db270a8SFrederic Weisbecker bt->pid = buts->pid; 5772db270a8SFrederic Weisbecker bt->trace_state = Blktrace_setup; 5782db270a8SFrederic Weisbecker 579c3dbe541SJan Kara rcu_assign_pointer(q->blk_trace, bt); 580a6da0024SJens Axboe get_probe_ref(); 581cbe28296SLi Zefan 5826ac93117SOmar Sandoval ret = 0; 5832db270a8SFrederic Weisbecker err: 5846ac93117SOmar Sandoval if (ret) 585*30939293SYu Kuai blk_trace_free(q, bt); 5862db270a8SFrederic Weisbecker return ret; 5872db270a8SFrederic Weisbecker } 5882db270a8SFrederic Weisbecker 5891f2cac10SJens Axboe static int __blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 5901f2cac10SJens Axboe struct block_device *bdev, char __user *arg) 5912db270a8SFrederic Weisbecker { 5922db270a8SFrederic Weisbecker struct blk_user_trace_setup buts; 5932db270a8SFrederic Weisbecker int ret; 5942db270a8SFrederic Weisbecker 5952db270a8SFrederic Weisbecker ret = copy_from_user(&buts, arg, sizeof(buts)); 5962db270a8SFrederic Weisbecker if (ret) 5972db270a8SFrederic Weisbecker return -EFAULT; 5982db270a8SFrederic Weisbecker 599d0deef5bSShawn Du ret = do_blk_trace_setup(q, name, dev, bdev, &buts); 6002db270a8SFrederic Weisbecker if (ret) 6012db270a8SFrederic Weisbecker return ret; 6022db270a8SFrederic Weisbecker 6039a8c28c8SDmitry Monakhov if (copy_to_user(arg, &buts, sizeof(buts))) { 6042967acbbSJens Axboe __blk_trace_remove(q); 6052db270a8SFrederic Weisbecker return -EFAULT; 6069a8c28c8SDmitry Monakhov } 6072db270a8SFrederic Weisbecker return 0; 6082db270a8SFrederic Weisbecker } 6091f2cac10SJens Axboe 6101f2cac10SJens Axboe int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 6111f2cac10SJens Axboe struct block_device *bdev, 6121f2cac10SJens Axboe char __user *arg) 6131f2cac10SJens Axboe { 6141f2cac10SJens Axboe int ret; 6151f2cac10SJens Axboe 61685e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 6171f2cac10SJens Axboe ret = __blk_trace_setup(q, name, dev, bdev, arg); 61885e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 6191f2cac10SJens Axboe 6201f2cac10SJens Axboe return ret; 6211f2cac10SJens Axboe } 6222db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_setup); 6232db270a8SFrederic Weisbecker 62462c2a7d9SArnd Bergmann #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 62562c2a7d9SArnd Bergmann static int compat_blk_trace_setup(struct request_queue *q, char *name, 62662c2a7d9SArnd Bergmann dev_t dev, struct block_device *bdev, 62762c2a7d9SArnd Bergmann char __user *arg) 62862c2a7d9SArnd Bergmann { 62962c2a7d9SArnd Bergmann struct blk_user_trace_setup buts; 63062c2a7d9SArnd Bergmann struct compat_blk_user_trace_setup cbuts; 63162c2a7d9SArnd Bergmann int ret; 63262c2a7d9SArnd Bergmann 63362c2a7d9SArnd Bergmann if (copy_from_user(&cbuts, arg, sizeof(cbuts))) 63462c2a7d9SArnd Bergmann return -EFAULT; 63562c2a7d9SArnd Bergmann 63662c2a7d9SArnd Bergmann buts = (struct blk_user_trace_setup) { 63762c2a7d9SArnd Bergmann .act_mask = cbuts.act_mask, 63862c2a7d9SArnd Bergmann .buf_size = cbuts.buf_size, 63962c2a7d9SArnd Bergmann .buf_nr = cbuts.buf_nr, 64062c2a7d9SArnd Bergmann .start_lba = cbuts.start_lba, 64162c2a7d9SArnd Bergmann .end_lba = cbuts.end_lba, 64262c2a7d9SArnd Bergmann .pid = cbuts.pid, 64362c2a7d9SArnd Bergmann }; 64462c2a7d9SArnd Bergmann 64562c2a7d9SArnd Bergmann ret = do_blk_trace_setup(q, name, dev, bdev, &buts); 64662c2a7d9SArnd Bergmann if (ret) 64762c2a7d9SArnd Bergmann return ret; 64862c2a7d9SArnd Bergmann 649f8c5e944SChen Gang if (copy_to_user(arg, &buts.name, ARRAY_SIZE(buts.name))) { 6502967acbbSJens Axboe __blk_trace_remove(q); 65162c2a7d9SArnd Bergmann return -EFAULT; 65262c2a7d9SArnd Bergmann } 65362c2a7d9SArnd Bergmann 65462c2a7d9SArnd Bergmann return 0; 65562c2a7d9SArnd Bergmann } 65662c2a7d9SArnd Bergmann #endif 65762c2a7d9SArnd Bergmann 6581f2cac10SJens Axboe static int __blk_trace_startstop(struct request_queue *q, int start) 6592db270a8SFrederic Weisbecker { 6602db270a8SFrederic Weisbecker int ret; 661c780e86dSJan Kara struct blk_trace *bt; 6622db270a8SFrederic Weisbecker 663c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 66485e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 6652db270a8SFrederic Weisbecker if (bt == NULL) 6662db270a8SFrederic Weisbecker return -EINVAL; 6672db270a8SFrederic Weisbecker 6682db270a8SFrederic Weisbecker /* 6692db270a8SFrederic Weisbecker * For starting a trace, we can transition from a setup or stopped 6702db270a8SFrederic Weisbecker * trace. For stopping a trace, the state must be running 6712db270a8SFrederic Weisbecker */ 6722db270a8SFrederic Weisbecker ret = -EINVAL; 6732db270a8SFrederic Weisbecker if (start) { 6742db270a8SFrederic Weisbecker if (bt->trace_state == Blktrace_setup || 6752db270a8SFrederic Weisbecker bt->trace_state == Blktrace_stopped) { 6762db270a8SFrederic Weisbecker blktrace_seq++; 6772db270a8SFrederic Weisbecker smp_mb(); 6782db270a8SFrederic Weisbecker bt->trace_state = Blktrace_running; 679361c81dbSWander Lairson Costa raw_spin_lock_irq(&running_trace_lock); 680a404d557SJan Kara list_add(&bt->running_list, &running_trace_list); 681361c81dbSWander Lairson Costa raw_spin_unlock_irq(&running_trace_lock); 6822db270a8SFrederic Weisbecker 6832db270a8SFrederic Weisbecker trace_note_time(bt); 6842db270a8SFrederic Weisbecker ret = 0; 6852db270a8SFrederic Weisbecker } 6862db270a8SFrederic Weisbecker } else { 6872db270a8SFrederic Weisbecker if (bt->trace_state == Blktrace_running) { 6882db270a8SFrederic Weisbecker bt->trace_state = Blktrace_stopped; 689361c81dbSWander Lairson Costa raw_spin_lock_irq(&running_trace_lock); 690a404d557SJan Kara list_del_init(&bt->running_list); 691361c81dbSWander Lairson Costa raw_spin_unlock_irq(&running_trace_lock); 6922db270a8SFrederic Weisbecker relay_flush(bt->rchan); 6932db270a8SFrederic Weisbecker ret = 0; 6942db270a8SFrederic Weisbecker } 6952db270a8SFrederic Weisbecker } 6962db270a8SFrederic Weisbecker 6972db270a8SFrederic Weisbecker return ret; 6982db270a8SFrederic Weisbecker } 6991f2cac10SJens Axboe 7001f2cac10SJens Axboe int blk_trace_startstop(struct request_queue *q, int start) 7011f2cac10SJens Axboe { 7021f2cac10SJens Axboe int ret; 7031f2cac10SJens Axboe 70485e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 7051f2cac10SJens Axboe ret = __blk_trace_startstop(q, start); 70685e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 7071f2cac10SJens Axboe 7081f2cac10SJens Axboe return ret; 7091f2cac10SJens Axboe } 7102db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_startstop); 7112db270a8SFrederic Weisbecker 7125acb3cc2SWaiman Long /* 7135acb3cc2SWaiman Long * When reading or writing the blktrace sysfs files, the references to the 7145acb3cc2SWaiman Long * opened sysfs or device files should prevent the underlying block device 7155acb3cc2SWaiman Long * from being removed. So no further delete protection is really needed. 7165acb3cc2SWaiman Long */ 7175acb3cc2SWaiman Long 7182db270a8SFrederic Weisbecker /** 7192db270a8SFrederic Weisbecker * blk_trace_ioctl: - handle the ioctls associated with tracing 7202db270a8SFrederic Weisbecker * @bdev: the block device 7212db270a8SFrederic Weisbecker * @cmd: the ioctl cmd 7222db270a8SFrederic Weisbecker * @arg: the argument data, if any 7232db270a8SFrederic Weisbecker * 7242db270a8SFrederic Weisbecker **/ 7252db270a8SFrederic Weisbecker int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) 7262db270a8SFrederic Weisbecker { 7272db270a8SFrederic Weisbecker struct request_queue *q; 7282db270a8SFrederic Weisbecker int ret, start = 0; 7292db270a8SFrederic Weisbecker char b[BDEVNAME_SIZE]; 7302db270a8SFrederic Weisbecker 7312db270a8SFrederic Weisbecker q = bdev_get_queue(bdev); 7322db270a8SFrederic Weisbecker if (!q) 7332db270a8SFrederic Weisbecker return -ENXIO; 7342db270a8SFrederic Weisbecker 73585e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 7362db270a8SFrederic Weisbecker 7372db270a8SFrederic Weisbecker switch (cmd) { 7382db270a8SFrederic Weisbecker case BLKTRACESETUP: 7392db270a8SFrederic Weisbecker bdevname(bdev, b); 7401f2cac10SJens Axboe ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 7412db270a8SFrederic Weisbecker break; 74262c2a7d9SArnd Bergmann #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 74362c2a7d9SArnd Bergmann case BLKTRACESETUP32: 74462c2a7d9SArnd Bergmann bdevname(bdev, b); 74562c2a7d9SArnd Bergmann ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 74662c2a7d9SArnd Bergmann break; 74762c2a7d9SArnd Bergmann #endif 7482db270a8SFrederic Weisbecker case BLKTRACESTART: 7492db270a8SFrederic Weisbecker start = 1; 750df561f66SGustavo A. R. Silva fallthrough; 7512db270a8SFrederic Weisbecker case BLKTRACESTOP: 7521f2cac10SJens Axboe ret = __blk_trace_startstop(q, start); 7532db270a8SFrederic Weisbecker break; 7542db270a8SFrederic Weisbecker case BLKTRACETEARDOWN: 7551f2cac10SJens Axboe ret = __blk_trace_remove(q); 7562db270a8SFrederic Weisbecker break; 7572db270a8SFrederic Weisbecker default: 7582db270a8SFrederic Weisbecker ret = -ENOTTY; 7592db270a8SFrederic Weisbecker break; 7602db270a8SFrederic Weisbecker } 7612db270a8SFrederic Weisbecker 76285e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 7632db270a8SFrederic Weisbecker return ret; 7642db270a8SFrederic Weisbecker } 7652db270a8SFrederic Weisbecker 7662db270a8SFrederic Weisbecker /** 7672db270a8SFrederic Weisbecker * blk_trace_shutdown: - stop and cleanup trace structures 7682db270a8SFrederic Weisbecker * @q: the request queue associated with the device 7692db270a8SFrederic Weisbecker * 7702db270a8SFrederic Weisbecker **/ 7712db270a8SFrederic Weisbecker void blk_trace_shutdown(struct request_queue *q) 7722db270a8SFrederic Weisbecker { 77385e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 774c780e86dSJan Kara if (rcu_dereference_protected(q->blk_trace, 77585e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex))) { 7761f2cac10SJens Axboe __blk_trace_startstop(q, 0); 7771f2cac10SJens Axboe __blk_trace_remove(q); 7782db270a8SFrederic Weisbecker } 7791f2cac10SJens Axboe 78085e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 7812db270a8SFrederic Weisbecker } 7822db270a8SFrederic Weisbecker 783ca1136c9SShaohua Li #ifdef CONFIG_BLK_CGROUP 78467c0496eSTejun Heo static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 785ca1136c9SShaohua Li { 786c780e86dSJan Kara struct blk_trace *bt; 787ca1136c9SShaohua Li 788c780e86dSJan Kara /* We don't use the 'bt' value here except as an optimization... */ 789c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 1); 790ca1136c9SShaohua Li if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 79167c0496eSTejun Heo return 0; 792ca1136c9SShaohua Li 793db6638d7SDennis Zhou if (!bio->bi_blkg) 79467c0496eSTejun Heo return 0; 79574321038STejun Heo return cgroup_id(bio_blkcg(bio)->css.cgroup); 796ca1136c9SShaohua Li } 797ca1136c9SShaohua Li #else 798e75ad2ccSWang Hai static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 799ca1136c9SShaohua Li { 80067c0496eSTejun Heo return 0; 801ca1136c9SShaohua Li } 802ca1136c9SShaohua Li #endif 803ca1136c9SShaohua Li 80467c0496eSTejun Heo static u64 805a54895faSChristoph Hellwig blk_trace_request_get_cgid(struct request *rq) 806ca1136c9SShaohua Li { 807ca1136c9SShaohua Li if (!rq->bio) 80867c0496eSTejun Heo return 0; 809ca1136c9SShaohua Li /* Use the first bio */ 810a54895faSChristoph Hellwig return blk_trace_bio_get_cgid(rq->q, rq->bio); 811ca1136c9SShaohua Li } 812ca1136c9SShaohua Li 8132db270a8SFrederic Weisbecker /* 8142db270a8SFrederic Weisbecker * blktrace probes 8152db270a8SFrederic Weisbecker */ 8162db270a8SFrederic Weisbecker 8172db270a8SFrederic Weisbecker /** 8182db270a8SFrederic Weisbecker * blk_add_trace_rq - Add a trace for a request oriented action 8192db270a8SFrederic Weisbecker * @rq: the source request 820caf7df12SChristoph Hellwig * @error: return status to log 821af5040daSRoman Pen * @nr_bytes: number of completed bytes 8222db270a8SFrederic Weisbecker * @what: the action 823ca1136c9SShaohua Li * @cgid: the cgroup info 8242db270a8SFrederic Weisbecker * 8252db270a8SFrederic Weisbecker * Description: 8262db270a8SFrederic Weisbecker * Records an action against a request. Will log the bio offset + size. 8272db270a8SFrederic Weisbecker * 8282db270a8SFrederic Weisbecker **/ 8298a7d267bSChristoph Hellwig static void blk_add_trace_rq(struct request *rq, blk_status_t error, 83067c0496eSTejun Heo unsigned int nr_bytes, u32 what, u64 cgid) 8312db270a8SFrederic Weisbecker { 832c780e86dSJan Kara struct blk_trace *bt; 8332db270a8SFrederic Weisbecker 834c780e86dSJan Kara rcu_read_lock(); 835c780e86dSJan Kara bt = rcu_dereference(rq->q->blk_trace); 836c780e86dSJan Kara if (likely(!bt)) { 837c780e86dSJan Kara rcu_read_unlock(); 8382db270a8SFrederic Weisbecker return; 839c780e86dSJan Kara } 8402db270a8SFrederic Weisbecker 84157292b58SChristoph Hellwig if (blk_rq_is_passthrough(rq)) 8422db270a8SFrederic Weisbecker what |= BLK_TC_ACT(BLK_TC_PC); 84348b77ad6SChristoph Hellwig else 8442db270a8SFrederic Weisbecker what |= BLK_TC_ACT(BLK_TC_FS); 84548b77ad6SChristoph Hellwig 84648b77ad6SChristoph Hellwig __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, req_op(rq), 8478a7d267bSChristoph Hellwig rq->cmd_flags, what, blk_status_to_errno(error), 0, 8488a7d267bSChristoph Hellwig NULL, cgid); 849c780e86dSJan Kara rcu_read_unlock(); 8502db270a8SFrederic Weisbecker } 8512db270a8SFrederic Weisbecker 852a54895faSChristoph Hellwig static void blk_add_trace_rq_insert(void *ignore, struct request *rq) 8532db270a8SFrederic Weisbecker { 854ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT, 855a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 8562db270a8SFrederic Weisbecker } 8572db270a8SFrederic Weisbecker 858a54895faSChristoph Hellwig static void blk_add_trace_rq_issue(void *ignore, struct request *rq) 8592db270a8SFrederic Weisbecker { 860ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ISSUE, 861a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 8622db270a8SFrederic Weisbecker } 8632db270a8SFrederic Weisbecker 864a54895faSChristoph Hellwig static void blk_add_trace_rq_merge(void *ignore, struct request *rq) 865f3bdc62fSJan Kara { 866f3bdc62fSJan Kara blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_BACKMERGE, 867a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 868f3bdc62fSJan Kara } 869f3bdc62fSJan Kara 870a54895faSChristoph Hellwig static void blk_add_trace_rq_requeue(void *ignore, struct request *rq) 8712db270a8SFrederic Weisbecker { 872ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_REQUEUE, 873a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 8742db270a8SFrederic Weisbecker } 8752db270a8SFrederic Weisbecker 876caf7df12SChristoph Hellwig static void blk_add_trace_rq_complete(void *ignore, struct request *rq, 8778a7d267bSChristoph Hellwig blk_status_t error, unsigned int nr_bytes) 8782db270a8SFrederic Weisbecker { 879ca1136c9SShaohua Li blk_add_trace_rq(rq, error, nr_bytes, BLK_TA_COMPLETE, 880a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 8812db270a8SFrederic Weisbecker } 8822db270a8SFrederic Weisbecker 8832db270a8SFrederic Weisbecker /** 8842db270a8SFrederic Weisbecker * blk_add_trace_bio - Add a trace for a bio oriented action 8852db270a8SFrederic Weisbecker * @q: queue the io is for 8862db270a8SFrederic Weisbecker * @bio: the source bio 8872db270a8SFrederic Weisbecker * @what: the action 888797a455dSJens Axboe * @error: error, if any 8892db270a8SFrederic Weisbecker * 8902db270a8SFrederic Weisbecker * Description: 8912db270a8SFrederic Weisbecker * Records an action against a bio. Will log the bio offset + size. 8922db270a8SFrederic Weisbecker * 8932db270a8SFrederic Weisbecker **/ 8942db270a8SFrederic Weisbecker static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, 8951690102dSMarcos Paulo de Souza u32 what, int error) 8962db270a8SFrederic Weisbecker { 897c780e86dSJan Kara struct blk_trace *bt; 8982db270a8SFrederic Weisbecker 899c780e86dSJan Kara rcu_read_lock(); 900c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 901c780e86dSJan Kara if (likely(!bt)) { 902c780e86dSJan Kara rcu_read_unlock(); 9032db270a8SFrederic Weisbecker return; 904c780e86dSJan Kara } 9052db270a8SFrederic Weisbecker 9064f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 9071690102dSMarcos Paulo de Souza bio_op(bio), bio->bi_opf, what, error, 0, NULL, 9081690102dSMarcos Paulo de Souza blk_trace_bio_get_cgid(q, bio)); 909c780e86dSJan Kara rcu_read_unlock(); 9102db270a8SFrederic Weisbecker } 9112db270a8SFrederic Weisbecker 912e8a676d6SChristoph Hellwig static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio) 9132db270a8SFrederic Weisbecker { 914309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BOUNCE, 0); 9152db270a8SFrederic Weisbecker } 9162db270a8SFrederic Weisbecker 9170a82a8d1SLinus Torvalds static void blk_add_trace_bio_complete(void *ignore, 918d24de76aSChristoph Hellwig struct request_queue *q, struct bio *bio) 9192db270a8SFrederic Weisbecker { 920d24de76aSChristoph Hellwig blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, 921d24de76aSChristoph Hellwig blk_status_to_errno(bio->bi_status)); 9222db270a8SFrederic Weisbecker } 9232db270a8SFrederic Weisbecker 924e8a676d6SChristoph Hellwig static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio) 9252db270a8SFrederic Weisbecker { 926309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE, 927309dca30SChristoph Hellwig 0); 9282db270a8SFrederic Weisbecker } 9292db270a8SFrederic Weisbecker 930e8a676d6SChristoph Hellwig static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio) 9312db270a8SFrederic Weisbecker { 932309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE, 933309dca30SChristoph Hellwig 0); 9342db270a8SFrederic Weisbecker } 9352db270a8SFrederic Weisbecker 936e8a676d6SChristoph Hellwig static void blk_add_trace_bio_queue(void *ignore, struct bio *bio) 9372db270a8SFrederic Weisbecker { 938309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0); 9392db270a8SFrederic Weisbecker } 9402db270a8SFrederic Weisbecker 941e8a676d6SChristoph Hellwig static void blk_add_trace_getrq(void *ignore, struct bio *bio) 9422db270a8SFrederic Weisbecker { 943309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0); 9442db270a8SFrederic Weisbecker } 9452db270a8SFrederic Weisbecker 94638516ab5SSteven Rostedt static void blk_add_trace_plug(void *ignore, struct request_queue *q) 9472db270a8SFrederic Weisbecker { 948c780e86dSJan Kara struct blk_trace *bt; 9492db270a8SFrederic Weisbecker 950c780e86dSJan Kara rcu_read_lock(); 951c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9522db270a8SFrederic Weisbecker if (bt) 95367c0496eSTejun Heo __blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); 954c780e86dSJan Kara rcu_read_unlock(); 9552db270a8SFrederic Weisbecker } 9562db270a8SFrederic Weisbecker 95749cac01eSJens Axboe static void blk_add_trace_unplug(void *ignore, struct request_queue *q, 95849cac01eSJens Axboe unsigned int depth, bool explicit) 9592db270a8SFrederic Weisbecker { 960c780e86dSJan Kara struct blk_trace *bt; 9612db270a8SFrederic Weisbecker 962c780e86dSJan Kara rcu_read_lock(); 963c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9642db270a8SFrederic Weisbecker if (bt) { 96594b5eb28SJens Axboe __be64 rpdu = cpu_to_be64(depth); 96649cac01eSJens Axboe u32 what; 9672db270a8SFrederic Weisbecker 96849cac01eSJens Axboe if (explicit) 96949cac01eSJens Axboe what = BLK_TA_UNPLUG_IO; 97049cac01eSJens Axboe else 97149cac01eSJens Axboe what = BLK_TA_UNPLUG_TIMER; 97249cac01eSJens Axboe 97367c0496eSTejun Heo __blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); 9742db270a8SFrederic Weisbecker } 975c780e86dSJan Kara rcu_read_unlock(); 9762db270a8SFrederic Weisbecker } 9772db270a8SFrederic Weisbecker 978eb6f7f7cSChristoph Hellwig static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) 9792db270a8SFrederic Weisbecker { 980309dca30SChristoph Hellwig struct request_queue *q = bio->bi_bdev->bd_disk->queue; 981c780e86dSJan Kara struct blk_trace *bt; 9822db270a8SFrederic Weisbecker 983c780e86dSJan Kara rcu_read_lock(); 984c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9852db270a8SFrederic Weisbecker if (bt) { 9862db270a8SFrederic Weisbecker __be64 rpdu = cpu_to_be64(pdu); 9872db270a8SFrederic Weisbecker 9884f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, 9891eff9d32SJens Axboe bio->bi_iter.bi_size, bio_op(bio), bio->bi_opf, 99048bc3cd3SChaitanya Kulkarni BLK_TA_SPLIT, 99148bc3cd3SChaitanya Kulkarni blk_status_to_errno(bio->bi_status), 99248bc3cd3SChaitanya Kulkarni sizeof(rpdu), &rpdu, 99348bc3cd3SChaitanya Kulkarni blk_trace_bio_get_cgid(q, bio)); 9942db270a8SFrederic Weisbecker } 995c780e86dSJan Kara rcu_read_unlock(); 9962db270a8SFrederic Weisbecker } 9972db270a8SFrederic Weisbecker 9982db270a8SFrederic Weisbecker /** 999d07335e5SMike Snitzer * blk_add_trace_bio_remap - Add a trace for a bio-remap operation 1000546cf44aSRandy Dunlap * @ignore: trace callback data parameter (not used) 10012db270a8SFrederic Weisbecker * @bio: the source bio 10021c02fca6SChristoph Hellwig * @dev: source device 1003a42aaa3bSAlan D. Brunelle * @from: source sector 10042db270a8SFrederic Weisbecker * 10051c02fca6SChristoph Hellwig * Called after a bio is remapped to a different device and/or sector. 10062db270a8SFrederic Weisbecker **/ 10071c02fca6SChristoph Hellwig static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, 10081c02fca6SChristoph Hellwig sector_t from) 10092db270a8SFrederic Weisbecker { 1010309dca30SChristoph Hellwig struct request_queue *q = bio->bi_bdev->bd_disk->queue; 1011c780e86dSJan Kara struct blk_trace *bt; 10122db270a8SFrederic Weisbecker struct blk_io_trace_remap r; 10132db270a8SFrederic Weisbecker 1014c780e86dSJan Kara rcu_read_lock(); 1015c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 1016c780e86dSJan Kara if (likely(!bt)) { 1017c780e86dSJan Kara rcu_read_unlock(); 10182db270a8SFrederic Weisbecker return; 1019c780e86dSJan Kara } 10202db270a8SFrederic Weisbecker 1021a42aaa3bSAlan D. Brunelle r.device_from = cpu_to_be32(dev); 102274d46992SChristoph Hellwig r.device_to = cpu_to_be32(bio_dev(bio)); 1023a42aaa3bSAlan D. Brunelle r.sector_from = cpu_to_be64(from); 10242db270a8SFrederic Weisbecker 10254f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 102648bc3cd3SChaitanya Kulkarni bio_op(bio), bio->bi_opf, BLK_TA_REMAP, 102748bc3cd3SChaitanya Kulkarni blk_status_to_errno(bio->bi_status), 1028ca1136c9SShaohua Li sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); 1029c780e86dSJan Kara rcu_read_unlock(); 10302db270a8SFrederic Weisbecker } 10312db270a8SFrederic Weisbecker 10322db270a8SFrederic Weisbecker /** 1033b0da3f0dSJun'ichi Nomura * blk_add_trace_rq_remap - Add a trace for a request-remap operation 1034546cf44aSRandy Dunlap * @ignore: trace callback data parameter (not used) 1035b0da3f0dSJun'ichi Nomura * @rq: the source request 1036b0da3f0dSJun'ichi Nomura * @dev: target device 1037b0da3f0dSJun'ichi Nomura * @from: source sector 1038b0da3f0dSJun'ichi Nomura * 1039b0da3f0dSJun'ichi Nomura * Description: 1040b0da3f0dSJun'ichi Nomura * Device mapper remaps request to other devices. 1041b0da3f0dSJun'ichi Nomura * Add a trace for that action. 1042b0da3f0dSJun'ichi Nomura * 1043b0da3f0dSJun'ichi Nomura **/ 1044a54895faSChristoph Hellwig static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev, 1045b0da3f0dSJun'ichi Nomura sector_t from) 1046b0da3f0dSJun'ichi Nomura { 1047c780e86dSJan Kara struct blk_trace *bt; 1048b0da3f0dSJun'ichi Nomura struct blk_io_trace_remap r; 1049b0da3f0dSJun'ichi Nomura 1050c780e86dSJan Kara rcu_read_lock(); 1051a54895faSChristoph Hellwig bt = rcu_dereference(rq->q->blk_trace); 1052c780e86dSJan Kara if (likely(!bt)) { 1053c780e86dSJan Kara rcu_read_unlock(); 1054b0da3f0dSJun'ichi Nomura return; 1055c780e86dSJan Kara } 1056b0da3f0dSJun'ichi Nomura 1057b0da3f0dSJun'ichi Nomura r.device_from = cpu_to_be32(dev); 1058f3fa33acSChristoph Hellwig r.device_to = cpu_to_be32(disk_devt(rq->q->disk)); 1059b0da3f0dSJun'ichi Nomura r.sector_from = cpu_to_be64(from); 1060b0da3f0dSJun'ichi Nomura 1061b0da3f0dSJun'ichi Nomura __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 1062caf7df12SChristoph Hellwig rq_data_dir(rq), 0, BLK_TA_REMAP, 0, 1063a54895faSChristoph Hellwig sizeof(r), &r, blk_trace_request_get_cgid(rq)); 1064c780e86dSJan Kara rcu_read_unlock(); 1065b0da3f0dSJun'ichi Nomura } 1066b0da3f0dSJun'ichi Nomura 1067b0da3f0dSJun'ichi Nomura /** 10682db270a8SFrederic Weisbecker * blk_add_driver_data - Add binary message with driver-specific data 10692db270a8SFrederic Weisbecker * @rq: io request 10702db270a8SFrederic Weisbecker * @data: driver-specific data 10712db270a8SFrederic Weisbecker * @len: length of driver-specific data 10722db270a8SFrederic Weisbecker * 10732db270a8SFrederic Weisbecker * Description: 10742db270a8SFrederic Weisbecker * Some drivers might want to write driver-specific data per request. 10752db270a8SFrederic Weisbecker * 10762db270a8SFrederic Weisbecker **/ 1077a54895faSChristoph Hellwig void blk_add_driver_data(struct request *rq, void *data, size_t len) 10782db270a8SFrederic Weisbecker { 1079c780e86dSJan Kara struct blk_trace *bt; 10802db270a8SFrederic Weisbecker 1081c780e86dSJan Kara rcu_read_lock(); 1082a54895faSChristoph Hellwig bt = rcu_dereference(rq->q->blk_trace); 1083c780e86dSJan Kara if (likely(!bt)) { 1084c780e86dSJan Kara rcu_read_unlock(); 10852db270a8SFrederic Weisbecker return; 1086c780e86dSJan Kara } 10872db270a8SFrederic Weisbecker 108848b77ad6SChristoph Hellwig __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 0, 1089ca1136c9SShaohua Li BLK_TA_DRV_DATA, 0, len, data, 1090a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 1091c780e86dSJan Kara rcu_read_unlock(); 10922db270a8SFrederic Weisbecker } 10932db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_add_driver_data); 10942db270a8SFrederic Weisbecker 10953c289ba7SLi Zefan static void blk_register_tracepoints(void) 10962db270a8SFrederic Weisbecker { 10972db270a8SFrederic Weisbecker int ret; 10982db270a8SFrederic Weisbecker 109938516ab5SSteven Rostedt ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 11002db270a8SFrederic Weisbecker WARN_ON(ret); 110138516ab5SSteven Rostedt ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 11022db270a8SFrederic Weisbecker WARN_ON(ret); 1103f3bdc62fSJan Kara ret = register_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 1104f3bdc62fSJan Kara WARN_ON(ret); 110538516ab5SSteven Rostedt ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 11062db270a8SFrederic Weisbecker WARN_ON(ret); 110738516ab5SSteven Rostedt ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 11082db270a8SFrederic Weisbecker WARN_ON(ret); 110938516ab5SSteven Rostedt ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); 11102db270a8SFrederic Weisbecker WARN_ON(ret); 111138516ab5SSteven Rostedt ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 11122db270a8SFrederic Weisbecker WARN_ON(ret); 111338516ab5SSteven Rostedt ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 11142db270a8SFrederic Weisbecker WARN_ON(ret); 111538516ab5SSteven Rostedt ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 11162db270a8SFrederic Weisbecker WARN_ON(ret); 111738516ab5SSteven Rostedt ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 11182db270a8SFrederic Weisbecker WARN_ON(ret); 111938516ab5SSteven Rostedt ret = register_trace_block_getrq(blk_add_trace_getrq, NULL); 11202db270a8SFrederic Weisbecker WARN_ON(ret); 112138516ab5SSteven Rostedt ret = register_trace_block_plug(blk_add_trace_plug, NULL); 11222db270a8SFrederic Weisbecker WARN_ON(ret); 112349cac01eSJens Axboe ret = register_trace_block_unplug(blk_add_trace_unplug, NULL); 11242db270a8SFrederic Weisbecker WARN_ON(ret); 112538516ab5SSteven Rostedt ret = register_trace_block_split(blk_add_trace_split, NULL); 11262db270a8SFrederic Weisbecker WARN_ON(ret); 1127d07335e5SMike Snitzer ret = register_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 11282db270a8SFrederic Weisbecker WARN_ON(ret); 112938516ab5SSteven Rostedt ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1130b0da3f0dSJun'ichi Nomura WARN_ON(ret); 11312db270a8SFrederic Weisbecker } 11322db270a8SFrederic Weisbecker 11332db270a8SFrederic Weisbecker static void blk_unregister_tracepoints(void) 11342db270a8SFrederic Weisbecker { 113538516ab5SSteven Rostedt unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1136d07335e5SMike Snitzer unregister_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 113738516ab5SSteven Rostedt unregister_trace_block_split(blk_add_trace_split, NULL); 113849cac01eSJens Axboe unregister_trace_block_unplug(blk_add_trace_unplug, NULL); 113938516ab5SSteven Rostedt unregister_trace_block_plug(blk_add_trace_plug, NULL); 114038516ab5SSteven Rostedt unregister_trace_block_getrq(blk_add_trace_getrq, NULL); 114138516ab5SSteven Rostedt unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 114238516ab5SSteven Rostedt unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 114338516ab5SSteven Rostedt unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 114438516ab5SSteven Rostedt unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 114538516ab5SSteven Rostedt unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); 114638516ab5SSteven Rostedt unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 114738516ab5SSteven Rostedt unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 1148f3bdc62fSJan Kara unregister_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 114938516ab5SSteven Rostedt unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 115038516ab5SSteven Rostedt unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 11512db270a8SFrederic Weisbecker 11522db270a8SFrederic Weisbecker tracepoint_synchronize_unregister(); 11532db270a8SFrederic Weisbecker } 11542db270a8SFrederic Weisbecker 11552db270a8SFrederic Weisbecker /* 11562db270a8SFrederic Weisbecker * struct blk_io_tracer formatting routines 11572db270a8SFrederic Weisbecker */ 11582db270a8SFrederic Weisbecker 11592db270a8SFrederic Weisbecker static void fill_rwbs(char *rwbs, const struct blk_io_trace *t) 11602db270a8SFrederic Weisbecker { 11612db270a8SFrederic Weisbecker int i = 0; 116265796348SLi Zefan int tc = t->action >> BLK_TC_SHIFT; 11632db270a8SFrederic Weisbecker 1164ca1136c9SShaohua Li if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 116518cea459SLi Zefan rwbs[i++] = 'N'; 116618cea459SLi Zefan goto out; 116718cea459SLi Zefan } 116818cea459SLi Zefan 1169c09c47caSNamhyung Kim if (tc & BLK_TC_FLUSH) 1170c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1171c09c47caSNamhyung Kim 117265796348SLi Zefan if (tc & BLK_TC_DISCARD) 11732db270a8SFrederic Weisbecker rwbs[i++] = 'D'; 117465796348SLi Zefan else if (tc & BLK_TC_WRITE) 11752db270a8SFrederic Weisbecker rwbs[i++] = 'W'; 11762db270a8SFrederic Weisbecker else if (t->bytes) 11772db270a8SFrederic Weisbecker rwbs[i++] = 'R'; 11782db270a8SFrederic Weisbecker else 11792db270a8SFrederic Weisbecker rwbs[i++] = 'N'; 11802db270a8SFrederic Weisbecker 1181c09c47caSNamhyung Kim if (tc & BLK_TC_FUA) 1182c09c47caSNamhyung Kim rwbs[i++] = 'F'; 118365796348SLi Zefan if (tc & BLK_TC_AHEAD) 11842db270a8SFrederic Weisbecker rwbs[i++] = 'A'; 118565796348SLi Zefan if (tc & BLK_TC_SYNC) 11862db270a8SFrederic Weisbecker rwbs[i++] = 'S'; 118765796348SLi Zefan if (tc & BLK_TC_META) 11882db270a8SFrederic Weisbecker rwbs[i++] = 'M'; 118918cea459SLi Zefan out: 11902db270a8SFrederic Weisbecker rwbs[i] = '\0'; 11912db270a8SFrederic Weisbecker } 11922db270a8SFrederic Weisbecker 11932db270a8SFrederic Weisbecker static inline 11942db270a8SFrederic Weisbecker const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent) 11952db270a8SFrederic Weisbecker { 11962db270a8SFrederic Weisbecker return (const struct blk_io_trace *)ent; 11972db270a8SFrederic Weisbecker } 11982db270a8SFrederic Weisbecker 1199ca1136c9SShaohua Li static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg) 12002db270a8SFrederic Weisbecker { 120167c0496eSTejun Heo return (void *)(te_blk_io_trace(ent) + 1) + (has_cg ? sizeof(u64) : 0); 1202ca1136c9SShaohua Li } 1203ca1136c9SShaohua Li 120467c0496eSTejun Heo static inline u64 t_cgid(const struct trace_entry *ent) 1205ca1136c9SShaohua Li { 120667c0496eSTejun Heo return *(u64 *)(te_blk_io_trace(ent) + 1); 1207ca1136c9SShaohua Li } 1208ca1136c9SShaohua Li 1209ca1136c9SShaohua Li static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg) 1210ca1136c9SShaohua Li { 121167c0496eSTejun Heo return te_blk_io_trace(ent)->pdu_len - (has_cg ? sizeof(u64) : 0); 12122db270a8SFrederic Weisbecker } 12132db270a8SFrederic Weisbecker 121466de7792SLi Zefan static inline u32 t_action(const struct trace_entry *ent) 121566de7792SLi Zefan { 121666de7792SLi Zefan return te_blk_io_trace(ent)->action; 121766de7792SLi Zefan } 121866de7792SLi Zefan 121966de7792SLi Zefan static inline u32 t_bytes(const struct trace_entry *ent) 122066de7792SLi Zefan { 122166de7792SLi Zefan return te_blk_io_trace(ent)->bytes; 122266de7792SLi Zefan } 122366de7792SLi Zefan 12242db270a8SFrederic Weisbecker static inline u32 t_sec(const struct trace_entry *ent) 12252db270a8SFrederic Weisbecker { 12262db270a8SFrederic Weisbecker return te_blk_io_trace(ent)->bytes >> 9; 12272db270a8SFrederic Weisbecker } 12282db270a8SFrederic Weisbecker 12292db270a8SFrederic Weisbecker static inline unsigned long long t_sector(const struct trace_entry *ent) 12302db270a8SFrederic Weisbecker { 12312db270a8SFrederic Weisbecker return te_blk_io_trace(ent)->sector; 12322db270a8SFrederic Weisbecker } 12332db270a8SFrederic Weisbecker 12342db270a8SFrederic Weisbecker static inline __u16 t_error(const struct trace_entry *ent) 12352db270a8SFrederic Weisbecker { 1236e0dc81beSLi Zefan return te_blk_io_trace(ent)->error; 12372db270a8SFrederic Weisbecker } 12382db270a8SFrederic Weisbecker 1239ca1136c9SShaohua Li static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg) 12402db270a8SFrederic Weisbecker { 124171df3fd8SChaitanya Kulkarni const __be64 *val = pdu_start(ent, has_cg); 12422db270a8SFrederic Weisbecker return be64_to_cpu(*val); 12432db270a8SFrederic Weisbecker } 12442db270a8SFrederic Weisbecker 1245ca1136c9SShaohua Li typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act, 1246ca1136c9SShaohua Li bool has_cg); 1247b6a4b0c3SLi Zefan 1248ca1136c9SShaohua Li static void blk_log_action_classic(struct trace_iterator *iter, const char *act, 1249ca1136c9SShaohua Li bool has_cg) 12502db270a8SFrederic Weisbecker { 1251c09c47caSNamhyung Kim char rwbs[RWBS_LEN]; 125235ac51bfSLi Zefan unsigned long long ts = iter->ts; 125335ac51bfSLi Zefan unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC); 12542db270a8SFrederic Weisbecker unsigned secs = (unsigned long)ts; 1255b6a4b0c3SLi Zefan const struct blk_io_trace *t = te_blk_io_trace(iter->ent); 12562db270a8SFrederic Weisbecker 12572db270a8SFrederic Weisbecker fill_rwbs(rwbs, t); 12582db270a8SFrederic Weisbecker 1259f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(&iter->seq, 126035ac51bfSLi Zefan "%3d,%-3d %2d %5d.%09lu %5u %2s %3s ", 12612db270a8SFrederic Weisbecker MAJOR(t->device), MINOR(t->device), iter->cpu, 1262b6a4b0c3SLi Zefan secs, nsec_rem, iter->ent->pid, act, rwbs); 12632db270a8SFrederic Weisbecker } 12642db270a8SFrederic Weisbecker 1265ca1136c9SShaohua Li static void blk_log_action(struct trace_iterator *iter, const char *act, 1266ca1136c9SShaohua Li bool has_cg) 12672db270a8SFrederic Weisbecker { 1268c09c47caSNamhyung Kim char rwbs[RWBS_LEN]; 1269b6a4b0c3SLi Zefan const struct blk_io_trace *t = te_blk_io_trace(iter->ent); 1270b6a4b0c3SLi Zefan 12712db270a8SFrederic Weisbecker fill_rwbs(rwbs, t); 1272ca1136c9SShaohua Li if (has_cg) { 127367c0496eSTejun Heo u64 id = t_cgid(iter->ent); 1274ca1136c9SShaohua Li 127569fd5c39SShaohua Li if (blk_tracer_flags.val & TRACE_BLK_OPT_CGNAME) { 127669fd5c39SShaohua Li char blkcg_name_buf[NAME_MAX + 1] = "<...>"; 127769fd5c39SShaohua Li 127869fd5c39SShaohua Li cgroup_path_from_kernfs_id(id, blkcg_name_buf, 127969fd5c39SShaohua Li sizeof(blkcg_name_buf)); 128069fd5c39SShaohua Li trace_seq_printf(&iter->seq, "%3d,%-3d %s %2s %3s ", 128169fd5c39SShaohua Li MAJOR(t->device), MINOR(t->device), 128269fd5c39SShaohua Li blkcg_name_buf, act, rwbs); 128340430452STejun Heo } else { 128440430452STejun Heo /* 128540430452STejun Heo * The cgid portion used to be "INO,GEN". Userland 128640430452STejun Heo * builds a FILEID_INO32_GEN fid out of them and 128740430452STejun Heo * opens the cgroup using open_by_handle_at(2). 128840430452STejun Heo * While 32bit ino setups are still the same, 64bit 128940430452STejun Heo * ones now use the 64bit ino as the whole ID and 129040430452STejun Heo * no longer use generation. 129140430452STejun Heo * 12922b5894ccSQiujun Huang * Regardless of the content, always output 129340430452STejun Heo * "LOW32,HIGH32" so that FILEID_INO32_GEN fid can 129440430452STejun Heo * be mapped back to @id on both 64 and 32bit ino 129540430452STejun Heo * setups. See __kernfs_fh_to_dentry(). 129640430452STejun Heo */ 129769fd5c39SShaohua Li trace_seq_printf(&iter->seq, 129840430452STejun Heo "%3d,%-3d %llx,%-llx %2s %3s ", 1299ca1136c9SShaohua Li MAJOR(t->device), MINOR(t->device), 130040430452STejun Heo id & U32_MAX, id >> 32, act, rwbs); 130140430452STejun Heo } 1302ca1136c9SShaohua Li } else 1303f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(&iter->seq, "%3d,%-3d %2s %3s ", 13042db270a8SFrederic Weisbecker MAJOR(t->device), MINOR(t->device), act, rwbs); 13052db270a8SFrederic Weisbecker } 13062db270a8SFrederic Weisbecker 1307ca1136c9SShaohua Li static void blk_log_dump_pdu(struct trace_seq *s, 1308ca1136c9SShaohua Li const struct trace_entry *ent, bool has_cg) 130966de7792SLi Zefan { 131004986257SLi Zefan const unsigned char *pdu_buf; 131166de7792SLi Zefan int pdu_len; 1312f4a1d08cSSteven Rostedt (Red Hat) int i, end; 131366de7792SLi Zefan 1314ca1136c9SShaohua Li pdu_buf = pdu_start(ent, has_cg); 1315ca1136c9SShaohua Li pdu_len = pdu_real_len(ent, has_cg); 131666de7792SLi Zefan 131766de7792SLi Zefan if (!pdu_len) 1318f4a1d08cSSteven Rostedt (Red Hat) return; 131966de7792SLi Zefan 132066de7792SLi Zefan /* find the last zero that needs to be printed */ 132166de7792SLi Zefan for (end = pdu_len - 1; end >= 0; end--) 132266de7792SLi Zefan if (pdu_buf[end]) 132366de7792SLi Zefan break; 132466de7792SLi Zefan end++; 132566de7792SLi Zefan 1326f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putc(s, '('); 132766de7792SLi Zefan 132866de7792SLi Zefan for (i = 0; i < pdu_len; i++) { 132966de7792SLi Zefan 1330f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%s%02x", 133166de7792SLi Zefan i == 0 ? "" : " ", pdu_buf[i]); 133266de7792SLi Zefan 133366de7792SLi Zefan /* 13342b5894ccSQiujun Huang * stop when the rest is just zeros and indicate so 133566de7792SLi Zefan * with a ".." appended 133666de7792SLi Zefan */ 1337f4a1d08cSSteven Rostedt (Red Hat) if (i == end && end != pdu_len - 1) { 1338f4a1d08cSSteven Rostedt (Red Hat) trace_seq_puts(s, " ..) "); 1339f4a1d08cSSteven Rostedt (Red Hat) return; 1340f4a1d08cSSteven Rostedt (Red Hat) } 134166de7792SLi Zefan } 134266de7792SLi Zefan 1343f4a1d08cSSteven Rostedt (Red Hat) trace_seq_puts(s, ") "); 134466de7792SLi Zefan } 134566de7792SLi Zefan 1346ca1136c9SShaohua Li static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13472db270a8SFrederic Weisbecker { 13484ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 13494ca53085SSteven Rostedt 13504ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 13512db270a8SFrederic Weisbecker 135266de7792SLi Zefan if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1353f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%u ", t_bytes(ent)); 1354ca1136c9SShaohua Li blk_log_dump_pdu(s, ent, has_cg); 1355f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 135666de7792SLi Zefan } else { 13572db270a8SFrederic Weisbecker if (t_sec(ent)) 1358f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u [%s]\n", 13592db270a8SFrederic Weisbecker t_sector(ent), t_sec(ent), cmd); 1360f4a1d08cSSteven Rostedt (Red Hat) else 1361f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 13622db270a8SFrederic Weisbecker } 136366de7792SLi Zefan } 13642db270a8SFrederic Weisbecker 1365f4a1d08cSSteven Rostedt (Red Hat) static void blk_log_with_error(struct trace_seq *s, 1366ca1136c9SShaohua Li const struct trace_entry *ent, bool has_cg) 13672db270a8SFrederic Weisbecker { 136866de7792SLi Zefan if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1369ca1136c9SShaohua Li blk_log_dump_pdu(s, ent, has_cg); 1370f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%d]\n", t_error(ent)); 137166de7792SLi Zefan } else { 13722db270a8SFrederic Weisbecker if (t_sec(ent)) 1373f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u [%d]\n", 137466de7792SLi Zefan t_sector(ent), 13752db270a8SFrederic Weisbecker t_sec(ent), t_error(ent)); 1376f4a1d08cSSteven Rostedt (Red Hat) else 1377f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu [%d]\n", 137866de7792SLi Zefan t_sector(ent), t_error(ent)); 137966de7792SLi Zefan } 13802db270a8SFrederic Weisbecker } 13812db270a8SFrederic Weisbecker 1382ca1136c9SShaohua Li static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13832db270a8SFrederic Weisbecker { 13845aec598cSChaitanya Kulkarni const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg); 13852db270a8SFrederic Weisbecker 1386f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n", 1387a42aaa3bSAlan D. Brunelle t_sector(ent), t_sec(ent), 13885aec598cSChaitanya Kulkarni MAJOR(be32_to_cpu(__r->device_from)), 13895aec598cSChaitanya Kulkarni MINOR(be32_to_cpu(__r->device_from)), 13905aec598cSChaitanya Kulkarni be64_to_cpu(__r->sector_from)); 13912db270a8SFrederic Weisbecker } 13922db270a8SFrederic Weisbecker 1393ca1136c9SShaohua Li static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13942db270a8SFrederic Weisbecker { 13954ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 13964ca53085SSteven Rostedt 13974ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 13984ca53085SSteven Rostedt 1399f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 14002db270a8SFrederic Weisbecker } 14012db270a8SFrederic Weisbecker 1402ca1136c9SShaohua Li static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 14032db270a8SFrederic Weisbecker { 14044ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 14054ca53085SSteven Rostedt 14064ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 14074ca53085SSteven Rostedt 1408ca1136c9SShaohua Li trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent, has_cg)); 14092db270a8SFrederic Weisbecker } 14102db270a8SFrederic Weisbecker 1411ca1136c9SShaohua Li static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 14122db270a8SFrederic Weisbecker { 14134ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 14144ca53085SSteven Rostedt 14154ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 14164ca53085SSteven Rostedt 1417f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent), 1418ca1136c9SShaohua Li get_pdu_int(ent, has_cg), cmd); 14192db270a8SFrederic Weisbecker } 14202db270a8SFrederic Weisbecker 1421ca1136c9SShaohua Li static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent, 1422ca1136c9SShaohua Li bool has_cg) 142318cea459SLi Zefan { 142418cea459SLi Zefan 1425ca1136c9SShaohua Li trace_seq_putmem(s, pdu_start(ent, has_cg), 1426ca1136c9SShaohua Li pdu_real_len(ent, has_cg)); 1427f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putc(s, '\n'); 142818cea459SLi Zefan } 142918cea459SLi Zefan 14302db270a8SFrederic Weisbecker /* 14312db270a8SFrederic Weisbecker * struct tracer operations 14322db270a8SFrederic Weisbecker */ 14332db270a8SFrederic Weisbecker 14342db270a8SFrederic Weisbecker static void blk_tracer_print_header(struct seq_file *m) 14352db270a8SFrederic Weisbecker { 14362db270a8SFrederic Weisbecker if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 14372db270a8SFrederic Weisbecker return; 14382db270a8SFrederic Weisbecker seq_puts(m, "# DEV CPU TIMESTAMP PID ACT FLG\n" 14392db270a8SFrederic Weisbecker "# | | | | | |\n"); 14402db270a8SFrederic Weisbecker } 14412db270a8SFrederic Weisbecker 14422db270a8SFrederic Weisbecker static void blk_tracer_start(struct trace_array *tr) 14432db270a8SFrederic Weisbecker { 1444ad5dd549SLi Zefan blk_tracer_enabled = true; 14452db270a8SFrederic Weisbecker } 14462db270a8SFrederic Weisbecker 14472db270a8SFrederic Weisbecker static int blk_tracer_init(struct trace_array *tr) 14482db270a8SFrederic Weisbecker { 14492db270a8SFrederic Weisbecker blk_tr = tr; 14502db270a8SFrederic Weisbecker blk_tracer_start(tr); 14512db270a8SFrederic Weisbecker return 0; 14522db270a8SFrederic Weisbecker } 14532db270a8SFrederic Weisbecker 14542db270a8SFrederic Weisbecker static void blk_tracer_stop(struct trace_array *tr) 14552db270a8SFrederic Weisbecker { 1456ad5dd549SLi Zefan blk_tracer_enabled = false; 14572db270a8SFrederic Weisbecker } 14582db270a8SFrederic Weisbecker 14592db270a8SFrederic Weisbecker static void blk_tracer_reset(struct trace_array *tr) 14602db270a8SFrederic Weisbecker { 14612db270a8SFrederic Weisbecker blk_tracer_stop(tr); 14622db270a8SFrederic Weisbecker } 14632db270a8SFrederic Weisbecker 1464e4955c99SLi Zefan static const struct { 14652db270a8SFrederic Weisbecker const char *act[2]; 1466ca1136c9SShaohua Li void (*print)(struct trace_seq *s, const struct trace_entry *ent, 1467ca1136c9SShaohua Li bool has_cg); 1468e4955c99SLi Zefan } what2act[] = { 14692db270a8SFrederic Weisbecker [__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic }, 14702db270a8SFrederic Weisbecker [__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic }, 14712db270a8SFrederic Weisbecker [__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic }, 14722db270a8SFrederic Weisbecker [__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic }, 14732db270a8SFrederic Weisbecker [__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic }, 14742db270a8SFrederic Weisbecker [__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error }, 14752db270a8SFrederic Weisbecker [__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic }, 14762db270a8SFrederic Weisbecker [__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error }, 14772db270a8SFrederic Weisbecker [__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug }, 14782db270a8SFrederic Weisbecker [__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug }, 147949cac01eSJens Axboe [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug }, 14802db270a8SFrederic Weisbecker [__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic }, 14812db270a8SFrederic Weisbecker [__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split }, 14822db270a8SFrederic Weisbecker [__BLK_TA_BOUNCE] = {{ "B", "bounce" }, blk_log_generic }, 14832db270a8SFrederic Weisbecker [__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap }, 14842db270a8SFrederic Weisbecker }; 14852db270a8SFrederic Weisbecker 1486b6a4b0c3SLi Zefan static enum print_line_t print_one_line(struct trace_iterator *iter, 1487b6a4b0c3SLi Zefan bool classic) 14882db270a8SFrederic Weisbecker { 1489983f938aSSteven Rostedt (Red Hat) struct trace_array *tr = iter->tr; 14902db270a8SFrederic Weisbecker struct trace_seq *s = &iter->seq; 1491b6a4b0c3SLi Zefan const struct blk_io_trace *t; 1492b6a4b0c3SLi Zefan u16 what; 1493b6a4b0c3SLi Zefan bool long_act; 1494b6a4b0c3SLi Zefan blk_log_action_t *log_action; 1495ca1136c9SShaohua Li bool has_cg; 14962db270a8SFrederic Weisbecker 1497b6a4b0c3SLi Zefan t = te_blk_io_trace(iter->ent); 1498ca1136c9SShaohua Li what = (t->action & ((1 << BLK_TC_SHIFT) - 1)) & ~__BLK_TA_CGROUP; 1499983f938aSSteven Rostedt (Red Hat) long_act = !!(tr->trace_flags & TRACE_ITER_VERBOSE); 1500b6a4b0c3SLi Zefan log_action = classic ? &blk_log_action_classic : &blk_log_action; 1501ca1136c9SShaohua Li has_cg = t->action & __BLK_TA_CGROUP; 15022db270a8SFrederic Weisbecker 1503ca1136c9SShaohua Li if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 1504ca1136c9SShaohua Li log_action(iter, long_act ? "message" : "m", has_cg); 1505ca1136c9SShaohua Li blk_log_msg(s, iter->ent, has_cg); 1506b7d7641eSShaohua Li return trace_handle_return(s); 150718cea459SLi Zefan } 150818cea459SLi Zefan 1509eb08f8ebSLi Zefan if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act))) 1510f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "Unknown action %x\n", what); 15112db270a8SFrederic Weisbecker else { 1512ca1136c9SShaohua Li log_action(iter, what2act[what].act[long_act], has_cg); 1513ca1136c9SShaohua Li what2act[what].print(s, iter->ent, has_cg); 15142db270a8SFrederic Weisbecker } 1515f4a1d08cSSteven Rostedt (Red Hat) 1516f4a1d08cSSteven Rostedt (Red Hat) return trace_handle_return(s); 15172db270a8SFrederic Weisbecker } 15182db270a8SFrederic Weisbecker 1519b6a4b0c3SLi Zefan static enum print_line_t blk_trace_event_print(struct trace_iterator *iter, 1520a9a57763SSteven Rostedt int flags, struct trace_event *event) 1521b6a4b0c3SLi Zefan { 1522b6a4b0c3SLi Zefan return print_one_line(iter, false); 1523b6a4b0c3SLi Zefan } 1524b6a4b0c3SLi Zefan 1525f4a1d08cSSteven Rostedt (Red Hat) static void blk_trace_synthesize_old_trace(struct trace_iterator *iter) 15262db270a8SFrederic Weisbecker { 15272db270a8SFrederic Weisbecker struct trace_seq *s = &iter->seq; 15282db270a8SFrederic Weisbecker struct blk_io_trace *t = (struct blk_io_trace *)iter->ent; 15292db270a8SFrederic Weisbecker const int offset = offsetof(struct blk_io_trace, sector); 15302db270a8SFrederic Weisbecker struct blk_io_trace old = { 15312db270a8SFrederic Weisbecker .magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION, 15326c051ce0SLi Zefan .time = iter->ts, 15332db270a8SFrederic Weisbecker }; 15342db270a8SFrederic Weisbecker 1535f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putmem(s, &old, offset); 1536f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putmem(s, &t->sector, 15372db270a8SFrederic Weisbecker sizeof(old) - offset + t->pdu_len); 15382db270a8SFrederic Weisbecker } 15392db270a8SFrederic Weisbecker 15402db270a8SFrederic Weisbecker static enum print_line_t 1541a9a57763SSteven Rostedt blk_trace_event_print_binary(struct trace_iterator *iter, int flags, 1542a9a57763SSteven Rostedt struct trace_event *event) 15432db270a8SFrederic Weisbecker { 1544f4a1d08cSSteven Rostedt (Red Hat) blk_trace_synthesize_old_trace(iter); 1545f4a1d08cSSteven Rostedt (Red Hat) 1546f4a1d08cSSteven Rostedt (Red Hat) return trace_handle_return(&iter->seq); 15472db270a8SFrederic Weisbecker } 15482db270a8SFrederic Weisbecker 15492db270a8SFrederic Weisbecker static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter) 15502db270a8SFrederic Weisbecker { 15512db270a8SFrederic Weisbecker if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 15522db270a8SFrederic Weisbecker return TRACE_TYPE_UNHANDLED; 15532db270a8SFrederic Weisbecker 1554b6a4b0c3SLi Zefan return print_one_line(iter, true); 15552db270a8SFrederic Weisbecker } 15562db270a8SFrederic Weisbecker 15578c1a49aeSSteven Rostedt (Red Hat) static int 15588c1a49aeSSteven Rostedt (Red Hat) blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1559f3948f88SLi Zefan { 1560f3948f88SLi Zefan /* don't output context-info for blk_classic output */ 1561f3948f88SLi Zefan if (bit == TRACE_BLK_OPT_CLASSIC) { 1562f3948f88SLi Zefan if (set) 1563983f938aSSteven Rostedt (Red Hat) tr->trace_flags &= ~TRACE_ITER_CONTEXT_INFO; 1564f3948f88SLi Zefan else 1565983f938aSSteven Rostedt (Red Hat) tr->trace_flags |= TRACE_ITER_CONTEXT_INFO; 1566f3948f88SLi Zefan } 1567f3948f88SLi Zefan return 0; 1568f3948f88SLi Zefan } 1569f3948f88SLi Zefan 15702db270a8SFrederic Weisbecker static struct tracer blk_tracer __read_mostly = { 15712db270a8SFrederic Weisbecker .name = "blk", 15722db270a8SFrederic Weisbecker .init = blk_tracer_init, 15732db270a8SFrederic Weisbecker .reset = blk_tracer_reset, 15742db270a8SFrederic Weisbecker .start = blk_tracer_start, 15752db270a8SFrederic Weisbecker .stop = blk_tracer_stop, 15762db270a8SFrederic Weisbecker .print_header = blk_tracer_print_header, 15772db270a8SFrederic Weisbecker .print_line = blk_tracer_print_line, 15782db270a8SFrederic Weisbecker .flags = &blk_tracer_flags, 1579f3948f88SLi Zefan .set_flag = blk_tracer_set_flag, 15802db270a8SFrederic Weisbecker }; 15812db270a8SFrederic Weisbecker 1582a9a57763SSteven Rostedt static struct trace_event_functions trace_blk_event_funcs = { 15832db270a8SFrederic Weisbecker .trace = blk_trace_event_print, 15842db270a8SFrederic Weisbecker .binary = blk_trace_event_print_binary, 15852db270a8SFrederic Weisbecker }; 15862db270a8SFrederic Weisbecker 1587a9a57763SSteven Rostedt static struct trace_event trace_blk_event = { 1588a9a57763SSteven Rostedt .type = TRACE_BLK, 1589a9a57763SSteven Rostedt .funcs = &trace_blk_event_funcs, 1590a9a57763SSteven Rostedt }; 1591a9a57763SSteven Rostedt 15922db270a8SFrederic Weisbecker static int __init init_blk_tracer(void) 15932db270a8SFrederic Weisbecker { 15949023c930SSteven Rostedt (Red Hat) if (!register_trace_event(&trace_blk_event)) { 1595a395d6a7SJoe Perches pr_warn("Warning: could not register block events\n"); 15962db270a8SFrederic Weisbecker return 1; 15972db270a8SFrederic Weisbecker } 15982db270a8SFrederic Weisbecker 15992db270a8SFrederic Weisbecker if (register_tracer(&blk_tracer) != 0) { 1600a395d6a7SJoe Perches pr_warn("Warning: could not register the block tracer\n"); 16019023c930SSteven Rostedt (Red Hat) unregister_trace_event(&trace_blk_event); 16022db270a8SFrederic Weisbecker return 1; 16032db270a8SFrederic Weisbecker } 16042db270a8SFrederic Weisbecker 16052db270a8SFrederic Weisbecker return 0; 16062db270a8SFrederic Weisbecker } 16072db270a8SFrederic Weisbecker 16082db270a8SFrederic Weisbecker device_initcall(init_blk_tracer); 16092db270a8SFrederic Weisbecker 16102db270a8SFrederic Weisbecker static int blk_trace_remove_queue(struct request_queue *q) 16112db270a8SFrederic Weisbecker { 16122db270a8SFrederic Weisbecker struct blk_trace *bt; 16132db270a8SFrederic Weisbecker 1614c3dbe541SJan Kara bt = rcu_replace_pointer(q->blk_trace, NULL, 161585e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 16162db270a8SFrederic Weisbecker if (bt == NULL) 16172db270a8SFrederic Weisbecker return -EINVAL; 16182db270a8SFrederic Weisbecker 16195afedf67SZhihao Cheng if (bt->trace_state == Blktrace_running) { 16205afedf67SZhihao Cheng bt->trace_state = Blktrace_stopped; 1621361c81dbSWander Lairson Costa raw_spin_lock_irq(&running_trace_lock); 16225afedf67SZhihao Cheng list_del_init(&bt->running_list); 1623361c81dbSWander Lairson Costa raw_spin_unlock_irq(&running_trace_lock); 16245afedf67SZhihao Cheng relay_flush(bt->rchan); 16255afedf67SZhihao Cheng } 16265afedf67SZhihao Cheng 1627a6da0024SJens Axboe put_probe_ref(); 1628c780e86dSJan Kara synchronize_rcu(); 1629*30939293SYu Kuai blk_trace_free(q, bt); 16302db270a8SFrederic Weisbecker return 0; 16312db270a8SFrederic Weisbecker } 16322db270a8SFrederic Weisbecker 16332db270a8SFrederic Weisbecker /* 16342db270a8SFrederic Weisbecker * Setup everything required to start tracing 16352db270a8SFrederic Weisbecker */ 16369908c309SLi Zefan static int blk_trace_setup_queue(struct request_queue *q, 16379908c309SLi Zefan struct block_device *bdev) 16382db270a8SFrederic Weisbecker { 1639cdea01b2SDavidlohr Bueso struct blk_trace *bt = NULL; 164018cea459SLi Zefan int ret = -ENOMEM; 16412db270a8SFrederic Weisbecker 16422db270a8SFrederic Weisbecker bt = kzalloc(sizeof(*bt), GFP_KERNEL); 16432db270a8SFrederic Weisbecker if (!bt) 164415152e44SLi Zefan return -ENOMEM; 16452db270a8SFrederic Weisbecker 164618cea459SLi Zefan bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 164718cea459SLi Zefan if (!bt->msg_data) 164818cea459SLi Zefan goto free_bt; 164918cea459SLi Zefan 16509908c309SLi Zefan bt->dev = bdev->bd_dev; 16512db270a8SFrederic Weisbecker bt->act_mask = (u16)-1; 16529908c309SLi Zefan 16539908c309SLi Zefan blk_trace_setup_lba(bt, bdev); 16542db270a8SFrederic Weisbecker 1655c3dbe541SJan Kara rcu_assign_pointer(q->blk_trace, bt); 1656a6da0024SJens Axboe get_probe_ref(); 16572db270a8SFrederic Weisbecker return 0; 165818cea459SLi Zefan 165918cea459SLi Zefan free_bt: 1660*30939293SYu Kuai blk_trace_free(q, bt); 166118cea459SLi Zefan return ret; 16622db270a8SFrederic Weisbecker } 16632db270a8SFrederic Weisbecker 16642db270a8SFrederic Weisbecker /* 16652db270a8SFrederic Weisbecker * sysfs interface to enable and configure tracing 16662db270a8SFrederic Weisbecker */ 16672db270a8SFrederic Weisbecker 16682db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 16692db270a8SFrederic Weisbecker struct device_attribute *attr, 16702db270a8SFrederic Weisbecker char *buf); 16712db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 16722db270a8SFrederic Weisbecker struct device_attribute *attr, 16732db270a8SFrederic Weisbecker const char *buf, size_t count); 16742db270a8SFrederic Weisbecker #define BLK_TRACE_DEVICE_ATTR(_name) \ 16752db270a8SFrederic Weisbecker DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \ 16762db270a8SFrederic Weisbecker sysfs_blk_trace_attr_show, \ 16772db270a8SFrederic Weisbecker sysfs_blk_trace_attr_store) 16782db270a8SFrederic Weisbecker 1679cd649b8bSLi Zefan static BLK_TRACE_DEVICE_ATTR(enable); 16802db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(act_mask); 16812db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(pid); 16822db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(start_lba); 16832db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(end_lba); 16842db270a8SFrederic Weisbecker 16852db270a8SFrederic Weisbecker static struct attribute *blk_trace_attrs[] = { 16862db270a8SFrederic Weisbecker &dev_attr_enable.attr, 16872db270a8SFrederic Weisbecker &dev_attr_act_mask.attr, 16882db270a8SFrederic Weisbecker &dev_attr_pid.attr, 16892db270a8SFrederic Weisbecker &dev_attr_start_lba.attr, 16902db270a8SFrederic Weisbecker &dev_attr_end_lba.attr, 16912db270a8SFrederic Weisbecker NULL 16922db270a8SFrederic Weisbecker }; 16932db270a8SFrederic Weisbecker 16942db270a8SFrederic Weisbecker struct attribute_group blk_trace_attr_group = { 16952db270a8SFrederic Weisbecker .name = "trace", 16962db270a8SFrederic Weisbecker .attrs = blk_trace_attrs, 16972db270a8SFrederic Weisbecker }; 16982db270a8SFrederic Weisbecker 169909341997SLi Zefan static const struct { 170009341997SLi Zefan int mask; 170109341997SLi Zefan const char *str; 170209341997SLi Zefan } mask_maps[] = { 170309341997SLi Zefan { BLK_TC_READ, "read" }, 170409341997SLi Zefan { BLK_TC_WRITE, "write" }, 1705c09c47caSNamhyung Kim { BLK_TC_FLUSH, "flush" }, 170609341997SLi Zefan { BLK_TC_SYNC, "sync" }, 170709341997SLi Zefan { BLK_TC_QUEUE, "queue" }, 170809341997SLi Zefan { BLK_TC_REQUEUE, "requeue" }, 170909341997SLi Zefan { BLK_TC_ISSUE, "issue" }, 171009341997SLi Zefan { BLK_TC_COMPLETE, "complete" }, 171109341997SLi Zefan { BLK_TC_FS, "fs" }, 171209341997SLi Zefan { BLK_TC_PC, "pc" }, 17138d1547e0SShaohua Li { BLK_TC_NOTIFY, "notify" }, 171409341997SLi Zefan { BLK_TC_AHEAD, "ahead" }, 171509341997SLi Zefan { BLK_TC_META, "meta" }, 171609341997SLi Zefan { BLK_TC_DISCARD, "discard" }, 171709341997SLi Zefan { BLK_TC_DRV_DATA, "drv_data" }, 1718c09c47caSNamhyung Kim { BLK_TC_FUA, "fua" }, 171909341997SLi Zefan }; 172009341997SLi Zefan 172109341997SLi Zefan static int blk_trace_str2mask(const char *str) 17222db270a8SFrederic Weisbecker { 172309341997SLi Zefan int i; 17242db270a8SFrederic Weisbecker int mask = 0; 17259eb85125SLi Zefan char *buf, *s, *token; 17262db270a8SFrederic Weisbecker 17279eb85125SLi Zefan buf = kstrdup(str, GFP_KERNEL); 17289eb85125SLi Zefan if (buf == NULL) 17292db270a8SFrederic Weisbecker return -ENOMEM; 17309eb85125SLi Zefan s = strstrip(buf); 17312db270a8SFrederic Weisbecker 17322db270a8SFrederic Weisbecker while (1) { 173309341997SLi Zefan token = strsep(&s, ","); 173409341997SLi Zefan if (token == NULL) 17352db270a8SFrederic Weisbecker break; 17362db270a8SFrederic Weisbecker 173709341997SLi Zefan if (*token == '\0') 173809341997SLi Zefan continue; 173909341997SLi Zefan 174009341997SLi Zefan for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 174109341997SLi Zefan if (strcasecmp(token, mask_maps[i].str) == 0) { 174209341997SLi Zefan mask |= mask_maps[i].mask; 174309341997SLi Zefan break; 17442db270a8SFrederic Weisbecker } 174509341997SLi Zefan } 174609341997SLi Zefan if (i == ARRAY_SIZE(mask_maps)) { 174709341997SLi Zefan mask = -EINVAL; 174809341997SLi Zefan break; 174909341997SLi Zefan } 175009341997SLi Zefan } 17519eb85125SLi Zefan kfree(buf); 17522db270a8SFrederic Weisbecker 17532db270a8SFrederic Weisbecker return mask; 17542db270a8SFrederic Weisbecker } 17552db270a8SFrederic Weisbecker 175609341997SLi Zefan static ssize_t blk_trace_mask2str(char *buf, int mask) 175709341997SLi Zefan { 175809341997SLi Zefan int i; 175909341997SLi Zefan char *p = buf; 176009341997SLi Zefan 176109341997SLi Zefan for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 176209341997SLi Zefan if (mask & mask_maps[i].mask) { 176309341997SLi Zefan p += sprintf(p, "%s%s", 176409341997SLi Zefan (p == buf) ? "" : ",", mask_maps[i].str); 176509341997SLi Zefan } 176609341997SLi Zefan } 176709341997SLi Zefan *p++ = '\n'; 176809341997SLi Zefan 176909341997SLi Zefan return p - buf; 177009341997SLi Zefan } 177109341997SLi Zefan 17722db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 17732db270a8SFrederic Weisbecker struct device_attribute *attr, 17742db270a8SFrederic Weisbecker char *buf) 17752db270a8SFrederic Weisbecker { 17760d02129eSChristoph Hellwig struct block_device *bdev = dev_to_bdev(dev); 17770d02129eSChristoph Hellwig struct request_queue *q = bdev_get_queue(bdev); 1778c780e86dSJan Kara struct blk_trace *bt; 17792db270a8SFrederic Weisbecker ssize_t ret = -ENXIO; 17802db270a8SFrederic Weisbecker 178185e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 1782cd649b8bSLi Zefan 1783c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 178485e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1785cd649b8bSLi Zefan if (attr == &dev_attr_enable) { 1786c780e86dSJan Kara ret = sprintf(buf, "%u\n", !!bt); 1787cd649b8bSLi Zefan goto out_unlock_bdev; 1788cd649b8bSLi Zefan } 1789cd649b8bSLi Zefan 1790c780e86dSJan Kara if (bt == NULL) 17912db270a8SFrederic Weisbecker ret = sprintf(buf, "disabled\n"); 17922db270a8SFrederic Weisbecker else if (attr == &dev_attr_act_mask) 1793c780e86dSJan Kara ret = blk_trace_mask2str(buf, bt->act_mask); 17942db270a8SFrederic Weisbecker else if (attr == &dev_attr_pid) 1795c780e86dSJan Kara ret = sprintf(buf, "%u\n", bt->pid); 17962db270a8SFrederic Weisbecker else if (attr == &dev_attr_start_lba) 1797c780e86dSJan Kara ret = sprintf(buf, "%llu\n", bt->start_lba); 17982db270a8SFrederic Weisbecker else if (attr == &dev_attr_end_lba) 1799c780e86dSJan Kara ret = sprintf(buf, "%llu\n", bt->end_lba); 1800cd649b8bSLi Zefan 1801cd649b8bSLi Zefan out_unlock_bdev: 180285e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 18032db270a8SFrederic Weisbecker return ret; 18042db270a8SFrederic Weisbecker } 18052db270a8SFrederic Weisbecker 18062db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 18072db270a8SFrederic Weisbecker struct device_attribute *attr, 18082db270a8SFrederic Weisbecker const char *buf, size_t count) 18092db270a8SFrederic Weisbecker { 18100d02129eSChristoph Hellwig struct block_device *bdev = dev_to_bdev(dev); 18110d02129eSChristoph Hellwig struct request_queue *q = bdev_get_queue(bdev); 1812c780e86dSJan Kara struct blk_trace *bt; 18132db270a8SFrederic Weisbecker u64 value; 181409341997SLi Zefan ssize_t ret = -EINVAL; 18152db270a8SFrederic Weisbecker 18162db270a8SFrederic Weisbecker if (count == 0) 18172db270a8SFrederic Weisbecker goto out; 18182db270a8SFrederic Weisbecker 18192db270a8SFrederic Weisbecker if (attr == &dev_attr_act_mask) { 18205f339453SShaohua Li if (kstrtoull(buf, 0, &value)) { 18212db270a8SFrederic Weisbecker /* Assume it is a list of trace category names */ 182209341997SLi Zefan ret = blk_trace_str2mask(buf); 182309341997SLi Zefan if (ret < 0) 18242db270a8SFrederic Weisbecker goto out; 182509341997SLi Zefan value = ret; 18262db270a8SFrederic Weisbecker } 18270d02129eSChristoph Hellwig } else { 18280d02129eSChristoph Hellwig if (kstrtoull(buf, 0, &value)) 18292db270a8SFrederic Weisbecker goto out; 18300d02129eSChristoph Hellwig } 18312db270a8SFrederic Weisbecker 183285e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 1833cd649b8bSLi Zefan 1834c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 183585e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1836cd649b8bSLi Zefan if (attr == &dev_attr_enable) { 1837c780e86dSJan Kara if (!!value == !!bt) { 1838757d9140SSteven Rostedt (VMware) ret = 0; 1839757d9140SSteven Rostedt (VMware) goto out_unlock_bdev; 1840757d9140SSteven Rostedt (VMware) } 1841cd649b8bSLi Zefan if (value) 18429908c309SLi Zefan ret = blk_trace_setup_queue(q, bdev); 1843cd649b8bSLi Zefan else 1844cd649b8bSLi Zefan ret = blk_trace_remove_queue(q); 1845cd649b8bSLi Zefan goto out_unlock_bdev; 1846cd649b8bSLi Zefan } 1847cd649b8bSLi Zefan 18482db270a8SFrederic Weisbecker ret = 0; 1849153031a3SCengiz Can if (bt == NULL) { 18509908c309SLi Zefan ret = blk_trace_setup_queue(q, bdev); 1851153031a3SCengiz Can bt = rcu_dereference_protected(q->blk_trace, 185285e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1853153031a3SCengiz Can } 18542db270a8SFrederic Weisbecker 18552db270a8SFrederic Weisbecker if (ret == 0) { 18562db270a8SFrederic Weisbecker if (attr == &dev_attr_act_mask) 1857c780e86dSJan Kara bt->act_mask = value; 18582db270a8SFrederic Weisbecker else if (attr == &dev_attr_pid) 1859c780e86dSJan Kara bt->pid = value; 18602db270a8SFrederic Weisbecker else if (attr == &dev_attr_start_lba) 1861c780e86dSJan Kara bt->start_lba = value; 18622db270a8SFrederic Weisbecker else if (attr == &dev_attr_end_lba) 1863c780e86dSJan Kara bt->end_lba = value; 18642db270a8SFrederic Weisbecker } 1865cd649b8bSLi Zefan 1866cd649b8bSLi Zefan out_unlock_bdev: 186785e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 18682db270a8SFrederic Weisbecker out: 1869cd649b8bSLi Zefan return ret ? ret : count; 18702db270a8SFrederic Weisbecker } 1871cd649b8bSLi Zefan 18721d54ad6dSLi Zefan int blk_trace_init_sysfs(struct device *dev) 18731d54ad6dSLi Zefan { 18741d54ad6dSLi Zefan return sysfs_create_group(&dev->kobj, &blk_trace_attr_group); 18751d54ad6dSLi Zefan } 18761d54ad6dSLi Zefan 187748c0d4d4SZdenek Kabelac void blk_trace_remove_sysfs(struct device *dev) 187848c0d4d4SZdenek Kabelac { 187948c0d4d4SZdenek Kabelac sysfs_remove_group(&dev->kobj, &blk_trace_attr_group); 188048c0d4d4SZdenek Kabelac } 188148c0d4d4SZdenek Kabelac 188255782138SLi Zefan #endif /* CONFIG_BLK_DEV_IO_TRACE */ 188355782138SLi Zefan 188455782138SLi Zefan #ifdef CONFIG_EVENT_TRACING 188555782138SLi Zefan 18861f83bb4bSChaitanya Kulkarni /** 18871f83bb4bSChaitanya Kulkarni * blk_fill_rwbs - Fill the buffer rwbs by mapping op to character string. 188894d4bffdSChaitanya Kulkarni * @rwbs: buffer to be filled 18891f83bb4bSChaitanya Kulkarni * @op: REQ_OP_XXX for the tracepoint 18901f83bb4bSChaitanya Kulkarni * 18911f83bb4bSChaitanya Kulkarni * Description: 18921f83bb4bSChaitanya Kulkarni * Maps the REQ_OP_XXX to character and fills the buffer provided by the 18931f83bb4bSChaitanya Kulkarni * caller with resulting string. 18941f83bb4bSChaitanya Kulkarni * 18951f83bb4bSChaitanya Kulkarni **/ 1896179d1600SChaitanya Kulkarni void blk_fill_rwbs(char *rwbs, unsigned int op) 189755782138SLi Zefan { 189855782138SLi Zefan int i = 0; 189955782138SLi Zefan 1900ef295ecfSChristoph Hellwig if (op & REQ_PREFLUSH) 1901c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1902c09c47caSNamhyung Kim 1903ef295ecfSChristoph Hellwig switch (op & REQ_OP_MASK) { 19041b9a9ab7SMike Christie case REQ_OP_WRITE: 19051b9a9ab7SMike Christie case REQ_OP_WRITE_SAME: 190655782138SLi Zefan rwbs[i++] = 'W'; 19071b9a9ab7SMike Christie break; 19081b9a9ab7SMike Christie case REQ_OP_DISCARD: 190955782138SLi Zefan rwbs[i++] = 'D'; 19101b9a9ab7SMike Christie break; 1911288dab8aSChristoph Hellwig case REQ_OP_SECURE_ERASE: 1912288dab8aSChristoph Hellwig rwbs[i++] = 'D'; 1913288dab8aSChristoph Hellwig rwbs[i++] = 'E'; 1914288dab8aSChristoph Hellwig break; 19153a5e02ceSMike Christie case REQ_OP_FLUSH: 19163a5e02ceSMike Christie rwbs[i++] = 'F'; 19173a5e02ceSMike Christie break; 19181b9a9ab7SMike Christie case REQ_OP_READ: 191955782138SLi Zefan rwbs[i++] = 'R'; 19201b9a9ab7SMike Christie break; 19211b9a9ab7SMike Christie default: 192255782138SLi Zefan rwbs[i++] = 'N'; 19231b9a9ab7SMike Christie } 192455782138SLi Zefan 1925ef295ecfSChristoph Hellwig if (op & REQ_FUA) 1926c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1927ef295ecfSChristoph Hellwig if (op & REQ_RAHEAD) 192855782138SLi Zefan rwbs[i++] = 'A'; 1929ef295ecfSChristoph Hellwig if (op & REQ_SYNC) 193055782138SLi Zefan rwbs[i++] = 'S'; 1931ef295ecfSChristoph Hellwig if (op & REQ_META) 193255782138SLi Zefan rwbs[i++] = 'M'; 193355782138SLi Zefan 193455782138SLi Zefan rwbs[i] = '\0'; 193555782138SLi Zefan } 19369ca8f8e5SKent Overstreet EXPORT_SYMBOL_GPL(blk_fill_rwbs); 193755782138SLi Zefan 193855782138SLi Zefan #endif /* CONFIG_EVENT_TRACING */ 193955782138SLi Zefan 1940