191c1e6baSSteven Rostedt (VMware) // SPDX-License-Identifier: GPL-2.0 22db270a8SFrederic Weisbecker /* 32db270a8SFrederic Weisbecker * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk> 42db270a8SFrederic Weisbecker * 52db270a8SFrederic Weisbecker */ 61b0b2836SLuis Chamberlain 71b0b2836SLuis Chamberlain #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 81b0b2836SLuis Chamberlain 92db270a8SFrederic Weisbecker #include <linux/kernel.h> 102db270a8SFrederic Weisbecker #include <linux/blkdev.h> 112db270a8SFrederic Weisbecker #include <linux/blktrace_api.h> 122db270a8SFrederic Weisbecker #include <linux/percpu.h> 132db270a8SFrederic Weisbecker #include <linux/init.h> 142db270a8SFrederic Weisbecker #include <linux/mutex.h> 155a0e3ad6STejun Heo #include <linux/slab.h> 162db270a8SFrederic Weisbecker #include <linux/debugfs.h> 176e5fdeedSPaul Gortmaker #include <linux/export.h> 182db270a8SFrederic Weisbecker #include <linux/time.h> 192db270a8SFrederic Weisbecker #include <linux/uaccess.h> 20a404d557SJan Kara #include <linux/list.h> 21ca1136c9SShaohua Li #include <linux/blk-cgroup.h> 2255782138SLi Zefan 2318fbda91SOmar Sandoval #include "../../block/blk.h" 2418fbda91SOmar Sandoval 2555782138SLi Zefan #include <trace/events/block.h> 2655782138SLi Zefan 272db270a8SFrederic Weisbecker #include "trace_output.h" 282db270a8SFrederic Weisbecker 2955782138SLi Zefan #ifdef CONFIG_BLK_DEV_IO_TRACE 3055782138SLi Zefan 312db270a8SFrederic Weisbecker static unsigned int blktrace_seq __read_mostly = 1; 322db270a8SFrederic Weisbecker 332db270a8SFrederic Weisbecker static struct trace_array *blk_tr; 345006ea73SLi Zefan static bool blk_tracer_enabled __read_mostly; 352db270a8SFrederic Weisbecker 36a404d557SJan Kara static LIST_HEAD(running_trace_list); 37361c81dbSWander Lairson Costa static __cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(running_trace_lock); 38a404d557SJan Kara 392db270a8SFrederic Weisbecker /* Select an alternative, minimalistic output than the original one */ 402db270a8SFrederic Weisbecker #define TRACE_BLK_OPT_CLASSIC 0x1 41ca1136c9SShaohua Li #define TRACE_BLK_OPT_CGROUP 0x2 4269fd5c39SShaohua Li #define TRACE_BLK_OPT_CGNAME 0x4 432db270a8SFrederic Weisbecker 442db270a8SFrederic Weisbecker static struct tracer_opt blk_tracer_opts[] = { 452db270a8SFrederic Weisbecker /* Default disable the minimalistic output */ 462db270a8SFrederic Weisbecker { TRACER_OPT(blk_classic, TRACE_BLK_OPT_CLASSIC) }, 47ca1136c9SShaohua Li #ifdef CONFIG_BLK_CGROUP 48ca1136c9SShaohua Li { TRACER_OPT(blk_cgroup, TRACE_BLK_OPT_CGROUP) }, 4969fd5c39SShaohua Li { TRACER_OPT(blk_cgname, TRACE_BLK_OPT_CGNAME) }, 50ca1136c9SShaohua Li #endif 512db270a8SFrederic Weisbecker { } 522db270a8SFrederic Weisbecker }; 532db270a8SFrederic Weisbecker 542db270a8SFrederic Weisbecker static struct tracer_flags blk_tracer_flags = { 552db270a8SFrederic Weisbecker .val = 0, 562db270a8SFrederic Weisbecker .opts = blk_tracer_opts, 572db270a8SFrederic Weisbecker }; 582db270a8SFrederic Weisbecker 592db270a8SFrederic Weisbecker /* Global reference count of probes */ 60a6da0024SJens Axboe static DEFINE_MUTEX(blk_probe_mutex); 61a6da0024SJens Axboe static int blk_probes_ref; 622db270a8SFrederic Weisbecker 633c289ba7SLi Zefan static void blk_register_tracepoints(void); 642db270a8SFrederic Weisbecker static void blk_unregister_tracepoints(void); 652db270a8SFrederic Weisbecker 662db270a8SFrederic Weisbecker /* 672db270a8SFrederic Weisbecker * Send out a notify message. 682db270a8SFrederic Weisbecker */ 692db270a8SFrederic Weisbecker static void trace_note(struct blk_trace *bt, pid_t pid, int action, 7067c0496eSTejun Heo const void *data, size_t len, u64 cgid) 712db270a8SFrederic Weisbecker { 722db270a8SFrederic Weisbecker struct blk_io_trace *t; 7318cea459SLi Zefan struct ring_buffer_event *event = NULL; 7413292494SSteven Rostedt (VMware) struct trace_buffer *buffer = NULL; 7536590c50SSebastian Andrzej Siewior unsigned int trace_ctx = 0; 7618cea459SLi Zefan int cpu = smp_processor_id(); 7718cea459SLi Zefan bool blk_tracer = blk_tracer_enabled; 7867c0496eSTejun Heo ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 7918cea459SLi Zefan 8018cea459SLi Zefan if (blk_tracer) { 811c5eb448SSteven Rostedt (VMware) buffer = blk_tr->array_buffer.buffer; 8236590c50SSebastian Andrzej Siewior trace_ctx = tracing_gen_ctx_flags(0); 83e77405adSSteven Rostedt event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 84ca1136c9SShaohua Li sizeof(*t) + len + cgid_len, 8536590c50SSebastian Andrzej Siewior trace_ctx); 8618cea459SLi Zefan if (!event) 8718cea459SLi Zefan return; 8818cea459SLi Zefan t = ring_buffer_event_data(event); 8918cea459SLi Zefan goto record_it; 9018cea459SLi Zefan } 912db270a8SFrederic Weisbecker 922db270a8SFrederic Weisbecker if (!bt->rchan) 932db270a8SFrederic Weisbecker return; 942db270a8SFrederic Weisbecker 95ca1136c9SShaohua Li t = relay_reserve(bt->rchan, sizeof(*t) + len + cgid_len); 962db270a8SFrederic Weisbecker if (t) { 972db270a8SFrederic Weisbecker t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 982db270a8SFrederic Weisbecker t->time = ktime_to_ns(ktime_get()); 9918cea459SLi Zefan record_it: 1002db270a8SFrederic Weisbecker t->device = bt->dev; 101ca1136c9SShaohua Li t->action = action | (cgid ? __BLK_TN_CGROUP : 0); 1022db270a8SFrederic Weisbecker t->pid = pid; 1032db270a8SFrederic Weisbecker t->cpu = cpu; 104ca1136c9SShaohua Li t->pdu_len = len + cgid_len; 10567c0496eSTejun Heo if (cgid_len) 10667c0496eSTejun Heo memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 107ca1136c9SShaohua Li memcpy((void *) t + sizeof(*t) + cgid_len, data, len); 10818cea459SLi Zefan 10918cea459SLi Zefan if (blk_tracer) 11036590c50SSebastian Andrzej Siewior trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx); 1112db270a8SFrederic Weisbecker } 1122db270a8SFrederic Weisbecker } 1132db270a8SFrederic Weisbecker 1142db270a8SFrederic Weisbecker /* 1152db270a8SFrederic Weisbecker * Send out a notify for this process, if we haven't done so since a trace 1162db270a8SFrederic Weisbecker * started 1172db270a8SFrederic Weisbecker */ 118a404d557SJan Kara static void trace_note_tsk(struct task_struct *tsk) 1192db270a8SFrederic Weisbecker { 120a404d557SJan Kara unsigned long flags; 121a404d557SJan Kara struct blk_trace *bt; 122a404d557SJan Kara 1232db270a8SFrederic Weisbecker tsk->btrace_seq = blktrace_seq; 124361c81dbSWander Lairson Costa raw_spin_lock_irqsave(&running_trace_lock, flags); 125a404d557SJan Kara list_for_each_entry(bt, &running_trace_list, running_list) { 126a404d557SJan Kara trace_note(bt, tsk->pid, BLK_TN_PROCESS, tsk->comm, 12767c0496eSTejun Heo sizeof(tsk->comm), 0); 128a404d557SJan Kara } 129361c81dbSWander Lairson Costa raw_spin_unlock_irqrestore(&running_trace_lock, flags); 1302db270a8SFrederic Weisbecker } 1312db270a8SFrederic Weisbecker 1322db270a8SFrederic Weisbecker static void trace_note_time(struct blk_trace *bt) 1332db270a8SFrederic Weisbecker { 13459a37f8bSArnd Bergmann struct timespec64 now; 1352db270a8SFrederic Weisbecker unsigned long flags; 1362db270a8SFrederic Weisbecker u32 words[2]; 1372db270a8SFrederic Weisbecker 13859a37f8bSArnd Bergmann /* need to check user space to see if this breaks in y2038 or y2106 */ 13959a37f8bSArnd Bergmann ktime_get_real_ts64(&now); 14059a37f8bSArnd Bergmann words[0] = (u32)now.tv_sec; 1412db270a8SFrederic Weisbecker words[1] = now.tv_nsec; 1422db270a8SFrederic Weisbecker 1432db270a8SFrederic Weisbecker local_irq_save(flags); 14467c0496eSTejun Heo trace_note(bt, 0, BLK_TN_TIMESTAMP, words, sizeof(words), 0); 1452db270a8SFrederic Weisbecker local_irq_restore(flags); 1462db270a8SFrederic Weisbecker } 1472db270a8SFrederic Weisbecker 148f4a6a61cSChristoph Hellwig void __blk_trace_note_message(struct blk_trace *bt, 149f4a6a61cSChristoph Hellwig struct cgroup_subsys_state *css, const char *fmt, ...) 1502db270a8SFrederic Weisbecker { 1512db270a8SFrederic Weisbecker int n; 1522db270a8SFrederic Weisbecker va_list args; 1532db270a8SFrederic Weisbecker unsigned long flags; 1542db270a8SFrederic Weisbecker char *buf; 155f4a6a61cSChristoph Hellwig u64 cgid = 0; 1562db270a8SFrederic Weisbecker 15718cea459SLi Zefan if (unlikely(bt->trace_state != Blktrace_running && 15818cea459SLi Zefan !blk_tracer_enabled)) 1592db270a8SFrederic Weisbecker return; 1602db270a8SFrederic Weisbecker 161490da40dSTao Ma /* 162490da40dSTao Ma * If the BLK_TC_NOTIFY action mask isn't set, don't send any note 163490da40dSTao Ma * message to the trace. 164490da40dSTao Ma */ 165490da40dSTao Ma if (!(bt->act_mask & BLK_TC_NOTIFY)) 166490da40dSTao Ma return; 167490da40dSTao Ma 1682db270a8SFrederic Weisbecker local_irq_save(flags); 169d8a0349cSShan Wei buf = this_cpu_ptr(bt->msg_data); 1702db270a8SFrederic Weisbecker va_start(args, fmt); 1712db270a8SFrederic Weisbecker n = vscnprintf(buf, BLK_TN_MAX_MSG, fmt, args); 1722db270a8SFrederic Weisbecker va_end(args); 1732db270a8SFrederic Weisbecker 17435fe6d76SShaohua Li #ifdef CONFIG_BLK_CGROUP 175f4a6a61cSChristoph Hellwig if (css && (blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 176f4a6a61cSChristoph Hellwig cgid = cgroup_id(css->cgroup); 177f4a6a61cSChristoph Hellwig else 178f4a6a61cSChristoph Hellwig cgid = 1; 17935fe6d76SShaohua Li #endif 180f4a6a61cSChristoph Hellwig trace_note(bt, current->pid, BLK_TN_MESSAGE, buf, n, cgid); 1812db270a8SFrederic Weisbecker local_irq_restore(flags); 1822db270a8SFrederic Weisbecker } 183f4a6a61cSChristoph Hellwig EXPORT_SYMBOL_GPL(__blk_trace_note_message); 1842db270a8SFrederic Weisbecker 1852db270a8SFrederic Weisbecker static int act_log_check(struct blk_trace *bt, u32 what, sector_t sector, 1862db270a8SFrederic Weisbecker pid_t pid) 1872db270a8SFrederic Weisbecker { 1882db270a8SFrederic Weisbecker if (((bt->act_mask << BLK_TC_SHIFT) & what) == 0) 1892db270a8SFrederic Weisbecker return 1; 190d0deef5bSShawn Du if (sector && (sector < bt->start_lba || sector > bt->end_lba)) 1912db270a8SFrederic Weisbecker return 1; 1922db270a8SFrederic Weisbecker if (bt->pid && pid != bt->pid) 1932db270a8SFrederic Weisbecker return 1; 1942db270a8SFrederic Weisbecker 1952db270a8SFrederic Weisbecker return 0; 1962db270a8SFrederic Weisbecker } 1972db270a8SFrederic Weisbecker 1982db270a8SFrederic Weisbecker /* 1992db270a8SFrederic Weisbecker * Data direction bit lookup 2002db270a8SFrederic Weisbecker */ 201e4955c99SLi Zefan static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ), 2022db270a8SFrederic Weisbecker BLK_TC_ACT(BLK_TC_WRITE) }; 2032db270a8SFrederic Weisbecker 2047b6d91daSChristoph Hellwig #define BLK_TC_RAHEAD BLK_TC_AHEAD 20528a8f0d3SMike Christie #define BLK_TC_PREFLUSH BLK_TC_FLUSH 2067b6d91daSChristoph Hellwig 2072db270a8SFrederic Weisbecker /* The ilog2() calls fall out because they're constant */ 208919dbca8SBart Van Assche #define MASK_TC_BIT(rw, __name) ((__force u32)(rw & REQ_ ## __name) << \ 2097b6d91daSChristoph Hellwig (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name)) 2102db270a8SFrederic Weisbecker 2112db270a8SFrederic Weisbecker /* 2122db270a8SFrederic Weisbecker * The worker for the various blk_add_trace*() types. Fills out a 2132db270a8SFrederic Weisbecker * blk_io_trace structure and places it in a per-cpu subbuffer. 2142db270a8SFrederic Weisbecker */ 2152db270a8SFrederic Weisbecker static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes, 216919dbca8SBart Van Assche const blk_opf_t opf, u32 what, int error, 217919dbca8SBart Van Assche int pdu_len, void *pdu_data, u64 cgid) 2182db270a8SFrederic Weisbecker { 2192db270a8SFrederic Weisbecker struct task_struct *tsk = current; 2202db270a8SFrederic Weisbecker struct ring_buffer_event *event = NULL; 22113292494SSteven Rostedt (VMware) struct trace_buffer *buffer = NULL; 2222db270a8SFrederic Weisbecker struct blk_io_trace *t; 2232db270a8SFrederic Weisbecker unsigned long flags = 0; 2242db270a8SFrederic Weisbecker unsigned long *sequence; 22536590c50SSebastian Andrzej Siewior unsigned int trace_ctx = 0; 2262db270a8SFrederic Weisbecker pid_t pid; 22736590c50SSebastian Andrzej Siewior int cpu; 22818cea459SLi Zefan bool blk_tracer = blk_tracer_enabled; 22967c0496eSTejun Heo ssize_t cgid_len = cgid ? sizeof(cgid) : 0; 230919dbca8SBart Van Assche const enum req_op op = opf & REQ_OP_MASK; 2312db270a8SFrederic Weisbecker 23218cea459SLi Zefan if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer)) 2332db270a8SFrederic Weisbecker return; 2342db270a8SFrederic Weisbecker 2351b9a9ab7SMike Christie what |= ddir_act[op_is_write(op) ? WRITE : READ]; 236919dbca8SBart Van Assche what |= MASK_TC_BIT(opf, SYNC); 237919dbca8SBart Van Assche what |= MASK_TC_BIT(opf, RAHEAD); 238919dbca8SBart Van Assche what |= MASK_TC_BIT(opf, META); 239919dbca8SBart Van Assche what |= MASK_TC_BIT(opf, PREFLUSH); 240919dbca8SBart Van Assche what |= MASK_TC_BIT(opf, FUA); 2417afafc8aSAdrian Hunter if (op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE) 2421b9a9ab7SMike Christie what |= BLK_TC_ACT(BLK_TC_DISCARD); 2433a5e02ceSMike Christie if (op == REQ_OP_FLUSH) 2443a5e02ceSMike Christie what |= BLK_TC_ACT(BLK_TC_FLUSH); 245ca1136c9SShaohua Li if (cgid) 246ca1136c9SShaohua Li what |= __BLK_TA_CGROUP; 2472db270a8SFrederic Weisbecker 2482db270a8SFrederic Weisbecker pid = tsk->pid; 249d0deef5bSShawn Du if (act_log_check(bt, what, sector, pid)) 2502db270a8SFrederic Weisbecker return; 2512db270a8SFrederic Weisbecker cpu = raw_smp_processor_id(); 2522db270a8SFrederic Weisbecker 25318cea459SLi Zefan if (blk_tracer) { 2542db270a8SFrederic Weisbecker tracing_record_cmdline(current); 2552db270a8SFrederic Weisbecker 2561c5eb448SSteven Rostedt (VMware) buffer = blk_tr->array_buffer.buffer; 25736590c50SSebastian Andrzej Siewior trace_ctx = tracing_gen_ctx_flags(0); 258e77405adSSteven Rostedt event = trace_buffer_lock_reserve(buffer, TRACE_BLK, 259ca1136c9SShaohua Li sizeof(*t) + pdu_len + cgid_len, 26036590c50SSebastian Andrzej Siewior trace_ctx); 2612db270a8SFrederic Weisbecker if (!event) 2622db270a8SFrederic Weisbecker return; 2632db270a8SFrederic Weisbecker t = ring_buffer_event_data(event); 2642db270a8SFrederic Weisbecker goto record_it; 2652db270a8SFrederic Weisbecker } 2662db270a8SFrederic Weisbecker 267a404d557SJan Kara if (unlikely(tsk->btrace_seq != blktrace_seq)) 268a404d557SJan Kara trace_note_tsk(tsk); 269a404d557SJan Kara 2702db270a8SFrederic Weisbecker /* 2712db270a8SFrederic Weisbecker * A word about the locking here - we disable interrupts to reserve 2722db270a8SFrederic Weisbecker * some space in the relay per-cpu buffer, to prevent an irq 2732db270a8SFrederic Weisbecker * from coming in and stepping on our toes. 2742db270a8SFrederic Weisbecker */ 2752db270a8SFrederic Weisbecker local_irq_save(flags); 276ca1136c9SShaohua Li t = relay_reserve(bt->rchan, sizeof(*t) + pdu_len + cgid_len); 2772db270a8SFrederic Weisbecker if (t) { 2782db270a8SFrederic Weisbecker sequence = per_cpu_ptr(bt->sequence, cpu); 2792db270a8SFrederic Weisbecker 2802db270a8SFrederic Weisbecker t->magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION; 2812db270a8SFrederic Weisbecker t->sequence = ++(*sequence); 2822db270a8SFrederic Weisbecker t->time = ktime_to_ns(ktime_get()); 2832db270a8SFrederic Weisbecker record_it: 2842db270a8SFrederic Weisbecker /* 2852db270a8SFrederic Weisbecker * These two are not needed in ftrace as they are in the 2862db270a8SFrederic Weisbecker * generic trace_entry, filled by tracing_generic_entry_update, 2872db270a8SFrederic Weisbecker * but for the trace_event->bin() synthesizer benefit we do it 2882db270a8SFrederic Weisbecker * here too. 2892db270a8SFrederic Weisbecker */ 2902db270a8SFrederic Weisbecker t->cpu = cpu; 2912db270a8SFrederic Weisbecker t->pid = pid; 2922db270a8SFrederic Weisbecker 2932db270a8SFrederic Weisbecker t->sector = sector; 2942db270a8SFrederic Weisbecker t->bytes = bytes; 2952db270a8SFrederic Weisbecker t->action = what; 2962db270a8SFrederic Weisbecker t->device = bt->dev; 2972db270a8SFrederic Weisbecker t->error = error; 298ca1136c9SShaohua Li t->pdu_len = pdu_len + cgid_len; 2992db270a8SFrederic Weisbecker 300ca1136c9SShaohua Li if (cgid_len) 30167c0496eSTejun Heo memcpy((void *)t + sizeof(*t), &cgid, cgid_len); 3022db270a8SFrederic Weisbecker if (pdu_len) 303ca1136c9SShaohua Li memcpy((void *)t + sizeof(*t) + cgid_len, pdu_data, pdu_len); 3042db270a8SFrederic Weisbecker 30518cea459SLi Zefan if (blk_tracer) { 30636590c50SSebastian Andrzej Siewior trace_buffer_unlock_commit(blk_tr, buffer, event, trace_ctx); 3072db270a8SFrederic Weisbecker return; 3082db270a8SFrederic Weisbecker } 3092db270a8SFrederic Weisbecker } 3102db270a8SFrederic Weisbecker 3112db270a8SFrederic Weisbecker local_irq_restore(flags); 3122db270a8SFrederic Weisbecker } 3132db270a8SFrederic Weisbecker 31430939293SYu Kuai static void blk_trace_free(struct request_queue *q, struct blk_trace *bt) 3152db270a8SFrederic Weisbecker { 3162db270a8SFrederic Weisbecker relay_close(bt->rchan); 31730939293SYu Kuai 31830939293SYu Kuai /* 31930939293SYu Kuai * If 'bt->dir' is not set, then both 'dropped' and 'msg' are created 32030939293SYu Kuai * under 'q->debugfs_dir', thus lookup and remove them. 32130939293SYu Kuai */ 32230939293SYu Kuai if (!bt->dir) { 32330939293SYu Kuai debugfs_remove(debugfs_lookup("dropped", q->debugfs_dir)); 32430939293SYu Kuai debugfs_remove(debugfs_lookup("msg", q->debugfs_dir)); 32530939293SYu Kuai } else { 32639cbb602SAlan D. Brunelle debugfs_remove(bt->dir); 32730939293SYu Kuai } 3282db270a8SFrederic Weisbecker free_percpu(bt->sequence); 3292db270a8SFrederic Weisbecker free_percpu(bt->msg_data); 3302db270a8SFrederic Weisbecker kfree(bt); 331ad5dd549SLi Zefan } 332ad5dd549SLi Zefan 333a6da0024SJens Axboe static void get_probe_ref(void) 334a6da0024SJens Axboe { 335a6da0024SJens Axboe mutex_lock(&blk_probe_mutex); 336a6da0024SJens Axboe if (++blk_probes_ref == 1) 337a6da0024SJens Axboe blk_register_tracepoints(); 338a6da0024SJens Axboe mutex_unlock(&blk_probe_mutex); 339a6da0024SJens Axboe } 340a6da0024SJens Axboe 341a6da0024SJens Axboe static void put_probe_ref(void) 342a6da0024SJens Axboe { 343a6da0024SJens Axboe mutex_lock(&blk_probe_mutex); 344a6da0024SJens Axboe if (!--blk_probes_ref) 345a6da0024SJens Axboe blk_unregister_tracepoints(); 346a6da0024SJens Axboe mutex_unlock(&blk_probe_mutex); 347a6da0024SJens Axboe } 348a6da0024SJens Axboe 34960a9bb90SYe Bin static int blk_trace_start(struct blk_trace *bt) 35060a9bb90SYe Bin { 35160a9bb90SYe Bin if (bt->trace_state != Blktrace_setup && 35260a9bb90SYe Bin bt->trace_state != Blktrace_stopped) 35360a9bb90SYe Bin return -EINVAL; 35460a9bb90SYe Bin 35560a9bb90SYe Bin blktrace_seq++; 35660a9bb90SYe Bin smp_mb(); 35760a9bb90SYe Bin bt->trace_state = Blktrace_running; 35860a9bb90SYe Bin raw_spin_lock_irq(&running_trace_lock); 35960a9bb90SYe Bin list_add(&bt->running_list, &running_trace_list); 36060a9bb90SYe Bin raw_spin_unlock_irq(&running_trace_lock); 36160a9bb90SYe Bin trace_note_time(bt); 36260a9bb90SYe Bin 36360a9bb90SYe Bin return 0; 36460a9bb90SYe Bin } 36560a9bb90SYe Bin 36660a9bb90SYe Bin static int blk_trace_stop(struct blk_trace *bt) 36760a9bb90SYe Bin { 36860a9bb90SYe Bin if (bt->trace_state != Blktrace_running) 36960a9bb90SYe Bin return -EINVAL; 37060a9bb90SYe Bin 37160a9bb90SYe Bin bt->trace_state = Blktrace_stopped; 37260a9bb90SYe Bin raw_spin_lock_irq(&running_trace_lock); 37360a9bb90SYe Bin list_del_init(&bt->running_list); 37460a9bb90SYe Bin raw_spin_unlock_irq(&running_trace_lock); 37560a9bb90SYe Bin relay_flush(bt->rchan); 37660a9bb90SYe Bin 37760a9bb90SYe Bin return 0; 37860a9bb90SYe Bin } 37960a9bb90SYe Bin 38030939293SYu Kuai static void blk_trace_cleanup(struct request_queue *q, struct blk_trace *bt) 381ad5dd549SLi Zefan { 382*dcd1a59cSYe Bin blk_trace_stop(bt); 383c780e86dSJan Kara synchronize_rcu(); 38430939293SYu Kuai blk_trace_free(q, bt); 385a6da0024SJens Axboe put_probe_ref(); 3862db270a8SFrederic Weisbecker } 3872db270a8SFrederic Weisbecker 3881f2cac10SJens Axboe static int __blk_trace_remove(struct request_queue *q) 3892db270a8SFrederic Weisbecker { 3902db270a8SFrederic Weisbecker struct blk_trace *bt; 3912db270a8SFrederic Weisbecker 392c3dbe541SJan Kara bt = rcu_replace_pointer(q->blk_trace, NULL, 39385e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 3942db270a8SFrederic Weisbecker if (!bt) 3952db270a8SFrederic Weisbecker return -EINVAL; 3962db270a8SFrederic Weisbecker 39730939293SYu Kuai blk_trace_cleanup(q, bt); 3982db270a8SFrederic Weisbecker 3992db270a8SFrederic Weisbecker return 0; 4002db270a8SFrederic Weisbecker } 4011f2cac10SJens Axboe 4021f2cac10SJens Axboe int blk_trace_remove(struct request_queue *q) 4031f2cac10SJens Axboe { 4041f2cac10SJens Axboe int ret; 4051f2cac10SJens Axboe 40685e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 4071f2cac10SJens Axboe ret = __blk_trace_remove(q); 40885e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 4091f2cac10SJens Axboe 4101f2cac10SJens Axboe return ret; 4111f2cac10SJens Axboe } 4122db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_remove); 4132db270a8SFrederic Weisbecker 4142db270a8SFrederic Weisbecker static ssize_t blk_dropped_read(struct file *filp, char __user *buffer, 4152db270a8SFrederic Weisbecker size_t count, loff_t *ppos) 4162db270a8SFrederic Weisbecker { 4172db270a8SFrederic Weisbecker struct blk_trace *bt = filp->private_data; 4182db270a8SFrederic Weisbecker char buf[16]; 4192db270a8SFrederic Weisbecker 4202db270a8SFrederic Weisbecker snprintf(buf, sizeof(buf), "%u\n", atomic_read(&bt->dropped)); 4212db270a8SFrederic Weisbecker 4222db270a8SFrederic Weisbecker return simple_read_from_buffer(buffer, count, ppos, buf, strlen(buf)); 4232db270a8SFrederic Weisbecker } 4242db270a8SFrederic Weisbecker 4252db270a8SFrederic Weisbecker static const struct file_operations blk_dropped_fops = { 4262db270a8SFrederic Weisbecker .owner = THIS_MODULE, 427234e3405SStephen Boyd .open = simple_open, 4282db270a8SFrederic Weisbecker .read = blk_dropped_read, 4296038f373SArnd Bergmann .llseek = default_llseek, 4302db270a8SFrederic Weisbecker }; 4312db270a8SFrederic Weisbecker 4322db270a8SFrederic Weisbecker static ssize_t blk_msg_write(struct file *filp, const char __user *buffer, 4332db270a8SFrederic Weisbecker size_t count, loff_t *ppos) 4342db270a8SFrederic Weisbecker { 4352db270a8SFrederic Weisbecker char *msg; 4362db270a8SFrederic Weisbecker struct blk_trace *bt; 4372db270a8SFrederic Weisbecker 4387635b03aSLi Zefan if (count >= BLK_TN_MAX_MSG) 4392db270a8SFrederic Weisbecker return -EINVAL; 4402db270a8SFrederic Weisbecker 44116e5c1fcSAl Viro msg = memdup_user_nul(buffer, count); 44216e5c1fcSAl Viro if (IS_ERR(msg)) 44316e5c1fcSAl Viro return PTR_ERR(msg); 4442db270a8SFrederic Weisbecker 4452db270a8SFrederic Weisbecker bt = filp->private_data; 446f4a6a61cSChristoph Hellwig __blk_trace_note_message(bt, NULL, "%s", msg); 4472db270a8SFrederic Weisbecker kfree(msg); 4482db270a8SFrederic Weisbecker 4492db270a8SFrederic Weisbecker return count; 4502db270a8SFrederic Weisbecker } 4512db270a8SFrederic Weisbecker 4522db270a8SFrederic Weisbecker static const struct file_operations blk_msg_fops = { 4532db270a8SFrederic Weisbecker .owner = THIS_MODULE, 454234e3405SStephen Boyd .open = simple_open, 4552db270a8SFrederic Weisbecker .write = blk_msg_write, 4566038f373SArnd Bergmann .llseek = noop_llseek, 4572db270a8SFrederic Weisbecker }; 4582db270a8SFrederic Weisbecker 4592db270a8SFrederic Weisbecker /* 4602db270a8SFrederic Weisbecker * Keep track of how many times we encountered a full subbuffer, to aid 4612db270a8SFrederic Weisbecker * the user space app in telling how many lost events there were. 4622db270a8SFrederic Weisbecker */ 4632db270a8SFrederic Weisbecker static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf, 4642db270a8SFrederic Weisbecker void *prev_subbuf, size_t prev_padding) 4652db270a8SFrederic Weisbecker { 4662db270a8SFrederic Weisbecker struct blk_trace *bt; 4672db270a8SFrederic Weisbecker 4682db270a8SFrederic Weisbecker if (!relay_buf_full(buf)) 4692db270a8SFrederic Weisbecker return 1; 4702db270a8SFrederic Weisbecker 4712db270a8SFrederic Weisbecker bt = buf->chan->private_data; 4722db270a8SFrederic Weisbecker atomic_inc(&bt->dropped); 4732db270a8SFrederic Weisbecker return 0; 4742db270a8SFrederic Weisbecker } 4752db270a8SFrederic Weisbecker 4762db270a8SFrederic Weisbecker static int blk_remove_buf_file_callback(struct dentry *dentry) 4772db270a8SFrederic Weisbecker { 4782db270a8SFrederic Weisbecker debugfs_remove(dentry); 4792db270a8SFrederic Weisbecker 4802db270a8SFrederic Weisbecker return 0; 4812db270a8SFrederic Weisbecker } 4822db270a8SFrederic Weisbecker 4832db270a8SFrederic Weisbecker static struct dentry *blk_create_buf_file_callback(const char *filename, 4842db270a8SFrederic Weisbecker struct dentry *parent, 485f4ae40a6SAl Viro umode_t mode, 4862db270a8SFrederic Weisbecker struct rchan_buf *buf, 4872db270a8SFrederic Weisbecker int *is_global) 4882db270a8SFrederic Weisbecker { 4892db270a8SFrederic Weisbecker return debugfs_create_file(filename, mode, parent, buf, 4902db270a8SFrederic Weisbecker &relay_file_operations); 4912db270a8SFrederic Weisbecker } 4922db270a8SFrederic Weisbecker 493abf4e00cSJani Nikula static const struct rchan_callbacks blk_relay_callbacks = { 4942db270a8SFrederic Weisbecker .subbuf_start = blk_subbuf_start_callback, 4952db270a8SFrederic Weisbecker .create_buf_file = blk_create_buf_file_callback, 4962db270a8SFrederic Weisbecker .remove_buf_file = blk_remove_buf_file_callback, 4972db270a8SFrederic Weisbecker }; 4982db270a8SFrederic Weisbecker 4999908c309SLi Zefan static void blk_trace_setup_lba(struct blk_trace *bt, 5009908c309SLi Zefan struct block_device *bdev) 5019908c309SLi Zefan { 50229ff57c6SChristoph Hellwig if (bdev) { 50329ff57c6SChristoph Hellwig bt->start_lba = bdev->bd_start_sect; 50429ff57c6SChristoph Hellwig bt->end_lba = bdev->bd_start_sect + bdev_nr_sectors(bdev); 5059908c309SLi Zefan } else { 5069908c309SLi Zefan bt->start_lba = 0; 5079908c309SLi Zefan bt->end_lba = -1ULL; 5089908c309SLi Zefan } 5099908c309SLi Zefan } 5109908c309SLi Zefan 5112db270a8SFrederic Weisbecker /* 5122db270a8SFrederic Weisbecker * Setup everything required to start tracing 5132db270a8SFrederic Weisbecker */ 514a428d314SOmar Sandoval static int do_blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 515d0deef5bSShawn Du struct block_device *bdev, 5162db270a8SFrederic Weisbecker struct blk_user_trace_setup *buts) 5172db270a8SFrederic Weisbecker { 518cdea01b2SDavidlohr Bueso struct blk_trace *bt = NULL; 5192db270a8SFrederic Weisbecker struct dentry *dir = NULL; 520ff14417cSRasmus Villemoes int ret; 5212db270a8SFrederic Weisbecker 52285e0cbbbSLuis Chamberlain lockdep_assert_held(&q->debugfs_mutex); 523a67549c8SLuis Chamberlain 5242db270a8SFrederic Weisbecker if (!buts->buf_size || !buts->buf_nr) 5252db270a8SFrederic Weisbecker return -EINVAL; 5262db270a8SFrederic Weisbecker 5272db270a8SFrederic Weisbecker strncpy(buts->name, name, BLKTRACE_BDEV_SIZE); 5282db270a8SFrederic Weisbecker buts->name[BLKTRACE_BDEV_SIZE - 1] = '\0'; 5292db270a8SFrederic Weisbecker 5302db270a8SFrederic Weisbecker /* 5312db270a8SFrederic Weisbecker * some device names have larger paths - convert the slashes 5322db270a8SFrederic Weisbecker * to underscores for this to work as expected 5332db270a8SFrederic Weisbecker */ 534ff14417cSRasmus Villemoes strreplace(buts->name, '/', '_'); 5352db270a8SFrederic Weisbecker 5361b0b2836SLuis Chamberlain /* 5371b0b2836SLuis Chamberlain * bdev can be NULL, as with scsi-generic, this is a helpful as 5381b0b2836SLuis Chamberlain * we can be. 5391b0b2836SLuis Chamberlain */ 540c3dbe541SJan Kara if (rcu_dereference_protected(q->blk_trace, 54185e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex))) { 5421b0b2836SLuis Chamberlain pr_warn("Concurrent blktraces are not allowed on %s\n", 5431b0b2836SLuis Chamberlain buts->name); 5441b0b2836SLuis Chamberlain return -EBUSY; 5451b0b2836SLuis Chamberlain } 5461b0b2836SLuis Chamberlain 5472db270a8SFrederic Weisbecker bt = kzalloc(sizeof(*bt), GFP_KERNEL); 5482db270a8SFrederic Weisbecker if (!bt) 549ad5dd549SLi Zefan return -ENOMEM; 5502db270a8SFrederic Weisbecker 551ad5dd549SLi Zefan ret = -ENOMEM; 5522db270a8SFrederic Weisbecker bt->sequence = alloc_percpu(unsigned long); 5532db270a8SFrederic Weisbecker if (!bt->sequence) 5542db270a8SFrederic Weisbecker goto err; 5552db270a8SFrederic Weisbecker 556f0ef0398SIngo Molnar bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 5572db270a8SFrederic Weisbecker if (!bt->msg_data) 5582db270a8SFrederic Weisbecker goto err; 5592db270a8SFrederic Weisbecker 560bad8e64fSLuis Chamberlain /* 56185e0cbbbSLuis Chamberlain * When tracing the whole disk reuse the existing debugfs directory 56285e0cbbbSLuis Chamberlain * created by the block layer on init. For partitions block devices, 563bad8e64fSLuis Chamberlain * and scsi-generic block devices we create a temporary new debugfs 564bad8e64fSLuis Chamberlain * directory that will be removed once the trace ends. 565bad8e64fSLuis Chamberlain */ 566fa01b1e9SChristoph Hellwig if (bdev && !bdev_is_partition(bdev)) 567bad8e64fSLuis Chamberlain dir = q->debugfs_dir; 568bad8e64fSLuis Chamberlain else 5696ac93117SOmar Sandoval bt->dir = dir = debugfs_create_dir(buts->name, blk_debugfs_root); 5702db270a8SFrederic Weisbecker 571b431ef83SLuis Chamberlain /* 572b431ef83SLuis Chamberlain * As blktrace relies on debugfs for its interface the debugfs directory 573b431ef83SLuis Chamberlain * is required, contrary to the usual mantra of not checking for debugfs 574b431ef83SLuis Chamberlain * files or directories. 575b431ef83SLuis Chamberlain */ 576b431ef83SLuis Chamberlain if (IS_ERR_OR_NULL(dir)) { 577b431ef83SLuis Chamberlain pr_warn("debugfs_dir not present for %s so skipping\n", 578b431ef83SLuis Chamberlain buts->name); 579b431ef83SLuis Chamberlain ret = -ENOENT; 580b431ef83SLuis Chamberlain goto err; 581b431ef83SLuis Chamberlain } 582b431ef83SLuis Chamberlain 5832db270a8SFrederic Weisbecker bt->dev = dev; 5842db270a8SFrederic Weisbecker atomic_set(&bt->dropped, 0); 585a404d557SJan Kara INIT_LIST_HEAD(&bt->running_list); 5862db270a8SFrederic Weisbecker 5872db270a8SFrederic Weisbecker ret = -EIO; 588c0ea5760SGreg Kroah-Hartman debugfs_create_file("dropped", 0444, dir, bt, &blk_dropped_fops); 589c0ea5760SGreg Kroah-Hartman debugfs_create_file("msg", 0222, dir, bt, &blk_msg_fops); 5902db270a8SFrederic Weisbecker 5912db270a8SFrederic Weisbecker bt->rchan = relay_open("trace", dir, buts->buf_size, 5922db270a8SFrederic Weisbecker buts->buf_nr, &blk_relay_callbacks, bt); 5932db270a8SFrederic Weisbecker if (!bt->rchan) 5942db270a8SFrederic Weisbecker goto err; 5952db270a8SFrederic Weisbecker 5962db270a8SFrederic Weisbecker bt->act_mask = buts->act_mask; 5972db270a8SFrederic Weisbecker if (!bt->act_mask) 5982db270a8SFrederic Weisbecker bt->act_mask = (u16) -1; 5992db270a8SFrederic Weisbecker 6009908c309SLi Zefan blk_trace_setup_lba(bt, bdev); 6012db270a8SFrederic Weisbecker 602d0deef5bSShawn Du /* overwrite with user settings */ 603d0deef5bSShawn Du if (buts->start_lba) 604d0deef5bSShawn Du bt->start_lba = buts->start_lba; 605d0deef5bSShawn Du if (buts->end_lba) 606d0deef5bSShawn Du bt->end_lba = buts->end_lba; 607d0deef5bSShawn Du 6082db270a8SFrederic Weisbecker bt->pid = buts->pid; 6092db270a8SFrederic Weisbecker bt->trace_state = Blktrace_setup; 6102db270a8SFrederic Weisbecker 611c3dbe541SJan Kara rcu_assign_pointer(q->blk_trace, bt); 612a6da0024SJens Axboe get_probe_ref(); 613cbe28296SLi Zefan 6146ac93117SOmar Sandoval ret = 0; 6152db270a8SFrederic Weisbecker err: 6166ac93117SOmar Sandoval if (ret) 61730939293SYu Kuai blk_trace_free(q, bt); 6182db270a8SFrederic Weisbecker return ret; 6192db270a8SFrederic Weisbecker } 6202db270a8SFrederic Weisbecker 6211f2cac10SJens Axboe static int __blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 6221f2cac10SJens Axboe struct block_device *bdev, char __user *arg) 6232db270a8SFrederic Weisbecker { 6242db270a8SFrederic Weisbecker struct blk_user_trace_setup buts; 6252db270a8SFrederic Weisbecker int ret; 6262db270a8SFrederic Weisbecker 6272db270a8SFrederic Weisbecker ret = copy_from_user(&buts, arg, sizeof(buts)); 6282db270a8SFrederic Weisbecker if (ret) 6292db270a8SFrederic Weisbecker return -EFAULT; 6302db270a8SFrederic Weisbecker 631d0deef5bSShawn Du ret = do_blk_trace_setup(q, name, dev, bdev, &buts); 6322db270a8SFrederic Weisbecker if (ret) 6332db270a8SFrederic Weisbecker return ret; 6342db270a8SFrederic Weisbecker 6359a8c28c8SDmitry Monakhov if (copy_to_user(arg, &buts, sizeof(buts))) { 6362967acbbSJens Axboe __blk_trace_remove(q); 6372db270a8SFrederic Weisbecker return -EFAULT; 6389a8c28c8SDmitry Monakhov } 6392db270a8SFrederic Weisbecker return 0; 6402db270a8SFrederic Weisbecker } 6411f2cac10SJens Axboe 6421f2cac10SJens Axboe int blk_trace_setup(struct request_queue *q, char *name, dev_t dev, 6431f2cac10SJens Axboe struct block_device *bdev, 6441f2cac10SJens Axboe char __user *arg) 6451f2cac10SJens Axboe { 6461f2cac10SJens Axboe int ret; 6471f2cac10SJens Axboe 64885e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 6491f2cac10SJens Axboe ret = __blk_trace_setup(q, name, dev, bdev, arg); 65085e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 6511f2cac10SJens Axboe 6521f2cac10SJens Axboe return ret; 6531f2cac10SJens Axboe } 6542db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_setup); 6552db270a8SFrederic Weisbecker 65662c2a7d9SArnd Bergmann #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 65762c2a7d9SArnd Bergmann static int compat_blk_trace_setup(struct request_queue *q, char *name, 65862c2a7d9SArnd Bergmann dev_t dev, struct block_device *bdev, 65962c2a7d9SArnd Bergmann char __user *arg) 66062c2a7d9SArnd Bergmann { 66162c2a7d9SArnd Bergmann struct blk_user_trace_setup buts; 66262c2a7d9SArnd Bergmann struct compat_blk_user_trace_setup cbuts; 66362c2a7d9SArnd Bergmann int ret; 66462c2a7d9SArnd Bergmann 66562c2a7d9SArnd Bergmann if (copy_from_user(&cbuts, arg, sizeof(cbuts))) 66662c2a7d9SArnd Bergmann return -EFAULT; 66762c2a7d9SArnd Bergmann 66862c2a7d9SArnd Bergmann buts = (struct blk_user_trace_setup) { 66962c2a7d9SArnd Bergmann .act_mask = cbuts.act_mask, 67062c2a7d9SArnd Bergmann .buf_size = cbuts.buf_size, 67162c2a7d9SArnd Bergmann .buf_nr = cbuts.buf_nr, 67262c2a7d9SArnd Bergmann .start_lba = cbuts.start_lba, 67362c2a7d9SArnd Bergmann .end_lba = cbuts.end_lba, 67462c2a7d9SArnd Bergmann .pid = cbuts.pid, 67562c2a7d9SArnd Bergmann }; 67662c2a7d9SArnd Bergmann 67762c2a7d9SArnd Bergmann ret = do_blk_trace_setup(q, name, dev, bdev, &buts); 67862c2a7d9SArnd Bergmann if (ret) 67962c2a7d9SArnd Bergmann return ret; 68062c2a7d9SArnd Bergmann 681f8c5e944SChen Gang if (copy_to_user(arg, &buts.name, ARRAY_SIZE(buts.name))) { 6822967acbbSJens Axboe __blk_trace_remove(q); 68362c2a7d9SArnd Bergmann return -EFAULT; 68462c2a7d9SArnd Bergmann } 68562c2a7d9SArnd Bergmann 68662c2a7d9SArnd Bergmann return 0; 68762c2a7d9SArnd Bergmann } 68862c2a7d9SArnd Bergmann #endif 68962c2a7d9SArnd Bergmann 6901f2cac10SJens Axboe static int __blk_trace_startstop(struct request_queue *q, int start) 6912db270a8SFrederic Weisbecker { 692c780e86dSJan Kara struct blk_trace *bt; 6932db270a8SFrederic Weisbecker 694c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 69585e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 6962db270a8SFrederic Weisbecker if (bt == NULL) 6972db270a8SFrederic Weisbecker return -EINVAL; 6982db270a8SFrederic Weisbecker 69960a9bb90SYe Bin if (start) 70060a9bb90SYe Bin return blk_trace_start(bt); 70160a9bb90SYe Bin else 70260a9bb90SYe Bin return blk_trace_stop(bt); 7032db270a8SFrederic Weisbecker } 7041f2cac10SJens Axboe 7051f2cac10SJens Axboe int blk_trace_startstop(struct request_queue *q, int start) 7061f2cac10SJens Axboe { 7071f2cac10SJens Axboe int ret; 7081f2cac10SJens Axboe 70985e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 7101f2cac10SJens Axboe ret = __blk_trace_startstop(q, start); 71185e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 7121f2cac10SJens Axboe 7131f2cac10SJens Axboe return ret; 7141f2cac10SJens Axboe } 7152db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_trace_startstop); 7162db270a8SFrederic Weisbecker 7175acb3cc2SWaiman Long /* 7185acb3cc2SWaiman Long * When reading or writing the blktrace sysfs files, the references to the 7195acb3cc2SWaiman Long * opened sysfs or device files should prevent the underlying block device 7205acb3cc2SWaiman Long * from being removed. So no further delete protection is really needed. 7215acb3cc2SWaiman Long */ 7225acb3cc2SWaiman Long 7232db270a8SFrederic Weisbecker /** 7242db270a8SFrederic Weisbecker * blk_trace_ioctl: - handle the ioctls associated with tracing 7252db270a8SFrederic Weisbecker * @bdev: the block device 7262db270a8SFrederic Weisbecker * @cmd: the ioctl cmd 7272db270a8SFrederic Weisbecker * @arg: the argument data, if any 7282db270a8SFrederic Weisbecker * 7292db270a8SFrederic Weisbecker **/ 7302db270a8SFrederic Weisbecker int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg) 7312db270a8SFrederic Weisbecker { 7322db270a8SFrederic Weisbecker struct request_queue *q; 7332db270a8SFrederic Weisbecker int ret, start = 0; 7342db270a8SFrederic Weisbecker char b[BDEVNAME_SIZE]; 7352db270a8SFrederic Weisbecker 7362db270a8SFrederic Weisbecker q = bdev_get_queue(bdev); 7372db270a8SFrederic Weisbecker if (!q) 7382db270a8SFrederic Weisbecker return -ENXIO; 7392db270a8SFrederic Weisbecker 74085e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 7412db270a8SFrederic Weisbecker 7422db270a8SFrederic Weisbecker switch (cmd) { 7432db270a8SFrederic Weisbecker case BLKTRACESETUP: 744900d156bSChristoph Hellwig snprintf(b, sizeof(b), "%pg", bdev); 7451f2cac10SJens Axboe ret = __blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 7462db270a8SFrederic Weisbecker break; 74762c2a7d9SArnd Bergmann #if defined(CONFIG_COMPAT) && defined(CONFIG_X86_64) 74862c2a7d9SArnd Bergmann case BLKTRACESETUP32: 749900d156bSChristoph Hellwig snprintf(b, sizeof(b), "%pg", bdev); 75062c2a7d9SArnd Bergmann ret = compat_blk_trace_setup(q, b, bdev->bd_dev, bdev, arg); 75162c2a7d9SArnd Bergmann break; 75262c2a7d9SArnd Bergmann #endif 7532db270a8SFrederic Weisbecker case BLKTRACESTART: 7542db270a8SFrederic Weisbecker start = 1; 755df561f66SGustavo A. R. Silva fallthrough; 7562db270a8SFrederic Weisbecker case BLKTRACESTOP: 7571f2cac10SJens Axboe ret = __blk_trace_startstop(q, start); 7582db270a8SFrederic Weisbecker break; 7592db270a8SFrederic Weisbecker case BLKTRACETEARDOWN: 7601f2cac10SJens Axboe ret = __blk_trace_remove(q); 7612db270a8SFrederic Weisbecker break; 7622db270a8SFrederic Weisbecker default: 7632db270a8SFrederic Weisbecker ret = -ENOTTY; 7642db270a8SFrederic Weisbecker break; 7652db270a8SFrederic Weisbecker } 7662db270a8SFrederic Weisbecker 76785e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 7682db270a8SFrederic Weisbecker return ret; 7692db270a8SFrederic Weisbecker } 7702db270a8SFrederic Weisbecker 7712db270a8SFrederic Weisbecker /** 7722db270a8SFrederic Weisbecker * blk_trace_shutdown: - stop and cleanup trace structures 7732db270a8SFrederic Weisbecker * @q: the request queue associated with the device 7742db270a8SFrederic Weisbecker * 7752db270a8SFrederic Weisbecker **/ 7762db270a8SFrederic Weisbecker void blk_trace_shutdown(struct request_queue *q) 7772db270a8SFrederic Weisbecker { 778c780e86dSJan Kara if (rcu_dereference_protected(q->blk_trace, 77985e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex))) { 7801f2cac10SJens Axboe __blk_trace_startstop(q, 0); 7811f2cac10SJens Axboe __blk_trace_remove(q); 7822db270a8SFrederic Weisbecker } 7832db270a8SFrederic Weisbecker } 7842db270a8SFrederic Weisbecker 785ca1136c9SShaohua Li #ifdef CONFIG_BLK_CGROUP 78667c0496eSTejun Heo static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 787ca1136c9SShaohua Li { 788bbb1ebe7SChristoph Hellwig struct cgroup_subsys_state *blkcg_css; 789c780e86dSJan Kara struct blk_trace *bt; 790ca1136c9SShaohua Li 791c780e86dSJan Kara /* We don't use the 'bt' value here except as an optimization... */ 792c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 1); 793ca1136c9SShaohua Li if (!bt || !(blk_tracer_flags.val & TRACE_BLK_OPT_CGROUP)) 79467c0496eSTejun Heo return 0; 795ca1136c9SShaohua Li 796bbb1ebe7SChristoph Hellwig blkcg_css = bio_blkcg_css(bio); 797bbb1ebe7SChristoph Hellwig if (!blkcg_css) 79867c0496eSTejun Heo return 0; 799bbb1ebe7SChristoph Hellwig return cgroup_id(blkcg_css->cgroup); 800ca1136c9SShaohua Li } 801ca1136c9SShaohua Li #else 802e75ad2ccSWang Hai static u64 blk_trace_bio_get_cgid(struct request_queue *q, struct bio *bio) 803ca1136c9SShaohua Li { 80467c0496eSTejun Heo return 0; 805ca1136c9SShaohua Li } 806ca1136c9SShaohua Li #endif 807ca1136c9SShaohua Li 80867c0496eSTejun Heo static u64 809a54895faSChristoph Hellwig blk_trace_request_get_cgid(struct request *rq) 810ca1136c9SShaohua Li { 811ca1136c9SShaohua Li if (!rq->bio) 81267c0496eSTejun Heo return 0; 813ca1136c9SShaohua Li /* Use the first bio */ 814a54895faSChristoph Hellwig return blk_trace_bio_get_cgid(rq->q, rq->bio); 815ca1136c9SShaohua Li } 816ca1136c9SShaohua Li 8172db270a8SFrederic Weisbecker /* 8182db270a8SFrederic Weisbecker * blktrace probes 8192db270a8SFrederic Weisbecker */ 8202db270a8SFrederic Weisbecker 8212db270a8SFrederic Weisbecker /** 8222db270a8SFrederic Weisbecker * blk_add_trace_rq - Add a trace for a request oriented action 8232db270a8SFrederic Weisbecker * @rq: the source request 824caf7df12SChristoph Hellwig * @error: return status to log 825af5040daSRoman Pen * @nr_bytes: number of completed bytes 8262db270a8SFrederic Weisbecker * @what: the action 827ca1136c9SShaohua Li * @cgid: the cgroup info 8282db270a8SFrederic Weisbecker * 8292db270a8SFrederic Weisbecker * Description: 8302db270a8SFrederic Weisbecker * Records an action against a request. Will log the bio offset + size. 8312db270a8SFrederic Weisbecker * 8322db270a8SFrederic Weisbecker **/ 8338a7d267bSChristoph Hellwig static void blk_add_trace_rq(struct request *rq, blk_status_t error, 83467c0496eSTejun Heo unsigned int nr_bytes, u32 what, u64 cgid) 8352db270a8SFrederic Weisbecker { 836c780e86dSJan Kara struct blk_trace *bt; 8372db270a8SFrederic Weisbecker 838c780e86dSJan Kara rcu_read_lock(); 839c780e86dSJan Kara bt = rcu_dereference(rq->q->blk_trace); 840c780e86dSJan Kara if (likely(!bt)) { 841c780e86dSJan Kara rcu_read_unlock(); 8422db270a8SFrederic Weisbecker return; 843c780e86dSJan Kara } 8442db270a8SFrederic Weisbecker 84557292b58SChristoph Hellwig if (blk_rq_is_passthrough(rq)) 8462db270a8SFrederic Weisbecker what |= BLK_TC_ACT(BLK_TC_PC); 84748b77ad6SChristoph Hellwig else 8482db270a8SFrederic Weisbecker what |= BLK_TC_ACT(BLK_TC_FS); 84948b77ad6SChristoph Hellwig 850919dbca8SBart Van Assche __blk_add_trace(bt, blk_rq_trace_sector(rq), nr_bytes, rq->cmd_flags, 851919dbca8SBart Van Assche what, blk_status_to_errno(error), 0, NULL, cgid); 852c780e86dSJan Kara rcu_read_unlock(); 8532db270a8SFrederic Weisbecker } 8542db270a8SFrederic Weisbecker 855a54895faSChristoph Hellwig static void blk_add_trace_rq_insert(void *ignore, struct request *rq) 8562db270a8SFrederic Weisbecker { 857ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_INSERT, 858a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 8592db270a8SFrederic Weisbecker } 8602db270a8SFrederic Weisbecker 861a54895faSChristoph Hellwig static void blk_add_trace_rq_issue(void *ignore, struct request *rq) 8622db270a8SFrederic Weisbecker { 863ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_ISSUE, 864a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 8652db270a8SFrederic Weisbecker } 8662db270a8SFrederic Weisbecker 867a54895faSChristoph Hellwig static void blk_add_trace_rq_merge(void *ignore, struct request *rq) 868f3bdc62fSJan Kara { 869f3bdc62fSJan Kara blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_BACKMERGE, 870a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 871f3bdc62fSJan Kara } 872f3bdc62fSJan Kara 873a54895faSChristoph Hellwig static void blk_add_trace_rq_requeue(void *ignore, struct request *rq) 8742db270a8SFrederic Weisbecker { 875ca1136c9SShaohua Li blk_add_trace_rq(rq, 0, blk_rq_bytes(rq), BLK_TA_REQUEUE, 876a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 8772db270a8SFrederic Weisbecker } 8782db270a8SFrederic Weisbecker 879caf7df12SChristoph Hellwig static void blk_add_trace_rq_complete(void *ignore, struct request *rq, 8808a7d267bSChristoph Hellwig blk_status_t error, unsigned int nr_bytes) 8812db270a8SFrederic Weisbecker { 882ca1136c9SShaohua Li blk_add_trace_rq(rq, error, nr_bytes, BLK_TA_COMPLETE, 883a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 8842db270a8SFrederic Weisbecker } 8852db270a8SFrederic Weisbecker 8862db270a8SFrederic Weisbecker /** 8872db270a8SFrederic Weisbecker * blk_add_trace_bio - Add a trace for a bio oriented action 8882db270a8SFrederic Weisbecker * @q: queue the io is for 8892db270a8SFrederic Weisbecker * @bio: the source bio 8902db270a8SFrederic Weisbecker * @what: the action 891797a455dSJens Axboe * @error: error, if any 8922db270a8SFrederic Weisbecker * 8932db270a8SFrederic Weisbecker * Description: 8942db270a8SFrederic Weisbecker * Records an action against a bio. Will log the bio offset + size. 8952db270a8SFrederic Weisbecker * 8962db270a8SFrederic Weisbecker **/ 8972db270a8SFrederic Weisbecker static void blk_add_trace_bio(struct request_queue *q, struct bio *bio, 8981690102dSMarcos Paulo de Souza u32 what, int error) 8992db270a8SFrederic Weisbecker { 900c780e86dSJan Kara struct blk_trace *bt; 9012db270a8SFrederic Weisbecker 902c780e86dSJan Kara rcu_read_lock(); 903c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 904c780e86dSJan Kara if (likely(!bt)) { 905c780e86dSJan Kara rcu_read_unlock(); 9062db270a8SFrederic Weisbecker return; 907c780e86dSJan Kara } 9082db270a8SFrederic Weisbecker 9094f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 910919dbca8SBart Van Assche bio->bi_opf, what, error, 0, NULL, 9111690102dSMarcos Paulo de Souza blk_trace_bio_get_cgid(q, bio)); 912c780e86dSJan Kara rcu_read_unlock(); 9132db270a8SFrederic Weisbecker } 9142db270a8SFrederic Weisbecker 915e8a676d6SChristoph Hellwig static void blk_add_trace_bio_bounce(void *ignore, struct bio *bio) 9162db270a8SFrederic Weisbecker { 917309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BOUNCE, 0); 9182db270a8SFrederic Weisbecker } 9192db270a8SFrederic Weisbecker 9200a82a8d1SLinus Torvalds static void blk_add_trace_bio_complete(void *ignore, 921d24de76aSChristoph Hellwig struct request_queue *q, struct bio *bio) 9222db270a8SFrederic Weisbecker { 923d24de76aSChristoph Hellwig blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, 924d24de76aSChristoph Hellwig blk_status_to_errno(bio->bi_status)); 9252db270a8SFrederic Weisbecker } 9262db270a8SFrederic Weisbecker 927e8a676d6SChristoph Hellwig static void blk_add_trace_bio_backmerge(void *ignore, struct bio *bio) 9282db270a8SFrederic Weisbecker { 929309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_BACKMERGE, 930309dca30SChristoph Hellwig 0); 9312db270a8SFrederic Weisbecker } 9322db270a8SFrederic Weisbecker 933e8a676d6SChristoph Hellwig static void blk_add_trace_bio_frontmerge(void *ignore, struct bio *bio) 9342db270a8SFrederic Weisbecker { 935309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_FRONTMERGE, 936309dca30SChristoph Hellwig 0); 9372db270a8SFrederic Weisbecker } 9382db270a8SFrederic Weisbecker 939e8a676d6SChristoph Hellwig static void blk_add_trace_bio_queue(void *ignore, struct bio *bio) 9402db270a8SFrederic Weisbecker { 941309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_QUEUE, 0); 9422db270a8SFrederic Weisbecker } 9432db270a8SFrederic Weisbecker 944e8a676d6SChristoph Hellwig static void blk_add_trace_getrq(void *ignore, struct bio *bio) 9452db270a8SFrederic Weisbecker { 946309dca30SChristoph Hellwig blk_add_trace_bio(bio->bi_bdev->bd_disk->queue, bio, BLK_TA_GETRQ, 0); 9472db270a8SFrederic Weisbecker } 9482db270a8SFrederic Weisbecker 94938516ab5SSteven Rostedt static void blk_add_trace_plug(void *ignore, struct request_queue *q) 9502db270a8SFrederic Weisbecker { 951c780e86dSJan Kara struct blk_trace *bt; 9522db270a8SFrederic Weisbecker 953c780e86dSJan Kara rcu_read_lock(); 954c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9552db270a8SFrederic Weisbecker if (bt) 956919dbca8SBart Van Assche __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL, 0); 957c780e86dSJan Kara rcu_read_unlock(); 9582db270a8SFrederic Weisbecker } 9592db270a8SFrederic Weisbecker 96049cac01eSJens Axboe static void blk_add_trace_unplug(void *ignore, struct request_queue *q, 96149cac01eSJens Axboe unsigned int depth, bool explicit) 9622db270a8SFrederic Weisbecker { 963c780e86dSJan Kara struct blk_trace *bt; 9642db270a8SFrederic Weisbecker 965c780e86dSJan Kara rcu_read_lock(); 966c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9672db270a8SFrederic Weisbecker if (bt) { 96894b5eb28SJens Axboe __be64 rpdu = cpu_to_be64(depth); 96949cac01eSJens Axboe u32 what; 9702db270a8SFrederic Weisbecker 97149cac01eSJens Axboe if (explicit) 97249cac01eSJens Axboe what = BLK_TA_UNPLUG_IO; 97349cac01eSJens Axboe else 97449cac01eSJens Axboe what = BLK_TA_UNPLUG_TIMER; 97549cac01eSJens Axboe 976919dbca8SBart Van Assche __blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu, 0); 9772db270a8SFrederic Weisbecker } 978c780e86dSJan Kara rcu_read_unlock(); 9792db270a8SFrederic Weisbecker } 9802db270a8SFrederic Weisbecker 981eb6f7f7cSChristoph Hellwig static void blk_add_trace_split(void *ignore, struct bio *bio, unsigned int pdu) 9822db270a8SFrederic Weisbecker { 983309dca30SChristoph Hellwig struct request_queue *q = bio->bi_bdev->bd_disk->queue; 984c780e86dSJan Kara struct blk_trace *bt; 9852db270a8SFrederic Weisbecker 986c780e86dSJan Kara rcu_read_lock(); 987c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 9882db270a8SFrederic Weisbecker if (bt) { 9892db270a8SFrederic Weisbecker __be64 rpdu = cpu_to_be64(pdu); 9902db270a8SFrederic Weisbecker 9914f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, 992919dbca8SBart Van Assche bio->bi_iter.bi_size, bio->bi_opf, BLK_TA_SPLIT, 99348bc3cd3SChaitanya Kulkarni blk_status_to_errno(bio->bi_status), 99448bc3cd3SChaitanya Kulkarni sizeof(rpdu), &rpdu, 99548bc3cd3SChaitanya Kulkarni blk_trace_bio_get_cgid(q, bio)); 9962db270a8SFrederic Weisbecker } 997c780e86dSJan Kara rcu_read_unlock(); 9982db270a8SFrederic Weisbecker } 9992db270a8SFrederic Weisbecker 10002db270a8SFrederic Weisbecker /** 1001d07335e5SMike Snitzer * blk_add_trace_bio_remap - Add a trace for a bio-remap operation 1002546cf44aSRandy Dunlap * @ignore: trace callback data parameter (not used) 10032db270a8SFrederic Weisbecker * @bio: the source bio 10041c02fca6SChristoph Hellwig * @dev: source device 1005a42aaa3bSAlan D. Brunelle * @from: source sector 10062db270a8SFrederic Weisbecker * 10071c02fca6SChristoph Hellwig * Called after a bio is remapped to a different device and/or sector. 10082db270a8SFrederic Weisbecker **/ 10091c02fca6SChristoph Hellwig static void blk_add_trace_bio_remap(void *ignore, struct bio *bio, dev_t dev, 10101c02fca6SChristoph Hellwig sector_t from) 10112db270a8SFrederic Weisbecker { 1012309dca30SChristoph Hellwig struct request_queue *q = bio->bi_bdev->bd_disk->queue; 1013c780e86dSJan Kara struct blk_trace *bt; 10142db270a8SFrederic Weisbecker struct blk_io_trace_remap r; 10152db270a8SFrederic Weisbecker 1016c780e86dSJan Kara rcu_read_lock(); 1017c780e86dSJan Kara bt = rcu_dereference(q->blk_trace); 1018c780e86dSJan Kara if (likely(!bt)) { 1019c780e86dSJan Kara rcu_read_unlock(); 10202db270a8SFrederic Weisbecker return; 1021c780e86dSJan Kara } 10222db270a8SFrederic Weisbecker 1023a42aaa3bSAlan D. Brunelle r.device_from = cpu_to_be32(dev); 102474d46992SChristoph Hellwig r.device_to = cpu_to_be32(bio_dev(bio)); 1025a42aaa3bSAlan D. Brunelle r.sector_from = cpu_to_be64(from); 10262db270a8SFrederic Weisbecker 10274f024f37SKent Overstreet __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, 1028919dbca8SBart Van Assche bio->bi_opf, BLK_TA_REMAP, 102948bc3cd3SChaitanya Kulkarni blk_status_to_errno(bio->bi_status), 1030ca1136c9SShaohua Li sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); 1031c780e86dSJan Kara rcu_read_unlock(); 10322db270a8SFrederic Weisbecker } 10332db270a8SFrederic Weisbecker 10342db270a8SFrederic Weisbecker /** 1035b0da3f0dSJun'ichi Nomura * blk_add_trace_rq_remap - Add a trace for a request-remap operation 1036546cf44aSRandy Dunlap * @ignore: trace callback data parameter (not used) 1037b0da3f0dSJun'ichi Nomura * @rq: the source request 1038b0da3f0dSJun'ichi Nomura * @dev: target device 1039b0da3f0dSJun'ichi Nomura * @from: source sector 1040b0da3f0dSJun'ichi Nomura * 1041b0da3f0dSJun'ichi Nomura * Description: 1042b0da3f0dSJun'ichi Nomura * Device mapper remaps request to other devices. 1043b0da3f0dSJun'ichi Nomura * Add a trace for that action. 1044b0da3f0dSJun'ichi Nomura * 1045b0da3f0dSJun'ichi Nomura **/ 1046a54895faSChristoph Hellwig static void blk_add_trace_rq_remap(void *ignore, struct request *rq, dev_t dev, 1047b0da3f0dSJun'ichi Nomura sector_t from) 1048b0da3f0dSJun'ichi Nomura { 1049c780e86dSJan Kara struct blk_trace *bt; 1050b0da3f0dSJun'ichi Nomura struct blk_io_trace_remap r; 1051b0da3f0dSJun'ichi Nomura 1052c780e86dSJan Kara rcu_read_lock(); 1053a54895faSChristoph Hellwig bt = rcu_dereference(rq->q->blk_trace); 1054c780e86dSJan Kara if (likely(!bt)) { 1055c780e86dSJan Kara rcu_read_unlock(); 1056b0da3f0dSJun'ichi Nomura return; 1057c780e86dSJan Kara } 1058b0da3f0dSJun'ichi Nomura 1059b0da3f0dSJun'ichi Nomura r.device_from = cpu_to_be32(dev); 1060f3fa33acSChristoph Hellwig r.device_to = cpu_to_be32(disk_devt(rq->q->disk)); 1061b0da3f0dSJun'ichi Nomura r.sector_from = cpu_to_be64(from); 1062b0da3f0dSJun'ichi Nomura 1063b0da3f0dSJun'ichi Nomura __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 1064919dbca8SBart Van Assche rq->cmd_flags, BLK_TA_REMAP, 0, 1065a54895faSChristoph Hellwig sizeof(r), &r, blk_trace_request_get_cgid(rq)); 1066c780e86dSJan Kara rcu_read_unlock(); 1067b0da3f0dSJun'ichi Nomura } 1068b0da3f0dSJun'ichi Nomura 1069b0da3f0dSJun'ichi Nomura /** 10702db270a8SFrederic Weisbecker * blk_add_driver_data - Add binary message with driver-specific data 10712db270a8SFrederic Weisbecker * @rq: io request 10722db270a8SFrederic Weisbecker * @data: driver-specific data 10732db270a8SFrederic Weisbecker * @len: length of driver-specific data 10742db270a8SFrederic Weisbecker * 10752db270a8SFrederic Weisbecker * Description: 10762db270a8SFrederic Weisbecker * Some drivers might want to write driver-specific data per request. 10772db270a8SFrederic Weisbecker * 10782db270a8SFrederic Weisbecker **/ 1079a54895faSChristoph Hellwig void blk_add_driver_data(struct request *rq, void *data, size_t len) 10802db270a8SFrederic Weisbecker { 1081c780e86dSJan Kara struct blk_trace *bt; 10822db270a8SFrederic Weisbecker 1083c780e86dSJan Kara rcu_read_lock(); 1084a54895faSChristoph Hellwig bt = rcu_dereference(rq->q->blk_trace); 1085c780e86dSJan Kara if (likely(!bt)) { 1086c780e86dSJan Kara rcu_read_unlock(); 10872db270a8SFrederic Weisbecker return; 1088c780e86dSJan Kara } 10892db270a8SFrederic Weisbecker 1090919dbca8SBart Van Assche __blk_add_trace(bt, blk_rq_trace_sector(rq), blk_rq_bytes(rq), 0, 1091ca1136c9SShaohua Li BLK_TA_DRV_DATA, 0, len, data, 1092a54895faSChristoph Hellwig blk_trace_request_get_cgid(rq)); 1093c780e86dSJan Kara rcu_read_unlock(); 10942db270a8SFrederic Weisbecker } 10952db270a8SFrederic Weisbecker EXPORT_SYMBOL_GPL(blk_add_driver_data); 10962db270a8SFrederic Weisbecker 10973c289ba7SLi Zefan static void blk_register_tracepoints(void) 10982db270a8SFrederic Weisbecker { 10992db270a8SFrederic Weisbecker int ret; 11002db270a8SFrederic Weisbecker 110138516ab5SSteven Rostedt ret = register_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 11022db270a8SFrederic Weisbecker WARN_ON(ret); 110338516ab5SSteven Rostedt ret = register_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 11042db270a8SFrederic Weisbecker WARN_ON(ret); 1105f3bdc62fSJan Kara ret = register_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 1106f3bdc62fSJan Kara WARN_ON(ret); 110738516ab5SSteven Rostedt ret = register_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 11082db270a8SFrederic Weisbecker WARN_ON(ret); 110938516ab5SSteven Rostedt ret = register_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 11102db270a8SFrederic Weisbecker WARN_ON(ret); 111138516ab5SSteven Rostedt ret = register_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); 11122db270a8SFrederic Weisbecker WARN_ON(ret); 111338516ab5SSteven Rostedt ret = register_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 11142db270a8SFrederic Weisbecker WARN_ON(ret); 111538516ab5SSteven Rostedt ret = register_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 11162db270a8SFrederic Weisbecker WARN_ON(ret); 111738516ab5SSteven Rostedt ret = register_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 11182db270a8SFrederic Weisbecker WARN_ON(ret); 111938516ab5SSteven Rostedt ret = register_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 11202db270a8SFrederic Weisbecker WARN_ON(ret); 112138516ab5SSteven Rostedt ret = register_trace_block_getrq(blk_add_trace_getrq, NULL); 11222db270a8SFrederic Weisbecker WARN_ON(ret); 112338516ab5SSteven Rostedt ret = register_trace_block_plug(blk_add_trace_plug, NULL); 11242db270a8SFrederic Weisbecker WARN_ON(ret); 112549cac01eSJens Axboe ret = register_trace_block_unplug(blk_add_trace_unplug, NULL); 11262db270a8SFrederic Weisbecker WARN_ON(ret); 112738516ab5SSteven Rostedt ret = register_trace_block_split(blk_add_trace_split, NULL); 11282db270a8SFrederic Weisbecker WARN_ON(ret); 1129d07335e5SMike Snitzer ret = register_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 11302db270a8SFrederic Weisbecker WARN_ON(ret); 113138516ab5SSteven Rostedt ret = register_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1132b0da3f0dSJun'ichi Nomura WARN_ON(ret); 11332db270a8SFrederic Weisbecker } 11342db270a8SFrederic Weisbecker 11352db270a8SFrederic Weisbecker static void blk_unregister_tracepoints(void) 11362db270a8SFrederic Weisbecker { 113738516ab5SSteven Rostedt unregister_trace_block_rq_remap(blk_add_trace_rq_remap, NULL); 1138d07335e5SMike Snitzer unregister_trace_block_bio_remap(blk_add_trace_bio_remap, NULL); 113938516ab5SSteven Rostedt unregister_trace_block_split(blk_add_trace_split, NULL); 114049cac01eSJens Axboe unregister_trace_block_unplug(blk_add_trace_unplug, NULL); 114138516ab5SSteven Rostedt unregister_trace_block_plug(blk_add_trace_plug, NULL); 114238516ab5SSteven Rostedt unregister_trace_block_getrq(blk_add_trace_getrq, NULL); 114338516ab5SSteven Rostedt unregister_trace_block_bio_queue(blk_add_trace_bio_queue, NULL); 114438516ab5SSteven Rostedt unregister_trace_block_bio_frontmerge(blk_add_trace_bio_frontmerge, NULL); 114538516ab5SSteven Rostedt unregister_trace_block_bio_backmerge(blk_add_trace_bio_backmerge, NULL); 114638516ab5SSteven Rostedt unregister_trace_block_bio_complete(blk_add_trace_bio_complete, NULL); 114738516ab5SSteven Rostedt unregister_trace_block_bio_bounce(blk_add_trace_bio_bounce, NULL); 114838516ab5SSteven Rostedt unregister_trace_block_rq_complete(blk_add_trace_rq_complete, NULL); 114938516ab5SSteven Rostedt unregister_trace_block_rq_requeue(blk_add_trace_rq_requeue, NULL); 1150f3bdc62fSJan Kara unregister_trace_block_rq_merge(blk_add_trace_rq_merge, NULL); 115138516ab5SSteven Rostedt unregister_trace_block_rq_issue(blk_add_trace_rq_issue, NULL); 115238516ab5SSteven Rostedt unregister_trace_block_rq_insert(blk_add_trace_rq_insert, NULL); 11532db270a8SFrederic Weisbecker 11542db270a8SFrederic Weisbecker tracepoint_synchronize_unregister(); 11552db270a8SFrederic Weisbecker } 11562db270a8SFrederic Weisbecker 11572db270a8SFrederic Weisbecker /* 11582db270a8SFrederic Weisbecker * struct blk_io_tracer formatting routines 11592db270a8SFrederic Weisbecker */ 11602db270a8SFrederic Weisbecker 11612db270a8SFrederic Weisbecker static void fill_rwbs(char *rwbs, const struct blk_io_trace *t) 11622db270a8SFrederic Weisbecker { 11632db270a8SFrederic Weisbecker int i = 0; 116465796348SLi Zefan int tc = t->action >> BLK_TC_SHIFT; 11652db270a8SFrederic Weisbecker 1166ca1136c9SShaohua Li if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 116718cea459SLi Zefan rwbs[i++] = 'N'; 116818cea459SLi Zefan goto out; 116918cea459SLi Zefan } 117018cea459SLi Zefan 1171c09c47caSNamhyung Kim if (tc & BLK_TC_FLUSH) 1172c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1173c09c47caSNamhyung Kim 117465796348SLi Zefan if (tc & BLK_TC_DISCARD) 11752db270a8SFrederic Weisbecker rwbs[i++] = 'D'; 117665796348SLi Zefan else if (tc & BLK_TC_WRITE) 11772db270a8SFrederic Weisbecker rwbs[i++] = 'W'; 11782db270a8SFrederic Weisbecker else if (t->bytes) 11792db270a8SFrederic Weisbecker rwbs[i++] = 'R'; 11802db270a8SFrederic Weisbecker else 11812db270a8SFrederic Weisbecker rwbs[i++] = 'N'; 11822db270a8SFrederic Weisbecker 1183c09c47caSNamhyung Kim if (tc & BLK_TC_FUA) 1184c09c47caSNamhyung Kim rwbs[i++] = 'F'; 118565796348SLi Zefan if (tc & BLK_TC_AHEAD) 11862db270a8SFrederic Weisbecker rwbs[i++] = 'A'; 118765796348SLi Zefan if (tc & BLK_TC_SYNC) 11882db270a8SFrederic Weisbecker rwbs[i++] = 'S'; 118965796348SLi Zefan if (tc & BLK_TC_META) 11902db270a8SFrederic Weisbecker rwbs[i++] = 'M'; 119118cea459SLi Zefan out: 11922db270a8SFrederic Weisbecker rwbs[i] = '\0'; 11932db270a8SFrederic Weisbecker } 11942db270a8SFrederic Weisbecker 11952db270a8SFrederic Weisbecker static inline 11962db270a8SFrederic Weisbecker const struct blk_io_trace *te_blk_io_trace(const struct trace_entry *ent) 11972db270a8SFrederic Weisbecker { 11982db270a8SFrederic Weisbecker return (const struct blk_io_trace *)ent; 11992db270a8SFrederic Weisbecker } 12002db270a8SFrederic Weisbecker 1201ca1136c9SShaohua Li static inline const void *pdu_start(const struct trace_entry *ent, bool has_cg) 12022db270a8SFrederic Weisbecker { 120367c0496eSTejun Heo return (void *)(te_blk_io_trace(ent) + 1) + (has_cg ? sizeof(u64) : 0); 1204ca1136c9SShaohua Li } 1205ca1136c9SShaohua Li 120667c0496eSTejun Heo static inline u64 t_cgid(const struct trace_entry *ent) 1207ca1136c9SShaohua Li { 120867c0496eSTejun Heo return *(u64 *)(te_blk_io_trace(ent) + 1); 1209ca1136c9SShaohua Li } 1210ca1136c9SShaohua Li 1211ca1136c9SShaohua Li static inline int pdu_real_len(const struct trace_entry *ent, bool has_cg) 1212ca1136c9SShaohua Li { 121367c0496eSTejun Heo return te_blk_io_trace(ent)->pdu_len - (has_cg ? sizeof(u64) : 0); 12142db270a8SFrederic Weisbecker } 12152db270a8SFrederic Weisbecker 121666de7792SLi Zefan static inline u32 t_action(const struct trace_entry *ent) 121766de7792SLi Zefan { 121866de7792SLi Zefan return te_blk_io_trace(ent)->action; 121966de7792SLi Zefan } 122066de7792SLi Zefan 122166de7792SLi Zefan static inline u32 t_bytes(const struct trace_entry *ent) 122266de7792SLi Zefan { 122366de7792SLi Zefan return te_blk_io_trace(ent)->bytes; 122466de7792SLi Zefan } 122566de7792SLi Zefan 12262db270a8SFrederic Weisbecker static inline u32 t_sec(const struct trace_entry *ent) 12272db270a8SFrederic Weisbecker { 12282db270a8SFrederic Weisbecker return te_blk_io_trace(ent)->bytes >> 9; 12292db270a8SFrederic Weisbecker } 12302db270a8SFrederic Weisbecker 12312db270a8SFrederic Weisbecker static inline unsigned long long t_sector(const struct trace_entry *ent) 12322db270a8SFrederic Weisbecker { 12332db270a8SFrederic Weisbecker return te_blk_io_trace(ent)->sector; 12342db270a8SFrederic Weisbecker } 12352db270a8SFrederic Weisbecker 12362db270a8SFrederic Weisbecker static inline __u16 t_error(const struct trace_entry *ent) 12372db270a8SFrederic Weisbecker { 1238e0dc81beSLi Zefan return te_blk_io_trace(ent)->error; 12392db270a8SFrederic Weisbecker } 12402db270a8SFrederic Weisbecker 1241ca1136c9SShaohua Li static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg) 12422db270a8SFrederic Weisbecker { 124371df3fd8SChaitanya Kulkarni const __be64 *val = pdu_start(ent, has_cg); 12442db270a8SFrederic Weisbecker return be64_to_cpu(*val); 12452db270a8SFrederic Weisbecker } 12462db270a8SFrederic Weisbecker 1247ca1136c9SShaohua Li typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act, 1248ca1136c9SShaohua Li bool has_cg); 1249b6a4b0c3SLi Zefan 1250ca1136c9SShaohua Li static void blk_log_action_classic(struct trace_iterator *iter, const char *act, 1251ca1136c9SShaohua Li bool has_cg) 12522db270a8SFrederic Weisbecker { 1253c09c47caSNamhyung Kim char rwbs[RWBS_LEN]; 125435ac51bfSLi Zefan unsigned long long ts = iter->ts; 125535ac51bfSLi Zefan unsigned long nsec_rem = do_div(ts, NSEC_PER_SEC); 12562db270a8SFrederic Weisbecker unsigned secs = (unsigned long)ts; 1257b6a4b0c3SLi Zefan const struct blk_io_trace *t = te_blk_io_trace(iter->ent); 12582db270a8SFrederic Weisbecker 12592db270a8SFrederic Weisbecker fill_rwbs(rwbs, t); 12602db270a8SFrederic Weisbecker 1261f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(&iter->seq, 126235ac51bfSLi Zefan "%3d,%-3d %2d %5d.%09lu %5u %2s %3s ", 12632db270a8SFrederic Weisbecker MAJOR(t->device), MINOR(t->device), iter->cpu, 1264b6a4b0c3SLi Zefan secs, nsec_rem, iter->ent->pid, act, rwbs); 12652db270a8SFrederic Weisbecker } 12662db270a8SFrederic Weisbecker 1267ca1136c9SShaohua Li static void blk_log_action(struct trace_iterator *iter, const char *act, 1268ca1136c9SShaohua Li bool has_cg) 12692db270a8SFrederic Weisbecker { 1270c09c47caSNamhyung Kim char rwbs[RWBS_LEN]; 1271b6a4b0c3SLi Zefan const struct blk_io_trace *t = te_blk_io_trace(iter->ent); 1272b6a4b0c3SLi Zefan 12732db270a8SFrederic Weisbecker fill_rwbs(rwbs, t); 1274ca1136c9SShaohua Li if (has_cg) { 127567c0496eSTejun Heo u64 id = t_cgid(iter->ent); 1276ca1136c9SShaohua Li 127769fd5c39SShaohua Li if (blk_tracer_flags.val & TRACE_BLK_OPT_CGNAME) { 127869fd5c39SShaohua Li char blkcg_name_buf[NAME_MAX + 1] = "<...>"; 127969fd5c39SShaohua Li 128069fd5c39SShaohua Li cgroup_path_from_kernfs_id(id, blkcg_name_buf, 128169fd5c39SShaohua Li sizeof(blkcg_name_buf)); 128269fd5c39SShaohua Li trace_seq_printf(&iter->seq, "%3d,%-3d %s %2s %3s ", 128369fd5c39SShaohua Li MAJOR(t->device), MINOR(t->device), 128469fd5c39SShaohua Li blkcg_name_buf, act, rwbs); 128540430452STejun Heo } else { 128640430452STejun Heo /* 128740430452STejun Heo * The cgid portion used to be "INO,GEN". Userland 128840430452STejun Heo * builds a FILEID_INO32_GEN fid out of them and 128940430452STejun Heo * opens the cgroup using open_by_handle_at(2). 129040430452STejun Heo * While 32bit ino setups are still the same, 64bit 129140430452STejun Heo * ones now use the 64bit ino as the whole ID and 129240430452STejun Heo * no longer use generation. 129340430452STejun Heo * 12942b5894ccSQiujun Huang * Regardless of the content, always output 129540430452STejun Heo * "LOW32,HIGH32" so that FILEID_INO32_GEN fid can 129640430452STejun Heo * be mapped back to @id on both 64 and 32bit ino 129740430452STejun Heo * setups. See __kernfs_fh_to_dentry(). 129840430452STejun Heo */ 129969fd5c39SShaohua Li trace_seq_printf(&iter->seq, 130040430452STejun Heo "%3d,%-3d %llx,%-llx %2s %3s ", 1301ca1136c9SShaohua Li MAJOR(t->device), MINOR(t->device), 130240430452STejun Heo id & U32_MAX, id >> 32, act, rwbs); 130340430452STejun Heo } 1304ca1136c9SShaohua Li } else 1305f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(&iter->seq, "%3d,%-3d %2s %3s ", 13062db270a8SFrederic Weisbecker MAJOR(t->device), MINOR(t->device), act, rwbs); 13072db270a8SFrederic Weisbecker } 13082db270a8SFrederic Weisbecker 1309ca1136c9SShaohua Li static void blk_log_dump_pdu(struct trace_seq *s, 1310ca1136c9SShaohua Li const struct trace_entry *ent, bool has_cg) 131166de7792SLi Zefan { 131204986257SLi Zefan const unsigned char *pdu_buf; 131366de7792SLi Zefan int pdu_len; 1314f4a1d08cSSteven Rostedt (Red Hat) int i, end; 131566de7792SLi Zefan 1316ca1136c9SShaohua Li pdu_buf = pdu_start(ent, has_cg); 1317ca1136c9SShaohua Li pdu_len = pdu_real_len(ent, has_cg); 131866de7792SLi Zefan 131966de7792SLi Zefan if (!pdu_len) 1320f4a1d08cSSteven Rostedt (Red Hat) return; 132166de7792SLi Zefan 132266de7792SLi Zefan /* find the last zero that needs to be printed */ 132366de7792SLi Zefan for (end = pdu_len - 1; end >= 0; end--) 132466de7792SLi Zefan if (pdu_buf[end]) 132566de7792SLi Zefan break; 132666de7792SLi Zefan end++; 132766de7792SLi Zefan 1328f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putc(s, '('); 132966de7792SLi Zefan 133066de7792SLi Zefan for (i = 0; i < pdu_len; i++) { 133166de7792SLi Zefan 1332f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%s%02x", 133366de7792SLi Zefan i == 0 ? "" : " ", pdu_buf[i]); 133466de7792SLi Zefan 133566de7792SLi Zefan /* 13362b5894ccSQiujun Huang * stop when the rest is just zeros and indicate so 133766de7792SLi Zefan * with a ".." appended 133866de7792SLi Zefan */ 1339f4a1d08cSSteven Rostedt (Red Hat) if (i == end && end != pdu_len - 1) { 1340f4a1d08cSSteven Rostedt (Red Hat) trace_seq_puts(s, " ..) "); 1341f4a1d08cSSteven Rostedt (Red Hat) return; 1342f4a1d08cSSteven Rostedt (Red Hat) } 134366de7792SLi Zefan } 134466de7792SLi Zefan 1345f4a1d08cSSteven Rostedt (Red Hat) trace_seq_puts(s, ") "); 134666de7792SLi Zefan } 134766de7792SLi Zefan 1348ca1136c9SShaohua Li static void blk_log_generic(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13492db270a8SFrederic Weisbecker { 13504ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 13514ca53085SSteven Rostedt 13524ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 13532db270a8SFrederic Weisbecker 135466de7792SLi Zefan if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1355f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%u ", t_bytes(ent)); 1356ca1136c9SShaohua Li blk_log_dump_pdu(s, ent, has_cg); 1357f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 135866de7792SLi Zefan } else { 13592db270a8SFrederic Weisbecker if (t_sec(ent)) 1360f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u [%s]\n", 13612db270a8SFrederic Weisbecker t_sector(ent), t_sec(ent), cmd); 1362f4a1d08cSSteven Rostedt (Red Hat) else 1363f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 13642db270a8SFrederic Weisbecker } 136566de7792SLi Zefan } 13662db270a8SFrederic Weisbecker 1367f4a1d08cSSteven Rostedt (Red Hat) static void blk_log_with_error(struct trace_seq *s, 1368ca1136c9SShaohua Li const struct trace_entry *ent, bool has_cg) 13692db270a8SFrederic Weisbecker { 137066de7792SLi Zefan if (t_action(ent) & BLK_TC_ACT(BLK_TC_PC)) { 1371ca1136c9SShaohua Li blk_log_dump_pdu(s, ent, has_cg); 1372f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%d]\n", t_error(ent)); 137366de7792SLi Zefan } else { 13742db270a8SFrederic Weisbecker if (t_sec(ent)) 1375f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u [%d]\n", 137666de7792SLi Zefan t_sector(ent), 13772db270a8SFrederic Weisbecker t_sec(ent), t_error(ent)); 1378f4a1d08cSSteven Rostedt (Red Hat) else 1379f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu [%d]\n", 138066de7792SLi Zefan t_sector(ent), t_error(ent)); 138166de7792SLi Zefan } 13822db270a8SFrederic Weisbecker } 13832db270a8SFrederic Weisbecker 1384ca1136c9SShaohua Li static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13852db270a8SFrederic Weisbecker { 13865aec598cSChaitanya Kulkarni const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg); 13872db270a8SFrederic Weisbecker 1388f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n", 1389a42aaa3bSAlan D. Brunelle t_sector(ent), t_sec(ent), 13905aec598cSChaitanya Kulkarni MAJOR(be32_to_cpu(__r->device_from)), 13915aec598cSChaitanya Kulkarni MINOR(be32_to_cpu(__r->device_from)), 13925aec598cSChaitanya Kulkarni be64_to_cpu(__r->sector_from)); 13932db270a8SFrederic Weisbecker } 13942db270a8SFrederic Weisbecker 1395ca1136c9SShaohua Li static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 13962db270a8SFrederic Weisbecker { 13974ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 13984ca53085SSteven Rostedt 13994ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 14004ca53085SSteven Rostedt 1401f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "[%s]\n", cmd); 14022db270a8SFrederic Weisbecker } 14032db270a8SFrederic Weisbecker 1404ca1136c9SShaohua Li static void blk_log_unplug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 14052db270a8SFrederic Weisbecker { 14064ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 14074ca53085SSteven Rostedt 14084ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 14094ca53085SSteven Rostedt 1410ca1136c9SShaohua Li trace_seq_printf(s, "[%s] %llu\n", cmd, get_pdu_int(ent, has_cg)); 14112db270a8SFrederic Weisbecker } 14122db270a8SFrederic Weisbecker 1413ca1136c9SShaohua Li static void blk_log_split(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) 14142db270a8SFrederic Weisbecker { 14154ca53085SSteven Rostedt char cmd[TASK_COMM_LEN]; 14164ca53085SSteven Rostedt 14174ca53085SSteven Rostedt trace_find_cmdline(ent->pid, cmd); 14184ca53085SSteven Rostedt 1419f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "%llu / %llu [%s]\n", t_sector(ent), 1420ca1136c9SShaohua Li get_pdu_int(ent, has_cg), cmd); 14212db270a8SFrederic Weisbecker } 14222db270a8SFrederic Weisbecker 1423ca1136c9SShaohua Li static void blk_log_msg(struct trace_seq *s, const struct trace_entry *ent, 1424ca1136c9SShaohua Li bool has_cg) 142518cea459SLi Zefan { 142618cea459SLi Zefan 1427ca1136c9SShaohua Li trace_seq_putmem(s, pdu_start(ent, has_cg), 1428ca1136c9SShaohua Li pdu_real_len(ent, has_cg)); 1429f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putc(s, '\n'); 143018cea459SLi Zefan } 143118cea459SLi Zefan 14322db270a8SFrederic Weisbecker /* 14332db270a8SFrederic Weisbecker * struct tracer operations 14342db270a8SFrederic Weisbecker */ 14352db270a8SFrederic Weisbecker 14362db270a8SFrederic Weisbecker static void blk_tracer_print_header(struct seq_file *m) 14372db270a8SFrederic Weisbecker { 14382db270a8SFrederic Weisbecker if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 14392db270a8SFrederic Weisbecker return; 14402db270a8SFrederic Weisbecker seq_puts(m, "# DEV CPU TIMESTAMP PID ACT FLG\n" 14412db270a8SFrederic Weisbecker "# | | | | | |\n"); 14422db270a8SFrederic Weisbecker } 14432db270a8SFrederic Weisbecker 14442db270a8SFrederic Weisbecker static void blk_tracer_start(struct trace_array *tr) 14452db270a8SFrederic Weisbecker { 1446ad5dd549SLi Zefan blk_tracer_enabled = true; 14472db270a8SFrederic Weisbecker } 14482db270a8SFrederic Weisbecker 14492db270a8SFrederic Weisbecker static int blk_tracer_init(struct trace_array *tr) 14502db270a8SFrederic Weisbecker { 14512db270a8SFrederic Weisbecker blk_tr = tr; 14522db270a8SFrederic Weisbecker blk_tracer_start(tr); 14532db270a8SFrederic Weisbecker return 0; 14542db270a8SFrederic Weisbecker } 14552db270a8SFrederic Weisbecker 14562db270a8SFrederic Weisbecker static void blk_tracer_stop(struct trace_array *tr) 14572db270a8SFrederic Weisbecker { 1458ad5dd549SLi Zefan blk_tracer_enabled = false; 14592db270a8SFrederic Weisbecker } 14602db270a8SFrederic Weisbecker 14612db270a8SFrederic Weisbecker static void blk_tracer_reset(struct trace_array *tr) 14622db270a8SFrederic Weisbecker { 14632db270a8SFrederic Weisbecker blk_tracer_stop(tr); 14642db270a8SFrederic Weisbecker } 14652db270a8SFrederic Weisbecker 1466e4955c99SLi Zefan static const struct { 14672db270a8SFrederic Weisbecker const char *act[2]; 1468ca1136c9SShaohua Li void (*print)(struct trace_seq *s, const struct trace_entry *ent, 1469ca1136c9SShaohua Li bool has_cg); 1470e4955c99SLi Zefan } what2act[] = { 14712db270a8SFrederic Weisbecker [__BLK_TA_QUEUE] = {{ "Q", "queue" }, blk_log_generic }, 14722db270a8SFrederic Weisbecker [__BLK_TA_BACKMERGE] = {{ "M", "backmerge" }, blk_log_generic }, 14732db270a8SFrederic Weisbecker [__BLK_TA_FRONTMERGE] = {{ "F", "frontmerge" }, blk_log_generic }, 14742db270a8SFrederic Weisbecker [__BLK_TA_GETRQ] = {{ "G", "getrq" }, blk_log_generic }, 14752db270a8SFrederic Weisbecker [__BLK_TA_SLEEPRQ] = {{ "S", "sleeprq" }, blk_log_generic }, 14762db270a8SFrederic Weisbecker [__BLK_TA_REQUEUE] = {{ "R", "requeue" }, blk_log_with_error }, 14772db270a8SFrederic Weisbecker [__BLK_TA_ISSUE] = {{ "D", "issue" }, blk_log_generic }, 14782db270a8SFrederic Weisbecker [__BLK_TA_COMPLETE] = {{ "C", "complete" }, blk_log_with_error }, 14792db270a8SFrederic Weisbecker [__BLK_TA_PLUG] = {{ "P", "plug" }, blk_log_plug }, 14802db270a8SFrederic Weisbecker [__BLK_TA_UNPLUG_IO] = {{ "U", "unplug_io" }, blk_log_unplug }, 148149cac01eSJens Axboe [__BLK_TA_UNPLUG_TIMER] = {{ "UT", "unplug_timer" }, blk_log_unplug }, 14822db270a8SFrederic Weisbecker [__BLK_TA_INSERT] = {{ "I", "insert" }, blk_log_generic }, 14832db270a8SFrederic Weisbecker [__BLK_TA_SPLIT] = {{ "X", "split" }, blk_log_split }, 14842db270a8SFrederic Weisbecker [__BLK_TA_BOUNCE] = {{ "B", "bounce" }, blk_log_generic }, 14852db270a8SFrederic Weisbecker [__BLK_TA_REMAP] = {{ "A", "remap" }, blk_log_remap }, 14862db270a8SFrederic Weisbecker }; 14872db270a8SFrederic Weisbecker 1488b6a4b0c3SLi Zefan static enum print_line_t print_one_line(struct trace_iterator *iter, 1489b6a4b0c3SLi Zefan bool classic) 14902db270a8SFrederic Weisbecker { 1491983f938aSSteven Rostedt (Red Hat) struct trace_array *tr = iter->tr; 14922db270a8SFrederic Weisbecker struct trace_seq *s = &iter->seq; 1493b6a4b0c3SLi Zefan const struct blk_io_trace *t; 1494b6a4b0c3SLi Zefan u16 what; 1495b6a4b0c3SLi Zefan bool long_act; 1496b6a4b0c3SLi Zefan blk_log_action_t *log_action; 1497ca1136c9SShaohua Li bool has_cg; 14982db270a8SFrederic Weisbecker 1499b6a4b0c3SLi Zefan t = te_blk_io_trace(iter->ent); 1500ca1136c9SShaohua Li what = (t->action & ((1 << BLK_TC_SHIFT) - 1)) & ~__BLK_TA_CGROUP; 1501983f938aSSteven Rostedt (Red Hat) long_act = !!(tr->trace_flags & TRACE_ITER_VERBOSE); 1502b6a4b0c3SLi Zefan log_action = classic ? &blk_log_action_classic : &blk_log_action; 1503ca1136c9SShaohua Li has_cg = t->action & __BLK_TA_CGROUP; 15042db270a8SFrederic Weisbecker 1505ca1136c9SShaohua Li if ((t->action & ~__BLK_TN_CGROUP) == BLK_TN_MESSAGE) { 1506ca1136c9SShaohua Li log_action(iter, long_act ? "message" : "m", has_cg); 1507ca1136c9SShaohua Li blk_log_msg(s, iter->ent, has_cg); 1508b7d7641eSShaohua Li return trace_handle_return(s); 150918cea459SLi Zefan } 151018cea459SLi Zefan 1511eb08f8ebSLi Zefan if (unlikely(what == 0 || what >= ARRAY_SIZE(what2act))) 1512f4a1d08cSSteven Rostedt (Red Hat) trace_seq_printf(s, "Unknown action %x\n", what); 15132db270a8SFrederic Weisbecker else { 1514ca1136c9SShaohua Li log_action(iter, what2act[what].act[long_act], has_cg); 1515ca1136c9SShaohua Li what2act[what].print(s, iter->ent, has_cg); 15162db270a8SFrederic Weisbecker } 1517f4a1d08cSSteven Rostedt (Red Hat) 1518f4a1d08cSSteven Rostedt (Red Hat) return trace_handle_return(s); 15192db270a8SFrederic Weisbecker } 15202db270a8SFrederic Weisbecker 1521b6a4b0c3SLi Zefan static enum print_line_t blk_trace_event_print(struct trace_iterator *iter, 1522a9a57763SSteven Rostedt int flags, struct trace_event *event) 1523b6a4b0c3SLi Zefan { 1524b6a4b0c3SLi Zefan return print_one_line(iter, false); 1525b6a4b0c3SLi Zefan } 1526b6a4b0c3SLi Zefan 1527f4a1d08cSSteven Rostedt (Red Hat) static void blk_trace_synthesize_old_trace(struct trace_iterator *iter) 15282db270a8SFrederic Weisbecker { 15292db270a8SFrederic Weisbecker struct trace_seq *s = &iter->seq; 15302db270a8SFrederic Weisbecker struct blk_io_trace *t = (struct blk_io_trace *)iter->ent; 15312db270a8SFrederic Weisbecker const int offset = offsetof(struct blk_io_trace, sector); 15322db270a8SFrederic Weisbecker struct blk_io_trace old = { 15332db270a8SFrederic Weisbecker .magic = BLK_IO_TRACE_MAGIC | BLK_IO_TRACE_VERSION, 15346c051ce0SLi Zefan .time = iter->ts, 15352db270a8SFrederic Weisbecker }; 15362db270a8SFrederic Weisbecker 1537f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putmem(s, &old, offset); 1538f4a1d08cSSteven Rostedt (Red Hat) trace_seq_putmem(s, &t->sector, 15392db270a8SFrederic Weisbecker sizeof(old) - offset + t->pdu_len); 15402db270a8SFrederic Weisbecker } 15412db270a8SFrederic Weisbecker 15422db270a8SFrederic Weisbecker static enum print_line_t 1543a9a57763SSteven Rostedt blk_trace_event_print_binary(struct trace_iterator *iter, int flags, 1544a9a57763SSteven Rostedt struct trace_event *event) 15452db270a8SFrederic Weisbecker { 1546f4a1d08cSSteven Rostedt (Red Hat) blk_trace_synthesize_old_trace(iter); 1547f4a1d08cSSteven Rostedt (Red Hat) 1548f4a1d08cSSteven Rostedt (Red Hat) return trace_handle_return(&iter->seq); 15492db270a8SFrederic Weisbecker } 15502db270a8SFrederic Weisbecker 15512db270a8SFrederic Weisbecker static enum print_line_t blk_tracer_print_line(struct trace_iterator *iter) 15522db270a8SFrederic Weisbecker { 15532db270a8SFrederic Weisbecker if (!(blk_tracer_flags.val & TRACE_BLK_OPT_CLASSIC)) 15542db270a8SFrederic Weisbecker return TRACE_TYPE_UNHANDLED; 15552db270a8SFrederic Weisbecker 1556b6a4b0c3SLi Zefan return print_one_line(iter, true); 15572db270a8SFrederic Weisbecker } 15582db270a8SFrederic Weisbecker 15598c1a49aeSSteven Rostedt (Red Hat) static int 15608c1a49aeSSteven Rostedt (Red Hat) blk_tracer_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set) 1561f3948f88SLi Zefan { 1562f3948f88SLi Zefan /* don't output context-info for blk_classic output */ 1563f3948f88SLi Zefan if (bit == TRACE_BLK_OPT_CLASSIC) { 1564f3948f88SLi Zefan if (set) 1565983f938aSSteven Rostedt (Red Hat) tr->trace_flags &= ~TRACE_ITER_CONTEXT_INFO; 1566f3948f88SLi Zefan else 1567983f938aSSteven Rostedt (Red Hat) tr->trace_flags |= TRACE_ITER_CONTEXT_INFO; 1568f3948f88SLi Zefan } 1569f3948f88SLi Zefan return 0; 1570f3948f88SLi Zefan } 1571f3948f88SLi Zefan 15722db270a8SFrederic Weisbecker static struct tracer blk_tracer __read_mostly = { 15732db270a8SFrederic Weisbecker .name = "blk", 15742db270a8SFrederic Weisbecker .init = blk_tracer_init, 15752db270a8SFrederic Weisbecker .reset = blk_tracer_reset, 15762db270a8SFrederic Weisbecker .start = blk_tracer_start, 15772db270a8SFrederic Weisbecker .stop = blk_tracer_stop, 15782db270a8SFrederic Weisbecker .print_header = blk_tracer_print_header, 15792db270a8SFrederic Weisbecker .print_line = blk_tracer_print_line, 15802db270a8SFrederic Weisbecker .flags = &blk_tracer_flags, 1581f3948f88SLi Zefan .set_flag = blk_tracer_set_flag, 15822db270a8SFrederic Weisbecker }; 15832db270a8SFrederic Weisbecker 1584a9a57763SSteven Rostedt static struct trace_event_functions trace_blk_event_funcs = { 15852db270a8SFrederic Weisbecker .trace = blk_trace_event_print, 15862db270a8SFrederic Weisbecker .binary = blk_trace_event_print_binary, 15872db270a8SFrederic Weisbecker }; 15882db270a8SFrederic Weisbecker 1589a9a57763SSteven Rostedt static struct trace_event trace_blk_event = { 1590a9a57763SSteven Rostedt .type = TRACE_BLK, 1591a9a57763SSteven Rostedt .funcs = &trace_blk_event_funcs, 1592a9a57763SSteven Rostedt }; 1593a9a57763SSteven Rostedt 15942db270a8SFrederic Weisbecker static int __init init_blk_tracer(void) 15952db270a8SFrederic Weisbecker { 15969023c930SSteven Rostedt (Red Hat) if (!register_trace_event(&trace_blk_event)) { 1597a395d6a7SJoe Perches pr_warn("Warning: could not register block events\n"); 15982db270a8SFrederic Weisbecker return 1; 15992db270a8SFrederic Weisbecker } 16002db270a8SFrederic Weisbecker 16012db270a8SFrederic Weisbecker if (register_tracer(&blk_tracer) != 0) { 1602a395d6a7SJoe Perches pr_warn("Warning: could not register the block tracer\n"); 16039023c930SSteven Rostedt (Red Hat) unregister_trace_event(&trace_blk_event); 16042db270a8SFrederic Weisbecker return 1; 16052db270a8SFrederic Weisbecker } 16062db270a8SFrederic Weisbecker 16072db270a8SFrederic Weisbecker return 0; 16082db270a8SFrederic Weisbecker } 16092db270a8SFrederic Weisbecker 16102db270a8SFrederic Weisbecker device_initcall(init_blk_tracer); 16112db270a8SFrederic Weisbecker 16122db270a8SFrederic Weisbecker static int blk_trace_remove_queue(struct request_queue *q) 16132db270a8SFrederic Weisbecker { 16142db270a8SFrederic Weisbecker struct blk_trace *bt; 16152db270a8SFrederic Weisbecker 1616c3dbe541SJan Kara bt = rcu_replace_pointer(q->blk_trace, NULL, 161785e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 16182db270a8SFrederic Weisbecker if (bt == NULL) 16192db270a8SFrederic Weisbecker return -EINVAL; 16202db270a8SFrederic Weisbecker 162160a9bb90SYe Bin blk_trace_stop(bt); 16225afedf67SZhihao Cheng 1623a6da0024SJens Axboe put_probe_ref(); 1624c780e86dSJan Kara synchronize_rcu(); 162530939293SYu Kuai blk_trace_free(q, bt); 16262db270a8SFrederic Weisbecker return 0; 16272db270a8SFrederic Weisbecker } 16282db270a8SFrederic Weisbecker 16292db270a8SFrederic Weisbecker /* 16302db270a8SFrederic Weisbecker * Setup everything required to start tracing 16312db270a8SFrederic Weisbecker */ 16329908c309SLi Zefan static int blk_trace_setup_queue(struct request_queue *q, 16339908c309SLi Zefan struct block_device *bdev) 16342db270a8SFrederic Weisbecker { 1635cdea01b2SDavidlohr Bueso struct blk_trace *bt = NULL; 163618cea459SLi Zefan int ret = -ENOMEM; 16372db270a8SFrederic Weisbecker 16382db270a8SFrederic Weisbecker bt = kzalloc(sizeof(*bt), GFP_KERNEL); 16392db270a8SFrederic Weisbecker if (!bt) 164015152e44SLi Zefan return -ENOMEM; 16412db270a8SFrederic Weisbecker 164218cea459SLi Zefan bt->msg_data = __alloc_percpu(BLK_TN_MAX_MSG, __alignof__(char)); 164318cea459SLi Zefan if (!bt->msg_data) 164418cea459SLi Zefan goto free_bt; 164518cea459SLi Zefan 16469908c309SLi Zefan bt->dev = bdev->bd_dev; 16472db270a8SFrederic Weisbecker bt->act_mask = (u16)-1; 16489908c309SLi Zefan 16499908c309SLi Zefan blk_trace_setup_lba(bt, bdev); 16502db270a8SFrederic Weisbecker 1651c3dbe541SJan Kara rcu_assign_pointer(q->blk_trace, bt); 1652a6da0024SJens Axboe get_probe_ref(); 16532db270a8SFrederic Weisbecker return 0; 165418cea459SLi Zefan 165518cea459SLi Zefan free_bt: 165630939293SYu Kuai blk_trace_free(q, bt); 165718cea459SLi Zefan return ret; 16582db270a8SFrederic Weisbecker } 16592db270a8SFrederic Weisbecker 16602db270a8SFrederic Weisbecker /* 16612db270a8SFrederic Weisbecker * sysfs interface to enable and configure tracing 16622db270a8SFrederic Weisbecker */ 16632db270a8SFrederic Weisbecker 16642db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 16652db270a8SFrederic Weisbecker struct device_attribute *attr, 16662db270a8SFrederic Weisbecker char *buf); 16672db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 16682db270a8SFrederic Weisbecker struct device_attribute *attr, 16692db270a8SFrederic Weisbecker const char *buf, size_t count); 16702db270a8SFrederic Weisbecker #define BLK_TRACE_DEVICE_ATTR(_name) \ 16712db270a8SFrederic Weisbecker DEVICE_ATTR(_name, S_IRUGO | S_IWUSR, \ 16722db270a8SFrederic Weisbecker sysfs_blk_trace_attr_show, \ 16732db270a8SFrederic Weisbecker sysfs_blk_trace_attr_store) 16742db270a8SFrederic Weisbecker 1675cd649b8bSLi Zefan static BLK_TRACE_DEVICE_ATTR(enable); 16762db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(act_mask); 16772db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(pid); 16782db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(start_lba); 16792db270a8SFrederic Weisbecker static BLK_TRACE_DEVICE_ATTR(end_lba); 16802db270a8SFrederic Weisbecker 16812db270a8SFrederic Weisbecker static struct attribute *blk_trace_attrs[] = { 16822db270a8SFrederic Weisbecker &dev_attr_enable.attr, 16832db270a8SFrederic Weisbecker &dev_attr_act_mask.attr, 16842db270a8SFrederic Weisbecker &dev_attr_pid.attr, 16852db270a8SFrederic Weisbecker &dev_attr_start_lba.attr, 16862db270a8SFrederic Weisbecker &dev_attr_end_lba.attr, 16872db270a8SFrederic Weisbecker NULL 16882db270a8SFrederic Weisbecker }; 16892db270a8SFrederic Weisbecker 16902db270a8SFrederic Weisbecker struct attribute_group blk_trace_attr_group = { 16912db270a8SFrederic Weisbecker .name = "trace", 16922db270a8SFrederic Weisbecker .attrs = blk_trace_attrs, 16932db270a8SFrederic Weisbecker }; 16942db270a8SFrederic Weisbecker 169509341997SLi Zefan static const struct { 169609341997SLi Zefan int mask; 169709341997SLi Zefan const char *str; 169809341997SLi Zefan } mask_maps[] = { 169909341997SLi Zefan { BLK_TC_READ, "read" }, 170009341997SLi Zefan { BLK_TC_WRITE, "write" }, 1701c09c47caSNamhyung Kim { BLK_TC_FLUSH, "flush" }, 170209341997SLi Zefan { BLK_TC_SYNC, "sync" }, 170309341997SLi Zefan { BLK_TC_QUEUE, "queue" }, 170409341997SLi Zefan { BLK_TC_REQUEUE, "requeue" }, 170509341997SLi Zefan { BLK_TC_ISSUE, "issue" }, 170609341997SLi Zefan { BLK_TC_COMPLETE, "complete" }, 170709341997SLi Zefan { BLK_TC_FS, "fs" }, 170809341997SLi Zefan { BLK_TC_PC, "pc" }, 17098d1547e0SShaohua Li { BLK_TC_NOTIFY, "notify" }, 171009341997SLi Zefan { BLK_TC_AHEAD, "ahead" }, 171109341997SLi Zefan { BLK_TC_META, "meta" }, 171209341997SLi Zefan { BLK_TC_DISCARD, "discard" }, 171309341997SLi Zefan { BLK_TC_DRV_DATA, "drv_data" }, 1714c09c47caSNamhyung Kim { BLK_TC_FUA, "fua" }, 171509341997SLi Zefan }; 171609341997SLi Zefan 171709341997SLi Zefan static int blk_trace_str2mask(const char *str) 17182db270a8SFrederic Weisbecker { 171909341997SLi Zefan int i; 17202db270a8SFrederic Weisbecker int mask = 0; 17219eb85125SLi Zefan char *buf, *s, *token; 17222db270a8SFrederic Weisbecker 17239eb85125SLi Zefan buf = kstrdup(str, GFP_KERNEL); 17249eb85125SLi Zefan if (buf == NULL) 17252db270a8SFrederic Weisbecker return -ENOMEM; 17269eb85125SLi Zefan s = strstrip(buf); 17272db270a8SFrederic Weisbecker 17282db270a8SFrederic Weisbecker while (1) { 172909341997SLi Zefan token = strsep(&s, ","); 173009341997SLi Zefan if (token == NULL) 17312db270a8SFrederic Weisbecker break; 17322db270a8SFrederic Weisbecker 173309341997SLi Zefan if (*token == '\0') 173409341997SLi Zefan continue; 173509341997SLi Zefan 173609341997SLi Zefan for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 173709341997SLi Zefan if (strcasecmp(token, mask_maps[i].str) == 0) { 173809341997SLi Zefan mask |= mask_maps[i].mask; 173909341997SLi Zefan break; 17402db270a8SFrederic Weisbecker } 174109341997SLi Zefan } 174209341997SLi Zefan if (i == ARRAY_SIZE(mask_maps)) { 174309341997SLi Zefan mask = -EINVAL; 174409341997SLi Zefan break; 174509341997SLi Zefan } 174609341997SLi Zefan } 17479eb85125SLi Zefan kfree(buf); 17482db270a8SFrederic Weisbecker 17492db270a8SFrederic Weisbecker return mask; 17502db270a8SFrederic Weisbecker } 17512db270a8SFrederic Weisbecker 175209341997SLi Zefan static ssize_t blk_trace_mask2str(char *buf, int mask) 175309341997SLi Zefan { 175409341997SLi Zefan int i; 175509341997SLi Zefan char *p = buf; 175609341997SLi Zefan 175709341997SLi Zefan for (i = 0; i < ARRAY_SIZE(mask_maps); i++) { 175809341997SLi Zefan if (mask & mask_maps[i].mask) { 175909341997SLi Zefan p += sprintf(p, "%s%s", 176009341997SLi Zefan (p == buf) ? "" : ",", mask_maps[i].str); 176109341997SLi Zefan } 176209341997SLi Zefan } 176309341997SLi Zefan *p++ = '\n'; 176409341997SLi Zefan 176509341997SLi Zefan return p - buf; 176609341997SLi Zefan } 176709341997SLi Zefan 17682db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_show(struct device *dev, 17692db270a8SFrederic Weisbecker struct device_attribute *attr, 17702db270a8SFrederic Weisbecker char *buf) 17712db270a8SFrederic Weisbecker { 17720d02129eSChristoph Hellwig struct block_device *bdev = dev_to_bdev(dev); 17730d02129eSChristoph Hellwig struct request_queue *q = bdev_get_queue(bdev); 1774c780e86dSJan Kara struct blk_trace *bt; 17752db270a8SFrederic Weisbecker ssize_t ret = -ENXIO; 17762db270a8SFrederic Weisbecker 177785e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 1778cd649b8bSLi Zefan 1779c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 178085e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1781cd649b8bSLi Zefan if (attr == &dev_attr_enable) { 1782c780e86dSJan Kara ret = sprintf(buf, "%u\n", !!bt); 1783cd649b8bSLi Zefan goto out_unlock_bdev; 1784cd649b8bSLi Zefan } 1785cd649b8bSLi Zefan 1786c780e86dSJan Kara if (bt == NULL) 17872db270a8SFrederic Weisbecker ret = sprintf(buf, "disabled\n"); 17882db270a8SFrederic Weisbecker else if (attr == &dev_attr_act_mask) 1789c780e86dSJan Kara ret = blk_trace_mask2str(buf, bt->act_mask); 17902db270a8SFrederic Weisbecker else if (attr == &dev_attr_pid) 1791c780e86dSJan Kara ret = sprintf(buf, "%u\n", bt->pid); 17922db270a8SFrederic Weisbecker else if (attr == &dev_attr_start_lba) 1793c780e86dSJan Kara ret = sprintf(buf, "%llu\n", bt->start_lba); 17942db270a8SFrederic Weisbecker else if (attr == &dev_attr_end_lba) 1795c780e86dSJan Kara ret = sprintf(buf, "%llu\n", bt->end_lba); 1796cd649b8bSLi Zefan 1797cd649b8bSLi Zefan out_unlock_bdev: 179885e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 17992db270a8SFrederic Weisbecker return ret; 18002db270a8SFrederic Weisbecker } 18012db270a8SFrederic Weisbecker 18022db270a8SFrederic Weisbecker static ssize_t sysfs_blk_trace_attr_store(struct device *dev, 18032db270a8SFrederic Weisbecker struct device_attribute *attr, 18042db270a8SFrederic Weisbecker const char *buf, size_t count) 18052db270a8SFrederic Weisbecker { 18060d02129eSChristoph Hellwig struct block_device *bdev = dev_to_bdev(dev); 18070d02129eSChristoph Hellwig struct request_queue *q = bdev_get_queue(bdev); 1808c780e86dSJan Kara struct blk_trace *bt; 18092db270a8SFrederic Weisbecker u64 value; 181009341997SLi Zefan ssize_t ret = -EINVAL; 18112db270a8SFrederic Weisbecker 18122db270a8SFrederic Weisbecker if (count == 0) 18132db270a8SFrederic Weisbecker goto out; 18142db270a8SFrederic Weisbecker 18152db270a8SFrederic Weisbecker if (attr == &dev_attr_act_mask) { 18165f339453SShaohua Li if (kstrtoull(buf, 0, &value)) { 18172db270a8SFrederic Weisbecker /* Assume it is a list of trace category names */ 181809341997SLi Zefan ret = blk_trace_str2mask(buf); 181909341997SLi Zefan if (ret < 0) 18202db270a8SFrederic Weisbecker goto out; 182109341997SLi Zefan value = ret; 18222db270a8SFrederic Weisbecker } 18230d02129eSChristoph Hellwig } else { 18240d02129eSChristoph Hellwig if (kstrtoull(buf, 0, &value)) 18252db270a8SFrederic Weisbecker goto out; 18260d02129eSChristoph Hellwig } 18272db270a8SFrederic Weisbecker 182885e0cbbbSLuis Chamberlain mutex_lock(&q->debugfs_mutex); 1829cd649b8bSLi Zefan 1830c780e86dSJan Kara bt = rcu_dereference_protected(q->blk_trace, 183185e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1832cd649b8bSLi Zefan if (attr == &dev_attr_enable) { 1833c780e86dSJan Kara if (!!value == !!bt) { 1834757d9140SSteven Rostedt (VMware) ret = 0; 1835757d9140SSteven Rostedt (VMware) goto out_unlock_bdev; 1836757d9140SSteven Rostedt (VMware) } 1837cd649b8bSLi Zefan if (value) 18389908c309SLi Zefan ret = blk_trace_setup_queue(q, bdev); 1839cd649b8bSLi Zefan else 1840cd649b8bSLi Zefan ret = blk_trace_remove_queue(q); 1841cd649b8bSLi Zefan goto out_unlock_bdev; 1842cd649b8bSLi Zefan } 1843cd649b8bSLi Zefan 18442db270a8SFrederic Weisbecker ret = 0; 1845153031a3SCengiz Can if (bt == NULL) { 18469908c309SLi Zefan ret = blk_trace_setup_queue(q, bdev); 1847153031a3SCengiz Can bt = rcu_dereference_protected(q->blk_trace, 184885e0cbbbSLuis Chamberlain lockdep_is_held(&q->debugfs_mutex)); 1849153031a3SCengiz Can } 18502db270a8SFrederic Weisbecker 18512db270a8SFrederic Weisbecker if (ret == 0) { 18522db270a8SFrederic Weisbecker if (attr == &dev_attr_act_mask) 1853c780e86dSJan Kara bt->act_mask = value; 18542db270a8SFrederic Weisbecker else if (attr == &dev_attr_pid) 1855c780e86dSJan Kara bt->pid = value; 18562db270a8SFrederic Weisbecker else if (attr == &dev_attr_start_lba) 1857c780e86dSJan Kara bt->start_lba = value; 18582db270a8SFrederic Weisbecker else if (attr == &dev_attr_end_lba) 1859c780e86dSJan Kara bt->end_lba = value; 18602db270a8SFrederic Weisbecker } 1861cd649b8bSLi Zefan 1862cd649b8bSLi Zefan out_unlock_bdev: 186385e0cbbbSLuis Chamberlain mutex_unlock(&q->debugfs_mutex); 18642db270a8SFrederic Weisbecker out: 1865cd649b8bSLi Zefan return ret ? ret : count; 18662db270a8SFrederic Weisbecker } 186755782138SLi Zefan #endif /* CONFIG_BLK_DEV_IO_TRACE */ 186855782138SLi Zefan 186955782138SLi Zefan #ifdef CONFIG_EVENT_TRACING 187055782138SLi Zefan 18711f83bb4bSChaitanya Kulkarni /** 18721f83bb4bSChaitanya Kulkarni * blk_fill_rwbs - Fill the buffer rwbs by mapping op to character string. 187394d4bffdSChaitanya Kulkarni * @rwbs: buffer to be filled 1874020e3618SBart Van Assche * @opf: request operation type (REQ_OP_XXX) and flags for the tracepoint 18751f83bb4bSChaitanya Kulkarni * 18761f83bb4bSChaitanya Kulkarni * Description: 1877020e3618SBart Van Assche * Maps each request operation and flag to a single character and fills the 1878020e3618SBart Van Assche * buffer provided by the caller with resulting string. 18791f83bb4bSChaitanya Kulkarni * 18801f83bb4bSChaitanya Kulkarni **/ 1881919dbca8SBart Van Assche void blk_fill_rwbs(char *rwbs, blk_opf_t opf) 188255782138SLi Zefan { 188355782138SLi Zefan int i = 0; 188455782138SLi Zefan 1885919dbca8SBart Van Assche if (opf & REQ_PREFLUSH) 1886c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1887c09c47caSNamhyung Kim 1888919dbca8SBart Van Assche switch (opf & REQ_OP_MASK) { 18891b9a9ab7SMike Christie case REQ_OP_WRITE: 189055782138SLi Zefan rwbs[i++] = 'W'; 18911b9a9ab7SMike Christie break; 18921b9a9ab7SMike Christie case REQ_OP_DISCARD: 189355782138SLi Zefan rwbs[i++] = 'D'; 18941b9a9ab7SMike Christie break; 1895288dab8aSChristoph Hellwig case REQ_OP_SECURE_ERASE: 1896288dab8aSChristoph Hellwig rwbs[i++] = 'D'; 1897288dab8aSChristoph Hellwig rwbs[i++] = 'E'; 1898288dab8aSChristoph Hellwig break; 18993a5e02ceSMike Christie case REQ_OP_FLUSH: 19003a5e02ceSMike Christie rwbs[i++] = 'F'; 19013a5e02ceSMike Christie break; 19021b9a9ab7SMike Christie case REQ_OP_READ: 190355782138SLi Zefan rwbs[i++] = 'R'; 19041b9a9ab7SMike Christie break; 19051b9a9ab7SMike Christie default: 190655782138SLi Zefan rwbs[i++] = 'N'; 19071b9a9ab7SMike Christie } 190855782138SLi Zefan 1909919dbca8SBart Van Assche if (opf & REQ_FUA) 1910c09c47caSNamhyung Kim rwbs[i++] = 'F'; 1911919dbca8SBart Van Assche if (opf & REQ_RAHEAD) 191255782138SLi Zefan rwbs[i++] = 'A'; 1913919dbca8SBart Van Assche if (opf & REQ_SYNC) 191455782138SLi Zefan rwbs[i++] = 'S'; 1915919dbca8SBart Van Assche if (opf & REQ_META) 191655782138SLi Zefan rwbs[i++] = 'M'; 191755782138SLi Zefan 191855782138SLi Zefan rwbs[i] = '\0'; 191955782138SLi Zefan } 19209ca8f8e5SKent Overstreet EXPORT_SYMBOL_GPL(blk_fill_rwbs); 192155782138SLi Zefan 192255782138SLi Zefan #endif /* CONFIG_EVENT_TRACING */ 192355782138SLi Zefan 1924