blktrace.c (8e8c668927b029f6ccc350eb1aa936864cc4eb6f) blktrace.c (288dab8a35a0bde426a09870943c8d3ee3a50dab)
1/*
2 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,

--- 175 unchanged lines hidden (view full) ---

184
185/*
186 * Data direction bit lookup
187 */
188static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ),
189 BLK_TC_ACT(BLK_TC_WRITE) };
190
191#define BLK_TC_RAHEAD BLK_TC_AHEAD
1/*
2 * Copyright (C) 2006 Jens Axboe <axboe@kernel.dk>
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,

--- 175 unchanged lines hidden (view full) ---

184
185/*
186 * Data direction bit lookup
187 */
188static const u32 ddir_act[2] = { BLK_TC_ACT(BLK_TC_READ),
189 BLK_TC_ACT(BLK_TC_WRITE) };
190
191#define BLK_TC_RAHEAD BLK_TC_AHEAD
192#define BLK_TC_PREFLUSH BLK_TC_FLUSH
192
193/* The ilog2() calls fall out because they're constant */
194#define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \
195 (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name))
196
197/*
198 * The worker for the various blk_add_trace*() types. Fills out a
199 * blk_io_trace structure and places it in a per-cpu subbuffer.
200 */
201static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
193
194/* The ilog2() calls fall out because they're constant */
195#define MASK_TC_BIT(rw, __name) ((rw & REQ_ ## __name) << \
196 (ilog2(BLK_TC_ ## __name) + BLK_TC_SHIFT - __REQ_ ## __name))
197
198/*
199 * The worker for the various blk_add_trace*() types. Fills out a
200 * blk_io_trace structure and places it in a per-cpu subbuffer.
201 */
202static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
202 int rw, u32 what, int error, int pdu_len, void *pdu_data)
203 int op, int op_flags, u32 what, int error, int pdu_len,
204 void *pdu_data)
203{
204 struct task_struct *tsk = current;
205 struct ring_buffer_event *event = NULL;
206 struct ring_buffer *buffer = NULL;
207 struct blk_io_trace *t;
208 unsigned long flags = 0;
209 unsigned long *sequence;
210 pid_t pid;
211 int cpu, pc = 0;
212 bool blk_tracer = blk_tracer_enabled;
213
214 if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))
215 return;
216
205{
206 struct task_struct *tsk = current;
207 struct ring_buffer_event *event = NULL;
208 struct ring_buffer *buffer = NULL;
209 struct blk_io_trace *t;
210 unsigned long flags = 0;
211 unsigned long *sequence;
212 pid_t pid;
213 int cpu, pc = 0;
214 bool blk_tracer = blk_tracer_enabled;
215
216 if (unlikely(bt->trace_state != Blktrace_running && !blk_tracer))
217 return;
218
217 what |= ddir_act[rw & WRITE];
218 what |= MASK_TC_BIT(rw, SYNC);
219 what |= MASK_TC_BIT(rw, RAHEAD);
220 what |= MASK_TC_BIT(rw, META);
221 what |= MASK_TC_BIT(rw, DISCARD);
222 what |= MASK_TC_BIT(rw, FLUSH);
223 what |= MASK_TC_BIT(rw, FUA);
219 what |= ddir_act[op_is_write(op) ? WRITE : READ];
220 what |= MASK_TC_BIT(op_flags, SYNC);
221 what |= MASK_TC_BIT(op_flags, RAHEAD);
222 what |= MASK_TC_BIT(op_flags, META);
223 what |= MASK_TC_BIT(op_flags, PREFLUSH);
224 what |= MASK_TC_BIT(op_flags, FUA);
225 if (op == REQ_OP_DISCARD)
226 what |= BLK_TC_ACT(BLK_TC_DISCARD);
227 if (op == REQ_OP_FLUSH)
228 what |= BLK_TC_ACT(BLK_TC_FLUSH);
224
225 pid = tsk->pid;
226 if (act_log_check(bt, what, sector, pid))
227 return;
228 cpu = raw_smp_processor_id();
229
230 if (blk_tracer) {
231 tracing_record_cmdline(current);

--- 471 unchanged lines hidden (view full) ---

703{
704 struct blk_trace *bt = q->blk_trace;
705
706 if (likely(!bt))
707 return;
708
709 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
710 what |= BLK_TC_ACT(BLK_TC_PC);
229
230 pid = tsk->pid;
231 if (act_log_check(bt, what, sector, pid))
232 return;
233 cpu = raw_smp_processor_id();
234
235 if (blk_tracer) {
236 tracing_record_cmdline(current);

--- 471 unchanged lines hidden (view full) ---

708{
709 struct blk_trace *bt = q->blk_trace;
710
711 if (likely(!bt))
712 return;
713
714 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
715 what |= BLK_TC_ACT(BLK_TC_PC);
711 __blk_add_trace(bt, 0, nr_bytes, rq->cmd_flags,
716 __blk_add_trace(bt, 0, nr_bytes, req_op(rq), rq->cmd_flags,
712 what, rq->errors, rq->cmd_len, rq->cmd);
713 } else {
714 what |= BLK_TC_ACT(BLK_TC_FS);
717 what, rq->errors, rq->cmd_len, rq->cmd);
718 } else {
719 what |= BLK_TC_ACT(BLK_TC_FS);
715 __blk_add_trace(bt, blk_rq_pos(rq), nr_bytes,
720 __blk_add_trace(bt, blk_rq_pos(rq), nr_bytes, req_op(rq),
716 rq->cmd_flags, what, rq->errors, 0, NULL);
717 }
718}
719
720static void blk_add_trace_rq_abort(void *ignore,
721 struct request_queue *q, struct request *rq)
722{
723 blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_ABORT);

--- 41 unchanged lines hidden (view full) ---

765 u32 what, int error)
766{
767 struct blk_trace *bt = q->blk_trace;
768
769 if (likely(!bt))
770 return;
771
772 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
721 rq->cmd_flags, what, rq->errors, 0, NULL);
722 }
723}
724
725static void blk_add_trace_rq_abort(void *ignore,
726 struct request_queue *q, struct request *rq)
727{
728 blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_ABORT);

--- 41 unchanged lines hidden (view full) ---

770 u32 what, int error)
771{
772 struct blk_trace *bt = q->blk_trace;
773
774 if (likely(!bt))
775 return;
776
777 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
773 bio->bi_rw, what, error, 0, NULL);
778 bio_op(bio), bio->bi_rw, what, error, 0, NULL);
774}
775
776static void blk_add_trace_bio_bounce(void *ignore,
777 struct request_queue *q, struct bio *bio)
778{
779 blk_add_trace_bio(q, bio, BLK_TA_BOUNCE, 0);
780}
781

--- 31 unchanged lines hidden (view full) ---

813 struct bio *bio, int rw)
814{
815 if (bio)
816 blk_add_trace_bio(q, bio, BLK_TA_GETRQ, 0);
817 else {
818 struct blk_trace *bt = q->blk_trace;
819
820 if (bt)
779}
780
781static void blk_add_trace_bio_bounce(void *ignore,
782 struct request_queue *q, struct bio *bio)
783{
784 blk_add_trace_bio(q, bio, BLK_TA_BOUNCE, 0);
785}
786

--- 31 unchanged lines hidden (view full) ---

818 struct bio *bio, int rw)
819{
820 if (bio)
821 blk_add_trace_bio(q, bio, BLK_TA_GETRQ, 0);
822 else {
823 struct blk_trace *bt = q->blk_trace;
824
825 if (bt)
821 __blk_add_trace(bt, 0, 0, rw, BLK_TA_GETRQ, 0, 0, NULL);
826 __blk_add_trace(bt, 0, 0, rw, 0, BLK_TA_GETRQ, 0, 0,
827 NULL);
822 }
823}
824
825
826static void blk_add_trace_sleeprq(void *ignore,
827 struct request_queue *q,
828 struct bio *bio, int rw)
829{
830 if (bio)
831 blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ, 0);
832 else {
833 struct blk_trace *bt = q->blk_trace;
834
835 if (bt)
828 }
829}
830
831
832static void blk_add_trace_sleeprq(void *ignore,
833 struct request_queue *q,
834 struct bio *bio, int rw)
835{
836 if (bio)
837 blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ, 0);
838 else {
839 struct blk_trace *bt = q->blk_trace;
840
841 if (bt)
836 __blk_add_trace(bt, 0, 0, rw, BLK_TA_SLEEPRQ,
842 __blk_add_trace(bt, 0, 0, rw, 0, BLK_TA_SLEEPRQ,
837 0, 0, NULL);
838 }
839}
840
841static void blk_add_trace_plug(void *ignore, struct request_queue *q)
842{
843 struct blk_trace *bt = q->blk_trace;
844
845 if (bt)
843 0, 0, NULL);
844 }
845}
846
847static void blk_add_trace_plug(void *ignore, struct request_queue *q)
848{
849 struct blk_trace *bt = q->blk_trace;
850
851 if (bt)
846 __blk_add_trace(bt, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);
852 __blk_add_trace(bt, 0, 0, 0, 0, BLK_TA_PLUG, 0, 0, NULL);
847}
848
849static void blk_add_trace_unplug(void *ignore, struct request_queue *q,
850 unsigned int depth, bool explicit)
851{
852 struct blk_trace *bt = q->blk_trace;
853
854 if (bt) {
855 __be64 rpdu = cpu_to_be64(depth);
856 u32 what;
857
858 if (explicit)
859 what = BLK_TA_UNPLUG_IO;
860 else
861 what = BLK_TA_UNPLUG_TIMER;
862
853}
854
855static void blk_add_trace_unplug(void *ignore, struct request_queue *q,
856 unsigned int depth, bool explicit)
857{
858 struct blk_trace *bt = q->blk_trace;
859
860 if (bt) {
861 __be64 rpdu = cpu_to_be64(depth);
862 u32 what;
863
864 if (explicit)
865 what = BLK_TA_UNPLUG_IO;
866 else
867 what = BLK_TA_UNPLUG_TIMER;
868
863 __blk_add_trace(bt, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu);
869 __blk_add_trace(bt, 0, 0, 0, 0, what, 0, sizeof(rpdu), &rpdu);
864 }
865}
866
867static void blk_add_trace_split(void *ignore,
868 struct request_queue *q, struct bio *bio,
869 unsigned int pdu)
870{
871 struct blk_trace *bt = q->blk_trace;
872
873 if (bt) {
874 __be64 rpdu = cpu_to_be64(pdu);
875
876 __blk_add_trace(bt, bio->bi_iter.bi_sector,
870 }
871}
872
873static void blk_add_trace_split(void *ignore,
874 struct request_queue *q, struct bio *bio,
875 unsigned int pdu)
876{
877 struct blk_trace *bt = q->blk_trace;
878
879 if (bt) {
880 __be64 rpdu = cpu_to_be64(pdu);
881
882 __blk_add_trace(bt, bio->bi_iter.bi_sector,
877 bio->bi_iter.bi_size, bio->bi_rw, BLK_TA_SPLIT,
878 bio->bi_error, sizeof(rpdu), &rpdu);
883 bio->bi_iter.bi_size, bio_op(bio), bio->bi_rw,
884 BLK_TA_SPLIT, bio->bi_error, sizeof(rpdu),
885 &rpdu);
879 }
880}
881
882/**
883 * blk_add_trace_bio_remap - Add a trace for a bio-remap operation
884 * @ignore: trace callback data parameter (not used)
885 * @q: queue the io is for
886 * @bio: the source bio

--- 15 unchanged lines hidden (view full) ---

902 if (likely(!bt))
903 return;
904
905 r.device_from = cpu_to_be32(dev);
906 r.device_to = cpu_to_be32(bio->bi_bdev->bd_dev);
907 r.sector_from = cpu_to_be64(from);
908
909 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
886 }
887}
888
889/**
890 * blk_add_trace_bio_remap - Add a trace for a bio-remap operation
891 * @ignore: trace callback data parameter (not used)
892 * @q: queue the io is for
893 * @bio: the source bio

--- 15 unchanged lines hidden (view full) ---

909 if (likely(!bt))
910 return;
911
912 r.device_from = cpu_to_be32(dev);
913 r.device_to = cpu_to_be32(bio->bi_bdev->bd_dev);
914 r.sector_from = cpu_to_be64(from);
915
916 __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size,
910 bio->bi_rw, BLK_TA_REMAP, bio->bi_error,
917 bio_op(bio), bio->bi_rw, BLK_TA_REMAP, bio->bi_error,
911 sizeof(r), &r);
912}
913
914/**
915 * blk_add_trace_rq_remap - Add a trace for a request-remap operation
916 * @ignore: trace callback data parameter (not used)
917 * @q: queue the io is for
918 * @rq: the source request

--- 16 unchanged lines hidden (view full) ---

935 if (likely(!bt))
936 return;
937
938 r.device_from = cpu_to_be32(dev);
939 r.device_to = cpu_to_be32(disk_devt(rq->rq_disk));
940 r.sector_from = cpu_to_be64(from);
941
942 __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),
918 sizeof(r), &r);
919}
920
921/**
922 * blk_add_trace_rq_remap - Add a trace for a request-remap operation
923 * @ignore: trace callback data parameter (not used)
924 * @q: queue the io is for
925 * @rq: the source request

--- 16 unchanged lines hidden (view full) ---

942 if (likely(!bt))
943 return;
944
945 r.device_from = cpu_to_be32(dev);
946 r.device_to = cpu_to_be32(disk_devt(rq->rq_disk));
947 r.sector_from = cpu_to_be64(from);
948
949 __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq),
943 rq_data_dir(rq), BLK_TA_REMAP, !!rq->errors,
950 rq_data_dir(rq), 0, BLK_TA_REMAP, !!rq->errors,
944 sizeof(r), &r);
945}
946
947/**
948 * blk_add_driver_data - Add binary message with driver-specific data
949 * @q: queue the io is for
950 * @rq: io request
951 * @data: driver-specific data

--- 8 unchanged lines hidden (view full) ---

960 void *data, size_t len)
961{
962 struct blk_trace *bt = q->blk_trace;
963
964 if (likely(!bt))
965 return;
966
967 if (rq->cmd_type == REQ_TYPE_BLOCK_PC)
951 sizeof(r), &r);
952}
953
954/**
955 * blk_add_driver_data - Add binary message with driver-specific data
956 * @q: queue the io is for
957 * @rq: io request
958 * @data: driver-specific data

--- 8 unchanged lines hidden (view full) ---

967 void *data, size_t len)
968{
969 struct blk_trace *bt = q->blk_trace;
970
971 if (likely(!bt))
972 return;
973
974 if (rq->cmd_type == REQ_TYPE_BLOCK_PC)
968 __blk_add_trace(bt, 0, blk_rq_bytes(rq), 0,
975 __blk_add_trace(bt, 0, blk_rq_bytes(rq), 0, 0,
969 BLK_TA_DRV_DATA, rq->errors, len, data);
970 else
976 BLK_TA_DRV_DATA, rq->errors, len, data);
977 else
971 __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 0,
978 __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 0, 0,
972 BLK_TA_DRV_DATA, rq->errors, len, data);
973}
974EXPORT_SYMBOL_GPL(blk_add_driver_data);
975
976static void blk_register_tracepoints(void)
977{
978 int ret;
979

--- 784 unchanged lines hidden (view full) ---

1764 buf += sprintf(buf, "%s%02x", i == 0 ? "" : " ", cmd[i]);
1765 if (i == end && end != len - 1) {
1766 sprintf(buf, " ..");
1767 break;
1768 }
1769 }
1770}
1771
979 BLK_TA_DRV_DATA, rq->errors, len, data);
980}
981EXPORT_SYMBOL_GPL(blk_add_driver_data);
982
983static void blk_register_tracepoints(void)
984{
985 int ret;
986

--- 784 unchanged lines hidden (view full) ---

1771 buf += sprintf(buf, "%s%02x", i == 0 ? "" : " ", cmd[i]);
1772 if (i == end && end != len - 1) {
1773 sprintf(buf, " ..");
1774 break;
1775 }
1776 }
1777}
1778
1772void blk_fill_rwbs(char *rwbs, u32 rw, int bytes)
1779void blk_fill_rwbs(char *rwbs, int op, u32 rw, int bytes)
1773{
1774 int i = 0;
1775
1780{
1781 int i = 0;
1782
1776 if (rw & REQ_FLUSH)
1783 if (rw & REQ_PREFLUSH)
1777 rwbs[i++] = 'F';
1778
1784 rwbs[i++] = 'F';
1785
1779 if (rw & WRITE)
1786 switch (op) {
1787 case REQ_OP_WRITE:
1788 case REQ_OP_WRITE_SAME:
1780 rwbs[i++] = 'W';
1789 rwbs[i++] = 'W';
1781 else if (rw & REQ_DISCARD)
1790 break;
1791 case REQ_OP_DISCARD:
1782 rwbs[i++] = 'D';
1792 rwbs[i++] = 'D';
1783 else if (bytes)
1793 break;
1794 case REQ_OP_SECURE_ERASE:
1795 rwbs[i++] = 'D';
1796 rwbs[i++] = 'E';
1797 break;
1798 case REQ_OP_FLUSH:
1799 rwbs[i++] = 'F';
1800 break;
1801 case REQ_OP_READ:
1784 rwbs[i++] = 'R';
1802 rwbs[i++] = 'R';
1785 else
1803 break;
1804 default:
1786 rwbs[i++] = 'N';
1805 rwbs[i++] = 'N';
1806 }
1787
1788 if (rw & REQ_FUA)
1789 rwbs[i++] = 'F';
1790 if (rw & REQ_RAHEAD)
1791 rwbs[i++] = 'A';
1792 if (rw & REQ_SYNC)
1793 rwbs[i++] = 'S';
1794 if (rw & REQ_META)
1795 rwbs[i++] = 'M';
1807
1808 if (rw & REQ_FUA)
1809 rwbs[i++] = 'F';
1810 if (rw & REQ_RAHEAD)
1811 rwbs[i++] = 'A';
1812 if (rw & REQ_SYNC)
1813 rwbs[i++] = 'S';
1814 if (rw & REQ_META)
1815 rwbs[i++] = 'M';
1796 if (rw & REQ_SECURE)
1797 rwbs[i++] = 'E';
1798
1799 rwbs[i] = '\0';
1800}
1801EXPORT_SYMBOL_GPL(blk_fill_rwbs);
1802
1803#endif /* CONFIG_EVENT_TRACING */
1804
1816
1817 rwbs[i] = '\0';
1818}
1819EXPORT_SYMBOL_GPL(blk_fill_rwbs);
1820
1821#endif /* CONFIG_EVENT_TRACING */
1822