1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef BLK_STAT_H 3 #define BLK_STAT_H 4 5 #include <linux/kernel.h> 6 #include <linux/blkdev.h> 7 #include <linux/ktime.h> 8 #include <linux/rcupdate.h> 9 #include <linux/timer.h> 10 11 /* 12 * from upper: 13 * 3 bits: reserved for other usage 14 * 12 bits: size 15 * 49 bits: time 16 */ 17 #define BLK_STAT_RES_BITS 3 18 #define BLK_STAT_SIZE_BITS 12 19 #define BLK_STAT_RES_SHIFT (64 - BLK_STAT_RES_BITS) 20 #define BLK_STAT_SIZE_SHIFT (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS) 21 #define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SIZE_SHIFT) - 1) 22 #define BLK_STAT_SIZE_MASK \ 23 (((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT) 24 #define BLK_STAT_RES_MASK (~((1ULL << BLK_STAT_RES_SHIFT) - 1)) 25 26 /** 27 * struct blk_stat_callback - Block statistics callback. 28 * 29 * A &struct blk_stat_callback is associated with a &struct request_queue. While 30 * @timer is active, that queue's request completion latencies are sorted into 31 * buckets by @bucket_fn and added to a per-cpu buffer, @cpu_stat. When the 32 * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked. 33 */ 34 struct blk_stat_callback { 35 /* 36 * @list: RCU list of callbacks for a &struct request_queue. 37 */ 38 struct list_head list; 39 40 /** 41 * @timer: Timer for the next callback invocation. 42 */ 43 struct timer_list timer; 44 45 /** 46 * @cpu_stat: Per-cpu statistics buckets. 47 */ 48 struct blk_rq_stat __percpu *cpu_stat; 49 50 /** 51 * @bucket_fn: Given a request, returns which statistics bucket it 52 * should be accounted under. Return -1 for no bucket for this 53 * request. 54 */ 55 int (*bucket_fn)(const struct request *); 56 57 /** 58 * @buckets: Number of statistics buckets. 59 */ 60 unsigned int buckets; 61 62 /** 63 * @stat: Array of statistics buckets. 64 */ 65 struct blk_rq_stat *stat; 66 67 /** 68 * @fn: Callback function. 69 */ 70 void (*timer_fn)(struct blk_stat_callback *); 71 72 /** 73 * @data: Private pointer for the user. 74 */ 75 void *data; 76 77 struct rcu_head rcu; 78 }; 79 80 struct blk_queue_stats *blk_alloc_queue_stats(void); 81 void blk_free_queue_stats(struct blk_queue_stats *); 82 83 void blk_stat_add(struct request *); 84 85 static inline u64 __blk_stat_time(u64 time) 86 { 87 return time & BLK_STAT_TIME_MASK; 88 } 89 90 static inline u64 blk_stat_time(struct blk_issue_stat *stat) 91 { 92 return __blk_stat_time(stat->stat); 93 } 94 95 static inline sector_t blk_capped_size(sector_t size) 96 { 97 return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1); 98 } 99 100 static inline sector_t blk_stat_size(struct blk_issue_stat *stat) 101 { 102 return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT; 103 } 104 105 static inline void blk_stat_set_issue(struct blk_issue_stat *stat, 106 sector_t size) 107 { 108 stat->stat = (stat->stat & BLK_STAT_RES_MASK) | 109 (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) | 110 (((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT); 111 } 112 113 /* record time/size info in request but not add a callback */ 114 void blk_stat_enable_accounting(struct request_queue *q); 115 116 /** 117 * blk_stat_alloc_callback() - Allocate a block statistics callback. 118 * @timer_fn: Timer callback function. 119 * @bucket_fn: Bucket callback function. 120 * @buckets: Number of statistics buckets. 121 * @data: Value for the @data field of the &struct blk_stat_callback. 122 * 123 * See &struct blk_stat_callback for details on the callback functions. 124 * 125 * Return: &struct blk_stat_callback on success or NULL on ENOMEM. 126 */ 127 struct blk_stat_callback * 128 blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), 129 int (*bucket_fn)(const struct request *), 130 unsigned int buckets, void *data); 131 132 /** 133 * blk_stat_add_callback() - Add a block statistics callback to be run on a 134 * request queue. 135 * @q: The request queue. 136 * @cb: The callback. 137 * 138 * Note that a single &struct blk_stat_callback can only be added to a single 139 * &struct request_queue. 140 */ 141 void blk_stat_add_callback(struct request_queue *q, 142 struct blk_stat_callback *cb); 143 144 /** 145 * blk_stat_remove_callback() - Remove a block statistics callback from a 146 * request queue. 147 * @q: The request queue. 148 * @cb: The callback. 149 * 150 * When this returns, the callback is not running on any CPUs and will not be 151 * called again unless readded. 152 */ 153 void blk_stat_remove_callback(struct request_queue *q, 154 struct blk_stat_callback *cb); 155 156 /** 157 * blk_stat_free_callback() - Free a block statistics callback. 158 * @cb: The callback. 159 * 160 * @cb may be NULL, in which case this does nothing. If it is not NULL, @cb must 161 * not be associated with a request queue. I.e., if it was previously added with 162 * blk_stat_add_callback(), it must also have been removed since then with 163 * blk_stat_remove_callback(). 164 */ 165 void blk_stat_free_callback(struct blk_stat_callback *cb); 166 167 /** 168 * blk_stat_is_active() - Check if a block statistics callback is currently 169 * gathering statistics. 170 * @cb: The callback. 171 */ 172 static inline bool blk_stat_is_active(struct blk_stat_callback *cb) 173 { 174 return timer_pending(&cb->timer); 175 } 176 177 /** 178 * blk_stat_activate_nsecs() - Gather block statistics during a time window in 179 * nanoseconds. 180 * @cb: The callback. 181 * @nsecs: Number of nanoseconds to gather statistics for. 182 * 183 * The timer callback will be called when the window expires. 184 */ 185 static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb, 186 u64 nsecs) 187 { 188 mod_timer(&cb->timer, jiffies + nsecs_to_jiffies(nsecs)); 189 } 190 191 /** 192 * blk_stat_activate_msecs() - Gather block statistics during a time window in 193 * milliseconds. 194 * @cb: The callback. 195 * @msecs: Number of milliseconds to gather statistics for. 196 * 197 * The timer callback will be called when the window expires. 198 */ 199 static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb, 200 unsigned int msecs) 201 { 202 mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs)); 203 } 204 205 #endif 206