1 #ifndef BLK_STAT_H 2 #define BLK_STAT_H 3 4 #include <linux/kernel.h> 5 #include <linux/blkdev.h> 6 #include <linux/ktime.h> 7 #include <linux/rcupdate.h> 8 #include <linux/timer.h> 9 10 /* 11 * from upper: 12 * 3 bits: reserved for other usage 13 * 12 bits: size 14 * 49 bits: time 15 */ 16 #define BLK_STAT_RES_BITS 3 17 #define BLK_STAT_SIZE_BITS 12 18 #define BLK_STAT_RES_SHIFT (64 - BLK_STAT_RES_BITS) 19 #define BLK_STAT_SIZE_SHIFT (BLK_STAT_RES_SHIFT - BLK_STAT_SIZE_BITS) 20 #define BLK_STAT_TIME_MASK ((1ULL << BLK_STAT_SIZE_SHIFT) - 1) 21 #define BLK_STAT_SIZE_MASK \ 22 (((1ULL << BLK_STAT_SIZE_BITS) - 1) << BLK_STAT_SIZE_SHIFT) 23 #define BLK_STAT_RES_MASK (~((1ULL << BLK_STAT_RES_SHIFT) - 1)) 24 25 /** 26 * struct blk_stat_callback - Block statistics callback. 27 * 28 * A &struct blk_stat_callback is associated with a &struct request_queue. While 29 * @timer is active, that queue's request completion latencies are sorted into 30 * buckets by @bucket_fn and added to a per-cpu buffer, @cpu_stat. When the 31 * timer fires, @cpu_stat is flushed to @stat and @timer_fn is invoked. 32 */ 33 struct blk_stat_callback { 34 /* 35 * @list: RCU list of callbacks for a &struct request_queue. 36 */ 37 struct list_head list; 38 39 /** 40 * @timer: Timer for the next callback invocation. 41 */ 42 struct timer_list timer; 43 44 /** 45 * @cpu_stat: Per-cpu statistics buckets. 46 */ 47 struct blk_rq_stat __percpu *cpu_stat; 48 49 /** 50 * @bucket_fn: Given a request, returns which statistics bucket it 51 * should be accounted under. Return -1 for no bucket for this 52 * request. 53 */ 54 int (*bucket_fn)(const struct request *); 55 56 /** 57 * @buckets: Number of statistics buckets. 58 */ 59 unsigned int buckets; 60 61 /** 62 * @stat: Array of statistics buckets. 63 */ 64 struct blk_rq_stat *stat; 65 66 /** 67 * @fn: Callback function. 68 */ 69 void (*timer_fn)(struct blk_stat_callback *); 70 71 /** 72 * @data: Private pointer for the user. 73 */ 74 void *data; 75 76 struct rcu_head rcu; 77 }; 78 79 struct blk_queue_stats *blk_alloc_queue_stats(void); 80 void blk_free_queue_stats(struct blk_queue_stats *); 81 82 void blk_stat_add(struct request *); 83 84 static inline u64 __blk_stat_time(u64 time) 85 { 86 return time & BLK_STAT_TIME_MASK; 87 } 88 89 static inline u64 blk_stat_time(struct blk_issue_stat *stat) 90 { 91 return __blk_stat_time(stat->stat); 92 } 93 94 static inline sector_t blk_capped_size(sector_t size) 95 { 96 return size & ((1ULL << BLK_STAT_SIZE_BITS) - 1); 97 } 98 99 static inline sector_t blk_stat_size(struct blk_issue_stat *stat) 100 { 101 return (stat->stat & BLK_STAT_SIZE_MASK) >> BLK_STAT_SIZE_SHIFT; 102 } 103 104 static inline void blk_stat_set_issue(struct blk_issue_stat *stat, 105 sector_t size) 106 { 107 stat->stat = (stat->stat & BLK_STAT_RES_MASK) | 108 (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK) | 109 (((u64)blk_capped_size(size)) << BLK_STAT_SIZE_SHIFT); 110 } 111 112 /* record time/size info in request but not add a callback */ 113 void blk_stat_enable_accounting(struct request_queue *q); 114 115 /** 116 * blk_stat_alloc_callback() - Allocate a block statistics callback. 117 * @timer_fn: Timer callback function. 118 * @bucket_fn: Bucket callback function. 119 * @buckets: Number of statistics buckets. 120 * @data: Value for the @data field of the &struct blk_stat_callback. 121 * 122 * See &struct blk_stat_callback for details on the callback functions. 123 * 124 * Return: &struct blk_stat_callback on success or NULL on ENOMEM. 125 */ 126 struct blk_stat_callback * 127 blk_stat_alloc_callback(void (*timer_fn)(struct blk_stat_callback *), 128 int (*bucket_fn)(const struct request *), 129 unsigned int buckets, void *data); 130 131 /** 132 * blk_stat_add_callback() - Add a block statistics callback to be run on a 133 * request queue. 134 * @q: The request queue. 135 * @cb: The callback. 136 * 137 * Note that a single &struct blk_stat_callback can only be added to a single 138 * &struct request_queue. 139 */ 140 void blk_stat_add_callback(struct request_queue *q, 141 struct blk_stat_callback *cb); 142 143 /** 144 * blk_stat_remove_callback() - Remove a block statistics callback from a 145 * request queue. 146 * @q: The request queue. 147 * @cb: The callback. 148 * 149 * When this returns, the callback is not running on any CPUs and will not be 150 * called again unless readded. 151 */ 152 void blk_stat_remove_callback(struct request_queue *q, 153 struct blk_stat_callback *cb); 154 155 /** 156 * blk_stat_free_callback() - Free a block statistics callback. 157 * @cb: The callback. 158 * 159 * @cb may be NULL, in which case this does nothing. If it is not NULL, @cb must 160 * not be associated with a request queue. I.e., if it was previously added with 161 * blk_stat_add_callback(), it must also have been removed since then with 162 * blk_stat_remove_callback(). 163 */ 164 void blk_stat_free_callback(struct blk_stat_callback *cb); 165 166 /** 167 * blk_stat_is_active() - Check if a block statistics callback is currently 168 * gathering statistics. 169 * @cb: The callback. 170 */ 171 static inline bool blk_stat_is_active(struct blk_stat_callback *cb) 172 { 173 return timer_pending(&cb->timer); 174 } 175 176 /** 177 * blk_stat_activate_nsecs() - Gather block statistics during a time window in 178 * nanoseconds. 179 * @cb: The callback. 180 * @nsecs: Number of nanoseconds to gather statistics for. 181 * 182 * The timer callback will be called when the window expires. 183 */ 184 static inline void blk_stat_activate_nsecs(struct blk_stat_callback *cb, 185 u64 nsecs) 186 { 187 mod_timer(&cb->timer, jiffies + nsecs_to_jiffies(nsecs)); 188 } 189 190 /** 191 * blk_stat_activate_msecs() - Gather block statistics during a time window in 192 * milliseconds. 193 * @cb: The callback. 194 * @msecs: Number of milliseconds to gather statistics for. 195 * 196 * The timer callback will be called when the window expires. 197 */ 198 static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb, 199 unsigned int msecs) 200 { 201 mod_timer(&cb->timer, jiffies + msecs_to_jiffies(msecs)); 202 } 203 204 #endif 205