1 /* 2 * Block stat tracking code 3 * 4 * Copyright (C) 2016 Jens Axboe 5 */ 6 #include <linux/kernel.h> 7 #include <linux/blk-mq.h> 8 9 #include "blk-stat.h" 10 #include "blk-mq.h" 11 12 static void blk_stat_flush_batch(struct blk_rq_stat *stat) 13 { 14 const s32 nr_batch = READ_ONCE(stat->nr_batch); 15 const s32 nr_samples = READ_ONCE(stat->nr_samples); 16 17 if (!nr_batch) 18 return; 19 if (!nr_samples) 20 stat->mean = div64_s64(stat->batch, nr_batch); 21 else { 22 stat->mean = div64_s64((stat->mean * nr_samples) + 23 stat->batch, 24 nr_batch + nr_samples); 25 } 26 27 stat->nr_samples += nr_batch; 28 stat->nr_batch = stat->batch = 0; 29 } 30 31 static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src) 32 { 33 blk_stat_flush_batch(src); 34 35 if (!src->nr_samples) 36 return; 37 38 dst->min = min(dst->min, src->min); 39 dst->max = max(dst->max, src->max); 40 41 if (!dst->nr_samples) 42 dst->mean = src->mean; 43 else { 44 dst->mean = div64_s64((src->mean * src->nr_samples) + 45 (dst->mean * dst->nr_samples), 46 dst->nr_samples + src->nr_samples); 47 } 48 dst->nr_samples += src->nr_samples; 49 } 50 51 static void blk_mq_stat_get(struct request_queue *q, struct blk_rq_stat *dst) 52 { 53 struct blk_mq_hw_ctx *hctx; 54 struct blk_mq_ctx *ctx; 55 uint64_t latest = 0; 56 int i, j, nr; 57 58 blk_stat_init(&dst[BLK_STAT_READ]); 59 blk_stat_init(&dst[BLK_STAT_WRITE]); 60 61 nr = 0; 62 do { 63 uint64_t newest = 0; 64 65 queue_for_each_hw_ctx(q, hctx, i) { 66 hctx_for_each_ctx(hctx, ctx, j) { 67 blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]); 68 blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]); 69 70 if (!ctx->stat[BLK_STAT_READ].nr_samples && 71 !ctx->stat[BLK_STAT_WRITE].nr_samples) 72 continue; 73 if (ctx->stat[BLK_STAT_READ].time > newest) 74 newest = ctx->stat[BLK_STAT_READ].time; 75 if (ctx->stat[BLK_STAT_WRITE].time > newest) 76 newest = ctx->stat[BLK_STAT_WRITE].time; 77 } 78 } 79 80 /* 81 * No samples 82 */ 83 if (!newest) 84 break; 85 86 if (newest > latest) 87 latest = newest; 88 89 queue_for_each_hw_ctx(q, hctx, i) { 90 hctx_for_each_ctx(hctx, ctx, j) { 91 if (ctx->stat[BLK_STAT_READ].time == newest) { 92 blk_stat_sum(&dst[BLK_STAT_READ], 93 &ctx->stat[BLK_STAT_READ]); 94 nr++; 95 } 96 if (ctx->stat[BLK_STAT_WRITE].time == newest) { 97 blk_stat_sum(&dst[BLK_STAT_WRITE], 98 &ctx->stat[BLK_STAT_WRITE]); 99 nr++; 100 } 101 } 102 } 103 /* 104 * If we race on finding an entry, just loop back again. 105 * Should be very rare. 106 */ 107 } while (!nr); 108 109 dst[BLK_STAT_READ].time = dst[BLK_STAT_WRITE].time = latest; 110 } 111 112 void blk_queue_stat_get(struct request_queue *q, struct blk_rq_stat *dst) 113 { 114 if (q->mq_ops) 115 blk_mq_stat_get(q, dst); 116 else { 117 blk_stat_flush_batch(&q->rq_stats[BLK_STAT_READ]); 118 blk_stat_flush_batch(&q->rq_stats[BLK_STAT_WRITE]); 119 memcpy(&dst[BLK_STAT_READ], &q->rq_stats[BLK_STAT_READ], 120 sizeof(struct blk_rq_stat)); 121 memcpy(&dst[BLK_STAT_WRITE], &q->rq_stats[BLK_STAT_WRITE], 122 sizeof(struct blk_rq_stat)); 123 } 124 } 125 126 void blk_hctx_stat_get(struct blk_mq_hw_ctx *hctx, struct blk_rq_stat *dst) 127 { 128 struct blk_mq_ctx *ctx; 129 unsigned int i, nr; 130 131 nr = 0; 132 do { 133 uint64_t newest = 0; 134 135 hctx_for_each_ctx(hctx, ctx, i) { 136 blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]); 137 blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]); 138 139 if (!ctx->stat[BLK_STAT_READ].nr_samples && 140 !ctx->stat[BLK_STAT_WRITE].nr_samples) 141 continue; 142 143 if (ctx->stat[BLK_STAT_READ].time > newest) 144 newest = ctx->stat[BLK_STAT_READ].time; 145 if (ctx->stat[BLK_STAT_WRITE].time > newest) 146 newest = ctx->stat[BLK_STAT_WRITE].time; 147 } 148 149 if (!newest) 150 break; 151 152 hctx_for_each_ctx(hctx, ctx, i) { 153 if (ctx->stat[BLK_STAT_READ].time == newest) { 154 blk_stat_sum(&dst[BLK_STAT_READ], 155 &ctx->stat[BLK_STAT_READ]); 156 nr++; 157 } 158 if (ctx->stat[BLK_STAT_WRITE].time == newest) { 159 blk_stat_sum(&dst[BLK_STAT_WRITE], 160 &ctx->stat[BLK_STAT_WRITE]); 161 nr++; 162 } 163 } 164 /* 165 * If we race on finding an entry, just loop back again. 166 * Should be very rare, as the window is only updated 167 * occasionally 168 */ 169 } while (!nr); 170 } 171 172 static void __blk_stat_init(struct blk_rq_stat *stat, s64 time_now) 173 { 174 stat->min = -1ULL; 175 stat->max = stat->nr_samples = stat->mean = 0; 176 stat->batch = stat->nr_batch = 0; 177 stat->time = time_now & BLK_STAT_NSEC_MASK; 178 } 179 180 void blk_stat_init(struct blk_rq_stat *stat) 181 { 182 __blk_stat_init(stat, ktime_to_ns(ktime_get())); 183 } 184 185 static bool __blk_stat_is_current(struct blk_rq_stat *stat, s64 now) 186 { 187 return (now & BLK_STAT_NSEC_MASK) == (stat->time & BLK_STAT_NSEC_MASK); 188 } 189 190 bool blk_stat_is_current(struct blk_rq_stat *stat) 191 { 192 return __blk_stat_is_current(stat, ktime_to_ns(ktime_get())); 193 } 194 195 void blk_stat_add(struct blk_rq_stat *stat, struct request *rq) 196 { 197 s64 now, value; 198 199 now = __blk_stat_time(ktime_to_ns(ktime_get())); 200 if (now < blk_stat_time(&rq->issue_stat)) 201 return; 202 203 if (!__blk_stat_is_current(stat, now)) 204 __blk_stat_init(stat, now); 205 206 value = now - blk_stat_time(&rq->issue_stat); 207 if (value > stat->max) 208 stat->max = value; 209 if (value < stat->min) 210 stat->min = value; 211 212 if (stat->batch + value < stat->batch || 213 stat->nr_batch + 1 == BLK_RQ_STAT_BATCH) 214 blk_stat_flush_batch(stat); 215 216 stat->batch += value; 217 stat->nr_batch++; 218 } 219 220 void blk_stat_clear(struct request_queue *q) 221 { 222 if (q->mq_ops) { 223 struct blk_mq_hw_ctx *hctx; 224 struct blk_mq_ctx *ctx; 225 int i, j; 226 227 queue_for_each_hw_ctx(q, hctx, i) { 228 hctx_for_each_ctx(hctx, ctx, j) { 229 blk_stat_init(&ctx->stat[BLK_STAT_READ]); 230 blk_stat_init(&ctx->stat[BLK_STAT_WRITE]); 231 } 232 } 233 } else { 234 blk_stat_init(&q->rq_stats[BLK_STAT_READ]); 235 blk_stat_init(&q->rq_stats[BLK_STAT_WRITE]); 236 } 237 } 238 239 void blk_stat_set_issue_time(struct blk_issue_stat *stat) 240 { 241 stat->time = (stat->time & BLK_STAT_MASK) | 242 (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK); 243 } 244 245 /* 246 * Enable stat tracking, return whether it was enabled 247 */ 248 bool blk_stat_enable(struct request_queue *q) 249 { 250 if (!test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) { 251 set_bit(QUEUE_FLAG_STATS, &q->queue_flags); 252 return false; 253 } 254 255 return true; 256 } 257