1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * f2fs iostat support 4 * 5 * Copyright 2021 Google LLC 6 * Author: Daeho Jeong <daehojeong@google.com> 7 */ 8 9 #include <linux/fs.h> 10 #include <linux/f2fs_fs.h> 11 #include <linux/seq_file.h> 12 13 #include "f2fs.h" 14 #include "iostat.h" 15 #include <trace/events/f2fs.h> 16 17 static struct kmem_cache *bio_iostat_ctx_cache; 18 static mempool_t *bio_iostat_ctx_pool; 19 20 static inline unsigned long long iostat_get_avg_bytes(struct f2fs_sb_info *sbi, 21 enum iostat_type type) 22 { 23 return sbi->iostat_count[type] ? div64_u64(sbi->iostat_bytes[type], 24 sbi->iostat_count[type]) : 0; 25 } 26 27 #define IOSTAT_INFO_SHOW(name, type) \ 28 seq_printf(seq, "%-23s %-16llu %-16llu %-16llu\n", \ 29 name":", sbi->iostat_bytes[type], \ 30 sbi->iostat_count[type], \ 31 iostat_get_avg_bytes(sbi, type)) 32 33 int __maybe_unused iostat_info_seq_show(struct seq_file *seq, void *offset) 34 { 35 struct super_block *sb = seq->private; 36 struct f2fs_sb_info *sbi = F2FS_SB(sb); 37 38 if (!sbi->iostat_enable) 39 return 0; 40 41 seq_printf(seq, "time: %-16llu\n", ktime_get_real_seconds()); 42 seq_printf(seq, "\t\t\t%-16s %-16s %-16s\n", 43 "io_bytes", "count", "avg_bytes"); 44 45 /* print app write IOs */ 46 seq_puts(seq, "[WRITE]\n"); 47 IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_IO); 48 IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_IO); 49 IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_IO); 50 IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_IO); 51 IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_IO); 52 53 /* print fs write IOs */ 54 IOSTAT_INFO_SHOW("fs data", FS_DATA_IO); 55 IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_IO); 56 IOSTAT_INFO_SHOW("fs node", FS_NODE_IO); 57 IOSTAT_INFO_SHOW("fs meta", FS_META_IO); 58 IOSTAT_INFO_SHOW("fs gc data", FS_GC_DATA_IO); 59 IOSTAT_INFO_SHOW("fs gc node", FS_GC_NODE_IO); 60 IOSTAT_INFO_SHOW("fs cp data", FS_CP_DATA_IO); 61 IOSTAT_INFO_SHOW("fs cp node", FS_CP_NODE_IO); 62 IOSTAT_INFO_SHOW("fs cp meta", FS_CP_META_IO); 63 64 /* print app read IOs */ 65 seq_puts(seq, "[READ]\n"); 66 IOSTAT_INFO_SHOW("app buffered data", APP_BUFFERED_READ_IO); 67 IOSTAT_INFO_SHOW("app direct data", APP_DIRECT_READ_IO); 68 IOSTAT_INFO_SHOW("app mapped data", APP_MAPPED_READ_IO); 69 IOSTAT_INFO_SHOW("app buffered cdata", APP_BUFFERED_CDATA_READ_IO); 70 IOSTAT_INFO_SHOW("app mapped cdata", APP_MAPPED_CDATA_READ_IO); 71 72 /* print fs read IOs */ 73 IOSTAT_INFO_SHOW("fs data", FS_DATA_READ_IO); 74 IOSTAT_INFO_SHOW("fs gc data", FS_GDATA_READ_IO); 75 IOSTAT_INFO_SHOW("fs cdata", FS_CDATA_READ_IO); 76 IOSTAT_INFO_SHOW("fs node", FS_NODE_READ_IO); 77 IOSTAT_INFO_SHOW("fs meta", FS_META_READ_IO); 78 79 /* print other IOs */ 80 seq_puts(seq, "[OTHER]\n"); 81 IOSTAT_INFO_SHOW("fs discard", FS_DISCARD_IO); 82 IOSTAT_INFO_SHOW("fs flush", FS_FLUSH_IO); 83 84 return 0; 85 } 86 87 static inline void __record_iostat_latency(struct f2fs_sb_info *sbi) 88 { 89 int io, idx; 90 struct f2fs_iostat_latency iostat_lat[MAX_IO_TYPE][NR_PAGE_TYPE]; 91 struct iostat_lat_info *io_lat = sbi->iostat_io_lat; 92 unsigned long flags; 93 94 spin_lock_irqsave(&sbi->iostat_lat_lock, flags); 95 for (idx = 0; idx < MAX_IO_TYPE; idx++) { 96 for (io = 0; io < NR_PAGE_TYPE; io++) { 97 iostat_lat[idx][io].peak_lat = 98 jiffies_to_msecs(io_lat->peak_lat[idx][io]); 99 iostat_lat[idx][io].cnt = io_lat->bio_cnt[idx][io]; 100 iostat_lat[idx][io].avg_lat = iostat_lat[idx][io].cnt ? 101 jiffies_to_msecs(io_lat->sum_lat[idx][io]) / iostat_lat[idx][io].cnt : 0; 102 io_lat->sum_lat[idx][io] = 0; 103 io_lat->peak_lat[idx][io] = 0; 104 io_lat->bio_cnt[idx][io] = 0; 105 } 106 } 107 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags); 108 109 trace_f2fs_iostat_latency(sbi, iostat_lat); 110 } 111 112 static inline void f2fs_record_iostat(struct f2fs_sb_info *sbi) 113 { 114 unsigned long long iostat_diff[NR_IO_TYPE]; 115 int i; 116 unsigned long flags; 117 118 if (time_is_after_jiffies(sbi->iostat_next_period)) 119 return; 120 121 /* Need double check under the lock */ 122 spin_lock_irqsave(&sbi->iostat_lock, flags); 123 if (time_is_after_jiffies(sbi->iostat_next_period)) { 124 spin_unlock_irqrestore(&sbi->iostat_lock, flags); 125 return; 126 } 127 sbi->iostat_next_period = jiffies + 128 msecs_to_jiffies(sbi->iostat_period_ms); 129 130 for (i = 0; i < NR_IO_TYPE; i++) { 131 iostat_diff[i] = sbi->iostat_bytes[i] - 132 sbi->prev_iostat_bytes[i]; 133 sbi->prev_iostat_bytes[i] = sbi->iostat_bytes[i]; 134 } 135 spin_unlock_irqrestore(&sbi->iostat_lock, flags); 136 137 trace_f2fs_iostat(sbi, iostat_diff); 138 139 __record_iostat_latency(sbi); 140 } 141 142 void f2fs_reset_iostat(struct f2fs_sb_info *sbi) 143 { 144 struct iostat_lat_info *io_lat = sbi->iostat_io_lat; 145 int i; 146 147 spin_lock_irq(&sbi->iostat_lock); 148 for (i = 0; i < NR_IO_TYPE; i++) { 149 sbi->iostat_count[i] = 0; 150 sbi->iostat_bytes[i] = 0; 151 sbi->prev_iostat_bytes[i] = 0; 152 } 153 spin_unlock_irq(&sbi->iostat_lock); 154 155 spin_lock_irq(&sbi->iostat_lat_lock); 156 memset(io_lat, 0, sizeof(struct iostat_lat_info)); 157 spin_unlock_irq(&sbi->iostat_lat_lock); 158 } 159 160 static inline void __f2fs_update_iostat(struct f2fs_sb_info *sbi, 161 enum iostat_type type, unsigned long long io_bytes) 162 { 163 sbi->iostat_bytes[type] += io_bytes; 164 sbi->iostat_count[type]++; 165 } 166 167 void f2fs_update_iostat(struct f2fs_sb_info *sbi, struct inode *inode, 168 enum iostat_type type, unsigned long long io_bytes) 169 { 170 unsigned long flags; 171 172 if (!sbi->iostat_enable) 173 return; 174 175 spin_lock_irqsave(&sbi->iostat_lock, flags); 176 __f2fs_update_iostat(sbi, type, io_bytes); 177 178 if (type == APP_BUFFERED_IO || type == APP_DIRECT_IO) 179 __f2fs_update_iostat(sbi, APP_WRITE_IO, io_bytes); 180 181 if (type == APP_BUFFERED_READ_IO || type == APP_DIRECT_READ_IO) 182 __f2fs_update_iostat(sbi, APP_READ_IO, io_bytes); 183 184 #ifdef CONFIG_F2FS_FS_COMPRESSION 185 if (inode && f2fs_compressed_file(inode)) { 186 if (type == APP_BUFFERED_IO) 187 __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_IO, io_bytes); 188 189 if (type == APP_BUFFERED_READ_IO) 190 __f2fs_update_iostat(sbi, APP_BUFFERED_CDATA_READ_IO, io_bytes); 191 192 if (type == APP_MAPPED_READ_IO) 193 __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_READ_IO, io_bytes); 194 195 if (type == APP_MAPPED_IO) 196 __f2fs_update_iostat(sbi, APP_MAPPED_CDATA_IO, io_bytes); 197 198 if (type == FS_DATA_READ_IO) 199 __f2fs_update_iostat(sbi, FS_CDATA_READ_IO, io_bytes); 200 201 if (type == FS_DATA_IO) 202 __f2fs_update_iostat(sbi, FS_CDATA_IO, io_bytes); 203 } 204 #endif 205 206 spin_unlock_irqrestore(&sbi->iostat_lock, flags); 207 208 f2fs_record_iostat(sbi); 209 } 210 211 static inline void __update_iostat_latency(struct bio_iostat_ctx *iostat_ctx, 212 enum iostat_lat_type lat_type) 213 { 214 unsigned long ts_diff; 215 unsigned int page_type = iostat_ctx->type; 216 struct f2fs_sb_info *sbi = iostat_ctx->sbi; 217 struct iostat_lat_info *io_lat = sbi->iostat_io_lat; 218 unsigned long flags; 219 220 if (!sbi->iostat_enable) 221 return; 222 223 ts_diff = jiffies - iostat_ctx->submit_ts; 224 if (page_type == META_FLUSH) { 225 page_type = META; 226 } else if (page_type >= NR_PAGE_TYPE) { 227 f2fs_warn(sbi, "%s: %d over NR_PAGE_TYPE", __func__, page_type); 228 return; 229 } 230 231 spin_lock_irqsave(&sbi->iostat_lat_lock, flags); 232 io_lat->sum_lat[lat_type][page_type] += ts_diff; 233 io_lat->bio_cnt[lat_type][page_type]++; 234 if (ts_diff > io_lat->peak_lat[lat_type][page_type]) 235 io_lat->peak_lat[lat_type][page_type] = ts_diff; 236 spin_unlock_irqrestore(&sbi->iostat_lat_lock, flags); 237 } 238 239 void iostat_update_and_unbind_ctx(struct bio *bio) 240 { 241 struct bio_iostat_ctx *iostat_ctx = bio->bi_private; 242 enum iostat_lat_type lat_type; 243 244 if (op_is_write(bio_op(bio))) { 245 lat_type = bio->bi_opf & REQ_SYNC ? 246 WRITE_SYNC_IO : WRITE_ASYNC_IO; 247 bio->bi_private = iostat_ctx->sbi; 248 } else { 249 lat_type = READ_IO; 250 bio->bi_private = iostat_ctx->post_read_ctx; 251 } 252 253 __update_iostat_latency(iostat_ctx, lat_type); 254 mempool_free(iostat_ctx, bio_iostat_ctx_pool); 255 } 256 257 void iostat_alloc_and_bind_ctx(struct f2fs_sb_info *sbi, 258 struct bio *bio, struct bio_post_read_ctx *ctx) 259 { 260 struct bio_iostat_ctx *iostat_ctx; 261 /* Due to the mempool, this never fails. */ 262 iostat_ctx = mempool_alloc(bio_iostat_ctx_pool, GFP_NOFS); 263 iostat_ctx->sbi = sbi; 264 iostat_ctx->submit_ts = 0; 265 iostat_ctx->type = 0; 266 iostat_ctx->post_read_ctx = ctx; 267 bio->bi_private = iostat_ctx; 268 } 269 270 int __init f2fs_init_iostat_processing(void) 271 { 272 bio_iostat_ctx_cache = 273 kmem_cache_create("f2fs_bio_iostat_ctx", 274 sizeof(struct bio_iostat_ctx), 0, 0, NULL); 275 if (!bio_iostat_ctx_cache) 276 goto fail; 277 bio_iostat_ctx_pool = 278 mempool_create_slab_pool(NUM_PREALLOC_IOSTAT_CTXS, 279 bio_iostat_ctx_cache); 280 if (!bio_iostat_ctx_pool) 281 goto fail_free_cache; 282 return 0; 283 284 fail_free_cache: 285 kmem_cache_destroy(bio_iostat_ctx_cache); 286 fail: 287 return -ENOMEM; 288 } 289 290 void f2fs_destroy_iostat_processing(void) 291 { 292 mempool_destroy(bio_iostat_ctx_pool); 293 kmem_cache_destroy(bio_iostat_ctx_cache); 294 } 295 296 int f2fs_init_iostat(struct f2fs_sb_info *sbi) 297 { 298 /* init iostat info */ 299 spin_lock_init(&sbi->iostat_lock); 300 spin_lock_init(&sbi->iostat_lat_lock); 301 sbi->iostat_enable = false; 302 sbi->iostat_period_ms = DEFAULT_IOSTAT_PERIOD_MS; 303 sbi->iostat_io_lat = f2fs_kzalloc(sbi, sizeof(struct iostat_lat_info), 304 GFP_KERNEL); 305 if (!sbi->iostat_io_lat) 306 return -ENOMEM; 307 308 return 0; 309 } 310 311 void f2fs_destroy_iostat(struct f2fs_sb_info *sbi) 312 { 313 kfree(sbi->iostat_io_lat); 314 } 315