11d156646STejun Heo /* SPDX-License-Identifier: GPL-2.0
21d156646STejun Heo *
31d156646STejun Heo * Legacy blkg rwstat helpers enabled by CONFIG_BLK_CGROUP_RWSTAT.
41d156646STejun Heo * Do not use in new code.
51d156646STejun Heo */
61d156646STejun Heo #ifndef _BLK_CGROUP_RWSTAT_H
71d156646STejun Heo #define _BLK_CGROUP_RWSTAT_H
81d156646STejun Heo
9672fdcf0SMing Lei #include "blk-cgroup.h"
101d156646STejun Heo
111d156646STejun Heo enum blkg_rwstat_type {
121d156646STejun Heo BLKG_RWSTAT_READ,
131d156646STejun Heo BLKG_RWSTAT_WRITE,
141d156646STejun Heo BLKG_RWSTAT_SYNC,
151d156646STejun Heo BLKG_RWSTAT_ASYNC,
161d156646STejun Heo BLKG_RWSTAT_DISCARD,
171d156646STejun Heo
181d156646STejun Heo BLKG_RWSTAT_NR,
191d156646STejun Heo BLKG_RWSTAT_TOTAL = BLKG_RWSTAT_NR,
201d156646STejun Heo };
211d156646STejun Heo
221d156646STejun Heo /*
231d156646STejun Heo * blkg_[rw]stat->aux_cnt is excluded for local stats but included for
241d156646STejun Heo * recursive. Used to carry stats of dead children.
251d156646STejun Heo */
261d156646STejun Heo struct blkg_rwstat {
271d156646STejun Heo struct percpu_counter cpu_cnt[BLKG_RWSTAT_NR];
281d156646STejun Heo atomic64_t aux_cnt[BLKG_RWSTAT_NR];
291d156646STejun Heo };
301d156646STejun Heo
311d156646STejun Heo struct blkg_rwstat_sample {
321d156646STejun Heo u64 cnt[BLKG_RWSTAT_NR];
331d156646STejun Heo };
341d156646STejun Heo
blkg_rwstat_read_counter(struct blkg_rwstat * rwstat,unsigned int idx)351d156646STejun Heo static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
361d156646STejun Heo unsigned int idx)
371d156646STejun Heo {
381d156646STejun Heo return atomic64_read(&rwstat->aux_cnt[idx]) +
391d156646STejun Heo percpu_counter_sum_positive(&rwstat->cpu_cnt[idx]);
401d156646STejun Heo }
411d156646STejun Heo
421d156646STejun Heo int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
431d156646STejun Heo void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
441d156646STejun Heo u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
451d156646STejun Heo const struct blkg_rwstat_sample *rwstat);
461d156646STejun Heo u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
471d156646STejun Heo int off);
481d156646STejun Heo void blkg_rwstat_recursive_sum(struct blkcg_gq *blkg, struct blkcg_policy *pol,
491d156646STejun Heo int off, struct blkg_rwstat_sample *sum);
501d156646STejun Heo
511d156646STejun Heo
521d156646STejun Heo /**
531d156646STejun Heo * blkg_rwstat_add - add a value to a blkg_rwstat
541d156646STejun Heo * @rwstat: target blkg_rwstat
551d156646STejun Heo * @op: REQ_OP and flags
561d156646STejun Heo * @val: value to add
571d156646STejun Heo *
581d156646STejun Heo * Add @val to @rwstat. The counters are chosen according to @rw. The
591d156646STejun Heo * caller is responsible for synchronizing calls to this function.
601d156646STejun Heo */
blkg_rwstat_add(struct blkg_rwstat * rwstat,blk_opf_t opf,uint64_t val)611d156646STejun Heo static inline void blkg_rwstat_add(struct blkg_rwstat *rwstat,
62*16458cf3SBart Van Assche blk_opf_t opf, uint64_t val)
631d156646STejun Heo {
641d156646STejun Heo struct percpu_counter *cnt;
651d156646STejun Heo
66*16458cf3SBart Van Assche if (op_is_discard(opf))
671d156646STejun Heo cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_DISCARD];
68*16458cf3SBart Van Assche else if (op_is_write(opf))
691d156646STejun Heo cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_WRITE];
701d156646STejun Heo else
711d156646STejun Heo cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_READ];
721d156646STejun Heo
731d156646STejun Heo percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
741d156646STejun Heo
75*16458cf3SBart Van Assche if (op_is_sync(opf))
761d156646STejun Heo cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_SYNC];
771d156646STejun Heo else
781d156646STejun Heo cnt = &rwstat->cpu_cnt[BLKG_RWSTAT_ASYNC];
791d156646STejun Heo
801d156646STejun Heo percpu_counter_add_batch(cnt, val, BLKG_STAT_CPU_BATCH);
811d156646STejun Heo }
821d156646STejun Heo
831d156646STejun Heo /**
841d156646STejun Heo * blkg_rwstat_read - read the current values of a blkg_rwstat
851d156646STejun Heo * @rwstat: blkg_rwstat to read
861d156646STejun Heo *
871d156646STejun Heo * Read the current snapshot of @rwstat and return it in the aux counts.
881d156646STejun Heo */
blkg_rwstat_read(struct blkg_rwstat * rwstat,struct blkg_rwstat_sample * result)891d156646STejun Heo static inline void blkg_rwstat_read(struct blkg_rwstat *rwstat,
901d156646STejun Heo struct blkg_rwstat_sample *result)
911d156646STejun Heo {
921d156646STejun Heo int i;
931d156646STejun Heo
941d156646STejun Heo for (i = 0; i < BLKG_RWSTAT_NR; i++)
951d156646STejun Heo result->cnt[i] =
961d156646STejun Heo percpu_counter_sum_positive(&rwstat->cpu_cnt[i]);
971d156646STejun Heo }
981d156646STejun Heo
991d156646STejun Heo /**
1001d156646STejun Heo * blkg_rwstat_total - read the total count of a blkg_rwstat
1011d156646STejun Heo * @rwstat: blkg_rwstat to read
1021d156646STejun Heo *
1031d156646STejun Heo * Return the total count of @rwstat regardless of the IO direction. This
1041d156646STejun Heo * function can be called without synchronization and takes care of u64
1051d156646STejun Heo * atomicity.
1061d156646STejun Heo */
blkg_rwstat_total(struct blkg_rwstat * rwstat)1071d156646STejun Heo static inline uint64_t blkg_rwstat_total(struct blkg_rwstat *rwstat)
1081d156646STejun Heo {
1091d156646STejun Heo struct blkg_rwstat_sample tmp = { };
1101d156646STejun Heo
1111d156646STejun Heo blkg_rwstat_read(rwstat, &tmp);
1121d156646STejun Heo return tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE];
1131d156646STejun Heo }
1141d156646STejun Heo
1151d156646STejun Heo /**
1161d156646STejun Heo * blkg_rwstat_reset - reset a blkg_rwstat
1171d156646STejun Heo * @rwstat: blkg_rwstat to reset
1181d156646STejun Heo */
blkg_rwstat_reset(struct blkg_rwstat * rwstat)1191d156646STejun Heo static inline void blkg_rwstat_reset(struct blkg_rwstat *rwstat)
1201d156646STejun Heo {
1211d156646STejun Heo int i;
1221d156646STejun Heo
1231d156646STejun Heo for (i = 0; i < BLKG_RWSTAT_NR; i++) {
1241d156646STejun Heo percpu_counter_set(&rwstat->cpu_cnt[i], 0);
1251d156646STejun Heo atomic64_set(&rwstat->aux_cnt[i], 0);
1261d156646STejun Heo }
1271d156646STejun Heo }
1281d156646STejun Heo
1291d156646STejun Heo /**
1301d156646STejun Heo * blkg_rwstat_add_aux - add a blkg_rwstat into another's aux count
1311d156646STejun Heo * @to: the destination blkg_rwstat
1321d156646STejun Heo * @from: the source
1331d156646STejun Heo *
1341d156646STejun Heo * Add @from's count including the aux one to @to's aux count.
1351d156646STejun Heo */
blkg_rwstat_add_aux(struct blkg_rwstat * to,struct blkg_rwstat * from)1361d156646STejun Heo static inline void blkg_rwstat_add_aux(struct blkg_rwstat *to,
1371d156646STejun Heo struct blkg_rwstat *from)
1381d156646STejun Heo {
1391d156646STejun Heo u64 sum[BLKG_RWSTAT_NR];
1401d156646STejun Heo int i;
1411d156646STejun Heo
1421d156646STejun Heo for (i = 0; i < BLKG_RWSTAT_NR; i++)
1431d156646STejun Heo sum[i] = percpu_counter_sum_positive(&from->cpu_cnt[i]);
1441d156646STejun Heo
1451d156646STejun Heo for (i = 0; i < BLKG_RWSTAT_NR; i++)
1461d156646STejun Heo atomic64_add(sum[i] + atomic64_read(&from->aux_cnt[i]),
1471d156646STejun Heo &to->aux_cnt[i]);
1481d156646STejun Heo }
1491d156646STejun Heo #endif /* _BLK_CGROUP_RWSTAT_H */
150