96 /* bfqg stats flags */
104 static void bfqg_stats_mark_##name(struct bfqg_stats *stats)	\
106 	stats->flags |= (1 << BFQG_stats_##name);			\
108 static void bfqg_stats_clear_##name(struct bfqg_stats *stats)	\
110 	stats->flags &= ~(1 << BFQG_stats_##name);			\
112 static int bfqg_stats_##name(struct bfqg_stats *stats)		\
114 	return (stats->flags & (1 << BFQG_stats_##name)) != 0;		\
123 static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)  in BFQG_FLAG_FNS()
127 	if (!bfqg_stats_waiting(stats))  in BFQG_FLAG_FNS()
131 	if (now > stats->start_group_wait_time)  in BFQG_FLAG_FNS()
132 		bfq_stat_add(&stats->group_wait_time,  in BFQG_FLAG_FNS()
133 			      now - stats->start_group_wait_time);  in BFQG_FLAG_FNS()
134 	bfqg_stats_clear_waiting(stats);  in BFQG_FLAG_FNS()
141 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_group_wait_time()  local
143 	if (bfqg_stats_waiting(stats))  in bfqg_stats_set_start_group_wait_time()
147 	stats->start_group_wait_time = ktime_get_ns();  in bfqg_stats_set_start_group_wait_time()
148 	bfqg_stats_mark_waiting(stats);  in bfqg_stats_set_start_group_wait_time()
152 static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)  in bfqg_stats_end_empty_time()  argument
156 	if (!bfqg_stats_empty(stats))  in bfqg_stats_end_empty_time()
160 	if (now > stats->start_empty_time)  in bfqg_stats_end_empty_time()
161 		bfq_stat_add(&stats->empty_time,  in bfqg_stats_end_empty_time()
162 			      now - stats->start_empty_time);  in bfqg_stats_end_empty_time()
163 	bfqg_stats_clear_empty(stats);  in bfqg_stats_end_empty_time()
168 	bfq_stat_add(&bfqg->stats.dequeue, 1);  in bfqg_stats_update_dequeue()
173 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_empty_time()  local
175 	if (blkg_rwstat_total(&stats->queued))  in bfqg_stats_set_start_empty_time()
183 	if (bfqg_stats_empty(stats))  in bfqg_stats_set_start_empty_time()
186 	stats->start_empty_time = ktime_get_ns();  in bfqg_stats_set_start_empty_time()
187 	bfqg_stats_mark_empty(stats);  in bfqg_stats_set_start_empty_time()
192 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_idle_time()  local
194 	if (bfqg_stats_idling(stats)) {  in bfqg_stats_update_idle_time()
197 		if (now > stats->start_idle_time)  in bfqg_stats_update_idle_time()
198 			bfq_stat_add(&stats->idle_time,  in bfqg_stats_update_idle_time()
199 				      now - stats->start_idle_time);  in bfqg_stats_update_idle_time()
200 		bfqg_stats_clear_idling(stats);  in bfqg_stats_update_idle_time()
206 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_idle_time()  local
208 	stats->start_idle_time = ktime_get_ns();  in bfqg_stats_set_start_idle_time()
209 	bfqg_stats_mark_idling(stats);  in bfqg_stats_set_start_idle_time()
214 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_avg_queue_size()  local
216 	bfq_stat_add(&stats->avg_queue_size_sum,  in bfqg_stats_update_avg_queue_size()
217 		      blkg_rwstat_total(&stats->queued));  in bfqg_stats_update_avg_queue_size()
218 	bfq_stat_add(&stats->avg_queue_size_samples, 1);  in bfqg_stats_update_avg_queue_size()
219 	bfqg_stats_update_group_wait_time(stats);  in bfqg_stats_update_avg_queue_size()
225 	blkg_rwstat_add(&bfqg->stats.queued, opf, 1);  in bfqg_stats_update_io_add()
226 	bfqg_stats_end_empty_time(&bfqg->stats);  in bfqg_stats_update_io_add()
233 	blkg_rwstat_add(&bfqg->stats.queued, opf, -1);  in bfqg_stats_update_io_remove()
238 	blkg_rwstat_add(&bfqg->stats.merged, opf, 1);  in bfqg_stats_update_io_merged()
244 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_completion()  local
248 		blkg_rwstat_add(&stats->service_time, opf,  in bfqg_stats_update_completion()
251 		blkg_rwstat_add(&stats->wait_time, opf,  in bfqg_stats_update_completion()
350 	blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));  in bfqg_stats_update_legacy_io()
351 	blkg_rwstat_add(&bfqg->stats.ios, rq->cmd_flags, 1);  in bfqg_stats_update_legacy_io()
354 /* @stats = 0 */
355 static void bfqg_stats_reset(struct bfqg_stats *stats)  in bfqg_stats_reset()  argument
358 	/* queued stats shouldn't be cleared */  in bfqg_stats_reset()
359 	blkg_rwstat_reset(&stats->merged);  in bfqg_stats_reset()
360 	blkg_rwstat_reset(&stats->service_time);  in bfqg_stats_reset()
361 	blkg_rwstat_reset(&stats->wait_time);  in bfqg_stats_reset()
362 	bfq_stat_reset(&stats->time);  in bfqg_stats_reset()
363 	bfq_stat_reset(&stats->avg_queue_size_sum);  in bfqg_stats_reset()
364 	bfq_stat_reset(&stats->avg_queue_size_samples);  in bfqg_stats_reset()
365 	bfq_stat_reset(&stats->dequeue);  in bfqg_stats_reset()
366 	bfq_stat_reset(&stats->group_wait_time);  in bfqg_stats_reset()
367 	bfq_stat_reset(&stats->idle_time);  in bfqg_stats_reset()
368 	bfq_stat_reset(&stats->empty_time);  in bfqg_stats_reset()
379 	/* queued stats shouldn't be cleared */  in bfqg_stats_add_aux()
395  * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
396  * recursive stats can still account for the amount used by this bfqg after
413 	bfqg_stats_add_aux(&parent->stats, &bfqg->stats);  in bfqg_stats_xfer_dead()
414 	bfqg_stats_reset(&bfqg->stats);  in bfqg_stats_xfer_dead()
436 static void bfqg_stats_exit(struct bfqg_stats *stats)  in bfqg_stats_exit()  argument
438 	blkg_rwstat_exit(&stats->bytes);  in bfqg_stats_exit()
439 	blkg_rwstat_exit(&stats->ios);  in bfqg_stats_exit()
441 	blkg_rwstat_exit(&stats->merged);  in bfqg_stats_exit()
442 	blkg_rwstat_exit(&stats->service_time);  in bfqg_stats_exit()
443 	blkg_rwstat_exit(&stats->wait_time);  in bfqg_stats_exit()
444 	blkg_rwstat_exit(&stats->queued);  in bfqg_stats_exit()
445 	bfq_stat_exit(&stats->time);  in bfqg_stats_exit()
446 	bfq_stat_exit(&stats->avg_queue_size_sum);  in bfqg_stats_exit()
447 	bfq_stat_exit(&stats->avg_queue_size_samples);  in bfqg_stats_exit()
448 	bfq_stat_exit(&stats->dequeue);  in bfqg_stats_exit()
449 	bfq_stat_exit(&stats->group_wait_time);  in bfqg_stats_exit()
450 	bfq_stat_exit(&stats->idle_time);  in bfqg_stats_exit()
451 	bfq_stat_exit(&stats->empty_time);  in bfqg_stats_exit()
455 static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)  in bfqg_stats_init()  argument
457 	if (blkg_rwstat_init(&stats->bytes, gfp) ||  in bfqg_stats_init()
458 	    blkg_rwstat_init(&stats->ios, gfp))  in bfqg_stats_init()
462 	if (blkg_rwstat_init(&stats->merged, gfp) ||  in bfqg_stats_init()
463 	    blkg_rwstat_init(&stats->service_time, gfp) ||  in bfqg_stats_init()
464 	    blkg_rwstat_init(&stats->wait_time, gfp) ||  in bfqg_stats_init()
465 	    blkg_rwstat_init(&stats->queued, gfp) ||  in bfqg_stats_init()
466 	    bfq_stat_init(&stats->time, gfp) ||  in bfqg_stats_init()
467 	    bfq_stat_init(&stats->avg_queue_size_sum, gfp) ||  in bfqg_stats_init()
468 	    bfq_stat_init(&stats->avg_queue_size_samples, gfp) ||  in bfqg_stats_init()
469 	    bfq_stat_init(&stats->dequeue, gfp) ||  in bfqg_stats_init()
470 	    bfq_stat_init(&stats->group_wait_time, gfp) ||  in bfqg_stats_init()
471 	    bfq_stat_init(&stats->idle_time, gfp) ||  in bfqg_stats_init()
472 	    bfq_stat_init(&stats->empty_time, gfp))  in bfqg_stats_init()
479 	bfqg_stats_exit(stats);  in bfqg_stats_init()
519 	if (bfqg_stats_init(&bfqg->stats, gfp)) {  in bfq_pd_alloc()
555 	bfqg_stats_exit(&bfqg->stats);  in bfq_pd_free()
563 	bfqg_stats_reset(&bfqg->stats);  in bfq_pd_reset_stats()
985 	 * blkg_[rw]stat_recursive_sum().  Transfer stats to the parent so  in bfq_pd_offline()
987 	 * stats for them will be lost.  Oh well...  in bfq_pd_offline()
1227 	u64 sum = blkg_rwstat_total(&bfqg->stats.bytes);  in bfqg_prfill_sectors()
1245 			offsetof(struct bfq_group, stats.bytes), &tmp);  in bfqg_prfill_sectors_recursive()
1263 	u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);  in bfqg_prfill_avg_queue_size()
1267 		v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);  in bfqg_prfill_avg_queue_size()
1326 		.private = offsetof(struct bfq_group, stats.bytes),
1331 		.private = offsetof(struct bfq_group, stats.ios),
1337 		.private = offsetof(struct bfq_group, stats.time),
1346 		.private = offsetof(struct bfq_group, stats.service_time),
1351 		.private = offsetof(struct bfq_group, stats.wait_time),
1356 		.private = offsetof(struct bfq_group, stats.merged),
1361 		.private = offsetof(struct bfq_group, stats.queued),
1369 		.private = offsetof(struct bfq_group, stats.bytes),
1374 		.private = offsetof(struct bfq_group, stats.ios),
1380 		.private = offsetof(struct bfq_group, stats.time),
1389 		.private = offsetof(struct bfq_group, stats.service_time),
1394 		.private = offsetof(struct bfq_group, stats.wait_time),
1399 		.private = offsetof(struct bfq_group, stats.merged),
1404 		.private = offsetof(struct bfq_group, stats.queued),
1413 		.private = offsetof(struct bfq_group, stats.group_wait_time),
1418 		.private = offsetof(struct bfq_group, stats.idle_time),
1423 		.private = offsetof(struct bfq_group, stats.empty_time),
1428 		.private = offsetof(struct bfq_group, stats.dequeue),