138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,  in bfqg_stats_set_start_group_wait_time()
139 						 struct bfq_group *curr_bfqg)  in bfqg_stats_set_start_group_wait_time()
166 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)  in bfqg_stats_update_dequeue()
171 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_empty_time()
190 void bfqg_stats_update_idle_time(struct bfq_group *bfqg)  in bfqg_stats_update_idle_time()
204 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_idle_time()
212 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)  in bfqg_stats_update_avg_queue_size()
222 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()
231 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf)  in bfqg_stats_update_io_remove()
236 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf)  in bfqg_stats_update_io_merged()
241 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()
257 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { }  in bfqg_stats_update_io_remove()
258 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { }  in bfqg_stats_update_io_merged()
259 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()
261 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }  in bfqg_stats_update_dequeue()
262 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_idle_time()
274 static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd)  in pd_to_bfqg()
276 	return pd ? container_of(pd, struct bfq_group, pd) : NULL;  in pd_to_bfqg()
279 struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)  in bfqg_to_blkg()
284 static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)  in blkg_to_bfqg()
296 static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)  in bfqg_parent()
303 struct bfq_group *bfqq_group(struct bfq_queue *bfqq)  in bfqq_group()
307 	return group_entity ? container_of(group_entity, struct bfq_group,  in bfqq_group()
317 static void bfqg_get(struct bfq_group *bfqg)  in bfqg_get()
322 static void bfqg_put(struct bfq_group *bfqg)  in bfqg_put()
328 static void bfqg_and_blkg_get(struct bfq_group *bfqg)  in bfqg_and_blkg_get()
336 void bfqg_and_blkg_put(struct bfq_group *bfqg)  in bfqg_and_blkg_put()
345 	struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);  in bfqg_stats_update_legacy_io()
399 static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)  in bfqg_stats_xfer_dead()
401 	struct bfq_group *parent;  in bfqg_stats_xfer_dead()
417 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()
513 	struct bfq_group *bfqg;  in bfq_pd_alloc()
532 	struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_pd_init()
553 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_free()
561 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_reset_stats()
566 static void bfq_group_set_parent(struct bfq_group *bfqg,  in bfq_group_set_parent()
567 					struct bfq_group *parent)  in bfq_group_set_parent()
576 static void bfq_link_bfqg(struct bfq_data *bfqd, struct bfq_group *bfqg)  in bfq_link_bfqg()
578 	struct bfq_group *parent;  in bfq_link_bfqg()
588 		struct bfq_group *curr_bfqg = container_of(entity,  in bfq_link_bfqg()
589 						struct bfq_group, entity);  in bfq_link_bfqg()
599 struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)  in bfq_bio_bfqg()
602 	struct bfq_group *bfqg;  in bfq_bio_bfqg()
636 		   struct bfq_group *bfqg)  in bfq_bfqq_move()
639 	struct bfq_group *old_parent = bfqq_group(bfqq);  in bfq_bfqq_move()
711 			       struct bfq_group *bfqg,  in bfq_sync_bfqq_move()
760 				    struct bfq_group *bfqg)  in __bfq_bic_change_cgroup()
782 	struct bfq_group *bfqg = bfq_bio_bfqg(bfqd, bio);  in bfq_bic_update_cgroup()
904 				       struct bfq_group *bfqg,  in bfq_reparent_active_queues()
931 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_offline()
997 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_end_wr_async()
1021 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_weight_device()
1039 static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)  in bfq_group_set_weight()
1088 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_io_set_weight_legacy()
1105 	struct bfq_group *bfqg;  in bfq_io_set_device_weight()
1226 	struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);  in bfqg_prfill_sectors()
1245 			offsetof(struct bfq_group, stats.bytes), &tmp);  in bfqg_prfill_sectors_recursive()
1262 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_avg_queue_size()
1284 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()
1326 		.private = offsetof(struct bfq_group, stats.bytes),
1331 		.private = offsetof(struct bfq_group, stats.ios),
1337 		.private = offsetof(struct bfq_group, stats.time),
1346 		.private = offsetof(struct bfq_group, stats.service_time),
1351 		.private = offsetof(struct bfq_group, stats.wait_time),
1356 		.private = offsetof(struct bfq_group, stats.merged),
1361 		.private = offsetof(struct bfq_group, stats.queued),
1369 		.private = offsetof(struct bfq_group, stats.bytes),
1374 		.private = offsetof(struct bfq_group, stats.ios),
1380 		.private = offsetof(struct bfq_group, stats.time),
1389 		.private = offsetof(struct bfq_group, stats.service_time),
1394 		.private = offsetof(struct bfq_group, stats.wait_time),
1399 		.private = offsetof(struct bfq_group, stats.merged),
1404 		.private = offsetof(struct bfq_group, stats.queued),
1413 		.private = offsetof(struct bfq_group, stats.group_wait_time),
1418 		.private = offsetof(struct bfq_group, stats.idle_time),
1423 		.private = offsetof(struct bfq_group, stats.empty_time),
1428 		.private = offsetof(struct bfq_group, stats.dequeue),
1448 		   struct bfq_group *bfqg) {}  in bfq_bfqq_move()
1450 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()
1470 struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)  in bfq_bio_bfqg()
1475 struct bfq_group *bfqq_group(struct bfq_queue *bfqq)  in bfqq_group()
1480 void bfqg_and_blkg_put(struct bfq_group *bfqg) {}  in bfqg_and_blkg_put()
1482 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()
1484 	struct bfq_group *bfqg;  in bfq_create_group_hierarchy()