138 static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,  in bfqg_stats_set_start_group_wait_time()  argument
141 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_group_wait_time()
145 	if (bfqg == curr_bfqg)  in bfqg_stats_set_start_group_wait_time()
166 void bfqg_stats_update_dequeue(struct bfq_group *bfqg)  in bfqg_stats_update_dequeue()  argument
168 	bfq_stat_add(&bfqg->stats.dequeue, 1);  in bfqg_stats_update_dequeue()
171 void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_empty_time()  argument
173 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_empty_time()
190 void bfqg_stats_update_idle_time(struct bfq_group *bfqg)  in bfqg_stats_update_idle_time()  argument
192 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_idle_time()
204 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)  in bfqg_stats_set_start_idle_time()  argument
206 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_set_start_idle_time()
212 void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)  in bfqg_stats_update_avg_queue_size()  argument
214 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_avg_queue_size()
222 void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,  in bfqg_stats_update_io_add()  argument
225 	blkg_rwstat_add(&bfqg->stats.queued, opf, 1);  in bfqg_stats_update_io_add()
226 	bfqg_stats_end_empty_time(&bfqg->stats);  in bfqg_stats_update_io_add()
227 	if (!(bfqq == bfqg->bfqd->in_service_queue))  in bfqg_stats_update_io_add()
228 		bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));  in bfqg_stats_update_io_add()
231 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf)  in bfqg_stats_update_io_remove()  argument
233 	blkg_rwstat_add(&bfqg->stats.queued, opf, -1);  in bfqg_stats_update_io_remove()
236 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf)  in bfqg_stats_update_io_merged()  argument
238 	blkg_rwstat_add(&bfqg->stats.merged, opf, 1);  in bfqg_stats_update_io_merged()
241 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()  argument
244 	struct bfqg_stats *stats = &bfqg->stats;  in bfqg_stats_update_completion()
257 void bfqg_stats_update_io_remove(struct bfq_group *bfqg, blk_opf_t opf) { }  in bfqg_stats_update_io_remove()  argument
258 void bfqg_stats_update_io_merged(struct bfq_group *bfqg, blk_opf_t opf) { }  in bfqg_stats_update_io_merged()  argument
259 void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,  in bfqg_stats_update_completion()  argument
261 void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }  in bfqg_stats_update_dequeue()  argument
262 void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }  in bfqg_stats_set_start_idle_time()  argument
279 struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)  in bfqg_to_blkg()  argument
281 	return pd_to_blkg(&bfqg->pd);  in bfqg_to_blkg()
296 static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)  in bfqg_parent()  argument
298 	struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;  in bfqg_parent()
317 static void bfqg_get(struct bfq_group *bfqg)  in bfqg_get()  argument
319 	refcount_inc(&bfqg->ref);  in bfqg_get()
322 static void bfqg_put(struct bfq_group *bfqg)  in bfqg_put()  argument
324 	if (refcount_dec_and_test(&bfqg->ref))  in bfqg_put()
325 		kfree(bfqg);  in bfqg_put()
328 static void bfqg_and_blkg_get(struct bfq_group *bfqg)  in bfqg_and_blkg_get()  argument
331 	bfqg_get(bfqg);  in bfqg_and_blkg_get()
333 	blkg_get(bfqg_to_blkg(bfqg));  in bfqg_and_blkg_get()
336 void bfqg_and_blkg_put(struct bfq_group *bfqg)  in bfqg_and_blkg_put()  argument
338 	blkg_put(bfqg_to_blkg(bfqg));  in bfqg_and_blkg_put()
340 	bfqg_put(bfqg);  in bfqg_and_blkg_put()
345 	struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);  in bfqg_stats_update_legacy_io()  local
347 	if (!bfqg)  in bfqg_stats_update_legacy_io()
350 	blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));  in bfqg_stats_update_legacy_io()
351 	blkg_rwstat_add(&bfqg->stats.ios, rq->cmd_flags, 1);  in bfqg_stats_update_legacy_io()
399 static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)  in bfqg_stats_xfer_dead()  argument
403 	if (!bfqg) /* root_group */  in bfqg_stats_xfer_dead()
406 	parent = bfqg_parent(bfqg);  in bfqg_stats_xfer_dead()
408 	lockdep_assert_held(&bfqg_to_blkg(bfqg)->q->queue_lock);  in bfqg_stats_xfer_dead()
413 	bfqg_stats_add_aux(&parent->stats, &bfqg->stats);  in bfqg_stats_xfer_dead()
414 	bfqg_stats_reset(&bfqg->stats);  in bfqg_stats_xfer_dead()
417 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()  argument
430 		bfqg_and_blkg_get(bfqg);  in bfq_init_entity()
432 	entity->parent = bfqg->my_entity; /* NULL for root group */  in bfq_init_entity()
433 	entity->sched_data = &bfqg->sched_data;  in bfq_init_entity()
513 	struct bfq_group *bfqg;  in bfq_pd_alloc()  local
515 	bfqg = kzalloc_node(sizeof(*bfqg), gfp, disk->node_id);  in bfq_pd_alloc()
516 	if (!bfqg)  in bfq_pd_alloc()
519 	if (bfqg_stats_init(&bfqg->stats, gfp)) {  in bfq_pd_alloc()
520 		kfree(bfqg);  in bfq_pd_alloc()
525 	refcount_set(&bfqg->ref, 1);  in bfq_pd_alloc()
526 	return &bfqg->pd;  in bfq_pd_alloc()
532 	struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_pd_init()  local
534 	struct bfq_entity *entity = &bfqg->entity;  in bfq_pd_init()
538 	entity->my_sched_data = &bfqg->sched_data;  in bfq_pd_init()
541 	bfqg->my_entity = entity; /*  in bfq_pd_init()
545 	bfqg->bfqd = bfqd;  in bfq_pd_init()
546 	bfqg->active_entities = 0;  in bfq_pd_init()
547 	bfqg->num_queues_with_pending_reqs = 0;  in bfq_pd_init()
548 	bfqg->rq_pos_tree = RB_ROOT;  in bfq_pd_init()
553 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_free()  local
555 	bfqg_stats_exit(&bfqg->stats);  in bfq_pd_free()
556 	bfqg_put(bfqg);  in bfq_pd_free()
561 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_reset_stats()  local
563 	bfqg_stats_reset(&bfqg->stats);  in bfq_pd_reset_stats()
566 static void bfq_group_set_parent(struct bfq_group *bfqg,  in bfq_group_set_parent()  argument
571 	entity = &bfqg->entity;  in bfq_group_set_parent()
576 static void bfq_link_bfqg(struct bfq_data *bfqd, struct bfq_group *bfqg)  in bfq_link_bfqg()  argument
586 	entity = &bfqg->entity;  in bfq_link_bfqg()
602 	struct bfq_group *bfqg;  in bfq_bio_bfqg()  local
609 		bfqg = blkg_to_bfqg(blkg);  in bfq_bio_bfqg()
610 		if (bfqg->pd.online) {  in bfq_bio_bfqg()
612 			return bfqg;  in bfq_bio_bfqg()
636 		   struct bfq_group *bfqg)  in bfq_bfqq_move()  argument
646 	if (old_parent == bfqg)  in bfq_bfqq_move()
688 	entity->parent = bfqg->my_entity;  in bfq_bfqq_move()
689 	entity->sched_data = &bfqg->sched_data;  in bfq_bfqq_move()
691 	bfqg_and_blkg_get(bfqg);  in bfq_bfqq_move()
711 			       struct bfq_group *bfqg,  in bfq_sync_bfqq_move()  argument
718 		if (sync_bfqq->entity.sched_data != &bfqg->sched_data)  in bfq_sync_bfqq_move()
719 			bfq_bfqq_move(bfqd, sync_bfqq, bfqg);  in bfq_sync_bfqq_move()
729 		if (bfqq->entity.sched_data != &bfqg->sched_data)  in bfq_sync_bfqq_move()
760 				    struct bfq_group *bfqg)  in __bfq_bic_change_cgroup()  argument
769 		    async_bfqq->entity.sched_data != &bfqg->sched_data) {  in __bfq_bic_change_cgroup()
775 			bfq_sync_bfqq_move(bfqd, sync_bfqq, bic, bfqg, act_idx);  in __bfq_bic_change_cgroup()
782 	struct bfq_group *bfqg = bfq_bio_bfqg(bfqd, bio);  in bfq_bic_update_cgroup()  local
785 	serial_nr = bfqg_to_blkg(bfqg)->blkcg->css.serial_nr;  in bfq_bic_update_cgroup()
798 	bfq_link_bfqg(bfqd, bfqg);  in bfq_bic_update_cgroup()
799 	__bfq_bic_change_cgroup(bfqd, bic, bfqg);  in bfq_bic_update_cgroup()
850 	blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));  in bfq_bic_update_cgroup()
904 				       struct bfq_group *bfqg,  in bfq_reparent_active_queues()  argument
914 	if (bfqg->sched_data.in_service_entity)  in bfq_reparent_active_queues()
916 					 bfqg->sched_data.in_service_entity,  in bfq_reparent_active_queues()
931 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfq_pd_offline()  local
932 	struct bfq_data *bfqd = bfqg->bfqd;  in bfq_pd_offline()
933 	struct bfq_entity *entity = bfqg->my_entity;  in bfq_pd_offline()
947 		st = bfqg->sched_data.service_tree + i;  in bfq_pd_offline()
961 		bfq_reparent_active_queues(bfqd, bfqg, st, i);  in bfq_pd_offline()
980 	bfq_put_async_queues(bfqd, bfqg);  in bfq_pd_offline()
989 	bfqg_stats_xfer_dead(bfqg);  in bfq_pd_offline()
997 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_end_wr_async()  local
999 		bfq_end_wr_async_queues(bfqd, bfqg);  in bfq_end_wr_async()
1021 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_weight_device()  local
1023 	if (!bfqg->entity.dev_weight)  in bfqg_prfill_weight_device()
1025 	return __blkg_prfill_u64(sf, pd, bfqg->entity.dev_weight);  in bfqg_prfill_weight_device()
1039 static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)  in bfq_group_set_weight()  argument
1043 	bfqg->entity.dev_weight = dev_weight;  in bfq_group_set_weight()
1050 	if ((unsigned short)weight != bfqg->entity.new_weight) {  in bfq_group_set_weight()
1051 		bfqg->entity.new_weight = (unsigned short)weight;  in bfq_group_set_weight()
1068 		bfqg->entity.prio_changed = 1;  in bfq_group_set_weight()
1088 		struct bfq_group *bfqg = blkg_to_bfqg(blkg);  in bfq_io_set_weight_legacy()  local
1090 		if (bfqg)  in bfq_io_set_weight_legacy()
1091 			bfq_group_set_weight(bfqg, val, 0);  in bfq_io_set_weight_legacy()
1105 	struct bfq_group *bfqg;  in bfq_io_set_device_weight()  local
1126 	bfqg = blkg_to_bfqg(ctx.blkg);  in bfq_io_set_device_weight()
1130 		bfq_group_set_weight(bfqg, bfqg->entity.weight, v);  in bfq_io_set_device_weight()
1226 	struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);  in bfqg_prfill_sectors()  local
1227 	u64 sum = blkg_rwstat_total(&bfqg->stats.bytes);  in bfqg_prfill_sectors()
1262 	struct bfq_group *bfqg = pd_to_bfqg(pd);  in bfqg_prfill_avg_queue_size()  local
1263 	u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);  in bfqg_prfill_avg_queue_size()
1267 		v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);  in bfqg_prfill_avg_queue_size()
1448 		   struct bfq_group *bfqg) {}  in bfq_bfqq_move()  argument
1450 void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)  in bfq_init_entity()  argument
1460 	entity->sched_data = &bfqg->sched_data;  in bfq_init_entity()
1480 void bfqg_and_blkg_put(struct bfq_group *bfqg) {}  in bfqg_and_blkg_put()  argument
1484 	struct bfq_group *bfqg;  in bfq_create_group_hierarchy()  local
1487 	bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);  in bfq_create_group_hierarchy()
1488 	if (!bfqg)  in bfq_create_group_hierarchy()
1492 		bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;  in bfq_create_group_hierarchy()
1494 	return bfqg;  in bfq_create_group_hierarchy()