227 	if (!(bfqq == bfqg->bfqd->in_service_queue))  in bfqg_stats_update_io_add()
309 			      bfqq->bfqd->root_group;  in bfqq_group()
533 	struct bfq_data *bfqd = blkg->q->elevator->elevator_data;  in bfq_pd_init()  local
545 	bfqg->bfqd = bfqd;  in bfq_pd_init()
576 static void bfq_link_bfqg(struct bfq_data *bfqd, struct bfq_group *bfqg)  in bfq_link_bfqg()  argument
590 		if (curr_bfqg != bfqd->root_group) {  in bfq_link_bfqg()
593 				parent = bfqd->root_group;  in bfq_link_bfqg()
599 struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)  in bfq_bio_bfqg()  argument
617 				&bfqg_to_blkg(bfqd->root_group)->blkcg->css);  in bfq_bio_bfqg()
618 	return bfqd->root_group;  in bfq_bio_bfqg()
635 void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_bfqq_move()  argument
653 	if (bfqq == &bfqd->oom_bfqq)  in bfq_bfqq_move()
672 	if (bfqq == bfqd->in_service_queue)  in bfq_bfqq_move()
673 		bfq_bfqq_expire(bfqd, bfqd->in_service_queue,  in bfq_bfqq_move()
677 		bfq_deactivate_bfqq(bfqd, bfqq, false, false);  in bfq_bfqq_move()
685 	else if (bfqd->last_bfqq_created == bfqq)  in bfq_bfqq_move()
686 		bfqd->last_bfqq_created = NULL;  in bfq_bfqq_move()
697 		if (unlikely(!bfqd->nonrot_with_queueing))  in bfq_bfqq_move()
698 			bfq_pos_tree_add_move(bfqd, bfqq);  in bfq_bfqq_move()
699 		bfq_activate_bfqq(bfqd, bfqq);  in bfq_bfqq_move()
702 	if (!bfqd->in_service_queue && !bfqd->tot_rq_in_driver)  in bfq_bfqq_move()
703 		bfq_schedule_dispatch(bfqd);  in bfq_bfqq_move()
708 static void bfq_sync_bfqq_move(struct bfq_data *bfqd,  in bfq_sync_bfqq_move()  argument
719 			bfq_bfqq_move(bfqd, sync_bfqq, bfqg);  in bfq_sync_bfqq_move()
744 		bfq_release_process_ref(bfqd, sync_bfqq);  in bfq_sync_bfqq_move()
758 static void __bfq_bic_change_cgroup(struct bfq_data *bfqd,  in __bfq_bic_change_cgroup()  argument
764 	for (act_idx = 0; act_idx < bfqd->num_actuators; act_idx++) {  in __bfq_bic_change_cgroup()
771 			bfq_release_process_ref(bfqd, async_bfqq);  in __bfq_bic_change_cgroup()
775 			bfq_sync_bfqq_move(bfqd, sync_bfqq, bic, bfqg, act_idx);  in __bfq_bic_change_cgroup()
781 	struct bfq_data *bfqd = bic_to_bfqd(bic);  in bfq_bic_update_cgroup()  local
782 	struct bfq_group *bfqg = bfq_bio_bfqg(bfqd, bio);  in bfq_bic_update_cgroup()
791 	if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr))  in bfq_bic_update_cgroup()
798 	bfq_link_bfqg(bfqd, bfqg);  in bfq_bic_update_cgroup()
799 	__bfq_bic_change_cgroup(bfqd, bic, bfqg);  in bfq_bic_update_cgroup()
873 static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,  in bfq_reparent_leaf_entity()  argument
893 	bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);  in bfq_reparent_leaf_entity()
903 static void bfq_reparent_active_queues(struct bfq_data *bfqd,  in bfq_reparent_active_queues()  argument
912 		bfq_reparent_leaf_entity(bfqd, entity, ioprio_class);  in bfq_reparent_active_queues()
915 		bfq_reparent_leaf_entity(bfqd,  in bfq_reparent_active_queues()
932 	struct bfq_data *bfqd = bfqg->bfqd;  in bfq_pd_offline()  local
937 	spin_lock_irqsave(&bfqd->lock, flags);  in bfq_pd_offline()
961 		bfq_reparent_active_queues(bfqd, bfqg, st, i);  in bfq_pd_offline()
980 	bfq_put_async_queues(bfqd, bfqg);  in bfq_pd_offline()
982 	spin_unlock_irqrestore(&bfqd->lock, flags);  in bfq_pd_offline()
992 void bfq_end_wr_async(struct bfq_data *bfqd)  in bfq_end_wr_async()  argument
996 	list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) {  in bfq_end_wr_async()
999 		bfq_end_wr_async_queues(bfqd, bfqg);  in bfq_end_wr_async()
1001 	bfq_end_wr_async_queues(bfqd, bfqd->root_group);  in bfq_end_wr_async()
1284 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()  argument
1288 	ret = blkcg_activate_policy(bfqd->queue->disk, &blkcg_policy_bfq);  in bfq_create_group_hierarchy()
1292 	return blkg_to_bfqg(bfqd->queue->root_blkg);  in bfq_create_group_hierarchy()
1447 void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,  in bfq_bfqq_move()  argument
1465 void bfq_end_wr_async(struct bfq_data *bfqd)  in bfq_end_wr_async()  argument
1467 	bfq_end_wr_async_queues(bfqd, bfqd->root_group);  in bfq_end_wr_async()
1470 struct bfq_group *bfq_bio_bfqg(struct bfq_data *bfqd, struct bio *bio)  in bfq_bio_bfqg()  argument
1472 	return bfqd->root_group;  in bfq_bio_bfqg()
1477 	return bfqq->bfqd->root_group;  in bfqq_group()
1482 struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)  in bfq_create_group_hierarchy()  argument