379 struct bfq_queue *bic_to_bfqq(struct bfq_io_cq *bic, bool is_sync,  in bic_to_bfqq()  argument
383 		return bic->bfqq[1][actuator_idx];  in bic_to_bfqq()
385 	return bic->bfqq[0][actuator_idx];  in bic_to_bfqq()
390 void bic_set_bfqq(struct bfq_io_cq *bic,  in bic_set_bfqq()  argument
395 	struct bfq_queue *old_bfqq = bic->bfqq[is_sync][actuator_idx];  in bic_set_bfqq()
410 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[actuator_idx];  in bic_set_bfqq()
413 	if (old_bfqq && old_bfqq->bic == bic)  in bic_set_bfqq()
414 		old_bfqq->bic = NULL;  in bic_set_bfqq()
417 		bic->bfqq[1][actuator_idx] = bfqq;  in bic_set_bfqq()
419 		bic->bfqq[0][actuator_idx] = bfqq;  in bic_set_bfqq()
436 struct bfq_data *bic_to_bfqd(struct bfq_io_cq *bic)  in bic_to_bfqd()  argument
438 	return bic->icq.q->elevator->elevator_data;  in bic_to_bfqd()
586 				    struct bfq_io_cq *bic, blk_opf_t opf,  in bfqq_request_over_limit()  argument
602 	bfqq = bic_to_bfqq(bic, op_is_sync(opf), act_idx);  in bfqq_request_over_limit()
678 				    struct bfq_io_cq *bic, blk_opf_t opf,  in bfqq_request_over_limit()  argument
703 	struct bfq_io_cq *bic = bfq_bic_lookup(data->q);  in bfq_limit_depth()  local
716 	for (act_idx = 0; bic && act_idx < bfqd->num_actuators; act_idx++) {  in bfq_limit_depth()
718 		if (!bic_to_bfqq(bic, op_is_sync(opf), act_idx))  in bfq_limit_depth()
727 		if (bfqq_request_over_limit(bfqd, bic, opf, act_idx, limit)) {  in bfq_limit_depth()
1145 		      struct bfq_io_cq *bic, bool bfq_already_existing)  in bfq_bfqq_resume_state()  argument
1150 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];  in bfq_bfqq_resume_state()
1848 		bfqq->bic || RQ_BIC(rq)->bfqq_data[act_idx].stably_merged;  in bfq_bfqq_handle_idle_busy_switch()
2468 	struct bfq_io_cq *bic = bfq_bic_lookup(q);  in bfq_bio_merge()  local
2473 	if (bic) {  in bfq_bio_merge()
2478 		bfq_bic_update_cgroup(bic, bio);  in bfq_bio_merge()
2480 		bfqd->bio_bfqq = bic_to_bfqq(bic, op_is_sync(bio->bi_opf),  in bfq_bio_merge()
2485 	bfqd->bio_bic = bic;  in bfq_bio_merge()
2880 		if (new_bfqq->bic) {  in bfq_setup_stable_merge()
2883 				&new_bfqq->bic->bfqq_data[new_a_idx];  in bfq_setup_stable_merge()
2918 		     void *io_struct, bool request, struct bfq_io_cq *bic)  in bfq_setup_cooperator()  argument
2922 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];  in bfq_setup_cooperator()
3053 	struct bfq_io_cq *bic = bfqq->bic;  in bfq_bfqq_save_state()  local
3055 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[a_idx];  in bfq_bfqq_save_state()
3062 	if (!bic)  in bfq_bfqq_save_state()
3144 					 struct bfq_io_cq *bic,  in bfq_merge_bfqqs()  argument
3214 	bic_set_bfqq(bic, new_bfqq, true, bfqq->actuator_idx);  in bfq_merge_bfqqs()
3226 	new_bfqq->bic = NULL;  in bfq_merge_bfqqs()
3237 	bfqq->bic = NULL;  in bfq_merge_bfqqs()
4915 		if (bfqq->bic && bfqq->bic->bfqq[0][act_idx] &&  in bfq_select_queue()
4916 		    bfq_bfqq_busy(bfqq->bic->bfqq[0][act_idx]) &&  in bfq_select_queue()
4917 		    bfqq->bic->bfqq[0][act_idx]->next_rq)  in bfq_select_queue()
4918 			async_bfqq = bfqq->bic->bfqq[0][act_idx];  in bfq_select_queue()
5000 		    icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic &&  in bfq_select_queue()
5386 		if (bfqq->bic && bfqq->bfqd->burst_size > 0)  in bfq_put_queue()
5464 static void bfq_exit_icq_bfqq(struct bfq_io_cq *bic, bool is_sync,  in bfq_exit_icq_bfqq()  argument
5467 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, actuator_idx);  in bfq_exit_icq_bfqq()
5474 		bic_set_bfqq(bic, NULL, is_sync, actuator_idx);  in bfq_exit_icq_bfqq()
5481 	struct bfq_io_cq *bic = icq_to_bic(icq);  in bfq_exit_icq()  local
5482 	struct bfq_data *bfqd = bic_to_bfqd(bic);  in bfq_exit_icq()
5492 	struct bfq_iocq_bfqq_data *bfqq_data = bic->bfqq_data;  in bfq_exit_icq()
5507 		bfq_exit_icq_bfqq(bic, true, act_idx);  in bfq_exit_icq()
5508 		bfq_exit_icq_bfqq(bic, false, act_idx);  in bfq_exit_icq()
5520 bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic)  in bfq_set_next_ioprio_data()  argument
5529 	ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);  in bfq_set_next_ioprio_data()
5544 		bfqq->new_ioprio = IOPRIO_PRIO_LEVEL(bic->ioprio);  in bfq_set_next_ioprio_data()
5548 		bfqq->new_ioprio = IOPRIO_PRIO_LEVEL(bic->ioprio);  in bfq_set_next_ioprio_data()
5571 				       struct bfq_io_cq *bic,
5574 static void bfq_check_ioprio_change(struct bfq_io_cq *bic, struct bio *bio)  in bfq_check_ioprio_change()  argument
5576 	struct bfq_data *bfqd = bic_to_bfqd(bic);  in bfq_check_ioprio_change()
5578 	int ioprio = bic->icq.ioc->ioprio;  in bfq_check_ioprio_change()
5584 	if (unlikely(!bfqd) || likely(bic->ioprio == ioprio))  in bfq_check_ioprio_change()
5587 	bic->ioprio = ioprio;  in bfq_check_ioprio_change()
5589 	bfqq = bic_to_bfqq(bic, false, bfq_actuator_index(bfqd, bio));  in bfq_check_ioprio_change()
5593 		bfqq = bfq_get_queue(bfqd, bio, false, bic, true);  in bfq_check_ioprio_change()
5594 		bic_set_bfqq(bic, bfqq, false, bfq_actuator_index(bfqd, bio));  in bfq_check_ioprio_change()
5598 	bfqq = bic_to_bfqq(bic, true, bfq_actuator_index(bfqd, bio));  in bfq_check_ioprio_change()
5600 		bfq_set_next_ioprio_data(bfqq, bic);  in bfq_check_ioprio_change()
5604 			  struct bfq_io_cq *bic, pid_t pid, int is_sync,  in bfq_init_bfqq()  argument
5619 	if (bic)  in bfq_init_bfqq()
5620 		bfq_set_next_ioprio_data(bfqq, bic);  in bfq_init_bfqq()
5694 			  struct bfq_io_cq *bic,  in bfq_do_early_stable_merge()  argument
5704 	if (new_bfqq->bic)  in bfq_do_early_stable_merge()
5705 		new_bfqq->bic->bfqq_data[a_idx].stably_merged = true;  in bfq_do_early_stable_merge()
5706 	bic->bfqq_data[a_idx].stably_merged = true;  in bfq_do_early_stable_merge()
5714 	bfqq->bic = bic;  in bfq_do_early_stable_merge()
5715 	return bfq_merge_bfqqs(bfqd, bic, bfqq);  in bfq_do_early_stable_merge()
5766 						      struct bfq_io_cq *bic)  in bfq_do_or_sched_stable_merge()  argument
5817 							 bic,  in bfq_do_or_sched_stable_merge()
5834 			bic->bfqq_data[last_bfqq_created->actuator_idx].stable_merge_bfqq =  in bfq_do_or_sched_stable_merge()
5845 				       struct bfq_io_cq *bic,  in bfq_get_queue()  argument
5848 	const int ioprio = IOPRIO_PRIO_LEVEL(bic->ioprio);  in bfq_get_queue()
5849 	const int ioprio_class = IOPRIO_PRIO_CLASS(bic->ioprio);  in bfq_get_queue()
5869 		bfq_init_bfqq(bfqd, bfqq, bic, current->pid,  in bfq_get_queue()
5900 		bfqq = bfq_do_or_sched_stable_merge(bfqd, bfqq, bic);  in bfq_get_queue()
5959 				       struct bfq_io_cq *bic)  in bfq_update_has_short_ttime()  argument
5982 	if (atomic_read(&bic->icq.ioc->active_ref) == 0 ||  in bfq_update_has_short_ttime()
6732 bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq)  in bfq_split_bfqq()  argument
6743 	bic_set_bfqq(bic, NULL, true, bfqq->actuator_idx);  in bfq_split_bfqq()
6752 						   struct bfq_io_cq *bic,  in bfq_get_bfqq_handle_split()  argument
6758 	struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, act_idx);  in bfq_get_bfqq_handle_split()
6759 	struct bfq_iocq_bfqq_data *bfqq_data = &bic->bfqq_data[act_idx];  in bfq_get_bfqq_handle_split()
6769 	bfqq = bfq_get_queue(bfqd, bio, is_sync, bic, split);  in bfq_get_bfqq_handle_split()
6771 	bic_set_bfqq(bic, bfqq, is_sync, act_idx);  in bfq_get_bfqq_handle_split()
6895 	struct bfq_io_cq *bic;  in bfq_init_rq()  local
6915 	bic = icq_to_bic(rq->elv.icq);  in bfq_init_rq()
6917 	bfq_check_ioprio_change(bic, bio);  in bfq_init_rq()
6919 	bfq_bic_update_cgroup(bic, bio);  in bfq_init_rq()
6921 	bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync,  in bfq_init_rq()
6927 			!bic->bfqq_data[a_idx].stably_merged) {  in bfq_init_rq()
6932 				bic->bfqq_data[a_idx].saved_in_large_burst =  in bfq_init_rq()
6935 			bfqq = bfq_split_bfqq(bic, bfqq);  in bfq_init_rq()
6939 				bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio,  in bfq_init_rq()
6967 	bic->requests++;  in bfq_init_rq()
6971 	rq->elv.priv[0] = bic;  in bfq_init_rq()
6982 		bfqq->bic = bic;  in bfq_init_rq()
6989 			bfq_bfqq_resume_state(bfqq, bfqd, bic,  in bfq_init_rq()