Lines Matching refs:bfqq
137 void bfq_mark_bfqq_##name(struct bfq_queue *bfqq) \
139 __set_bit(BFQQF_##name, &(bfqq)->flags); \
141 void bfq_clear_bfqq_##name(struct bfq_queue *bfqq) \
143 __clear_bit(BFQQF_##name, &(bfqq)->flags); \
145 int bfq_bfqq_##name(const struct bfq_queue *bfqq) \
147 return test_bit(BFQQF_##name, &(bfqq)->flags); \
237 #define BFQQ_SEEKY(bfqq) (hweight32(bfqq->seek_history) > 19) argument
245 #define BFQQ_TOTALLY_SEEKY(bfqq) (bfqq->seek_history == -1) argument
383 return bic->bfqq[1][actuator_idx]; in bic_to_bfqq()
385 return bic->bfqq[0][actuator_idx]; in bic_to_bfqq()
388 static void bfq_put_stable_ref(struct bfq_queue *bfqq);
391 struct bfq_queue *bfqq, in bic_set_bfqq() argument
395 struct bfq_queue *old_bfqq = bic->bfqq[is_sync][actuator_idx]; in bic_set_bfqq()
417 bic->bfqq[1][actuator_idx] = bfqq; in bic_set_bfqq()
419 bic->bfqq[0][actuator_idx] = bfqq; in bic_set_bfqq()
421 if (bfqq && bfqq_data->stable_merge_bfqq == bfqq) { in bic_set_bfqq()
484 #define bfq_class_idle(bfqq) ((bfqq)->ioprio_class == IOPRIO_CLASS_IDLE) argument
594 struct bfq_queue *bfqq; in bfqq_request_over_limit() local
602 bfqq = bic_to_bfqq(bic, op_is_sync(opf), act_idx); in bfqq_request_over_limit()
603 if (!bfqq) in bfqq_request_over_limit()
606 entity = &bfqq->entity; in bfqq_request_over_limit()
611 depth = bfqg_to_blkg(bfqq_group(bfqq))->blkcg->css.cgroup->level + 1; in bfqq_request_over_limit()
654 for (i = 0; i <= bfqq->ioprio_class - 1; i++) { in bfqq_request_over_limit()
663 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfqq_request_over_limit()
744 struct bfq_queue *bfqq = NULL; in bfq_rq_pos_tree_lookup() local
752 bfqq = rb_entry(parent, struct bfq_queue, pos_node); in bfq_rq_pos_tree_lookup()
758 if (sector > blk_rq_pos(bfqq->next_rq)) in bfq_rq_pos_tree_lookup()
760 else if (sector < blk_rq_pos(bfqq->next_rq)) in bfq_rq_pos_tree_lookup()
765 bfqq = NULL; in bfq_rq_pos_tree_lookup()
774 bfqq ? bfqq->pid : 0); in bfq_rq_pos_tree_lookup()
776 return bfqq; in bfq_rq_pos_tree_lookup()
779 static bool bfq_too_late_for_merging(struct bfq_queue *bfqq) in bfq_too_late_for_merging() argument
781 return bfqq->service_from_backlogged > 0 && in bfq_too_late_for_merging()
782 time_is_before_jiffies(bfqq->first_IO_time + in bfq_too_late_for_merging()
795 bfq_pos_tree_add_move(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_pos_tree_add_move() argument
800 if (bfqq->pos_root) { in bfq_pos_tree_add_move()
801 rb_erase(&bfqq->pos_node, bfqq->pos_root); in bfq_pos_tree_add_move()
802 bfqq->pos_root = NULL; in bfq_pos_tree_add_move()
806 if (bfqq == &bfqd->oom_bfqq) in bfq_pos_tree_add_move()
814 if (bfq_too_late_for_merging(bfqq)) in bfq_pos_tree_add_move()
817 if (bfq_class_idle(bfqq)) in bfq_pos_tree_add_move()
819 if (!bfqq->next_rq) in bfq_pos_tree_add_move()
822 bfqq->pos_root = &bfqq_group(bfqq)->rq_pos_tree; in bfq_pos_tree_add_move()
823 __bfqq = bfq_rq_pos_tree_lookup(bfqd, bfqq->pos_root, in bfq_pos_tree_add_move()
824 blk_rq_pos(bfqq->next_rq), &parent, &p); in bfq_pos_tree_add_move()
826 rb_link_node(&bfqq->pos_node, parent, p); in bfq_pos_tree_add_move()
827 rb_insert_color(&bfqq->pos_node, bfqq->pos_root); in bfq_pos_tree_add_move()
829 bfqq->pos_root = NULL; in bfq_pos_tree_add_move()
866 struct bfq_queue *bfqq) in bfq_asymmetric_scenario() argument
868 bool smallest_weight = bfqq && in bfq_asymmetric_scenario()
869 bfqq->weight_counter && in bfq_asymmetric_scenario()
870 bfqq->weight_counter == in bfq_asymmetric_scenario()
910 void bfq_weights_tree_add(struct bfq_queue *bfqq) in bfq_weights_tree_add() argument
912 struct rb_root_cached *root = &bfqq->bfqd->queue_weights_tree; in bfq_weights_tree_add()
913 struct bfq_entity *entity = &bfqq->entity; in bfq_weights_tree_add()
929 if (bfqq->weight_counter) in bfq_weights_tree_add()
939 bfqq->weight_counter = __counter; in bfq_weights_tree_add()
950 bfqq->weight_counter = kzalloc(sizeof(struct bfq_weight_counter), in bfq_weights_tree_add()
965 if (unlikely(!bfqq->weight_counter)) in bfq_weights_tree_add()
968 bfqq->weight_counter->weight = entity->weight; in bfq_weights_tree_add()
969 rb_link_node(&bfqq->weight_counter->weights_node, parent, new); in bfq_weights_tree_add()
970 rb_insert_color_cached(&bfqq->weight_counter->weights_node, root, in bfq_weights_tree_add()
974 bfqq->weight_counter->num_active++; in bfq_weights_tree_add()
975 bfqq->ref++; in bfq_weights_tree_add()
984 void bfq_weights_tree_remove(struct bfq_queue *bfqq) in bfq_weights_tree_remove() argument
988 if (!bfqq->weight_counter) in bfq_weights_tree_remove()
991 root = &bfqq->bfqd->queue_weights_tree; in bfq_weights_tree_remove()
992 bfqq->weight_counter->num_active--; in bfq_weights_tree_remove()
993 if (bfqq->weight_counter->num_active > 0) in bfq_weights_tree_remove()
996 rb_erase_cached(&bfqq->weight_counter->weights_node, root); in bfq_weights_tree_remove()
997 kfree(bfqq->weight_counter); in bfq_weights_tree_remove()
1000 bfqq->weight_counter = NULL; in bfq_weights_tree_remove()
1001 bfq_put_queue(bfqq); in bfq_weights_tree_remove()
1007 static struct request *bfq_check_fifo(struct bfq_queue *bfqq, in bfq_check_fifo() argument
1012 if (bfq_bfqq_fifo_expire(bfqq)) in bfq_check_fifo()
1015 bfq_mark_bfqq_fifo_expire(bfqq); in bfq_check_fifo()
1017 rq = rq_entry_fifo(bfqq->fifo.next); in bfq_check_fifo()
1022 bfq_log_bfqq(bfqq->bfqd, bfqq, "check_fifo: returned %p", rq); in bfq_check_fifo()
1027 struct bfq_queue *bfqq, in bfq_find_next_rq() argument
1035 next = bfq_check_fifo(bfqq, last); in bfq_find_next_rq()
1045 rbnext = rb_first(&bfqq->sort_list); in bfq_find_next_rq()
1055 struct bfq_queue *bfqq) in bfq_serv_to_charge() argument
1057 if (bfq_bfqq_sync(bfqq) || bfqq->wr_coeff > 1 || in bfq_serv_to_charge()
1058 bfq_asymmetric_scenario(bfqq->bfqd, bfqq)) in bfq_serv_to_charge()
1076 struct bfq_queue *bfqq) in bfq_updated_next_req() argument
1078 struct bfq_entity *entity = &bfqq->entity; in bfq_updated_next_req()
1079 struct request *next_rq = bfqq->next_rq; in bfq_updated_next_req()
1085 if (bfqq == bfqd->in_service_queue) in bfq_updated_next_req()
1093 max_t(unsigned long, bfqq->max_budget, in bfq_updated_next_req()
1094 bfq_serv_to_charge(next_rq, bfqq)), in bfq_updated_next_req()
1098 bfq_log_bfqq(bfqd, bfqq, "updated next rq: new budget %lu", in bfq_updated_next_req()
1100 bfq_requeue_bfqq(bfqd, bfqq, false); in bfq_updated_next_req()
1135 static void switch_back_to_interactive_wr(struct bfq_queue *bfqq, in switch_back_to_interactive_wr() argument
1138 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in switch_back_to_interactive_wr()
1139 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in switch_back_to_interactive_wr()
1140 bfqq->last_wr_start_finish = bfqq->wr_start_at_switch_to_srt; in switch_back_to_interactive_wr()
1144 bfq_bfqq_resume_state(struct bfq_queue *bfqq, struct bfq_data *bfqd, in bfq_bfqq_resume_state() argument
1148 bool busy = bfq_already_existing && bfq_bfqq_busy(bfqq); in bfq_bfqq_resume_state()
1149 unsigned int a_idx = bfqq->actuator_idx; in bfq_bfqq_resume_state()
1153 bfq_mark_bfqq_has_short_ttime(bfqq); in bfq_bfqq_resume_state()
1155 bfq_clear_bfqq_has_short_ttime(bfqq); in bfq_bfqq_resume_state()
1158 bfq_mark_bfqq_IO_bound(bfqq); in bfq_bfqq_resume_state()
1160 bfq_clear_bfqq_IO_bound(bfqq); in bfq_bfqq_resume_state()
1162 bfqq->last_serv_time_ns = bfqq_data->saved_last_serv_time_ns; in bfq_bfqq_resume_state()
1163 bfqq->inject_limit = bfqq_data->saved_inject_limit; in bfq_bfqq_resume_state()
1164 bfqq->decrease_time_jif = bfqq_data->saved_decrease_time_jif; in bfq_bfqq_resume_state()
1166 bfqq->entity.new_weight = bfqq_data->saved_weight; in bfq_bfqq_resume_state()
1167 bfqq->ttime = bfqq_data->saved_ttime; in bfq_bfqq_resume_state()
1168 bfqq->io_start_time = bfqq_data->saved_io_start_time; in bfq_bfqq_resume_state()
1169 bfqq->tot_idle_time = bfqq_data->saved_tot_idle_time; in bfq_bfqq_resume_state()
1174 old_wr_coeff = bfqq->wr_coeff; in bfq_bfqq_resume_state()
1175 bfqq->wr_coeff = bfqq_data->saved_wr_coeff; in bfq_bfqq_resume_state()
1177 bfqq->service_from_wr = bfqq_data->saved_service_from_wr; in bfq_bfqq_resume_state()
1178 bfqq->wr_start_at_switch_to_srt = in bfq_bfqq_resume_state()
1180 bfqq->last_wr_start_finish = bfqq_data->saved_last_wr_start_finish; in bfq_bfqq_resume_state()
1181 bfqq->wr_cur_max_time = bfqq_data->saved_wr_cur_max_time; in bfq_bfqq_resume_state()
1183 if (bfqq->wr_coeff > 1 && (bfq_bfqq_in_large_burst(bfqq) || in bfq_bfqq_resume_state()
1184 time_is_before_jiffies(bfqq->last_wr_start_finish + in bfq_bfqq_resume_state()
1185 bfqq->wr_cur_max_time))) { in bfq_bfqq_resume_state()
1186 if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in bfq_bfqq_resume_state()
1187 !bfq_bfqq_in_large_burst(bfqq) && in bfq_bfqq_resume_state()
1188 time_is_after_eq_jiffies(bfqq->wr_start_at_switch_to_srt + in bfq_bfqq_resume_state()
1190 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_bfqq_resume_state()
1192 bfqq->wr_coeff = 1; in bfq_bfqq_resume_state()
1193 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfq_bfqq_resume_state()
1199 bfqq->entity.prio_changed = 1; in bfq_bfqq_resume_state()
1204 if (old_wr_coeff == 1 && bfqq->wr_coeff > 1) in bfq_bfqq_resume_state()
1206 else if (old_wr_coeff > 1 && bfqq->wr_coeff == 1) in bfq_bfqq_resume_state()
1210 static int bfqq_process_refs(struct bfq_queue *bfqq) in bfqq_process_refs() argument
1212 return bfqq->ref - bfqq->entity.allocated - in bfqq_process_refs()
1213 bfqq->entity.on_st_or_in_serv - in bfqq_process_refs()
1214 (bfqq->weight_counter != NULL) - bfqq->stable_ref; in bfqq_process_refs()
1218 static void bfq_reset_burst_list(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_reset_burst_list() argument
1232 hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); in bfq_reset_burst_list()
1237 bfqd->burst_parent_entity = bfqq->entity.parent; in bfq_reset_burst_list()
1241 static void bfq_add_to_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_add_to_burst() argument
1263 bfq_mark_bfqq_in_large_burst(bfqq); in bfq_add_to_burst()
1281 hlist_add_head(&bfqq->burst_list_node, &bfqd->burst_list); in bfq_add_to_burst()
1393 static void bfq_handle_burst(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_handle_burst() argument
1400 if (!hlist_unhashed(&bfqq->burst_list_node) || in bfq_handle_burst()
1401 bfq_bfqq_in_large_burst(bfqq) || in bfq_handle_burst()
1402 time_is_after_eq_jiffies(bfqq->split_time + in bfq_handle_burst()
1425 bfqq->entity.parent != bfqd->burst_parent_entity) { in bfq_handle_burst()
1427 bfq_reset_burst_list(bfqd, bfqq); in bfq_handle_burst()
1437 bfq_mark_bfqq_in_large_burst(bfqq); in bfq_handle_burst()
1446 bfq_add_to_burst(bfqd, bfqq); in bfq_handle_burst()
1459 static int bfq_bfqq_budget_left(struct bfq_queue *bfqq) in bfq_bfqq_budget_left() argument
1461 struct bfq_entity *entity = &bfqq->entity; in bfq_bfqq_budget_left()
1596 struct bfq_queue *bfqq, in bfq_bfqq_update_budg_for_activation() argument
1599 struct bfq_entity *entity = &bfqq->entity; in bfq_bfqq_update_budg_for_activation()
1608 if (bfq_bfqq_non_blocking_wait_rq(bfqq) && arrived_in_time && in bfq_bfqq_update_budg_for_activation()
1609 bfq_bfqq_budget_left(bfqq) > 0) { in bfq_bfqq_update_budg_for_activation()
1628 bfq_bfqq_budget_left(bfqq), in bfq_bfqq_update_budg_for_activation()
1629 bfqq->max_budget); in bfq_bfqq_update_budg_for_activation()
1649 entity->budget = max_t(unsigned long, bfqq->max_budget, in bfq_bfqq_update_budg_for_activation()
1650 bfq_serv_to_charge(bfqq->next_rq, bfqq)); in bfq_bfqq_update_budg_for_activation()
1651 bfq_clear_bfqq_non_blocking_wait_rq(bfqq); in bfq_bfqq_update_budg_for_activation()
1665 struct bfq_queue *bfqq, in bfq_update_bfqq_wr_on_rq_arrival() argument
1675 bfqq->service_from_wr = 0; in bfq_update_bfqq_wr_on_rq_arrival()
1676 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_update_bfqq_wr_on_rq_arrival()
1677 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_update_bfqq_wr_on_rq_arrival()
1690 bfqq->wr_start_at_switch_to_srt = in bfq_update_bfqq_wr_on_rq_arrival()
1692 bfqq->wr_coeff = bfqd->bfq_wr_coeff * in bfq_update_bfqq_wr_on_rq_arrival()
1694 bfqq->wr_cur_max_time = in bfq_update_bfqq_wr_on_rq_arrival()
1707 bfqq->entity.budget = min_t(unsigned long, in bfq_update_bfqq_wr_on_rq_arrival()
1708 bfqq->entity.budget, in bfq_update_bfqq_wr_on_rq_arrival()
1712 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_update_bfqq_wr_on_rq_arrival()
1713 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_update_bfqq_wr_on_rq_arrival()
1715 bfqq->wr_coeff = 1; in bfq_update_bfqq_wr_on_rq_arrival()
1746 if (bfqq->wr_cur_max_time != in bfq_update_bfqq_wr_on_rq_arrival()
1748 bfqq->wr_start_at_switch_to_srt = in bfq_update_bfqq_wr_on_rq_arrival()
1749 bfqq->last_wr_start_finish; in bfq_update_bfqq_wr_on_rq_arrival()
1751 bfqq->wr_cur_max_time = in bfq_update_bfqq_wr_on_rq_arrival()
1753 bfqq->wr_coeff = bfqd->bfq_wr_coeff * in bfq_update_bfqq_wr_on_rq_arrival()
1756 bfqq->last_wr_start_finish = jiffies; in bfq_update_bfqq_wr_on_rq_arrival()
1762 struct bfq_queue *bfqq) in bfq_bfqq_idle_for_long_time() argument
1764 return bfqq->dispatched == 0 && in bfq_bfqq_idle_for_long_time()
1766 bfqq->budget_timeout + in bfq_bfqq_idle_for_long_time()
1775 static bool bfq_bfqq_higher_class_or_weight(struct bfq_queue *bfqq, in bfq_bfqq_higher_class_or_weight() argument
1780 if (bfqq->ioprio_class < in_serv_bfqq->ioprio_class) in bfq_bfqq_higher_class_or_weight()
1783 if (in_serv_bfqq->entity.parent == bfqq->entity.parent) { in bfq_bfqq_higher_class_or_weight()
1784 bfqq_weight = bfqq->entity.weight; in bfq_bfqq_higher_class_or_weight()
1787 if (bfqq->entity.parent) in bfq_bfqq_higher_class_or_weight()
1788 bfqq_weight = bfqq->entity.parent->weight; in bfq_bfqq_higher_class_or_weight()
1790 bfqq_weight = bfqq->entity.weight; in bfq_bfqq_higher_class_or_weight()
1827 static bool bfq_better_to_idle(struct bfq_queue *bfqq);
1830 struct bfq_queue *bfqq, in bfq_bfqq_handle_idle_busy_switch() argument
1837 idle_for_long_time = bfq_bfqq_idle_for_long_time(bfqd, bfqq), in bfq_bfqq_handle_idle_busy_switch()
1844 bfqq->ttime.last_end_request + in bfq_bfqq_handle_idle_busy_switch()
1848 bfqq->bic || RQ_BIC(rq)->bfqq_data[act_idx].stably_merged; in bfq_bfqq_handle_idle_busy_switch()
1859 in_burst = bfq_bfqq_in_large_burst(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1861 !BFQQ_TOTALLY_SEEKY(bfqq) && in bfq_bfqq_handle_idle_busy_switch()
1863 time_is_before_jiffies(bfqq->soft_rt_next_start) && in bfq_bfqq_handle_idle_busy_switch()
1864 bfqq->dispatched == 0 && in bfq_bfqq_handle_idle_busy_switch()
1865 bfqq->entity.new_weight == 40; in bfq_bfqq_handle_idle_busy_switch()
1867 bfqq->entity.new_weight == 40; in bfq_bfqq_handle_idle_busy_switch()
1881 (bfqq->wr_coeff > 1 || in bfq_bfqq_handle_idle_busy_switch()
1882 (bfq_bfqq_sync(bfqq) && bfqq_non_merged_or_stably_merged && in bfq_bfqq_handle_idle_busy_switch()
1890 bfq_bfqq_update_budg_for_activation(bfqd, bfqq, in bfq_bfqq_handle_idle_busy_switch()
1906 if (likely(!bfq_bfqq_just_created(bfqq)) && in bfq_bfqq_handle_idle_busy_switch()
1909 bfqq->budget_timeout + in bfq_bfqq_handle_idle_busy_switch()
1911 hlist_del_init(&bfqq->burst_list_node); in bfq_bfqq_handle_idle_busy_switch()
1912 bfq_clear_bfqq_in_large_burst(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1915 bfq_clear_bfqq_just_created(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1918 if (unlikely(time_is_after_jiffies(bfqq->split_time))) in bfq_bfqq_handle_idle_busy_switch()
1920 bfqq->split_time = in bfq_bfqq_handle_idle_busy_switch()
1923 if (time_is_before_jiffies(bfqq->split_time + in bfq_bfqq_handle_idle_busy_switch()
1925 bfq_update_bfqq_wr_on_rq_arrival(bfqd, bfqq, in bfq_bfqq_handle_idle_busy_switch()
1932 if (old_wr_coeff != bfqq->wr_coeff) in bfq_bfqq_handle_idle_busy_switch()
1933 bfqq->entity.prio_changed = 1; in bfq_bfqq_handle_idle_busy_switch()
1937 bfqq->last_idle_bklogged = jiffies; in bfq_bfqq_handle_idle_busy_switch()
1938 bfqq->service_from_backlogged = 0; in bfq_bfqq_handle_idle_busy_switch()
1939 bfq_clear_bfqq_softrt_update(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1941 bfq_add_bfqq_busy(bfqq); in bfq_bfqq_handle_idle_busy_switch()
1988 bfqq->wr_coeff >= bfqd->in_service_queue->wr_coeff) || in bfq_bfqq_handle_idle_busy_switch()
1989 bfq_bfqq_higher_class_or_weight(bfqq, bfqd->in_service_queue) || in bfq_bfqq_handle_idle_busy_switch()
1997 struct bfq_queue *bfqq) in bfq_reset_inject_limit() argument
2000 bfqq->last_serv_time_ns = 0; in bfq_reset_inject_limit()
2054 if (bfq_bfqq_has_short_ttime(bfqq)) in bfq_reset_inject_limit()
2055 bfqq->inject_limit = 0; in bfq_reset_inject_limit()
2057 bfqq->inject_limit = 1; in bfq_reset_inject_limit()
2059 bfqq->decrease_time_jif = jiffies; in bfq_reset_inject_limit()
2062 static void bfq_update_io_intensity(struct bfq_queue *bfqq, u64 now_ns) in bfq_update_io_intensity() argument
2064 u64 tot_io_time = now_ns - bfqq->io_start_time; in bfq_update_io_intensity()
2066 if (RB_EMPTY_ROOT(&bfqq->sort_list) && bfqq->dispatched == 0) in bfq_update_io_intensity()
2067 bfqq->tot_idle_time += in bfq_update_io_intensity()
2068 now_ns - bfqq->ttime.last_end_request; in bfq_update_io_intensity()
2070 if (unlikely(bfq_bfqq_just_created(bfqq))) in bfq_update_io_intensity()
2077 if (bfqq->tot_idle_time * 5 > tot_io_time) in bfq_update_io_intensity()
2078 bfq_clear_bfqq_IO_bound(bfqq); in bfq_update_io_intensity()
2080 bfq_mark_bfqq_IO_bound(bfqq); in bfq_update_io_intensity()
2087 bfqq->io_start_time = now_ns - (tot_io_time>>1); in bfq_update_io_intensity()
2088 bfqq->tot_idle_time >>= 1; in bfq_update_io_intensity()
2143 static void bfq_check_waker(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_check_waker() argument
2149 bfqd->last_completed_rq_bfqq == bfqq || in bfq_check_waker()
2150 bfq_bfqq_has_short_ttime(bfqq) || in bfq_check_waker()
2153 bfqq == &bfqd->oom_bfqq) in bfq_check_waker()
2163 bfqq->tentative_waker_bfqq || in bfq_check_waker()
2164 now_ns > bfqq->waker_detection_started + in bfq_check_waker()
2171 bfqq->tentative_waker_bfqq = in bfq_check_waker()
2173 bfqq->num_waker_detections = 1; in bfq_check_waker()
2174 bfqq->waker_detection_started = now_ns; in bfq_check_waker()
2175 bfq_bfqq_name(bfqq->tentative_waker_bfqq, waker_name, in bfq_check_waker()
2177 bfq_log_bfqq(bfqd, bfqq, "set tentative waker %s", waker_name); in bfq_check_waker()
2179 bfqq->num_waker_detections++; in bfq_check_waker()
2181 if (bfqq->num_waker_detections == 3) { in bfq_check_waker()
2182 bfqq->waker_bfqq = bfqd->last_completed_rq_bfqq; in bfq_check_waker()
2183 bfqq->tentative_waker_bfqq = NULL; in bfq_check_waker()
2184 bfq_bfqq_name(bfqq->waker_bfqq, waker_name, in bfq_check_waker()
2186 bfq_log_bfqq(bfqd, bfqq, "set waker %s", waker_name); in bfq_check_waker()
2208 if (!hlist_unhashed(&bfqq->woken_list_node)) in bfq_check_waker()
2209 hlist_del_init(&bfqq->woken_list_node); in bfq_check_waker()
2210 hlist_add_head(&bfqq->woken_list_node, in bfq_check_waker()
2217 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_add_request() local
2218 struct bfq_data *bfqd = bfqq->bfqd; in bfq_add_request()
2220 unsigned int old_wr_coeff = bfqq->wr_coeff; in bfq_add_request()
2224 bfq_log_bfqq(bfqd, bfqq, "add_request %d", rq_is_sync(rq)); in bfq_add_request()
2225 bfqq->queued[rq_is_sync(rq)]++; in bfq_add_request()
2232 if (bfq_bfqq_sync(bfqq) && RQ_BIC(rq)->requests <= 1) { in bfq_add_request()
2233 bfq_check_waker(bfqd, bfqq, now_ns); in bfq_add_request()
2241 if (time_is_before_eq_jiffies(bfqq->decrease_time_jif + in bfq_add_request()
2243 bfq_reset_inject_limit(bfqd, bfqq); in bfq_add_request()
2270 if (bfqq == bfqd->in_service_queue && in bfq_add_request()
2272 (bfqq->last_serv_time_ns > 0 && in bfq_add_request()
2274 time_is_before_eq_jiffies(bfqq->decrease_time_jif + in bfq_add_request()
2302 if (bfq_bfqq_sync(bfqq)) in bfq_add_request()
2303 bfq_update_io_intensity(bfqq, now_ns); in bfq_add_request()
2305 elv_rb_add(&bfqq->sort_list, rq); in bfq_add_request()
2310 prev = bfqq->next_rq; in bfq_add_request()
2311 next_rq = bfq_choose_req(bfqd, bfqq->next_rq, rq, bfqd->last_position); in bfq_add_request()
2312 bfqq->next_rq = next_rq; in bfq_add_request()
2318 if (unlikely(!bfqd->nonrot_with_queueing && prev != bfqq->next_rq)) in bfq_add_request()
2319 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_add_request()
2321 if (!bfq_bfqq_busy(bfqq)) /* switching to busy ... */ in bfq_add_request()
2322 bfq_bfqq_handle_idle_busy_switch(bfqd, bfqq, old_wr_coeff, in bfq_add_request()
2327 bfqq->last_wr_start_finish + in bfq_add_request()
2329 bfqq->wr_coeff = bfqd->bfq_wr_coeff; in bfq_add_request()
2330 bfqq->wr_cur_max_time = bfq_wr_duration(bfqd); in bfq_add_request()
2333 bfqq->entity.prio_changed = 1; in bfq_add_request()
2335 if (prev != bfqq->next_rq) in bfq_add_request()
2336 bfq_updated_next_req(bfqd, bfqq); in bfq_add_request()
2366 (old_wr_coeff == 1 || bfqq->wr_coeff == 1 || interactive)) in bfq_add_request()
2367 bfqq->last_wr_start_finish = jiffies; in bfq_add_request()
2374 struct bfq_queue *bfqq = bfqd->bio_bfqq; in bfq_find_rq_fmerge() local
2377 if (bfqq) in bfq_find_rq_fmerge()
2378 return elv_rb_find(&bfqq->sort_list, bio_end_sector(bio)); in bfq_find_rq_fmerge()
2394 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_remove_request() local
2395 struct bfq_data *bfqd = bfqq->bfqd; in bfq_remove_request()
2398 if (bfqq->next_rq == rq) { in bfq_remove_request()
2399 bfqq->next_rq = bfq_find_next_rq(bfqd, bfqq, rq); in bfq_remove_request()
2400 bfq_updated_next_req(bfqd, bfqq); in bfq_remove_request()
2405 bfqq->queued[sync]--; in bfq_remove_request()
2411 elv_rb_del(&bfqq->sort_list, rq); in bfq_remove_request()
2417 if (RB_EMPTY_ROOT(&bfqq->sort_list)) { in bfq_remove_request()
2418 bfqq->next_rq = NULL; in bfq_remove_request()
2420 if (bfq_bfqq_busy(bfqq) && bfqq != bfqd->in_service_queue) { in bfq_remove_request()
2421 bfq_del_bfqq_busy(bfqq, false); in bfq_remove_request()
2435 bfqq->entity.budget = bfqq->entity.service = 0; in bfq_remove_request()
2441 if (bfqq->pos_root) { in bfq_remove_request()
2442 rb_erase(&bfqq->pos_node, bfqq->pos_root); in bfq_remove_request()
2443 bfqq->pos_root = NULL; in bfq_remove_request()
2448 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_remove_request()
2452 bfqq->meta_pending--; in bfq_remove_request()
2522 struct bfq_queue *bfqq = RQ_BFQQ(req); in bfq_request_merged() local
2526 if (!bfqq) in bfq_request_merged()
2529 bfqd = bfqq->bfqd; in bfq_request_merged()
2532 elv_rb_del(&bfqq->sort_list, req); in bfq_request_merged()
2533 elv_rb_add(&bfqq->sort_list, req); in bfq_request_merged()
2536 prev = bfqq->next_rq; in bfq_request_merged()
2537 next_rq = bfq_choose_req(bfqd, bfqq->next_rq, req, in bfq_request_merged()
2539 bfqq->next_rq = next_rq; in bfq_request_merged()
2545 if (prev != bfqq->next_rq) { in bfq_request_merged()
2546 bfq_updated_next_req(bfqd, bfqq); in bfq_request_merged()
2552 bfq_pos_tree_add_move(bfqd, bfqq); in bfq_request_merged()
2574 struct bfq_queue *bfqq = RQ_BFQQ(rq), in bfq_requests_merged() local
2577 if (!bfqq) in bfq_requests_merged()
2589 if (bfqq == next_bfqq && in bfq_requests_merged()
2597 if (bfqq->next_rq == next) in bfq_requests_merged()
2598 bfqq->next_rq = rq; in bfq_requests_merged()
2600 bfqg_stats_update_io_merged(bfqq_group(bfqq), next->cmd_flags); in bfq_requests_merged()
2612 static void bfq_bfqq_end_wr(struct bfq_queue *bfqq) in bfq_bfqq_end_wr() argument
2628 if (bfqq->wr_cur_max_time != in bfq_bfqq_end_wr()
2629 bfqq->bfqd->bfq_wr_rt_max_time) in bfq_bfqq_end_wr()
2630 bfqq->soft_rt_next_start = jiffies; in bfq_bfqq_end_wr()
2632 if (bfq_bfqq_busy(bfqq)) in bfq_bfqq_end_wr()
2633 bfqq->bfqd->wr_busy_queues--; in bfq_bfqq_end_wr()
2634 bfqq->wr_coeff = 1; in bfq_bfqq_end_wr()
2635 bfqq->wr_cur_max_time = 0; in bfq_bfqq_end_wr()
2636 bfqq->last_wr_start_finish = jiffies; in bfq_bfqq_end_wr()
2641 bfqq->entity.prio_changed = 1; in bfq_bfqq_end_wr()
2661 struct bfq_queue *bfqq; in bfq_end_wr() local
2667 list_for_each_entry(bfqq, &bfqd->active_list[i], bfqq_list) in bfq_end_wr()
2668 bfq_bfqq_end_wr(bfqq); in bfq_end_wr()
2670 list_for_each_entry(bfqq, &bfqd->idle_list, bfqq_list) in bfq_end_wr()
2671 bfq_bfqq_end_wr(bfqq); in bfq_end_wr()
2693 struct bfq_queue *bfqq, in bfqq_find_close()
2696 struct rb_root *root = &bfqq_group(bfqq)->rq_pos_tree; in bfqq_find_close()
2738 struct bfq_queue *bfqq; in bfq_find_close_cooperator() local
2747 bfqq = bfqq_find_close(bfqd, cur_bfqq, sector); in bfq_find_close_cooperator()
2748 if (!bfqq || bfqq == cur_bfqq) in bfq_find_close_cooperator()
2751 return bfqq; in bfq_find_close_cooperator()
2755 bfq_setup_merge(struct bfq_queue *bfqq, struct bfq_queue *new_bfqq) in bfq_setup_merge() argument
2771 if (__bfqq == bfqq) in bfq_setup_merge()
2776 process_refs = bfqq_process_refs(bfqq); in bfq_setup_merge()
2790 if (new_bfqq->entity.parent != bfqq->entity.parent) in bfq_setup_merge()
2793 bfq_log_bfqq(bfqq->bfqd, bfqq, "scheduling merge with queue %d", in bfq_setup_merge()
2816 bfqq->new_bfqq = new_bfqq; in bfq_setup_merge()
2830 static bool bfq_may_be_close_cooperator(struct bfq_queue *bfqq, in bfq_may_be_close_cooperator() argument
2836 if (bfq_class_idle(bfqq) || bfq_class_idle(new_bfqq) || in bfq_may_be_close_cooperator()
2837 (bfqq->ioprio_class != new_bfqq->ioprio_class)) in bfq_may_be_close_cooperator()
2845 if (BFQQ_SEEKY(bfqq) || BFQQ_SEEKY(new_bfqq)) in bfq_may_be_close_cooperator()
2853 if (!bfq_bfqq_sync(bfqq) || !bfq_bfqq_sync(new_bfqq)) in bfq_may_be_close_cooperator()
2860 struct bfq_queue *bfqq);
2863 bfq_setup_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_setup_stable_merge() argument
2867 int proc_ref = min(bfqq_process_refs(bfqq), in bfq_setup_stable_merge()
2872 if (idling_boosts_thr_without_issues(bfqd, bfqq) || proc_ref == 0) in bfq_setup_stable_merge()
2876 new_bfqq = bfq_setup_merge(bfqq, stable_merge_bfqq); in bfq_setup_stable_merge()
2917 bfq_setup_cooperator(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_setup_cooperator() argument
2921 unsigned int a_idx = bfqq->actuator_idx; in bfq_setup_cooperator()
2925 new_bfqq = bfqq->new_bfqq; in bfq_setup_cooperator()
2948 if (bfq_bfqq_sync(bfqq) && bfqq_data->stable_merge_bfqq && in bfq_setup_cooperator()
2949 !bfq_bfqq_just_created(bfqq) && in bfq_setup_cooperator()
2950 time_is_before_jiffies(bfqq->split_time + in bfq_setup_cooperator()
2952 time_is_before_jiffies(bfqq->creation_time + in bfq_setup_cooperator()
2957 return bfq_setup_stable_merge(bfqd, bfqq, in bfq_setup_cooperator()
3014 if (bfq_too_late_for_merging(bfqq)) in bfq_setup_cooperator()
3017 if (!io_struct || unlikely(bfqq == &bfqd->oom_bfqq)) in bfq_setup_cooperator()
3026 if (in_service_bfqq && in_service_bfqq != bfqq && in bfq_setup_cooperator()
3030 bfqq->entity.parent == in_service_bfqq->entity.parent && in bfq_setup_cooperator()
3031 bfq_may_be_close_cooperator(bfqq, in_service_bfqq)) { in bfq_setup_cooperator()
3032 new_bfqq = bfq_setup_merge(bfqq, in_service_bfqq); in bfq_setup_cooperator()
3041 new_bfqq = bfq_find_close_cooperator(bfqd, bfqq, in bfq_setup_cooperator()
3045 bfq_may_be_close_cooperator(bfqq, new_bfqq)) in bfq_setup_cooperator()
3046 return bfq_setup_merge(bfqq, new_bfqq); in bfq_setup_cooperator()
3051 static void bfq_bfqq_save_state(struct bfq_queue *bfqq) in bfq_bfqq_save_state() argument
3053 struct bfq_io_cq *bic = bfqq->bic; in bfq_bfqq_save_state()
3054 unsigned int a_idx = bfqq->actuator_idx; in bfq_bfqq_save_state()
3065 bfqq_data->saved_last_serv_time_ns = bfqq->last_serv_time_ns; in bfq_bfqq_save_state()
3066 bfqq_data->saved_inject_limit = bfqq->inject_limit; in bfq_bfqq_save_state()
3067 bfqq_data->saved_decrease_time_jif = bfqq->decrease_time_jif; in bfq_bfqq_save_state()
3069 bfqq_data->saved_weight = bfqq->entity.orig_weight; in bfq_bfqq_save_state()
3070 bfqq_data->saved_ttime = bfqq->ttime; in bfq_bfqq_save_state()
3072 bfq_bfqq_has_short_ttime(bfqq); in bfq_bfqq_save_state()
3073 bfqq_data->saved_IO_bound = bfq_bfqq_IO_bound(bfqq); in bfq_bfqq_save_state()
3074 bfqq_data->saved_io_start_time = bfqq->io_start_time; in bfq_bfqq_save_state()
3075 bfqq_data->saved_tot_idle_time = bfqq->tot_idle_time; in bfq_bfqq_save_state()
3076 bfqq_data->saved_in_large_burst = bfq_bfqq_in_large_burst(bfqq); in bfq_bfqq_save_state()
3078 !hlist_unhashed(&bfqq->burst_list_node); in bfq_bfqq_save_state()
3080 if (unlikely(bfq_bfqq_just_created(bfqq) && in bfq_bfqq_save_state()
3081 !bfq_bfqq_in_large_burst(bfqq) && in bfq_bfqq_save_state()
3082 bfqq->bfqd->low_latency)) { in bfq_bfqq_save_state()
3092 bfqq_data->saved_wr_coeff = bfqq->bfqd->bfq_wr_coeff; in bfq_bfqq_save_state()
3096 bfq_wr_duration(bfqq->bfqd); in bfq_bfqq_save_state()
3099 bfqq_data->saved_wr_coeff = bfqq->wr_coeff; in bfq_bfqq_save_state()
3101 bfqq->wr_start_at_switch_to_srt; in bfq_bfqq_save_state()
3103 bfqq->service_from_wr; in bfq_bfqq_save_state()
3105 bfqq->last_wr_start_finish; in bfq_bfqq_save_state()
3106 bfqq_data->saved_wr_cur_max_time = bfqq->wr_cur_max_time; in bfq_bfqq_save_state()
3121 void bfq_release_process_ref(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_release_process_ref() argument
3134 if (bfq_bfqq_busy(bfqq) && RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_release_process_ref()
3135 bfqq != bfqd->in_service_queue) in bfq_release_process_ref()
3136 bfq_del_bfqq_busy(bfqq, false); in bfq_release_process_ref()
3138 bfq_reassign_last_bfqq(bfqq, NULL); in bfq_release_process_ref()
3140 bfq_put_queue(bfqq); in bfq_release_process_ref()
3145 struct bfq_queue *bfqq) in bfq_merge_bfqqs() argument
3147 struct bfq_queue *new_bfqq = bfqq->new_bfqq; in bfq_merge_bfqqs()
3149 bfq_log_bfqq(bfqd, bfqq, "merging with queue %lu", in bfq_merge_bfqqs()
3152 bfq_bfqq_save_state(bfqq); in bfq_merge_bfqqs()
3154 if (bfq_bfqq_IO_bound(bfqq)) in bfq_merge_bfqqs()
3156 bfq_clear_bfqq_IO_bound(bfqq); in bfq_merge_bfqqs()
3165 if (bfqq->waker_bfqq && !new_bfqq->waker_bfqq && in bfq_merge_bfqqs()
3166 bfqq->waker_bfqq != new_bfqq) { in bfq_merge_bfqqs()
3167 new_bfqq->waker_bfqq = bfqq->waker_bfqq; in bfq_merge_bfqqs()
3190 if (new_bfqq->wr_coeff == 1 && bfqq->wr_coeff > 1) { in bfq_merge_bfqqs()
3191 new_bfqq->wr_coeff = bfqq->wr_coeff; in bfq_merge_bfqqs()
3192 new_bfqq->wr_cur_max_time = bfqq->wr_cur_max_time; in bfq_merge_bfqqs()
3193 new_bfqq->last_wr_start_finish = bfqq->last_wr_start_finish; in bfq_merge_bfqqs()
3195 bfqq->wr_start_at_switch_to_srt; in bfq_merge_bfqqs()
3201 if (bfqq->wr_coeff > 1) { /* bfqq has given its wr to new_bfqq */ in bfq_merge_bfqqs()
3202 bfqq->wr_coeff = 1; in bfq_merge_bfqqs()
3203 bfqq->entity.prio_changed = 1; in bfq_merge_bfqqs()
3204 if (bfq_bfqq_busy(bfqq)) in bfq_merge_bfqqs()
3214 bic_set_bfqq(bic, new_bfqq, true, bfqq->actuator_idx); in bfq_merge_bfqqs()
3237 bfqq->bic = NULL; in bfq_merge_bfqqs()
3239 bfq_reassign_last_bfqq(bfqq, new_bfqq); in bfq_merge_bfqqs()
3241 bfq_release_process_ref(bfqd, bfqq); in bfq_merge_bfqqs()
3251 struct bfq_queue *bfqq = bfqd->bio_bfqq, *new_bfqq; in bfq_allow_bio_merge() local
3263 if (!bfqq) in bfq_allow_bio_merge()
3270 new_bfqq = bfq_setup_cooperator(bfqd, bfqq, bio, false, bfqd->bio_bic); in bfq_allow_bio_merge()
3279 while (bfqq != new_bfqq) in bfq_allow_bio_merge()
3280 bfqq = bfq_merge_bfqqs(bfqd, bfqd->bio_bic, bfqq); in bfq_allow_bio_merge()
3288 bfqd->bio_bfqq = bfqq; in bfq_allow_bio_merge()
3291 return bfqq == RQ_BFQQ(rq); in bfq_allow_bio_merge()
3301 struct bfq_queue *bfqq) in bfq_set_budget_timeout() argument
3305 if (bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time) in bfq_set_budget_timeout()
3308 timeout_coeff = bfqq->entity.weight / bfqq->entity.orig_weight; in bfq_set_budget_timeout()
3312 bfqq->budget_timeout = jiffies + in bfq_set_budget_timeout()
3317 struct bfq_queue *bfqq) in __bfq_set_in_service_queue() argument
3319 if (bfqq) { in __bfq_set_in_service_queue()
3320 bfq_clear_bfqq_fifo_expire(bfqq); in __bfq_set_in_service_queue()
3324 if (time_is_before_jiffies(bfqq->last_wr_start_finish) && in __bfq_set_in_service_queue()
3325 bfqq->wr_coeff > 1 && in __bfq_set_in_service_queue()
3326 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in __bfq_set_in_service_queue()
3327 time_is_before_jiffies(bfqq->budget_timeout)) { in __bfq_set_in_service_queue()
3352 if (time_after(bfqq->budget_timeout, in __bfq_set_in_service_queue()
3353 bfqq->last_wr_start_finish)) in __bfq_set_in_service_queue()
3354 bfqq->last_wr_start_finish += in __bfq_set_in_service_queue()
3355 jiffies - bfqq->budget_timeout; in __bfq_set_in_service_queue()
3357 bfqq->last_wr_start_finish = jiffies; in __bfq_set_in_service_queue()
3360 bfq_set_budget_timeout(bfqd, bfqq); in __bfq_set_in_service_queue()
3361 bfq_log_bfqq(bfqd, bfqq, in __bfq_set_in_service_queue()
3363 bfqq->entity.budget); in __bfq_set_in_service_queue()
3366 bfqd->in_service_queue = bfqq; in __bfq_set_in_service_queue()
3375 struct bfq_queue *bfqq = bfq_get_next_queue(bfqd); in bfq_set_in_service_queue() local
3377 __bfq_set_in_service_queue(bfqd, bfqq); in bfq_set_in_service_queue()
3378 return bfqq; in bfq_set_in_service_queue()
3383 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_arm_slice_timer() local
3386 bfq_mark_bfqq_wait_request(bfqq); in bfq_arm_slice_timer()
3404 if (BFQQ_SEEKY(bfqq) && bfqq->wr_coeff == 1 && in bfq_arm_slice_timer()
3405 !bfq_asymmetric_scenario(bfqd, bfqq)) in bfq_arm_slice_timer()
3407 else if (bfqq->wr_coeff > 1) in bfq_arm_slice_timer()
3415 bfqg_stats_set_start_idle_time(bfqq_group(bfqq)); in bfq_arm_slice_timer()
3668 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_dispatch_remove() local
3682 bfqq->dispatched++; in bfq_dispatch_remove()
3890 struct bfq_queue *bfqq) in idling_needed_for_service_guarantees() argument
3895 if (unlikely(!bfqq_process_refs(bfqq))) in idling_needed_for_service_guarantees()
3898 return (bfqq->wr_coeff > 1 && in idling_needed_for_service_guarantees()
3900 bfqd->tot_rq_in_driver >= bfqq->dispatched + 4)) || in idling_needed_for_service_guarantees()
3901 bfq_asymmetric_scenario(bfqd, bfqq) || in idling_needed_for_service_guarantees()
3905 static bool __bfq_bfqq_expire(struct bfq_data *bfqd, struct bfq_queue *bfqq, in __bfq_bfqq_expire() argument
3914 if (bfq_bfqq_coop(bfqq) && BFQQ_SEEKY(bfqq)) in __bfq_bfqq_expire()
3915 bfq_mark_bfqq_split_coop(bfqq); in __bfq_bfqq_expire()
3930 if (RB_EMPTY_ROOT(&bfqq->sort_list) && in __bfq_bfqq_expire()
3932 idling_needed_for_service_guarantees(bfqd, bfqq))) { in __bfq_bfqq_expire()
3933 if (bfqq->dispatched == 0) in __bfq_bfqq_expire()
3940 bfqq->budget_timeout = jiffies; in __bfq_bfqq_expire()
3942 bfq_del_bfqq_busy(bfqq, true); in __bfq_bfqq_expire()
3944 bfq_requeue_bfqq(bfqd, bfqq, true); in __bfq_bfqq_expire()
3950 !RB_EMPTY_ROOT(&bfqq->sort_list))) in __bfq_bfqq_expire()
3951 bfq_pos_tree_add_move(bfqd, bfqq); in __bfq_bfqq_expire()
3974 struct bfq_queue *bfqq, in __bfq_bfqq_recalc_budget() argument
3982 if (bfqq->wr_coeff == 1) in __bfq_bfqq_recalc_budget()
3983 budget = bfqq->max_budget; in __bfq_bfqq_recalc_budget()
3992 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last budg %d, budg left %d", in __bfq_bfqq_recalc_budget()
3993 bfqq->entity.budget, bfq_bfqq_budget_left(bfqq)); in __bfq_bfqq_recalc_budget()
3994 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: last max_budg %d, min budg %d", in __bfq_bfqq_recalc_budget()
3996 bfq_log_bfqq(bfqd, bfqq, "recalc_budg: sync %d, seeky %d", in __bfq_bfqq_recalc_budget()
3997 bfq_bfqq_sync(bfqq), BFQQ_SEEKY(bfqd->in_service_queue)); in __bfq_bfqq_recalc_budget()
3999 if (bfq_bfqq_sync(bfqq) && bfqq->wr_coeff == 1) { in __bfq_bfqq_recalc_budget()
4030 if (bfqq->dispatched > 0) /* still outstanding reqs */ in __bfq_bfqq_recalc_budget()
4093 budget = max_t(int, bfqq->entity.service, min_budget); in __bfq_bfqq_recalc_budget()
4098 } else if (!bfq_bfqq_sync(bfqq)) { in __bfq_bfqq_recalc_budget()
4108 bfqq->max_budget = budget; in __bfq_bfqq_recalc_budget()
4112 bfqq->max_budget = min(bfqq->max_budget, bfqd->bfq_max_budget); in __bfq_bfqq_recalc_budget()
4124 next_rq = bfqq->next_rq; in __bfq_bfqq_recalc_budget()
4126 bfqq->entity.budget = max_t(unsigned long, bfqq->max_budget, in __bfq_bfqq_recalc_budget()
4127 bfq_serv_to_charge(next_rq, bfqq)); in __bfq_bfqq_recalc_budget()
4129 bfq_log_bfqq(bfqd, bfqq, "head sect: %u, new budget %d", in __bfq_bfqq_recalc_budget()
4131 bfqq->entity.budget); in __bfq_bfqq_recalc_budget()
4165 static bool bfq_bfqq_is_slow(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_bfqq_is_slow() argument
4170 bool slow = BFQQ_SEEKY(bfqq); /* if delta too short, use seekyness */ in bfq_bfqq_is_slow()
4172 if (!bfq_bfqq_sync(bfqq)) in bfq_bfqq_is_slow()
4213 slow = bfqq->entity.service < bfqd->bfq_max_budget / 2; in bfq_bfqq_is_slow()
4216 bfq_log_bfqq(bfqd, bfqq, "bfq_bfqq_is_slow: slow %d", slow); in bfq_bfqq_is_slow()
4315 struct bfq_queue *bfqq) in bfq_bfqq_softrt_next_start() argument
4317 return max3(bfqq->soft_rt_next_start, in bfq_bfqq_softrt_next_start()
4318 bfqq->last_idle_bklogged + in bfq_bfqq_softrt_next_start()
4319 HZ * bfqq->service_from_backlogged / in bfq_bfqq_softrt_next_start()
4321 jiffies + nsecs_to_jiffies(bfqq->bfqd->bfq_slice_idle) + 4); in bfq_bfqq_softrt_next_start()
4351 struct bfq_queue *bfqq, in bfq_bfqq_expire() argument
4357 struct bfq_entity *entity = &bfqq->entity; in bfq_bfqq_expire()
4362 slow = bfq_bfqq_is_slow(bfqd, bfqq, compensate, &delta); in bfq_bfqq_expire()
4379 if (bfqq->wr_coeff == 1 && in bfq_bfqq_expire()
4382 bfq_bfqq_budget_left(bfqq) >= entity->budget / 3))) in bfq_bfqq_expire()
4383 bfq_bfqq_charge_time(bfqd, bfqq, delta); in bfq_bfqq_expire()
4385 if (bfqd->low_latency && bfqq->wr_coeff == 1) in bfq_bfqq_expire()
4386 bfqq->last_wr_start_finish = jiffies; in bfq_bfqq_expire()
4389 RB_EMPTY_ROOT(&bfqq->sort_list)) { in bfq_bfqq_expire()
4402 if (bfqq->dispatched == 0) in bfq_bfqq_expire()
4403 bfqq->soft_rt_next_start = in bfq_bfqq_expire()
4404 bfq_bfqq_softrt_next_start(bfqd, bfqq); in bfq_bfqq_expire()
4405 else if (bfqq->dispatched > 0) { in bfq_bfqq_expire()
4410 bfq_mark_bfqq_softrt_update(bfqq); in bfq_bfqq_expire()
4414 bfq_log_bfqq(bfqd, bfqq, in bfq_bfqq_expire()
4416 slow, bfqq->dispatched, bfq_bfqq_has_short_ttime(bfqq)); in bfq_bfqq_expire()
4430 __bfq_bfqq_recalc_budget(bfqd, bfqq, reason); in bfq_bfqq_expire()
4431 if (__bfq_bfqq_expire(bfqd, bfqq, reason)) in bfq_bfqq_expire()
4436 if (!bfq_bfqq_busy(bfqq) && in bfq_bfqq_expire()
4439 bfq_mark_bfqq_non_blocking_wait_rq(bfqq); in bfq_bfqq_expire()
4475 static bool bfq_bfqq_budget_timeout(struct bfq_queue *bfqq) in bfq_bfqq_budget_timeout() argument
4477 return time_is_before_eq_jiffies(bfqq->budget_timeout); in bfq_bfqq_budget_timeout()
4488 static bool bfq_may_expire_for_budg_timeout(struct bfq_queue *bfqq) in bfq_may_expire_for_budg_timeout() argument
4490 bfq_log_bfqq(bfqq->bfqd, bfqq, in bfq_may_expire_for_budg_timeout()
4492 bfq_bfqq_wait_request(bfqq), in bfq_may_expire_for_budg_timeout()
4493 bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3, in bfq_may_expire_for_budg_timeout()
4494 bfq_bfqq_budget_timeout(bfqq)); in bfq_may_expire_for_budg_timeout()
4496 return (!bfq_bfqq_wait_request(bfqq) || in bfq_may_expire_for_budg_timeout()
4497 bfq_bfqq_budget_left(bfqq) >= bfqq->entity.budget / 3) in bfq_may_expire_for_budg_timeout()
4499 bfq_bfqq_budget_timeout(bfqq); in bfq_may_expire_for_budg_timeout()
4503 struct bfq_queue *bfqq) in idling_boosts_thr_without_issues() argument
4511 if (unlikely(!bfqq_process_refs(bfqq))) in idling_boosts_thr_without_issues()
4514 bfqq_sequential_and_IO_bound = !BFQQ_SEEKY(bfqq) && in idling_boosts_thr_without_issues()
4515 bfq_bfqq_IO_bound(bfqq) && bfq_bfqq_has_short_ttime(bfqq); in idling_boosts_thr_without_issues()
4602 static bool bfq_better_to_idle(struct bfq_queue *bfqq) in bfq_better_to_idle() argument
4604 struct bfq_data *bfqd = bfqq->bfqd; in bfq_better_to_idle()
4608 if (unlikely(!bfqq_process_refs(bfqq))) in bfq_better_to_idle()
4622 if (bfqd->bfq_slice_idle == 0 || !bfq_bfqq_sync(bfqq) || in bfq_better_to_idle()
4623 bfq_class_idle(bfqq)) in bfq_better_to_idle()
4627 idling_boosts_thr_without_issues(bfqd, bfqq); in bfq_better_to_idle()
4630 idling_needed_for_service_guarantees(bfqd, bfqq); in bfq_better_to_idle()
4653 static bool bfq_bfqq_must_idle(struct bfq_queue *bfqq) in bfq_bfqq_must_idle() argument
4655 return RB_EMPTY_ROOT(&bfqq->sort_list) && bfq_better_to_idle(bfqq); in bfq_bfqq_must_idle()
4668 struct bfq_queue *bfqq, *in_serv_bfqq = bfqd->in_service_queue; in bfq_choose_bfqq_for_injection() local
4719 list_for_each_entry(bfqq, &bfqd->active_list[i], bfqq_list) in bfq_choose_bfqq_for_injection()
4720 if (!RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_choose_bfqq_for_injection()
4721 (in_serv_always_inject || bfqq->wr_coeff > 1) && in bfq_choose_bfqq_for_injection()
4722 bfq_serv_to_charge(bfqq->next_rq, bfqq) <= in bfq_choose_bfqq_for_injection()
4723 bfq_bfqq_budget_left(bfqq)) { in bfq_choose_bfqq_for_injection()
4742 blk_rq_sectors(bfqq->next_rq) >= in bfq_choose_bfqq_for_injection()
4748 return bfqq; in bfq_choose_bfqq_for_injection()
4759 struct bfq_queue *bfqq; in bfq_find_active_bfqq_for_actuator() local
4765 list_for_each_entry(bfqq, &bfqd->active_list[idx], bfqq_list) { in bfq_find_active_bfqq_for_actuator()
4766 if (!RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_find_active_bfqq_for_actuator()
4767 bfq_serv_to_charge(bfqq->next_rq, bfqq) <= in bfq_find_active_bfqq_for_actuator()
4768 bfq_bfqq_budget_left(bfqq)) { in bfq_find_active_bfqq_for_actuator()
4769 return bfqq; in bfq_find_active_bfqq_for_actuator()
4798 struct bfq_queue *bfqq = in bfq_find_bfqq_for_underused_actuator() local
4801 if (bfqq) in bfq_find_bfqq_for_underused_actuator()
4802 return bfqq; in bfq_find_bfqq_for_underused_actuator()
4816 struct bfq_queue *bfqq, *inject_bfqq; in bfq_select_queue() local
4820 bfqq = bfqd->in_service_queue; in bfq_select_queue()
4821 if (!bfqq) in bfq_select_queue()
4824 bfq_log_bfqq(bfqd, bfqq, "select_queue: already in-service queue"); in bfq_select_queue()
4833 if (bfq_may_expire_for_budg_timeout(bfqq) && in bfq_select_queue()
4834 !bfq_bfqq_must_idle(bfqq)) in bfq_select_queue()
4844 if (inject_bfqq && inject_bfqq != bfqq) in bfq_select_queue()
4853 next_rq = bfqq->next_rq; in bfq_select_queue()
4859 if (bfq_serv_to_charge(next_rq, bfqq) > in bfq_select_queue()
4860 bfq_bfqq_budget_left(bfqq)) { in bfq_select_queue()
4875 if (bfq_bfqq_wait_request(bfqq)) { in bfq_select_queue()
4889 bfq_clear_bfqq_wait_request(bfqq); in bfq_select_queue()
4904 if (bfq_bfqq_wait_request(bfqq) || in bfq_select_queue()
4905 (bfqq->dispatched != 0 && bfq_better_to_idle(bfqq))) { in bfq_select_queue()
4906 unsigned int act_idx = bfqq->actuator_idx; in bfq_select_queue()
4909 !hlist_empty(&bfqq->woken_list) ? in bfq_select_queue()
4910 container_of(bfqq->woken_list.first, in bfq_select_queue()
4915 if (bfqq->bic && bfqq->bic->bfqq[0][act_idx] && in bfq_select_queue()
4916 bfq_bfqq_busy(bfqq->bic->bfqq[0][act_idx]) && in bfq_select_queue()
4917 bfqq->bic->bfqq[0][act_idx]->next_rq) in bfq_select_queue()
4918 async_bfqq = bfqq->bic->bfqq[0][act_idx]; in bfq_select_queue()
5000 icq_to_bic(async_bfqq->next_rq->elv.icq) == bfqq->bic && in bfq_select_queue()
5003 bfqq = async_bfqq; in bfq_select_queue()
5004 else if (bfqq->waker_bfqq && in bfq_select_queue()
5005 bfq_bfqq_busy(bfqq->waker_bfqq) && in bfq_select_queue()
5006 bfqq->waker_bfqq->next_rq && in bfq_select_queue()
5007 bfq_serv_to_charge(bfqq->waker_bfqq->next_rq, in bfq_select_queue()
5008 bfqq->waker_bfqq) <= in bfq_select_queue()
5009 bfq_bfqq_budget_left(bfqq->waker_bfqq) in bfq_select_queue()
5011 bfqq = bfqq->waker_bfqq; in bfq_select_queue()
5019 bfqq = blocked_bfqq; in bfq_select_queue()
5020 else if (!idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_select_queue()
5021 (bfqq->wr_coeff == 1 || bfqd->wr_busy_queues > 1 || in bfq_select_queue()
5022 !bfq_bfqq_has_short_ttime(bfqq))) in bfq_select_queue()
5023 bfqq = bfq_choose_bfqq_for_injection(bfqd); in bfq_select_queue()
5025 bfqq = NULL; in bfq_select_queue()
5032 bfq_bfqq_expire(bfqd, bfqq, false, reason); in bfq_select_queue()
5034 bfqq = bfq_set_in_service_queue(bfqd); in bfq_select_queue()
5035 if (bfqq) { in bfq_select_queue()
5036 bfq_log_bfqq(bfqd, bfqq, "select_queue: checking new queue"); in bfq_select_queue()
5040 if (bfqq) in bfq_select_queue()
5041 bfq_log_bfqq(bfqd, bfqq, "select_queue: returned this queue"); in bfq_select_queue()
5045 return bfqq; in bfq_select_queue()
5048 static void bfq_update_wr_data(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_update_wr_data() argument
5050 struct bfq_entity *entity = &bfqq->entity; in bfq_update_wr_data()
5052 if (bfqq->wr_coeff > 1) { /* queue is being weight-raised */ in bfq_update_wr_data()
5053 bfq_log_bfqq(bfqd, bfqq, in bfq_update_wr_data()
5055 jiffies_to_msecs(jiffies - bfqq->last_wr_start_finish), in bfq_update_wr_data()
5056 jiffies_to_msecs(bfqq->wr_cur_max_time), in bfq_update_wr_data()
5057 bfqq->wr_coeff, in bfq_update_wr_data()
5058 bfqq->entity.weight, bfqq->entity.orig_weight); in bfq_update_wr_data()
5061 bfq_log_bfqq(bfqd, bfqq, "WARN: pending prio change"); in bfq_update_wr_data()
5068 if (bfq_bfqq_in_large_burst(bfqq)) in bfq_update_wr_data()
5069 bfq_bfqq_end_wr(bfqq); in bfq_update_wr_data()
5070 else if (time_is_before_jiffies(bfqq->last_wr_start_finish + in bfq_update_wr_data()
5071 bfqq->wr_cur_max_time)) { in bfq_update_wr_data()
5072 if (bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time || in bfq_update_wr_data()
5073 time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt + in bfq_update_wr_data()
5083 bfq_bfqq_end_wr(bfqq); in bfq_update_wr_data()
5089 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_update_wr_data()
5090 bfqq->entity.prio_changed = 1; in bfq_update_wr_data()
5093 if (bfqq->wr_coeff > 1 && in bfq_update_wr_data()
5094 bfqq->wr_cur_max_time != bfqd->bfq_wr_rt_max_time && in bfq_update_wr_data()
5095 bfqq->service_from_wr > max_service_from_wr) { in bfq_update_wr_data()
5097 bfq_bfqq_end_wr(bfqq); in bfq_update_wr_data()
5108 if ((entity->weight > entity->orig_weight) != (bfqq->wr_coeff > 1)) in bfq_update_wr_data()
5117 struct bfq_queue *bfqq) in bfq_dispatch_rq_from_bfqq() argument
5119 struct request *rq = bfqq->next_rq; in bfq_dispatch_rq_from_bfqq()
5122 service_to_charge = bfq_serv_to_charge(rq, bfqq); in bfq_dispatch_rq_from_bfqq()
5124 bfq_bfqq_served(bfqq, service_to_charge); in bfq_dispatch_rq_from_bfqq()
5126 if (bfqq == bfqd->in_service_queue && bfqd->wait_dispatch) { in bfq_dispatch_rq_from_bfqq()
5133 if (bfqq != bfqd->in_service_queue) in bfq_dispatch_rq_from_bfqq()
5147 bfq_update_wr_data(bfqd, bfqq); in bfq_dispatch_rq_from_bfqq()
5154 if (bfq_tot_busy_queues(bfqd) > 1 && bfq_class_idle(bfqq)) in bfq_dispatch_rq_from_bfqq()
5155 bfq_bfqq_expire(bfqd, bfqq, false, BFQQE_BUDGET_EXHAUSTED); in bfq_dispatch_rq_from_bfqq()
5176 struct bfq_queue *bfqq = NULL; in __bfq_dispatch_request() local
5183 bfqq = RQ_BFQQ(rq); in __bfq_dispatch_request()
5185 if (bfqq) { in __bfq_dispatch_request()
5192 bfqq->dispatched++; in __bfq_dispatch_request()
5244 bfqq = bfq_select_queue(bfqd); in __bfq_dispatch_request()
5245 if (!bfqq) in __bfq_dispatch_request()
5248 rq = bfq_dispatch_rq_from_bfqq(bfqd, bfqq); in __bfq_dispatch_request()
5252 bfqd->rq_in_driver[bfqq->actuator_idx]++; in __bfq_dispatch_request()
5267 struct bfq_queue *bfqq = rq ? RQ_BFQQ(rq) : NULL; in bfq_update_dispatch_stats() local
5269 if (!idle_timer_disabled && !bfqq) in bfq_update_dispatch_stats()
5297 if (bfqq) { in bfq_update_dispatch_stats()
5298 struct bfq_group *bfqg = bfqq_group(bfqq); in bfq_update_dispatch_stats()
5346 void bfq_put_queue(struct bfq_queue *bfqq) in bfq_put_queue() argument
5350 struct bfq_group *bfqg = bfqq_group(bfqq); in bfq_put_queue()
5352 bfq_log_bfqq(bfqq->bfqd, bfqq, "put_queue: %p %d", bfqq, bfqq->ref); in bfq_put_queue()
5354 bfqq->ref--; in bfq_put_queue()
5355 if (bfqq->ref) in bfq_put_queue()
5358 if (!hlist_unhashed(&bfqq->burst_list_node)) { in bfq_put_queue()
5359 hlist_del_init(&bfqq->burst_list_node); in bfq_put_queue()
5386 if (bfqq->bic && bfqq->bfqd->burst_size > 0) in bfq_put_queue()
5387 bfqq->bfqd->burst_size--; in bfq_put_queue()
5406 if (!hlist_unhashed(&bfqq->woken_list_node)) in bfq_put_queue()
5407 hlist_del_init(&bfqq->woken_list_node); in bfq_put_queue()
5410 hlist_for_each_entry_safe(item, n, &bfqq->woken_list, in bfq_put_queue()
5416 if (bfqq->bfqd->last_completed_rq_bfqq == bfqq) in bfq_put_queue()
5417 bfqq->bfqd->last_completed_rq_bfqq = NULL; in bfq_put_queue()
5419 WARN_ON_ONCE(!list_empty(&bfqq->fifo)); in bfq_put_queue()
5420 WARN_ON_ONCE(!RB_EMPTY_ROOT(&bfqq->sort_list)); in bfq_put_queue()
5421 WARN_ON_ONCE(bfqq->dispatched); in bfq_put_queue()
5423 kmem_cache_free(bfq_pool, bfqq); in bfq_put_queue()
5427 static void bfq_put_stable_ref(struct bfq_queue *bfqq) in bfq_put_stable_ref() argument
5429 bfqq->stable_ref--; in bfq_put_stable_ref()
5430 bfq_put_queue(bfqq); in bfq_put_stable_ref()
5433 void bfq_put_cooperator(struct bfq_queue *bfqq) in bfq_put_cooperator() argument
5442 __bfqq = bfqq->new_bfqq; in bfq_put_cooperator()
5450 static void bfq_exit_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_exit_bfqq() argument
5452 if (bfqq == bfqd->in_service_queue) { in bfq_exit_bfqq()
5453 __bfq_bfqq_expire(bfqd, bfqq, BFQQE_BUDGET_TIMEOUT); in bfq_exit_bfqq()
5457 bfq_log_bfqq(bfqd, bfqq, "exit_bfqq: %p, %d", bfqq, bfqq->ref); in bfq_exit_bfqq()
5459 bfq_put_cooperator(bfqq); in bfq_exit_bfqq()
5461 bfq_release_process_ref(bfqd, bfqq); in bfq_exit_bfqq()
5467 struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, actuator_idx); in bfq_exit_icq_bfqq() local
5470 if (bfqq) in bfq_exit_icq_bfqq()
5471 bfqd = bfqq->bfqd; /* NULL if scheduler already exited */ in bfq_exit_icq_bfqq()
5473 if (bfqq && bfqd) { in bfq_exit_icq_bfqq()
5475 bfq_exit_bfqq(bfqd, bfqq); in bfq_exit_icq_bfqq()
5520 bfq_set_next_ioprio_data(struct bfq_queue *bfqq, struct bfq_io_cq *bic) in bfq_set_next_ioprio_data() argument
5524 struct bfq_data *bfqd = bfqq->bfqd; in bfq_set_next_ioprio_data()
5533 bdi_dev_name(bfqq->bfqd->queue->disk->bdi), in bfq_set_next_ioprio_data()
5540 bfqq->new_ioprio = task_nice_ioprio(tsk); in bfq_set_next_ioprio_data()
5541 bfqq->new_ioprio_class = task_nice_ioclass(tsk); in bfq_set_next_ioprio_data()
5544 bfqq->new_ioprio = IOPRIO_PRIO_LEVEL(bic->ioprio); in bfq_set_next_ioprio_data()
5545 bfqq->new_ioprio_class = IOPRIO_CLASS_RT; in bfq_set_next_ioprio_data()
5548 bfqq->new_ioprio = IOPRIO_PRIO_LEVEL(bic->ioprio); in bfq_set_next_ioprio_data()
5549 bfqq->new_ioprio_class = IOPRIO_CLASS_BE; in bfq_set_next_ioprio_data()
5552 bfqq->new_ioprio_class = IOPRIO_CLASS_IDLE; in bfq_set_next_ioprio_data()
5553 bfqq->new_ioprio = IOPRIO_NR_LEVELS - 1; in bfq_set_next_ioprio_data()
5557 if (bfqq->new_ioprio >= IOPRIO_NR_LEVELS) { in bfq_set_next_ioprio_data()
5559 bfqq->new_ioprio); in bfq_set_next_ioprio_data()
5560 bfqq->new_ioprio = IOPRIO_NR_LEVELS - 1; in bfq_set_next_ioprio_data()
5563 bfqq->entity.new_weight = bfq_ioprio_to_weight(bfqq->new_ioprio); in bfq_set_next_ioprio_data()
5564 bfq_log_bfqq(bfqd, bfqq, "new_ioprio %d new_weight %d", in bfq_set_next_ioprio_data()
5565 bfqq->new_ioprio, bfqq->entity.new_weight); in bfq_set_next_ioprio_data()
5566 bfqq->entity.prio_changed = 1; in bfq_set_next_ioprio_data()
5577 struct bfq_queue *bfqq; in bfq_check_ioprio_change() local
5589 bfqq = bic_to_bfqq(bic, false, bfq_actuator_index(bfqd, bio)); in bfq_check_ioprio_change()
5590 if (bfqq) { in bfq_check_ioprio_change()
5591 struct bfq_queue *old_bfqq = bfqq; in bfq_check_ioprio_change()
5593 bfqq = bfq_get_queue(bfqd, bio, false, bic, true); in bfq_check_ioprio_change()
5594 bic_set_bfqq(bic, bfqq, false, bfq_actuator_index(bfqd, bio)); in bfq_check_ioprio_change()
5598 bfqq = bic_to_bfqq(bic, true, bfq_actuator_index(bfqd, bio)); in bfq_check_ioprio_change()
5599 if (bfqq) in bfq_check_ioprio_change()
5600 bfq_set_next_ioprio_data(bfqq, bic); in bfq_check_ioprio_change()
5603 static void bfq_init_bfqq(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_init_bfqq() argument
5609 bfqq->actuator_idx = act_idx; in bfq_init_bfqq()
5610 RB_CLEAR_NODE(&bfqq->entity.rb_node); in bfq_init_bfqq()
5611 INIT_LIST_HEAD(&bfqq->fifo); in bfq_init_bfqq()
5612 INIT_HLIST_NODE(&bfqq->burst_list_node); in bfq_init_bfqq()
5613 INIT_HLIST_NODE(&bfqq->woken_list_node); in bfq_init_bfqq()
5614 INIT_HLIST_HEAD(&bfqq->woken_list); in bfq_init_bfqq()
5616 bfqq->ref = 0; in bfq_init_bfqq()
5617 bfqq->bfqd = bfqd; in bfq_init_bfqq()
5620 bfq_set_next_ioprio_data(bfqq, bic); in bfq_init_bfqq()
5628 if (!bfq_class_idle(bfqq)) in bfq_init_bfqq()
5630 bfq_mark_bfqq_has_short_ttime(bfqq); in bfq_init_bfqq()
5631 bfq_mark_bfqq_sync(bfqq); in bfq_init_bfqq()
5632 bfq_mark_bfqq_just_created(bfqq); in bfq_init_bfqq()
5634 bfq_clear_bfqq_sync(bfqq); in bfq_init_bfqq()
5637 bfqq->ttime.last_end_request = now_ns + 1; in bfq_init_bfqq()
5639 bfqq->creation_time = jiffies; in bfq_init_bfqq()
5641 bfqq->io_start_time = now_ns; in bfq_init_bfqq()
5643 bfq_mark_bfqq_IO_bound(bfqq); in bfq_init_bfqq()
5645 bfqq->pid = pid; in bfq_init_bfqq()
5648 bfqq->max_budget = (2 * bfq_max_budget(bfqd)) / 3; in bfq_init_bfqq()
5649 bfqq->budget_timeout = bfq_smallest_from_now(); in bfq_init_bfqq()
5651 bfqq->wr_coeff = 1; in bfq_init_bfqq()
5652 bfqq->last_wr_start_finish = jiffies; in bfq_init_bfqq()
5653 bfqq->wr_start_at_switch_to_srt = bfq_smallest_from_now(); in bfq_init_bfqq()
5654 bfqq->split_time = bfq_smallest_from_now(); in bfq_init_bfqq()
5665 bfqq->soft_rt_next_start = jiffies; in bfq_init_bfqq()
5668 bfqq->seek_history = 1; in bfq_init_bfqq()
5670 bfqq->decrease_time_jif = jiffies; in bfq_init_bfqq()
5693 bfq_do_early_stable_merge(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_do_early_stable_merge() argument
5699 bfq_setup_merge(bfqq, last_bfqq_created); in bfq_do_early_stable_merge()
5702 return bfqq; in bfq_do_early_stable_merge()
5714 bfqq->bic = bic; in bfq_do_early_stable_merge()
5715 return bfq_merge_bfqqs(bfqd, bic, bfqq); in bfq_do_early_stable_merge()
5765 struct bfq_queue *bfqq, in bfq_do_or_sched_stable_merge() argument
5768 struct bfq_queue **source_bfqq = bfqq->entity.parent ? in bfq_do_or_sched_stable_merge()
5769 &bfqq->entity.parent->last_bfqq_created : in bfq_do_or_sched_stable_merge()
5800 bfqq->creation_time) || in bfq_do_or_sched_stable_merge()
5801 bfqq->entity.parent != last_bfqq_created->entity.parent || in bfq_do_or_sched_stable_merge()
5802 bfqq->ioprio != last_bfqq_created->ioprio || in bfq_do_or_sched_stable_merge()
5803 bfqq->ioprio_class != last_bfqq_created->ioprio_class || in bfq_do_or_sched_stable_merge()
5804 bfqq->actuator_idx != last_bfqq_created->actuator_idx) in bfq_do_or_sched_stable_merge()
5805 *source_bfqq = bfqq; in bfq_do_or_sched_stable_merge()
5808 bfqq->creation_time)) { in bfq_do_or_sched_stable_merge()
5816 bfqq = bfq_do_early_stable_merge(bfqd, bfqq, in bfq_do_or_sched_stable_merge()
5839 return bfqq; in bfq_do_or_sched_stable_merge()
5851 struct bfq_queue *bfqq; in bfq_get_queue() local
5859 bfqq = *async_bfqq; in bfq_get_queue()
5860 if (bfqq) in bfq_get_queue()
5864 bfqq = kmem_cache_alloc_node(bfq_pool, in bfq_get_queue()
5868 if (bfqq) { in bfq_get_queue()
5869 bfq_init_bfqq(bfqd, bfqq, bic, current->pid, in bfq_get_queue()
5871 bfq_init_entity(&bfqq->entity, bfqg); in bfq_get_queue()
5872 bfq_log_bfqq(bfqd, bfqq, "allocated"); in bfq_get_queue()
5874 bfqq = &bfqd->oom_bfqq; in bfq_get_queue()
5875 bfq_log_bfqq(bfqd, bfqq, "using oom bfqq"); in bfq_get_queue()
5884 bfqq->ref++; /* in bfq_get_queue()
5891 bfq_log_bfqq(bfqd, bfqq, "get_queue, bfqq not in async: %p, %d", in bfq_get_queue()
5892 bfqq, bfqq->ref); in bfq_get_queue()
5893 *async_bfqq = bfqq; in bfq_get_queue()
5897 bfqq->ref++; /* get a process reference to this queue */ in bfq_get_queue()
5899 if (bfqq != &bfqd->oom_bfqq && is_sync && !respawn) in bfq_get_queue()
5900 bfqq = bfq_do_or_sched_stable_merge(bfqd, bfqq, bic); in bfq_get_queue()
5901 return bfqq; in bfq_get_queue()
5905 struct bfq_queue *bfqq) in bfq_update_io_thinktime() argument
5907 struct bfq_ttime *ttime = &bfqq->ttime; in bfq_update_io_thinktime()
5915 if (bfqq->dispatched || bfq_bfqq_busy(bfqq)) in bfq_update_io_thinktime()
5917 elapsed = ktime_get_ns() - bfqq->ttime.last_end_request; in bfq_update_io_thinktime()
5927 bfq_update_io_seektime(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_update_io_seektime() argument
5930 bfqq->seek_history <<= 1; in bfq_update_io_seektime()
5931 bfqq->seek_history |= BFQ_RQ_SEEKY(bfqd, bfqq->last_request_pos, rq); in bfq_update_io_seektime()
5933 if (bfqq->wr_coeff > 1 && in bfq_update_io_seektime()
5934 bfqq->wr_cur_max_time == bfqd->bfq_wr_rt_max_time && in bfq_update_io_seektime()
5935 BFQQ_TOTALLY_SEEKY(bfqq)) { in bfq_update_io_seektime()
5936 if (time_is_before_jiffies(bfqq->wr_start_at_switch_to_srt + in bfq_update_io_seektime()
5944 bfq_bfqq_end_wr(bfqq); in bfq_update_io_seektime()
5951 switch_back_to_interactive_wr(bfqq, bfqd); in bfq_update_io_seektime()
5952 bfqq->entity.prio_changed = 1; in bfq_update_io_seektime()
5958 struct bfq_queue *bfqq, in bfq_update_has_short_ttime() argument
5968 if (!bfq_bfqq_sync(bfqq) || bfq_class_idle(bfqq) || in bfq_update_has_short_ttime()
5973 if (time_is_after_eq_jiffies(bfqq->split_time + in bfq_update_has_short_ttime()
5983 (bfq_sample_valid(bfqq->ttime.ttime_samples) && in bfq_update_has_short_ttime()
5984 bfqq->ttime.ttime_mean > bfqd->bfq_slice_idle>>1)) in bfq_update_has_short_ttime()
5987 state_changed = has_short_ttime != bfq_bfqq_has_short_ttime(bfqq); in bfq_update_has_short_ttime()
5990 bfq_mark_bfqq_has_short_ttime(bfqq); in bfq_update_has_short_ttime()
5992 bfq_clear_bfqq_has_short_ttime(bfqq); in bfq_update_has_short_ttime()
6078 if (state_changed && bfqq->last_serv_time_ns == 0 && in bfq_update_has_short_ttime()
6079 (time_is_before_eq_jiffies(bfqq->decrease_time_jif + in bfq_update_has_short_ttime()
6082 bfq_reset_inject_limit(bfqd, bfqq); in bfq_update_has_short_ttime()
6089 static void bfq_rq_enqueued(struct bfq_data *bfqd, struct bfq_queue *bfqq, in bfq_rq_enqueued() argument
6093 bfqq->meta_pending++; in bfq_rq_enqueued()
6095 bfqq->last_request_pos = blk_rq_pos(rq) + blk_rq_sectors(rq); in bfq_rq_enqueued()
6097 if (bfqq == bfqd->in_service_queue && bfq_bfqq_wait_request(bfqq)) { in bfq_rq_enqueued()
6098 bool small_req = bfqq->queued[rq_is_sync(rq)] == 1 && in bfq_rq_enqueued()
6100 bool budget_timeout = bfq_bfqq_budget_timeout(bfqq); in bfq_rq_enqueued()
6118 if (small_req && idling_boosts_thr_without_issues(bfqd, bfqq) && in bfq_rq_enqueued()
6129 bfq_clear_bfqq_wait_request(bfqq); in bfq_rq_enqueued()
6140 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_rq_enqueued()
6145 static void bfqq_request_allocated(struct bfq_queue *bfqq) in bfqq_request_allocated() argument
6147 struct bfq_entity *entity = &bfqq->entity; in bfqq_request_allocated()
6153 static void bfqq_request_freed(struct bfq_queue *bfqq) in bfqq_request_freed() argument
6155 struct bfq_entity *entity = &bfqq->entity; in bfqq_request_freed()
6164 struct bfq_queue *bfqq = RQ_BFQQ(rq), in __bfq_insert_request() local
6165 *new_bfqq = bfq_setup_cooperator(bfqd, bfqq, rq, true, in __bfq_insert_request()
6170 struct bfq_queue *old_bfqq = bfqq; in __bfq_insert_request()
6176 bfqq_request_freed(bfqq); in __bfq_insert_request()
6187 bfq_actuator_index(bfqd, rq->bio)) == bfqq) { in __bfq_insert_request()
6188 while (bfqq != new_bfqq) in __bfq_insert_request()
6189 bfqq = bfq_merge_bfqqs(bfqd, RQ_BIC(rq), bfqq); in __bfq_insert_request()
6201 bfq_update_io_thinktime(bfqd, bfqq); in __bfq_insert_request()
6202 bfq_update_has_short_ttime(bfqd, bfqq, RQ_BIC(rq)); in __bfq_insert_request()
6203 bfq_update_io_seektime(bfqd, bfqq, rq); in __bfq_insert_request()
6205 waiting = bfqq && bfq_bfqq_wait_request(bfqq); in __bfq_insert_request()
6207 idle_timer_disabled = waiting && !bfq_bfqq_wait_request(bfqq); in __bfq_insert_request()
6210 list_add_tail(&rq->queuelist, &bfqq->fifo); in __bfq_insert_request()
6212 bfq_rq_enqueued(bfqd, bfqq, rq); in __bfq_insert_request()
6219 struct bfq_queue *bfqq, in bfq_update_insert_stats() argument
6223 if (!bfqq) in bfq_update_insert_stats()
6237 bfqg_stats_update_io_add(bfqq_group(bfqq), bfqq, cmd_flags); in bfq_update_insert_stats()
6239 bfqg_stats_update_idle_time(bfqq_group(bfqq)); in bfq_update_insert_stats()
6244 struct bfq_queue *bfqq, in bfq_update_insert_stats() argument
6256 struct bfq_queue *bfqq; in bfq_insert_request() local
6266 bfqq = bfq_init_rq(rq); in bfq_insert_request()
6277 } else if (!bfqq) { in bfq_insert_request()
6286 bfqq = RQ_BFQQ(rq); in bfq_insert_request()
6303 bfq_update_insert_stats(q, bfqq, idle_timer_disabled, in bfq_insert_request()
6322 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_update_hw_tag() local
6344 if (bfqq && bfq_bfqq_has_short_ttime(bfqq) && in bfq_update_hw_tag()
6345 bfqq->dispatched + bfqq->queued[0] + bfqq->queued[1] < in bfq_update_hw_tag()
6361 static void bfq_completed_request(struct bfq_queue *bfqq, struct bfq_data *bfqd) in bfq_completed_request() argument
6368 bfqd->rq_in_driver[bfqq->actuator_idx]--; in bfq_completed_request()
6370 bfqq->dispatched--; in bfq_completed_request()
6372 if (!bfqq->dispatched && !bfq_bfqq_busy(bfqq)) { in bfq_completed_request()
6379 bfqq->budget_timeout = jiffies; in bfq_completed_request()
6381 bfq_del_bfqq_in_groups_with_pending_reqs(bfqq); in bfq_completed_request()
6382 bfq_weights_tree_remove(bfqq); in bfq_completed_request()
6387 bfqq->ttime.last_end_request = now_ns; in bfq_completed_request()
6425 if (!bfq_bfqq_coop(bfqq)) in bfq_completed_request()
6426 bfqd->last_completed_rq_bfqq = bfqq; in bfq_completed_request()
6441 if (bfq_bfqq_softrt_update(bfqq) && bfqq->dispatched == 0 && in bfq_completed_request()
6442 RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_completed_request()
6443 bfqq->wr_coeff != bfqd->bfq_wr_coeff) in bfq_completed_request()
6444 bfqq->soft_rt_next_start = in bfq_completed_request()
6445 bfq_bfqq_softrt_next_start(bfqd, bfqq); in bfq_completed_request()
6451 if (bfqd->in_service_queue == bfqq) { in bfq_completed_request()
6452 if (bfq_bfqq_must_idle(bfqq)) { in bfq_completed_request()
6453 if (bfqq->dispatched == 0) in bfq_completed_request()
6479 } else if (bfq_may_expire_for_budg_timeout(bfqq)) in bfq_completed_request()
6480 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_completed_request()
6482 else if (RB_EMPTY_ROOT(&bfqq->sort_list) && in bfq_completed_request()
6483 (bfqq->dispatched == 0 || in bfq_completed_request()
6484 !bfq_better_to_idle(bfqq))) in bfq_completed_request()
6485 bfq_bfqq_expire(bfqd, bfqq, false, in bfq_completed_request()
6598 struct bfq_queue *bfqq) in bfq_update_inject_limit() argument
6601 unsigned int old_limit = bfqq->inject_limit; in bfq_update_inject_limit()
6603 if (bfqq->last_serv_time_ns > 0 && bfqd->rqs_injected) { in bfq_update_inject_limit()
6604 u64 threshold = (bfqq->last_serv_time_ns * 3)>>1; in bfq_update_inject_limit()
6607 bfqq->inject_limit--; in bfq_update_inject_limit()
6608 bfqq->decrease_time_jif = jiffies; in bfq_update_inject_limit()
6611 bfqq->inject_limit++; in bfq_update_inject_limit()
6626 if ((bfqq->last_serv_time_ns == 0 && bfqd->tot_rq_in_driver == 1) || in bfq_update_inject_limit()
6627 tot_time_ns < bfqq->last_serv_time_ns) { in bfq_update_inject_limit()
6628 if (bfqq->last_serv_time_ns == 0) { in bfq_update_inject_limit()
6633 bfqq->inject_limit = max_t(unsigned int, 1, old_limit); in bfq_update_inject_limit()
6635 bfqq->last_serv_time_ns = tot_time_ns; in bfq_update_inject_limit()
6646 bfqq->last_serv_time_ns = tot_time_ns; in bfq_update_inject_limit()
6662 struct bfq_queue *bfqq = RQ_BFQQ(rq); in bfq_finish_requeue_request() local
6671 if (!rq->elv.icq || !bfqq) in bfq_finish_requeue_request()
6674 bfqd = bfqq->bfqd; in bfq_finish_requeue_request()
6677 bfqg_stats_update_completion(bfqq_group(bfqq), in bfq_finish_requeue_request()
6685 bfq_update_inject_limit(bfqd, bfqq); in bfq_finish_requeue_request()
6687 bfq_completed_request(bfqq, bfqd); in bfq_finish_requeue_request()
6689 bfqq_request_freed(bfqq); in bfq_finish_requeue_request()
6690 bfq_put_queue(bfqq); in bfq_finish_requeue_request()
6732 bfq_split_bfqq(struct bfq_io_cq *bic, struct bfq_queue *bfqq) in bfq_split_bfqq() argument
6734 bfq_log_bfqq(bfqq->bfqd, bfqq, "splitting queue"); in bfq_split_bfqq()
6736 if (bfqq_process_refs(bfqq) == 1 && !bfqq->new_bfqq) { in bfq_split_bfqq()
6737 bfqq->pid = current->pid; in bfq_split_bfqq()
6738 bfq_clear_bfqq_coop(bfqq); in bfq_split_bfqq()
6739 bfq_clear_bfqq_split_coop(bfqq); in bfq_split_bfqq()
6740 return bfqq; in bfq_split_bfqq()
6743 bic_set_bfqq(bic, NULL, true, bfqq->actuator_idx); in bfq_split_bfqq()
6745 bfq_put_cooperator(bfqq); in bfq_split_bfqq()
6747 bfq_release_process_ref(bfqq->bfqd, bfqq); in bfq_split_bfqq()
6758 struct bfq_queue *bfqq = bic_to_bfqq(bic, is_sync, act_idx); in bfq_get_bfqq_handle_split() local
6761 if (likely(bfqq && bfqq != &bfqd->oom_bfqq)) in bfq_get_bfqq_handle_split()
6762 return bfqq; in bfq_get_bfqq_handle_split()
6767 if (bfqq) in bfq_get_bfqq_handle_split()
6768 bfq_put_queue(bfqq); in bfq_get_bfqq_handle_split()
6769 bfqq = bfq_get_queue(bfqd, bio, is_sync, bic, split); in bfq_get_bfqq_handle_split()
6771 bic_set_bfqq(bic, bfqq, is_sync, act_idx); in bfq_get_bfqq_handle_split()
6775 bfq_mark_bfqq_in_large_burst(bfqq); in bfq_get_bfqq_handle_split()
6777 bfq_clear_bfqq_in_large_burst(bfqq); in bfq_get_bfqq_handle_split()
6807 hlist_add_head(&bfqq->burst_list_node, in bfq_get_bfqq_handle_split()
6810 bfqq->split_time = jiffies; in bfq_get_bfqq_handle_split()
6813 return bfqq; in bfq_get_bfqq_handle_split()
6834 static struct bfq_queue *bfq_waker_bfqq(struct bfq_queue *bfqq) in bfq_waker_bfqq() argument
6836 struct bfq_queue *new_bfqq = bfqq->new_bfqq; in bfq_waker_bfqq()
6837 struct bfq_queue *waker_bfqq = bfqq->waker_bfqq; in bfq_waker_bfqq()
6897 struct bfq_queue *bfqq; in bfq_init_rq() local
6921 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, false, is_sync, in bfq_init_rq()
6926 if (bfq_bfqq_coop(bfqq) && bfq_bfqq_split_coop(bfqq) && in bfq_init_rq()
6928 struct bfq_queue *waker_bfqq = bfq_waker_bfqq(bfqq); in bfq_init_rq()
6931 if (bfq_bfqq_in_large_burst(bfqq)) in bfq_init_rq()
6935 bfqq = bfq_split_bfqq(bic, bfqq); in bfq_init_rq()
6938 if (!bfqq) { in bfq_init_rq()
6939 bfqq = bfq_get_bfqq_handle_split(bfqd, bic, bio, in bfq_init_rq()
6942 if (unlikely(bfqq == &bfqd->oom_bfqq)) in bfq_init_rq()
6948 bfqq->waker_bfqq = waker_bfqq; in bfq_init_rq()
6949 bfqq->tentative_waker_bfqq = NULL; in bfq_init_rq()
6959 hlist_add_head(&bfqq->woken_list_node, in bfq_init_rq()
6960 &bfqq->waker_bfqq->woken_list); in bfq_init_rq()
6965 bfqq_request_allocated(bfqq); in bfq_init_rq()
6966 bfqq->ref++; in bfq_init_rq()
6968 bfq_log_bfqq(bfqd, bfqq, "get_request %p: bfqq %p, %d", in bfq_init_rq()
6969 rq, bfqq, bfqq->ref); in bfq_init_rq()
6972 rq->elv.priv[1] = bfqq; in bfq_init_rq()
6980 if (likely(bfqq != &bfqd->oom_bfqq) && !bfqq->new_bfqq && in bfq_init_rq()
6981 bfqq_process_refs(bfqq) == 1) { in bfq_init_rq()
6982 bfqq->bic = bic; in bfq_init_rq()
6989 bfq_bfqq_resume_state(bfqq, bfqd, bic, in bfq_init_rq()
7014 if (unlikely(bfq_bfqq_just_created(bfqq) && in bfq_init_rq()
7017 bfq_handle_burst(bfqd, bfqq); in bfq_init_rq()
7019 return bfqq; in bfq_init_rq()
7023 bfq_idle_slice_timer_body(struct bfq_data *bfqd, struct bfq_queue *bfqq) in bfq_idle_slice_timer_body() argument
7037 if (bfqq != bfqd->in_service_queue) { in bfq_idle_slice_timer_body()
7042 bfq_clear_bfqq_wait_request(bfqq); in bfq_idle_slice_timer_body()
7044 if (bfq_bfqq_budget_timeout(bfqq)) in bfq_idle_slice_timer_body()
7051 else if (bfqq->queued[0] == 0 && bfqq->queued[1] == 0) in bfq_idle_slice_timer_body()
7062 bfq_bfqq_expire(bfqd, bfqq, true, reason); in bfq_idle_slice_timer_body()
7077 struct bfq_queue *bfqq = bfqd->in_service_queue; in bfq_idle_slice_timer() local
7087 if (bfqq) in bfq_idle_slice_timer()
7088 bfq_idle_slice_timer_body(bfqd, bfqq); in bfq_idle_slice_timer()
7096 struct bfq_queue *bfqq = *bfqq_ptr; in __bfq_put_async_bfqq() local
7098 bfq_log(bfqd, "put_async_bfqq: %p", bfqq); in __bfq_put_async_bfqq()
7099 if (bfqq) { in __bfq_put_async_bfqq()
7100 bfq_bfqq_move(bfqd, bfqq, bfqd->root_group); in __bfq_put_async_bfqq()
7102 bfq_log_bfqq(bfqd, bfqq, "put_async_bfqq: putting %p, %d", in __bfq_put_async_bfqq()
7103 bfqq, bfqq->ref); in __bfq_put_async_bfqq()
7104 bfq_put_queue(bfqq); in __bfq_put_async_bfqq()
7187 struct bfq_queue *bfqq, *n; in bfq_exit_queue() local
7193 list_for_each_entry_safe(bfqq, n, &bfqd->idle_list, bfqq_list) in bfq_exit_queue()
7194 bfq_deactivate_bfqq(bfqd, bfqq, false, false); in bfq_exit_queue()