Lines Matching +full:lock +full:- +full:latency +full:- +full:ns
1 // SPDX-License-Identifier: GPL-2.0
14 #include "blk-cgroup-rwstat.h"
15 #include "blk-stat.h"
16 #include "blk-throttle.h"
31 #define DFL_LATENCY_TARGET (-1L)
36 * For HD, very small latency comes from sequential IO. Such IO is helpless to
46 /* We measure latency for request size from <= 4k to >= 1M */
50 unsigned long total_latency; /* ns / 1024 */
55 unsigned long latency; /* ns / 1024 */ member
94 return pd_to_blkg(&tg->pd); in tg_to_blkg()
98 * sq_to_tg - return the throl_grp the specified service queue belongs to
101 * Return the throtl_grp @sq belongs to. If @sq is the top-level one
106 if (sq && sq->parent_sq) in sq_to_tg()
113 * sq_to_td - return throtl_data the specified service queue belongs to
124 return tg->td; in sq_to_td()
140 if (td->scale < 4096 && time_after_eq(jiffies, in throtl_adjusted_limit()
141 td->low_upgrade_time + td->scale * td->throtl_slice)) in throtl_adjusted_limit()
142 td->scale = (jiffies - td->low_upgrade_time) / td->throtl_slice; in throtl_adjusted_limit()
144 return low + (low >> 1) * td->scale; in throtl_adjusted_limit()
153 if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent) in tg_bps_limit()
156 td = tg->td; in tg_bps_limit()
157 ret = tg->bps[rw][td->limit_index]; in tg_bps_limit()
158 if (ret == 0 && td->limit_index == LIMIT_LOW) { in tg_bps_limit()
160 if (!list_empty(&blkg->blkcg->css.children) || in tg_bps_limit()
161 tg->iops[rw][td->limit_index]) in tg_bps_limit()
167 if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] && in tg_bps_limit()
168 tg->bps[rw][LIMIT_LOW] != tg->bps[rw][LIMIT_MAX]) { in tg_bps_limit()
171 adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td); in tg_bps_limit()
172 ret = min(tg->bps[rw][LIMIT_MAX], adjusted); in tg_bps_limit()
183 if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent) in tg_iops_limit()
186 td = tg->td; in tg_iops_limit()
187 ret = tg->iops[rw][td->limit_index]; in tg_iops_limit()
188 if (ret == 0 && tg->td->limit_index == LIMIT_LOW) { in tg_iops_limit()
190 if (!list_empty(&blkg->blkcg->css.children) || in tg_iops_limit()
191 tg->bps[rw][td->limit_index]) in tg_iops_limit()
197 if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_LOW] && in tg_iops_limit()
198 tg->iops[rw][LIMIT_LOW] != tg->iops[rw][LIMIT_MAX]) { in tg_iops_limit()
201 adjusted = throtl_adjusted_limit(tg->iops[rw][LIMIT_LOW], td); in tg_iops_limit()
204 ret = min_t(unsigned int, tg->iops[rw][LIMIT_MAX], adjusted); in tg_iops_limit()
210 clamp_t(int, order_base_2(sectors) - 3, 0, LATENCY_BUCKET_SIZE - 1)
213 * throtl_log - log debug message via blktrace
226 if (likely(!blk_trace_note_message_enabled(__td->queue))) \
229 blk_add_cgroup_trace_msg(__td->queue, \
230 &tg_to_blkg(__tg)->blkcg->css, "throtl " fmt, ##args);\
232 blk_add_trace_msg(__td->queue, "throtl " fmt, ##args); \
241 return bio->bi_iter.bi_size; in throtl_bio_data_size()
246 INIT_LIST_HEAD(&qn->node); in throtl_qnode_init()
247 bio_list_init(&qn->bios); in throtl_qnode_init()
248 qn->tg = tg; in throtl_qnode_init()
252 * throtl_qnode_add_bio - add a bio to a throtl_qnode and activate it
255 * @queued: the service_queue->queued[] list @qn belongs to
258 * @qn->tg's reference count is bumped when @qn is activated. See the
264 bio_list_add(&qn->bios, bio); in throtl_qnode_add_bio()
265 if (list_empty(&qn->node)) { in throtl_qnode_add_bio()
266 list_add_tail(&qn->node, queued); in throtl_qnode_add_bio()
267 blkg_get(tg_to_blkg(qn->tg)); in throtl_qnode_add_bio()
272 * throtl_peek_queued - peek the first bio on a qnode list
284 bio = bio_list_peek(&qn->bios); in throtl_peek_queued()
290 * throtl_pop_queued - pop the first bio form a qnode list
296 * that the popping order is round-robin.
313 bio = bio_list_pop(&qn->bios); in throtl_pop_queued()
316 if (bio_list_empty(&qn->bios)) { in throtl_pop_queued()
317 list_del_init(&qn->node); in throtl_pop_queued()
319 *tg_to_put = qn->tg; in throtl_pop_queued()
321 blkg_put(tg_to_blkg(qn->tg)); in throtl_pop_queued()
323 list_move_tail(&qn->node, queued); in throtl_pop_queued()
332 INIT_LIST_HEAD(&sq->queued[READ]); in throtl_service_queue_init()
333 INIT_LIST_HEAD(&sq->queued[WRITE]); in throtl_service_queue_init()
334 sq->pending_tree = RB_ROOT_CACHED; in throtl_service_queue_init()
335 timer_setup(&sq->pending_timer, throtl_pending_timer_fn, 0); in throtl_service_queue_init()
344 tg = kzalloc_node(sizeof(*tg), gfp, disk->node_id); in throtl_pd_alloc()
348 if (blkg_rwstat_init(&tg->stat_bytes, gfp)) in throtl_pd_alloc()
351 if (blkg_rwstat_init(&tg->stat_ios, gfp)) in throtl_pd_alloc()
354 throtl_service_queue_init(&tg->service_queue); in throtl_pd_alloc()
357 throtl_qnode_init(&tg->qnode_on_self[rw], tg); in throtl_pd_alloc()
358 throtl_qnode_init(&tg->qnode_on_parent[rw], tg); in throtl_pd_alloc()
361 RB_CLEAR_NODE(&tg->rb_node); in throtl_pd_alloc()
362 tg->bps[READ][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
363 tg->bps[WRITE][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
364 tg->iops[READ][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
365 tg->iops[WRITE][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
366 tg->bps_conf[READ][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
367 tg->bps_conf[WRITE][LIMIT_MAX] = U64_MAX; in throtl_pd_alloc()
368 tg->iops_conf[READ][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
369 tg->iops_conf[WRITE][LIMIT_MAX] = UINT_MAX; in throtl_pd_alloc()
372 tg->latency_target = DFL_LATENCY_TARGET; in throtl_pd_alloc()
373 tg->latency_target_conf = DFL_LATENCY_TARGET; in throtl_pd_alloc()
374 tg->idletime_threshold = DFL_IDLE_THRESHOLD; in throtl_pd_alloc()
375 tg->idletime_threshold_conf = DFL_IDLE_THRESHOLD; in throtl_pd_alloc()
377 return &tg->pd; in throtl_pd_alloc()
380 blkg_rwstat_exit(&tg->stat_bytes); in throtl_pd_alloc()
390 struct throtl_data *td = blkg->q->td; in throtl_pd_init()
391 struct throtl_service_queue *sq = &tg->service_queue; in throtl_pd_init()
407 sq->parent_sq = &td->service_queue; in throtl_pd_init()
408 if (cgroup_subsys_on_dfl(io_cgrp_subsys) && blkg->parent) in throtl_pd_init()
409 sq->parent_sq = &blkg_to_tg(blkg->parent)->service_queue; in throtl_pd_init()
410 tg->td = td; in throtl_pd_init()
420 struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq); in tg_update_has_rules()
421 struct throtl_data *td = tg->td; in tg_update_has_rules()
425 tg->has_rules_iops[rw] = in tg_update_has_rules()
426 (parent_tg && parent_tg->has_rules_iops[rw]) || in tg_update_has_rules()
427 (td->limit_valid[td->limit_index] && in tg_update_has_rules()
429 tg->has_rules_bps[rw] = in tg_update_has_rules()
430 (parent_tg && parent_tg->has_rules_bps[rw]) || in tg_update_has_rules()
431 (td->limit_valid[td->limit_index] && in tg_update_has_rules()
454 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in blk_throtl_update_limit_valid()
457 if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] || in blk_throtl_update_limit_valid()
458 tg->iops[READ][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) { in blk_throtl_update_limit_valid()
465 td->limit_valid[LIMIT_LOW] = low_valid; in blk_throtl_update_limit_valid()
478 tg->bps[READ][LIMIT_LOW] = 0; in throtl_pd_offline()
479 tg->bps[WRITE][LIMIT_LOW] = 0; in throtl_pd_offline()
480 tg->iops[READ][LIMIT_LOW] = 0; in throtl_pd_offline()
481 tg->iops[WRITE][LIMIT_LOW] = 0; in throtl_pd_offline()
483 blk_throtl_update_limit_valid(tg->td); in throtl_pd_offline()
485 if (!tg->td->limit_valid[tg->td->limit_index]) in throtl_pd_offline()
486 throtl_upgrade_state(tg->td); in throtl_pd_offline()
493 del_timer_sync(&tg->service_queue.pending_timer); in throtl_pd_free()
494 blkg_rwstat_exit(&tg->stat_bytes); in throtl_pd_free()
495 blkg_rwstat_exit(&tg->stat_ios); in throtl_pd_free()
504 n = rb_first_cached(&parent_sq->pending_tree); in throtl_rb_first()
514 rb_erase_cached(n, &parent_sq->pending_tree); in throtl_rb_erase()
526 parent_sq->first_pending_disptime = tg->disptime; in update_min_dispatch_time()
531 struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq; in tg_service_queue_add()
532 struct rb_node **node = &parent_sq->pending_tree.rb_root.rb_node; in tg_service_queue_add()
535 unsigned long key = tg->disptime; in tg_service_queue_add()
542 if (time_before(key, __tg->disptime)) in tg_service_queue_add()
543 node = &parent->rb_left; in tg_service_queue_add()
545 node = &parent->rb_right; in tg_service_queue_add()
550 rb_link_node(&tg->rb_node, parent, node); in tg_service_queue_add()
551 rb_insert_color_cached(&tg->rb_node, &parent_sq->pending_tree, in tg_service_queue_add()
557 if (!(tg->flags & THROTL_TG_PENDING)) { in throtl_enqueue_tg()
559 tg->flags |= THROTL_TG_PENDING; in throtl_enqueue_tg()
560 tg->service_queue.parent_sq->nr_pending++; in throtl_enqueue_tg()
566 if (tg->flags & THROTL_TG_PENDING) { in throtl_dequeue_tg()
568 tg->service_queue.parent_sq; in throtl_dequeue_tg()
570 throtl_rb_erase(&tg->rb_node, parent_sq); in throtl_dequeue_tg()
571 --parent_sq->nr_pending; in throtl_dequeue_tg()
572 tg->flags &= ~THROTL_TG_PENDING; in throtl_dequeue_tg()
576 /* Call with queue lock held */
580 unsigned long max_expire = jiffies + 8 * sq_to_td(sq)->throtl_slice; in throtl_schedule_pending_timer()
591 mod_timer(&sq->pending_timer, expires); in throtl_schedule_pending_timer()
593 expires - jiffies, jiffies); in throtl_schedule_pending_timer()
597 * throtl_schedule_next_dispatch - schedule the next dispatch cycle
601 * Arm @sq->pending_timer so that the next dispatch cycle starts on the
611 * delay before dispatch starts even if @sq->first_pending_disptime is not
618 if (!sq->nr_pending) in throtl_schedule_next_dispatch()
624 if (force || time_after(sq->first_pending_disptime, jiffies)) { in throtl_schedule_next_dispatch()
625 throtl_schedule_pending_timer(sq, sq->first_pending_disptime); in throtl_schedule_next_dispatch()
636 tg->bytes_disp[rw] = 0; in throtl_start_new_slice_with_credit()
637 tg->io_disp[rw] = 0; in throtl_start_new_slice_with_credit()
638 tg->carryover_bytes[rw] = 0; in throtl_start_new_slice_with_credit()
639 tg->carryover_ios[rw] = 0; in throtl_start_new_slice_with_credit()
647 if (time_after(start, tg->slice_start[rw])) in throtl_start_new_slice_with_credit()
648 tg->slice_start[rw] = start; in throtl_start_new_slice_with_credit()
650 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice_with_credit()
651 throtl_log(&tg->service_queue, in throtl_start_new_slice_with_credit()
653 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice_with_credit()
654 tg->slice_end[rw], jiffies); in throtl_start_new_slice_with_credit()
660 tg->bytes_disp[rw] = 0; in throtl_start_new_slice()
661 tg->io_disp[rw] = 0; in throtl_start_new_slice()
662 tg->slice_start[rw] = jiffies; in throtl_start_new_slice()
663 tg->slice_end[rw] = jiffies + tg->td->throtl_slice; in throtl_start_new_slice()
665 tg->carryover_bytes[rw] = 0; in throtl_start_new_slice()
666 tg->carryover_ios[rw] = 0; in throtl_start_new_slice()
669 throtl_log(&tg->service_queue, in throtl_start_new_slice()
671 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_start_new_slice()
672 tg->slice_end[rw], jiffies); in throtl_start_new_slice()
678 tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice); in throtl_set_slice_end()
685 throtl_log(&tg->service_queue, in throtl_extend_slice()
687 rw == READ ? 'R' : 'W', tg->slice_start[rw], in throtl_extend_slice()
688 tg->slice_end[rw], jiffies); in throtl_extend_slice()
694 if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw])) in throtl_slice_used()
730 if (ilog2(bps_limit) + ilog2(jiffy_elapsed) - ilog2(HZ) > 62) in calculate_bytes_allowed()
742 BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw])); in throtl_trim_slice()
745 * If bps are unlimited (-1), then time slice don't get in throtl_trim_slice()
760 throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice); in throtl_trim_slice()
762 time_elapsed = rounddown(jiffies - tg->slice_start[rw], in throtl_trim_slice()
763 tg->td->throtl_slice); in throtl_trim_slice()
769 tg->carryover_bytes[rw]; in throtl_trim_slice()
771 tg->carryover_ios[rw]; in throtl_trim_slice()
775 tg->carryover_bytes[rw] = 0; in throtl_trim_slice()
776 if ((long long)tg->bytes_disp[rw] >= bytes_trim) in throtl_trim_slice()
777 tg->bytes_disp[rw] -= bytes_trim; in throtl_trim_slice()
779 tg->bytes_disp[rw] = 0; in throtl_trim_slice()
781 tg->carryover_ios[rw] = 0; in throtl_trim_slice()
782 if ((int)tg->io_disp[rw] >= io_trim) in throtl_trim_slice()
783 tg->io_disp[rw] -= io_trim; in throtl_trim_slice()
785 tg->io_disp[rw] = 0; in throtl_trim_slice()
787 tg->slice_start[rw] += time_elapsed; in throtl_trim_slice()
789 throtl_log(&tg->service_queue, in throtl_trim_slice()
791 rw == READ ? 'R' : 'W', time_elapsed / tg->td->throtl_slice, in throtl_trim_slice()
792 bytes_trim, io_trim, tg->slice_start[rw], tg->slice_end[rw], in throtl_trim_slice()
798 unsigned long jiffy_elapsed = jiffies - tg->slice_start[rw]; in __tg_update_carryover()
809 tg->carryover_bytes[rw] += in __tg_update_carryover()
810 calculate_bytes_allowed(bps_limit, jiffy_elapsed) - in __tg_update_carryover()
811 tg->bytes_disp[rw]; in __tg_update_carryover()
813 tg->carryover_ios[rw] += in __tg_update_carryover()
814 calculate_io_allowed(iops_limit, jiffy_elapsed) - in __tg_update_carryover()
815 tg->io_disp[rw]; in __tg_update_carryover()
820 if (tg->service_queue.nr_queued[READ]) in tg_update_carryover()
822 if (tg->service_queue.nr_queued[WRITE]) in tg_update_carryover()
826 throtl_log(&tg->service_queue, "%s: %lld %lld %d %d\n", __func__, in tg_update_carryover()
827 tg->carryover_bytes[READ], tg->carryover_bytes[WRITE], in tg_update_carryover()
828 tg->carryover_ios[READ], tg->carryover_ios[WRITE]); in tg_update_carryover()
842 jiffy_elapsed = jiffies - tg->slice_start[rw]; in tg_within_iops_limit()
845 jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice); in tg_within_iops_limit()
847 tg->carryover_ios[rw]; in tg_within_iops_limit()
848 if (io_allowed > 0 && tg->io_disp[rw] + 1 <= io_allowed) in tg_within_iops_limit()
852 jiffy_wait = jiffy_elapsed_rnd - jiffy_elapsed; in tg_within_iops_limit()
870 jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw]; in tg_within_bps_limit()
874 jiffy_elapsed_rnd = tg->td->throtl_slice; in tg_within_bps_limit()
876 jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice); in tg_within_bps_limit()
878 tg->carryover_bytes[rw]; in tg_within_bps_limit()
879 if (bytes_allowed > 0 && tg->bytes_disp[rw] + bio_size <= bytes_allowed) in tg_within_bps_limit()
883 extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed; in tg_within_bps_limit()
893 jiffy_wait = jiffy_wait + (jiffy_elapsed_rnd - jiffy_elapsed); in tg_within_bps_limit()
899 * of jiffies to wait before this bio is with-in IO rate and can be dispatched
915 BUG_ON(tg->service_queue.nr_queued[rw] && in tg_may_dispatch()
916 bio != throtl_peek_queued(&tg->service_queue.queued[rw])); in tg_may_dispatch()
918 /* If tg->bps = -1, then BW is unlimited */ in tg_may_dispatch()
920 tg->flags & THROTL_TG_CANCELING) { in tg_may_dispatch()
933 if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw])) in tg_may_dispatch()
936 if (time_before(tg->slice_end[rw], in tg_may_dispatch()
937 jiffies + tg->td->throtl_slice)) in tg_may_dispatch()
939 jiffies + tg->td->throtl_slice); in tg_may_dispatch()
955 if (time_before(tg->slice_end[rw], jiffies + max_wait)) in tg_may_dispatch()
968 tg->bytes_disp[rw] += bio_size; in throtl_charge_bio()
969 tg->last_bytes_disp[rw] += bio_size; in throtl_charge_bio()
972 tg->io_disp[rw]++; in throtl_charge_bio()
973 tg->last_io_disp[rw]++; in throtl_charge_bio()
977 * throtl_add_bio_tg - add a bio to the specified throtl_grp
983 * tg->qnode_on_self[] is used.
988 struct throtl_service_queue *sq = &tg->service_queue; in throtl_add_bio_tg()
992 qn = &tg->qnode_on_self[rw]; in throtl_add_bio_tg()
1000 if (!sq->nr_queued[rw]) in throtl_add_bio_tg()
1001 tg->flags |= THROTL_TG_WAS_EMPTY; in throtl_add_bio_tg()
1003 throtl_qnode_add_bio(bio, qn, &sq->queued[rw]); in throtl_add_bio_tg()
1005 sq->nr_queued[rw]++; in throtl_add_bio_tg()
1011 struct throtl_service_queue *sq = &tg->service_queue; in tg_update_disptime()
1012 unsigned long read_wait = -1, write_wait = -1, min_wait = -1, disptime; in tg_update_disptime()
1015 bio = throtl_peek_queued(&sq->queued[READ]); in tg_update_disptime()
1019 bio = throtl_peek_queued(&sq->queued[WRITE]); in tg_update_disptime()
1027 throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq); in tg_update_disptime()
1028 tg->disptime = disptime; in tg_update_disptime()
1032 tg->flags &= ~THROTL_TG_WAS_EMPTY; in tg_update_disptime()
1040 child_tg->slice_start[rw]); in start_parent_slice_with_credit()
1047 struct throtl_service_queue *sq = &tg->service_queue; in tg_dispatch_one_bio()
1048 struct throtl_service_queue *parent_sq = sq->parent_sq; in tg_dispatch_one_bio()
1059 bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put); in tg_dispatch_one_bio()
1060 sq->nr_queued[rw]--; in tg_dispatch_one_bio()
1067 * @td->service_queue, @bio is ready to be issued. Put it on its in tg_dispatch_one_bio()
1072 throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg); in tg_dispatch_one_bio()
1076 throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw], in tg_dispatch_one_bio()
1077 &parent_sq->queued[rw]); in tg_dispatch_one_bio()
1078 BUG_ON(tg->td->nr_queued[rw] <= 0); in tg_dispatch_one_bio()
1079 tg->td->nr_queued[rw]--; in tg_dispatch_one_bio()
1090 struct throtl_service_queue *sq = &tg->service_queue; in throtl_dispatch_tg()
1093 unsigned int max_nr_writes = THROTL_GRP_QUANTUM - max_nr_reads; in throtl_dispatch_tg()
1098 while ((bio = throtl_peek_queued(&sq->queued[READ])) && in throtl_dispatch_tg()
1108 while ((bio = throtl_peek_queued(&sq->queued[WRITE])) && in throtl_dispatch_tg()
1129 if (!parent_sq->nr_pending) in throtl_select_dispatch()
1136 if (time_before(jiffies, tg->disptime)) in throtl_select_dispatch()
1141 sq = &tg->service_queue; in throtl_select_dispatch()
1142 if (sq->nr_queued[READ] || sq->nr_queued[WRITE]) in throtl_select_dispatch()
1157 * throtl_pending_timer_fn - timer function for service_queue->pending_timer
1168 * the top-level service_tree is reached, throtl_data->dispatch_work is
1183 q = tg->pd.blkg->q; in throtl_pending_timer_fn()
1185 q = td->queue; in throtl_pending_timer_fn()
1187 spin_lock_irq(&q->queue_lock); in throtl_pending_timer_fn()
1189 if (!q->root_blkg) in throtl_pending_timer_fn()
1196 parent_sq = sq->parent_sq; in throtl_pending_timer_fn()
1201 sq->nr_queued[READ] + sq->nr_queued[WRITE], in throtl_pending_timer_fn()
1202 sq->nr_queued[READ], sq->nr_queued[WRITE]); in throtl_pending_timer_fn()
1214 spin_unlock_irq(&q->queue_lock); in throtl_pending_timer_fn()
1216 spin_lock_irq(&q->queue_lock); in throtl_pending_timer_fn()
1224 if (tg->flags & THROTL_TG_WAS_EMPTY) { in throtl_pending_timer_fn()
1234 /* reached the top-level, queue issuing */ in throtl_pending_timer_fn()
1235 queue_work(kthrotld_workqueue, &td->dispatch_work); in throtl_pending_timer_fn()
1238 spin_unlock_irq(&q->queue_lock); in throtl_pending_timer_fn()
1242 * blk_throtl_dispatch_work_fn - work function for throtl_data->dispatch_work
1246 * of throtl_data->service_queue. Those bios are ready and issued by this
1253 struct throtl_service_queue *td_sq = &td->service_queue; in blk_throtl_dispatch_work_fn()
1254 struct request_queue *q = td->queue; in blk_throtl_dispatch_work_fn()
1262 spin_lock_irq(&q->queue_lock); in blk_throtl_dispatch_work_fn()
1264 while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL))) in blk_throtl_dispatch_work_fn()
1266 spin_unlock_irq(&q->queue_lock); in blk_throtl_dispatch_work_fn()
1301 &blkcg_policy_throtl, seq_cft(sf)->private, false); in tg_print_conf_u64()
1308 &blkcg_policy_throtl, seq_cft(sf)->private, false); in tg_print_conf_uint()
1314 struct throtl_service_queue *sq = &tg->service_queue; in tg_conf_updated()
1318 throtl_log(&tg->service_queue, in tg_conf_updated()
1329 * blk-throttle. in tg_conf_updated()
1332 global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) { in tg_conf_updated()
1338 if (!cgroup_subsys_on_dfl(io_cgrp_subsys) || !blkg->parent || in tg_conf_updated()
1339 !blkg->parent->parent) in tg_conf_updated()
1341 parent_tg = blkg_to_tg(blkg->parent); in tg_conf_updated()
1344 * higher latency target in tg_conf_updated()
1346 this_tg->idletime_threshold = min(this_tg->idletime_threshold, in tg_conf_updated()
1347 parent_tg->idletime_threshold); in tg_conf_updated()
1348 this_tg->latency_target = max(this_tg->latency_target, in tg_conf_updated()
1349 parent_tg->latency_target); in tg_conf_updated()
1364 if (tg->flags & THROTL_TG_PENDING) { in tg_conf_updated()
1366 throtl_schedule_next_dispatch(sq->parent_sq, true); in tg_conf_updated()
1385 ret = -EINVAL; in tg_set_conf()
1395 *(u64 *)((void *)tg + of_cft(of)->private) = v; in tg_set_conf()
1397 *(unsigned int *)((void *)tg + of_cft(of)->private) = v; in tg_set_conf()
1422 seq_cft(sf)->private, true); in tg_print_rwstat()
1440 seq_cft(sf)->private, true); in tg_print_rwstat_recursive()
1496 const char *dname = blkg_dev_name(pd->blkg); in tg_prfill_limit()
1514 if (tg->bps_conf[READ][off] == bps_dft && in tg_prfill_limit()
1515 tg->bps_conf[WRITE][off] == bps_dft && in tg_prfill_limit()
1516 tg->iops_conf[READ][off] == iops_dft && in tg_prfill_limit()
1517 tg->iops_conf[WRITE][off] == iops_dft && in tg_prfill_limit()
1519 (tg->idletime_threshold_conf == DFL_IDLE_THRESHOLD && in tg_prfill_limit()
1520 tg->latency_target_conf == DFL_LATENCY_TARGET))) in tg_prfill_limit()
1523 if (tg->bps_conf[READ][off] != U64_MAX) in tg_prfill_limit()
1525 tg->bps_conf[READ][off]); in tg_prfill_limit()
1526 if (tg->bps_conf[WRITE][off] != U64_MAX) in tg_prfill_limit()
1528 tg->bps_conf[WRITE][off]); in tg_prfill_limit()
1529 if (tg->iops_conf[READ][off] != UINT_MAX) in tg_prfill_limit()
1531 tg->iops_conf[READ][off]); in tg_prfill_limit()
1532 if (tg->iops_conf[WRITE][off] != UINT_MAX) in tg_prfill_limit()
1534 tg->iops_conf[WRITE][off]); in tg_prfill_limit()
1536 if (tg->idletime_threshold_conf == ULONG_MAX) in tg_prfill_limit()
1540 tg->idletime_threshold_conf); in tg_prfill_limit()
1542 if (tg->latency_target_conf == ULONG_MAX) in tg_prfill_limit()
1543 strcpy(latency_time, " latency=max"); in tg_prfill_limit()
1546 " latency=%lu", tg->latency_target_conf); in tg_prfill_limit()
1558 &blkcg_policy_throtl, seq_cft(sf)->private, false); in tg_print_limit()
1572 int index = of_cft(of)->private; in tg_set_limit()
1583 v[0] = tg->bps_conf[READ][index]; in tg_set_limit()
1584 v[1] = tg->bps_conf[WRITE][index]; in tg_set_limit()
1585 v[2] = tg->iops_conf[READ][index]; in tg_set_limit()
1586 v[3] = tg->iops_conf[WRITE][index]; in tg_set_limit()
1588 idle_time = tg->idletime_threshold_conf; in tg_set_limit()
1589 latency_time = tg->latency_target_conf; in tg_set_limit()
1602 ret = -EINVAL; in tg_set_limit()
1608 ret = -ERANGE; in tg_set_limit()
1612 ret = -EINVAL; in tg_set_limit()
1623 else if (off == LIMIT_LOW && !strcmp(tok, "latency")) in tg_set_limit()
1629 tg->bps_conf[READ][index] = v[0]; in tg_set_limit()
1630 tg->bps_conf[WRITE][index] = v[1]; in tg_set_limit()
1631 tg->iops_conf[READ][index] = v[2]; in tg_set_limit()
1632 tg->iops_conf[WRITE][index] = v[3]; in tg_set_limit()
1635 tg->bps[READ][index] = v[0]; in tg_set_limit()
1636 tg->bps[WRITE][index] = v[1]; in tg_set_limit()
1637 tg->iops[READ][index] = v[2]; in tg_set_limit()
1638 tg->iops[WRITE][index] = v[3]; in tg_set_limit()
1640 tg->bps[READ][LIMIT_LOW] = min(tg->bps_conf[READ][LIMIT_LOW], in tg_set_limit()
1641 tg->bps_conf[READ][LIMIT_MAX]); in tg_set_limit()
1642 tg->bps[WRITE][LIMIT_LOW] = min(tg->bps_conf[WRITE][LIMIT_LOW], in tg_set_limit()
1643 tg->bps_conf[WRITE][LIMIT_MAX]); in tg_set_limit()
1644 tg->iops[READ][LIMIT_LOW] = min(tg->iops_conf[READ][LIMIT_LOW], in tg_set_limit()
1645 tg->iops_conf[READ][LIMIT_MAX]); in tg_set_limit()
1646 tg->iops[WRITE][LIMIT_LOW] = min(tg->iops_conf[WRITE][LIMIT_LOW], in tg_set_limit()
1647 tg->iops_conf[WRITE][LIMIT_MAX]); in tg_set_limit()
1648 tg->idletime_threshold_conf = idle_time; in tg_set_limit()
1649 tg->latency_target_conf = latency_time; in tg_set_limit()
1652 if (!(tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW] || in tg_set_limit()
1653 tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) || in tg_set_limit()
1654 tg->idletime_threshold_conf == DFL_IDLE_THRESHOLD || in tg_set_limit()
1655 tg->latency_target_conf == DFL_LATENCY_TARGET) { in tg_set_limit()
1656 tg->bps[READ][LIMIT_LOW] = 0; in tg_set_limit()
1657 tg->bps[WRITE][LIMIT_LOW] = 0; in tg_set_limit()
1658 tg->iops[READ][LIMIT_LOW] = 0; in tg_set_limit()
1659 tg->iops[WRITE][LIMIT_LOW] = 0; in tg_set_limit()
1660 tg->idletime_threshold = DFL_IDLE_THRESHOLD; in tg_set_limit()
1661 tg->latency_target = DFL_LATENCY_TARGET; in tg_set_limit()
1663 tg->idletime_threshold = tg->idletime_threshold_conf; in tg_set_limit()
1664 tg->latency_target = tg->latency_target_conf; in tg_set_limit()
1667 blk_throtl_update_limit_valid(tg->td); in tg_set_limit()
1668 if (tg->td->limit_valid[LIMIT_LOW]) { in tg_set_limit()
1670 tg->td->limit_index = LIMIT_LOW; in tg_set_limit()
1672 tg->td->limit_index = LIMIT_MAX; in tg_set_limit()
1674 tg->td->limit_valid[LIMIT_LOW]); in tg_set_limit()
1703 struct throtl_data *td = q->td; in throtl_shutdown_wq()
1705 cancel_work_sync(&td->dispatch_work); in throtl_shutdown_wq()
1721 struct request_queue *q = disk->queue; in blk_throtl_cancel_bios()
1725 spin_lock_irq(&q->queue_lock); in blk_throtl_cancel_bios()
1727 * queue_lock is held, rcu lock is not needed here technically. in blk_throtl_cancel_bios()
1728 * However, rcu lock is still held to emphasize that following in blk_throtl_cancel_bios()
1732 blkg_for_each_descendant_post(blkg, pos_css, q->root_blkg) { in blk_throtl_cancel_bios()
1734 struct throtl_service_queue *sq = &tg->service_queue; in blk_throtl_cancel_bios()
1740 tg->flags |= THROTL_TG_CANCELING; in blk_throtl_cancel_bios()
1749 if (!(tg->flags & THROTL_TG_PENDING)) in blk_throtl_cancel_bios()
1761 spin_unlock_irq(&q->queue_lock); in blk_throtl_cancel_bios()
1769 if (tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW]) in __tg_last_low_overflow_time()
1770 rtime = tg->last_low_overflow_time[READ]; in __tg_last_low_overflow_time()
1771 if (tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) in __tg_last_low_overflow_time()
1772 wtime = tg->last_low_overflow_time[WRITE]; in __tg_last_low_overflow_time()
1783 parent_sq = parent->service_queue.parent_sq; in tg_last_low_overflow_time()
1792 if (!parent->bps[READ][LIMIT_LOW] && in tg_last_low_overflow_time()
1793 !parent->iops[READ][LIMIT_LOW] && in tg_last_low_overflow_time()
1794 !parent->bps[WRITE][LIMIT_LOW] && in tg_last_low_overflow_time()
1795 !parent->iops[WRITE][LIMIT_LOW]) in tg_last_low_overflow_time()
1807 * - single idle is too long, longer than a fixed value (in case user in throtl_tg_is_idle()
1809 * - average think time is more than threshold in throtl_tg_is_idle()
1810 * - IO latency is largely below threshold in throtl_tg_is_idle()
1815 time = min_t(unsigned long, MAX_IDLE_TIME, 4 * tg->idletime_threshold); in throtl_tg_is_idle()
1816 ret = tg->latency_target == DFL_LATENCY_TARGET || in throtl_tg_is_idle()
1817 tg->idletime_threshold == DFL_IDLE_THRESHOLD || in throtl_tg_is_idle()
1818 (ktime_get_ns() >> 10) - tg->last_finish_time > time || in throtl_tg_is_idle()
1819 tg->avg_idletime > tg->idletime_threshold || in throtl_tg_is_idle()
1820 (tg->latency_target && tg->bio_cnt && in throtl_tg_is_idle()
1821 tg->bad_bio_cnt * 5 < tg->bio_cnt); in throtl_tg_is_idle()
1822 throtl_log(&tg->service_queue, in throtl_tg_is_idle()
1824 tg->avg_idletime, tg->idletime_threshold, tg->bad_bio_cnt, in throtl_tg_is_idle()
1825 tg->bio_cnt, ret, tg->td->scale); in throtl_tg_is_idle()
1831 struct throtl_service_queue *sq = &tg->service_queue; in throtl_low_limit_reached()
1832 bool limit = tg->bps[rw][LIMIT_LOW] || tg->iops[rw][LIMIT_LOW]; in throtl_low_limit_reached()
1836 * if low limit is non-zero, we can check if there is any request in throtl_low_limit_reached()
1840 return !limit || sq->nr_queued[rw]; in throtl_low_limit_reached()
1855 tg_last_low_overflow_time(tg) + tg->td->throtl_slice) && in throtl_tg_can_upgrade()
1866 tg = sq_to_tg(tg->service_queue.parent_sq); in throtl_hierarchy_can_upgrade()
1867 if (!tg || !tg_to_blkg(tg)->parent) in throtl_hierarchy_can_upgrade()
1879 if (td->limit_index != LIMIT_LOW) in throtl_can_upgrade()
1882 if (time_before(jiffies, td->low_downgrade_time + td->throtl_slice)) in throtl_can_upgrade()
1886 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in throtl_can_upgrade()
1891 if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children)) in throtl_can_upgrade()
1906 if (tg->td->limit_index != LIMIT_LOW) in throtl_upgrade_check()
1909 if (time_after(tg->last_check_time + tg->td->throtl_slice, now)) in throtl_upgrade_check()
1912 tg->last_check_time = now; in throtl_upgrade_check()
1915 __tg_last_low_overflow_time(tg) + tg->td->throtl_slice)) in throtl_upgrade_check()
1918 if (throtl_can_upgrade(tg->td, NULL)) in throtl_upgrade_check()
1919 throtl_upgrade_state(tg->td); in throtl_upgrade_check()
1927 throtl_log(&td->service_queue, "upgrade to max"); in throtl_upgrade_state()
1928 td->limit_index = LIMIT_MAX; in throtl_upgrade_state()
1929 td->low_upgrade_time = jiffies; in throtl_upgrade_state()
1930 td->scale = 0; in throtl_upgrade_state()
1932 blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) { in throtl_upgrade_state()
1934 struct throtl_service_queue *sq = &tg->service_queue; in throtl_upgrade_state()
1936 tg->disptime = jiffies - 1; in throtl_upgrade_state()
1941 throtl_select_dispatch(&td->service_queue); in throtl_upgrade_state()
1942 throtl_schedule_next_dispatch(&td->service_queue, true); in throtl_upgrade_state()
1943 queue_work(kthrotld_workqueue, &td->dispatch_work); in throtl_upgrade_state()
1948 td->scale /= 2; in throtl_downgrade_state()
1950 throtl_log(&td->service_queue, "downgrade, scale %d", td->scale); in throtl_downgrade_state()
1951 if (td->scale) { in throtl_downgrade_state()
1952 td->low_upgrade_time = jiffies - td->scale * td->throtl_slice; in throtl_downgrade_state()
1956 td->limit_index = LIMIT_LOW; in throtl_downgrade_state()
1957 td->low_downgrade_time = jiffies; in throtl_downgrade_state()
1962 struct throtl_data *td = tg->td; in throtl_tg_can_downgrade()
1970 td->throtl_slice) && in throtl_tg_can_downgrade()
1972 !list_empty(&tg_to_blkg(tg)->blkcg->css.children))) in throtl_tg_can_downgrade()
1979 struct throtl_data *td = tg->td; in throtl_hierarchy_can_downgrade()
1981 if (time_before(jiffies, td->low_upgrade_time + td->throtl_slice)) in throtl_hierarchy_can_downgrade()
1987 tg = sq_to_tg(tg->service_queue.parent_sq); in throtl_hierarchy_can_downgrade()
1988 if (!tg || !tg_to_blkg(tg)->parent) in throtl_hierarchy_can_downgrade()
2001 if (tg->td->limit_index != LIMIT_MAX || in throtl_downgrade_check()
2002 !tg->td->limit_valid[LIMIT_LOW]) in throtl_downgrade_check()
2004 if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children)) in throtl_downgrade_check()
2006 if (time_after(tg->last_check_time + tg->td->throtl_slice, now)) in throtl_downgrade_check()
2009 elapsed_time = now - tg->last_check_time; in throtl_downgrade_check()
2010 tg->last_check_time = now; in throtl_downgrade_check()
2013 tg->td->throtl_slice)) in throtl_downgrade_check()
2016 if (tg->bps[READ][LIMIT_LOW]) { in throtl_downgrade_check()
2017 bps = tg->last_bytes_disp[READ] * HZ; in throtl_downgrade_check()
2019 if (bps >= tg->bps[READ][LIMIT_LOW]) in throtl_downgrade_check()
2020 tg->last_low_overflow_time[READ] = now; in throtl_downgrade_check()
2023 if (tg->bps[WRITE][LIMIT_LOW]) { in throtl_downgrade_check()
2024 bps = tg->last_bytes_disp[WRITE] * HZ; in throtl_downgrade_check()
2026 if (bps >= tg->bps[WRITE][LIMIT_LOW]) in throtl_downgrade_check()
2027 tg->last_low_overflow_time[WRITE] = now; in throtl_downgrade_check()
2030 if (tg->iops[READ][LIMIT_LOW]) { in throtl_downgrade_check()
2031 iops = tg->last_io_disp[READ] * HZ / elapsed_time; in throtl_downgrade_check()
2032 if (iops >= tg->iops[READ][LIMIT_LOW]) in throtl_downgrade_check()
2033 tg->last_low_overflow_time[READ] = now; in throtl_downgrade_check()
2036 if (tg->iops[WRITE][LIMIT_LOW]) { in throtl_downgrade_check()
2037 iops = tg->last_io_disp[WRITE] * HZ / elapsed_time; in throtl_downgrade_check()
2038 if (iops >= tg->iops[WRITE][LIMIT_LOW]) in throtl_downgrade_check()
2039 tg->last_low_overflow_time[WRITE] = now; in throtl_downgrade_check()
2047 throtl_downgrade_state(tg->td); in throtl_downgrade_check()
2049 tg->last_bytes_disp[READ] = 0; in throtl_downgrade_check()
2050 tg->last_bytes_disp[WRITE] = 0; in throtl_downgrade_check()
2051 tg->last_io_disp[READ] = 0; in throtl_downgrade_check()
2052 tg->last_io_disp[WRITE] = 0; in throtl_downgrade_check()
2058 unsigned long last_finish_time = tg->last_finish_time; in blk_throtl_update_idletime()
2065 last_finish_time == tg->checked_last_finish_time) in blk_throtl_update_idletime()
2068 tg->avg_idletime = (tg->avg_idletime * 7 + now - last_finish_time) >> 3; in blk_throtl_update_idletime()
2069 tg->checked_last_finish_time = last_finish_time; in blk_throtl_update_idletime()
2077 unsigned long latency[2]; in throtl_update_latency_buckets() local
2079 if (!blk_queue_nonrot(td->queue) || !td->limit_valid[LIMIT_LOW]) in throtl_update_latency_buckets()
2081 if (time_before(jiffies, td->last_calculate_time + HZ)) in throtl_update_latency_buckets()
2083 td->last_calculate_time = jiffies; in throtl_update_latency_buckets()
2088 struct latency_bucket *tmp = &td->tmp_buckets[rw][i]; in throtl_update_latency_buckets()
2094 bucket = per_cpu_ptr(td->latency_buckets[rw], in throtl_update_latency_buckets()
2096 tmp->total_latency += bucket[i].total_latency; in throtl_update_latency_buckets()
2097 tmp->samples += bucket[i].samples; in throtl_update_latency_buckets()
2102 if (tmp->samples >= 32) { in throtl_update_latency_buckets()
2103 int samples = tmp->samples; in throtl_update_latency_buckets()
2105 latency[rw] = tmp->total_latency; in throtl_update_latency_buckets()
2107 tmp->total_latency = 0; in throtl_update_latency_buckets()
2108 tmp->samples = 0; in throtl_update_latency_buckets()
2109 latency[rw] /= samples; in throtl_update_latency_buckets()
2110 if (latency[rw] == 0) in throtl_update_latency_buckets()
2112 avg_latency[rw][i].latency = latency[rw]; in throtl_update_latency_buckets()
2119 if (!avg_latency[rw][i].latency) { in throtl_update_latency_buckets()
2120 if (td->avg_buckets[rw][i].latency < last_latency[rw]) in throtl_update_latency_buckets()
2121 td->avg_buckets[rw][i].latency = in throtl_update_latency_buckets()
2126 if (!td->avg_buckets[rw][i].valid) in throtl_update_latency_buckets()
2127 latency[rw] = avg_latency[rw][i].latency; in throtl_update_latency_buckets()
2129 latency[rw] = (td->avg_buckets[rw][i].latency * 7 + in throtl_update_latency_buckets()
2130 avg_latency[rw][i].latency) >> 3; in throtl_update_latency_buckets()
2132 td->avg_buckets[rw][i].latency = max(latency[rw], in throtl_update_latency_buckets()
2134 td->avg_buckets[rw][i].valid = true; in throtl_update_latency_buckets()
2135 last_latency[rw] = td->avg_buckets[rw][i].latency; in throtl_update_latency_buckets()
2140 throtl_log(&td->service_queue, in throtl_update_latency_buckets()
2141 "Latency bucket %d: read latency=%ld, read valid=%d, " in throtl_update_latency_buckets()
2142 "write latency=%ld, write valid=%d", i, in throtl_update_latency_buckets()
2143 td->avg_buckets[READ][i].latency, in throtl_update_latency_buckets()
2144 td->avg_buckets[READ][i].valid, in throtl_update_latency_buckets()
2145 td->avg_buckets[WRITE][i].latency, in throtl_update_latency_buckets()
2146 td->avg_buckets[WRITE][i].valid); in throtl_update_latency_buckets()
2178 struct request_queue *q = bdev_get_queue(bio->bi_bdev); in __blk_throtl_bio()
2179 struct blkcg_gq *blkg = bio->bi_blkg; in __blk_throtl_bio()
2185 struct throtl_data *td = tg->td; in __blk_throtl_bio()
2189 spin_lock_irq(&q->queue_lock); in __blk_throtl_bio()
2195 sq = &tg->service_queue; in __blk_throtl_bio()
2199 if (tg->last_low_overflow_time[rw] == 0) in __blk_throtl_bio()
2200 tg->last_low_overflow_time[rw] = jiffies; in __blk_throtl_bio()
2203 /* throtl is FIFO - if bios are already queued, should queue */ in __blk_throtl_bio()
2204 if (sq->nr_queued[rw]) in __blk_throtl_bio()
2209 tg->last_low_overflow_time[rw] = jiffies; in __blk_throtl_bio()
2238 qn = &tg->qnode_on_parent[rw]; in __blk_throtl_bio()
2239 sq = sq->parent_sq; in __blk_throtl_bio()
2247 /* out-of-limit, queue to @tg */ in __blk_throtl_bio()
2250 tg->bytes_disp[rw], bio->bi_iter.bi_size, in __blk_throtl_bio()
2252 tg->io_disp[rw], tg_iops_limit(tg, rw), in __blk_throtl_bio()
2253 sq->nr_queued[READ], sq->nr_queued[WRITE]); in __blk_throtl_bio()
2255 tg->last_low_overflow_time[rw] = jiffies; in __blk_throtl_bio()
2257 td->nr_queued[rw]++; in __blk_throtl_bio()
2267 if (tg->flags & THROTL_TG_WAS_EMPTY) { in __blk_throtl_bio()
2269 throtl_schedule_next_dispatch(tg->service_queue.parent_sq, true); in __blk_throtl_bio()
2274 if (throttled || !td->track_bio_latency) in __blk_throtl_bio()
2275 bio->bi_issue.value |= BIO_ISSUE_THROTL_SKIP_LATENCY; in __blk_throtl_bio()
2277 spin_unlock_irq(&q->queue_lock); in __blk_throtl_bio()
2288 struct latency_bucket *latency; in throtl_track_latency() local
2291 if (!td || td->limit_index != LIMIT_LOW || in throtl_track_latency()
2293 !blk_queue_nonrot(td->queue)) in throtl_track_latency()
2298 latency = get_cpu_ptr(td->latency_buckets[rw]); in throtl_track_latency()
2299 latency[index].total_latency += time; in throtl_track_latency()
2300 latency[index].samples++; in throtl_track_latency()
2301 put_cpu_ptr(td->latency_buckets[rw]); in throtl_track_latency()
2306 struct request_queue *q = rq->q; in blk_throtl_stat_add()
2307 struct throtl_data *td = q->td; in blk_throtl_stat_add()
2323 blkg = bio->bi_blkg; in blk_throtl_bio_endio()
2327 if (!tg->td->limit_valid[LIMIT_LOW]) in blk_throtl_bio_endio()
2331 tg->last_finish_time = finish_time_ns >> 10; in blk_throtl_bio_endio()
2333 start_time = bio_issue_time(&bio->bi_issue) >> 10; in blk_throtl_bio_endio()
2338 lat = finish_time - start_time; in blk_throtl_bio_endio()
2340 if (!(bio->bi_issue.value & BIO_ISSUE_THROTL_SKIP_LATENCY)) in blk_throtl_bio_endio()
2341 throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue), in blk_throtl_bio_endio()
2344 if (tg->latency_target && lat >= tg->td->filtered_latency) { in blk_throtl_bio_endio()
2348 bucket = request_bucket_index(bio_issue_size(&bio->bi_issue)); in blk_throtl_bio_endio()
2349 threshold = tg->td->avg_buckets[rw][bucket].latency + in blk_throtl_bio_endio()
2350 tg->latency_target; in blk_throtl_bio_endio()
2352 tg->bad_bio_cnt++; in blk_throtl_bio_endio()
2357 tg->bio_cnt++; in blk_throtl_bio_endio()
2360 if (time_after(jiffies, tg->bio_cnt_reset_time) || tg->bio_cnt > 1024) { in blk_throtl_bio_endio()
2361 tg->bio_cnt_reset_time = tg->td->throtl_slice + jiffies; in blk_throtl_bio_endio()
2362 tg->bio_cnt /= 2; in blk_throtl_bio_endio()
2363 tg->bad_bio_cnt /= 2; in blk_throtl_bio_endio()
2370 struct request_queue *q = disk->queue; in blk_throtl_init()
2374 td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node); in blk_throtl_init()
2376 return -ENOMEM; in blk_throtl_init()
2377 td->latency_buckets[READ] = __alloc_percpu(sizeof(struct latency_bucket) * in blk_throtl_init()
2379 if (!td->latency_buckets[READ]) { in blk_throtl_init()
2381 return -ENOMEM; in blk_throtl_init()
2383 td->latency_buckets[WRITE] = __alloc_percpu(sizeof(struct latency_bucket) * in blk_throtl_init()
2385 if (!td->latency_buckets[WRITE]) { in blk_throtl_init()
2386 free_percpu(td->latency_buckets[READ]); in blk_throtl_init()
2388 return -ENOMEM; in blk_throtl_init()
2391 INIT_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn); in blk_throtl_init()
2392 throtl_service_queue_init(&td->service_queue); in blk_throtl_init()
2394 q->td = td; in blk_throtl_init()
2395 td->queue = q; in blk_throtl_init()
2397 td->limit_valid[LIMIT_MAX] = true; in blk_throtl_init()
2398 td->limit_index = LIMIT_MAX; in blk_throtl_init()
2399 td->low_upgrade_time = jiffies; in blk_throtl_init()
2400 td->low_downgrade_time = jiffies; in blk_throtl_init()
2405 free_percpu(td->latency_buckets[READ]); in blk_throtl_init()
2406 free_percpu(td->latency_buckets[WRITE]); in blk_throtl_init()
2414 struct request_queue *q = disk->queue; in blk_throtl_exit()
2416 BUG_ON(!q->td); in blk_throtl_exit()
2417 del_timer_sync(&q->td->service_queue.pending_timer); in blk_throtl_exit()
2420 free_percpu(q->td->latency_buckets[READ]); in blk_throtl_exit()
2421 free_percpu(q->td->latency_buckets[WRITE]); in blk_throtl_exit()
2422 kfree(q->td); in blk_throtl_exit()
2427 struct request_queue *q = disk->queue; in blk_throtl_register()
2431 td = q->td; in blk_throtl_register()
2435 td->throtl_slice = DFL_THROTL_SLICE_SSD; in blk_throtl_register()
2436 td->filtered_latency = LATENCY_FILTERED_SSD; in blk_throtl_register()
2438 td->throtl_slice = DFL_THROTL_SLICE_HD; in blk_throtl_register()
2439 td->filtered_latency = LATENCY_FILTERED_HD; in blk_throtl_register()
2441 td->avg_buckets[READ][i].latency = DFL_HD_BASELINE_LATENCY; in blk_throtl_register()
2442 td->avg_buckets[WRITE][i].latency = DFL_HD_BASELINE_LATENCY; in blk_throtl_register()
2447 td->throtl_slice = DFL_THROTL_SLICE_HD; in blk_throtl_register()
2450 td->track_bio_latency = !queue_is_mq(q); in blk_throtl_register()
2451 if (!td->track_bio_latency) in blk_throtl_register()
2459 if (!q->td) in blk_throtl_sample_time_show()
2460 return -EINVAL; in blk_throtl_sample_time_show()
2461 return sprintf(page, "%u\n", jiffies_to_msecs(q->td->throtl_slice)); in blk_throtl_sample_time_show()
2470 if (!q->td) in blk_throtl_sample_time_store()
2471 return -EINVAL; in blk_throtl_sample_time_store()
2473 return -EINVAL; in blk_throtl_sample_time_store()
2476 return -EINVAL; in blk_throtl_sample_time_store()
2477 q->td->throtl_slice = t; in blk_throtl_sample_time_store()