blk-wbt.c (8a2b20a997a3779ae9fcae268f2959eb82ec05a1) blk-wbt.c (a13bd91be22318768d55470cbc0b0f4488ef9edf)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * buffered writeback throttling. loosely based on CoDel. We can't drop
4 * packets for IO scheduling, so the logic is something like this:
5 *
6 * - Monitor latencies in a defined window of time.
7 * - If the minimum latency in the above window exceeds some target, increment
8 * scaling step and scale down queue depth by a factor of 2x. The monitoring

--- 716 unchanged lines hidden (view full) ---

725
726/*
727 * Enable wbt if defaults are configured that way
728 */
729void wbt_enable_default(struct gendisk *disk)
730{
731 struct request_queue *q = disk->queue;
732 struct rq_qos *rqos;
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * buffered writeback throttling. loosely based on CoDel. We can't drop
4 * packets for IO scheduling, so the logic is something like this:
5 *
6 * - Monitor latencies in a defined window of time.
7 * - If the minimum latency in the above window exceeds some target, increment
8 * scaling step and scale down queue depth by a factor of 2x. The monitoring

--- 716 unchanged lines hidden (view full) ---

725
726/*
727 * Enable wbt if defaults are configured that way
728 */
729void wbt_enable_default(struct gendisk *disk)
730{
731 struct request_queue *q = disk->queue;
732 struct rq_qos *rqos;
733 bool enable = IS_ENABLED(CONFIG_BLK_WBT_MQ);
733 bool disable_flag = q->elevator &&
734 test_bit(ELEVATOR_FLAG_DISABLE_WBT, &q->elevator->flags);
734
735
735 if (q->elevator &&
736 test_bit(ELEVATOR_FLAG_DISABLE_WBT, &q->elevator->flags))
737 enable = false;
738
739 /* Throttling already enabled? */
740 rqos = wbt_rq_qos(q);
741 if (rqos) {
736 /* Throttling already enabled? */
737 rqos = wbt_rq_qos(q);
738 if (rqos) {
742 if (enable && RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT)
739 if (!disable_flag &&
740 RQWB(rqos)->enable_state == WBT_STATE_OFF_DEFAULT)
743 RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT;
744 return;
745 }
746
747 /* Queue not registered? Maybe shutting down... */
748 if (!blk_queue_registered(q))
749 return;
750
741 RQWB(rqos)->enable_state = WBT_STATE_ON_DEFAULT;
742 return;
743 }
744
745 /* Queue not registered? Maybe shutting down... */
746 if (!blk_queue_registered(q))
747 return;
748
751 if (queue_is_mq(q) && enable)
749 if (queue_is_mq(q) && !disable_flag)
752 wbt_init(disk);
753}
754EXPORT_SYMBOL_GPL(wbt_enable_default);
755
756u64 wbt_default_latency_nsec(struct request_queue *q)
757{
758 /*
759 * We default to 2msec for non-rotational storage, and 75msec

--- 179 unchanged lines hidden (view full) ---

939 rwb->rq_depth.default_depth = RWB_DEF_DEPTH;
940 rwb->min_lat_nsec = wbt_default_latency_nsec(q);
941 rwb->rq_depth.queue_depth = blk_queue_depth(q);
942 wbt_update_limits(rwb);
943
944 /*
945 * Assign rwb and add the stats callback.
946 */
750 wbt_init(disk);
751}
752EXPORT_SYMBOL_GPL(wbt_enable_default);
753
754u64 wbt_default_latency_nsec(struct request_queue *q)
755{
756 /*
757 * We default to 2msec for non-rotational storage, and 75msec

--- 179 unchanged lines hidden (view full) ---

937 rwb->rq_depth.default_depth = RWB_DEF_DEPTH;
938 rwb->min_lat_nsec = wbt_default_latency_nsec(q);
939 rwb->rq_depth.queue_depth = blk_queue_depth(q);
940 wbt_update_limits(rwb);
941
942 /*
943 * Assign rwb and add the stats callback.
944 */
945 mutex_lock(&q->rq_qos_mutex);
947 ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops);
946 ret = rq_qos_add(&rwb->rqos, disk, RQ_QOS_WBT, &wbt_rqos_ops);
947 mutex_unlock(&q->rq_qos_mutex);
948 if (ret)
949 goto err_free;
950
951 blk_stat_add_callback(q, rwb->cb);
952
953 return 0;
954
955err_free:
956 blk_stat_free_callback(rwb->cb);
957 kfree(rwb);
958 return ret;
959
960}
948 if (ret)
949 goto err_free;
950
951 blk_stat_add_callback(q, rwb->cb);
952
953 return 0;
954
955err_free:
956 blk_stat_free_callback(rwb->cb);
957 kfree(rwb);
958 return ret;
959
960}