1556910e3SBart Van Assche // SPDX-License-Identifier: GPL-2.0
2556910e3SBart Van Assche /*
3556910e3SBart Van Assche * Block rq-qos policy for assigning an I/O priority class to requests.
4556910e3SBart Van Assche *
5556910e3SBart Van Assche * Using an rq-qos policy for assigning I/O priority class has two advantages
6556910e3SBart Van Assche * over using the ioprio_set() system call:
7556910e3SBart Van Assche *
8556910e3SBart Van Assche * - This policy is cgroup based so it has all the advantages of cgroups.
9556910e3SBart Van Assche * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos
10556910e3SBart Van Assche * controller affects page cache writeback I/O for filesystems that support
11556910e3SBart Van Assche * assiociating a cgroup with writeback I/O. See also
12556910e3SBart Van Assche * Documentation/admin-guide/cgroup-v2.rst.
13556910e3SBart Van Assche */
14556910e3SBart Van Assche
15556910e3SBart Van Assche #include <linux/blk-mq.h>
16556910e3SBart Van Assche #include <linux/blk_types.h>
17556910e3SBart Van Assche #include <linux/kernel.h>
18556910e3SBart Van Assche #include <linux/module.h>
19672fdcf0SMing Lei #include "blk-cgroup.h"
20556910e3SBart Van Assche #include "blk-ioprio.h"
21556910e3SBart Van Assche #include "blk-rq-qos.h"
22556910e3SBart Van Assche
23556910e3SBart Van Assche /**
24556910e3SBart Van Assche * enum prio_policy - I/O priority class policy.
25556910e3SBart Van Assche * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class.
26*ddf63516SHou Tao * @POLICY_PROMOTE_TO_RT: modify no-IOPRIO_CLASS_RT to IOPRIO_CLASS_RT.
27556910e3SBart Van Assche * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into
28556910e3SBart Van Assche * IOPRIO_CLASS_BE.
29556910e3SBart Van Assche * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE.
30*ddf63516SHou Tao * @POLICY_NONE_TO_RT: an alias for POLICY_PROMOTE_TO_RT.
31556910e3SBart Van Assche *
32556910e3SBart Van Assche * See also <linux/ioprio.h>.
33556910e3SBart Van Assche */
34556910e3SBart Van Assche enum prio_policy {
35556910e3SBart Van Assche POLICY_NO_CHANGE = 0,
36*ddf63516SHou Tao POLICY_PROMOTE_TO_RT = 1,
37556910e3SBart Van Assche POLICY_RESTRICT_TO_BE = 2,
38556910e3SBart Van Assche POLICY_ALL_TO_IDLE = 3,
39*ddf63516SHou Tao POLICY_NONE_TO_RT = 4,
40556910e3SBart Van Assche };
41556910e3SBart Van Assche
42556910e3SBart Van Assche static const char *policy_name[] = {
43556910e3SBart Van Assche [POLICY_NO_CHANGE] = "no-change",
44*ddf63516SHou Tao [POLICY_PROMOTE_TO_RT] = "promote-to-rt",
45556910e3SBart Van Assche [POLICY_RESTRICT_TO_BE] = "restrict-to-be",
46556910e3SBart Van Assche [POLICY_ALL_TO_IDLE] = "idle",
47*ddf63516SHou Tao [POLICY_NONE_TO_RT] = "none-to-rt",
48556910e3SBart Van Assche };
49556910e3SBart Van Assche
50556910e3SBart Van Assche static struct blkcg_policy ioprio_policy;
51556910e3SBart Van Assche
52556910e3SBart Van Assche /**
53556910e3SBart Van Assche * struct ioprio_blkg - Per (cgroup, request queue) data.
54556910e3SBart Van Assche * @pd: blkg_policy_data structure.
55556910e3SBart Van Assche */
56556910e3SBart Van Assche struct ioprio_blkg {
57556910e3SBart Van Assche struct blkg_policy_data pd;
58556910e3SBart Van Assche };
59556910e3SBart Van Assche
60556910e3SBart Van Assche /**
61556910e3SBart Van Assche * struct ioprio_blkcg - Per cgroup data.
62556910e3SBart Van Assche * @cpd: blkcg_policy_data structure.
63556910e3SBart Van Assche * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>.
64556910e3SBart Van Assche */
65556910e3SBart Van Assche struct ioprio_blkcg {
66556910e3SBart Van Assche struct blkcg_policy_data cpd;
67556910e3SBart Van Assche enum prio_policy prio_policy;
68556910e3SBart Van Assche };
69556910e3SBart Van Assche
pd_to_ioprio(struct blkg_policy_data * pd)70556910e3SBart Van Assche static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd)
71556910e3SBart Van Assche {
72556910e3SBart Van Assche return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL;
73556910e3SBart Van Assche }
74556910e3SBart Van Assche
blkcg_to_ioprio_blkcg(struct blkcg * blkcg)75556910e3SBart Van Assche static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg)
76556910e3SBart Van Assche {
77556910e3SBart Van Assche return container_of(blkcg_to_cpd(blkcg, &ioprio_policy),
78556910e3SBart Van Assche struct ioprio_blkcg, cpd);
79556910e3SBart Van Assche }
80556910e3SBart Van Assche
81556910e3SBart Van Assche static struct ioprio_blkcg *
ioprio_blkcg_from_css(struct cgroup_subsys_state * css)82556910e3SBart Van Assche ioprio_blkcg_from_css(struct cgroup_subsys_state *css)
83556910e3SBart Van Assche {
84556910e3SBart Van Assche return blkcg_to_ioprio_blkcg(css_to_blkcg(css));
85556910e3SBart Van Assche }
86556910e3SBart Van Assche
ioprio_blkcg_from_bio(struct bio * bio)87556910e3SBart Van Assche static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio)
88556910e3SBart Van Assche {
89556910e3SBart Van Assche struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy);
90556910e3SBart Van Assche
91556910e3SBart Van Assche if (!pd)
92556910e3SBart Van Assche return NULL;
93556910e3SBart Van Assche
94556910e3SBart Van Assche return blkcg_to_ioprio_blkcg(pd->blkg->blkcg);
95556910e3SBart Van Assche }
96556910e3SBart Van Assche
ioprio_show_prio_policy(struct seq_file * sf,void * v)97556910e3SBart Van Assche static int ioprio_show_prio_policy(struct seq_file *sf, void *v)
98556910e3SBart Van Assche {
99556910e3SBart Van Assche struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf));
100556910e3SBart Van Assche
101556910e3SBart Van Assche seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]);
102556910e3SBart Van Assche return 0;
103556910e3SBart Van Assche }
104556910e3SBart Van Assche
ioprio_set_prio_policy(struct kernfs_open_file * of,char * buf,size_t nbytes,loff_t off)105556910e3SBart Van Assche static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf,
106556910e3SBart Van Assche size_t nbytes, loff_t off)
107556910e3SBart Van Assche {
108556910e3SBart Van Assche struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of));
109556910e3SBart Van Assche int ret;
110556910e3SBart Van Assche
111556910e3SBart Van Assche if (off != 0)
112556910e3SBart Van Assche return -EIO;
113556910e3SBart Van Assche /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */
114556910e3SBart Van Assche ret = sysfs_match_string(policy_name, buf);
115556910e3SBart Van Assche if (ret < 0)
116556910e3SBart Van Assche return ret;
117556910e3SBart Van Assche blkcg->prio_policy = ret;
118556910e3SBart Van Assche return nbytes;
119556910e3SBart Van Assche }
120556910e3SBart Van Assche
121556910e3SBart Van Assche static struct blkg_policy_data *
ioprio_alloc_pd(struct gendisk * disk,struct blkcg * blkcg,gfp_t gfp)1220a0b4f79SChristoph Hellwig ioprio_alloc_pd(struct gendisk *disk, struct blkcg *blkcg, gfp_t gfp)
123556910e3SBart Van Assche {
124556910e3SBart Van Assche struct ioprio_blkg *ioprio_blkg;
125556910e3SBart Van Assche
126556910e3SBart Van Assche ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp);
127556910e3SBart Van Assche if (!ioprio_blkg)
128556910e3SBart Van Assche return NULL;
129556910e3SBart Van Assche
130556910e3SBart Van Assche return &ioprio_blkg->pd;
131556910e3SBart Van Assche }
132556910e3SBart Van Assche
ioprio_free_pd(struct blkg_policy_data * pd)133556910e3SBart Van Assche static void ioprio_free_pd(struct blkg_policy_data *pd)
134556910e3SBart Van Assche {
135556910e3SBart Van Assche struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd);
136556910e3SBart Van Assche
137556910e3SBart Van Assche kfree(ioprio_blkg);
138556910e3SBart Van Assche }
139556910e3SBart Van Assche
ioprio_alloc_cpd(gfp_t gfp)140556910e3SBart Van Assche static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp)
141556910e3SBart Van Assche {
142556910e3SBart Van Assche struct ioprio_blkcg *blkcg;
143556910e3SBart Van Assche
144556910e3SBart Van Assche blkcg = kzalloc(sizeof(*blkcg), gfp);
145556910e3SBart Van Assche if (!blkcg)
146556910e3SBart Van Assche return NULL;
147556910e3SBart Van Assche blkcg->prio_policy = POLICY_NO_CHANGE;
148556910e3SBart Van Assche return &blkcg->cpd;
149556910e3SBart Van Assche }
150556910e3SBart Van Assche
ioprio_free_cpd(struct blkcg_policy_data * cpd)151556910e3SBart Van Assche static void ioprio_free_cpd(struct blkcg_policy_data *cpd)
152556910e3SBart Van Assche {
153556910e3SBart Van Assche struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd);
154556910e3SBart Van Assche
155556910e3SBart Van Assche kfree(blkcg);
156556910e3SBart Van Assche }
157556910e3SBart Van Assche
158556910e3SBart Van Assche #define IOPRIO_ATTRS \
159556910e3SBart Van Assche { \
160556910e3SBart Van Assche .name = "prio.class", \
161556910e3SBart Van Assche .seq_show = ioprio_show_prio_policy, \
162556910e3SBart Van Assche .write = ioprio_set_prio_policy, \
163556910e3SBart Van Assche }, \
164556910e3SBart Van Assche { } /* sentinel */
165556910e3SBart Van Assche
166556910e3SBart Van Assche /* cgroup v2 attributes */
167556910e3SBart Van Assche static struct cftype ioprio_files[] = {
168556910e3SBart Van Assche IOPRIO_ATTRS
169556910e3SBart Van Assche };
170556910e3SBart Van Assche
171556910e3SBart Van Assche /* cgroup v1 attributes */
172556910e3SBart Van Assche static struct cftype ioprio_legacy_files[] = {
173556910e3SBart Van Assche IOPRIO_ATTRS
174556910e3SBart Van Assche };
175556910e3SBart Van Assche
176556910e3SBart Van Assche static struct blkcg_policy ioprio_policy = {
177556910e3SBart Van Assche .dfl_cftypes = ioprio_files,
178556910e3SBart Van Assche .legacy_cftypes = ioprio_legacy_files,
179556910e3SBart Van Assche
180556910e3SBart Van Assche .cpd_alloc_fn = ioprio_alloc_cpd,
181556910e3SBart Van Assche .cpd_free_fn = ioprio_free_cpd,
182556910e3SBart Van Assche
183556910e3SBart Van Assche .pd_alloc_fn = ioprio_alloc_pd,
184556910e3SBart Van Assche .pd_free_fn = ioprio_free_pd,
185556910e3SBart Van Assche };
186556910e3SBart Van Assche
blkcg_set_ioprio(struct bio * bio)18782b74cacSJan Kara void blkcg_set_ioprio(struct bio *bio)
188556910e3SBart Van Assche {
189556910e3SBart Van Assche struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio);
19025c4b5e0SJens Axboe u16 prio;
19125c4b5e0SJens Axboe
19282b74cacSJan Kara if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE)
19325c4b5e0SJens Axboe return;
194556910e3SBart Van Assche
195*ddf63516SHou Tao if (blkcg->prio_policy == POLICY_PROMOTE_TO_RT ||
196*ddf63516SHou Tao blkcg->prio_policy == POLICY_NONE_TO_RT) {
197*ddf63516SHou Tao /*
198*ddf63516SHou Tao * For RT threads, the default priority level is 4 because
199*ddf63516SHou Tao * task_nice is 0. By promoting non-RT io-priority to RT-class
200*ddf63516SHou Tao * and default level 4, those requests that are already
201*ddf63516SHou Tao * RT-class but need a higher io-priority can use ioprio_set()
202*ddf63516SHou Tao * to achieve this.
203*ddf63516SHou Tao */
204*ddf63516SHou Tao if (IOPRIO_PRIO_CLASS(bio->bi_ioprio) != IOPRIO_CLASS_RT)
205*ddf63516SHou Tao bio->bi_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_RT, 4);
206*ddf63516SHou Tao return;
207*ddf63516SHou Tao }
208*ddf63516SHou Tao
209556910e3SBart Van Assche /*
210556910e3SBart Van Assche * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers
211556910e3SBart Van Assche * correspond to a lower priority. Hence, the max_t() below selects
212556910e3SBart Van Assche * the lower priority of bi_ioprio and the cgroup I/O priority class.
213f2586544SJan Kara * If the bio I/O priority equals IOPRIO_CLASS_NONE, the cgroup I/O
214f2586544SJan Kara * priority is assigned to the bio.
215556910e3SBart Van Assche */
21625c4b5e0SJens Axboe prio = max_t(u16, bio->bi_ioprio,
217556910e3SBart Van Assche IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0));
21825c4b5e0SJens Axboe if (prio > bio->bi_ioprio)
21925c4b5e0SJens Axboe bio->bi_ioprio = prio;
220556910e3SBart Van Assche }
221556910e3SBart Van Assche
blk_ioprio_exit(struct gendisk * disk)222b0dde3f5SChristoph Hellwig void blk_ioprio_exit(struct gendisk *disk)
223556910e3SBart Van Assche {
22440e4996eSChristoph Hellwig blkcg_deactivate_policy(disk, &ioprio_policy);
225556910e3SBart Van Assche }
226556910e3SBart Van Assche
blk_ioprio_init(struct gendisk * disk)227b0dde3f5SChristoph Hellwig int blk_ioprio_init(struct gendisk *disk)
228556910e3SBart Van Assche {
22940e4996eSChristoph Hellwig return blkcg_activate_policy(disk, &ioprio_policy);
230556910e3SBart Van Assche }
231556910e3SBart Van Assche
ioprio_init(void)232556910e3SBart Van Assche static int __init ioprio_init(void)
233556910e3SBart Van Assche {
234556910e3SBart Van Assche return blkcg_policy_register(&ioprio_policy);
235556910e3SBart Van Assche }
236556910e3SBart Van Assche
ioprio_exit(void)237556910e3SBart Van Assche static void __exit ioprio_exit(void)
238556910e3SBart Van Assche {
239556910e3SBart Van Assche blkcg_policy_unregister(&ioprio_policy);
240556910e3SBart Van Assche }
241556910e3SBart Van Assche
242556910e3SBart Van Assche module_init(ioprio_init);
243556910e3SBart Van Assche module_exit(ioprio_exit);
244