xref: /openbmc/linux/block/blk-ioprio.c (revision 10756dc5)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Block rq-qos policy for assigning an I/O priority class to requests.
4  *
5  * Using an rq-qos policy for assigning I/O priority class has two advantages
6  * over using the ioprio_set() system call:
7  *
8  * - This policy is cgroup based so it has all the advantages of cgroups.
9  * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos
10  *   controller affects page cache writeback I/O for filesystems that support
11  *   assiociating a cgroup with writeback I/O. See also
12  *   Documentation/admin-guide/cgroup-v2.rst.
13  */
14 
15 #include <linux/blk-cgroup.h>
16 #include <linux/blk-mq.h>
17 #include <linux/blk_types.h>
18 #include <linux/kernel.h>
19 #include <linux/module.h>
20 #include "blk-ioprio.h"
21 #include "blk-rq-qos.h"
22 
23 /**
24  * enum prio_policy - I/O priority class policy.
25  * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class.
26  * @POLICY_NONE_TO_RT: modify IOPRIO_CLASS_NONE into IOPRIO_CLASS_RT.
27  * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into
28  *		IOPRIO_CLASS_BE.
29  * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE.
30  *
31  * See also <linux/ioprio.h>.
32  */
33 enum prio_policy {
34 	POLICY_NO_CHANGE	= 0,
35 	POLICY_NONE_TO_RT	= 1,
36 	POLICY_RESTRICT_TO_BE	= 2,
37 	POLICY_ALL_TO_IDLE	= 3,
38 };
39 
40 static const char *policy_name[] = {
41 	[POLICY_NO_CHANGE]	= "no-change",
42 	[POLICY_NONE_TO_RT]	= "none-to-rt",
43 	[POLICY_RESTRICT_TO_BE]	= "restrict-to-be",
44 	[POLICY_ALL_TO_IDLE]	= "idle",
45 };
46 
47 static struct blkcg_policy ioprio_policy;
48 
49 /**
50  * struct ioprio_blkg - Per (cgroup, request queue) data.
51  * @pd: blkg_policy_data structure.
52  */
53 struct ioprio_blkg {
54 	struct blkg_policy_data pd;
55 };
56 
57 /**
58  * struct ioprio_blkcg - Per cgroup data.
59  * @cpd: blkcg_policy_data structure.
60  * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>.
61  */
62 struct ioprio_blkcg {
63 	struct blkcg_policy_data cpd;
64 	enum prio_policy	 prio_policy;
65 	bool			 prio_set;
66 };
67 
68 static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd)
69 {
70 	return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL;
71 }
72 
73 static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg)
74 {
75 	return container_of(blkcg_to_cpd(blkcg, &ioprio_policy),
76 			    struct ioprio_blkcg, cpd);
77 }
78 
79 static struct ioprio_blkcg *
80 ioprio_blkcg_from_css(struct cgroup_subsys_state *css)
81 {
82 	return blkcg_to_ioprio_blkcg(css_to_blkcg(css));
83 }
84 
85 static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio)
86 {
87 	struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy);
88 
89 	if (!pd)
90 		return NULL;
91 
92 	return blkcg_to_ioprio_blkcg(pd->blkg->blkcg);
93 }
94 
95 static int ioprio_show_prio_policy(struct seq_file *sf, void *v)
96 {
97 	struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf));
98 
99 	seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]);
100 	return 0;
101 }
102 
103 static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf,
104 				      size_t nbytes, loff_t off)
105 {
106 	struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of));
107 	int ret;
108 
109 	if (off != 0)
110 		return -EIO;
111 	/* kernfs_fop_write_iter() terminates 'buf' with '\0'. */
112 	ret = sysfs_match_string(policy_name, buf);
113 	if (ret < 0)
114 		return ret;
115 	blkcg->prio_policy = ret;
116 	blkcg->prio_set = true;
117 	return nbytes;
118 }
119 
120 static struct blkg_policy_data *
121 ioprio_alloc_pd(gfp_t gfp, struct request_queue *q, struct blkcg *blkcg)
122 {
123 	struct ioprio_blkg *ioprio_blkg;
124 
125 	ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp);
126 	if (!ioprio_blkg)
127 		return NULL;
128 
129 	return &ioprio_blkg->pd;
130 }
131 
132 static void ioprio_free_pd(struct blkg_policy_data *pd)
133 {
134 	struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd);
135 
136 	kfree(ioprio_blkg);
137 }
138 
139 static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp)
140 {
141 	struct ioprio_blkcg *blkcg;
142 
143 	blkcg = kzalloc(sizeof(*blkcg), gfp);
144 	if (!blkcg)
145 		return NULL;
146 	blkcg->prio_policy = POLICY_NO_CHANGE;
147 	return &blkcg->cpd;
148 }
149 
150 static void ioprio_free_cpd(struct blkcg_policy_data *cpd)
151 {
152 	struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd);
153 
154 	kfree(blkcg);
155 }
156 
157 #define IOPRIO_ATTRS						\
158 	{							\
159 		.name		= "prio.class",			\
160 		.seq_show	= ioprio_show_prio_policy,	\
161 		.write		= ioprio_set_prio_policy,	\
162 	},							\
163 	{ } /* sentinel */
164 
165 /* cgroup v2 attributes */
166 static struct cftype ioprio_files[] = {
167 	IOPRIO_ATTRS
168 };
169 
170 /* cgroup v1 attributes */
171 static struct cftype ioprio_legacy_files[] = {
172 	IOPRIO_ATTRS
173 };
174 
175 static struct blkcg_policy ioprio_policy = {
176 	.dfl_cftypes	= ioprio_files,
177 	.legacy_cftypes = ioprio_legacy_files,
178 
179 	.cpd_alloc_fn	= ioprio_alloc_cpd,
180 	.cpd_free_fn	= ioprio_free_cpd,
181 
182 	.pd_alloc_fn	= ioprio_alloc_pd,
183 	.pd_free_fn	= ioprio_free_pd,
184 };
185 
186 struct blk_ioprio {
187 	struct rq_qos rqos;
188 };
189 
190 static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq,
191 			       struct bio *bio)
192 {
193 	struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio);
194 	u16 prio;
195 
196 	if (!blkcg->prio_set)
197 		return;
198 
199 	/*
200 	 * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers
201 	 * correspond to a lower priority. Hence, the max_t() below selects
202 	 * the lower priority of bi_ioprio and the cgroup I/O priority class.
203 	 * If the cgroup policy has been set to POLICY_NO_CHANGE == 0, the
204 	 * bio I/O priority is not modified. If the bio I/O priority equals
205 	 * IOPRIO_CLASS_NONE, the cgroup I/O priority is assigned to the bio.
206 	 */
207 	prio = max_t(u16, bio->bi_ioprio,
208 			IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0));
209 	if (prio > bio->bi_ioprio)
210 		bio->bi_ioprio = prio;
211 }
212 
213 static void blkcg_ioprio_exit(struct rq_qos *rqos)
214 {
215 	struct blk_ioprio *blkioprio_blkg =
216 		container_of(rqos, typeof(*blkioprio_blkg), rqos);
217 
218 	blkcg_deactivate_policy(rqos->q, &ioprio_policy);
219 	kfree(blkioprio_blkg);
220 }
221 
222 static struct rq_qos_ops blkcg_ioprio_ops = {
223 	.track	= blkcg_ioprio_track,
224 	.exit	= blkcg_ioprio_exit,
225 };
226 
227 int blk_ioprio_init(struct request_queue *q)
228 {
229 	struct blk_ioprio *blkioprio_blkg;
230 	struct rq_qos *rqos;
231 	int ret;
232 
233 	blkioprio_blkg = kzalloc(sizeof(*blkioprio_blkg), GFP_KERNEL);
234 	if (!blkioprio_blkg)
235 		return -ENOMEM;
236 
237 	ret = blkcg_activate_policy(q, &ioprio_policy);
238 	if (ret) {
239 		kfree(blkioprio_blkg);
240 		return ret;
241 	}
242 
243 	rqos = &blkioprio_blkg->rqos;
244 	rqos->id = RQ_QOS_IOPRIO;
245 	rqos->ops = &blkcg_ioprio_ops;
246 	rqos->q = q;
247 
248 	/*
249 	 * Registering the rq-qos policy after activating the blk-cgroup
250 	 * policy guarantees that ioprio_blkcg_from_bio(bio) != NULL in the
251 	 * rq-qos callbacks.
252 	 */
253 	rq_qos_add(q, rqos);
254 
255 	return 0;
256 }
257 
258 static int __init ioprio_init(void)
259 {
260 	return blkcg_policy_register(&ioprio_policy);
261 }
262 
263 static void __exit ioprio_exit(void)
264 {
265 	blkcg_policy_unregister(&ioprio_policy);
266 }
267 
268 module_init(ioprio_init);
269 module_exit(ioprio_exit);
270