1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020 Oracle Corporation
4  *
5  * Module Author: Mike Christie
6  */
7 #include "dm-path-selector.h"
8 
9 #include <linux/device-mapper.h>
10 #include <linux/module.h>
11 
12 #define DM_MSG_PREFIX "multipath io-affinity"
13 
14 struct path_info {
15 	struct dm_path *path;
16 	cpumask_var_t cpumask;
17 	refcount_t refcount;
18 	bool failed;
19 };
20 
21 struct selector {
22 	struct path_info **path_map;
23 	cpumask_var_t path_mask;
24 	atomic_t map_misses;
25 };
26 
27 static void ioa_free_path(struct selector *s, unsigned int cpu)
28 {
29 	struct path_info *pi = s->path_map[cpu];
30 
31 	if (!pi)
32 		return;
33 
34 	if (refcount_dec_and_test(&pi->refcount)) {
35 		cpumask_clear_cpu(cpu, s->path_mask);
36 		free_cpumask_var(pi->cpumask);
37 		kfree(pi);
38 
39 		s->path_map[cpu] = NULL;
40 	}
41 }
42 
43 static int ioa_add_path(struct path_selector *ps, struct dm_path *path,
44 			int argc, char **argv, char **error)
45 {
46 	struct selector *s = ps->context;
47 	struct path_info *pi = NULL;
48 	unsigned int cpu;
49 	int ret;
50 
51 	if (argc != 1) {
52 		*error = "io-affinity ps: invalid number of arguments";
53 		return -EINVAL;
54 	}
55 
56 	pi = kzalloc(sizeof(*pi), GFP_KERNEL);
57 	if (!pi) {
58 		*error = "io-affinity ps: Error allocating path context";
59 		return -ENOMEM;
60 	}
61 
62 	pi->path = path;
63 	path->pscontext = pi;
64 	refcount_set(&pi->refcount, 1);
65 
66 	if (!zalloc_cpumask_var(&pi->cpumask, GFP_KERNEL)) {
67 		*error = "io-affinity ps: Error allocating cpumask context";
68 		ret = -ENOMEM;
69 		goto free_pi;
70 	}
71 
72 	ret = cpumask_parse(argv[0], pi->cpumask);
73 	if (ret) {
74 		*error = "io-affinity ps: invalid cpumask";
75 		ret = -EINVAL;
76 		goto free_mask;
77 	}
78 
79 	for_each_cpu(cpu, pi->cpumask) {
80 		if (cpu >= nr_cpu_ids) {
81 			DMWARN_LIMIT("Ignoring mapping for CPU %u. Max CPU is %u",
82 				     cpu, nr_cpu_ids);
83 			break;
84 		}
85 
86 		if (s->path_map[cpu]) {
87 			DMWARN("CPU mapping for %u exists. Ignoring.", cpu);
88 			continue;
89 		}
90 
91 		cpumask_set_cpu(cpu, s->path_mask);
92 		s->path_map[cpu] = pi;
93 		refcount_inc(&pi->refcount);
94 	}
95 
96 	if (refcount_dec_and_test(&pi->refcount)) {
97 		*error = "io-affinity ps: No new/valid CPU mapping found";
98 		ret = -EINVAL;
99 		goto free_mask;
100 	}
101 
102 	return 0;
103 
104 free_mask:
105 	free_cpumask_var(pi->cpumask);
106 free_pi:
107 	kfree(pi);
108 	return ret;
109 }
110 
111 static int ioa_create(struct path_selector *ps, unsigned int argc, char **argv)
112 {
113 	struct selector *s;
114 
115 	s = kmalloc(sizeof(*s), GFP_KERNEL);
116 	if (!s)
117 		return -ENOMEM;
118 
119 	s->path_map = kzalloc(nr_cpu_ids * sizeof(struct path_info *),
120 			      GFP_KERNEL);
121 	if (!s->path_map)
122 		goto free_selector;
123 
124 	if (!zalloc_cpumask_var(&s->path_mask, GFP_KERNEL))
125 		goto free_map;
126 
127 	atomic_set(&s->map_misses, 0);
128 	ps->context = s;
129 	return 0;
130 
131 free_map:
132 	kfree(s->path_map);
133 free_selector:
134 	kfree(s);
135 	return -ENOMEM;
136 }
137 
138 static void ioa_destroy(struct path_selector *ps)
139 {
140 	struct selector *s = ps->context;
141 	unsigned int cpu;
142 
143 	for_each_cpu(cpu, s->path_mask)
144 		ioa_free_path(s, cpu);
145 
146 	free_cpumask_var(s->path_mask);
147 	kfree(s->path_map);
148 	kfree(s);
149 
150 	ps->context = NULL;
151 }
152 
153 static int ioa_status(struct path_selector *ps, struct dm_path *path,
154 		      status_type_t type, char *result, unsigned int maxlen)
155 {
156 	struct selector *s = ps->context;
157 	struct path_info *pi;
158 	int sz = 0;
159 
160 	if (!path) {
161 		DMEMIT("0 ");
162 		return sz;
163 	}
164 
165 	switch (type) {
166 	case STATUSTYPE_INFO:
167 		DMEMIT("%d ", atomic_read(&s->map_misses));
168 		break;
169 	case STATUSTYPE_TABLE:
170 		pi = path->pscontext;
171 		DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask));
172 		break;
173 	case STATUSTYPE_IMA:
174 		*result = '\0';
175 		break;
176 	}
177 
178 	return sz;
179 }
180 
181 static void ioa_fail_path(struct path_selector *ps, struct dm_path *p)
182 {
183 	struct path_info *pi = p->pscontext;
184 
185 	pi->failed = true;
186 }
187 
188 static int ioa_reinstate_path(struct path_selector *ps, struct dm_path *p)
189 {
190 	struct path_info *pi = p->pscontext;
191 
192 	pi->failed = false;
193 	return 0;
194 }
195 
196 static struct dm_path *ioa_select_path(struct path_selector *ps,
197 				       size_t nr_bytes)
198 {
199 	unsigned int cpu, node;
200 	struct selector *s = ps->context;
201 	const struct cpumask *cpumask;
202 	struct path_info *pi;
203 	int i;
204 
205 	cpu = get_cpu();
206 
207 	pi = s->path_map[cpu];
208 	if (pi && !pi->failed)
209 		goto done;
210 
211 	/*
212 	 * Perf is not optimal, but we at least try the local node then just
213 	 * try not to fail.
214 	 */
215 	if (!pi)
216 		atomic_inc(&s->map_misses);
217 
218 	node = cpu_to_node(cpu);
219 	cpumask = cpumask_of_node(node);
220 	for_each_cpu(i, cpumask) {
221 		pi = s->path_map[i];
222 		if (pi && !pi->failed)
223 			goto done;
224 	}
225 
226 	for_each_cpu(i, s->path_mask) {
227 		pi = s->path_map[i];
228 		if (pi && !pi->failed)
229 			goto done;
230 	}
231 	pi = NULL;
232 
233 done:
234 	put_cpu();
235 	return pi ? pi->path : NULL;
236 }
237 
238 static struct path_selector_type ioa_ps = {
239 	.name		= "io-affinity",
240 	.module		= THIS_MODULE,
241 	.table_args	= 1,
242 	.info_args	= 1,
243 	.create		= ioa_create,
244 	.destroy	= ioa_destroy,
245 	.status		= ioa_status,
246 	.add_path	= ioa_add_path,
247 	.fail_path	= ioa_fail_path,
248 	.reinstate_path	= ioa_reinstate_path,
249 	.select_path	= ioa_select_path,
250 };
251 
252 static int __init dm_ioa_init(void)
253 {
254 	int ret = dm_register_path_selector(&ioa_ps);
255 
256 	if (ret < 0)
257 		DMERR("register failed %d", ret);
258 	return ret;
259 }
260 
261 static void __exit dm_ioa_exit(void)
262 {
263 	int ret = dm_unregister_path_selector(&ioa_ps);
264 
265 	if (ret < 0)
266 		DMERR("unregister failed %d", ret);
267 }
268 
269 module_init(dm_ioa_init);
270 module_exit(dm_ioa_exit);
271 
272 MODULE_DESCRIPTION(DM_NAME " multipath path selector that selects paths based on the CPU IO is being executed on");
273 MODULE_AUTHOR("Mike Christie <michael.christie@oracle.com>");
274 MODULE_LICENSE("GPL");
275