1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Oracle Corporation 4 * 5 * Module Author: Mike Christie 6 */ 7 #include "dm-path-selector.h" 8 9 #include <linux/device-mapper.h> 10 #include <linux/module.h> 11 12 #define DM_MSG_PREFIX "multipath io-affinity" 13 14 struct path_info { 15 struct dm_path *path; 16 cpumask_var_t cpumask; 17 refcount_t refcount; 18 bool failed; 19 }; 20 21 struct selector { 22 struct path_info **path_map; 23 cpumask_var_t path_mask; 24 atomic_t map_misses; 25 }; 26 27 static void ioa_free_path(struct selector *s, unsigned int cpu) 28 { 29 struct path_info *pi = s->path_map[cpu]; 30 31 if (!pi) 32 return; 33 34 if (refcount_dec_and_test(&pi->refcount)) { 35 cpumask_clear_cpu(cpu, s->path_mask); 36 free_cpumask_var(pi->cpumask); 37 kfree(pi); 38 39 s->path_map[cpu] = NULL; 40 } 41 } 42 43 static int ioa_add_path(struct path_selector *ps, struct dm_path *path, 44 int argc, char **argv, char **error) 45 { 46 struct selector *s = ps->context; 47 struct path_info *pi = NULL; 48 unsigned int cpu; 49 int ret; 50 51 if (argc != 1) { 52 *error = "io-affinity ps: invalid number of arguments"; 53 return -EINVAL; 54 } 55 56 pi = kzalloc(sizeof(*pi), GFP_KERNEL); 57 if (!pi) { 58 *error = "io-affinity ps: Error allocating path context"; 59 return -ENOMEM; 60 } 61 62 pi->path = path; 63 path->pscontext = pi; 64 refcount_set(&pi->refcount, 1); 65 66 if (!zalloc_cpumask_var(&pi->cpumask, GFP_KERNEL)) { 67 *error = "io-affinity ps: Error allocating cpumask context"; 68 ret = -ENOMEM; 69 goto free_pi; 70 } 71 72 ret = cpumask_parse(argv[0], pi->cpumask); 73 if (ret) { 74 *error = "io-affinity ps: invalid cpumask"; 75 ret = -EINVAL; 76 goto free_mask; 77 } 78 79 for_each_cpu(cpu, pi->cpumask) { 80 if (cpu >= nr_cpu_ids) { 81 DMWARN_LIMIT("Ignoring mapping for CPU %u. Max CPU is %u", 82 cpu, nr_cpu_ids); 83 break; 84 } 85 86 if (s->path_map[cpu]) { 87 DMWARN("CPU mapping for %u exists. Ignoring.", cpu); 88 continue; 89 } 90 91 cpumask_set_cpu(cpu, s->path_mask); 92 s->path_map[cpu] = pi; 93 refcount_inc(&pi->refcount); 94 } 95 96 if (refcount_dec_and_test(&pi->refcount)) { 97 *error = "io-affinity ps: No new/valid CPU mapping found"; 98 ret = -EINVAL; 99 goto free_mask; 100 } 101 102 return 0; 103 104 free_mask: 105 free_cpumask_var(pi->cpumask); 106 free_pi: 107 kfree(pi); 108 return ret; 109 } 110 111 static int ioa_create(struct path_selector *ps, unsigned int argc, char **argv) 112 { 113 struct selector *s; 114 115 s = kmalloc(sizeof(*s), GFP_KERNEL); 116 if (!s) 117 return -ENOMEM; 118 119 s->path_map = kzalloc(nr_cpu_ids * sizeof(struct path_info *), 120 GFP_KERNEL); 121 if (!s->path_map) 122 goto free_selector; 123 124 if (!zalloc_cpumask_var(&s->path_mask, GFP_KERNEL)) 125 goto free_map; 126 127 atomic_set(&s->map_misses, 0); 128 ps->context = s; 129 return 0; 130 131 free_map: 132 kfree(s->path_map); 133 free_selector: 134 kfree(s); 135 return -ENOMEM; 136 } 137 138 static void ioa_destroy(struct path_selector *ps) 139 { 140 struct selector *s = ps->context; 141 unsigned int cpu; 142 143 for_each_cpu(cpu, s->path_mask) 144 ioa_free_path(s, cpu); 145 146 free_cpumask_var(s->path_mask); 147 kfree(s->path_map); 148 kfree(s); 149 150 ps->context = NULL; 151 } 152 153 static int ioa_status(struct path_selector *ps, struct dm_path *path, 154 status_type_t type, char *result, unsigned int maxlen) 155 { 156 struct selector *s = ps->context; 157 struct path_info *pi; 158 int sz = 0; 159 160 if (!path) { 161 DMEMIT("0 "); 162 return sz; 163 } 164 165 switch (type) { 166 case STATUSTYPE_INFO: 167 DMEMIT("%d ", atomic_read(&s->map_misses)); 168 break; 169 case STATUSTYPE_TABLE: 170 pi = path->pscontext; 171 DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask)); 172 break; 173 case STATUSTYPE_IMA: 174 *result = '\0'; 175 break; 176 } 177 178 return sz; 179 } 180 181 static void ioa_fail_path(struct path_selector *ps, struct dm_path *p) 182 { 183 struct path_info *pi = p->pscontext; 184 185 pi->failed = true; 186 } 187 188 static int ioa_reinstate_path(struct path_selector *ps, struct dm_path *p) 189 { 190 struct path_info *pi = p->pscontext; 191 192 pi->failed = false; 193 return 0; 194 } 195 196 static struct dm_path *ioa_select_path(struct path_selector *ps, 197 size_t nr_bytes) 198 { 199 unsigned int cpu, node; 200 struct selector *s = ps->context; 201 const struct cpumask *cpumask; 202 struct path_info *pi; 203 int i; 204 205 cpu = get_cpu(); 206 207 pi = s->path_map[cpu]; 208 if (pi && !pi->failed) 209 goto done; 210 211 /* 212 * Perf is not optimal, but we at least try the local node then just 213 * try not to fail. 214 */ 215 if (!pi) 216 atomic_inc(&s->map_misses); 217 218 node = cpu_to_node(cpu); 219 cpumask = cpumask_of_node(node); 220 for_each_cpu(i, cpumask) { 221 pi = s->path_map[i]; 222 if (pi && !pi->failed) 223 goto done; 224 } 225 226 for_each_cpu(i, s->path_mask) { 227 pi = s->path_map[i]; 228 if (pi && !pi->failed) 229 goto done; 230 } 231 pi = NULL; 232 233 done: 234 put_cpu(); 235 return pi ? pi->path : NULL; 236 } 237 238 static struct path_selector_type ioa_ps = { 239 .name = "io-affinity", 240 .module = THIS_MODULE, 241 .table_args = 1, 242 .info_args = 1, 243 .create = ioa_create, 244 .destroy = ioa_destroy, 245 .status = ioa_status, 246 .add_path = ioa_add_path, 247 .fail_path = ioa_fail_path, 248 .reinstate_path = ioa_reinstate_path, 249 .select_path = ioa_select_path, 250 }; 251 252 static int __init dm_ioa_init(void) 253 { 254 int ret = dm_register_path_selector(&ioa_ps); 255 256 if (ret < 0) 257 DMERR("register failed %d", ret); 258 return ret; 259 } 260 261 static void __exit dm_ioa_exit(void) 262 { 263 int ret = dm_unregister_path_selector(&ioa_ps); 264 265 if (ret < 0) 266 DMERR("unregister failed %d", ret); 267 } 268 269 module_init(dm_ioa_init); 270 module_exit(dm_ioa_exit); 271 272 MODULE_DESCRIPTION(DM_NAME " multipath path selector that selects paths based on the CPU IO is being executed on"); 273 MODULE_AUTHOR("Mike Christie <michael.christie@oracle.com>"); 274 MODULE_LICENSE("GPL"); 275