1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Oracle Corporation 4 * 5 * Module Author: Mike Christie 6 */ 7 #include "dm-path-selector.h" 8 9 #include <linux/device-mapper.h> 10 #include <linux/module.h> 11 12 #define DM_MSG_PREFIX "multipath io-affinity" 13 14 struct path_info { 15 struct dm_path *path; 16 cpumask_var_t cpumask; 17 refcount_t refcount; 18 bool failed; 19 }; 20 21 struct selector { 22 struct path_info **path_map; 23 cpumask_var_t path_mask; 24 atomic_t map_misses; 25 }; 26 27 static void ioa_free_path(struct selector *s, unsigned int cpu) 28 { 29 struct path_info *pi = s->path_map[cpu]; 30 31 if (!pi) 32 return; 33 34 if (refcount_dec_and_test(&pi->refcount)) { 35 cpumask_clear_cpu(cpu, s->path_mask); 36 free_cpumask_var(pi->cpumask); 37 kfree(pi); 38 39 s->path_map[cpu] = NULL; 40 } 41 } 42 43 static int ioa_add_path(struct path_selector *ps, struct dm_path *path, 44 int argc, char **argv, char **error) 45 { 46 struct selector *s = ps->context; 47 struct path_info *pi = NULL; 48 unsigned int cpu; 49 int ret; 50 51 if (argc != 1) { 52 *error = "io-affinity ps: invalid number of arguments"; 53 return -EINVAL; 54 } 55 56 pi = kzalloc(sizeof(*pi), GFP_KERNEL); 57 if (!pi) { 58 *error = "io-affinity ps: Error allocating path context"; 59 return -ENOMEM; 60 } 61 62 pi->path = path; 63 path->pscontext = pi; 64 refcount_set(&pi->refcount, 1); 65 66 if (!zalloc_cpumask_var(&pi->cpumask, GFP_KERNEL)) { 67 *error = "io-affinity ps: Error allocating cpumask context"; 68 ret = -ENOMEM; 69 goto free_pi; 70 } 71 72 ret = cpumask_parse(argv[0], pi->cpumask); 73 if (ret) { 74 *error = "io-affinity ps: invalid cpumask"; 75 ret = -EINVAL; 76 goto free_mask; 77 } 78 79 for_each_cpu(cpu, pi->cpumask) { 80 if (cpu >= nr_cpu_ids) { 81 DMWARN_LIMIT("Ignoring mapping for CPU %u. Max CPU is %u", 82 cpu, nr_cpu_ids); 83 break; 84 } 85 86 if (s->path_map[cpu]) { 87 DMWARN("CPU mapping for %u exists. Ignoring.", cpu); 88 continue; 89 } 90 91 cpumask_set_cpu(cpu, s->path_mask); 92 s->path_map[cpu] = pi; 93 refcount_inc(&pi->refcount); 94 continue; 95 } 96 97 if (refcount_dec_and_test(&pi->refcount)) { 98 *error = "io-affinity ps: No new/valid CPU mapping found"; 99 ret = -EINVAL; 100 goto free_mask; 101 } 102 103 return 0; 104 105 free_mask: 106 free_cpumask_var(pi->cpumask); 107 free_pi: 108 kfree(pi); 109 return ret; 110 } 111 112 static int ioa_create(struct path_selector *ps, unsigned argc, char **argv) 113 { 114 struct selector *s; 115 116 s = kmalloc(sizeof(*s), GFP_KERNEL); 117 if (!s) 118 return -ENOMEM; 119 120 s->path_map = kzalloc(nr_cpu_ids * sizeof(struct path_info *), 121 GFP_KERNEL); 122 if (!s->path_map) 123 goto free_selector; 124 125 if (!zalloc_cpumask_var(&s->path_mask, GFP_KERNEL)) 126 goto free_map; 127 128 atomic_set(&s->map_misses, 0); 129 ps->context = s; 130 return 0; 131 132 free_map: 133 kfree(s->path_map); 134 free_selector: 135 kfree(s); 136 return -ENOMEM; 137 } 138 139 static void ioa_destroy(struct path_selector *ps) 140 { 141 struct selector *s = ps->context; 142 unsigned cpu; 143 144 for_each_cpu(cpu, s->path_mask) 145 ioa_free_path(s, cpu); 146 147 free_cpumask_var(s->path_mask); 148 kfree(s->path_map); 149 kfree(s); 150 151 ps->context = NULL; 152 } 153 154 static int ioa_status(struct path_selector *ps, struct dm_path *path, 155 status_type_t type, char *result, unsigned int maxlen) 156 { 157 struct selector *s = ps->context; 158 struct path_info *pi; 159 int sz = 0; 160 161 if (!path) { 162 DMEMIT("0 "); 163 return sz; 164 } 165 166 switch(type) { 167 case STATUSTYPE_INFO: 168 DMEMIT("%d ", atomic_read(&s->map_misses)); 169 break; 170 case STATUSTYPE_TABLE: 171 pi = path->pscontext; 172 DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask)); 173 break; 174 } 175 176 return sz; 177 } 178 179 static void ioa_fail_path(struct path_selector *ps, struct dm_path *p) 180 { 181 struct path_info *pi = p->pscontext; 182 183 pi->failed = true; 184 } 185 186 static int ioa_reinstate_path(struct path_selector *ps, struct dm_path *p) 187 { 188 struct path_info *pi = p->pscontext; 189 190 pi->failed = false; 191 return 0; 192 } 193 194 static struct dm_path *ioa_select_path(struct path_selector *ps, 195 size_t nr_bytes) 196 { 197 unsigned int cpu, node; 198 struct selector *s = ps->context; 199 const struct cpumask *cpumask; 200 struct path_info *pi; 201 int i; 202 203 cpu = get_cpu(); 204 205 pi = s->path_map[cpu]; 206 if (pi && !pi->failed) 207 goto done; 208 209 /* 210 * Perf is not optimal, but we at least try the local node then just 211 * try not to fail. 212 */ 213 if (!pi) 214 atomic_inc(&s->map_misses); 215 216 node = cpu_to_node(cpu); 217 cpumask = cpumask_of_node(node); 218 for_each_cpu(i, cpumask) { 219 pi = s->path_map[i]; 220 if (pi && !pi->failed) 221 goto done; 222 } 223 224 for_each_cpu(i, s->path_mask) { 225 pi = s->path_map[i]; 226 if (pi && !pi->failed) 227 goto done; 228 } 229 pi = NULL; 230 231 done: 232 put_cpu(); 233 return pi ? pi->path : NULL; 234 } 235 236 static struct path_selector_type ioa_ps = { 237 .name = "io-affinity", 238 .module = THIS_MODULE, 239 .table_args = 1, 240 .info_args = 1, 241 .create = ioa_create, 242 .destroy = ioa_destroy, 243 .status = ioa_status, 244 .add_path = ioa_add_path, 245 .fail_path = ioa_fail_path, 246 .reinstate_path = ioa_reinstate_path, 247 .select_path = ioa_select_path, 248 }; 249 250 static int __init dm_ioa_init(void) 251 { 252 int ret = dm_register_path_selector(&ioa_ps); 253 254 if (ret < 0) 255 DMERR("register failed %d", ret); 256 return ret; 257 } 258 259 static void __exit dm_ioa_exit(void) 260 { 261 int ret = dm_unregister_path_selector(&ioa_ps); 262 263 if (ret < 0) 264 DMERR("unregister failed %d", ret); 265 } 266 267 module_init(dm_ioa_init); 268 module_exit(dm_ioa_exit); 269 270 MODULE_DESCRIPTION(DM_NAME " multipath path selector that selects paths based on the CPU IO is being executed on"); 271 MODULE_AUTHOR("Mike Christie <michael.christie@oracle.com>"); 272 MODULE_LICENSE("GPL"); 273