1 /* 2 * Copyright (C) 2016 Thomas Gleixner. 3 * Copyright (C) 2016-2017 Christoph Hellwig. 4 */ 5 #include <linux/interrupt.h> 6 #include <linux/kernel.h> 7 #include <linux/slab.h> 8 #include <linux/cpu.h> 9 10 static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, 11 int cpus_per_vec) 12 { 13 const struct cpumask *siblmsk; 14 int cpu, sibl; 15 16 for ( ; cpus_per_vec > 0; ) { 17 cpu = cpumask_first(nmsk); 18 19 /* Should not happen, but I'm too lazy to think about it */ 20 if (cpu >= nr_cpu_ids) 21 return; 22 23 cpumask_clear_cpu(cpu, nmsk); 24 cpumask_set_cpu(cpu, irqmsk); 25 cpus_per_vec--; 26 27 /* If the cpu has siblings, use them first */ 28 siblmsk = topology_sibling_cpumask(cpu); 29 for (sibl = -1; cpus_per_vec > 0; ) { 30 sibl = cpumask_next(sibl, siblmsk); 31 if (sibl >= nr_cpu_ids) 32 break; 33 if (!cpumask_test_and_clear_cpu(sibl, nmsk)) 34 continue; 35 cpumask_set_cpu(sibl, irqmsk); 36 cpus_per_vec--; 37 } 38 } 39 } 40 41 static cpumask_var_t *alloc_node_to_present_cpumask(void) 42 { 43 cpumask_var_t *masks; 44 int node; 45 46 masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL); 47 if (!masks) 48 return NULL; 49 50 for (node = 0; node < nr_node_ids; node++) { 51 if (!zalloc_cpumask_var(&masks[node], GFP_KERNEL)) 52 goto out_unwind; 53 } 54 55 return masks; 56 57 out_unwind: 58 while (--node >= 0) 59 free_cpumask_var(masks[node]); 60 kfree(masks); 61 return NULL; 62 } 63 64 static void free_node_to_present_cpumask(cpumask_var_t *masks) 65 { 66 int node; 67 68 for (node = 0; node < nr_node_ids; node++) 69 free_cpumask_var(masks[node]); 70 kfree(masks); 71 } 72 73 static void build_node_to_present_cpumask(cpumask_var_t *masks) 74 { 75 int cpu; 76 77 for_each_present_cpu(cpu) 78 cpumask_set_cpu(cpu, masks[cpu_to_node(cpu)]); 79 } 80 81 static int get_nodes_in_cpumask(cpumask_var_t *node_to_present_cpumask, 82 const struct cpumask *mask, nodemask_t *nodemsk) 83 { 84 int n, nodes = 0; 85 86 /* Calculate the number of nodes in the supplied affinity mask */ 87 for_each_node(n) { 88 if (cpumask_intersects(mask, node_to_present_cpumask[n])) { 89 node_set(n, *nodemsk); 90 nodes++; 91 } 92 } 93 return nodes; 94 } 95 96 /** 97 * irq_create_affinity_masks - Create affinity masks for multiqueue spreading 98 * @nvecs: The total number of vectors 99 * @affd: Description of the affinity requirements 100 * 101 * Returns the masks pointer or NULL if allocation failed. 102 */ 103 struct cpumask * 104 irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) 105 { 106 int n, nodes, cpus_per_vec, extra_vecs, curvec; 107 int affv = nvecs - affd->pre_vectors - affd->post_vectors; 108 int last_affv = affv + affd->pre_vectors; 109 nodemask_t nodemsk = NODE_MASK_NONE; 110 struct cpumask *masks; 111 cpumask_var_t nmsk, *node_to_present_cpumask; 112 113 /* 114 * If there aren't any vectors left after applying the pre/post 115 * vectors don't bother with assigning affinity. 116 */ 117 if (!affv) 118 return NULL; 119 120 if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) 121 return NULL; 122 123 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); 124 if (!masks) 125 goto out; 126 127 node_to_present_cpumask = alloc_node_to_present_cpumask(); 128 if (!node_to_present_cpumask) 129 goto out; 130 131 /* Fill out vectors at the beginning that don't need affinity */ 132 for (curvec = 0; curvec < affd->pre_vectors; curvec++) 133 cpumask_copy(masks + curvec, irq_default_affinity); 134 135 /* Stabilize the cpumasks */ 136 get_online_cpus(); 137 build_node_to_present_cpumask(node_to_present_cpumask); 138 nodes = get_nodes_in_cpumask(node_to_present_cpumask, cpu_present_mask, 139 &nodemsk); 140 141 /* 142 * If the number of nodes in the mask is greater than or equal the 143 * number of vectors we just spread the vectors across the nodes. 144 */ 145 if (affv <= nodes) { 146 for_each_node_mask(n, nodemsk) { 147 cpumask_copy(masks + curvec, 148 node_to_present_cpumask[n]); 149 if (++curvec == last_affv) 150 break; 151 } 152 goto done; 153 } 154 155 for_each_node_mask(n, nodemsk) { 156 int ncpus, v, vecs_to_assign, vecs_per_node; 157 158 /* Spread the vectors per node */ 159 vecs_per_node = (affv - (curvec - affd->pre_vectors)) / nodes; 160 161 /* Get the cpus on this node which are in the mask */ 162 cpumask_and(nmsk, cpu_present_mask, node_to_present_cpumask[n]); 163 164 /* Calculate the number of cpus per vector */ 165 ncpus = cpumask_weight(nmsk); 166 vecs_to_assign = min(vecs_per_node, ncpus); 167 168 /* Account for rounding errors */ 169 extra_vecs = ncpus - vecs_to_assign * (ncpus / vecs_to_assign); 170 171 for (v = 0; curvec < last_affv && v < vecs_to_assign; 172 curvec++, v++) { 173 cpus_per_vec = ncpus / vecs_to_assign; 174 175 /* Account for extra vectors to compensate rounding errors */ 176 if (extra_vecs) { 177 cpus_per_vec++; 178 --extra_vecs; 179 } 180 irq_spread_init_one(masks + curvec, nmsk, cpus_per_vec); 181 } 182 183 if (curvec >= last_affv) 184 break; 185 --nodes; 186 } 187 188 done: 189 put_online_cpus(); 190 191 /* Fill out vectors at the end that don't need affinity */ 192 for (; curvec < nvecs; curvec++) 193 cpumask_copy(masks + curvec, irq_default_affinity); 194 free_node_to_present_cpumask(node_to_present_cpumask); 195 out: 196 free_cpumask_var(nmsk); 197 return masks; 198 } 199 200 /** 201 * irq_calc_affinity_vectors - Calculate the optimal number of vectors 202 * @minvec: The minimum number of vectors available 203 * @maxvec: The maximum number of vectors available 204 * @affd: Description of the affinity requirements 205 */ 206 int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd) 207 { 208 int resv = affd->pre_vectors + affd->post_vectors; 209 int vecs = maxvec - resv; 210 int ret; 211 212 if (resv > minvec) 213 return 0; 214 215 get_online_cpus(); 216 ret = min_t(int, cpumask_weight(cpu_present_mask), vecs) + resv; 217 put_online_cpus(); 218 return ret; 219 } 220