1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include <linux/interrupt.h> 5 #include <linux/notifier.h> 6 #include <linux/module.h> 7 #include <linux/mlx5/driver.h> 8 #include "mlx5_core.h" 9 #ifdef CONFIG_RFS_ACCEL 10 #include <linux/cpu_rmap.h> 11 #endif 12 13 #define MLX5_MAX_IRQ_NAME (32) 14 15 struct mlx5_irq { 16 struct atomic_notifier_head nh; 17 cpumask_var_t mask; 18 char name[MLX5_MAX_IRQ_NAME]; 19 }; 20 21 struct mlx5_irq_table { 22 struct mlx5_irq *irq; 23 int nvec; 24 #ifdef CONFIG_RFS_ACCEL 25 struct cpu_rmap *rmap; 26 #endif 27 }; 28 29 int mlx5_irq_table_init(struct mlx5_core_dev *dev) 30 { 31 struct mlx5_irq_table *irq_table; 32 33 if (mlx5_core_is_sf(dev)) 34 return 0; 35 36 irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL); 37 if (!irq_table) 38 return -ENOMEM; 39 40 dev->priv.irq_table = irq_table; 41 return 0; 42 } 43 44 void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev) 45 { 46 if (mlx5_core_is_sf(dev)) 47 return; 48 49 kvfree(dev->priv.irq_table); 50 } 51 52 int mlx5_irq_get_num_comp(struct mlx5_irq_table *table) 53 { 54 return table->nvec - MLX5_IRQ_VEC_COMP_BASE; 55 } 56 57 static struct mlx5_irq *mlx5_irq_get(struct mlx5_core_dev *dev, int vecidx) 58 { 59 struct mlx5_irq_table *irq_table = dev->priv.irq_table; 60 61 return &irq_table->irq[vecidx]; 62 } 63 64 int mlx5_irq_attach_nb(struct mlx5_irq_table *irq_table, int vecidx, 65 struct notifier_block *nb) 66 { 67 struct mlx5_irq *irq; 68 69 irq = &irq_table->irq[vecidx]; 70 return atomic_notifier_chain_register(&irq->nh, nb); 71 } 72 73 int mlx5_irq_detach_nb(struct mlx5_irq_table *irq_table, int vecidx, 74 struct notifier_block *nb) 75 { 76 struct mlx5_irq *irq; 77 78 irq = &irq_table->irq[vecidx]; 79 return atomic_notifier_chain_unregister(&irq->nh, nb); 80 } 81 82 static irqreturn_t mlx5_irq_int_handler(int irq, void *nh) 83 { 84 atomic_notifier_call_chain(nh, 0, NULL); 85 return IRQ_HANDLED; 86 } 87 88 static void irq_set_name(char *name, int vecidx) 89 { 90 if (vecidx == 0) { 91 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async"); 92 return; 93 } 94 95 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", 96 vecidx - MLX5_IRQ_VEC_COMP_BASE); 97 return; 98 } 99 100 static int request_irqs(struct mlx5_core_dev *dev, int nvec) 101 { 102 char name[MLX5_MAX_IRQ_NAME]; 103 int err; 104 int i; 105 106 for (i = 0; i < nvec; i++) { 107 struct mlx5_irq *irq = mlx5_irq_get(dev, i); 108 int irqn = pci_irq_vector(dev->pdev, i); 109 110 irq_set_name(name, i); 111 ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh); 112 snprintf(irq->name, MLX5_MAX_IRQ_NAME, 113 "%s@pci:%s", name, pci_name(dev->pdev)); 114 err = request_irq(irqn, mlx5_irq_int_handler, 0, irq->name, 115 &irq->nh); 116 if (err) { 117 mlx5_core_err(dev, "Failed to request irq\n"); 118 goto err_request_irq; 119 } 120 } 121 return 0; 122 123 err_request_irq: 124 while (i--) { 125 struct mlx5_irq *irq = mlx5_irq_get(dev, i); 126 int irqn = pci_irq_vector(dev->pdev, i); 127 128 free_irq(irqn, &irq->nh); 129 } 130 return err; 131 } 132 133 static void irq_clear_rmap(struct mlx5_core_dev *dev) 134 { 135 #ifdef CONFIG_RFS_ACCEL 136 struct mlx5_irq_table *irq_table = dev->priv.irq_table; 137 138 free_irq_cpu_rmap(irq_table->rmap); 139 #endif 140 } 141 142 static int irq_set_rmap(struct mlx5_core_dev *mdev) 143 { 144 int err = 0; 145 #ifdef CONFIG_RFS_ACCEL 146 struct mlx5_irq_table *irq_table = mdev->priv.irq_table; 147 int num_affinity_vec; 148 int vecidx; 149 150 num_affinity_vec = mlx5_irq_get_num_comp(irq_table); 151 irq_table->rmap = alloc_irq_cpu_rmap(num_affinity_vec); 152 if (!irq_table->rmap) { 153 err = -ENOMEM; 154 mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err); 155 goto err_out; 156 } 157 158 vecidx = MLX5_IRQ_VEC_COMP_BASE; 159 for (; vecidx < irq_table->nvec; vecidx++) { 160 err = irq_cpu_rmap_add(irq_table->rmap, 161 pci_irq_vector(mdev->pdev, vecidx)); 162 if (err) { 163 mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d", 164 err); 165 goto err_irq_cpu_rmap_add; 166 } 167 } 168 return 0; 169 170 err_irq_cpu_rmap_add: 171 irq_clear_rmap(mdev); 172 err_out: 173 #endif 174 return err; 175 } 176 177 /* Completion IRQ vectors */ 178 179 static int set_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i) 180 { 181 int vecidx = MLX5_IRQ_VEC_COMP_BASE + i; 182 struct mlx5_irq *irq; 183 int irqn; 184 185 irq = mlx5_irq_get(mdev, vecidx); 186 irqn = pci_irq_vector(mdev->pdev, vecidx); 187 if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) { 188 mlx5_core_warn(mdev, "zalloc_cpumask_var failed"); 189 return -ENOMEM; 190 } 191 192 cpumask_set_cpu(cpumask_local_spread(i, mdev->priv.numa_node), 193 irq->mask); 194 if (IS_ENABLED(CONFIG_SMP) && 195 irq_set_affinity_hint(irqn, irq->mask)) 196 mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x", 197 irqn); 198 199 return 0; 200 } 201 202 static void clear_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i) 203 { 204 int vecidx = MLX5_IRQ_VEC_COMP_BASE + i; 205 struct mlx5_irq *irq; 206 int irqn; 207 208 irq = mlx5_irq_get(mdev, vecidx); 209 irqn = pci_irq_vector(mdev->pdev, vecidx); 210 irq_set_affinity_hint(irqn, NULL); 211 free_cpumask_var(irq->mask); 212 } 213 214 static int set_comp_irq_affinity_hints(struct mlx5_core_dev *mdev) 215 { 216 int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table); 217 int err; 218 int i; 219 220 for (i = 0; i < nvec; i++) { 221 err = set_comp_irq_affinity_hint(mdev, i); 222 if (err) 223 goto err_out; 224 } 225 226 return 0; 227 228 err_out: 229 for (i--; i >= 0; i--) 230 clear_comp_irq_affinity_hint(mdev, i); 231 232 return err; 233 } 234 235 static void clear_comp_irqs_affinity_hints(struct mlx5_core_dev *mdev) 236 { 237 int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table); 238 int i; 239 240 for (i = 0; i < nvec; i++) 241 clear_comp_irq_affinity_hint(mdev, i); 242 } 243 244 struct cpumask * 245 mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx) 246 { 247 return irq_table->irq[vecidx].mask; 248 } 249 250 #ifdef CONFIG_RFS_ACCEL 251 struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *irq_table) 252 { 253 return irq_table->rmap; 254 } 255 #endif 256 257 static void unrequest_irqs(struct mlx5_core_dev *dev) 258 { 259 struct mlx5_irq_table *table = dev->priv.irq_table; 260 int i; 261 262 for (i = 0; i < table->nvec; i++) 263 free_irq(pci_irq_vector(dev->pdev, i), 264 &mlx5_irq_get(dev, i)->nh); 265 } 266 267 int mlx5_irq_table_create(struct mlx5_core_dev *dev) 268 { 269 struct mlx5_priv *priv = &dev->priv; 270 struct mlx5_irq_table *table = priv->irq_table; 271 int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ? 272 MLX5_CAP_GEN(dev, max_num_eqs) : 273 1 << MLX5_CAP_GEN(dev, log_max_eq); 274 int nvec; 275 int err; 276 277 if (mlx5_core_is_sf(dev)) 278 return 0; 279 280 nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 281 MLX5_IRQ_VEC_COMP_BASE; 282 nvec = min_t(int, nvec, num_eqs); 283 if (nvec <= MLX5_IRQ_VEC_COMP_BASE) 284 return -ENOMEM; 285 286 table->irq = kcalloc(nvec, sizeof(*table->irq), GFP_KERNEL); 287 if (!table->irq) 288 return -ENOMEM; 289 290 nvec = pci_alloc_irq_vectors(dev->pdev, MLX5_IRQ_VEC_COMP_BASE + 1, 291 nvec, PCI_IRQ_MSIX); 292 if (nvec < 0) { 293 err = nvec; 294 goto err_free_irq; 295 } 296 297 table->nvec = nvec; 298 299 err = irq_set_rmap(dev); 300 if (err) 301 goto err_set_rmap; 302 303 err = request_irqs(dev, nvec); 304 if (err) 305 goto err_request_irqs; 306 307 err = set_comp_irq_affinity_hints(dev); 308 if (err) { 309 mlx5_core_err(dev, "Failed to alloc affinity hint cpumask\n"); 310 goto err_set_affinity; 311 } 312 313 return 0; 314 315 err_set_affinity: 316 unrequest_irqs(dev); 317 err_request_irqs: 318 irq_clear_rmap(dev); 319 err_set_rmap: 320 pci_free_irq_vectors(dev->pdev); 321 err_free_irq: 322 kfree(table->irq); 323 return err; 324 } 325 326 void mlx5_irq_table_destroy(struct mlx5_core_dev *dev) 327 { 328 struct mlx5_irq_table *table = dev->priv.irq_table; 329 int i; 330 331 if (mlx5_core_is_sf(dev)) 332 return; 333 334 /* free_irq requires that affinity and rmap will be cleared 335 * before calling it. This is why there is asymmetry with set_rmap 336 * which should be called after alloc_irq but before request_irq. 337 */ 338 irq_clear_rmap(dev); 339 clear_comp_irqs_affinity_hints(dev); 340 for (i = 0; i < table->nvec; i++) 341 free_irq(pci_irq_vector(dev->pdev, i), 342 &mlx5_irq_get(dev, i)->nh); 343 pci_free_irq_vectors(dev->pdev); 344 kfree(table->irq); 345 } 346 347 struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev) 348 { 349 #ifdef CONFIG_MLX5_SF 350 if (mlx5_core_is_sf(dev)) 351 return dev->priv.parent_mdev->priv.irq_table; 352 #endif 353 return dev->priv.irq_table; 354 } 355