1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3 
4 #include <linux/interrupt.h>
5 #include <linux/notifier.h>
6 #include <linux/module.h>
7 #include <linux/mlx5/driver.h>
8 #include "mlx5_core.h"
9 #ifdef CONFIG_RFS_ACCEL
10 #include <linux/cpu_rmap.h>
11 #endif
12 
13 #define MLX5_MAX_IRQ_NAME (32)
14 
15 struct mlx5_irq {
16 	struct atomic_notifier_head nh;
17 	cpumask_var_t mask;
18 	char name[MLX5_MAX_IRQ_NAME];
19 };
20 
21 struct mlx5_irq_table {
22 	struct mlx5_irq *irq;
23 	int nvec;
24 #ifdef CONFIG_RFS_ACCEL
25 	struct cpu_rmap *rmap;
26 #endif
27 };
28 
29 int mlx5_irq_table_init(struct mlx5_core_dev *dev)
30 {
31 	struct mlx5_irq_table *irq_table;
32 
33 	if (mlx5_core_is_sf(dev))
34 		return 0;
35 
36 	irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL);
37 	if (!irq_table)
38 		return -ENOMEM;
39 
40 	dev->priv.irq_table = irq_table;
41 	return 0;
42 }
43 
44 void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev)
45 {
46 	if (mlx5_core_is_sf(dev))
47 		return;
48 
49 	kvfree(dev->priv.irq_table);
50 }
51 
52 int mlx5_irq_get_num_comp(struct mlx5_irq_table *table)
53 {
54 	return table->nvec - MLX5_IRQ_VEC_COMP_BASE;
55 }
56 
57 static struct mlx5_irq *mlx5_irq_get(struct mlx5_core_dev *dev, int vecidx)
58 {
59 	struct mlx5_irq_table *irq_table = dev->priv.irq_table;
60 
61 	return &irq_table->irq[vecidx];
62 }
63 
64 int mlx5_irq_attach_nb(struct mlx5_irq_table *irq_table, int vecidx,
65 		       struct notifier_block *nb)
66 {
67 	struct mlx5_irq *irq;
68 
69 	irq = &irq_table->irq[vecidx];
70 	return atomic_notifier_chain_register(&irq->nh, nb);
71 }
72 
73 int mlx5_irq_detach_nb(struct mlx5_irq_table *irq_table, int vecidx,
74 		       struct notifier_block *nb)
75 {
76 	struct mlx5_irq *irq;
77 
78 	irq = &irq_table->irq[vecidx];
79 	return atomic_notifier_chain_unregister(&irq->nh, nb);
80 }
81 
82 static irqreturn_t mlx5_irq_int_handler(int irq, void *nh)
83 {
84 	atomic_notifier_call_chain(nh, 0, NULL);
85 	return IRQ_HANDLED;
86 }
87 
88 static void irq_set_name(char *name, int vecidx)
89 {
90 	if (vecidx == 0) {
91 		snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async");
92 		return;
93 	}
94 
95 	snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d",
96 		 vecidx - MLX5_IRQ_VEC_COMP_BASE);
97 	return;
98 }
99 
100 static int request_irqs(struct mlx5_core_dev *dev, int nvec)
101 {
102 	char name[MLX5_MAX_IRQ_NAME];
103 	int err;
104 	int i;
105 
106 	for (i = 0; i < nvec; i++) {
107 		struct mlx5_irq *irq = mlx5_irq_get(dev, i);
108 		int irqn = pci_irq_vector(dev->pdev, i);
109 
110 		irq_set_name(name, i);
111 		ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh);
112 		snprintf(irq->name, MLX5_MAX_IRQ_NAME,
113 			 "%s@pci:%s", name, pci_name(dev->pdev));
114 		err = request_irq(irqn, mlx5_irq_int_handler, 0, irq->name,
115 				  &irq->nh);
116 		if (err) {
117 			mlx5_core_err(dev, "Failed to request irq\n");
118 			goto err_request_irq;
119 		}
120 	}
121 	return 0;
122 
123 err_request_irq:
124 	while (i--) {
125 		struct mlx5_irq *irq = mlx5_irq_get(dev, i);
126 		int irqn = pci_irq_vector(dev->pdev, i);
127 
128 		free_irq(irqn, &irq->nh);
129 	}
130 	return  err;
131 }
132 
133 static void irq_clear_rmap(struct mlx5_core_dev *dev)
134 {
135 #ifdef CONFIG_RFS_ACCEL
136 	struct mlx5_irq_table *irq_table = dev->priv.irq_table;
137 
138 	free_irq_cpu_rmap(irq_table->rmap);
139 #endif
140 }
141 
142 static int irq_set_rmap(struct mlx5_core_dev *mdev)
143 {
144 	int err = 0;
145 #ifdef CONFIG_RFS_ACCEL
146 	struct mlx5_irq_table *irq_table = mdev->priv.irq_table;
147 	int num_affinity_vec;
148 	int vecidx;
149 
150 	num_affinity_vec = mlx5_irq_get_num_comp(irq_table);
151 	irq_table->rmap = alloc_irq_cpu_rmap(num_affinity_vec);
152 	if (!irq_table->rmap) {
153 		err = -ENOMEM;
154 		mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err);
155 		goto err_out;
156 	}
157 
158 	vecidx = MLX5_IRQ_VEC_COMP_BASE;
159 	for (; vecidx < irq_table->nvec; vecidx++) {
160 		err = irq_cpu_rmap_add(irq_table->rmap,
161 				       pci_irq_vector(mdev->pdev, vecidx));
162 		if (err) {
163 			mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d",
164 				      err);
165 			goto err_irq_cpu_rmap_add;
166 		}
167 	}
168 	return 0;
169 
170 err_irq_cpu_rmap_add:
171 	irq_clear_rmap(mdev);
172 err_out:
173 #endif
174 	return err;
175 }
176 
177 /* Completion IRQ vectors */
178 
179 static int set_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)
180 {
181 	int vecidx = MLX5_IRQ_VEC_COMP_BASE + i;
182 	struct mlx5_irq *irq;
183 	int irqn;
184 
185 	irq = mlx5_irq_get(mdev, vecidx);
186 	irqn = pci_irq_vector(mdev->pdev, vecidx);
187 	if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) {
188 		mlx5_core_warn(mdev, "zalloc_cpumask_var failed");
189 		return -ENOMEM;
190 	}
191 
192 	cpumask_set_cpu(cpumask_local_spread(i, mdev->priv.numa_node),
193 			irq->mask);
194 	if (IS_ENABLED(CONFIG_SMP) &&
195 	    irq_set_affinity_hint(irqn, irq->mask))
196 		mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x",
197 			       irqn);
198 
199 	return 0;
200 }
201 
202 static void clear_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)
203 {
204 	int vecidx = MLX5_IRQ_VEC_COMP_BASE + i;
205 	struct mlx5_irq *irq;
206 	int irqn;
207 
208 	irq = mlx5_irq_get(mdev, vecidx);
209 	irqn = pci_irq_vector(mdev->pdev, vecidx);
210 	irq_set_affinity_hint(irqn, NULL);
211 	free_cpumask_var(irq->mask);
212 }
213 
214 static int set_comp_irq_affinity_hints(struct mlx5_core_dev *mdev)
215 {
216 	int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table);
217 	int err;
218 	int i;
219 
220 	for (i = 0; i < nvec; i++) {
221 		err = set_comp_irq_affinity_hint(mdev, i);
222 		if (err)
223 			goto err_out;
224 	}
225 
226 	return 0;
227 
228 err_out:
229 	for (i--; i >= 0; i--)
230 		clear_comp_irq_affinity_hint(mdev, i);
231 
232 	return err;
233 }
234 
235 static void clear_comp_irqs_affinity_hints(struct mlx5_core_dev *mdev)
236 {
237 	int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table);
238 	int i;
239 
240 	for (i = 0; i < nvec; i++)
241 		clear_comp_irq_affinity_hint(mdev, i);
242 }
243 
244 struct cpumask *
245 mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx)
246 {
247 	return irq_table->irq[vecidx].mask;
248 }
249 
250 #ifdef CONFIG_RFS_ACCEL
251 struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *irq_table)
252 {
253 	return irq_table->rmap;
254 }
255 #endif
256 
257 static void unrequest_irqs(struct mlx5_core_dev *dev)
258 {
259 	struct mlx5_irq_table *table = dev->priv.irq_table;
260 	int i;
261 
262 	for (i = 0; i < table->nvec; i++)
263 		free_irq(pci_irq_vector(dev->pdev, i),
264 			 &mlx5_irq_get(dev, i)->nh);
265 }
266 
267 int mlx5_irq_table_create(struct mlx5_core_dev *dev)
268 {
269 	struct mlx5_priv *priv = &dev->priv;
270 	struct mlx5_irq_table *table = priv->irq_table;
271 	int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
272 		      MLX5_CAP_GEN(dev, max_num_eqs) :
273 		      1 << MLX5_CAP_GEN(dev, log_max_eq);
274 	int nvec;
275 	int err;
276 
277 	if (mlx5_core_is_sf(dev))
278 		return 0;
279 
280 	nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() +
281 	       MLX5_IRQ_VEC_COMP_BASE;
282 	nvec = min_t(int, nvec, num_eqs);
283 	if (nvec <= MLX5_IRQ_VEC_COMP_BASE)
284 		return -ENOMEM;
285 
286 	table->irq = kcalloc(nvec, sizeof(*table->irq), GFP_KERNEL);
287 	if (!table->irq)
288 		return -ENOMEM;
289 
290 	nvec = pci_alloc_irq_vectors(dev->pdev, MLX5_IRQ_VEC_COMP_BASE + 1,
291 				     nvec, PCI_IRQ_MSIX);
292 	if (nvec < 0) {
293 		err = nvec;
294 		goto err_free_irq;
295 	}
296 
297 	table->nvec = nvec;
298 
299 	err = irq_set_rmap(dev);
300 	if (err)
301 		goto err_set_rmap;
302 
303 	err = request_irqs(dev, nvec);
304 	if (err)
305 		goto err_request_irqs;
306 
307 	err = set_comp_irq_affinity_hints(dev);
308 	if (err) {
309 		mlx5_core_err(dev, "Failed to alloc affinity hint cpumask\n");
310 		goto err_set_affinity;
311 	}
312 
313 	return 0;
314 
315 err_set_affinity:
316 	unrequest_irqs(dev);
317 err_request_irqs:
318 	irq_clear_rmap(dev);
319 err_set_rmap:
320 	pci_free_irq_vectors(dev->pdev);
321 err_free_irq:
322 	kfree(table->irq);
323 	return err;
324 }
325 
326 void mlx5_irq_table_destroy(struct mlx5_core_dev *dev)
327 {
328 	struct mlx5_irq_table *table = dev->priv.irq_table;
329 	int i;
330 
331 	if (mlx5_core_is_sf(dev))
332 		return;
333 
334 	/* free_irq requires that affinity and rmap will be cleared
335 	 * before calling it. This is why there is asymmetry with set_rmap
336 	 * which should be called after alloc_irq but before request_irq.
337 	 */
338 	irq_clear_rmap(dev);
339 	clear_comp_irqs_affinity_hints(dev);
340 	for (i = 0; i < table->nvec; i++)
341 		free_irq(pci_irq_vector(dev->pdev, i),
342 			 &mlx5_irq_get(dev, i)->nh);
343 	pci_free_irq_vectors(dev->pdev);
344 	kfree(table->irq);
345 }
346 
347 struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev)
348 {
349 #ifdef CONFIG_MLX5_SF
350 	if (mlx5_core_is_sf(dev))
351 		return dev->priv.parent_mdev->priv.irq_table;
352 #endif
353 	return dev->priv.irq_table;
354 }
355