1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2 /*
3  * Copyright(c) 2020 Intel Corporation.
4  *
5  */
6 
7 /*
8  * This file contains HFI1 support for netdev RX functionality
9  */
10 
11 #include "sdma.h"
12 #include "verbs.h"
13 #include "netdev.h"
14 #include "hfi.h"
15 
16 #include <linux/netdevice.h>
17 #include <linux/etherdevice.h>
18 #include <rdma/ib_verbs.h>
19 
20 static int hfi1_netdev_setup_ctxt(struct hfi1_netdev_priv *priv,
21 				  struct hfi1_ctxtdata *uctxt)
22 {
23 	unsigned int rcvctrl_ops;
24 	struct hfi1_devdata *dd = priv->dd;
25 	int ret;
26 
27 	uctxt->rhf_rcv_function_map = netdev_rhf_rcv_functions;
28 	uctxt->do_interrupt = &handle_receive_interrupt_napi_sp;
29 
30 	/* Now allocate the RcvHdr queue and eager buffers. */
31 	ret = hfi1_create_rcvhdrq(dd, uctxt);
32 	if (ret)
33 		goto done;
34 
35 	ret = hfi1_setup_eagerbufs(uctxt);
36 	if (ret)
37 		goto done;
38 
39 	clear_rcvhdrtail(uctxt);
40 
41 	rcvctrl_ops = HFI1_RCVCTRL_CTXT_DIS;
42 	rcvctrl_ops |= HFI1_RCVCTRL_INTRAVAIL_DIS;
43 
44 	if (!HFI1_CAP_KGET_MASK(uctxt->flags, MULTI_PKT_EGR))
45 		rcvctrl_ops |= HFI1_RCVCTRL_ONE_PKT_EGR_ENB;
46 	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_EGR_FULL))
47 		rcvctrl_ops |= HFI1_RCVCTRL_NO_EGR_DROP_ENB;
48 	if (HFI1_CAP_KGET_MASK(uctxt->flags, NODROP_RHQ_FULL))
49 		rcvctrl_ops |= HFI1_RCVCTRL_NO_RHQ_DROP_ENB;
50 	if (HFI1_CAP_KGET_MASK(uctxt->flags, DMA_RTAIL))
51 		rcvctrl_ops |= HFI1_RCVCTRL_TAILUPD_ENB;
52 
53 	hfi1_rcvctrl(uctxt->dd, rcvctrl_ops, uctxt);
54 done:
55 	return ret;
56 }
57 
58 static int hfi1_netdev_allocate_ctxt(struct hfi1_devdata *dd,
59 				     struct hfi1_ctxtdata **ctxt)
60 {
61 	struct hfi1_ctxtdata *uctxt;
62 	int ret;
63 
64 	if (dd->flags & HFI1_FROZEN)
65 		return -EIO;
66 
67 	ret = hfi1_create_ctxtdata(dd->pport, dd->node, &uctxt);
68 	if (ret < 0) {
69 		dd_dev_err(dd, "Unable to create ctxtdata, failing open\n");
70 		return -ENOMEM;
71 	}
72 
73 	uctxt->flags = HFI1_CAP_KGET(MULTI_PKT_EGR) |
74 		HFI1_CAP_KGET(NODROP_RHQ_FULL) |
75 		HFI1_CAP_KGET(NODROP_EGR_FULL) |
76 		HFI1_CAP_KGET(DMA_RTAIL);
77 	/* Netdev contexts are always NO_RDMA_RTAIL */
78 	uctxt->fast_handler = handle_receive_interrupt_napi_fp;
79 	uctxt->slow_handler = handle_receive_interrupt_napi_sp;
80 	hfi1_set_seq_cnt(uctxt, 1);
81 	uctxt->is_vnic = true;
82 
83 	hfi1_stats.sps_ctxts++;
84 
85 	dd_dev_info(dd, "created netdev context %d\n", uctxt->ctxt);
86 	*ctxt = uctxt;
87 
88 	return 0;
89 }
90 
91 static void hfi1_netdev_deallocate_ctxt(struct hfi1_devdata *dd,
92 					struct hfi1_ctxtdata *uctxt)
93 {
94 	flush_wc();
95 
96 	/*
97 	 * Disable receive context and interrupt available, reset all
98 	 * RcvCtxtCtrl bits to default values.
99 	 */
100 	hfi1_rcvctrl(dd, HFI1_RCVCTRL_CTXT_DIS |
101 		     HFI1_RCVCTRL_TIDFLOW_DIS |
102 		     HFI1_RCVCTRL_INTRAVAIL_DIS |
103 		     HFI1_RCVCTRL_ONE_PKT_EGR_DIS |
104 		     HFI1_RCVCTRL_NO_RHQ_DROP_DIS |
105 		     HFI1_RCVCTRL_NO_EGR_DROP_DIS, uctxt);
106 
107 	if (uctxt->msix_intr != CCE_NUM_MSIX_VECTORS)
108 		msix_free_irq(dd, uctxt->msix_intr);
109 
110 	uctxt->msix_intr = CCE_NUM_MSIX_VECTORS;
111 	uctxt->event_flags = 0;
112 
113 	hfi1_clear_tids(uctxt);
114 	hfi1_clear_ctxt_pkey(dd, uctxt);
115 
116 	hfi1_stats.sps_ctxts--;
117 
118 	hfi1_free_ctxt(uctxt);
119 }
120 
121 static int hfi1_netdev_allot_ctxt(struct hfi1_netdev_priv *priv,
122 				  struct hfi1_ctxtdata **ctxt)
123 {
124 	int rc;
125 	struct hfi1_devdata *dd = priv->dd;
126 
127 	rc = hfi1_netdev_allocate_ctxt(dd, ctxt);
128 	if (rc) {
129 		dd_dev_err(dd, "netdev ctxt alloc failed %d\n", rc);
130 		return rc;
131 	}
132 
133 	rc = hfi1_netdev_setup_ctxt(priv, *ctxt);
134 	if (rc) {
135 		dd_dev_err(dd, "netdev ctxt setup failed %d\n", rc);
136 		hfi1_netdev_deallocate_ctxt(dd, *ctxt);
137 		*ctxt = NULL;
138 	}
139 
140 	return rc;
141 }
142 
143 /**
144  * hfi1_num_netdev_contexts - Count of netdev recv contexts to use.
145  * @dd: device on which to allocate netdev contexts
146  * @available_contexts: count of available receive contexts
147  * @cpu_mask: mask of possible cpus to include for contexts
148  *
149  * Return: count of physical cores on a node or the remaining available recv
150  * contexts for netdev recv context usage up to the maximum of
151  * HFI1_MAX_NETDEV_CTXTS.
152  * A value of 0 can be returned when acceleration is explicitly turned off,
153  * a memory allocation error occurs or when there are no available contexts.
154  *
155  */
156 u32 hfi1_num_netdev_contexts(struct hfi1_devdata *dd, u32 available_contexts,
157 			     struct cpumask *cpu_mask)
158 {
159 	cpumask_var_t node_cpu_mask;
160 	unsigned int available_cpus;
161 
162 	if (!HFI1_CAP_IS_KSET(AIP))
163 		return 0;
164 
165 	/* Always give user contexts priority over netdev contexts */
166 	if (available_contexts == 0) {
167 		dd_dev_info(dd, "No receive contexts available for netdevs.\n");
168 		return 0;
169 	}
170 
171 	if (!zalloc_cpumask_var(&node_cpu_mask, GFP_KERNEL)) {
172 		dd_dev_err(dd, "Unable to allocate cpu_mask for netdevs.\n");
173 		return 0;
174 	}
175 
176 	cpumask_and(node_cpu_mask, cpu_mask,
177 		    cpumask_of_node(pcibus_to_node(dd->pcidev->bus)));
178 
179 	available_cpus = cpumask_weight(node_cpu_mask);
180 
181 	free_cpumask_var(node_cpu_mask);
182 
183 	return min3(available_cpus, available_contexts,
184 		    (u32)HFI1_MAX_NETDEV_CTXTS);
185 }
186 
187 static int hfi1_netdev_rxq_init(struct net_device *dev)
188 {
189 	int i;
190 	int rc;
191 	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev);
192 	struct hfi1_devdata *dd = priv->dd;
193 
194 	priv->num_rx_q = dd->num_netdev_contexts;
195 	priv->rxq = kcalloc_node(priv->num_rx_q, sizeof(struct hfi1_netdev_rxq),
196 				 GFP_KERNEL, dd->node);
197 
198 	if (!priv->rxq) {
199 		dd_dev_err(dd, "Unable to allocate netdev queue data\n");
200 		return (-ENOMEM);
201 	}
202 
203 	for (i = 0; i < priv->num_rx_q; i++) {
204 		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
205 
206 		rc = hfi1_netdev_allot_ctxt(priv, &rxq->rcd);
207 		if (rc)
208 			goto bail_context_irq_failure;
209 
210 		hfi1_rcd_get(rxq->rcd);
211 		rxq->priv = priv;
212 		rxq->rcd->napi = &rxq->napi;
213 		dd_dev_info(dd, "Setting rcv queue %d napi to context %d\n",
214 			    i, rxq->rcd->ctxt);
215 		/*
216 		 * Disable BUSY_POLL on this NAPI as this is not supported
217 		 * right now.
218 		 */
219 		set_bit(NAPI_STATE_NO_BUSY_POLL, &rxq->napi.state);
220 		netif_napi_add(dev, &rxq->napi, hfi1_netdev_rx_napi, 64);
221 		rc = msix_netdev_request_rcd_irq(rxq->rcd);
222 		if (rc)
223 			goto bail_context_irq_failure;
224 	}
225 
226 	return 0;
227 
228 bail_context_irq_failure:
229 	dd_dev_err(dd, "Unable to allot receive context\n");
230 	for (; i >= 0; i--) {
231 		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
232 
233 		if (rxq->rcd) {
234 			hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
235 			hfi1_rcd_put(rxq->rcd);
236 			rxq->rcd = NULL;
237 		}
238 	}
239 	kfree(priv->rxq);
240 	priv->rxq = NULL;
241 
242 	return rc;
243 }
244 
245 static void hfi1_netdev_rxq_deinit(struct net_device *dev)
246 {
247 	int i;
248 	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dev);
249 	struct hfi1_devdata *dd = priv->dd;
250 
251 	for (i = 0; i < priv->num_rx_q; i++) {
252 		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
253 
254 		netif_napi_del(&rxq->napi);
255 		hfi1_netdev_deallocate_ctxt(dd, rxq->rcd);
256 		hfi1_rcd_put(rxq->rcd);
257 		rxq->rcd = NULL;
258 	}
259 
260 	kfree(priv->rxq);
261 	priv->rxq = NULL;
262 	priv->num_rx_q = 0;
263 }
264 
265 static void enable_queues(struct hfi1_netdev_priv *priv)
266 {
267 	int i;
268 
269 	for (i = 0; i < priv->num_rx_q; i++) {
270 		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
271 
272 		dd_dev_info(priv->dd, "enabling queue %d on context %d\n", i,
273 			    rxq->rcd->ctxt);
274 		napi_enable(&rxq->napi);
275 		hfi1_rcvctrl(priv->dd,
276 			     HFI1_RCVCTRL_CTXT_ENB | HFI1_RCVCTRL_INTRAVAIL_ENB,
277 			     rxq->rcd);
278 	}
279 }
280 
281 static void disable_queues(struct hfi1_netdev_priv *priv)
282 {
283 	int i;
284 
285 	msix_netdev_synchronize_irq(priv->dd);
286 
287 	for (i = 0; i < priv->num_rx_q; i++) {
288 		struct hfi1_netdev_rxq *rxq = &priv->rxq[i];
289 
290 		dd_dev_info(priv->dd, "disabling queue %d on context %d\n", i,
291 			    rxq->rcd->ctxt);
292 
293 		/* wait for napi if it was scheduled */
294 		hfi1_rcvctrl(priv->dd,
295 			     HFI1_RCVCTRL_CTXT_DIS | HFI1_RCVCTRL_INTRAVAIL_DIS,
296 			     rxq->rcd);
297 		napi_synchronize(&rxq->napi);
298 		napi_disable(&rxq->napi);
299 	}
300 }
301 
302 /**
303  * hfi1_netdev_rx_init - Incrememnts netdevs counter. When called first time,
304  * it allocates receive queue data and calls netif_napi_add
305  * for each queue.
306  *
307  * @dd: hfi1 dev data
308  */
309 int hfi1_netdev_rx_init(struct hfi1_devdata *dd)
310 {
311 	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
312 	int res;
313 
314 	if (atomic_fetch_inc(&priv->netdevs))
315 		return 0;
316 
317 	mutex_lock(&hfi1_mutex);
318 	init_dummy_netdev(dd->dummy_netdev);
319 	res = hfi1_netdev_rxq_init(dd->dummy_netdev);
320 	mutex_unlock(&hfi1_mutex);
321 	return res;
322 }
323 
324 /**
325  * hfi1_netdev_rx_destroy - Decrements netdevs counter, when it reaches 0
326  * napi is deleted and receive queses memory is freed.
327  *
328  * @dd: hfi1 dev data
329  */
330 int hfi1_netdev_rx_destroy(struct hfi1_devdata *dd)
331 {
332 	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
333 
334 	/* destroy the RX queues only if it is the last netdev going away */
335 	if (atomic_fetch_add_unless(&priv->netdevs, -1, 0) == 1) {
336 		mutex_lock(&hfi1_mutex);
337 		hfi1_netdev_rxq_deinit(dd->dummy_netdev);
338 		mutex_unlock(&hfi1_mutex);
339 	}
340 
341 	return 0;
342 }
343 
344 /**
345  * hfi1_netdev_alloc - Allocates netdev and private data. It is required
346  * because RMT index and MSI-X interrupt can be set only
347  * during driver initialization.
348  *
349  * @dd: hfi1 dev data
350  */
351 int hfi1_netdev_alloc(struct hfi1_devdata *dd)
352 {
353 	struct hfi1_netdev_priv *priv;
354 	const int netdev_size = sizeof(*dd->dummy_netdev) +
355 		sizeof(struct hfi1_netdev_priv);
356 
357 	dd_dev_info(dd, "allocating netdev size %d\n", netdev_size);
358 	dd->dummy_netdev = kcalloc_node(1, netdev_size, GFP_KERNEL, dd->node);
359 
360 	if (!dd->dummy_netdev)
361 		return -ENOMEM;
362 
363 	priv = hfi1_netdev_priv(dd->dummy_netdev);
364 	priv->dd = dd;
365 	xa_init(&priv->dev_tbl);
366 	atomic_set(&priv->enabled, 0);
367 	atomic_set(&priv->netdevs, 0);
368 
369 	return 0;
370 }
371 
372 void hfi1_netdev_free(struct hfi1_devdata *dd)
373 {
374 	if (dd->dummy_netdev) {
375 		dd_dev_info(dd, "hfi1 netdev freed\n");
376 		kfree(dd->dummy_netdev);
377 		dd->dummy_netdev = NULL;
378 	}
379 }
380 
381 /**
382  * hfi1_netdev_enable_queues - This is napi enable function.
383  * It enables napi objects associated with queues.
384  * When at least one device has called it it increments atomic counter.
385  * Disable function decrements counter and when it is 0,
386  * calls napi_disable for every queue.
387  *
388  * @dd: hfi1 dev data
389  */
390 void hfi1_netdev_enable_queues(struct hfi1_devdata *dd)
391 {
392 	struct hfi1_netdev_priv *priv;
393 
394 	if (!dd->dummy_netdev)
395 		return;
396 
397 	priv = hfi1_netdev_priv(dd->dummy_netdev);
398 	if (atomic_fetch_inc(&priv->enabled))
399 		return;
400 
401 	mutex_lock(&hfi1_mutex);
402 	enable_queues(priv);
403 	mutex_unlock(&hfi1_mutex);
404 }
405 
406 void hfi1_netdev_disable_queues(struct hfi1_devdata *dd)
407 {
408 	struct hfi1_netdev_priv *priv;
409 
410 	if (!dd->dummy_netdev)
411 		return;
412 
413 	priv = hfi1_netdev_priv(dd->dummy_netdev);
414 	if (atomic_dec_if_positive(&priv->enabled))
415 		return;
416 
417 	mutex_lock(&hfi1_mutex);
418 	disable_queues(priv);
419 	mutex_unlock(&hfi1_mutex);
420 }
421 
422 /**
423  * hfi1_netdev_add_data - Registers data with unique identifier
424  * to be requested later this is needed for VNIC and IPoIB VLANs
425  * implementations.
426  * This call is protected by mutex idr_lock.
427  *
428  * @dd: hfi1 dev data
429  * @id: requested integer id up to INT_MAX
430  * @data: data to be associated with index
431  */
432 int hfi1_netdev_add_data(struct hfi1_devdata *dd, int id, void *data)
433 {
434 	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
435 
436 	return xa_insert(&priv->dev_tbl, id, data, GFP_NOWAIT);
437 }
438 
439 /**
440  * hfi1_netdev_remove_data - Removes data with previously given id.
441  * Returns the reference to removed entry.
442  *
443  * @dd: hfi1 dev data
444  * @id: requested integer id up to INT_MAX
445  */
446 void *hfi1_netdev_remove_data(struct hfi1_devdata *dd, int id)
447 {
448 	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
449 
450 	return xa_erase(&priv->dev_tbl, id);
451 }
452 
453 /**
454  * hfi1_netdev_get_data - Gets data with given id
455  *
456  * @dd: hfi1 dev data
457  * @id: requested integer id up to INT_MAX
458  */
459 void *hfi1_netdev_get_data(struct hfi1_devdata *dd, int id)
460 {
461 	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
462 
463 	return xa_load(&priv->dev_tbl, id);
464 }
465 
466 /**
467  * hfi1_netdev_get_first_dat - Gets first entry with greater or equal id.
468  *
469  * @dd: hfi1 dev data
470  * @id: requested integer id up to INT_MAX
471  */
472 void *hfi1_netdev_get_first_data(struct hfi1_devdata *dd, int *start_id)
473 {
474 	struct hfi1_netdev_priv *priv = hfi1_netdev_priv(dd->dummy_netdev);
475 	unsigned long index = *start_id;
476 	void *ret;
477 
478 	ret = xa_find(&priv->dev_tbl, &index, UINT_MAX, XA_PRESENT);
479 	*start_id = (int)index;
480 	return ret;
481 }
482