1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 
3 #include <linux/bpf.h>
4 #include <linux/crash_dump.h>
5 #include <linux/etherdevice.h>
6 #include <linux/ethtool.h>
7 #include <linux/filter.h>
8 #include <linux/idr.h>
9 #include <linux/if_vlan.h>
10 #include <linux/module.h>
11 #include <linux/netdevice.h>
12 #include <linux/pci.h>
13 #include <linux/rtnetlink.h>
14 #include <linux/inetdevice.h>
15 
16 #include "funeth.h"
17 #include "funeth_devlink.h"
18 #include "funeth_ktls.h"
19 #include "fun_port.h"
20 #include "fun_queue.h"
21 #include "funeth_txrx.h"
22 
23 #define ADMIN_SQ_DEPTH 32
24 #define ADMIN_CQ_DEPTH 64
25 #define ADMIN_RQ_DEPTH 16
26 
27 /* Default number of Tx/Rx queues. */
28 #define FUN_DFLT_QUEUES 16U
29 
30 enum {
31 	FUN_SERV_RES_CHANGE = FUN_SERV_FIRST_AVAIL,
32 	FUN_SERV_DEL_PORTS,
33 };
34 
35 static const struct pci_device_id funeth_id_table[] = {
36 	{ PCI_VDEVICE(FUNGIBLE, 0x0101) },
37 	{ PCI_VDEVICE(FUNGIBLE, 0x0181) },
38 	{ 0, }
39 };
40 
41 /* Issue a port write admin command with @n key/value pairs. */
42 static int fun_port_write_cmds(struct funeth_priv *fp, unsigned int n,
43 			       const int *keys, const u64 *data)
44 {
45 	unsigned int cmd_size, i;
46 	union {
47 		struct fun_admin_port_req req;
48 		struct fun_admin_port_rsp rsp;
49 		u8 v[ADMIN_SQE_SIZE];
50 	} cmd;
51 
52 	cmd_size = offsetof(struct fun_admin_port_req, u.write.write48) +
53 		n * sizeof(struct fun_admin_write48_req);
54 	if (cmd_size > sizeof(cmd) || cmd_size > ADMIN_RSP_MAX_LEN)
55 		return -EINVAL;
56 
57 	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
58 						    cmd_size);
59 	cmd.req.u.write =
60 		FUN_ADMIN_PORT_WRITE_REQ_INIT(FUN_ADMIN_SUBOP_WRITE, 0,
61 					      fp->netdev->dev_port);
62 	for (i = 0; i < n; i++)
63 		cmd.req.u.write.write48[i] =
64 			FUN_ADMIN_WRITE48_REQ_INIT(keys[i], data[i]);
65 
66 	return fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
67 					 &cmd.rsp, cmd_size, 0);
68 }
69 
70 int fun_port_write_cmd(struct funeth_priv *fp, int key, u64 data)
71 {
72 	return fun_port_write_cmds(fp, 1, &key, &data);
73 }
74 
75 /* Issue a port read admin command with @n key/value pairs. */
76 static int fun_port_read_cmds(struct funeth_priv *fp, unsigned int n,
77 			      const int *keys, u64 *data)
78 {
79 	const struct fun_admin_read48_rsp *r48rsp;
80 	unsigned int cmd_size, i;
81 	int rc;
82 	union {
83 		struct fun_admin_port_req req;
84 		struct fun_admin_port_rsp rsp;
85 		u8 v[ADMIN_SQE_SIZE];
86 	} cmd;
87 
88 	cmd_size = offsetof(struct fun_admin_port_req, u.read.read48) +
89 		n * sizeof(struct fun_admin_read48_req);
90 	if (cmd_size > sizeof(cmd) || cmd_size > ADMIN_RSP_MAX_LEN)
91 		return -EINVAL;
92 
93 	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
94 						    cmd_size);
95 	cmd.req.u.read =
96 		FUN_ADMIN_PORT_READ_REQ_INIT(FUN_ADMIN_SUBOP_READ, 0,
97 					     fp->netdev->dev_port);
98 	for (i = 0; i < n; i++)
99 		cmd.req.u.read.read48[i] = FUN_ADMIN_READ48_REQ_INIT(keys[i]);
100 
101 	rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
102 				       &cmd.rsp, cmd_size, 0);
103 	if (rc)
104 		return rc;
105 
106 	for (r48rsp = cmd.rsp.u.read.read48, i = 0; i < n; i++, r48rsp++) {
107 		data[i] = FUN_ADMIN_READ48_RSP_DATA_G(r48rsp->key_to_data);
108 		dev_dbg(fp->fdev->dev,
109 			"port_read_rsp lport=%u (key_to_data=0x%llx) key=%d data:%lld retval:%lld",
110 			fp->lport, r48rsp->key_to_data, keys[i], data[i],
111 			FUN_ADMIN_READ48_RSP_RET_G(r48rsp->key_to_data));
112 	}
113 	return 0;
114 }
115 
116 int fun_port_read_cmd(struct funeth_priv *fp, int key, u64 *data)
117 {
118 	return fun_port_read_cmds(fp, 1, &key, data);
119 }
120 
121 static void fun_report_link(struct net_device *netdev)
122 {
123 	if (netif_carrier_ok(netdev)) {
124 		const struct funeth_priv *fp = netdev_priv(netdev);
125 		const char *fec = "", *pause = "";
126 		int speed = fp->link_speed;
127 		char unit = 'M';
128 
129 		if (fp->link_speed >= SPEED_1000) {
130 			speed /= 1000;
131 			unit = 'G';
132 		}
133 
134 		if (fp->active_fec & FUN_PORT_FEC_RS)
135 			fec = ", RS-FEC";
136 		else if (fp->active_fec & FUN_PORT_FEC_FC)
137 			fec = ", BASER-FEC";
138 
139 		if ((fp->active_fc & FUN_PORT_CAP_PAUSE_MASK) == FUN_PORT_CAP_PAUSE_MASK)
140 			pause = ", Tx/Rx PAUSE";
141 		else if (fp->active_fc & FUN_PORT_CAP_RX_PAUSE)
142 			pause = ", Rx PAUSE";
143 		else if (fp->active_fc & FUN_PORT_CAP_TX_PAUSE)
144 			pause = ", Tx PAUSE";
145 
146 		netdev_info(netdev, "Link up at %d %cb/s full-duplex%s%s\n",
147 			    speed, unit, pause, fec);
148 	} else {
149 		netdev_info(netdev, "Link down\n");
150 	}
151 }
152 
153 static int fun_adi_write(struct fun_dev *fdev, enum fun_admin_adi_attr attr,
154 			 unsigned int adi_id, const struct fun_adi_param *param)
155 {
156 	struct fun_admin_adi_req req = {
157 		.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_ADI,
158 						     sizeof(req)),
159 		.u.write.subop = FUN_ADMIN_SUBOP_WRITE,
160 		.u.write.attribute = attr,
161 		.u.write.id = cpu_to_be32(adi_id),
162 		.u.write.param = *param
163 	};
164 
165 	return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0);
166 }
167 
168 /* Configure RSS for the given port. @op determines whether a new RSS context
169  * is to be created or whether an existing one should be reconfigured. The
170  * remaining parameters specify the hashing algorithm, key, and indirection
171  * table.
172  *
173  * This initiates packet delivery to the Rx queues set in the indirection
174  * table.
175  */
176 int fun_config_rss(struct net_device *dev, int algo, const u8 *key,
177 		   const u32 *qtable, u8 op)
178 {
179 	struct funeth_priv *fp = netdev_priv(dev);
180 	unsigned int table_len = fp->indir_table_nentries;
181 	unsigned int len = FUN_ETH_RSS_MAX_KEY_SIZE + sizeof(u32) * table_len;
182 	struct funeth_rxq **rxqs = rtnl_dereference(fp->rxqs);
183 	union {
184 		struct {
185 			struct fun_admin_rss_req req;
186 			struct fun_dataop_gl gl;
187 		};
188 		struct fun_admin_generic_create_rsp rsp;
189 	} cmd;
190 	__be32 *indir_tab;
191 	u16 flags;
192 	int rc;
193 
194 	if (op != FUN_ADMIN_SUBOP_CREATE && fp->rss_hw_id == FUN_HCI_ID_INVALID)
195 		return -EINVAL;
196 
197 	flags = op == FUN_ADMIN_SUBOP_CREATE ?
198 			FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR : 0;
199 	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_RSS,
200 						    sizeof(cmd));
201 	cmd.req.u.create =
202 		FUN_ADMIN_RSS_CREATE_REQ_INIT(op, flags, fp->rss_hw_id,
203 					      dev->dev_port, algo,
204 					      FUN_ETH_RSS_MAX_KEY_SIZE,
205 					      table_len, 0,
206 					      FUN_ETH_RSS_MAX_KEY_SIZE);
207 	cmd.req.u.create.dataop = FUN_DATAOP_HDR_INIT(1, 0, 1, 0, len);
208 	fun_dataop_gl_init(&cmd.gl, 0, 0, len, fp->rss_dma_addr);
209 
210 	/* write the key and indirection table into the RSS DMA area */
211 	memcpy(fp->rss_cfg, key, FUN_ETH_RSS_MAX_KEY_SIZE);
212 	indir_tab = fp->rss_cfg + FUN_ETH_RSS_MAX_KEY_SIZE;
213 	for (rc = 0; rc < table_len; rc++)
214 		*indir_tab++ = cpu_to_be32(rxqs[*qtable++]->hw_cqid);
215 
216 	rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common,
217 				       &cmd.rsp, sizeof(cmd.rsp), 0);
218 	if (!rc && op == FUN_ADMIN_SUBOP_CREATE)
219 		fp->rss_hw_id = be32_to_cpu(cmd.rsp.id);
220 	return rc;
221 }
222 
223 /* Destroy the HW RSS conntext associated with the given port. This also stops
224  * all packet delivery to our Rx queues.
225  */
226 static void fun_destroy_rss(struct funeth_priv *fp)
227 {
228 	if (fp->rss_hw_id != FUN_HCI_ID_INVALID) {
229 		fun_res_destroy(fp->fdev, FUN_ADMIN_OP_RSS, 0, fp->rss_hw_id);
230 		fp->rss_hw_id = FUN_HCI_ID_INVALID;
231 	}
232 }
233 
234 static void fun_irq_aff_notify(struct irq_affinity_notify *notify,
235 			       const cpumask_t *mask)
236 {
237 	struct fun_irq *p = container_of(notify, struct fun_irq, aff_notify);
238 
239 	cpumask_copy(&p->affinity_mask, mask);
240 }
241 
242 static void fun_irq_aff_release(struct kref __always_unused *ref)
243 {
244 }
245 
246 /* Allocate an IRQ structure, assign an MSI-X index and initial affinity to it,
247  * and add it to the IRQ XArray.
248  */
249 static struct fun_irq *fun_alloc_qirq(struct funeth_priv *fp, unsigned int idx,
250 				      int node, unsigned int xa_idx_offset)
251 {
252 	struct fun_irq *irq;
253 	int cpu, res;
254 
255 	cpu = cpumask_local_spread(idx, node);
256 	node = cpu_to_mem(cpu);
257 
258 	irq = kzalloc_node(sizeof(*irq), GFP_KERNEL, node);
259 	if (!irq)
260 		return ERR_PTR(-ENOMEM);
261 
262 	res = fun_reserve_irqs(fp->fdev, 1, &irq->irq_idx);
263 	if (res != 1)
264 		goto free_irq;
265 
266 	res = xa_insert(&fp->irqs, idx + xa_idx_offset, irq, GFP_KERNEL);
267 	if (res)
268 		goto release_irq;
269 
270 	irq->irq = pci_irq_vector(fp->pdev, irq->irq_idx);
271 	cpumask_set_cpu(cpu, &irq->affinity_mask);
272 	irq->aff_notify.notify = fun_irq_aff_notify;
273 	irq->aff_notify.release = fun_irq_aff_release;
274 	irq->state = FUN_IRQ_INIT;
275 	return irq;
276 
277 release_irq:
278 	fun_release_irqs(fp->fdev, 1, &irq->irq_idx);
279 free_irq:
280 	kfree(irq);
281 	return ERR_PTR(res);
282 }
283 
284 static void fun_free_qirq(struct funeth_priv *fp, struct fun_irq *irq)
285 {
286 	netif_napi_del(&irq->napi);
287 	fun_release_irqs(fp->fdev, 1, &irq->irq_idx);
288 	kfree(irq);
289 }
290 
291 /* Release the IRQs reserved for Tx/Rx queues that aren't being used. */
292 static void fun_prune_queue_irqs(struct net_device *dev)
293 {
294 	struct funeth_priv *fp = netdev_priv(dev);
295 	unsigned int nreleased = 0;
296 	struct fun_irq *irq;
297 	unsigned long idx;
298 
299 	xa_for_each(&fp->irqs, idx, irq) {
300 		if (irq->txq || irq->rxq)  /* skip those in use */
301 			continue;
302 
303 		xa_erase(&fp->irqs, idx);
304 		fun_free_qirq(fp, irq);
305 		nreleased++;
306 		if (idx < fp->rx_irq_ofst)
307 			fp->num_tx_irqs--;
308 		else
309 			fp->num_rx_irqs--;
310 	}
311 	netif_info(fp, intr, dev, "Released %u queue IRQs\n", nreleased);
312 }
313 
314 /* Reserve IRQs, one per queue, to acommodate the requested queue numbers @ntx
315  * and @nrx. IRQs are added incrementally to those we already have.
316  * We hold on to allocated IRQs until garbage collection of unused IRQs is
317  * separately requested.
318  */
319 static int fun_alloc_queue_irqs(struct net_device *dev, unsigned int ntx,
320 				unsigned int nrx)
321 {
322 	struct funeth_priv *fp = netdev_priv(dev);
323 	int node = dev_to_node(&fp->pdev->dev);
324 	struct fun_irq *irq;
325 	unsigned int i;
326 
327 	for (i = fp->num_tx_irqs; i < ntx; i++) {
328 		irq = fun_alloc_qirq(fp, i, node, 0);
329 		if (IS_ERR(irq))
330 			return PTR_ERR(irq);
331 
332 		fp->num_tx_irqs++;
333 		netif_napi_add_tx(dev, &irq->napi, fun_txq_napi_poll);
334 	}
335 
336 	for (i = fp->num_rx_irqs; i < nrx; i++) {
337 		irq = fun_alloc_qirq(fp, i, node, fp->rx_irq_ofst);
338 		if (IS_ERR(irq))
339 			return PTR_ERR(irq);
340 
341 		fp->num_rx_irqs++;
342 		netif_napi_add(dev, &irq->napi, fun_rxq_napi_poll);
343 	}
344 
345 	netif_info(fp, intr, dev, "Reserved %u/%u IRQs for Tx/Rx queues\n",
346 		   ntx, nrx);
347 	return 0;
348 }
349 
350 static void free_txqs(struct funeth_txq **txqs, unsigned int nqs,
351 		      unsigned int start, int state)
352 {
353 	unsigned int i;
354 
355 	for (i = start; i < nqs && txqs[i]; i++)
356 		txqs[i] = funeth_txq_free(txqs[i], state);
357 }
358 
359 static int alloc_txqs(struct net_device *dev, struct funeth_txq **txqs,
360 		      unsigned int nqs, unsigned int depth, unsigned int start,
361 		      int state)
362 {
363 	struct funeth_priv *fp = netdev_priv(dev);
364 	unsigned int i;
365 	int err;
366 
367 	for (i = start; i < nqs; i++) {
368 		err = funeth_txq_create(dev, i, depth, xa_load(&fp->irqs, i),
369 					state, &txqs[i]);
370 		if (err) {
371 			free_txqs(txqs, nqs, start, FUN_QSTATE_DESTROYED);
372 			return err;
373 		}
374 	}
375 	return 0;
376 }
377 
378 static void free_rxqs(struct funeth_rxq **rxqs, unsigned int nqs,
379 		      unsigned int start, int state)
380 {
381 	unsigned int i;
382 
383 	for (i = start; i < nqs && rxqs[i]; i++)
384 		rxqs[i] = funeth_rxq_free(rxqs[i], state);
385 }
386 
387 static int alloc_rxqs(struct net_device *dev, struct funeth_rxq **rxqs,
388 		      unsigned int nqs, unsigned int ncqe, unsigned int nrqe,
389 		      unsigned int start, int state)
390 {
391 	struct funeth_priv *fp = netdev_priv(dev);
392 	unsigned int i;
393 	int err;
394 
395 	for (i = start; i < nqs; i++) {
396 		err = funeth_rxq_create(dev, i, ncqe, nrqe,
397 					xa_load(&fp->irqs, i + fp->rx_irq_ofst),
398 					state, &rxqs[i]);
399 		if (err) {
400 			free_rxqs(rxqs, nqs, start, FUN_QSTATE_DESTROYED);
401 			return err;
402 		}
403 	}
404 	return 0;
405 }
406 
407 static void free_xdpqs(struct funeth_txq **xdpqs, unsigned int nqs,
408 		       unsigned int start, int state)
409 {
410 	unsigned int i;
411 
412 	for (i = start; i < nqs && xdpqs[i]; i++)
413 		xdpqs[i] = funeth_txq_free(xdpqs[i], state);
414 
415 	if (state == FUN_QSTATE_DESTROYED)
416 		kfree(xdpqs);
417 }
418 
419 static struct funeth_txq **alloc_xdpqs(struct net_device *dev, unsigned int nqs,
420 				       unsigned int depth, unsigned int start,
421 				       int state)
422 {
423 	struct funeth_txq **xdpqs;
424 	unsigned int i;
425 	int err;
426 
427 	xdpqs = kcalloc(nqs, sizeof(*xdpqs), GFP_KERNEL);
428 	if (!xdpqs)
429 		return ERR_PTR(-ENOMEM);
430 
431 	for (i = start; i < nqs; i++) {
432 		err = funeth_txq_create(dev, i, depth, NULL, state, &xdpqs[i]);
433 		if (err) {
434 			free_xdpqs(xdpqs, nqs, start, FUN_QSTATE_DESTROYED);
435 			return ERR_PTR(err);
436 		}
437 	}
438 	return xdpqs;
439 }
440 
441 static void fun_free_rings(struct net_device *netdev, struct fun_qset *qset)
442 {
443 	struct funeth_priv *fp = netdev_priv(netdev);
444 	struct funeth_txq **xdpqs = qset->xdpqs;
445 	struct funeth_rxq **rxqs = qset->rxqs;
446 
447 	/* qset may not specify any queues to operate on. In that case the
448 	 * currently installed queues are implied.
449 	 */
450 	if (!rxqs) {
451 		rxqs = rtnl_dereference(fp->rxqs);
452 		xdpqs = rtnl_dereference(fp->xdpqs);
453 		qset->txqs = fp->txqs;
454 		qset->nrxqs = netdev->real_num_rx_queues;
455 		qset->ntxqs = netdev->real_num_tx_queues;
456 		qset->nxdpqs = fp->num_xdpqs;
457 	}
458 	if (!rxqs)
459 		return;
460 
461 	if (rxqs == rtnl_dereference(fp->rxqs)) {
462 		rcu_assign_pointer(fp->rxqs, NULL);
463 		rcu_assign_pointer(fp->xdpqs, NULL);
464 		synchronize_net();
465 		fp->txqs = NULL;
466 	}
467 
468 	free_rxqs(rxqs, qset->nrxqs, qset->rxq_start, qset->state);
469 	free_txqs(qset->txqs, qset->ntxqs, qset->txq_start, qset->state);
470 	free_xdpqs(xdpqs, qset->nxdpqs, qset->xdpq_start, qset->state);
471 	if (qset->state == FUN_QSTATE_DESTROYED)
472 		kfree(rxqs);
473 
474 	/* Tell the caller which queues were operated on. */
475 	qset->rxqs = rxqs;
476 	qset->xdpqs = xdpqs;
477 }
478 
479 static int fun_alloc_rings(struct net_device *netdev, struct fun_qset *qset)
480 {
481 	struct funeth_txq **xdpqs = NULL, **txqs;
482 	struct funeth_rxq **rxqs;
483 	int err;
484 
485 	err = fun_alloc_queue_irqs(netdev, qset->ntxqs, qset->nrxqs);
486 	if (err)
487 		return err;
488 
489 	rxqs = kcalloc(qset->ntxqs + qset->nrxqs, sizeof(*rxqs), GFP_KERNEL);
490 	if (!rxqs)
491 		return -ENOMEM;
492 
493 	if (qset->nxdpqs) {
494 		xdpqs = alloc_xdpqs(netdev, qset->nxdpqs, qset->sq_depth,
495 				    qset->xdpq_start, qset->state);
496 		if (IS_ERR(xdpqs)) {
497 			err = PTR_ERR(xdpqs);
498 			goto free_qvec;
499 		}
500 	}
501 
502 	txqs = (struct funeth_txq **)&rxqs[qset->nrxqs];
503 	err = alloc_txqs(netdev, txqs, qset->ntxqs, qset->sq_depth,
504 			 qset->txq_start, qset->state);
505 	if (err)
506 		goto free_xdpqs;
507 
508 	err = alloc_rxqs(netdev, rxqs, qset->nrxqs, qset->cq_depth,
509 			 qset->rq_depth, qset->rxq_start, qset->state);
510 	if (err)
511 		goto free_txqs;
512 
513 	qset->rxqs = rxqs;
514 	qset->txqs = txqs;
515 	qset->xdpqs = xdpqs;
516 	return 0;
517 
518 free_txqs:
519 	free_txqs(txqs, qset->ntxqs, qset->txq_start, FUN_QSTATE_DESTROYED);
520 free_xdpqs:
521 	free_xdpqs(xdpqs, qset->nxdpqs, qset->xdpq_start, FUN_QSTATE_DESTROYED);
522 free_qvec:
523 	kfree(rxqs);
524 	return err;
525 }
526 
527 /* Take queues to the next level. Presently this means creating them on the
528  * device.
529  */
530 static int fun_advance_ring_state(struct net_device *dev, struct fun_qset *qset)
531 {
532 	struct funeth_priv *fp = netdev_priv(dev);
533 	int i, err;
534 
535 	for (i = 0; i < qset->nrxqs; i++) {
536 		err = fun_rxq_create_dev(qset->rxqs[i],
537 					 xa_load(&fp->irqs,
538 						 i + fp->rx_irq_ofst));
539 		if (err)
540 			goto out;
541 	}
542 
543 	for (i = 0; i < qset->ntxqs; i++) {
544 		err = fun_txq_create_dev(qset->txqs[i], xa_load(&fp->irqs, i));
545 		if (err)
546 			goto out;
547 	}
548 
549 	for (i = 0; i < qset->nxdpqs; i++) {
550 		err = fun_txq_create_dev(qset->xdpqs[i], NULL);
551 		if (err)
552 			goto out;
553 	}
554 
555 	return 0;
556 
557 out:
558 	fun_free_rings(dev, qset);
559 	return err;
560 }
561 
562 static int fun_port_create(struct net_device *netdev)
563 {
564 	struct funeth_priv *fp = netdev_priv(netdev);
565 	union {
566 		struct fun_admin_port_req req;
567 		struct fun_admin_port_rsp rsp;
568 	} cmd;
569 	int rc;
570 
571 	if (fp->lport != INVALID_LPORT)
572 		return 0;
573 
574 	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_PORT,
575 						    sizeof(cmd.req));
576 	cmd.req.u.create =
577 		FUN_ADMIN_PORT_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, 0,
578 					       netdev->dev_port);
579 
580 	rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common, &cmd.rsp,
581 				       sizeof(cmd.rsp), 0);
582 
583 	if (!rc)
584 		fp->lport = be16_to_cpu(cmd.rsp.u.create.lport);
585 	return rc;
586 }
587 
588 static int fun_port_destroy(struct net_device *netdev)
589 {
590 	struct funeth_priv *fp = netdev_priv(netdev);
591 
592 	if (fp->lport == INVALID_LPORT)
593 		return 0;
594 
595 	fp->lport = INVALID_LPORT;
596 	return fun_res_destroy(fp->fdev, FUN_ADMIN_OP_PORT, 0,
597 			       netdev->dev_port);
598 }
599 
600 static int fun_eth_create(struct funeth_priv *fp)
601 {
602 	union {
603 		struct fun_admin_eth_req req;
604 		struct fun_admin_generic_create_rsp rsp;
605 	} cmd;
606 	int rc;
607 
608 	cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_ETH,
609 						    sizeof(cmd.req));
610 	cmd.req.u.create = FUN_ADMIN_ETH_CREATE_REQ_INIT(
611 				FUN_ADMIN_SUBOP_CREATE,
612 				FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR,
613 				0, fp->netdev->dev_port);
614 
615 	rc = fun_submit_admin_sync_cmd(fp->fdev, &cmd.req.common, &cmd.rsp,
616 				       sizeof(cmd.rsp), 0);
617 	return rc ? rc : be32_to_cpu(cmd.rsp.id);
618 }
619 
620 static int fun_vi_create(struct funeth_priv *fp)
621 {
622 	struct fun_admin_vi_req req = {
623 		.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_VI,
624 						     sizeof(req)),
625 		.u.create = FUN_ADMIN_VI_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE,
626 							 0,
627 							 fp->netdev->dev_port,
628 							 fp->netdev->dev_port)
629 	};
630 
631 	return fun_submit_admin_sync_cmd(fp->fdev, &req.common, NULL, 0, 0);
632 }
633 
634 /* Helper to create an ETH flow and bind an SQ to it.
635  * Returns the ETH id (>= 0) on success or a negative error.
636  */
637 int fun_create_and_bind_tx(struct funeth_priv *fp, u32 sqid)
638 {
639 	int rc, ethid;
640 
641 	ethid = fun_eth_create(fp);
642 	if (ethid >= 0) {
643 		rc = fun_bind(fp->fdev, FUN_ADMIN_BIND_TYPE_EPSQ, sqid,
644 			      FUN_ADMIN_BIND_TYPE_ETH, ethid);
645 		if (rc) {
646 			fun_res_destroy(fp->fdev, FUN_ADMIN_OP_ETH, 0, ethid);
647 			ethid = rc;
648 		}
649 	}
650 	return ethid;
651 }
652 
653 static irqreturn_t fun_queue_irq_handler(int irq, void *data)
654 {
655 	struct fun_irq *p = data;
656 
657 	if (p->rxq) {
658 		prefetch(p->rxq->next_cqe_info);
659 		p->rxq->irq_cnt++;
660 	}
661 	napi_schedule_irqoff(&p->napi);
662 	return IRQ_HANDLED;
663 }
664 
665 static int fun_enable_irqs(struct net_device *dev)
666 {
667 	struct funeth_priv *fp = netdev_priv(dev);
668 	unsigned long idx, last;
669 	unsigned int qidx;
670 	struct fun_irq *p;
671 	const char *qtype;
672 	int err;
673 
674 	xa_for_each(&fp->irqs, idx, p) {
675 		if (p->txq) {
676 			qtype = "tx";
677 			qidx = p->txq->qidx;
678 		} else if (p->rxq) {
679 			qtype = "rx";
680 			qidx = p->rxq->qidx;
681 		} else {
682 			continue;
683 		}
684 
685 		if (p->state != FUN_IRQ_INIT)
686 			continue;
687 
688 		snprintf(p->name, sizeof(p->name) - 1, "%s-%s-%u", dev->name,
689 			 qtype, qidx);
690 		err = request_irq(p->irq, fun_queue_irq_handler, 0, p->name, p);
691 		if (err) {
692 			netdev_err(dev, "Failed to allocate IRQ %u, err %d\n",
693 				   p->irq, err);
694 			goto unroll;
695 		}
696 		p->state = FUN_IRQ_REQUESTED;
697 	}
698 
699 	xa_for_each(&fp->irqs, idx, p) {
700 		if (p->state != FUN_IRQ_REQUESTED)
701 			continue;
702 		irq_set_affinity_notifier(p->irq, &p->aff_notify);
703 		irq_set_affinity_and_hint(p->irq, &p->affinity_mask);
704 		napi_enable(&p->napi);
705 		p->state = FUN_IRQ_ENABLED;
706 	}
707 
708 	return 0;
709 
710 unroll:
711 	last = idx - 1;
712 	xa_for_each_range(&fp->irqs, idx, p, 0, last)
713 		if (p->state == FUN_IRQ_REQUESTED) {
714 			free_irq(p->irq, p);
715 			p->state = FUN_IRQ_INIT;
716 		}
717 
718 	return err;
719 }
720 
721 static void fun_disable_one_irq(struct fun_irq *irq)
722 {
723 	napi_disable(&irq->napi);
724 	irq_set_affinity_notifier(irq->irq, NULL);
725 	irq_update_affinity_hint(irq->irq, NULL);
726 	free_irq(irq->irq, irq);
727 	irq->state = FUN_IRQ_INIT;
728 }
729 
730 static void fun_disable_irqs(struct net_device *dev)
731 {
732 	struct funeth_priv *fp = netdev_priv(dev);
733 	struct fun_irq *p;
734 	unsigned long idx;
735 
736 	xa_for_each(&fp->irqs, idx, p)
737 		if (p->state == FUN_IRQ_ENABLED)
738 			fun_disable_one_irq(p);
739 }
740 
741 static void fun_down(struct net_device *dev, struct fun_qset *qset)
742 {
743 	struct funeth_priv *fp = netdev_priv(dev);
744 
745 	/* If we don't have queues the data path is already down.
746 	 * Note netif_running(dev) may be true.
747 	 */
748 	if (!rcu_access_pointer(fp->rxqs))
749 		return;
750 
751 	/* It is also down if the queues aren't on the device. */
752 	if (fp->txqs[0]->init_state >= FUN_QSTATE_INIT_FULL) {
753 		netif_info(fp, ifdown, dev,
754 			   "Tearing down data path on device\n");
755 		fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_DISABLE, 0);
756 
757 		netif_carrier_off(dev);
758 		netif_tx_disable(dev);
759 
760 		fun_destroy_rss(fp);
761 		fun_res_destroy(fp->fdev, FUN_ADMIN_OP_VI, 0, dev->dev_port);
762 		fun_disable_irqs(dev);
763 	}
764 
765 	fun_free_rings(dev, qset);
766 }
767 
768 static int fun_up(struct net_device *dev, struct fun_qset *qset)
769 {
770 	static const int port_keys[] = {
771 		FUN_ADMIN_PORT_KEY_STATS_DMA_LOW,
772 		FUN_ADMIN_PORT_KEY_STATS_DMA_HIGH,
773 		FUN_ADMIN_PORT_KEY_ENABLE
774 	};
775 
776 	struct funeth_priv *fp = netdev_priv(dev);
777 	u64 vals[] = {
778 		lower_32_bits(fp->stats_dma_addr),
779 		upper_32_bits(fp->stats_dma_addr),
780 		FUN_PORT_FLAG_ENABLE_NOTIFY
781 	};
782 	int err;
783 
784 	netif_info(fp, ifup, dev, "Setting up data path on device\n");
785 
786 	if (qset->rxqs[0]->init_state < FUN_QSTATE_INIT_FULL) {
787 		err = fun_advance_ring_state(dev, qset);
788 		if (err)
789 			return err;
790 	}
791 
792 	err = fun_vi_create(fp);
793 	if (err)
794 		goto free_queues;
795 
796 	fp->txqs = qset->txqs;
797 	rcu_assign_pointer(fp->rxqs, qset->rxqs);
798 	rcu_assign_pointer(fp->xdpqs, qset->xdpqs);
799 
800 	err = fun_enable_irqs(dev);
801 	if (err)
802 		goto destroy_vi;
803 
804 	if (fp->rss_cfg) {
805 		err = fun_config_rss(dev, fp->hash_algo, fp->rss_key,
806 				     fp->indir_table, FUN_ADMIN_SUBOP_CREATE);
807 	} else {
808 		/* The non-RSS case has only 1 queue. */
809 		err = fun_bind(fp->fdev, FUN_ADMIN_BIND_TYPE_VI, dev->dev_port,
810 			       FUN_ADMIN_BIND_TYPE_EPCQ,
811 			       qset->rxqs[0]->hw_cqid);
812 	}
813 	if (err)
814 		goto disable_irqs;
815 
816 	err = fun_port_write_cmds(fp, 3, port_keys, vals);
817 	if (err)
818 		goto free_rss;
819 
820 	netif_tx_start_all_queues(dev);
821 	return 0;
822 
823 free_rss:
824 	fun_destroy_rss(fp);
825 disable_irqs:
826 	fun_disable_irqs(dev);
827 destroy_vi:
828 	fun_res_destroy(fp->fdev, FUN_ADMIN_OP_VI, 0, dev->dev_port);
829 free_queues:
830 	fun_free_rings(dev, qset);
831 	return err;
832 }
833 
834 static int funeth_open(struct net_device *netdev)
835 {
836 	struct funeth_priv *fp = netdev_priv(netdev);
837 	struct fun_qset qset = {
838 		.nrxqs = netdev->real_num_rx_queues,
839 		.ntxqs = netdev->real_num_tx_queues,
840 		.nxdpqs = fp->num_xdpqs,
841 		.cq_depth = fp->cq_depth,
842 		.rq_depth = fp->rq_depth,
843 		.sq_depth = fp->sq_depth,
844 		.state = FUN_QSTATE_INIT_FULL,
845 	};
846 	int rc;
847 
848 	rc = fun_alloc_rings(netdev, &qset);
849 	if (rc)
850 		return rc;
851 
852 	rc = fun_up(netdev, &qset);
853 	if (rc) {
854 		qset.state = FUN_QSTATE_DESTROYED;
855 		fun_free_rings(netdev, &qset);
856 	}
857 
858 	return rc;
859 }
860 
861 static int funeth_close(struct net_device *netdev)
862 {
863 	struct fun_qset qset = { .state = FUN_QSTATE_DESTROYED };
864 
865 	fun_down(netdev, &qset);
866 	return 0;
867 }
868 
869 static void fun_get_stats64(struct net_device *netdev,
870 			    struct rtnl_link_stats64 *stats)
871 {
872 	struct funeth_priv *fp = netdev_priv(netdev);
873 	struct funeth_txq **xdpqs;
874 	struct funeth_rxq **rxqs;
875 	unsigned int i, start;
876 
877 	stats->tx_packets = fp->tx_packets;
878 	stats->tx_bytes   = fp->tx_bytes;
879 	stats->tx_dropped = fp->tx_dropped;
880 
881 	stats->rx_packets = fp->rx_packets;
882 	stats->rx_bytes   = fp->rx_bytes;
883 	stats->rx_dropped = fp->rx_dropped;
884 
885 	rcu_read_lock();
886 	rxqs = rcu_dereference(fp->rxqs);
887 	if (!rxqs)
888 		goto unlock;
889 
890 	for (i = 0; i < netdev->real_num_tx_queues; i++) {
891 		struct funeth_txq_stats txs;
892 
893 		FUN_QSTAT_READ(fp->txqs[i], start, txs);
894 		stats->tx_packets += txs.tx_pkts;
895 		stats->tx_bytes   += txs.tx_bytes;
896 		stats->tx_dropped += txs.tx_map_err;
897 	}
898 
899 	for (i = 0; i < netdev->real_num_rx_queues; i++) {
900 		struct funeth_rxq_stats rxs;
901 
902 		FUN_QSTAT_READ(rxqs[i], start, rxs);
903 		stats->rx_packets += rxs.rx_pkts;
904 		stats->rx_bytes   += rxs.rx_bytes;
905 		stats->rx_dropped += rxs.rx_map_err + rxs.rx_mem_drops;
906 	}
907 
908 	xdpqs = rcu_dereference(fp->xdpqs);
909 	if (!xdpqs)
910 		goto unlock;
911 
912 	for (i = 0; i < fp->num_xdpqs; i++) {
913 		struct funeth_txq_stats txs;
914 
915 		FUN_QSTAT_READ(xdpqs[i], start, txs);
916 		stats->tx_packets += txs.tx_pkts;
917 		stats->tx_bytes   += txs.tx_bytes;
918 	}
919 unlock:
920 	rcu_read_unlock();
921 }
922 
923 static int fun_change_mtu(struct net_device *netdev, int new_mtu)
924 {
925 	struct funeth_priv *fp = netdev_priv(netdev);
926 	int rc;
927 
928 	rc = fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MTU, new_mtu);
929 	if (!rc)
930 		netdev->mtu = new_mtu;
931 	return rc;
932 }
933 
934 static int fun_set_macaddr(struct net_device *netdev, void *addr)
935 {
936 	struct funeth_priv *fp = netdev_priv(netdev);
937 	struct sockaddr *saddr = addr;
938 	int rc;
939 
940 	if (!is_valid_ether_addr(saddr->sa_data))
941 		return -EADDRNOTAVAIL;
942 
943 	if (ether_addr_equal(netdev->dev_addr, saddr->sa_data))
944 		return 0;
945 
946 	rc = fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MACADDR,
947 				ether_addr_to_u64(saddr->sa_data));
948 	if (!rc)
949 		eth_hw_addr_set(netdev, saddr->sa_data);
950 	return rc;
951 }
952 
953 static int fun_get_port_attributes(struct net_device *netdev)
954 {
955 	static const int keys[] = {
956 		FUN_ADMIN_PORT_KEY_MACADDR, FUN_ADMIN_PORT_KEY_CAPABILITIES,
957 		FUN_ADMIN_PORT_KEY_ADVERT, FUN_ADMIN_PORT_KEY_MTU
958 	};
959 	static const int phys_keys[] = {
960 		FUN_ADMIN_PORT_KEY_LANE_ATTRS,
961 	};
962 
963 	struct funeth_priv *fp = netdev_priv(netdev);
964 	u64 data[ARRAY_SIZE(keys)];
965 	u8 mac[ETH_ALEN];
966 	int i, rc;
967 
968 	rc = fun_port_read_cmds(fp, ARRAY_SIZE(keys), keys, data);
969 	if (rc)
970 		return rc;
971 
972 	for (i = 0; i < ARRAY_SIZE(keys); i++) {
973 		switch (keys[i]) {
974 		case FUN_ADMIN_PORT_KEY_MACADDR:
975 			u64_to_ether_addr(data[i], mac);
976 			if (is_zero_ether_addr(mac)) {
977 				eth_hw_addr_random(netdev);
978 			} else if (is_valid_ether_addr(mac)) {
979 				eth_hw_addr_set(netdev, mac);
980 			} else {
981 				netdev_err(netdev,
982 					   "device provided a bad MAC address %pM\n",
983 					   mac);
984 				return -EINVAL;
985 			}
986 			break;
987 
988 		case FUN_ADMIN_PORT_KEY_CAPABILITIES:
989 			fp->port_caps = data[i];
990 			break;
991 
992 		case FUN_ADMIN_PORT_KEY_ADVERT:
993 			fp->advertising = data[i];
994 			break;
995 
996 		case FUN_ADMIN_PORT_KEY_MTU:
997 			netdev->mtu = data[i];
998 			break;
999 		}
1000 	}
1001 
1002 	if (!(fp->port_caps & FUN_PORT_CAP_VPORT)) {
1003 		rc = fun_port_read_cmds(fp, ARRAY_SIZE(phys_keys), phys_keys,
1004 					data);
1005 		if (rc)
1006 			return rc;
1007 
1008 		fp->lane_attrs = data[0];
1009 	}
1010 
1011 	if (netdev->addr_assign_type == NET_ADDR_RANDOM)
1012 		return fun_port_write_cmd(fp, FUN_ADMIN_PORT_KEY_MACADDR,
1013 					  ether_addr_to_u64(netdev->dev_addr));
1014 	return 0;
1015 }
1016 
1017 static int fun_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
1018 {
1019 	const struct funeth_priv *fp = netdev_priv(dev);
1020 
1021 	return copy_to_user(ifr->ifr_data, &fp->hwtstamp_cfg,
1022 			    sizeof(fp->hwtstamp_cfg)) ? -EFAULT : 0;
1023 }
1024 
1025 static int fun_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
1026 {
1027 	struct funeth_priv *fp = netdev_priv(dev);
1028 	struct hwtstamp_config cfg;
1029 
1030 	if (copy_from_user(&cfg, ifr->ifr_data, sizeof(cfg)))
1031 		return -EFAULT;
1032 
1033 	/* no TX HW timestamps */
1034 	cfg.tx_type = HWTSTAMP_TX_OFF;
1035 
1036 	switch (cfg.rx_filter) {
1037 	case HWTSTAMP_FILTER_NONE:
1038 		break;
1039 	case HWTSTAMP_FILTER_ALL:
1040 	case HWTSTAMP_FILTER_SOME:
1041 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
1042 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
1043 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
1044 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
1045 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
1046 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
1047 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
1048 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
1049 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
1050 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
1051 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
1052 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
1053 	case HWTSTAMP_FILTER_NTP_ALL:
1054 		cfg.rx_filter = HWTSTAMP_FILTER_ALL;
1055 		break;
1056 	default:
1057 		return -ERANGE;
1058 	}
1059 
1060 	fp->hwtstamp_cfg = cfg;
1061 	return copy_to_user(ifr->ifr_data, &cfg, sizeof(cfg)) ? -EFAULT : 0;
1062 }
1063 
1064 static int fun_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
1065 {
1066 	switch (cmd) {
1067 	case SIOCSHWTSTAMP:
1068 		return fun_hwtstamp_set(dev, ifr);
1069 	case SIOCGHWTSTAMP:
1070 		return fun_hwtstamp_get(dev, ifr);
1071 	default:
1072 		return -EOPNOTSUPP;
1073 	}
1074 }
1075 
1076 /* Prepare the queues for XDP. */
1077 static int fun_enter_xdp(struct net_device *dev, struct bpf_prog *prog)
1078 {
1079 	struct funeth_priv *fp = netdev_priv(dev);
1080 	unsigned int i, nqs = num_online_cpus();
1081 	struct funeth_txq **xdpqs;
1082 	struct funeth_rxq **rxqs;
1083 	int err;
1084 
1085 	xdpqs = alloc_xdpqs(dev, nqs, fp->sq_depth, 0, FUN_QSTATE_INIT_FULL);
1086 	if (IS_ERR(xdpqs))
1087 		return PTR_ERR(xdpqs);
1088 
1089 	rxqs = rtnl_dereference(fp->rxqs);
1090 	for (i = 0; i < dev->real_num_rx_queues; i++) {
1091 		err = fun_rxq_set_bpf(rxqs[i], prog);
1092 		if (err)
1093 			goto out;
1094 	}
1095 
1096 	fp->num_xdpqs = nqs;
1097 	rcu_assign_pointer(fp->xdpqs, xdpqs);
1098 	return 0;
1099 out:
1100 	while (i--)
1101 		fun_rxq_set_bpf(rxqs[i], NULL);
1102 
1103 	free_xdpqs(xdpqs, nqs, 0, FUN_QSTATE_DESTROYED);
1104 	return err;
1105 }
1106 
1107 /* Set the queues for non-XDP operation. */
1108 static void fun_end_xdp(struct net_device *dev)
1109 {
1110 	struct funeth_priv *fp = netdev_priv(dev);
1111 	struct funeth_txq **xdpqs;
1112 	struct funeth_rxq **rxqs;
1113 	unsigned int i;
1114 
1115 	xdpqs = rtnl_dereference(fp->xdpqs);
1116 	rcu_assign_pointer(fp->xdpqs, NULL);
1117 	synchronize_net();
1118 	/* at this point both Rx and Tx XDP processing has ended */
1119 
1120 	free_xdpqs(xdpqs, fp->num_xdpqs, 0, FUN_QSTATE_DESTROYED);
1121 	fp->num_xdpqs = 0;
1122 
1123 	rxqs = rtnl_dereference(fp->rxqs);
1124 	for (i = 0; i < dev->real_num_rx_queues; i++)
1125 		fun_rxq_set_bpf(rxqs[i], NULL);
1126 }
1127 
1128 #define XDP_MAX_MTU \
1129 	(PAGE_SIZE - FUN_XDP_HEADROOM - VLAN_ETH_HLEN - FUN_RX_TAILROOM)
1130 
1131 static int fun_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp)
1132 {
1133 	struct bpf_prog *old_prog, *prog = xdp->prog;
1134 	struct funeth_priv *fp = netdev_priv(dev);
1135 	int i, err;
1136 
1137 	/* XDP uses at most one buffer */
1138 	if (prog && dev->mtu > XDP_MAX_MTU) {
1139 		netdev_err(dev, "device MTU %u too large for XDP\n", dev->mtu);
1140 		NL_SET_ERR_MSG_MOD(xdp->extack,
1141 				   "Device MTU too large for XDP");
1142 		return -EINVAL;
1143 	}
1144 
1145 	if (!netif_running(dev)) {
1146 		fp->num_xdpqs = prog ? num_online_cpus() : 0;
1147 	} else if (prog && !fp->xdp_prog) {
1148 		err = fun_enter_xdp(dev, prog);
1149 		if (err) {
1150 			NL_SET_ERR_MSG_MOD(xdp->extack,
1151 					   "Failed to set queues for XDP.");
1152 			return err;
1153 		}
1154 	} else if (!prog && fp->xdp_prog) {
1155 		fun_end_xdp(dev);
1156 	} else {
1157 		struct funeth_rxq **rxqs = rtnl_dereference(fp->rxqs);
1158 
1159 		for (i = 0; i < dev->real_num_rx_queues; i++)
1160 			WRITE_ONCE(rxqs[i]->xdp_prog, prog);
1161 	}
1162 
1163 	if (prog)
1164 		xdp_features_set_redirect_target(dev, true);
1165 	else
1166 		xdp_features_clear_redirect_target(dev);
1167 
1168 	dev->max_mtu = prog ? XDP_MAX_MTU : FUN_MAX_MTU;
1169 	old_prog = xchg(&fp->xdp_prog, prog);
1170 	if (old_prog)
1171 		bpf_prog_put(old_prog);
1172 
1173 	return 0;
1174 }
1175 
1176 static int fun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1177 {
1178 	switch (xdp->command) {
1179 	case XDP_SETUP_PROG:
1180 		return fun_xdp_setup(dev, xdp);
1181 	default:
1182 		return -EINVAL;
1183 	}
1184 }
1185 
1186 static int fun_init_vports(struct fun_ethdev *ed, unsigned int n)
1187 {
1188 	if (ed->num_vports)
1189 		return -EINVAL;
1190 
1191 	ed->vport_info = kvcalloc(n, sizeof(*ed->vport_info), GFP_KERNEL);
1192 	if (!ed->vport_info)
1193 		return -ENOMEM;
1194 	ed->num_vports = n;
1195 	return 0;
1196 }
1197 
1198 static void fun_free_vports(struct fun_ethdev *ed)
1199 {
1200 	kvfree(ed->vport_info);
1201 	ed->vport_info = NULL;
1202 	ed->num_vports = 0;
1203 }
1204 
1205 static struct fun_vport_info *fun_get_vport(struct fun_ethdev *ed,
1206 					    unsigned int vport)
1207 {
1208 	if (!ed->vport_info || vport >= ed->num_vports)
1209 		return NULL;
1210 
1211 	return ed->vport_info + vport;
1212 }
1213 
1214 static int fun_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
1215 {
1216 	struct funeth_priv *fp = netdev_priv(dev);
1217 	struct fun_adi_param mac_param = {};
1218 	struct fun_dev *fdev = fp->fdev;
1219 	struct fun_ethdev *ed = to_fun_ethdev(fdev);
1220 	struct fun_vport_info *vi;
1221 	int rc = -EINVAL;
1222 
1223 	if (is_multicast_ether_addr(mac))
1224 		return -EINVAL;
1225 
1226 	mutex_lock(&ed->state_mutex);
1227 	vi = fun_get_vport(ed, vf);
1228 	if (!vi)
1229 		goto unlock;
1230 
1231 	mac_param.u.mac = FUN_ADI_MAC_INIT(ether_addr_to_u64(mac));
1232 	rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_MACADDR, vf + 1,
1233 			   &mac_param);
1234 	if (!rc)
1235 		ether_addr_copy(vi->mac, mac);
1236 unlock:
1237 	mutex_unlock(&ed->state_mutex);
1238 	return rc;
1239 }
1240 
1241 static int fun_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
1242 			   __be16 vlan_proto)
1243 {
1244 	struct funeth_priv *fp = netdev_priv(dev);
1245 	struct fun_adi_param vlan_param = {};
1246 	struct fun_dev *fdev = fp->fdev;
1247 	struct fun_ethdev *ed = to_fun_ethdev(fdev);
1248 	struct fun_vport_info *vi;
1249 	int rc = -EINVAL;
1250 
1251 	if (vlan > 4095 || qos > 7)
1252 		return -EINVAL;
1253 	if (vlan_proto && vlan_proto != htons(ETH_P_8021Q) &&
1254 	    vlan_proto != htons(ETH_P_8021AD))
1255 		return -EINVAL;
1256 
1257 	mutex_lock(&ed->state_mutex);
1258 	vi = fun_get_vport(ed, vf);
1259 	if (!vi)
1260 		goto unlock;
1261 
1262 	vlan_param.u.vlan = FUN_ADI_VLAN_INIT(be16_to_cpu(vlan_proto),
1263 					      ((u16)qos << VLAN_PRIO_SHIFT) | vlan);
1264 	rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_VLAN, vf + 1, &vlan_param);
1265 	if (!rc) {
1266 		vi->vlan = vlan;
1267 		vi->qos = qos;
1268 		vi->vlan_proto = vlan_proto;
1269 	}
1270 unlock:
1271 	mutex_unlock(&ed->state_mutex);
1272 	return rc;
1273 }
1274 
1275 static int fun_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
1276 			   int max_tx_rate)
1277 {
1278 	struct funeth_priv *fp = netdev_priv(dev);
1279 	struct fun_adi_param rate_param = {};
1280 	struct fun_dev *fdev = fp->fdev;
1281 	struct fun_ethdev *ed = to_fun_ethdev(fdev);
1282 	struct fun_vport_info *vi;
1283 	int rc = -EINVAL;
1284 
1285 	if (min_tx_rate)
1286 		return -EINVAL;
1287 
1288 	mutex_lock(&ed->state_mutex);
1289 	vi = fun_get_vport(ed, vf);
1290 	if (!vi)
1291 		goto unlock;
1292 
1293 	rate_param.u.rate = FUN_ADI_RATE_INIT(max_tx_rate);
1294 	rc = fun_adi_write(fdev, FUN_ADMIN_ADI_ATTR_RATE, vf + 1, &rate_param);
1295 	if (!rc)
1296 		vi->max_rate = max_tx_rate;
1297 unlock:
1298 	mutex_unlock(&ed->state_mutex);
1299 	return rc;
1300 }
1301 
1302 static int fun_get_vf_config(struct net_device *dev, int vf,
1303 			     struct ifla_vf_info *ivi)
1304 {
1305 	struct funeth_priv *fp = netdev_priv(dev);
1306 	struct fun_ethdev *ed = to_fun_ethdev(fp->fdev);
1307 	const struct fun_vport_info *vi;
1308 
1309 	mutex_lock(&ed->state_mutex);
1310 	vi = fun_get_vport(ed, vf);
1311 	if (!vi)
1312 		goto unlock;
1313 
1314 	memset(ivi, 0, sizeof(*ivi));
1315 	ivi->vf = vf;
1316 	ether_addr_copy(ivi->mac, vi->mac);
1317 	ivi->vlan = vi->vlan;
1318 	ivi->qos = vi->qos;
1319 	ivi->vlan_proto = vi->vlan_proto;
1320 	ivi->max_tx_rate = vi->max_rate;
1321 	ivi->spoofchk = vi->spoofchk;
1322 unlock:
1323 	mutex_unlock(&ed->state_mutex);
1324 	return vi ? 0 : -EINVAL;
1325 }
1326 
1327 static void fun_uninit(struct net_device *dev)
1328 {
1329 	struct funeth_priv *fp = netdev_priv(dev);
1330 
1331 	fun_prune_queue_irqs(dev);
1332 	xa_destroy(&fp->irqs);
1333 }
1334 
1335 static const struct net_device_ops fun_netdev_ops = {
1336 	.ndo_open		= funeth_open,
1337 	.ndo_stop		= funeth_close,
1338 	.ndo_start_xmit		= fun_start_xmit,
1339 	.ndo_get_stats64	= fun_get_stats64,
1340 	.ndo_change_mtu		= fun_change_mtu,
1341 	.ndo_set_mac_address	= fun_set_macaddr,
1342 	.ndo_validate_addr	= eth_validate_addr,
1343 	.ndo_eth_ioctl		= fun_ioctl,
1344 	.ndo_uninit		= fun_uninit,
1345 	.ndo_bpf		= fun_xdp,
1346 	.ndo_xdp_xmit		= fun_xdp_xmit_frames,
1347 	.ndo_set_vf_mac		= fun_set_vf_mac,
1348 	.ndo_set_vf_vlan	= fun_set_vf_vlan,
1349 	.ndo_set_vf_rate	= fun_set_vf_rate,
1350 	.ndo_get_vf_config	= fun_get_vf_config,
1351 };
1352 
1353 #define GSO_ENCAP_FLAGS (NETIF_F_GSO_GRE | NETIF_F_GSO_IPXIP4 | \
1354 			 NETIF_F_GSO_IPXIP6 | NETIF_F_GSO_UDP_TUNNEL | \
1355 			 NETIF_F_GSO_UDP_TUNNEL_CSUM)
1356 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN | \
1357 		   NETIF_F_GSO_UDP_L4)
1358 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_HW_CSUM | TSO_FLAGS | \
1359 		   GSO_ENCAP_FLAGS | NETIF_F_HIGHDMA)
1360 
1361 static void fun_dflt_rss_indir(struct funeth_priv *fp, unsigned int nrx)
1362 {
1363 	unsigned int i;
1364 
1365 	for (i = 0; i < fp->indir_table_nentries; i++)
1366 		fp->indir_table[i] = ethtool_rxfh_indir_default(i, nrx);
1367 }
1368 
1369 /* Reset the RSS indirection table to equal distribution across the current
1370  * number of Rx queues. Called at init time and whenever the number of Rx
1371  * queues changes subsequently. Note that this may also resize the indirection
1372  * table.
1373  */
1374 static void fun_reset_rss_indir(struct net_device *dev, unsigned int nrx)
1375 {
1376 	struct funeth_priv *fp = netdev_priv(dev);
1377 
1378 	if (!fp->rss_cfg)
1379 		return;
1380 
1381 	/* Set the table size to the max possible that allows an equal number
1382 	 * of occurrences of each CQ.
1383 	 */
1384 	fp->indir_table_nentries = rounddown(FUN_ETH_RSS_MAX_INDIR_ENT, nrx);
1385 	fun_dflt_rss_indir(fp, nrx);
1386 }
1387 
1388 /* Update the RSS LUT to contain only queues in [0, nrx). Normally this will
1389  * update the LUT to an equal distribution among nrx queues, If @only_if_needed
1390  * is set the LUT is left unchanged if it already does not reference any queues
1391  * >= nrx.
1392  */
1393 static int fun_rss_set_qnum(struct net_device *dev, unsigned int nrx,
1394 			    bool only_if_needed)
1395 {
1396 	struct funeth_priv *fp = netdev_priv(dev);
1397 	u32 old_lut[FUN_ETH_RSS_MAX_INDIR_ENT];
1398 	unsigned int i, oldsz;
1399 	int err;
1400 
1401 	if (!fp->rss_cfg)
1402 		return 0;
1403 
1404 	if (only_if_needed) {
1405 		for (i = 0; i < fp->indir_table_nentries; i++)
1406 			if (fp->indir_table[i] >= nrx)
1407 				break;
1408 
1409 		if (i >= fp->indir_table_nentries)
1410 			return 0;
1411 	}
1412 
1413 	memcpy(old_lut, fp->indir_table, sizeof(old_lut));
1414 	oldsz = fp->indir_table_nentries;
1415 	fun_reset_rss_indir(dev, nrx);
1416 
1417 	err = fun_config_rss(dev, fp->hash_algo, fp->rss_key,
1418 			     fp->indir_table, FUN_ADMIN_SUBOP_MODIFY);
1419 	if (!err)
1420 		return 0;
1421 
1422 	memcpy(fp->indir_table, old_lut, sizeof(old_lut));
1423 	fp->indir_table_nentries = oldsz;
1424 	return err;
1425 }
1426 
1427 /* Allocate the DMA area for the RSS configuration commands to the device, and
1428  * initialize the hash, hash key, indirection table size and its entries to
1429  * their defaults. The indirection table defaults to equal distribution across
1430  * the Rx queues.
1431  */
1432 static int fun_init_rss(struct net_device *dev)
1433 {
1434 	struct funeth_priv *fp = netdev_priv(dev);
1435 	size_t size = sizeof(fp->rss_key) + sizeof(fp->indir_table);
1436 
1437 	fp->rss_hw_id = FUN_HCI_ID_INVALID;
1438 	if (!(fp->port_caps & FUN_PORT_CAP_OFFLOADS))
1439 		return 0;
1440 
1441 	fp->rss_cfg = dma_alloc_coherent(&fp->pdev->dev, size,
1442 					 &fp->rss_dma_addr, GFP_KERNEL);
1443 	if (!fp->rss_cfg)
1444 		return -ENOMEM;
1445 
1446 	fp->hash_algo = FUN_ETH_RSS_ALG_TOEPLITZ;
1447 	netdev_rss_key_fill(fp->rss_key, sizeof(fp->rss_key));
1448 	fun_reset_rss_indir(dev, dev->real_num_rx_queues);
1449 	return 0;
1450 }
1451 
1452 static void fun_free_rss(struct funeth_priv *fp)
1453 {
1454 	if (fp->rss_cfg) {
1455 		dma_free_coherent(&fp->pdev->dev,
1456 				  sizeof(fp->rss_key) + sizeof(fp->indir_table),
1457 				  fp->rss_cfg, fp->rss_dma_addr);
1458 		fp->rss_cfg = NULL;
1459 	}
1460 }
1461 
1462 void fun_set_ring_count(struct net_device *netdev, unsigned int ntx,
1463 			unsigned int nrx)
1464 {
1465 	netif_set_real_num_tx_queues(netdev, ntx);
1466 	if (nrx != netdev->real_num_rx_queues) {
1467 		netif_set_real_num_rx_queues(netdev, nrx);
1468 		fun_reset_rss_indir(netdev, nrx);
1469 	}
1470 }
1471 
1472 static int fun_init_stats_area(struct funeth_priv *fp)
1473 {
1474 	unsigned int nstats;
1475 
1476 	if (!(fp->port_caps & FUN_PORT_CAP_STATS))
1477 		return 0;
1478 
1479 	nstats = PORT_MAC_RX_STATS_MAX + PORT_MAC_TX_STATS_MAX +
1480 		 PORT_MAC_FEC_STATS_MAX;
1481 
1482 	fp->stats = dma_alloc_coherent(&fp->pdev->dev, nstats * sizeof(u64),
1483 				       &fp->stats_dma_addr, GFP_KERNEL);
1484 	if (!fp->stats)
1485 		return -ENOMEM;
1486 	return 0;
1487 }
1488 
1489 static void fun_free_stats_area(struct funeth_priv *fp)
1490 {
1491 	unsigned int nstats;
1492 
1493 	if (fp->stats) {
1494 		nstats = PORT_MAC_RX_STATS_MAX + PORT_MAC_TX_STATS_MAX;
1495 		dma_free_coherent(&fp->pdev->dev, nstats * sizeof(u64),
1496 				  fp->stats, fp->stats_dma_addr);
1497 		fp->stats = NULL;
1498 	}
1499 }
1500 
1501 static int fun_dl_port_register(struct net_device *netdev)
1502 {
1503 	struct funeth_priv *fp = netdev_priv(netdev);
1504 	struct devlink *dl = priv_to_devlink(fp->fdev);
1505 	struct devlink_port_attrs attrs = {};
1506 	unsigned int idx;
1507 
1508 	if (fp->port_caps & FUN_PORT_CAP_VPORT) {
1509 		attrs.flavour = DEVLINK_PORT_FLAVOUR_VIRTUAL;
1510 		idx = fp->lport;
1511 	} else {
1512 		idx = netdev->dev_port;
1513 		attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL;
1514 		attrs.lanes = fp->lane_attrs & 7;
1515 		if (fp->lane_attrs & FUN_PORT_LANE_SPLIT) {
1516 			attrs.split = 1;
1517 			attrs.phys.port_number = fp->lport & ~3;
1518 			attrs.phys.split_subport_number = fp->lport & 3;
1519 		} else {
1520 			attrs.phys.port_number = fp->lport;
1521 		}
1522 	}
1523 
1524 	devlink_port_attrs_set(&fp->dl_port, &attrs);
1525 
1526 	return devlink_port_register(dl, &fp->dl_port, idx);
1527 }
1528 
1529 /* Determine the max Tx/Rx queues for a port. */
1530 static int fun_max_qs(struct fun_ethdev *ed, unsigned int *ntx,
1531 		      unsigned int *nrx)
1532 {
1533 	int neth;
1534 
1535 	if (ed->num_ports > 1 || is_kdump_kernel()) {
1536 		*ntx = 1;
1537 		*nrx = 1;
1538 		return 0;
1539 	}
1540 
1541 	neth = fun_get_res_count(&ed->fdev, FUN_ADMIN_OP_ETH);
1542 	if (neth < 0)
1543 		return neth;
1544 
1545 	/* We determine the max number of queues based on the CPU
1546 	 * cores, device interrupts and queues, RSS size, and device Tx flows.
1547 	 *
1548 	 * - At least 1 Rx and 1 Tx queues.
1549 	 * - At most 1 Rx/Tx queue per core.
1550 	 * - Each Rx/Tx queue needs 1 SQ.
1551 	 */
1552 	*ntx = min(ed->nsqs_per_port - 1, num_online_cpus());
1553 	*nrx = *ntx;
1554 	if (*ntx > neth)
1555 		*ntx = neth;
1556 	if (*nrx > FUN_ETH_RSS_MAX_INDIR_ENT)
1557 		*nrx = FUN_ETH_RSS_MAX_INDIR_ENT;
1558 	return 0;
1559 }
1560 
1561 static void fun_queue_defaults(struct net_device *dev, unsigned int nsqs)
1562 {
1563 	unsigned int ntx, nrx;
1564 
1565 	ntx = min(dev->num_tx_queues, FUN_DFLT_QUEUES);
1566 	nrx = min(dev->num_rx_queues, FUN_DFLT_QUEUES);
1567 	if (ntx <= nrx) {
1568 		ntx = min(ntx, nsqs / 2);
1569 		nrx = min(nrx, nsqs - ntx);
1570 	} else {
1571 		nrx = min(nrx, nsqs / 2);
1572 		ntx = min(ntx, nsqs - nrx);
1573 	}
1574 
1575 	netif_set_real_num_tx_queues(dev, ntx);
1576 	netif_set_real_num_rx_queues(dev, nrx);
1577 }
1578 
1579 /* Replace the existing Rx/Tx/XDP queues with equal number of queues with
1580  * different settings, e.g. depth. This is a disruptive replacement that
1581  * temporarily shuts down the data path and should be limited to changes that
1582  * can't be applied to live queues. The old queues are always discarded.
1583  */
1584 int fun_replace_queues(struct net_device *dev, struct fun_qset *newqs,
1585 		       struct netlink_ext_ack *extack)
1586 {
1587 	struct fun_qset oldqs = { .state = FUN_QSTATE_DESTROYED };
1588 	struct funeth_priv *fp = netdev_priv(dev);
1589 	int err;
1590 
1591 	newqs->nrxqs = dev->real_num_rx_queues;
1592 	newqs->ntxqs = dev->real_num_tx_queues;
1593 	newqs->nxdpqs = fp->num_xdpqs;
1594 	newqs->state = FUN_QSTATE_INIT_SW;
1595 	err = fun_alloc_rings(dev, newqs);
1596 	if (err) {
1597 		NL_SET_ERR_MSG_MOD(extack,
1598 				   "Unable to allocate memory for new queues, keeping current settings");
1599 		return err;
1600 	}
1601 
1602 	fun_down(dev, &oldqs);
1603 
1604 	err = fun_up(dev, newqs);
1605 	if (!err)
1606 		return 0;
1607 
1608 	/* The new queues couldn't be installed. We do not retry the old queues
1609 	 * as they are the same to the device as the new queues and would
1610 	 * similarly fail.
1611 	 */
1612 	newqs->state = FUN_QSTATE_DESTROYED;
1613 	fun_free_rings(dev, newqs);
1614 	NL_SET_ERR_MSG_MOD(extack, "Unable to restore the data path with the new queues.");
1615 	return err;
1616 }
1617 
1618 /* Change the number of Rx/Tx queues of a device while it is up. This is done
1619  * by incrementally adding/removing queues to meet the new requirements while
1620  * handling ongoing traffic.
1621  */
1622 int fun_change_num_queues(struct net_device *dev, unsigned int ntx,
1623 			  unsigned int nrx)
1624 {
1625 	unsigned int keep_tx = min(dev->real_num_tx_queues, ntx);
1626 	unsigned int keep_rx = min(dev->real_num_rx_queues, nrx);
1627 	struct funeth_priv *fp = netdev_priv(dev);
1628 	struct fun_qset oldqs = {
1629 		.rxqs = rtnl_dereference(fp->rxqs),
1630 		.txqs = fp->txqs,
1631 		.nrxqs = dev->real_num_rx_queues,
1632 		.ntxqs = dev->real_num_tx_queues,
1633 		.rxq_start = keep_rx,
1634 		.txq_start = keep_tx,
1635 		.state = FUN_QSTATE_DESTROYED
1636 	};
1637 	struct fun_qset newqs = {
1638 		.nrxqs = nrx,
1639 		.ntxqs = ntx,
1640 		.rxq_start = keep_rx,
1641 		.txq_start = keep_tx,
1642 		.cq_depth = fp->cq_depth,
1643 		.rq_depth = fp->rq_depth,
1644 		.sq_depth = fp->sq_depth,
1645 		.state = FUN_QSTATE_INIT_FULL
1646 	};
1647 	int i, err;
1648 
1649 	err = fun_alloc_rings(dev, &newqs);
1650 	if (err)
1651 		goto free_irqs;
1652 
1653 	err = fun_enable_irqs(dev); /* of any newly added queues */
1654 	if (err)
1655 		goto free_rings;
1656 
1657 	/* copy the queues we are keeping to the new set */
1658 	memcpy(newqs.rxqs, oldqs.rxqs, keep_rx * sizeof(*oldqs.rxqs));
1659 	memcpy(newqs.txqs, fp->txqs, keep_tx * sizeof(*fp->txqs));
1660 
1661 	if (nrx < dev->real_num_rx_queues) {
1662 		err = fun_rss_set_qnum(dev, nrx, true);
1663 		if (err)
1664 			goto disable_tx_irqs;
1665 
1666 		for (i = nrx; i < dev->real_num_rx_queues; i++)
1667 			fun_disable_one_irq(container_of(oldqs.rxqs[i]->napi,
1668 							 struct fun_irq, napi));
1669 
1670 		netif_set_real_num_rx_queues(dev, nrx);
1671 	}
1672 
1673 	if (ntx < dev->real_num_tx_queues)
1674 		netif_set_real_num_tx_queues(dev, ntx);
1675 
1676 	rcu_assign_pointer(fp->rxqs, newqs.rxqs);
1677 	fp->txqs = newqs.txqs;
1678 	synchronize_net();
1679 
1680 	if (ntx > dev->real_num_tx_queues)
1681 		netif_set_real_num_tx_queues(dev, ntx);
1682 
1683 	if (nrx > dev->real_num_rx_queues) {
1684 		netif_set_real_num_rx_queues(dev, nrx);
1685 		fun_rss_set_qnum(dev, nrx, false);
1686 	}
1687 
1688 	/* disable interrupts of any excess Tx queues */
1689 	for (i = keep_tx; i < oldqs.ntxqs; i++)
1690 		fun_disable_one_irq(oldqs.txqs[i]->irq);
1691 
1692 	fun_free_rings(dev, &oldqs);
1693 	fun_prune_queue_irqs(dev);
1694 	return 0;
1695 
1696 disable_tx_irqs:
1697 	for (i = oldqs.ntxqs; i < ntx; i++)
1698 		fun_disable_one_irq(newqs.txqs[i]->irq);
1699 free_rings:
1700 	newqs.state = FUN_QSTATE_DESTROYED;
1701 	fun_free_rings(dev, &newqs);
1702 free_irqs:
1703 	fun_prune_queue_irqs(dev);
1704 	return err;
1705 }
1706 
1707 static int fun_create_netdev(struct fun_ethdev *ed, unsigned int portid)
1708 {
1709 	struct fun_dev *fdev = &ed->fdev;
1710 	struct net_device *netdev;
1711 	struct funeth_priv *fp;
1712 	unsigned int ntx, nrx;
1713 	int rc;
1714 
1715 	rc = fun_max_qs(ed, &ntx, &nrx);
1716 	if (rc)
1717 		return rc;
1718 
1719 	netdev = alloc_etherdev_mqs(sizeof(*fp), ntx, nrx);
1720 	if (!netdev) {
1721 		rc = -ENOMEM;
1722 		goto done;
1723 	}
1724 
1725 	netdev->dev_port = portid;
1726 	fun_queue_defaults(netdev, ed->nsqs_per_port);
1727 
1728 	fp = netdev_priv(netdev);
1729 	fp->fdev = fdev;
1730 	fp->pdev = to_pci_dev(fdev->dev);
1731 	fp->netdev = netdev;
1732 	xa_init(&fp->irqs);
1733 	fp->rx_irq_ofst = ntx;
1734 	seqcount_init(&fp->link_seq);
1735 
1736 	fp->lport = INVALID_LPORT;
1737 	rc = fun_port_create(netdev);
1738 	if (rc)
1739 		goto free_netdev;
1740 
1741 	/* bind port to admin CQ for async events */
1742 	rc = fun_bind(fdev, FUN_ADMIN_BIND_TYPE_PORT, portid,
1743 		      FUN_ADMIN_BIND_TYPE_EPCQ, 0);
1744 	if (rc)
1745 		goto destroy_port;
1746 
1747 	rc = fun_get_port_attributes(netdev);
1748 	if (rc)
1749 		goto destroy_port;
1750 
1751 	rc = fun_init_rss(netdev);
1752 	if (rc)
1753 		goto destroy_port;
1754 
1755 	rc = fun_init_stats_area(fp);
1756 	if (rc)
1757 		goto free_rss;
1758 
1759 	SET_NETDEV_DEV(netdev, fdev->dev);
1760 	SET_NETDEV_DEVLINK_PORT(netdev, &fp->dl_port);
1761 	netdev->netdev_ops = &fun_netdev_ops;
1762 
1763 	netdev->hw_features = NETIF_F_SG | NETIF_F_RXHASH | NETIF_F_RXCSUM;
1764 	if (fp->port_caps & FUN_PORT_CAP_OFFLOADS)
1765 		netdev->hw_features |= NETIF_F_HW_CSUM | TSO_FLAGS;
1766 	if (fp->port_caps & FUN_PORT_CAP_ENCAP_OFFLOADS)
1767 		netdev->hw_features |= GSO_ENCAP_FLAGS;
1768 
1769 	netdev->features |= netdev->hw_features | NETIF_F_HIGHDMA;
1770 	netdev->vlan_features = netdev->features & VLAN_FEAT;
1771 	netdev->mpls_features = netdev->vlan_features;
1772 	netdev->hw_enc_features = netdev->hw_features;
1773 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
1774 
1775 	netdev->min_mtu = ETH_MIN_MTU;
1776 	netdev->max_mtu = FUN_MAX_MTU;
1777 
1778 	fun_set_ethtool_ops(netdev);
1779 
1780 	/* configurable parameters */
1781 	fp->sq_depth = min(SQ_DEPTH, fdev->q_depth);
1782 	fp->cq_depth = min(CQ_DEPTH, fdev->q_depth);
1783 	fp->rq_depth = min_t(unsigned int, RQ_DEPTH, fdev->q_depth);
1784 	fp->rx_coal_usec  = CQ_INTCOAL_USEC;
1785 	fp->rx_coal_count = CQ_INTCOAL_NPKT;
1786 	fp->tx_coal_usec  = SQ_INTCOAL_USEC;
1787 	fp->tx_coal_count = SQ_INTCOAL_NPKT;
1788 	fp->cq_irq_db = FUN_IRQ_CQ_DB(fp->rx_coal_usec, fp->rx_coal_count);
1789 
1790 	rc = fun_dl_port_register(netdev);
1791 	if (rc)
1792 		goto free_stats;
1793 
1794 	fp->ktls_id = FUN_HCI_ID_INVALID;
1795 	fun_ktls_init(netdev);            /* optional, failure OK */
1796 
1797 	netif_carrier_off(netdev);
1798 	ed->netdevs[portid] = netdev;
1799 	rc = register_netdev(netdev);
1800 	if (rc)
1801 		goto unreg_devlink;
1802 	return 0;
1803 
1804 unreg_devlink:
1805 	ed->netdevs[portid] = NULL;
1806 	fun_ktls_cleanup(fp);
1807 	devlink_port_unregister(&fp->dl_port);
1808 free_stats:
1809 	fun_free_stats_area(fp);
1810 free_rss:
1811 	fun_free_rss(fp);
1812 destroy_port:
1813 	fun_port_destroy(netdev);
1814 free_netdev:
1815 	free_netdev(netdev);
1816 done:
1817 	dev_err(fdev->dev, "couldn't allocate port %u, error %d", portid, rc);
1818 	return rc;
1819 }
1820 
1821 static void fun_destroy_netdev(struct net_device *netdev)
1822 {
1823 	struct funeth_priv *fp;
1824 
1825 	fp = netdev_priv(netdev);
1826 	unregister_netdev(netdev);
1827 	devlink_port_unregister(&fp->dl_port);
1828 	fun_ktls_cleanup(fp);
1829 	fun_free_stats_area(fp);
1830 	fun_free_rss(fp);
1831 	fun_port_destroy(netdev);
1832 	free_netdev(netdev);
1833 }
1834 
1835 static int fun_create_ports(struct fun_ethdev *ed, unsigned int nports)
1836 {
1837 	struct fun_dev *fd = &ed->fdev;
1838 	int i, rc;
1839 
1840 	/* The admin queue takes 1 IRQ and 2 SQs. */
1841 	ed->nsqs_per_port = min(fd->num_irqs - 1,
1842 				fd->kern_end_qid - 2) / nports;
1843 	if (ed->nsqs_per_port < 2) {
1844 		dev_err(fd->dev, "Too few SQs for %u ports", nports);
1845 		return -EINVAL;
1846 	}
1847 
1848 	ed->netdevs = kcalloc(nports, sizeof(*ed->netdevs), GFP_KERNEL);
1849 	if (!ed->netdevs)
1850 		return -ENOMEM;
1851 
1852 	ed->num_ports = nports;
1853 	for (i = 0; i < nports; i++) {
1854 		rc = fun_create_netdev(ed, i);
1855 		if (rc)
1856 			goto free_netdevs;
1857 	}
1858 
1859 	return 0;
1860 
1861 free_netdevs:
1862 	while (i)
1863 		fun_destroy_netdev(ed->netdevs[--i]);
1864 	kfree(ed->netdevs);
1865 	ed->netdevs = NULL;
1866 	ed->num_ports = 0;
1867 	return rc;
1868 }
1869 
1870 static void fun_destroy_ports(struct fun_ethdev *ed)
1871 {
1872 	unsigned int i;
1873 
1874 	for (i = 0; i < ed->num_ports; i++)
1875 		fun_destroy_netdev(ed->netdevs[i]);
1876 
1877 	kfree(ed->netdevs);
1878 	ed->netdevs = NULL;
1879 	ed->num_ports = 0;
1880 }
1881 
1882 static void fun_update_link_state(const struct fun_ethdev *ed,
1883 				  const struct fun_admin_port_notif *notif)
1884 {
1885 	unsigned int port_idx = be16_to_cpu(notif->id);
1886 	struct net_device *netdev;
1887 	struct funeth_priv *fp;
1888 
1889 	if (port_idx >= ed->num_ports)
1890 		return;
1891 
1892 	netdev = ed->netdevs[port_idx];
1893 	fp = netdev_priv(netdev);
1894 
1895 	write_seqcount_begin(&fp->link_seq);
1896 	fp->link_speed = be32_to_cpu(notif->speed) * 10;  /* 10 Mbps->Mbps */
1897 	fp->active_fc = notif->flow_ctrl;
1898 	fp->active_fec = notif->fec;
1899 	fp->xcvr_type = notif->xcvr_type;
1900 	fp->link_down_reason = notif->link_down_reason;
1901 	fp->lp_advertising = be64_to_cpu(notif->lp_advertising);
1902 
1903 	if ((notif->link_state | notif->missed_events) & FUN_PORT_FLAG_MAC_DOWN)
1904 		netif_carrier_off(netdev);
1905 	if (notif->link_state & FUN_PORT_FLAG_MAC_UP)
1906 		netif_carrier_on(netdev);
1907 
1908 	write_seqcount_end(&fp->link_seq);
1909 	fun_report_link(netdev);
1910 }
1911 
1912 /* handler for async events delivered through the admin CQ */
1913 static void fun_event_cb(struct fun_dev *fdev, void *entry)
1914 {
1915 	u8 op = ((struct fun_admin_rsp_common *)entry)->op;
1916 
1917 	if (op == FUN_ADMIN_OP_PORT) {
1918 		const struct fun_admin_port_notif *rsp = entry;
1919 
1920 		if (rsp->subop == FUN_ADMIN_SUBOP_NOTIFY) {
1921 			fun_update_link_state(to_fun_ethdev(fdev), rsp);
1922 		} else if (rsp->subop == FUN_ADMIN_SUBOP_RES_COUNT) {
1923 			const struct fun_admin_res_count_rsp *r = entry;
1924 
1925 			if (r->count.data)
1926 				set_bit(FUN_SERV_RES_CHANGE, &fdev->service_flags);
1927 			else
1928 				set_bit(FUN_SERV_DEL_PORTS, &fdev->service_flags);
1929 			fun_serv_sched(fdev);
1930 		} else {
1931 			dev_info(fdev->dev, "adminq event unexpected op %u subop %u",
1932 				 op, rsp->subop);
1933 		}
1934 	} else {
1935 		dev_info(fdev->dev, "adminq event unexpected op %u", op);
1936 	}
1937 }
1938 
1939 /* handler for pending work managed by the service task */
1940 static void fun_service_cb(struct fun_dev *fdev)
1941 {
1942 	struct fun_ethdev *ed = to_fun_ethdev(fdev);
1943 	int rc;
1944 
1945 	if (test_and_clear_bit(FUN_SERV_DEL_PORTS, &fdev->service_flags))
1946 		fun_destroy_ports(ed);
1947 
1948 	if (!test_and_clear_bit(FUN_SERV_RES_CHANGE, &fdev->service_flags))
1949 		return;
1950 
1951 	rc = fun_get_res_count(fdev, FUN_ADMIN_OP_PORT);
1952 	if (rc < 0 || rc == ed->num_ports)
1953 		return;
1954 
1955 	if (ed->num_ports)
1956 		fun_destroy_ports(ed);
1957 	if (rc)
1958 		fun_create_ports(ed, rc);
1959 }
1960 
1961 static int funeth_sriov_configure(struct pci_dev *pdev, int nvfs)
1962 {
1963 	struct fun_dev *fdev = pci_get_drvdata(pdev);
1964 	struct fun_ethdev *ed = to_fun_ethdev(fdev);
1965 	int rc;
1966 
1967 	if (nvfs == 0) {
1968 		if (pci_vfs_assigned(pdev)) {
1969 			dev_warn(&pdev->dev,
1970 				 "Cannot disable SR-IOV while VFs are assigned\n");
1971 			return -EPERM;
1972 		}
1973 
1974 		mutex_lock(&ed->state_mutex);
1975 		fun_free_vports(ed);
1976 		mutex_unlock(&ed->state_mutex);
1977 		pci_disable_sriov(pdev);
1978 		return 0;
1979 	}
1980 
1981 	rc = pci_enable_sriov(pdev, nvfs);
1982 	if (rc)
1983 		return rc;
1984 
1985 	mutex_lock(&ed->state_mutex);
1986 	rc = fun_init_vports(ed, nvfs);
1987 	mutex_unlock(&ed->state_mutex);
1988 	if (rc) {
1989 		pci_disable_sriov(pdev);
1990 		return rc;
1991 	}
1992 
1993 	return nvfs;
1994 }
1995 
1996 static int funeth_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1997 {
1998 	struct fun_dev_params aqreq = {
1999 		.cqe_size_log2 = ilog2(ADMIN_CQE_SIZE),
2000 		.sqe_size_log2 = ilog2(ADMIN_SQE_SIZE),
2001 		.cq_depth      = ADMIN_CQ_DEPTH,
2002 		.sq_depth      = ADMIN_SQ_DEPTH,
2003 		.rq_depth      = ADMIN_RQ_DEPTH,
2004 		.min_msix      = 2,              /* 1 Rx + 1 Tx */
2005 		.event_cb      = fun_event_cb,
2006 		.serv_cb       = fun_service_cb,
2007 	};
2008 	struct devlink *devlink;
2009 	struct fun_ethdev *ed;
2010 	struct fun_dev *fdev;
2011 	int rc;
2012 
2013 	devlink = fun_devlink_alloc(&pdev->dev);
2014 	if (!devlink) {
2015 		dev_err(&pdev->dev, "devlink alloc failed\n");
2016 		return -ENOMEM;
2017 	}
2018 
2019 	ed = devlink_priv(devlink);
2020 	mutex_init(&ed->state_mutex);
2021 
2022 	fdev = &ed->fdev;
2023 	rc = fun_dev_enable(fdev, pdev, &aqreq, KBUILD_MODNAME);
2024 	if (rc)
2025 		goto free_devlink;
2026 
2027 	rc = fun_get_res_count(fdev, FUN_ADMIN_OP_PORT);
2028 	if (rc > 0)
2029 		rc = fun_create_ports(ed, rc);
2030 	if (rc < 0)
2031 		goto disable_dev;
2032 
2033 	fun_serv_restart(fdev);
2034 	fun_devlink_register(devlink);
2035 	return 0;
2036 
2037 disable_dev:
2038 	fun_dev_disable(fdev);
2039 free_devlink:
2040 	mutex_destroy(&ed->state_mutex);
2041 	fun_devlink_free(devlink);
2042 	return rc;
2043 }
2044 
2045 static void funeth_remove(struct pci_dev *pdev)
2046 {
2047 	struct fun_dev *fdev = pci_get_drvdata(pdev);
2048 	struct devlink *devlink;
2049 	struct fun_ethdev *ed;
2050 
2051 	ed = to_fun_ethdev(fdev);
2052 	devlink = priv_to_devlink(ed);
2053 	fun_devlink_unregister(devlink);
2054 
2055 #ifdef CONFIG_PCI_IOV
2056 	funeth_sriov_configure(pdev, 0);
2057 #endif
2058 
2059 	fun_serv_stop(fdev);
2060 	fun_destroy_ports(ed);
2061 	fun_dev_disable(fdev);
2062 	mutex_destroy(&ed->state_mutex);
2063 
2064 	fun_devlink_free(devlink);
2065 }
2066 
2067 static struct pci_driver funeth_driver = {
2068 	.name		 = KBUILD_MODNAME,
2069 	.id_table	 = funeth_id_table,
2070 	.probe		 = funeth_probe,
2071 	.remove		 = funeth_remove,
2072 	.shutdown	 = funeth_remove,
2073 	.sriov_configure = funeth_sriov_configure,
2074 };
2075 
2076 module_pci_driver(funeth_driver);
2077 
2078 MODULE_AUTHOR("Dimitris Michailidis <dmichail@fungible.com>");
2079 MODULE_DESCRIPTION("Fungible Ethernet Network Driver");
2080 MODULE_LICENSE("Dual BSD/GPL");
2081 MODULE_DEVICE_TABLE(pci, funeth_id_table);
2082