1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 /* Copyright (C) 2015-2019 Netronome Systems, Inc. */
3 
4 /*
5  * nfp_net_common.c
6  * Netronome network device driver: Common functions between PF and VF
7  * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
8  *          Jason McMullan <jason.mcmullan@netronome.com>
9  *          Rolf Neugebauer <rolf.neugebauer@netronome.com>
10  *          Brad Petrus <brad.petrus@netronome.com>
11  *          Chris Telfer <chris.telfer@netronome.com>
12  */
13 
14 #include <linux/bitfield.h>
15 #include <linux/bpf.h>
16 #include <linux/module.h>
17 #include <linux/kernel.h>
18 #include <linux/init.h>
19 #include <linux/fs.h>
20 #include <linux/netdevice.h>
21 #include <linux/etherdevice.h>
22 #include <linux/interrupt.h>
23 #include <linux/ip.h>
24 #include <linux/ipv6.h>
25 #include <linux/mm.h>
26 #include <linux/overflow.h>
27 #include <linux/page_ref.h>
28 #include <linux/pci.h>
29 #include <linux/pci_regs.h>
30 #include <linux/ethtool.h>
31 #include <linux/log2.h>
32 #include <linux/if_vlan.h>
33 #include <linux/if_bridge.h>
34 #include <linux/random.h>
35 #include <linux/vmalloc.h>
36 #include <linux/ktime.h>
37 
38 #include <net/tls.h>
39 #include <net/vxlan.h>
40 #include <net/xdp_sock_drv.h>
41 #include <net/xfrm.h>
42 
43 #include "nfpcore/nfp_dev.h"
44 #include "nfpcore/nfp_nsp.h"
45 #include "ccm.h"
46 #include "nfp_app.h"
47 #include "nfp_net_ctrl.h"
48 #include "nfp_net.h"
49 #include "nfp_net_dp.h"
50 #include "nfp_net_sriov.h"
51 #include "nfp_net_xsk.h"
52 #include "nfp_port.h"
53 #include "crypto/crypto.h"
54 #include "crypto/fw.h"
55 
56 static int nfp_net_mc_unsync(struct net_device *netdev, const unsigned char *addr);
57 
58 /**
59  * nfp_net_get_fw_version() - Read and parse the FW version
60  * @fw_ver:	Output fw_version structure to read to
61  * @ctrl_bar:	Mapped address of the control BAR
62  */
63 void nfp_net_get_fw_version(struct nfp_net_fw_version *fw_ver,
64 			    void __iomem *ctrl_bar)
65 {
66 	u32 reg;
67 
68 	reg = readl(ctrl_bar + NFP_NET_CFG_VERSION);
69 	put_unaligned_le32(reg, fw_ver);
70 }
71 
72 u32 nfp_qcp_queue_offset(const struct nfp_dev_info *dev_info, u16 queue)
73 {
74 	queue &= dev_info->qc_idx_mask;
75 	return dev_info->qc_addr_offset + NFP_QCP_QUEUE_ADDR_SZ * queue;
76 }
77 
78 /* Firmware reconfig
79  *
80  * Firmware reconfig may take a while so we have two versions of it -
81  * synchronous and asynchronous (posted).  All synchronous callers are holding
82  * RTNL so we don't have to worry about serializing them.
83  */
84 static void nfp_net_reconfig_start(struct nfp_net *nn, u32 update)
85 {
86 	nn_writel(nn, NFP_NET_CFG_UPDATE, update);
87 	/* ensure update is written before pinging HW */
88 	nn_pci_flush(nn);
89 	nfp_qcp_wr_ptr_add(nn->qcp_cfg, 1);
90 	nn->reconfig_in_progress_update = update;
91 }
92 
93 /* Pass 0 as update to run posted reconfigs. */
94 static void nfp_net_reconfig_start_async(struct nfp_net *nn, u32 update)
95 {
96 	update |= nn->reconfig_posted;
97 	nn->reconfig_posted = 0;
98 
99 	nfp_net_reconfig_start(nn, update);
100 
101 	nn->reconfig_timer_active = true;
102 	mod_timer(&nn->reconfig_timer, jiffies + NFP_NET_POLL_TIMEOUT * HZ);
103 }
104 
105 static bool nfp_net_reconfig_check_done(struct nfp_net *nn, bool last_check)
106 {
107 	u32 reg;
108 
109 	reg = nn_readl(nn, NFP_NET_CFG_UPDATE);
110 	if (reg == 0)
111 		return true;
112 	if (reg & NFP_NET_CFG_UPDATE_ERR) {
113 		nn_err(nn, "Reconfig error (status: 0x%08x update: 0x%08x ctrl: 0x%08x)\n",
114 		       reg, nn->reconfig_in_progress_update,
115 		       nn_readl(nn, NFP_NET_CFG_CTRL));
116 		return true;
117 	} else if (last_check) {
118 		nn_err(nn, "Reconfig timeout (status: 0x%08x update: 0x%08x ctrl: 0x%08x)\n",
119 		       reg, nn->reconfig_in_progress_update,
120 		       nn_readl(nn, NFP_NET_CFG_CTRL));
121 		return true;
122 	}
123 
124 	return false;
125 }
126 
127 static bool __nfp_net_reconfig_wait(struct nfp_net *nn, unsigned long deadline)
128 {
129 	bool timed_out = false;
130 	int i;
131 
132 	/* Poll update field, waiting for NFP to ack the config.
133 	 * Do an opportunistic wait-busy loop, afterward sleep.
134 	 */
135 	for (i = 0; i < 50; i++) {
136 		if (nfp_net_reconfig_check_done(nn, false))
137 			return false;
138 		udelay(4);
139 	}
140 
141 	while (!nfp_net_reconfig_check_done(nn, timed_out)) {
142 		usleep_range(250, 500);
143 		timed_out = time_is_before_eq_jiffies(deadline);
144 	}
145 
146 	return timed_out;
147 }
148 
149 static int nfp_net_reconfig_wait(struct nfp_net *nn, unsigned long deadline)
150 {
151 	if (__nfp_net_reconfig_wait(nn, deadline))
152 		return -EIO;
153 
154 	if (nn_readl(nn, NFP_NET_CFG_UPDATE) & NFP_NET_CFG_UPDATE_ERR)
155 		return -EIO;
156 
157 	return 0;
158 }
159 
160 static void nfp_net_reconfig_timer(struct timer_list *t)
161 {
162 	struct nfp_net *nn = from_timer(nn, t, reconfig_timer);
163 
164 	spin_lock_bh(&nn->reconfig_lock);
165 
166 	nn->reconfig_timer_active = false;
167 
168 	/* If sync caller is present it will take over from us */
169 	if (nn->reconfig_sync_present)
170 		goto done;
171 
172 	/* Read reconfig status and report errors */
173 	nfp_net_reconfig_check_done(nn, true);
174 
175 	if (nn->reconfig_posted)
176 		nfp_net_reconfig_start_async(nn, 0);
177 done:
178 	spin_unlock_bh(&nn->reconfig_lock);
179 }
180 
181 /**
182  * nfp_net_reconfig_post() - Post async reconfig request
183  * @nn:      NFP Net device to reconfigure
184  * @update:  The value for the update field in the BAR config
185  *
186  * Record FW reconfiguration request.  Reconfiguration will be kicked off
187  * whenever reconfiguration machinery is idle.  Multiple requests can be
188  * merged together!
189  */
190 static void nfp_net_reconfig_post(struct nfp_net *nn, u32 update)
191 {
192 	spin_lock_bh(&nn->reconfig_lock);
193 
194 	/* Sync caller will kick off async reconf when it's done, just post */
195 	if (nn->reconfig_sync_present) {
196 		nn->reconfig_posted |= update;
197 		goto done;
198 	}
199 
200 	/* Opportunistically check if the previous command is done */
201 	if (!nn->reconfig_timer_active ||
202 	    nfp_net_reconfig_check_done(nn, false))
203 		nfp_net_reconfig_start_async(nn, update);
204 	else
205 		nn->reconfig_posted |= update;
206 done:
207 	spin_unlock_bh(&nn->reconfig_lock);
208 }
209 
210 static void nfp_net_reconfig_sync_enter(struct nfp_net *nn)
211 {
212 	bool cancelled_timer = false;
213 	u32 pre_posted_requests;
214 
215 	spin_lock_bh(&nn->reconfig_lock);
216 
217 	WARN_ON(nn->reconfig_sync_present);
218 	nn->reconfig_sync_present = true;
219 
220 	if (nn->reconfig_timer_active) {
221 		nn->reconfig_timer_active = false;
222 		cancelled_timer = true;
223 	}
224 	pre_posted_requests = nn->reconfig_posted;
225 	nn->reconfig_posted = 0;
226 
227 	spin_unlock_bh(&nn->reconfig_lock);
228 
229 	if (cancelled_timer) {
230 		del_timer_sync(&nn->reconfig_timer);
231 		nfp_net_reconfig_wait(nn, nn->reconfig_timer.expires);
232 	}
233 
234 	/* Run the posted reconfigs which were issued before we started */
235 	if (pre_posted_requests) {
236 		nfp_net_reconfig_start(nn, pre_posted_requests);
237 		nfp_net_reconfig_wait(nn, jiffies + HZ * NFP_NET_POLL_TIMEOUT);
238 	}
239 }
240 
241 static void nfp_net_reconfig_wait_posted(struct nfp_net *nn)
242 {
243 	nfp_net_reconfig_sync_enter(nn);
244 
245 	spin_lock_bh(&nn->reconfig_lock);
246 	nn->reconfig_sync_present = false;
247 	spin_unlock_bh(&nn->reconfig_lock);
248 }
249 
250 /**
251  * __nfp_net_reconfig() - Reconfigure the firmware
252  * @nn:      NFP Net device to reconfigure
253  * @update:  The value for the update field in the BAR config
254  *
255  * Write the update word to the BAR and ping the reconfig queue.  The
256  * poll until the firmware has acknowledged the update by zeroing the
257  * update word.
258  *
259  * Return: Negative errno on error, 0 on success
260  */
261 int __nfp_net_reconfig(struct nfp_net *nn, u32 update)
262 {
263 	int ret;
264 
265 	nfp_net_reconfig_sync_enter(nn);
266 
267 	nfp_net_reconfig_start(nn, update);
268 	ret = nfp_net_reconfig_wait(nn, jiffies + HZ * NFP_NET_POLL_TIMEOUT);
269 
270 	spin_lock_bh(&nn->reconfig_lock);
271 
272 	if (nn->reconfig_posted)
273 		nfp_net_reconfig_start_async(nn, 0);
274 
275 	nn->reconfig_sync_present = false;
276 
277 	spin_unlock_bh(&nn->reconfig_lock);
278 
279 	return ret;
280 }
281 
282 int nfp_net_reconfig(struct nfp_net *nn, u32 update)
283 {
284 	int ret;
285 
286 	nn_ctrl_bar_lock(nn);
287 	ret = __nfp_net_reconfig(nn, update);
288 	nn_ctrl_bar_unlock(nn);
289 
290 	return ret;
291 }
292 
293 int nfp_net_mbox_lock(struct nfp_net *nn, unsigned int data_size)
294 {
295 	if (nn->tlv_caps.mbox_len < NFP_NET_CFG_MBOX_SIMPLE_VAL + data_size) {
296 		nn_err(nn, "mailbox too small for %u of data (%u)\n",
297 		       data_size, nn->tlv_caps.mbox_len);
298 		return -EIO;
299 	}
300 
301 	nn_ctrl_bar_lock(nn);
302 	return 0;
303 }
304 
305 /**
306  * nfp_net_mbox_reconfig() - Reconfigure the firmware via the mailbox
307  * @nn:        NFP Net device to reconfigure
308  * @mbox_cmd:  The value for the mailbox command
309  *
310  * Helper function for mailbox updates
311  *
312  * Return: Negative errno on error, 0 on success
313  */
314 int nfp_net_mbox_reconfig(struct nfp_net *nn, u32 mbox_cmd)
315 {
316 	u32 mbox = nn->tlv_caps.mbox_off;
317 	int ret;
318 
319 	nn_writeq(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_CMD, mbox_cmd);
320 
321 	ret = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_MBOX);
322 	if (ret) {
323 		nn_err(nn, "Mailbox update error\n");
324 		return ret;
325 	}
326 
327 	return -nn_readl(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_RET);
328 }
329 
330 void nfp_net_mbox_reconfig_post(struct nfp_net *nn, u32 mbox_cmd)
331 {
332 	u32 mbox = nn->tlv_caps.mbox_off;
333 
334 	nn_writeq(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_CMD, mbox_cmd);
335 
336 	nfp_net_reconfig_post(nn, NFP_NET_CFG_UPDATE_MBOX);
337 }
338 
339 int nfp_net_mbox_reconfig_wait_posted(struct nfp_net *nn)
340 {
341 	u32 mbox = nn->tlv_caps.mbox_off;
342 
343 	nfp_net_reconfig_wait_posted(nn);
344 
345 	return -nn_readl(nn, mbox + NFP_NET_CFG_MBOX_SIMPLE_RET);
346 }
347 
348 int nfp_net_mbox_reconfig_and_unlock(struct nfp_net *nn, u32 mbox_cmd)
349 {
350 	int ret;
351 
352 	ret = nfp_net_mbox_reconfig(nn, mbox_cmd);
353 	nn_ctrl_bar_unlock(nn);
354 	return ret;
355 }
356 
357 /* Interrupt configuration and handling
358  */
359 
360 /**
361  * nfp_net_irqs_alloc() - allocates MSI-X irqs
362  * @pdev:        PCI device structure
363  * @irq_entries: Array to be initialized and used to hold the irq entries
364  * @min_irqs:    Minimal acceptable number of interrupts
365  * @wanted_irqs: Target number of interrupts to allocate
366  *
367  * Return: Number of irqs obtained or 0 on error.
368  */
369 unsigned int
370 nfp_net_irqs_alloc(struct pci_dev *pdev, struct msix_entry *irq_entries,
371 		   unsigned int min_irqs, unsigned int wanted_irqs)
372 {
373 	unsigned int i;
374 	int got_irqs;
375 
376 	for (i = 0; i < wanted_irqs; i++)
377 		irq_entries[i].entry = i;
378 
379 	got_irqs = pci_enable_msix_range(pdev, irq_entries,
380 					 min_irqs, wanted_irqs);
381 	if (got_irqs < 0) {
382 		dev_err(&pdev->dev, "Failed to enable %d-%d MSI-X (err=%d)\n",
383 			min_irqs, wanted_irqs, got_irqs);
384 		return 0;
385 	}
386 
387 	if (got_irqs < wanted_irqs)
388 		dev_warn(&pdev->dev, "Unable to allocate %d IRQs got only %d\n",
389 			 wanted_irqs, got_irqs);
390 
391 	return got_irqs;
392 }
393 
394 /**
395  * nfp_net_irqs_assign() - Assign interrupts allocated externally to netdev
396  * @nn:		 NFP Network structure
397  * @irq_entries: Table of allocated interrupts
398  * @n:		 Size of @irq_entries (number of entries to grab)
399  *
400  * After interrupts are allocated with nfp_net_irqs_alloc() this function
401  * should be called to assign them to a specific netdev (port).
402  */
403 void
404 nfp_net_irqs_assign(struct nfp_net *nn, struct msix_entry *irq_entries,
405 		    unsigned int n)
406 {
407 	struct nfp_net_dp *dp = &nn->dp;
408 
409 	nn->max_r_vecs = n - NFP_NET_NON_Q_VECTORS;
410 	dp->num_r_vecs = nn->max_r_vecs;
411 
412 	memcpy(nn->irq_entries, irq_entries, sizeof(*irq_entries) * n);
413 
414 	if (dp->num_rx_rings > dp->num_r_vecs ||
415 	    dp->num_tx_rings > dp->num_r_vecs)
416 		dev_warn(nn->dp.dev, "More rings (%d,%d) than vectors (%d).\n",
417 			 dp->num_rx_rings, dp->num_tx_rings,
418 			 dp->num_r_vecs);
419 
420 	dp->num_rx_rings = min(dp->num_r_vecs, dp->num_rx_rings);
421 	dp->num_tx_rings = min(dp->num_r_vecs, dp->num_tx_rings);
422 	dp->num_stack_tx_rings = dp->num_tx_rings;
423 }
424 
425 /**
426  * nfp_net_irqs_disable() - Disable interrupts
427  * @pdev:        PCI device structure
428  *
429  * Undoes what @nfp_net_irqs_alloc() does.
430  */
431 void nfp_net_irqs_disable(struct pci_dev *pdev)
432 {
433 	pci_disable_msix(pdev);
434 }
435 
436 /**
437  * nfp_net_irq_rxtx() - Interrupt service routine for RX/TX rings.
438  * @irq:      Interrupt
439  * @data:     Opaque data structure
440  *
441  * Return: Indicate if the interrupt has been handled.
442  */
443 static irqreturn_t nfp_net_irq_rxtx(int irq, void *data)
444 {
445 	struct nfp_net_r_vector *r_vec = data;
446 
447 	/* Currently we cannot tell if it's a rx or tx interrupt,
448 	 * since dim does not need accurate event_ctr to calculate,
449 	 * we just use this counter for both rx and tx dim.
450 	 */
451 	r_vec->event_ctr++;
452 
453 	napi_schedule_irqoff(&r_vec->napi);
454 
455 	/* The FW auto-masks any interrupt, either via the MASK bit in
456 	 * the MSI-X table or via the per entry ICR field.  So there
457 	 * is no need to disable interrupts here.
458 	 */
459 	return IRQ_HANDLED;
460 }
461 
462 static irqreturn_t nfp_ctrl_irq_rxtx(int irq, void *data)
463 {
464 	struct nfp_net_r_vector *r_vec = data;
465 
466 	tasklet_schedule(&r_vec->tasklet);
467 
468 	return IRQ_HANDLED;
469 }
470 
471 /**
472  * nfp_net_read_link_status() - Reread link status from control BAR
473  * @nn:       NFP Network structure
474  */
475 static void nfp_net_read_link_status(struct nfp_net *nn)
476 {
477 	unsigned long flags;
478 	bool link_up;
479 	u16 sts;
480 
481 	spin_lock_irqsave(&nn->link_status_lock, flags);
482 
483 	sts = nn_readw(nn, NFP_NET_CFG_STS);
484 	link_up = !!(sts & NFP_NET_CFG_STS_LINK);
485 
486 	if (nn->link_up == link_up)
487 		goto out;
488 
489 	nn->link_up = link_up;
490 	if (nn->port) {
491 		set_bit(NFP_PORT_CHANGED, &nn->port->flags);
492 		if (nn->port->link_cb)
493 			nn->port->link_cb(nn->port);
494 	}
495 
496 	if (nn->link_up) {
497 		netif_carrier_on(nn->dp.netdev);
498 		netdev_info(nn->dp.netdev, "NIC Link is Up\n");
499 	} else {
500 		netif_carrier_off(nn->dp.netdev);
501 		netdev_info(nn->dp.netdev, "NIC Link is Down\n");
502 	}
503 out:
504 	spin_unlock_irqrestore(&nn->link_status_lock, flags);
505 }
506 
507 /**
508  * nfp_net_irq_lsc() - Interrupt service routine for link state changes
509  * @irq:      Interrupt
510  * @data:     Opaque data structure
511  *
512  * Return: Indicate if the interrupt has been handled.
513  */
514 static irqreturn_t nfp_net_irq_lsc(int irq, void *data)
515 {
516 	struct nfp_net *nn = data;
517 	struct msix_entry *entry;
518 
519 	entry = &nn->irq_entries[NFP_NET_IRQ_LSC_IDX];
520 
521 	nfp_net_read_link_status(nn);
522 
523 	nfp_net_irq_unmask(nn, entry->entry);
524 
525 	return IRQ_HANDLED;
526 }
527 
528 /**
529  * nfp_net_irq_exn() - Interrupt service routine for exceptions
530  * @irq:      Interrupt
531  * @data:     Opaque data structure
532  *
533  * Return: Indicate if the interrupt has been handled.
534  */
535 static irqreturn_t nfp_net_irq_exn(int irq, void *data)
536 {
537 	struct nfp_net *nn = data;
538 
539 	nn_err(nn, "%s: UNIMPLEMENTED.\n", __func__);
540 	/* XXX TO BE IMPLEMENTED */
541 	return IRQ_HANDLED;
542 }
543 
544 /**
545  * nfp_net_aux_irq_request() - Request an auxiliary interrupt (LSC or EXN)
546  * @nn:		NFP Network structure
547  * @ctrl_offset: Control BAR offset where IRQ configuration should be written
548  * @format:	printf-style format to construct the interrupt name
549  * @name:	Pointer to allocated space for interrupt name
550  * @name_sz:	Size of space for interrupt name
551  * @vector_idx:	Index of MSI-X vector used for this interrupt
552  * @handler:	IRQ handler to register for this interrupt
553  */
554 static int
555 nfp_net_aux_irq_request(struct nfp_net *nn, u32 ctrl_offset,
556 			const char *format, char *name, size_t name_sz,
557 			unsigned int vector_idx, irq_handler_t handler)
558 {
559 	struct msix_entry *entry;
560 	int err;
561 
562 	entry = &nn->irq_entries[vector_idx];
563 
564 	snprintf(name, name_sz, format, nfp_net_name(nn));
565 	err = request_irq(entry->vector, handler, 0, name, nn);
566 	if (err) {
567 		nn_err(nn, "Failed to request IRQ %d (err=%d).\n",
568 		       entry->vector, err);
569 		return err;
570 	}
571 	nn_writeb(nn, ctrl_offset, entry->entry);
572 	nfp_net_irq_unmask(nn, entry->entry);
573 
574 	return 0;
575 }
576 
577 /**
578  * nfp_net_aux_irq_free() - Free an auxiliary interrupt (LSC or EXN)
579  * @nn:		NFP Network structure
580  * @ctrl_offset: Control BAR offset where IRQ configuration should be written
581  * @vector_idx:	Index of MSI-X vector used for this interrupt
582  */
583 static void nfp_net_aux_irq_free(struct nfp_net *nn, u32 ctrl_offset,
584 				 unsigned int vector_idx)
585 {
586 	nn_writeb(nn, ctrl_offset, 0xff);
587 	nn_pci_flush(nn);
588 	free_irq(nn->irq_entries[vector_idx].vector, nn);
589 }
590 
591 struct sk_buff *
592 nfp_net_tls_tx(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec,
593 	       struct sk_buff *skb, u64 *tls_handle, int *nr_frags)
594 {
595 #ifdef CONFIG_TLS_DEVICE
596 	struct nfp_net_tls_offload_ctx *ntls;
597 	struct sk_buff *nskb;
598 	bool resync_pending;
599 	u32 datalen, seq;
600 
601 	if (likely(!dp->ktls_tx))
602 		return skb;
603 	if (!tls_is_skb_tx_device_offloaded(skb))
604 		return skb;
605 
606 	datalen = skb->len - skb_tcp_all_headers(skb);
607 	seq = ntohl(tcp_hdr(skb)->seq);
608 	ntls = tls_driver_ctx(skb->sk, TLS_OFFLOAD_CTX_DIR_TX);
609 	resync_pending = tls_offload_tx_resync_pending(skb->sk);
610 	if (unlikely(resync_pending || ntls->next_seq != seq)) {
611 		/* Pure ACK out of order already */
612 		if (!datalen)
613 			return skb;
614 
615 		u64_stats_update_begin(&r_vec->tx_sync);
616 		r_vec->tls_tx_fallback++;
617 		u64_stats_update_end(&r_vec->tx_sync);
618 
619 		nskb = tls_encrypt_skb(skb);
620 		if (!nskb) {
621 			u64_stats_update_begin(&r_vec->tx_sync);
622 			r_vec->tls_tx_no_fallback++;
623 			u64_stats_update_end(&r_vec->tx_sync);
624 			return NULL;
625 		}
626 		/* encryption wasn't necessary */
627 		if (nskb == skb)
628 			return skb;
629 		/* we don't re-check ring space */
630 		if (unlikely(skb_is_nonlinear(nskb))) {
631 			nn_dp_warn(dp, "tls_encrypt_skb() produced fragmented frame\n");
632 			u64_stats_update_begin(&r_vec->tx_sync);
633 			r_vec->tx_errors++;
634 			u64_stats_update_end(&r_vec->tx_sync);
635 			dev_kfree_skb_any(nskb);
636 			return NULL;
637 		}
638 
639 		/* jump forward, a TX may have gotten lost, need to sync TX */
640 		if (!resync_pending && seq - ntls->next_seq < U32_MAX / 4)
641 			tls_offload_tx_resync_request(nskb->sk, seq,
642 						      ntls->next_seq);
643 
644 		*nr_frags = 0;
645 		return nskb;
646 	}
647 
648 	if (datalen) {
649 		u64_stats_update_begin(&r_vec->tx_sync);
650 		if (!skb_is_gso(skb))
651 			r_vec->hw_tls_tx++;
652 		else
653 			r_vec->hw_tls_tx += skb_shinfo(skb)->gso_segs;
654 		u64_stats_update_end(&r_vec->tx_sync);
655 	}
656 
657 	memcpy(tls_handle, ntls->fw_handle, sizeof(ntls->fw_handle));
658 	ntls->next_seq += datalen;
659 #endif
660 	return skb;
661 }
662 
663 void nfp_net_tls_tx_undo(struct sk_buff *skb, u64 tls_handle)
664 {
665 #ifdef CONFIG_TLS_DEVICE
666 	struct nfp_net_tls_offload_ctx *ntls;
667 	u32 datalen, seq;
668 
669 	if (!tls_handle)
670 		return;
671 	if (WARN_ON_ONCE(!tls_is_skb_tx_device_offloaded(skb)))
672 		return;
673 
674 	datalen = skb->len - skb_tcp_all_headers(skb);
675 	seq = ntohl(tcp_hdr(skb)->seq);
676 
677 	ntls = tls_driver_ctx(skb->sk, TLS_OFFLOAD_CTX_DIR_TX);
678 	if (ntls->next_seq == seq + datalen)
679 		ntls->next_seq = seq;
680 	else
681 		WARN_ON_ONCE(1);
682 #endif
683 }
684 
685 static void nfp_net_tx_timeout(struct net_device *netdev, unsigned int txqueue)
686 {
687 	struct nfp_net *nn = netdev_priv(netdev);
688 
689 	nn_warn(nn, "TX watchdog timeout on ring: %u\n", txqueue);
690 }
691 
692 /* Receive processing */
693 static unsigned int
694 nfp_net_calc_fl_bufsz_data(struct nfp_net_dp *dp)
695 {
696 	unsigned int fl_bufsz = 0;
697 
698 	if (dp->rx_offset == NFP_NET_CFG_RX_OFFSET_DYNAMIC)
699 		fl_bufsz += NFP_NET_MAX_PREPEND;
700 	else
701 		fl_bufsz += dp->rx_offset;
702 	fl_bufsz += ETH_HLEN + VLAN_HLEN * 2 + dp->mtu;
703 
704 	return fl_bufsz;
705 }
706 
707 static unsigned int nfp_net_calc_fl_bufsz(struct nfp_net_dp *dp)
708 {
709 	unsigned int fl_bufsz;
710 
711 	fl_bufsz = NFP_NET_RX_BUF_HEADROOM;
712 	fl_bufsz += dp->rx_dma_off;
713 	fl_bufsz += nfp_net_calc_fl_bufsz_data(dp);
714 
715 	fl_bufsz = SKB_DATA_ALIGN(fl_bufsz);
716 	fl_bufsz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
717 
718 	return fl_bufsz;
719 }
720 
721 static unsigned int nfp_net_calc_fl_bufsz_xsk(struct nfp_net_dp *dp)
722 {
723 	unsigned int fl_bufsz;
724 
725 	fl_bufsz = XDP_PACKET_HEADROOM;
726 	fl_bufsz += nfp_net_calc_fl_bufsz_data(dp);
727 
728 	return fl_bufsz;
729 }
730 
731 /* Setup and Configuration
732  */
733 
734 /**
735  * nfp_net_vecs_init() - Assign IRQs and setup rvecs.
736  * @nn:		NFP Network structure
737  */
738 static void nfp_net_vecs_init(struct nfp_net *nn)
739 {
740 	int numa_node = dev_to_node(&nn->pdev->dev);
741 	struct nfp_net_r_vector *r_vec;
742 	unsigned int r;
743 
744 	nn->lsc_handler = nfp_net_irq_lsc;
745 	nn->exn_handler = nfp_net_irq_exn;
746 
747 	for (r = 0; r < nn->max_r_vecs; r++) {
748 		struct msix_entry *entry;
749 
750 		entry = &nn->irq_entries[NFP_NET_NON_Q_VECTORS + r];
751 
752 		r_vec = &nn->r_vecs[r];
753 		r_vec->nfp_net = nn;
754 		r_vec->irq_entry = entry->entry;
755 		r_vec->irq_vector = entry->vector;
756 
757 		if (nn->dp.netdev) {
758 			r_vec->handler = nfp_net_irq_rxtx;
759 		} else {
760 			r_vec->handler = nfp_ctrl_irq_rxtx;
761 
762 			__skb_queue_head_init(&r_vec->queue);
763 			spin_lock_init(&r_vec->lock);
764 			tasklet_setup(&r_vec->tasklet, nn->dp.ops->ctrl_poll);
765 			tasklet_disable(&r_vec->tasklet);
766 		}
767 
768 		cpumask_set_cpu(cpumask_local_spread(r, numa_node), &r_vec->affinity_mask);
769 	}
770 }
771 
772 static void
773 nfp_net_napi_add(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec, int idx)
774 {
775 	if (dp->netdev)
776 		netif_napi_add(dp->netdev, &r_vec->napi,
777 			       nfp_net_has_xsk_pool_slow(dp, idx) ? dp->ops->xsk_poll : dp->ops->poll);
778 	else
779 		tasklet_enable(&r_vec->tasklet);
780 }
781 
782 static void
783 nfp_net_napi_del(struct nfp_net_dp *dp, struct nfp_net_r_vector *r_vec)
784 {
785 	if (dp->netdev)
786 		netif_napi_del(&r_vec->napi);
787 	else
788 		tasklet_disable(&r_vec->tasklet);
789 }
790 
791 static void
792 nfp_net_vector_assign_rings(struct nfp_net_dp *dp,
793 			    struct nfp_net_r_vector *r_vec, int idx)
794 {
795 	r_vec->rx_ring = idx < dp->num_rx_rings ? &dp->rx_rings[idx] : NULL;
796 	r_vec->tx_ring =
797 		idx < dp->num_stack_tx_rings ? &dp->tx_rings[idx] : NULL;
798 
799 	r_vec->xdp_ring = idx < dp->num_tx_rings - dp->num_stack_tx_rings ?
800 		&dp->tx_rings[dp->num_stack_tx_rings + idx] : NULL;
801 
802 	if (nfp_net_has_xsk_pool_slow(dp, idx) || r_vec->xsk_pool) {
803 		r_vec->xsk_pool = dp->xdp_prog ? dp->xsk_pools[idx] : NULL;
804 
805 		if (r_vec->xsk_pool)
806 			xsk_pool_set_rxq_info(r_vec->xsk_pool,
807 					      &r_vec->rx_ring->xdp_rxq);
808 
809 		nfp_net_napi_del(dp, r_vec);
810 		nfp_net_napi_add(dp, r_vec, idx);
811 	}
812 }
813 
814 static int
815 nfp_net_prepare_vector(struct nfp_net *nn, struct nfp_net_r_vector *r_vec,
816 		       int idx)
817 {
818 	int err;
819 
820 	nfp_net_napi_add(&nn->dp, r_vec, idx);
821 
822 	snprintf(r_vec->name, sizeof(r_vec->name),
823 		 "%s-rxtx-%d", nfp_net_name(nn), idx);
824 	err = request_irq(r_vec->irq_vector, r_vec->handler, 0, r_vec->name,
825 			  r_vec);
826 	if (err) {
827 		nfp_net_napi_del(&nn->dp, r_vec);
828 		nn_err(nn, "Error requesting IRQ %d\n", r_vec->irq_vector);
829 		return err;
830 	}
831 	disable_irq(r_vec->irq_vector);
832 
833 	irq_set_affinity_hint(r_vec->irq_vector, &r_vec->affinity_mask);
834 
835 	nn_dbg(nn, "RV%02d: irq=%03d/%03d\n", idx, r_vec->irq_vector,
836 	       r_vec->irq_entry);
837 
838 	return 0;
839 }
840 
841 static void
842 nfp_net_cleanup_vector(struct nfp_net *nn, struct nfp_net_r_vector *r_vec)
843 {
844 	irq_set_affinity_hint(r_vec->irq_vector, NULL);
845 	nfp_net_napi_del(&nn->dp, r_vec);
846 	free_irq(r_vec->irq_vector, r_vec);
847 }
848 
849 /**
850  * nfp_net_rss_write_itbl() - Write RSS indirection table to device
851  * @nn:      NFP Net device to reconfigure
852  */
853 void nfp_net_rss_write_itbl(struct nfp_net *nn)
854 {
855 	int i;
856 
857 	for (i = 0; i < NFP_NET_CFG_RSS_ITBL_SZ; i += 4)
858 		nn_writel(nn, NFP_NET_CFG_RSS_ITBL + i,
859 			  get_unaligned_le32(nn->rss_itbl + i));
860 }
861 
862 /**
863  * nfp_net_rss_write_key() - Write RSS hash key to device
864  * @nn:      NFP Net device to reconfigure
865  */
866 void nfp_net_rss_write_key(struct nfp_net *nn)
867 {
868 	int i;
869 
870 	for (i = 0; i < nfp_net_rss_key_sz(nn); i += 4)
871 		nn_writel(nn, NFP_NET_CFG_RSS_KEY + i,
872 			  get_unaligned_le32(nn->rss_key + i));
873 }
874 
875 /**
876  * nfp_net_coalesce_write_cfg() - Write irq coalescence configuration to HW
877  * @nn:      NFP Net device to reconfigure
878  */
879 void nfp_net_coalesce_write_cfg(struct nfp_net *nn)
880 {
881 	u8 i;
882 	u32 factor;
883 	u32 value;
884 
885 	/* Compute factor used to convert coalesce '_usecs' parameters to
886 	 * ME timestamp ticks.  There are 16 ME clock cycles for each timestamp
887 	 * count.
888 	 */
889 	factor = nn->tlv_caps.me_freq_mhz / 16;
890 
891 	/* copy RX interrupt coalesce parameters */
892 	value = (nn->rx_coalesce_max_frames << 16) |
893 		(factor * nn->rx_coalesce_usecs);
894 	for (i = 0; i < nn->dp.num_rx_rings; i++)
895 		nn_writel(nn, NFP_NET_CFG_RXR_IRQ_MOD(i), value);
896 
897 	/* copy TX interrupt coalesce parameters */
898 	value = (nn->tx_coalesce_max_frames << 16) |
899 		(factor * nn->tx_coalesce_usecs);
900 	for (i = 0; i < nn->dp.num_tx_rings; i++)
901 		nn_writel(nn, NFP_NET_CFG_TXR_IRQ_MOD(i), value);
902 }
903 
904 /**
905  * nfp_net_write_mac_addr() - Write mac address to the device control BAR
906  * @nn:      NFP Net device to reconfigure
907  * @addr:    MAC address to write
908  *
909  * Writes the MAC address from the netdev to the device control BAR.  Does not
910  * perform the required reconfig.  We do a bit of byte swapping dance because
911  * firmware is LE.
912  */
913 static void nfp_net_write_mac_addr(struct nfp_net *nn, const u8 *addr)
914 {
915 	nn_writel(nn, NFP_NET_CFG_MACADDR + 0, get_unaligned_be32(addr));
916 	nn_writew(nn, NFP_NET_CFG_MACADDR + 6, get_unaligned_be16(addr + 4));
917 }
918 
919 /**
920  * nfp_net_clear_config_and_disable() - Clear control BAR and disable NFP
921  * @nn:      NFP Net device to reconfigure
922  *
923  * Warning: must be fully idempotent.
924  */
925 static void nfp_net_clear_config_and_disable(struct nfp_net *nn)
926 {
927 	u32 new_ctrl, new_ctrl_w1, update;
928 	unsigned int r;
929 	int err;
930 
931 	new_ctrl = nn->dp.ctrl;
932 	new_ctrl &= ~NFP_NET_CFG_CTRL_ENABLE;
933 	update = NFP_NET_CFG_UPDATE_GEN;
934 	update |= NFP_NET_CFG_UPDATE_MSIX;
935 	update |= NFP_NET_CFG_UPDATE_RING;
936 
937 	if (nn->cap & NFP_NET_CFG_CTRL_RINGCFG)
938 		new_ctrl &= ~NFP_NET_CFG_CTRL_RINGCFG;
939 
940 	if (!(nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN)) {
941 		nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0);
942 		nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0);
943 	}
944 
945 	nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);
946 	err = nfp_net_reconfig(nn, update);
947 	if (err)
948 		nn_err(nn, "Could not disable device: %d\n", err);
949 
950 	if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN) {
951 		new_ctrl_w1 = nn->dp.ctrl_w1;
952 		new_ctrl_w1 &= ~NFP_NET_CFG_CTRL_FREELIST_EN;
953 		nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0);
954 		nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0);
955 
956 		nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1);
957 		err = nfp_net_reconfig(nn, update);
958 		if (err)
959 			nn_err(nn, "Could not disable FREELIST_EN: %d\n", err);
960 		nn->dp.ctrl_w1 = new_ctrl_w1;
961 	}
962 
963 	for (r = 0; r < nn->dp.num_rx_rings; r++) {
964 		nfp_net_rx_ring_reset(&nn->dp.rx_rings[r]);
965 		if (nfp_net_has_xsk_pool_slow(&nn->dp, nn->dp.rx_rings[r].idx))
966 			nfp_net_xsk_rx_bufs_free(&nn->dp.rx_rings[r]);
967 	}
968 	for (r = 0; r < nn->dp.num_tx_rings; r++)
969 		nfp_net_tx_ring_reset(&nn->dp, &nn->dp.tx_rings[r]);
970 	for (r = 0; r < nn->dp.num_r_vecs; r++)
971 		nfp_net_vec_clear_ring_data(nn, r);
972 
973 	nn->dp.ctrl = new_ctrl;
974 }
975 
976 /**
977  * nfp_net_set_config_and_enable() - Write control BAR and enable NFP
978  * @nn:      NFP Net device to reconfigure
979  */
980 static int nfp_net_set_config_and_enable(struct nfp_net *nn)
981 {
982 	u32 bufsz, new_ctrl, new_ctrl_w1, update = 0;
983 	unsigned int r;
984 	int err;
985 
986 	new_ctrl = nn->dp.ctrl;
987 	new_ctrl_w1 = nn->dp.ctrl_w1;
988 
989 	if (nn->dp.ctrl & NFP_NET_CFG_CTRL_RSS_ANY) {
990 		nfp_net_rss_write_key(nn);
991 		nfp_net_rss_write_itbl(nn);
992 		nn_writel(nn, NFP_NET_CFG_RSS_CTRL, nn->rss_cfg);
993 		update |= NFP_NET_CFG_UPDATE_RSS;
994 	}
995 
996 	if (nn->dp.ctrl & NFP_NET_CFG_CTRL_IRQMOD) {
997 		nfp_net_coalesce_write_cfg(nn);
998 		update |= NFP_NET_CFG_UPDATE_IRQMOD;
999 	}
1000 
1001 	for (r = 0; r < nn->dp.num_tx_rings; r++)
1002 		nfp_net_tx_ring_hw_cfg_write(nn, &nn->dp.tx_rings[r], r);
1003 	for (r = 0; r < nn->dp.num_rx_rings; r++)
1004 		nfp_net_rx_ring_hw_cfg_write(nn, &nn->dp.rx_rings[r], r);
1005 
1006 	nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE,
1007 		  U64_MAX >> (64 - nn->dp.num_tx_rings));
1008 
1009 	nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE,
1010 		  U64_MAX >> (64 - nn->dp.num_rx_rings));
1011 
1012 	if (nn->dp.netdev)
1013 		nfp_net_write_mac_addr(nn, nn->dp.netdev->dev_addr);
1014 
1015 	nn_writel(nn, NFP_NET_CFG_MTU, nn->dp.mtu);
1016 
1017 	bufsz = nn->dp.fl_bufsz - nn->dp.rx_dma_off - NFP_NET_RX_BUF_NON_DATA;
1018 	nn_writel(nn, NFP_NET_CFG_FLBUFSZ, bufsz);
1019 
1020 	/* Enable device
1021 	 * Step 1: Replace the CTRL_ENABLE by NFP_NET_CFG_CTRL_FREELIST_EN if
1022 	 * FREELIST_EN exits.
1023 	 */
1024 	if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN)
1025 		new_ctrl_w1 |= NFP_NET_CFG_CTRL_FREELIST_EN;
1026 	else
1027 		new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
1028 	update |= NFP_NET_CFG_UPDATE_GEN;
1029 	update |= NFP_NET_CFG_UPDATE_MSIX;
1030 	update |= NFP_NET_CFG_UPDATE_RING;
1031 	if (nn->cap & NFP_NET_CFG_CTRL_RINGCFG)
1032 		new_ctrl |= NFP_NET_CFG_CTRL_RINGCFG;
1033 
1034 	/* Step 2: Send the configuration and write the freelist.
1035 	 * - The freelist only need to be written once.
1036 	 */
1037 	nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);
1038 	nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1);
1039 	err = nfp_net_reconfig(nn, update);
1040 	if (err) {
1041 		nfp_net_clear_config_and_disable(nn);
1042 		return err;
1043 	}
1044 
1045 	nn->dp.ctrl = new_ctrl;
1046 	nn->dp.ctrl_w1 = new_ctrl_w1;
1047 
1048 	for (r = 0; r < nn->dp.num_rx_rings; r++)
1049 		nfp_net_rx_ring_fill_freelist(&nn->dp, &nn->dp.rx_rings[r]);
1050 
1051 	/* Step 3: Do the NFP_NET_CFG_CTRL_ENABLE. Send the configuration.
1052 	 */
1053 	if (nn->cap_w1 & NFP_NET_CFG_CTRL_FREELIST_EN) {
1054 		new_ctrl |= NFP_NET_CFG_CTRL_ENABLE;
1055 		nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);
1056 
1057 		err = nfp_net_reconfig(nn, update);
1058 		if (err) {
1059 			nfp_net_clear_config_and_disable(nn);
1060 			return err;
1061 		}
1062 		nn->dp.ctrl = new_ctrl;
1063 	}
1064 
1065 	return 0;
1066 }
1067 
1068 /**
1069  * nfp_net_close_stack() - Quiesce the stack (part of close)
1070  * @nn:	     NFP Net device to reconfigure
1071  */
1072 static void nfp_net_close_stack(struct nfp_net *nn)
1073 {
1074 	struct nfp_net_r_vector *r_vec;
1075 	unsigned int r;
1076 
1077 	disable_irq(nn->irq_entries[NFP_NET_IRQ_LSC_IDX].vector);
1078 	netif_carrier_off(nn->dp.netdev);
1079 	nn->link_up = false;
1080 
1081 	for (r = 0; r < nn->dp.num_r_vecs; r++) {
1082 		r_vec = &nn->r_vecs[r];
1083 
1084 		disable_irq(r_vec->irq_vector);
1085 		napi_disable(&r_vec->napi);
1086 
1087 		if (r_vec->rx_ring)
1088 			cancel_work_sync(&r_vec->rx_dim.work);
1089 
1090 		if (r_vec->tx_ring)
1091 			cancel_work_sync(&r_vec->tx_dim.work);
1092 	}
1093 
1094 	netif_tx_disable(nn->dp.netdev);
1095 }
1096 
1097 /**
1098  * nfp_net_close_free_all() - Free all runtime resources
1099  * @nn:      NFP Net device to reconfigure
1100  */
1101 static void nfp_net_close_free_all(struct nfp_net *nn)
1102 {
1103 	unsigned int r;
1104 
1105 	nfp_net_tx_rings_free(&nn->dp);
1106 	nfp_net_rx_rings_free(&nn->dp);
1107 
1108 	for (r = 0; r < nn->dp.num_r_vecs; r++)
1109 		nfp_net_cleanup_vector(nn, &nn->r_vecs[r]);
1110 
1111 	nfp_net_aux_irq_free(nn, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
1112 	nfp_net_aux_irq_free(nn, NFP_NET_CFG_EXN, NFP_NET_IRQ_EXN_IDX);
1113 }
1114 
1115 /**
1116  * nfp_net_netdev_close() - Called when the device is downed
1117  * @netdev:      netdev structure
1118  */
1119 static int nfp_net_netdev_close(struct net_device *netdev)
1120 {
1121 	struct nfp_net *nn = netdev_priv(netdev);
1122 
1123 	/* Step 1: Disable RX and TX rings from the Linux kernel perspective
1124 	 */
1125 	nfp_net_close_stack(nn);
1126 
1127 	/* Step 2: Tell NFP
1128 	 */
1129 	if (nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER)
1130 		__dev_mc_unsync(netdev, nfp_net_mc_unsync);
1131 
1132 	nfp_net_clear_config_and_disable(nn);
1133 	nfp_port_configure(netdev, false);
1134 
1135 	/* Step 3: Free resources
1136 	 */
1137 	nfp_net_close_free_all(nn);
1138 
1139 	nn_dbg(nn, "%s down", netdev->name);
1140 	return 0;
1141 }
1142 
1143 void nfp_ctrl_close(struct nfp_net *nn)
1144 {
1145 	int r;
1146 
1147 	rtnl_lock();
1148 
1149 	for (r = 0; r < nn->dp.num_r_vecs; r++) {
1150 		disable_irq(nn->r_vecs[r].irq_vector);
1151 		tasklet_disable(&nn->r_vecs[r].tasklet);
1152 	}
1153 
1154 	nfp_net_clear_config_and_disable(nn);
1155 
1156 	nfp_net_close_free_all(nn);
1157 
1158 	rtnl_unlock();
1159 }
1160 
1161 static void nfp_net_rx_dim_work(struct work_struct *work)
1162 {
1163 	struct nfp_net_r_vector *r_vec;
1164 	unsigned int factor, value;
1165 	struct dim_cq_moder moder;
1166 	struct nfp_net *nn;
1167 	struct dim *dim;
1168 
1169 	dim = container_of(work, struct dim, work);
1170 	moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1171 	r_vec = container_of(dim, struct nfp_net_r_vector, rx_dim);
1172 	nn = r_vec->nfp_net;
1173 
1174 	/* Compute factor used to convert coalesce '_usecs' parameters to
1175 	 * ME timestamp ticks.  There are 16 ME clock cycles for each timestamp
1176 	 * count.
1177 	 */
1178 	factor = nn->tlv_caps.me_freq_mhz / 16;
1179 	if (nfp_net_coalesce_para_check(factor * moder.usec, moder.pkts))
1180 		return;
1181 
1182 	/* copy RX interrupt coalesce parameters */
1183 	value = (moder.pkts << 16) | (factor * moder.usec);
1184 	nn_writel(nn, NFP_NET_CFG_RXR_IRQ_MOD(r_vec->rx_ring->idx), value);
1185 	(void)nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_IRQMOD);
1186 
1187 	dim->state = DIM_START_MEASURE;
1188 }
1189 
1190 static void nfp_net_tx_dim_work(struct work_struct *work)
1191 {
1192 	struct nfp_net_r_vector *r_vec;
1193 	unsigned int factor, value;
1194 	struct dim_cq_moder moder;
1195 	struct nfp_net *nn;
1196 	struct dim *dim;
1197 
1198 	dim = container_of(work, struct dim, work);
1199 	moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
1200 	r_vec = container_of(dim, struct nfp_net_r_vector, tx_dim);
1201 	nn = r_vec->nfp_net;
1202 
1203 	/* Compute factor used to convert coalesce '_usecs' parameters to
1204 	 * ME timestamp ticks.  There are 16 ME clock cycles for each timestamp
1205 	 * count.
1206 	 */
1207 	factor = nn->tlv_caps.me_freq_mhz / 16;
1208 	if (nfp_net_coalesce_para_check(factor * moder.usec, moder.pkts))
1209 		return;
1210 
1211 	/* copy TX interrupt coalesce parameters */
1212 	value = (moder.pkts << 16) | (factor * moder.usec);
1213 	nn_writel(nn, NFP_NET_CFG_TXR_IRQ_MOD(r_vec->tx_ring->idx), value);
1214 	(void)nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_IRQMOD);
1215 
1216 	dim->state = DIM_START_MEASURE;
1217 }
1218 
1219 /**
1220  * nfp_net_open_stack() - Start the device from stack's perspective
1221  * @nn:      NFP Net device to reconfigure
1222  */
1223 static void nfp_net_open_stack(struct nfp_net *nn)
1224 {
1225 	struct nfp_net_r_vector *r_vec;
1226 	unsigned int r;
1227 
1228 	for (r = 0; r < nn->dp.num_r_vecs; r++) {
1229 		r_vec = &nn->r_vecs[r];
1230 
1231 		if (r_vec->rx_ring) {
1232 			INIT_WORK(&r_vec->rx_dim.work, nfp_net_rx_dim_work);
1233 			r_vec->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1234 		}
1235 
1236 		if (r_vec->tx_ring) {
1237 			INIT_WORK(&r_vec->tx_dim.work, nfp_net_tx_dim_work);
1238 			r_vec->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1239 		}
1240 
1241 		napi_enable(&r_vec->napi);
1242 		enable_irq(r_vec->irq_vector);
1243 	}
1244 
1245 	netif_tx_wake_all_queues(nn->dp.netdev);
1246 
1247 	enable_irq(nn->irq_entries[NFP_NET_IRQ_LSC_IDX].vector);
1248 	nfp_net_read_link_status(nn);
1249 }
1250 
1251 static int nfp_net_open_alloc_all(struct nfp_net *nn)
1252 {
1253 	int err, r;
1254 
1255 	err = nfp_net_aux_irq_request(nn, NFP_NET_CFG_EXN, "%s-exn",
1256 				      nn->exn_name, sizeof(nn->exn_name),
1257 				      NFP_NET_IRQ_EXN_IDX, nn->exn_handler);
1258 	if (err)
1259 		return err;
1260 	err = nfp_net_aux_irq_request(nn, NFP_NET_CFG_LSC, "%s-lsc",
1261 				      nn->lsc_name, sizeof(nn->lsc_name),
1262 				      NFP_NET_IRQ_LSC_IDX, nn->lsc_handler);
1263 	if (err)
1264 		goto err_free_exn;
1265 	disable_irq(nn->irq_entries[NFP_NET_IRQ_LSC_IDX].vector);
1266 
1267 	for (r = 0; r < nn->dp.num_r_vecs; r++) {
1268 		err = nfp_net_prepare_vector(nn, &nn->r_vecs[r], r);
1269 		if (err)
1270 			goto err_cleanup_vec_p;
1271 	}
1272 
1273 	err = nfp_net_rx_rings_prepare(nn, &nn->dp);
1274 	if (err)
1275 		goto err_cleanup_vec;
1276 
1277 	err = nfp_net_tx_rings_prepare(nn, &nn->dp);
1278 	if (err)
1279 		goto err_free_rx_rings;
1280 
1281 	for (r = 0; r < nn->max_r_vecs; r++)
1282 		nfp_net_vector_assign_rings(&nn->dp, &nn->r_vecs[r], r);
1283 
1284 	return 0;
1285 
1286 err_free_rx_rings:
1287 	nfp_net_rx_rings_free(&nn->dp);
1288 err_cleanup_vec:
1289 	r = nn->dp.num_r_vecs;
1290 err_cleanup_vec_p:
1291 	while (r--)
1292 		nfp_net_cleanup_vector(nn, &nn->r_vecs[r]);
1293 	nfp_net_aux_irq_free(nn, NFP_NET_CFG_LSC, NFP_NET_IRQ_LSC_IDX);
1294 err_free_exn:
1295 	nfp_net_aux_irq_free(nn, NFP_NET_CFG_EXN, NFP_NET_IRQ_EXN_IDX);
1296 	return err;
1297 }
1298 
1299 static int nfp_net_netdev_open(struct net_device *netdev)
1300 {
1301 	struct nfp_net *nn = netdev_priv(netdev);
1302 	int err;
1303 
1304 	/* Step 1: Allocate resources for rings and the like
1305 	 * - Request interrupts
1306 	 * - Allocate RX and TX ring resources
1307 	 * - Setup initial RSS table
1308 	 */
1309 	err = nfp_net_open_alloc_all(nn);
1310 	if (err)
1311 		return err;
1312 
1313 	err = netif_set_real_num_tx_queues(netdev, nn->dp.num_stack_tx_rings);
1314 	if (err)
1315 		goto err_free_all;
1316 
1317 	err = netif_set_real_num_rx_queues(netdev, nn->dp.num_rx_rings);
1318 	if (err)
1319 		goto err_free_all;
1320 
1321 	/* Step 2: Configure the NFP
1322 	 * - Ifup the physical interface if it exists
1323 	 * - Enable rings from 0 to tx_rings/rx_rings - 1.
1324 	 * - Write MAC address (in case it changed)
1325 	 * - Set the MTU
1326 	 * - Set the Freelist buffer size
1327 	 * - Enable the FW
1328 	 */
1329 	err = nfp_port_configure(netdev, true);
1330 	if (err)
1331 		goto err_free_all;
1332 
1333 	err = nfp_net_set_config_and_enable(nn);
1334 	if (err)
1335 		goto err_port_disable;
1336 
1337 	/* Step 3: Enable for kernel
1338 	 * - put some freelist descriptors on each RX ring
1339 	 * - enable NAPI on each ring
1340 	 * - enable all TX queues
1341 	 * - set link state
1342 	 */
1343 	nfp_net_open_stack(nn);
1344 
1345 	return 0;
1346 
1347 err_port_disable:
1348 	nfp_port_configure(netdev, false);
1349 err_free_all:
1350 	nfp_net_close_free_all(nn);
1351 	return err;
1352 }
1353 
1354 int nfp_ctrl_open(struct nfp_net *nn)
1355 {
1356 	int err, r;
1357 
1358 	/* ring dumping depends on vNICs being opened/closed under rtnl */
1359 	rtnl_lock();
1360 
1361 	err = nfp_net_open_alloc_all(nn);
1362 	if (err)
1363 		goto err_unlock;
1364 
1365 	err = nfp_net_set_config_and_enable(nn);
1366 	if (err)
1367 		goto err_free_all;
1368 
1369 	for (r = 0; r < nn->dp.num_r_vecs; r++)
1370 		enable_irq(nn->r_vecs[r].irq_vector);
1371 
1372 	rtnl_unlock();
1373 
1374 	return 0;
1375 
1376 err_free_all:
1377 	nfp_net_close_free_all(nn);
1378 err_unlock:
1379 	rtnl_unlock();
1380 	return err;
1381 }
1382 
1383 int nfp_net_sched_mbox_amsg_work(struct nfp_net *nn, u32 cmd, const void *data, size_t len,
1384 				 int (*cb)(struct nfp_net *, struct nfp_mbox_amsg_entry *))
1385 {
1386 	struct nfp_mbox_amsg_entry *entry;
1387 
1388 	entry = kmalloc(sizeof(*entry) + len, GFP_ATOMIC);
1389 	if (!entry)
1390 		return -ENOMEM;
1391 
1392 	memcpy(entry->msg, data, len);
1393 	entry->cmd = cmd;
1394 	entry->cfg = cb;
1395 
1396 	spin_lock_bh(&nn->mbox_amsg.lock);
1397 	list_add_tail(&entry->list, &nn->mbox_amsg.list);
1398 	spin_unlock_bh(&nn->mbox_amsg.lock);
1399 
1400 	schedule_work(&nn->mbox_amsg.work);
1401 
1402 	return 0;
1403 }
1404 
1405 static void nfp_net_mbox_amsg_work(struct work_struct *work)
1406 {
1407 	struct nfp_net *nn = container_of(work, struct nfp_net, mbox_amsg.work);
1408 	struct nfp_mbox_amsg_entry *entry, *tmp;
1409 	struct list_head tmp_list;
1410 
1411 	INIT_LIST_HEAD(&tmp_list);
1412 
1413 	spin_lock_bh(&nn->mbox_amsg.lock);
1414 	list_splice_init(&nn->mbox_amsg.list, &tmp_list);
1415 	spin_unlock_bh(&nn->mbox_amsg.lock);
1416 
1417 	list_for_each_entry_safe(entry, tmp, &tmp_list, list) {
1418 		int err = entry->cfg(nn, entry);
1419 
1420 		if (err)
1421 			nn_err(nn, "Config cmd %d to HW failed %d.\n", entry->cmd, err);
1422 
1423 		list_del(&entry->list);
1424 		kfree(entry);
1425 	}
1426 }
1427 
1428 static int nfp_net_mc_cfg(struct nfp_net *nn, struct nfp_mbox_amsg_entry *entry)
1429 {
1430 	unsigned char *addr = entry->msg;
1431 	int ret;
1432 
1433 	ret = nfp_net_mbox_lock(nn, NFP_NET_CFG_MULTICAST_SZ);
1434 	if (ret)
1435 		return ret;
1436 
1437 	nn_writel(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_MULTICAST_MAC_HI,
1438 		  get_unaligned_be32(addr));
1439 	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_MULTICAST_MAC_LO,
1440 		  get_unaligned_be16(addr + 4));
1441 
1442 	return nfp_net_mbox_reconfig_and_unlock(nn, entry->cmd);
1443 }
1444 
1445 static int nfp_net_mc_sync(struct net_device *netdev, const unsigned char *addr)
1446 {
1447 	struct nfp_net *nn = netdev_priv(netdev);
1448 
1449 	if (netdev_mc_count(netdev) > NFP_NET_CFG_MAC_MC_MAX) {
1450 		nn_err(nn, "Requested number of MC addresses (%d) exceeds maximum (%d).\n",
1451 		       netdev_mc_count(netdev), NFP_NET_CFG_MAC_MC_MAX);
1452 		return -EINVAL;
1453 	}
1454 
1455 	return nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_MULTICAST_ADD, addr,
1456 					    NFP_NET_CFG_MULTICAST_SZ, nfp_net_mc_cfg);
1457 }
1458 
1459 static int nfp_net_mc_unsync(struct net_device *netdev, const unsigned char *addr)
1460 {
1461 	struct nfp_net *nn = netdev_priv(netdev);
1462 
1463 	return nfp_net_sched_mbox_amsg_work(nn, NFP_NET_CFG_MBOX_CMD_MULTICAST_DEL, addr,
1464 					    NFP_NET_CFG_MULTICAST_SZ, nfp_net_mc_cfg);
1465 }
1466 
1467 static void nfp_net_set_rx_mode(struct net_device *netdev)
1468 {
1469 	struct nfp_net *nn = netdev_priv(netdev);
1470 	u32 new_ctrl, new_ctrl_w1;
1471 
1472 	new_ctrl = nn->dp.ctrl;
1473 	new_ctrl_w1 = nn->dp.ctrl_w1;
1474 
1475 	if (!netdev_mc_empty(netdev) || netdev->flags & IFF_ALLMULTI)
1476 		new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_L2MC;
1477 	else
1478 		new_ctrl &= ~NFP_NET_CFG_CTRL_L2MC;
1479 
1480 	if (netdev->flags & IFF_ALLMULTI)
1481 		new_ctrl_w1 &= ~NFP_NET_CFG_CTRL_MCAST_FILTER;
1482 	else
1483 		new_ctrl_w1 |= nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER;
1484 
1485 	if (netdev->flags & IFF_PROMISC) {
1486 		if (nn->cap & NFP_NET_CFG_CTRL_PROMISC)
1487 			new_ctrl |= NFP_NET_CFG_CTRL_PROMISC;
1488 		else
1489 			nn_warn(nn, "FW does not support promiscuous mode\n");
1490 	} else {
1491 		new_ctrl &= ~NFP_NET_CFG_CTRL_PROMISC;
1492 	}
1493 
1494 	if ((nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER) &&
1495 	    __dev_mc_sync(netdev, nfp_net_mc_sync, nfp_net_mc_unsync))
1496 		netdev_err(netdev, "Sync mc address failed\n");
1497 
1498 	if (new_ctrl == nn->dp.ctrl && new_ctrl_w1 == nn->dp.ctrl_w1)
1499 		return;
1500 
1501 	if (new_ctrl != nn->dp.ctrl)
1502 		nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);
1503 	if (new_ctrl_w1 != nn->dp.ctrl_w1)
1504 		nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, new_ctrl_w1);
1505 	nfp_net_reconfig_post(nn, NFP_NET_CFG_UPDATE_GEN);
1506 
1507 	nn->dp.ctrl = new_ctrl;
1508 	nn->dp.ctrl_w1 = new_ctrl_w1;
1509 }
1510 
1511 static void nfp_net_rss_init_itbl(struct nfp_net *nn)
1512 {
1513 	int i;
1514 
1515 	for (i = 0; i < sizeof(nn->rss_itbl); i++)
1516 		nn->rss_itbl[i] =
1517 			ethtool_rxfh_indir_default(i, nn->dp.num_rx_rings);
1518 }
1519 
1520 static void nfp_net_dp_swap(struct nfp_net *nn, struct nfp_net_dp *dp)
1521 {
1522 	struct nfp_net_dp new_dp = *dp;
1523 
1524 	*dp = nn->dp;
1525 	nn->dp = new_dp;
1526 
1527 	nn->dp.netdev->mtu = new_dp.mtu;
1528 
1529 	if (!netif_is_rxfh_configured(nn->dp.netdev))
1530 		nfp_net_rss_init_itbl(nn);
1531 }
1532 
1533 static int nfp_net_dp_swap_enable(struct nfp_net *nn, struct nfp_net_dp *dp)
1534 {
1535 	unsigned int r;
1536 	int err;
1537 
1538 	nfp_net_dp_swap(nn, dp);
1539 
1540 	for (r = 0; r <	nn->max_r_vecs; r++)
1541 		nfp_net_vector_assign_rings(&nn->dp, &nn->r_vecs[r], r);
1542 
1543 	err = netif_set_real_num_queues(nn->dp.netdev,
1544 					nn->dp.num_stack_tx_rings,
1545 					nn->dp.num_rx_rings);
1546 	if (err)
1547 		return err;
1548 
1549 	return nfp_net_set_config_and_enable(nn);
1550 }
1551 
1552 struct nfp_net_dp *nfp_net_clone_dp(struct nfp_net *nn)
1553 {
1554 	struct nfp_net_dp *new;
1555 
1556 	new = kmalloc(sizeof(*new), GFP_KERNEL);
1557 	if (!new)
1558 		return NULL;
1559 
1560 	*new = nn->dp;
1561 
1562 	new->xsk_pools = kmemdup(new->xsk_pools,
1563 				 array_size(nn->max_r_vecs,
1564 					    sizeof(new->xsk_pools)),
1565 				 GFP_KERNEL);
1566 	if (!new->xsk_pools) {
1567 		kfree(new);
1568 		return NULL;
1569 	}
1570 
1571 	/* Clear things which need to be recomputed */
1572 	new->fl_bufsz = 0;
1573 	new->tx_rings = NULL;
1574 	new->rx_rings = NULL;
1575 	new->num_r_vecs = 0;
1576 	new->num_stack_tx_rings = 0;
1577 	new->txrwb = NULL;
1578 	new->txrwb_dma = 0;
1579 
1580 	return new;
1581 }
1582 
1583 static void nfp_net_free_dp(struct nfp_net_dp *dp)
1584 {
1585 	kfree(dp->xsk_pools);
1586 	kfree(dp);
1587 }
1588 
1589 static int
1590 nfp_net_check_config(struct nfp_net *nn, struct nfp_net_dp *dp,
1591 		     struct netlink_ext_ack *extack)
1592 {
1593 	unsigned int r, xsk_min_fl_bufsz;
1594 
1595 	/* XDP-enabled tests */
1596 	if (!dp->xdp_prog)
1597 		return 0;
1598 	if (dp->fl_bufsz > PAGE_SIZE) {
1599 		NL_SET_ERR_MSG_MOD(extack, "MTU too large w/ XDP enabled");
1600 		return -EINVAL;
1601 	}
1602 	if (dp->num_tx_rings > nn->max_tx_rings) {
1603 		NL_SET_ERR_MSG_MOD(extack, "Insufficient number of TX rings w/ XDP enabled");
1604 		return -EINVAL;
1605 	}
1606 
1607 	xsk_min_fl_bufsz = nfp_net_calc_fl_bufsz_xsk(dp);
1608 	for (r = 0; r < nn->max_r_vecs; r++) {
1609 		if (!dp->xsk_pools[r])
1610 			continue;
1611 
1612 		if (xsk_pool_get_rx_frame_size(dp->xsk_pools[r]) < xsk_min_fl_bufsz) {
1613 			NL_SET_ERR_MSG_MOD(extack,
1614 					   "XSK buffer pool chunk size too small");
1615 			return -EINVAL;
1616 		}
1617 	}
1618 
1619 	return 0;
1620 }
1621 
1622 int nfp_net_ring_reconfig(struct nfp_net *nn, struct nfp_net_dp *dp,
1623 			  struct netlink_ext_ack *extack)
1624 {
1625 	int r, err;
1626 
1627 	dp->fl_bufsz = nfp_net_calc_fl_bufsz(dp);
1628 
1629 	dp->num_stack_tx_rings = dp->num_tx_rings;
1630 	if (dp->xdp_prog)
1631 		dp->num_stack_tx_rings -= dp->num_rx_rings;
1632 
1633 	dp->num_r_vecs = max(dp->num_rx_rings, dp->num_stack_tx_rings);
1634 
1635 	err = nfp_net_check_config(nn, dp, extack);
1636 	if (err)
1637 		goto exit_free_dp;
1638 
1639 	if (!netif_running(dp->netdev)) {
1640 		nfp_net_dp_swap(nn, dp);
1641 		err = 0;
1642 		goto exit_free_dp;
1643 	}
1644 
1645 	/* Prepare new rings */
1646 	for (r = nn->dp.num_r_vecs; r < dp->num_r_vecs; r++) {
1647 		err = nfp_net_prepare_vector(nn, &nn->r_vecs[r], r);
1648 		if (err) {
1649 			dp->num_r_vecs = r;
1650 			goto err_cleanup_vecs;
1651 		}
1652 	}
1653 
1654 	err = nfp_net_rx_rings_prepare(nn, dp);
1655 	if (err)
1656 		goto err_cleanup_vecs;
1657 
1658 	err = nfp_net_tx_rings_prepare(nn, dp);
1659 	if (err)
1660 		goto err_free_rx;
1661 
1662 	/* Stop device, swap in new rings, try to start the firmware */
1663 	nfp_net_close_stack(nn);
1664 	nfp_net_clear_config_and_disable(nn);
1665 
1666 	err = nfp_net_dp_swap_enable(nn, dp);
1667 	if (err) {
1668 		int err2;
1669 
1670 		nfp_net_clear_config_and_disable(nn);
1671 
1672 		/* Try with old configuration and old rings */
1673 		err2 = nfp_net_dp_swap_enable(nn, dp);
1674 		if (err2)
1675 			nn_err(nn, "Can't restore ring config - FW communication failed (%d,%d)\n",
1676 			       err, err2);
1677 	}
1678 	for (r = dp->num_r_vecs - 1; r >= nn->dp.num_r_vecs; r--)
1679 		nfp_net_cleanup_vector(nn, &nn->r_vecs[r]);
1680 
1681 	nfp_net_rx_rings_free(dp);
1682 	nfp_net_tx_rings_free(dp);
1683 
1684 	nfp_net_open_stack(nn);
1685 exit_free_dp:
1686 	nfp_net_free_dp(dp);
1687 
1688 	return err;
1689 
1690 err_free_rx:
1691 	nfp_net_rx_rings_free(dp);
1692 err_cleanup_vecs:
1693 	for (r = dp->num_r_vecs - 1; r >= nn->dp.num_r_vecs; r--)
1694 		nfp_net_cleanup_vector(nn, &nn->r_vecs[r]);
1695 	nfp_net_free_dp(dp);
1696 	return err;
1697 }
1698 
1699 static int nfp_net_change_mtu(struct net_device *netdev, int new_mtu)
1700 {
1701 	struct nfp_net *nn = netdev_priv(netdev);
1702 	struct nfp_net_dp *dp;
1703 	int err;
1704 
1705 	err = nfp_app_check_mtu(nn->app, netdev, new_mtu);
1706 	if (err)
1707 		return err;
1708 
1709 	dp = nfp_net_clone_dp(nn);
1710 	if (!dp)
1711 		return -ENOMEM;
1712 
1713 	dp->mtu = new_mtu;
1714 
1715 	return nfp_net_ring_reconfig(nn, dp, NULL);
1716 }
1717 
1718 static int
1719 nfp_net_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1720 {
1721 	const u32 cmd = NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_ADD;
1722 	struct nfp_net *nn = netdev_priv(netdev);
1723 	int err;
1724 
1725 	/* Priority tagged packets with vlan id 0 are processed by the
1726 	 * NFP as untagged packets
1727 	 */
1728 	if (!vid)
1729 		return 0;
1730 
1731 	err = nfp_net_mbox_lock(nn, NFP_NET_CFG_VLAN_FILTER_SZ);
1732 	if (err)
1733 		return err;
1734 
1735 	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_VID, vid);
1736 	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_PROTO,
1737 		  ETH_P_8021Q);
1738 
1739 	return nfp_net_mbox_reconfig_and_unlock(nn, cmd);
1740 }
1741 
1742 static int
1743 nfp_net_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
1744 {
1745 	const u32 cmd = NFP_NET_CFG_MBOX_CMD_CTAG_FILTER_KILL;
1746 	struct nfp_net *nn = netdev_priv(netdev);
1747 	int err;
1748 
1749 	/* Priority tagged packets with vlan id 0 are processed by the
1750 	 * NFP as untagged packets
1751 	 */
1752 	if (!vid)
1753 		return 0;
1754 
1755 	err = nfp_net_mbox_lock(nn, NFP_NET_CFG_VLAN_FILTER_SZ);
1756 	if (err)
1757 		return err;
1758 
1759 	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_VID, vid);
1760 	nn_writew(nn, nn->tlv_caps.mbox_off + NFP_NET_CFG_VLAN_FILTER_PROTO,
1761 		  ETH_P_8021Q);
1762 
1763 	return nfp_net_mbox_reconfig_and_unlock(nn, cmd);
1764 }
1765 
1766 static void nfp_net_stat64(struct net_device *netdev,
1767 			   struct rtnl_link_stats64 *stats)
1768 {
1769 	struct nfp_net *nn = netdev_priv(netdev);
1770 	int r;
1771 
1772 	/* Collect software stats */
1773 	for (r = 0; r < nn->max_r_vecs; r++) {
1774 		struct nfp_net_r_vector *r_vec = &nn->r_vecs[r];
1775 		u64 data[3];
1776 		unsigned int start;
1777 
1778 		do {
1779 			start = u64_stats_fetch_begin(&r_vec->rx_sync);
1780 			data[0] = r_vec->rx_pkts;
1781 			data[1] = r_vec->rx_bytes;
1782 			data[2] = r_vec->rx_drops;
1783 		} while (u64_stats_fetch_retry(&r_vec->rx_sync, start));
1784 		stats->rx_packets += data[0];
1785 		stats->rx_bytes += data[1];
1786 		stats->rx_dropped += data[2];
1787 
1788 		do {
1789 			start = u64_stats_fetch_begin(&r_vec->tx_sync);
1790 			data[0] = r_vec->tx_pkts;
1791 			data[1] = r_vec->tx_bytes;
1792 			data[2] = r_vec->tx_errors;
1793 		} while (u64_stats_fetch_retry(&r_vec->tx_sync, start));
1794 		stats->tx_packets += data[0];
1795 		stats->tx_bytes += data[1];
1796 		stats->tx_errors += data[2];
1797 	}
1798 
1799 	/* Add in device stats */
1800 	stats->multicast += nn_readq(nn, NFP_NET_CFG_STATS_RX_MC_FRAMES);
1801 	stats->rx_dropped += nn_readq(nn, NFP_NET_CFG_STATS_RX_DISCARDS);
1802 	stats->rx_errors += nn_readq(nn, NFP_NET_CFG_STATS_RX_ERRORS);
1803 
1804 	stats->tx_dropped += nn_readq(nn, NFP_NET_CFG_STATS_TX_DISCARDS);
1805 	stats->tx_errors += nn_readq(nn, NFP_NET_CFG_STATS_TX_ERRORS);
1806 }
1807 
1808 static int nfp_net_set_features(struct net_device *netdev,
1809 				netdev_features_t features)
1810 {
1811 	netdev_features_t changed = netdev->features ^ features;
1812 	struct nfp_net *nn = netdev_priv(netdev);
1813 	u32 new_ctrl;
1814 	int err;
1815 
1816 	/* Assume this is not called with features we have not advertised */
1817 
1818 	new_ctrl = nn->dp.ctrl;
1819 
1820 	if (changed & NETIF_F_RXCSUM) {
1821 		if (features & NETIF_F_RXCSUM)
1822 			new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_RXCSUM_ANY;
1823 		else
1824 			new_ctrl &= ~NFP_NET_CFG_CTRL_RXCSUM_ANY;
1825 	}
1826 
1827 	if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
1828 		if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM))
1829 			new_ctrl |= NFP_NET_CFG_CTRL_TXCSUM;
1830 		else
1831 			new_ctrl &= ~NFP_NET_CFG_CTRL_TXCSUM;
1832 	}
1833 
1834 	if (changed & (NETIF_F_TSO | NETIF_F_TSO6)) {
1835 		if (features & (NETIF_F_TSO | NETIF_F_TSO6))
1836 			new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_LSO2 ?:
1837 					      NFP_NET_CFG_CTRL_LSO;
1838 		else
1839 			new_ctrl &= ~NFP_NET_CFG_CTRL_LSO_ANY;
1840 	}
1841 
1842 	if (changed & NETIF_F_HW_VLAN_CTAG_RX) {
1843 		if (features & NETIF_F_HW_VLAN_CTAG_RX)
1844 			new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_RXVLAN_V2 ?:
1845 				    NFP_NET_CFG_CTRL_RXVLAN;
1846 		else
1847 			new_ctrl &= ~NFP_NET_CFG_CTRL_RXVLAN_ANY;
1848 	}
1849 
1850 	if (changed & NETIF_F_HW_VLAN_CTAG_TX) {
1851 		if (features & NETIF_F_HW_VLAN_CTAG_TX)
1852 			new_ctrl |= nn->cap & NFP_NET_CFG_CTRL_TXVLAN_V2 ?:
1853 				    NFP_NET_CFG_CTRL_TXVLAN;
1854 		else
1855 			new_ctrl &= ~NFP_NET_CFG_CTRL_TXVLAN_ANY;
1856 	}
1857 
1858 	if (changed & NETIF_F_HW_VLAN_CTAG_FILTER) {
1859 		if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1860 			new_ctrl |= NFP_NET_CFG_CTRL_CTAG_FILTER;
1861 		else
1862 			new_ctrl &= ~NFP_NET_CFG_CTRL_CTAG_FILTER;
1863 	}
1864 
1865 	if (changed & NETIF_F_HW_VLAN_STAG_RX) {
1866 		if (features & NETIF_F_HW_VLAN_STAG_RX)
1867 			new_ctrl |= NFP_NET_CFG_CTRL_RXQINQ;
1868 		else
1869 			new_ctrl &= ~NFP_NET_CFG_CTRL_RXQINQ;
1870 	}
1871 
1872 	if (changed & NETIF_F_SG) {
1873 		if (features & NETIF_F_SG)
1874 			new_ctrl |= NFP_NET_CFG_CTRL_GATHER;
1875 		else
1876 			new_ctrl &= ~NFP_NET_CFG_CTRL_GATHER;
1877 	}
1878 
1879 	err = nfp_port_set_features(netdev, features);
1880 	if (err)
1881 		return err;
1882 
1883 	nn_dbg(nn, "Feature change 0x%llx -> 0x%llx (changed=0x%llx)\n",
1884 	       netdev->features, features, changed);
1885 
1886 	if (new_ctrl == nn->dp.ctrl)
1887 		return 0;
1888 
1889 	nn_dbg(nn, "NIC ctrl: 0x%x -> 0x%x\n", nn->dp.ctrl, new_ctrl);
1890 	nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);
1891 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
1892 	if (err)
1893 		return err;
1894 
1895 	nn->dp.ctrl = new_ctrl;
1896 
1897 	return 0;
1898 }
1899 
1900 static netdev_features_t
1901 nfp_net_fix_features(struct net_device *netdev,
1902 		     netdev_features_t features)
1903 {
1904 	if ((features & NETIF_F_HW_VLAN_CTAG_RX) &&
1905 	    (features & NETIF_F_HW_VLAN_STAG_RX)) {
1906 		if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) {
1907 			features &= ~NETIF_F_HW_VLAN_CTAG_RX;
1908 			netdev->wanted_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
1909 			netdev_warn(netdev,
1910 				    "S-tag and C-tag stripping can't be enabled at the same time. Enabling S-tag stripping and disabling C-tag stripping\n");
1911 		} else if (netdev->features & NETIF_F_HW_VLAN_STAG_RX) {
1912 			features &= ~NETIF_F_HW_VLAN_STAG_RX;
1913 			netdev->wanted_features &= ~NETIF_F_HW_VLAN_STAG_RX;
1914 			netdev_warn(netdev,
1915 				    "S-tag and C-tag stripping can't be enabled at the same time. Enabling C-tag stripping and disabling S-tag stripping\n");
1916 		}
1917 	}
1918 	return features;
1919 }
1920 
1921 static netdev_features_t
1922 nfp_net_features_check(struct sk_buff *skb, struct net_device *dev,
1923 		       netdev_features_t features)
1924 {
1925 	u8 l4_hdr;
1926 
1927 	/* We can't do TSO over double tagged packets (802.1AD) */
1928 	features &= vlan_features_check(skb, features);
1929 
1930 	if (!skb->encapsulation)
1931 		return features;
1932 
1933 	/* Ensure that inner L4 header offset fits into TX descriptor field */
1934 	if (skb_is_gso(skb)) {
1935 		u32 hdrlen;
1936 
1937 		hdrlen = skb_inner_tcp_all_headers(skb);
1938 
1939 		/* Assume worst case scenario of having longest possible
1940 		 * metadata prepend - 8B
1941 		 */
1942 		if (unlikely(hdrlen > NFP_NET_LSO_MAX_HDR_SZ - 8))
1943 			features &= ~NETIF_F_GSO_MASK;
1944 	}
1945 
1946 	if (xfrm_offload(skb))
1947 		return features;
1948 
1949 	/* VXLAN/GRE check */
1950 	switch (vlan_get_protocol(skb)) {
1951 	case htons(ETH_P_IP):
1952 		l4_hdr = ip_hdr(skb)->protocol;
1953 		break;
1954 	case htons(ETH_P_IPV6):
1955 		l4_hdr = ipv6_hdr(skb)->nexthdr;
1956 		break;
1957 	default:
1958 		return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
1959 	}
1960 
1961 	if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
1962 	    skb->inner_protocol != htons(ETH_P_TEB) ||
1963 	    (l4_hdr != IPPROTO_UDP && l4_hdr != IPPROTO_GRE) ||
1964 	    (l4_hdr == IPPROTO_UDP &&
1965 	     (skb_inner_mac_header(skb) - skb_transport_header(skb) !=
1966 	      sizeof(struct udphdr) + sizeof(struct vxlanhdr))))
1967 		return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
1968 
1969 	return features;
1970 }
1971 
1972 static int
1973 nfp_net_get_phys_port_name(struct net_device *netdev, char *name, size_t len)
1974 {
1975 	struct nfp_net *nn = netdev_priv(netdev);
1976 	int n;
1977 
1978 	/* If port is defined, devlink_port is registered and devlink core
1979 	 * is taking care of name formatting.
1980 	 */
1981 	if (nn->port)
1982 		return -EOPNOTSUPP;
1983 
1984 	if (nn->dp.is_vf || nn->vnic_no_name)
1985 		return -EOPNOTSUPP;
1986 
1987 	n = snprintf(name, len, "n%d", nn->id);
1988 	if (n >= len)
1989 		return -EINVAL;
1990 
1991 	return 0;
1992 }
1993 
1994 static int nfp_net_xdp_setup_drv(struct nfp_net *nn, struct netdev_bpf *bpf)
1995 {
1996 	struct bpf_prog *prog = bpf->prog;
1997 	struct nfp_net_dp *dp;
1998 	int err;
1999 
2000 	if (!prog == !nn->dp.xdp_prog) {
2001 		WRITE_ONCE(nn->dp.xdp_prog, prog);
2002 		xdp_attachment_setup(&nn->xdp, bpf);
2003 		return 0;
2004 	}
2005 
2006 	dp = nfp_net_clone_dp(nn);
2007 	if (!dp)
2008 		return -ENOMEM;
2009 
2010 	dp->xdp_prog = prog;
2011 	dp->num_tx_rings += prog ? nn->dp.num_rx_rings : -nn->dp.num_rx_rings;
2012 	dp->rx_dma_dir = prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
2013 	dp->rx_dma_off = prog ? XDP_PACKET_HEADROOM - nn->dp.rx_offset : 0;
2014 
2015 	/* We need RX reconfig to remap the buffers (BIDIR vs FROM_DEV) */
2016 	err = nfp_net_ring_reconfig(nn, dp, bpf->extack);
2017 	if (err)
2018 		return err;
2019 
2020 	xdp_attachment_setup(&nn->xdp, bpf);
2021 	return 0;
2022 }
2023 
2024 static int nfp_net_xdp_setup_hw(struct nfp_net *nn, struct netdev_bpf *bpf)
2025 {
2026 	int err;
2027 
2028 	err = nfp_app_xdp_offload(nn->app, nn, bpf->prog, bpf->extack);
2029 	if (err)
2030 		return err;
2031 
2032 	xdp_attachment_setup(&nn->xdp_hw, bpf);
2033 	return 0;
2034 }
2035 
2036 static int nfp_net_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
2037 {
2038 	struct nfp_net *nn = netdev_priv(netdev);
2039 
2040 	switch (xdp->command) {
2041 	case XDP_SETUP_PROG:
2042 		return nfp_net_xdp_setup_drv(nn, xdp);
2043 	case XDP_SETUP_PROG_HW:
2044 		return nfp_net_xdp_setup_hw(nn, xdp);
2045 	case XDP_SETUP_XSK_POOL:
2046 		return nfp_net_xsk_setup_pool(netdev, xdp->xsk.pool,
2047 					      xdp->xsk.queue_id);
2048 	default:
2049 		return nfp_app_bpf(nn->app, nn, xdp);
2050 	}
2051 }
2052 
2053 static int nfp_net_set_mac_address(struct net_device *netdev, void *addr)
2054 {
2055 	struct nfp_net *nn = netdev_priv(netdev);
2056 	struct sockaddr *saddr = addr;
2057 	int err;
2058 
2059 	err = eth_prepare_mac_addr_change(netdev, addr);
2060 	if (err)
2061 		return err;
2062 
2063 	nfp_net_write_mac_addr(nn, saddr->sa_data);
2064 
2065 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_MACADDR);
2066 	if (err)
2067 		return err;
2068 
2069 	eth_commit_mac_addr_change(netdev, addr);
2070 
2071 	return 0;
2072 }
2073 
2074 static int nfp_net_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
2075 				  struct net_device *dev, u32 filter_mask,
2076 				  int nlflags)
2077 {
2078 	struct nfp_net *nn = netdev_priv(dev);
2079 	u16 mode;
2080 
2081 	if (!(nn->cap & NFP_NET_CFG_CTRL_VEPA))
2082 		return -EOPNOTSUPP;
2083 
2084 	mode = (nn->dp.ctrl & NFP_NET_CFG_CTRL_VEPA) ?
2085 	       BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB;
2086 
2087 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, mode, 0, 0,
2088 				       nlflags, filter_mask, NULL);
2089 }
2090 
2091 static int nfp_net_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
2092 				  u16 flags, struct netlink_ext_ack *extack)
2093 {
2094 	struct nfp_net *nn = netdev_priv(dev);
2095 	struct nlattr *attr, *br_spec;
2096 	int rem, err;
2097 	u32 new_ctrl;
2098 	u16 mode;
2099 
2100 	if (!(nn->cap & NFP_NET_CFG_CTRL_VEPA))
2101 		return -EOPNOTSUPP;
2102 
2103 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
2104 	if (!br_spec)
2105 		return -EINVAL;
2106 
2107 	nla_for_each_nested(attr, br_spec, rem) {
2108 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
2109 			continue;
2110 
2111 		new_ctrl = nn->dp.ctrl;
2112 		mode = nla_get_u16(attr);
2113 		if (mode == BRIDGE_MODE_VEPA)
2114 			new_ctrl |= NFP_NET_CFG_CTRL_VEPA;
2115 		else if (mode == BRIDGE_MODE_VEB)
2116 			new_ctrl &= ~NFP_NET_CFG_CTRL_VEPA;
2117 		else
2118 			return -EOPNOTSUPP;
2119 
2120 		if (new_ctrl == nn->dp.ctrl)
2121 			return 0;
2122 
2123 		nn_writel(nn, NFP_NET_CFG_CTRL, new_ctrl);
2124 		err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
2125 		if (!err)
2126 			nn->dp.ctrl = new_ctrl;
2127 
2128 		return err;
2129 	}
2130 
2131 	return -EINVAL;
2132 }
2133 
2134 const struct net_device_ops nfp_nfd3_netdev_ops = {
2135 	.ndo_init		= nfp_app_ndo_init,
2136 	.ndo_uninit		= nfp_app_ndo_uninit,
2137 	.ndo_open		= nfp_net_netdev_open,
2138 	.ndo_stop		= nfp_net_netdev_close,
2139 	.ndo_start_xmit		= nfp_net_tx,
2140 	.ndo_get_stats64	= nfp_net_stat64,
2141 	.ndo_vlan_rx_add_vid	= nfp_net_vlan_rx_add_vid,
2142 	.ndo_vlan_rx_kill_vid	= nfp_net_vlan_rx_kill_vid,
2143 	.ndo_set_vf_mac         = nfp_app_set_vf_mac,
2144 	.ndo_set_vf_vlan        = nfp_app_set_vf_vlan,
2145 	.ndo_set_vf_rate	= nfp_app_set_vf_rate,
2146 	.ndo_set_vf_spoofchk    = nfp_app_set_vf_spoofchk,
2147 	.ndo_set_vf_trust	= nfp_app_set_vf_trust,
2148 	.ndo_get_vf_config	= nfp_app_get_vf_config,
2149 	.ndo_set_vf_link_state  = nfp_app_set_vf_link_state,
2150 	.ndo_setup_tc		= nfp_port_setup_tc,
2151 	.ndo_tx_timeout		= nfp_net_tx_timeout,
2152 	.ndo_set_rx_mode	= nfp_net_set_rx_mode,
2153 	.ndo_change_mtu		= nfp_net_change_mtu,
2154 	.ndo_set_mac_address	= nfp_net_set_mac_address,
2155 	.ndo_set_features	= nfp_net_set_features,
2156 	.ndo_fix_features	= nfp_net_fix_features,
2157 	.ndo_features_check	= nfp_net_features_check,
2158 	.ndo_get_phys_port_name	= nfp_net_get_phys_port_name,
2159 	.ndo_bpf		= nfp_net_xdp,
2160 	.ndo_xsk_wakeup		= nfp_net_xsk_wakeup,
2161 	.ndo_bridge_getlink     = nfp_net_bridge_getlink,
2162 	.ndo_bridge_setlink     = nfp_net_bridge_setlink,
2163 };
2164 
2165 const struct net_device_ops nfp_nfdk_netdev_ops = {
2166 	.ndo_init		= nfp_app_ndo_init,
2167 	.ndo_uninit		= nfp_app_ndo_uninit,
2168 	.ndo_open		= nfp_net_netdev_open,
2169 	.ndo_stop		= nfp_net_netdev_close,
2170 	.ndo_start_xmit		= nfp_net_tx,
2171 	.ndo_get_stats64	= nfp_net_stat64,
2172 	.ndo_vlan_rx_add_vid	= nfp_net_vlan_rx_add_vid,
2173 	.ndo_vlan_rx_kill_vid	= nfp_net_vlan_rx_kill_vid,
2174 	.ndo_set_vf_mac         = nfp_app_set_vf_mac,
2175 	.ndo_set_vf_vlan        = nfp_app_set_vf_vlan,
2176 	.ndo_set_vf_rate	= nfp_app_set_vf_rate,
2177 	.ndo_set_vf_spoofchk    = nfp_app_set_vf_spoofchk,
2178 	.ndo_set_vf_trust	= nfp_app_set_vf_trust,
2179 	.ndo_get_vf_config	= nfp_app_get_vf_config,
2180 	.ndo_set_vf_link_state  = nfp_app_set_vf_link_state,
2181 	.ndo_setup_tc		= nfp_port_setup_tc,
2182 	.ndo_tx_timeout		= nfp_net_tx_timeout,
2183 	.ndo_set_rx_mode	= nfp_net_set_rx_mode,
2184 	.ndo_change_mtu		= nfp_net_change_mtu,
2185 	.ndo_set_mac_address	= nfp_net_set_mac_address,
2186 	.ndo_set_features	= nfp_net_set_features,
2187 	.ndo_fix_features	= nfp_net_fix_features,
2188 	.ndo_features_check	= nfp_net_features_check,
2189 	.ndo_get_phys_port_name	= nfp_net_get_phys_port_name,
2190 	.ndo_bpf		= nfp_net_xdp,
2191 	.ndo_bridge_getlink     = nfp_net_bridge_getlink,
2192 	.ndo_bridge_setlink     = nfp_net_bridge_setlink,
2193 };
2194 
2195 static int nfp_udp_tunnel_sync(struct net_device *netdev, unsigned int table)
2196 {
2197 	struct nfp_net *nn = netdev_priv(netdev);
2198 	int i;
2199 
2200 	BUILD_BUG_ON(NFP_NET_N_VXLAN_PORTS & 1);
2201 	for (i = 0; i < NFP_NET_N_VXLAN_PORTS; i += 2) {
2202 		struct udp_tunnel_info ti0, ti1;
2203 
2204 		udp_tunnel_nic_get_port(netdev, table, i, &ti0);
2205 		udp_tunnel_nic_get_port(netdev, table, i + 1, &ti1);
2206 
2207 		nn_writel(nn, NFP_NET_CFG_VXLAN_PORT + i * sizeof(ti0.port),
2208 			  be16_to_cpu(ti1.port) << 16 | be16_to_cpu(ti0.port));
2209 	}
2210 
2211 	return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_VXLAN);
2212 }
2213 
2214 static const struct udp_tunnel_nic_info nfp_udp_tunnels = {
2215 	.sync_table     = nfp_udp_tunnel_sync,
2216 	.flags          = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
2217 			  UDP_TUNNEL_NIC_INFO_OPEN_ONLY,
2218 	.tables         = {
2219 		{
2220 			.n_entries      = NFP_NET_N_VXLAN_PORTS,
2221 			.tunnel_types   = UDP_TUNNEL_TYPE_VXLAN,
2222 		},
2223 	},
2224 };
2225 
2226 /**
2227  * nfp_net_info() - Print general info about the NIC
2228  * @nn:      NFP Net device to reconfigure
2229  */
2230 void nfp_net_info(struct nfp_net *nn)
2231 {
2232 	nn_info(nn, "NFP-6xxx %sNetdev: TxQs=%d/%d RxQs=%d/%d\n",
2233 		nn->dp.is_vf ? "VF " : "",
2234 		nn->dp.num_tx_rings, nn->max_tx_rings,
2235 		nn->dp.num_rx_rings, nn->max_rx_rings);
2236 	nn_info(nn, "VER: %d.%d.%d.%d, Maximum supported MTU: %d\n",
2237 		nn->fw_ver.extend, nn->fw_ver.class,
2238 		nn->fw_ver.major, nn->fw_ver.minor,
2239 		nn->max_mtu);
2240 	nn_info(nn, "CAP: %#x %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s\n",
2241 		nn->cap,
2242 		nn->cap & NFP_NET_CFG_CTRL_PROMISC  ? "PROMISC "  : "",
2243 		nn->cap & NFP_NET_CFG_CTRL_L2BC     ? "L2BCFILT " : "",
2244 		nn->cap & NFP_NET_CFG_CTRL_L2MC     ? "L2MCFILT " : "",
2245 		nn->cap & NFP_NET_CFG_CTRL_RXCSUM   ? "RXCSUM "   : "",
2246 		nn->cap & NFP_NET_CFG_CTRL_TXCSUM   ? "TXCSUM "   : "",
2247 		nn->cap & NFP_NET_CFG_CTRL_RXVLAN   ? "RXVLAN "   : "",
2248 		nn->cap & NFP_NET_CFG_CTRL_TXVLAN   ? "TXVLAN "   : "",
2249 		nn->cap & NFP_NET_CFG_CTRL_RXQINQ   ? "RXQINQ "   : "",
2250 		nn->cap & NFP_NET_CFG_CTRL_RXVLAN_V2 ? "RXVLANv2 "   : "",
2251 		nn->cap & NFP_NET_CFG_CTRL_TXVLAN_V2   ? "TXVLANv2 "   : "",
2252 		nn->cap & NFP_NET_CFG_CTRL_SCATTER  ? "SCATTER "  : "",
2253 		nn->cap & NFP_NET_CFG_CTRL_GATHER   ? "GATHER "   : "",
2254 		nn->cap & NFP_NET_CFG_CTRL_LSO      ? "TSO1 "     : "",
2255 		nn->cap & NFP_NET_CFG_CTRL_LSO2     ? "TSO2 "     : "",
2256 		nn->cap & NFP_NET_CFG_CTRL_RSS      ? "RSS1 "     : "",
2257 		nn->cap & NFP_NET_CFG_CTRL_RSS2     ? "RSS2 "     : "",
2258 		nn->cap & NFP_NET_CFG_CTRL_CTAG_FILTER ? "CTAG_FILTER " : "",
2259 		nn->cap & NFP_NET_CFG_CTRL_MSIXAUTO ? "AUTOMASK " : "",
2260 		nn->cap & NFP_NET_CFG_CTRL_IRQMOD   ? "IRQMOD "   : "",
2261 		nn->cap & NFP_NET_CFG_CTRL_TXRWB    ? "TXRWB "    : "",
2262 		nn->cap & NFP_NET_CFG_CTRL_VEPA     ? "VEPA "     : "",
2263 		nn->cap & NFP_NET_CFG_CTRL_VXLAN    ? "VXLAN "    : "",
2264 		nn->cap & NFP_NET_CFG_CTRL_NVGRE    ? "NVGRE "	  : "",
2265 		nn->cap & NFP_NET_CFG_CTRL_CSUM_COMPLETE ?
2266 						      "RXCSUM_COMPLETE " : "",
2267 		nn->cap & NFP_NET_CFG_CTRL_LIVE_ADDR ? "LIVE_ADDR " : "",
2268 		nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER ? "MULTICAST_FILTER " : "",
2269 		nfp_app_extra_cap(nn->app, nn));
2270 }
2271 
2272 /**
2273  * nfp_net_alloc() - Allocate netdev and related structure
2274  * @pdev:         PCI device
2275  * @dev_info:     NFP ASIC params
2276  * @ctrl_bar:     PCI IOMEM with vNIC config memory
2277  * @needs_netdev: Whether to allocate a netdev for this vNIC
2278  * @max_tx_rings: Maximum number of TX rings supported by device
2279  * @max_rx_rings: Maximum number of RX rings supported by device
2280  *
2281  * This function allocates a netdev device and fills in the initial
2282  * part of the @struct nfp_net structure.  In case of control device
2283  * nfp_net structure is allocated without the netdev.
2284  *
2285  * Return: NFP Net device structure, or ERR_PTR on error.
2286  */
2287 struct nfp_net *
2288 nfp_net_alloc(struct pci_dev *pdev, const struct nfp_dev_info *dev_info,
2289 	      void __iomem *ctrl_bar, bool needs_netdev,
2290 	      unsigned int max_tx_rings, unsigned int max_rx_rings)
2291 {
2292 	u64 dma_mask = dma_get_mask(&pdev->dev);
2293 	struct nfp_net *nn;
2294 	int err;
2295 
2296 	if (needs_netdev) {
2297 		struct net_device *netdev;
2298 
2299 		netdev = alloc_etherdev_mqs(sizeof(struct nfp_net),
2300 					    max_tx_rings, max_rx_rings);
2301 		if (!netdev)
2302 			return ERR_PTR(-ENOMEM);
2303 
2304 		SET_NETDEV_DEV(netdev, &pdev->dev);
2305 		nn = netdev_priv(netdev);
2306 		nn->dp.netdev = netdev;
2307 	} else {
2308 		nn = vzalloc(sizeof(*nn));
2309 		if (!nn)
2310 			return ERR_PTR(-ENOMEM);
2311 	}
2312 
2313 	nn->dp.dev = &pdev->dev;
2314 	nn->dp.ctrl_bar = ctrl_bar;
2315 	nn->dev_info = dev_info;
2316 	nn->pdev = pdev;
2317 	nfp_net_get_fw_version(&nn->fw_ver, ctrl_bar);
2318 
2319 	switch (FIELD_GET(NFP_NET_CFG_VERSION_DP_MASK, nn->fw_ver.extend)) {
2320 	case NFP_NET_CFG_VERSION_DP_NFD3:
2321 		nn->dp.ops = &nfp_nfd3_ops;
2322 		break;
2323 	case NFP_NET_CFG_VERSION_DP_NFDK:
2324 		if (nn->fw_ver.major < 5) {
2325 			dev_err(&pdev->dev,
2326 				"NFDK must use ABI 5 or newer, found: %d\n",
2327 				nn->fw_ver.major);
2328 			err = -EINVAL;
2329 			goto err_free_nn;
2330 		}
2331 		nn->dp.ops = &nfp_nfdk_ops;
2332 		break;
2333 	default:
2334 		err = -EINVAL;
2335 		goto err_free_nn;
2336 	}
2337 
2338 	if ((dma_mask & nn->dp.ops->dma_mask) != dma_mask) {
2339 		dev_err(&pdev->dev,
2340 			"DMA mask of loaded firmware: %llx, required DMA mask: %llx\n",
2341 			nn->dp.ops->dma_mask, dma_mask);
2342 		err = -EINVAL;
2343 		goto err_free_nn;
2344 	}
2345 
2346 	nn->max_tx_rings = max_tx_rings;
2347 	nn->max_rx_rings = max_rx_rings;
2348 
2349 	nn->dp.num_tx_rings = min_t(unsigned int,
2350 				    max_tx_rings, num_online_cpus());
2351 	nn->dp.num_rx_rings = min_t(unsigned int, max_rx_rings,
2352 				 netif_get_num_default_rss_queues());
2353 
2354 	nn->dp.num_r_vecs = max(nn->dp.num_tx_rings, nn->dp.num_rx_rings);
2355 	nn->dp.num_r_vecs = min_t(unsigned int,
2356 				  nn->dp.num_r_vecs, num_online_cpus());
2357 	nn->max_r_vecs = nn->dp.num_r_vecs;
2358 
2359 	nn->dp.xsk_pools = kcalloc(nn->max_r_vecs, sizeof(nn->dp.xsk_pools),
2360 				   GFP_KERNEL);
2361 	if (!nn->dp.xsk_pools) {
2362 		err = -ENOMEM;
2363 		goto err_free_nn;
2364 	}
2365 
2366 	nn->dp.txd_cnt = NFP_NET_TX_DESCS_DEFAULT;
2367 	nn->dp.rxd_cnt = NFP_NET_RX_DESCS_DEFAULT;
2368 
2369 	sema_init(&nn->bar_lock, 1);
2370 
2371 	spin_lock_init(&nn->reconfig_lock);
2372 	spin_lock_init(&nn->link_status_lock);
2373 
2374 	timer_setup(&nn->reconfig_timer, nfp_net_reconfig_timer, 0);
2375 
2376 	err = nfp_net_tlv_caps_parse(&nn->pdev->dev, nn->dp.ctrl_bar,
2377 				     &nn->tlv_caps);
2378 	if (err)
2379 		goto err_free_nn;
2380 
2381 	err = nfp_ccm_mbox_alloc(nn);
2382 	if (err)
2383 		goto err_free_nn;
2384 
2385 	return nn;
2386 
2387 err_free_nn:
2388 	if (nn->dp.netdev)
2389 		free_netdev(nn->dp.netdev);
2390 	else
2391 		vfree(nn);
2392 	return ERR_PTR(err);
2393 }
2394 
2395 /**
2396  * nfp_net_free() - Undo what @nfp_net_alloc() did
2397  * @nn:      NFP Net device to reconfigure
2398  */
2399 void nfp_net_free(struct nfp_net *nn)
2400 {
2401 	WARN_ON(timer_pending(&nn->reconfig_timer) || nn->reconfig_posted);
2402 	nfp_ccm_mbox_free(nn);
2403 
2404 	kfree(nn->dp.xsk_pools);
2405 	if (nn->dp.netdev)
2406 		free_netdev(nn->dp.netdev);
2407 	else
2408 		vfree(nn);
2409 }
2410 
2411 /**
2412  * nfp_net_rss_key_sz() - Get current size of the RSS key
2413  * @nn:		NFP Net device instance
2414  *
2415  * Return: size of the RSS key for currently selected hash function.
2416  */
2417 unsigned int nfp_net_rss_key_sz(struct nfp_net *nn)
2418 {
2419 	switch (nn->rss_hfunc) {
2420 	case ETH_RSS_HASH_TOP:
2421 		return NFP_NET_CFG_RSS_KEY_SZ;
2422 	case ETH_RSS_HASH_XOR:
2423 		return 0;
2424 	case ETH_RSS_HASH_CRC32:
2425 		return 4;
2426 	}
2427 
2428 	nn_warn(nn, "Unknown hash function: %u\n", nn->rss_hfunc);
2429 	return 0;
2430 }
2431 
2432 /**
2433  * nfp_net_rss_init() - Set the initial RSS parameters
2434  * @nn:	     NFP Net device to reconfigure
2435  */
2436 static void nfp_net_rss_init(struct nfp_net *nn)
2437 {
2438 	unsigned long func_bit, rss_cap_hfunc;
2439 	u32 reg;
2440 
2441 	/* Read the RSS function capability and select first supported func */
2442 	reg = nn_readl(nn, NFP_NET_CFG_RSS_CAP);
2443 	rss_cap_hfunc =	FIELD_GET(NFP_NET_CFG_RSS_CAP_HFUNC, reg);
2444 	if (!rss_cap_hfunc)
2445 		rss_cap_hfunc =	FIELD_GET(NFP_NET_CFG_RSS_CAP_HFUNC,
2446 					  NFP_NET_CFG_RSS_TOEPLITZ);
2447 
2448 	func_bit = find_first_bit(&rss_cap_hfunc, NFP_NET_CFG_RSS_HFUNCS);
2449 	if (func_bit == NFP_NET_CFG_RSS_HFUNCS) {
2450 		dev_warn(nn->dp.dev,
2451 			 "Bad RSS config, defaulting to Toeplitz hash\n");
2452 		func_bit = ETH_RSS_HASH_TOP_BIT;
2453 	}
2454 	nn->rss_hfunc = 1 << func_bit;
2455 
2456 	netdev_rss_key_fill(nn->rss_key, nfp_net_rss_key_sz(nn));
2457 
2458 	nfp_net_rss_init_itbl(nn);
2459 
2460 	/* Enable IPv4/IPv6 TCP by default */
2461 	nn->rss_cfg = NFP_NET_CFG_RSS_IPV4_TCP |
2462 		      NFP_NET_CFG_RSS_IPV6_TCP |
2463 		      NFP_NET_CFG_RSS_IPV4_UDP |
2464 		      NFP_NET_CFG_RSS_IPV6_UDP |
2465 		      FIELD_PREP(NFP_NET_CFG_RSS_HFUNC, nn->rss_hfunc) |
2466 		      NFP_NET_CFG_RSS_MASK;
2467 }
2468 
2469 /**
2470  * nfp_net_irqmod_init() - Set the initial IRQ moderation parameters
2471  * @nn:	     NFP Net device to reconfigure
2472  */
2473 static void nfp_net_irqmod_init(struct nfp_net *nn)
2474 {
2475 	nn->rx_coalesce_usecs      = 50;
2476 	nn->rx_coalesce_max_frames = 64;
2477 	nn->tx_coalesce_usecs      = 50;
2478 	nn->tx_coalesce_max_frames = 64;
2479 
2480 	nn->rx_coalesce_adapt_on   = true;
2481 	nn->tx_coalesce_adapt_on   = true;
2482 }
2483 
2484 static void nfp_net_netdev_init(struct nfp_net *nn)
2485 {
2486 	struct net_device *netdev = nn->dp.netdev;
2487 
2488 	nfp_net_write_mac_addr(nn, nn->dp.netdev->dev_addr);
2489 
2490 	netdev->mtu = nn->dp.mtu;
2491 
2492 	/* Advertise/enable offloads based on capabilities
2493 	 *
2494 	 * Note: netdev->features show the currently enabled features
2495 	 * and netdev->hw_features advertises which features are
2496 	 * supported.  By default we enable most features.
2497 	 */
2498 	if (nn->cap & NFP_NET_CFG_CTRL_LIVE_ADDR)
2499 		netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
2500 
2501 	netdev->hw_features = NETIF_F_HIGHDMA;
2502 	if (nn->cap & NFP_NET_CFG_CTRL_RXCSUM_ANY) {
2503 		netdev->hw_features |= NETIF_F_RXCSUM;
2504 		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_RXCSUM_ANY;
2505 	}
2506 	if (nn->cap & NFP_NET_CFG_CTRL_TXCSUM) {
2507 		netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2508 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_TXCSUM;
2509 	}
2510 	if (nn->cap & NFP_NET_CFG_CTRL_GATHER) {
2511 		netdev->hw_features |= NETIF_F_SG;
2512 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_GATHER;
2513 	}
2514 	if ((nn->cap & NFP_NET_CFG_CTRL_LSO && nn->fw_ver.major > 2) ||
2515 	    nn->cap & NFP_NET_CFG_CTRL_LSO2) {
2516 		netdev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
2517 		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_LSO2 ?:
2518 					 NFP_NET_CFG_CTRL_LSO;
2519 	}
2520 	if (nn->cap & NFP_NET_CFG_CTRL_RSS_ANY)
2521 		netdev->hw_features |= NETIF_F_RXHASH;
2522 
2523 #ifdef CONFIG_NFP_NET_IPSEC
2524 	if (nn->cap_w1 & NFP_NET_CFG_CTRL_IPSEC)
2525 		netdev->hw_features |= NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM;
2526 #endif
2527 
2528 	if (nn->cap & NFP_NET_CFG_CTRL_VXLAN) {
2529 		if (nn->cap & NFP_NET_CFG_CTRL_LSO) {
2530 			netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
2531 					       NETIF_F_GSO_UDP_TUNNEL_CSUM |
2532 					       NETIF_F_GSO_PARTIAL;
2533 			netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
2534 		}
2535 		netdev->udp_tunnel_nic_info = &nfp_udp_tunnels;
2536 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_VXLAN;
2537 	}
2538 	if (nn->cap & NFP_NET_CFG_CTRL_NVGRE) {
2539 		if (nn->cap & NFP_NET_CFG_CTRL_LSO)
2540 			netdev->hw_features |= NETIF_F_GSO_GRE;
2541 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_NVGRE;
2542 	}
2543 	if (nn->cap & (NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE))
2544 		netdev->hw_enc_features = netdev->hw_features;
2545 
2546 	netdev->vlan_features = netdev->hw_features;
2547 
2548 	if (nn->cap & NFP_NET_CFG_CTRL_RXVLAN_ANY) {
2549 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
2550 		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_RXVLAN_V2 ?:
2551 			       NFP_NET_CFG_CTRL_RXVLAN;
2552 	}
2553 	if (nn->cap & NFP_NET_CFG_CTRL_TXVLAN_ANY) {
2554 		if (nn->cap & NFP_NET_CFG_CTRL_LSO2) {
2555 			nn_warn(nn, "Device advertises both TSO2 and TXVLAN. Refusing to enable TXVLAN.\n");
2556 		} else {
2557 			netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
2558 			nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_TXVLAN_V2 ?:
2559 				       NFP_NET_CFG_CTRL_TXVLAN;
2560 		}
2561 	}
2562 	if (nn->cap & NFP_NET_CFG_CTRL_CTAG_FILTER) {
2563 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
2564 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_CTAG_FILTER;
2565 	}
2566 	if (nn->cap & NFP_NET_CFG_CTRL_RXQINQ) {
2567 		netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX;
2568 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_RXQINQ;
2569 	}
2570 
2571 	netdev->features = netdev->hw_features;
2572 
2573 	if (nfp_app_has_tc(nn->app) && nn->port)
2574 		netdev->hw_features |= NETIF_F_HW_TC;
2575 
2576 	/* C-Tag strip and S-Tag strip can't be supported simultaneously,
2577 	 * so enable C-Tag strip and disable S-Tag strip by default.
2578 	 */
2579 	netdev->features &= ~NETIF_F_HW_VLAN_STAG_RX;
2580 	nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_RXQINQ;
2581 
2582 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC;
2583 	if (nn->app && nn->app->type->id == NFP_APP_BPF_NIC)
2584 		netdev->xdp_features |= NETDEV_XDP_ACT_HW_OFFLOAD;
2585 
2586 	/* Finalise the netdev setup */
2587 	switch (nn->dp.ops->version) {
2588 	case NFP_NFD_VER_NFD3:
2589 		netdev->netdev_ops = &nfp_nfd3_netdev_ops;
2590 		netdev->xdp_features |= NETDEV_XDP_ACT_XSK_ZEROCOPY;
2591 		netdev->xdp_features |= NETDEV_XDP_ACT_REDIRECT;
2592 		break;
2593 	case NFP_NFD_VER_NFDK:
2594 		netdev->netdev_ops = &nfp_nfdk_netdev_ops;
2595 		break;
2596 	}
2597 
2598 	netdev->watchdog_timeo = msecs_to_jiffies(5 * 1000);
2599 
2600 	/* MTU range: 68 - hw-specific max */
2601 	netdev->min_mtu = ETH_MIN_MTU;
2602 	netdev->max_mtu = nn->max_mtu;
2603 
2604 	netif_set_tso_max_segs(netdev, NFP_NET_LSO_MAX_SEGS);
2605 
2606 	netif_carrier_off(netdev);
2607 
2608 	nfp_net_set_ethtool_ops(netdev);
2609 }
2610 
2611 static int nfp_net_read_caps(struct nfp_net *nn)
2612 {
2613 	/* Get some of the read-only fields from the BAR */
2614 	nn->cap = nn_readl(nn, NFP_NET_CFG_CAP);
2615 	nn->cap_w1 = nn_readl(nn, NFP_NET_CFG_CAP_WORD1);
2616 	nn->max_mtu = nn_readl(nn, NFP_NET_CFG_MAX_MTU);
2617 
2618 	/* ABI 4.x and ctrl vNIC always use chained metadata, in other cases
2619 	 * we allow use of non-chained metadata if RSS(v1) is the only
2620 	 * advertised capability requiring metadata.
2621 	 */
2622 	nn->dp.chained_metadata_format = nn->fw_ver.major == 4 ||
2623 					 !nn->dp.netdev ||
2624 					 !(nn->cap & NFP_NET_CFG_CTRL_RSS) ||
2625 					 nn->cap & NFP_NET_CFG_CTRL_CHAIN_META;
2626 	/* RSS(v1) uses non-chained metadata format, except in ABI 4.x where
2627 	 * it has the same meaning as RSSv2.
2628 	 */
2629 	if (nn->dp.chained_metadata_format && nn->fw_ver.major != 4)
2630 		nn->cap &= ~NFP_NET_CFG_CTRL_RSS;
2631 
2632 	/* Determine RX packet/metadata boundary offset */
2633 	if (nn->fw_ver.major >= 2) {
2634 		u32 reg;
2635 
2636 		reg = nn_readl(nn, NFP_NET_CFG_RX_OFFSET);
2637 		if (reg > NFP_NET_MAX_PREPEND) {
2638 			nn_err(nn, "Invalid rx offset: %d\n", reg);
2639 			return -EINVAL;
2640 		}
2641 		nn->dp.rx_offset = reg;
2642 	} else {
2643 		nn->dp.rx_offset = NFP_NET_RX_OFFSET;
2644 	}
2645 
2646 	/* Mask out NFD-version-specific features */
2647 	nn->cap &= nn->dp.ops->cap_mask;
2648 
2649 	/* For control vNICs mask out the capabilities app doesn't want. */
2650 	if (!nn->dp.netdev)
2651 		nn->cap &= nn->app->type->ctrl_cap_mask;
2652 
2653 	return 0;
2654 }
2655 
2656 /**
2657  * nfp_net_init() - Initialise/finalise the nfp_net structure
2658  * @nn:		NFP Net device structure
2659  *
2660  * Return: 0 on success or negative errno on error.
2661  */
2662 int nfp_net_init(struct nfp_net *nn)
2663 {
2664 	int err;
2665 
2666 	nn->dp.rx_dma_dir = DMA_FROM_DEVICE;
2667 
2668 	err = nfp_net_read_caps(nn);
2669 	if (err)
2670 		return err;
2671 
2672 	/* Set default MTU and Freelist buffer size */
2673 	if (!nfp_net_is_data_vnic(nn) && nn->app->ctrl_mtu) {
2674 		nn->dp.mtu = min(nn->app->ctrl_mtu, nn->max_mtu);
2675 	} else if (nn->max_mtu < NFP_NET_DEFAULT_MTU) {
2676 		nn->dp.mtu = nn->max_mtu;
2677 	} else {
2678 		nn->dp.mtu = NFP_NET_DEFAULT_MTU;
2679 	}
2680 	nn->dp.fl_bufsz = nfp_net_calc_fl_bufsz(&nn->dp);
2681 
2682 	if (nfp_app_ctrl_uses_data_vnics(nn->app))
2683 		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_CMSG_DATA;
2684 
2685 	if (nn->cap & NFP_NET_CFG_CTRL_RSS_ANY) {
2686 		nfp_net_rss_init(nn);
2687 		nn->dp.ctrl |= nn->cap & NFP_NET_CFG_CTRL_RSS2 ?:
2688 					 NFP_NET_CFG_CTRL_RSS;
2689 	}
2690 
2691 	/* Allow L2 Broadcast and Multicast through by default, if supported */
2692 	if (nn->cap & NFP_NET_CFG_CTRL_L2BC)
2693 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_L2BC;
2694 
2695 	/* Allow IRQ moderation, if supported */
2696 	if (nn->cap & NFP_NET_CFG_CTRL_IRQMOD) {
2697 		nfp_net_irqmod_init(nn);
2698 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_IRQMOD;
2699 	}
2700 
2701 	/* Enable TX pointer writeback, if supported */
2702 	if (nn->cap & NFP_NET_CFG_CTRL_TXRWB)
2703 		nn->dp.ctrl |= NFP_NET_CFG_CTRL_TXRWB;
2704 
2705 	if (nn->cap_w1 & NFP_NET_CFG_CTRL_MCAST_FILTER)
2706 		nn->dp.ctrl_w1 |= NFP_NET_CFG_CTRL_MCAST_FILTER;
2707 
2708 	/* Stash the re-configuration queue away.  First odd queue in TX Bar */
2709 	nn->qcp_cfg = nn->tx_bar + NFP_QCP_QUEUE_ADDR_SZ;
2710 
2711 	/* Make sure the FW knows the netdev is supposed to be disabled here */
2712 	nn_writel(nn, NFP_NET_CFG_CTRL, 0);
2713 	nn_writeq(nn, NFP_NET_CFG_TXRS_ENABLE, 0);
2714 	nn_writeq(nn, NFP_NET_CFG_RXRS_ENABLE, 0);
2715 	nn_writel(nn, NFP_NET_CFG_CTRL_WORD1, 0);
2716 	err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_RING |
2717 				   NFP_NET_CFG_UPDATE_GEN);
2718 	if (err)
2719 		return err;
2720 
2721 	if (nn->dp.netdev) {
2722 		nfp_net_netdev_init(nn);
2723 
2724 		err = nfp_ccm_mbox_init(nn);
2725 		if (err)
2726 			return err;
2727 
2728 		err = nfp_net_tls_init(nn);
2729 		if (err)
2730 			goto err_clean_mbox;
2731 
2732 		nfp_net_ipsec_init(nn);
2733 	}
2734 
2735 	nfp_net_vecs_init(nn);
2736 
2737 	if (!nn->dp.netdev)
2738 		return 0;
2739 
2740 	spin_lock_init(&nn->mbox_amsg.lock);
2741 	INIT_LIST_HEAD(&nn->mbox_amsg.list);
2742 	INIT_WORK(&nn->mbox_amsg.work, nfp_net_mbox_amsg_work);
2743 
2744 	return register_netdev(nn->dp.netdev);
2745 
2746 err_clean_mbox:
2747 	nfp_ccm_mbox_clean(nn);
2748 	return err;
2749 }
2750 
2751 /**
2752  * nfp_net_clean() - Undo what nfp_net_init() did.
2753  * @nn:		NFP Net device structure
2754  */
2755 void nfp_net_clean(struct nfp_net *nn)
2756 {
2757 	if (!nn->dp.netdev)
2758 		return;
2759 
2760 	unregister_netdev(nn->dp.netdev);
2761 	nfp_net_ipsec_clean(nn);
2762 	nfp_ccm_mbox_clean(nn);
2763 	flush_work(&nn->mbox_amsg.work);
2764 	nfp_net_reconfig_wait_posted(nn);
2765 }
2766