1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2021 Intel Corporation. */
3 
4 #include <linux/etherdevice.h>
5 #include <linux/of_net.h>
6 #include <linux/pci.h>
7 #include <linux/bpf.h>
8 #include <generated/utsrelease.h>
9 #include <linux/crash_dump.h>
10 
11 /* Local includes */
12 #include "i40e.h"
13 #include "i40e_diag.h"
14 #include "i40e_xsk.h"
15 #include <net/udp_tunnel.h>
16 #include <net/xdp_sock_drv.h>
17 /* All i40e tracepoints are defined by the include below, which
18  * must be included exactly once across the whole kernel with
19  * CREATE_TRACE_POINTS defined
20  */
21 #define CREATE_TRACE_POINTS
22 #include "i40e_trace.h"
23 
24 const char i40e_driver_name[] = "i40e";
25 static const char i40e_driver_string[] =
26 			"Intel(R) Ethernet Connection XL710 Network Driver";
27 
28 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
29 
30 /* a bit of forward declarations */
31 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
32 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
33 static int i40e_add_vsi(struct i40e_vsi *vsi);
34 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
35 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired);
36 static int i40e_setup_misc_vector(struct i40e_pf *pf);
37 static void i40e_determine_queue_usage(struct i40e_pf *pf);
38 static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
39 static void i40e_prep_for_reset(struct i40e_pf *pf);
40 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
41 				   bool lock_acquired);
42 static int i40e_reset(struct i40e_pf *pf);
43 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
44 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
45 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
46 static bool i40e_check_recovery_mode(struct i40e_pf *pf);
47 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
48 static void i40e_fdir_sb_setup(struct i40e_pf *pf);
49 static int i40e_veb_get_bw_info(struct i40e_veb *veb);
50 static int i40e_get_capabilities(struct i40e_pf *pf,
51 				 enum i40e_admin_queue_opc list_type);
52 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
53 
54 /* i40e_pci_tbl - PCI Device ID Table
55  *
56  * Last entry must be all 0s
57  *
58  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
59  *   Class, Class Mask, private data (not used) }
60  */
61 static const struct pci_device_id i40e_pci_tbl[] = {
62 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
63 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
64 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
65 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
66 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
67 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
68 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
69 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},
70 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
71 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
72 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
73 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
74 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
75 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
76 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
77 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
78 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
79 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
80 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
81 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0},
82 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
83 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
84 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
85 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
86 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
87 	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
88 	/* required last entry */
89 	{0, }
90 };
91 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
92 
93 #define I40E_MAX_VF_COUNT 128
94 static int debug = -1;
95 module_param(debug, uint, 0);
96 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
97 
98 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
99 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
100 MODULE_LICENSE("GPL v2");
101 
102 static struct workqueue_struct *i40e_wq;
103 
104 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f,
105 				  struct net_device *netdev, int delta)
106 {
107 	struct netdev_hw_addr *ha;
108 
109 	if (!f || !netdev)
110 		return;
111 
112 	netdev_for_each_mc_addr(ha, netdev) {
113 		if (ether_addr_equal(ha->addr, f->macaddr)) {
114 			ha->refcount += delta;
115 			if (ha->refcount <= 0)
116 				ha->refcount = 1;
117 			break;
118 		}
119 	}
120 }
121 
122 /**
123  * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code
124  * @hw:   pointer to the HW structure
125  * @mem:  ptr to mem struct to fill out
126  * @size: size of memory requested
127  * @alignment: what to align the allocation to
128  **/
129 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
130 			    u64 size, u32 alignment)
131 {
132 	struct i40e_pf *pf = (struct i40e_pf *)hw->back;
133 
134 	mem->size = ALIGN(size, alignment);
135 	mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
136 				     GFP_KERNEL);
137 	if (!mem->va)
138 		return -ENOMEM;
139 
140 	return 0;
141 }
142 
143 /**
144  * i40e_free_dma_mem_d - OS specific memory free for shared code
145  * @hw:   pointer to the HW structure
146  * @mem:  ptr to mem struct to free
147  **/
148 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
149 {
150 	struct i40e_pf *pf = (struct i40e_pf *)hw->back;
151 
152 	dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
153 	mem->va = NULL;
154 	mem->pa = 0;
155 	mem->size = 0;
156 
157 	return 0;
158 }
159 
160 /**
161  * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code
162  * @hw:   pointer to the HW structure
163  * @mem:  ptr to mem struct to fill out
164  * @size: size of memory requested
165  **/
166 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
167 			     u32 size)
168 {
169 	mem->size = size;
170 	mem->va = kzalloc(size, GFP_KERNEL);
171 
172 	if (!mem->va)
173 		return -ENOMEM;
174 
175 	return 0;
176 }
177 
178 /**
179  * i40e_free_virt_mem_d - OS specific memory free for shared code
180  * @hw:   pointer to the HW structure
181  * @mem:  ptr to mem struct to free
182  **/
183 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
184 {
185 	/* it's ok to kfree a NULL pointer */
186 	kfree(mem->va);
187 	mem->va = NULL;
188 	mem->size = 0;
189 
190 	return 0;
191 }
192 
193 /**
194  * i40e_get_lump - find a lump of free generic resource
195  * @pf: board private structure
196  * @pile: the pile of resource to search
197  * @needed: the number of items needed
198  * @id: an owner id to stick on the items assigned
199  *
200  * Returns the base item index of the lump, or negative for error
201  **/
202 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
203 			 u16 needed, u16 id)
204 {
205 	int ret = -ENOMEM;
206 	int i, j;
207 
208 	if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
209 		dev_info(&pf->pdev->dev,
210 			 "param err: pile=%s needed=%d id=0x%04x\n",
211 			 pile ? "<valid>" : "<null>", needed, id);
212 		return -EINVAL;
213 	}
214 
215 	/* Allocate last queue in the pile for FDIR VSI queue
216 	 * so it doesn't fragment the qp_pile
217 	 */
218 	if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) {
219 		if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) {
220 			dev_err(&pf->pdev->dev,
221 				"Cannot allocate queue %d for I40E_VSI_FDIR\n",
222 				pile->num_entries - 1);
223 			return -ENOMEM;
224 		}
225 		pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT;
226 		return pile->num_entries - 1;
227 	}
228 
229 	i = 0;
230 	while (i < pile->num_entries) {
231 		/* skip already allocated entries */
232 		if (pile->list[i] & I40E_PILE_VALID_BIT) {
233 			i++;
234 			continue;
235 		}
236 
237 		/* do we have enough in this lump? */
238 		for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
239 			if (pile->list[i+j] & I40E_PILE_VALID_BIT)
240 				break;
241 		}
242 
243 		if (j == needed) {
244 			/* there was enough, so assign it to the requestor */
245 			for (j = 0; j < needed; j++)
246 				pile->list[i+j] = id | I40E_PILE_VALID_BIT;
247 			ret = i;
248 			break;
249 		}
250 
251 		/* not enough, so skip over it and continue looking */
252 		i += j;
253 	}
254 
255 	return ret;
256 }
257 
258 /**
259  * i40e_put_lump - return a lump of generic resource
260  * @pile: the pile of resource to search
261  * @index: the base item index
262  * @id: the owner id of the items assigned
263  *
264  * Returns the count of items in the lump
265  **/
266 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
267 {
268 	int valid_id = (id | I40E_PILE_VALID_BIT);
269 	int count = 0;
270 	u16 i;
271 
272 	if (!pile || index >= pile->num_entries)
273 		return -EINVAL;
274 
275 	for (i = index;
276 	     i < pile->num_entries && pile->list[i] == valid_id;
277 	     i++) {
278 		pile->list[i] = 0;
279 		count++;
280 	}
281 
282 
283 	return count;
284 }
285 
286 /**
287  * i40e_find_vsi_from_id - searches for the vsi with the given id
288  * @pf: the pf structure to search for the vsi
289  * @id: id of the vsi it is searching for
290  **/
291 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
292 {
293 	int i;
294 
295 	for (i = 0; i < pf->num_alloc_vsi; i++)
296 		if (pf->vsi[i] && (pf->vsi[i]->id == id))
297 			return pf->vsi[i];
298 
299 	return NULL;
300 }
301 
302 /**
303  * i40e_service_event_schedule - Schedule the service task to wake up
304  * @pf: board private structure
305  *
306  * If not already scheduled, this puts the task into the work queue
307  **/
308 void i40e_service_event_schedule(struct i40e_pf *pf)
309 {
310 	if ((!test_bit(__I40E_DOWN, pf->state) &&
311 	     !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
312 	      test_bit(__I40E_RECOVERY_MODE, pf->state))
313 		queue_work(i40e_wq, &pf->service_task);
314 }
315 
316 /**
317  * i40e_tx_timeout - Respond to a Tx Hang
318  * @netdev: network interface device structure
319  * @txqueue: queue number timing out
320  *
321  * If any port has noticed a Tx timeout, it is likely that the whole
322  * device is munged, not just the one netdev port, so go for the full
323  * reset.
324  **/
325 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue)
326 {
327 	struct i40e_netdev_priv *np = netdev_priv(netdev);
328 	struct i40e_vsi *vsi = np->vsi;
329 	struct i40e_pf *pf = vsi->back;
330 	struct i40e_ring *tx_ring = NULL;
331 	unsigned int i;
332 	u32 head, val;
333 
334 	pf->tx_timeout_count++;
335 
336 	/* with txqueue index, find the tx_ring struct */
337 	for (i = 0; i < vsi->num_queue_pairs; i++) {
338 		if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
339 			if (txqueue ==
340 			    vsi->tx_rings[i]->queue_index) {
341 				tx_ring = vsi->tx_rings[i];
342 				break;
343 			}
344 		}
345 	}
346 
347 	if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
348 		pf->tx_timeout_recovery_level = 1;  /* reset after some time */
349 	else if (time_before(jiffies,
350 		      (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
351 		return;   /* don't do any new action before the next timeout */
352 
353 	/* don't kick off another recovery if one is already pending */
354 	if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
355 		return;
356 
357 	if (tx_ring) {
358 		head = i40e_get_head(tx_ring);
359 		/* Read interrupt register */
360 		if (pf->flags & I40E_FLAG_MSIX_ENABLED)
361 			val = rd32(&pf->hw,
362 			     I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
363 						tx_ring->vsi->base_vector - 1));
364 		else
365 			val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
366 
367 		netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
368 			    vsi->seid, txqueue, tx_ring->next_to_clean,
369 			    head, tx_ring->next_to_use,
370 			    readl(tx_ring->tail), val);
371 	}
372 
373 	pf->tx_timeout_last_recovery = jiffies;
374 	netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n",
375 		    pf->tx_timeout_recovery_level, txqueue);
376 
377 	switch (pf->tx_timeout_recovery_level) {
378 	case 1:
379 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
380 		break;
381 	case 2:
382 		set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
383 		break;
384 	case 3:
385 		set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
386 		break;
387 	default:
388 		netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n");
389 		set_bit(__I40E_DOWN_REQUESTED, pf->state);
390 		set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state);
391 		break;
392 	}
393 
394 	i40e_service_event_schedule(pf);
395 	pf->tx_timeout_recovery_level++;
396 }
397 
398 /**
399  * i40e_get_vsi_stats_struct - Get System Network Statistics
400  * @vsi: the VSI we care about
401  *
402  * Returns the address of the device statistics structure.
403  * The statistics are actually updated from the service task.
404  **/
405 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
406 {
407 	return &vsi->net_stats;
408 }
409 
410 /**
411  * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring
412  * @ring: Tx ring to get statistics from
413  * @stats: statistics entry to be updated
414  **/
415 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
416 					    struct rtnl_link_stats64 *stats)
417 {
418 	u64 bytes, packets;
419 	unsigned int start;
420 
421 	do {
422 		start = u64_stats_fetch_begin(&ring->syncp);
423 		packets = ring->stats.packets;
424 		bytes   = ring->stats.bytes;
425 	} while (u64_stats_fetch_retry(&ring->syncp, start));
426 
427 	stats->tx_packets += packets;
428 	stats->tx_bytes   += bytes;
429 }
430 
431 /**
432  * i40e_get_netdev_stats_struct - Get statistics for netdev interface
433  * @netdev: network interface device structure
434  * @stats: data structure to store statistics
435  *
436  * Returns the address of the device statistics structure.
437  * The statistics are actually updated from the service task.
438  **/
439 static void i40e_get_netdev_stats_struct(struct net_device *netdev,
440 				  struct rtnl_link_stats64 *stats)
441 {
442 	struct i40e_netdev_priv *np = netdev_priv(netdev);
443 	struct i40e_vsi *vsi = np->vsi;
444 	struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
445 	struct i40e_ring *ring;
446 	int i;
447 
448 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
449 		return;
450 
451 	if (!vsi->tx_rings)
452 		return;
453 
454 	rcu_read_lock();
455 	for (i = 0; i < vsi->num_queue_pairs; i++) {
456 		u64 bytes, packets;
457 		unsigned int start;
458 
459 		ring = READ_ONCE(vsi->tx_rings[i]);
460 		if (!ring)
461 			continue;
462 		i40e_get_netdev_stats_struct_tx(ring, stats);
463 
464 		if (i40e_enabled_xdp_vsi(vsi)) {
465 			ring = READ_ONCE(vsi->xdp_rings[i]);
466 			if (!ring)
467 				continue;
468 			i40e_get_netdev_stats_struct_tx(ring, stats);
469 		}
470 
471 		ring = READ_ONCE(vsi->rx_rings[i]);
472 		if (!ring)
473 			continue;
474 		do {
475 			start   = u64_stats_fetch_begin(&ring->syncp);
476 			packets = ring->stats.packets;
477 			bytes   = ring->stats.bytes;
478 		} while (u64_stats_fetch_retry(&ring->syncp, start));
479 
480 		stats->rx_packets += packets;
481 		stats->rx_bytes   += bytes;
482 
483 	}
484 	rcu_read_unlock();
485 
486 	/* following stats updated by i40e_watchdog_subtask() */
487 	stats->multicast	= vsi_stats->multicast;
488 	stats->tx_errors	= vsi_stats->tx_errors;
489 	stats->tx_dropped	= vsi_stats->tx_dropped;
490 	stats->rx_errors	= vsi_stats->rx_errors;
491 	stats->rx_dropped	= vsi_stats->rx_dropped;
492 	stats->rx_crc_errors	= vsi_stats->rx_crc_errors;
493 	stats->rx_length_errors	= vsi_stats->rx_length_errors;
494 }
495 
496 /**
497  * i40e_vsi_reset_stats - Resets all stats of the given vsi
498  * @vsi: the VSI to have its stats reset
499  **/
500 void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
501 {
502 	struct rtnl_link_stats64 *ns;
503 	int i;
504 
505 	if (!vsi)
506 		return;
507 
508 	ns = i40e_get_vsi_stats_struct(vsi);
509 	memset(ns, 0, sizeof(*ns));
510 	memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
511 	memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
512 	memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
513 	if (vsi->rx_rings && vsi->rx_rings[0]) {
514 		for (i = 0; i < vsi->num_queue_pairs; i++) {
515 			memset(&vsi->rx_rings[i]->stats, 0,
516 			       sizeof(vsi->rx_rings[i]->stats));
517 			memset(&vsi->rx_rings[i]->rx_stats, 0,
518 			       sizeof(vsi->rx_rings[i]->rx_stats));
519 			memset(&vsi->tx_rings[i]->stats, 0,
520 			       sizeof(vsi->tx_rings[i]->stats));
521 			memset(&vsi->tx_rings[i]->tx_stats, 0,
522 			       sizeof(vsi->tx_rings[i]->tx_stats));
523 		}
524 	}
525 	vsi->stat_offsets_loaded = false;
526 }
527 
528 /**
529  * i40e_pf_reset_stats - Reset all of the stats for the given PF
530  * @pf: the PF to be reset
531  **/
532 void i40e_pf_reset_stats(struct i40e_pf *pf)
533 {
534 	int i;
535 
536 	memset(&pf->stats, 0, sizeof(pf->stats));
537 	memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
538 	pf->stat_offsets_loaded = false;
539 
540 	for (i = 0; i < I40E_MAX_VEB; i++) {
541 		if (pf->veb[i]) {
542 			memset(&pf->veb[i]->stats, 0,
543 			       sizeof(pf->veb[i]->stats));
544 			memset(&pf->veb[i]->stats_offsets, 0,
545 			       sizeof(pf->veb[i]->stats_offsets));
546 			memset(&pf->veb[i]->tc_stats, 0,
547 			       sizeof(pf->veb[i]->tc_stats));
548 			memset(&pf->veb[i]->tc_stats_offsets, 0,
549 			       sizeof(pf->veb[i]->tc_stats_offsets));
550 			pf->veb[i]->stat_offsets_loaded = false;
551 		}
552 	}
553 	pf->hw_csum_rx_error = 0;
554 }
555 
556 /**
557  * i40e_compute_pci_to_hw_id - compute index form PCI function.
558  * @vsi: ptr to the VSI to read from.
559  * @hw: ptr to the hardware info.
560  **/
561 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw)
562 {
563 	int pf_count = i40e_get_pf_count(hw);
564 
565 	if (vsi->type == I40E_VSI_SRIOV)
566 		return (hw->port * BIT(7)) / pf_count + vsi->vf_id;
567 
568 	return hw->port + BIT(7);
569 }
570 
571 /**
572  * i40e_stat_update64 - read and update a 64 bit stat from the chip.
573  * @hw: ptr to the hardware info.
574  * @hireg: the high 32 bit reg to read.
575  * @loreg: the low 32 bit reg to read.
576  * @offset_loaded: has the initial offset been loaded yet.
577  * @offset: ptr to current offset value.
578  * @stat: ptr to the stat.
579  *
580  * Since the device stats are not reset at PFReset, they will not
581  * be zeroed when the driver starts.  We'll save the first values read
582  * and use them as offsets to be subtracted from the raw values in order
583  * to report stats that count from zero.
584  **/
585 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg,
586 			       bool offset_loaded, u64 *offset, u64 *stat)
587 {
588 	u64 new_data;
589 
590 	new_data = rd64(hw, loreg);
591 
592 	if (!offset_loaded || new_data < *offset)
593 		*offset = new_data;
594 	*stat = new_data - *offset;
595 }
596 
597 /**
598  * i40e_stat_update48 - read and update a 48 bit stat from the chip
599  * @hw: ptr to the hardware info
600  * @hireg: the high 32 bit reg to read
601  * @loreg: the low 32 bit reg to read
602  * @offset_loaded: has the initial offset been loaded yet
603  * @offset: ptr to current offset value
604  * @stat: ptr to the stat
605  *
606  * Since the device stats are not reset at PFReset, they likely will not
607  * be zeroed when the driver starts.  We'll save the first values read
608  * and use them as offsets to be subtracted from the raw values in order
609  * to report stats that count from zero.  In the process, we also manage
610  * the potential roll-over.
611  **/
612 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
613 			       bool offset_loaded, u64 *offset, u64 *stat)
614 {
615 	u64 new_data;
616 
617 	if (hw->device_id == I40E_DEV_ID_QEMU) {
618 		new_data = rd32(hw, loreg);
619 		new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
620 	} else {
621 		new_data = rd64(hw, loreg);
622 	}
623 	if (!offset_loaded)
624 		*offset = new_data;
625 	if (likely(new_data >= *offset))
626 		*stat = new_data - *offset;
627 	else
628 		*stat = (new_data + BIT_ULL(48)) - *offset;
629 	*stat &= 0xFFFFFFFFFFFFULL;
630 }
631 
632 /**
633  * i40e_stat_update32 - read and update a 32 bit stat from the chip
634  * @hw: ptr to the hardware info
635  * @reg: the hw reg to read
636  * @offset_loaded: has the initial offset been loaded yet
637  * @offset: ptr to current offset value
638  * @stat: ptr to the stat
639  **/
640 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
641 			       bool offset_loaded, u64 *offset, u64 *stat)
642 {
643 	u32 new_data;
644 
645 	new_data = rd32(hw, reg);
646 	if (!offset_loaded)
647 		*offset = new_data;
648 	if (likely(new_data >= *offset))
649 		*stat = (u32)(new_data - *offset);
650 	else
651 		*stat = (u32)((new_data + BIT_ULL(32)) - *offset);
652 }
653 
654 /**
655  * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat
656  * @hw: ptr to the hardware info
657  * @reg: the hw reg to read and clear
658  * @stat: ptr to the stat
659  **/
660 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
661 {
662 	u32 new_data = rd32(hw, reg);
663 
664 	wr32(hw, reg, 1); /* must write a nonzero value to clear register */
665 	*stat += new_data;
666 }
667 
668 /**
669  * i40e_stats_update_rx_discards - update rx_discards.
670  * @vsi: ptr to the VSI to be updated.
671  * @hw: ptr to the hardware info.
672  * @stat_idx: VSI's stat_counter_idx.
673  * @offset_loaded: ptr to the VSI's stat_offsets_loaded.
674  * @stat_offset: ptr to stat_offset to store first read of specific register.
675  * @stat: ptr to VSI's stat to be updated.
676  **/
677 static void
678 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw,
679 			      int stat_idx, bool offset_loaded,
680 			      struct i40e_eth_stats *stat_offset,
681 			      struct i40e_eth_stats *stat)
682 {
683 	u64 rx_rdpc, rx_rxerr;
684 
685 	i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded,
686 			   &stat_offset->rx_discards, &rx_rdpc);
687 	i40e_stat_update64(hw,
688 			   I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)),
689 			   I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)),
690 			   offset_loaded, &stat_offset->rx_discards_other,
691 			   &rx_rxerr);
692 
693 	stat->rx_discards = rx_rdpc + rx_rxerr;
694 }
695 
696 /**
697  * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters.
698  * @vsi: the VSI to be updated
699  **/
700 void i40e_update_eth_stats(struct i40e_vsi *vsi)
701 {
702 	int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
703 	struct i40e_pf *pf = vsi->back;
704 	struct i40e_hw *hw = &pf->hw;
705 	struct i40e_eth_stats *oes;
706 	struct i40e_eth_stats *es;     /* device's eth stats */
707 
708 	es = &vsi->eth_stats;
709 	oes = &vsi->eth_stats_offsets;
710 
711 	/* Gather up the stats that the hw collects */
712 	i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
713 			   vsi->stat_offsets_loaded,
714 			   &oes->tx_errors, &es->tx_errors);
715 	i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
716 			   vsi->stat_offsets_loaded,
717 			   &oes->rx_discards, &es->rx_discards);
718 	i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
719 			   vsi->stat_offsets_loaded,
720 			   &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
721 
722 	i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
723 			   I40E_GLV_GORCL(stat_idx),
724 			   vsi->stat_offsets_loaded,
725 			   &oes->rx_bytes, &es->rx_bytes);
726 	i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
727 			   I40E_GLV_UPRCL(stat_idx),
728 			   vsi->stat_offsets_loaded,
729 			   &oes->rx_unicast, &es->rx_unicast);
730 	i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
731 			   I40E_GLV_MPRCL(stat_idx),
732 			   vsi->stat_offsets_loaded,
733 			   &oes->rx_multicast, &es->rx_multicast);
734 	i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
735 			   I40E_GLV_BPRCL(stat_idx),
736 			   vsi->stat_offsets_loaded,
737 			   &oes->rx_broadcast, &es->rx_broadcast);
738 
739 	i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
740 			   I40E_GLV_GOTCL(stat_idx),
741 			   vsi->stat_offsets_loaded,
742 			   &oes->tx_bytes, &es->tx_bytes);
743 	i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
744 			   I40E_GLV_UPTCL(stat_idx),
745 			   vsi->stat_offsets_loaded,
746 			   &oes->tx_unicast, &es->tx_unicast);
747 	i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
748 			   I40E_GLV_MPTCL(stat_idx),
749 			   vsi->stat_offsets_loaded,
750 			   &oes->tx_multicast, &es->tx_multicast);
751 	i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
752 			   I40E_GLV_BPTCL(stat_idx),
753 			   vsi->stat_offsets_loaded,
754 			   &oes->tx_broadcast, &es->tx_broadcast);
755 
756 	i40e_stats_update_rx_discards(vsi, hw, stat_idx,
757 				      vsi->stat_offsets_loaded, oes, es);
758 
759 	vsi->stat_offsets_loaded = true;
760 }
761 
762 /**
763  * i40e_update_veb_stats - Update Switch component statistics
764  * @veb: the VEB being updated
765  **/
766 void i40e_update_veb_stats(struct i40e_veb *veb)
767 {
768 	struct i40e_pf *pf = veb->pf;
769 	struct i40e_hw *hw = &pf->hw;
770 	struct i40e_eth_stats *oes;
771 	struct i40e_eth_stats *es;     /* device's eth stats */
772 	struct i40e_veb_tc_stats *veb_oes;
773 	struct i40e_veb_tc_stats *veb_es;
774 	int i, idx = 0;
775 
776 	idx = veb->stats_idx;
777 	es = &veb->stats;
778 	oes = &veb->stats_offsets;
779 	veb_es = &veb->tc_stats;
780 	veb_oes = &veb->tc_stats_offsets;
781 
782 	/* Gather up the stats that the hw collects */
783 	i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
784 			   veb->stat_offsets_loaded,
785 			   &oes->tx_discards, &es->tx_discards);
786 	if (hw->revision_id > 0)
787 		i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
788 				   veb->stat_offsets_loaded,
789 				   &oes->rx_unknown_protocol,
790 				   &es->rx_unknown_protocol);
791 	i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
792 			   veb->stat_offsets_loaded,
793 			   &oes->rx_bytes, &es->rx_bytes);
794 	i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
795 			   veb->stat_offsets_loaded,
796 			   &oes->rx_unicast, &es->rx_unicast);
797 	i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
798 			   veb->stat_offsets_loaded,
799 			   &oes->rx_multicast, &es->rx_multicast);
800 	i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
801 			   veb->stat_offsets_loaded,
802 			   &oes->rx_broadcast, &es->rx_broadcast);
803 
804 	i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
805 			   veb->stat_offsets_loaded,
806 			   &oes->tx_bytes, &es->tx_bytes);
807 	i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
808 			   veb->stat_offsets_loaded,
809 			   &oes->tx_unicast, &es->tx_unicast);
810 	i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
811 			   veb->stat_offsets_loaded,
812 			   &oes->tx_multicast, &es->tx_multicast);
813 	i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
814 			   veb->stat_offsets_loaded,
815 			   &oes->tx_broadcast, &es->tx_broadcast);
816 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
817 		i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
818 				   I40E_GLVEBTC_RPCL(i, idx),
819 				   veb->stat_offsets_loaded,
820 				   &veb_oes->tc_rx_packets[i],
821 				   &veb_es->tc_rx_packets[i]);
822 		i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
823 				   I40E_GLVEBTC_RBCL(i, idx),
824 				   veb->stat_offsets_loaded,
825 				   &veb_oes->tc_rx_bytes[i],
826 				   &veb_es->tc_rx_bytes[i]);
827 		i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
828 				   I40E_GLVEBTC_TPCL(i, idx),
829 				   veb->stat_offsets_loaded,
830 				   &veb_oes->tc_tx_packets[i],
831 				   &veb_es->tc_tx_packets[i]);
832 		i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
833 				   I40E_GLVEBTC_TBCL(i, idx),
834 				   veb->stat_offsets_loaded,
835 				   &veb_oes->tc_tx_bytes[i],
836 				   &veb_es->tc_tx_bytes[i]);
837 	}
838 	veb->stat_offsets_loaded = true;
839 }
840 
841 /**
842  * i40e_update_vsi_stats - Update the vsi statistics counters.
843  * @vsi: the VSI to be updated
844  *
845  * There are a few instances where we store the same stat in a
846  * couple of different structs.  This is partly because we have
847  * the netdev stats that need to be filled out, which is slightly
848  * different from the "eth_stats" defined by the chip and used in
849  * VF communications.  We sort it out here.
850  **/
851 static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
852 {
853 	u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy;
854 	struct i40e_pf *pf = vsi->back;
855 	struct rtnl_link_stats64 *ons;
856 	struct rtnl_link_stats64 *ns;   /* netdev stats */
857 	struct i40e_eth_stats *oes;
858 	struct i40e_eth_stats *es;     /* device's eth stats */
859 	u64 tx_restart, tx_busy;
860 	struct i40e_ring *p;
861 	u64 bytes, packets;
862 	unsigned int start;
863 	u64 tx_linearize;
864 	u64 tx_force_wb;
865 	u64 tx_stopped;
866 	u64 rx_p, rx_b;
867 	u64 tx_p, tx_b;
868 	u16 q;
869 
870 	if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
871 	    test_bit(__I40E_CONFIG_BUSY, pf->state))
872 		return;
873 
874 	ns = i40e_get_vsi_stats_struct(vsi);
875 	ons = &vsi->net_stats_offsets;
876 	es = &vsi->eth_stats;
877 	oes = &vsi->eth_stats_offsets;
878 
879 	/* Gather up the netdev and vsi stats that the driver collects
880 	 * on the fly during packet processing
881 	 */
882 	rx_b = rx_p = 0;
883 	tx_b = tx_p = 0;
884 	tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
885 	tx_stopped = 0;
886 	rx_page = 0;
887 	rx_buf = 0;
888 	rx_reuse = 0;
889 	rx_alloc = 0;
890 	rx_waive = 0;
891 	rx_busy = 0;
892 	rcu_read_lock();
893 	for (q = 0; q < vsi->num_queue_pairs; q++) {
894 		/* locate Tx ring */
895 		p = READ_ONCE(vsi->tx_rings[q]);
896 		if (!p)
897 			continue;
898 
899 		do {
900 			start = u64_stats_fetch_begin(&p->syncp);
901 			packets = p->stats.packets;
902 			bytes = p->stats.bytes;
903 		} while (u64_stats_fetch_retry(&p->syncp, start));
904 		tx_b += bytes;
905 		tx_p += packets;
906 		tx_restart += p->tx_stats.restart_queue;
907 		tx_busy += p->tx_stats.tx_busy;
908 		tx_linearize += p->tx_stats.tx_linearize;
909 		tx_force_wb += p->tx_stats.tx_force_wb;
910 		tx_stopped += p->tx_stats.tx_stopped;
911 
912 		/* locate Rx ring */
913 		p = READ_ONCE(vsi->rx_rings[q]);
914 		if (!p)
915 			continue;
916 
917 		do {
918 			start = u64_stats_fetch_begin(&p->syncp);
919 			packets = p->stats.packets;
920 			bytes = p->stats.bytes;
921 		} while (u64_stats_fetch_retry(&p->syncp, start));
922 		rx_b += bytes;
923 		rx_p += packets;
924 		rx_buf += p->rx_stats.alloc_buff_failed;
925 		rx_page += p->rx_stats.alloc_page_failed;
926 		rx_reuse += p->rx_stats.page_reuse_count;
927 		rx_alloc += p->rx_stats.page_alloc_count;
928 		rx_waive += p->rx_stats.page_waive_count;
929 		rx_busy += p->rx_stats.page_busy_count;
930 
931 		if (i40e_enabled_xdp_vsi(vsi)) {
932 			/* locate XDP ring */
933 			p = READ_ONCE(vsi->xdp_rings[q]);
934 			if (!p)
935 				continue;
936 
937 			do {
938 				start = u64_stats_fetch_begin(&p->syncp);
939 				packets = p->stats.packets;
940 				bytes = p->stats.bytes;
941 			} while (u64_stats_fetch_retry(&p->syncp, start));
942 			tx_b += bytes;
943 			tx_p += packets;
944 			tx_restart += p->tx_stats.restart_queue;
945 			tx_busy += p->tx_stats.tx_busy;
946 			tx_linearize += p->tx_stats.tx_linearize;
947 			tx_force_wb += p->tx_stats.tx_force_wb;
948 		}
949 	}
950 	rcu_read_unlock();
951 	vsi->tx_restart = tx_restart;
952 	vsi->tx_busy = tx_busy;
953 	vsi->tx_linearize = tx_linearize;
954 	vsi->tx_force_wb = tx_force_wb;
955 	vsi->tx_stopped = tx_stopped;
956 	vsi->rx_page_failed = rx_page;
957 	vsi->rx_buf_failed = rx_buf;
958 	vsi->rx_page_reuse = rx_reuse;
959 	vsi->rx_page_alloc = rx_alloc;
960 	vsi->rx_page_waive = rx_waive;
961 	vsi->rx_page_busy = rx_busy;
962 
963 	ns->rx_packets = rx_p;
964 	ns->rx_bytes = rx_b;
965 	ns->tx_packets = tx_p;
966 	ns->tx_bytes = tx_b;
967 
968 	/* update netdev stats from eth stats */
969 	i40e_update_eth_stats(vsi);
970 	ons->tx_errors = oes->tx_errors;
971 	ns->tx_errors = es->tx_errors;
972 	ons->multicast = oes->rx_multicast;
973 	ns->multicast = es->rx_multicast;
974 	ons->rx_dropped = oes->rx_discards;
975 	ns->rx_dropped = es->rx_discards;
976 	ons->tx_dropped = oes->tx_discards;
977 	ns->tx_dropped = es->tx_discards;
978 
979 	/* pull in a couple PF stats if this is the main vsi */
980 	if (vsi == pf->vsi[pf->lan_vsi]) {
981 		ns->rx_crc_errors = pf->stats.crc_errors;
982 		ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
983 		ns->rx_length_errors = pf->stats.rx_length_errors;
984 	}
985 }
986 
987 /**
988  * i40e_update_pf_stats - Update the PF statistics counters.
989  * @pf: the PF to be updated
990  **/
991 static void i40e_update_pf_stats(struct i40e_pf *pf)
992 {
993 	struct i40e_hw_port_stats *osd = &pf->stats_offsets;
994 	struct i40e_hw_port_stats *nsd = &pf->stats;
995 	struct i40e_hw *hw = &pf->hw;
996 	u32 val;
997 	int i;
998 
999 	i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
1000 			   I40E_GLPRT_GORCL(hw->port),
1001 			   pf->stat_offsets_loaded,
1002 			   &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
1003 	i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
1004 			   I40E_GLPRT_GOTCL(hw->port),
1005 			   pf->stat_offsets_loaded,
1006 			   &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
1007 	i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
1008 			   pf->stat_offsets_loaded,
1009 			   &osd->eth.rx_discards,
1010 			   &nsd->eth.rx_discards);
1011 	i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
1012 			   I40E_GLPRT_UPRCL(hw->port),
1013 			   pf->stat_offsets_loaded,
1014 			   &osd->eth.rx_unicast,
1015 			   &nsd->eth.rx_unicast);
1016 	i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
1017 			   I40E_GLPRT_MPRCL(hw->port),
1018 			   pf->stat_offsets_loaded,
1019 			   &osd->eth.rx_multicast,
1020 			   &nsd->eth.rx_multicast);
1021 	i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
1022 			   I40E_GLPRT_BPRCL(hw->port),
1023 			   pf->stat_offsets_loaded,
1024 			   &osd->eth.rx_broadcast,
1025 			   &nsd->eth.rx_broadcast);
1026 	i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
1027 			   I40E_GLPRT_UPTCL(hw->port),
1028 			   pf->stat_offsets_loaded,
1029 			   &osd->eth.tx_unicast,
1030 			   &nsd->eth.tx_unicast);
1031 	i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
1032 			   I40E_GLPRT_MPTCL(hw->port),
1033 			   pf->stat_offsets_loaded,
1034 			   &osd->eth.tx_multicast,
1035 			   &nsd->eth.tx_multicast);
1036 	i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
1037 			   I40E_GLPRT_BPTCL(hw->port),
1038 			   pf->stat_offsets_loaded,
1039 			   &osd->eth.tx_broadcast,
1040 			   &nsd->eth.tx_broadcast);
1041 
1042 	i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
1043 			   pf->stat_offsets_loaded,
1044 			   &osd->tx_dropped_link_down,
1045 			   &nsd->tx_dropped_link_down);
1046 
1047 	i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
1048 			   pf->stat_offsets_loaded,
1049 			   &osd->crc_errors, &nsd->crc_errors);
1050 
1051 	i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
1052 			   pf->stat_offsets_loaded,
1053 			   &osd->illegal_bytes, &nsd->illegal_bytes);
1054 
1055 	i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
1056 			   pf->stat_offsets_loaded,
1057 			   &osd->mac_local_faults,
1058 			   &nsd->mac_local_faults);
1059 	i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
1060 			   pf->stat_offsets_loaded,
1061 			   &osd->mac_remote_faults,
1062 			   &nsd->mac_remote_faults);
1063 
1064 	i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
1065 			   pf->stat_offsets_loaded,
1066 			   &osd->rx_length_errors,
1067 			   &nsd->rx_length_errors);
1068 
1069 	i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
1070 			   pf->stat_offsets_loaded,
1071 			   &osd->link_xon_rx, &nsd->link_xon_rx);
1072 	i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
1073 			   pf->stat_offsets_loaded,
1074 			   &osd->link_xon_tx, &nsd->link_xon_tx);
1075 	i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
1076 			   pf->stat_offsets_loaded,
1077 			   &osd->link_xoff_rx, &nsd->link_xoff_rx);
1078 	i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
1079 			   pf->stat_offsets_loaded,
1080 			   &osd->link_xoff_tx, &nsd->link_xoff_tx);
1081 
1082 	for (i = 0; i < 8; i++) {
1083 		i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
1084 				   pf->stat_offsets_loaded,
1085 				   &osd->priority_xoff_rx[i],
1086 				   &nsd->priority_xoff_rx[i]);
1087 		i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
1088 				   pf->stat_offsets_loaded,
1089 				   &osd->priority_xon_rx[i],
1090 				   &nsd->priority_xon_rx[i]);
1091 		i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
1092 				   pf->stat_offsets_loaded,
1093 				   &osd->priority_xon_tx[i],
1094 				   &nsd->priority_xon_tx[i]);
1095 		i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
1096 				   pf->stat_offsets_loaded,
1097 				   &osd->priority_xoff_tx[i],
1098 				   &nsd->priority_xoff_tx[i]);
1099 		i40e_stat_update32(hw,
1100 				   I40E_GLPRT_RXON2OFFCNT(hw->port, i),
1101 				   pf->stat_offsets_loaded,
1102 				   &osd->priority_xon_2_xoff[i],
1103 				   &nsd->priority_xon_2_xoff[i]);
1104 	}
1105 
1106 	i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
1107 			   I40E_GLPRT_PRC64L(hw->port),
1108 			   pf->stat_offsets_loaded,
1109 			   &osd->rx_size_64, &nsd->rx_size_64);
1110 	i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
1111 			   I40E_GLPRT_PRC127L(hw->port),
1112 			   pf->stat_offsets_loaded,
1113 			   &osd->rx_size_127, &nsd->rx_size_127);
1114 	i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
1115 			   I40E_GLPRT_PRC255L(hw->port),
1116 			   pf->stat_offsets_loaded,
1117 			   &osd->rx_size_255, &nsd->rx_size_255);
1118 	i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
1119 			   I40E_GLPRT_PRC511L(hw->port),
1120 			   pf->stat_offsets_loaded,
1121 			   &osd->rx_size_511, &nsd->rx_size_511);
1122 	i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
1123 			   I40E_GLPRT_PRC1023L(hw->port),
1124 			   pf->stat_offsets_loaded,
1125 			   &osd->rx_size_1023, &nsd->rx_size_1023);
1126 	i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1127 			   I40E_GLPRT_PRC1522L(hw->port),
1128 			   pf->stat_offsets_loaded,
1129 			   &osd->rx_size_1522, &nsd->rx_size_1522);
1130 	i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1131 			   I40E_GLPRT_PRC9522L(hw->port),
1132 			   pf->stat_offsets_loaded,
1133 			   &osd->rx_size_big, &nsd->rx_size_big);
1134 
1135 	i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1136 			   I40E_GLPRT_PTC64L(hw->port),
1137 			   pf->stat_offsets_loaded,
1138 			   &osd->tx_size_64, &nsd->tx_size_64);
1139 	i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1140 			   I40E_GLPRT_PTC127L(hw->port),
1141 			   pf->stat_offsets_loaded,
1142 			   &osd->tx_size_127, &nsd->tx_size_127);
1143 	i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1144 			   I40E_GLPRT_PTC255L(hw->port),
1145 			   pf->stat_offsets_loaded,
1146 			   &osd->tx_size_255, &nsd->tx_size_255);
1147 	i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1148 			   I40E_GLPRT_PTC511L(hw->port),
1149 			   pf->stat_offsets_loaded,
1150 			   &osd->tx_size_511, &nsd->tx_size_511);
1151 	i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1152 			   I40E_GLPRT_PTC1023L(hw->port),
1153 			   pf->stat_offsets_loaded,
1154 			   &osd->tx_size_1023, &nsd->tx_size_1023);
1155 	i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1156 			   I40E_GLPRT_PTC1522L(hw->port),
1157 			   pf->stat_offsets_loaded,
1158 			   &osd->tx_size_1522, &nsd->tx_size_1522);
1159 	i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1160 			   I40E_GLPRT_PTC9522L(hw->port),
1161 			   pf->stat_offsets_loaded,
1162 			   &osd->tx_size_big, &nsd->tx_size_big);
1163 
1164 	i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1165 			   pf->stat_offsets_loaded,
1166 			   &osd->rx_undersize, &nsd->rx_undersize);
1167 	i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1168 			   pf->stat_offsets_loaded,
1169 			   &osd->rx_fragments, &nsd->rx_fragments);
1170 	i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1171 			   pf->stat_offsets_loaded,
1172 			   &osd->rx_oversize, &nsd->rx_oversize);
1173 	i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1174 			   pf->stat_offsets_loaded,
1175 			   &osd->rx_jabber, &nsd->rx_jabber);
1176 
1177 	/* FDIR stats */
1178 	i40e_stat_update_and_clear32(hw,
1179 			I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1180 			&nsd->fd_atr_match);
1181 	i40e_stat_update_and_clear32(hw,
1182 			I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1183 			&nsd->fd_sb_match);
1184 	i40e_stat_update_and_clear32(hw,
1185 			I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1186 			&nsd->fd_atr_tunnel_match);
1187 
1188 	val = rd32(hw, I40E_PRTPM_EEE_STAT);
1189 	nsd->tx_lpi_status =
1190 		       (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1191 			I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1192 	nsd->rx_lpi_status =
1193 		       (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1194 			I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1195 	i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1196 			   pf->stat_offsets_loaded,
1197 			   &osd->tx_lpi_count, &nsd->tx_lpi_count);
1198 	i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1199 			   pf->stat_offsets_loaded,
1200 			   &osd->rx_lpi_count, &nsd->rx_lpi_count);
1201 
1202 	if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1203 	    !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1204 		nsd->fd_sb_status = true;
1205 	else
1206 		nsd->fd_sb_status = false;
1207 
1208 	if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1209 	    !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1210 		nsd->fd_atr_status = true;
1211 	else
1212 		nsd->fd_atr_status = false;
1213 
1214 	pf->stat_offsets_loaded = true;
1215 }
1216 
1217 /**
1218  * i40e_update_stats - Update the various statistics counters.
1219  * @vsi: the VSI to be updated
1220  *
1221  * Update the various stats for this VSI and its related entities.
1222  **/
1223 void i40e_update_stats(struct i40e_vsi *vsi)
1224 {
1225 	struct i40e_pf *pf = vsi->back;
1226 
1227 	if (vsi == pf->vsi[pf->lan_vsi])
1228 		i40e_update_pf_stats(pf);
1229 
1230 	i40e_update_vsi_stats(vsi);
1231 }
1232 
1233 /**
1234  * i40e_count_filters - counts VSI mac filters
1235  * @vsi: the VSI to be searched
1236  *
1237  * Returns count of mac filters
1238  **/
1239 int i40e_count_filters(struct i40e_vsi *vsi)
1240 {
1241 	struct i40e_mac_filter *f;
1242 	struct hlist_node *h;
1243 	int bkt;
1244 	int cnt = 0;
1245 
1246 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
1247 		++cnt;
1248 
1249 	return cnt;
1250 }
1251 
1252 /**
1253  * i40e_find_filter - Search VSI filter list for specific mac/vlan filter
1254  * @vsi: the VSI to be searched
1255  * @macaddr: the MAC address
1256  * @vlan: the vlan
1257  *
1258  * Returns ptr to the filter object or NULL
1259  **/
1260 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1261 						const u8 *macaddr, s16 vlan)
1262 {
1263 	struct i40e_mac_filter *f;
1264 	u64 key;
1265 
1266 	if (!vsi || !macaddr)
1267 		return NULL;
1268 
1269 	key = i40e_addr_to_hkey(macaddr);
1270 	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1271 		if ((ether_addr_equal(macaddr, f->macaddr)) &&
1272 		    (vlan == f->vlan))
1273 			return f;
1274 	}
1275 	return NULL;
1276 }
1277 
1278 /**
1279  * i40e_find_mac - Find a mac addr in the macvlan filters list
1280  * @vsi: the VSI to be searched
1281  * @macaddr: the MAC address we are searching for
1282  *
1283  * Returns the first filter with the provided MAC address or NULL if
1284  * MAC address was not found
1285  **/
1286 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1287 {
1288 	struct i40e_mac_filter *f;
1289 	u64 key;
1290 
1291 	if (!vsi || !macaddr)
1292 		return NULL;
1293 
1294 	key = i40e_addr_to_hkey(macaddr);
1295 	hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1296 		if ((ether_addr_equal(macaddr, f->macaddr)))
1297 			return f;
1298 	}
1299 	return NULL;
1300 }
1301 
1302 /**
1303  * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode
1304  * @vsi: the VSI to be searched
1305  *
1306  * Returns true if VSI is in vlan mode or false otherwise
1307  **/
1308 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1309 {
1310 	/* If we have a PVID, always operate in VLAN mode */
1311 	if (vsi->info.pvid)
1312 		return true;
1313 
1314 	/* We need to operate in VLAN mode whenever we have any filters with
1315 	 * a VLAN other than I40E_VLAN_ALL. We could check the table each
1316 	 * time, incurring search cost repeatedly. However, we can notice two
1317 	 * things:
1318 	 *
1319 	 * 1) the only place where we can gain a VLAN filter is in
1320 	 *    i40e_add_filter.
1321 	 *
1322 	 * 2) the only place where filters are actually removed is in
1323 	 *    i40e_sync_filters_subtask.
1324 	 *
1325 	 * Thus, we can simply use a boolean value, has_vlan_filters which we
1326 	 * will set to true when we add a VLAN filter in i40e_add_filter. Then
1327 	 * we have to perform the full search after deleting filters in
1328 	 * i40e_sync_filters_subtask, but we already have to search
1329 	 * filters here and can perform the check at the same time. This
1330 	 * results in avoiding embedding a loop for VLAN mode inside another
1331 	 * loop over all the filters, and should maintain correctness as noted
1332 	 * above.
1333 	 */
1334 	return vsi->has_vlan_filter;
1335 }
1336 
1337 /**
1338  * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary
1339  * @vsi: the VSI to configure
1340  * @tmp_add_list: list of filters ready to be added
1341  * @tmp_del_list: list of filters ready to be deleted
1342  * @vlan_filters: the number of active VLAN filters
1343  *
1344  * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they
1345  * behave as expected. If we have any active VLAN filters remaining or about
1346  * to be added then we need to update non-VLAN filters to be marked as VLAN=0
1347  * so that they only match against untagged traffic. If we no longer have any
1348  * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1
1349  * so that they match against both tagged and untagged traffic. In this way,
1350  * we ensure that we correctly receive the desired traffic. This ensures that
1351  * when we have an active VLAN we will receive only untagged traffic and
1352  * traffic matching active VLANs. If we have no active VLANs then we will
1353  * operate in non-VLAN mode and receive all traffic, tagged or untagged.
1354  *
1355  * Finally, in a similar fashion, this function also corrects filters when
1356  * there is an active PVID assigned to this VSI.
1357  *
1358  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1359  *
1360  * This function is only expected to be called from within
1361  * i40e_sync_vsi_filters.
1362  *
1363  * NOTE: This function expects to be called while under the
1364  * mac_filter_hash_lock
1365  */
1366 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1367 					 struct hlist_head *tmp_add_list,
1368 					 struct hlist_head *tmp_del_list,
1369 					 int vlan_filters)
1370 {
1371 	s16 pvid = le16_to_cpu(vsi->info.pvid);
1372 	struct i40e_mac_filter *f, *add_head;
1373 	struct i40e_new_mac_filter *new;
1374 	struct hlist_node *h;
1375 	int bkt, new_vlan;
1376 
1377 	/* To determine if a particular filter needs to be replaced we
1378 	 * have the three following conditions:
1379 	 *
1380 	 * a) if we have a PVID assigned, then all filters which are
1381 	 *    not marked as VLAN=PVID must be replaced with filters that
1382 	 *    are.
1383 	 * b) otherwise, if we have any active VLANS, all filters
1384 	 *    which are marked as VLAN=-1 must be replaced with
1385 	 *    filters marked as VLAN=0
1386 	 * c) finally, if we do not have any active VLANS, all filters
1387 	 *    which are marked as VLAN=0 must be replaced with filters
1388 	 *    marked as VLAN=-1
1389 	 */
1390 
1391 	/* Update the filters about to be added in place */
1392 	hlist_for_each_entry(new, tmp_add_list, hlist) {
1393 		if (pvid && new->f->vlan != pvid)
1394 			new->f->vlan = pvid;
1395 		else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1396 			new->f->vlan = 0;
1397 		else if (!vlan_filters && new->f->vlan == 0)
1398 			new->f->vlan = I40E_VLAN_ANY;
1399 	}
1400 
1401 	/* Update the remaining active filters */
1402 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1403 		/* Combine the checks for whether a filter needs to be changed
1404 		 * and then determine the new VLAN inside the if block, in
1405 		 * order to avoid duplicating code for adding the new filter
1406 		 * then deleting the old filter.
1407 		 */
1408 		if ((pvid && f->vlan != pvid) ||
1409 		    (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1410 		    (!vlan_filters && f->vlan == 0)) {
1411 			/* Determine the new vlan we will be adding */
1412 			if (pvid)
1413 				new_vlan = pvid;
1414 			else if (vlan_filters)
1415 				new_vlan = 0;
1416 			else
1417 				new_vlan = I40E_VLAN_ANY;
1418 
1419 			/* Create the new filter */
1420 			add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1421 			if (!add_head)
1422 				return -ENOMEM;
1423 
1424 			/* Create a temporary i40e_new_mac_filter */
1425 			new = kzalloc(sizeof(*new), GFP_ATOMIC);
1426 			if (!new)
1427 				return -ENOMEM;
1428 
1429 			new->f = add_head;
1430 			new->state = add_head->state;
1431 
1432 			/* Add the new filter to the tmp list */
1433 			hlist_add_head(&new->hlist, tmp_add_list);
1434 
1435 			/* Put the original filter into the delete list */
1436 			f->state = I40E_FILTER_REMOVE;
1437 			hash_del(&f->hlist);
1438 			hlist_add_head(&f->hlist, tmp_del_list);
1439 		}
1440 	}
1441 
1442 	vsi->has_vlan_filter = !!vlan_filters;
1443 
1444 	return 0;
1445 }
1446 
1447 /**
1448  * i40e_get_vf_new_vlan - Get new vlan id on a vf
1449  * @vsi: the vsi to configure
1450  * @new_mac: new mac filter to be added
1451  * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL
1452  * @vlan_filters: the number of active VLAN filters
1453  * @trusted: flag if the VF is trusted
1454  *
1455  * Get new VLAN id based on current VLAN filters, trust, PVID
1456  * and vf-vlan-prune-disable flag.
1457  *
1458  * Returns the value of the new vlan filter or
1459  * the old value if no new filter is needed.
1460  */
1461 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi,
1462 				struct i40e_new_mac_filter *new_mac,
1463 				struct i40e_mac_filter *f,
1464 				int vlan_filters,
1465 				bool trusted)
1466 {
1467 	s16 pvid = le16_to_cpu(vsi->info.pvid);
1468 	struct i40e_pf *pf = vsi->back;
1469 	bool is_any;
1470 
1471 	if (new_mac)
1472 		f = new_mac->f;
1473 
1474 	if (pvid && f->vlan != pvid)
1475 		return pvid;
1476 
1477 	is_any = (trusted ||
1478 		  !(pf->flags & I40E_FLAG_VF_VLAN_PRUNING));
1479 
1480 	if ((vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1481 	    (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1482 	    (is_any && !vlan_filters && f->vlan == 0)) {
1483 		if (is_any)
1484 			return I40E_VLAN_ANY;
1485 		else
1486 			return 0;
1487 	}
1488 
1489 	return f->vlan;
1490 }
1491 
1492 /**
1493  * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary
1494  * @vsi: the vsi to configure
1495  * @tmp_add_list: list of filters ready to be added
1496  * @tmp_del_list: list of filters ready to be deleted
1497  * @vlan_filters: the number of active VLAN filters
1498  * @trusted: flag if the VF is trusted
1499  *
1500  * Correct VF VLAN filters based on current VLAN filters, trust, PVID
1501  * and vf-vlan-prune-disable flag.
1502  *
1503  * In case of memory allocation failure return -ENOMEM. Otherwise, return 0.
1504  *
1505  * This function is only expected to be called from within
1506  * i40e_sync_vsi_filters.
1507  *
1508  * NOTE: This function expects to be called while under the
1509  * mac_filter_hash_lock
1510  */
1511 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi,
1512 					    struct hlist_head *tmp_add_list,
1513 					    struct hlist_head *tmp_del_list,
1514 					    int vlan_filters,
1515 					    bool trusted)
1516 {
1517 	struct i40e_mac_filter *f, *add_head;
1518 	struct i40e_new_mac_filter *new_mac;
1519 	struct hlist_node *h;
1520 	int bkt, new_vlan;
1521 
1522 	hlist_for_each_entry(new_mac, tmp_add_list, hlist) {
1523 		new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL,
1524 							vlan_filters, trusted);
1525 	}
1526 
1527 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1528 		new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters,
1529 						trusted);
1530 		if (new_vlan != f->vlan) {
1531 			add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1532 			if (!add_head)
1533 				return -ENOMEM;
1534 			/* Create a temporary i40e_new_mac_filter */
1535 			new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC);
1536 			if (!new_mac)
1537 				return -ENOMEM;
1538 			new_mac->f = add_head;
1539 			new_mac->state = add_head->state;
1540 
1541 			/* Add the new filter to the tmp list */
1542 			hlist_add_head(&new_mac->hlist, tmp_add_list);
1543 
1544 			/* Put the original filter into the delete list */
1545 			f->state = I40E_FILTER_REMOVE;
1546 			hash_del(&f->hlist);
1547 			hlist_add_head(&f->hlist, tmp_del_list);
1548 		}
1549 	}
1550 
1551 	vsi->has_vlan_filter = !!vlan_filters;
1552 	return 0;
1553 }
1554 
1555 /**
1556  * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM
1557  * @vsi: the PF Main VSI - inappropriate for any other VSI
1558  * @macaddr: the MAC address
1559  *
1560  * Remove whatever filter the firmware set up so the driver can manage
1561  * its own filtering intelligently.
1562  **/
1563 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1564 {
1565 	struct i40e_aqc_remove_macvlan_element_data element;
1566 	struct i40e_pf *pf = vsi->back;
1567 
1568 	/* Only appropriate for the PF main VSI */
1569 	if (vsi->type != I40E_VSI_MAIN)
1570 		return;
1571 
1572 	memset(&element, 0, sizeof(element));
1573 	ether_addr_copy(element.mac_addr, macaddr);
1574 	element.vlan_tag = 0;
1575 	/* Ignore error returns, some firmware does it this way... */
1576 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1577 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1578 
1579 	memset(&element, 0, sizeof(element));
1580 	ether_addr_copy(element.mac_addr, macaddr);
1581 	element.vlan_tag = 0;
1582 	/* ...and some firmware does it this way. */
1583 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1584 			I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1585 	i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1586 }
1587 
1588 /**
1589  * i40e_add_filter - Add a mac/vlan filter to the VSI
1590  * @vsi: the VSI to be searched
1591  * @macaddr: the MAC address
1592  * @vlan: the vlan
1593  *
1594  * Returns ptr to the filter object or NULL when no memory available.
1595  *
1596  * NOTE: This function is expected to be called with mac_filter_hash_lock
1597  * being held.
1598  **/
1599 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1600 					const u8 *macaddr, s16 vlan)
1601 {
1602 	struct i40e_mac_filter *f;
1603 	u64 key;
1604 
1605 	if (!vsi || !macaddr)
1606 		return NULL;
1607 
1608 	f = i40e_find_filter(vsi, macaddr, vlan);
1609 	if (!f) {
1610 		f = kzalloc(sizeof(*f), GFP_ATOMIC);
1611 		if (!f)
1612 			return NULL;
1613 
1614 		/* Update the boolean indicating if we need to function in
1615 		 * VLAN mode.
1616 		 */
1617 		if (vlan >= 0)
1618 			vsi->has_vlan_filter = true;
1619 
1620 		ether_addr_copy(f->macaddr, macaddr);
1621 		f->vlan = vlan;
1622 		f->state = I40E_FILTER_NEW;
1623 		INIT_HLIST_NODE(&f->hlist);
1624 
1625 		key = i40e_addr_to_hkey(macaddr);
1626 		hash_add(vsi->mac_filter_hash, &f->hlist, key);
1627 
1628 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1629 		set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1630 	}
1631 
1632 	/* If we're asked to add a filter that has been marked for removal, it
1633 	 * is safe to simply restore it to active state. __i40e_del_filter
1634 	 * will have simply deleted any filters which were previously marked
1635 	 * NEW or FAILED, so if it is currently marked REMOVE it must have
1636 	 * previously been ACTIVE. Since we haven't yet run the sync filters
1637 	 * task, just restore this filter to the ACTIVE state so that the
1638 	 * sync task leaves it in place
1639 	 */
1640 	if (f->state == I40E_FILTER_REMOVE)
1641 		f->state = I40E_FILTER_ACTIVE;
1642 
1643 	return f;
1644 }
1645 
1646 /**
1647  * __i40e_del_filter - Remove a specific filter from the VSI
1648  * @vsi: VSI to remove from
1649  * @f: the filter to remove from the list
1650  *
1651  * This function should be called instead of i40e_del_filter only if you know
1652  * the exact filter you will remove already, such as via i40e_find_filter or
1653  * i40e_find_mac.
1654  *
1655  * NOTE: This function is expected to be called with mac_filter_hash_lock
1656  * being held.
1657  * ANOTHER NOTE: This function MUST be called from within the context of
1658  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1659  * instead of list_for_each_entry().
1660  **/
1661 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1662 {
1663 	if (!f)
1664 		return;
1665 
1666 	/* If the filter was never added to firmware then we can just delete it
1667 	 * directly and we don't want to set the status to remove or else an
1668 	 * admin queue command will unnecessarily fire.
1669 	 */
1670 	if ((f->state == I40E_FILTER_FAILED) ||
1671 	    (f->state == I40E_FILTER_NEW)) {
1672 		hash_del(&f->hlist);
1673 		kfree(f);
1674 	} else {
1675 		f->state = I40E_FILTER_REMOVE;
1676 	}
1677 
1678 	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1679 	set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1680 }
1681 
1682 /**
1683  * i40e_del_filter - Remove a MAC/VLAN filter from the VSI
1684  * @vsi: the VSI to be searched
1685  * @macaddr: the MAC address
1686  * @vlan: the VLAN
1687  *
1688  * NOTE: This function is expected to be called with mac_filter_hash_lock
1689  * being held.
1690  * ANOTHER NOTE: This function MUST be called from within the context of
1691  * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe()
1692  * instead of list_for_each_entry().
1693  **/
1694 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1695 {
1696 	struct i40e_mac_filter *f;
1697 
1698 	if (!vsi || !macaddr)
1699 		return;
1700 
1701 	f = i40e_find_filter(vsi, macaddr, vlan);
1702 	__i40e_del_filter(vsi, f);
1703 }
1704 
1705 /**
1706  * i40e_add_mac_filter - Add a MAC filter for all active VLANs
1707  * @vsi: the VSI to be searched
1708  * @macaddr: the mac address to be filtered
1709  *
1710  * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise,
1711  * go through all the macvlan filters and add a macvlan filter for each
1712  * unique vlan that already exists. If a PVID has been assigned, instead only
1713  * add the macaddr to that VLAN.
1714  *
1715  * Returns last filter added on success, else NULL
1716  **/
1717 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1718 					    const u8 *macaddr)
1719 {
1720 	struct i40e_mac_filter *f, *add = NULL;
1721 	struct hlist_node *h;
1722 	int bkt;
1723 
1724 	if (vsi->info.pvid)
1725 		return i40e_add_filter(vsi, macaddr,
1726 				       le16_to_cpu(vsi->info.pvid));
1727 
1728 	if (!i40e_is_vsi_in_vlan(vsi))
1729 		return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1730 
1731 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1732 		if (f->state == I40E_FILTER_REMOVE)
1733 			continue;
1734 		add = i40e_add_filter(vsi, macaddr, f->vlan);
1735 		if (!add)
1736 			return NULL;
1737 	}
1738 
1739 	return add;
1740 }
1741 
1742 /**
1743  * i40e_del_mac_filter - Remove a MAC filter from all VLANs
1744  * @vsi: the VSI to be searched
1745  * @macaddr: the mac address to be removed
1746  *
1747  * Removes a given MAC address from a VSI regardless of what VLAN it has been
1748  * associated with.
1749  *
1750  * Returns 0 for success, or error
1751  **/
1752 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1753 {
1754 	struct i40e_mac_filter *f;
1755 	struct hlist_node *h;
1756 	bool found = false;
1757 	int bkt;
1758 
1759 	lockdep_assert_held(&vsi->mac_filter_hash_lock);
1760 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1761 		if (ether_addr_equal(macaddr, f->macaddr)) {
1762 			__i40e_del_filter(vsi, f);
1763 			found = true;
1764 		}
1765 	}
1766 
1767 	if (found)
1768 		return 0;
1769 	else
1770 		return -ENOENT;
1771 }
1772 
1773 /**
1774  * i40e_set_mac - NDO callback to set mac address
1775  * @netdev: network interface device structure
1776  * @p: pointer to an address structure
1777  *
1778  * Returns 0 on success, negative on failure
1779  **/
1780 static int i40e_set_mac(struct net_device *netdev, void *p)
1781 {
1782 	struct i40e_netdev_priv *np = netdev_priv(netdev);
1783 	struct i40e_vsi *vsi = np->vsi;
1784 	struct i40e_pf *pf = vsi->back;
1785 	struct i40e_hw *hw = &pf->hw;
1786 	struct sockaddr *addr = p;
1787 
1788 	if (!is_valid_ether_addr(addr->sa_data))
1789 		return -EADDRNOTAVAIL;
1790 
1791 	if (test_bit(__I40E_DOWN, pf->state) ||
1792 	    test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1793 		return -EADDRNOTAVAIL;
1794 
1795 	if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1796 		netdev_info(netdev, "returning to hw mac address %pM\n",
1797 			    hw->mac.addr);
1798 	else
1799 		netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1800 
1801 	/* Copy the address first, so that we avoid a possible race with
1802 	 * .set_rx_mode().
1803 	 * - Remove old address from MAC filter
1804 	 * - Copy new address
1805 	 * - Add new address to MAC filter
1806 	 */
1807 	spin_lock_bh(&vsi->mac_filter_hash_lock);
1808 	i40e_del_mac_filter(vsi, netdev->dev_addr);
1809 	eth_hw_addr_set(netdev, addr->sa_data);
1810 	i40e_add_mac_filter(vsi, netdev->dev_addr);
1811 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
1812 
1813 	if (vsi->type == I40E_VSI_MAIN) {
1814 		int ret;
1815 
1816 		ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1817 						addr->sa_data, NULL);
1818 		if (ret)
1819 			netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n",
1820 				    ERR_PTR(ret),
1821 				    i40e_aq_str(hw, hw->aq.asq_last_status));
1822 	}
1823 
1824 	/* schedule our worker thread which will take care of
1825 	 * applying the new filter changes
1826 	 */
1827 	i40e_service_event_schedule(pf);
1828 	return 0;
1829 }
1830 
1831 /**
1832  * i40e_config_rss_aq - Prepare for RSS using AQ commands
1833  * @vsi: vsi structure
1834  * @seed: RSS hash seed
1835  * @lut: pointer to lookup table of lut_size
1836  * @lut_size: size of the lookup table
1837  **/
1838 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1839 			      u8 *lut, u16 lut_size)
1840 {
1841 	struct i40e_pf *pf = vsi->back;
1842 	struct i40e_hw *hw = &pf->hw;
1843 	int ret = 0;
1844 
1845 	if (seed) {
1846 		struct i40e_aqc_get_set_rss_key_data *seed_dw =
1847 			(struct i40e_aqc_get_set_rss_key_data *)seed;
1848 		ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1849 		if (ret) {
1850 			dev_info(&pf->pdev->dev,
1851 				 "Cannot set RSS key, err %pe aq_err %s\n",
1852 				 ERR_PTR(ret),
1853 				 i40e_aq_str(hw, hw->aq.asq_last_status));
1854 			return ret;
1855 		}
1856 	}
1857 	if (lut) {
1858 		bool pf_lut = vsi->type == I40E_VSI_MAIN;
1859 
1860 		ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1861 		if (ret) {
1862 			dev_info(&pf->pdev->dev,
1863 				 "Cannot set RSS lut, err %pe aq_err %s\n",
1864 				 ERR_PTR(ret),
1865 				 i40e_aq_str(hw, hw->aq.asq_last_status));
1866 			return ret;
1867 		}
1868 	}
1869 	return ret;
1870 }
1871 
1872 /**
1873  * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used
1874  * @vsi: VSI structure
1875  **/
1876 static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1877 {
1878 	struct i40e_pf *pf = vsi->back;
1879 	u8 seed[I40E_HKEY_ARRAY_SIZE];
1880 	u8 *lut;
1881 	int ret;
1882 
1883 	if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1884 		return 0;
1885 	if (!vsi->rss_size)
1886 		vsi->rss_size = min_t(int, pf->alloc_rss_size,
1887 				      vsi->num_queue_pairs);
1888 	if (!vsi->rss_size)
1889 		return -EINVAL;
1890 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1891 	if (!lut)
1892 		return -ENOMEM;
1893 
1894 	/* Use the user configured hash keys and lookup table if there is one,
1895 	 * otherwise use default
1896 	 */
1897 	if (vsi->rss_lut_user)
1898 		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1899 	else
1900 		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1901 	if (vsi->rss_hkey_user)
1902 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1903 	else
1904 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1905 	ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1906 	kfree(lut);
1907 	return ret;
1908 }
1909 
1910 /**
1911  * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config
1912  * @vsi: the VSI being configured,
1913  * @ctxt: VSI context structure
1914  * @enabled_tc: number of traffic classes to enable
1915  *
1916  * Prepares VSI tc_config to have queue configurations based on MQPRIO options.
1917  **/
1918 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1919 					   struct i40e_vsi_context *ctxt,
1920 					   u8 enabled_tc)
1921 {
1922 	u16 qcount = 0, max_qcount, qmap, sections = 0;
1923 	int i, override_q, pow, num_qps, ret;
1924 	u8 netdev_tc = 0, offset = 0;
1925 
1926 	if (vsi->type != I40E_VSI_MAIN)
1927 		return -EINVAL;
1928 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1929 	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1930 	vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1931 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1932 	num_qps = vsi->mqprio_qopt.qopt.count[0];
1933 
1934 	/* find the next higher power-of-2 of num queue pairs */
1935 	pow = ilog2(num_qps);
1936 	if (!is_power_of_2(num_qps))
1937 		pow++;
1938 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1939 		(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1940 
1941 	/* Setup queue offset/count for all TCs for given VSI */
1942 	max_qcount = vsi->mqprio_qopt.qopt.count[0];
1943 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1944 		/* See if the given TC is enabled for the given VSI */
1945 		if (vsi->tc_config.enabled_tc & BIT(i)) {
1946 			offset = vsi->mqprio_qopt.qopt.offset[i];
1947 			qcount = vsi->mqprio_qopt.qopt.count[i];
1948 			if (qcount > max_qcount)
1949 				max_qcount = qcount;
1950 			vsi->tc_config.tc_info[i].qoffset = offset;
1951 			vsi->tc_config.tc_info[i].qcount = qcount;
1952 			vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1953 		} else {
1954 			/* TC is not enabled so set the offset to
1955 			 * default queue and allocate one queue
1956 			 * for the given TC.
1957 			 */
1958 			vsi->tc_config.tc_info[i].qoffset = 0;
1959 			vsi->tc_config.tc_info[i].qcount = 1;
1960 			vsi->tc_config.tc_info[i].netdev_tc = 0;
1961 		}
1962 	}
1963 
1964 	/* Set actual Tx/Rx queue pairs */
1965 	vsi->num_queue_pairs = offset + qcount;
1966 
1967 	/* Setup queue TC[0].qmap for given VSI context */
1968 	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1969 	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1970 	ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1971 	ctxt->info.valid_sections |= cpu_to_le16(sections);
1972 
1973 	/* Reconfigure RSS for main VSI with max queue count */
1974 	vsi->rss_size = max_qcount;
1975 	ret = i40e_vsi_config_rss(vsi);
1976 	if (ret) {
1977 		dev_info(&vsi->back->pdev->dev,
1978 			 "Failed to reconfig rss for num_queues (%u)\n",
1979 			 max_qcount);
1980 		return ret;
1981 	}
1982 	vsi->reconfig_rss = true;
1983 	dev_dbg(&vsi->back->pdev->dev,
1984 		"Reconfigured rss with num_queues (%u)\n", max_qcount);
1985 
1986 	/* Find queue count available for channel VSIs and starting offset
1987 	 * for channel VSIs
1988 	 */
1989 	override_q = vsi->mqprio_qopt.qopt.count[0];
1990 	if (override_q && override_q < vsi->num_queue_pairs) {
1991 		vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1992 		vsi->next_base_queue = override_q;
1993 	}
1994 	return 0;
1995 }
1996 
1997 /**
1998  * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc
1999  * @vsi: the VSI being setup
2000  * @ctxt: VSI context structure
2001  * @enabled_tc: Enabled TCs bitmap
2002  * @is_add: True if called before Add VSI
2003  *
2004  * Setup VSI queue mapping for enabled traffic classes.
2005  **/
2006 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
2007 				     struct i40e_vsi_context *ctxt,
2008 				     u8 enabled_tc,
2009 				     bool is_add)
2010 {
2011 	struct i40e_pf *pf = vsi->back;
2012 	u16 num_tc_qps = 0;
2013 	u16 sections = 0;
2014 	u8 netdev_tc = 0;
2015 	u16 numtc = 1;
2016 	u16 qcount;
2017 	u8 offset;
2018 	u16 qmap;
2019 	int i;
2020 
2021 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
2022 	offset = 0;
2023 	/* zero out queue mapping, it will get updated on the end of the function */
2024 	memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping));
2025 
2026 	if (vsi->type == I40E_VSI_MAIN) {
2027 		/* This code helps add more queue to the VSI if we have
2028 		 * more cores than RSS can support, the higher cores will
2029 		 * be served by ATR or other filters. Furthermore, the
2030 		 * non-zero req_queue_pairs says that user requested a new
2031 		 * queue count via ethtool's set_channels, so use this
2032 		 * value for queues distribution across traffic classes
2033 		 * We need at least one queue pair for the interface
2034 		 * to be usable as we see in else statement.
2035 		 */
2036 		if (vsi->req_queue_pairs > 0)
2037 			vsi->num_queue_pairs = vsi->req_queue_pairs;
2038 		else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
2039 			vsi->num_queue_pairs = pf->num_lan_msix;
2040 		else
2041 			vsi->num_queue_pairs = 1;
2042 	}
2043 
2044 	/* Number of queues per enabled TC */
2045 	if (vsi->type == I40E_VSI_MAIN ||
2046 	    (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0))
2047 		num_tc_qps = vsi->num_queue_pairs;
2048 	else
2049 		num_tc_qps = vsi->alloc_queue_pairs;
2050 
2051 	if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
2052 		/* Find numtc from enabled TC bitmap */
2053 		for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2054 			if (enabled_tc & BIT(i)) /* TC is enabled */
2055 				numtc++;
2056 		}
2057 		if (!numtc) {
2058 			dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
2059 			numtc = 1;
2060 		}
2061 		num_tc_qps = num_tc_qps / numtc;
2062 		num_tc_qps = min_t(int, num_tc_qps,
2063 				   i40e_pf_get_max_q_per_tc(pf));
2064 	}
2065 
2066 	vsi->tc_config.numtc = numtc;
2067 	vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
2068 
2069 	/* Do not allow use more TC queue pairs than MSI-X vectors exist */
2070 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
2071 		num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
2072 
2073 	/* Setup queue offset/count for all TCs for given VSI */
2074 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2075 		/* See if the given TC is enabled for the given VSI */
2076 		if (vsi->tc_config.enabled_tc & BIT(i)) {
2077 			/* TC is enabled */
2078 			int pow, num_qps;
2079 
2080 			switch (vsi->type) {
2081 			case I40E_VSI_MAIN:
2082 				if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
2083 				    I40E_FLAG_FD_ATR_ENABLED)) ||
2084 				    vsi->tc_config.enabled_tc != 1) {
2085 					qcount = min_t(int, pf->alloc_rss_size,
2086 						       num_tc_qps);
2087 					break;
2088 				}
2089 				fallthrough;
2090 			case I40E_VSI_FDIR:
2091 			case I40E_VSI_SRIOV:
2092 			case I40E_VSI_VMDQ2:
2093 			default:
2094 				qcount = num_tc_qps;
2095 				WARN_ON(i != 0);
2096 				break;
2097 			}
2098 			vsi->tc_config.tc_info[i].qoffset = offset;
2099 			vsi->tc_config.tc_info[i].qcount = qcount;
2100 
2101 			/* find the next higher power-of-2 of num queue pairs */
2102 			num_qps = qcount;
2103 			pow = 0;
2104 			while (num_qps && (BIT_ULL(pow) < qcount)) {
2105 				pow++;
2106 				num_qps >>= 1;
2107 			}
2108 
2109 			vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
2110 			qmap =
2111 			    (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
2112 			    (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
2113 
2114 			offset += qcount;
2115 		} else {
2116 			/* TC is not enabled so set the offset to
2117 			 * default queue and allocate one queue
2118 			 * for the given TC.
2119 			 */
2120 			vsi->tc_config.tc_info[i].qoffset = 0;
2121 			vsi->tc_config.tc_info[i].qcount = 1;
2122 			vsi->tc_config.tc_info[i].netdev_tc = 0;
2123 
2124 			qmap = 0;
2125 		}
2126 		ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
2127 	}
2128 	/* Do not change previously set num_queue_pairs for PFs and VFs*/
2129 	if ((vsi->type == I40E_VSI_MAIN && numtc != 1) ||
2130 	    (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) ||
2131 	    (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV))
2132 		vsi->num_queue_pairs = offset;
2133 
2134 	/* Scheduler section valid can only be set for ADD VSI */
2135 	if (is_add) {
2136 		sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
2137 
2138 		ctxt->info.up_enable_bits = enabled_tc;
2139 	}
2140 	if (vsi->type == I40E_VSI_SRIOV) {
2141 		ctxt->info.mapping_flags |=
2142 				     cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
2143 		for (i = 0; i < vsi->num_queue_pairs; i++)
2144 			ctxt->info.queue_mapping[i] =
2145 					       cpu_to_le16(vsi->base_queue + i);
2146 	} else {
2147 		ctxt->info.mapping_flags |=
2148 					cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
2149 		ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
2150 	}
2151 	ctxt->info.valid_sections |= cpu_to_le16(sections);
2152 }
2153 
2154 /**
2155  * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address
2156  * @netdev: the netdevice
2157  * @addr: address to add
2158  *
2159  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
2160  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2161  */
2162 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
2163 {
2164 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2165 	struct i40e_vsi *vsi = np->vsi;
2166 
2167 	if (i40e_add_mac_filter(vsi, addr))
2168 		return 0;
2169 	else
2170 		return -ENOMEM;
2171 }
2172 
2173 /**
2174  * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
2175  * @netdev: the netdevice
2176  * @addr: address to add
2177  *
2178  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
2179  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
2180  */
2181 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
2182 {
2183 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2184 	struct i40e_vsi *vsi = np->vsi;
2185 
2186 	/* Under some circumstances, we might receive a request to delete
2187 	 * our own device address from our uc list. Because we store the
2188 	 * device address in the VSI's MAC/VLAN filter list, we need to ignore
2189 	 * such requests and not delete our device address from this list.
2190 	 */
2191 	if (ether_addr_equal(addr, netdev->dev_addr))
2192 		return 0;
2193 
2194 	i40e_del_mac_filter(vsi, addr);
2195 
2196 	return 0;
2197 }
2198 
2199 /**
2200  * i40e_set_rx_mode - NDO callback to set the netdev filters
2201  * @netdev: network interface device structure
2202  **/
2203 static void i40e_set_rx_mode(struct net_device *netdev)
2204 {
2205 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2206 	struct i40e_vsi *vsi = np->vsi;
2207 
2208 	spin_lock_bh(&vsi->mac_filter_hash_lock);
2209 
2210 	__dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2211 	__dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
2212 
2213 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
2214 
2215 	/* check for other flag changes */
2216 	if (vsi->current_netdev_flags != vsi->netdev->flags) {
2217 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2218 		set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
2219 	}
2220 }
2221 
2222 /**
2223  * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries
2224  * @vsi: Pointer to VSI struct
2225  * @from: Pointer to list which contains MAC filter entries - changes to
2226  *        those entries needs to be undone.
2227  *
2228  * MAC filter entries from this list were slated for deletion.
2229  **/
2230 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
2231 					 struct hlist_head *from)
2232 {
2233 	struct i40e_mac_filter *f;
2234 	struct hlist_node *h;
2235 
2236 	hlist_for_each_entry_safe(f, h, from, hlist) {
2237 		u64 key = i40e_addr_to_hkey(f->macaddr);
2238 
2239 		/* Move the element back into MAC filter list*/
2240 		hlist_del(&f->hlist);
2241 		hash_add(vsi->mac_filter_hash, &f->hlist, key);
2242 	}
2243 }
2244 
2245 /**
2246  * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries
2247  * @vsi: Pointer to vsi struct
2248  * @from: Pointer to list which contains MAC filter entries - changes to
2249  *        those entries needs to be undone.
2250  *
2251  * MAC filter entries from this list were slated for addition.
2252  **/
2253 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
2254 					 struct hlist_head *from)
2255 {
2256 	struct i40e_new_mac_filter *new;
2257 	struct hlist_node *h;
2258 
2259 	hlist_for_each_entry_safe(new, h, from, hlist) {
2260 		/* We can simply free the wrapper structure */
2261 		hlist_del(&new->hlist);
2262 		netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2263 		kfree(new);
2264 	}
2265 }
2266 
2267 /**
2268  * i40e_next_filter - Get the next non-broadcast filter from a list
2269  * @next: pointer to filter in list
2270  *
2271  * Returns the next non-broadcast filter in the list. Required so that we
2272  * ignore broadcast filters within the list, since these are not handled via
2273  * the normal firmware update path.
2274  */
2275 static
2276 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2277 {
2278 	hlist_for_each_entry_continue(next, hlist) {
2279 		if (!is_broadcast_ether_addr(next->f->macaddr))
2280 			return next;
2281 	}
2282 
2283 	return NULL;
2284 }
2285 
2286 /**
2287  * i40e_update_filter_state - Update filter state based on return data
2288  * from firmware
2289  * @count: Number of filters added
2290  * @add_list: return data from fw
2291  * @add_head: pointer to first filter in current batch
2292  *
2293  * MAC filter entries from list were slated to be added to device. Returns
2294  * number of successful filters. Note that 0 does NOT mean success!
2295  **/
2296 static int
2297 i40e_update_filter_state(int count,
2298 			 struct i40e_aqc_add_macvlan_element_data *add_list,
2299 			 struct i40e_new_mac_filter *add_head)
2300 {
2301 	int retval = 0;
2302 	int i;
2303 
2304 	for (i = 0; i < count; i++) {
2305 		/* Always check status of each filter. We don't need to check
2306 		 * the firmware return status because we pre-set the filter
2307 		 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter
2308 		 * request to the adminq. Thus, if it no longer matches then
2309 		 * we know the filter is active.
2310 		 */
2311 		if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2312 			add_head->state = I40E_FILTER_FAILED;
2313 		} else {
2314 			add_head->state = I40E_FILTER_ACTIVE;
2315 			retval++;
2316 		}
2317 
2318 		add_head = i40e_next_filter(add_head);
2319 		if (!add_head)
2320 			break;
2321 	}
2322 
2323 	return retval;
2324 }
2325 
2326 /**
2327  * i40e_aqc_del_filters - Request firmware to delete a set of filters
2328  * @vsi: ptr to the VSI
2329  * @vsi_name: name to display in messages
2330  * @list: the list of filters to send to firmware
2331  * @num_del: the number of filters to delete
2332  * @retval: Set to -EIO on failure to delete
2333  *
2334  * Send a request to firmware via AdminQ to delete a set of filters. Uses
2335  * *retval instead of a return value so that success does not force ret_val to
2336  * be set to 0. This ensures that a sequence of calls to this function
2337  * preserve the previous value of *retval on successful delete.
2338  */
2339 static
2340 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2341 			  struct i40e_aqc_remove_macvlan_element_data *list,
2342 			  int num_del, int *retval)
2343 {
2344 	struct i40e_hw *hw = &vsi->back->hw;
2345 	enum i40e_admin_queue_err aq_status;
2346 	int aq_ret;
2347 
2348 	aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL,
2349 					   &aq_status);
2350 
2351 	/* Explicitly ignore and do not report when firmware returns ENOENT */
2352 	if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) {
2353 		*retval = -EIO;
2354 		dev_info(&vsi->back->pdev->dev,
2355 			 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n",
2356 			 vsi_name, ERR_PTR(aq_ret),
2357 			 i40e_aq_str(hw, aq_status));
2358 	}
2359 }
2360 
2361 /**
2362  * i40e_aqc_add_filters - Request firmware to add a set of filters
2363  * @vsi: ptr to the VSI
2364  * @vsi_name: name to display in messages
2365  * @list: the list of filters to send to firmware
2366  * @add_head: Position in the add hlist
2367  * @num_add: the number of filters to add
2368  *
2369  * Send a request to firmware via AdminQ to add a chunk of filters. Will set
2370  * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of
2371  * space for more filters.
2372  */
2373 static
2374 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2375 			  struct i40e_aqc_add_macvlan_element_data *list,
2376 			  struct i40e_new_mac_filter *add_head,
2377 			  int num_add)
2378 {
2379 	struct i40e_hw *hw = &vsi->back->hw;
2380 	enum i40e_admin_queue_err aq_status;
2381 	int fcnt;
2382 
2383 	i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status);
2384 	fcnt = i40e_update_filter_state(num_add, list, add_head);
2385 
2386 	if (fcnt != num_add) {
2387 		if (vsi->type == I40E_VSI_MAIN) {
2388 			set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2389 			dev_warn(&vsi->back->pdev->dev,
2390 				 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2391 				 i40e_aq_str(hw, aq_status), vsi_name);
2392 		} else if (vsi->type == I40E_VSI_SRIOV ||
2393 			   vsi->type == I40E_VSI_VMDQ1 ||
2394 			   vsi->type == I40E_VSI_VMDQ2) {
2395 			dev_warn(&vsi->back->pdev->dev,
2396 				 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2397 				 i40e_aq_str(hw, aq_status), vsi_name,
2398 					     vsi_name);
2399 		} else {
2400 			dev_warn(&vsi->back->pdev->dev,
2401 				 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2402 				 i40e_aq_str(hw, aq_status), vsi_name,
2403 					     vsi->type);
2404 		}
2405 	}
2406 }
2407 
2408 /**
2409  * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags
2410  * @vsi: pointer to the VSI
2411  * @vsi_name: the VSI name
2412  * @f: filter data
2413  *
2414  * This function sets or clears the promiscuous broadcast flags for VLAN
2415  * filters in order to properly receive broadcast frames. Assumes that only
2416  * broadcast filters are passed.
2417  *
2418  * Returns status indicating success or failure;
2419  **/
2420 static int
2421 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2422 			  struct i40e_mac_filter *f)
2423 {
2424 	bool enable = f->state == I40E_FILTER_NEW;
2425 	struct i40e_hw *hw = &vsi->back->hw;
2426 	int aq_ret;
2427 
2428 	if (f->vlan == I40E_VLAN_ANY) {
2429 		aq_ret = i40e_aq_set_vsi_broadcast(hw,
2430 						   vsi->seid,
2431 						   enable,
2432 						   NULL);
2433 	} else {
2434 		aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2435 							    vsi->seid,
2436 							    enable,
2437 							    f->vlan,
2438 							    NULL);
2439 	}
2440 
2441 	if (aq_ret) {
2442 		set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2443 		dev_warn(&vsi->back->pdev->dev,
2444 			 "Error %s, forcing overflow promiscuous on %s\n",
2445 			 i40e_aq_str(hw, hw->aq.asq_last_status),
2446 			 vsi_name);
2447 	}
2448 
2449 	return aq_ret;
2450 }
2451 
2452 /**
2453  * i40e_set_promiscuous - set promiscuous mode
2454  * @pf: board private structure
2455  * @promisc: promisc on or off
2456  *
2457  * There are different ways of setting promiscuous mode on a PF depending on
2458  * what state/environment we're in.  This identifies and sets it appropriately.
2459  * Returns 0 on success.
2460  **/
2461 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2462 {
2463 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2464 	struct i40e_hw *hw = &pf->hw;
2465 	int aq_ret;
2466 
2467 	if (vsi->type == I40E_VSI_MAIN &&
2468 	    pf->lan_veb != I40E_NO_VEB &&
2469 	    !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2470 		/* set defport ON for Main VSI instead of true promisc
2471 		 * this way we will get all unicast/multicast and VLAN
2472 		 * promisc behavior but will not get VF or VMDq traffic
2473 		 * replicated on the Main VSI.
2474 		 */
2475 		if (promisc)
2476 			aq_ret = i40e_aq_set_default_vsi(hw,
2477 							 vsi->seid,
2478 							 NULL);
2479 		else
2480 			aq_ret = i40e_aq_clear_default_vsi(hw,
2481 							   vsi->seid,
2482 							   NULL);
2483 		if (aq_ret) {
2484 			dev_info(&pf->pdev->dev,
2485 				 "Set default VSI failed, err %pe, aq_err %s\n",
2486 				 ERR_PTR(aq_ret),
2487 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2488 		}
2489 	} else {
2490 		aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2491 						  hw,
2492 						  vsi->seid,
2493 						  promisc, NULL,
2494 						  true);
2495 		if (aq_ret) {
2496 			dev_info(&pf->pdev->dev,
2497 				 "set unicast promisc failed, err %pe, aq_err %s\n",
2498 				 ERR_PTR(aq_ret),
2499 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2500 		}
2501 		aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2502 						  hw,
2503 						  vsi->seid,
2504 						  promisc, NULL);
2505 		if (aq_ret) {
2506 			dev_info(&pf->pdev->dev,
2507 				 "set multicast promisc failed, err %pe, aq_err %s\n",
2508 				 ERR_PTR(aq_ret),
2509 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2510 		}
2511 	}
2512 
2513 	if (!aq_ret)
2514 		pf->cur_promisc = promisc;
2515 
2516 	return aq_ret;
2517 }
2518 
2519 /**
2520  * i40e_sync_vsi_filters - Update the VSI filter list to the HW
2521  * @vsi: ptr to the VSI
2522  *
2523  * Push any outstanding VSI filter changes through the AdminQ.
2524  *
2525  * Returns 0 or error value
2526  **/
2527 int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2528 {
2529 	struct hlist_head tmp_add_list, tmp_del_list;
2530 	struct i40e_mac_filter *f;
2531 	struct i40e_new_mac_filter *new, *add_head = NULL;
2532 	struct i40e_hw *hw = &vsi->back->hw;
2533 	bool old_overflow, new_overflow;
2534 	unsigned int failed_filters = 0;
2535 	unsigned int vlan_filters = 0;
2536 	char vsi_name[16] = "PF";
2537 	int filter_list_len = 0;
2538 	u32 changed_flags = 0;
2539 	struct hlist_node *h;
2540 	struct i40e_pf *pf;
2541 	int num_add = 0;
2542 	int num_del = 0;
2543 	int aq_ret = 0;
2544 	int retval = 0;
2545 	u16 cmd_flags;
2546 	int list_size;
2547 	int bkt;
2548 
2549 	/* empty array typed pointers, kcalloc later */
2550 	struct i40e_aqc_add_macvlan_element_data *add_list;
2551 	struct i40e_aqc_remove_macvlan_element_data *del_list;
2552 
2553 	while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2554 		usleep_range(1000, 2000);
2555 	pf = vsi->back;
2556 
2557 	old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2558 
2559 	if (vsi->netdev) {
2560 		changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2561 		vsi->current_netdev_flags = vsi->netdev->flags;
2562 	}
2563 
2564 	INIT_HLIST_HEAD(&tmp_add_list);
2565 	INIT_HLIST_HEAD(&tmp_del_list);
2566 
2567 	if (vsi->type == I40E_VSI_SRIOV)
2568 		snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2569 	else if (vsi->type != I40E_VSI_MAIN)
2570 		snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2571 
2572 	if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2573 		vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2574 
2575 		spin_lock_bh(&vsi->mac_filter_hash_lock);
2576 		/* Create a list of filters to delete. */
2577 		hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2578 			if (f->state == I40E_FILTER_REMOVE) {
2579 				/* Move the element into temporary del_list */
2580 				hash_del(&f->hlist);
2581 				hlist_add_head(&f->hlist, &tmp_del_list);
2582 
2583 				/* Avoid counting removed filters */
2584 				continue;
2585 			}
2586 			if (f->state == I40E_FILTER_NEW) {
2587 				/* Create a temporary i40e_new_mac_filter */
2588 				new = kzalloc(sizeof(*new), GFP_ATOMIC);
2589 				if (!new)
2590 					goto err_no_memory_locked;
2591 
2592 				/* Store pointer to the real filter */
2593 				new->f = f;
2594 				new->state = f->state;
2595 
2596 				/* Add it to the hash list */
2597 				hlist_add_head(&new->hlist, &tmp_add_list);
2598 			}
2599 
2600 			/* Count the number of active (current and new) VLAN
2601 			 * filters we have now. Does not count filters which
2602 			 * are marked for deletion.
2603 			 */
2604 			if (f->vlan > 0)
2605 				vlan_filters++;
2606 		}
2607 
2608 		if (vsi->type != I40E_VSI_SRIOV)
2609 			retval = i40e_correct_mac_vlan_filters
2610 				(vsi, &tmp_add_list, &tmp_del_list,
2611 				 vlan_filters);
2612 		else
2613 			retval = i40e_correct_vf_mac_vlan_filters
2614 				(vsi, &tmp_add_list, &tmp_del_list,
2615 				 vlan_filters, pf->vf[vsi->vf_id].trusted);
2616 
2617 		hlist_for_each_entry(new, &tmp_add_list, hlist)
2618 			netdev_hw_addr_refcnt(new->f, vsi->netdev, 1);
2619 
2620 		if (retval)
2621 			goto err_no_memory_locked;
2622 
2623 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
2624 	}
2625 
2626 	/* Now process 'del_list' outside the lock */
2627 	if (!hlist_empty(&tmp_del_list)) {
2628 		filter_list_len = hw->aq.asq_buf_size /
2629 			    sizeof(struct i40e_aqc_remove_macvlan_element_data);
2630 		list_size = filter_list_len *
2631 			    sizeof(struct i40e_aqc_remove_macvlan_element_data);
2632 		del_list = kzalloc(list_size, GFP_ATOMIC);
2633 		if (!del_list)
2634 			goto err_no_memory;
2635 
2636 		hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2637 			cmd_flags = 0;
2638 
2639 			/* handle broadcast filters by updating the broadcast
2640 			 * promiscuous flag and release filter list.
2641 			 */
2642 			if (is_broadcast_ether_addr(f->macaddr)) {
2643 				i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2644 
2645 				hlist_del(&f->hlist);
2646 				kfree(f);
2647 				continue;
2648 			}
2649 
2650 			/* add to delete list */
2651 			ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2652 			if (f->vlan == I40E_VLAN_ANY) {
2653 				del_list[num_del].vlan_tag = 0;
2654 				cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2655 			} else {
2656 				del_list[num_del].vlan_tag =
2657 					cpu_to_le16((u16)(f->vlan));
2658 			}
2659 
2660 			cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2661 			del_list[num_del].flags = cmd_flags;
2662 			num_del++;
2663 
2664 			/* flush a full buffer */
2665 			if (num_del == filter_list_len) {
2666 				i40e_aqc_del_filters(vsi, vsi_name, del_list,
2667 						     num_del, &retval);
2668 				memset(del_list, 0, list_size);
2669 				num_del = 0;
2670 			}
2671 			/* Release memory for MAC filter entries which were
2672 			 * synced up with HW.
2673 			 */
2674 			hlist_del(&f->hlist);
2675 			kfree(f);
2676 		}
2677 
2678 		if (num_del) {
2679 			i40e_aqc_del_filters(vsi, vsi_name, del_list,
2680 					     num_del, &retval);
2681 		}
2682 
2683 		kfree(del_list);
2684 		del_list = NULL;
2685 	}
2686 
2687 	if (!hlist_empty(&tmp_add_list)) {
2688 		/* Do all the adds now. */
2689 		filter_list_len = hw->aq.asq_buf_size /
2690 			       sizeof(struct i40e_aqc_add_macvlan_element_data);
2691 		list_size = filter_list_len *
2692 			       sizeof(struct i40e_aqc_add_macvlan_element_data);
2693 		add_list = kzalloc(list_size, GFP_ATOMIC);
2694 		if (!add_list)
2695 			goto err_no_memory;
2696 
2697 		num_add = 0;
2698 		hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2699 			/* handle broadcast filters by updating the broadcast
2700 			 * promiscuous flag instead of adding a MAC filter.
2701 			 */
2702 			if (is_broadcast_ether_addr(new->f->macaddr)) {
2703 				if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2704 							      new->f))
2705 					new->state = I40E_FILTER_FAILED;
2706 				else
2707 					new->state = I40E_FILTER_ACTIVE;
2708 				continue;
2709 			}
2710 
2711 			/* add to add array */
2712 			if (num_add == 0)
2713 				add_head = new;
2714 			cmd_flags = 0;
2715 			ether_addr_copy(add_list[num_add].mac_addr,
2716 					new->f->macaddr);
2717 			if (new->f->vlan == I40E_VLAN_ANY) {
2718 				add_list[num_add].vlan_tag = 0;
2719 				cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2720 			} else {
2721 				add_list[num_add].vlan_tag =
2722 					cpu_to_le16((u16)(new->f->vlan));
2723 			}
2724 			add_list[num_add].queue_number = 0;
2725 			/* set invalid match method for later detection */
2726 			add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2727 			cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2728 			add_list[num_add].flags = cpu_to_le16(cmd_flags);
2729 			num_add++;
2730 
2731 			/* flush a full buffer */
2732 			if (num_add == filter_list_len) {
2733 				i40e_aqc_add_filters(vsi, vsi_name, add_list,
2734 						     add_head, num_add);
2735 				memset(add_list, 0, list_size);
2736 				num_add = 0;
2737 			}
2738 		}
2739 		if (num_add) {
2740 			i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2741 					     num_add);
2742 		}
2743 		/* Now move all of the filters from the temp add list back to
2744 		 * the VSI's list.
2745 		 */
2746 		spin_lock_bh(&vsi->mac_filter_hash_lock);
2747 		hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2748 			/* Only update the state if we're still NEW */
2749 			if (new->f->state == I40E_FILTER_NEW)
2750 				new->f->state = new->state;
2751 			hlist_del(&new->hlist);
2752 			netdev_hw_addr_refcnt(new->f, vsi->netdev, -1);
2753 			kfree(new);
2754 		}
2755 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
2756 		kfree(add_list);
2757 		add_list = NULL;
2758 	}
2759 
2760 	/* Determine the number of active and failed filters. */
2761 	spin_lock_bh(&vsi->mac_filter_hash_lock);
2762 	vsi->active_filters = 0;
2763 	hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2764 		if (f->state == I40E_FILTER_ACTIVE)
2765 			vsi->active_filters++;
2766 		else if (f->state == I40E_FILTER_FAILED)
2767 			failed_filters++;
2768 	}
2769 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
2770 
2771 	/* Check if we are able to exit overflow promiscuous mode. We can
2772 	 * safely exit if we didn't just enter, we no longer have any failed
2773 	 * filters, and we have reduced filters below the threshold value.
2774 	 */
2775 	if (old_overflow && !failed_filters &&
2776 	    vsi->active_filters < vsi->promisc_threshold) {
2777 		dev_info(&pf->pdev->dev,
2778 			 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2779 			 vsi_name);
2780 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2781 		vsi->promisc_threshold = 0;
2782 	}
2783 
2784 	/* if the VF is not trusted do not do promisc */
2785 	if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2786 		clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2787 		goto out;
2788 	}
2789 
2790 	new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2791 
2792 	/* If we are entering overflow promiscuous, we need to calculate a new
2793 	 * threshold for when we are safe to exit
2794 	 */
2795 	if (!old_overflow && new_overflow)
2796 		vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2797 
2798 	/* check for changes in promiscuous modes */
2799 	if (changed_flags & IFF_ALLMULTI) {
2800 		bool cur_multipromisc;
2801 
2802 		cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2803 		aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2804 							       vsi->seid,
2805 							       cur_multipromisc,
2806 							       NULL);
2807 		if (aq_ret) {
2808 			retval = i40e_aq_rc_to_posix(aq_ret,
2809 						     hw->aq.asq_last_status);
2810 			dev_info(&pf->pdev->dev,
2811 				 "set multi promisc failed on %s, err %pe aq_err %s\n",
2812 				 vsi_name,
2813 				 ERR_PTR(aq_ret),
2814 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2815 		} else {
2816 			dev_info(&pf->pdev->dev, "%s allmulti mode.\n",
2817 				 cur_multipromisc ? "entering" : "leaving");
2818 		}
2819 	}
2820 
2821 	if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2822 		bool cur_promisc;
2823 
2824 		cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2825 			       new_overflow);
2826 		aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2827 		if (aq_ret) {
2828 			retval = i40e_aq_rc_to_posix(aq_ret,
2829 						     hw->aq.asq_last_status);
2830 			dev_info(&pf->pdev->dev,
2831 				 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n",
2832 				 cur_promisc ? "on" : "off",
2833 				 vsi_name,
2834 				 ERR_PTR(aq_ret),
2835 				 i40e_aq_str(hw, hw->aq.asq_last_status));
2836 		}
2837 	}
2838 out:
2839 	/* if something went wrong then set the changed flag so we try again */
2840 	if (retval)
2841 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2842 
2843 	clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2844 	return retval;
2845 
2846 err_no_memory:
2847 	/* Restore elements on the temporary add and delete lists */
2848 	spin_lock_bh(&vsi->mac_filter_hash_lock);
2849 err_no_memory_locked:
2850 	i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2851 	i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2852 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
2853 
2854 	vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2855 	clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2856 	return -ENOMEM;
2857 }
2858 
2859 /**
2860  * i40e_sync_filters_subtask - Sync the VSI filter list with HW
2861  * @pf: board private structure
2862  **/
2863 static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2864 {
2865 	int v;
2866 
2867 	if (!pf)
2868 		return;
2869 	if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2870 		return;
2871 	if (test_bit(__I40E_VF_DISABLE, pf->state)) {
2872 		set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
2873 		return;
2874 	}
2875 
2876 	for (v = 0; v < pf->num_alloc_vsi; v++) {
2877 		if (pf->vsi[v] &&
2878 		    (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED) &&
2879 		    !test_bit(__I40E_VSI_RELEASING, pf->vsi[v]->state)) {
2880 			int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2881 
2882 			if (ret) {
2883 				/* come back and try again later */
2884 				set_bit(__I40E_MACVLAN_SYNC_PENDING,
2885 					pf->state);
2886 				break;
2887 			}
2888 		}
2889 	}
2890 }
2891 
2892 /**
2893  * i40e_calculate_vsi_rx_buf_len - Calculates buffer length
2894  *
2895  * @vsi: VSI to calculate rx_buf_len from
2896  */
2897 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi)
2898 {
2899 	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2900 		return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048);
2901 
2902 	return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048;
2903 }
2904 
2905 /**
2906  * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI
2907  * @vsi: the vsi
2908  * @xdp_prog: XDP program
2909  **/
2910 static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi,
2911 				   struct bpf_prog *xdp_prog)
2912 {
2913 	u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
2914 	u16 chain_len;
2915 
2916 	if (xdp_prog && !xdp_prog->aux->xdp_has_frags)
2917 		chain_len = 1;
2918 	else
2919 		chain_len = I40E_MAX_CHAINED_RX_BUFFERS;
2920 
2921 	return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER);
2922 }
2923 
2924 /**
2925  * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit
2926  * @netdev: network interface device structure
2927  * @new_mtu: new value for maximum frame size
2928  *
2929  * Returns 0 on success, negative on failure
2930  **/
2931 static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2932 {
2933 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2934 	struct i40e_vsi *vsi = np->vsi;
2935 	struct i40e_pf *pf = vsi->back;
2936 	int frame_size;
2937 
2938 	frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
2939 	if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) {
2940 		netdev_err(netdev, "Error changing mtu to %d, Max is %d\n",
2941 			   new_mtu, frame_size - I40E_PACKET_HDR_PAD);
2942 		return -EINVAL;
2943 	}
2944 
2945 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
2946 		   netdev->mtu, new_mtu);
2947 	netdev->mtu = new_mtu;
2948 	if (netif_running(netdev))
2949 		i40e_vsi_reinit_locked(vsi);
2950 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2951 	set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2952 	return 0;
2953 }
2954 
2955 /**
2956  * i40e_ioctl - Access the hwtstamp interface
2957  * @netdev: network interface device structure
2958  * @ifr: interface request data
2959  * @cmd: ioctl command
2960  **/
2961 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2962 {
2963 	struct i40e_netdev_priv *np = netdev_priv(netdev);
2964 	struct i40e_pf *pf = np->vsi->back;
2965 
2966 	switch (cmd) {
2967 	case SIOCGHWTSTAMP:
2968 		return i40e_ptp_get_ts_config(pf, ifr);
2969 	case SIOCSHWTSTAMP:
2970 		return i40e_ptp_set_ts_config(pf, ifr);
2971 	default:
2972 		return -EOPNOTSUPP;
2973 	}
2974 }
2975 
2976 /**
2977  * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI
2978  * @vsi: the vsi being adjusted
2979  **/
2980 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2981 {
2982 	struct i40e_vsi_context ctxt;
2983 	int ret;
2984 
2985 	/* Don't modify stripping options if a port VLAN is active */
2986 	if (vsi->info.pvid)
2987 		return;
2988 
2989 	if ((vsi->info.valid_sections &
2990 	     cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2991 	    ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2992 		return;  /* already enabled */
2993 
2994 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2995 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2996 				    I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2997 
2998 	ctxt.seid = vsi->seid;
2999 	ctxt.info = vsi->info;
3000 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3001 	if (ret) {
3002 		dev_info(&vsi->back->pdev->dev,
3003 			 "update vlan stripping failed, err %pe aq_err %s\n",
3004 			 ERR_PTR(ret),
3005 			 i40e_aq_str(&vsi->back->hw,
3006 				     vsi->back->hw.aq.asq_last_status));
3007 	}
3008 }
3009 
3010 /**
3011  * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI
3012  * @vsi: the vsi being adjusted
3013  **/
3014 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
3015 {
3016 	struct i40e_vsi_context ctxt;
3017 	int ret;
3018 
3019 	/* Don't modify stripping options if a port VLAN is active */
3020 	if (vsi->info.pvid)
3021 		return;
3022 
3023 	if ((vsi->info.valid_sections &
3024 	     cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
3025 	    ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
3026 	     I40E_AQ_VSI_PVLAN_EMOD_MASK))
3027 		return;  /* already disabled */
3028 
3029 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3030 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
3031 				    I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
3032 
3033 	ctxt.seid = vsi->seid;
3034 	ctxt.info = vsi->info;
3035 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3036 	if (ret) {
3037 		dev_info(&vsi->back->pdev->dev,
3038 			 "update vlan stripping failed, err %pe aq_err %s\n",
3039 			 ERR_PTR(ret),
3040 			 i40e_aq_str(&vsi->back->hw,
3041 				     vsi->back->hw.aq.asq_last_status));
3042 	}
3043 }
3044 
3045 /**
3046  * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address
3047  * @vsi: the vsi being configured
3048  * @vid: vlan id to be added (0 = untagged only , -1 = any)
3049  *
3050  * This is a helper function for adding a new MAC/VLAN filter with the
3051  * specified VLAN for each existing MAC address already in the hash table.
3052  * This function does *not* perform any accounting to update filters based on
3053  * VLAN mode.
3054  *
3055  * NOTE: this function expects to be called while under the
3056  * mac_filter_hash_lock
3057  **/
3058 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3059 {
3060 	struct i40e_mac_filter *f, *add_f;
3061 	struct hlist_node *h;
3062 	int bkt;
3063 
3064 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3065 		/* If we're asked to add a filter that has been marked for
3066 		 * removal, it is safe to simply restore it to active state.
3067 		 * __i40e_del_filter will have simply deleted any filters which
3068 		 * were previously marked NEW or FAILED, so if it is currently
3069 		 * marked REMOVE it must have previously been ACTIVE. Since we
3070 		 * haven't yet run the sync filters task, just restore this
3071 		 * filter to the ACTIVE state so that the sync task leaves it
3072 		 * in place.
3073 		 */
3074 		if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) {
3075 			f->state = I40E_FILTER_ACTIVE;
3076 			continue;
3077 		} else if (f->state == I40E_FILTER_REMOVE) {
3078 			continue;
3079 		}
3080 		add_f = i40e_add_filter(vsi, f->macaddr, vid);
3081 		if (!add_f) {
3082 			dev_info(&vsi->back->pdev->dev,
3083 				 "Could not add vlan filter %d for %pM\n",
3084 				 vid, f->macaddr);
3085 			return -ENOMEM;
3086 		}
3087 	}
3088 
3089 	return 0;
3090 }
3091 
3092 /**
3093  * i40e_vsi_add_vlan - Add VSI membership for given VLAN
3094  * @vsi: the VSI being configured
3095  * @vid: VLAN id to be added
3096  **/
3097 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
3098 {
3099 	int err;
3100 
3101 	if (vsi->info.pvid)
3102 		return -EINVAL;
3103 
3104 	/* The network stack will attempt to add VID=0, with the intention to
3105 	 * receive priority tagged packets with a VLAN of 0. Our HW receives
3106 	 * these packets by default when configured to receive untagged
3107 	 * packets, so we don't need to add a filter for this case.
3108 	 * Additionally, HW interprets adding a VID=0 filter as meaning to
3109 	 * receive *only* tagged traffic and stops receiving untagged traffic.
3110 	 * Thus, we do not want to actually add a filter for VID=0
3111 	 */
3112 	if (!vid)
3113 		return 0;
3114 
3115 	/* Locked once because all functions invoked below iterates list*/
3116 	spin_lock_bh(&vsi->mac_filter_hash_lock);
3117 	err = i40e_add_vlan_all_mac(vsi, vid);
3118 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
3119 	if (err)
3120 		return err;
3121 
3122 	/* schedule our worker thread which will take care of
3123 	 * applying the new filter changes
3124 	 */
3125 	i40e_service_event_schedule(vsi->back);
3126 	return 0;
3127 }
3128 
3129 /**
3130  * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN
3131  * @vsi: the vsi being configured
3132  * @vid: vlan id to be removed (0 = untagged only , -1 = any)
3133  *
3134  * This function should be used to remove all VLAN filters which match the
3135  * given VID. It does not schedule the service event and does not take the
3136  * mac_filter_hash_lock so it may be combined with other operations under
3137  * a single invocation of the mac_filter_hash_lock.
3138  *
3139  * NOTE: this function expects to be called while under the
3140  * mac_filter_hash_lock
3141  */
3142 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
3143 {
3144 	struct i40e_mac_filter *f;
3145 	struct hlist_node *h;
3146 	int bkt;
3147 
3148 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
3149 		if (f->vlan == vid)
3150 			__i40e_del_filter(vsi, f);
3151 	}
3152 }
3153 
3154 /**
3155  * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN
3156  * @vsi: the VSI being configured
3157  * @vid: VLAN id to be removed
3158  **/
3159 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
3160 {
3161 	if (!vid || vsi->info.pvid)
3162 		return;
3163 
3164 	spin_lock_bh(&vsi->mac_filter_hash_lock);
3165 	i40e_rm_vlan_all_mac(vsi, vid);
3166 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
3167 
3168 	/* schedule our worker thread which will take care of
3169 	 * applying the new filter changes
3170 	 */
3171 	i40e_service_event_schedule(vsi->back);
3172 }
3173 
3174 /**
3175  * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload
3176  * @netdev: network interface to be adjusted
3177  * @proto: unused protocol value
3178  * @vid: vlan id to be added
3179  *
3180  * net_device_ops implementation for adding vlan ids
3181  **/
3182 static int i40e_vlan_rx_add_vid(struct net_device *netdev,
3183 				__always_unused __be16 proto, u16 vid)
3184 {
3185 	struct i40e_netdev_priv *np = netdev_priv(netdev);
3186 	struct i40e_vsi *vsi = np->vsi;
3187 	int ret = 0;
3188 
3189 	if (vid >= VLAN_N_VID)
3190 		return -EINVAL;
3191 
3192 	ret = i40e_vsi_add_vlan(vsi, vid);
3193 	if (!ret)
3194 		set_bit(vid, vsi->active_vlans);
3195 
3196 	return ret;
3197 }
3198 
3199 /**
3200  * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path
3201  * @netdev: network interface to be adjusted
3202  * @proto: unused protocol value
3203  * @vid: vlan id to be added
3204  **/
3205 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
3206 				    __always_unused __be16 proto, u16 vid)
3207 {
3208 	struct i40e_netdev_priv *np = netdev_priv(netdev);
3209 	struct i40e_vsi *vsi = np->vsi;
3210 
3211 	if (vid >= VLAN_N_VID)
3212 		return;
3213 	set_bit(vid, vsi->active_vlans);
3214 }
3215 
3216 /**
3217  * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
3218  * @netdev: network interface to be adjusted
3219  * @proto: unused protocol value
3220  * @vid: vlan id to be removed
3221  *
3222  * net_device_ops implementation for removing vlan ids
3223  **/
3224 static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
3225 				 __always_unused __be16 proto, u16 vid)
3226 {
3227 	struct i40e_netdev_priv *np = netdev_priv(netdev);
3228 	struct i40e_vsi *vsi = np->vsi;
3229 
3230 	/* return code is ignored as there is nothing a user
3231 	 * can do about failure to remove and a log message was
3232 	 * already printed from the other function
3233 	 */
3234 	i40e_vsi_kill_vlan(vsi, vid);
3235 
3236 	clear_bit(vid, vsi->active_vlans);
3237 
3238 	return 0;
3239 }
3240 
3241 /**
3242  * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up
3243  * @vsi: the vsi being brought back up
3244  **/
3245 static void i40e_restore_vlan(struct i40e_vsi *vsi)
3246 {
3247 	u16 vid;
3248 
3249 	if (!vsi->netdev)
3250 		return;
3251 
3252 	if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3253 		i40e_vlan_stripping_enable(vsi);
3254 	else
3255 		i40e_vlan_stripping_disable(vsi);
3256 
3257 	for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
3258 		i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
3259 					vid);
3260 }
3261 
3262 /**
3263  * i40e_vsi_add_pvid - Add pvid for the VSI
3264  * @vsi: the vsi being adjusted
3265  * @vid: the vlan id to set as a PVID
3266  **/
3267 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
3268 {
3269 	struct i40e_vsi_context ctxt;
3270 	int ret;
3271 
3272 	vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
3273 	vsi->info.pvid = cpu_to_le16(vid);
3274 	vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
3275 				    I40E_AQ_VSI_PVLAN_INSERT_PVID |
3276 				    I40E_AQ_VSI_PVLAN_EMOD_STR;
3277 
3278 	ctxt.seid = vsi->seid;
3279 	ctxt.info = vsi->info;
3280 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
3281 	if (ret) {
3282 		dev_info(&vsi->back->pdev->dev,
3283 			 "add pvid failed, err %pe aq_err %s\n",
3284 			 ERR_PTR(ret),
3285 			 i40e_aq_str(&vsi->back->hw,
3286 				     vsi->back->hw.aq.asq_last_status));
3287 		return -ENOENT;
3288 	}
3289 
3290 	return 0;
3291 }
3292 
3293 /**
3294  * i40e_vsi_remove_pvid - Remove the pvid from the VSI
3295  * @vsi: the vsi being adjusted
3296  *
3297  * Just use the vlan_rx_register() service to put it back to normal
3298  **/
3299 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
3300 {
3301 	vsi->info.pvid = 0;
3302 
3303 	i40e_vlan_stripping_disable(vsi);
3304 }
3305 
3306 /**
3307  * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources
3308  * @vsi: ptr to the VSI
3309  *
3310  * If this function returns with an error, then it's possible one or
3311  * more of the rings is populated (while the rest are not).  It is the
3312  * callers duty to clean those orphaned rings.
3313  *
3314  * Return 0 on success, negative on failure
3315  **/
3316 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
3317 {
3318 	int i, err = 0;
3319 
3320 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3321 		err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
3322 
3323 	if (!i40e_enabled_xdp_vsi(vsi))
3324 		return err;
3325 
3326 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3327 		err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3328 
3329 	return err;
3330 }
3331 
3332 /**
3333  * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues
3334  * @vsi: ptr to the VSI
3335  *
3336  * Free VSI's transmit software resources
3337  **/
3338 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3339 {
3340 	int i;
3341 
3342 	if (vsi->tx_rings) {
3343 		for (i = 0; i < vsi->num_queue_pairs; i++)
3344 			if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3345 				i40e_free_tx_resources(vsi->tx_rings[i]);
3346 	}
3347 
3348 	if (vsi->xdp_rings) {
3349 		for (i = 0; i < vsi->num_queue_pairs; i++)
3350 			if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3351 				i40e_free_tx_resources(vsi->xdp_rings[i]);
3352 	}
3353 }
3354 
3355 /**
3356  * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources
3357  * @vsi: ptr to the VSI
3358  *
3359  * If this function returns with an error, then it's possible one or
3360  * more of the rings is populated (while the rest are not).  It is the
3361  * callers duty to clean those orphaned rings.
3362  *
3363  * Return 0 on success, negative on failure
3364  **/
3365 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3366 {
3367 	int i, err = 0;
3368 
3369 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3370 		err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3371 	return err;
3372 }
3373 
3374 /**
3375  * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues
3376  * @vsi: ptr to the VSI
3377  *
3378  * Free all receive software resources
3379  **/
3380 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3381 {
3382 	int i;
3383 
3384 	if (!vsi->rx_rings)
3385 		return;
3386 
3387 	for (i = 0; i < vsi->num_queue_pairs; i++)
3388 		if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3389 			i40e_free_rx_resources(vsi->rx_rings[i]);
3390 }
3391 
3392 /**
3393  * i40e_config_xps_tx_ring - Configure XPS for a Tx ring
3394  * @ring: The Tx ring to configure
3395  *
3396  * This enables/disables XPS for a given Tx descriptor ring
3397  * based on the TCs enabled for the VSI that ring belongs to.
3398  **/
3399 static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3400 {
3401 	int cpu;
3402 
3403 	if (!ring->q_vector || !ring->netdev || ring->ch)
3404 		return;
3405 
3406 	/* We only initialize XPS once, so as not to overwrite user settings */
3407 	if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3408 		return;
3409 
3410 	cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3411 	netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3412 			    ring->queue_index);
3413 }
3414 
3415 /**
3416  * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled
3417  * @ring: The Tx or Rx ring
3418  *
3419  * Returns the AF_XDP buffer pool or NULL.
3420  **/
3421 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring)
3422 {
3423 	bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3424 	int qid = ring->queue_index;
3425 
3426 	if (ring_is_xdp(ring))
3427 		qid -= ring->vsi->alloc_queue_pairs;
3428 
3429 	if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3430 		return NULL;
3431 
3432 	return xsk_get_pool_from_qid(ring->vsi->netdev, qid);
3433 }
3434 
3435 /**
3436  * i40e_configure_tx_ring - Configure a transmit ring context and rest
3437  * @ring: The Tx ring to configure
3438  *
3439  * Configure the Tx descriptor ring in the HMC context.
3440  **/
3441 static int i40e_configure_tx_ring(struct i40e_ring *ring)
3442 {
3443 	struct i40e_vsi *vsi = ring->vsi;
3444 	u16 pf_q = vsi->base_queue + ring->queue_index;
3445 	struct i40e_hw *hw = &vsi->back->hw;
3446 	struct i40e_hmc_obj_txq tx_ctx;
3447 	u32 qtx_ctl = 0;
3448 	int err = 0;
3449 
3450 	if (ring_is_xdp(ring))
3451 		ring->xsk_pool = i40e_xsk_pool(ring);
3452 
3453 	/* some ATR related tx ring init */
3454 	if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3455 		ring->atr_sample_rate = vsi->back->atr_sample_rate;
3456 		ring->atr_count = 0;
3457 	} else {
3458 		ring->atr_sample_rate = 0;
3459 	}
3460 
3461 	/* configure XPS */
3462 	i40e_config_xps_tx_ring(ring);
3463 
3464 	/* clear the context structure first */
3465 	memset(&tx_ctx, 0, sizeof(tx_ctx));
3466 
3467 	tx_ctx.new_context = 1;
3468 	tx_ctx.base = (ring->dma / 128);
3469 	tx_ctx.qlen = ring->count;
3470 	tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3471 					       I40E_FLAG_FD_ATR_ENABLED));
3472 	tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3473 	/* FDIR VSI tx ring can still use RS bit and writebacks */
3474 	if (vsi->type != I40E_VSI_FDIR)
3475 		tx_ctx.head_wb_ena = 1;
3476 	tx_ctx.head_wb_addr = ring->dma +
3477 			      (ring->count * sizeof(struct i40e_tx_desc));
3478 
3479 	/* As part of VSI creation/update, FW allocates certain
3480 	 * Tx arbitration queue sets for each TC enabled for
3481 	 * the VSI. The FW returns the handles to these queue
3482 	 * sets as part of the response buffer to Add VSI,
3483 	 * Update VSI, etc. AQ commands. It is expected that
3484 	 * these queue set handles be associated with the Tx
3485 	 * queues by the driver as part of the TX queue context
3486 	 * initialization. This has to be done regardless of
3487 	 * DCB as by default everything is mapped to TC0.
3488 	 */
3489 
3490 	if (ring->ch)
3491 		tx_ctx.rdylist =
3492 			le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3493 
3494 	else
3495 		tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3496 
3497 	tx_ctx.rdylist_act = 0;
3498 
3499 	/* clear the context in the HMC */
3500 	err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3501 	if (err) {
3502 		dev_info(&vsi->back->pdev->dev,
3503 			 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3504 			 ring->queue_index, pf_q, err);
3505 		return -ENOMEM;
3506 	}
3507 
3508 	/* set the context in the HMC */
3509 	err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3510 	if (err) {
3511 		dev_info(&vsi->back->pdev->dev,
3512 			 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3513 			 ring->queue_index, pf_q, err);
3514 		return -ENOMEM;
3515 	}
3516 
3517 	/* Now associate this queue with this PCI function */
3518 	if (ring->ch) {
3519 		if (ring->ch->type == I40E_VSI_VMDQ2)
3520 			qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3521 		else
3522 			return -EINVAL;
3523 
3524 		qtx_ctl |= (ring->ch->vsi_number <<
3525 			    I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3526 			    I40E_QTX_CTL_VFVM_INDX_MASK;
3527 	} else {
3528 		if (vsi->type == I40E_VSI_VMDQ2) {
3529 			qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3530 			qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3531 				    I40E_QTX_CTL_VFVM_INDX_MASK;
3532 		} else {
3533 			qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3534 		}
3535 	}
3536 
3537 	qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3538 		    I40E_QTX_CTL_PF_INDX_MASK);
3539 	wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3540 	i40e_flush(hw);
3541 
3542 	/* cache tail off for easier writes later */
3543 	ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3544 
3545 	return 0;
3546 }
3547 
3548 /**
3549  * i40e_rx_offset - Return expected offset into page to access data
3550  * @rx_ring: Ring we are requesting offset of
3551  *
3552  * Returns the offset value for ring into the data buffer.
3553  */
3554 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
3555 {
3556 	return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
3557 }
3558 
3559 /**
3560  * i40e_configure_rx_ring - Configure a receive ring context
3561  * @ring: The Rx ring to configure
3562  *
3563  * Configure the Rx descriptor ring in the HMC context.
3564  **/
3565 static int i40e_configure_rx_ring(struct i40e_ring *ring)
3566 {
3567 	struct i40e_vsi *vsi = ring->vsi;
3568 	u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3569 	u16 pf_q = vsi->base_queue + ring->queue_index;
3570 	struct i40e_hw *hw = &vsi->back->hw;
3571 	struct i40e_hmc_obj_rxq rx_ctx;
3572 	int err = 0;
3573 	bool ok;
3574 	int ret;
3575 
3576 	bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3577 
3578 	/* clear the context structure first */
3579 	memset(&rx_ctx, 0, sizeof(rx_ctx));
3580 
3581 	if (ring->vsi->type == I40E_VSI_MAIN)
3582 		xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3583 
3584 	ring->xsk_pool = i40e_xsk_pool(ring);
3585 	if (ring->xsk_pool) {
3586 		ring->rx_buf_len =
3587 		  xsk_pool_get_rx_frame_size(ring->xsk_pool);
3588 		ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3589 						 MEM_TYPE_XSK_BUFF_POOL,
3590 						 NULL);
3591 		if (ret)
3592 			return ret;
3593 		dev_info(&vsi->back->pdev->dev,
3594 			 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n",
3595 			 ring->queue_index);
3596 
3597 	} else {
3598 		ring->rx_buf_len = vsi->rx_buf_len;
3599 		if (ring->vsi->type == I40E_VSI_MAIN) {
3600 			ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3601 							 MEM_TYPE_PAGE_SHARED,
3602 							 NULL);
3603 			if (ret)
3604 				return ret;
3605 		}
3606 	}
3607 
3608 	xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq);
3609 
3610 	rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3611 				    BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3612 
3613 	rx_ctx.base = (ring->dma / 128);
3614 	rx_ctx.qlen = ring->count;
3615 
3616 	/* use 16 byte descriptors */
3617 	rx_ctx.dsize = 0;
3618 
3619 	/* descriptor type is always zero
3620 	 * rx_ctx.dtype = 0;
3621 	 */
3622 	rx_ctx.hsplit_0 = 0;
3623 
3624 	rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3625 	if (hw->revision_id == 0)
3626 		rx_ctx.lrxqthresh = 0;
3627 	else
3628 		rx_ctx.lrxqthresh = 1;
3629 	rx_ctx.crcstrip = 1;
3630 	rx_ctx.l2tsel = 1;
3631 	/* this controls whether VLAN is stripped from inner headers */
3632 	rx_ctx.showiv = 0;
3633 	/* set the prefena field to 1 because the manual says to */
3634 	rx_ctx.prefena = 1;
3635 
3636 	/* clear the context in the HMC */
3637 	err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3638 	if (err) {
3639 		dev_info(&vsi->back->pdev->dev,
3640 			 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3641 			 ring->queue_index, pf_q, err);
3642 		return -ENOMEM;
3643 	}
3644 
3645 	/* set the context in the HMC */
3646 	err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3647 	if (err) {
3648 		dev_info(&vsi->back->pdev->dev,
3649 			 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3650 			 ring->queue_index, pf_q, err);
3651 		return -ENOMEM;
3652 	}
3653 
3654 	/* configure Rx buffer alignment */
3655 	if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3656 		if (I40E_2K_TOO_SMALL_WITH_PADDING) {
3657 			dev_info(&vsi->back->pdev->dev,
3658 				 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n");
3659 			return -EOPNOTSUPP;
3660 		}
3661 		clear_ring_build_skb_enabled(ring);
3662 	} else {
3663 		set_ring_build_skb_enabled(ring);
3664 	}
3665 
3666 	ring->rx_offset = i40e_rx_offset(ring);
3667 
3668 	/* cache tail for quicker writes, and clear the reg before use */
3669 	ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3670 	writel(0, ring->tail);
3671 
3672 	if (ring->xsk_pool) {
3673 		xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
3674 		ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring));
3675 	} else {
3676 		ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3677 	}
3678 	if (!ok) {
3679 		/* Log this in case the user has forgotten to give the kernel
3680 		 * any buffers, even later in the application.
3681 		 */
3682 		dev_info(&vsi->back->pdev->dev,
3683 			 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3684 			 ring->xsk_pool ? "AF_XDP ZC enabled " : "",
3685 			 ring->queue_index, pf_q);
3686 	}
3687 
3688 	return 0;
3689 }
3690 
3691 /**
3692  * i40e_vsi_configure_tx - Configure the VSI for Tx
3693  * @vsi: VSI structure describing this set of rings and resources
3694  *
3695  * Configure the Tx VSI for operation.
3696  **/
3697 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3698 {
3699 	int err = 0;
3700 	u16 i;
3701 
3702 	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3703 		err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3704 
3705 	if (err || !i40e_enabled_xdp_vsi(vsi))
3706 		return err;
3707 
3708 	for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3709 		err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3710 
3711 	return err;
3712 }
3713 
3714 /**
3715  * i40e_vsi_configure_rx - Configure the VSI for Rx
3716  * @vsi: the VSI being configured
3717  *
3718  * Configure the Rx VSI for operation.
3719  **/
3720 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3721 {
3722 	int err = 0;
3723 	u16 i;
3724 
3725 	vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog);
3726 	vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi);
3727 
3728 #if (PAGE_SIZE < 8192)
3729 	if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING &&
3730 	    vsi->netdev->mtu <= ETH_DATA_LEN) {
3731 		vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3732 		vsi->max_frame = vsi->rx_buf_len;
3733 	}
3734 #endif
3735 
3736 	/* set up individual rings */
3737 	for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3738 		err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3739 
3740 	return err;
3741 }
3742 
3743 /**
3744  * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC
3745  * @vsi: ptr to the VSI
3746  **/
3747 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3748 {
3749 	struct i40e_ring *tx_ring, *rx_ring;
3750 	u16 qoffset, qcount;
3751 	int i, n;
3752 
3753 	if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3754 		/* Reset the TC information */
3755 		for (i = 0; i < vsi->num_queue_pairs; i++) {
3756 			rx_ring = vsi->rx_rings[i];
3757 			tx_ring = vsi->tx_rings[i];
3758 			rx_ring->dcb_tc = 0;
3759 			tx_ring->dcb_tc = 0;
3760 		}
3761 		return;
3762 	}
3763 
3764 	for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3765 		if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3766 			continue;
3767 
3768 		qoffset = vsi->tc_config.tc_info[n].qoffset;
3769 		qcount = vsi->tc_config.tc_info[n].qcount;
3770 		for (i = qoffset; i < (qoffset + qcount); i++) {
3771 			rx_ring = vsi->rx_rings[i];
3772 			tx_ring = vsi->tx_rings[i];
3773 			rx_ring->dcb_tc = n;
3774 			tx_ring->dcb_tc = n;
3775 		}
3776 	}
3777 }
3778 
3779 /**
3780  * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI
3781  * @vsi: ptr to the VSI
3782  **/
3783 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3784 {
3785 	if (vsi->netdev)
3786 		i40e_set_rx_mode(vsi->netdev);
3787 }
3788 
3789 /**
3790  * i40e_reset_fdir_filter_cnt - Reset flow director filter counters
3791  * @pf: Pointer to the targeted PF
3792  *
3793  * Set all flow director counters to 0.
3794  */
3795 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf)
3796 {
3797 	pf->fd_tcp4_filter_cnt = 0;
3798 	pf->fd_udp4_filter_cnt = 0;
3799 	pf->fd_sctp4_filter_cnt = 0;
3800 	pf->fd_ip4_filter_cnt = 0;
3801 	pf->fd_tcp6_filter_cnt = 0;
3802 	pf->fd_udp6_filter_cnt = 0;
3803 	pf->fd_sctp6_filter_cnt = 0;
3804 	pf->fd_ip6_filter_cnt = 0;
3805 }
3806 
3807 /**
3808  * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters
3809  * @vsi: Pointer to the targeted VSI
3810  *
3811  * This function replays the hlist on the hw where all the SB Flow Director
3812  * filters were saved.
3813  **/
3814 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3815 {
3816 	struct i40e_fdir_filter *filter;
3817 	struct i40e_pf *pf = vsi->back;
3818 	struct hlist_node *node;
3819 
3820 	if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3821 		return;
3822 
3823 	/* Reset FDir counters as we're replaying all existing filters */
3824 	i40e_reset_fdir_filter_cnt(pf);
3825 
3826 	hlist_for_each_entry_safe(filter, node,
3827 				  &pf->fdir_filter_list, fdir_node) {
3828 		i40e_add_del_fdir(vsi, filter, true);
3829 	}
3830 }
3831 
3832 /**
3833  * i40e_vsi_configure - Set up the VSI for action
3834  * @vsi: the VSI being configured
3835  **/
3836 static int i40e_vsi_configure(struct i40e_vsi *vsi)
3837 {
3838 	int err;
3839 
3840 	i40e_set_vsi_rx_mode(vsi);
3841 	i40e_restore_vlan(vsi);
3842 	i40e_vsi_config_dcb_rings(vsi);
3843 	err = i40e_vsi_configure_tx(vsi);
3844 	if (!err)
3845 		err = i40e_vsi_configure_rx(vsi);
3846 
3847 	return err;
3848 }
3849 
3850 /**
3851  * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW
3852  * @vsi: the VSI being configured
3853  **/
3854 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3855 {
3856 	bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3857 	struct i40e_pf *pf = vsi->back;
3858 	struct i40e_hw *hw = &pf->hw;
3859 	u16 vector;
3860 	int i, q;
3861 	u32 qp;
3862 
3863 	/* The interrupt indexing is offset by 1 in the PFINT_ITRn
3864 	 * and PFINT_LNKLSTn registers, e.g.:
3865 	 *   PFINT_ITRn[0..n-1] gets msix-1..msix-n  (qpair interrupts)
3866 	 */
3867 	qp = vsi->base_queue;
3868 	vector = vsi->base_vector;
3869 	for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3870 		struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3871 
3872 		q_vector->rx.next_update = jiffies + 1;
3873 		q_vector->rx.target_itr =
3874 			ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3875 		wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3876 		     q_vector->rx.target_itr >> 1);
3877 		q_vector->rx.current_itr = q_vector->rx.target_itr;
3878 
3879 		q_vector->tx.next_update = jiffies + 1;
3880 		q_vector->tx.target_itr =
3881 			ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3882 		wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3883 		     q_vector->tx.target_itr >> 1);
3884 		q_vector->tx.current_itr = q_vector->tx.target_itr;
3885 
3886 		wr32(hw, I40E_PFINT_RATEN(vector - 1),
3887 		     i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3888 
3889 		/* begin of linked list for RX queue assigned to this vector */
3890 		wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3891 		for (q = 0; q < q_vector->num_ringpairs; q++) {
3892 			u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3893 			u32 val;
3894 
3895 			val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3896 			      (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3897 			      (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3898 			      (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3899 			      (I40E_QUEUE_TYPE_TX <<
3900 			       I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3901 
3902 			wr32(hw, I40E_QINT_RQCTL(qp), val);
3903 
3904 			if (has_xdp) {
3905 				/* TX queue with next queue set to TX */
3906 				val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3907 				      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3908 				      (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3909 				      (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3910 				      (I40E_QUEUE_TYPE_TX <<
3911 				       I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3912 
3913 				wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3914 			}
3915 			/* TX queue with next RX or end of linked list */
3916 			val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3917 			      (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3918 			      (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3919 			      ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3920 			      (I40E_QUEUE_TYPE_RX <<
3921 			       I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3922 
3923 			/* Terminate the linked list */
3924 			if (q == (q_vector->num_ringpairs - 1))
3925 				val |= (I40E_QUEUE_END_OF_LIST <<
3926 					I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3927 
3928 			wr32(hw, I40E_QINT_TQCTL(qp), val);
3929 			qp++;
3930 		}
3931 	}
3932 
3933 	i40e_flush(hw);
3934 }
3935 
3936 /**
3937  * i40e_enable_misc_int_causes - enable the non-queue interrupts
3938  * @pf: pointer to private device data structure
3939  **/
3940 static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3941 {
3942 	struct i40e_hw *hw = &pf->hw;
3943 	u32 val;
3944 
3945 	/* clear things first */
3946 	wr32(hw, I40E_PFINT_ICR0_ENA, 0);  /* disable all */
3947 	rd32(hw, I40E_PFINT_ICR0);         /* read to clear */
3948 
3949 	val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK       |
3950 	      I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK    |
3951 	      I40E_PFINT_ICR0_ENA_GRST_MASK          |
3952 	      I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3953 	      I40E_PFINT_ICR0_ENA_GPIO_MASK          |
3954 	      I40E_PFINT_ICR0_ENA_HMC_ERR_MASK       |
3955 	      I40E_PFINT_ICR0_ENA_VFLR_MASK          |
3956 	      I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3957 
3958 	if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3959 		val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3960 
3961 	if (pf->flags & I40E_FLAG_PTP)
3962 		val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3963 
3964 	wr32(hw, I40E_PFINT_ICR0_ENA, val);
3965 
3966 	/* SW_ITR_IDX = 0, but don't change INTENA */
3967 	wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3968 					I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3969 
3970 	/* OTHER_ITR_IDX = 0 */
3971 	wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3972 }
3973 
3974 /**
3975  * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW
3976  * @vsi: the VSI being configured
3977  **/
3978 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3979 {
3980 	u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3981 	struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3982 	struct i40e_pf *pf = vsi->back;
3983 	struct i40e_hw *hw = &pf->hw;
3984 
3985 	/* set the ITR configuration */
3986 	q_vector->rx.next_update = jiffies + 1;
3987 	q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3988 	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1);
3989 	q_vector->rx.current_itr = q_vector->rx.target_itr;
3990 	q_vector->tx.next_update = jiffies + 1;
3991 	q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3992 	wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1);
3993 	q_vector->tx.current_itr = q_vector->tx.target_itr;
3994 
3995 	i40e_enable_misc_int_causes(pf);
3996 
3997 	/* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */
3998 	wr32(hw, I40E_PFINT_LNKLST0, 0);
3999 
4000 	/* Associate the queue pair to the vector and enable the queue
4001 	 * interrupt RX queue in linked list with next queue set to TX
4002 	 */
4003 	wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX));
4004 
4005 	if (i40e_enabled_xdp_vsi(vsi)) {
4006 		/* TX queue in linked list with next queue set to TX */
4007 		wr32(hw, I40E_QINT_TQCTL(nextqp),
4008 		     I40E_QINT_TQCTL_VAL(nextqp, 0, TX));
4009 	}
4010 
4011 	/* last TX queue so the next RX queue doesn't matter */
4012 	wr32(hw, I40E_QINT_TQCTL(0),
4013 	     I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX));
4014 	i40e_flush(hw);
4015 }
4016 
4017 /**
4018  * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0
4019  * @pf: board private structure
4020  **/
4021 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
4022 {
4023 	struct i40e_hw *hw = &pf->hw;
4024 
4025 	wr32(hw, I40E_PFINT_DYN_CTL0,
4026 	     I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
4027 	i40e_flush(hw);
4028 }
4029 
4030 /**
4031  * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0
4032  * @pf: board private structure
4033  **/
4034 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
4035 {
4036 	struct i40e_hw *hw = &pf->hw;
4037 	u32 val;
4038 
4039 	val = I40E_PFINT_DYN_CTL0_INTENA_MASK   |
4040 	      I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
4041 	      (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
4042 
4043 	wr32(hw, I40E_PFINT_DYN_CTL0, val);
4044 	i40e_flush(hw);
4045 }
4046 
4047 /**
4048  * i40e_msix_clean_rings - MSIX mode Interrupt Handler
4049  * @irq: interrupt number
4050  * @data: pointer to a q_vector
4051  **/
4052 static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
4053 {
4054 	struct i40e_q_vector *q_vector = data;
4055 
4056 	if (!q_vector->tx.ring && !q_vector->rx.ring)
4057 		return IRQ_HANDLED;
4058 
4059 	napi_schedule_irqoff(&q_vector->napi);
4060 
4061 	return IRQ_HANDLED;
4062 }
4063 
4064 /**
4065  * i40e_irq_affinity_notify - Callback for affinity changes
4066  * @notify: context as to what irq was changed
4067  * @mask: the new affinity mask
4068  *
4069  * This is a callback function used by the irq_set_affinity_notifier function
4070  * so that we may register to receive changes to the irq affinity masks.
4071  **/
4072 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
4073 				     const cpumask_t *mask)
4074 {
4075 	struct i40e_q_vector *q_vector =
4076 		container_of(notify, struct i40e_q_vector, affinity_notify);
4077 
4078 	cpumask_copy(&q_vector->affinity_mask, mask);
4079 }
4080 
4081 /**
4082  * i40e_irq_affinity_release - Callback for affinity notifier release
4083  * @ref: internal core kernel usage
4084  *
4085  * This is a callback function used by the irq_set_affinity_notifier function
4086  * to inform the current notification subscriber that they will no longer
4087  * receive notifications.
4088  **/
4089 static void i40e_irq_affinity_release(struct kref *ref) {}
4090 
4091 /**
4092  * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts
4093  * @vsi: the VSI being configured
4094  * @basename: name for the vector
4095  *
4096  * Allocates MSI-X vectors and requests interrupts from the kernel.
4097  **/
4098 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
4099 {
4100 	int q_vectors = vsi->num_q_vectors;
4101 	struct i40e_pf *pf = vsi->back;
4102 	int base = vsi->base_vector;
4103 	int rx_int_idx = 0;
4104 	int tx_int_idx = 0;
4105 	int vector, err;
4106 	int irq_num;
4107 	int cpu;
4108 
4109 	for (vector = 0; vector < q_vectors; vector++) {
4110 		struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
4111 
4112 		irq_num = pf->msix_entries[base + vector].vector;
4113 
4114 		if (q_vector->tx.ring && q_vector->rx.ring) {
4115 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4116 				 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
4117 			tx_int_idx++;
4118 		} else if (q_vector->rx.ring) {
4119 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4120 				 "%s-%s-%d", basename, "rx", rx_int_idx++);
4121 		} else if (q_vector->tx.ring) {
4122 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
4123 				 "%s-%s-%d", basename, "tx", tx_int_idx++);
4124 		} else {
4125 			/* skip this unused q_vector */
4126 			continue;
4127 		}
4128 		err = request_irq(irq_num,
4129 				  vsi->irq_handler,
4130 				  0,
4131 				  q_vector->name,
4132 				  q_vector);
4133 		if (err) {
4134 			dev_info(&pf->pdev->dev,
4135 				 "MSIX request_irq failed, error: %d\n", err);
4136 			goto free_queue_irqs;
4137 		}
4138 
4139 		/* register for affinity change notifications */
4140 		q_vector->irq_num = irq_num;
4141 		q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
4142 		q_vector->affinity_notify.release = i40e_irq_affinity_release;
4143 		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
4144 		/* Spread affinity hints out across online CPUs.
4145 		 *
4146 		 * get_cpu_mask returns a static constant mask with
4147 		 * a permanent lifetime so it's ok to pass to
4148 		 * irq_update_affinity_hint without making a copy.
4149 		 */
4150 		cpu = cpumask_local_spread(q_vector->v_idx, -1);
4151 		irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
4152 	}
4153 
4154 	vsi->irqs_ready = true;
4155 	return 0;
4156 
4157 free_queue_irqs:
4158 	while (vector) {
4159 		vector--;
4160 		irq_num = pf->msix_entries[base + vector].vector;
4161 		irq_set_affinity_notifier(irq_num, NULL);
4162 		irq_update_affinity_hint(irq_num, NULL);
4163 		free_irq(irq_num, &vsi->q_vectors[vector]);
4164 	}
4165 	return err;
4166 }
4167 
4168 /**
4169  * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI
4170  * @vsi: the VSI being un-configured
4171  **/
4172 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
4173 {
4174 	struct i40e_pf *pf = vsi->back;
4175 	struct i40e_hw *hw = &pf->hw;
4176 	int base = vsi->base_vector;
4177 	int i;
4178 
4179 	/* disable interrupt causation from each queue */
4180 	for (i = 0; i < vsi->num_queue_pairs; i++) {
4181 		u32 val;
4182 
4183 		val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4184 		val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
4185 		wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4186 
4187 		val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
4188 		val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
4189 		wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
4190 
4191 		if (!i40e_enabled_xdp_vsi(vsi))
4192 			continue;
4193 		wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
4194 	}
4195 
4196 	/* disable each interrupt */
4197 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4198 		for (i = vsi->base_vector;
4199 		     i < (vsi->num_q_vectors + vsi->base_vector); i++)
4200 			wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
4201 
4202 		i40e_flush(hw);
4203 		for (i = 0; i < vsi->num_q_vectors; i++)
4204 			synchronize_irq(pf->msix_entries[i + base].vector);
4205 	} else {
4206 		/* Legacy and MSI mode - this stops all interrupt handling */
4207 		wr32(hw, I40E_PFINT_ICR0_ENA, 0);
4208 		wr32(hw, I40E_PFINT_DYN_CTL0, 0);
4209 		i40e_flush(hw);
4210 		synchronize_irq(pf->pdev->irq);
4211 	}
4212 }
4213 
4214 /**
4215  * i40e_vsi_enable_irq - Enable IRQ for the given VSI
4216  * @vsi: the VSI being configured
4217  **/
4218 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
4219 {
4220 	struct i40e_pf *pf = vsi->back;
4221 	int i;
4222 
4223 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4224 		for (i = 0; i < vsi->num_q_vectors; i++)
4225 			i40e_irq_dynamic_enable(vsi, i);
4226 	} else {
4227 		i40e_irq_dynamic_enable_icr0(pf);
4228 	}
4229 
4230 	i40e_flush(&pf->hw);
4231 	return 0;
4232 }
4233 
4234 /**
4235  * i40e_free_misc_vector - Free the vector that handles non-queue events
4236  * @pf: board private structure
4237  **/
4238 static void i40e_free_misc_vector(struct i40e_pf *pf)
4239 {
4240 	/* Disable ICR 0 */
4241 	wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
4242 	i40e_flush(&pf->hw);
4243 
4244 	if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
4245 		free_irq(pf->msix_entries[0].vector, pf);
4246 		clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
4247 	}
4248 }
4249 
4250 /**
4251  * i40e_intr - MSI/Legacy and non-queue interrupt handler
4252  * @irq: interrupt number
4253  * @data: pointer to a q_vector
4254  *
4255  * This is the handler used for all MSI/Legacy interrupts, and deals
4256  * with both queue and non-queue interrupts.  This is also used in
4257  * MSIX mode to handle the non-queue interrupts.
4258  **/
4259 static irqreturn_t i40e_intr(int irq, void *data)
4260 {
4261 	struct i40e_pf *pf = (struct i40e_pf *)data;
4262 	struct i40e_hw *hw = &pf->hw;
4263 	irqreturn_t ret = IRQ_NONE;
4264 	u32 icr0, icr0_remaining;
4265 	u32 val, ena_mask;
4266 
4267 	icr0 = rd32(hw, I40E_PFINT_ICR0);
4268 	ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
4269 
4270 	/* if sharing a legacy IRQ, we might get called w/o an intr pending */
4271 	if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
4272 		goto enable_intr;
4273 
4274 	/* if interrupt but no bits showing, must be SWINT */
4275 	if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
4276 	    (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
4277 		pf->sw_int_count++;
4278 
4279 	if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
4280 	    (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
4281 		ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
4282 		dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
4283 		set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
4284 	}
4285 
4286 	/* only q0 is used in MSI/Legacy mode, and none are used in MSIX */
4287 	if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
4288 		struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
4289 		struct i40e_q_vector *q_vector = vsi->q_vectors[0];
4290 
4291 		/* We do not have a way to disarm Queue causes while leaving
4292 		 * interrupt enabled for all other causes, ideally
4293 		 * interrupt should be disabled while we are in NAPI but
4294 		 * this is not a performance path and napi_schedule()
4295 		 * can deal with rescheduling.
4296 		 */
4297 		if (!test_bit(__I40E_DOWN, pf->state))
4298 			napi_schedule_irqoff(&q_vector->napi);
4299 	}
4300 
4301 	if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
4302 		ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
4303 		set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
4304 		i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
4305 	}
4306 
4307 	if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
4308 		ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
4309 		set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
4310 	}
4311 
4312 	if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
4313 		/* disable any further VFLR event notifications */
4314 		if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) {
4315 			u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
4316 
4317 			reg &= ~I40E_PFINT_ICR0_VFLR_MASK;
4318 			wr32(hw, I40E_PFINT_ICR0_ENA, reg);
4319 		} else {
4320 			ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
4321 			set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
4322 		}
4323 	}
4324 
4325 	if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
4326 		if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4327 			set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
4328 		ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
4329 		val = rd32(hw, I40E_GLGEN_RSTAT);
4330 		val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
4331 		       >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
4332 		if (val == I40E_RESET_CORER) {
4333 			pf->corer_count++;
4334 		} else if (val == I40E_RESET_GLOBR) {
4335 			pf->globr_count++;
4336 		} else if (val == I40E_RESET_EMPR) {
4337 			pf->empr_count++;
4338 			set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
4339 		}
4340 	}
4341 
4342 	if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
4343 		icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
4344 		dev_info(&pf->pdev->dev, "HMC error interrupt\n");
4345 		dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
4346 			 rd32(hw, I40E_PFHMC_ERRORINFO),
4347 			 rd32(hw, I40E_PFHMC_ERRORDATA));
4348 	}
4349 
4350 	if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
4351 		u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
4352 
4353 		if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK)
4354 			schedule_work(&pf->ptp_extts0_work);
4355 
4356 		if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK)
4357 			i40e_ptp_tx_hwtstamp(pf);
4358 
4359 		icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4360 	}
4361 
4362 	/* If a critical error is pending we have no choice but to reset the
4363 	 * device.
4364 	 * Report and mask out any remaining unexpected interrupts.
4365 	 */
4366 	icr0_remaining = icr0 & ena_mask;
4367 	if (icr0_remaining) {
4368 		dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4369 			 icr0_remaining);
4370 		if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4371 		    (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4372 		    (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4373 			dev_info(&pf->pdev->dev, "device will be reset\n");
4374 			set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4375 			i40e_service_event_schedule(pf);
4376 		}
4377 		ena_mask &= ~icr0_remaining;
4378 	}
4379 	ret = IRQ_HANDLED;
4380 
4381 enable_intr:
4382 	/* re-enable interrupt causes */
4383 	wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4384 	if (!test_bit(__I40E_DOWN, pf->state) ||
4385 	    test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4386 		i40e_service_event_schedule(pf);
4387 		i40e_irq_dynamic_enable_icr0(pf);
4388 	}
4389 
4390 	return ret;
4391 }
4392 
4393 /**
4394  * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes
4395  * @tx_ring:  tx ring to clean
4396  * @budget:   how many cleans we're allowed
4397  *
4398  * Returns true if there's any budget left (e.g. the clean is finished)
4399  **/
4400 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4401 {
4402 	struct i40e_vsi *vsi = tx_ring->vsi;
4403 	u16 i = tx_ring->next_to_clean;
4404 	struct i40e_tx_buffer *tx_buf;
4405 	struct i40e_tx_desc *tx_desc;
4406 
4407 	tx_buf = &tx_ring->tx_bi[i];
4408 	tx_desc = I40E_TX_DESC(tx_ring, i);
4409 	i -= tx_ring->count;
4410 
4411 	do {
4412 		struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4413 
4414 		/* if next_to_watch is not set then there is no work pending */
4415 		if (!eop_desc)
4416 			break;
4417 
4418 		/* prevent any other reads prior to eop_desc */
4419 		smp_rmb();
4420 
4421 		/* if the descriptor isn't done, no work yet to do */
4422 		if (!(eop_desc->cmd_type_offset_bsz &
4423 		      cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4424 			break;
4425 
4426 		/* clear next_to_watch to prevent false hangs */
4427 		tx_buf->next_to_watch = NULL;
4428 
4429 		tx_desc->buffer_addr = 0;
4430 		tx_desc->cmd_type_offset_bsz = 0;
4431 		/* move past filter desc */
4432 		tx_buf++;
4433 		tx_desc++;
4434 		i++;
4435 		if (unlikely(!i)) {
4436 			i -= tx_ring->count;
4437 			tx_buf = tx_ring->tx_bi;
4438 			tx_desc = I40E_TX_DESC(tx_ring, 0);
4439 		}
4440 		/* unmap skb header data */
4441 		dma_unmap_single(tx_ring->dev,
4442 				 dma_unmap_addr(tx_buf, dma),
4443 				 dma_unmap_len(tx_buf, len),
4444 				 DMA_TO_DEVICE);
4445 		if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4446 			kfree(tx_buf->raw_buf);
4447 
4448 		tx_buf->raw_buf = NULL;
4449 		tx_buf->tx_flags = 0;
4450 		tx_buf->next_to_watch = NULL;
4451 		dma_unmap_len_set(tx_buf, len, 0);
4452 		tx_desc->buffer_addr = 0;
4453 		tx_desc->cmd_type_offset_bsz = 0;
4454 
4455 		/* move us past the eop_desc for start of next FD desc */
4456 		tx_buf++;
4457 		tx_desc++;
4458 		i++;
4459 		if (unlikely(!i)) {
4460 			i -= tx_ring->count;
4461 			tx_buf = tx_ring->tx_bi;
4462 			tx_desc = I40E_TX_DESC(tx_ring, 0);
4463 		}
4464 
4465 		/* update budget accounting */
4466 		budget--;
4467 	} while (likely(budget));
4468 
4469 	i += tx_ring->count;
4470 	tx_ring->next_to_clean = i;
4471 
4472 	if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4473 		i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4474 
4475 	return budget > 0;
4476 }
4477 
4478 /**
4479  * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring
4480  * @irq: interrupt number
4481  * @data: pointer to a q_vector
4482  **/
4483 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4484 {
4485 	struct i40e_q_vector *q_vector = data;
4486 	struct i40e_vsi *vsi;
4487 
4488 	if (!q_vector->tx.ring)
4489 		return IRQ_HANDLED;
4490 
4491 	vsi = q_vector->tx.ring->vsi;
4492 	i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4493 
4494 	return IRQ_HANDLED;
4495 }
4496 
4497 /**
4498  * i40e_map_vector_to_qp - Assigns the queue pair to the vector
4499  * @vsi: the VSI being configured
4500  * @v_idx: vector index
4501  * @qp_idx: queue pair index
4502  **/
4503 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4504 {
4505 	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4506 	struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4507 	struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4508 
4509 	tx_ring->q_vector = q_vector;
4510 	tx_ring->next = q_vector->tx.ring;
4511 	q_vector->tx.ring = tx_ring;
4512 	q_vector->tx.count++;
4513 
4514 	/* Place XDP Tx ring in the same q_vector ring list as regular Tx */
4515 	if (i40e_enabled_xdp_vsi(vsi)) {
4516 		struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4517 
4518 		xdp_ring->q_vector = q_vector;
4519 		xdp_ring->next = q_vector->tx.ring;
4520 		q_vector->tx.ring = xdp_ring;
4521 		q_vector->tx.count++;
4522 	}
4523 
4524 	rx_ring->q_vector = q_vector;
4525 	rx_ring->next = q_vector->rx.ring;
4526 	q_vector->rx.ring = rx_ring;
4527 	q_vector->rx.count++;
4528 }
4529 
4530 /**
4531  * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors
4532  * @vsi: the VSI being configured
4533  *
4534  * This function maps descriptor rings to the queue-specific vectors
4535  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
4536  * one vector per queue pair, but on a constrained vector budget, we
4537  * group the queue pairs as "efficiently" as possible.
4538  **/
4539 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4540 {
4541 	int qp_remaining = vsi->num_queue_pairs;
4542 	int q_vectors = vsi->num_q_vectors;
4543 	int num_ringpairs;
4544 	int v_start = 0;
4545 	int qp_idx = 0;
4546 
4547 	/* If we don't have enough vectors for a 1-to-1 mapping, we'll have to
4548 	 * group them so there are multiple queues per vector.
4549 	 * It is also important to go through all the vectors available to be
4550 	 * sure that if we don't use all the vectors, that the remaining vectors
4551 	 * are cleared. This is especially important when decreasing the
4552 	 * number of queues in use.
4553 	 */
4554 	for (; v_start < q_vectors; v_start++) {
4555 		struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4556 
4557 		num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4558 
4559 		q_vector->num_ringpairs = num_ringpairs;
4560 		q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4561 
4562 		q_vector->rx.count = 0;
4563 		q_vector->tx.count = 0;
4564 		q_vector->rx.ring = NULL;
4565 		q_vector->tx.ring = NULL;
4566 
4567 		while (num_ringpairs--) {
4568 			i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4569 			qp_idx++;
4570 			qp_remaining--;
4571 		}
4572 	}
4573 }
4574 
4575 /**
4576  * i40e_vsi_request_irq - Request IRQ from the OS
4577  * @vsi: the VSI being configured
4578  * @basename: name for the vector
4579  **/
4580 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4581 {
4582 	struct i40e_pf *pf = vsi->back;
4583 	int err;
4584 
4585 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4586 		err = i40e_vsi_request_irq_msix(vsi, basename);
4587 	else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4588 		err = request_irq(pf->pdev->irq, i40e_intr, 0,
4589 				  pf->int_name, pf);
4590 	else
4591 		err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4592 				  pf->int_name, pf);
4593 
4594 	if (err)
4595 		dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4596 
4597 	return err;
4598 }
4599 
4600 #ifdef CONFIG_NET_POLL_CONTROLLER
4601 /**
4602  * i40e_netpoll - A Polling 'interrupt' handler
4603  * @netdev: network interface device structure
4604  *
4605  * This is used by netconsole to send skbs without having to re-enable
4606  * interrupts.  It's not called while the normal interrupt routine is executing.
4607  **/
4608 static void i40e_netpoll(struct net_device *netdev)
4609 {
4610 	struct i40e_netdev_priv *np = netdev_priv(netdev);
4611 	struct i40e_vsi *vsi = np->vsi;
4612 	struct i40e_pf *pf = vsi->back;
4613 	int i;
4614 
4615 	/* if interface is down do nothing */
4616 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
4617 		return;
4618 
4619 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4620 		for (i = 0; i < vsi->num_q_vectors; i++)
4621 			i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4622 	} else {
4623 		i40e_intr(pf->pdev->irq, netdev);
4624 	}
4625 }
4626 #endif
4627 
4628 #define I40E_QTX_ENA_WAIT_COUNT 50
4629 
4630 /**
4631  * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled
4632  * @pf: the PF being configured
4633  * @pf_q: the PF queue
4634  * @enable: enable or disable state of the queue
4635  *
4636  * This routine will wait for the given Tx queue of the PF to reach the
4637  * enabled or disabled state.
4638  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4639  * multiple retries; else will return 0 in case of success.
4640  **/
4641 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4642 {
4643 	int i;
4644 	u32 tx_reg;
4645 
4646 	for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4647 		tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4648 		if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4649 			break;
4650 
4651 		usleep_range(10, 20);
4652 	}
4653 	if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4654 		return -ETIMEDOUT;
4655 
4656 	return 0;
4657 }
4658 
4659 /**
4660  * i40e_control_tx_q - Start or stop a particular Tx queue
4661  * @pf: the PF structure
4662  * @pf_q: the PF queue to configure
4663  * @enable: start or stop the queue
4664  *
4665  * This function enables or disables a single queue. Note that any delay
4666  * required after the operation is expected to be handled by the caller of
4667  * this function.
4668  **/
4669 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4670 {
4671 	struct i40e_hw *hw = &pf->hw;
4672 	u32 tx_reg;
4673 	int i;
4674 
4675 	/* warn the TX unit of coming changes */
4676 	i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4677 	if (!enable)
4678 		usleep_range(10, 20);
4679 
4680 	for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4681 		tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4682 		if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4683 		    ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4684 			break;
4685 		usleep_range(1000, 2000);
4686 	}
4687 
4688 	/* Skip if the queue is already in the requested state */
4689 	if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4690 		return;
4691 
4692 	/* turn on/off the queue */
4693 	if (enable) {
4694 		wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4695 		tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4696 	} else {
4697 		tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4698 	}
4699 
4700 	wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4701 }
4702 
4703 /**
4704  * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion
4705  * @seid: VSI SEID
4706  * @pf: the PF structure
4707  * @pf_q: the PF queue to configure
4708  * @is_xdp: true if the queue is used for XDP
4709  * @enable: start or stop the queue
4710  **/
4711 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4712 			   bool is_xdp, bool enable)
4713 {
4714 	int ret;
4715 
4716 	i40e_control_tx_q(pf, pf_q, enable);
4717 
4718 	/* wait for the change to finish */
4719 	ret = i40e_pf_txq_wait(pf, pf_q, enable);
4720 	if (ret) {
4721 		dev_info(&pf->pdev->dev,
4722 			 "VSI seid %d %sTx ring %d %sable timeout\n",
4723 			 seid, (is_xdp ? "XDP " : ""), pf_q,
4724 			 (enable ? "en" : "dis"));
4725 	}
4726 
4727 	return ret;
4728 }
4729 
4730 /**
4731  * i40e_vsi_enable_tx - Start a VSI's rings
4732  * @vsi: the VSI being configured
4733  **/
4734 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi)
4735 {
4736 	struct i40e_pf *pf = vsi->back;
4737 	int i, pf_q, ret = 0;
4738 
4739 	pf_q = vsi->base_queue;
4740 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4741 		ret = i40e_control_wait_tx_q(vsi->seid, pf,
4742 					     pf_q,
4743 					     false /*is xdp*/, true);
4744 		if (ret)
4745 			break;
4746 
4747 		if (!i40e_enabled_xdp_vsi(vsi))
4748 			continue;
4749 
4750 		ret = i40e_control_wait_tx_q(vsi->seid, pf,
4751 					     pf_q + vsi->alloc_queue_pairs,
4752 					     true /*is xdp*/, true);
4753 		if (ret)
4754 			break;
4755 	}
4756 	return ret;
4757 }
4758 
4759 /**
4760  * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4761  * @pf: the PF being configured
4762  * @pf_q: the PF queue
4763  * @enable: enable or disable state of the queue
4764  *
4765  * This routine will wait for the given Rx queue of the PF to reach the
4766  * enabled or disabled state.
4767  * Returns -ETIMEDOUT in case of failing to reach the requested state after
4768  * multiple retries; else will return 0 in case of success.
4769  **/
4770 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4771 {
4772 	int i;
4773 	u32 rx_reg;
4774 
4775 	for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4776 		rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4777 		if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4778 			break;
4779 
4780 		usleep_range(10, 20);
4781 	}
4782 	if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4783 		return -ETIMEDOUT;
4784 
4785 	return 0;
4786 }
4787 
4788 /**
4789  * i40e_control_rx_q - Start or stop a particular Rx queue
4790  * @pf: the PF structure
4791  * @pf_q: the PF queue to configure
4792  * @enable: start or stop the queue
4793  *
4794  * This function enables or disables a single queue. Note that
4795  * any delay required after the operation is expected to be
4796  * handled by the caller of this function.
4797  **/
4798 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4799 {
4800 	struct i40e_hw *hw = &pf->hw;
4801 	u32 rx_reg;
4802 	int i;
4803 
4804 	for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4805 		rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4806 		if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4807 		    ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4808 			break;
4809 		usleep_range(1000, 2000);
4810 	}
4811 
4812 	/* Skip if the queue is already in the requested state */
4813 	if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4814 		return;
4815 
4816 	/* turn on/off the queue */
4817 	if (enable)
4818 		rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4819 	else
4820 		rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4821 
4822 	wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4823 }
4824 
4825 /**
4826  * i40e_control_wait_rx_q
4827  * @pf: the PF structure
4828  * @pf_q: queue being configured
4829  * @enable: start or stop the rings
4830  *
4831  * This function enables or disables a single queue along with waiting
4832  * for the change to finish. The caller of this function should handle
4833  * the delays needed in the case of disabling queues.
4834  **/
4835 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4836 {
4837 	int ret = 0;
4838 
4839 	i40e_control_rx_q(pf, pf_q, enable);
4840 
4841 	/* wait for the change to finish */
4842 	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4843 	if (ret)
4844 		return ret;
4845 
4846 	return ret;
4847 }
4848 
4849 /**
4850  * i40e_vsi_enable_rx - Start a VSI's rings
4851  * @vsi: the VSI being configured
4852  **/
4853 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi)
4854 {
4855 	struct i40e_pf *pf = vsi->back;
4856 	int i, pf_q, ret = 0;
4857 
4858 	pf_q = vsi->base_queue;
4859 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4860 		ret = i40e_control_wait_rx_q(pf, pf_q, true);
4861 		if (ret) {
4862 			dev_info(&pf->pdev->dev,
4863 				 "VSI seid %d Rx ring %d enable timeout\n",
4864 				 vsi->seid, pf_q);
4865 			break;
4866 		}
4867 	}
4868 
4869 	return ret;
4870 }
4871 
4872 /**
4873  * i40e_vsi_start_rings - Start a VSI's rings
4874  * @vsi: the VSI being configured
4875  **/
4876 int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4877 {
4878 	int ret = 0;
4879 
4880 	/* do rx first for enable and last for disable */
4881 	ret = i40e_vsi_enable_rx(vsi);
4882 	if (ret)
4883 		return ret;
4884 	ret = i40e_vsi_enable_tx(vsi);
4885 
4886 	return ret;
4887 }
4888 
4889 #define I40E_DISABLE_TX_GAP_MSEC	50
4890 
4891 /**
4892  * i40e_vsi_stop_rings - Stop a VSI's rings
4893  * @vsi: the VSI being configured
4894  **/
4895 void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4896 {
4897 	struct i40e_pf *pf = vsi->back;
4898 	int pf_q, err, q_end;
4899 
4900 	/* When port TX is suspended, don't wait */
4901 	if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4902 		return i40e_vsi_stop_rings_no_wait(vsi);
4903 
4904 	q_end = vsi->base_queue + vsi->num_queue_pairs;
4905 	for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4906 		i40e_pre_tx_queue_cfg(&pf->hw, (u32)pf_q, false);
4907 
4908 	for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) {
4909 		err = i40e_control_wait_rx_q(pf, pf_q, false);
4910 		if (err)
4911 			dev_info(&pf->pdev->dev,
4912 				 "VSI seid %d Rx ring %d disable timeout\n",
4913 				 vsi->seid, pf_q);
4914 	}
4915 
4916 	msleep(I40E_DISABLE_TX_GAP_MSEC);
4917 	pf_q = vsi->base_queue;
4918 	for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++)
4919 		wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0);
4920 
4921 	i40e_vsi_wait_queues_disabled(vsi);
4922 }
4923 
4924 /**
4925  * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay
4926  * @vsi: the VSI being shutdown
4927  *
4928  * This function stops all the rings for a VSI but does not delay to verify
4929  * that rings have been disabled. It is expected that the caller is shutting
4930  * down multiple VSIs at once and will delay together for all the VSIs after
4931  * initiating the shutdown. This is particularly useful for shutting down lots
4932  * of VFs together. Otherwise, a large delay can be incurred while configuring
4933  * each VSI in serial.
4934  **/
4935 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4936 {
4937 	struct i40e_pf *pf = vsi->back;
4938 	int i, pf_q;
4939 
4940 	pf_q = vsi->base_queue;
4941 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4942 		i40e_control_tx_q(pf, pf_q, false);
4943 		i40e_control_rx_q(pf, pf_q, false);
4944 	}
4945 }
4946 
4947 /**
4948  * i40e_vsi_free_irq - Free the irq association with the OS
4949  * @vsi: the VSI being configured
4950  **/
4951 static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4952 {
4953 	struct i40e_pf *pf = vsi->back;
4954 	struct i40e_hw *hw = &pf->hw;
4955 	int base = vsi->base_vector;
4956 	u32 val, qp;
4957 	int i;
4958 
4959 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4960 		if (!vsi->q_vectors)
4961 			return;
4962 
4963 		if (!vsi->irqs_ready)
4964 			return;
4965 
4966 		vsi->irqs_ready = false;
4967 		for (i = 0; i < vsi->num_q_vectors; i++) {
4968 			int irq_num;
4969 			u16 vector;
4970 
4971 			vector = i + base;
4972 			irq_num = pf->msix_entries[vector].vector;
4973 
4974 			/* free only the irqs that were actually requested */
4975 			if (!vsi->q_vectors[i] ||
4976 			    !vsi->q_vectors[i]->num_ringpairs)
4977 				continue;
4978 
4979 			/* clear the affinity notifier in the IRQ descriptor */
4980 			irq_set_affinity_notifier(irq_num, NULL);
4981 			/* remove our suggested affinity mask for this IRQ */
4982 			irq_update_affinity_hint(irq_num, NULL);
4983 			free_irq(irq_num, vsi->q_vectors[i]);
4984 
4985 			/* Tear down the interrupt queue link list
4986 			 *
4987 			 * We know that they come in pairs and always
4988 			 * the Rx first, then the Tx.  To clear the
4989 			 * link list, stick the EOL value into the
4990 			 * next_q field of the registers.
4991 			 */
4992 			val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4993 			qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4994 				>> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4995 			val |= I40E_QUEUE_END_OF_LIST
4996 				<< I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4997 			wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4998 
4999 			while (qp != I40E_QUEUE_END_OF_LIST) {
5000 				u32 next;
5001 
5002 				val = rd32(hw, I40E_QINT_RQCTL(qp));
5003 
5004 				val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
5005 					 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5006 					 I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
5007 					 I40E_QINT_RQCTL_INTEVENT_MASK);
5008 
5009 				val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5010 					 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5011 
5012 				wr32(hw, I40E_QINT_RQCTL(qp), val);
5013 
5014 				val = rd32(hw, I40E_QINT_TQCTL(qp));
5015 
5016 				next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
5017 					>> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
5018 
5019 				val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
5020 					 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5021 					 I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
5022 					 I40E_QINT_TQCTL_INTEVENT_MASK);
5023 
5024 				val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5025 					 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5026 
5027 				wr32(hw, I40E_QINT_TQCTL(qp), val);
5028 				qp = next;
5029 			}
5030 		}
5031 	} else {
5032 		free_irq(pf->pdev->irq, pf);
5033 
5034 		val = rd32(hw, I40E_PFINT_LNKLST0);
5035 		qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
5036 			>> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
5037 		val |= I40E_QUEUE_END_OF_LIST
5038 			<< I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
5039 		wr32(hw, I40E_PFINT_LNKLST0, val);
5040 
5041 		val = rd32(hw, I40E_QINT_RQCTL(qp));
5042 		val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK  |
5043 			 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
5044 			 I40E_QINT_RQCTL_CAUSE_ENA_MASK  |
5045 			 I40E_QINT_RQCTL_INTEVENT_MASK);
5046 
5047 		val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
5048 			I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
5049 
5050 		wr32(hw, I40E_QINT_RQCTL(qp), val);
5051 
5052 		val = rd32(hw, I40E_QINT_TQCTL(qp));
5053 
5054 		val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK  |
5055 			 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
5056 			 I40E_QINT_TQCTL_CAUSE_ENA_MASK  |
5057 			 I40E_QINT_TQCTL_INTEVENT_MASK);
5058 
5059 		val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
5060 			I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
5061 
5062 		wr32(hw, I40E_QINT_TQCTL(qp), val);
5063 	}
5064 }
5065 
5066 /**
5067  * i40e_free_q_vector - Free memory allocated for specific interrupt vector
5068  * @vsi: the VSI being configured
5069  * @v_idx: Index of vector to be freed
5070  *
5071  * This function frees the memory allocated to the q_vector.  In addition if
5072  * NAPI is enabled it will delete any references to the NAPI struct prior
5073  * to freeing the q_vector.
5074  **/
5075 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
5076 {
5077 	struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
5078 	struct i40e_ring *ring;
5079 
5080 	if (!q_vector)
5081 		return;
5082 
5083 	/* disassociate q_vector from rings */
5084 	i40e_for_each_ring(ring, q_vector->tx)
5085 		ring->q_vector = NULL;
5086 
5087 	i40e_for_each_ring(ring, q_vector->rx)
5088 		ring->q_vector = NULL;
5089 
5090 	/* only VSI w/ an associated netdev is set up w/ NAPI */
5091 	if (vsi->netdev)
5092 		netif_napi_del(&q_vector->napi);
5093 
5094 	vsi->q_vectors[v_idx] = NULL;
5095 
5096 	kfree_rcu(q_vector, rcu);
5097 }
5098 
5099 /**
5100  * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors
5101  * @vsi: the VSI being un-configured
5102  *
5103  * This frees the memory allocated to the q_vectors and
5104  * deletes references to the NAPI struct.
5105  **/
5106 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
5107 {
5108 	int v_idx;
5109 
5110 	for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
5111 		i40e_free_q_vector(vsi, v_idx);
5112 }
5113 
5114 /**
5115  * i40e_reset_interrupt_capability - Disable interrupt setup in OS
5116  * @pf: board private structure
5117  **/
5118 static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
5119 {
5120 	/* If we're in Legacy mode, the interrupt was cleaned in vsi_close */
5121 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
5122 		pci_disable_msix(pf->pdev);
5123 		kfree(pf->msix_entries);
5124 		pf->msix_entries = NULL;
5125 		kfree(pf->irq_pile);
5126 		pf->irq_pile = NULL;
5127 	} else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
5128 		pci_disable_msi(pf->pdev);
5129 	}
5130 	pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
5131 }
5132 
5133 /**
5134  * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings
5135  * @pf: board private structure
5136  *
5137  * We go through and clear interrupt specific resources and reset the structure
5138  * to pre-load conditions
5139  **/
5140 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
5141 {
5142 	int i;
5143 
5144 	if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state))
5145 		i40e_free_misc_vector(pf);
5146 
5147 	i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
5148 		      I40E_IWARP_IRQ_PILE_ID);
5149 
5150 	i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
5151 	for (i = 0; i < pf->num_alloc_vsi; i++)
5152 		if (pf->vsi[i])
5153 			i40e_vsi_free_q_vectors(pf->vsi[i]);
5154 	i40e_reset_interrupt_capability(pf);
5155 }
5156 
5157 /**
5158  * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI
5159  * @vsi: the VSI being configured
5160  **/
5161 static void i40e_napi_enable_all(struct i40e_vsi *vsi)
5162 {
5163 	int q_idx;
5164 
5165 	if (!vsi->netdev)
5166 		return;
5167 
5168 	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5169 		struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5170 
5171 		if (q_vector->rx.ring || q_vector->tx.ring)
5172 			napi_enable(&q_vector->napi);
5173 	}
5174 }
5175 
5176 /**
5177  * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI
5178  * @vsi: the VSI being configured
5179  **/
5180 static void i40e_napi_disable_all(struct i40e_vsi *vsi)
5181 {
5182 	int q_idx;
5183 
5184 	if (!vsi->netdev)
5185 		return;
5186 
5187 	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
5188 		struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
5189 
5190 		if (q_vector->rx.ring || q_vector->tx.ring)
5191 			napi_disable(&q_vector->napi);
5192 	}
5193 }
5194 
5195 /**
5196  * i40e_vsi_close - Shut down a VSI
5197  * @vsi: the vsi to be quelled
5198  **/
5199 static void i40e_vsi_close(struct i40e_vsi *vsi)
5200 {
5201 	struct i40e_pf *pf = vsi->back;
5202 	if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
5203 		i40e_down(vsi);
5204 	i40e_vsi_free_irq(vsi);
5205 	i40e_vsi_free_tx_resources(vsi);
5206 	i40e_vsi_free_rx_resources(vsi);
5207 	vsi->current_netdev_flags = 0;
5208 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
5209 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
5210 		set_bit(__I40E_CLIENT_RESET, pf->state);
5211 }
5212 
5213 /**
5214  * i40e_quiesce_vsi - Pause a given VSI
5215  * @vsi: the VSI being paused
5216  **/
5217 static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
5218 {
5219 	if (test_bit(__I40E_VSI_DOWN, vsi->state))
5220 		return;
5221 
5222 	set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
5223 	if (vsi->netdev && netif_running(vsi->netdev))
5224 		vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5225 	else
5226 		i40e_vsi_close(vsi);
5227 }
5228 
5229 /**
5230  * i40e_unquiesce_vsi - Resume a given VSI
5231  * @vsi: the VSI being resumed
5232  **/
5233 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
5234 {
5235 	if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
5236 		return;
5237 
5238 	if (vsi->netdev && netif_running(vsi->netdev))
5239 		vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5240 	else
5241 		i40e_vsi_open(vsi);   /* this clears the DOWN bit */
5242 }
5243 
5244 /**
5245  * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF
5246  * @pf: the PF
5247  **/
5248 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
5249 {
5250 	int v;
5251 
5252 	for (v = 0; v < pf->num_alloc_vsi; v++) {
5253 		if (pf->vsi[v])
5254 			i40e_quiesce_vsi(pf->vsi[v]);
5255 	}
5256 }
5257 
5258 /**
5259  * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF
5260  * @pf: the PF
5261  **/
5262 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
5263 {
5264 	int v;
5265 
5266 	for (v = 0; v < pf->num_alloc_vsi; v++) {
5267 		if (pf->vsi[v])
5268 			i40e_unquiesce_vsi(pf->vsi[v]);
5269 	}
5270 }
5271 
5272 /**
5273  * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled
5274  * @vsi: the VSI being configured
5275  *
5276  * Wait until all queues on a given VSI have been disabled.
5277  **/
5278 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
5279 {
5280 	struct i40e_pf *pf = vsi->back;
5281 	int i, pf_q, ret;
5282 
5283 	pf_q = vsi->base_queue;
5284 	for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
5285 		/* Check and wait for the Tx queue */
5286 		ret = i40e_pf_txq_wait(pf, pf_q, false);
5287 		if (ret) {
5288 			dev_info(&pf->pdev->dev,
5289 				 "VSI seid %d Tx ring %d disable timeout\n",
5290 				 vsi->seid, pf_q);
5291 			return ret;
5292 		}
5293 
5294 		if (!i40e_enabled_xdp_vsi(vsi))
5295 			goto wait_rx;
5296 
5297 		/* Check and wait for the XDP Tx queue */
5298 		ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
5299 				       false);
5300 		if (ret) {
5301 			dev_info(&pf->pdev->dev,
5302 				 "VSI seid %d XDP Tx ring %d disable timeout\n",
5303 				 vsi->seid, pf_q);
5304 			return ret;
5305 		}
5306 wait_rx:
5307 		/* Check and wait for the Rx queue */
5308 		ret = i40e_pf_rxq_wait(pf, pf_q, false);
5309 		if (ret) {
5310 			dev_info(&pf->pdev->dev,
5311 				 "VSI seid %d Rx ring %d disable timeout\n",
5312 				 vsi->seid, pf_q);
5313 			return ret;
5314 		}
5315 	}
5316 
5317 	return 0;
5318 }
5319 
5320 #ifdef CONFIG_I40E_DCB
5321 /**
5322  * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled
5323  * @pf: the PF
5324  *
5325  * This function waits for the queues to be in disabled state for all the
5326  * VSIs that are managed by this PF.
5327  **/
5328 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
5329 {
5330 	int v, ret = 0;
5331 
5332 	for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
5333 		if (pf->vsi[v]) {
5334 			ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
5335 			if (ret)
5336 				break;
5337 		}
5338 	}
5339 
5340 	return ret;
5341 }
5342 
5343 #endif
5344 
5345 /**
5346  * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP
5347  * @pf: pointer to PF
5348  *
5349  * Get TC map for ISCSI PF type that will include iSCSI TC
5350  * and LAN TC.
5351  **/
5352 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
5353 {
5354 	struct i40e_dcb_app_priority_table app;
5355 	struct i40e_hw *hw = &pf->hw;
5356 	u8 enabled_tc = 1; /* TC0 is always enabled */
5357 	u8 tc, i;
5358 	/* Get the iSCSI APP TLV */
5359 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5360 
5361 	for (i = 0; i < dcbcfg->numapps; i++) {
5362 		app = dcbcfg->app[i];
5363 		if (app.selector == I40E_APP_SEL_TCPIP &&
5364 		    app.protocolid == I40E_APP_PROTOID_ISCSI) {
5365 			tc = dcbcfg->etscfg.prioritytable[app.priority];
5366 			enabled_tc |= BIT(tc);
5367 			break;
5368 		}
5369 	}
5370 
5371 	return enabled_tc;
5372 }
5373 
5374 /**
5375  * i40e_dcb_get_num_tc -  Get the number of TCs from DCBx config
5376  * @dcbcfg: the corresponding DCBx configuration structure
5377  *
5378  * Return the number of TCs from given DCBx configuration
5379  **/
5380 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5381 {
5382 	int i, tc_unused = 0;
5383 	u8 num_tc = 0;
5384 	u8 ret = 0;
5385 
5386 	/* Scan the ETS Config Priority Table to find
5387 	 * traffic class enabled for a given priority
5388 	 * and create a bitmask of enabled TCs
5389 	 */
5390 	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5391 		num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5392 
5393 	/* Now scan the bitmask to check for
5394 	 * contiguous TCs starting with TC0
5395 	 */
5396 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5397 		if (num_tc & BIT(i)) {
5398 			if (!tc_unused) {
5399 				ret++;
5400 			} else {
5401 				pr_err("Non-contiguous TC - Disabling DCB\n");
5402 				return 1;
5403 			}
5404 		} else {
5405 			tc_unused = 1;
5406 		}
5407 	}
5408 
5409 	/* There is always at least TC0 */
5410 	if (!ret)
5411 		ret = 1;
5412 
5413 	return ret;
5414 }
5415 
5416 /**
5417  * i40e_dcb_get_enabled_tc - Get enabled traffic classes
5418  * @dcbcfg: the corresponding DCBx configuration structure
5419  *
5420  * Query the current DCB configuration and return the number of
5421  * traffic classes enabled from the given DCBX config
5422  **/
5423 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5424 {
5425 	u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5426 	u8 enabled_tc = 1;
5427 	u8 i;
5428 
5429 	for (i = 0; i < num_tc; i++)
5430 		enabled_tc |= BIT(i);
5431 
5432 	return enabled_tc;
5433 }
5434 
5435 /**
5436  * i40e_mqprio_get_enabled_tc - Get enabled traffic classes
5437  * @pf: PF being queried
5438  *
5439  * Query the current MQPRIO configuration and return the number of
5440  * traffic classes enabled.
5441  **/
5442 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5443 {
5444 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5445 	u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5446 	u8 enabled_tc = 1, i;
5447 
5448 	for (i = 1; i < num_tc; i++)
5449 		enabled_tc |= BIT(i);
5450 	return enabled_tc;
5451 }
5452 
5453 /**
5454  * i40e_pf_get_num_tc - Get enabled traffic classes for PF
5455  * @pf: PF being queried
5456  *
5457  * Return number of traffic classes enabled for the given PF
5458  **/
5459 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5460 {
5461 	struct i40e_hw *hw = &pf->hw;
5462 	u8 i, enabled_tc = 1;
5463 	u8 num_tc = 0;
5464 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5465 
5466 	if (i40e_is_tc_mqprio_enabled(pf))
5467 		return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5468 
5469 	/* If neither MQPRIO nor DCB is enabled, then always use single TC */
5470 	if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5471 		return 1;
5472 
5473 	/* SFP mode will be enabled for all TCs on port */
5474 	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5475 		return i40e_dcb_get_num_tc(dcbcfg);
5476 
5477 	/* MFP mode return count of enabled TCs for this PF */
5478 	if (pf->hw.func_caps.iscsi)
5479 		enabled_tc =  i40e_get_iscsi_tc_map(pf);
5480 	else
5481 		return 1; /* Only TC0 */
5482 
5483 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5484 		if (enabled_tc & BIT(i))
5485 			num_tc++;
5486 	}
5487 	return num_tc;
5488 }
5489 
5490 /**
5491  * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes
5492  * @pf: PF being queried
5493  *
5494  * Return a bitmap for enabled traffic classes for this PF.
5495  **/
5496 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5497 {
5498 	if (i40e_is_tc_mqprio_enabled(pf))
5499 		return i40e_mqprio_get_enabled_tc(pf);
5500 
5501 	/* If neither MQPRIO nor DCB is enabled for this PF then just return
5502 	 * default TC
5503 	 */
5504 	if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5505 		return I40E_DEFAULT_TRAFFIC_CLASS;
5506 
5507 	/* SFP mode we want PF to be enabled for all TCs */
5508 	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5509 		return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5510 
5511 	/* MFP enabled and iSCSI PF type */
5512 	if (pf->hw.func_caps.iscsi)
5513 		return i40e_get_iscsi_tc_map(pf);
5514 	else
5515 		return I40E_DEFAULT_TRAFFIC_CLASS;
5516 }
5517 
5518 /**
5519  * i40e_vsi_get_bw_info - Query VSI BW Information
5520  * @vsi: the VSI being queried
5521  *
5522  * Returns 0 on success, negative value on failure
5523  **/
5524 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5525 {
5526 	struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5527 	struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5528 	struct i40e_pf *pf = vsi->back;
5529 	struct i40e_hw *hw = &pf->hw;
5530 	u32 tc_bw_max;
5531 	int ret;
5532 	int i;
5533 
5534 	/* Get the VSI level BW configuration */
5535 	ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5536 	if (ret) {
5537 		dev_info(&pf->pdev->dev,
5538 			 "couldn't get PF vsi bw config, err %pe aq_err %s\n",
5539 			 ERR_PTR(ret),
5540 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5541 		return -EINVAL;
5542 	}
5543 
5544 	/* Get the VSI level BW configuration per TC */
5545 	ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5546 					       NULL);
5547 	if (ret) {
5548 		dev_info(&pf->pdev->dev,
5549 			 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n",
5550 			 ERR_PTR(ret),
5551 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5552 		return -EINVAL;
5553 	}
5554 
5555 	if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5556 		dev_info(&pf->pdev->dev,
5557 			 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5558 			 bw_config.tc_valid_bits,
5559 			 bw_ets_config.tc_valid_bits);
5560 		/* Still continuing */
5561 	}
5562 
5563 	vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5564 	vsi->bw_max_quanta = bw_config.max_bw;
5565 	tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5566 		    (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5567 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5568 		vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5569 		vsi->bw_ets_limit_credits[i] =
5570 					le16_to_cpu(bw_ets_config.credits[i]);
5571 		/* 3 bits out of 4 for each TC */
5572 		vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5573 	}
5574 
5575 	return 0;
5576 }
5577 
5578 /**
5579  * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC
5580  * @vsi: the VSI being configured
5581  * @enabled_tc: TC bitmap
5582  * @bw_share: BW shared credits per TC
5583  *
5584  * Returns 0 on success, negative value on failure
5585  **/
5586 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5587 				       u8 *bw_share)
5588 {
5589 	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5590 	struct i40e_pf *pf = vsi->back;
5591 	int ret;
5592 	int i;
5593 
5594 	/* There is no need to reset BW when mqprio mode is on.  */
5595 	if (i40e_is_tc_mqprio_enabled(pf))
5596 		return 0;
5597 	if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5598 		ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5599 		if (ret)
5600 			dev_info(&pf->pdev->dev,
5601 				 "Failed to reset tx rate for vsi->seid %u\n",
5602 				 vsi->seid);
5603 		return ret;
5604 	}
5605 	memset(&bw_data, 0, sizeof(bw_data));
5606 	bw_data.tc_valid_bits = enabled_tc;
5607 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5608 		bw_data.tc_bw_credits[i] = bw_share[i];
5609 
5610 	ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5611 	if (ret) {
5612 		dev_info(&pf->pdev->dev,
5613 			 "AQ command Config VSI BW allocation per TC failed = %d\n",
5614 			 pf->hw.aq.asq_last_status);
5615 		return -EINVAL;
5616 	}
5617 
5618 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5619 		vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5620 
5621 	return 0;
5622 }
5623 
5624 /**
5625  * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration
5626  * @vsi: the VSI being configured
5627  * @enabled_tc: TC map to be enabled
5628  *
5629  **/
5630 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5631 {
5632 	struct net_device *netdev = vsi->netdev;
5633 	struct i40e_pf *pf = vsi->back;
5634 	struct i40e_hw *hw = &pf->hw;
5635 	u8 netdev_tc = 0;
5636 	int i;
5637 	struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5638 
5639 	if (!netdev)
5640 		return;
5641 
5642 	if (!enabled_tc) {
5643 		netdev_reset_tc(netdev);
5644 		return;
5645 	}
5646 
5647 	/* Set up actual enabled TCs on the VSI */
5648 	if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5649 		return;
5650 
5651 	/* set per TC queues for the VSI */
5652 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5653 		/* Only set TC queues for enabled tcs
5654 		 *
5655 		 * e.g. For a VSI that has TC0 and TC3 enabled the
5656 		 * enabled_tc bitmap would be 0x00001001; the driver
5657 		 * will set the numtc for netdev as 2 that will be
5658 		 * referenced by the netdev layer as TC 0 and 1.
5659 		 */
5660 		if (vsi->tc_config.enabled_tc & BIT(i))
5661 			netdev_set_tc_queue(netdev,
5662 					vsi->tc_config.tc_info[i].netdev_tc,
5663 					vsi->tc_config.tc_info[i].qcount,
5664 					vsi->tc_config.tc_info[i].qoffset);
5665 	}
5666 
5667 	if (i40e_is_tc_mqprio_enabled(pf))
5668 		return;
5669 
5670 	/* Assign UP2TC map for the VSI */
5671 	for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5672 		/* Get the actual TC# for the UP */
5673 		u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5674 		/* Get the mapped netdev TC# for the UP */
5675 		netdev_tc =  vsi->tc_config.tc_info[ets_tc].netdev_tc;
5676 		netdev_set_prio_tc_map(netdev, i, netdev_tc);
5677 	}
5678 }
5679 
5680 /**
5681  * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map
5682  * @vsi: the VSI being configured
5683  * @ctxt: the ctxt buffer returned from AQ VSI update param command
5684  **/
5685 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5686 				      struct i40e_vsi_context *ctxt)
5687 {
5688 	/* copy just the sections touched not the entire info
5689 	 * since not all sections are valid as returned by
5690 	 * update vsi params
5691 	 */
5692 	vsi->info.mapping_flags = ctxt->info.mapping_flags;
5693 	memcpy(&vsi->info.queue_mapping,
5694 	       &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5695 	memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5696 	       sizeof(vsi->info.tc_mapping));
5697 }
5698 
5699 /**
5700  * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI
5701  * @vsi: the VSI being reconfigured
5702  * @vsi_offset: offset from main VF VSI
5703  */
5704 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset)
5705 {
5706 	struct i40e_vsi_context ctxt = {};
5707 	struct i40e_pf *pf;
5708 	struct i40e_hw *hw;
5709 	int ret;
5710 
5711 	if (!vsi)
5712 		return I40E_ERR_PARAM;
5713 	pf = vsi->back;
5714 	hw = &pf->hw;
5715 
5716 	ctxt.seid = vsi->seid;
5717 	ctxt.pf_num = hw->pf_id;
5718 	ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset;
5719 	ctxt.uplink_seid = vsi->uplink_seid;
5720 	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5721 	ctxt.flags = I40E_AQ_VSI_TYPE_VF;
5722 	ctxt.info = vsi->info;
5723 
5724 	i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc,
5725 				 false);
5726 	if (vsi->reconfig_rss) {
5727 		vsi->rss_size = min_t(int, pf->alloc_rss_size,
5728 				      vsi->num_queue_pairs);
5729 		ret = i40e_vsi_config_rss(vsi);
5730 		if (ret) {
5731 			dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n");
5732 			return ret;
5733 		}
5734 		vsi->reconfig_rss = false;
5735 	}
5736 
5737 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5738 	if (ret) {
5739 		dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n",
5740 			 ERR_PTR(ret),
5741 			 i40e_aq_str(hw, hw->aq.asq_last_status));
5742 		return ret;
5743 	}
5744 	/* update the local VSI info with updated queue map */
5745 	i40e_vsi_update_queue_map(vsi, &ctxt);
5746 	vsi->info.valid_sections = 0;
5747 
5748 	return ret;
5749 }
5750 
5751 /**
5752  * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map
5753  * @vsi: VSI to be configured
5754  * @enabled_tc: TC bitmap
5755  *
5756  * This configures a particular VSI for TCs that are mapped to the
5757  * given TC bitmap. It uses default bandwidth share for TCs across
5758  * VSIs to configure TC for a particular VSI.
5759  *
5760  * NOTE:
5761  * It is expected that the VSI queues have been quisced before calling
5762  * this function.
5763  **/
5764 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5765 {
5766 	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5767 	struct i40e_pf *pf = vsi->back;
5768 	struct i40e_hw *hw = &pf->hw;
5769 	struct i40e_vsi_context ctxt;
5770 	int ret = 0;
5771 	int i;
5772 
5773 	/* Check if enabled_tc is same as existing or new TCs */
5774 	if (vsi->tc_config.enabled_tc == enabled_tc &&
5775 	    vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5776 		return ret;
5777 
5778 	/* Enable ETS TCs with equal BW Share for now across all VSIs */
5779 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5780 		if (enabled_tc & BIT(i))
5781 			bw_share[i] = 1;
5782 	}
5783 
5784 	ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5785 	if (ret) {
5786 		struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5787 
5788 		dev_info(&pf->pdev->dev,
5789 			 "Failed configuring TC map %d for VSI %d\n",
5790 			 enabled_tc, vsi->seid);
5791 		ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5792 						  &bw_config, NULL);
5793 		if (ret) {
5794 			dev_info(&pf->pdev->dev,
5795 				 "Failed querying vsi bw info, err %pe aq_err %s\n",
5796 				 ERR_PTR(ret),
5797 				 i40e_aq_str(hw, hw->aq.asq_last_status));
5798 			goto out;
5799 		}
5800 		if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5801 			u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5802 
5803 			if (!valid_tc)
5804 				valid_tc = bw_config.tc_valid_bits;
5805 			/* Always enable TC0, no matter what */
5806 			valid_tc |= 1;
5807 			dev_info(&pf->pdev->dev,
5808 				 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5809 				 enabled_tc, bw_config.tc_valid_bits, valid_tc);
5810 			enabled_tc = valid_tc;
5811 		}
5812 
5813 		ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5814 		if (ret) {
5815 			dev_err(&pf->pdev->dev,
5816 				"Unable to  configure TC map %d for VSI %d\n",
5817 				enabled_tc, vsi->seid);
5818 			goto out;
5819 		}
5820 	}
5821 
5822 	/* Update Queue Pairs Mapping for currently enabled UPs */
5823 	ctxt.seid = vsi->seid;
5824 	ctxt.pf_num = vsi->back->hw.pf_id;
5825 	ctxt.vf_num = 0;
5826 	ctxt.uplink_seid = vsi->uplink_seid;
5827 	ctxt.info = vsi->info;
5828 	if (i40e_is_tc_mqprio_enabled(pf)) {
5829 		ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5830 		if (ret)
5831 			goto out;
5832 	} else {
5833 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5834 	}
5835 
5836 	/* On destroying the qdisc, reset vsi->rss_size, as number of enabled
5837 	 * queues changed.
5838 	 */
5839 	if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5840 		vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5841 				      vsi->num_queue_pairs);
5842 		ret = i40e_vsi_config_rss(vsi);
5843 		if (ret) {
5844 			dev_info(&vsi->back->pdev->dev,
5845 				 "Failed to reconfig rss for num_queues\n");
5846 			return ret;
5847 		}
5848 		vsi->reconfig_rss = false;
5849 	}
5850 	if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5851 		ctxt.info.valid_sections |=
5852 				cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5853 		ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5854 	}
5855 
5856 	/* Update the VSI after updating the VSI queue-mapping
5857 	 * information
5858 	 */
5859 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5860 	if (ret) {
5861 		dev_info(&pf->pdev->dev,
5862 			 "Update vsi tc config failed, err %pe aq_err %s\n",
5863 			 ERR_PTR(ret),
5864 			 i40e_aq_str(hw, hw->aq.asq_last_status));
5865 		goto out;
5866 	}
5867 	/* update the local VSI info with updated queue map */
5868 	i40e_vsi_update_queue_map(vsi, &ctxt);
5869 	vsi->info.valid_sections = 0;
5870 
5871 	/* Update current VSI BW information */
5872 	ret = i40e_vsi_get_bw_info(vsi);
5873 	if (ret) {
5874 		dev_info(&pf->pdev->dev,
5875 			 "Failed updating vsi bw info, err %pe aq_err %s\n",
5876 			 ERR_PTR(ret),
5877 			 i40e_aq_str(hw, hw->aq.asq_last_status));
5878 		goto out;
5879 	}
5880 
5881 	/* Update the netdev TC setup */
5882 	i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5883 out:
5884 	return ret;
5885 }
5886 
5887 /**
5888  * i40e_get_link_speed - Returns link speed for the interface
5889  * @vsi: VSI to be configured
5890  *
5891  **/
5892 static int i40e_get_link_speed(struct i40e_vsi *vsi)
5893 {
5894 	struct i40e_pf *pf = vsi->back;
5895 
5896 	switch (pf->hw.phy.link_info.link_speed) {
5897 	case I40E_LINK_SPEED_40GB:
5898 		return 40000;
5899 	case I40E_LINK_SPEED_25GB:
5900 		return 25000;
5901 	case I40E_LINK_SPEED_20GB:
5902 		return 20000;
5903 	case I40E_LINK_SPEED_10GB:
5904 		return 10000;
5905 	case I40E_LINK_SPEED_1GB:
5906 		return 1000;
5907 	default:
5908 		return -EINVAL;
5909 	}
5910 }
5911 
5912 /**
5913  * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits
5914  * @vsi: Pointer to vsi structure
5915  * @max_tx_rate: max TX rate in bytes to be converted into Mbits
5916  *
5917  * Helper function to convert units before send to set BW limit
5918  **/
5919 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate)
5920 {
5921 	if (max_tx_rate < I40E_BW_MBPS_DIVISOR) {
5922 		dev_warn(&vsi->back->pdev->dev,
5923 			 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5924 		max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5925 	} else {
5926 		do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
5927 	}
5928 
5929 	return max_tx_rate;
5930 }
5931 
5932 /**
5933  * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
5934  * @vsi: VSI to be configured
5935  * @seid: seid of the channel/VSI
5936  * @max_tx_rate: max TX rate to be configured as BW limit
5937  *
5938  * Helper function to set BW limit for a given VSI
5939  **/
5940 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5941 {
5942 	struct i40e_pf *pf = vsi->back;
5943 	u64 credits = 0;
5944 	int speed = 0;
5945 	int ret = 0;
5946 
5947 	speed = i40e_get_link_speed(vsi);
5948 	if (max_tx_rate > speed) {
5949 		dev_err(&pf->pdev->dev,
5950 			"Invalid max tx rate %llu specified for VSI seid %d.",
5951 			max_tx_rate, seid);
5952 		return -EINVAL;
5953 	}
5954 	if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) {
5955 		dev_warn(&pf->pdev->dev,
5956 			 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5957 		max_tx_rate = I40E_BW_CREDIT_DIVISOR;
5958 	}
5959 
5960 	/* Tx rate credits are in values of 50Mbps, 0 is disabled */
5961 	credits = max_tx_rate;
5962 	do_div(credits, I40E_BW_CREDIT_DIVISOR);
5963 	ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5964 					  I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5965 	if (ret)
5966 		dev_err(&pf->pdev->dev,
5967 			"Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n",
5968 			max_tx_rate, seid, ERR_PTR(ret),
5969 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5970 	return ret;
5971 }
5972 
5973 /**
5974  * i40e_remove_queue_channels - Remove queue channels for the TCs
5975  * @vsi: VSI to be configured
5976  *
5977  * Remove queue channels for the TCs
5978  **/
5979 static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5980 {
5981 	enum i40e_admin_queue_err last_aq_status;
5982 	struct i40e_cloud_filter *cfilter;
5983 	struct i40e_channel *ch, *ch_tmp;
5984 	struct i40e_pf *pf = vsi->back;
5985 	struct hlist_node *node;
5986 	int ret, i;
5987 
5988 	/* Reset rss size that was stored when reconfiguring rss for
5989 	 * channel VSIs with non-power-of-2 queue count.
5990 	 */
5991 	vsi->current_rss_size = 0;
5992 
5993 	/* perform cleanup for channels if they exist */
5994 	if (list_empty(&vsi->ch_list))
5995 		return;
5996 
5997 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5998 		struct i40e_vsi *p_vsi;
5999 
6000 		list_del(&ch->list);
6001 		p_vsi = ch->parent_vsi;
6002 		if (!p_vsi || !ch->initialized) {
6003 			kfree(ch);
6004 			continue;
6005 		}
6006 		/* Reset queue contexts */
6007 		for (i = 0; i < ch->num_queue_pairs; i++) {
6008 			struct i40e_ring *tx_ring, *rx_ring;
6009 			u16 pf_q;
6010 
6011 			pf_q = ch->base_queue + i;
6012 			tx_ring = vsi->tx_rings[pf_q];
6013 			tx_ring->ch = NULL;
6014 
6015 			rx_ring = vsi->rx_rings[pf_q];
6016 			rx_ring->ch = NULL;
6017 		}
6018 
6019 		/* Reset BW configured for this VSI via mqprio */
6020 		ret = i40e_set_bw_limit(vsi, ch->seid, 0);
6021 		if (ret)
6022 			dev_info(&vsi->back->pdev->dev,
6023 				 "Failed to reset tx rate for ch->seid %u\n",
6024 				 ch->seid);
6025 
6026 		/* delete cloud filters associated with this channel */
6027 		hlist_for_each_entry_safe(cfilter, node,
6028 					  &pf->cloud_filter_list, cloud_node) {
6029 			if (cfilter->seid != ch->seid)
6030 				continue;
6031 
6032 			hash_del(&cfilter->cloud_node);
6033 			if (cfilter->dst_port)
6034 				ret = i40e_add_del_cloud_filter_big_buf(vsi,
6035 									cfilter,
6036 									false);
6037 			else
6038 				ret = i40e_add_del_cloud_filter(vsi, cfilter,
6039 								false);
6040 			last_aq_status = pf->hw.aq.asq_last_status;
6041 			if (ret)
6042 				dev_info(&pf->pdev->dev,
6043 					 "Failed to delete cloud filter, err %pe aq_err %s\n",
6044 					 ERR_PTR(ret),
6045 					 i40e_aq_str(&pf->hw, last_aq_status));
6046 			kfree(cfilter);
6047 		}
6048 
6049 		/* delete VSI from FW */
6050 		ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
6051 					     NULL);
6052 		if (ret)
6053 			dev_err(&vsi->back->pdev->dev,
6054 				"unable to remove channel (%d) for parent VSI(%d)\n",
6055 				ch->seid, p_vsi->seid);
6056 		kfree(ch);
6057 	}
6058 	INIT_LIST_HEAD(&vsi->ch_list);
6059 }
6060 
6061 /**
6062  * i40e_get_max_queues_for_channel
6063  * @vsi: ptr to VSI to which channels are associated with
6064  *
6065  * Helper function which returns max value among the queue counts set on the
6066  * channels/TCs created.
6067  **/
6068 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
6069 {
6070 	struct i40e_channel *ch, *ch_tmp;
6071 	int max = 0;
6072 
6073 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
6074 		if (!ch->initialized)
6075 			continue;
6076 		if (ch->num_queue_pairs > max)
6077 			max = ch->num_queue_pairs;
6078 	}
6079 
6080 	return max;
6081 }
6082 
6083 /**
6084  * i40e_validate_num_queues - validate num_queues w.r.t channel
6085  * @pf: ptr to PF device
6086  * @num_queues: number of queues
6087  * @vsi: the parent VSI
6088  * @reconfig_rss: indicates should the RSS be reconfigured or not
6089  *
6090  * This function validates number of queues in the context of new channel
6091  * which is being established and determines if RSS should be reconfigured
6092  * or not for parent VSI.
6093  **/
6094 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
6095 				    struct i40e_vsi *vsi, bool *reconfig_rss)
6096 {
6097 	int max_ch_queues;
6098 
6099 	if (!reconfig_rss)
6100 		return -EINVAL;
6101 
6102 	*reconfig_rss = false;
6103 	if (vsi->current_rss_size) {
6104 		if (num_queues > vsi->current_rss_size) {
6105 			dev_dbg(&pf->pdev->dev,
6106 				"Error: num_queues (%d) > vsi's current_size(%d)\n",
6107 				num_queues, vsi->current_rss_size);
6108 			return -EINVAL;
6109 		} else if ((num_queues < vsi->current_rss_size) &&
6110 			   (!is_power_of_2(num_queues))) {
6111 			dev_dbg(&pf->pdev->dev,
6112 				"Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
6113 				num_queues, vsi->current_rss_size);
6114 			return -EINVAL;
6115 		}
6116 	}
6117 
6118 	if (!is_power_of_2(num_queues)) {
6119 		/* Find the max num_queues configured for channel if channel
6120 		 * exist.
6121 		 * if channel exist, then enforce 'num_queues' to be more than
6122 		 * max ever queues configured for channel.
6123 		 */
6124 		max_ch_queues = i40e_get_max_queues_for_channel(vsi);
6125 		if (num_queues < max_ch_queues) {
6126 			dev_dbg(&pf->pdev->dev,
6127 				"Error: num_queues (%d) < max queues configured for channel(%d)\n",
6128 				num_queues, max_ch_queues);
6129 			return -EINVAL;
6130 		}
6131 		*reconfig_rss = true;
6132 	}
6133 
6134 	return 0;
6135 }
6136 
6137 /**
6138  * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size
6139  * @vsi: the VSI being setup
6140  * @rss_size: size of RSS, accordingly LUT gets reprogrammed
6141  *
6142  * This function reconfigures RSS by reprogramming LUTs using 'rss_size'
6143  **/
6144 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
6145 {
6146 	struct i40e_pf *pf = vsi->back;
6147 	u8 seed[I40E_HKEY_ARRAY_SIZE];
6148 	struct i40e_hw *hw = &pf->hw;
6149 	int local_rss_size;
6150 	u8 *lut;
6151 	int ret;
6152 
6153 	if (!vsi->rss_size)
6154 		return -EINVAL;
6155 
6156 	if (rss_size > vsi->rss_size)
6157 		return -EINVAL;
6158 
6159 	local_rss_size = min_t(int, vsi->rss_size, rss_size);
6160 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
6161 	if (!lut)
6162 		return -ENOMEM;
6163 
6164 	/* Ignoring user configured lut if there is one */
6165 	i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
6166 
6167 	/* Use user configured hash key if there is one, otherwise
6168 	 * use default.
6169 	 */
6170 	if (vsi->rss_hkey_user)
6171 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
6172 	else
6173 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
6174 
6175 	ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
6176 	if (ret) {
6177 		dev_info(&pf->pdev->dev,
6178 			 "Cannot set RSS lut, err %pe aq_err %s\n",
6179 			 ERR_PTR(ret),
6180 			 i40e_aq_str(hw, hw->aq.asq_last_status));
6181 		kfree(lut);
6182 		return ret;
6183 	}
6184 	kfree(lut);
6185 
6186 	/* Do the update w.r.t. storing rss_size */
6187 	if (!vsi->orig_rss_size)
6188 		vsi->orig_rss_size = vsi->rss_size;
6189 	vsi->current_rss_size = local_rss_size;
6190 
6191 	return ret;
6192 }
6193 
6194 /**
6195  * i40e_channel_setup_queue_map - Setup a channel queue map
6196  * @pf: ptr to PF device
6197  * @ctxt: VSI context structure
6198  * @ch: ptr to channel structure
6199  *
6200  * Setup queue map for a specific channel
6201  **/
6202 static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
6203 					 struct i40e_vsi_context *ctxt,
6204 					 struct i40e_channel *ch)
6205 {
6206 	u16 qcount, qmap, sections = 0;
6207 	u8 offset = 0;
6208 	int pow;
6209 
6210 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
6211 	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
6212 
6213 	qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
6214 	ch->num_queue_pairs = qcount;
6215 
6216 	/* find the next higher power-of-2 of num queue pairs */
6217 	pow = ilog2(qcount);
6218 	if (!is_power_of_2(qcount))
6219 		pow++;
6220 
6221 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
6222 		(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
6223 
6224 	/* Setup queue TC[0].qmap for given VSI context */
6225 	ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
6226 
6227 	ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */
6228 	ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
6229 	ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
6230 	ctxt->info.valid_sections |= cpu_to_le16(sections);
6231 }
6232 
6233 /**
6234  * i40e_add_channel - add a channel by adding VSI
6235  * @pf: ptr to PF device
6236  * @uplink_seid: underlying HW switching element (VEB) ID
6237  * @ch: ptr to channel structure
6238  *
6239  * Add a channel (VSI) using add_vsi and queue_map
6240  **/
6241 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
6242 			    struct i40e_channel *ch)
6243 {
6244 	struct i40e_hw *hw = &pf->hw;
6245 	struct i40e_vsi_context ctxt;
6246 	u8 enabled_tc = 0x1; /* TC0 enabled */
6247 	int ret;
6248 
6249 	if (ch->type != I40E_VSI_VMDQ2) {
6250 		dev_info(&pf->pdev->dev,
6251 			 "add new vsi failed, ch->type %d\n", ch->type);
6252 		return -EINVAL;
6253 	}
6254 
6255 	memset(&ctxt, 0, sizeof(ctxt));
6256 	ctxt.pf_num = hw->pf_id;
6257 	ctxt.vf_num = 0;
6258 	ctxt.uplink_seid = uplink_seid;
6259 	ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
6260 	if (ch->type == I40E_VSI_VMDQ2)
6261 		ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
6262 
6263 	if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
6264 		ctxt.info.valid_sections |=
6265 		     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
6266 		ctxt.info.switch_id =
6267 		   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
6268 	}
6269 
6270 	/* Set queue map for a given VSI context */
6271 	i40e_channel_setup_queue_map(pf, &ctxt, ch);
6272 
6273 	/* Now time to create VSI */
6274 	ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
6275 	if (ret) {
6276 		dev_info(&pf->pdev->dev,
6277 			 "add new vsi failed, err %pe aq_err %s\n",
6278 			 ERR_PTR(ret),
6279 			 i40e_aq_str(&pf->hw,
6280 				     pf->hw.aq.asq_last_status));
6281 		return -ENOENT;
6282 	}
6283 
6284 	/* Success, update channel, set enabled_tc only if the channel
6285 	 * is not a macvlan
6286 	 */
6287 	ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
6288 	ch->seid = ctxt.seid;
6289 	ch->vsi_number = ctxt.vsi_number;
6290 	ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx);
6291 
6292 	/* copy just the sections touched not the entire info
6293 	 * since not all sections are valid as returned by
6294 	 * update vsi params
6295 	 */
6296 	ch->info.mapping_flags = ctxt.info.mapping_flags;
6297 	memcpy(&ch->info.queue_mapping,
6298 	       &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
6299 	memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
6300 	       sizeof(ctxt.info.tc_mapping));
6301 
6302 	return 0;
6303 }
6304 
6305 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
6306 				  u8 *bw_share)
6307 {
6308 	struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
6309 	int ret;
6310 	int i;
6311 
6312 	memset(&bw_data, 0, sizeof(bw_data));
6313 	bw_data.tc_valid_bits = ch->enabled_tc;
6314 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6315 		bw_data.tc_bw_credits[i] = bw_share[i];
6316 
6317 	ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
6318 				       &bw_data, NULL);
6319 	if (ret) {
6320 		dev_info(&vsi->back->pdev->dev,
6321 			 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
6322 			 vsi->back->hw.aq.asq_last_status, ch->seid);
6323 		return -EINVAL;
6324 	}
6325 
6326 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
6327 		ch->info.qs_handle[i] = bw_data.qs_handles[i];
6328 
6329 	return 0;
6330 }
6331 
6332 /**
6333  * i40e_channel_config_tx_ring - config TX ring associated with new channel
6334  * @pf: ptr to PF device
6335  * @vsi: the VSI being setup
6336  * @ch: ptr to channel structure
6337  *
6338  * Configure TX rings associated with channel (VSI) since queues are being
6339  * from parent VSI.
6340  **/
6341 static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
6342 				       struct i40e_vsi *vsi,
6343 				       struct i40e_channel *ch)
6344 {
6345 	u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
6346 	int ret;
6347 	int i;
6348 
6349 	/* Enable ETS TCs with equal BW Share for now across all VSIs */
6350 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6351 		if (ch->enabled_tc & BIT(i))
6352 			bw_share[i] = 1;
6353 	}
6354 
6355 	/* configure BW for new VSI */
6356 	ret = i40e_channel_config_bw(vsi, ch, bw_share);
6357 	if (ret) {
6358 		dev_info(&vsi->back->pdev->dev,
6359 			 "Failed configuring TC map %d for channel (seid %u)\n",
6360 			 ch->enabled_tc, ch->seid);
6361 		return ret;
6362 	}
6363 
6364 	for (i = 0; i < ch->num_queue_pairs; i++) {
6365 		struct i40e_ring *tx_ring, *rx_ring;
6366 		u16 pf_q;
6367 
6368 		pf_q = ch->base_queue + i;
6369 
6370 		/* Get to TX ring ptr of main VSI, for re-setup TX queue
6371 		 * context
6372 		 */
6373 		tx_ring = vsi->tx_rings[pf_q];
6374 		tx_ring->ch = ch;
6375 
6376 		/* Get the RX ring ptr */
6377 		rx_ring = vsi->rx_rings[pf_q];
6378 		rx_ring->ch = ch;
6379 	}
6380 
6381 	return 0;
6382 }
6383 
6384 /**
6385  * i40e_setup_hw_channel - setup new channel
6386  * @pf: ptr to PF device
6387  * @vsi: the VSI being setup
6388  * @ch: ptr to channel structure
6389  * @uplink_seid: underlying HW switching element (VEB) ID
6390  * @type: type of channel to be created (VMDq2/VF)
6391  *
6392  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6393  * and configures TX rings accordingly
6394  **/
6395 static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
6396 					struct i40e_vsi *vsi,
6397 					struct i40e_channel *ch,
6398 					u16 uplink_seid, u8 type)
6399 {
6400 	int ret;
6401 
6402 	ch->initialized = false;
6403 	ch->base_queue = vsi->next_base_queue;
6404 	ch->type = type;
6405 
6406 	/* Proceed with creation of channel (VMDq2) VSI */
6407 	ret = i40e_add_channel(pf, uplink_seid, ch);
6408 	if (ret) {
6409 		dev_info(&pf->pdev->dev,
6410 			 "failed to add_channel using uplink_seid %u\n",
6411 			 uplink_seid);
6412 		return ret;
6413 	}
6414 
6415 	/* Mark the successful creation of channel */
6416 	ch->initialized = true;
6417 
6418 	/* Reconfigure TX queues using QTX_CTL register */
6419 	ret = i40e_channel_config_tx_ring(pf, vsi, ch);
6420 	if (ret) {
6421 		dev_info(&pf->pdev->dev,
6422 			 "failed to configure TX rings for channel %u\n",
6423 			 ch->seid);
6424 		return ret;
6425 	}
6426 
6427 	/* update 'next_base_queue' */
6428 	vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6429 	dev_dbg(&pf->pdev->dev,
6430 		"Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6431 		ch->seid, ch->vsi_number, ch->stat_counter_idx,
6432 		ch->num_queue_pairs,
6433 		vsi->next_base_queue);
6434 	return ret;
6435 }
6436 
6437 /**
6438  * i40e_setup_channel - setup new channel using uplink element
6439  * @pf: ptr to PF device
6440  * @vsi: pointer to the VSI to set up the channel within
6441  * @ch: ptr to channel structure
6442  *
6443  * Setup new channel (VSI) based on specified type (VMDq2/VF)
6444  * and uplink switching element (uplink_seid)
6445  **/
6446 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6447 			       struct i40e_channel *ch)
6448 {
6449 	u8 vsi_type;
6450 	u16 seid;
6451 	int ret;
6452 
6453 	if (vsi->type == I40E_VSI_MAIN) {
6454 		vsi_type = I40E_VSI_VMDQ2;
6455 	} else {
6456 		dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6457 			vsi->type);
6458 		return false;
6459 	}
6460 
6461 	/* underlying switching element */
6462 	seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6463 
6464 	/* create channel (VSI), configure TX rings */
6465 	ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6466 	if (ret) {
6467 		dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6468 		return false;
6469 	}
6470 
6471 	return ch->initialized ? true : false;
6472 }
6473 
6474 /**
6475  * i40e_validate_and_set_switch_mode - sets up switch mode correctly
6476  * @vsi: ptr to VSI which has PF backing
6477  *
6478  * Sets up switch mode correctly if it needs to be changed and perform
6479  * what are allowed modes.
6480  **/
6481 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6482 {
6483 	u8 mode;
6484 	struct i40e_pf *pf = vsi->back;
6485 	struct i40e_hw *hw = &pf->hw;
6486 	int ret;
6487 
6488 	ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6489 	if (ret)
6490 		return -EINVAL;
6491 
6492 	if (hw->dev_caps.switch_mode) {
6493 		/* if switch mode is set, support mode2 (non-tunneled for
6494 		 * cloud filter) for now
6495 		 */
6496 		u32 switch_mode = hw->dev_caps.switch_mode &
6497 				  I40E_SWITCH_MODE_MASK;
6498 		if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6499 			if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6500 				return 0;
6501 			dev_err(&pf->pdev->dev,
6502 				"Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6503 				hw->dev_caps.switch_mode);
6504 			return -EINVAL;
6505 		}
6506 	}
6507 
6508 	/* Set Bit 7 to be valid */
6509 	mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6510 
6511 	/* Set L4type for TCP support */
6512 	mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6513 
6514 	/* Set cloud filter mode */
6515 	mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6516 
6517 	/* Prep mode field for set_switch_config */
6518 	ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6519 					pf->last_sw_conf_valid_flags,
6520 					mode, NULL);
6521 	if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6522 		dev_err(&pf->pdev->dev,
6523 			"couldn't set switch config bits, err %pe aq_err %s\n",
6524 			ERR_PTR(ret),
6525 			i40e_aq_str(hw,
6526 				    hw->aq.asq_last_status));
6527 
6528 	return ret;
6529 }
6530 
6531 /**
6532  * i40e_create_queue_channel - function to create channel
6533  * @vsi: VSI to be configured
6534  * @ch: ptr to channel (it contains channel specific params)
6535  *
6536  * This function creates channel (VSI) using num_queues specified by user,
6537  * reconfigs RSS if needed.
6538  **/
6539 int i40e_create_queue_channel(struct i40e_vsi *vsi,
6540 			      struct i40e_channel *ch)
6541 {
6542 	struct i40e_pf *pf = vsi->back;
6543 	bool reconfig_rss;
6544 	int err;
6545 
6546 	if (!ch)
6547 		return -EINVAL;
6548 
6549 	if (!ch->num_queue_pairs) {
6550 		dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6551 			ch->num_queue_pairs);
6552 		return -EINVAL;
6553 	}
6554 
6555 	/* validate user requested num_queues for channel */
6556 	err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6557 				       &reconfig_rss);
6558 	if (err) {
6559 		dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6560 			 ch->num_queue_pairs);
6561 		return -EINVAL;
6562 	}
6563 
6564 	/* By default we are in VEPA mode, if this is the first VF/VMDq
6565 	 * VSI to be added switch to VEB mode.
6566 	 */
6567 
6568 	if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6569 		pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6570 
6571 		if (vsi->type == I40E_VSI_MAIN) {
6572 			if (i40e_is_tc_mqprio_enabled(pf))
6573 				i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
6574 			else
6575 				i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG);
6576 		}
6577 		/* now onwards for main VSI, number of queues will be value
6578 		 * of TC0's queue count
6579 		 */
6580 	}
6581 
6582 	/* By this time, vsi->cnt_q_avail shall be set to non-zero and
6583 	 * it should be more than num_queues
6584 	 */
6585 	if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6586 		dev_dbg(&pf->pdev->dev,
6587 			"Error: cnt_q_avail (%u) less than num_queues %d\n",
6588 			vsi->cnt_q_avail, ch->num_queue_pairs);
6589 		return -EINVAL;
6590 	}
6591 
6592 	/* reconfig_rss only if vsi type is MAIN_VSI */
6593 	if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6594 		err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6595 		if (err) {
6596 			dev_info(&pf->pdev->dev,
6597 				 "Error: unable to reconfig rss for num_queues (%u)\n",
6598 				 ch->num_queue_pairs);
6599 			return -EINVAL;
6600 		}
6601 	}
6602 
6603 	if (!i40e_setup_channel(pf, vsi, ch)) {
6604 		dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6605 		return -EINVAL;
6606 	}
6607 
6608 	dev_info(&pf->pdev->dev,
6609 		 "Setup channel (id:%u) utilizing num_queues %d\n",
6610 		 ch->seid, ch->num_queue_pairs);
6611 
6612 	/* configure VSI for BW limit */
6613 	if (ch->max_tx_rate) {
6614 		u64 credits = ch->max_tx_rate;
6615 
6616 		if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6617 			return -EINVAL;
6618 
6619 		do_div(credits, I40E_BW_CREDIT_DIVISOR);
6620 		dev_dbg(&pf->pdev->dev,
6621 			"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6622 			ch->max_tx_rate,
6623 			credits,
6624 			ch->seid);
6625 	}
6626 
6627 	/* in case of VF, this will be main SRIOV VSI */
6628 	ch->parent_vsi = vsi;
6629 
6630 	/* and update main_vsi's count for queue_available to use */
6631 	vsi->cnt_q_avail -= ch->num_queue_pairs;
6632 
6633 	return 0;
6634 }
6635 
6636 /**
6637  * i40e_configure_queue_channels - Add queue channel for the given TCs
6638  * @vsi: VSI to be configured
6639  *
6640  * Configures queue channel mapping to the given TCs
6641  **/
6642 static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6643 {
6644 	struct i40e_channel *ch;
6645 	u64 max_rate = 0;
6646 	int ret = 0, i;
6647 
6648 	/* Create app vsi with the TCs. Main VSI with TC0 is already set up */
6649 	vsi->tc_seid_map[0] = vsi->seid;
6650 	for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6651 		if (vsi->tc_config.enabled_tc & BIT(i)) {
6652 			ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6653 			if (!ch) {
6654 				ret = -ENOMEM;
6655 				goto err_free;
6656 			}
6657 
6658 			INIT_LIST_HEAD(&ch->list);
6659 			ch->num_queue_pairs =
6660 				vsi->tc_config.tc_info[i].qcount;
6661 			ch->base_queue =
6662 				vsi->tc_config.tc_info[i].qoffset;
6663 
6664 			/* Bandwidth limit through tc interface is in bytes/s,
6665 			 * change to Mbit/s
6666 			 */
6667 			max_rate = vsi->mqprio_qopt.max_rate[i];
6668 			do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6669 			ch->max_tx_rate = max_rate;
6670 
6671 			list_add_tail(&ch->list, &vsi->ch_list);
6672 
6673 			ret = i40e_create_queue_channel(vsi, ch);
6674 			if (ret) {
6675 				dev_err(&vsi->back->pdev->dev,
6676 					"Failed creating queue channel with TC%d: queues %d\n",
6677 					i, ch->num_queue_pairs);
6678 				goto err_free;
6679 			}
6680 			vsi->tc_seid_map[i] = ch->seid;
6681 		}
6682 	}
6683 
6684 	/* reset to reconfigure TX queue contexts */
6685 	i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true);
6686 	return ret;
6687 
6688 err_free:
6689 	i40e_remove_queue_channels(vsi);
6690 	return ret;
6691 }
6692 
6693 /**
6694  * i40e_veb_config_tc - Configure TCs for given VEB
6695  * @veb: given VEB
6696  * @enabled_tc: TC bitmap
6697  *
6698  * Configures given TC bitmap for VEB (switching) element
6699  **/
6700 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6701 {
6702 	struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6703 	struct i40e_pf *pf = veb->pf;
6704 	int ret = 0;
6705 	int i;
6706 
6707 	/* No TCs or already enabled TCs just return */
6708 	if (!enabled_tc || veb->enabled_tc == enabled_tc)
6709 		return ret;
6710 
6711 	bw_data.tc_valid_bits = enabled_tc;
6712 	/* bw_data.absolute_credits is not set (relative) */
6713 
6714 	/* Enable ETS TCs with equal BW Share for now */
6715 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6716 		if (enabled_tc & BIT(i))
6717 			bw_data.tc_bw_share_credits[i] = 1;
6718 	}
6719 
6720 	ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6721 						   &bw_data, NULL);
6722 	if (ret) {
6723 		dev_info(&pf->pdev->dev,
6724 			 "VEB bw config failed, err %pe aq_err %s\n",
6725 			 ERR_PTR(ret),
6726 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6727 		goto out;
6728 	}
6729 
6730 	/* Update the BW information */
6731 	ret = i40e_veb_get_bw_info(veb);
6732 	if (ret) {
6733 		dev_info(&pf->pdev->dev,
6734 			 "Failed getting veb bw config, err %pe aq_err %s\n",
6735 			 ERR_PTR(ret),
6736 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6737 	}
6738 
6739 out:
6740 	return ret;
6741 }
6742 
6743 #ifdef CONFIG_I40E_DCB
6744 /**
6745  * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs
6746  * @pf: PF struct
6747  *
6748  * Reconfigure VEB/VSIs on a given PF; it is assumed that
6749  * the caller would've quiesce all the VSIs before calling
6750  * this function
6751  **/
6752 static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6753 {
6754 	u8 tc_map = 0;
6755 	int ret;
6756 	u8 v;
6757 
6758 	/* Enable the TCs available on PF to all VEBs */
6759 	tc_map = i40e_pf_get_tc_map(pf);
6760 	if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS)
6761 		return;
6762 
6763 	for (v = 0; v < I40E_MAX_VEB; v++) {
6764 		if (!pf->veb[v])
6765 			continue;
6766 		ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6767 		if (ret) {
6768 			dev_info(&pf->pdev->dev,
6769 				 "Failed configuring TC for VEB seid=%d\n",
6770 				 pf->veb[v]->seid);
6771 			/* Will try to configure as many components */
6772 		}
6773 	}
6774 
6775 	/* Update each VSI */
6776 	for (v = 0; v < pf->num_alloc_vsi; v++) {
6777 		if (!pf->vsi[v])
6778 			continue;
6779 
6780 		/* - Enable all TCs for the LAN VSI
6781 		 * - For all others keep them at TC0 for now
6782 		 */
6783 		if (v == pf->lan_vsi)
6784 			tc_map = i40e_pf_get_tc_map(pf);
6785 		else
6786 			tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6787 
6788 		ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6789 		if (ret) {
6790 			dev_info(&pf->pdev->dev,
6791 				 "Failed configuring TC for VSI seid=%d\n",
6792 				 pf->vsi[v]->seid);
6793 			/* Will try to configure as many components */
6794 		} else {
6795 			/* Re-configure VSI vectors based on updated TC map */
6796 			i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6797 			if (pf->vsi[v]->netdev)
6798 				i40e_dcbnl_set_all(pf->vsi[v]);
6799 		}
6800 	}
6801 }
6802 
6803 /**
6804  * i40e_resume_port_tx - Resume port Tx
6805  * @pf: PF struct
6806  *
6807  * Resume a port's Tx and issue a PF reset in case of failure to
6808  * resume.
6809  **/
6810 static int i40e_resume_port_tx(struct i40e_pf *pf)
6811 {
6812 	struct i40e_hw *hw = &pf->hw;
6813 	int ret;
6814 
6815 	ret = i40e_aq_resume_port_tx(hw, NULL);
6816 	if (ret) {
6817 		dev_info(&pf->pdev->dev,
6818 			 "Resume Port Tx failed, err %pe aq_err %s\n",
6819 			  ERR_PTR(ret),
6820 			  i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6821 		/* Schedule PF reset to recover */
6822 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6823 		i40e_service_event_schedule(pf);
6824 	}
6825 
6826 	return ret;
6827 }
6828 
6829 /**
6830  * i40e_suspend_port_tx - Suspend port Tx
6831  * @pf: PF struct
6832  *
6833  * Suspend a port's Tx and issue a PF reset in case of failure.
6834  **/
6835 static int i40e_suspend_port_tx(struct i40e_pf *pf)
6836 {
6837 	struct i40e_hw *hw = &pf->hw;
6838 	int ret;
6839 
6840 	ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL);
6841 	if (ret) {
6842 		dev_info(&pf->pdev->dev,
6843 			 "Suspend Port Tx failed, err %pe aq_err %s\n",
6844 			 ERR_PTR(ret),
6845 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6846 		/* Schedule PF reset to recover */
6847 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6848 		i40e_service_event_schedule(pf);
6849 	}
6850 
6851 	return ret;
6852 }
6853 
6854 /**
6855  * i40e_hw_set_dcb_config - Program new DCBX settings into HW
6856  * @pf: PF being configured
6857  * @new_cfg: New DCBX configuration
6858  *
6859  * Program DCB settings into HW and reconfigure VEB/VSIs on
6860  * given PF. Uses "Set LLDP MIB" AQC to program the hardware.
6861  **/
6862 static int i40e_hw_set_dcb_config(struct i40e_pf *pf,
6863 				  struct i40e_dcbx_config *new_cfg)
6864 {
6865 	struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config;
6866 	int ret;
6867 
6868 	/* Check if need reconfiguration */
6869 	if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) {
6870 		dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n");
6871 		return 0;
6872 	}
6873 
6874 	/* Config change disable all VSIs */
6875 	i40e_pf_quiesce_all_vsi(pf);
6876 
6877 	/* Copy the new config to the current config */
6878 	*old_cfg = *new_cfg;
6879 	old_cfg->etsrec = old_cfg->etscfg;
6880 	ret = i40e_set_dcb_config(&pf->hw);
6881 	if (ret) {
6882 		dev_info(&pf->pdev->dev,
6883 			 "Set DCB Config failed, err %pe aq_err %s\n",
6884 			 ERR_PTR(ret),
6885 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6886 		goto out;
6887 	}
6888 
6889 	/* Changes in configuration update VEB/VSI */
6890 	i40e_dcb_reconfigure(pf);
6891 out:
6892 	/* In case of reset do not try to resume anything */
6893 	if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) {
6894 		/* Re-start the VSIs if disabled */
6895 		ret = i40e_resume_port_tx(pf);
6896 		/* In case of error no point in resuming VSIs */
6897 		if (ret)
6898 			goto err;
6899 		i40e_pf_unquiesce_all_vsi(pf);
6900 	}
6901 err:
6902 	return ret;
6903 }
6904 
6905 /**
6906  * i40e_hw_dcb_config - Program new DCBX settings into HW
6907  * @pf: PF being configured
6908  * @new_cfg: New DCBX configuration
6909  *
6910  * Program DCB settings into HW and reconfigure VEB/VSIs on
6911  * given PF
6912  **/
6913 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg)
6914 {
6915 	struct i40e_aqc_configure_switching_comp_ets_data ets_data;
6916 	u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0};
6917 	u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS];
6918 	struct i40e_dcbx_config *old_cfg;
6919 	u8 mode[I40E_MAX_TRAFFIC_CLASS];
6920 	struct i40e_rx_pb_config pb_cfg;
6921 	struct i40e_hw *hw = &pf->hw;
6922 	u8 num_ports = hw->num_ports;
6923 	bool need_reconfig;
6924 	int ret = -EINVAL;
6925 	u8 lltc_map = 0;
6926 	u8 tc_map = 0;
6927 	u8 new_numtc;
6928 	u8 i;
6929 
6930 	dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n");
6931 	/* Un-pack information to Program ETS HW via shared API
6932 	 * numtc, tcmap
6933 	 * LLTC map
6934 	 * ETS/NON-ETS arbiter mode
6935 	 * max exponent (credit refills)
6936 	 * Total number of ports
6937 	 * PFC priority bit-map
6938 	 * Priority Table
6939 	 * BW % per TC
6940 	 * Arbiter mode between UPs sharing same TC
6941 	 * TSA table (ETS or non-ETS)
6942 	 * EEE enabled or not
6943 	 * MFS TC table
6944 	 */
6945 
6946 	new_numtc = i40e_dcb_get_num_tc(new_cfg);
6947 
6948 	memset(&ets_data, 0, sizeof(ets_data));
6949 	for (i = 0; i < new_numtc; i++) {
6950 		tc_map |= BIT(i);
6951 		switch (new_cfg->etscfg.tsatable[i]) {
6952 		case I40E_IEEE_TSA_ETS:
6953 			prio_type[i] = I40E_DCB_PRIO_TYPE_ETS;
6954 			ets_data.tc_bw_share_credits[i] =
6955 					new_cfg->etscfg.tcbwtable[i];
6956 			break;
6957 		case I40E_IEEE_TSA_STRICT:
6958 			prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT;
6959 			lltc_map |= BIT(i);
6960 			ets_data.tc_bw_share_credits[i] =
6961 					I40E_DCB_STRICT_PRIO_CREDITS;
6962 			break;
6963 		default:
6964 			/* Invalid TSA type */
6965 			need_reconfig = false;
6966 			goto out;
6967 		}
6968 	}
6969 
6970 	old_cfg = &hw->local_dcbx_config;
6971 	/* Check if need reconfiguration */
6972 	need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg);
6973 
6974 	/* If needed, enable/disable frame tagging, disable all VSIs
6975 	 * and suspend port tx
6976 	 */
6977 	if (need_reconfig) {
6978 		/* Enable DCB tagging only when more than one TC */
6979 		if (new_numtc > 1)
6980 			pf->flags |= I40E_FLAG_DCB_ENABLED;
6981 		else
6982 			pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6983 
6984 		set_bit(__I40E_PORT_SUSPENDED, pf->state);
6985 		/* Reconfiguration needed quiesce all VSIs */
6986 		i40e_pf_quiesce_all_vsi(pf);
6987 		ret = i40e_suspend_port_tx(pf);
6988 		if (ret)
6989 			goto err;
6990 	}
6991 
6992 	/* Configure Port ETS Tx Scheduler */
6993 	ets_data.tc_valid_bits = tc_map;
6994 	ets_data.tc_strict_priority_flags = lltc_map;
6995 	ret = i40e_aq_config_switch_comp_ets
6996 		(hw, pf->mac_seid, &ets_data,
6997 		 i40e_aqc_opc_modify_switching_comp_ets, NULL);
6998 	if (ret) {
6999 		dev_info(&pf->pdev->dev,
7000 			 "Modify Port ETS failed, err %pe aq_err %s\n",
7001 			 ERR_PTR(ret),
7002 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7003 		goto out;
7004 	}
7005 
7006 	/* Configure Rx ETS HW */
7007 	memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode));
7008 	i40e_dcb_hw_set_num_tc(hw, new_numtc);
7009 	i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN,
7010 				   I40E_DCB_ARB_MODE_STRICT_PRIORITY,
7011 				   I40E_DCB_DEFAULT_MAX_EXPONENT,
7012 				   lltc_map);
7013 	i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports);
7014 	i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode,
7015 				     prio_type);
7016 	i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable,
7017 			       new_cfg->etscfg.prioritytable);
7018 	i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable);
7019 
7020 	/* Configure Rx Packet Buffers in HW */
7021 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7022 		mfs_tc[i] = pf->vsi[pf->lan_vsi]->netdev->mtu;
7023 		mfs_tc[i] += I40E_PACKET_HDR_PAD;
7024 	}
7025 
7026 	i40e_dcb_hw_calculate_pool_sizes(hw, num_ports,
7027 					 false, new_cfg->pfc.pfcenable,
7028 					 mfs_tc, &pb_cfg);
7029 	i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg);
7030 
7031 	/* Update the local Rx Packet buffer config */
7032 	pf->pb_cfg = pb_cfg;
7033 
7034 	/* Inform the FW about changes to DCB configuration */
7035 	ret = i40e_aq_dcb_updated(&pf->hw, NULL);
7036 	if (ret) {
7037 		dev_info(&pf->pdev->dev,
7038 			 "DCB Updated failed, err %pe aq_err %s\n",
7039 			 ERR_PTR(ret),
7040 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7041 		goto out;
7042 	}
7043 
7044 	/* Update the port DCBx configuration */
7045 	*old_cfg = *new_cfg;
7046 
7047 	/* Changes in configuration update VEB/VSI */
7048 	i40e_dcb_reconfigure(pf);
7049 out:
7050 	/* Re-start the VSIs if disabled */
7051 	if (need_reconfig) {
7052 		ret = i40e_resume_port_tx(pf);
7053 
7054 		clear_bit(__I40E_PORT_SUSPENDED, pf->state);
7055 		/* In case of error no point in resuming VSIs */
7056 		if (ret)
7057 			goto err;
7058 
7059 		/* Wait for the PF's queues to be disabled */
7060 		ret = i40e_pf_wait_queues_disabled(pf);
7061 		if (ret) {
7062 			/* Schedule PF reset to recover */
7063 			set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
7064 			i40e_service_event_schedule(pf);
7065 			goto err;
7066 		} else {
7067 			i40e_pf_unquiesce_all_vsi(pf);
7068 			set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7069 			set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
7070 		}
7071 		/* registers are set, lets apply */
7072 		if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB)
7073 			ret = i40e_hw_set_dcb_config(pf, new_cfg);
7074 	}
7075 
7076 err:
7077 	return ret;
7078 }
7079 
7080 /**
7081  * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW
7082  * @pf: PF being queried
7083  *
7084  * Set default DCB configuration in case DCB is to be done in SW.
7085  **/
7086 int i40e_dcb_sw_default_config(struct i40e_pf *pf)
7087 {
7088 	struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config;
7089 	struct i40e_aqc_configure_switching_comp_ets_data ets_data;
7090 	struct i40e_hw *hw = &pf->hw;
7091 	int err;
7092 
7093 	if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB) {
7094 		/* Update the local cached instance with TC0 ETS */
7095 		memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config));
7096 		pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7097 		pf->tmp_cfg.etscfg.maxtcs = 0;
7098 		pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7099 		pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS;
7100 		pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING;
7101 		pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS;
7102 		/* FW needs one App to configure HW */
7103 		pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS;
7104 		pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE;
7105 		pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO;
7106 		pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE;
7107 
7108 		return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg);
7109 	}
7110 
7111 	memset(&ets_data, 0, sizeof(ets_data));
7112 	ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */
7113 	ets_data.tc_strict_priority_flags = 0; /* ETS */
7114 	ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */
7115 
7116 	/* Enable ETS on the Physical port */
7117 	err = i40e_aq_config_switch_comp_ets
7118 		(hw, pf->mac_seid, &ets_data,
7119 		 i40e_aqc_opc_enable_switching_comp_ets, NULL);
7120 	if (err) {
7121 		dev_info(&pf->pdev->dev,
7122 			 "Enable Port ETS failed, err %pe aq_err %s\n",
7123 			 ERR_PTR(err),
7124 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7125 		err = -ENOENT;
7126 		goto out;
7127 	}
7128 
7129 	/* Update the local cached instance with TC0 ETS */
7130 	dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING;
7131 	dcb_cfg->etscfg.cbs = 0;
7132 	dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS;
7133 	dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW;
7134 
7135 out:
7136 	return err;
7137 }
7138 
7139 /**
7140  * i40e_init_pf_dcb - Initialize DCB configuration
7141  * @pf: PF being configured
7142  *
7143  * Query the current DCB configuration and cache it
7144  * in the hardware structure
7145  **/
7146 static int i40e_init_pf_dcb(struct i40e_pf *pf)
7147 {
7148 	struct i40e_hw *hw = &pf->hw;
7149 	int err;
7150 
7151 	/* Do not enable DCB for SW1 and SW2 images even if the FW is capable
7152 	 * Also do not enable DCBx if FW LLDP agent is disabled
7153 	 */
7154 	if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) {
7155 		dev_info(&pf->pdev->dev, "DCB is not supported.\n");
7156 		err = I40E_NOT_SUPPORTED;
7157 		goto out;
7158 	}
7159 	if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) {
7160 		dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n");
7161 		err = i40e_dcb_sw_default_config(pf);
7162 		if (err) {
7163 			dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n");
7164 			goto out;
7165 		}
7166 		dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n");
7167 		pf->dcbx_cap = DCB_CAP_DCBX_HOST |
7168 			       DCB_CAP_DCBX_VER_IEEE;
7169 		/* at init capable but disabled */
7170 		pf->flags |= I40E_FLAG_DCB_CAPABLE;
7171 		pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7172 		goto out;
7173 	}
7174 	err = i40e_init_dcb(hw, true);
7175 	if (!err) {
7176 		/* Device/Function is not DCBX capable */
7177 		if ((!hw->func_caps.dcb) ||
7178 		    (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
7179 			dev_info(&pf->pdev->dev,
7180 				 "DCBX offload is not supported or is disabled for this PF.\n");
7181 		} else {
7182 			/* When status is not DISABLED then DCBX in FW */
7183 			pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
7184 				       DCB_CAP_DCBX_VER_IEEE;
7185 
7186 			pf->flags |= I40E_FLAG_DCB_CAPABLE;
7187 			/* Enable DCB tagging only when more than one TC
7188 			 * or explicitly disable if only one TC
7189 			 */
7190 			if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
7191 				pf->flags |= I40E_FLAG_DCB_ENABLED;
7192 			else
7193 				pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7194 			dev_dbg(&pf->pdev->dev,
7195 				"DCBX offload is supported for this PF.\n");
7196 		}
7197 	} else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
7198 		dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
7199 		pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
7200 	} else {
7201 		dev_info(&pf->pdev->dev,
7202 			 "Query for DCB configuration failed, err %pe aq_err %s\n",
7203 			 ERR_PTR(err),
7204 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7205 	}
7206 
7207 out:
7208 	return err;
7209 }
7210 #endif /* CONFIG_I40E_DCB */
7211 
7212 /**
7213  * i40e_print_link_message - print link up or down
7214  * @vsi: the VSI for which link needs a message
7215  * @isup: true of link is up, false otherwise
7216  */
7217 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
7218 {
7219 	enum i40e_aq_link_speed new_speed;
7220 	struct i40e_pf *pf = vsi->back;
7221 	char *speed = "Unknown";
7222 	char *fc = "Unknown";
7223 	char *fec = "";
7224 	char *req_fec = "";
7225 	char *an = "";
7226 
7227 	if (isup)
7228 		new_speed = pf->hw.phy.link_info.link_speed;
7229 	else
7230 		new_speed = I40E_LINK_SPEED_UNKNOWN;
7231 
7232 	if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
7233 		return;
7234 	vsi->current_isup = isup;
7235 	vsi->current_speed = new_speed;
7236 	if (!isup) {
7237 		netdev_info(vsi->netdev, "NIC Link is Down\n");
7238 		return;
7239 	}
7240 
7241 	/* Warn user if link speed on NPAR enabled partition is not at
7242 	 * least 10GB
7243 	 */
7244 	if (pf->hw.func_caps.npar_enable &&
7245 	    (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
7246 	     pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
7247 		netdev_warn(vsi->netdev,
7248 			    "The partition detected link speed that is less than 10Gbps\n");
7249 
7250 	switch (pf->hw.phy.link_info.link_speed) {
7251 	case I40E_LINK_SPEED_40GB:
7252 		speed = "40 G";
7253 		break;
7254 	case I40E_LINK_SPEED_20GB:
7255 		speed = "20 G";
7256 		break;
7257 	case I40E_LINK_SPEED_25GB:
7258 		speed = "25 G";
7259 		break;
7260 	case I40E_LINK_SPEED_10GB:
7261 		speed = "10 G";
7262 		break;
7263 	case I40E_LINK_SPEED_5GB:
7264 		speed = "5 G";
7265 		break;
7266 	case I40E_LINK_SPEED_2_5GB:
7267 		speed = "2.5 G";
7268 		break;
7269 	case I40E_LINK_SPEED_1GB:
7270 		speed = "1000 M";
7271 		break;
7272 	case I40E_LINK_SPEED_100MB:
7273 		speed = "100 M";
7274 		break;
7275 	default:
7276 		break;
7277 	}
7278 
7279 	switch (pf->hw.fc.current_mode) {
7280 	case I40E_FC_FULL:
7281 		fc = "RX/TX";
7282 		break;
7283 	case I40E_FC_TX_PAUSE:
7284 		fc = "TX";
7285 		break;
7286 	case I40E_FC_RX_PAUSE:
7287 		fc = "RX";
7288 		break;
7289 	default:
7290 		fc = "None";
7291 		break;
7292 	}
7293 
7294 	if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
7295 		req_fec = "None";
7296 		fec = "None";
7297 		an = "False";
7298 
7299 		if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7300 			an = "True";
7301 
7302 		if (pf->hw.phy.link_info.fec_info &
7303 		    I40E_AQ_CONFIG_FEC_KR_ENA)
7304 			fec = "CL74 FC-FEC/BASE-R";
7305 		else if (pf->hw.phy.link_info.fec_info &
7306 			 I40E_AQ_CONFIG_FEC_RS_ENA)
7307 			fec = "CL108 RS-FEC";
7308 
7309 		/* 'CL108 RS-FEC' should be displayed when RS is requested, or
7310 		 * both RS and FC are requested
7311 		 */
7312 		if (vsi->back->hw.phy.link_info.req_fec_info &
7313 		    (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
7314 			if (vsi->back->hw.phy.link_info.req_fec_info &
7315 			    I40E_AQ_REQUEST_FEC_RS)
7316 				req_fec = "CL108 RS-FEC";
7317 			else
7318 				req_fec = "CL74 FC-FEC/BASE-R";
7319 		}
7320 		netdev_info(vsi->netdev,
7321 			    "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7322 			    speed, req_fec, fec, an, fc);
7323 	} else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) {
7324 		req_fec = "None";
7325 		fec = "None";
7326 		an = "False";
7327 
7328 		if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
7329 			an = "True";
7330 
7331 		if (pf->hw.phy.link_info.fec_info &
7332 		    I40E_AQ_CONFIG_FEC_KR_ENA)
7333 			fec = "CL74 FC-FEC/BASE-R";
7334 
7335 		if (pf->hw.phy.link_info.req_fec_info &
7336 		    I40E_AQ_REQUEST_FEC_KR)
7337 			req_fec = "CL74 FC-FEC/BASE-R";
7338 
7339 		netdev_info(vsi->netdev,
7340 			    "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
7341 			    speed, req_fec, fec, an, fc);
7342 	} else {
7343 		netdev_info(vsi->netdev,
7344 			    "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n",
7345 			    speed, fc);
7346 	}
7347 
7348 }
7349 
7350 /**
7351  * i40e_up_complete - Finish the last steps of bringing up a connection
7352  * @vsi: the VSI being configured
7353  **/
7354 static int i40e_up_complete(struct i40e_vsi *vsi)
7355 {
7356 	struct i40e_pf *pf = vsi->back;
7357 	int err;
7358 
7359 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
7360 		i40e_vsi_configure_msix(vsi);
7361 	else
7362 		i40e_configure_msi_and_legacy(vsi);
7363 
7364 	/* start rings */
7365 	err = i40e_vsi_start_rings(vsi);
7366 	if (err)
7367 		return err;
7368 
7369 	clear_bit(__I40E_VSI_DOWN, vsi->state);
7370 	i40e_napi_enable_all(vsi);
7371 	i40e_vsi_enable_irq(vsi);
7372 
7373 	if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
7374 	    (vsi->netdev)) {
7375 		i40e_print_link_message(vsi, true);
7376 		netif_tx_start_all_queues(vsi->netdev);
7377 		netif_carrier_on(vsi->netdev);
7378 	}
7379 
7380 	/* replay FDIR SB filters */
7381 	if (vsi->type == I40E_VSI_FDIR) {
7382 		/* reset fd counters */
7383 		pf->fd_add_err = 0;
7384 		pf->fd_atr_cnt = 0;
7385 		i40e_fdir_filter_restore(vsi);
7386 	}
7387 
7388 	/* On the next run of the service_task, notify any clients of the new
7389 	 * opened netdev
7390 	 */
7391 	set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
7392 	i40e_service_event_schedule(pf);
7393 
7394 	return 0;
7395 }
7396 
7397 /**
7398  * i40e_vsi_reinit_locked - Reset the VSI
7399  * @vsi: the VSI being configured
7400  *
7401  * Rebuild the ring structs after some configuration
7402  * has changed, e.g. MTU size.
7403  **/
7404 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
7405 {
7406 	struct i40e_pf *pf = vsi->back;
7407 
7408 	while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
7409 		usleep_range(1000, 2000);
7410 	i40e_down(vsi);
7411 
7412 	i40e_up(vsi);
7413 	clear_bit(__I40E_CONFIG_BUSY, pf->state);
7414 }
7415 
7416 /**
7417  * i40e_force_link_state - Force the link status
7418  * @pf: board private structure
7419  * @is_up: whether the link state should be forced up or down
7420  **/
7421 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up)
7422 {
7423 	struct i40e_aq_get_phy_abilities_resp abilities;
7424 	struct i40e_aq_set_phy_config config = {0};
7425 	bool non_zero_phy_type = is_up;
7426 	struct i40e_hw *hw = &pf->hw;
7427 	u64 mask;
7428 	u8 speed;
7429 	int err;
7430 
7431 	/* Card might've been put in an unstable state by other drivers
7432 	 * and applications, which causes incorrect speed values being
7433 	 * set on startup. In order to clear speed registers, we call
7434 	 * get_phy_capabilities twice, once to get initial state of
7435 	 * available speeds, and once to get current PHY config.
7436 	 */
7437 	err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
7438 					   NULL);
7439 	if (err) {
7440 		dev_err(&pf->pdev->dev,
7441 			"failed to get phy cap., ret =  %pe last_status =  %s\n",
7442 			ERR_PTR(err),
7443 			i40e_aq_str(hw, hw->aq.asq_last_status));
7444 		return err;
7445 	}
7446 	speed = abilities.link_speed;
7447 
7448 	/* Get the current phy config */
7449 	err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
7450 					   NULL);
7451 	if (err) {
7452 		dev_err(&pf->pdev->dev,
7453 			"failed to get phy cap., ret =  %pe last_status =  %s\n",
7454 			ERR_PTR(err),
7455 			i40e_aq_str(hw, hw->aq.asq_last_status));
7456 		return err;
7457 	}
7458 
7459 	/* If link needs to go up, but was not forced to go down,
7460 	 * and its speed values are OK, no need for a flap
7461 	 * if non_zero_phy_type was set, still need to force up
7462 	 */
7463 	if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)
7464 		non_zero_phy_type = true;
7465 	else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
7466 		return I40E_SUCCESS;
7467 
7468 	/* To force link we need to set bits for all supported PHY types,
7469 	 * but there are now more than 32, so we need to split the bitmap
7470 	 * across two fields.
7471 	 */
7472 	mask = I40E_PHY_TYPES_BITMASK;
7473 	config.phy_type =
7474 		non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
7475 	config.phy_type_ext =
7476 		non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0;
7477 	/* Copy the old settings, except of phy_type */
7478 	config.abilities = abilities.abilities;
7479 	if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) {
7480 		if (is_up)
7481 			config.abilities |= I40E_AQ_PHY_ENABLE_LINK;
7482 		else
7483 			config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK);
7484 	}
7485 	if (abilities.link_speed != 0)
7486 		config.link_speed = abilities.link_speed;
7487 	else
7488 		config.link_speed = speed;
7489 	config.eee_capability = abilities.eee_capability;
7490 	config.eeer = abilities.eeer_val;
7491 	config.low_power_ctrl = abilities.d3_lpan;
7492 	config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
7493 			    I40E_AQ_PHY_FEC_CONFIG_MASK;
7494 	err = i40e_aq_set_phy_config(hw, &config, NULL);
7495 
7496 	if (err) {
7497 		dev_err(&pf->pdev->dev,
7498 			"set phy config ret =  %pe last_status =  %s\n",
7499 			ERR_PTR(err),
7500 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
7501 		return err;
7502 	}
7503 
7504 	/* Update the link info */
7505 	err = i40e_update_link_info(hw);
7506 	if (err) {
7507 		/* Wait a little bit (on 40G cards it sometimes takes a really
7508 		 * long time for link to come back from the atomic reset)
7509 		 * and try once more
7510 		 */
7511 		msleep(1000);
7512 		i40e_update_link_info(hw);
7513 	}
7514 
7515 	i40e_aq_set_link_restart_an(hw, is_up, NULL);
7516 
7517 	return I40E_SUCCESS;
7518 }
7519 
7520 /**
7521  * i40e_up - Bring the connection back up after being down
7522  * @vsi: the VSI being configured
7523  **/
7524 int i40e_up(struct i40e_vsi *vsi)
7525 {
7526 	int err;
7527 
7528 	if (vsi->type == I40E_VSI_MAIN &&
7529 	    (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7530 	     vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7531 		i40e_force_link_state(vsi->back, true);
7532 
7533 	err = i40e_vsi_configure(vsi);
7534 	if (!err)
7535 		err = i40e_up_complete(vsi);
7536 
7537 	return err;
7538 }
7539 
7540 /**
7541  * i40e_down - Shutdown the connection processing
7542  * @vsi: the VSI being stopped
7543  **/
7544 void i40e_down(struct i40e_vsi *vsi)
7545 {
7546 	int i;
7547 
7548 	/* It is assumed that the caller of this function
7549 	 * sets the vsi->state __I40E_VSI_DOWN bit.
7550 	 */
7551 	if (vsi->netdev) {
7552 		netif_carrier_off(vsi->netdev);
7553 		netif_tx_disable(vsi->netdev);
7554 	}
7555 	i40e_vsi_disable_irq(vsi);
7556 	i40e_vsi_stop_rings(vsi);
7557 	if (vsi->type == I40E_VSI_MAIN &&
7558 	   (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED ||
7559 	    vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED))
7560 		i40e_force_link_state(vsi->back, false);
7561 	i40e_napi_disable_all(vsi);
7562 
7563 	for (i = 0; i < vsi->num_queue_pairs; i++) {
7564 		i40e_clean_tx_ring(vsi->tx_rings[i]);
7565 		if (i40e_enabled_xdp_vsi(vsi)) {
7566 			/* Make sure that in-progress ndo_xdp_xmit and
7567 			 * ndo_xsk_wakeup calls are completed.
7568 			 */
7569 			synchronize_rcu();
7570 			i40e_clean_tx_ring(vsi->xdp_rings[i]);
7571 		}
7572 		i40e_clean_rx_ring(vsi->rx_rings[i]);
7573 	}
7574 
7575 }
7576 
7577 /**
7578  * i40e_validate_mqprio_qopt- validate queue mapping info
7579  * @vsi: the VSI being configured
7580  * @mqprio_qopt: queue parametrs
7581  **/
7582 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
7583 				     struct tc_mqprio_qopt_offload *mqprio_qopt)
7584 {
7585 	u64 sum_max_rate = 0;
7586 	u64 max_rate = 0;
7587 	int i;
7588 
7589 	if (mqprio_qopt->qopt.offset[0] != 0 ||
7590 	    mqprio_qopt->qopt.num_tc < 1 ||
7591 	    mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
7592 		return -EINVAL;
7593 	for (i = 0; ; i++) {
7594 		if (!mqprio_qopt->qopt.count[i])
7595 			return -EINVAL;
7596 		if (mqprio_qopt->min_rate[i]) {
7597 			dev_err(&vsi->back->pdev->dev,
7598 				"Invalid min tx rate (greater than 0) specified\n");
7599 			return -EINVAL;
7600 		}
7601 		max_rate = mqprio_qopt->max_rate[i];
7602 		do_div(max_rate, I40E_BW_MBPS_DIVISOR);
7603 		sum_max_rate += max_rate;
7604 
7605 		if (i >= mqprio_qopt->qopt.num_tc - 1)
7606 			break;
7607 		if (mqprio_qopt->qopt.offset[i + 1] !=
7608 		    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7609 			return -EINVAL;
7610 	}
7611 	if (vsi->num_queue_pairs <
7612 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
7613 		dev_err(&vsi->back->pdev->dev,
7614 			"Failed to create traffic channel, insufficient number of queues.\n");
7615 		return -EINVAL;
7616 	}
7617 	if (sum_max_rate > i40e_get_link_speed(vsi)) {
7618 		dev_err(&vsi->back->pdev->dev,
7619 			"Invalid max tx rate specified\n");
7620 		return -EINVAL;
7621 	}
7622 	return 0;
7623 }
7624 
7625 /**
7626  * i40e_vsi_set_default_tc_config - set default values for tc configuration
7627  * @vsi: the VSI being configured
7628  **/
7629 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
7630 {
7631 	u16 qcount;
7632 	int i;
7633 
7634 	/* Only TC0 is enabled */
7635 	vsi->tc_config.numtc = 1;
7636 	vsi->tc_config.enabled_tc = 1;
7637 	qcount = min_t(int, vsi->alloc_queue_pairs,
7638 		       i40e_pf_get_max_q_per_tc(vsi->back));
7639 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
7640 		/* For the TC that is not enabled set the offset to default
7641 		 * queue and allocate one queue for the given TC.
7642 		 */
7643 		vsi->tc_config.tc_info[i].qoffset = 0;
7644 		if (i == 0)
7645 			vsi->tc_config.tc_info[i].qcount = qcount;
7646 		else
7647 			vsi->tc_config.tc_info[i].qcount = 1;
7648 		vsi->tc_config.tc_info[i].netdev_tc = 0;
7649 	}
7650 }
7651 
7652 /**
7653  * i40e_del_macvlan_filter
7654  * @hw: pointer to the HW structure
7655  * @seid: seid of the channel VSI
7656  * @macaddr: the mac address to apply as a filter
7657  * @aq_err: store the admin Q error
7658  *
7659  * This function deletes a mac filter on the channel VSI which serves as the
7660  * macvlan. Returns 0 on success.
7661  **/
7662 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
7663 				   const u8 *macaddr, int *aq_err)
7664 {
7665 	struct i40e_aqc_remove_macvlan_element_data element;
7666 	int status;
7667 
7668 	memset(&element, 0, sizeof(element));
7669 	ether_addr_copy(element.mac_addr, macaddr);
7670 	element.vlan_tag = 0;
7671 	element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
7672 	status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
7673 	*aq_err = hw->aq.asq_last_status;
7674 
7675 	return status;
7676 }
7677 
7678 /**
7679  * i40e_add_macvlan_filter
7680  * @hw: pointer to the HW structure
7681  * @seid: seid of the channel VSI
7682  * @macaddr: the mac address to apply as a filter
7683  * @aq_err: store the admin Q error
7684  *
7685  * This function adds a mac filter on the channel VSI which serves as the
7686  * macvlan. Returns 0 on success.
7687  **/
7688 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
7689 				   const u8 *macaddr, int *aq_err)
7690 {
7691 	struct i40e_aqc_add_macvlan_element_data element;
7692 	u16 cmd_flags = 0;
7693 	int status;
7694 
7695 	ether_addr_copy(element.mac_addr, macaddr);
7696 	element.vlan_tag = 0;
7697 	element.queue_number = 0;
7698 	element.match_method = I40E_AQC_MM_ERR_NO_RES;
7699 	cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
7700 	element.flags = cpu_to_le16(cmd_flags);
7701 	status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
7702 	*aq_err = hw->aq.asq_last_status;
7703 
7704 	return status;
7705 }
7706 
7707 /**
7708  * i40e_reset_ch_rings - Reset the queue contexts in a channel
7709  * @vsi: the VSI we want to access
7710  * @ch: the channel we want to access
7711  */
7712 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
7713 {
7714 	struct i40e_ring *tx_ring, *rx_ring;
7715 	u16 pf_q;
7716 	int i;
7717 
7718 	for (i = 0; i < ch->num_queue_pairs; i++) {
7719 		pf_q = ch->base_queue + i;
7720 		tx_ring = vsi->tx_rings[pf_q];
7721 		tx_ring->ch = NULL;
7722 		rx_ring = vsi->rx_rings[pf_q];
7723 		rx_ring->ch = NULL;
7724 	}
7725 }
7726 
7727 /**
7728  * i40e_free_macvlan_channels
7729  * @vsi: the VSI we want to access
7730  *
7731  * This function frees the Qs of the channel VSI from
7732  * the stack and also deletes the channel VSIs which
7733  * serve as macvlans.
7734  */
7735 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7736 {
7737 	struct i40e_channel *ch, *ch_tmp;
7738 	int ret;
7739 
7740 	if (list_empty(&vsi->macvlan_list))
7741 		return;
7742 
7743 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7744 		struct i40e_vsi *parent_vsi;
7745 
7746 		if (i40e_is_channel_macvlan(ch)) {
7747 			i40e_reset_ch_rings(vsi, ch);
7748 			clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7749 			netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7750 			netdev_set_sb_channel(ch->fwd->netdev, 0);
7751 			kfree(ch->fwd);
7752 			ch->fwd = NULL;
7753 		}
7754 
7755 		list_del(&ch->list);
7756 		parent_vsi = ch->parent_vsi;
7757 		if (!parent_vsi || !ch->initialized) {
7758 			kfree(ch);
7759 			continue;
7760 		}
7761 
7762 		/* remove the VSI */
7763 		ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7764 					     NULL);
7765 		if (ret)
7766 			dev_err(&vsi->back->pdev->dev,
7767 				"unable to remove channel (%d) for parent VSI(%d)\n",
7768 				ch->seid, parent_vsi->seid);
7769 		kfree(ch);
7770 	}
7771 	vsi->macvlan_cnt = 0;
7772 }
7773 
7774 /**
7775  * i40e_fwd_ring_up - bring the macvlan device up
7776  * @vsi: the VSI we want to access
7777  * @vdev: macvlan netdevice
7778  * @fwd: the private fwd structure
7779  */
7780 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7781 			    struct i40e_fwd_adapter *fwd)
7782 {
7783 	struct i40e_channel *ch = NULL, *ch_tmp, *iter;
7784 	int ret = 0, num_tc = 1,  i, aq_err;
7785 	struct i40e_pf *pf = vsi->back;
7786 	struct i40e_hw *hw = &pf->hw;
7787 
7788 	/* Go through the list and find an available channel */
7789 	list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) {
7790 		if (!i40e_is_channel_macvlan(iter)) {
7791 			iter->fwd = fwd;
7792 			/* record configuration for macvlan interface in vdev */
7793 			for (i = 0; i < num_tc; i++)
7794 				netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7795 							     i,
7796 							     iter->num_queue_pairs,
7797 							     iter->base_queue);
7798 			for (i = 0; i < iter->num_queue_pairs; i++) {
7799 				struct i40e_ring *tx_ring, *rx_ring;
7800 				u16 pf_q;
7801 
7802 				pf_q = iter->base_queue + i;
7803 
7804 				/* Get to TX ring ptr */
7805 				tx_ring = vsi->tx_rings[pf_q];
7806 				tx_ring->ch = iter;
7807 
7808 				/* Get the RX ring ptr */
7809 				rx_ring = vsi->rx_rings[pf_q];
7810 				rx_ring->ch = iter;
7811 			}
7812 			ch = iter;
7813 			break;
7814 		}
7815 	}
7816 
7817 	if (!ch)
7818 		return -EINVAL;
7819 
7820 	/* Guarantee all rings are updated before we update the
7821 	 * MAC address filter.
7822 	 */
7823 	wmb();
7824 
7825 	/* Add a mac filter */
7826 	ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7827 	if (ret) {
7828 		/* if we cannot add the MAC rule then disable the offload */
7829 		macvlan_release_l2fw_offload(vdev);
7830 		for (i = 0; i < ch->num_queue_pairs; i++) {
7831 			struct i40e_ring *rx_ring;
7832 			u16 pf_q;
7833 
7834 			pf_q = ch->base_queue + i;
7835 			rx_ring = vsi->rx_rings[pf_q];
7836 			rx_ring->netdev = NULL;
7837 		}
7838 		dev_info(&pf->pdev->dev,
7839 			 "Error adding mac filter on macvlan err %pe, aq_err %s\n",
7840 			  ERR_PTR(ret),
7841 			  i40e_aq_str(hw, aq_err));
7842 		netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7843 	}
7844 
7845 	return ret;
7846 }
7847 
7848 /**
7849  * i40e_setup_macvlans - create the channels which will be macvlans
7850  * @vsi: the VSI we want to access
7851  * @macvlan_cnt: no. of macvlans to be setup
7852  * @qcnt: no. of Qs per macvlan
7853  * @vdev: macvlan netdevice
7854  */
7855 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7856 			       struct net_device *vdev)
7857 {
7858 	struct i40e_pf *pf = vsi->back;
7859 	struct i40e_hw *hw = &pf->hw;
7860 	struct i40e_vsi_context ctxt;
7861 	u16 sections, qmap, num_qps;
7862 	struct i40e_channel *ch;
7863 	int i, pow, ret = 0;
7864 	u8 offset = 0;
7865 
7866 	if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7867 		return -EINVAL;
7868 
7869 	num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7870 
7871 	/* find the next higher power-of-2 of num queue pairs */
7872 	pow = fls(roundup_pow_of_two(num_qps) - 1);
7873 
7874 	qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7875 		(pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7876 
7877 	/* Setup context bits for the main VSI */
7878 	sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7879 	sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7880 	memset(&ctxt, 0, sizeof(ctxt));
7881 	ctxt.seid = vsi->seid;
7882 	ctxt.pf_num = vsi->back->hw.pf_id;
7883 	ctxt.vf_num = 0;
7884 	ctxt.uplink_seid = vsi->uplink_seid;
7885 	ctxt.info = vsi->info;
7886 	ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7887 	ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7888 	ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7889 	ctxt.info.valid_sections |= cpu_to_le16(sections);
7890 
7891 	/* Reconfigure RSS for main VSI with new max queue count */
7892 	vsi->rss_size = max_t(u16, num_qps, qcnt);
7893 	ret = i40e_vsi_config_rss(vsi);
7894 	if (ret) {
7895 		dev_info(&pf->pdev->dev,
7896 			 "Failed to reconfig RSS for num_queues (%u)\n",
7897 			 vsi->rss_size);
7898 		return ret;
7899 	}
7900 	vsi->reconfig_rss = true;
7901 	dev_dbg(&vsi->back->pdev->dev,
7902 		"Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7903 	vsi->next_base_queue = num_qps;
7904 	vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7905 
7906 	/* Update the VSI after updating the VSI queue-mapping
7907 	 * information
7908 	 */
7909 	ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7910 	if (ret) {
7911 		dev_info(&pf->pdev->dev,
7912 			 "Update vsi tc config failed, err %pe aq_err %s\n",
7913 			 ERR_PTR(ret),
7914 			 i40e_aq_str(hw, hw->aq.asq_last_status));
7915 		return ret;
7916 	}
7917 	/* update the local VSI info with updated queue map */
7918 	i40e_vsi_update_queue_map(vsi, &ctxt);
7919 	vsi->info.valid_sections = 0;
7920 
7921 	/* Create channels for macvlans */
7922 	INIT_LIST_HEAD(&vsi->macvlan_list);
7923 	for (i = 0; i < macvlan_cnt; i++) {
7924 		ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7925 		if (!ch) {
7926 			ret = -ENOMEM;
7927 			goto err_free;
7928 		}
7929 		INIT_LIST_HEAD(&ch->list);
7930 		ch->num_queue_pairs = qcnt;
7931 		if (!i40e_setup_channel(pf, vsi, ch)) {
7932 			ret = -EINVAL;
7933 			kfree(ch);
7934 			goto err_free;
7935 		}
7936 		ch->parent_vsi = vsi;
7937 		vsi->cnt_q_avail -= ch->num_queue_pairs;
7938 		vsi->macvlan_cnt++;
7939 		list_add_tail(&ch->list, &vsi->macvlan_list);
7940 	}
7941 
7942 	return ret;
7943 
7944 err_free:
7945 	dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7946 	i40e_free_macvlan_channels(vsi);
7947 
7948 	return ret;
7949 }
7950 
7951 /**
7952  * i40e_fwd_add - configure macvlans
7953  * @netdev: net device to configure
7954  * @vdev: macvlan netdevice
7955  **/
7956 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7957 {
7958 	struct i40e_netdev_priv *np = netdev_priv(netdev);
7959 	u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7960 	struct i40e_vsi *vsi = np->vsi;
7961 	struct i40e_pf *pf = vsi->back;
7962 	struct i40e_fwd_adapter *fwd;
7963 	int avail_macvlan, ret;
7964 
7965 	if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
7966 		netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7967 		return ERR_PTR(-EINVAL);
7968 	}
7969 	if (i40e_is_tc_mqprio_enabled(pf)) {
7970 		netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7971 		return ERR_PTR(-EINVAL);
7972 	}
7973 	if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
7974 		netdev_info(netdev, "Not enough vectors available to support macvlans\n");
7975 		return ERR_PTR(-EINVAL);
7976 	}
7977 
7978 	/* The macvlan device has to be a single Q device so that the
7979 	 * tc_to_txq field can be reused to pick the tx queue.
7980 	 */
7981 	if (netif_is_multiqueue(vdev))
7982 		return ERR_PTR(-ERANGE);
7983 
7984 	if (!vsi->macvlan_cnt) {
7985 		/* reserve bit 0 for the pf device */
7986 		set_bit(0, vsi->fwd_bitmask);
7987 
7988 		/* Try to reserve as many queues as possible for macvlans. First
7989 		 * reserve 3/4th of max vectors, then half, then quarter and
7990 		 * calculate Qs per macvlan as you go
7991 		 */
7992 		vectors = pf->num_lan_msix;
7993 		if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
7994 			/* allocate 4 Qs per macvlan and 32 Qs to the PF*/
7995 			q_per_macvlan = 4;
7996 			macvlan_cnt = (vectors - 32) / 4;
7997 		} else if (vectors <= 64 && vectors > 32) {
7998 			/* allocate 2 Qs per macvlan and 16 Qs to the PF*/
7999 			q_per_macvlan = 2;
8000 			macvlan_cnt = (vectors - 16) / 2;
8001 		} else if (vectors <= 32 && vectors > 16) {
8002 			/* allocate 1 Q per macvlan and 16 Qs to the PF*/
8003 			q_per_macvlan = 1;
8004 			macvlan_cnt = vectors - 16;
8005 		} else if (vectors <= 16 && vectors > 8) {
8006 			/* allocate 1 Q per macvlan and 8 Qs to the PF */
8007 			q_per_macvlan = 1;
8008 			macvlan_cnt = vectors - 8;
8009 		} else {
8010 			/* allocate 1 Q per macvlan and 1 Q to the PF */
8011 			q_per_macvlan = 1;
8012 			macvlan_cnt = vectors - 1;
8013 		}
8014 
8015 		if (macvlan_cnt == 0)
8016 			return ERR_PTR(-EBUSY);
8017 
8018 		/* Quiesce VSI queues */
8019 		i40e_quiesce_vsi(vsi);
8020 
8021 		/* sets up the macvlans but does not "enable" them */
8022 		ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
8023 					  vdev);
8024 		if (ret)
8025 			return ERR_PTR(ret);
8026 
8027 		/* Unquiesce VSI */
8028 		i40e_unquiesce_vsi(vsi);
8029 	}
8030 	avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
8031 					    vsi->macvlan_cnt);
8032 	if (avail_macvlan >= I40E_MAX_MACVLANS)
8033 		return ERR_PTR(-EBUSY);
8034 
8035 	/* create the fwd struct */
8036 	fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
8037 	if (!fwd)
8038 		return ERR_PTR(-ENOMEM);
8039 
8040 	set_bit(avail_macvlan, vsi->fwd_bitmask);
8041 	fwd->bit_no = avail_macvlan;
8042 	netdev_set_sb_channel(vdev, avail_macvlan);
8043 	fwd->netdev = vdev;
8044 
8045 	if (!netif_running(netdev))
8046 		return fwd;
8047 
8048 	/* Set fwd ring up */
8049 	ret = i40e_fwd_ring_up(vsi, vdev, fwd);
8050 	if (ret) {
8051 		/* unbind the queues and drop the subordinate channel config */
8052 		netdev_unbind_sb_channel(netdev, vdev);
8053 		netdev_set_sb_channel(vdev, 0);
8054 
8055 		kfree(fwd);
8056 		return ERR_PTR(-EINVAL);
8057 	}
8058 
8059 	return fwd;
8060 }
8061 
8062 /**
8063  * i40e_del_all_macvlans - Delete all the mac filters on the channels
8064  * @vsi: the VSI we want to access
8065  */
8066 static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
8067 {
8068 	struct i40e_channel *ch, *ch_tmp;
8069 	struct i40e_pf *pf = vsi->back;
8070 	struct i40e_hw *hw = &pf->hw;
8071 	int aq_err, ret = 0;
8072 
8073 	if (list_empty(&vsi->macvlan_list))
8074 		return;
8075 
8076 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8077 		if (i40e_is_channel_macvlan(ch)) {
8078 			ret = i40e_del_macvlan_filter(hw, ch->seid,
8079 						      i40e_channel_mac(ch),
8080 						      &aq_err);
8081 			if (!ret) {
8082 				/* Reset queue contexts */
8083 				i40e_reset_ch_rings(vsi, ch);
8084 				clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8085 				netdev_unbind_sb_channel(vsi->netdev,
8086 							 ch->fwd->netdev);
8087 				netdev_set_sb_channel(ch->fwd->netdev, 0);
8088 				kfree(ch->fwd);
8089 				ch->fwd = NULL;
8090 			}
8091 		}
8092 	}
8093 }
8094 
8095 /**
8096  * i40e_fwd_del - delete macvlan interfaces
8097  * @netdev: net device to configure
8098  * @vdev: macvlan netdevice
8099  */
8100 static void i40e_fwd_del(struct net_device *netdev, void *vdev)
8101 {
8102 	struct i40e_netdev_priv *np = netdev_priv(netdev);
8103 	struct i40e_fwd_adapter *fwd = vdev;
8104 	struct i40e_channel *ch, *ch_tmp;
8105 	struct i40e_vsi *vsi = np->vsi;
8106 	struct i40e_pf *pf = vsi->back;
8107 	struct i40e_hw *hw = &pf->hw;
8108 	int aq_err, ret = 0;
8109 
8110 	/* Find the channel associated with the macvlan and del mac filter */
8111 	list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
8112 		if (i40e_is_channel_macvlan(ch) &&
8113 		    ether_addr_equal(i40e_channel_mac(ch),
8114 				     fwd->netdev->dev_addr)) {
8115 			ret = i40e_del_macvlan_filter(hw, ch->seid,
8116 						      i40e_channel_mac(ch),
8117 						      &aq_err);
8118 			if (!ret) {
8119 				/* Reset queue contexts */
8120 				i40e_reset_ch_rings(vsi, ch);
8121 				clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
8122 				netdev_unbind_sb_channel(netdev, fwd->netdev);
8123 				netdev_set_sb_channel(fwd->netdev, 0);
8124 				kfree(ch->fwd);
8125 				ch->fwd = NULL;
8126 			} else {
8127 				dev_info(&pf->pdev->dev,
8128 					 "Error deleting mac filter on macvlan err %pe, aq_err %s\n",
8129 					  ERR_PTR(ret),
8130 					  i40e_aq_str(hw, aq_err));
8131 			}
8132 			break;
8133 		}
8134 	}
8135 }
8136 
8137 /**
8138  * i40e_setup_tc - configure multiple traffic classes
8139  * @netdev: net device to configure
8140  * @type_data: tc offload data
8141  **/
8142 static int i40e_setup_tc(struct net_device *netdev, void *type_data)
8143 {
8144 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8145 	struct i40e_netdev_priv *np = netdev_priv(netdev);
8146 	struct i40e_vsi *vsi = np->vsi;
8147 	struct i40e_pf *pf = vsi->back;
8148 	u8 enabled_tc = 0, num_tc, hw;
8149 	bool need_reset = false;
8150 	int old_queue_pairs;
8151 	int ret = -EINVAL;
8152 	u16 mode;
8153 	int i;
8154 
8155 	old_queue_pairs = vsi->num_queue_pairs;
8156 	num_tc = mqprio_qopt->qopt.num_tc;
8157 	hw = mqprio_qopt->qopt.hw;
8158 	mode = mqprio_qopt->mode;
8159 	if (!hw) {
8160 		pf->flags &= ~I40E_FLAG_TC_MQPRIO;
8161 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8162 		goto config_tc;
8163 	}
8164 
8165 	/* Check if MFP enabled */
8166 	if (pf->flags & I40E_FLAG_MFP_ENABLED) {
8167 		netdev_info(netdev,
8168 			    "Configuring TC not supported in MFP mode\n");
8169 		return ret;
8170 	}
8171 	switch (mode) {
8172 	case TC_MQPRIO_MODE_DCB:
8173 		pf->flags &= ~I40E_FLAG_TC_MQPRIO;
8174 
8175 		/* Check if DCB enabled to continue */
8176 		if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
8177 			netdev_info(netdev,
8178 				    "DCB is not enabled for adapter\n");
8179 			return ret;
8180 		}
8181 
8182 		/* Check whether tc count is within enabled limit */
8183 		if (num_tc > i40e_pf_get_num_tc(pf)) {
8184 			netdev_info(netdev,
8185 				    "TC count greater than enabled on link for adapter\n");
8186 			return ret;
8187 		}
8188 		break;
8189 	case TC_MQPRIO_MODE_CHANNEL:
8190 		if (pf->flags & I40E_FLAG_DCB_ENABLED) {
8191 			netdev_info(netdev,
8192 				    "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
8193 			return ret;
8194 		}
8195 		if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
8196 			return ret;
8197 		ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
8198 		if (ret)
8199 			return ret;
8200 		memcpy(&vsi->mqprio_qopt, mqprio_qopt,
8201 		       sizeof(*mqprio_qopt));
8202 		pf->flags |= I40E_FLAG_TC_MQPRIO;
8203 		pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8204 		break;
8205 	default:
8206 		return -EINVAL;
8207 	}
8208 
8209 config_tc:
8210 	/* Generate TC map for number of tc requested */
8211 	for (i = 0; i < num_tc; i++)
8212 		enabled_tc |= BIT(i);
8213 
8214 	/* Requesting same TC configuration as already enabled */
8215 	if (enabled_tc == vsi->tc_config.enabled_tc &&
8216 	    mode != TC_MQPRIO_MODE_CHANNEL)
8217 		return 0;
8218 
8219 	/* Quiesce VSI queues */
8220 	i40e_quiesce_vsi(vsi);
8221 
8222 	if (!hw && !i40e_is_tc_mqprio_enabled(pf))
8223 		i40e_remove_queue_channels(vsi);
8224 
8225 	/* Configure VSI for enabled TCs */
8226 	ret = i40e_vsi_config_tc(vsi, enabled_tc);
8227 	if (ret) {
8228 		netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
8229 			    vsi->seid);
8230 		need_reset = true;
8231 		goto exit;
8232 	} else if (enabled_tc &&
8233 		   (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) {
8234 		netdev_info(netdev,
8235 			    "Failed to create channel. Override queues (%u) not power of 2\n",
8236 			    vsi->tc_config.tc_info[0].qcount);
8237 		ret = -EINVAL;
8238 		need_reset = true;
8239 		goto exit;
8240 	}
8241 
8242 	dev_info(&vsi->back->pdev->dev,
8243 		 "Setup channel (id:%u) utilizing num_queues %d\n",
8244 		 vsi->seid, vsi->tc_config.tc_info[0].qcount);
8245 
8246 	if (i40e_is_tc_mqprio_enabled(pf)) {
8247 		if (vsi->mqprio_qopt.max_rate[0]) {
8248 			u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
8249 						  vsi->mqprio_qopt.max_rate[0]);
8250 
8251 			ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
8252 			if (!ret) {
8253 				u64 credits = max_tx_rate;
8254 
8255 				do_div(credits, I40E_BW_CREDIT_DIVISOR);
8256 				dev_dbg(&vsi->back->pdev->dev,
8257 					"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
8258 					max_tx_rate,
8259 					credits,
8260 					vsi->seid);
8261 			} else {
8262 				need_reset = true;
8263 				goto exit;
8264 			}
8265 		}
8266 		ret = i40e_configure_queue_channels(vsi);
8267 		if (ret) {
8268 			vsi->num_queue_pairs = old_queue_pairs;
8269 			netdev_info(netdev,
8270 				    "Failed configuring queue channels\n");
8271 			need_reset = true;
8272 			goto exit;
8273 		}
8274 	}
8275 
8276 exit:
8277 	/* Reset the configuration data to defaults, only TC0 is enabled */
8278 	if (need_reset) {
8279 		i40e_vsi_set_default_tc_config(vsi);
8280 		need_reset = false;
8281 	}
8282 
8283 	/* Unquiesce VSI */
8284 	i40e_unquiesce_vsi(vsi);
8285 	return ret;
8286 }
8287 
8288 /**
8289  * i40e_set_cld_element - sets cloud filter element data
8290  * @filter: cloud filter rule
8291  * @cld: ptr to cloud filter element data
8292  *
8293  * This is helper function to copy data into cloud filter element
8294  **/
8295 static inline void
8296 i40e_set_cld_element(struct i40e_cloud_filter *filter,
8297 		     struct i40e_aqc_cloud_filters_element_data *cld)
8298 {
8299 	u32 ipa;
8300 	int i;
8301 
8302 	memset(cld, 0, sizeof(*cld));
8303 	ether_addr_copy(cld->outer_mac, filter->dst_mac);
8304 	ether_addr_copy(cld->inner_mac, filter->src_mac);
8305 
8306 	if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
8307 		return;
8308 
8309 	if (filter->n_proto == ETH_P_IPV6) {
8310 #define IPV6_MAX_INDEX	(ARRAY_SIZE(filter->dst_ipv6) - 1)
8311 		for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) {
8312 			ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
8313 
8314 			*(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa);
8315 		}
8316 	} else {
8317 		ipa = be32_to_cpu(filter->dst_ipv4);
8318 
8319 		memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
8320 	}
8321 
8322 	cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
8323 
8324 	/* tenant_id is not supported by FW now, once the support is enabled
8325 	 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id)
8326 	 */
8327 	if (filter->tenant_id)
8328 		return;
8329 }
8330 
8331 /**
8332  * i40e_add_del_cloud_filter - Add/del cloud filter
8333  * @vsi: pointer to VSI
8334  * @filter: cloud filter rule
8335  * @add: if true, add, if false, delete
8336  *
8337  * Add or delete a cloud filter for a specific flow spec.
8338  * Returns 0 if the filter were successfully added.
8339  **/
8340 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
8341 			      struct i40e_cloud_filter *filter, bool add)
8342 {
8343 	struct i40e_aqc_cloud_filters_element_data cld_filter;
8344 	struct i40e_pf *pf = vsi->back;
8345 	int ret;
8346 	static const u16 flag_table[128] = {
8347 		[I40E_CLOUD_FILTER_FLAGS_OMAC]  =
8348 			I40E_AQC_ADD_CLOUD_FILTER_OMAC,
8349 		[I40E_CLOUD_FILTER_FLAGS_IMAC]  =
8350 			I40E_AQC_ADD_CLOUD_FILTER_IMAC,
8351 		[I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN]  =
8352 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
8353 		[I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
8354 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
8355 		[I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
8356 			I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
8357 		[I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
8358 			I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
8359 		[I40E_CLOUD_FILTER_FLAGS_IIP] =
8360 			I40E_AQC_ADD_CLOUD_FILTER_IIP,
8361 	};
8362 
8363 	if (filter->flags >= ARRAY_SIZE(flag_table))
8364 		return I40E_ERR_CONFIG;
8365 
8366 	memset(&cld_filter, 0, sizeof(cld_filter));
8367 
8368 	/* copy element needed to add cloud filter from filter */
8369 	i40e_set_cld_element(filter, &cld_filter);
8370 
8371 	if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
8372 		cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
8373 					     I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
8374 
8375 	if (filter->n_proto == ETH_P_IPV6)
8376 		cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8377 						I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8378 	else
8379 		cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
8380 						I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8381 
8382 	if (add)
8383 		ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
8384 						&cld_filter, 1);
8385 	else
8386 		ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
8387 						&cld_filter, 1);
8388 	if (ret)
8389 		dev_dbg(&pf->pdev->dev,
8390 			"Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
8391 			add ? "add" : "delete", filter->dst_port, ret,
8392 			pf->hw.aq.asq_last_status);
8393 	else
8394 		dev_info(&pf->pdev->dev,
8395 			 "%s cloud filter for VSI: %d\n",
8396 			 add ? "Added" : "Deleted", filter->seid);
8397 	return ret;
8398 }
8399 
8400 /**
8401  * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf
8402  * @vsi: pointer to VSI
8403  * @filter: cloud filter rule
8404  * @add: if true, add, if false, delete
8405  *
8406  * Add or delete a cloud filter for a specific flow spec using big buffer.
8407  * Returns 0 if the filter were successfully added.
8408  **/
8409 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
8410 				      struct i40e_cloud_filter *filter,
8411 				      bool add)
8412 {
8413 	struct i40e_aqc_cloud_filters_element_bb cld_filter;
8414 	struct i40e_pf *pf = vsi->back;
8415 	int ret;
8416 
8417 	/* Both (src/dst) valid mac_addr are not supported */
8418 	if ((is_valid_ether_addr(filter->dst_mac) &&
8419 	     is_valid_ether_addr(filter->src_mac)) ||
8420 	    (is_multicast_ether_addr(filter->dst_mac) &&
8421 	     is_multicast_ether_addr(filter->src_mac)))
8422 		return -EOPNOTSUPP;
8423 
8424 	/* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP
8425 	 * ports are not supported via big buffer now.
8426 	 */
8427 	if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
8428 		return -EOPNOTSUPP;
8429 
8430 	/* adding filter using src_port/src_ip is not supported at this stage */
8431 	if (filter->src_port ||
8432 	    (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8433 	    !ipv6_addr_any(&filter->ip.v6.src_ip6))
8434 		return -EOPNOTSUPP;
8435 
8436 	memset(&cld_filter, 0, sizeof(cld_filter));
8437 
8438 	/* copy element needed to add cloud filter from filter */
8439 	i40e_set_cld_element(filter, &cld_filter.element);
8440 
8441 	if (is_valid_ether_addr(filter->dst_mac) ||
8442 	    is_valid_ether_addr(filter->src_mac) ||
8443 	    is_multicast_ether_addr(filter->dst_mac) ||
8444 	    is_multicast_ether_addr(filter->src_mac)) {
8445 		/* MAC + IP : unsupported mode */
8446 		if (filter->dst_ipv4)
8447 			return -EOPNOTSUPP;
8448 
8449 		/* since we validated that L4 port must be valid before
8450 		 * we get here, start with respective "flags" value
8451 		 * and update if vlan is present or not
8452 		 */
8453 		cld_filter.element.flags =
8454 			cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
8455 
8456 		if (filter->vlan_id) {
8457 			cld_filter.element.flags =
8458 			cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
8459 		}
8460 
8461 	} else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) ||
8462 		   !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
8463 		cld_filter.element.flags =
8464 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
8465 		if (filter->n_proto == ETH_P_IPV6)
8466 			cld_filter.element.flags |=
8467 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
8468 		else
8469 			cld_filter.element.flags |=
8470 				cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
8471 	} else {
8472 		dev_err(&pf->pdev->dev,
8473 			"either mac or ip has to be valid for cloud filter\n");
8474 		return -EINVAL;
8475 	}
8476 
8477 	/* Now copy L4 port in Byte 6..7 in general fields */
8478 	cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
8479 						be16_to_cpu(filter->dst_port);
8480 
8481 	if (add) {
8482 		/* Validate current device switch mode, change if necessary */
8483 		ret = i40e_validate_and_set_switch_mode(vsi);
8484 		if (ret) {
8485 			dev_err(&pf->pdev->dev,
8486 				"failed to set switch mode, ret %d\n",
8487 				ret);
8488 			return ret;
8489 		}
8490 
8491 		ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
8492 						   &cld_filter, 1);
8493 	} else {
8494 		ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
8495 						   &cld_filter, 1);
8496 	}
8497 
8498 	if (ret)
8499 		dev_dbg(&pf->pdev->dev,
8500 			"Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
8501 			add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
8502 	else
8503 		dev_info(&pf->pdev->dev,
8504 			 "%s cloud filter for VSI: %d, L4 port: %d\n",
8505 			 add ? "add" : "delete", filter->seid,
8506 			 ntohs(filter->dst_port));
8507 	return ret;
8508 }
8509 
8510 /**
8511  * i40e_parse_cls_flower - Parse tc flower filters provided by kernel
8512  * @vsi: Pointer to VSI
8513  * @f: Pointer to struct flow_cls_offload
8514  * @filter: Pointer to cloud filter structure
8515  *
8516  **/
8517 static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
8518 				 struct flow_cls_offload *f,
8519 				 struct i40e_cloud_filter *filter)
8520 {
8521 	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
8522 	struct flow_dissector *dissector = rule->match.dissector;
8523 	u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
8524 	struct i40e_pf *pf = vsi->back;
8525 	u8 field_flags = 0;
8526 
8527 	if (dissector->used_keys &
8528 	    ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
8529 	      BIT(FLOW_DISSECTOR_KEY_BASIC) |
8530 	      BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
8531 	      BIT(FLOW_DISSECTOR_KEY_VLAN) |
8532 	      BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
8533 	      BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
8534 	      BIT(FLOW_DISSECTOR_KEY_PORTS) |
8535 	      BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
8536 		dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
8537 			dissector->used_keys);
8538 		return -EOPNOTSUPP;
8539 	}
8540 
8541 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
8542 		struct flow_match_enc_keyid match;
8543 
8544 		flow_rule_match_enc_keyid(rule, &match);
8545 		if (match.mask->keyid != 0)
8546 			field_flags |= I40E_CLOUD_FIELD_TEN_ID;
8547 
8548 		filter->tenant_id = be32_to_cpu(match.key->keyid);
8549 	}
8550 
8551 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
8552 		struct flow_match_basic match;
8553 
8554 		flow_rule_match_basic(rule, &match);
8555 		n_proto_key = ntohs(match.key->n_proto);
8556 		n_proto_mask = ntohs(match.mask->n_proto);
8557 
8558 		if (n_proto_key == ETH_P_ALL) {
8559 			n_proto_key = 0;
8560 			n_proto_mask = 0;
8561 		}
8562 		filter->n_proto = n_proto_key & n_proto_mask;
8563 		filter->ip_proto = match.key->ip_proto;
8564 	}
8565 
8566 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
8567 		struct flow_match_eth_addrs match;
8568 
8569 		flow_rule_match_eth_addrs(rule, &match);
8570 
8571 		/* use is_broadcast and is_zero to check for all 0xf or 0 */
8572 		if (!is_zero_ether_addr(match.mask->dst)) {
8573 			if (is_broadcast_ether_addr(match.mask->dst)) {
8574 				field_flags |= I40E_CLOUD_FIELD_OMAC;
8575 			} else {
8576 				dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
8577 					match.mask->dst);
8578 				return I40E_ERR_CONFIG;
8579 			}
8580 		}
8581 
8582 		if (!is_zero_ether_addr(match.mask->src)) {
8583 			if (is_broadcast_ether_addr(match.mask->src)) {
8584 				field_flags |= I40E_CLOUD_FIELD_IMAC;
8585 			} else {
8586 				dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
8587 					match.mask->src);
8588 				return I40E_ERR_CONFIG;
8589 			}
8590 		}
8591 		ether_addr_copy(filter->dst_mac, match.key->dst);
8592 		ether_addr_copy(filter->src_mac, match.key->src);
8593 	}
8594 
8595 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
8596 		struct flow_match_vlan match;
8597 
8598 		flow_rule_match_vlan(rule, &match);
8599 		if (match.mask->vlan_id) {
8600 			if (match.mask->vlan_id == VLAN_VID_MASK) {
8601 				field_flags |= I40E_CLOUD_FIELD_IVLAN;
8602 
8603 			} else {
8604 				dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
8605 					match.mask->vlan_id);
8606 				return I40E_ERR_CONFIG;
8607 			}
8608 		}
8609 
8610 		filter->vlan_id = cpu_to_be16(match.key->vlan_id);
8611 	}
8612 
8613 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
8614 		struct flow_match_control match;
8615 
8616 		flow_rule_match_control(rule, &match);
8617 		addr_type = match.key->addr_type;
8618 	}
8619 
8620 	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
8621 		struct flow_match_ipv4_addrs match;
8622 
8623 		flow_rule_match_ipv4_addrs(rule, &match);
8624 		if (match.mask->dst) {
8625 			if (match.mask->dst == cpu_to_be32(0xffffffff)) {
8626 				field_flags |= I40E_CLOUD_FIELD_IIP;
8627 			} else {
8628 				dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
8629 					&match.mask->dst);
8630 				return I40E_ERR_CONFIG;
8631 			}
8632 		}
8633 
8634 		if (match.mask->src) {
8635 			if (match.mask->src == cpu_to_be32(0xffffffff)) {
8636 				field_flags |= I40E_CLOUD_FIELD_IIP;
8637 			} else {
8638 				dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
8639 					&match.mask->src);
8640 				return I40E_ERR_CONFIG;
8641 			}
8642 		}
8643 
8644 		if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
8645 			dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
8646 			return I40E_ERR_CONFIG;
8647 		}
8648 		filter->dst_ipv4 = match.key->dst;
8649 		filter->src_ipv4 = match.key->src;
8650 	}
8651 
8652 	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
8653 		struct flow_match_ipv6_addrs match;
8654 
8655 		flow_rule_match_ipv6_addrs(rule, &match);
8656 
8657 		/* src and dest IPV6 address should not be LOOPBACK
8658 		 * (0:0:0:0:0:0:0:1), which can be represented as ::1
8659 		 */
8660 		if (ipv6_addr_loopback(&match.key->dst) ||
8661 		    ipv6_addr_loopback(&match.key->src)) {
8662 			dev_err(&pf->pdev->dev,
8663 				"Bad ipv6, addr is LOOPBACK\n");
8664 			return I40E_ERR_CONFIG;
8665 		}
8666 		if (!ipv6_addr_any(&match.mask->dst) ||
8667 		    !ipv6_addr_any(&match.mask->src))
8668 			field_flags |= I40E_CLOUD_FIELD_IIP;
8669 
8670 		memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
8671 		       sizeof(filter->src_ipv6));
8672 		memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
8673 		       sizeof(filter->dst_ipv6));
8674 	}
8675 
8676 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
8677 		struct flow_match_ports match;
8678 
8679 		flow_rule_match_ports(rule, &match);
8680 		if (match.mask->src) {
8681 			if (match.mask->src == cpu_to_be16(0xffff)) {
8682 				field_flags |= I40E_CLOUD_FIELD_IIP;
8683 			} else {
8684 				dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
8685 					be16_to_cpu(match.mask->src));
8686 				return I40E_ERR_CONFIG;
8687 			}
8688 		}
8689 
8690 		if (match.mask->dst) {
8691 			if (match.mask->dst == cpu_to_be16(0xffff)) {
8692 				field_flags |= I40E_CLOUD_FIELD_IIP;
8693 			} else {
8694 				dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
8695 					be16_to_cpu(match.mask->dst));
8696 				return I40E_ERR_CONFIG;
8697 			}
8698 		}
8699 
8700 		filter->dst_port = match.key->dst;
8701 		filter->src_port = match.key->src;
8702 
8703 		switch (filter->ip_proto) {
8704 		case IPPROTO_TCP:
8705 		case IPPROTO_UDP:
8706 			break;
8707 		default:
8708 			dev_err(&pf->pdev->dev,
8709 				"Only UDP and TCP transport are supported\n");
8710 			return -EINVAL;
8711 		}
8712 	}
8713 	filter->flags = field_flags;
8714 	return 0;
8715 }
8716 
8717 /**
8718  * i40e_handle_tclass: Forward to a traffic class on the device
8719  * @vsi: Pointer to VSI
8720  * @tc: traffic class index on the device
8721  * @filter: Pointer to cloud filter structure
8722  *
8723  **/
8724 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
8725 			      struct i40e_cloud_filter *filter)
8726 {
8727 	struct i40e_channel *ch, *ch_tmp;
8728 
8729 	/* direct to a traffic class on the same device */
8730 	if (tc == 0) {
8731 		filter->seid = vsi->seid;
8732 		return 0;
8733 	} else if (vsi->tc_config.enabled_tc & BIT(tc)) {
8734 		if (!filter->dst_port) {
8735 			dev_err(&vsi->back->pdev->dev,
8736 				"Specify destination port to direct to traffic class that is not default\n");
8737 			return -EINVAL;
8738 		}
8739 		if (list_empty(&vsi->ch_list))
8740 			return -EINVAL;
8741 		list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
8742 					 list) {
8743 			if (ch->seid == vsi->tc_seid_map[tc])
8744 				filter->seid = ch->seid;
8745 		}
8746 		return 0;
8747 	}
8748 	dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
8749 	return -EINVAL;
8750 }
8751 
8752 /**
8753  * i40e_configure_clsflower - Configure tc flower filters
8754  * @vsi: Pointer to VSI
8755  * @cls_flower: Pointer to struct flow_cls_offload
8756  *
8757  **/
8758 static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8759 				    struct flow_cls_offload *cls_flower)
8760 {
8761 	int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8762 	struct i40e_cloud_filter *filter = NULL;
8763 	struct i40e_pf *pf = vsi->back;
8764 	int err = 0;
8765 
8766 	if (tc < 0) {
8767 		dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8768 		return -EOPNOTSUPP;
8769 	}
8770 
8771 	if (!tc) {
8772 		dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination");
8773 		return -EINVAL;
8774 	}
8775 
8776 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8777 	    test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8778 		return -EBUSY;
8779 
8780 	if (pf->fdir_pf_active_filters ||
8781 	    (!hlist_empty(&pf->fdir_filter_list))) {
8782 		dev_err(&vsi->back->pdev->dev,
8783 			"Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8784 		return -EINVAL;
8785 	}
8786 
8787 	if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
8788 		dev_err(&vsi->back->pdev->dev,
8789 			"Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8790 		vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8791 		vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8792 	}
8793 
8794 	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8795 	if (!filter)
8796 		return -ENOMEM;
8797 
8798 	filter->cookie = cls_flower->cookie;
8799 
8800 	err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8801 	if (err < 0)
8802 		goto err;
8803 
8804 	err = i40e_handle_tclass(vsi, tc, filter);
8805 	if (err < 0)
8806 		goto err;
8807 
8808 	/* Add cloud filter */
8809 	if (filter->dst_port)
8810 		err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8811 	else
8812 		err = i40e_add_del_cloud_filter(vsi, filter, true);
8813 
8814 	if (err) {
8815 		dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n",
8816 			err);
8817 		goto err;
8818 	}
8819 
8820 	/* add filter to the ordered list */
8821 	INIT_HLIST_NODE(&filter->cloud_node);
8822 
8823 	hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8824 
8825 	pf->num_cloud_filters++;
8826 
8827 	return err;
8828 err:
8829 	kfree(filter);
8830 	return err;
8831 }
8832 
8833 /**
8834  * i40e_find_cloud_filter - Find the could filter in the list
8835  * @vsi: Pointer to VSI
8836  * @cookie: filter specific cookie
8837  *
8838  **/
8839 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8840 							unsigned long *cookie)
8841 {
8842 	struct i40e_cloud_filter *filter = NULL;
8843 	struct hlist_node *node2;
8844 
8845 	hlist_for_each_entry_safe(filter, node2,
8846 				  &vsi->back->cloud_filter_list, cloud_node)
8847 		if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8848 			return filter;
8849 	return NULL;
8850 }
8851 
8852 /**
8853  * i40e_delete_clsflower - Remove tc flower filters
8854  * @vsi: Pointer to VSI
8855  * @cls_flower: Pointer to struct flow_cls_offload
8856  *
8857  **/
8858 static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8859 				 struct flow_cls_offload *cls_flower)
8860 {
8861 	struct i40e_cloud_filter *filter = NULL;
8862 	struct i40e_pf *pf = vsi->back;
8863 	int err = 0;
8864 
8865 	filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8866 
8867 	if (!filter)
8868 		return -EINVAL;
8869 
8870 	hash_del(&filter->cloud_node);
8871 
8872 	if (filter->dst_port)
8873 		err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8874 	else
8875 		err = i40e_add_del_cloud_filter(vsi, filter, false);
8876 
8877 	kfree(filter);
8878 	if (err) {
8879 		dev_err(&pf->pdev->dev,
8880 			"Failed to delete cloud filter, err %pe\n",
8881 			ERR_PTR(err));
8882 		return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8883 	}
8884 
8885 	pf->num_cloud_filters--;
8886 	if (!pf->num_cloud_filters)
8887 		if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8888 		    !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8889 			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8890 			pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8891 			pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8892 		}
8893 	return 0;
8894 }
8895 
8896 /**
8897  * i40e_setup_tc_cls_flower - flower classifier offloads
8898  * @np: net device to configure
8899  * @cls_flower: offload data
8900  **/
8901 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8902 				    struct flow_cls_offload *cls_flower)
8903 {
8904 	struct i40e_vsi *vsi = np->vsi;
8905 
8906 	switch (cls_flower->command) {
8907 	case FLOW_CLS_REPLACE:
8908 		return i40e_configure_clsflower(vsi, cls_flower);
8909 	case FLOW_CLS_DESTROY:
8910 		return i40e_delete_clsflower(vsi, cls_flower);
8911 	case FLOW_CLS_STATS:
8912 		return -EOPNOTSUPP;
8913 	default:
8914 		return -EOPNOTSUPP;
8915 	}
8916 }
8917 
8918 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8919 				  void *cb_priv)
8920 {
8921 	struct i40e_netdev_priv *np = cb_priv;
8922 
8923 	if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8924 		return -EOPNOTSUPP;
8925 
8926 	switch (type) {
8927 	case TC_SETUP_CLSFLOWER:
8928 		return i40e_setup_tc_cls_flower(np, type_data);
8929 
8930 	default:
8931 		return -EOPNOTSUPP;
8932 	}
8933 }
8934 
8935 static LIST_HEAD(i40e_block_cb_list);
8936 
8937 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8938 			   void *type_data)
8939 {
8940 	struct i40e_netdev_priv *np = netdev_priv(netdev);
8941 
8942 	switch (type) {
8943 	case TC_SETUP_QDISC_MQPRIO:
8944 		return i40e_setup_tc(netdev, type_data);
8945 	case TC_SETUP_BLOCK:
8946 		return flow_block_cb_setup_simple(type_data,
8947 						  &i40e_block_cb_list,
8948 						  i40e_setup_tc_block_cb,
8949 						  np, np, true);
8950 	default:
8951 		return -EOPNOTSUPP;
8952 	}
8953 }
8954 
8955 /**
8956  * i40e_open - Called when a network interface is made active
8957  * @netdev: network interface device structure
8958  *
8959  * The open entry point is called when a network interface is made
8960  * active by the system (IFF_UP).  At this point all resources needed
8961  * for transmit and receive operations are allocated, the interrupt
8962  * handler is registered with the OS, the netdev watchdog subtask is
8963  * enabled, and the stack is notified that the interface is ready.
8964  *
8965  * Returns 0 on success, negative value on failure
8966  **/
8967 int i40e_open(struct net_device *netdev)
8968 {
8969 	struct i40e_netdev_priv *np = netdev_priv(netdev);
8970 	struct i40e_vsi *vsi = np->vsi;
8971 	struct i40e_pf *pf = vsi->back;
8972 	int err;
8973 
8974 	/* disallow open during test or if eeprom is broken */
8975 	if (test_bit(__I40E_TESTING, pf->state) ||
8976 	    test_bit(__I40E_BAD_EEPROM, pf->state))
8977 		return -EBUSY;
8978 
8979 	netif_carrier_off(netdev);
8980 
8981 	if (i40e_force_link_state(pf, true))
8982 		return -EAGAIN;
8983 
8984 	err = i40e_vsi_open(vsi);
8985 	if (err)
8986 		return err;
8987 
8988 	/* configure global TSO hardware offload settings */
8989 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
8990 						       TCP_FLAG_FIN) >> 16);
8991 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
8992 						       TCP_FLAG_FIN |
8993 						       TCP_FLAG_CWR) >> 16);
8994 	wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
8995 	udp_tunnel_get_rx_info(netdev);
8996 
8997 	return 0;
8998 }
8999 
9000 /**
9001  * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues
9002  * @vsi: vsi structure
9003  *
9004  * This updates netdev's number of tx/rx queues
9005  *
9006  * Returns status of setting tx/rx queues
9007  **/
9008 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi)
9009 {
9010 	int ret;
9011 
9012 	ret = netif_set_real_num_rx_queues(vsi->netdev,
9013 					   vsi->num_queue_pairs);
9014 	if (ret)
9015 		return ret;
9016 
9017 	return netif_set_real_num_tx_queues(vsi->netdev,
9018 					    vsi->num_queue_pairs);
9019 }
9020 
9021 /**
9022  * i40e_vsi_open -
9023  * @vsi: the VSI to open
9024  *
9025  * Finish initialization of the VSI.
9026  *
9027  * Returns 0 on success, negative value on failure
9028  *
9029  * Note: expects to be called while under rtnl_lock()
9030  **/
9031 int i40e_vsi_open(struct i40e_vsi *vsi)
9032 {
9033 	struct i40e_pf *pf = vsi->back;
9034 	char int_name[I40E_INT_NAME_STR_LEN];
9035 	int err;
9036 
9037 	/* allocate descriptors */
9038 	err = i40e_vsi_setup_tx_resources(vsi);
9039 	if (err)
9040 		goto err_setup_tx;
9041 	err = i40e_vsi_setup_rx_resources(vsi);
9042 	if (err)
9043 		goto err_setup_rx;
9044 
9045 	err = i40e_vsi_configure(vsi);
9046 	if (err)
9047 		goto err_setup_rx;
9048 
9049 	if (vsi->netdev) {
9050 		snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
9051 			 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
9052 		err = i40e_vsi_request_irq(vsi, int_name);
9053 		if (err)
9054 			goto err_setup_rx;
9055 
9056 		/* Notify the stack of the actual queue counts. */
9057 		err = i40e_netif_set_realnum_tx_rx_queues(vsi);
9058 		if (err)
9059 			goto err_set_queues;
9060 
9061 	} else if (vsi->type == I40E_VSI_FDIR) {
9062 		snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
9063 			 dev_driver_string(&pf->pdev->dev),
9064 			 dev_name(&pf->pdev->dev));
9065 		err = i40e_vsi_request_irq(vsi, int_name);
9066 		if (err)
9067 			goto err_setup_rx;
9068 
9069 	} else {
9070 		err = -EINVAL;
9071 		goto err_setup_rx;
9072 	}
9073 
9074 	err = i40e_up_complete(vsi);
9075 	if (err)
9076 		goto err_up_complete;
9077 
9078 	return 0;
9079 
9080 err_up_complete:
9081 	i40e_down(vsi);
9082 err_set_queues:
9083 	i40e_vsi_free_irq(vsi);
9084 err_setup_rx:
9085 	i40e_vsi_free_rx_resources(vsi);
9086 err_setup_tx:
9087 	i40e_vsi_free_tx_resources(vsi);
9088 	if (vsi == pf->vsi[pf->lan_vsi])
9089 		i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
9090 
9091 	return err;
9092 }
9093 
9094 /**
9095  * i40e_fdir_filter_exit - Cleans up the Flow Director accounting
9096  * @pf: Pointer to PF
9097  *
9098  * This function destroys the hlist where all the Flow Director
9099  * filters were saved.
9100  **/
9101 static void i40e_fdir_filter_exit(struct i40e_pf *pf)
9102 {
9103 	struct i40e_fdir_filter *filter;
9104 	struct i40e_flex_pit *pit_entry, *tmp;
9105 	struct hlist_node *node2;
9106 
9107 	hlist_for_each_entry_safe(filter, node2,
9108 				  &pf->fdir_filter_list, fdir_node) {
9109 		hlist_del(&filter->fdir_node);
9110 		kfree(filter);
9111 	}
9112 
9113 	list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
9114 		list_del(&pit_entry->list);
9115 		kfree(pit_entry);
9116 	}
9117 	INIT_LIST_HEAD(&pf->l3_flex_pit_list);
9118 
9119 	list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
9120 		list_del(&pit_entry->list);
9121 		kfree(pit_entry);
9122 	}
9123 	INIT_LIST_HEAD(&pf->l4_flex_pit_list);
9124 
9125 	pf->fdir_pf_active_filters = 0;
9126 	i40e_reset_fdir_filter_cnt(pf);
9127 
9128 	/* Reprogram the default input set for TCP/IPv4 */
9129 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9130 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9131 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9132 
9133 	/* Reprogram the default input set for TCP/IPv6 */
9134 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP,
9135 				I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9136 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9137 
9138 	/* Reprogram the default input set for UDP/IPv4 */
9139 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
9140 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9141 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9142 
9143 	/* Reprogram the default input set for UDP/IPv6 */
9144 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP,
9145 				I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9146 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9147 
9148 	/* Reprogram the default input set for SCTP/IPv4 */
9149 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
9150 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9151 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9152 
9153 	/* Reprogram the default input set for SCTP/IPv6 */
9154 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP,
9155 				I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK |
9156 				I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9157 
9158 	/* Reprogram the default input set for Other/IPv4 */
9159 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
9160 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9161 
9162 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
9163 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9164 
9165 	/* Reprogram the default input set for Other/IPv6 */
9166 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER,
9167 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9168 
9169 	i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6,
9170 				I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
9171 }
9172 
9173 /**
9174  * i40e_cloud_filter_exit - Cleans up the cloud filters
9175  * @pf: Pointer to PF
9176  *
9177  * This function destroys the hlist where all the cloud filters
9178  * were saved.
9179  **/
9180 static void i40e_cloud_filter_exit(struct i40e_pf *pf)
9181 {
9182 	struct i40e_cloud_filter *cfilter;
9183 	struct hlist_node *node;
9184 
9185 	hlist_for_each_entry_safe(cfilter, node,
9186 				  &pf->cloud_filter_list, cloud_node) {
9187 		hlist_del(&cfilter->cloud_node);
9188 		kfree(cfilter);
9189 	}
9190 	pf->num_cloud_filters = 0;
9191 
9192 	if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
9193 	    !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
9194 		pf->flags |= I40E_FLAG_FD_SB_ENABLED;
9195 		pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
9196 		pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
9197 	}
9198 }
9199 
9200 /**
9201  * i40e_close - Disables a network interface
9202  * @netdev: network interface device structure
9203  *
9204  * The close entry point is called when an interface is de-activated
9205  * by the OS.  The hardware is still under the driver's control, but
9206  * this netdev interface is disabled.
9207  *
9208  * Returns 0, this is not allowed to fail
9209  **/
9210 int i40e_close(struct net_device *netdev)
9211 {
9212 	struct i40e_netdev_priv *np = netdev_priv(netdev);
9213 	struct i40e_vsi *vsi = np->vsi;
9214 
9215 	i40e_vsi_close(vsi);
9216 
9217 	return 0;
9218 }
9219 
9220 /**
9221  * i40e_do_reset - Start a PF or Core Reset sequence
9222  * @pf: board private structure
9223  * @reset_flags: which reset is requested
9224  * @lock_acquired: indicates whether or not the lock has been acquired
9225  * before this function was called.
9226  *
9227  * The essential difference in resets is that the PF Reset
9228  * doesn't clear the packet buffers, doesn't reset the PE
9229  * firmware, and doesn't bother the other PFs on the chip.
9230  **/
9231 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
9232 {
9233 	u32 val;
9234 
9235 	/* do the biggest reset indicated */
9236 	if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
9237 
9238 		/* Request a Global Reset
9239 		 *
9240 		 * This will start the chip's countdown to the actual full
9241 		 * chip reset event, and a warning interrupt to be sent
9242 		 * to all PFs, including the requestor.  Our handler
9243 		 * for the warning interrupt will deal with the shutdown
9244 		 * and recovery of the switch setup.
9245 		 */
9246 		dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
9247 		val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9248 		val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
9249 		wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9250 
9251 	} else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
9252 
9253 		/* Request a Core Reset
9254 		 *
9255 		 * Same as Global Reset, except does *not* include the MAC/PHY
9256 		 */
9257 		dev_dbg(&pf->pdev->dev, "CoreR requested\n");
9258 		val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
9259 		val |= I40E_GLGEN_RTRIG_CORER_MASK;
9260 		wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
9261 		i40e_flush(&pf->hw);
9262 
9263 	} else if (reset_flags & I40E_PF_RESET_FLAG) {
9264 
9265 		/* Request a PF Reset
9266 		 *
9267 		 * Resets only the PF-specific registers
9268 		 *
9269 		 * This goes directly to the tear-down and rebuild of
9270 		 * the switch, since we need to do all the recovery as
9271 		 * for the Core Reset.
9272 		 */
9273 		dev_dbg(&pf->pdev->dev, "PFR requested\n");
9274 		i40e_handle_reset_warning(pf, lock_acquired);
9275 
9276 	} else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) {
9277 		/* Request a PF Reset
9278 		 *
9279 		 * Resets PF and reinitializes PFs VSI.
9280 		 */
9281 		i40e_prep_for_reset(pf);
9282 		i40e_reset_and_rebuild(pf, true, lock_acquired);
9283 		dev_info(&pf->pdev->dev,
9284 			 pf->flags & I40E_FLAG_DISABLE_FW_LLDP ?
9285 			 "FW LLDP is disabled\n" :
9286 			 "FW LLDP is enabled\n");
9287 
9288 	} else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
9289 		int v;
9290 
9291 		/* Find the VSI(s) that requested a re-init */
9292 		dev_info(&pf->pdev->dev,
9293 			 "VSI reinit requested\n");
9294 		for (v = 0; v < pf->num_alloc_vsi; v++) {
9295 			struct i40e_vsi *vsi = pf->vsi[v];
9296 
9297 			if (vsi != NULL &&
9298 			    test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
9299 					       vsi->state))
9300 				i40e_vsi_reinit_locked(pf->vsi[v]);
9301 		}
9302 	} else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
9303 		int v;
9304 
9305 		/* Find the VSI(s) that needs to be brought down */
9306 		dev_info(&pf->pdev->dev, "VSI down requested\n");
9307 		for (v = 0; v < pf->num_alloc_vsi; v++) {
9308 			struct i40e_vsi *vsi = pf->vsi[v];
9309 
9310 			if (vsi != NULL &&
9311 			    test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
9312 					       vsi->state)) {
9313 				set_bit(__I40E_VSI_DOWN, vsi->state);
9314 				i40e_down(vsi);
9315 			}
9316 		}
9317 	} else {
9318 		dev_info(&pf->pdev->dev,
9319 			 "bad reset request 0x%08x\n", reset_flags);
9320 	}
9321 }
9322 
9323 #ifdef CONFIG_I40E_DCB
9324 /**
9325  * i40e_dcb_need_reconfig - Check if DCB needs reconfig
9326  * @pf: board private structure
9327  * @old_cfg: current DCB config
9328  * @new_cfg: new DCB config
9329  **/
9330 bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
9331 			    struct i40e_dcbx_config *old_cfg,
9332 			    struct i40e_dcbx_config *new_cfg)
9333 {
9334 	bool need_reconfig = false;
9335 
9336 	/* Check if ETS configuration has changed */
9337 	if (memcmp(&new_cfg->etscfg,
9338 		   &old_cfg->etscfg,
9339 		   sizeof(new_cfg->etscfg))) {
9340 		/* If Priority Table has changed reconfig is needed */
9341 		if (memcmp(&new_cfg->etscfg.prioritytable,
9342 			   &old_cfg->etscfg.prioritytable,
9343 			   sizeof(new_cfg->etscfg.prioritytable))) {
9344 			need_reconfig = true;
9345 			dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
9346 		}
9347 
9348 		if (memcmp(&new_cfg->etscfg.tcbwtable,
9349 			   &old_cfg->etscfg.tcbwtable,
9350 			   sizeof(new_cfg->etscfg.tcbwtable)))
9351 			dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
9352 
9353 		if (memcmp(&new_cfg->etscfg.tsatable,
9354 			   &old_cfg->etscfg.tsatable,
9355 			   sizeof(new_cfg->etscfg.tsatable)))
9356 			dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
9357 	}
9358 
9359 	/* Check if PFC configuration has changed */
9360 	if (memcmp(&new_cfg->pfc,
9361 		   &old_cfg->pfc,
9362 		   sizeof(new_cfg->pfc))) {
9363 		need_reconfig = true;
9364 		dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
9365 	}
9366 
9367 	/* Check if APP Table has changed */
9368 	if (memcmp(&new_cfg->app,
9369 		   &old_cfg->app,
9370 		   sizeof(new_cfg->app))) {
9371 		need_reconfig = true;
9372 		dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
9373 	}
9374 
9375 	dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
9376 	return need_reconfig;
9377 }
9378 
9379 /**
9380  * i40e_handle_lldp_event - Handle LLDP Change MIB event
9381  * @pf: board private structure
9382  * @e: event info posted on ARQ
9383  **/
9384 static int i40e_handle_lldp_event(struct i40e_pf *pf,
9385 				  struct i40e_arq_event_info *e)
9386 {
9387 	struct i40e_aqc_lldp_get_mib *mib =
9388 		(struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
9389 	struct i40e_hw *hw = &pf->hw;
9390 	struct i40e_dcbx_config tmp_dcbx_cfg;
9391 	bool need_reconfig = false;
9392 	int ret = 0;
9393 	u8 type;
9394 
9395 	/* X710-T*L 2.5G and 5G speeds don't support DCB */
9396 	if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9397 	    (hw->phy.link_info.link_speed &
9398 	     ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) &&
9399 	     !(pf->flags & I40E_FLAG_DCB_CAPABLE))
9400 		/* let firmware decide if the DCB should be disabled */
9401 		pf->flags |= I40E_FLAG_DCB_CAPABLE;
9402 
9403 	/* Not DCB capable or capability disabled */
9404 	if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
9405 		return ret;
9406 
9407 	/* Ignore if event is not for Nearest Bridge */
9408 	type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
9409 		& I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
9410 	dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
9411 	if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
9412 		return ret;
9413 
9414 	/* Check MIB Type and return if event for Remote MIB update */
9415 	type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
9416 	dev_dbg(&pf->pdev->dev,
9417 		"LLDP event mib type %s\n", type ? "remote" : "local");
9418 	if (type == I40E_AQ_LLDP_MIB_REMOTE) {
9419 		/* Update the remote cached instance and return */
9420 		ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
9421 				I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
9422 				&hw->remote_dcbx_config);
9423 		goto exit;
9424 	}
9425 
9426 	/* Store the old configuration */
9427 	tmp_dcbx_cfg = hw->local_dcbx_config;
9428 
9429 	/* Reset the old DCBx configuration data */
9430 	memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
9431 	/* Get updated DCBX data from firmware */
9432 	ret = i40e_get_dcb_config(&pf->hw);
9433 	if (ret) {
9434 		/* X710-T*L 2.5G and 5G speeds don't support DCB */
9435 		if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
9436 		    (hw->phy.link_info.link_speed &
9437 		     (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
9438 			dev_warn(&pf->pdev->dev,
9439 				 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
9440 			pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
9441 		} else {
9442 			dev_info(&pf->pdev->dev,
9443 				 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n",
9444 				 ERR_PTR(ret),
9445 				 i40e_aq_str(&pf->hw,
9446 					     pf->hw.aq.asq_last_status));
9447 		}
9448 		goto exit;
9449 	}
9450 
9451 	/* No change detected in DCBX configs */
9452 	if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
9453 		    sizeof(tmp_dcbx_cfg))) {
9454 		dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
9455 		goto exit;
9456 	}
9457 
9458 	need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
9459 					       &hw->local_dcbx_config);
9460 
9461 	i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
9462 
9463 	if (!need_reconfig)
9464 		goto exit;
9465 
9466 	/* Enable DCB tagging only when more than one TC */
9467 	if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
9468 		pf->flags |= I40E_FLAG_DCB_ENABLED;
9469 	else
9470 		pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9471 
9472 	set_bit(__I40E_PORT_SUSPENDED, pf->state);
9473 	/* Reconfiguration needed quiesce all VSIs */
9474 	i40e_pf_quiesce_all_vsi(pf);
9475 
9476 	/* Changes in configuration update VEB/VSI */
9477 	i40e_dcb_reconfigure(pf);
9478 
9479 	ret = i40e_resume_port_tx(pf);
9480 
9481 	clear_bit(__I40E_PORT_SUSPENDED, pf->state);
9482 	/* In case of error no point in resuming VSIs */
9483 	if (ret)
9484 		goto exit;
9485 
9486 	/* Wait for the PF's queues to be disabled */
9487 	ret = i40e_pf_wait_queues_disabled(pf);
9488 	if (ret) {
9489 		/* Schedule PF reset to recover */
9490 		set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9491 		i40e_service_event_schedule(pf);
9492 	} else {
9493 		i40e_pf_unquiesce_all_vsi(pf);
9494 		set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
9495 		set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
9496 	}
9497 
9498 exit:
9499 	return ret;
9500 }
9501 #endif /* CONFIG_I40E_DCB */
9502 
9503 /**
9504  * i40e_do_reset_safe - Protected reset path for userland calls.
9505  * @pf: board private structure
9506  * @reset_flags: which reset is requested
9507  *
9508  **/
9509 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
9510 {
9511 	rtnl_lock();
9512 	i40e_do_reset(pf, reset_flags, true);
9513 	rtnl_unlock();
9514 }
9515 
9516 /**
9517  * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event
9518  * @pf: board private structure
9519  * @e: event info posted on ARQ
9520  *
9521  * Handler for LAN Queue Overflow Event generated by the firmware for PF
9522  * and VF queues
9523  **/
9524 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
9525 					   struct i40e_arq_event_info *e)
9526 {
9527 	struct i40e_aqc_lan_overflow *data =
9528 		(struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
9529 	u32 queue = le32_to_cpu(data->prtdcb_rupto);
9530 	u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
9531 	struct i40e_hw *hw = &pf->hw;
9532 	struct i40e_vf *vf;
9533 	u16 vf_id;
9534 
9535 	dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
9536 		queue, qtx_ctl);
9537 
9538 	/* Queue belongs to VF, find the VF and issue VF reset */
9539 	if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
9540 	    >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
9541 		vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
9542 			 >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
9543 		vf_id -= hw->func_caps.vf_base_id;
9544 		vf = &pf->vf[vf_id];
9545 		i40e_vc_notify_vf_reset(vf);
9546 		/* Allow VF to process pending reset notification */
9547 		msleep(20);
9548 		i40e_reset_vf(vf, false);
9549 	}
9550 }
9551 
9552 /**
9553  * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters
9554  * @pf: board private structure
9555  **/
9556 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
9557 {
9558 	u32 val, fcnt_prog;
9559 
9560 	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9561 	fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
9562 	return fcnt_prog;
9563 }
9564 
9565 /**
9566  * i40e_get_current_fd_count - Get total FD filters programmed for this PF
9567  * @pf: board private structure
9568  **/
9569 u32 i40e_get_current_fd_count(struct i40e_pf *pf)
9570 {
9571 	u32 val, fcnt_prog;
9572 
9573 	val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
9574 	fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
9575 		    ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
9576 		      I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
9577 	return fcnt_prog;
9578 }
9579 
9580 /**
9581  * i40e_get_global_fd_count - Get total FD filters programmed on device
9582  * @pf: board private structure
9583  **/
9584 u32 i40e_get_global_fd_count(struct i40e_pf *pf)
9585 {
9586 	u32 val, fcnt_prog;
9587 
9588 	val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
9589 	fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
9590 		    ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
9591 		     I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
9592 	return fcnt_prog;
9593 }
9594 
9595 /**
9596  * i40e_reenable_fdir_sb - Restore FDir SB capability
9597  * @pf: board private structure
9598  **/
9599 static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
9600 {
9601 	if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
9602 		if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
9603 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
9604 			dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
9605 }
9606 
9607 /**
9608  * i40e_reenable_fdir_atr - Restore FDir ATR capability
9609  * @pf: board private structure
9610  **/
9611 static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
9612 {
9613 	if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
9614 		/* ATR uses the same filtering logic as SB rules. It only
9615 		 * functions properly if the input set mask is at the default
9616 		 * settings. It is safe to restore the default input set
9617 		 * because there are no active TCPv4 filter rules.
9618 		 */
9619 		i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
9620 					I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
9621 					I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
9622 
9623 		if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
9624 		    (I40E_DEBUG_FD & pf->hw.debug_mask))
9625 			dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
9626 	}
9627 }
9628 
9629 /**
9630  * i40e_delete_invalid_filter - Delete an invalid FDIR filter
9631  * @pf: board private structure
9632  * @filter: FDir filter to remove
9633  */
9634 static void i40e_delete_invalid_filter(struct i40e_pf *pf,
9635 				       struct i40e_fdir_filter *filter)
9636 {
9637 	/* Update counters */
9638 	pf->fdir_pf_active_filters--;
9639 	pf->fd_inv = 0;
9640 
9641 	switch (filter->flow_type) {
9642 	case TCP_V4_FLOW:
9643 		pf->fd_tcp4_filter_cnt--;
9644 		break;
9645 	case UDP_V4_FLOW:
9646 		pf->fd_udp4_filter_cnt--;
9647 		break;
9648 	case SCTP_V4_FLOW:
9649 		pf->fd_sctp4_filter_cnt--;
9650 		break;
9651 	case TCP_V6_FLOW:
9652 		pf->fd_tcp6_filter_cnt--;
9653 		break;
9654 	case UDP_V6_FLOW:
9655 		pf->fd_udp6_filter_cnt--;
9656 		break;
9657 	case SCTP_V6_FLOW:
9658 		pf->fd_udp6_filter_cnt--;
9659 		break;
9660 	case IP_USER_FLOW:
9661 		switch (filter->ipl4_proto) {
9662 		case IPPROTO_TCP:
9663 			pf->fd_tcp4_filter_cnt--;
9664 			break;
9665 		case IPPROTO_UDP:
9666 			pf->fd_udp4_filter_cnt--;
9667 			break;
9668 		case IPPROTO_SCTP:
9669 			pf->fd_sctp4_filter_cnt--;
9670 			break;
9671 		case IPPROTO_IP:
9672 			pf->fd_ip4_filter_cnt--;
9673 			break;
9674 		}
9675 		break;
9676 	case IPV6_USER_FLOW:
9677 		switch (filter->ipl4_proto) {
9678 		case IPPROTO_TCP:
9679 			pf->fd_tcp6_filter_cnt--;
9680 			break;
9681 		case IPPROTO_UDP:
9682 			pf->fd_udp6_filter_cnt--;
9683 			break;
9684 		case IPPROTO_SCTP:
9685 			pf->fd_sctp6_filter_cnt--;
9686 			break;
9687 		case IPPROTO_IP:
9688 			pf->fd_ip6_filter_cnt--;
9689 			break;
9690 		}
9691 		break;
9692 	}
9693 
9694 	/* Remove the filter from the list and free memory */
9695 	hlist_del(&filter->fdir_node);
9696 	kfree(filter);
9697 }
9698 
9699 /**
9700  * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled
9701  * @pf: board private structure
9702  **/
9703 void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
9704 {
9705 	struct i40e_fdir_filter *filter;
9706 	u32 fcnt_prog, fcnt_avail;
9707 	struct hlist_node *node;
9708 
9709 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9710 		return;
9711 
9712 	/* Check if we have enough room to re-enable FDir SB capability. */
9713 	fcnt_prog = i40e_get_global_fd_count(pf);
9714 	fcnt_avail = pf->fdir_pf_filter_count;
9715 	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
9716 	    (pf->fd_add_err == 0) ||
9717 	    (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
9718 		i40e_reenable_fdir_sb(pf);
9719 
9720 	/* We should wait for even more space before re-enabling ATR.
9721 	 * Additionally, we cannot enable ATR as long as we still have TCP SB
9722 	 * rules active.
9723 	 */
9724 	if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
9725 	    pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0)
9726 		i40e_reenable_fdir_atr(pf);
9727 
9728 	/* if hw had a problem adding a filter, delete it */
9729 	if (pf->fd_inv > 0) {
9730 		hlist_for_each_entry_safe(filter, node,
9731 					  &pf->fdir_filter_list, fdir_node)
9732 			if (filter->fd_id == pf->fd_inv)
9733 				i40e_delete_invalid_filter(pf, filter);
9734 	}
9735 }
9736 
9737 #define I40E_MIN_FD_FLUSH_INTERVAL 10
9738 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
9739 /**
9740  * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB
9741  * @pf: board private structure
9742  **/
9743 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
9744 {
9745 	unsigned long min_flush_time;
9746 	int flush_wait_retry = 50;
9747 	bool disable_atr = false;
9748 	int fd_room;
9749 	int reg;
9750 
9751 	if (!time_after(jiffies, pf->fd_flush_timestamp +
9752 				 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
9753 		return;
9754 
9755 	/* If the flush is happening too quick and we have mostly SB rules we
9756 	 * should not re-enable ATR for some time.
9757 	 */
9758 	min_flush_time = pf->fd_flush_timestamp +
9759 			 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
9760 	fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
9761 
9762 	if (!(time_after(jiffies, min_flush_time)) &&
9763 	    (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
9764 		if (I40E_DEBUG_FD & pf->hw.debug_mask)
9765 			dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
9766 		disable_atr = true;
9767 	}
9768 
9769 	pf->fd_flush_timestamp = jiffies;
9770 	set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9771 	/* flush all filters */
9772 	wr32(&pf->hw, I40E_PFQF_CTL_1,
9773 	     I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
9774 	i40e_flush(&pf->hw);
9775 	pf->fd_flush_cnt++;
9776 	pf->fd_add_err = 0;
9777 	do {
9778 		/* Check FD flush status every 5-6msec */
9779 		usleep_range(5000, 6000);
9780 		reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
9781 		if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
9782 			break;
9783 	} while (flush_wait_retry--);
9784 	if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
9785 		dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
9786 	} else {
9787 		/* replay sideband filters */
9788 		i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
9789 		if (!disable_atr && !pf->fd_tcp4_filter_cnt)
9790 			clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
9791 		clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
9792 		if (I40E_DEBUG_FD & pf->hw.debug_mask)
9793 			dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
9794 	}
9795 }
9796 
9797 /**
9798  * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed
9799  * @pf: board private structure
9800  **/
9801 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
9802 {
9803 	return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
9804 }
9805 
9806 /**
9807  * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table
9808  * @pf: board private structure
9809  **/
9810 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
9811 {
9812 
9813 	/* if interface is down do nothing */
9814 	if (test_bit(__I40E_DOWN, pf->state))
9815 		return;
9816 
9817 	if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
9818 		i40e_fdir_flush_and_replay(pf);
9819 
9820 	i40e_fdir_check_and_reenable(pf);
9821 
9822 }
9823 
9824 /**
9825  * i40e_vsi_link_event - notify VSI of a link event
9826  * @vsi: vsi to be notified
9827  * @link_up: link up or down
9828  **/
9829 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
9830 {
9831 	if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
9832 		return;
9833 
9834 	switch (vsi->type) {
9835 	case I40E_VSI_MAIN:
9836 		if (!vsi->netdev || !vsi->netdev_registered)
9837 			break;
9838 
9839 		if (link_up) {
9840 			netif_carrier_on(vsi->netdev);
9841 			netif_tx_wake_all_queues(vsi->netdev);
9842 		} else {
9843 			netif_carrier_off(vsi->netdev);
9844 			netif_tx_stop_all_queues(vsi->netdev);
9845 		}
9846 		break;
9847 
9848 	case I40E_VSI_SRIOV:
9849 	case I40E_VSI_VMDQ2:
9850 	case I40E_VSI_CTRL:
9851 	case I40E_VSI_IWARP:
9852 	case I40E_VSI_MIRROR:
9853 	default:
9854 		/* there is no notification for other VSIs */
9855 		break;
9856 	}
9857 }
9858 
9859 /**
9860  * i40e_veb_link_event - notify elements on the veb of a link event
9861  * @veb: veb to be notified
9862  * @link_up: link up or down
9863  **/
9864 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9865 {
9866 	struct i40e_pf *pf;
9867 	int i;
9868 
9869 	if (!veb || !veb->pf)
9870 		return;
9871 	pf = veb->pf;
9872 
9873 	/* depth first... */
9874 	for (i = 0; i < I40E_MAX_VEB; i++)
9875 		if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
9876 			i40e_veb_link_event(pf->veb[i], link_up);
9877 
9878 	/* ... now the local VSIs */
9879 	for (i = 0; i < pf->num_alloc_vsi; i++)
9880 		if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
9881 			i40e_vsi_link_event(pf->vsi[i], link_up);
9882 }
9883 
9884 /**
9885  * i40e_link_event - Update netif_carrier status
9886  * @pf: board private structure
9887  **/
9888 static void i40e_link_event(struct i40e_pf *pf)
9889 {
9890 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9891 	u8 new_link_speed, old_link_speed;
9892 	bool new_link, old_link;
9893 	int status;
9894 #ifdef CONFIG_I40E_DCB
9895 	int err;
9896 #endif /* CONFIG_I40E_DCB */
9897 
9898 	/* set this to force the get_link_status call to refresh state */
9899 	pf->hw.phy.get_link_info = true;
9900 	old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9901 	status = i40e_get_link_status(&pf->hw, &new_link);
9902 
9903 	/* On success, disable temp link polling */
9904 	if (status == I40E_SUCCESS) {
9905 		clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9906 	} else {
9907 		/* Enable link polling temporarily until i40e_get_link_status
9908 		 * returns I40E_SUCCESS
9909 		 */
9910 		set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9911 		dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9912 			status);
9913 		return;
9914 	}
9915 
9916 	old_link_speed = pf->hw.phy.link_info_old.link_speed;
9917 	new_link_speed = pf->hw.phy.link_info.link_speed;
9918 
9919 	if (new_link == old_link &&
9920 	    new_link_speed == old_link_speed &&
9921 	    (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9922 	     new_link == netif_carrier_ok(vsi->netdev)))
9923 		return;
9924 
9925 	i40e_print_link_message(vsi, new_link);
9926 
9927 	/* Notify the base of the switch tree connected to
9928 	 * the link.  Floating VEBs are not notified.
9929 	 */
9930 	if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9931 		i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9932 	else
9933 		i40e_vsi_link_event(vsi, new_link);
9934 
9935 	if (pf->vf)
9936 		i40e_vc_notify_link_state(pf);
9937 
9938 	if (pf->flags & I40E_FLAG_PTP)
9939 		i40e_ptp_set_increment(pf);
9940 #ifdef CONFIG_I40E_DCB
9941 	if (new_link == old_link)
9942 		return;
9943 	/* Not SW DCB so firmware will take care of default settings */
9944 	if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED)
9945 		return;
9946 
9947 	/* We cover here only link down, as after link up in case of SW DCB
9948 	 * SW LLDP agent will take care of setting it up
9949 	 */
9950 	if (!new_link) {
9951 		dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n");
9952 		memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg));
9953 		err = i40e_dcb_sw_default_config(pf);
9954 		if (err) {
9955 			pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
9956 				       I40E_FLAG_DCB_ENABLED);
9957 		} else {
9958 			pf->dcbx_cap = DCB_CAP_DCBX_HOST |
9959 				       DCB_CAP_DCBX_VER_IEEE;
9960 			pf->flags |= I40E_FLAG_DCB_CAPABLE;
9961 			pf->flags &= ~I40E_FLAG_DCB_ENABLED;
9962 		}
9963 	}
9964 #endif /* CONFIG_I40E_DCB */
9965 }
9966 
9967 /**
9968  * i40e_watchdog_subtask - periodic checks not using event driven response
9969  * @pf: board private structure
9970  **/
9971 static void i40e_watchdog_subtask(struct i40e_pf *pf)
9972 {
9973 	int i;
9974 
9975 	/* if interface is down do nothing */
9976 	if (test_bit(__I40E_DOWN, pf->state) ||
9977 	    test_bit(__I40E_CONFIG_BUSY, pf->state))
9978 		return;
9979 
9980 	/* make sure we don't do these things too often */
9981 	if (time_before(jiffies, (pf->service_timer_previous +
9982 				  pf->service_timer_period)))
9983 		return;
9984 	pf->service_timer_previous = jiffies;
9985 
9986 	if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
9987 	    test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
9988 		i40e_link_event(pf);
9989 
9990 	/* Update the stats for active netdevs so the network stack
9991 	 * can look at updated numbers whenever it cares to
9992 	 */
9993 	for (i = 0; i < pf->num_alloc_vsi; i++)
9994 		if (pf->vsi[i] && pf->vsi[i]->netdev)
9995 			i40e_update_stats(pf->vsi[i]);
9996 
9997 	if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
9998 		/* Update the stats for the active switching components */
9999 		for (i = 0; i < I40E_MAX_VEB; i++)
10000 			if (pf->veb[i])
10001 				i40e_update_veb_stats(pf->veb[i]);
10002 	}
10003 
10004 	i40e_ptp_rx_hang(pf);
10005 	i40e_ptp_tx_hang(pf);
10006 }
10007 
10008 /**
10009  * i40e_reset_subtask - Set up for resetting the device and driver
10010  * @pf: board private structure
10011  **/
10012 static void i40e_reset_subtask(struct i40e_pf *pf)
10013 {
10014 	u32 reset_flags = 0;
10015 
10016 	if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
10017 		reset_flags |= BIT(__I40E_REINIT_REQUESTED);
10018 		clear_bit(__I40E_REINIT_REQUESTED, pf->state);
10019 	}
10020 	if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
10021 		reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
10022 		clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
10023 	}
10024 	if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
10025 		reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
10026 		clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
10027 	}
10028 	if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
10029 		reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
10030 		clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
10031 	}
10032 	if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
10033 		reset_flags |= BIT(__I40E_DOWN_REQUESTED);
10034 		clear_bit(__I40E_DOWN_REQUESTED, pf->state);
10035 	}
10036 
10037 	/* If there's a recovery already waiting, it takes
10038 	 * precedence before starting a new reset sequence.
10039 	 */
10040 	if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
10041 		i40e_prep_for_reset(pf);
10042 		i40e_reset(pf);
10043 		i40e_rebuild(pf, false, false);
10044 	}
10045 
10046 	/* If we're already down or resetting, just bail */
10047 	if (reset_flags &&
10048 	    !test_bit(__I40E_DOWN, pf->state) &&
10049 	    !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
10050 		i40e_do_reset(pf, reset_flags, false);
10051 	}
10052 }
10053 
10054 /**
10055  * i40e_handle_link_event - Handle link event
10056  * @pf: board private structure
10057  * @e: event info posted on ARQ
10058  **/
10059 static void i40e_handle_link_event(struct i40e_pf *pf,
10060 				   struct i40e_arq_event_info *e)
10061 {
10062 	struct i40e_aqc_get_link_status *status =
10063 		(struct i40e_aqc_get_link_status *)&e->desc.params.raw;
10064 
10065 	/* Do a new status request to re-enable LSE reporting
10066 	 * and load new status information into the hw struct
10067 	 * This completely ignores any state information
10068 	 * in the ARQ event info, instead choosing to always
10069 	 * issue the AQ update link status command.
10070 	 */
10071 	i40e_link_event(pf);
10072 
10073 	/* Check if module meets thermal requirements */
10074 	if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
10075 		dev_err(&pf->pdev->dev,
10076 			"Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
10077 		dev_err(&pf->pdev->dev,
10078 			"Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10079 	} else {
10080 		/* check for unqualified module, if link is down, suppress
10081 		 * the message if link was forced to be down.
10082 		 */
10083 		if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
10084 		    (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
10085 		    (!(status->link_info & I40E_AQ_LINK_UP)) &&
10086 		    (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
10087 			dev_err(&pf->pdev->dev,
10088 				"Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
10089 			dev_err(&pf->pdev->dev,
10090 				"Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
10091 		}
10092 	}
10093 }
10094 
10095 /**
10096  * i40e_clean_adminq_subtask - Clean the AdminQ rings
10097  * @pf: board private structure
10098  **/
10099 static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
10100 {
10101 	struct i40e_arq_event_info event;
10102 	struct i40e_hw *hw = &pf->hw;
10103 	u16 pending, i = 0;
10104 	u16 opcode;
10105 	u32 oldval;
10106 	int ret;
10107 	u32 val;
10108 
10109 	/* Do not run clean AQ when PF reset fails */
10110 	if (test_bit(__I40E_RESET_FAILED, pf->state))
10111 		return;
10112 
10113 	/* check for error indications */
10114 	val = rd32(&pf->hw, pf->hw.aq.arq.len);
10115 	oldval = val;
10116 	if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
10117 		if (hw->debug_mask & I40E_DEBUG_AQ)
10118 			dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
10119 		val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
10120 	}
10121 	if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
10122 		if (hw->debug_mask & I40E_DEBUG_AQ)
10123 			dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
10124 		val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
10125 		pf->arq_overflows++;
10126 	}
10127 	if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
10128 		if (hw->debug_mask & I40E_DEBUG_AQ)
10129 			dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
10130 		val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
10131 	}
10132 	if (oldval != val)
10133 		wr32(&pf->hw, pf->hw.aq.arq.len, val);
10134 
10135 	val = rd32(&pf->hw, pf->hw.aq.asq.len);
10136 	oldval = val;
10137 	if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
10138 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10139 			dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
10140 		val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
10141 	}
10142 	if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
10143 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10144 			dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
10145 		val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
10146 	}
10147 	if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
10148 		if (pf->hw.debug_mask & I40E_DEBUG_AQ)
10149 			dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
10150 		val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
10151 	}
10152 	if (oldval != val)
10153 		wr32(&pf->hw, pf->hw.aq.asq.len, val);
10154 
10155 	event.buf_len = I40E_MAX_AQ_BUF_SIZE;
10156 	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
10157 	if (!event.msg_buf)
10158 		return;
10159 
10160 	do {
10161 		ret = i40e_clean_arq_element(hw, &event, &pending);
10162 		if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
10163 			break;
10164 		else if (ret) {
10165 			dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
10166 			break;
10167 		}
10168 
10169 		opcode = le16_to_cpu(event.desc.opcode);
10170 		switch (opcode) {
10171 
10172 		case i40e_aqc_opc_get_link_status:
10173 			rtnl_lock();
10174 			i40e_handle_link_event(pf, &event);
10175 			rtnl_unlock();
10176 			break;
10177 		case i40e_aqc_opc_send_msg_to_pf:
10178 			ret = i40e_vc_process_vf_msg(pf,
10179 					le16_to_cpu(event.desc.retval),
10180 					le32_to_cpu(event.desc.cookie_high),
10181 					le32_to_cpu(event.desc.cookie_low),
10182 					event.msg_buf,
10183 					event.msg_len);
10184 			break;
10185 		case i40e_aqc_opc_lldp_update_mib:
10186 			dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
10187 #ifdef CONFIG_I40E_DCB
10188 			rtnl_lock();
10189 			i40e_handle_lldp_event(pf, &event);
10190 			rtnl_unlock();
10191 #endif /* CONFIG_I40E_DCB */
10192 			break;
10193 		case i40e_aqc_opc_event_lan_overflow:
10194 			dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
10195 			i40e_handle_lan_overflow_event(pf, &event);
10196 			break;
10197 		case i40e_aqc_opc_send_msg_to_peer:
10198 			dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
10199 			break;
10200 		case i40e_aqc_opc_nvm_erase:
10201 		case i40e_aqc_opc_nvm_update:
10202 		case i40e_aqc_opc_oem_post_update:
10203 			i40e_debug(&pf->hw, I40E_DEBUG_NVM,
10204 				   "ARQ NVM operation 0x%04x completed\n",
10205 				   opcode);
10206 			break;
10207 		default:
10208 			dev_info(&pf->pdev->dev,
10209 				 "ARQ: Unknown event 0x%04x ignored\n",
10210 				 opcode);
10211 			break;
10212 		}
10213 	} while (i++ < pf->adminq_work_limit);
10214 
10215 	if (i < pf->adminq_work_limit)
10216 		clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
10217 
10218 	/* re-enable Admin queue interrupt cause */
10219 	val = rd32(hw, I40E_PFINT_ICR0_ENA);
10220 	val |=  I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
10221 	wr32(hw, I40E_PFINT_ICR0_ENA, val);
10222 	i40e_flush(hw);
10223 
10224 	kfree(event.msg_buf);
10225 }
10226 
10227 /**
10228  * i40e_verify_eeprom - make sure eeprom is good to use
10229  * @pf: board private structure
10230  **/
10231 static void i40e_verify_eeprom(struct i40e_pf *pf)
10232 {
10233 	int err;
10234 
10235 	err = i40e_diag_eeprom_test(&pf->hw);
10236 	if (err) {
10237 		/* retry in case of garbage read */
10238 		err = i40e_diag_eeprom_test(&pf->hw);
10239 		if (err) {
10240 			dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
10241 				 err);
10242 			set_bit(__I40E_BAD_EEPROM, pf->state);
10243 		}
10244 	}
10245 
10246 	if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
10247 		dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
10248 		clear_bit(__I40E_BAD_EEPROM, pf->state);
10249 	}
10250 }
10251 
10252 /**
10253  * i40e_enable_pf_switch_lb
10254  * @pf: pointer to the PF structure
10255  *
10256  * enable switch loop back or die - no point in a return value
10257  **/
10258 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
10259 {
10260 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10261 	struct i40e_vsi_context ctxt;
10262 	int ret;
10263 
10264 	ctxt.seid = pf->main_vsi_seid;
10265 	ctxt.pf_num = pf->hw.pf_id;
10266 	ctxt.vf_num = 0;
10267 	ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10268 	if (ret) {
10269 		dev_info(&pf->pdev->dev,
10270 			 "couldn't get PF vsi config, err %pe aq_err %s\n",
10271 			 ERR_PTR(ret),
10272 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10273 		return;
10274 	}
10275 	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10276 	ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10277 	ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10278 
10279 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10280 	if (ret) {
10281 		dev_info(&pf->pdev->dev,
10282 			 "update vsi switch failed, err %pe aq_err %s\n",
10283 			 ERR_PTR(ret),
10284 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10285 	}
10286 }
10287 
10288 /**
10289  * i40e_disable_pf_switch_lb
10290  * @pf: pointer to the PF structure
10291  *
10292  * disable switch loop back or die - no point in a return value
10293  **/
10294 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
10295 {
10296 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10297 	struct i40e_vsi_context ctxt;
10298 	int ret;
10299 
10300 	ctxt.seid = pf->main_vsi_seid;
10301 	ctxt.pf_num = pf->hw.pf_id;
10302 	ctxt.vf_num = 0;
10303 	ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
10304 	if (ret) {
10305 		dev_info(&pf->pdev->dev,
10306 			 "couldn't get PF vsi config, err %pe aq_err %s\n",
10307 			 ERR_PTR(ret),
10308 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10309 		return;
10310 	}
10311 	ctxt.flags = I40E_AQ_VSI_TYPE_PF;
10312 	ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
10313 	ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
10314 
10315 	ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
10316 	if (ret) {
10317 		dev_info(&pf->pdev->dev,
10318 			 "update vsi switch failed, err %pe aq_err %s\n",
10319 			 ERR_PTR(ret),
10320 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10321 	}
10322 }
10323 
10324 /**
10325  * i40e_config_bridge_mode - Configure the HW bridge mode
10326  * @veb: pointer to the bridge instance
10327  *
10328  * Configure the loop back mode for the LAN VSI that is downlink to the
10329  * specified HW bridge instance. It is expected this function is called
10330  * when a new HW bridge is instantiated.
10331  **/
10332 static void i40e_config_bridge_mode(struct i40e_veb *veb)
10333 {
10334 	struct i40e_pf *pf = veb->pf;
10335 
10336 	if (pf->hw.debug_mask & I40E_DEBUG_LAN)
10337 		dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
10338 			 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
10339 	if (veb->bridge_mode & BRIDGE_MODE_VEPA)
10340 		i40e_disable_pf_switch_lb(pf);
10341 	else
10342 		i40e_enable_pf_switch_lb(pf);
10343 }
10344 
10345 /**
10346  * i40e_reconstitute_veb - rebuild the VEB and anything connected to it
10347  * @veb: pointer to the VEB instance
10348  *
10349  * This is a recursive function that first builds the attached VSIs then
10350  * recurses in to build the next layer of VEB.  We track the connections
10351  * through our own index numbers because the seid's from the HW could
10352  * change across the reset.
10353  **/
10354 static int i40e_reconstitute_veb(struct i40e_veb *veb)
10355 {
10356 	struct i40e_vsi *ctl_vsi = NULL;
10357 	struct i40e_pf *pf = veb->pf;
10358 	int v, veb_idx;
10359 	int ret;
10360 
10361 	/* build VSI that owns this VEB, temporarily attached to base VEB */
10362 	for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
10363 		if (pf->vsi[v] &&
10364 		    pf->vsi[v]->veb_idx == veb->idx &&
10365 		    pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
10366 			ctl_vsi = pf->vsi[v];
10367 			break;
10368 		}
10369 	}
10370 	if (!ctl_vsi) {
10371 		dev_info(&pf->pdev->dev,
10372 			 "missing owner VSI for veb_idx %d\n", veb->idx);
10373 		ret = -ENOENT;
10374 		goto end_reconstitute;
10375 	}
10376 	if (ctl_vsi != pf->vsi[pf->lan_vsi])
10377 		ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
10378 	ret = i40e_add_vsi(ctl_vsi);
10379 	if (ret) {
10380 		dev_info(&pf->pdev->dev,
10381 			 "rebuild of veb_idx %d owner VSI failed: %d\n",
10382 			 veb->idx, ret);
10383 		goto end_reconstitute;
10384 	}
10385 	i40e_vsi_reset_stats(ctl_vsi);
10386 
10387 	/* create the VEB in the switch and move the VSI onto the VEB */
10388 	ret = i40e_add_veb(veb, ctl_vsi);
10389 	if (ret)
10390 		goto end_reconstitute;
10391 
10392 	if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
10393 		veb->bridge_mode = BRIDGE_MODE_VEB;
10394 	else
10395 		veb->bridge_mode = BRIDGE_MODE_VEPA;
10396 	i40e_config_bridge_mode(veb);
10397 
10398 	/* create the remaining VSIs attached to this VEB */
10399 	for (v = 0; v < pf->num_alloc_vsi; v++) {
10400 		if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
10401 			continue;
10402 
10403 		if (pf->vsi[v]->veb_idx == veb->idx) {
10404 			struct i40e_vsi *vsi = pf->vsi[v];
10405 
10406 			vsi->uplink_seid = veb->seid;
10407 			ret = i40e_add_vsi(vsi);
10408 			if (ret) {
10409 				dev_info(&pf->pdev->dev,
10410 					 "rebuild of vsi_idx %d failed: %d\n",
10411 					 v, ret);
10412 				goto end_reconstitute;
10413 			}
10414 			i40e_vsi_reset_stats(vsi);
10415 		}
10416 	}
10417 
10418 	/* create any VEBs attached to this VEB - RECURSION */
10419 	for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
10420 		if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
10421 			pf->veb[veb_idx]->uplink_seid = veb->seid;
10422 			ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
10423 			if (ret)
10424 				break;
10425 		}
10426 	}
10427 
10428 end_reconstitute:
10429 	return ret;
10430 }
10431 
10432 /**
10433  * i40e_get_capabilities - get info about the HW
10434  * @pf: the PF struct
10435  * @list_type: AQ capability to be queried
10436  **/
10437 static int i40e_get_capabilities(struct i40e_pf *pf,
10438 				 enum i40e_admin_queue_opc list_type)
10439 {
10440 	struct i40e_aqc_list_capabilities_element_resp *cap_buf;
10441 	u16 data_size;
10442 	int buf_len;
10443 	int err;
10444 
10445 	buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
10446 	do {
10447 		cap_buf = kzalloc(buf_len, GFP_KERNEL);
10448 		if (!cap_buf)
10449 			return -ENOMEM;
10450 
10451 		/* this loads the data into the hw struct for us */
10452 		err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
10453 						    &data_size, list_type,
10454 						    NULL);
10455 		/* data loaded, buffer no longer needed */
10456 		kfree(cap_buf);
10457 
10458 		if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
10459 			/* retry with a larger buffer */
10460 			buf_len = data_size;
10461 		} else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) {
10462 			dev_info(&pf->pdev->dev,
10463 				 "capability discovery failed, err %pe aq_err %s\n",
10464 				 ERR_PTR(err),
10465 				 i40e_aq_str(&pf->hw,
10466 					     pf->hw.aq.asq_last_status));
10467 			return -ENODEV;
10468 		}
10469 	} while (err);
10470 
10471 	if (pf->hw.debug_mask & I40E_DEBUG_USER) {
10472 		if (list_type == i40e_aqc_opc_list_func_capabilities) {
10473 			dev_info(&pf->pdev->dev,
10474 				 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
10475 				 pf->hw.pf_id, pf->hw.func_caps.num_vfs,
10476 				 pf->hw.func_caps.num_msix_vectors,
10477 				 pf->hw.func_caps.num_msix_vectors_vf,
10478 				 pf->hw.func_caps.fd_filters_guaranteed,
10479 				 pf->hw.func_caps.fd_filters_best_effort,
10480 				 pf->hw.func_caps.num_tx_qp,
10481 				 pf->hw.func_caps.num_vsis);
10482 		} else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
10483 			dev_info(&pf->pdev->dev,
10484 				 "switch_mode=0x%04x, function_valid=0x%08x\n",
10485 				 pf->hw.dev_caps.switch_mode,
10486 				 pf->hw.dev_caps.valid_functions);
10487 			dev_info(&pf->pdev->dev,
10488 				 "SR-IOV=%d, num_vfs for all function=%u\n",
10489 				 pf->hw.dev_caps.sr_iov_1_1,
10490 				 pf->hw.dev_caps.num_vfs);
10491 			dev_info(&pf->pdev->dev,
10492 				 "num_vsis=%u, num_rx:%u, num_tx=%u\n",
10493 				 pf->hw.dev_caps.num_vsis,
10494 				 pf->hw.dev_caps.num_rx_qp,
10495 				 pf->hw.dev_caps.num_tx_qp);
10496 		}
10497 	}
10498 	if (list_type == i40e_aqc_opc_list_func_capabilities) {
10499 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
10500 		       + pf->hw.func_caps.num_vfs)
10501 		if (pf->hw.revision_id == 0 &&
10502 		    pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
10503 			dev_info(&pf->pdev->dev,
10504 				 "got num_vsis %d, setting num_vsis to %d\n",
10505 				 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
10506 			pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
10507 		}
10508 	}
10509 	return 0;
10510 }
10511 
10512 static int i40e_vsi_clear(struct i40e_vsi *vsi);
10513 
10514 /**
10515  * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband
10516  * @pf: board private structure
10517  **/
10518 static void i40e_fdir_sb_setup(struct i40e_pf *pf)
10519 {
10520 	struct i40e_vsi *vsi;
10521 
10522 	/* quick workaround for an NVM issue that leaves a critical register
10523 	 * uninitialized
10524 	 */
10525 	if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
10526 		static const u32 hkey[] = {
10527 			0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
10528 			0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
10529 			0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
10530 			0x95b3a76d};
10531 		int i;
10532 
10533 		for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
10534 			wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
10535 	}
10536 
10537 	if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
10538 		return;
10539 
10540 	/* find existing VSI and see if it needs configuring */
10541 	vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10542 
10543 	/* create a new VSI if none exists */
10544 	if (!vsi) {
10545 		vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
10546 				     pf->vsi[pf->lan_vsi]->seid, 0);
10547 		if (!vsi) {
10548 			dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
10549 			pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
10550 			pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
10551 			return;
10552 		}
10553 	}
10554 
10555 	i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
10556 }
10557 
10558 /**
10559  * i40e_fdir_teardown - release the Flow Director resources
10560  * @pf: board private structure
10561  **/
10562 static void i40e_fdir_teardown(struct i40e_pf *pf)
10563 {
10564 	struct i40e_vsi *vsi;
10565 
10566 	i40e_fdir_filter_exit(pf);
10567 	vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
10568 	if (vsi)
10569 		i40e_vsi_release(vsi);
10570 }
10571 
10572 /**
10573  * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs
10574  * @vsi: PF main vsi
10575  * @seid: seid of main or channel VSIs
10576  *
10577  * Rebuilds cloud filters associated with main VSI and channel VSIs if they
10578  * existed before reset
10579  **/
10580 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
10581 {
10582 	struct i40e_cloud_filter *cfilter;
10583 	struct i40e_pf *pf = vsi->back;
10584 	struct hlist_node *node;
10585 	int ret;
10586 
10587 	/* Add cloud filters back if they exist */
10588 	hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
10589 				  cloud_node) {
10590 		if (cfilter->seid != seid)
10591 			continue;
10592 
10593 		if (cfilter->dst_port)
10594 			ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
10595 								true);
10596 		else
10597 			ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
10598 
10599 		if (ret) {
10600 			dev_dbg(&pf->pdev->dev,
10601 				"Failed to rebuild cloud filter, err %pe aq_err %s\n",
10602 				ERR_PTR(ret),
10603 				i40e_aq_str(&pf->hw,
10604 					    pf->hw.aq.asq_last_status));
10605 			return ret;
10606 		}
10607 	}
10608 	return 0;
10609 }
10610 
10611 /**
10612  * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset
10613  * @vsi: PF main vsi
10614  *
10615  * Rebuilds channel VSIs if they existed before reset
10616  **/
10617 static int i40e_rebuild_channels(struct i40e_vsi *vsi)
10618 {
10619 	struct i40e_channel *ch, *ch_tmp;
10620 	int ret;
10621 
10622 	if (list_empty(&vsi->ch_list))
10623 		return 0;
10624 
10625 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
10626 		if (!ch->initialized)
10627 			break;
10628 		/* Proceed with creation of channel (VMDq2) VSI */
10629 		ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
10630 		if (ret) {
10631 			dev_info(&vsi->back->pdev->dev,
10632 				 "failed to rebuild channels using uplink_seid %u\n",
10633 				 vsi->uplink_seid);
10634 			return ret;
10635 		}
10636 		/* Reconfigure TX queues using QTX_CTL register */
10637 		ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
10638 		if (ret) {
10639 			dev_info(&vsi->back->pdev->dev,
10640 				 "failed to configure TX rings for channel %u\n",
10641 				 ch->seid);
10642 			return ret;
10643 		}
10644 		/* update 'next_base_queue' */
10645 		vsi->next_base_queue = vsi->next_base_queue +
10646 							ch->num_queue_pairs;
10647 		if (ch->max_tx_rate) {
10648 			u64 credits = ch->max_tx_rate;
10649 
10650 			if (i40e_set_bw_limit(vsi, ch->seid,
10651 					      ch->max_tx_rate))
10652 				return -EINVAL;
10653 
10654 			do_div(credits, I40E_BW_CREDIT_DIVISOR);
10655 			dev_dbg(&vsi->back->pdev->dev,
10656 				"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10657 				ch->max_tx_rate,
10658 				credits,
10659 				ch->seid);
10660 		}
10661 		ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
10662 		if (ret) {
10663 			dev_dbg(&vsi->back->pdev->dev,
10664 				"Failed to rebuild cloud filters for channel VSI %u\n",
10665 				ch->seid);
10666 			return ret;
10667 		}
10668 	}
10669 	return 0;
10670 }
10671 
10672 /**
10673  * i40e_clean_xps_state - clean xps state for every tx_ring
10674  * @vsi: ptr to the VSI
10675  **/
10676 static void i40e_clean_xps_state(struct i40e_vsi *vsi)
10677 {
10678 	int i;
10679 
10680 	if (vsi->tx_rings)
10681 		for (i = 0; i < vsi->num_queue_pairs; i++)
10682 			if (vsi->tx_rings[i])
10683 				clear_bit(__I40E_TX_XPS_INIT_DONE,
10684 					  vsi->tx_rings[i]->state);
10685 }
10686 
10687 /**
10688  * i40e_prep_for_reset - prep for the core to reset
10689  * @pf: board private structure
10690  *
10691  * Close up the VFs and other things in prep for PF Reset.
10692   **/
10693 static void i40e_prep_for_reset(struct i40e_pf *pf)
10694 {
10695 	struct i40e_hw *hw = &pf->hw;
10696 	int ret = 0;
10697 	u32 v;
10698 
10699 	clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
10700 	if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
10701 		return;
10702 	if (i40e_check_asq_alive(&pf->hw))
10703 		i40e_vc_notify_reset(pf);
10704 
10705 	dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
10706 
10707 	/* quiesce the VSIs and their queues that are not already DOWN */
10708 	i40e_pf_quiesce_all_vsi(pf);
10709 
10710 	for (v = 0; v < pf->num_alloc_vsi; v++) {
10711 		if (pf->vsi[v]) {
10712 			i40e_clean_xps_state(pf->vsi[v]);
10713 			pf->vsi[v]->seid = 0;
10714 		}
10715 	}
10716 
10717 	i40e_shutdown_adminq(&pf->hw);
10718 
10719 	/* call shutdown HMC */
10720 	if (hw->hmc.hmc_obj) {
10721 		ret = i40e_shutdown_lan_hmc(hw);
10722 		if (ret)
10723 			dev_warn(&pf->pdev->dev,
10724 				 "shutdown_lan_hmc failed: %d\n", ret);
10725 	}
10726 
10727 	/* Save the current PTP time so that we can restore the time after the
10728 	 * reset completes.
10729 	 */
10730 	i40e_ptp_save_hw_time(pf);
10731 }
10732 
10733 /**
10734  * i40e_send_version - update firmware with driver version
10735  * @pf: PF struct
10736  */
10737 static void i40e_send_version(struct i40e_pf *pf)
10738 {
10739 	struct i40e_driver_version dv;
10740 
10741 	dv.major_version = 0xff;
10742 	dv.minor_version = 0xff;
10743 	dv.build_version = 0xff;
10744 	dv.subbuild_version = 0;
10745 	strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string));
10746 	i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
10747 }
10748 
10749 /**
10750  * i40e_get_oem_version - get OEM specific version information
10751  * @hw: pointer to the hardware structure
10752  **/
10753 static void i40e_get_oem_version(struct i40e_hw *hw)
10754 {
10755 	u16 block_offset = 0xffff;
10756 	u16 block_length = 0;
10757 	u16 capabilities = 0;
10758 	u16 gen_snap = 0;
10759 	u16 release = 0;
10760 
10761 #define I40E_SR_NVM_OEM_VERSION_PTR		0x1B
10762 #define I40E_NVM_OEM_LENGTH_OFFSET		0x00
10763 #define I40E_NVM_OEM_CAPABILITIES_OFFSET	0x01
10764 #define I40E_NVM_OEM_GEN_OFFSET			0x02
10765 #define I40E_NVM_OEM_RELEASE_OFFSET		0x03
10766 #define I40E_NVM_OEM_CAPABILITIES_MASK		0x000F
10767 #define I40E_NVM_OEM_LENGTH			3
10768 
10769 	/* Check if pointer to OEM version block is valid. */
10770 	i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
10771 	if (block_offset == 0xffff)
10772 		return;
10773 
10774 	/* Check if OEM version block has correct length. */
10775 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
10776 			   &block_length);
10777 	if (block_length < I40E_NVM_OEM_LENGTH)
10778 		return;
10779 
10780 	/* Check if OEM version format is as expected. */
10781 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
10782 			   &capabilities);
10783 	if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
10784 		return;
10785 
10786 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
10787 			   &gen_snap);
10788 	i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
10789 			   &release);
10790 	hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
10791 	hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
10792 }
10793 
10794 /**
10795  * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen
10796  * @pf: board private structure
10797  **/
10798 static int i40e_reset(struct i40e_pf *pf)
10799 {
10800 	struct i40e_hw *hw = &pf->hw;
10801 	int ret;
10802 
10803 	ret = i40e_pf_reset(hw);
10804 	if (ret) {
10805 		dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
10806 		set_bit(__I40E_RESET_FAILED, pf->state);
10807 		clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10808 	} else {
10809 		pf->pfr_count++;
10810 	}
10811 	return ret;
10812 }
10813 
10814 /**
10815  * i40e_rebuild - rebuild using a saved config
10816  * @pf: board private structure
10817  * @reinit: if the Main VSI needs to re-initialized.
10818  * @lock_acquired: indicates whether or not the lock has been acquired
10819  * before this function was called.
10820  **/
10821 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
10822 {
10823 	const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf);
10824 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
10825 	struct i40e_hw *hw = &pf->hw;
10826 	int ret;
10827 	u32 val;
10828 	int v;
10829 
10830 	if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
10831 	    is_recovery_mode_reported)
10832 		i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
10833 
10834 	if (test_bit(__I40E_DOWN, pf->state) &&
10835 	    !test_bit(__I40E_RECOVERY_MODE, pf->state))
10836 		goto clear_recovery;
10837 	dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
10838 
10839 	/* rebuild the basics for the AdminQ, HMC, and initial HW switch */
10840 	ret = i40e_init_adminq(&pf->hw);
10841 	if (ret) {
10842 		dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n",
10843 			 ERR_PTR(ret),
10844 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10845 		goto clear_recovery;
10846 	}
10847 	i40e_get_oem_version(&pf->hw);
10848 
10849 	if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) {
10850 		/* The following delay is necessary for firmware update. */
10851 		mdelay(1000);
10852 	}
10853 
10854 	/* re-verify the eeprom if we just had an EMP reset */
10855 	if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
10856 		i40e_verify_eeprom(pf);
10857 
10858 	/* if we are going out of or into recovery mode we have to act
10859 	 * accordingly with regard to resources initialization
10860 	 * and deinitialization
10861 	 */
10862 	if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10863 		if (i40e_get_capabilities(pf,
10864 					  i40e_aqc_opc_list_func_capabilities))
10865 			goto end_unlock;
10866 
10867 		if (is_recovery_mode_reported) {
10868 			/* we're staying in recovery mode so we'll reinitialize
10869 			 * misc vector here
10870 			 */
10871 			if (i40e_setup_misc_vector_for_recovery_mode(pf))
10872 				goto end_unlock;
10873 		} else {
10874 			if (!lock_acquired)
10875 				rtnl_lock();
10876 			/* we're going out of recovery mode so we'll free
10877 			 * the IRQ allocated specifically for recovery mode
10878 			 * and restore the interrupt scheme
10879 			 */
10880 			free_irq(pf->pdev->irq, pf);
10881 			i40e_clear_interrupt_scheme(pf);
10882 			if (i40e_restore_interrupt_scheme(pf))
10883 				goto end_unlock;
10884 		}
10885 
10886 		/* tell the firmware that we're starting */
10887 		i40e_send_version(pf);
10888 
10889 		/* bail out in case recovery mode was detected, as there is
10890 		 * no need for further configuration.
10891 		 */
10892 		goto end_unlock;
10893 	}
10894 
10895 	i40e_clear_pxe_mode(hw);
10896 	ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10897 	if (ret)
10898 		goto end_core_reset;
10899 
10900 	ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10901 				hw->func_caps.num_rx_qp, 0, 0);
10902 	if (ret) {
10903 		dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10904 		goto end_core_reset;
10905 	}
10906 	ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10907 	if (ret) {
10908 		dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10909 		goto end_core_reset;
10910 	}
10911 
10912 #ifdef CONFIG_I40E_DCB
10913 	/* Enable FW to write a default DCB config on link-up
10914 	 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB
10915 	 * is not supported with new link speed
10916 	 */
10917 	if (i40e_is_tc_mqprio_enabled(pf)) {
10918 		i40e_aq_set_dcb_parameters(hw, false, NULL);
10919 	} else {
10920 		if (I40E_IS_X710TL_DEVICE(hw->device_id) &&
10921 		    (hw->phy.link_info.link_speed &
10922 		     (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) {
10923 			i40e_aq_set_dcb_parameters(hw, false, NULL);
10924 			dev_warn(&pf->pdev->dev,
10925 				 "DCB is not supported for X710-T*L 2.5/5G speeds\n");
10926 			pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10927 		} else {
10928 			i40e_aq_set_dcb_parameters(hw, true, NULL);
10929 			ret = i40e_init_pf_dcb(pf);
10930 			if (ret) {
10931 				dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n",
10932 					 ret);
10933 				pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10934 				/* Continue without DCB enabled */
10935 			}
10936 		}
10937 	}
10938 
10939 #endif /* CONFIG_I40E_DCB */
10940 	if (!lock_acquired)
10941 		rtnl_lock();
10942 	ret = i40e_setup_pf_switch(pf, reinit, true);
10943 	if (ret)
10944 		goto end_unlock;
10945 
10946 	/* The driver only wants link up/down and module qualification
10947 	 * reports from firmware.  Note the negative logic.
10948 	 */
10949 	ret = i40e_aq_set_phy_int_mask(&pf->hw,
10950 				       ~(I40E_AQ_EVENT_LINK_UPDOWN |
10951 					 I40E_AQ_EVENT_MEDIA_NA |
10952 					 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10953 	if (ret)
10954 		dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
10955 			 ERR_PTR(ret),
10956 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10957 
10958 	/* Rebuild the VSIs and VEBs that existed before reset.
10959 	 * They are still in our local switch element arrays, so only
10960 	 * need to rebuild the switch model in the HW.
10961 	 *
10962 	 * If there were VEBs but the reconstitution failed, we'll try
10963 	 * to recover minimal use by getting the basic PF VSI working.
10964 	 */
10965 	if (vsi->uplink_seid != pf->mac_seid) {
10966 		dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10967 		/* find the one VEB connected to the MAC, and find orphans */
10968 		for (v = 0; v < I40E_MAX_VEB; v++) {
10969 			if (!pf->veb[v])
10970 				continue;
10971 
10972 			if (pf->veb[v]->uplink_seid == pf->mac_seid ||
10973 			    pf->veb[v]->uplink_seid == 0) {
10974 				ret = i40e_reconstitute_veb(pf->veb[v]);
10975 
10976 				if (!ret)
10977 					continue;
10978 
10979 				/* If Main VEB failed, we're in deep doodoo,
10980 				 * so give up rebuilding the switch and set up
10981 				 * for minimal rebuild of PF VSI.
10982 				 * If orphan failed, we'll report the error
10983 				 * but try to keep going.
10984 				 */
10985 				if (pf->veb[v]->uplink_seid == pf->mac_seid) {
10986 					dev_info(&pf->pdev->dev,
10987 						 "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10988 						 ret);
10989 					vsi->uplink_seid = pf->mac_seid;
10990 					break;
10991 				} else if (pf->veb[v]->uplink_seid == 0) {
10992 					dev_info(&pf->pdev->dev,
10993 						 "rebuild of orphan VEB failed: %d\n",
10994 						 ret);
10995 				}
10996 			}
10997 		}
10998 	}
10999 
11000 	if (vsi->uplink_seid == pf->mac_seid) {
11001 		dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
11002 		/* no VEB, so rebuild only the Main VSI */
11003 		ret = i40e_add_vsi(vsi);
11004 		if (ret) {
11005 			dev_info(&pf->pdev->dev,
11006 				 "rebuild of Main VSI failed: %d\n", ret);
11007 			goto end_unlock;
11008 		}
11009 	}
11010 
11011 	if (vsi->mqprio_qopt.max_rate[0]) {
11012 		u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi,
11013 						  vsi->mqprio_qopt.max_rate[0]);
11014 		u64 credits = 0;
11015 
11016 		ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
11017 		if (ret)
11018 			goto end_unlock;
11019 
11020 		credits = max_tx_rate;
11021 		do_div(credits, I40E_BW_CREDIT_DIVISOR);
11022 		dev_dbg(&vsi->back->pdev->dev,
11023 			"Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
11024 			max_tx_rate,
11025 			credits,
11026 			vsi->seid);
11027 	}
11028 
11029 	ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
11030 	if (ret)
11031 		goto end_unlock;
11032 
11033 	/* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs
11034 	 * for this main VSI if they exist
11035 	 */
11036 	ret = i40e_rebuild_channels(vsi);
11037 	if (ret)
11038 		goto end_unlock;
11039 
11040 	/* Reconfigure hardware for allowing smaller MSS in the case
11041 	 * of TSO, so that we avoid the MDD being fired and causing
11042 	 * a reset in the case of small MSS+TSO.
11043 	 */
11044 #define I40E_REG_MSS          0x000E64DC
11045 #define I40E_REG_MSS_MIN_MASK 0x3FF0000
11046 #define I40E_64BYTE_MSS       0x400000
11047 	val = rd32(hw, I40E_REG_MSS);
11048 	if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
11049 		val &= ~I40E_REG_MSS_MIN_MASK;
11050 		val |= I40E_64BYTE_MSS;
11051 		wr32(hw, I40E_REG_MSS, val);
11052 	}
11053 
11054 	if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
11055 		msleep(75);
11056 		ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
11057 		if (ret)
11058 			dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
11059 				 ERR_PTR(ret),
11060 				 i40e_aq_str(&pf->hw,
11061 					     pf->hw.aq.asq_last_status));
11062 	}
11063 	/* reinit the misc interrupt */
11064 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11065 		ret = i40e_setup_misc_vector(pf);
11066 		if (ret)
11067 			goto end_unlock;
11068 	}
11069 
11070 	/* Add a filter to drop all Flow control frames from any VSI from being
11071 	 * transmitted. By doing so we stop a malicious VF from sending out
11072 	 * PAUSE or PFC frames and potentially controlling traffic for other
11073 	 * PF/VF VSIs.
11074 	 * The FW can still send Flow control frames if enabled.
11075 	 */
11076 	i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
11077 						       pf->main_vsi_seid);
11078 
11079 	/* restart the VSIs that were rebuilt and running before the reset */
11080 	i40e_pf_unquiesce_all_vsi(pf);
11081 
11082 	/* Release the RTNL lock before we start resetting VFs */
11083 	if (!lock_acquired)
11084 		rtnl_unlock();
11085 
11086 	/* Restore promiscuous settings */
11087 	ret = i40e_set_promiscuous(pf, pf->cur_promisc);
11088 	if (ret)
11089 		dev_warn(&pf->pdev->dev,
11090 			 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n",
11091 			 pf->cur_promisc ? "on" : "off",
11092 			 ERR_PTR(ret),
11093 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
11094 
11095 	i40e_reset_all_vfs(pf, true);
11096 
11097 	/* tell the firmware that we're starting */
11098 	i40e_send_version(pf);
11099 
11100 	/* We've already released the lock, so don't do it again */
11101 	goto end_core_reset;
11102 
11103 end_unlock:
11104 	if (!lock_acquired)
11105 		rtnl_unlock();
11106 end_core_reset:
11107 	clear_bit(__I40E_RESET_FAILED, pf->state);
11108 clear_recovery:
11109 	clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
11110 	clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
11111 }
11112 
11113 /**
11114  * i40e_reset_and_rebuild - reset and rebuild using a saved config
11115  * @pf: board private structure
11116  * @reinit: if the Main VSI needs to re-initialized.
11117  * @lock_acquired: indicates whether or not the lock has been acquired
11118  * before this function was called.
11119  **/
11120 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
11121 				   bool lock_acquired)
11122 {
11123 	int ret;
11124 
11125 	if (test_bit(__I40E_IN_REMOVE, pf->state))
11126 		return;
11127 	/* Now we wait for GRST to settle out.
11128 	 * We don't have to delete the VEBs or VSIs from the hw switch
11129 	 * because the reset will make them disappear.
11130 	 */
11131 	ret = i40e_reset(pf);
11132 	if (!ret)
11133 		i40e_rebuild(pf, reinit, lock_acquired);
11134 }
11135 
11136 /**
11137  * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild
11138  * @pf: board private structure
11139  *
11140  * Close up the VFs and other things in prep for a Core Reset,
11141  * then get ready to rebuild the world.
11142  * @lock_acquired: indicates whether or not the lock has been acquired
11143  * before this function was called.
11144  **/
11145 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
11146 {
11147 	i40e_prep_for_reset(pf);
11148 	i40e_reset_and_rebuild(pf, false, lock_acquired);
11149 }
11150 
11151 /**
11152  * i40e_handle_mdd_event
11153  * @pf: pointer to the PF structure
11154  *
11155  * Called from the MDD irq handler to identify possibly malicious vfs
11156  **/
11157 static void i40e_handle_mdd_event(struct i40e_pf *pf)
11158 {
11159 	struct i40e_hw *hw = &pf->hw;
11160 	bool mdd_detected = false;
11161 	struct i40e_vf *vf;
11162 	u32 reg;
11163 	int i;
11164 
11165 	if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
11166 		return;
11167 
11168 	/* find what triggered the MDD event */
11169 	reg = rd32(hw, I40E_GL_MDET_TX);
11170 	if (reg & I40E_GL_MDET_TX_VALID_MASK) {
11171 		u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
11172 				I40E_GL_MDET_TX_PF_NUM_SHIFT;
11173 		u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
11174 				I40E_GL_MDET_TX_VF_NUM_SHIFT;
11175 		u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
11176 				I40E_GL_MDET_TX_EVENT_SHIFT;
11177 		u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
11178 				I40E_GL_MDET_TX_QUEUE_SHIFT) -
11179 				pf->hw.func_caps.base_queue;
11180 		if (netif_msg_tx_err(pf))
11181 			dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
11182 				 event, queue, pf_num, vf_num);
11183 		wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
11184 		mdd_detected = true;
11185 	}
11186 	reg = rd32(hw, I40E_GL_MDET_RX);
11187 	if (reg & I40E_GL_MDET_RX_VALID_MASK) {
11188 		u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
11189 				I40E_GL_MDET_RX_FUNCTION_SHIFT;
11190 		u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
11191 				I40E_GL_MDET_RX_EVENT_SHIFT;
11192 		u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
11193 				I40E_GL_MDET_RX_QUEUE_SHIFT) -
11194 				pf->hw.func_caps.base_queue;
11195 		if (netif_msg_rx_err(pf))
11196 			dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
11197 				 event, queue, func);
11198 		wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
11199 		mdd_detected = true;
11200 	}
11201 
11202 	if (mdd_detected) {
11203 		reg = rd32(hw, I40E_PF_MDET_TX);
11204 		if (reg & I40E_PF_MDET_TX_VALID_MASK) {
11205 			wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
11206 			dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
11207 		}
11208 		reg = rd32(hw, I40E_PF_MDET_RX);
11209 		if (reg & I40E_PF_MDET_RX_VALID_MASK) {
11210 			wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
11211 			dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
11212 		}
11213 	}
11214 
11215 	/* see if one of the VFs needs its hand slapped */
11216 	for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
11217 		vf = &(pf->vf[i]);
11218 		reg = rd32(hw, I40E_VP_MDET_TX(i));
11219 		if (reg & I40E_VP_MDET_TX_VALID_MASK) {
11220 			wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
11221 			vf->num_mdd_events++;
11222 			dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
11223 				 i);
11224 			dev_info(&pf->pdev->dev,
11225 				 "Use PF Control I/F to re-enable the VF\n");
11226 			set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11227 		}
11228 
11229 		reg = rd32(hw, I40E_VP_MDET_RX(i));
11230 		if (reg & I40E_VP_MDET_RX_VALID_MASK) {
11231 			wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
11232 			vf->num_mdd_events++;
11233 			dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
11234 				 i);
11235 			dev_info(&pf->pdev->dev,
11236 				 "Use PF Control I/F to re-enable the VF\n");
11237 			set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
11238 		}
11239 	}
11240 
11241 	/* re-enable mdd interrupt cause */
11242 	clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
11243 	reg = rd32(hw, I40E_PFINT_ICR0_ENA);
11244 	reg |=  I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
11245 	wr32(hw, I40E_PFINT_ICR0_ENA, reg);
11246 	i40e_flush(hw);
11247 }
11248 
11249 /**
11250  * i40e_service_task - Run the driver's async subtasks
11251  * @work: pointer to work_struct containing our data
11252  **/
11253 static void i40e_service_task(struct work_struct *work)
11254 {
11255 	struct i40e_pf *pf = container_of(work,
11256 					  struct i40e_pf,
11257 					  service_task);
11258 	unsigned long start_time = jiffies;
11259 
11260 	/* don't bother with service tasks if a reset is in progress */
11261 	if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
11262 	    test_bit(__I40E_SUSPENDED, pf->state))
11263 		return;
11264 
11265 	if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
11266 		return;
11267 
11268 	if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
11269 		i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
11270 		i40e_sync_filters_subtask(pf);
11271 		i40e_reset_subtask(pf);
11272 		i40e_handle_mdd_event(pf);
11273 		i40e_vc_process_vflr_event(pf);
11274 		i40e_watchdog_subtask(pf);
11275 		i40e_fdir_reinit_subtask(pf);
11276 		if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
11277 			/* Client subtask will reopen next time through. */
11278 			i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
11279 							   true);
11280 		} else {
11281 			i40e_client_subtask(pf);
11282 			if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
11283 					       pf->state))
11284 				i40e_notify_client_of_l2_param_changes(
11285 								pf->vsi[pf->lan_vsi]);
11286 		}
11287 		i40e_sync_filters_subtask(pf);
11288 	} else {
11289 		i40e_reset_subtask(pf);
11290 	}
11291 
11292 	i40e_clean_adminq_subtask(pf);
11293 
11294 	/* flush memory to make sure state is correct before next watchdog */
11295 	smp_mb__before_atomic();
11296 	clear_bit(__I40E_SERVICE_SCHED, pf->state);
11297 
11298 	/* If the tasks have taken longer than one timer cycle or there
11299 	 * is more work to be done, reschedule the service task now
11300 	 * rather than wait for the timer to tick again.
11301 	 */
11302 	if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
11303 	    test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state)		 ||
11304 	    test_bit(__I40E_MDD_EVENT_PENDING, pf->state)		 ||
11305 	    test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
11306 		i40e_service_event_schedule(pf);
11307 }
11308 
11309 /**
11310  * i40e_service_timer - timer callback
11311  * @t: timer list pointer
11312  **/
11313 static void i40e_service_timer(struct timer_list *t)
11314 {
11315 	struct i40e_pf *pf = from_timer(pf, t, service_timer);
11316 
11317 	mod_timer(&pf->service_timer,
11318 		  round_jiffies(jiffies + pf->service_timer_period));
11319 	i40e_service_event_schedule(pf);
11320 }
11321 
11322 /**
11323  * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI
11324  * @vsi: the VSI being configured
11325  **/
11326 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
11327 {
11328 	struct i40e_pf *pf = vsi->back;
11329 
11330 	switch (vsi->type) {
11331 	case I40E_VSI_MAIN:
11332 		vsi->alloc_queue_pairs = pf->num_lan_qps;
11333 		if (!vsi->num_tx_desc)
11334 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11335 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11336 		if (!vsi->num_rx_desc)
11337 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11338 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11339 		if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11340 			vsi->num_q_vectors = pf->num_lan_msix;
11341 		else
11342 			vsi->num_q_vectors = 1;
11343 
11344 		break;
11345 
11346 	case I40E_VSI_FDIR:
11347 		vsi->alloc_queue_pairs = 1;
11348 		vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11349 					 I40E_REQ_DESCRIPTOR_MULTIPLE);
11350 		vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
11351 					 I40E_REQ_DESCRIPTOR_MULTIPLE);
11352 		vsi->num_q_vectors = pf->num_fdsb_msix;
11353 		break;
11354 
11355 	case I40E_VSI_VMDQ2:
11356 		vsi->alloc_queue_pairs = pf->num_vmdq_qps;
11357 		if (!vsi->num_tx_desc)
11358 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11359 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11360 		if (!vsi->num_rx_desc)
11361 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11362 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11363 		vsi->num_q_vectors = pf->num_vmdq_msix;
11364 		break;
11365 
11366 	case I40E_VSI_SRIOV:
11367 		vsi->alloc_queue_pairs = pf->num_vf_qps;
11368 		if (!vsi->num_tx_desc)
11369 			vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11370 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11371 		if (!vsi->num_rx_desc)
11372 			vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
11373 						 I40E_REQ_DESCRIPTOR_MULTIPLE);
11374 		break;
11375 
11376 	default:
11377 		WARN_ON(1);
11378 		return -ENODATA;
11379 	}
11380 
11381 	if (is_kdump_kernel()) {
11382 		vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS;
11383 		vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS;
11384 	}
11385 
11386 	return 0;
11387 }
11388 
11389 /**
11390  * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
11391  * @vsi: VSI pointer
11392  * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
11393  *
11394  * On error: returns error code (negative)
11395  * On success: returns 0
11396  **/
11397 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
11398 {
11399 	struct i40e_ring **next_rings;
11400 	int size;
11401 	int ret = 0;
11402 
11403 	/* allocate memory for both Tx, XDP Tx and Rx ring pointers */
11404 	size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
11405 	       (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
11406 	vsi->tx_rings = kzalloc(size, GFP_KERNEL);
11407 	if (!vsi->tx_rings)
11408 		return -ENOMEM;
11409 	next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
11410 	if (i40e_enabled_xdp_vsi(vsi)) {
11411 		vsi->xdp_rings = next_rings;
11412 		next_rings += vsi->alloc_queue_pairs;
11413 	}
11414 	vsi->rx_rings = next_rings;
11415 
11416 	if (alloc_qvectors) {
11417 		/* allocate memory for q_vector pointers */
11418 		size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
11419 		vsi->q_vectors = kzalloc(size, GFP_KERNEL);
11420 		if (!vsi->q_vectors) {
11421 			ret = -ENOMEM;
11422 			goto err_vectors;
11423 		}
11424 	}
11425 	return ret;
11426 
11427 err_vectors:
11428 	kfree(vsi->tx_rings);
11429 	return ret;
11430 }
11431 
11432 /**
11433  * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF
11434  * @pf: board private structure
11435  * @type: type of VSI
11436  *
11437  * On error: returns error code (negative)
11438  * On success: returns vsi index in PF (positive)
11439  **/
11440 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
11441 {
11442 	int ret = -ENODEV;
11443 	struct i40e_vsi *vsi;
11444 	int vsi_idx;
11445 	int i;
11446 
11447 	/* Need to protect the allocation of the VSIs at the PF level */
11448 	mutex_lock(&pf->switch_mutex);
11449 
11450 	/* VSI list may be fragmented if VSI creation/destruction has
11451 	 * been happening.  We can afford to do a quick scan to look
11452 	 * for any free VSIs in the list.
11453 	 *
11454 	 * find next empty vsi slot, looping back around if necessary
11455 	 */
11456 	i = pf->next_vsi;
11457 	while (i < pf->num_alloc_vsi && pf->vsi[i])
11458 		i++;
11459 	if (i >= pf->num_alloc_vsi) {
11460 		i = 0;
11461 		while (i < pf->next_vsi && pf->vsi[i])
11462 			i++;
11463 	}
11464 
11465 	if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
11466 		vsi_idx = i;             /* Found one! */
11467 	} else {
11468 		ret = -ENODEV;
11469 		goto unlock_pf;  /* out of VSI slots! */
11470 	}
11471 	pf->next_vsi = ++i;
11472 
11473 	vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
11474 	if (!vsi) {
11475 		ret = -ENOMEM;
11476 		goto unlock_pf;
11477 	}
11478 	vsi->type = type;
11479 	vsi->back = pf;
11480 	set_bit(__I40E_VSI_DOWN, vsi->state);
11481 	vsi->flags = 0;
11482 	vsi->idx = vsi_idx;
11483 	vsi->int_rate_limit = 0;
11484 	vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
11485 				pf->rss_table_size : 64;
11486 	vsi->netdev_registered = false;
11487 	vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
11488 	hash_init(vsi->mac_filter_hash);
11489 	vsi->irqs_ready = false;
11490 
11491 	if (type == I40E_VSI_MAIN) {
11492 		vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
11493 		if (!vsi->af_xdp_zc_qps)
11494 			goto err_rings;
11495 	}
11496 
11497 	ret = i40e_set_num_rings_in_vsi(vsi);
11498 	if (ret)
11499 		goto err_rings;
11500 
11501 	ret = i40e_vsi_alloc_arrays(vsi, true);
11502 	if (ret)
11503 		goto err_rings;
11504 
11505 	/* Setup default MSIX irq handler for VSI */
11506 	i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
11507 
11508 	/* Initialize VSI lock */
11509 	spin_lock_init(&vsi->mac_filter_hash_lock);
11510 	pf->vsi[vsi_idx] = vsi;
11511 	ret = vsi_idx;
11512 	goto unlock_pf;
11513 
11514 err_rings:
11515 	bitmap_free(vsi->af_xdp_zc_qps);
11516 	pf->next_vsi = i - 1;
11517 	kfree(vsi);
11518 unlock_pf:
11519 	mutex_unlock(&pf->switch_mutex);
11520 	return ret;
11521 }
11522 
11523 /**
11524  * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI
11525  * @vsi: VSI pointer
11526  * @free_qvectors: a bool to specify if q_vectors need to be freed.
11527  *
11528  * On error: returns error code (negative)
11529  * On success: returns 0
11530  **/
11531 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
11532 {
11533 	/* free the ring and vector containers */
11534 	if (free_qvectors) {
11535 		kfree(vsi->q_vectors);
11536 		vsi->q_vectors = NULL;
11537 	}
11538 	kfree(vsi->tx_rings);
11539 	vsi->tx_rings = NULL;
11540 	vsi->rx_rings = NULL;
11541 	vsi->xdp_rings = NULL;
11542 }
11543 
11544 /**
11545  * i40e_clear_rss_config_user - clear the user configured RSS hash keys
11546  * and lookup table
11547  * @vsi: Pointer to VSI structure
11548  */
11549 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
11550 {
11551 	if (!vsi)
11552 		return;
11553 
11554 	kfree(vsi->rss_hkey_user);
11555 	vsi->rss_hkey_user = NULL;
11556 
11557 	kfree(vsi->rss_lut_user);
11558 	vsi->rss_lut_user = NULL;
11559 }
11560 
11561 /**
11562  * i40e_vsi_clear - Deallocate the VSI provided
11563  * @vsi: the VSI being un-configured
11564  **/
11565 static int i40e_vsi_clear(struct i40e_vsi *vsi)
11566 {
11567 	struct i40e_pf *pf;
11568 
11569 	if (!vsi)
11570 		return 0;
11571 
11572 	if (!vsi->back)
11573 		goto free_vsi;
11574 	pf = vsi->back;
11575 
11576 	mutex_lock(&pf->switch_mutex);
11577 	if (!pf->vsi[vsi->idx]) {
11578 		dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
11579 			vsi->idx, vsi->idx, vsi->type);
11580 		goto unlock_vsi;
11581 	}
11582 
11583 	if (pf->vsi[vsi->idx] != vsi) {
11584 		dev_err(&pf->pdev->dev,
11585 			"pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
11586 			pf->vsi[vsi->idx]->idx,
11587 			pf->vsi[vsi->idx]->type,
11588 			vsi->idx, vsi->type);
11589 		goto unlock_vsi;
11590 	}
11591 
11592 	/* updates the PF for this cleared vsi */
11593 	i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
11594 	i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
11595 
11596 	bitmap_free(vsi->af_xdp_zc_qps);
11597 	i40e_vsi_free_arrays(vsi, true);
11598 	i40e_clear_rss_config_user(vsi);
11599 
11600 	pf->vsi[vsi->idx] = NULL;
11601 	if (vsi->idx < pf->next_vsi)
11602 		pf->next_vsi = vsi->idx;
11603 
11604 unlock_vsi:
11605 	mutex_unlock(&pf->switch_mutex);
11606 free_vsi:
11607 	kfree(vsi);
11608 
11609 	return 0;
11610 }
11611 
11612 /**
11613  * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI
11614  * @vsi: the VSI being cleaned
11615  **/
11616 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
11617 {
11618 	int i;
11619 
11620 	if (vsi->tx_rings && vsi->tx_rings[0]) {
11621 		for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11622 			kfree_rcu(vsi->tx_rings[i], rcu);
11623 			WRITE_ONCE(vsi->tx_rings[i], NULL);
11624 			WRITE_ONCE(vsi->rx_rings[i], NULL);
11625 			if (vsi->xdp_rings)
11626 				WRITE_ONCE(vsi->xdp_rings[i], NULL);
11627 		}
11628 	}
11629 }
11630 
11631 /**
11632  * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI
11633  * @vsi: the VSI being configured
11634  **/
11635 static int i40e_alloc_rings(struct i40e_vsi *vsi)
11636 {
11637 	int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
11638 	struct i40e_pf *pf = vsi->back;
11639 	struct i40e_ring *ring;
11640 
11641 	/* Set basic values in the rings to be used later during open() */
11642 	for (i = 0; i < vsi->alloc_queue_pairs; i++) {
11643 		/* allocate space for both Tx and Rx in one shot */
11644 		ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
11645 		if (!ring)
11646 			goto err_out;
11647 
11648 		ring->queue_index = i;
11649 		ring->reg_idx = vsi->base_queue + i;
11650 		ring->ring_active = false;
11651 		ring->vsi = vsi;
11652 		ring->netdev = vsi->netdev;
11653 		ring->dev = &pf->pdev->dev;
11654 		ring->count = vsi->num_tx_desc;
11655 		ring->size = 0;
11656 		ring->dcb_tc = 0;
11657 		if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11658 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11659 		ring->itr_setting = pf->tx_itr_default;
11660 		WRITE_ONCE(vsi->tx_rings[i], ring++);
11661 
11662 		if (!i40e_enabled_xdp_vsi(vsi))
11663 			goto setup_rx;
11664 
11665 		ring->queue_index = vsi->alloc_queue_pairs + i;
11666 		ring->reg_idx = vsi->base_queue + ring->queue_index;
11667 		ring->ring_active = false;
11668 		ring->vsi = vsi;
11669 		ring->netdev = NULL;
11670 		ring->dev = &pf->pdev->dev;
11671 		ring->count = vsi->num_tx_desc;
11672 		ring->size = 0;
11673 		ring->dcb_tc = 0;
11674 		if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
11675 			ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
11676 		set_ring_xdp(ring);
11677 		ring->itr_setting = pf->tx_itr_default;
11678 		WRITE_ONCE(vsi->xdp_rings[i], ring++);
11679 
11680 setup_rx:
11681 		ring->queue_index = i;
11682 		ring->reg_idx = vsi->base_queue + i;
11683 		ring->ring_active = false;
11684 		ring->vsi = vsi;
11685 		ring->netdev = vsi->netdev;
11686 		ring->dev = &pf->pdev->dev;
11687 		ring->count = vsi->num_rx_desc;
11688 		ring->size = 0;
11689 		ring->dcb_tc = 0;
11690 		ring->itr_setting = pf->rx_itr_default;
11691 		WRITE_ONCE(vsi->rx_rings[i], ring);
11692 	}
11693 
11694 	return 0;
11695 
11696 err_out:
11697 	i40e_vsi_clear_rings(vsi);
11698 	return -ENOMEM;
11699 }
11700 
11701 /**
11702  * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel
11703  * @pf: board private structure
11704  * @vectors: the number of MSI-X vectors to request
11705  *
11706  * Returns the number of vectors reserved, or error
11707  **/
11708 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
11709 {
11710 	vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
11711 					I40E_MIN_MSIX, vectors);
11712 	if (vectors < 0) {
11713 		dev_info(&pf->pdev->dev,
11714 			 "MSI-X vector reservation failed: %d\n", vectors);
11715 		vectors = 0;
11716 	}
11717 
11718 	return vectors;
11719 }
11720 
11721 /**
11722  * i40e_init_msix - Setup the MSIX capability
11723  * @pf: board private structure
11724  *
11725  * Work with the OS to set up the MSIX vectors needed.
11726  *
11727  * Returns the number of vectors reserved or negative on failure
11728  **/
11729 static int i40e_init_msix(struct i40e_pf *pf)
11730 {
11731 	struct i40e_hw *hw = &pf->hw;
11732 	int cpus, extra_vectors;
11733 	int vectors_left;
11734 	int v_budget, i;
11735 	int v_actual;
11736 	int iwarp_requested = 0;
11737 
11738 	if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
11739 		return -ENODEV;
11740 
11741 	/* The number of vectors we'll request will be comprised of:
11742 	 *   - Add 1 for "other" cause for Admin Queue events, etc.
11743 	 *   - The number of LAN queue pairs
11744 	 *	- Queues being used for RSS.
11745 	 *		We don't need as many as max_rss_size vectors.
11746 	 *		use rss_size instead in the calculation since that
11747 	 *		is governed by number of cpus in the system.
11748 	 *	- assumes symmetric Tx/Rx pairing
11749 	 *   - The number of VMDq pairs
11750 	 *   - The CPU count within the NUMA node if iWARP is enabled
11751 	 * Once we count this up, try the request.
11752 	 *
11753 	 * If we can't get what we want, we'll simplify to nearly nothing
11754 	 * and try again.  If that still fails, we punt.
11755 	 */
11756 	vectors_left = hw->func_caps.num_msix_vectors;
11757 	v_budget = 0;
11758 
11759 	/* reserve one vector for miscellaneous handler */
11760 	if (vectors_left) {
11761 		v_budget++;
11762 		vectors_left--;
11763 	}
11764 
11765 	/* reserve some vectors for the main PF traffic queues. Initially we
11766 	 * only reserve at most 50% of the available vectors, in the case that
11767 	 * the number of online CPUs is large. This ensures that we can enable
11768 	 * extra features as well. Once we've enabled the other features, we
11769 	 * will use any remaining vectors to reach as close as we can to the
11770 	 * number of online CPUs.
11771 	 */
11772 	cpus = num_online_cpus();
11773 	pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
11774 	vectors_left -= pf->num_lan_msix;
11775 
11776 	/* reserve one vector for sideband flow director */
11777 	if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11778 		if (vectors_left) {
11779 			pf->num_fdsb_msix = 1;
11780 			v_budget++;
11781 			vectors_left--;
11782 		} else {
11783 			pf->num_fdsb_msix = 0;
11784 		}
11785 	}
11786 
11787 	/* can we reserve enough for iWARP? */
11788 	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11789 		iwarp_requested = pf->num_iwarp_msix;
11790 
11791 		if (!vectors_left)
11792 			pf->num_iwarp_msix = 0;
11793 		else if (vectors_left < pf->num_iwarp_msix)
11794 			pf->num_iwarp_msix = 1;
11795 		v_budget += pf->num_iwarp_msix;
11796 		vectors_left -= pf->num_iwarp_msix;
11797 	}
11798 
11799 	/* any vectors left over go for VMDq support */
11800 	if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
11801 		if (!vectors_left) {
11802 			pf->num_vmdq_msix = 0;
11803 			pf->num_vmdq_qps = 0;
11804 		} else {
11805 			int vmdq_vecs_wanted =
11806 				pf->num_vmdq_vsis * pf->num_vmdq_qps;
11807 			int vmdq_vecs =
11808 				min_t(int, vectors_left, vmdq_vecs_wanted);
11809 
11810 			/* if we're short on vectors for what's desired, we limit
11811 			 * the queues per vmdq.  If this is still more than are
11812 			 * available, the user will need to change the number of
11813 			 * queues/vectors used by the PF later with the ethtool
11814 			 * channels command
11815 			 */
11816 			if (vectors_left < vmdq_vecs_wanted) {
11817 				pf->num_vmdq_qps = 1;
11818 				vmdq_vecs_wanted = pf->num_vmdq_vsis;
11819 				vmdq_vecs = min_t(int,
11820 						  vectors_left,
11821 						  vmdq_vecs_wanted);
11822 			}
11823 			pf->num_vmdq_msix = pf->num_vmdq_qps;
11824 
11825 			v_budget += vmdq_vecs;
11826 			vectors_left -= vmdq_vecs;
11827 		}
11828 	}
11829 
11830 	/* On systems with a large number of SMP cores, we previously limited
11831 	 * the number of vectors for num_lan_msix to be at most 50% of the
11832 	 * available vectors, to allow for other features. Now, we add back
11833 	 * the remaining vectors. However, we ensure that the total
11834 	 * num_lan_msix will not exceed num_online_cpus(). To do this, we
11835 	 * calculate the number of vectors we can add without going over the
11836 	 * cap of CPUs. For systems with a small number of CPUs this will be
11837 	 * zero.
11838 	 */
11839 	extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11840 	pf->num_lan_msix += extra_vectors;
11841 	vectors_left -= extra_vectors;
11842 
11843 	WARN(vectors_left < 0,
11844 	     "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11845 
11846 	v_budget += pf->num_lan_msix;
11847 	pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11848 				   GFP_KERNEL);
11849 	if (!pf->msix_entries)
11850 		return -ENOMEM;
11851 
11852 	for (i = 0; i < v_budget; i++)
11853 		pf->msix_entries[i].entry = i;
11854 	v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11855 
11856 	if (v_actual < I40E_MIN_MSIX) {
11857 		pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
11858 		kfree(pf->msix_entries);
11859 		pf->msix_entries = NULL;
11860 		pci_disable_msix(pf->pdev);
11861 		return -ENODEV;
11862 
11863 	} else if (v_actual == I40E_MIN_MSIX) {
11864 		/* Adjust for minimal MSIX use */
11865 		pf->num_vmdq_vsis = 0;
11866 		pf->num_vmdq_qps = 0;
11867 		pf->num_lan_qps = 1;
11868 		pf->num_lan_msix = 1;
11869 
11870 	} else if (v_actual != v_budget) {
11871 		/* If we have limited resources, we will start with no vectors
11872 		 * for the special features and then allocate vectors to some
11873 		 * of these features based on the policy and at the end disable
11874 		 * the features that did not get any vectors.
11875 		 */
11876 		int vec;
11877 
11878 		dev_info(&pf->pdev->dev,
11879 			 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11880 			 v_actual, v_budget);
11881 		/* reserve the misc vector */
11882 		vec = v_actual - 1;
11883 
11884 		/* Scale vector usage down */
11885 		pf->num_vmdq_msix = 1;    /* force VMDqs to only one vector */
11886 		pf->num_vmdq_vsis = 1;
11887 		pf->num_vmdq_qps = 1;
11888 
11889 		/* partition out the remaining vectors */
11890 		switch (vec) {
11891 		case 2:
11892 			pf->num_lan_msix = 1;
11893 			break;
11894 		case 3:
11895 			if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11896 				pf->num_lan_msix = 1;
11897 				pf->num_iwarp_msix = 1;
11898 			} else {
11899 				pf->num_lan_msix = 2;
11900 			}
11901 			break;
11902 		default:
11903 			if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11904 				pf->num_iwarp_msix = min_t(int, (vec / 3),
11905 						 iwarp_requested);
11906 				pf->num_vmdq_vsis = min_t(int, (vec / 3),
11907 						  I40E_DEFAULT_NUM_VMDQ_VSI);
11908 			} else {
11909 				pf->num_vmdq_vsis = min_t(int, (vec / 2),
11910 						  I40E_DEFAULT_NUM_VMDQ_VSI);
11911 			}
11912 			if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11913 				pf->num_fdsb_msix = 1;
11914 				vec--;
11915 			}
11916 			pf->num_lan_msix = min_t(int,
11917 			       (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11918 							      pf->num_lan_msix);
11919 			pf->num_lan_qps = pf->num_lan_msix;
11920 			break;
11921 		}
11922 	}
11923 
11924 	if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
11925 	    (pf->num_fdsb_msix == 0)) {
11926 		dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11927 		pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11928 		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11929 	}
11930 	if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11931 	    (pf->num_vmdq_msix == 0)) {
11932 		dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11933 		pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
11934 	}
11935 
11936 	if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
11937 	    (pf->num_iwarp_msix == 0)) {
11938 		dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11939 		pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11940 	}
11941 	i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11942 		   "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11943 		   pf->num_lan_msix,
11944 		   pf->num_vmdq_msix * pf->num_vmdq_vsis,
11945 		   pf->num_fdsb_msix,
11946 		   pf->num_iwarp_msix);
11947 
11948 	return v_actual;
11949 }
11950 
11951 /**
11952  * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
11953  * @vsi: the VSI being configured
11954  * @v_idx: index of the vector in the vsi struct
11955  *
11956  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
11957  **/
11958 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx)
11959 {
11960 	struct i40e_q_vector *q_vector;
11961 
11962 	/* allocate q_vector */
11963 	q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11964 	if (!q_vector)
11965 		return -ENOMEM;
11966 
11967 	q_vector->vsi = vsi;
11968 	q_vector->v_idx = v_idx;
11969 	cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11970 
11971 	if (vsi->netdev)
11972 		netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll);
11973 
11974 	/* tie q_vector and vsi together */
11975 	vsi->q_vectors[v_idx] = q_vector;
11976 
11977 	return 0;
11978 }
11979 
11980 /**
11981  * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
11982  * @vsi: the VSI being configured
11983  *
11984  * We allocate one q_vector per queue interrupt.  If allocation fails we
11985  * return -ENOMEM.
11986  **/
11987 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11988 {
11989 	struct i40e_pf *pf = vsi->back;
11990 	int err, v_idx, num_q_vectors;
11991 
11992 	/* if not MSIX, give the one vector only to the LAN VSI */
11993 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11994 		num_q_vectors = vsi->num_q_vectors;
11995 	else if (vsi == pf->vsi[pf->lan_vsi])
11996 		num_q_vectors = 1;
11997 	else
11998 		return -EINVAL;
11999 
12000 	for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
12001 		err = i40e_vsi_alloc_q_vector(vsi, v_idx);
12002 		if (err)
12003 			goto err_out;
12004 	}
12005 
12006 	return 0;
12007 
12008 err_out:
12009 	while (v_idx--)
12010 		i40e_free_q_vector(vsi, v_idx);
12011 
12012 	return err;
12013 }
12014 
12015 /**
12016  * i40e_init_interrupt_scheme - Determine proper interrupt scheme
12017  * @pf: board private structure to initialize
12018  **/
12019 static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
12020 {
12021 	int vectors = 0;
12022 	ssize_t size;
12023 
12024 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
12025 		vectors = i40e_init_msix(pf);
12026 		if (vectors < 0) {
12027 			pf->flags &= ~(I40E_FLAG_MSIX_ENABLED	|
12028 				       I40E_FLAG_IWARP_ENABLED	|
12029 				       I40E_FLAG_RSS_ENABLED	|
12030 				       I40E_FLAG_DCB_CAPABLE	|
12031 				       I40E_FLAG_DCB_ENABLED	|
12032 				       I40E_FLAG_SRIOV_ENABLED	|
12033 				       I40E_FLAG_FD_SB_ENABLED	|
12034 				       I40E_FLAG_FD_ATR_ENABLED	|
12035 				       I40E_FLAG_VMDQ_ENABLED);
12036 			pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12037 
12038 			/* rework the queue expectations without MSIX */
12039 			i40e_determine_queue_usage(pf);
12040 		}
12041 	}
12042 
12043 	if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
12044 	    (pf->flags & I40E_FLAG_MSI_ENABLED)) {
12045 		dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
12046 		vectors = pci_enable_msi(pf->pdev);
12047 		if (vectors < 0) {
12048 			dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
12049 				 vectors);
12050 			pf->flags &= ~I40E_FLAG_MSI_ENABLED;
12051 		}
12052 		vectors = 1;  /* one MSI or Legacy vector */
12053 	}
12054 
12055 	if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
12056 		dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
12057 
12058 	/* set up vector assignment tracking */
12059 	size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
12060 	pf->irq_pile = kzalloc(size, GFP_KERNEL);
12061 	if (!pf->irq_pile)
12062 		return -ENOMEM;
12063 
12064 	pf->irq_pile->num_entries = vectors;
12065 
12066 	/* track first vector for misc interrupts, ignore return */
12067 	(void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
12068 
12069 	return 0;
12070 }
12071 
12072 /**
12073  * i40e_restore_interrupt_scheme - Restore the interrupt scheme
12074  * @pf: private board data structure
12075  *
12076  * Restore the interrupt scheme that was cleared when we suspended the
12077  * device. This should be called during resume to re-allocate the q_vectors
12078  * and reacquire IRQs.
12079  */
12080 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
12081 {
12082 	int err, i;
12083 
12084 	/* We cleared the MSI and MSI-X flags when disabling the old interrupt
12085 	 * scheme. We need to re-enabled them here in order to attempt to
12086 	 * re-acquire the MSI or MSI-X vectors
12087 	 */
12088 	pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
12089 
12090 	err = i40e_init_interrupt_scheme(pf);
12091 	if (err)
12092 		return err;
12093 
12094 	/* Now that we've re-acquired IRQs, we need to remap the vectors and
12095 	 * rings together again.
12096 	 */
12097 	for (i = 0; i < pf->num_alloc_vsi; i++) {
12098 		if (pf->vsi[i]) {
12099 			err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
12100 			if (err)
12101 				goto err_unwind;
12102 			i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
12103 		}
12104 	}
12105 
12106 	err = i40e_setup_misc_vector(pf);
12107 	if (err)
12108 		goto err_unwind;
12109 
12110 	if (pf->flags & I40E_FLAG_IWARP_ENABLED)
12111 		i40e_client_update_msix_info(pf);
12112 
12113 	return 0;
12114 
12115 err_unwind:
12116 	while (i--) {
12117 		if (pf->vsi[i])
12118 			i40e_vsi_free_q_vectors(pf->vsi[i]);
12119 	}
12120 
12121 	return err;
12122 }
12123 
12124 /**
12125  * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle
12126  * non queue events in recovery mode
12127  * @pf: board private structure
12128  *
12129  * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage
12130  * the non-queue interrupts, e.g. AdminQ and errors in recovery mode.
12131  * This is handled differently than in recovery mode since no Tx/Rx resources
12132  * are being allocated.
12133  **/
12134 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
12135 {
12136 	int err;
12137 
12138 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
12139 		err = i40e_setup_misc_vector(pf);
12140 
12141 		if (err) {
12142 			dev_info(&pf->pdev->dev,
12143 				 "MSI-X misc vector request failed, error %d\n",
12144 				 err);
12145 			return err;
12146 		}
12147 	} else {
12148 		u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
12149 
12150 		err = request_irq(pf->pdev->irq, i40e_intr, flags,
12151 				  pf->int_name, pf);
12152 
12153 		if (err) {
12154 			dev_info(&pf->pdev->dev,
12155 				 "MSI/legacy misc vector request failed, error %d\n",
12156 				 err);
12157 			return err;
12158 		}
12159 		i40e_enable_misc_int_causes(pf);
12160 		i40e_irq_dynamic_enable_icr0(pf);
12161 	}
12162 
12163 	return 0;
12164 }
12165 
12166 /**
12167  * i40e_setup_misc_vector - Setup the misc vector to handle non queue events
12168  * @pf: board private structure
12169  *
12170  * This sets up the handler for MSIX 0, which is used to manage the
12171  * non-queue interrupts, e.g. AdminQ and errors.  This is not used
12172  * when in MSI or Legacy interrupt mode.
12173  **/
12174 static int i40e_setup_misc_vector(struct i40e_pf *pf)
12175 {
12176 	struct i40e_hw *hw = &pf->hw;
12177 	int err = 0;
12178 
12179 	/* Only request the IRQ once, the first time through. */
12180 	if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
12181 		err = request_irq(pf->msix_entries[0].vector,
12182 				  i40e_intr, 0, pf->int_name, pf);
12183 		if (err) {
12184 			clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
12185 			dev_info(&pf->pdev->dev,
12186 				 "request_irq for %s failed: %d\n",
12187 				 pf->int_name, err);
12188 			return -EFAULT;
12189 		}
12190 	}
12191 
12192 	i40e_enable_misc_int_causes(pf);
12193 
12194 	/* associate no queues to the misc vector */
12195 	wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
12196 	wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1);
12197 
12198 	i40e_flush(hw);
12199 
12200 	i40e_irq_dynamic_enable_icr0(pf);
12201 
12202 	return err;
12203 }
12204 
12205 /**
12206  * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands
12207  * @vsi: Pointer to vsi structure
12208  * @seed: Buffter to store the hash keys
12209  * @lut: Buffer to store the lookup table entries
12210  * @lut_size: Size of buffer to store the lookup table entries
12211  *
12212  * Return 0 on success, negative on failure
12213  */
12214 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
12215 			   u8 *lut, u16 lut_size)
12216 {
12217 	struct i40e_pf *pf = vsi->back;
12218 	struct i40e_hw *hw = &pf->hw;
12219 	int ret = 0;
12220 
12221 	if (seed) {
12222 		ret = i40e_aq_get_rss_key(hw, vsi->id,
12223 			(struct i40e_aqc_get_set_rss_key_data *)seed);
12224 		if (ret) {
12225 			dev_info(&pf->pdev->dev,
12226 				 "Cannot get RSS key, err %pe aq_err %s\n",
12227 				 ERR_PTR(ret),
12228 				 i40e_aq_str(&pf->hw,
12229 					     pf->hw.aq.asq_last_status));
12230 			return ret;
12231 		}
12232 	}
12233 
12234 	if (lut) {
12235 		bool pf_lut = vsi->type == I40E_VSI_MAIN;
12236 
12237 		ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
12238 		if (ret) {
12239 			dev_info(&pf->pdev->dev,
12240 				 "Cannot get RSS lut, err %pe aq_err %s\n",
12241 				 ERR_PTR(ret),
12242 				 i40e_aq_str(&pf->hw,
12243 					     pf->hw.aq.asq_last_status));
12244 			return ret;
12245 		}
12246 	}
12247 
12248 	return ret;
12249 }
12250 
12251 /**
12252  * i40e_config_rss_reg - Configure RSS keys and lut by writing registers
12253  * @vsi: Pointer to vsi structure
12254  * @seed: RSS hash seed
12255  * @lut: Lookup table
12256  * @lut_size: Lookup table size
12257  *
12258  * Returns 0 on success, negative on failure
12259  **/
12260 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
12261 			       const u8 *lut, u16 lut_size)
12262 {
12263 	struct i40e_pf *pf = vsi->back;
12264 	struct i40e_hw *hw = &pf->hw;
12265 	u16 vf_id = vsi->vf_id;
12266 	u8 i;
12267 
12268 	/* Fill out hash function seed */
12269 	if (seed) {
12270 		u32 *seed_dw = (u32 *)seed;
12271 
12272 		if (vsi->type == I40E_VSI_MAIN) {
12273 			for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12274 				wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
12275 		} else if (vsi->type == I40E_VSI_SRIOV) {
12276 			for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
12277 				wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
12278 		} else {
12279 			dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
12280 		}
12281 	}
12282 
12283 	if (lut) {
12284 		u32 *lut_dw = (u32 *)lut;
12285 
12286 		if (vsi->type == I40E_VSI_MAIN) {
12287 			if (lut_size != I40E_HLUT_ARRAY_SIZE)
12288 				return -EINVAL;
12289 			for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12290 				wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
12291 		} else if (vsi->type == I40E_VSI_SRIOV) {
12292 			if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
12293 				return -EINVAL;
12294 			for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12295 				wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
12296 		} else {
12297 			dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12298 		}
12299 	}
12300 	i40e_flush(hw);
12301 
12302 	return 0;
12303 }
12304 
12305 /**
12306  * i40e_get_rss_reg - Get the RSS keys and lut by reading registers
12307  * @vsi: Pointer to VSI structure
12308  * @seed: Buffer to store the keys
12309  * @lut: Buffer to store the lookup table entries
12310  * @lut_size: Size of buffer to store the lookup table entries
12311  *
12312  * Returns 0 on success, negative on failure
12313  */
12314 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
12315 			    u8 *lut, u16 lut_size)
12316 {
12317 	struct i40e_pf *pf = vsi->back;
12318 	struct i40e_hw *hw = &pf->hw;
12319 	u16 i;
12320 
12321 	if (seed) {
12322 		u32 *seed_dw = (u32 *)seed;
12323 
12324 		for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
12325 			seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
12326 	}
12327 	if (lut) {
12328 		u32 *lut_dw = (u32 *)lut;
12329 
12330 		if (lut_size != I40E_HLUT_ARRAY_SIZE)
12331 			return -EINVAL;
12332 		for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12333 			lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
12334 	}
12335 
12336 	return 0;
12337 }
12338 
12339 /**
12340  * i40e_config_rss - Configure RSS keys and lut
12341  * @vsi: Pointer to VSI structure
12342  * @seed: RSS hash seed
12343  * @lut: Lookup table
12344  * @lut_size: Lookup table size
12345  *
12346  * Returns 0 on success, negative on failure
12347  */
12348 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12349 {
12350 	struct i40e_pf *pf = vsi->back;
12351 
12352 	if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12353 		return i40e_config_rss_aq(vsi, seed, lut, lut_size);
12354 	else
12355 		return i40e_config_rss_reg(vsi, seed, lut, lut_size);
12356 }
12357 
12358 /**
12359  * i40e_get_rss - Get RSS keys and lut
12360  * @vsi: Pointer to VSI structure
12361  * @seed: Buffer to store the keys
12362  * @lut: Buffer to store the lookup table entries
12363  * @lut_size: Size of buffer to store the lookup table entries
12364  *
12365  * Returns 0 on success, negative on failure
12366  */
12367 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
12368 {
12369 	struct i40e_pf *pf = vsi->back;
12370 
12371 	if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
12372 		return i40e_get_rss_aq(vsi, seed, lut, lut_size);
12373 	else
12374 		return i40e_get_rss_reg(vsi, seed, lut, lut_size);
12375 }
12376 
12377 /**
12378  * i40e_fill_rss_lut - Fill the RSS lookup table with default values
12379  * @pf: Pointer to board private structure
12380  * @lut: Lookup table
12381  * @rss_table_size: Lookup table size
12382  * @rss_size: Range of queue number for hashing
12383  */
12384 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
12385 		       u16 rss_table_size, u16 rss_size)
12386 {
12387 	u16 i;
12388 
12389 	for (i = 0; i < rss_table_size; i++)
12390 		lut[i] = i % rss_size;
12391 }
12392 
12393 /**
12394  * i40e_pf_config_rss - Prepare for RSS if used
12395  * @pf: board private structure
12396  **/
12397 static int i40e_pf_config_rss(struct i40e_pf *pf)
12398 {
12399 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12400 	u8 seed[I40E_HKEY_ARRAY_SIZE];
12401 	u8 *lut;
12402 	struct i40e_hw *hw = &pf->hw;
12403 	u32 reg_val;
12404 	u64 hena;
12405 	int ret;
12406 
12407 	/* By default we enable TCP/UDP with IPv4/IPv6 ptypes */
12408 	hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
12409 		((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
12410 	hena |= i40e_pf_get_default_rss_hena(pf);
12411 
12412 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
12413 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
12414 
12415 	/* Determine the RSS table size based on the hardware capabilities */
12416 	reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
12417 	reg_val = (pf->rss_table_size == 512) ?
12418 			(reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
12419 			(reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
12420 	i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
12421 
12422 	/* Determine the RSS size of the VSI */
12423 	if (!vsi->rss_size) {
12424 		u16 qcount;
12425 		/* If the firmware does something weird during VSI init, we
12426 		 * could end up with zero TCs. Check for that to avoid
12427 		 * divide-by-zero. It probably won't pass traffic, but it also
12428 		 * won't panic.
12429 		 */
12430 		qcount = vsi->num_queue_pairs /
12431 			 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
12432 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12433 	}
12434 	if (!vsi->rss_size)
12435 		return -EINVAL;
12436 
12437 	lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
12438 	if (!lut)
12439 		return -ENOMEM;
12440 
12441 	/* Use user configured lut if there is one, otherwise use default */
12442 	if (vsi->rss_lut_user)
12443 		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
12444 	else
12445 		i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
12446 
12447 	/* Use user configured hash key if there is one, otherwise
12448 	 * use default.
12449 	 */
12450 	if (vsi->rss_hkey_user)
12451 		memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
12452 	else
12453 		netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
12454 	ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
12455 	kfree(lut);
12456 
12457 	return ret;
12458 }
12459 
12460 /**
12461  * i40e_reconfig_rss_queues - change number of queues for rss and rebuild
12462  * @pf: board private structure
12463  * @queue_count: the requested queue count for rss.
12464  *
12465  * returns 0 if rss is not enabled, if enabled returns the final rss queue
12466  * count which may be different from the requested queue count.
12467  * Note: expects to be called while under rtnl_lock()
12468  **/
12469 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
12470 {
12471 	struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
12472 	int new_rss_size;
12473 
12474 	if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
12475 		return 0;
12476 
12477 	queue_count = min_t(int, queue_count, num_online_cpus());
12478 	new_rss_size = min_t(int, queue_count, pf->rss_size_max);
12479 
12480 	if (queue_count != vsi->num_queue_pairs) {
12481 		u16 qcount;
12482 
12483 		vsi->req_queue_pairs = queue_count;
12484 		i40e_prep_for_reset(pf);
12485 		if (test_bit(__I40E_IN_REMOVE, pf->state))
12486 			return pf->alloc_rss_size;
12487 
12488 		pf->alloc_rss_size = new_rss_size;
12489 
12490 		i40e_reset_and_rebuild(pf, true, true);
12491 
12492 		/* Discard the user configured hash keys and lut, if less
12493 		 * queues are enabled.
12494 		 */
12495 		if (queue_count < vsi->rss_size) {
12496 			i40e_clear_rss_config_user(vsi);
12497 			dev_dbg(&pf->pdev->dev,
12498 				"discard user configured hash keys and lut\n");
12499 		}
12500 
12501 		/* Reset vsi->rss_size, as number of enabled queues changed */
12502 		qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
12503 		vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
12504 
12505 		i40e_pf_config_rss(pf);
12506 	}
12507 	dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count:  %d/%d\n",
12508 		 vsi->req_queue_pairs, pf->rss_size_max);
12509 	return pf->alloc_rss_size;
12510 }
12511 
12512 /**
12513  * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition
12514  * @pf: board private structure
12515  **/
12516 int i40e_get_partition_bw_setting(struct i40e_pf *pf)
12517 {
12518 	bool min_valid, max_valid;
12519 	u32 max_bw, min_bw;
12520 	int status;
12521 
12522 	status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
12523 					   &min_valid, &max_valid);
12524 
12525 	if (!status) {
12526 		if (min_valid)
12527 			pf->min_bw = min_bw;
12528 		if (max_valid)
12529 			pf->max_bw = max_bw;
12530 	}
12531 
12532 	return status;
12533 }
12534 
12535 /**
12536  * i40e_set_partition_bw_setting - Set BW settings for this PF partition
12537  * @pf: board private structure
12538  **/
12539 int i40e_set_partition_bw_setting(struct i40e_pf *pf)
12540 {
12541 	struct i40e_aqc_configure_partition_bw_data bw_data;
12542 	int status;
12543 
12544 	memset(&bw_data, 0, sizeof(bw_data));
12545 
12546 	/* Set the valid bit for this PF */
12547 	bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
12548 	bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
12549 	bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
12550 
12551 	/* Set the new bandwidths */
12552 	status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
12553 
12554 	return status;
12555 }
12556 
12557 /**
12558  * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition
12559  * @pf: board private structure
12560  **/
12561 int i40e_commit_partition_bw_setting(struct i40e_pf *pf)
12562 {
12563 	/* Commit temporary BW setting to permanent NVM image */
12564 	enum i40e_admin_queue_err last_aq_status;
12565 	u16 nvm_word;
12566 	int ret;
12567 
12568 	if (pf->hw.partition_id != 1) {
12569 		dev_info(&pf->pdev->dev,
12570 			 "Commit BW only works on partition 1! This is partition %d",
12571 			 pf->hw.partition_id);
12572 		ret = I40E_NOT_SUPPORTED;
12573 		goto bw_commit_out;
12574 	}
12575 
12576 	/* Acquire NVM for read access */
12577 	ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
12578 	last_aq_status = pf->hw.aq.asq_last_status;
12579 	if (ret) {
12580 		dev_info(&pf->pdev->dev,
12581 			 "Cannot acquire NVM for read access, err %pe aq_err %s\n",
12582 			 ERR_PTR(ret),
12583 			 i40e_aq_str(&pf->hw, last_aq_status));
12584 		goto bw_commit_out;
12585 	}
12586 
12587 	/* Read word 0x10 of NVM - SW compatibility word 1 */
12588 	ret = i40e_aq_read_nvm(&pf->hw,
12589 			       I40E_SR_NVM_CONTROL_WORD,
12590 			       0x10, sizeof(nvm_word), &nvm_word,
12591 			       false, NULL);
12592 	/* Save off last admin queue command status before releasing
12593 	 * the NVM
12594 	 */
12595 	last_aq_status = pf->hw.aq.asq_last_status;
12596 	i40e_release_nvm(&pf->hw);
12597 	if (ret) {
12598 		dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n",
12599 			 ERR_PTR(ret),
12600 			 i40e_aq_str(&pf->hw, last_aq_status));
12601 		goto bw_commit_out;
12602 	}
12603 
12604 	/* Wait a bit for NVM release to complete */
12605 	msleep(50);
12606 
12607 	/* Acquire NVM for write access */
12608 	ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
12609 	last_aq_status = pf->hw.aq.asq_last_status;
12610 	if (ret) {
12611 		dev_info(&pf->pdev->dev,
12612 			 "Cannot acquire NVM for write access, err %pe aq_err %s\n",
12613 			 ERR_PTR(ret),
12614 			 i40e_aq_str(&pf->hw, last_aq_status));
12615 		goto bw_commit_out;
12616 	}
12617 	/* Write it back out unchanged to initiate update NVM,
12618 	 * which will force a write of the shadow (alt) RAM to
12619 	 * the NVM - thus storing the bandwidth values permanently.
12620 	 */
12621 	ret = i40e_aq_update_nvm(&pf->hw,
12622 				 I40E_SR_NVM_CONTROL_WORD,
12623 				 0x10, sizeof(nvm_word),
12624 				 &nvm_word, true, 0, NULL);
12625 	/* Save off last admin queue command status before releasing
12626 	 * the NVM
12627 	 */
12628 	last_aq_status = pf->hw.aq.asq_last_status;
12629 	i40e_release_nvm(&pf->hw);
12630 	if (ret)
12631 		dev_info(&pf->pdev->dev,
12632 			 "BW settings NOT SAVED, err %pe aq_err %s\n",
12633 			 ERR_PTR(ret),
12634 			 i40e_aq_str(&pf->hw, last_aq_status));
12635 bw_commit_out:
12636 
12637 	return ret;
12638 }
12639 
12640 /**
12641  * i40e_is_total_port_shutdown_enabled - read NVM and return value
12642  * if total port shutdown feature is enabled for this PF
12643  * @pf: board private structure
12644  **/
12645 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf)
12646 {
12647 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED	BIT(4)
12648 #define I40E_FEATURES_ENABLE_PTR		0x2A
12649 #define I40E_CURRENT_SETTING_PTR		0x2B
12650 #define I40E_LINK_BEHAVIOR_WORD_OFFSET		0x2D
12651 #define I40E_LINK_BEHAVIOR_WORD_LENGTH		0x1
12652 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED	BIT(0)
12653 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH	4
12654 	int read_status = I40E_SUCCESS;
12655 	u16 sr_emp_sr_settings_ptr = 0;
12656 	u16 features_enable = 0;
12657 	u16 link_behavior = 0;
12658 	bool ret = false;
12659 
12660 	read_status = i40e_read_nvm_word(&pf->hw,
12661 					 I40E_SR_EMP_SR_SETTINGS_PTR,
12662 					 &sr_emp_sr_settings_ptr);
12663 	if (read_status)
12664 		goto err_nvm;
12665 	read_status = i40e_read_nvm_word(&pf->hw,
12666 					 sr_emp_sr_settings_ptr +
12667 					 I40E_FEATURES_ENABLE_PTR,
12668 					 &features_enable);
12669 	if (read_status)
12670 		goto err_nvm;
12671 	if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) {
12672 		read_status = i40e_read_nvm_module_data(&pf->hw,
12673 							I40E_SR_EMP_SR_SETTINGS_PTR,
12674 							I40E_CURRENT_SETTING_PTR,
12675 							I40E_LINK_BEHAVIOR_WORD_OFFSET,
12676 							I40E_LINK_BEHAVIOR_WORD_LENGTH,
12677 							&link_behavior);
12678 		if (read_status)
12679 			goto err_nvm;
12680 		link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH);
12681 		ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior;
12682 	}
12683 	return ret;
12684 
12685 err_nvm:
12686 	dev_warn(&pf->pdev->dev,
12687 		 "total-port-shutdown feature is off due to read nvm error: %pe\n",
12688 		 ERR_PTR(read_status));
12689 	return ret;
12690 }
12691 
12692 /**
12693  * i40e_sw_init - Initialize general software structures (struct i40e_pf)
12694  * @pf: board private structure to initialize
12695  *
12696  * i40e_sw_init initializes the Adapter private data structure.
12697  * Fields are initialized based on PCI device information and
12698  * OS network device settings (MTU size).
12699  **/
12700 static int i40e_sw_init(struct i40e_pf *pf)
12701 {
12702 	int err = 0;
12703 	int size;
12704 	u16 pow;
12705 
12706 	/* Set default capability flags */
12707 	pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
12708 		    I40E_FLAG_MSI_ENABLED     |
12709 		    I40E_FLAG_MSIX_ENABLED;
12710 
12711 	/* Set default ITR */
12712 	pf->rx_itr_default = I40E_ITR_RX_DEF;
12713 	pf->tx_itr_default = I40E_ITR_TX_DEF;
12714 
12715 	/* Depending on PF configurations, it is possible that the RSS
12716 	 * maximum might end up larger than the available queues
12717 	 */
12718 	pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
12719 	pf->alloc_rss_size = 1;
12720 	pf->rss_table_size = pf->hw.func_caps.rss_table_size;
12721 	pf->rss_size_max = min_t(int, pf->rss_size_max,
12722 				 pf->hw.func_caps.num_tx_qp);
12723 
12724 	/* find the next higher power-of-2 of num cpus */
12725 	pow = roundup_pow_of_two(num_online_cpus());
12726 	pf->rss_size_max = min_t(int, pf->rss_size_max, pow);
12727 
12728 	if (pf->hw.func_caps.rss) {
12729 		pf->flags |= I40E_FLAG_RSS_ENABLED;
12730 		pf->alloc_rss_size = min_t(int, pf->rss_size_max,
12731 					   num_online_cpus());
12732 	}
12733 
12734 	/* MFP mode enabled */
12735 	if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
12736 		pf->flags |= I40E_FLAG_MFP_ENABLED;
12737 		dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
12738 		if (i40e_get_partition_bw_setting(pf)) {
12739 			dev_warn(&pf->pdev->dev,
12740 				 "Could not get partition bw settings\n");
12741 		} else {
12742 			dev_info(&pf->pdev->dev,
12743 				 "Partition BW Min = %8.8x, Max = %8.8x\n",
12744 				 pf->min_bw, pf->max_bw);
12745 
12746 			/* nudge the Tx scheduler */
12747 			i40e_set_partition_bw_setting(pf);
12748 		}
12749 	}
12750 
12751 	if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
12752 	    (pf->hw.func_caps.fd_filters_best_effort > 0)) {
12753 		pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
12754 		pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
12755 		if (pf->flags & I40E_FLAG_MFP_ENABLED &&
12756 		    pf->hw.num_partitions > 1)
12757 			dev_info(&pf->pdev->dev,
12758 				 "Flow Director Sideband mode Disabled in MFP mode\n");
12759 		else
12760 			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12761 		pf->fdir_pf_filter_count =
12762 				 pf->hw.func_caps.fd_filters_guaranteed;
12763 		pf->hw.fdir_shared_filter_count =
12764 				 pf->hw.func_caps.fd_filters_best_effort;
12765 	}
12766 
12767 	if (pf->hw.mac.type == I40E_MAC_X722) {
12768 		pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
12769 				    I40E_HW_128_QP_RSS_CAPABLE |
12770 				    I40E_HW_ATR_EVICT_CAPABLE |
12771 				    I40E_HW_WB_ON_ITR_CAPABLE |
12772 				    I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
12773 				    I40E_HW_NO_PCI_LINK_CHECK |
12774 				    I40E_HW_USE_SET_LLDP_MIB |
12775 				    I40E_HW_GENEVE_OFFLOAD_CAPABLE |
12776 				    I40E_HW_PTP_L4_CAPABLE |
12777 				    I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
12778 				    I40E_HW_OUTER_UDP_CSUM_CAPABLE);
12779 
12780 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
12781 		if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
12782 		    I40E_FDEVICT_PCTYPE_DEFAULT) {
12783 			dev_warn(&pf->pdev->dev,
12784 				 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
12785 			pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
12786 		}
12787 	} else if ((pf->hw.aq.api_maj_ver > 1) ||
12788 		   ((pf->hw.aq.api_maj_ver == 1) &&
12789 		    (pf->hw.aq.api_min_ver > 4))) {
12790 		/* Supported in FW API version higher than 1.4 */
12791 		pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
12792 	}
12793 
12794 	/* Enable HW ATR eviction if possible */
12795 	if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
12796 		pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
12797 
12798 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12799 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
12800 	    (pf->hw.aq.fw_maj_ver < 4))) {
12801 		pf->hw_features |= I40E_HW_RESTART_AUTONEG;
12802 		/* No DCB support  for FW < v4.33 */
12803 		pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
12804 	}
12805 
12806 	/* Disable FW LLDP if FW < v4.3 */
12807 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12808 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
12809 	    (pf->hw.aq.fw_maj_ver < 4)))
12810 		pf->hw_features |= I40E_HW_STOP_FW_LLDP;
12811 
12812 	/* Use the FW Set LLDP MIB API if FW > v4.40 */
12813 	if ((pf->hw.mac.type == I40E_MAC_XL710) &&
12814 	    (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
12815 	    (pf->hw.aq.fw_maj_ver >= 5)))
12816 		pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
12817 
12818 	/* Enable PTP L4 if FW > v6.0 */
12819 	if (pf->hw.mac.type == I40E_MAC_XL710 &&
12820 	    pf->hw.aq.fw_maj_ver >= 6)
12821 		pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
12822 
12823 	if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
12824 		pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
12825 		pf->flags |= I40E_FLAG_VMDQ_ENABLED;
12826 		pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
12827 	}
12828 
12829 	if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
12830 		pf->flags |= I40E_FLAG_IWARP_ENABLED;
12831 		/* IWARP needs one extra vector for CQP just like MISC.*/
12832 		pf->num_iwarp_msix = (int)num_online_cpus() + 1;
12833 	}
12834 	/* Stopping FW LLDP engine is supported on XL710 and X722
12835 	 * starting from FW versions determined in i40e_init_adminq.
12836 	 * Stopping the FW LLDP engine is not supported on XL710
12837 	 * if NPAR is functioning so unset this hw flag in this case.
12838 	 */
12839 	if (pf->hw.mac.type == I40E_MAC_XL710 &&
12840 	    pf->hw.func_caps.npar_enable &&
12841 	    (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
12842 		pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
12843 
12844 #ifdef CONFIG_PCI_IOV
12845 	if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12846 		pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12847 		pf->flags |= I40E_FLAG_SRIOV_ENABLED;
12848 		pf->num_req_vfs = min_t(int,
12849 					pf->hw.func_caps.num_vfs,
12850 					I40E_MAX_VF_COUNT);
12851 	}
12852 #endif /* CONFIG_PCI_IOV */
12853 	pf->eeprom_version = 0xDEAD;
12854 	pf->lan_veb = I40E_NO_VEB;
12855 	pf->lan_vsi = I40E_NO_VSI;
12856 
12857 	/* By default FW has this off for performance reasons */
12858 	pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
12859 
12860 	/* set up queue assignment tracking */
12861 	size = sizeof(struct i40e_lump_tracking)
12862 		+ (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12863 	pf->qp_pile = kzalloc(size, GFP_KERNEL);
12864 	if (!pf->qp_pile) {
12865 		err = -ENOMEM;
12866 		goto sw_init_done;
12867 	}
12868 	pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12869 
12870 	pf->tx_timeout_recovery_level = 1;
12871 
12872 	if (pf->hw.mac.type != I40E_MAC_X722 &&
12873 	    i40e_is_total_port_shutdown_enabled(pf)) {
12874 		/* Link down on close must be on when total port shutdown
12875 		 * is enabled for a given port
12876 		 */
12877 		pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED |
12878 			      I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED);
12879 		dev_info(&pf->pdev->dev,
12880 			 "total-port-shutdown was enabled, link-down-on-close is forced on\n");
12881 	}
12882 	mutex_init(&pf->switch_mutex);
12883 
12884 sw_init_done:
12885 	return err;
12886 }
12887 
12888 /**
12889  * i40e_set_ntuple - set the ntuple feature flag and take action
12890  * @pf: board private structure to initialize
12891  * @features: the feature set that the stack is suggesting
12892  *
12893  * returns a bool to indicate if reset needs to happen
12894  **/
12895 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12896 {
12897 	bool need_reset = false;
12898 
12899 	/* Check if Flow Director n-tuple support was enabled or disabled.  If
12900 	 * the state changed, we need to reset.
12901 	 */
12902 	if (features & NETIF_F_NTUPLE) {
12903 		/* Enable filters and mark for reset */
12904 		if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
12905 			need_reset = true;
12906 		/* enable FD_SB only if there is MSI-X vector and no cloud
12907 		 * filters exist
12908 		 */
12909 		if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12910 			pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12911 			pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
12912 		}
12913 	} else {
12914 		/* turn off filters, mark for reset and clear SW filter list */
12915 		if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
12916 			need_reset = true;
12917 			i40e_fdir_filter_exit(pf);
12918 		}
12919 		pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
12920 		clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12921 		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12922 
12923 		/* reset fd counters */
12924 		pf->fd_add_err = 0;
12925 		pf->fd_atr_cnt = 0;
12926 		/* if ATR was auto disabled it can be re-enabled. */
12927 		if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12928 			if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
12929 			    (I40E_DEBUG_FD & pf->hw.debug_mask))
12930 				dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12931 	}
12932 	return need_reset;
12933 }
12934 
12935 /**
12936  * i40e_clear_rss_lut - clear the rx hash lookup table
12937  * @vsi: the VSI being configured
12938  **/
12939 static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12940 {
12941 	struct i40e_pf *pf = vsi->back;
12942 	struct i40e_hw *hw = &pf->hw;
12943 	u16 vf_id = vsi->vf_id;
12944 	u8 i;
12945 
12946 	if (vsi->type == I40E_VSI_MAIN) {
12947 		for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12948 			wr32(hw, I40E_PFQF_HLUT(i), 0);
12949 	} else if (vsi->type == I40E_VSI_SRIOV) {
12950 		for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12951 			i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12952 	} else {
12953 		dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12954 	}
12955 }
12956 
12957 /**
12958  * i40e_set_loopback - turn on/off loopback mode on underlying PF
12959  * @vsi: ptr to VSI
12960  * @ena: flag to indicate the on/off setting
12961  */
12962 static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena)
12963 {
12964 	bool if_running = netif_running(vsi->netdev) &&
12965 			  !test_and_set_bit(__I40E_VSI_DOWN, vsi->state);
12966 	int ret;
12967 
12968 	if (if_running)
12969 		i40e_down(vsi);
12970 
12971 	ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);
12972 	if (ret)
12973 		netdev_err(vsi->netdev, "Failed to toggle loopback state\n");
12974 	if (if_running)
12975 		i40e_up(vsi);
12976 
12977 	return ret;
12978 }
12979 
12980 /**
12981  * i40e_set_features - set the netdev feature flags
12982  * @netdev: ptr to the netdev being adjusted
12983  * @features: the feature set that the stack is suggesting
12984  * Note: expects to be called while under rtnl_lock()
12985  **/
12986 static int i40e_set_features(struct net_device *netdev,
12987 			     netdev_features_t features)
12988 {
12989 	struct i40e_netdev_priv *np = netdev_priv(netdev);
12990 	struct i40e_vsi *vsi = np->vsi;
12991 	struct i40e_pf *pf = vsi->back;
12992 	bool need_reset;
12993 
12994 	if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12995 		i40e_pf_config_rss(pf);
12996 	else if (!(features & NETIF_F_RXHASH) &&
12997 		 netdev->features & NETIF_F_RXHASH)
12998 		i40e_clear_rss_lut(vsi);
12999 
13000 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
13001 		i40e_vlan_stripping_enable(vsi);
13002 	else
13003 		i40e_vlan_stripping_disable(vsi);
13004 
13005 	if (!(features & NETIF_F_HW_TC) &&
13006 	    (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
13007 		dev_err(&pf->pdev->dev,
13008 			"Offloaded tc filters active, can't turn hw_tc_offload off");
13009 		return -EINVAL;
13010 	}
13011 
13012 	if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
13013 		i40e_del_all_macvlans(vsi);
13014 
13015 	need_reset = i40e_set_ntuple(pf, features);
13016 
13017 	if (need_reset)
13018 		i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
13019 
13020 	if ((features ^ netdev->features) & NETIF_F_LOOPBACK)
13021 		return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));
13022 
13023 	return 0;
13024 }
13025 
13026 static int i40e_udp_tunnel_set_port(struct net_device *netdev,
13027 				    unsigned int table, unsigned int idx,
13028 				    struct udp_tunnel_info *ti)
13029 {
13030 	struct i40e_netdev_priv *np = netdev_priv(netdev);
13031 	struct i40e_hw *hw = &np->vsi->back->hw;
13032 	u8 type, filter_index;
13033 	int ret;
13034 
13035 	type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN :
13036 						   I40E_AQC_TUNNEL_TYPE_NGE;
13037 
13038 	ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index,
13039 				     NULL);
13040 	if (ret) {
13041 		netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n",
13042 			    ERR_PTR(ret),
13043 			    i40e_aq_str(hw, hw->aq.asq_last_status));
13044 		return -EIO;
13045 	}
13046 
13047 	udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index);
13048 	return 0;
13049 }
13050 
13051 static int i40e_udp_tunnel_unset_port(struct net_device *netdev,
13052 				      unsigned int table, unsigned int idx,
13053 				      struct udp_tunnel_info *ti)
13054 {
13055 	struct i40e_netdev_priv *np = netdev_priv(netdev);
13056 	struct i40e_hw *hw = &np->vsi->back->hw;
13057 	int ret;
13058 
13059 	ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL);
13060 	if (ret) {
13061 		netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n",
13062 			    ERR_PTR(ret),
13063 			    i40e_aq_str(hw, hw->aq.asq_last_status));
13064 		return -EIO;
13065 	}
13066 
13067 	return 0;
13068 }
13069 
13070 static int i40e_get_phys_port_id(struct net_device *netdev,
13071 				 struct netdev_phys_item_id *ppid)
13072 {
13073 	struct i40e_netdev_priv *np = netdev_priv(netdev);
13074 	struct i40e_pf *pf = np->vsi->back;
13075 	struct i40e_hw *hw = &pf->hw;
13076 
13077 	if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
13078 		return -EOPNOTSUPP;
13079 
13080 	ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
13081 	memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
13082 
13083 	return 0;
13084 }
13085 
13086 /**
13087  * i40e_ndo_fdb_add - add an entry to the hardware database
13088  * @ndm: the input from the stack
13089  * @tb: pointer to array of nladdr (unused)
13090  * @dev: the net device pointer
13091  * @addr: the MAC address entry being added
13092  * @vid: VLAN ID
13093  * @flags: instructions from stack about fdb operation
13094  * @extack: netlink extended ack, unused currently
13095  */
13096 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
13097 			    struct net_device *dev,
13098 			    const unsigned char *addr, u16 vid,
13099 			    u16 flags,
13100 			    struct netlink_ext_ack *extack)
13101 {
13102 	struct i40e_netdev_priv *np = netdev_priv(dev);
13103 	struct i40e_pf *pf = np->vsi->back;
13104 	int err = 0;
13105 
13106 	if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
13107 		return -EOPNOTSUPP;
13108 
13109 	if (vid) {
13110 		pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
13111 		return -EINVAL;
13112 	}
13113 
13114 	/* Hardware does not support aging addresses so if a
13115 	 * ndm_state is given only allow permanent addresses
13116 	 */
13117 	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
13118 		netdev_info(dev, "FDB only supports static addresses\n");
13119 		return -EINVAL;
13120 	}
13121 
13122 	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
13123 		err = dev_uc_add_excl(dev, addr);
13124 	else if (is_multicast_ether_addr(addr))
13125 		err = dev_mc_add_excl(dev, addr);
13126 	else
13127 		err = -EINVAL;
13128 
13129 	/* Only return duplicate errors if NLM_F_EXCL is set */
13130 	if (err == -EEXIST && !(flags & NLM_F_EXCL))
13131 		err = 0;
13132 
13133 	return err;
13134 }
13135 
13136 /**
13137  * i40e_ndo_bridge_setlink - Set the hardware bridge mode
13138  * @dev: the netdev being configured
13139  * @nlh: RTNL message
13140  * @flags: bridge flags
13141  * @extack: netlink extended ack
13142  *
13143  * Inserts a new hardware bridge if not already created and
13144  * enables the bridging mode requested (VEB or VEPA). If the
13145  * hardware bridge has already been inserted and the request
13146  * is to change the mode then that requires a PF reset to
13147  * allow rebuild of the components with required hardware
13148  * bridge mode enabled.
13149  *
13150  * Note: expects to be called while under rtnl_lock()
13151  **/
13152 static int i40e_ndo_bridge_setlink(struct net_device *dev,
13153 				   struct nlmsghdr *nlh,
13154 				   u16 flags,
13155 				   struct netlink_ext_ack *extack)
13156 {
13157 	struct i40e_netdev_priv *np = netdev_priv(dev);
13158 	struct i40e_vsi *vsi = np->vsi;
13159 	struct i40e_pf *pf = vsi->back;
13160 	struct i40e_veb *veb = NULL;
13161 	struct nlattr *attr, *br_spec;
13162 	int i, rem;
13163 
13164 	/* Only for PF VSI for now */
13165 	if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
13166 		return -EOPNOTSUPP;
13167 
13168 	/* Find the HW bridge for PF VSI */
13169 	for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
13170 		if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
13171 			veb = pf->veb[i];
13172 	}
13173 
13174 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
13175 	if (!br_spec)
13176 		return -EINVAL;
13177 
13178 	nla_for_each_nested(attr, br_spec, rem) {
13179 		__u16 mode;
13180 
13181 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
13182 			continue;
13183 
13184 		mode = nla_get_u16(attr);
13185 		if ((mode != BRIDGE_MODE_VEPA) &&
13186 		    (mode != BRIDGE_MODE_VEB))
13187 			return -EINVAL;
13188 
13189 		/* Insert a new HW bridge */
13190 		if (!veb) {
13191 			veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
13192 					     vsi->tc_config.enabled_tc);
13193 			if (veb) {
13194 				veb->bridge_mode = mode;
13195 				i40e_config_bridge_mode(veb);
13196 			} else {
13197 				/* No Bridge HW offload available */
13198 				return -ENOENT;
13199 			}
13200 			break;
13201 		} else if (mode != veb->bridge_mode) {
13202 			/* Existing HW bridge but different mode needs reset */
13203 			veb->bridge_mode = mode;
13204 			/* TODO: If no VFs or VMDq VSIs, disallow VEB mode */
13205 			if (mode == BRIDGE_MODE_VEB)
13206 				pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
13207 			else
13208 				pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
13209 			i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
13210 			break;
13211 		}
13212 	}
13213 
13214 	return 0;
13215 }
13216 
13217 /**
13218  * i40e_ndo_bridge_getlink - Get the hardware bridge mode
13219  * @skb: skb buff
13220  * @pid: process id
13221  * @seq: RTNL message seq #
13222  * @dev: the netdev being configured
13223  * @filter_mask: unused
13224  * @nlflags: netlink flags passed in
13225  *
13226  * Return the mode in which the hardware bridge is operating in
13227  * i.e VEB or VEPA.
13228  **/
13229 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
13230 				   struct net_device *dev,
13231 				   u32 __always_unused filter_mask,
13232 				   int nlflags)
13233 {
13234 	struct i40e_netdev_priv *np = netdev_priv(dev);
13235 	struct i40e_vsi *vsi = np->vsi;
13236 	struct i40e_pf *pf = vsi->back;
13237 	struct i40e_veb *veb = NULL;
13238 	int i;
13239 
13240 	/* Only for PF VSI for now */
13241 	if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
13242 		return -EOPNOTSUPP;
13243 
13244 	/* Find the HW bridge for the PF VSI */
13245 	for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
13246 		if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
13247 			veb = pf->veb[i];
13248 	}
13249 
13250 	if (!veb)
13251 		return 0;
13252 
13253 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
13254 				       0, 0, nlflags, filter_mask, NULL);
13255 }
13256 
13257 /**
13258  * i40e_features_check - Validate encapsulated packet conforms to limits
13259  * @skb: skb buff
13260  * @dev: This physical port's netdev
13261  * @features: Offload features that the stack believes apply
13262  **/
13263 static netdev_features_t i40e_features_check(struct sk_buff *skb,
13264 					     struct net_device *dev,
13265 					     netdev_features_t features)
13266 {
13267 	size_t len;
13268 
13269 	/* No point in doing any of this if neither checksum nor GSO are
13270 	 * being requested for this frame.  We can rule out both by just
13271 	 * checking for CHECKSUM_PARTIAL
13272 	 */
13273 	if (skb->ip_summed != CHECKSUM_PARTIAL)
13274 		return features;
13275 
13276 	/* We cannot support GSO if the MSS is going to be less than
13277 	 * 64 bytes.  If it is then we need to drop support for GSO.
13278 	 */
13279 	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
13280 		features &= ~NETIF_F_GSO_MASK;
13281 
13282 	/* MACLEN can support at most 63 words */
13283 	len = skb_network_header(skb) - skb->data;
13284 	if (len & ~(63 * 2))
13285 		goto out_err;
13286 
13287 	/* IPLEN and EIPLEN can support at most 127 dwords */
13288 	len = skb_transport_header(skb) - skb_network_header(skb);
13289 	if (len & ~(127 * 4))
13290 		goto out_err;
13291 
13292 	if (skb->encapsulation) {
13293 		/* L4TUNLEN can support 127 words */
13294 		len = skb_inner_network_header(skb) - skb_transport_header(skb);
13295 		if (len & ~(127 * 2))
13296 			goto out_err;
13297 
13298 		/* IPLEN can support at most 127 dwords */
13299 		len = skb_inner_transport_header(skb) -
13300 		      skb_inner_network_header(skb);
13301 		if (len & ~(127 * 4))
13302 			goto out_err;
13303 	}
13304 
13305 	/* No need to validate L4LEN as TCP is the only protocol with a
13306 	 * flexible value and we support all possible values supported
13307 	 * by TCP, which is at most 15 dwords
13308 	 */
13309 
13310 	return features;
13311 out_err:
13312 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
13313 }
13314 
13315 /**
13316  * i40e_xdp_setup - add/remove an XDP program
13317  * @vsi: VSI to changed
13318  * @prog: XDP program
13319  * @extack: netlink extended ack
13320  **/
13321 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog,
13322 			  struct netlink_ext_ack *extack)
13323 {
13324 	int frame_size = i40e_max_vsi_frame_size(vsi, prog);
13325 	struct i40e_pf *pf = vsi->back;
13326 	struct bpf_prog *old_prog;
13327 	bool need_reset;
13328 	int i;
13329 
13330 	/* Don't allow frames that span over multiple buffers */
13331 	if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) {
13332 		NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags");
13333 		return -EINVAL;
13334 	}
13335 
13336 	/* When turning XDP on->off/off->on we reset and rebuild the rings. */
13337 	need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
13338 
13339 	if (need_reset)
13340 		i40e_prep_for_reset(pf);
13341 
13342 	/* VSI shall be deleted in a moment, just return EINVAL */
13343 	if (test_bit(__I40E_IN_REMOVE, pf->state))
13344 		return -EINVAL;
13345 
13346 	old_prog = xchg(&vsi->xdp_prog, prog);
13347 
13348 	if (need_reset) {
13349 		if (!prog) {
13350 			xdp_features_clear_redirect_target(vsi->netdev);
13351 			/* Wait until ndo_xsk_wakeup completes. */
13352 			synchronize_rcu();
13353 		}
13354 		i40e_reset_and_rebuild(pf, true, true);
13355 	}
13356 
13357 	if (!i40e_enabled_xdp_vsi(vsi) && prog) {
13358 		if (i40e_realloc_rx_bi_zc(vsi, true))
13359 			return -ENOMEM;
13360 	} else if (i40e_enabled_xdp_vsi(vsi) && !prog) {
13361 		if (i40e_realloc_rx_bi_zc(vsi, false))
13362 			return -ENOMEM;
13363 	}
13364 
13365 	for (i = 0; i < vsi->num_queue_pairs; i++)
13366 		WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
13367 
13368 	if (old_prog)
13369 		bpf_prog_put(old_prog);
13370 
13371 	/* Kick start the NAPI context if there is an AF_XDP socket open
13372 	 * on that queue id. This so that receiving will start.
13373 	 */
13374 	if (need_reset && prog) {
13375 		for (i = 0; i < vsi->num_queue_pairs; i++)
13376 			if (vsi->xdp_rings[i]->xsk_pool)
13377 				(void)i40e_xsk_wakeup(vsi->netdev, i,
13378 						      XDP_WAKEUP_RX);
13379 		xdp_features_set_redirect_target(vsi->netdev, true);
13380 	}
13381 
13382 	return 0;
13383 }
13384 
13385 /**
13386  * i40e_enter_busy_conf - Enters busy config state
13387  * @vsi: vsi
13388  *
13389  * Returns 0 on success, <0 for failure.
13390  **/
13391 static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
13392 {
13393 	struct i40e_pf *pf = vsi->back;
13394 	int timeout = 50;
13395 
13396 	while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
13397 		timeout--;
13398 		if (!timeout)
13399 			return -EBUSY;
13400 		usleep_range(1000, 2000);
13401 	}
13402 
13403 	return 0;
13404 }
13405 
13406 /**
13407  * i40e_exit_busy_conf - Exits busy config state
13408  * @vsi: vsi
13409  **/
13410 static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
13411 {
13412 	struct i40e_pf *pf = vsi->back;
13413 
13414 	clear_bit(__I40E_CONFIG_BUSY, pf->state);
13415 }
13416 
13417 /**
13418  * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair
13419  * @vsi: vsi
13420  * @queue_pair: queue pair
13421  **/
13422 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
13423 {
13424 	memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
13425 	       sizeof(vsi->rx_rings[queue_pair]->rx_stats));
13426 	memset(&vsi->tx_rings[queue_pair]->stats, 0,
13427 	       sizeof(vsi->tx_rings[queue_pair]->stats));
13428 	if (i40e_enabled_xdp_vsi(vsi)) {
13429 		memset(&vsi->xdp_rings[queue_pair]->stats, 0,
13430 		       sizeof(vsi->xdp_rings[queue_pair]->stats));
13431 	}
13432 }
13433 
13434 /**
13435  * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair
13436  * @vsi: vsi
13437  * @queue_pair: queue pair
13438  **/
13439 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
13440 {
13441 	i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
13442 	if (i40e_enabled_xdp_vsi(vsi)) {
13443 		/* Make sure that in-progress ndo_xdp_xmit calls are
13444 		 * completed.
13445 		 */
13446 		synchronize_rcu();
13447 		i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
13448 	}
13449 	i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13450 }
13451 
13452 /**
13453  * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair
13454  * @vsi: vsi
13455  * @queue_pair: queue pair
13456  * @enable: true for enable, false for disable
13457  **/
13458 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
13459 					bool enable)
13460 {
13461 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13462 	struct i40e_q_vector *q_vector = rxr->q_vector;
13463 
13464 	if (!vsi->netdev)
13465 		return;
13466 
13467 	/* All rings in a qp belong to the same qvector. */
13468 	if (q_vector->rx.ring || q_vector->tx.ring) {
13469 		if (enable)
13470 			napi_enable(&q_vector->napi);
13471 		else
13472 			napi_disable(&q_vector->napi);
13473 	}
13474 }
13475 
13476 /**
13477  * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair
13478  * @vsi: vsi
13479  * @queue_pair: queue pair
13480  * @enable: true for enable, false for disable
13481  *
13482  * Returns 0 on success, <0 on failure.
13483  **/
13484 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
13485 					bool enable)
13486 {
13487 	struct i40e_pf *pf = vsi->back;
13488 	int pf_q, ret = 0;
13489 
13490 	pf_q = vsi->base_queue + queue_pair;
13491 	ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
13492 				     false /*is xdp*/, enable);
13493 	if (ret) {
13494 		dev_info(&pf->pdev->dev,
13495 			 "VSI seid %d Tx ring %d %sable timeout\n",
13496 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
13497 		return ret;
13498 	}
13499 
13500 	i40e_control_rx_q(pf, pf_q, enable);
13501 	ret = i40e_pf_rxq_wait(pf, pf_q, enable);
13502 	if (ret) {
13503 		dev_info(&pf->pdev->dev,
13504 			 "VSI seid %d Rx ring %d %sable timeout\n",
13505 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
13506 		return ret;
13507 	}
13508 
13509 	/* Due to HW errata, on Rx disable only, the register can
13510 	 * indicate done before it really is. Needs 50ms to be sure
13511 	 */
13512 	if (!enable)
13513 		mdelay(50);
13514 
13515 	if (!i40e_enabled_xdp_vsi(vsi))
13516 		return ret;
13517 
13518 	ret = i40e_control_wait_tx_q(vsi->seid, pf,
13519 				     pf_q + vsi->alloc_queue_pairs,
13520 				     true /*is xdp*/, enable);
13521 	if (ret) {
13522 		dev_info(&pf->pdev->dev,
13523 			 "VSI seid %d XDP Tx ring %d %sable timeout\n",
13524 			 vsi->seid, pf_q, (enable ? "en" : "dis"));
13525 	}
13526 
13527 	return ret;
13528 }
13529 
13530 /**
13531  * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair
13532  * @vsi: vsi
13533  * @queue_pair: queue_pair
13534  **/
13535 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
13536 {
13537 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13538 	struct i40e_pf *pf = vsi->back;
13539 	struct i40e_hw *hw = &pf->hw;
13540 
13541 	/* All rings in a qp belong to the same qvector. */
13542 	if (pf->flags & I40E_FLAG_MSIX_ENABLED)
13543 		i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
13544 	else
13545 		i40e_irq_dynamic_enable_icr0(pf);
13546 
13547 	i40e_flush(hw);
13548 }
13549 
13550 /**
13551  * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair
13552  * @vsi: vsi
13553  * @queue_pair: queue_pair
13554  **/
13555 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
13556 {
13557 	struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
13558 	struct i40e_pf *pf = vsi->back;
13559 	struct i40e_hw *hw = &pf->hw;
13560 
13561 	/* For simplicity, instead of removing the qp interrupt causes
13562 	 * from the interrupt linked list, we simply disable the interrupt, and
13563 	 * leave the list intact.
13564 	 *
13565 	 * All rings in a qp belong to the same qvector.
13566 	 */
13567 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
13568 		u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
13569 
13570 		wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
13571 		i40e_flush(hw);
13572 		synchronize_irq(pf->msix_entries[intpf].vector);
13573 	} else {
13574 		/* Legacy and MSI mode - this stops all interrupt handling */
13575 		wr32(hw, I40E_PFINT_ICR0_ENA, 0);
13576 		wr32(hw, I40E_PFINT_DYN_CTL0, 0);
13577 		i40e_flush(hw);
13578 		synchronize_irq(pf->pdev->irq);
13579 	}
13580 }
13581 
13582 /**
13583  * i40e_queue_pair_disable - Disables a queue pair
13584  * @vsi: vsi
13585  * @queue_pair: queue pair
13586  *
13587  * Returns 0 on success, <0 on failure.
13588  **/
13589 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
13590 {
13591 	int err;
13592 
13593 	err = i40e_enter_busy_conf(vsi);
13594 	if (err)
13595 		return err;
13596 
13597 	i40e_queue_pair_disable_irq(vsi, queue_pair);
13598 	err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */);
13599 	i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
13600 	i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */);
13601 	i40e_queue_pair_clean_rings(vsi, queue_pair);
13602 	i40e_queue_pair_reset_stats(vsi, queue_pair);
13603 
13604 	return err;
13605 }
13606 
13607 /**
13608  * i40e_queue_pair_enable - Enables a queue pair
13609  * @vsi: vsi
13610  * @queue_pair: queue pair
13611  *
13612  * Returns 0 on success, <0 on failure.
13613  **/
13614 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
13615 {
13616 	int err;
13617 
13618 	err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
13619 	if (err)
13620 		return err;
13621 
13622 	if (i40e_enabled_xdp_vsi(vsi)) {
13623 		err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
13624 		if (err)
13625 			return err;
13626 	}
13627 
13628 	err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
13629 	if (err)
13630 		return err;
13631 
13632 	err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */);
13633 	i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */);
13634 	i40e_queue_pair_enable_irq(vsi, queue_pair);
13635 
13636 	i40e_exit_busy_conf(vsi);
13637 
13638 	return err;
13639 }
13640 
13641 /**
13642  * i40e_xdp - implements ndo_bpf for i40e
13643  * @dev: netdevice
13644  * @xdp: XDP command
13645  **/
13646 static int i40e_xdp(struct net_device *dev,
13647 		    struct netdev_bpf *xdp)
13648 {
13649 	struct i40e_netdev_priv *np = netdev_priv(dev);
13650 	struct i40e_vsi *vsi = np->vsi;
13651 
13652 	if (vsi->type != I40E_VSI_MAIN)
13653 		return -EINVAL;
13654 
13655 	switch (xdp->command) {
13656 	case XDP_SETUP_PROG:
13657 		return i40e_xdp_setup(vsi, xdp->prog, xdp->extack);
13658 	case XDP_SETUP_XSK_POOL:
13659 		return i40e_xsk_pool_setup(vsi, xdp->xsk.pool,
13660 					   xdp->xsk.queue_id);
13661 	default:
13662 		return -EINVAL;
13663 	}
13664 }
13665 
13666 static const struct net_device_ops i40e_netdev_ops = {
13667 	.ndo_open		= i40e_open,
13668 	.ndo_stop		= i40e_close,
13669 	.ndo_start_xmit		= i40e_lan_xmit_frame,
13670 	.ndo_get_stats64	= i40e_get_netdev_stats_struct,
13671 	.ndo_set_rx_mode	= i40e_set_rx_mode,
13672 	.ndo_validate_addr	= eth_validate_addr,
13673 	.ndo_set_mac_address	= i40e_set_mac,
13674 	.ndo_change_mtu		= i40e_change_mtu,
13675 	.ndo_eth_ioctl		= i40e_ioctl,
13676 	.ndo_tx_timeout		= i40e_tx_timeout,
13677 	.ndo_vlan_rx_add_vid	= i40e_vlan_rx_add_vid,
13678 	.ndo_vlan_rx_kill_vid	= i40e_vlan_rx_kill_vid,
13679 #ifdef CONFIG_NET_POLL_CONTROLLER
13680 	.ndo_poll_controller	= i40e_netpoll,
13681 #endif
13682 	.ndo_setup_tc		= __i40e_setup_tc,
13683 	.ndo_select_queue	= i40e_lan_select_queue,
13684 	.ndo_set_features	= i40e_set_features,
13685 	.ndo_set_vf_mac		= i40e_ndo_set_vf_mac,
13686 	.ndo_set_vf_vlan	= i40e_ndo_set_vf_port_vlan,
13687 	.ndo_get_vf_stats	= i40e_get_vf_stats,
13688 	.ndo_set_vf_rate	= i40e_ndo_set_vf_bw,
13689 	.ndo_get_vf_config	= i40e_ndo_get_vf_config,
13690 	.ndo_set_vf_link_state	= i40e_ndo_set_vf_link_state,
13691 	.ndo_set_vf_spoofchk	= i40e_ndo_set_vf_spoofchk,
13692 	.ndo_set_vf_trust	= i40e_ndo_set_vf_trust,
13693 	.ndo_get_phys_port_id	= i40e_get_phys_port_id,
13694 	.ndo_fdb_add		= i40e_ndo_fdb_add,
13695 	.ndo_features_check	= i40e_features_check,
13696 	.ndo_bridge_getlink	= i40e_ndo_bridge_getlink,
13697 	.ndo_bridge_setlink	= i40e_ndo_bridge_setlink,
13698 	.ndo_bpf		= i40e_xdp,
13699 	.ndo_xdp_xmit		= i40e_xdp_xmit,
13700 	.ndo_xsk_wakeup	        = i40e_xsk_wakeup,
13701 	.ndo_dfwd_add_station	= i40e_fwd_add,
13702 	.ndo_dfwd_del_station	= i40e_fwd_del,
13703 };
13704 
13705 /**
13706  * i40e_config_netdev - Setup the netdev flags
13707  * @vsi: the VSI being configured
13708  *
13709  * Returns 0 on success, negative value on failure
13710  **/
13711 static int i40e_config_netdev(struct i40e_vsi *vsi)
13712 {
13713 	struct i40e_pf *pf = vsi->back;
13714 	struct i40e_hw *hw = &pf->hw;
13715 	struct i40e_netdev_priv *np;
13716 	struct net_device *netdev;
13717 	u8 broadcast[ETH_ALEN];
13718 	u8 mac_addr[ETH_ALEN];
13719 	int etherdev_size;
13720 	netdev_features_t hw_enc_features;
13721 	netdev_features_t hw_features;
13722 
13723 	etherdev_size = sizeof(struct i40e_netdev_priv);
13724 	netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
13725 	if (!netdev)
13726 		return -ENOMEM;
13727 
13728 	vsi->netdev = netdev;
13729 	np = netdev_priv(netdev);
13730 	np->vsi = vsi;
13731 
13732 	hw_enc_features = NETIF_F_SG			|
13733 			  NETIF_F_HW_CSUM		|
13734 			  NETIF_F_HIGHDMA		|
13735 			  NETIF_F_SOFT_FEATURES		|
13736 			  NETIF_F_TSO			|
13737 			  NETIF_F_TSO_ECN		|
13738 			  NETIF_F_TSO6			|
13739 			  NETIF_F_GSO_GRE		|
13740 			  NETIF_F_GSO_GRE_CSUM		|
13741 			  NETIF_F_GSO_PARTIAL		|
13742 			  NETIF_F_GSO_IPXIP4		|
13743 			  NETIF_F_GSO_IPXIP6		|
13744 			  NETIF_F_GSO_UDP_TUNNEL	|
13745 			  NETIF_F_GSO_UDP_TUNNEL_CSUM	|
13746 			  NETIF_F_GSO_UDP_L4		|
13747 			  NETIF_F_SCTP_CRC		|
13748 			  NETIF_F_RXHASH		|
13749 			  NETIF_F_RXCSUM		|
13750 			  0;
13751 
13752 	if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
13753 		netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
13754 
13755 	netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic;
13756 
13757 	netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
13758 
13759 	netdev->hw_enc_features |= hw_enc_features;
13760 
13761 	/* record features VLANs can make use of */
13762 	netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
13763 
13764 #define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE |		\
13765 				   NETIF_F_GSO_GRE_CSUM |	\
13766 				   NETIF_F_GSO_IPXIP4 |		\
13767 				   NETIF_F_GSO_IPXIP6 |		\
13768 				   NETIF_F_GSO_UDP_TUNNEL |	\
13769 				   NETIF_F_GSO_UDP_TUNNEL_CSUM)
13770 
13771 	netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES;
13772 	netdev->features |= NETIF_F_GSO_PARTIAL |
13773 			    I40E_GSO_PARTIAL_FEATURES;
13774 
13775 	netdev->mpls_features |= NETIF_F_SG;
13776 	netdev->mpls_features |= NETIF_F_HW_CSUM;
13777 	netdev->mpls_features |= NETIF_F_TSO;
13778 	netdev->mpls_features |= NETIF_F_TSO6;
13779 	netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES;
13780 
13781 	/* enable macvlan offloads */
13782 	netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
13783 
13784 	hw_features = hw_enc_features		|
13785 		      NETIF_F_HW_VLAN_CTAG_TX	|
13786 		      NETIF_F_HW_VLAN_CTAG_RX;
13787 
13788 	if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
13789 		hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
13790 
13791 	netdev->hw_features |= hw_features | NETIF_F_LOOPBACK;
13792 
13793 	netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
13794 	netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
13795 
13796 	netdev->features &= ~NETIF_F_HW_TC;
13797 
13798 	if (vsi->type == I40E_VSI_MAIN) {
13799 		SET_NETDEV_DEV(netdev, &pf->pdev->dev);
13800 		ether_addr_copy(mac_addr, hw->mac.perm_addr);
13801 		/* The following steps are necessary for two reasons. First,
13802 		 * some older NVM configurations load a default MAC-VLAN
13803 		 * filter that will accept any tagged packet, and we want to
13804 		 * replace this with a normal filter. Additionally, it is
13805 		 * possible our MAC address was provided by the platform using
13806 		 * Open Firmware or similar.
13807 		 *
13808 		 * Thus, we need to remove the default filter and install one
13809 		 * specific to the MAC address.
13810 		 */
13811 		i40e_rm_default_mac_filter(vsi, mac_addr);
13812 		spin_lock_bh(&vsi->mac_filter_hash_lock);
13813 		i40e_add_mac_filter(vsi, mac_addr);
13814 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
13815 
13816 		netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
13817 				       NETDEV_XDP_ACT_REDIRECT |
13818 				       NETDEV_XDP_ACT_XSK_ZEROCOPY |
13819 				       NETDEV_XDP_ACT_RX_SG;
13820 		netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD;
13821 	} else {
13822 		/* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we
13823 		 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to
13824 		 * the end, which is 4 bytes long, so force truncation of the
13825 		 * original name by IFNAMSIZ - 4
13826 		 */
13827 		snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
13828 			 IFNAMSIZ - 4,
13829 			 pf->vsi[pf->lan_vsi]->netdev->name);
13830 		eth_random_addr(mac_addr);
13831 
13832 		spin_lock_bh(&vsi->mac_filter_hash_lock);
13833 		i40e_add_mac_filter(vsi, mac_addr);
13834 		spin_unlock_bh(&vsi->mac_filter_hash_lock);
13835 	}
13836 
13837 	/* Add the broadcast filter so that we initially will receive
13838 	 * broadcast packets. Note that when a new VLAN is first added the
13839 	 * driver will convert all filters marked I40E_VLAN_ANY into VLAN
13840 	 * specific filters as part of transitioning into "vlan" operation.
13841 	 * When more VLANs are added, the driver will copy each existing MAC
13842 	 * filter and add it for the new VLAN.
13843 	 *
13844 	 * Broadcast filters are handled specially by
13845 	 * i40e_sync_filters_subtask, as the driver must to set the broadcast
13846 	 * promiscuous bit instead of adding this directly as a MAC/VLAN
13847 	 * filter. The subtask will update the correct broadcast promiscuous
13848 	 * bits as VLANs become active or inactive.
13849 	 */
13850 	eth_broadcast_addr(broadcast);
13851 	spin_lock_bh(&vsi->mac_filter_hash_lock);
13852 	i40e_add_mac_filter(vsi, broadcast);
13853 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
13854 
13855 	eth_hw_addr_set(netdev, mac_addr);
13856 	ether_addr_copy(netdev->perm_addr, mac_addr);
13857 
13858 	/* i40iw_net_event() reads 16 bytes from neigh->primary_key */
13859 	netdev->neigh_priv_len = sizeof(u32) * 4;
13860 
13861 	netdev->priv_flags |= IFF_UNICAST_FLT;
13862 	netdev->priv_flags |= IFF_SUPP_NOFCS;
13863 	/* Setup netdev TC information */
13864 	i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13865 
13866 	netdev->netdev_ops = &i40e_netdev_ops;
13867 	netdev->watchdog_timeo = 5 * HZ;
13868 	i40e_set_ethtool_ops(netdev);
13869 
13870 	/* MTU range: 68 - 9706 */
13871 	netdev->min_mtu = ETH_MIN_MTU;
13872 	netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13873 
13874 	return 0;
13875 }
13876 
13877 /**
13878  * i40e_vsi_delete - Delete a VSI from the switch
13879  * @vsi: the VSI being removed
13880  *
13881  * Returns 0 on success, negative value on failure
13882  **/
13883 static void i40e_vsi_delete(struct i40e_vsi *vsi)
13884 {
13885 	/* remove default VSI is not allowed */
13886 	if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13887 		return;
13888 
13889 	i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13890 }
13891 
13892 /**
13893  * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB
13894  * @vsi: the VSI being queried
13895  *
13896  * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode
13897  **/
13898 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13899 {
13900 	struct i40e_veb *veb;
13901 	struct i40e_pf *pf = vsi->back;
13902 
13903 	/* Uplink is not a bridge so default to VEB */
13904 	if (vsi->veb_idx >= I40E_MAX_VEB)
13905 		return 1;
13906 
13907 	veb = pf->veb[vsi->veb_idx];
13908 	if (!veb) {
13909 		dev_info(&pf->pdev->dev,
13910 			 "There is no veb associated with the bridge\n");
13911 		return -ENOENT;
13912 	}
13913 
13914 	/* Uplink is a bridge in VEPA mode */
13915 	if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13916 		return 0;
13917 	} else {
13918 		/* Uplink is a bridge in VEB mode */
13919 		return 1;
13920 	}
13921 
13922 	/* VEPA is now default bridge, so return 0 */
13923 	return 0;
13924 }
13925 
13926 /**
13927  * i40e_add_vsi - Add a VSI to the switch
13928  * @vsi: the VSI being configured
13929  *
13930  * This initializes a VSI context depending on the VSI type to be added and
13931  * passes it down to the add_vsi aq command.
13932  **/
13933 static int i40e_add_vsi(struct i40e_vsi *vsi)
13934 {
13935 	int ret = -ENODEV;
13936 	struct i40e_pf *pf = vsi->back;
13937 	struct i40e_hw *hw = &pf->hw;
13938 	struct i40e_vsi_context ctxt;
13939 	struct i40e_mac_filter *f;
13940 	struct hlist_node *h;
13941 	int bkt;
13942 
13943 	u8 enabled_tc = 0x1; /* TC0 enabled */
13944 	int f_count = 0;
13945 
13946 	memset(&ctxt, 0, sizeof(ctxt));
13947 	switch (vsi->type) {
13948 	case I40E_VSI_MAIN:
13949 		/* The PF's main VSI is already setup as part of the
13950 		 * device initialization, so we'll not bother with
13951 		 * the add_vsi call, but we will retrieve the current
13952 		 * VSI context.
13953 		 */
13954 		ctxt.seid = pf->main_vsi_seid;
13955 		ctxt.pf_num = pf->hw.pf_id;
13956 		ctxt.vf_num = 0;
13957 		ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13958 		ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13959 		if (ret) {
13960 			dev_info(&pf->pdev->dev,
13961 				 "couldn't get PF vsi config, err %pe aq_err %s\n",
13962 				 ERR_PTR(ret),
13963 				 i40e_aq_str(&pf->hw,
13964 					     pf->hw.aq.asq_last_status));
13965 			return -ENOENT;
13966 		}
13967 		vsi->info = ctxt.info;
13968 		vsi->info.valid_sections = 0;
13969 
13970 		vsi->seid = ctxt.seid;
13971 		vsi->id = ctxt.vsi_number;
13972 
13973 		enabled_tc = i40e_pf_get_tc_map(pf);
13974 
13975 		/* Source pruning is enabled by default, so the flag is
13976 		 * negative logic - if it's set, we need to fiddle with
13977 		 * the VSI to disable source pruning.
13978 		 */
13979 		if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
13980 			memset(&ctxt, 0, sizeof(ctxt));
13981 			ctxt.seid = pf->main_vsi_seid;
13982 			ctxt.pf_num = pf->hw.pf_id;
13983 			ctxt.vf_num = 0;
13984 			ctxt.info.valid_sections |=
13985 				     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13986 			ctxt.info.switch_id =
13987 				   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13988 			ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13989 			if (ret) {
13990 				dev_info(&pf->pdev->dev,
13991 					 "update vsi failed, err %d aq_err %s\n",
13992 					 ret,
13993 					 i40e_aq_str(&pf->hw,
13994 						     pf->hw.aq.asq_last_status));
13995 				ret = -ENOENT;
13996 				goto err;
13997 			}
13998 		}
13999 
14000 		/* MFP mode setup queue map and update VSI */
14001 		if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
14002 		    !(pf->hw.func_caps.iscsi)) { /* NIC type PF */
14003 			memset(&ctxt, 0, sizeof(ctxt));
14004 			ctxt.seid = pf->main_vsi_seid;
14005 			ctxt.pf_num = pf->hw.pf_id;
14006 			ctxt.vf_num = 0;
14007 			i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
14008 			ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
14009 			if (ret) {
14010 				dev_info(&pf->pdev->dev,
14011 					 "update vsi failed, err %pe aq_err %s\n",
14012 					 ERR_PTR(ret),
14013 					 i40e_aq_str(&pf->hw,
14014 						    pf->hw.aq.asq_last_status));
14015 				ret = -ENOENT;
14016 				goto err;
14017 			}
14018 			/* update the local VSI info queue map */
14019 			i40e_vsi_update_queue_map(vsi, &ctxt);
14020 			vsi->info.valid_sections = 0;
14021 		} else {
14022 			/* Default/Main VSI is only enabled for TC0
14023 			 * reconfigure it to enable all TCs that are
14024 			 * available on the port in SFP mode.
14025 			 * For MFP case the iSCSI PF would use this
14026 			 * flow to enable LAN+iSCSI TC.
14027 			 */
14028 			ret = i40e_vsi_config_tc(vsi, enabled_tc);
14029 			if (ret) {
14030 				/* Single TC condition is not fatal,
14031 				 * message and continue
14032 				 */
14033 				dev_info(&pf->pdev->dev,
14034 					 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n",
14035 					 enabled_tc,
14036 					 ERR_PTR(ret),
14037 					 i40e_aq_str(&pf->hw,
14038 						    pf->hw.aq.asq_last_status));
14039 			}
14040 		}
14041 		break;
14042 
14043 	case I40E_VSI_FDIR:
14044 		ctxt.pf_num = hw->pf_id;
14045 		ctxt.vf_num = 0;
14046 		ctxt.uplink_seid = vsi->uplink_seid;
14047 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14048 		ctxt.flags = I40E_AQ_VSI_TYPE_PF;
14049 		if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
14050 		    (i40e_is_vsi_uplink_mode_veb(vsi))) {
14051 			ctxt.info.valid_sections |=
14052 			     cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14053 			ctxt.info.switch_id =
14054 			   cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14055 		}
14056 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14057 		break;
14058 
14059 	case I40E_VSI_VMDQ2:
14060 		ctxt.pf_num = hw->pf_id;
14061 		ctxt.vf_num = 0;
14062 		ctxt.uplink_seid = vsi->uplink_seid;
14063 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14064 		ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
14065 
14066 		/* This VSI is connected to VEB so the switch_id
14067 		 * should be set to zero by default.
14068 		 */
14069 		if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14070 			ctxt.info.valid_sections |=
14071 				cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14072 			ctxt.info.switch_id =
14073 				cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14074 		}
14075 
14076 		/* Setup the VSI tx/rx queue map for TC0 only for now */
14077 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14078 		break;
14079 
14080 	case I40E_VSI_SRIOV:
14081 		ctxt.pf_num = hw->pf_id;
14082 		ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
14083 		ctxt.uplink_seid = vsi->uplink_seid;
14084 		ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
14085 		ctxt.flags = I40E_AQ_VSI_TYPE_VF;
14086 
14087 		/* This VSI is connected to VEB so the switch_id
14088 		 * should be set to zero by default.
14089 		 */
14090 		if (i40e_is_vsi_uplink_mode_veb(vsi)) {
14091 			ctxt.info.valid_sections |=
14092 				cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
14093 			ctxt.info.switch_id =
14094 				cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
14095 		}
14096 
14097 		if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
14098 			ctxt.info.valid_sections |=
14099 				cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
14100 			ctxt.info.queueing_opt_flags |=
14101 				(I40E_AQ_VSI_QUE_OPT_TCP_ENA |
14102 				 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
14103 		}
14104 
14105 		ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
14106 		ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
14107 		if (pf->vf[vsi->vf_id].spoofchk) {
14108 			ctxt.info.valid_sections |=
14109 				cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
14110 			ctxt.info.sec_flags |=
14111 				(I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
14112 				 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
14113 		}
14114 		/* Setup the VSI tx/rx queue map for TC0 only for now */
14115 		i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
14116 		break;
14117 
14118 	case I40E_VSI_IWARP:
14119 		/* send down message to iWARP */
14120 		break;
14121 
14122 	default:
14123 		return -ENODEV;
14124 	}
14125 
14126 	if (vsi->type != I40E_VSI_MAIN) {
14127 		ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
14128 		if (ret) {
14129 			dev_info(&vsi->back->pdev->dev,
14130 				 "add vsi failed, err %pe aq_err %s\n",
14131 				 ERR_PTR(ret),
14132 				 i40e_aq_str(&pf->hw,
14133 					     pf->hw.aq.asq_last_status));
14134 			ret = -ENOENT;
14135 			goto err;
14136 		}
14137 		vsi->info = ctxt.info;
14138 		vsi->info.valid_sections = 0;
14139 		vsi->seid = ctxt.seid;
14140 		vsi->id = ctxt.vsi_number;
14141 	}
14142 
14143 	spin_lock_bh(&vsi->mac_filter_hash_lock);
14144 	vsi->active_filters = 0;
14145 	/* If macvlan filters already exist, force them to get loaded */
14146 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
14147 		f->state = I40E_FILTER_NEW;
14148 		f_count++;
14149 	}
14150 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
14151 	clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
14152 
14153 	if (f_count) {
14154 		vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
14155 		set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
14156 	}
14157 
14158 	/* Update VSI BW information */
14159 	ret = i40e_vsi_get_bw_info(vsi);
14160 	if (ret) {
14161 		dev_info(&pf->pdev->dev,
14162 			 "couldn't get vsi bw info, err %pe aq_err %s\n",
14163 			 ERR_PTR(ret),
14164 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14165 		/* VSI is already added so not tearing that up */
14166 		ret = 0;
14167 	}
14168 
14169 err:
14170 	return ret;
14171 }
14172 
14173 /**
14174  * i40e_vsi_release - Delete a VSI and free its resources
14175  * @vsi: the VSI being removed
14176  *
14177  * Returns 0 on success or < 0 on error
14178  **/
14179 int i40e_vsi_release(struct i40e_vsi *vsi)
14180 {
14181 	struct i40e_mac_filter *f;
14182 	struct hlist_node *h;
14183 	struct i40e_veb *veb = NULL;
14184 	struct i40e_pf *pf;
14185 	u16 uplink_seid;
14186 	int i, n, bkt;
14187 
14188 	pf = vsi->back;
14189 
14190 	/* release of a VEB-owner or last VSI is not allowed */
14191 	if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
14192 		dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
14193 			 vsi->seid, vsi->uplink_seid);
14194 		return -ENODEV;
14195 	}
14196 	if (vsi == pf->vsi[pf->lan_vsi] &&
14197 	    !test_bit(__I40E_DOWN, pf->state)) {
14198 		dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
14199 		return -ENODEV;
14200 	}
14201 	set_bit(__I40E_VSI_RELEASING, vsi->state);
14202 	uplink_seid = vsi->uplink_seid;
14203 	if (vsi->type != I40E_VSI_SRIOV) {
14204 		if (vsi->netdev_registered) {
14205 			vsi->netdev_registered = false;
14206 			if (vsi->netdev) {
14207 				/* results in a call to i40e_close() */
14208 				unregister_netdev(vsi->netdev);
14209 			}
14210 		} else {
14211 			i40e_vsi_close(vsi);
14212 		}
14213 		i40e_vsi_disable_irq(vsi);
14214 	}
14215 
14216 	spin_lock_bh(&vsi->mac_filter_hash_lock);
14217 
14218 	/* clear the sync flag on all filters */
14219 	if (vsi->netdev) {
14220 		__dev_uc_unsync(vsi->netdev, NULL);
14221 		__dev_mc_unsync(vsi->netdev, NULL);
14222 	}
14223 
14224 	/* make sure any remaining filters are marked for deletion */
14225 	hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
14226 		__i40e_del_filter(vsi, f);
14227 
14228 	spin_unlock_bh(&vsi->mac_filter_hash_lock);
14229 
14230 	i40e_sync_vsi_filters(vsi);
14231 
14232 	i40e_vsi_delete(vsi);
14233 	i40e_vsi_free_q_vectors(vsi);
14234 	if (vsi->netdev) {
14235 		free_netdev(vsi->netdev);
14236 		vsi->netdev = NULL;
14237 	}
14238 	i40e_vsi_clear_rings(vsi);
14239 	i40e_vsi_clear(vsi);
14240 
14241 	/* If this was the last thing on the VEB, except for the
14242 	 * controlling VSI, remove the VEB, which puts the controlling
14243 	 * VSI onto the next level down in the switch.
14244 	 *
14245 	 * Well, okay, there's one more exception here: don't remove
14246 	 * the orphan VEBs yet.  We'll wait for an explicit remove request
14247 	 * from up the network stack.
14248 	 */
14249 	for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
14250 		if (pf->vsi[i] &&
14251 		    pf->vsi[i]->uplink_seid == uplink_seid &&
14252 		    (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
14253 			n++;      /* count the VSIs */
14254 		}
14255 	}
14256 	for (i = 0; i < I40E_MAX_VEB; i++) {
14257 		if (!pf->veb[i])
14258 			continue;
14259 		if (pf->veb[i]->uplink_seid == uplink_seid)
14260 			n++;     /* count the VEBs */
14261 		if (pf->veb[i]->seid == uplink_seid)
14262 			veb = pf->veb[i];
14263 	}
14264 	if (n == 0 && veb && veb->uplink_seid != 0)
14265 		i40e_veb_release(veb);
14266 
14267 	return 0;
14268 }
14269 
14270 /**
14271  * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI
14272  * @vsi: ptr to the VSI
14273  *
14274  * This should only be called after i40e_vsi_mem_alloc() which allocates the
14275  * corresponding SW VSI structure and initializes num_queue_pairs for the
14276  * newly allocated VSI.
14277  *
14278  * Returns 0 on success or negative on failure
14279  **/
14280 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
14281 {
14282 	int ret = -ENOENT;
14283 	struct i40e_pf *pf = vsi->back;
14284 
14285 	if (vsi->q_vectors[0]) {
14286 		dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
14287 			 vsi->seid);
14288 		return -EEXIST;
14289 	}
14290 
14291 	if (vsi->base_vector) {
14292 		dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
14293 			 vsi->seid, vsi->base_vector);
14294 		return -EEXIST;
14295 	}
14296 
14297 	ret = i40e_vsi_alloc_q_vectors(vsi);
14298 	if (ret) {
14299 		dev_info(&pf->pdev->dev,
14300 			 "failed to allocate %d q_vector for VSI %d, ret=%d\n",
14301 			 vsi->num_q_vectors, vsi->seid, ret);
14302 		vsi->num_q_vectors = 0;
14303 		goto vector_setup_out;
14304 	}
14305 
14306 	/* In Legacy mode, we do not have to get any other vector since we
14307 	 * piggyback on the misc/ICR0 for queue interrupts.
14308 	*/
14309 	if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
14310 		return ret;
14311 	if (vsi->num_q_vectors)
14312 		vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
14313 						 vsi->num_q_vectors, vsi->idx);
14314 	if (vsi->base_vector < 0) {
14315 		dev_info(&pf->pdev->dev,
14316 			 "failed to get tracking for %d vectors for VSI %d, err=%d\n",
14317 			 vsi->num_q_vectors, vsi->seid, vsi->base_vector);
14318 		i40e_vsi_free_q_vectors(vsi);
14319 		ret = -ENOENT;
14320 		goto vector_setup_out;
14321 	}
14322 
14323 vector_setup_out:
14324 	return ret;
14325 }
14326 
14327 /**
14328  * i40e_vsi_reinit_setup - return and reallocate resources for a VSI
14329  * @vsi: pointer to the vsi.
14330  *
14331  * This re-allocates a vsi's queue resources.
14332  *
14333  * Returns pointer to the successfully allocated and configured VSI sw struct
14334  * on success, otherwise returns NULL on failure.
14335  **/
14336 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
14337 {
14338 	u16 alloc_queue_pairs;
14339 	struct i40e_pf *pf;
14340 	u8 enabled_tc;
14341 	int ret;
14342 
14343 	if (!vsi)
14344 		return NULL;
14345 
14346 	pf = vsi->back;
14347 
14348 	i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
14349 	i40e_vsi_clear_rings(vsi);
14350 
14351 	i40e_vsi_free_arrays(vsi, false);
14352 	i40e_set_num_rings_in_vsi(vsi);
14353 	ret = i40e_vsi_alloc_arrays(vsi, false);
14354 	if (ret)
14355 		goto err_vsi;
14356 
14357 	alloc_queue_pairs = vsi->alloc_queue_pairs *
14358 			    (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14359 
14360 	ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14361 	if (ret < 0) {
14362 		dev_info(&pf->pdev->dev,
14363 			 "failed to get tracking for %d queues for VSI %d err %d\n",
14364 			 alloc_queue_pairs, vsi->seid, ret);
14365 		goto err_vsi;
14366 	}
14367 	vsi->base_queue = ret;
14368 
14369 	/* Update the FW view of the VSI. Force a reset of TC and queue
14370 	 * layout configurations.
14371 	 */
14372 	enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14373 	pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14374 	pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14375 	i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14376 	if (vsi->type == I40E_VSI_MAIN)
14377 		i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
14378 
14379 	/* assign it some queues */
14380 	ret = i40e_alloc_rings(vsi);
14381 	if (ret)
14382 		goto err_rings;
14383 
14384 	/* map all of the rings to the q_vectors */
14385 	i40e_vsi_map_rings_to_vectors(vsi);
14386 	return vsi;
14387 
14388 err_rings:
14389 	i40e_vsi_free_q_vectors(vsi);
14390 	if (vsi->netdev_registered) {
14391 		vsi->netdev_registered = false;
14392 		unregister_netdev(vsi->netdev);
14393 		free_netdev(vsi->netdev);
14394 		vsi->netdev = NULL;
14395 	}
14396 	i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14397 err_vsi:
14398 	i40e_vsi_clear(vsi);
14399 	return NULL;
14400 }
14401 
14402 /**
14403  * i40e_vsi_setup - Set up a VSI by a given type
14404  * @pf: board private structure
14405  * @type: VSI type
14406  * @uplink_seid: the switch element to link to
14407  * @param1: usage depends upon VSI type. For VF types, indicates VF id
14408  *
14409  * This allocates the sw VSI structure and its queue resources, then add a VSI
14410  * to the identified VEB.
14411  *
14412  * Returns pointer to the successfully allocated and configure VSI sw struct on
14413  * success, otherwise returns NULL on failure.
14414  **/
14415 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
14416 				u16 uplink_seid, u32 param1)
14417 {
14418 	struct i40e_vsi *vsi = NULL;
14419 	struct i40e_veb *veb = NULL;
14420 	u16 alloc_queue_pairs;
14421 	int ret, i;
14422 	int v_idx;
14423 
14424 	/* The requested uplink_seid must be either
14425 	 *     - the PF's port seid
14426 	 *              no VEB is needed because this is the PF
14427 	 *              or this is a Flow Director special case VSI
14428 	 *     - seid of an existing VEB
14429 	 *     - seid of a VSI that owns an existing VEB
14430 	 *     - seid of a VSI that doesn't own a VEB
14431 	 *              a new VEB is created and the VSI becomes the owner
14432 	 *     - seid of the PF VSI, which is what creates the first VEB
14433 	 *              this is a special case of the previous
14434 	 *
14435 	 * Find which uplink_seid we were given and create a new VEB if needed
14436 	 */
14437 	for (i = 0; i < I40E_MAX_VEB; i++) {
14438 		if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
14439 			veb = pf->veb[i];
14440 			break;
14441 		}
14442 	}
14443 
14444 	if (!veb && uplink_seid != pf->mac_seid) {
14445 
14446 		for (i = 0; i < pf->num_alloc_vsi; i++) {
14447 			if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
14448 				vsi = pf->vsi[i];
14449 				break;
14450 			}
14451 		}
14452 		if (!vsi) {
14453 			dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
14454 				 uplink_seid);
14455 			return NULL;
14456 		}
14457 
14458 		if (vsi->uplink_seid == pf->mac_seid)
14459 			veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
14460 					     vsi->tc_config.enabled_tc);
14461 		else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
14462 			veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
14463 					     vsi->tc_config.enabled_tc);
14464 		if (veb) {
14465 			if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
14466 				dev_info(&vsi->back->pdev->dev,
14467 					 "New VSI creation error, uplink seid of LAN VSI expected.\n");
14468 				return NULL;
14469 			}
14470 			/* We come up by default in VEPA mode if SRIOV is not
14471 			 * already enabled, in which case we can't force VEPA
14472 			 * mode.
14473 			 */
14474 			if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
14475 				veb->bridge_mode = BRIDGE_MODE_VEPA;
14476 				pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
14477 			}
14478 			i40e_config_bridge_mode(veb);
14479 		}
14480 		for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
14481 			if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
14482 				veb = pf->veb[i];
14483 		}
14484 		if (!veb) {
14485 			dev_info(&pf->pdev->dev, "couldn't add VEB\n");
14486 			return NULL;
14487 		}
14488 
14489 		vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14490 		uplink_seid = veb->seid;
14491 	}
14492 
14493 	/* get vsi sw struct */
14494 	v_idx = i40e_vsi_mem_alloc(pf, type);
14495 	if (v_idx < 0)
14496 		goto err_alloc;
14497 	vsi = pf->vsi[v_idx];
14498 	if (!vsi)
14499 		goto err_alloc;
14500 	vsi->type = type;
14501 	vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
14502 
14503 	if (type == I40E_VSI_MAIN)
14504 		pf->lan_vsi = v_idx;
14505 	else if (type == I40E_VSI_SRIOV)
14506 		vsi->vf_id = param1;
14507 	/* assign it some queues */
14508 	alloc_queue_pairs = vsi->alloc_queue_pairs *
14509 			    (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
14510 
14511 	ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
14512 	if (ret < 0) {
14513 		dev_info(&pf->pdev->dev,
14514 			 "failed to get tracking for %d queues for VSI %d err=%d\n",
14515 			 alloc_queue_pairs, vsi->seid, ret);
14516 		goto err_vsi;
14517 	}
14518 	vsi->base_queue = ret;
14519 
14520 	/* get a VSI from the hardware */
14521 	vsi->uplink_seid = uplink_seid;
14522 	ret = i40e_add_vsi(vsi);
14523 	if (ret)
14524 		goto err_vsi;
14525 
14526 	switch (vsi->type) {
14527 	/* setup the netdev if needed */
14528 	case I40E_VSI_MAIN:
14529 	case I40E_VSI_VMDQ2:
14530 		ret = i40e_config_netdev(vsi);
14531 		if (ret)
14532 			goto err_netdev;
14533 		ret = i40e_netif_set_realnum_tx_rx_queues(vsi);
14534 		if (ret)
14535 			goto err_netdev;
14536 		ret = register_netdev(vsi->netdev);
14537 		if (ret)
14538 			goto err_netdev;
14539 		vsi->netdev_registered = true;
14540 		netif_carrier_off(vsi->netdev);
14541 #ifdef CONFIG_I40E_DCB
14542 		/* Setup DCB netlink interface */
14543 		i40e_dcbnl_setup(vsi);
14544 #endif /* CONFIG_I40E_DCB */
14545 		fallthrough;
14546 	case I40E_VSI_FDIR:
14547 		/* set up vectors and rings if needed */
14548 		ret = i40e_vsi_setup_vectors(vsi);
14549 		if (ret)
14550 			goto err_msix;
14551 
14552 		ret = i40e_alloc_rings(vsi);
14553 		if (ret)
14554 			goto err_rings;
14555 
14556 		/* map all of the rings to the q_vectors */
14557 		i40e_vsi_map_rings_to_vectors(vsi);
14558 
14559 		i40e_vsi_reset_stats(vsi);
14560 		break;
14561 	default:
14562 		/* no netdev or rings for the other VSI types */
14563 		break;
14564 	}
14565 
14566 	if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
14567 	    (vsi->type == I40E_VSI_VMDQ2)) {
14568 		ret = i40e_vsi_config_rss(vsi);
14569 	}
14570 	return vsi;
14571 
14572 err_rings:
14573 	i40e_vsi_free_q_vectors(vsi);
14574 err_msix:
14575 	if (vsi->netdev_registered) {
14576 		vsi->netdev_registered = false;
14577 		unregister_netdev(vsi->netdev);
14578 		free_netdev(vsi->netdev);
14579 		vsi->netdev = NULL;
14580 	}
14581 err_netdev:
14582 	i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
14583 err_vsi:
14584 	i40e_vsi_clear(vsi);
14585 err_alloc:
14586 	return NULL;
14587 }
14588 
14589 /**
14590  * i40e_veb_get_bw_info - Query VEB BW information
14591  * @veb: the veb to query
14592  *
14593  * Query the Tx scheduler BW configuration data for given VEB
14594  **/
14595 static int i40e_veb_get_bw_info(struct i40e_veb *veb)
14596 {
14597 	struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
14598 	struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
14599 	struct i40e_pf *pf = veb->pf;
14600 	struct i40e_hw *hw = &pf->hw;
14601 	u32 tc_bw_max;
14602 	int ret = 0;
14603 	int i;
14604 
14605 	ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
14606 						  &bw_data, NULL);
14607 	if (ret) {
14608 		dev_info(&pf->pdev->dev,
14609 			 "query veb bw config failed, err %pe aq_err %s\n",
14610 			 ERR_PTR(ret),
14611 			 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14612 		goto out;
14613 	}
14614 
14615 	ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
14616 						   &ets_data, NULL);
14617 	if (ret) {
14618 		dev_info(&pf->pdev->dev,
14619 			 "query veb bw ets config failed, err %pe aq_err %s\n",
14620 			 ERR_PTR(ret),
14621 			 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
14622 		goto out;
14623 	}
14624 
14625 	veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
14626 	veb->bw_max_quanta = ets_data.tc_bw_max;
14627 	veb->is_abs_credits = bw_data.absolute_credits_enable;
14628 	veb->enabled_tc = ets_data.tc_valid_bits;
14629 	tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
14630 		    (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
14631 	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
14632 		veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
14633 		veb->bw_tc_limit_credits[i] =
14634 					le16_to_cpu(bw_data.tc_bw_limits[i]);
14635 		veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
14636 	}
14637 
14638 out:
14639 	return ret;
14640 }
14641 
14642 /**
14643  * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF
14644  * @pf: board private structure
14645  *
14646  * On error: returns error code (negative)
14647  * On success: returns vsi index in PF (positive)
14648  **/
14649 static int i40e_veb_mem_alloc(struct i40e_pf *pf)
14650 {
14651 	int ret = -ENOENT;
14652 	struct i40e_veb *veb;
14653 	int i;
14654 
14655 	/* Need to protect the allocation of switch elements at the PF level */
14656 	mutex_lock(&pf->switch_mutex);
14657 
14658 	/* VEB list may be fragmented if VEB creation/destruction has
14659 	 * been happening.  We can afford to do a quick scan to look
14660 	 * for any free slots in the list.
14661 	 *
14662 	 * find next empty veb slot, looping back around if necessary
14663 	 */
14664 	i = 0;
14665 	while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
14666 		i++;
14667 	if (i >= I40E_MAX_VEB) {
14668 		ret = -ENOMEM;
14669 		goto err_alloc_veb;  /* out of VEB slots! */
14670 	}
14671 
14672 	veb = kzalloc(sizeof(*veb), GFP_KERNEL);
14673 	if (!veb) {
14674 		ret = -ENOMEM;
14675 		goto err_alloc_veb;
14676 	}
14677 	veb->pf = pf;
14678 	veb->idx = i;
14679 	veb->enabled_tc = 1;
14680 
14681 	pf->veb[i] = veb;
14682 	ret = i;
14683 err_alloc_veb:
14684 	mutex_unlock(&pf->switch_mutex);
14685 	return ret;
14686 }
14687 
14688 /**
14689  * i40e_switch_branch_release - Delete a branch of the switch tree
14690  * @branch: where to start deleting
14691  *
14692  * This uses recursion to find the tips of the branch to be
14693  * removed, deleting until we get back to and can delete this VEB.
14694  **/
14695 static void i40e_switch_branch_release(struct i40e_veb *branch)
14696 {
14697 	struct i40e_pf *pf = branch->pf;
14698 	u16 branch_seid = branch->seid;
14699 	u16 veb_idx = branch->idx;
14700 	int i;
14701 
14702 	/* release any VEBs on this VEB - RECURSION */
14703 	for (i = 0; i < I40E_MAX_VEB; i++) {
14704 		if (!pf->veb[i])
14705 			continue;
14706 		if (pf->veb[i]->uplink_seid == branch->seid)
14707 			i40e_switch_branch_release(pf->veb[i]);
14708 	}
14709 
14710 	/* Release the VSIs on this VEB, but not the owner VSI.
14711 	 *
14712 	 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing
14713 	 *       the VEB itself, so don't use (*branch) after this loop.
14714 	 */
14715 	for (i = 0; i < pf->num_alloc_vsi; i++) {
14716 		if (!pf->vsi[i])
14717 			continue;
14718 		if (pf->vsi[i]->uplink_seid == branch_seid &&
14719 		   (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
14720 			i40e_vsi_release(pf->vsi[i]);
14721 		}
14722 	}
14723 
14724 	/* There's one corner case where the VEB might not have been
14725 	 * removed, so double check it here and remove it if needed.
14726 	 * This case happens if the veb was created from the debugfs
14727 	 * commands and no VSIs were added to it.
14728 	 */
14729 	if (pf->veb[veb_idx])
14730 		i40e_veb_release(pf->veb[veb_idx]);
14731 }
14732 
14733 /**
14734  * i40e_veb_clear - remove veb struct
14735  * @veb: the veb to remove
14736  **/
14737 static void i40e_veb_clear(struct i40e_veb *veb)
14738 {
14739 	if (!veb)
14740 		return;
14741 
14742 	if (veb->pf) {
14743 		struct i40e_pf *pf = veb->pf;
14744 
14745 		mutex_lock(&pf->switch_mutex);
14746 		if (pf->veb[veb->idx] == veb)
14747 			pf->veb[veb->idx] = NULL;
14748 		mutex_unlock(&pf->switch_mutex);
14749 	}
14750 
14751 	kfree(veb);
14752 }
14753 
14754 /**
14755  * i40e_veb_release - Delete a VEB and free its resources
14756  * @veb: the VEB being removed
14757  **/
14758 void i40e_veb_release(struct i40e_veb *veb)
14759 {
14760 	struct i40e_vsi *vsi = NULL;
14761 	struct i40e_pf *pf;
14762 	int i, n = 0;
14763 
14764 	pf = veb->pf;
14765 
14766 	/* find the remaining VSI and check for extras */
14767 	for (i = 0; i < pf->num_alloc_vsi; i++) {
14768 		if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
14769 			n++;
14770 			vsi = pf->vsi[i];
14771 		}
14772 	}
14773 	if (n != 1) {
14774 		dev_info(&pf->pdev->dev,
14775 			 "can't remove VEB %d with %d VSIs left\n",
14776 			 veb->seid, n);
14777 		return;
14778 	}
14779 
14780 	/* move the remaining VSI to uplink veb */
14781 	vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
14782 	if (veb->uplink_seid) {
14783 		vsi->uplink_seid = veb->uplink_seid;
14784 		if (veb->uplink_seid == pf->mac_seid)
14785 			vsi->veb_idx = I40E_NO_VEB;
14786 		else
14787 			vsi->veb_idx = veb->veb_idx;
14788 	} else {
14789 		/* floating VEB */
14790 		vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
14791 		vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
14792 	}
14793 
14794 	i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14795 	i40e_veb_clear(veb);
14796 }
14797 
14798 /**
14799  * i40e_add_veb - create the VEB in the switch
14800  * @veb: the VEB to be instantiated
14801  * @vsi: the controlling VSI
14802  **/
14803 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
14804 {
14805 	struct i40e_pf *pf = veb->pf;
14806 	bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
14807 	int ret;
14808 
14809 	ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
14810 			      veb->enabled_tc, false,
14811 			      &veb->seid, enable_stats, NULL);
14812 
14813 	/* get a VEB from the hardware */
14814 	if (ret) {
14815 		dev_info(&pf->pdev->dev,
14816 			 "couldn't add VEB, err %pe aq_err %s\n",
14817 			 ERR_PTR(ret),
14818 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14819 		return -EPERM;
14820 	}
14821 
14822 	/* get statistics counter */
14823 	ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
14824 					 &veb->stats_idx, NULL, NULL, NULL);
14825 	if (ret) {
14826 		dev_info(&pf->pdev->dev,
14827 			 "couldn't get VEB statistics idx, err %pe aq_err %s\n",
14828 			 ERR_PTR(ret),
14829 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14830 		return -EPERM;
14831 	}
14832 	ret = i40e_veb_get_bw_info(veb);
14833 	if (ret) {
14834 		dev_info(&pf->pdev->dev,
14835 			 "couldn't get VEB bw info, err %pe aq_err %s\n",
14836 			 ERR_PTR(ret),
14837 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14838 		i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
14839 		return -ENOENT;
14840 	}
14841 
14842 	vsi->uplink_seid = veb->seid;
14843 	vsi->veb_idx = veb->idx;
14844 	vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14845 
14846 	return 0;
14847 }
14848 
14849 /**
14850  * i40e_veb_setup - Set up a VEB
14851  * @pf: board private structure
14852  * @flags: VEB setup flags
14853  * @uplink_seid: the switch element to link to
14854  * @vsi_seid: the initial VSI seid
14855  * @enabled_tc: Enabled TC bit-map
14856  *
14857  * This allocates the sw VEB structure and links it into the switch
14858  * It is possible and legal for this to be a duplicate of an already
14859  * existing VEB.  It is also possible for both uplink and vsi seids
14860  * to be zero, in order to create a floating VEB.
14861  *
14862  * Returns pointer to the successfully allocated VEB sw struct on
14863  * success, otherwise returns NULL on failure.
14864  **/
14865 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14866 				u16 uplink_seid, u16 vsi_seid,
14867 				u8 enabled_tc)
14868 {
14869 	struct i40e_veb *veb, *uplink_veb = NULL;
14870 	int vsi_idx, veb_idx;
14871 	int ret;
14872 
14873 	/* if one seid is 0, the other must be 0 to create a floating relay */
14874 	if ((uplink_seid == 0 || vsi_seid == 0) &&
14875 	    (uplink_seid + vsi_seid != 0)) {
14876 		dev_info(&pf->pdev->dev,
14877 			 "one, not both seid's are 0: uplink=%d vsi=%d\n",
14878 			 uplink_seid, vsi_seid);
14879 		return NULL;
14880 	}
14881 
14882 	/* make sure there is such a vsi and uplink */
14883 	for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
14884 		if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
14885 			break;
14886 	if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) {
14887 		dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
14888 			 vsi_seid);
14889 		return NULL;
14890 	}
14891 
14892 	if (uplink_seid && uplink_seid != pf->mac_seid) {
14893 		for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
14894 			if (pf->veb[veb_idx] &&
14895 			    pf->veb[veb_idx]->seid == uplink_seid) {
14896 				uplink_veb = pf->veb[veb_idx];
14897 				break;
14898 			}
14899 		}
14900 		if (!uplink_veb) {
14901 			dev_info(&pf->pdev->dev,
14902 				 "uplink seid %d not found\n", uplink_seid);
14903 			return NULL;
14904 		}
14905 	}
14906 
14907 	/* get veb sw struct */
14908 	veb_idx = i40e_veb_mem_alloc(pf);
14909 	if (veb_idx < 0)
14910 		goto err_alloc;
14911 	veb = pf->veb[veb_idx];
14912 	veb->flags = flags;
14913 	veb->uplink_seid = uplink_seid;
14914 	veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
14915 	veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14916 
14917 	/* create the VEB in the switch */
14918 	ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
14919 	if (ret)
14920 		goto err_veb;
14921 	if (vsi_idx == pf->lan_vsi)
14922 		pf->lan_veb = veb->idx;
14923 
14924 	return veb;
14925 
14926 err_veb:
14927 	i40e_veb_clear(veb);
14928 err_alloc:
14929 	return NULL;
14930 }
14931 
14932 /**
14933  * i40e_setup_pf_switch_element - set PF vars based on switch type
14934  * @pf: board private structure
14935  * @ele: element we are building info from
14936  * @num_reported: total number of elements
14937  * @printconfig: should we print the contents
14938  *
14939  * helper function to assist in extracting a few useful SEID values.
14940  **/
14941 static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14942 				struct i40e_aqc_switch_config_element_resp *ele,
14943 				u16 num_reported, bool printconfig)
14944 {
14945 	u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14946 	u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14947 	u8 element_type = ele->element_type;
14948 	u16 seid = le16_to_cpu(ele->seid);
14949 
14950 	if (printconfig)
14951 		dev_info(&pf->pdev->dev,
14952 			 "type=%d seid=%d uplink=%d downlink=%d\n",
14953 			 element_type, seid, uplink_seid, downlink_seid);
14954 
14955 	switch (element_type) {
14956 	case I40E_SWITCH_ELEMENT_TYPE_MAC:
14957 		pf->mac_seid = seid;
14958 		break;
14959 	case I40E_SWITCH_ELEMENT_TYPE_VEB:
14960 		/* Main VEB? */
14961 		if (uplink_seid != pf->mac_seid)
14962 			break;
14963 		if (pf->lan_veb >= I40E_MAX_VEB) {
14964 			int v;
14965 
14966 			/* find existing or else empty VEB */
14967 			for (v = 0; v < I40E_MAX_VEB; v++) {
14968 				if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
14969 					pf->lan_veb = v;
14970 					break;
14971 				}
14972 			}
14973 			if (pf->lan_veb >= I40E_MAX_VEB) {
14974 				v = i40e_veb_mem_alloc(pf);
14975 				if (v < 0)
14976 					break;
14977 				pf->lan_veb = v;
14978 			}
14979 		}
14980 		if (pf->lan_veb >= I40E_MAX_VEB)
14981 			break;
14982 
14983 		pf->veb[pf->lan_veb]->seid = seid;
14984 		pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14985 		pf->veb[pf->lan_veb]->pf = pf;
14986 		pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
14987 		break;
14988 	case I40E_SWITCH_ELEMENT_TYPE_VSI:
14989 		if (num_reported != 1)
14990 			break;
14991 		/* This is immediately after a reset so we can assume this is
14992 		 * the PF's VSI
14993 		 */
14994 		pf->mac_seid = uplink_seid;
14995 		pf->pf_seid = downlink_seid;
14996 		pf->main_vsi_seid = seid;
14997 		if (printconfig)
14998 			dev_info(&pf->pdev->dev,
14999 				 "pf_seid=%d main_vsi_seid=%d\n",
15000 				 pf->pf_seid, pf->main_vsi_seid);
15001 		break;
15002 	case I40E_SWITCH_ELEMENT_TYPE_PF:
15003 	case I40E_SWITCH_ELEMENT_TYPE_VF:
15004 	case I40E_SWITCH_ELEMENT_TYPE_EMP:
15005 	case I40E_SWITCH_ELEMENT_TYPE_BMC:
15006 	case I40E_SWITCH_ELEMENT_TYPE_PE:
15007 	case I40E_SWITCH_ELEMENT_TYPE_PA:
15008 		/* ignore these for now */
15009 		break;
15010 	default:
15011 		dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
15012 			 element_type, seid);
15013 		break;
15014 	}
15015 }
15016 
15017 /**
15018  * i40e_fetch_switch_configuration - Get switch config from firmware
15019  * @pf: board private structure
15020  * @printconfig: should we print the contents
15021  *
15022  * Get the current switch configuration from the device and
15023  * extract a few useful SEID values.
15024  **/
15025 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
15026 {
15027 	struct i40e_aqc_get_switch_config_resp *sw_config;
15028 	u16 next_seid = 0;
15029 	int ret = 0;
15030 	u8 *aq_buf;
15031 	int i;
15032 
15033 	aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
15034 	if (!aq_buf)
15035 		return -ENOMEM;
15036 
15037 	sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
15038 	do {
15039 		u16 num_reported, num_total;
15040 
15041 		ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
15042 						I40E_AQ_LARGE_BUF,
15043 						&next_seid, NULL);
15044 		if (ret) {
15045 			dev_info(&pf->pdev->dev,
15046 				 "get switch config failed err %d aq_err %s\n",
15047 				 ret,
15048 				 i40e_aq_str(&pf->hw,
15049 					     pf->hw.aq.asq_last_status));
15050 			kfree(aq_buf);
15051 			return -ENOENT;
15052 		}
15053 
15054 		num_reported = le16_to_cpu(sw_config->header.num_reported);
15055 		num_total = le16_to_cpu(sw_config->header.num_total);
15056 
15057 		if (printconfig)
15058 			dev_info(&pf->pdev->dev,
15059 				 "header: %d reported %d total\n",
15060 				 num_reported, num_total);
15061 
15062 		for (i = 0; i < num_reported; i++) {
15063 			struct i40e_aqc_switch_config_element_resp *ele =
15064 				&sw_config->element[i];
15065 
15066 			i40e_setup_pf_switch_element(pf, ele, num_reported,
15067 						     printconfig);
15068 		}
15069 	} while (next_seid != 0);
15070 
15071 	kfree(aq_buf);
15072 	return ret;
15073 }
15074 
15075 /**
15076  * i40e_setup_pf_switch - Setup the HW switch on startup or after reset
15077  * @pf: board private structure
15078  * @reinit: if the Main VSI needs to re-initialized.
15079  * @lock_acquired: indicates whether or not the lock has been acquired
15080  *
15081  * Returns 0 on success, negative value on failure
15082  **/
15083 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired)
15084 {
15085 	u16 flags = 0;
15086 	int ret;
15087 
15088 	/* find out what's out there already */
15089 	ret = i40e_fetch_switch_configuration(pf, false);
15090 	if (ret) {
15091 		dev_info(&pf->pdev->dev,
15092 			 "couldn't fetch switch config, err %pe aq_err %s\n",
15093 			 ERR_PTR(ret),
15094 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15095 		return ret;
15096 	}
15097 	i40e_pf_reset_stats(pf);
15098 
15099 	/* set the switch config bit for the whole device to
15100 	 * support limited promisc or true promisc
15101 	 * when user requests promisc. The default is limited
15102 	 * promisc.
15103 	*/
15104 
15105 	if ((pf->hw.pf_id == 0) &&
15106 	    !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
15107 		flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15108 		pf->last_sw_conf_flags = flags;
15109 	}
15110 
15111 	if (pf->hw.pf_id == 0) {
15112 		u16 valid_flags;
15113 
15114 		valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
15115 		ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
15116 						NULL);
15117 		if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
15118 			dev_info(&pf->pdev->dev,
15119 				 "couldn't set switch config bits, err %pe aq_err %s\n",
15120 				 ERR_PTR(ret),
15121 				 i40e_aq_str(&pf->hw,
15122 					     pf->hw.aq.asq_last_status));
15123 			/* not a fatal problem, just keep going */
15124 		}
15125 		pf->last_sw_conf_valid_flags = valid_flags;
15126 	}
15127 
15128 	/* first time setup */
15129 	if (pf->lan_vsi == I40E_NO_VSI || reinit) {
15130 		struct i40e_vsi *vsi = NULL;
15131 		u16 uplink_seid;
15132 
15133 		/* Set up the PF VSI associated with the PF's main VSI
15134 		 * that is already in the HW switch
15135 		 */
15136 		if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
15137 			uplink_seid = pf->veb[pf->lan_veb]->seid;
15138 		else
15139 			uplink_seid = pf->mac_seid;
15140 		if (pf->lan_vsi == I40E_NO_VSI)
15141 			vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
15142 		else if (reinit)
15143 			vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
15144 		if (!vsi) {
15145 			dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
15146 			i40e_cloud_filter_exit(pf);
15147 			i40e_fdir_teardown(pf);
15148 			return -EAGAIN;
15149 		}
15150 	} else {
15151 		/* force a reset of TC and queue layout configurations */
15152 		u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
15153 
15154 		pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
15155 		pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
15156 		i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
15157 	}
15158 	i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
15159 
15160 	i40e_fdir_sb_setup(pf);
15161 
15162 	/* Setup static PF queue filter control settings */
15163 	ret = i40e_setup_pf_filter_control(pf);
15164 	if (ret) {
15165 		dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
15166 			 ret);
15167 		/* Failure here should not stop continuing other steps */
15168 	}
15169 
15170 	/* enable RSS in the HW, even for only one queue, as the stack can use
15171 	 * the hash
15172 	 */
15173 	if ((pf->flags & I40E_FLAG_RSS_ENABLED))
15174 		i40e_pf_config_rss(pf);
15175 
15176 	/* fill in link information and enable LSE reporting */
15177 	i40e_link_event(pf);
15178 
15179 	/* Initialize user-specific link properties */
15180 	pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
15181 				  I40E_AQ_AN_COMPLETED) ? true : false);
15182 
15183 	i40e_ptp_init(pf);
15184 
15185 	if (!lock_acquired)
15186 		rtnl_lock();
15187 
15188 	/* repopulate tunnel port filters */
15189 	udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev);
15190 
15191 	if (!lock_acquired)
15192 		rtnl_unlock();
15193 
15194 	return ret;
15195 }
15196 
15197 /**
15198  * i40e_determine_queue_usage - Work out queue distribution
15199  * @pf: board private structure
15200  **/
15201 static void i40e_determine_queue_usage(struct i40e_pf *pf)
15202 {
15203 	int queues_left;
15204 	int q_max;
15205 
15206 	pf->num_lan_qps = 0;
15207 
15208 	/* Find the max queues to be put into basic use.  We'll always be
15209 	 * using TC0, whether or not DCB is running, and TC0 will get the
15210 	 * big RSS set.
15211 	 */
15212 	queues_left = pf->hw.func_caps.num_tx_qp;
15213 
15214 	if ((queues_left == 1) ||
15215 	    !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
15216 		/* one qp for PF, no queues for anything else */
15217 		queues_left = 0;
15218 		pf->alloc_rss_size = pf->num_lan_qps = 1;
15219 
15220 		/* make sure all the fancies are disabled */
15221 		pf->flags &= ~(I40E_FLAG_RSS_ENABLED	|
15222 			       I40E_FLAG_IWARP_ENABLED	|
15223 			       I40E_FLAG_FD_SB_ENABLED	|
15224 			       I40E_FLAG_FD_ATR_ENABLED	|
15225 			       I40E_FLAG_DCB_CAPABLE	|
15226 			       I40E_FLAG_DCB_ENABLED	|
15227 			       I40E_FLAG_SRIOV_ENABLED	|
15228 			       I40E_FLAG_VMDQ_ENABLED);
15229 		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
15230 	} else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
15231 				  I40E_FLAG_FD_SB_ENABLED |
15232 				  I40E_FLAG_FD_ATR_ENABLED |
15233 				  I40E_FLAG_DCB_CAPABLE))) {
15234 		/* one qp for PF */
15235 		pf->alloc_rss_size = pf->num_lan_qps = 1;
15236 		queues_left -= pf->num_lan_qps;
15237 
15238 		pf->flags &= ~(I40E_FLAG_RSS_ENABLED	|
15239 			       I40E_FLAG_IWARP_ENABLED	|
15240 			       I40E_FLAG_FD_SB_ENABLED	|
15241 			       I40E_FLAG_FD_ATR_ENABLED	|
15242 			       I40E_FLAG_DCB_ENABLED	|
15243 			       I40E_FLAG_VMDQ_ENABLED);
15244 		pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
15245 	} else {
15246 		/* Not enough queues for all TCs */
15247 		if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
15248 		    (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
15249 			pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
15250 					I40E_FLAG_DCB_ENABLED);
15251 			dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
15252 		}
15253 
15254 		/* limit lan qps to the smaller of qps, cpus or msix */
15255 		q_max = max_t(int, pf->rss_size_max, num_online_cpus());
15256 		q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
15257 		q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
15258 		pf->num_lan_qps = q_max;
15259 
15260 		queues_left -= pf->num_lan_qps;
15261 	}
15262 
15263 	if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
15264 		if (queues_left > 1) {
15265 			queues_left -= 1; /* save 1 queue for FD */
15266 		} else {
15267 			pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
15268 			pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
15269 			dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
15270 		}
15271 	}
15272 
15273 	if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15274 	    pf->num_vf_qps && pf->num_req_vfs && queues_left) {
15275 		pf->num_req_vfs = min_t(int, pf->num_req_vfs,
15276 					(queues_left / pf->num_vf_qps));
15277 		queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
15278 	}
15279 
15280 	if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
15281 	    pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
15282 		pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
15283 					  (queues_left / pf->num_vmdq_qps));
15284 		queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
15285 	}
15286 
15287 	pf->queues_left = queues_left;
15288 	dev_dbg(&pf->pdev->dev,
15289 		"qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
15290 		pf->hw.func_caps.num_tx_qp,
15291 		!!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
15292 		pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
15293 		pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
15294 		queues_left);
15295 }
15296 
15297 /**
15298  * i40e_setup_pf_filter_control - Setup PF static filter control
15299  * @pf: PF to be setup
15300  *
15301  * i40e_setup_pf_filter_control sets up a PF's initial filter control
15302  * settings. If PE/FCoE are enabled then it will also set the per PF
15303  * based filter sizes required for them. It also enables Flow director,
15304  * ethertype and macvlan type filter settings for the pf.
15305  *
15306  * Returns 0 on success, negative on failure
15307  **/
15308 static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
15309 {
15310 	struct i40e_filter_control_settings *settings = &pf->filter_settings;
15311 
15312 	settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
15313 
15314 	/* Flow Director is enabled */
15315 	if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
15316 		settings->enable_fdir = true;
15317 
15318 	/* Ethtype and MACVLAN filters enabled for PF */
15319 	settings->enable_ethtype = true;
15320 	settings->enable_macvlan = true;
15321 
15322 	if (i40e_set_filter_control(&pf->hw, settings))
15323 		return -ENOENT;
15324 
15325 	return 0;
15326 }
15327 
15328 #define INFO_STRING_LEN 255
15329 #define REMAIN(__x) (INFO_STRING_LEN - (__x))
15330 static void i40e_print_features(struct i40e_pf *pf)
15331 {
15332 	struct i40e_hw *hw = &pf->hw;
15333 	char *buf;
15334 	int i;
15335 
15336 	buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
15337 	if (!buf)
15338 		return;
15339 
15340 	i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
15341 #ifdef CONFIG_PCI_IOV
15342 	i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
15343 #endif
15344 	i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
15345 		      pf->hw.func_caps.num_vsis,
15346 		      pf->vsi[pf->lan_vsi]->num_queue_pairs);
15347 	if (pf->flags & I40E_FLAG_RSS_ENABLED)
15348 		i += scnprintf(&buf[i], REMAIN(i), " RSS");
15349 	if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
15350 		i += scnprintf(&buf[i], REMAIN(i), " FD_ATR");
15351 	if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
15352 		i += scnprintf(&buf[i], REMAIN(i), " FD_SB");
15353 		i += scnprintf(&buf[i], REMAIN(i), " NTUPLE");
15354 	}
15355 	if (pf->flags & I40E_FLAG_DCB_CAPABLE)
15356 		i += scnprintf(&buf[i], REMAIN(i), " DCB");
15357 	i += scnprintf(&buf[i], REMAIN(i), " VxLAN");
15358 	i += scnprintf(&buf[i], REMAIN(i), " Geneve");
15359 	if (pf->flags & I40E_FLAG_PTP)
15360 		i += scnprintf(&buf[i], REMAIN(i), " PTP");
15361 	if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
15362 		i += scnprintf(&buf[i], REMAIN(i), " VEB");
15363 	else
15364 		i += scnprintf(&buf[i], REMAIN(i), " VEPA");
15365 
15366 	dev_info(&pf->pdev->dev, "%s\n", buf);
15367 	kfree(buf);
15368 	WARN_ON(i > INFO_STRING_LEN);
15369 }
15370 
15371 /**
15372  * i40e_get_platform_mac_addr - get platform-specific MAC address
15373  * @pdev: PCI device information struct
15374  * @pf: board private structure
15375  *
15376  * Look up the MAC address for the device. First we'll try
15377  * eth_platform_get_mac_address, which will check Open Firmware, or arch
15378  * specific fallback. Otherwise, we'll default to the stored value in
15379  * firmware.
15380  **/
15381 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
15382 {
15383 	if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
15384 		i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
15385 }
15386 
15387 /**
15388  * i40e_set_fec_in_flags - helper function for setting FEC options in flags
15389  * @fec_cfg: FEC option to set in flags
15390  * @flags: ptr to flags in which we set FEC option
15391  **/
15392 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
15393 {
15394 	if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
15395 		*flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
15396 	if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
15397 	    (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
15398 		*flags |= I40E_FLAG_RS_FEC;
15399 		*flags &= ~I40E_FLAG_BASE_R_FEC;
15400 	}
15401 	if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
15402 	    (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
15403 		*flags |= I40E_FLAG_BASE_R_FEC;
15404 		*flags &= ~I40E_FLAG_RS_FEC;
15405 	}
15406 	if (fec_cfg == 0)
15407 		*flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
15408 }
15409 
15410 /**
15411  * i40e_check_recovery_mode - check if we are running transition firmware
15412  * @pf: board private structure
15413  *
15414  * Check registers indicating the firmware runs in recovery mode. Sets the
15415  * appropriate driver state.
15416  *
15417  * Returns true if the recovery mode was detected, false otherwise
15418  **/
15419 static bool i40e_check_recovery_mode(struct i40e_pf *pf)
15420 {
15421 	u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
15422 
15423 	if (val & I40E_GL_FWSTS_FWS1B_MASK) {
15424 		dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
15425 		dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
15426 		set_bit(__I40E_RECOVERY_MODE, pf->state);
15427 
15428 		return true;
15429 	}
15430 	if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15431 		dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n");
15432 
15433 	return false;
15434 }
15435 
15436 /**
15437  * i40e_pf_loop_reset - perform reset in a loop.
15438  * @pf: board private structure
15439  *
15440  * This function is useful when a NIC is about to enter recovery mode.
15441  * When a NIC's internal data structures are corrupted the NIC's
15442  * firmware is going to enter recovery mode.
15443  * Right after a POR it takes about 7 minutes for firmware to enter
15444  * recovery mode. Until that time a NIC is in some kind of intermediate
15445  * state. After that time period the NIC almost surely enters
15446  * recovery mode. The only way for a driver to detect intermediate
15447  * state is to issue a series of pf-resets and check a return value.
15448  * If a PF reset returns success then the firmware could be in recovery
15449  * mode so the caller of this code needs to check for recovery mode
15450  * if this function returns success. There is a little chance that
15451  * firmware will hang in intermediate state forever.
15452  * Since waiting 7 minutes is quite a lot of time this function waits
15453  * 10 seconds and then gives up by returning an error.
15454  *
15455  * Return 0 on success, negative on failure.
15456  **/
15457 static int i40e_pf_loop_reset(struct i40e_pf *pf)
15458 {
15459 	/* wait max 10 seconds for PF reset to succeed */
15460 	const unsigned long time_end = jiffies + 10 * HZ;
15461 	struct i40e_hw *hw = &pf->hw;
15462 	int ret;
15463 
15464 	ret = i40e_pf_reset(hw);
15465 	while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) {
15466 		usleep_range(10000, 20000);
15467 		ret = i40e_pf_reset(hw);
15468 	}
15469 
15470 	if (ret == I40E_SUCCESS)
15471 		pf->pfr_count++;
15472 	else
15473 		dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret);
15474 
15475 	return ret;
15476 }
15477 
15478 /**
15479  * i40e_check_fw_empr - check if FW issued unexpected EMP Reset
15480  * @pf: board private structure
15481  *
15482  * Check FW registers to determine if FW issued unexpected EMP Reset.
15483  * Every time when unexpected EMP Reset occurs the FW increments
15484  * a counter of unexpected EMP Resets. When the counter reaches 10
15485  * the FW should enter the Recovery mode
15486  *
15487  * Returns true if FW issued unexpected EMP Reset
15488  **/
15489 static bool i40e_check_fw_empr(struct i40e_pf *pf)
15490 {
15491 	const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) &
15492 			   I40E_GL_FWSTS_FWS1B_MASK;
15493 	return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) &&
15494 	       (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10);
15495 }
15496 
15497 /**
15498  * i40e_handle_resets - handle EMP resets and PF resets
15499  * @pf: board private structure
15500  *
15501  * Handle both EMP resets and PF resets and conclude whether there are
15502  * any issues regarding these resets. If there are any issues then
15503  * generate log entry.
15504  *
15505  * Return 0 if NIC is healthy or negative value when there are issues
15506  * with resets
15507  **/
15508 static int i40e_handle_resets(struct i40e_pf *pf)
15509 {
15510 	const int pfr = i40e_pf_loop_reset(pf);
15511 	const bool is_empr = i40e_check_fw_empr(pf);
15512 
15513 	if (is_empr || pfr != I40E_SUCCESS)
15514 		dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n");
15515 
15516 	return is_empr ? I40E_ERR_RESET_FAILED : pfr;
15517 }
15518 
15519 /**
15520  * i40e_init_recovery_mode - initialize subsystems needed in recovery mode
15521  * @pf: board private structure
15522  * @hw: ptr to the hardware info
15523  *
15524  * This function does a minimal setup of all subsystems needed for running
15525  * recovery mode.
15526  *
15527  * Returns 0 on success, negative on failure
15528  **/
15529 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
15530 {
15531 	struct i40e_vsi *vsi;
15532 	int err;
15533 	int v_idx;
15534 
15535 	pci_set_drvdata(pf->pdev, pf);
15536 	pci_save_state(pf->pdev);
15537 
15538 	/* set up periodic task facility */
15539 	timer_setup(&pf->service_timer, i40e_service_timer, 0);
15540 	pf->service_timer_period = HZ;
15541 
15542 	INIT_WORK(&pf->service_task, i40e_service_task);
15543 	clear_bit(__I40E_SERVICE_SCHED, pf->state);
15544 
15545 	err = i40e_init_interrupt_scheme(pf);
15546 	if (err)
15547 		goto err_switch_setup;
15548 
15549 	/* The number of VSIs reported by the FW is the minimum guaranteed
15550 	 * to us; HW supports far more and we share the remaining pool with
15551 	 * the other PFs. We allocate space for more than the guarantee with
15552 	 * the understanding that we might not get them all later.
15553 	 */
15554 	if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15555 		pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15556 	else
15557 		pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15558 
15559 	/* Set up the vsi struct and our local tracking of the MAIN PF vsi. */
15560 	pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15561 			  GFP_KERNEL);
15562 	if (!pf->vsi) {
15563 		err = -ENOMEM;
15564 		goto err_switch_setup;
15565 	}
15566 
15567 	/* We allocate one VSI which is needed as absolute minimum
15568 	 * in order to register the netdev
15569 	 */
15570 	v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
15571 	if (v_idx < 0) {
15572 		err = v_idx;
15573 		goto err_switch_setup;
15574 	}
15575 	pf->lan_vsi = v_idx;
15576 	vsi = pf->vsi[v_idx];
15577 	if (!vsi) {
15578 		err = -EFAULT;
15579 		goto err_switch_setup;
15580 	}
15581 	vsi->alloc_queue_pairs = 1;
15582 	err = i40e_config_netdev(vsi);
15583 	if (err)
15584 		goto err_switch_setup;
15585 	err = register_netdev(vsi->netdev);
15586 	if (err)
15587 		goto err_switch_setup;
15588 	vsi->netdev_registered = true;
15589 	i40e_dbg_pf_init(pf);
15590 
15591 	err = i40e_setup_misc_vector_for_recovery_mode(pf);
15592 	if (err)
15593 		goto err_switch_setup;
15594 
15595 	/* tell the firmware that we're starting */
15596 	i40e_send_version(pf);
15597 
15598 	/* since everything's happy, start the service_task timer */
15599 	mod_timer(&pf->service_timer,
15600 		  round_jiffies(jiffies + pf->service_timer_period));
15601 
15602 	return 0;
15603 
15604 err_switch_setup:
15605 	i40e_reset_interrupt_capability(pf);
15606 	timer_shutdown_sync(&pf->service_timer);
15607 	i40e_shutdown_adminq(hw);
15608 	iounmap(hw->hw_addr);
15609 	pci_release_mem_regions(pf->pdev);
15610 	pci_disable_device(pf->pdev);
15611 	kfree(pf);
15612 
15613 	return err;
15614 }
15615 
15616 /**
15617  * i40e_set_subsystem_device_id - set subsystem device id
15618  * @hw: pointer to the hardware info
15619  *
15620  * Set PCI subsystem device id either from a pci_dev structure or
15621  * a specific FW register.
15622  **/
15623 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw)
15624 {
15625 	struct pci_dev *pdev = ((struct i40e_pf *)hw->back)->pdev;
15626 
15627 	hw->subsystem_device_id = pdev->subsystem_device ?
15628 		pdev->subsystem_device :
15629 		(ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX);
15630 }
15631 
15632 /**
15633  * i40e_probe - Device initialization routine
15634  * @pdev: PCI device information struct
15635  * @ent: entry in i40e_pci_tbl
15636  *
15637  * i40e_probe initializes a PF identified by a pci_dev structure.
15638  * The OS initialization, configuring of the PF private structure,
15639  * and a hardware reset occur.
15640  *
15641  * Returns 0 on success, negative on failure
15642  **/
15643 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
15644 {
15645 	struct i40e_aq_get_phy_abilities_resp abilities;
15646 #ifdef CONFIG_I40E_DCB
15647 	enum i40e_get_fw_lldp_status_resp lldp_status;
15648 #endif /* CONFIG_I40E_DCB */
15649 	struct i40e_pf *pf;
15650 	struct i40e_hw *hw;
15651 	static u16 pfs_found;
15652 	u16 wol_nvm_bits;
15653 	u16 link_status;
15654 #ifdef CONFIG_I40E_DCB
15655 	int status;
15656 #endif /* CONFIG_I40E_DCB */
15657 	int err;
15658 	u32 val;
15659 	u32 i;
15660 
15661 	err = pci_enable_device_mem(pdev);
15662 	if (err)
15663 		return err;
15664 
15665 	/* set up for high or low dma */
15666 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
15667 	if (err) {
15668 		dev_err(&pdev->dev,
15669 			"DMA configuration failed: 0x%x\n", err);
15670 		goto err_dma;
15671 	}
15672 
15673 	/* set up pci connections */
15674 	err = pci_request_mem_regions(pdev, i40e_driver_name);
15675 	if (err) {
15676 		dev_info(&pdev->dev,
15677 			 "pci_request_selected_regions failed %d\n", err);
15678 		goto err_pci_reg;
15679 	}
15680 
15681 	pci_set_master(pdev);
15682 
15683 	/* Now that we have a PCI connection, we need to do the
15684 	 * low level device setup.  This is primarily setting up
15685 	 * the Admin Queue structures and then querying for the
15686 	 * device's current profile information.
15687 	 */
15688 	pf = kzalloc(sizeof(*pf), GFP_KERNEL);
15689 	if (!pf) {
15690 		err = -ENOMEM;
15691 		goto err_pf_alloc;
15692 	}
15693 	pf->next_vsi = 0;
15694 	pf->pdev = pdev;
15695 	set_bit(__I40E_DOWN, pf->state);
15696 
15697 	hw = &pf->hw;
15698 	hw->back = pf;
15699 
15700 	pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
15701 				I40E_MAX_CSR_SPACE);
15702 	/* We believe that the highest register to read is
15703 	 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size
15704 	 * is not less than that before mapping to prevent a
15705 	 * kernel panic.
15706 	 */
15707 	if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
15708 		dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
15709 			pf->ioremap_len);
15710 		err = -ENOMEM;
15711 		goto err_ioremap;
15712 	}
15713 	hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
15714 	if (!hw->hw_addr) {
15715 		err = -EIO;
15716 		dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
15717 			 (unsigned int)pci_resource_start(pdev, 0),
15718 			 pf->ioremap_len, err);
15719 		goto err_ioremap;
15720 	}
15721 	hw->vendor_id = pdev->vendor;
15722 	hw->device_id = pdev->device;
15723 	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
15724 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
15725 	i40e_set_subsystem_device_id(hw);
15726 	hw->bus.device = PCI_SLOT(pdev->devfn);
15727 	hw->bus.func = PCI_FUNC(pdev->devfn);
15728 	hw->bus.bus_id = pdev->bus->number;
15729 	pf->instance = pfs_found;
15730 
15731 	/* Select something other than the 802.1ad ethertype for the
15732 	 * switch to use internally and drop on ingress.
15733 	 */
15734 	hw->switch_tag = 0xffff;
15735 	hw->first_tag = ETH_P_8021AD;
15736 	hw->second_tag = ETH_P_8021Q;
15737 
15738 	INIT_LIST_HEAD(&pf->l3_flex_pit_list);
15739 	INIT_LIST_HEAD(&pf->l4_flex_pit_list);
15740 	INIT_LIST_HEAD(&pf->ddp_old_prof);
15741 
15742 	/* set up the locks for the AQ, do this only once in probe
15743 	 * and destroy them only once in remove
15744 	 */
15745 	mutex_init(&hw->aq.asq_mutex);
15746 	mutex_init(&hw->aq.arq_mutex);
15747 
15748 	pf->msg_enable = netif_msg_init(debug,
15749 					NETIF_MSG_DRV |
15750 					NETIF_MSG_PROBE |
15751 					NETIF_MSG_LINK);
15752 	if (debug < -1)
15753 		pf->hw.debug_mask = debug;
15754 
15755 	/* do a special CORER for clearing PXE mode once at init */
15756 	if (hw->revision_id == 0 &&
15757 	    (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
15758 		wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
15759 		i40e_flush(hw);
15760 		msleep(200);
15761 		pf->corer_count++;
15762 
15763 		i40e_clear_pxe_mode(hw);
15764 	}
15765 
15766 	/* Reset here to make sure all is clean and to define PF 'n' */
15767 	i40e_clear_hw(hw);
15768 
15769 	err = i40e_set_mac_type(hw);
15770 	if (err) {
15771 		dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15772 			 err);
15773 		goto err_pf_reset;
15774 	}
15775 
15776 	err = i40e_handle_resets(pf);
15777 	if (err)
15778 		goto err_pf_reset;
15779 
15780 	i40e_check_recovery_mode(pf);
15781 
15782 	if (is_kdump_kernel()) {
15783 		hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN;
15784 		hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN;
15785 	} else {
15786 		hw->aq.num_arq_entries = I40E_AQ_LEN;
15787 		hw->aq.num_asq_entries = I40E_AQ_LEN;
15788 	}
15789 	hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15790 	hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
15791 	pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
15792 
15793 	snprintf(pf->int_name, sizeof(pf->int_name) - 1,
15794 		 "%s-%s:misc",
15795 		 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
15796 
15797 	err = i40e_init_shared_code(hw);
15798 	if (err) {
15799 		dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
15800 			 err);
15801 		goto err_pf_reset;
15802 	}
15803 
15804 	/* set up a default setting for link flow control */
15805 	pf->hw.fc.requested_mode = I40E_FC_NONE;
15806 
15807 	err = i40e_init_adminq(hw);
15808 	if (err) {
15809 		if (err == I40E_ERR_FIRMWARE_API_VERSION)
15810 			dev_info(&pdev->dev,
15811 				 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
15812 				 hw->aq.api_maj_ver,
15813 				 hw->aq.api_min_ver,
15814 				 I40E_FW_API_VERSION_MAJOR,
15815 				 I40E_FW_MINOR_VERSION(hw));
15816 		else
15817 			dev_info(&pdev->dev,
15818 				 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
15819 
15820 		goto err_pf_reset;
15821 	}
15822 	i40e_get_oem_version(hw);
15823 
15824 	/* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */
15825 	dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
15826 		 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
15827 		 hw->aq.api_maj_ver, hw->aq.api_min_ver,
15828 		 i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id,
15829 		 hw->subsystem_vendor_id, hw->subsystem_device_id);
15830 
15831 	if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
15832 	    hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
15833 		dev_dbg(&pdev->dev,
15834 			"The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n",
15835 			 hw->aq.api_maj_ver,
15836 			 hw->aq.api_min_ver,
15837 			 I40E_FW_API_VERSION_MAJOR,
15838 			 I40E_FW_MINOR_VERSION(hw));
15839 	else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
15840 		dev_info(&pdev->dev,
15841 			 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
15842 			 hw->aq.api_maj_ver,
15843 			 hw->aq.api_min_ver,
15844 			 I40E_FW_API_VERSION_MAJOR,
15845 			 I40E_FW_MINOR_VERSION(hw));
15846 
15847 	i40e_verify_eeprom(pf);
15848 
15849 	/* Rev 0 hardware was never productized */
15850 	if (hw->revision_id < 1)
15851 		dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
15852 
15853 	i40e_clear_pxe_mode(hw);
15854 
15855 	err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
15856 	if (err)
15857 		goto err_adminq_setup;
15858 
15859 	err = i40e_sw_init(pf);
15860 	if (err) {
15861 		dev_info(&pdev->dev, "sw_init failed: %d\n", err);
15862 		goto err_sw_init;
15863 	}
15864 
15865 	if (test_bit(__I40E_RECOVERY_MODE, pf->state))
15866 		return i40e_init_recovery_mode(pf, hw);
15867 
15868 	err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
15869 				hw->func_caps.num_rx_qp, 0, 0);
15870 	if (err) {
15871 		dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
15872 		goto err_init_lan_hmc;
15873 	}
15874 
15875 	err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
15876 	if (err) {
15877 		dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
15878 		err = -ENOENT;
15879 		goto err_configure_lan_hmc;
15880 	}
15881 
15882 	/* Disable LLDP for NICs that have firmware versions lower than v4.3.
15883 	 * Ignore error return codes because if it was already disabled via
15884 	 * hardware settings this will fail
15885 	 */
15886 	if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
15887 		dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
15888 		i40e_aq_stop_lldp(hw, true, false, NULL);
15889 	}
15890 
15891 	/* allow a platform config to override the HW addr */
15892 	i40e_get_platform_mac_addr(pdev, pf);
15893 
15894 	if (!is_valid_ether_addr(hw->mac.addr)) {
15895 		dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
15896 		err = -EIO;
15897 		goto err_mac_addr;
15898 	}
15899 	dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
15900 	ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
15901 	i40e_get_port_mac_addr(hw, hw->mac.port_addr);
15902 	if (is_valid_ether_addr(hw->mac.port_addr))
15903 		pf->hw_features |= I40E_HW_PORT_ID_VALID;
15904 
15905 	i40e_ptp_alloc_pins(pf);
15906 	pci_set_drvdata(pdev, pf);
15907 	pci_save_state(pdev);
15908 
15909 #ifdef CONFIG_I40E_DCB
15910 	status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status);
15911 	(!status &&
15912 	 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ?
15913 		(pf->flags &= ~I40E_FLAG_DISABLE_FW_LLDP) :
15914 		(pf->flags |= I40E_FLAG_DISABLE_FW_LLDP);
15915 	dev_info(&pdev->dev,
15916 		 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
15917 			"FW LLDP is disabled\n" :
15918 			"FW LLDP is enabled\n");
15919 
15920 	/* Enable FW to write default DCB config on link-up */
15921 	i40e_aq_set_dcb_parameters(hw, true, NULL);
15922 
15923 	err = i40e_init_pf_dcb(pf);
15924 	if (err) {
15925 		dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
15926 		pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
15927 		/* Continue without DCB enabled */
15928 	}
15929 #endif /* CONFIG_I40E_DCB */
15930 
15931 	/* set up periodic task facility */
15932 	timer_setup(&pf->service_timer, i40e_service_timer, 0);
15933 	pf->service_timer_period = HZ;
15934 
15935 	INIT_WORK(&pf->service_task, i40e_service_task);
15936 	clear_bit(__I40E_SERVICE_SCHED, pf->state);
15937 
15938 	/* NVM bit on means WoL disabled for the port */
15939 	i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
15940 	if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
15941 		pf->wol_en = false;
15942 	else
15943 		pf->wol_en = true;
15944 	device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
15945 
15946 	/* set up the main switch operations */
15947 	i40e_determine_queue_usage(pf);
15948 	err = i40e_init_interrupt_scheme(pf);
15949 	if (err)
15950 		goto err_switch_setup;
15951 
15952 	/* Reduce Tx and Rx pairs for kdump
15953 	 * When MSI-X is enabled, it's not allowed to use more TC queue
15954 	 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus
15955 	 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1.
15956 	 */
15957 	if (is_kdump_kernel())
15958 		pf->num_lan_msix = 1;
15959 
15960 	pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port;
15961 	pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port;
15962 	pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
15963 	pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared;
15964 	pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS;
15965 	pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN |
15966 						    UDP_TUNNEL_TYPE_GENEVE;
15967 
15968 	/* The number of VSIs reported by the FW is the minimum guaranteed
15969 	 * to us; HW supports far more and we share the remaining pool with
15970 	 * the other PFs. We allocate space for more than the guarantee with
15971 	 * the understanding that we might not get them all later.
15972 	 */
15973 	if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
15974 		pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
15975 	else
15976 		pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
15977 	if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
15978 		dev_warn(&pf->pdev->dev,
15979 			 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
15980 			 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
15981 		pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
15982 	}
15983 
15984 	/* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */
15985 	pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
15986 			  GFP_KERNEL);
15987 	if (!pf->vsi) {
15988 		err = -ENOMEM;
15989 		goto err_switch_setup;
15990 	}
15991 
15992 #ifdef CONFIG_PCI_IOV
15993 	/* prep for VF support */
15994 	if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15995 	    (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15996 	    !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15997 		if (pci_num_vf(pdev))
15998 			pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
15999 	}
16000 #endif
16001 	err = i40e_setup_pf_switch(pf, false, false);
16002 	if (err) {
16003 		dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
16004 		goto err_vsis;
16005 	}
16006 	INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
16007 
16008 	/* if FDIR VSI was set up, start it now */
16009 	for (i = 0; i < pf->num_alloc_vsi; i++) {
16010 		if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
16011 			i40e_vsi_open(pf->vsi[i]);
16012 			break;
16013 		}
16014 	}
16015 
16016 	/* The driver only wants link up/down and module qualification
16017 	 * reports from firmware.  Note the negative logic.
16018 	 */
16019 	err = i40e_aq_set_phy_int_mask(&pf->hw,
16020 				       ~(I40E_AQ_EVENT_LINK_UPDOWN |
16021 					 I40E_AQ_EVENT_MEDIA_NA |
16022 					 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
16023 	if (err)
16024 		dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n",
16025 			 ERR_PTR(err),
16026 			 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16027 
16028 	/* Reconfigure hardware for allowing smaller MSS in the case
16029 	 * of TSO, so that we avoid the MDD being fired and causing
16030 	 * a reset in the case of small MSS+TSO.
16031 	 */
16032 	val = rd32(hw, I40E_REG_MSS);
16033 	if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
16034 		val &= ~I40E_REG_MSS_MIN_MASK;
16035 		val |= I40E_64BYTE_MSS;
16036 		wr32(hw, I40E_REG_MSS, val);
16037 	}
16038 
16039 	if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
16040 		msleep(75);
16041 		err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
16042 		if (err)
16043 			dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n",
16044 				 ERR_PTR(err),
16045 				 i40e_aq_str(&pf->hw,
16046 					     pf->hw.aq.asq_last_status));
16047 	}
16048 	/* The main driver is (mostly) up and happy. We need to set this state
16049 	 * before setting up the misc vector or we get a race and the vector
16050 	 * ends up disabled forever.
16051 	 */
16052 	clear_bit(__I40E_DOWN, pf->state);
16053 
16054 	/* In case of MSIX we are going to setup the misc vector right here
16055 	 * to handle admin queue events etc. In case of legacy and MSI
16056 	 * the misc functionality and queue processing is combined in
16057 	 * the same vector and that gets setup at open.
16058 	 */
16059 	if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
16060 		err = i40e_setup_misc_vector(pf);
16061 		if (err) {
16062 			dev_info(&pdev->dev,
16063 				 "setup of misc vector failed: %d\n", err);
16064 			i40e_cloud_filter_exit(pf);
16065 			i40e_fdir_teardown(pf);
16066 			goto err_vsis;
16067 		}
16068 	}
16069 
16070 #ifdef CONFIG_PCI_IOV
16071 	/* prep for VF support */
16072 	if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
16073 	    (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
16074 	    !test_bit(__I40E_BAD_EEPROM, pf->state)) {
16075 		/* disable link interrupts for VFs */
16076 		val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
16077 		val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
16078 		wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
16079 		i40e_flush(hw);
16080 
16081 		if (pci_num_vf(pdev)) {
16082 			dev_info(&pdev->dev,
16083 				 "Active VFs found, allocating resources.\n");
16084 			err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
16085 			if (err)
16086 				dev_info(&pdev->dev,
16087 					 "Error %d allocating resources for existing VFs\n",
16088 					 err);
16089 		}
16090 	}
16091 #endif /* CONFIG_PCI_IOV */
16092 
16093 	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
16094 		pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
16095 						      pf->num_iwarp_msix,
16096 						      I40E_IWARP_IRQ_PILE_ID);
16097 		if (pf->iwarp_base_vector < 0) {
16098 			dev_info(&pdev->dev,
16099 				 "failed to get tracking for %d vectors for IWARP err=%d\n",
16100 				 pf->num_iwarp_msix, pf->iwarp_base_vector);
16101 			pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
16102 		}
16103 	}
16104 
16105 	i40e_dbg_pf_init(pf);
16106 
16107 	/* tell the firmware that we're starting */
16108 	i40e_send_version(pf);
16109 
16110 	/* since everything's happy, start the service_task timer */
16111 	mod_timer(&pf->service_timer,
16112 		  round_jiffies(jiffies + pf->service_timer_period));
16113 
16114 	/* add this PF to client device list and launch a client service task */
16115 	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
16116 		err = i40e_lan_add_device(pf);
16117 		if (err)
16118 			dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
16119 				 err);
16120 	}
16121 
16122 #define PCI_SPEED_SIZE 8
16123 #define PCI_WIDTH_SIZE 8
16124 	/* Devices on the IOSF bus do not have this information
16125 	 * and will report PCI Gen 1 x 1 by default so don't bother
16126 	 * checking them.
16127 	 */
16128 	if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
16129 		char speed[PCI_SPEED_SIZE] = "Unknown";
16130 		char width[PCI_WIDTH_SIZE] = "Unknown";
16131 
16132 		/* Get the negotiated link width and speed from PCI config
16133 		 * space
16134 		 */
16135 		pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
16136 					  &link_status);
16137 
16138 		i40e_set_pci_config_data(hw, link_status);
16139 
16140 		switch (hw->bus.speed) {
16141 		case i40e_bus_speed_8000:
16142 			strscpy(speed, "8.0", PCI_SPEED_SIZE); break;
16143 		case i40e_bus_speed_5000:
16144 			strscpy(speed, "5.0", PCI_SPEED_SIZE); break;
16145 		case i40e_bus_speed_2500:
16146 			strscpy(speed, "2.5", PCI_SPEED_SIZE); break;
16147 		default:
16148 			break;
16149 		}
16150 		switch (hw->bus.width) {
16151 		case i40e_bus_width_pcie_x8:
16152 			strscpy(width, "8", PCI_WIDTH_SIZE); break;
16153 		case i40e_bus_width_pcie_x4:
16154 			strscpy(width, "4", PCI_WIDTH_SIZE); break;
16155 		case i40e_bus_width_pcie_x2:
16156 			strscpy(width, "2", PCI_WIDTH_SIZE); break;
16157 		case i40e_bus_width_pcie_x1:
16158 			strscpy(width, "1", PCI_WIDTH_SIZE); break;
16159 		default:
16160 			break;
16161 		}
16162 
16163 		dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
16164 			 speed, width);
16165 
16166 		if (hw->bus.width < i40e_bus_width_pcie_x8 ||
16167 		    hw->bus.speed < i40e_bus_speed_8000) {
16168 			dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
16169 			dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
16170 		}
16171 	}
16172 
16173 	/* get the requested speeds from the fw */
16174 	err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
16175 	if (err)
16176 		dev_dbg(&pf->pdev->dev, "get requested speeds ret =  %pe last_status =  %s\n",
16177 			ERR_PTR(err),
16178 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16179 	pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
16180 
16181 	/* set the FEC config due to the board capabilities */
16182 	i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
16183 
16184 	/* get the supported phy types from the fw */
16185 	err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
16186 	if (err)
16187 		dev_dbg(&pf->pdev->dev, "get supported phy types ret =  %pe last_status =  %s\n",
16188 			ERR_PTR(err),
16189 			i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
16190 
16191 	/* make sure the MFS hasn't been set lower than the default */
16192 #define MAX_FRAME_SIZE_DEFAULT 0x2600
16193 	val = (rd32(&pf->hw, I40E_PRTGL_SAH) &
16194 	       I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT;
16195 	if (val < MAX_FRAME_SIZE_DEFAULT)
16196 		dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n",
16197 			 i, val);
16198 
16199 	/* Add a filter to drop all Flow control frames from any VSI from being
16200 	 * transmitted. By doing so we stop a malicious VF from sending out
16201 	 * PAUSE or PFC frames and potentially controlling traffic for other
16202 	 * PF/VF VSIs.
16203 	 * The FW can still send Flow control frames if enabled.
16204 	 */
16205 	i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
16206 						       pf->main_vsi_seid);
16207 
16208 	if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
16209 		(pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
16210 		pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
16211 	if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
16212 		pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
16213 	/* print a string summarizing features */
16214 	i40e_print_features(pf);
16215 
16216 	return 0;
16217 
16218 	/* Unwind what we've done if something failed in the setup */
16219 err_vsis:
16220 	set_bit(__I40E_DOWN, pf->state);
16221 	i40e_clear_interrupt_scheme(pf);
16222 	kfree(pf->vsi);
16223 err_switch_setup:
16224 	i40e_reset_interrupt_capability(pf);
16225 	timer_shutdown_sync(&pf->service_timer);
16226 err_mac_addr:
16227 err_configure_lan_hmc:
16228 	(void)i40e_shutdown_lan_hmc(hw);
16229 err_init_lan_hmc:
16230 	kfree(pf->qp_pile);
16231 err_sw_init:
16232 err_adminq_setup:
16233 err_pf_reset:
16234 	iounmap(hw->hw_addr);
16235 err_ioremap:
16236 	kfree(pf);
16237 err_pf_alloc:
16238 	pci_release_mem_regions(pdev);
16239 err_pci_reg:
16240 err_dma:
16241 	pci_disable_device(pdev);
16242 	return err;
16243 }
16244 
16245 /**
16246  * i40e_remove - Device removal routine
16247  * @pdev: PCI device information struct
16248  *
16249  * i40e_remove is called by the PCI subsystem to alert the driver
16250  * that is should release a PCI device.  This could be caused by a
16251  * Hot-Plug event, or because the driver is going to be removed from
16252  * memory.
16253  **/
16254 static void i40e_remove(struct pci_dev *pdev)
16255 {
16256 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16257 	struct i40e_hw *hw = &pf->hw;
16258 	int ret_code;
16259 	int i;
16260 
16261 	i40e_dbg_pf_exit(pf);
16262 
16263 	i40e_ptp_stop(pf);
16264 
16265 	/* Disable RSS in hw */
16266 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
16267 	i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
16268 
16269 	/* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE
16270 	 * flags, once they are set, i40e_rebuild should not be called as
16271 	 * i40e_prep_for_reset always returns early.
16272 	 */
16273 	while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
16274 		usleep_range(1000, 2000);
16275 	set_bit(__I40E_IN_REMOVE, pf->state);
16276 
16277 	if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
16278 		set_bit(__I40E_VF_RESETS_DISABLED, pf->state);
16279 		i40e_free_vfs(pf);
16280 		pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
16281 	}
16282 	/* no more scheduling of any task */
16283 	set_bit(__I40E_SUSPENDED, pf->state);
16284 	set_bit(__I40E_DOWN, pf->state);
16285 	if (pf->service_timer.function)
16286 		timer_shutdown_sync(&pf->service_timer);
16287 	if (pf->service_task.func)
16288 		cancel_work_sync(&pf->service_task);
16289 
16290 	if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
16291 		struct i40e_vsi *vsi = pf->vsi[0];
16292 
16293 		/* We know that we have allocated only one vsi for this PF,
16294 		 * it was just for registering netdevice, so the interface
16295 		 * could be visible in the 'ifconfig' output
16296 		 */
16297 		unregister_netdev(vsi->netdev);
16298 		free_netdev(vsi->netdev);
16299 
16300 		goto unmap;
16301 	}
16302 
16303 	/* Client close must be called explicitly here because the timer
16304 	 * has been stopped.
16305 	 */
16306 	i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16307 
16308 	i40e_fdir_teardown(pf);
16309 
16310 	/* If there is a switch structure or any orphans, remove them.
16311 	 * This will leave only the PF's VSI remaining.
16312 	 */
16313 	for (i = 0; i < I40E_MAX_VEB; i++) {
16314 		if (!pf->veb[i])
16315 			continue;
16316 
16317 		if (pf->veb[i]->uplink_seid == pf->mac_seid ||
16318 		    pf->veb[i]->uplink_seid == 0)
16319 			i40e_switch_branch_release(pf->veb[i]);
16320 	}
16321 
16322 	/* Now we can shutdown the PF's VSI, just before we kill
16323 	 * adminq and hmc.
16324 	 */
16325 	if (pf->vsi[pf->lan_vsi])
16326 		i40e_vsi_release(pf->vsi[pf->lan_vsi]);
16327 
16328 	i40e_cloud_filter_exit(pf);
16329 
16330 	/* remove attached clients */
16331 	if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
16332 		ret_code = i40e_lan_del_device(pf);
16333 		if (ret_code)
16334 			dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
16335 				 ret_code);
16336 	}
16337 
16338 	/* shutdown and destroy the HMC */
16339 	if (hw->hmc.hmc_obj) {
16340 		ret_code = i40e_shutdown_lan_hmc(hw);
16341 		if (ret_code)
16342 			dev_warn(&pdev->dev,
16343 				 "Failed to destroy the HMC resources: %d\n",
16344 				 ret_code);
16345 	}
16346 
16347 unmap:
16348 	/* Free MSI/legacy interrupt 0 when in recovery mode. */
16349 	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16350 	    !(pf->flags & I40E_FLAG_MSIX_ENABLED))
16351 		free_irq(pf->pdev->irq, pf);
16352 
16353 	/* shutdown the adminq */
16354 	i40e_shutdown_adminq(hw);
16355 
16356 	/* destroy the locks only once, here */
16357 	mutex_destroy(&hw->aq.arq_mutex);
16358 	mutex_destroy(&hw->aq.asq_mutex);
16359 
16360 	/* Clear all dynamic memory lists of rings, q_vectors, and VSIs */
16361 	rtnl_lock();
16362 	i40e_clear_interrupt_scheme(pf);
16363 	for (i = 0; i < pf->num_alloc_vsi; i++) {
16364 		if (pf->vsi[i]) {
16365 			if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
16366 				i40e_vsi_clear_rings(pf->vsi[i]);
16367 			i40e_vsi_clear(pf->vsi[i]);
16368 			pf->vsi[i] = NULL;
16369 		}
16370 	}
16371 	rtnl_unlock();
16372 
16373 	for (i = 0; i < I40E_MAX_VEB; i++) {
16374 		kfree(pf->veb[i]);
16375 		pf->veb[i] = NULL;
16376 	}
16377 
16378 	kfree(pf->qp_pile);
16379 	kfree(pf->vsi);
16380 
16381 	iounmap(hw->hw_addr);
16382 	kfree(pf);
16383 	pci_release_mem_regions(pdev);
16384 
16385 	pci_disable_device(pdev);
16386 }
16387 
16388 /**
16389  * i40e_pci_error_detected - warning that something funky happened in PCI land
16390  * @pdev: PCI device information struct
16391  * @error: the type of PCI error
16392  *
16393  * Called to warn that something happened and the error handling steps
16394  * are in progress.  Allows the driver to quiesce things, be ready for
16395  * remediation.
16396  **/
16397 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
16398 						pci_channel_state_t error)
16399 {
16400 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16401 
16402 	dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
16403 
16404 	if (!pf) {
16405 		dev_info(&pdev->dev,
16406 			 "Cannot recover - error happened during device probe\n");
16407 		return PCI_ERS_RESULT_DISCONNECT;
16408 	}
16409 
16410 	/* shutdown all operations */
16411 	if (!test_bit(__I40E_SUSPENDED, pf->state))
16412 		i40e_prep_for_reset(pf);
16413 
16414 	/* Request a slot reset */
16415 	return PCI_ERS_RESULT_NEED_RESET;
16416 }
16417 
16418 /**
16419  * i40e_pci_error_slot_reset - a PCI slot reset just happened
16420  * @pdev: PCI device information struct
16421  *
16422  * Called to find if the driver can work with the device now that
16423  * the pci slot has been reset.  If a basic connection seems good
16424  * (registers are readable and have sane content) then return a
16425  * happy little PCI_ERS_RESULT_xxx.
16426  **/
16427 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
16428 {
16429 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16430 	pci_ers_result_t result;
16431 	u32 reg;
16432 
16433 	dev_dbg(&pdev->dev, "%s\n", __func__);
16434 	if (pci_enable_device_mem(pdev)) {
16435 		dev_info(&pdev->dev,
16436 			 "Cannot re-enable PCI device after reset.\n");
16437 		result = PCI_ERS_RESULT_DISCONNECT;
16438 	} else {
16439 		pci_set_master(pdev);
16440 		pci_restore_state(pdev);
16441 		pci_save_state(pdev);
16442 		pci_wake_from_d3(pdev, false);
16443 
16444 		reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
16445 		if (reg == 0)
16446 			result = PCI_ERS_RESULT_RECOVERED;
16447 		else
16448 			result = PCI_ERS_RESULT_DISCONNECT;
16449 	}
16450 
16451 	return result;
16452 }
16453 
16454 /**
16455  * i40e_pci_error_reset_prepare - prepare device driver for pci reset
16456  * @pdev: PCI device information struct
16457  */
16458 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
16459 {
16460 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16461 
16462 	i40e_prep_for_reset(pf);
16463 }
16464 
16465 /**
16466  * i40e_pci_error_reset_done - pci reset done, device driver reset can begin
16467  * @pdev: PCI device information struct
16468  */
16469 static void i40e_pci_error_reset_done(struct pci_dev *pdev)
16470 {
16471 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16472 
16473 	if (test_bit(__I40E_IN_REMOVE, pf->state))
16474 		return;
16475 
16476 	i40e_reset_and_rebuild(pf, false, false);
16477 }
16478 
16479 /**
16480  * i40e_pci_error_resume - restart operations after PCI error recovery
16481  * @pdev: PCI device information struct
16482  *
16483  * Called to allow the driver to bring things back up after PCI error
16484  * and/or reset recovery has finished.
16485  **/
16486 static void i40e_pci_error_resume(struct pci_dev *pdev)
16487 {
16488 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16489 
16490 	dev_dbg(&pdev->dev, "%s\n", __func__);
16491 	if (test_bit(__I40E_SUSPENDED, pf->state))
16492 		return;
16493 
16494 	i40e_handle_reset_warning(pf, false);
16495 }
16496 
16497 /**
16498  * i40e_enable_mc_magic_wake - enable multicast magic packet wake up
16499  * using the mac_address_write admin q function
16500  * @pf: pointer to i40e_pf struct
16501  **/
16502 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
16503 {
16504 	struct i40e_hw *hw = &pf->hw;
16505 	u8 mac_addr[6];
16506 	u16 flags = 0;
16507 	int ret;
16508 
16509 	/* Get current MAC address in case it's an LAA */
16510 	if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
16511 		ether_addr_copy(mac_addr,
16512 				pf->vsi[pf->lan_vsi]->netdev->dev_addr);
16513 	} else {
16514 		dev_err(&pf->pdev->dev,
16515 			"Failed to retrieve MAC address; using default\n");
16516 		ether_addr_copy(mac_addr, hw->mac.addr);
16517 	}
16518 
16519 	/* The FW expects the mac address write cmd to first be called with
16520 	 * one of these flags before calling it again with the multicast
16521 	 * enable flags.
16522 	 */
16523 	flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
16524 
16525 	if (hw->func_caps.flex10_enable && hw->partition_id != 1)
16526 		flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
16527 
16528 	ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16529 	if (ret) {
16530 		dev_err(&pf->pdev->dev,
16531 			"Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
16532 		return;
16533 	}
16534 
16535 	flags = I40E_AQC_MC_MAG_EN
16536 			| I40E_AQC_WOL_PRESERVE_ON_PFR
16537 			| I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
16538 	ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
16539 	if (ret)
16540 		dev_err(&pf->pdev->dev,
16541 			"Failed to enable Multicast Magic Packet wake up\n");
16542 }
16543 
16544 /**
16545  * i40e_shutdown - PCI callback for shutting down
16546  * @pdev: PCI device information struct
16547  **/
16548 static void i40e_shutdown(struct pci_dev *pdev)
16549 {
16550 	struct i40e_pf *pf = pci_get_drvdata(pdev);
16551 	struct i40e_hw *hw = &pf->hw;
16552 
16553 	set_bit(__I40E_SUSPENDED, pf->state);
16554 	set_bit(__I40E_DOWN, pf->state);
16555 
16556 	del_timer_sync(&pf->service_timer);
16557 	cancel_work_sync(&pf->service_task);
16558 	i40e_cloud_filter_exit(pf);
16559 	i40e_fdir_teardown(pf);
16560 
16561 	/* Client close must be called explicitly here because the timer
16562 	 * has been stopped.
16563 	 */
16564 	i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16565 
16566 	if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16567 		i40e_enable_mc_magic_wake(pf);
16568 
16569 	i40e_prep_for_reset(pf);
16570 
16571 	wr32(hw, I40E_PFPM_APM,
16572 	     (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16573 	wr32(hw, I40E_PFPM_WUFC,
16574 	     (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16575 
16576 	/* Free MSI/legacy interrupt 0 when in recovery mode. */
16577 	if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
16578 	    !(pf->flags & I40E_FLAG_MSIX_ENABLED))
16579 		free_irq(pf->pdev->irq, pf);
16580 
16581 	/* Since we're going to destroy queues during the
16582 	 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16583 	 * whole section
16584 	 */
16585 	rtnl_lock();
16586 	i40e_clear_interrupt_scheme(pf);
16587 	rtnl_unlock();
16588 
16589 	if (system_state == SYSTEM_POWER_OFF) {
16590 		pci_wake_from_d3(pdev, pf->wol_en);
16591 		pci_set_power_state(pdev, PCI_D3hot);
16592 	}
16593 }
16594 
16595 /**
16596  * i40e_suspend - PM callback for moving to D3
16597  * @dev: generic device information structure
16598  **/
16599 static int __maybe_unused i40e_suspend(struct device *dev)
16600 {
16601 	struct i40e_pf *pf = dev_get_drvdata(dev);
16602 	struct i40e_hw *hw = &pf->hw;
16603 
16604 	/* If we're already suspended, then there is nothing to do */
16605 	if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
16606 		return 0;
16607 
16608 	set_bit(__I40E_DOWN, pf->state);
16609 
16610 	/* Ensure service task will not be running */
16611 	del_timer_sync(&pf->service_timer);
16612 	cancel_work_sync(&pf->service_task);
16613 
16614 	/* Client close must be called explicitly here because the timer
16615 	 * has been stopped.
16616 	 */
16617 	i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
16618 
16619 	if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
16620 		i40e_enable_mc_magic_wake(pf);
16621 
16622 	/* Since we're going to destroy queues during the
16623 	 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this
16624 	 * whole section
16625 	 */
16626 	rtnl_lock();
16627 
16628 	i40e_prep_for_reset(pf);
16629 
16630 	wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
16631 	wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
16632 
16633 	/* Clear the interrupt scheme and release our IRQs so that the system
16634 	 * can safely hibernate even when there are a large number of CPUs.
16635 	 * Otherwise hibernation might fail when mapping all the vectors back
16636 	 * to CPU0.
16637 	 */
16638 	i40e_clear_interrupt_scheme(pf);
16639 
16640 	rtnl_unlock();
16641 
16642 	return 0;
16643 }
16644 
16645 /**
16646  * i40e_resume - PM callback for waking up from D3
16647  * @dev: generic device information structure
16648  **/
16649 static int __maybe_unused i40e_resume(struct device *dev)
16650 {
16651 	struct i40e_pf *pf = dev_get_drvdata(dev);
16652 	int err;
16653 
16654 	/* If we're not suspended, then there is nothing to do */
16655 	if (!test_bit(__I40E_SUSPENDED, pf->state))
16656 		return 0;
16657 
16658 	/* We need to hold the RTNL lock prior to restoring interrupt schemes,
16659 	 * since we're going to be restoring queues
16660 	 */
16661 	rtnl_lock();
16662 
16663 	/* We cleared the interrupt scheme when we suspended, so we need to
16664 	 * restore it now to resume device functionality.
16665 	 */
16666 	err = i40e_restore_interrupt_scheme(pf);
16667 	if (err) {
16668 		dev_err(dev, "Cannot restore interrupt scheme: %d\n",
16669 			err);
16670 	}
16671 
16672 	clear_bit(__I40E_DOWN, pf->state);
16673 	i40e_reset_and_rebuild(pf, false, true);
16674 
16675 	rtnl_unlock();
16676 
16677 	/* Clear suspended state last after everything is recovered */
16678 	clear_bit(__I40E_SUSPENDED, pf->state);
16679 
16680 	/* Restart the service task */
16681 	mod_timer(&pf->service_timer,
16682 		  round_jiffies(jiffies + pf->service_timer_period));
16683 
16684 	return 0;
16685 }
16686 
16687 static const struct pci_error_handlers i40e_err_handler = {
16688 	.error_detected = i40e_pci_error_detected,
16689 	.slot_reset = i40e_pci_error_slot_reset,
16690 	.reset_prepare = i40e_pci_error_reset_prepare,
16691 	.reset_done = i40e_pci_error_reset_done,
16692 	.resume = i40e_pci_error_resume,
16693 };
16694 
16695 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
16696 
16697 static struct pci_driver i40e_driver = {
16698 	.name     = i40e_driver_name,
16699 	.id_table = i40e_pci_tbl,
16700 	.probe    = i40e_probe,
16701 	.remove   = i40e_remove,
16702 	.driver   = {
16703 		.pm = &i40e_pm_ops,
16704 	},
16705 	.shutdown = i40e_shutdown,
16706 	.err_handler = &i40e_err_handler,
16707 	.sriov_configure = i40e_pci_sriov_configure,
16708 };
16709 
16710 /**
16711  * i40e_init_module - Driver registration routine
16712  *
16713  * i40e_init_module is the first routine called when the driver is
16714  * loaded. All it does is register with the PCI subsystem.
16715  **/
16716 static int __init i40e_init_module(void)
16717 {
16718 	int err;
16719 
16720 	pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string);
16721 	pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
16722 
16723 	/* There is no need to throttle the number of active tasks because
16724 	 * each device limits its own task using a state bit for scheduling
16725 	 * the service task, and the device tasks do not interfere with each
16726 	 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM
16727 	 * since we need to be able to guarantee forward progress even under
16728 	 * memory pressure.
16729 	 */
16730 	i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
16731 	if (!i40e_wq) {
16732 		pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
16733 		return -ENOMEM;
16734 	}
16735 
16736 	i40e_dbg_init();
16737 	err = pci_register_driver(&i40e_driver);
16738 	if (err) {
16739 		destroy_workqueue(i40e_wq);
16740 		i40e_dbg_exit();
16741 		return err;
16742 	}
16743 
16744 	return 0;
16745 }
16746 module_init(i40e_init_module);
16747 
16748 /**
16749  * i40e_exit_module - Driver exit cleanup routine
16750  *
16751  * i40e_exit_module is called just before the driver is removed
16752  * from memory.
16753  **/
16754 static void __exit i40e_exit_module(void)
16755 {
16756 	pci_unregister_driver(&i40e_driver);
16757 	destroy_workqueue(i40e_wq);
16758 	ida_destroy(&i40e_client_ida);
16759 	i40e_dbg_exit();
16760 }
16761 module_exit(i40e_exit_module);
16762