1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018, Intel Corporation. */
3 
4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #include <generated/utsrelease.h>
9 #include "ice.h"
10 #include "ice_base.h"
11 #include "ice_lib.h"
12 #include "ice_fltr.h"
13 #include "ice_dcb_lib.h"
14 #include "ice_dcb_nl.h"
15 #include "ice_devlink.h"
16 /* Including ice_trace.h with CREATE_TRACE_POINTS defined will generate the
17  * ice tracepoint functions. This must be done exactly once across the
18  * ice driver.
19  */
20 #define CREATE_TRACE_POINTS
21 #include "ice_trace.h"
22 #include "ice_eswitch.h"
23 #include "ice_tc_lib.h"
24 #include "ice_vsi_vlan_ops.h"
25 #include <net/xdp_sock_drv.h>
26 
27 #define DRV_SUMMARY	"Intel(R) Ethernet Connection E800 Series Linux Driver"
28 static const char ice_driver_string[] = DRV_SUMMARY;
29 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation.";
30 
31 /* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */
32 #define ICE_DDP_PKG_PATH	"intel/ice/ddp/"
33 #define ICE_DDP_PKG_FILE	ICE_DDP_PKG_PATH "ice.pkg"
34 
35 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
36 MODULE_DESCRIPTION(DRV_SUMMARY);
37 MODULE_LICENSE("GPL v2");
38 MODULE_FIRMWARE(ICE_DDP_PKG_FILE);
39 
40 static int debug = -1;
41 module_param(debug, int, 0644);
42 #ifndef CONFIG_DYNAMIC_DEBUG
43 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)");
44 #else
45 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)");
46 #endif /* !CONFIG_DYNAMIC_DEBUG */
47 
48 DEFINE_STATIC_KEY_FALSE(ice_xdp_locking_key);
49 EXPORT_SYMBOL(ice_xdp_locking_key);
50 
51 /**
52  * ice_hw_to_dev - Get device pointer from the hardware structure
53  * @hw: pointer to the device HW structure
54  *
55  * Used to access the device pointer from compilation units which can't easily
56  * include the definition of struct ice_pf without leading to circular header
57  * dependencies.
58  */
59 struct device *ice_hw_to_dev(struct ice_hw *hw)
60 {
61 	struct ice_pf *pf = container_of(hw, struct ice_pf, hw);
62 
63 	return &pf->pdev->dev;
64 }
65 
66 static struct workqueue_struct *ice_wq;
67 static const struct net_device_ops ice_netdev_safe_mode_ops;
68 static const struct net_device_ops ice_netdev_ops;
69 
70 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type);
71 
72 static void ice_vsi_release_all(struct ice_pf *pf);
73 
74 static int ice_rebuild_channels(struct ice_pf *pf);
75 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_adv_fltr);
76 
77 static int
78 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch,
79 		     void *cb_priv, enum tc_setup_type type, void *type_data,
80 		     void *data,
81 		     void (*cleanup)(struct flow_block_cb *block_cb));
82 
83 bool netif_is_ice(struct net_device *dev)
84 {
85 	return dev && (dev->netdev_ops == &ice_netdev_ops);
86 }
87 
88 /**
89  * ice_get_tx_pending - returns number of Tx descriptors not processed
90  * @ring: the ring of descriptors
91  */
92 static u16 ice_get_tx_pending(struct ice_tx_ring *ring)
93 {
94 	u16 head, tail;
95 
96 	head = ring->next_to_clean;
97 	tail = ring->next_to_use;
98 
99 	if (head != tail)
100 		return (head < tail) ?
101 			tail - head : (tail + ring->count - head);
102 	return 0;
103 }
104 
105 /**
106  * ice_check_for_hang_subtask - check for and recover hung queues
107  * @pf: pointer to PF struct
108  */
109 static void ice_check_for_hang_subtask(struct ice_pf *pf)
110 {
111 	struct ice_vsi *vsi = NULL;
112 	struct ice_hw *hw;
113 	unsigned int i;
114 	int packets;
115 	u32 v;
116 
117 	ice_for_each_vsi(pf, v)
118 		if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) {
119 			vsi = pf->vsi[v];
120 			break;
121 		}
122 
123 	if (!vsi || test_bit(ICE_VSI_DOWN, vsi->state))
124 		return;
125 
126 	if (!(vsi->netdev && netif_carrier_ok(vsi->netdev)))
127 		return;
128 
129 	hw = &vsi->back->hw;
130 
131 	ice_for_each_txq(vsi, i) {
132 		struct ice_tx_ring *tx_ring = vsi->tx_rings[i];
133 		struct ice_ring_stats *ring_stats;
134 
135 		if (!tx_ring)
136 			continue;
137 		if (ice_ring_ch_enabled(tx_ring))
138 			continue;
139 
140 		ring_stats = tx_ring->ring_stats;
141 		if (!ring_stats)
142 			continue;
143 
144 		if (tx_ring->desc) {
145 			/* If packet counter has not changed the queue is
146 			 * likely stalled, so force an interrupt for this
147 			 * queue.
148 			 *
149 			 * prev_pkt would be negative if there was no
150 			 * pending work.
151 			 */
152 			packets = ring_stats->stats.pkts & INT_MAX;
153 			if (ring_stats->tx_stats.prev_pkt == packets) {
154 				/* Trigger sw interrupt to revive the queue */
155 				ice_trigger_sw_intr(hw, tx_ring->q_vector);
156 				continue;
157 			}
158 
159 			/* Memory barrier between read of packet count and call
160 			 * to ice_get_tx_pending()
161 			 */
162 			smp_rmb();
163 			ring_stats->tx_stats.prev_pkt =
164 			    ice_get_tx_pending(tx_ring) ? packets : -1;
165 		}
166 	}
167 }
168 
169 /**
170  * ice_init_mac_fltr - Set initial MAC filters
171  * @pf: board private structure
172  *
173  * Set initial set of MAC filters for PF VSI; configure filters for permanent
174  * address and broadcast address. If an error is encountered, netdevice will be
175  * unregistered.
176  */
177 static int ice_init_mac_fltr(struct ice_pf *pf)
178 {
179 	struct ice_vsi *vsi;
180 	u8 *perm_addr;
181 
182 	vsi = ice_get_main_vsi(pf);
183 	if (!vsi)
184 		return -EINVAL;
185 
186 	perm_addr = vsi->port_info->mac.perm_addr;
187 	return ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI);
188 }
189 
190 /**
191  * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced
192  * @netdev: the net device on which the sync is happening
193  * @addr: MAC address to sync
194  *
195  * This is a callback function which is called by the in kernel device sync
196  * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only
197  * populates the tmp_sync_list, which is later used by ice_add_mac to add the
198  * MAC filters from the hardware.
199  */
200 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr)
201 {
202 	struct ice_netdev_priv *np = netdev_priv(netdev);
203 	struct ice_vsi *vsi = np->vsi;
204 
205 	if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr,
206 				     ICE_FWD_TO_VSI))
207 		return -EINVAL;
208 
209 	return 0;
210 }
211 
212 /**
213  * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced
214  * @netdev: the net device on which the unsync is happening
215  * @addr: MAC address to unsync
216  *
217  * This is a callback function which is called by the in kernel device unsync
218  * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only
219  * populates the tmp_unsync_list, which is later used by ice_remove_mac to
220  * delete the MAC filters from the hardware.
221  */
222 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr)
223 {
224 	struct ice_netdev_priv *np = netdev_priv(netdev);
225 	struct ice_vsi *vsi = np->vsi;
226 
227 	/* Under some circumstances, we might receive a request to delete our
228 	 * own device address from our uc list. Because we store the device
229 	 * address in the VSI's MAC filter list, we need to ignore such
230 	 * requests and not delete our device address from this list.
231 	 */
232 	if (ether_addr_equal(addr, netdev->dev_addr))
233 		return 0;
234 
235 	if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr,
236 				     ICE_FWD_TO_VSI))
237 		return -EINVAL;
238 
239 	return 0;
240 }
241 
242 /**
243  * ice_vsi_fltr_changed - check if filter state changed
244  * @vsi: VSI to be checked
245  *
246  * returns true if filter state has changed, false otherwise.
247  */
248 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi)
249 {
250 	return test_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state) ||
251 	       test_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
252 }
253 
254 /**
255  * ice_set_promisc - Enable promiscuous mode for a given PF
256  * @vsi: the VSI being configured
257  * @promisc_m: mask of promiscuous config bits
258  *
259  */
260 static int ice_set_promisc(struct ice_vsi *vsi, u8 promisc_m)
261 {
262 	int status;
263 
264 	if (vsi->type != ICE_VSI_PF)
265 		return 0;
266 
267 	if (ice_vsi_has_non_zero_vlans(vsi)) {
268 		promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX);
269 		status = ice_fltr_set_vlan_vsi_promisc(&vsi->back->hw, vsi,
270 						       promisc_m);
271 	} else {
272 		status = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
273 						  promisc_m, 0);
274 	}
275 	if (status && status != -EEXIST)
276 		return status;
277 
278 	netdev_dbg(vsi->netdev, "set promisc filter bits for VSI %i: 0x%x\n",
279 		   vsi->vsi_num, promisc_m);
280 	return 0;
281 }
282 
283 /**
284  * ice_clear_promisc - Disable promiscuous mode for a given PF
285  * @vsi: the VSI being configured
286  * @promisc_m: mask of promiscuous config bits
287  *
288  */
289 static int ice_clear_promisc(struct ice_vsi *vsi, u8 promisc_m)
290 {
291 	int status;
292 
293 	if (vsi->type != ICE_VSI_PF)
294 		return 0;
295 
296 	if (ice_vsi_has_non_zero_vlans(vsi)) {
297 		promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX);
298 		status = ice_fltr_clear_vlan_vsi_promisc(&vsi->back->hw, vsi,
299 							 promisc_m);
300 	} else {
301 		status = ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
302 						    promisc_m, 0);
303 	}
304 
305 	netdev_dbg(vsi->netdev, "clear promisc filter bits for VSI %i: 0x%x\n",
306 		   vsi->vsi_num, promisc_m);
307 	return status;
308 }
309 
310 /**
311  * ice_vsi_sync_fltr - Update the VSI filter list to the HW
312  * @vsi: ptr to the VSI
313  *
314  * Push any outstanding VSI filter changes through the AdminQ.
315  */
316 static int ice_vsi_sync_fltr(struct ice_vsi *vsi)
317 {
318 	struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
319 	struct device *dev = ice_pf_to_dev(vsi->back);
320 	struct net_device *netdev = vsi->netdev;
321 	bool promisc_forced_on = false;
322 	struct ice_pf *pf = vsi->back;
323 	struct ice_hw *hw = &pf->hw;
324 	u32 changed_flags = 0;
325 	int err;
326 
327 	if (!vsi->netdev)
328 		return -EINVAL;
329 
330 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
331 		usleep_range(1000, 2000);
332 
333 	changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
334 	vsi->current_netdev_flags = vsi->netdev->flags;
335 
336 	INIT_LIST_HEAD(&vsi->tmp_sync_list);
337 	INIT_LIST_HEAD(&vsi->tmp_unsync_list);
338 
339 	if (ice_vsi_fltr_changed(vsi)) {
340 		clear_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
341 		clear_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
342 
343 		/* grab the netdev's addr_list_lock */
344 		netif_addr_lock_bh(netdev);
345 		__dev_uc_sync(netdev, ice_add_mac_to_sync_list,
346 			      ice_add_mac_to_unsync_list);
347 		__dev_mc_sync(netdev, ice_add_mac_to_sync_list,
348 			      ice_add_mac_to_unsync_list);
349 		/* our temp lists are populated. release lock */
350 		netif_addr_unlock_bh(netdev);
351 	}
352 
353 	/* Remove MAC addresses in the unsync list */
354 	err = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list);
355 	ice_fltr_free_list(dev, &vsi->tmp_unsync_list);
356 	if (err) {
357 		netdev_err(netdev, "Failed to delete MAC filters\n");
358 		/* if we failed because of alloc failures, just bail */
359 		if (err == -ENOMEM)
360 			goto out;
361 	}
362 
363 	/* Add MAC addresses in the sync list */
364 	err = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list);
365 	ice_fltr_free_list(dev, &vsi->tmp_sync_list);
366 	/* If filter is added successfully or already exists, do not go into
367 	 * 'if' condition and report it as error. Instead continue processing
368 	 * rest of the function.
369 	 */
370 	if (err && err != -EEXIST) {
371 		netdev_err(netdev, "Failed to add MAC filters\n");
372 		/* If there is no more space for new umac filters, VSI
373 		 * should go into promiscuous mode. There should be some
374 		 * space reserved for promiscuous filters.
375 		 */
376 		if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC &&
377 		    !test_and_set_bit(ICE_FLTR_OVERFLOW_PROMISC,
378 				      vsi->state)) {
379 			promisc_forced_on = true;
380 			netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n",
381 				    vsi->vsi_num);
382 		} else {
383 			goto out;
384 		}
385 	}
386 	err = 0;
387 	/* check for changes in promiscuous modes */
388 	if (changed_flags & IFF_ALLMULTI) {
389 		if (vsi->current_netdev_flags & IFF_ALLMULTI) {
390 			err = ice_set_promisc(vsi, ICE_MCAST_PROMISC_BITS);
391 			if (err) {
392 				vsi->current_netdev_flags &= ~IFF_ALLMULTI;
393 				goto out_promisc;
394 			}
395 		} else {
396 			/* !(vsi->current_netdev_flags & IFF_ALLMULTI) */
397 			err = ice_clear_promisc(vsi, ICE_MCAST_PROMISC_BITS);
398 			if (err) {
399 				vsi->current_netdev_flags |= IFF_ALLMULTI;
400 				goto out_promisc;
401 			}
402 		}
403 	}
404 
405 	if (((changed_flags & IFF_PROMISC) || promisc_forced_on) ||
406 	    test_bit(ICE_VSI_PROMISC_CHANGED, vsi->state)) {
407 		clear_bit(ICE_VSI_PROMISC_CHANGED, vsi->state);
408 		if (vsi->current_netdev_flags & IFF_PROMISC) {
409 			/* Apply Rx filter rule to get traffic from wire */
410 			if (!ice_is_dflt_vsi_in_use(vsi->port_info)) {
411 				err = ice_set_dflt_vsi(vsi);
412 				if (err && err != -EEXIST) {
413 					netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n",
414 						   err, vsi->vsi_num);
415 					vsi->current_netdev_flags &=
416 						~IFF_PROMISC;
417 					goto out_promisc;
418 				}
419 				err = 0;
420 				vlan_ops->dis_rx_filtering(vsi);
421 
422 				/* promiscuous mode implies allmulticast so
423 				 * that VSIs that are in promiscuous mode are
424 				 * subscribed to multicast packets coming to
425 				 * the port
426 				 */
427 				err = ice_set_promisc(vsi,
428 						      ICE_MCAST_PROMISC_BITS);
429 				if (err)
430 					goto out_promisc;
431 			}
432 		} else {
433 			/* Clear Rx filter to remove traffic from wire */
434 			if (ice_is_vsi_dflt_vsi(vsi)) {
435 				err = ice_clear_dflt_vsi(vsi);
436 				if (err) {
437 					netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n",
438 						   err, vsi->vsi_num);
439 					vsi->current_netdev_flags |=
440 						IFF_PROMISC;
441 					goto out_promisc;
442 				}
443 				if (vsi->netdev->features &
444 				    NETIF_F_HW_VLAN_CTAG_FILTER)
445 					vlan_ops->ena_rx_filtering(vsi);
446 			}
447 
448 			/* disable allmulti here, but only if allmulti is not
449 			 * still enabled for the netdev
450 			 */
451 			if (!(vsi->current_netdev_flags & IFF_ALLMULTI)) {
452 				err = ice_clear_promisc(vsi,
453 							ICE_MCAST_PROMISC_BITS);
454 				if (err) {
455 					netdev_err(netdev, "Error %d clearing multicast promiscuous on VSI %i\n",
456 						   err, vsi->vsi_num);
457 				}
458 			}
459 		}
460 	}
461 	goto exit;
462 
463 out_promisc:
464 	set_bit(ICE_VSI_PROMISC_CHANGED, vsi->state);
465 	goto exit;
466 out:
467 	/* if something went wrong then set the changed flag so we try again */
468 	set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
469 	set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
470 exit:
471 	clear_bit(ICE_CFG_BUSY, vsi->state);
472 	return err;
473 }
474 
475 /**
476  * ice_sync_fltr_subtask - Sync the VSI filter list with HW
477  * @pf: board private structure
478  */
479 static void ice_sync_fltr_subtask(struct ice_pf *pf)
480 {
481 	int v;
482 
483 	if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags)))
484 		return;
485 
486 	clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
487 
488 	ice_for_each_vsi(pf, v)
489 		if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) &&
490 		    ice_vsi_sync_fltr(pf->vsi[v])) {
491 			/* come back and try again later */
492 			set_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
493 			break;
494 		}
495 }
496 
497 /**
498  * ice_pf_dis_all_vsi - Pause all VSIs on a PF
499  * @pf: the PF
500  * @locked: is the rtnl_lock already held
501  */
502 static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked)
503 {
504 	int node;
505 	int v;
506 
507 	ice_for_each_vsi(pf, v)
508 		if (pf->vsi[v])
509 			ice_dis_vsi(pf->vsi[v], locked);
510 
511 	for (node = 0; node < ICE_MAX_PF_AGG_NODES; node++)
512 		pf->pf_agg_node[node].num_vsis = 0;
513 
514 	for (node = 0; node < ICE_MAX_VF_AGG_NODES; node++)
515 		pf->vf_agg_node[node].num_vsis = 0;
516 }
517 
518 /**
519  * ice_clear_sw_switch_recipes - clear switch recipes
520  * @pf: board private structure
521  *
522  * Mark switch recipes as not created in sw structures. There are cases where
523  * rules (especially advanced rules) need to be restored, either re-read from
524  * hardware or added again. For example after the reset. 'recp_created' flag
525  * prevents from doing that and need to be cleared upfront.
526  */
527 static void ice_clear_sw_switch_recipes(struct ice_pf *pf)
528 {
529 	struct ice_sw_recipe *recp;
530 	u8 i;
531 
532 	recp = pf->hw.switch_info->recp_list;
533 	for (i = 0; i < ICE_MAX_NUM_RECIPES; i++)
534 		recp[i].recp_created = false;
535 }
536 
537 /**
538  * ice_prepare_for_reset - prep for reset
539  * @pf: board private structure
540  * @reset_type: reset type requested
541  *
542  * Inform or close all dependent features in prep for reset.
543  */
544 static void
545 ice_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
546 {
547 	struct ice_hw *hw = &pf->hw;
548 	struct ice_vsi *vsi;
549 	struct ice_vf *vf;
550 	unsigned int bkt;
551 
552 	dev_dbg(ice_pf_to_dev(pf), "reset_type=%d\n", reset_type);
553 
554 	/* already prepared for reset */
555 	if (test_bit(ICE_PREPARED_FOR_RESET, pf->state))
556 		return;
557 
558 	ice_unplug_aux_dev(pf);
559 
560 	/* Notify VFs of impending reset */
561 	if (ice_check_sq_alive(hw, &hw->mailboxq))
562 		ice_vc_notify_reset(pf);
563 
564 	/* Disable VFs until reset is completed */
565 	mutex_lock(&pf->vfs.table_lock);
566 	ice_for_each_vf(pf, bkt, vf)
567 		ice_set_vf_state_dis(vf);
568 	mutex_unlock(&pf->vfs.table_lock);
569 
570 	if (ice_is_eswitch_mode_switchdev(pf)) {
571 		if (reset_type != ICE_RESET_PFR)
572 			ice_clear_sw_switch_recipes(pf);
573 	}
574 
575 	/* release ADQ specific HW and SW resources */
576 	vsi = ice_get_main_vsi(pf);
577 	if (!vsi)
578 		goto skip;
579 
580 	/* to be on safe side, reset orig_rss_size so that normal flow
581 	 * of deciding rss_size can take precedence
582 	 */
583 	vsi->orig_rss_size = 0;
584 
585 	if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
586 		if (reset_type == ICE_RESET_PFR) {
587 			vsi->old_ena_tc = vsi->all_enatc;
588 			vsi->old_numtc = vsi->all_numtc;
589 		} else {
590 			ice_remove_q_channels(vsi, true);
591 
592 			/* for other reset type, do not support channel rebuild
593 			 * hence reset needed info
594 			 */
595 			vsi->old_ena_tc = 0;
596 			vsi->all_enatc = 0;
597 			vsi->old_numtc = 0;
598 			vsi->all_numtc = 0;
599 			vsi->req_txq = 0;
600 			vsi->req_rxq = 0;
601 			clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
602 			memset(&vsi->mqprio_qopt, 0, sizeof(vsi->mqprio_qopt));
603 		}
604 	}
605 skip:
606 
607 	/* clear SW filtering DB */
608 	ice_clear_hw_tbls(hw);
609 	/* disable the VSIs and their queues that are not already DOWN */
610 	ice_pf_dis_all_vsi(pf, false);
611 
612 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
613 		ice_ptp_prepare_for_reset(pf);
614 
615 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
616 		ice_gnss_exit(pf);
617 
618 	if (hw->port_info)
619 		ice_sched_clear_port(hw->port_info);
620 
621 	ice_shutdown_all_ctrlq(hw);
622 
623 	set_bit(ICE_PREPARED_FOR_RESET, pf->state);
624 }
625 
626 /**
627  * ice_do_reset - Initiate one of many types of resets
628  * @pf: board private structure
629  * @reset_type: reset type requested before this function was called.
630  */
631 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
632 {
633 	struct device *dev = ice_pf_to_dev(pf);
634 	struct ice_hw *hw = &pf->hw;
635 
636 	dev_dbg(dev, "reset_type 0x%x requested\n", reset_type);
637 
638 	ice_prepare_for_reset(pf, reset_type);
639 
640 	/* trigger the reset */
641 	if (ice_reset(hw, reset_type)) {
642 		dev_err(dev, "reset %d failed\n", reset_type);
643 		set_bit(ICE_RESET_FAILED, pf->state);
644 		clear_bit(ICE_RESET_OICR_RECV, pf->state);
645 		clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
646 		clear_bit(ICE_PFR_REQ, pf->state);
647 		clear_bit(ICE_CORER_REQ, pf->state);
648 		clear_bit(ICE_GLOBR_REQ, pf->state);
649 		wake_up(&pf->reset_wait_queue);
650 		return;
651 	}
652 
653 	/* PFR is a bit of a special case because it doesn't result in an OICR
654 	 * interrupt. So for PFR, rebuild after the reset and clear the reset-
655 	 * associated state bits.
656 	 */
657 	if (reset_type == ICE_RESET_PFR) {
658 		pf->pfr_count++;
659 		ice_rebuild(pf, reset_type);
660 		clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
661 		clear_bit(ICE_PFR_REQ, pf->state);
662 		wake_up(&pf->reset_wait_queue);
663 		ice_reset_all_vfs(pf);
664 	}
665 }
666 
667 /**
668  * ice_reset_subtask - Set up for resetting the device and driver
669  * @pf: board private structure
670  */
671 static void ice_reset_subtask(struct ice_pf *pf)
672 {
673 	enum ice_reset_req reset_type = ICE_RESET_INVAL;
674 
675 	/* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an
676 	 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type
677 	 * of reset is pending and sets bits in pf->state indicating the reset
678 	 * type and ICE_RESET_OICR_RECV. So, if the latter bit is set
679 	 * prepare for pending reset if not already (for PF software-initiated
680 	 * global resets the software should already be prepared for it as
681 	 * indicated by ICE_PREPARED_FOR_RESET; for global resets initiated
682 	 * by firmware or software on other PFs, that bit is not set so prepare
683 	 * for the reset now), poll for reset done, rebuild and return.
684 	 */
685 	if (test_bit(ICE_RESET_OICR_RECV, pf->state)) {
686 		/* Perform the largest reset requested */
687 		if (test_and_clear_bit(ICE_CORER_RECV, pf->state))
688 			reset_type = ICE_RESET_CORER;
689 		if (test_and_clear_bit(ICE_GLOBR_RECV, pf->state))
690 			reset_type = ICE_RESET_GLOBR;
691 		if (test_and_clear_bit(ICE_EMPR_RECV, pf->state))
692 			reset_type = ICE_RESET_EMPR;
693 		/* return if no valid reset type requested */
694 		if (reset_type == ICE_RESET_INVAL)
695 			return;
696 		ice_prepare_for_reset(pf, reset_type);
697 
698 		/* make sure we are ready to rebuild */
699 		if (ice_check_reset(&pf->hw)) {
700 			set_bit(ICE_RESET_FAILED, pf->state);
701 		} else {
702 			/* done with reset. start rebuild */
703 			pf->hw.reset_ongoing = false;
704 			ice_rebuild(pf, reset_type);
705 			/* clear bit to resume normal operations, but
706 			 * ICE_NEEDS_RESTART bit is set in case rebuild failed
707 			 */
708 			clear_bit(ICE_RESET_OICR_RECV, pf->state);
709 			clear_bit(ICE_PREPARED_FOR_RESET, pf->state);
710 			clear_bit(ICE_PFR_REQ, pf->state);
711 			clear_bit(ICE_CORER_REQ, pf->state);
712 			clear_bit(ICE_GLOBR_REQ, pf->state);
713 			wake_up(&pf->reset_wait_queue);
714 			ice_reset_all_vfs(pf);
715 		}
716 
717 		return;
718 	}
719 
720 	/* No pending resets to finish processing. Check for new resets */
721 	if (test_bit(ICE_PFR_REQ, pf->state))
722 		reset_type = ICE_RESET_PFR;
723 	if (test_bit(ICE_CORER_REQ, pf->state))
724 		reset_type = ICE_RESET_CORER;
725 	if (test_bit(ICE_GLOBR_REQ, pf->state))
726 		reset_type = ICE_RESET_GLOBR;
727 	/* If no valid reset type requested just return */
728 	if (reset_type == ICE_RESET_INVAL)
729 		return;
730 
731 	/* reset if not already down or busy */
732 	if (!test_bit(ICE_DOWN, pf->state) &&
733 	    !test_bit(ICE_CFG_BUSY, pf->state)) {
734 		ice_do_reset(pf, reset_type);
735 	}
736 }
737 
738 /**
739  * ice_print_topo_conflict - print topology conflict message
740  * @vsi: the VSI whose topology status is being checked
741  */
742 static void ice_print_topo_conflict(struct ice_vsi *vsi)
743 {
744 	switch (vsi->port_info->phy.link_info.topo_media_conflict) {
745 	case ICE_AQ_LINK_TOPO_CONFLICT:
746 	case ICE_AQ_LINK_MEDIA_CONFLICT:
747 	case ICE_AQ_LINK_TOPO_UNREACH_PRT:
748 	case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT:
749 	case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA:
750 		netdev_info(vsi->netdev, "Potential misconfiguration of the Ethernet port detected. If it was not intended, please use the Intel (R) Ethernet Port Configuration Tool to address the issue.\n");
751 		break;
752 	case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA:
753 		if (test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, vsi->back->flags))
754 			netdev_warn(vsi->netdev, "An unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules\n");
755 		else
756 			netdev_err(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
757 		break;
758 	default:
759 		break;
760 	}
761 }
762 
763 /**
764  * ice_print_link_msg - print link up or down message
765  * @vsi: the VSI whose link status is being queried
766  * @isup: boolean for if the link is now up or down
767  */
768 void ice_print_link_msg(struct ice_vsi *vsi, bool isup)
769 {
770 	struct ice_aqc_get_phy_caps_data *caps;
771 	const char *an_advertised;
772 	const char *fec_req;
773 	const char *speed;
774 	const char *fec;
775 	const char *fc;
776 	const char *an;
777 	int status;
778 
779 	if (!vsi)
780 		return;
781 
782 	if (vsi->current_isup == isup)
783 		return;
784 
785 	vsi->current_isup = isup;
786 
787 	if (!isup) {
788 		netdev_info(vsi->netdev, "NIC Link is Down\n");
789 		return;
790 	}
791 
792 	switch (vsi->port_info->phy.link_info.link_speed) {
793 	case ICE_AQ_LINK_SPEED_100GB:
794 		speed = "100 G";
795 		break;
796 	case ICE_AQ_LINK_SPEED_50GB:
797 		speed = "50 G";
798 		break;
799 	case ICE_AQ_LINK_SPEED_40GB:
800 		speed = "40 G";
801 		break;
802 	case ICE_AQ_LINK_SPEED_25GB:
803 		speed = "25 G";
804 		break;
805 	case ICE_AQ_LINK_SPEED_20GB:
806 		speed = "20 G";
807 		break;
808 	case ICE_AQ_LINK_SPEED_10GB:
809 		speed = "10 G";
810 		break;
811 	case ICE_AQ_LINK_SPEED_5GB:
812 		speed = "5 G";
813 		break;
814 	case ICE_AQ_LINK_SPEED_2500MB:
815 		speed = "2.5 G";
816 		break;
817 	case ICE_AQ_LINK_SPEED_1000MB:
818 		speed = "1 G";
819 		break;
820 	case ICE_AQ_LINK_SPEED_100MB:
821 		speed = "100 M";
822 		break;
823 	default:
824 		speed = "Unknown ";
825 		break;
826 	}
827 
828 	switch (vsi->port_info->fc.current_mode) {
829 	case ICE_FC_FULL:
830 		fc = "Rx/Tx";
831 		break;
832 	case ICE_FC_TX_PAUSE:
833 		fc = "Tx";
834 		break;
835 	case ICE_FC_RX_PAUSE:
836 		fc = "Rx";
837 		break;
838 	case ICE_FC_NONE:
839 		fc = "None";
840 		break;
841 	default:
842 		fc = "Unknown";
843 		break;
844 	}
845 
846 	/* Get FEC mode based on negotiated link info */
847 	switch (vsi->port_info->phy.link_info.fec_info) {
848 	case ICE_AQ_LINK_25G_RS_528_FEC_EN:
849 	case ICE_AQ_LINK_25G_RS_544_FEC_EN:
850 		fec = "RS-FEC";
851 		break;
852 	case ICE_AQ_LINK_25G_KR_FEC_EN:
853 		fec = "FC-FEC/BASE-R";
854 		break;
855 	default:
856 		fec = "NONE";
857 		break;
858 	}
859 
860 	/* check if autoneg completed, might be false due to not supported */
861 	if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED)
862 		an = "True";
863 	else
864 		an = "False";
865 
866 	/* Get FEC mode requested based on PHY caps last SW configuration */
867 	caps = kzalloc(sizeof(*caps), GFP_KERNEL);
868 	if (!caps) {
869 		fec_req = "Unknown";
870 		an_advertised = "Unknown";
871 		goto done;
872 	}
873 
874 	status = ice_aq_get_phy_caps(vsi->port_info, false,
875 				     ICE_AQC_REPORT_ACTIVE_CFG, caps, NULL);
876 	if (status)
877 		netdev_info(vsi->netdev, "Get phy capability failed.\n");
878 
879 	an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off";
880 
881 	if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ ||
882 	    caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ)
883 		fec_req = "RS-FEC";
884 	else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ ||
885 		 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ)
886 		fec_req = "FC-FEC/BASE-R";
887 	else
888 		fec_req = "NONE";
889 
890 	kfree(caps);
891 
892 done:
893 	netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n",
894 		    speed, fec_req, fec, an_advertised, an, fc);
895 	ice_print_topo_conflict(vsi);
896 }
897 
898 /**
899  * ice_vsi_link_event - update the VSI's netdev
900  * @vsi: the VSI on which the link event occurred
901  * @link_up: whether or not the VSI needs to be set up or down
902  */
903 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up)
904 {
905 	if (!vsi)
906 		return;
907 
908 	if (test_bit(ICE_VSI_DOWN, vsi->state) || !vsi->netdev)
909 		return;
910 
911 	if (vsi->type == ICE_VSI_PF) {
912 		if (link_up == netif_carrier_ok(vsi->netdev))
913 			return;
914 
915 		if (link_up) {
916 			netif_carrier_on(vsi->netdev);
917 			netif_tx_wake_all_queues(vsi->netdev);
918 		} else {
919 			netif_carrier_off(vsi->netdev);
920 			netif_tx_stop_all_queues(vsi->netdev);
921 		}
922 	}
923 }
924 
925 /**
926  * ice_set_dflt_mib - send a default config MIB to the FW
927  * @pf: private PF struct
928  *
929  * This function sends a default configuration MIB to the FW.
930  *
931  * If this function errors out at any point, the driver is still able to
932  * function.  The main impact is that LFC may not operate as expected.
933  * Therefore an error state in this function should be treated with a DBG
934  * message and continue on with driver rebuild/reenable.
935  */
936 static void ice_set_dflt_mib(struct ice_pf *pf)
937 {
938 	struct device *dev = ice_pf_to_dev(pf);
939 	u8 mib_type, *buf, *lldpmib = NULL;
940 	u16 len, typelen, offset = 0;
941 	struct ice_lldp_org_tlv *tlv;
942 	struct ice_hw *hw = &pf->hw;
943 	u32 ouisubtype;
944 
945 	mib_type = SET_LOCAL_MIB_TYPE_LOCAL_MIB;
946 	lldpmib = kzalloc(ICE_LLDPDU_SIZE, GFP_KERNEL);
947 	if (!lldpmib) {
948 		dev_dbg(dev, "%s Failed to allocate MIB memory\n",
949 			__func__);
950 		return;
951 	}
952 
953 	/* Add ETS CFG TLV */
954 	tlv = (struct ice_lldp_org_tlv *)lldpmib;
955 	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
956 		   ICE_IEEE_ETS_TLV_LEN);
957 	tlv->typelen = htons(typelen);
958 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
959 		      ICE_IEEE_SUBTYPE_ETS_CFG);
960 	tlv->ouisubtype = htonl(ouisubtype);
961 
962 	buf = tlv->tlvinfo;
963 	buf[0] = 0;
964 
965 	/* ETS CFG all UPs map to TC 0. Next 4 (1 - 4) Octets = 0.
966 	 * Octets 5 - 12 are BW values, set octet 5 to 100% BW.
967 	 * Octets 13 - 20 are TSA values - leave as zeros
968 	 */
969 	buf[5] = 0x64;
970 	len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S;
971 	offset += len + 2;
972 	tlv = (struct ice_lldp_org_tlv *)
973 		((char *)tlv + sizeof(tlv->typelen) + len);
974 
975 	/* Add ETS REC TLV */
976 	buf = tlv->tlvinfo;
977 	tlv->typelen = htons(typelen);
978 
979 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
980 		      ICE_IEEE_SUBTYPE_ETS_REC);
981 	tlv->ouisubtype = htonl(ouisubtype);
982 
983 	/* First octet of buf is reserved
984 	 * Octets 1 - 4 map UP to TC - all UPs map to zero
985 	 * Octets 5 - 12 are BW values - set TC 0 to 100%.
986 	 * Octets 13 - 20 are TSA value - leave as zeros
987 	 */
988 	buf[5] = 0x64;
989 	offset += len + 2;
990 	tlv = (struct ice_lldp_org_tlv *)
991 		((char *)tlv + sizeof(tlv->typelen) + len);
992 
993 	/* Add PFC CFG TLV */
994 	typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) |
995 		   ICE_IEEE_PFC_TLV_LEN);
996 	tlv->typelen = htons(typelen);
997 
998 	ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) |
999 		      ICE_IEEE_SUBTYPE_PFC_CFG);
1000 	tlv->ouisubtype = htonl(ouisubtype);
1001 
1002 	/* Octet 1 left as all zeros - PFC disabled */
1003 	buf[0] = 0x08;
1004 	len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S;
1005 	offset += len + 2;
1006 
1007 	if (ice_aq_set_lldp_mib(hw, mib_type, (void *)lldpmib, offset, NULL))
1008 		dev_dbg(dev, "%s Failed to set default LLDP MIB\n", __func__);
1009 
1010 	kfree(lldpmib);
1011 }
1012 
1013 /**
1014  * ice_check_phy_fw_load - check if PHY FW load failed
1015  * @pf: pointer to PF struct
1016  * @link_cfg_err: bitmap from the link info structure
1017  *
1018  * check if external PHY FW load failed and print an error message if it did
1019  */
1020 static void ice_check_phy_fw_load(struct ice_pf *pf, u8 link_cfg_err)
1021 {
1022 	if (!(link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE)) {
1023 		clear_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags);
1024 		return;
1025 	}
1026 
1027 	if (test_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags))
1028 		return;
1029 
1030 	if (link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE) {
1031 		dev_err(ice_pf_to_dev(pf), "Device failed to load the FW for the external PHY. Please download and install the latest NVM for your device and try again\n");
1032 		set_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags);
1033 	}
1034 }
1035 
1036 /**
1037  * ice_check_module_power
1038  * @pf: pointer to PF struct
1039  * @link_cfg_err: bitmap from the link info structure
1040  *
1041  * check module power level returned by a previous call to aq_get_link_info
1042  * and print error messages if module power level is not supported
1043  */
1044 static void ice_check_module_power(struct ice_pf *pf, u8 link_cfg_err)
1045 {
1046 	/* if module power level is supported, clear the flag */
1047 	if (!(link_cfg_err & (ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT |
1048 			      ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED))) {
1049 		clear_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1050 		return;
1051 	}
1052 
1053 	/* if ICE_FLAG_MOD_POWER_UNSUPPORTED was previously set and the
1054 	 * above block didn't clear this bit, there's nothing to do
1055 	 */
1056 	if (test_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags))
1057 		return;
1058 
1059 	if (link_cfg_err & ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT) {
1060 		dev_err(ice_pf_to_dev(pf), "The installed module is incompatible with the device's NVM image. Cannot start link\n");
1061 		set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1062 	} else if (link_cfg_err & ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED) {
1063 		dev_err(ice_pf_to_dev(pf), "The module's power requirements exceed the device's power supply. Cannot start link\n");
1064 		set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags);
1065 	}
1066 }
1067 
1068 /**
1069  * ice_check_link_cfg_err - check if link configuration failed
1070  * @pf: pointer to the PF struct
1071  * @link_cfg_err: bitmap from the link info structure
1072  *
1073  * print if any link configuration failure happens due to the value in the
1074  * link_cfg_err parameter in the link info structure
1075  */
1076 static void ice_check_link_cfg_err(struct ice_pf *pf, u8 link_cfg_err)
1077 {
1078 	ice_check_module_power(pf, link_cfg_err);
1079 	ice_check_phy_fw_load(pf, link_cfg_err);
1080 }
1081 
1082 /**
1083  * ice_link_event - process the link event
1084  * @pf: PF that the link event is associated with
1085  * @pi: port_info for the port that the link event is associated with
1086  * @link_up: true if the physical link is up and false if it is down
1087  * @link_speed: current link speed received from the link event
1088  *
1089  * Returns 0 on success and negative on failure
1090  */
1091 static int
1092 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up,
1093 	       u16 link_speed)
1094 {
1095 	struct device *dev = ice_pf_to_dev(pf);
1096 	struct ice_phy_info *phy_info;
1097 	struct ice_vsi *vsi;
1098 	u16 old_link_speed;
1099 	bool old_link;
1100 	int status;
1101 
1102 	phy_info = &pi->phy;
1103 	phy_info->link_info_old = phy_info->link_info;
1104 
1105 	old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP);
1106 	old_link_speed = phy_info->link_info_old.link_speed;
1107 
1108 	/* update the link info structures and re-enable link events,
1109 	 * don't bail on failure due to other book keeping needed
1110 	 */
1111 	status = ice_update_link_info(pi);
1112 	if (status)
1113 		dev_dbg(dev, "Failed to update link status on port %d, err %d aq_err %s\n",
1114 			pi->lport, status,
1115 			ice_aq_str(pi->hw->adminq.sq_last_status));
1116 
1117 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
1118 
1119 	/* Check if the link state is up after updating link info, and treat
1120 	 * this event as an UP event since the link is actually UP now.
1121 	 */
1122 	if (phy_info->link_info.link_info & ICE_AQ_LINK_UP)
1123 		link_up = true;
1124 
1125 	vsi = ice_get_main_vsi(pf);
1126 	if (!vsi || !vsi->port_info)
1127 		return -EINVAL;
1128 
1129 	/* turn off PHY if media was removed */
1130 	if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) &&
1131 	    !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) {
1132 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
1133 		ice_set_link(vsi, false);
1134 	}
1135 
1136 	/* if the old link up/down and speed is the same as the new */
1137 	if (link_up == old_link && link_speed == old_link_speed)
1138 		return 0;
1139 
1140 	ice_ptp_link_change(pf, pf->hw.pf_id, link_up);
1141 
1142 	if (ice_is_dcb_active(pf)) {
1143 		if (test_bit(ICE_FLAG_DCB_ENA, pf->flags))
1144 			ice_dcb_rebuild(pf);
1145 	} else {
1146 		if (link_up)
1147 			ice_set_dflt_mib(pf);
1148 	}
1149 	ice_vsi_link_event(vsi, link_up);
1150 	ice_print_link_msg(vsi, link_up);
1151 
1152 	ice_vc_notify_link_state(pf);
1153 
1154 	return 0;
1155 }
1156 
1157 /**
1158  * ice_watchdog_subtask - periodic tasks not using event driven scheduling
1159  * @pf: board private structure
1160  */
1161 static void ice_watchdog_subtask(struct ice_pf *pf)
1162 {
1163 	int i;
1164 
1165 	/* if interface is down do nothing */
1166 	if (test_bit(ICE_DOWN, pf->state) ||
1167 	    test_bit(ICE_CFG_BUSY, pf->state))
1168 		return;
1169 
1170 	/* make sure we don't do these things too often */
1171 	if (time_before(jiffies,
1172 			pf->serv_tmr_prev + pf->serv_tmr_period))
1173 		return;
1174 
1175 	pf->serv_tmr_prev = jiffies;
1176 
1177 	/* Update the stats for active netdevs so the network stack
1178 	 * can look at updated numbers whenever it cares to
1179 	 */
1180 	ice_update_pf_stats(pf);
1181 	ice_for_each_vsi(pf, i)
1182 		if (pf->vsi[i] && pf->vsi[i]->netdev)
1183 			ice_update_vsi_stats(pf->vsi[i]);
1184 }
1185 
1186 /**
1187  * ice_init_link_events - enable/initialize link events
1188  * @pi: pointer to the port_info instance
1189  *
1190  * Returns -EIO on failure, 0 on success
1191  */
1192 static int ice_init_link_events(struct ice_port_info *pi)
1193 {
1194 	u16 mask;
1195 
1196 	mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA |
1197 		       ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL |
1198 		       ICE_AQ_LINK_EVENT_PHY_FW_LOAD_FAIL));
1199 
1200 	if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) {
1201 		dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n",
1202 			pi->lport);
1203 		return -EIO;
1204 	}
1205 
1206 	if (ice_aq_get_link_info(pi, true, NULL, NULL)) {
1207 		dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n",
1208 			pi->lport);
1209 		return -EIO;
1210 	}
1211 
1212 	return 0;
1213 }
1214 
1215 /**
1216  * ice_handle_link_event - handle link event via ARQ
1217  * @pf: PF that the link event is associated with
1218  * @event: event structure containing link status info
1219  */
1220 static int
1221 ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event)
1222 {
1223 	struct ice_aqc_get_link_status_data *link_data;
1224 	struct ice_port_info *port_info;
1225 	int status;
1226 
1227 	link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf;
1228 	port_info = pf->hw.port_info;
1229 	if (!port_info)
1230 		return -EINVAL;
1231 
1232 	status = ice_link_event(pf, port_info,
1233 				!!(link_data->link_info & ICE_AQ_LINK_UP),
1234 				le16_to_cpu(link_data->link_speed));
1235 	if (status)
1236 		dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n",
1237 			status);
1238 
1239 	return status;
1240 }
1241 
1242 enum ice_aq_task_state {
1243 	ICE_AQ_TASK_WAITING = 0,
1244 	ICE_AQ_TASK_COMPLETE,
1245 	ICE_AQ_TASK_CANCELED,
1246 };
1247 
1248 struct ice_aq_task {
1249 	struct hlist_node entry;
1250 
1251 	u16 opcode;
1252 	struct ice_rq_event_info *event;
1253 	enum ice_aq_task_state state;
1254 };
1255 
1256 /**
1257  * ice_aq_wait_for_event - Wait for an AdminQ event from firmware
1258  * @pf: pointer to the PF private structure
1259  * @opcode: the opcode to wait for
1260  * @timeout: how long to wait, in jiffies
1261  * @event: storage for the event info
1262  *
1263  * Waits for a specific AdminQ completion event on the ARQ for a given PF. The
1264  * current thread will be put to sleep until the specified event occurs or
1265  * until the given timeout is reached.
1266  *
1267  * To obtain only the descriptor contents, pass an event without an allocated
1268  * msg_buf. If the complete data buffer is desired, allocate the
1269  * event->msg_buf with enough space ahead of time.
1270  *
1271  * Returns: zero on success, or a negative error code on failure.
1272  */
1273 int ice_aq_wait_for_event(struct ice_pf *pf, u16 opcode, unsigned long timeout,
1274 			  struct ice_rq_event_info *event)
1275 {
1276 	struct device *dev = ice_pf_to_dev(pf);
1277 	struct ice_aq_task *task;
1278 	unsigned long start;
1279 	long ret;
1280 	int err;
1281 
1282 	task = kzalloc(sizeof(*task), GFP_KERNEL);
1283 	if (!task)
1284 		return -ENOMEM;
1285 
1286 	INIT_HLIST_NODE(&task->entry);
1287 	task->opcode = opcode;
1288 	task->event = event;
1289 	task->state = ICE_AQ_TASK_WAITING;
1290 
1291 	spin_lock_bh(&pf->aq_wait_lock);
1292 	hlist_add_head(&task->entry, &pf->aq_wait_list);
1293 	spin_unlock_bh(&pf->aq_wait_lock);
1294 
1295 	start = jiffies;
1296 
1297 	ret = wait_event_interruptible_timeout(pf->aq_wait_queue, task->state,
1298 					       timeout);
1299 	switch (task->state) {
1300 	case ICE_AQ_TASK_WAITING:
1301 		err = ret < 0 ? ret : -ETIMEDOUT;
1302 		break;
1303 	case ICE_AQ_TASK_CANCELED:
1304 		err = ret < 0 ? ret : -ECANCELED;
1305 		break;
1306 	case ICE_AQ_TASK_COMPLETE:
1307 		err = ret < 0 ? ret : 0;
1308 		break;
1309 	default:
1310 		WARN(1, "Unexpected AdminQ wait task state %u", task->state);
1311 		err = -EINVAL;
1312 		break;
1313 	}
1314 
1315 	dev_dbg(dev, "Waited %u msecs (max %u msecs) for firmware response to op 0x%04x\n",
1316 		jiffies_to_msecs(jiffies - start),
1317 		jiffies_to_msecs(timeout),
1318 		opcode);
1319 
1320 	spin_lock_bh(&pf->aq_wait_lock);
1321 	hlist_del(&task->entry);
1322 	spin_unlock_bh(&pf->aq_wait_lock);
1323 	kfree(task);
1324 
1325 	return err;
1326 }
1327 
1328 /**
1329  * ice_aq_check_events - Check if any thread is waiting for an AdminQ event
1330  * @pf: pointer to the PF private structure
1331  * @opcode: the opcode of the event
1332  * @event: the event to check
1333  *
1334  * Loops over the current list of pending threads waiting for an AdminQ event.
1335  * For each matching task, copy the contents of the event into the task
1336  * structure and wake up the thread.
1337  *
1338  * If multiple threads wait for the same opcode, they will all be woken up.
1339  *
1340  * Note that event->msg_buf will only be duplicated if the event has a buffer
1341  * with enough space already allocated. Otherwise, only the descriptor and
1342  * message length will be copied.
1343  *
1344  * Returns: true if an event was found, false otherwise
1345  */
1346 static void ice_aq_check_events(struct ice_pf *pf, u16 opcode,
1347 				struct ice_rq_event_info *event)
1348 {
1349 	struct ice_aq_task *task;
1350 	bool found = false;
1351 
1352 	spin_lock_bh(&pf->aq_wait_lock);
1353 	hlist_for_each_entry(task, &pf->aq_wait_list, entry) {
1354 		if (task->state || task->opcode != opcode)
1355 			continue;
1356 
1357 		memcpy(&task->event->desc, &event->desc, sizeof(event->desc));
1358 		task->event->msg_len = event->msg_len;
1359 
1360 		/* Only copy the data buffer if a destination was set */
1361 		if (task->event->msg_buf &&
1362 		    task->event->buf_len > event->buf_len) {
1363 			memcpy(task->event->msg_buf, event->msg_buf,
1364 			       event->buf_len);
1365 			task->event->buf_len = event->buf_len;
1366 		}
1367 
1368 		task->state = ICE_AQ_TASK_COMPLETE;
1369 		found = true;
1370 	}
1371 	spin_unlock_bh(&pf->aq_wait_lock);
1372 
1373 	if (found)
1374 		wake_up(&pf->aq_wait_queue);
1375 }
1376 
1377 /**
1378  * ice_aq_cancel_waiting_tasks - Immediately cancel all waiting tasks
1379  * @pf: the PF private structure
1380  *
1381  * Set all waiting tasks to ICE_AQ_TASK_CANCELED, and wake up their threads.
1382  * This will then cause ice_aq_wait_for_event to exit with -ECANCELED.
1383  */
1384 static void ice_aq_cancel_waiting_tasks(struct ice_pf *pf)
1385 {
1386 	struct ice_aq_task *task;
1387 
1388 	spin_lock_bh(&pf->aq_wait_lock);
1389 	hlist_for_each_entry(task, &pf->aq_wait_list, entry)
1390 		task->state = ICE_AQ_TASK_CANCELED;
1391 	spin_unlock_bh(&pf->aq_wait_lock);
1392 
1393 	wake_up(&pf->aq_wait_queue);
1394 }
1395 
1396 #define ICE_MBX_OVERFLOW_WATERMARK 64
1397 
1398 /**
1399  * __ice_clean_ctrlq - helper function to clean controlq rings
1400  * @pf: ptr to struct ice_pf
1401  * @q_type: specific Control queue type
1402  */
1403 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
1404 {
1405 	struct device *dev = ice_pf_to_dev(pf);
1406 	struct ice_rq_event_info event;
1407 	struct ice_hw *hw = &pf->hw;
1408 	struct ice_ctl_q_info *cq;
1409 	u16 pending, i = 0;
1410 	const char *qtype;
1411 	u32 oldval, val;
1412 
1413 	/* Do not clean control queue if/when PF reset fails */
1414 	if (test_bit(ICE_RESET_FAILED, pf->state))
1415 		return 0;
1416 
1417 	switch (q_type) {
1418 	case ICE_CTL_Q_ADMIN:
1419 		cq = &hw->adminq;
1420 		qtype = "Admin";
1421 		break;
1422 	case ICE_CTL_Q_SB:
1423 		cq = &hw->sbq;
1424 		qtype = "Sideband";
1425 		break;
1426 	case ICE_CTL_Q_MAILBOX:
1427 		cq = &hw->mailboxq;
1428 		qtype = "Mailbox";
1429 		/* we are going to try to detect a malicious VF, so set the
1430 		 * state to begin detection
1431 		 */
1432 		hw->mbx_snapshot.mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT;
1433 		break;
1434 	default:
1435 		dev_warn(dev, "Unknown control queue type 0x%x\n", q_type);
1436 		return 0;
1437 	}
1438 
1439 	/* check for error indications - PF_xx_AxQLEN register layout for
1440 	 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN.
1441 	 */
1442 	val = rd32(hw, cq->rq.len);
1443 	if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
1444 		   PF_FW_ARQLEN_ARQCRIT_M)) {
1445 		oldval = val;
1446 		if (val & PF_FW_ARQLEN_ARQVFE_M)
1447 			dev_dbg(dev, "%s Receive Queue VF Error detected\n",
1448 				qtype);
1449 		if (val & PF_FW_ARQLEN_ARQOVFL_M) {
1450 			dev_dbg(dev, "%s Receive Queue Overflow Error detected\n",
1451 				qtype);
1452 		}
1453 		if (val & PF_FW_ARQLEN_ARQCRIT_M)
1454 			dev_dbg(dev, "%s Receive Queue Critical Error detected\n",
1455 				qtype);
1456 		val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
1457 			 PF_FW_ARQLEN_ARQCRIT_M);
1458 		if (oldval != val)
1459 			wr32(hw, cq->rq.len, val);
1460 	}
1461 
1462 	val = rd32(hw, cq->sq.len);
1463 	if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
1464 		   PF_FW_ATQLEN_ATQCRIT_M)) {
1465 		oldval = val;
1466 		if (val & PF_FW_ATQLEN_ATQVFE_M)
1467 			dev_dbg(dev, "%s Send Queue VF Error detected\n",
1468 				qtype);
1469 		if (val & PF_FW_ATQLEN_ATQOVFL_M) {
1470 			dev_dbg(dev, "%s Send Queue Overflow Error detected\n",
1471 				qtype);
1472 		}
1473 		if (val & PF_FW_ATQLEN_ATQCRIT_M)
1474 			dev_dbg(dev, "%s Send Queue Critical Error detected\n",
1475 				qtype);
1476 		val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
1477 			 PF_FW_ATQLEN_ATQCRIT_M);
1478 		if (oldval != val)
1479 			wr32(hw, cq->sq.len, val);
1480 	}
1481 
1482 	event.buf_len = cq->rq_buf_size;
1483 	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
1484 	if (!event.msg_buf)
1485 		return 0;
1486 
1487 	do {
1488 		struct ice_mbx_data data = {};
1489 		u16 opcode;
1490 		int ret;
1491 
1492 		ret = ice_clean_rq_elem(hw, cq, &event, &pending);
1493 		if (ret == -EALREADY)
1494 			break;
1495 		if (ret) {
1496 			dev_err(dev, "%s Receive Queue event error %d\n", qtype,
1497 				ret);
1498 			break;
1499 		}
1500 
1501 		opcode = le16_to_cpu(event.desc.opcode);
1502 
1503 		/* Notify any thread that might be waiting for this event */
1504 		ice_aq_check_events(pf, opcode, &event);
1505 
1506 		switch (opcode) {
1507 		case ice_aqc_opc_get_link_status:
1508 			if (ice_handle_link_event(pf, &event))
1509 				dev_err(dev, "Could not handle link event\n");
1510 			break;
1511 		case ice_aqc_opc_event_lan_overflow:
1512 			ice_vf_lan_overflow_event(pf, &event);
1513 			break;
1514 		case ice_mbx_opc_send_msg_to_pf:
1515 			data.num_msg_proc = i;
1516 			data.num_pending_arq = pending;
1517 			data.max_num_msgs_mbx = hw->mailboxq.num_rq_entries;
1518 			data.async_watermark_val = ICE_MBX_OVERFLOW_WATERMARK;
1519 
1520 			ice_vc_process_vf_msg(pf, &event, &data);
1521 			break;
1522 		case ice_aqc_opc_fw_logging:
1523 			ice_output_fw_log(hw, &event.desc, event.msg_buf);
1524 			break;
1525 		case ice_aqc_opc_lldp_set_mib_change:
1526 			ice_dcb_process_lldp_set_mib_change(pf, &event);
1527 			break;
1528 		default:
1529 			dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n",
1530 				qtype, opcode);
1531 			break;
1532 		}
1533 	} while (pending && (i++ < ICE_DFLT_IRQ_WORK));
1534 
1535 	kfree(event.msg_buf);
1536 
1537 	return pending && (i == ICE_DFLT_IRQ_WORK);
1538 }
1539 
1540 /**
1541  * ice_ctrlq_pending - check if there is a difference between ntc and ntu
1542  * @hw: pointer to hardware info
1543  * @cq: control queue information
1544  *
1545  * returns true if there are pending messages in a queue, false if there aren't
1546  */
1547 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq)
1548 {
1549 	u16 ntu;
1550 
1551 	ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask);
1552 	return cq->rq.next_to_clean != ntu;
1553 }
1554 
1555 /**
1556  * ice_clean_adminq_subtask - clean the AdminQ rings
1557  * @pf: board private structure
1558  */
1559 static void ice_clean_adminq_subtask(struct ice_pf *pf)
1560 {
1561 	struct ice_hw *hw = &pf->hw;
1562 
1563 	if (!test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state))
1564 		return;
1565 
1566 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN))
1567 		return;
1568 
1569 	clear_bit(ICE_ADMINQ_EVENT_PENDING, pf->state);
1570 
1571 	/* There might be a situation where new messages arrive to a control
1572 	 * queue between processing the last message and clearing the
1573 	 * EVENT_PENDING bit. So before exiting, check queue head again (using
1574 	 * ice_ctrlq_pending) and process new messages if any.
1575 	 */
1576 	if (ice_ctrlq_pending(hw, &hw->adminq))
1577 		__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN);
1578 
1579 	ice_flush(hw);
1580 }
1581 
1582 /**
1583  * ice_clean_mailboxq_subtask - clean the MailboxQ rings
1584  * @pf: board private structure
1585  */
1586 static void ice_clean_mailboxq_subtask(struct ice_pf *pf)
1587 {
1588 	struct ice_hw *hw = &pf->hw;
1589 
1590 	if (!test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state))
1591 		return;
1592 
1593 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX))
1594 		return;
1595 
1596 	clear_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state);
1597 
1598 	if (ice_ctrlq_pending(hw, &hw->mailboxq))
1599 		__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX);
1600 
1601 	ice_flush(hw);
1602 }
1603 
1604 /**
1605  * ice_clean_sbq_subtask - clean the Sideband Queue rings
1606  * @pf: board private structure
1607  */
1608 static void ice_clean_sbq_subtask(struct ice_pf *pf)
1609 {
1610 	struct ice_hw *hw = &pf->hw;
1611 
1612 	/* Nothing to do here if sideband queue is not supported */
1613 	if (!ice_is_sbq_supported(hw)) {
1614 		clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
1615 		return;
1616 	}
1617 
1618 	if (!test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state))
1619 		return;
1620 
1621 	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_SB))
1622 		return;
1623 
1624 	clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
1625 
1626 	if (ice_ctrlq_pending(hw, &hw->sbq))
1627 		__ice_clean_ctrlq(pf, ICE_CTL_Q_SB);
1628 
1629 	ice_flush(hw);
1630 }
1631 
1632 /**
1633  * ice_service_task_schedule - schedule the service task to wake up
1634  * @pf: board private structure
1635  *
1636  * If not already scheduled, this puts the task into the work queue.
1637  */
1638 void ice_service_task_schedule(struct ice_pf *pf)
1639 {
1640 	if (!test_bit(ICE_SERVICE_DIS, pf->state) &&
1641 	    !test_and_set_bit(ICE_SERVICE_SCHED, pf->state) &&
1642 	    !test_bit(ICE_NEEDS_RESTART, pf->state))
1643 		queue_work(ice_wq, &pf->serv_task);
1644 }
1645 
1646 /**
1647  * ice_service_task_complete - finish up the service task
1648  * @pf: board private structure
1649  */
1650 static void ice_service_task_complete(struct ice_pf *pf)
1651 {
1652 	WARN_ON(!test_bit(ICE_SERVICE_SCHED, pf->state));
1653 
1654 	/* force memory (pf->state) to sync before next service task */
1655 	smp_mb__before_atomic();
1656 	clear_bit(ICE_SERVICE_SCHED, pf->state);
1657 }
1658 
1659 /**
1660  * ice_service_task_stop - stop service task and cancel works
1661  * @pf: board private structure
1662  *
1663  * Return 0 if the ICE_SERVICE_DIS bit was not already set,
1664  * 1 otherwise.
1665  */
1666 static int ice_service_task_stop(struct ice_pf *pf)
1667 {
1668 	int ret;
1669 
1670 	ret = test_and_set_bit(ICE_SERVICE_DIS, pf->state);
1671 
1672 	if (pf->serv_tmr.function)
1673 		del_timer_sync(&pf->serv_tmr);
1674 	if (pf->serv_task.func)
1675 		cancel_work_sync(&pf->serv_task);
1676 
1677 	clear_bit(ICE_SERVICE_SCHED, pf->state);
1678 	return ret;
1679 }
1680 
1681 /**
1682  * ice_service_task_restart - restart service task and schedule works
1683  * @pf: board private structure
1684  *
1685  * This function is needed for suspend and resume works (e.g WoL scenario)
1686  */
1687 static void ice_service_task_restart(struct ice_pf *pf)
1688 {
1689 	clear_bit(ICE_SERVICE_DIS, pf->state);
1690 	ice_service_task_schedule(pf);
1691 }
1692 
1693 /**
1694  * ice_service_timer - timer callback to schedule service task
1695  * @t: pointer to timer_list
1696  */
1697 static void ice_service_timer(struct timer_list *t)
1698 {
1699 	struct ice_pf *pf = from_timer(pf, t, serv_tmr);
1700 
1701 	mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies));
1702 	ice_service_task_schedule(pf);
1703 }
1704 
1705 /**
1706  * ice_handle_mdd_event - handle malicious driver detect event
1707  * @pf: pointer to the PF structure
1708  *
1709  * Called from service task. OICR interrupt handler indicates MDD event.
1710  * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log
1711  * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events
1712  * disable the queue, the PF can be configured to reset the VF using ethtool
1713  * private flag mdd-auto-reset-vf.
1714  */
1715 static void ice_handle_mdd_event(struct ice_pf *pf)
1716 {
1717 	struct device *dev = ice_pf_to_dev(pf);
1718 	struct ice_hw *hw = &pf->hw;
1719 	struct ice_vf *vf;
1720 	unsigned int bkt;
1721 	u32 reg;
1722 
1723 	if (!test_and_clear_bit(ICE_MDD_EVENT_PENDING, pf->state)) {
1724 		/* Since the VF MDD event logging is rate limited, check if
1725 		 * there are pending MDD events.
1726 		 */
1727 		ice_print_vfs_mdd_events(pf);
1728 		return;
1729 	}
1730 
1731 	/* find what triggered an MDD event */
1732 	reg = rd32(hw, GL_MDET_TX_PQM);
1733 	if (reg & GL_MDET_TX_PQM_VALID_M) {
1734 		u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >>
1735 				GL_MDET_TX_PQM_PF_NUM_S;
1736 		u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >>
1737 				GL_MDET_TX_PQM_VF_NUM_S;
1738 		u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >>
1739 				GL_MDET_TX_PQM_MAL_TYPE_S;
1740 		u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >>
1741 				GL_MDET_TX_PQM_QNUM_S);
1742 
1743 		if (netif_msg_tx_err(pf))
1744 			dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",
1745 				 event, queue, pf_num, vf_num);
1746 		wr32(hw, GL_MDET_TX_PQM, 0xffffffff);
1747 	}
1748 
1749 	reg = rd32(hw, GL_MDET_TX_TCLAN);
1750 	if (reg & GL_MDET_TX_TCLAN_VALID_M) {
1751 		u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >>
1752 				GL_MDET_TX_TCLAN_PF_NUM_S;
1753 		u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >>
1754 				GL_MDET_TX_TCLAN_VF_NUM_S;
1755 		u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >>
1756 				GL_MDET_TX_TCLAN_MAL_TYPE_S;
1757 		u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >>
1758 				GL_MDET_TX_TCLAN_QNUM_S);
1759 
1760 		if (netif_msg_tx_err(pf))
1761 			dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n",
1762 				 event, queue, pf_num, vf_num);
1763 		wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff);
1764 	}
1765 
1766 	reg = rd32(hw, GL_MDET_RX);
1767 	if (reg & GL_MDET_RX_VALID_M) {
1768 		u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >>
1769 				GL_MDET_RX_PF_NUM_S;
1770 		u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >>
1771 				GL_MDET_RX_VF_NUM_S;
1772 		u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >>
1773 				GL_MDET_RX_MAL_TYPE_S;
1774 		u16 queue = ((reg & GL_MDET_RX_QNUM_M) >>
1775 				GL_MDET_RX_QNUM_S);
1776 
1777 		if (netif_msg_rx_err(pf))
1778 			dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n",
1779 				 event, queue, pf_num, vf_num);
1780 		wr32(hw, GL_MDET_RX, 0xffffffff);
1781 	}
1782 
1783 	/* check to see if this PF caused an MDD event */
1784 	reg = rd32(hw, PF_MDET_TX_PQM);
1785 	if (reg & PF_MDET_TX_PQM_VALID_M) {
1786 		wr32(hw, PF_MDET_TX_PQM, 0xFFFF);
1787 		if (netif_msg_tx_err(pf))
1788 			dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n");
1789 	}
1790 
1791 	reg = rd32(hw, PF_MDET_TX_TCLAN);
1792 	if (reg & PF_MDET_TX_TCLAN_VALID_M) {
1793 		wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF);
1794 		if (netif_msg_tx_err(pf))
1795 			dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n");
1796 	}
1797 
1798 	reg = rd32(hw, PF_MDET_RX);
1799 	if (reg & PF_MDET_RX_VALID_M) {
1800 		wr32(hw, PF_MDET_RX, 0xFFFF);
1801 		if (netif_msg_rx_err(pf))
1802 			dev_info(dev, "Malicious Driver Detection event RX detected on PF\n");
1803 	}
1804 
1805 	/* Check to see if one of the VFs caused an MDD event, and then
1806 	 * increment counters and set print pending
1807 	 */
1808 	mutex_lock(&pf->vfs.table_lock);
1809 	ice_for_each_vf(pf, bkt, vf) {
1810 		reg = rd32(hw, VP_MDET_TX_PQM(vf->vf_id));
1811 		if (reg & VP_MDET_TX_PQM_VALID_M) {
1812 			wr32(hw, VP_MDET_TX_PQM(vf->vf_id), 0xFFFF);
1813 			vf->mdd_tx_events.count++;
1814 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1815 			if (netif_msg_tx_err(pf))
1816 				dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n",
1817 					 vf->vf_id);
1818 		}
1819 
1820 		reg = rd32(hw, VP_MDET_TX_TCLAN(vf->vf_id));
1821 		if (reg & VP_MDET_TX_TCLAN_VALID_M) {
1822 			wr32(hw, VP_MDET_TX_TCLAN(vf->vf_id), 0xFFFF);
1823 			vf->mdd_tx_events.count++;
1824 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1825 			if (netif_msg_tx_err(pf))
1826 				dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n",
1827 					 vf->vf_id);
1828 		}
1829 
1830 		reg = rd32(hw, VP_MDET_TX_TDPU(vf->vf_id));
1831 		if (reg & VP_MDET_TX_TDPU_VALID_M) {
1832 			wr32(hw, VP_MDET_TX_TDPU(vf->vf_id), 0xFFFF);
1833 			vf->mdd_tx_events.count++;
1834 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1835 			if (netif_msg_tx_err(pf))
1836 				dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n",
1837 					 vf->vf_id);
1838 		}
1839 
1840 		reg = rd32(hw, VP_MDET_RX(vf->vf_id));
1841 		if (reg & VP_MDET_RX_VALID_M) {
1842 			wr32(hw, VP_MDET_RX(vf->vf_id), 0xFFFF);
1843 			vf->mdd_rx_events.count++;
1844 			set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state);
1845 			if (netif_msg_rx_err(pf))
1846 				dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n",
1847 					 vf->vf_id);
1848 
1849 			/* Since the queue is disabled on VF Rx MDD events, the
1850 			 * PF can be configured to reset the VF through ethtool
1851 			 * private flag mdd-auto-reset-vf.
1852 			 */
1853 			if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) {
1854 				/* VF MDD event counters will be cleared by
1855 				 * reset, so print the event prior to reset.
1856 				 */
1857 				ice_print_vf_rx_mdd_event(vf);
1858 				ice_reset_vf(vf, ICE_VF_RESET_LOCK);
1859 			}
1860 		}
1861 	}
1862 	mutex_unlock(&pf->vfs.table_lock);
1863 
1864 	ice_print_vfs_mdd_events(pf);
1865 }
1866 
1867 /**
1868  * ice_force_phys_link_state - Force the physical link state
1869  * @vsi: VSI to force the physical link state to up/down
1870  * @link_up: true/false indicates to set the physical link to up/down
1871  *
1872  * Force the physical link state by getting the current PHY capabilities from
1873  * hardware and setting the PHY config based on the determined capabilities. If
1874  * link changes a link event will be triggered because both the Enable Automatic
1875  * Link Update and LESM Enable bits are set when setting the PHY capabilities.
1876  *
1877  * Returns 0 on success, negative on failure
1878  */
1879 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up)
1880 {
1881 	struct ice_aqc_get_phy_caps_data *pcaps;
1882 	struct ice_aqc_set_phy_cfg_data *cfg;
1883 	struct ice_port_info *pi;
1884 	struct device *dev;
1885 	int retcode;
1886 
1887 	if (!vsi || !vsi->port_info || !vsi->back)
1888 		return -EINVAL;
1889 	if (vsi->type != ICE_VSI_PF)
1890 		return 0;
1891 
1892 	dev = ice_pf_to_dev(vsi->back);
1893 
1894 	pi = vsi->port_info;
1895 
1896 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
1897 	if (!pcaps)
1898 		return -ENOMEM;
1899 
1900 	retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps,
1901 				      NULL);
1902 	if (retcode) {
1903 		dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n",
1904 			vsi->vsi_num, retcode);
1905 		retcode = -EIO;
1906 		goto out;
1907 	}
1908 
1909 	/* No change in link */
1910 	if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) &&
1911 	    link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP))
1912 		goto out;
1913 
1914 	/* Use the current user PHY configuration. The current user PHY
1915 	 * configuration is initialized during probe from PHY capabilities
1916 	 * software mode, and updated on set PHY configuration.
1917 	 */
1918 	cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL);
1919 	if (!cfg) {
1920 		retcode = -ENOMEM;
1921 		goto out;
1922 	}
1923 
1924 	cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT;
1925 	if (link_up)
1926 		cfg->caps |= ICE_AQ_PHY_ENA_LINK;
1927 	else
1928 		cfg->caps &= ~ICE_AQ_PHY_ENA_LINK;
1929 
1930 	retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL);
1931 	if (retcode) {
1932 		dev_err(dev, "Failed to set phy config, VSI %d error %d\n",
1933 			vsi->vsi_num, retcode);
1934 		retcode = -EIO;
1935 	}
1936 
1937 	kfree(cfg);
1938 out:
1939 	kfree(pcaps);
1940 	return retcode;
1941 }
1942 
1943 /**
1944  * ice_init_nvm_phy_type - Initialize the NVM PHY type
1945  * @pi: port info structure
1946  *
1947  * Initialize nvm_phy_type_[low|high] for link lenient mode support
1948  */
1949 static int ice_init_nvm_phy_type(struct ice_port_info *pi)
1950 {
1951 	struct ice_aqc_get_phy_caps_data *pcaps;
1952 	struct ice_pf *pf = pi->hw->back;
1953 	int err;
1954 
1955 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
1956 	if (!pcaps)
1957 		return -ENOMEM;
1958 
1959 	err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA,
1960 				  pcaps, NULL);
1961 
1962 	if (err) {
1963 		dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n");
1964 		goto out;
1965 	}
1966 
1967 	pf->nvm_phy_type_hi = pcaps->phy_type_high;
1968 	pf->nvm_phy_type_lo = pcaps->phy_type_low;
1969 
1970 out:
1971 	kfree(pcaps);
1972 	return err;
1973 }
1974 
1975 /**
1976  * ice_init_link_dflt_override - Initialize link default override
1977  * @pi: port info structure
1978  *
1979  * Initialize link default override and PHY total port shutdown during probe
1980  */
1981 static void ice_init_link_dflt_override(struct ice_port_info *pi)
1982 {
1983 	struct ice_link_default_override_tlv *ldo;
1984 	struct ice_pf *pf = pi->hw->back;
1985 
1986 	ldo = &pf->link_dflt_override;
1987 	if (ice_get_link_default_override(ldo, pi))
1988 		return;
1989 
1990 	if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS))
1991 		return;
1992 
1993 	/* Enable Total Port Shutdown (override/replace link-down-on-close
1994 	 * ethtool private flag) for ports with Port Disable bit set.
1995 	 */
1996 	set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags);
1997 	set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags);
1998 }
1999 
2000 /**
2001  * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings
2002  * @pi: port info structure
2003  *
2004  * If default override is enabled, initialize the user PHY cfg speed and FEC
2005  * settings using the default override mask from the NVM.
2006  *
2007  * The PHY should only be configured with the default override settings the
2008  * first time media is available. The ICE_LINK_DEFAULT_OVERRIDE_PENDING state
2009  * is used to indicate that the user PHY cfg default override is initialized
2010  * and the PHY has not been configured with the default override settings. The
2011  * state is set here, and cleared in ice_configure_phy the first time the PHY is
2012  * configured.
2013  *
2014  * This function should be called only if the FW doesn't support default
2015  * configuration mode, as reported by ice_fw_supports_report_dflt_cfg.
2016  */
2017 static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi)
2018 {
2019 	struct ice_link_default_override_tlv *ldo;
2020 	struct ice_aqc_set_phy_cfg_data *cfg;
2021 	struct ice_phy_info *phy = &pi->phy;
2022 	struct ice_pf *pf = pi->hw->back;
2023 
2024 	ldo = &pf->link_dflt_override;
2025 
2026 	/* If link default override is enabled, use to mask NVM PHY capabilities
2027 	 * for speed and FEC default configuration.
2028 	 */
2029 	cfg = &phy->curr_user_phy_cfg;
2030 
2031 	if (ldo->phy_type_low || ldo->phy_type_high) {
2032 		cfg->phy_type_low = pf->nvm_phy_type_lo &
2033 				    cpu_to_le64(ldo->phy_type_low);
2034 		cfg->phy_type_high = pf->nvm_phy_type_hi &
2035 				     cpu_to_le64(ldo->phy_type_high);
2036 	}
2037 	cfg->link_fec_opt = ldo->fec_options;
2038 	phy->curr_user_fec_req = ICE_FEC_AUTO;
2039 
2040 	set_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state);
2041 }
2042 
2043 /**
2044  * ice_init_phy_user_cfg - Initialize the PHY user configuration
2045  * @pi: port info structure
2046  *
2047  * Initialize the current user PHY configuration, speed, FEC, and FC requested
2048  * mode to default. The PHY defaults are from get PHY capabilities topology
2049  * with media so call when media is first available. An error is returned if
2050  * called when media is not available. The PHY initialization completed state is
2051  * set here.
2052  *
2053  * These configurations are used when setting PHY
2054  * configuration. The user PHY configuration is updated on set PHY
2055  * configuration. Returns 0 on success, negative on failure
2056  */
2057 static int ice_init_phy_user_cfg(struct ice_port_info *pi)
2058 {
2059 	struct ice_aqc_get_phy_caps_data *pcaps;
2060 	struct ice_phy_info *phy = &pi->phy;
2061 	struct ice_pf *pf = pi->hw->back;
2062 	int err;
2063 
2064 	if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE))
2065 		return -EIO;
2066 
2067 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
2068 	if (!pcaps)
2069 		return -ENOMEM;
2070 
2071 	if (ice_fw_supports_report_dflt_cfg(pi->hw))
2072 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG,
2073 					  pcaps, NULL);
2074 	else
2075 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA,
2076 					  pcaps, NULL);
2077 	if (err) {
2078 		dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n");
2079 		goto err_out;
2080 	}
2081 
2082 	ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg);
2083 
2084 	/* check if lenient mode is supported and enabled */
2085 	if (ice_fw_supports_link_override(pi->hw) &&
2086 	    !(pcaps->module_compliance_enforcement &
2087 	      ICE_AQC_MOD_ENFORCE_STRICT_MODE)) {
2088 		set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags);
2089 
2090 		/* if the FW supports default PHY configuration mode, then the driver
2091 		 * does not have to apply link override settings. If not,
2092 		 * initialize user PHY configuration with link override values
2093 		 */
2094 		if (!ice_fw_supports_report_dflt_cfg(pi->hw) &&
2095 		    (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN)) {
2096 			ice_init_phy_cfg_dflt_override(pi);
2097 			goto out;
2098 		}
2099 	}
2100 
2101 	/* if link default override is not enabled, set user flow control and
2102 	 * FEC settings based on what get_phy_caps returned
2103 	 */
2104 	phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps,
2105 						      pcaps->link_fec_options);
2106 	phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps);
2107 
2108 out:
2109 	phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M;
2110 	set_bit(ICE_PHY_INIT_COMPLETE, pf->state);
2111 err_out:
2112 	kfree(pcaps);
2113 	return err;
2114 }
2115 
2116 /**
2117  * ice_configure_phy - configure PHY
2118  * @vsi: VSI of PHY
2119  *
2120  * Set the PHY configuration. If the current PHY configuration is the same as
2121  * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise
2122  * configure the based get PHY capabilities for topology with media.
2123  */
2124 static int ice_configure_phy(struct ice_vsi *vsi)
2125 {
2126 	struct device *dev = ice_pf_to_dev(vsi->back);
2127 	struct ice_port_info *pi = vsi->port_info;
2128 	struct ice_aqc_get_phy_caps_data *pcaps;
2129 	struct ice_aqc_set_phy_cfg_data *cfg;
2130 	struct ice_phy_info *phy = &pi->phy;
2131 	struct ice_pf *pf = vsi->back;
2132 	int err;
2133 
2134 	/* Ensure we have media as we cannot configure a medialess port */
2135 	if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE))
2136 		return -EPERM;
2137 
2138 	ice_print_topo_conflict(vsi);
2139 
2140 	if (!test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags) &&
2141 	    phy->link_info.topo_media_conflict == ICE_AQ_LINK_TOPO_UNSUPP_MEDIA)
2142 		return -EPERM;
2143 
2144 	if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags))
2145 		return ice_force_phys_link_state(vsi, true);
2146 
2147 	pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL);
2148 	if (!pcaps)
2149 		return -ENOMEM;
2150 
2151 	/* Get current PHY config */
2152 	err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps,
2153 				  NULL);
2154 	if (err) {
2155 		dev_err(dev, "Failed to get PHY configuration, VSI %d error %d\n",
2156 			vsi->vsi_num, err);
2157 		goto done;
2158 	}
2159 
2160 	/* If PHY enable link is configured and configuration has not changed,
2161 	 * there's nothing to do
2162 	 */
2163 	if (pcaps->caps & ICE_AQC_PHY_EN_LINK &&
2164 	    ice_phy_caps_equals_cfg(pcaps, &phy->curr_user_phy_cfg))
2165 		goto done;
2166 
2167 	/* Use PHY topology as baseline for configuration */
2168 	memset(pcaps, 0, sizeof(*pcaps));
2169 	if (ice_fw_supports_report_dflt_cfg(pi->hw))
2170 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG,
2171 					  pcaps, NULL);
2172 	else
2173 		err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA,
2174 					  pcaps, NULL);
2175 	if (err) {
2176 		dev_err(dev, "Failed to get PHY caps, VSI %d error %d\n",
2177 			vsi->vsi_num, err);
2178 		goto done;
2179 	}
2180 
2181 	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
2182 	if (!cfg) {
2183 		err = -ENOMEM;
2184 		goto done;
2185 	}
2186 
2187 	ice_copy_phy_caps_to_cfg(pi, pcaps, cfg);
2188 
2189 	/* Speed - If default override pending, use curr_user_phy_cfg set in
2190 	 * ice_init_phy_user_cfg_ldo.
2191 	 */
2192 	if (test_and_clear_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING,
2193 			       vsi->back->state)) {
2194 		cfg->phy_type_low = phy->curr_user_phy_cfg.phy_type_low;
2195 		cfg->phy_type_high = phy->curr_user_phy_cfg.phy_type_high;
2196 	} else {
2197 		u64 phy_low = 0, phy_high = 0;
2198 
2199 		ice_update_phy_type(&phy_low, &phy_high,
2200 				    pi->phy.curr_user_speed_req);
2201 		cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low);
2202 		cfg->phy_type_high = pcaps->phy_type_high &
2203 				     cpu_to_le64(phy_high);
2204 	}
2205 
2206 	/* Can't provide what was requested; use PHY capabilities */
2207 	if (!cfg->phy_type_low && !cfg->phy_type_high) {
2208 		cfg->phy_type_low = pcaps->phy_type_low;
2209 		cfg->phy_type_high = pcaps->phy_type_high;
2210 	}
2211 
2212 	/* FEC */
2213 	ice_cfg_phy_fec(pi, cfg, phy->curr_user_fec_req);
2214 
2215 	/* Can't provide what was requested; use PHY capabilities */
2216 	if (cfg->link_fec_opt !=
2217 	    (cfg->link_fec_opt & pcaps->link_fec_options)) {
2218 		cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC;
2219 		cfg->link_fec_opt = pcaps->link_fec_options;
2220 	}
2221 
2222 	/* Flow Control - always supported; no need to check against
2223 	 * capabilities
2224 	 */
2225 	ice_cfg_phy_fc(pi, cfg, phy->curr_user_fc_req);
2226 
2227 	/* Enable link and link update */
2228 	cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK;
2229 
2230 	err = ice_aq_set_phy_cfg(&pf->hw, pi, cfg, NULL);
2231 	if (err)
2232 		dev_err(dev, "Failed to set phy config, VSI %d error %d\n",
2233 			vsi->vsi_num, err);
2234 
2235 	kfree(cfg);
2236 done:
2237 	kfree(pcaps);
2238 	return err;
2239 }
2240 
2241 /**
2242  * ice_check_media_subtask - Check for media
2243  * @pf: pointer to PF struct
2244  *
2245  * If media is available, then initialize PHY user configuration if it is not
2246  * been, and configure the PHY if the interface is up.
2247  */
2248 static void ice_check_media_subtask(struct ice_pf *pf)
2249 {
2250 	struct ice_port_info *pi;
2251 	struct ice_vsi *vsi;
2252 	int err;
2253 
2254 	/* No need to check for media if it's already present */
2255 	if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags))
2256 		return;
2257 
2258 	vsi = ice_get_main_vsi(pf);
2259 	if (!vsi)
2260 		return;
2261 
2262 	/* Refresh link info and check if media is present */
2263 	pi = vsi->port_info;
2264 	err = ice_update_link_info(pi);
2265 	if (err)
2266 		return;
2267 
2268 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
2269 
2270 	if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
2271 		if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state))
2272 			ice_init_phy_user_cfg(pi);
2273 
2274 		/* PHY settings are reset on media insertion, reconfigure
2275 		 * PHY to preserve settings.
2276 		 */
2277 		if (test_bit(ICE_VSI_DOWN, vsi->state) &&
2278 		    test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags))
2279 			return;
2280 
2281 		err = ice_configure_phy(vsi);
2282 		if (!err)
2283 			clear_bit(ICE_FLAG_NO_MEDIA, pf->flags);
2284 
2285 		/* A Link Status Event will be generated; the event handler
2286 		 * will complete bringing the interface up
2287 		 */
2288 	}
2289 }
2290 
2291 /**
2292  * ice_service_task - manage and run subtasks
2293  * @work: pointer to work_struct contained by the PF struct
2294  */
2295 static void ice_service_task(struct work_struct *work)
2296 {
2297 	struct ice_pf *pf = container_of(work, struct ice_pf, serv_task);
2298 	unsigned long start_time = jiffies;
2299 
2300 	/* subtasks */
2301 
2302 	/* process reset requests first */
2303 	ice_reset_subtask(pf);
2304 
2305 	/* bail if a reset/recovery cycle is pending or rebuild failed */
2306 	if (ice_is_reset_in_progress(pf->state) ||
2307 	    test_bit(ICE_SUSPENDED, pf->state) ||
2308 	    test_bit(ICE_NEEDS_RESTART, pf->state)) {
2309 		ice_service_task_complete(pf);
2310 		return;
2311 	}
2312 
2313 	if (test_and_clear_bit(ICE_AUX_ERR_PENDING, pf->state)) {
2314 		struct iidc_event *event;
2315 
2316 		event = kzalloc(sizeof(*event), GFP_KERNEL);
2317 		if (event) {
2318 			set_bit(IIDC_EVENT_CRIT_ERR, event->type);
2319 			/* report the entire OICR value to AUX driver */
2320 			swap(event->reg, pf->oicr_err_reg);
2321 			ice_send_event_to_aux(pf, event);
2322 			kfree(event);
2323 		}
2324 	}
2325 
2326 	/* unplug aux dev per request, if an unplug request came in
2327 	 * while processing a plug request, this will handle it
2328 	 */
2329 	if (test_and_clear_bit(ICE_FLAG_UNPLUG_AUX_DEV, pf->flags))
2330 		ice_unplug_aux_dev(pf);
2331 
2332 	/* Plug aux device per request */
2333 	if (test_and_clear_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags))
2334 		ice_plug_aux_dev(pf);
2335 
2336 	if (test_and_clear_bit(ICE_FLAG_MTU_CHANGED, pf->flags)) {
2337 		struct iidc_event *event;
2338 
2339 		event = kzalloc(sizeof(*event), GFP_KERNEL);
2340 		if (event) {
2341 			set_bit(IIDC_EVENT_AFTER_MTU_CHANGE, event->type);
2342 			ice_send_event_to_aux(pf, event);
2343 			kfree(event);
2344 		}
2345 	}
2346 
2347 	ice_clean_adminq_subtask(pf);
2348 	ice_check_media_subtask(pf);
2349 	ice_check_for_hang_subtask(pf);
2350 	ice_sync_fltr_subtask(pf);
2351 	ice_handle_mdd_event(pf);
2352 	ice_watchdog_subtask(pf);
2353 
2354 	if (ice_is_safe_mode(pf)) {
2355 		ice_service_task_complete(pf);
2356 		return;
2357 	}
2358 
2359 	ice_process_vflr_event(pf);
2360 	ice_clean_mailboxq_subtask(pf);
2361 	ice_clean_sbq_subtask(pf);
2362 	ice_sync_arfs_fltrs(pf);
2363 	ice_flush_fdir_ctx(pf);
2364 
2365 	/* Clear ICE_SERVICE_SCHED flag to allow scheduling next event */
2366 	ice_service_task_complete(pf);
2367 
2368 	/* If the tasks have taken longer than one service timer period
2369 	 * or there is more work to be done, reset the service timer to
2370 	 * schedule the service task now.
2371 	 */
2372 	if (time_after(jiffies, (start_time + pf->serv_tmr_period)) ||
2373 	    test_bit(ICE_MDD_EVENT_PENDING, pf->state) ||
2374 	    test_bit(ICE_VFLR_EVENT_PENDING, pf->state) ||
2375 	    test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state) ||
2376 	    test_bit(ICE_FD_VF_FLUSH_CTX, pf->state) ||
2377 	    test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state) ||
2378 	    test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state))
2379 		mod_timer(&pf->serv_tmr, jiffies);
2380 }
2381 
2382 /**
2383  * ice_set_ctrlq_len - helper function to set controlq length
2384  * @hw: pointer to the HW instance
2385  */
2386 static void ice_set_ctrlq_len(struct ice_hw *hw)
2387 {
2388 	hw->adminq.num_rq_entries = ICE_AQ_LEN;
2389 	hw->adminq.num_sq_entries = ICE_AQ_LEN;
2390 	hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
2391 	hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;
2392 	hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M;
2393 	hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN;
2394 	hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
2395 	hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
2396 	hw->sbq.num_rq_entries = ICE_SBQ_LEN;
2397 	hw->sbq.num_sq_entries = ICE_SBQ_LEN;
2398 	hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN;
2399 	hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN;
2400 }
2401 
2402 /**
2403  * ice_schedule_reset - schedule a reset
2404  * @pf: board private structure
2405  * @reset: reset being requested
2406  */
2407 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset)
2408 {
2409 	struct device *dev = ice_pf_to_dev(pf);
2410 
2411 	/* bail out if earlier reset has failed */
2412 	if (test_bit(ICE_RESET_FAILED, pf->state)) {
2413 		dev_dbg(dev, "earlier reset has failed\n");
2414 		return -EIO;
2415 	}
2416 	/* bail if reset/recovery already in progress */
2417 	if (ice_is_reset_in_progress(pf->state)) {
2418 		dev_dbg(dev, "Reset already in progress\n");
2419 		return -EBUSY;
2420 	}
2421 
2422 	switch (reset) {
2423 	case ICE_RESET_PFR:
2424 		set_bit(ICE_PFR_REQ, pf->state);
2425 		break;
2426 	case ICE_RESET_CORER:
2427 		set_bit(ICE_CORER_REQ, pf->state);
2428 		break;
2429 	case ICE_RESET_GLOBR:
2430 		set_bit(ICE_GLOBR_REQ, pf->state);
2431 		break;
2432 	default:
2433 		return -EINVAL;
2434 	}
2435 
2436 	ice_service_task_schedule(pf);
2437 	return 0;
2438 }
2439 
2440 /**
2441  * ice_irq_affinity_notify - Callback for affinity changes
2442  * @notify: context as to what irq was changed
2443  * @mask: the new affinity mask
2444  *
2445  * This is a callback function used by the irq_set_affinity_notifier function
2446  * so that we may register to receive changes to the irq affinity masks.
2447  */
2448 static void
2449 ice_irq_affinity_notify(struct irq_affinity_notify *notify,
2450 			const cpumask_t *mask)
2451 {
2452 	struct ice_q_vector *q_vector =
2453 		container_of(notify, struct ice_q_vector, affinity_notify);
2454 
2455 	cpumask_copy(&q_vector->affinity_mask, mask);
2456 }
2457 
2458 /**
2459  * ice_irq_affinity_release - Callback for affinity notifier release
2460  * @ref: internal core kernel usage
2461  *
2462  * This is a callback function used by the irq_set_affinity_notifier function
2463  * to inform the current notification subscriber that they will no longer
2464  * receive notifications.
2465  */
2466 static void ice_irq_affinity_release(struct kref __always_unused *ref) {}
2467 
2468 /**
2469  * ice_vsi_ena_irq - Enable IRQ for the given VSI
2470  * @vsi: the VSI being configured
2471  */
2472 static int ice_vsi_ena_irq(struct ice_vsi *vsi)
2473 {
2474 	struct ice_hw *hw = &vsi->back->hw;
2475 	int i;
2476 
2477 	ice_for_each_q_vector(vsi, i)
2478 		ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]);
2479 
2480 	ice_flush(hw);
2481 	return 0;
2482 }
2483 
2484 /**
2485  * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI
2486  * @vsi: the VSI being configured
2487  * @basename: name for the vector
2488  */
2489 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename)
2490 {
2491 	int q_vectors = vsi->num_q_vectors;
2492 	struct ice_pf *pf = vsi->back;
2493 	struct device *dev;
2494 	int rx_int_idx = 0;
2495 	int tx_int_idx = 0;
2496 	int vector, err;
2497 	int irq_num;
2498 
2499 	dev = ice_pf_to_dev(pf);
2500 	for (vector = 0; vector < q_vectors; vector++) {
2501 		struct ice_q_vector *q_vector = vsi->q_vectors[vector];
2502 
2503 		irq_num = q_vector->irq.virq;
2504 
2505 		if (q_vector->tx.tx_ring && q_vector->rx.rx_ring) {
2506 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2507 				 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
2508 			tx_int_idx++;
2509 		} else if (q_vector->rx.rx_ring) {
2510 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2511 				 "%s-%s-%d", basename, "rx", rx_int_idx++);
2512 		} else if (q_vector->tx.tx_ring) {
2513 			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2514 				 "%s-%s-%d", basename, "tx", tx_int_idx++);
2515 		} else {
2516 			/* skip this unused q_vector */
2517 			continue;
2518 		}
2519 		if (vsi->type == ICE_VSI_CTRL && vsi->vf)
2520 			err = devm_request_irq(dev, irq_num, vsi->irq_handler,
2521 					       IRQF_SHARED, q_vector->name,
2522 					       q_vector);
2523 		else
2524 			err = devm_request_irq(dev, irq_num, vsi->irq_handler,
2525 					       0, q_vector->name, q_vector);
2526 		if (err) {
2527 			netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n",
2528 				   err);
2529 			goto free_q_irqs;
2530 		}
2531 
2532 		/* register for affinity change notifications */
2533 		if (!IS_ENABLED(CONFIG_RFS_ACCEL)) {
2534 			struct irq_affinity_notify *affinity_notify;
2535 
2536 			affinity_notify = &q_vector->affinity_notify;
2537 			affinity_notify->notify = ice_irq_affinity_notify;
2538 			affinity_notify->release = ice_irq_affinity_release;
2539 			irq_set_affinity_notifier(irq_num, affinity_notify);
2540 		}
2541 
2542 		/* assign the mask for this irq */
2543 		irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
2544 	}
2545 
2546 	err = ice_set_cpu_rx_rmap(vsi);
2547 	if (err) {
2548 		netdev_err(vsi->netdev, "Failed to setup CPU RMAP on VSI %u: %pe\n",
2549 			   vsi->vsi_num, ERR_PTR(err));
2550 		goto free_q_irqs;
2551 	}
2552 
2553 	vsi->irqs_ready = true;
2554 	return 0;
2555 
2556 free_q_irqs:
2557 	while (vector--) {
2558 		irq_num = vsi->q_vectors[vector]->irq.virq;
2559 		if (!IS_ENABLED(CONFIG_RFS_ACCEL))
2560 			irq_set_affinity_notifier(irq_num, NULL);
2561 		irq_set_affinity_hint(irq_num, NULL);
2562 		devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]);
2563 	}
2564 	return err;
2565 }
2566 
2567 /**
2568  * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP
2569  * @vsi: VSI to setup Tx rings used by XDP
2570  *
2571  * Return 0 on success and negative value on error
2572  */
2573 static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi)
2574 {
2575 	struct device *dev = ice_pf_to_dev(vsi->back);
2576 	struct ice_tx_desc *tx_desc;
2577 	int i, j;
2578 
2579 	ice_for_each_xdp_txq(vsi, i) {
2580 		u16 xdp_q_idx = vsi->alloc_txq + i;
2581 		struct ice_ring_stats *ring_stats;
2582 		struct ice_tx_ring *xdp_ring;
2583 
2584 		xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL);
2585 		if (!xdp_ring)
2586 			goto free_xdp_rings;
2587 
2588 		ring_stats = kzalloc(sizeof(*ring_stats), GFP_KERNEL);
2589 		if (!ring_stats) {
2590 			ice_free_tx_ring(xdp_ring);
2591 			goto free_xdp_rings;
2592 		}
2593 
2594 		xdp_ring->ring_stats = ring_stats;
2595 		xdp_ring->q_index = xdp_q_idx;
2596 		xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx];
2597 		xdp_ring->vsi = vsi;
2598 		xdp_ring->netdev = NULL;
2599 		xdp_ring->dev = dev;
2600 		xdp_ring->count = vsi->num_tx_desc;
2601 		WRITE_ONCE(vsi->xdp_rings[i], xdp_ring);
2602 		if (ice_setup_tx_ring(xdp_ring))
2603 			goto free_xdp_rings;
2604 		ice_set_ring_xdp(xdp_ring);
2605 		spin_lock_init(&xdp_ring->tx_lock);
2606 		for (j = 0; j < xdp_ring->count; j++) {
2607 			tx_desc = ICE_TX_DESC(xdp_ring, j);
2608 			tx_desc->cmd_type_offset_bsz = 0;
2609 		}
2610 	}
2611 
2612 	return 0;
2613 
2614 free_xdp_rings:
2615 	for (; i >= 0; i--) {
2616 		if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) {
2617 			kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu);
2618 			vsi->xdp_rings[i]->ring_stats = NULL;
2619 			ice_free_tx_ring(vsi->xdp_rings[i]);
2620 		}
2621 	}
2622 	return -ENOMEM;
2623 }
2624 
2625 /**
2626  * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI
2627  * @vsi: VSI to set the bpf prog on
2628  * @prog: the bpf prog pointer
2629  */
2630 static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog)
2631 {
2632 	struct bpf_prog *old_prog;
2633 	int i;
2634 
2635 	old_prog = xchg(&vsi->xdp_prog, prog);
2636 	ice_for_each_rxq(vsi, i)
2637 		WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
2638 
2639 	if (old_prog)
2640 		bpf_prog_put(old_prog);
2641 }
2642 
2643 /**
2644  * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP
2645  * @vsi: VSI to bring up Tx rings used by XDP
2646  * @prog: bpf program that will be assigned to VSI
2647  *
2648  * Return 0 on success and negative value on error
2649  */
2650 int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog)
2651 {
2652 	u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2653 	int xdp_rings_rem = vsi->num_xdp_txq;
2654 	struct ice_pf *pf = vsi->back;
2655 	struct ice_qs_cfg xdp_qs_cfg = {
2656 		.qs_mutex = &pf->avail_q_mutex,
2657 		.pf_map = pf->avail_txqs,
2658 		.pf_map_size = pf->max_pf_txqs,
2659 		.q_count = vsi->num_xdp_txq,
2660 		.scatter_count = ICE_MAX_SCATTER_TXQS,
2661 		.vsi_map = vsi->txq_map,
2662 		.vsi_map_offset = vsi->alloc_txq,
2663 		.mapping_mode = ICE_VSI_MAP_CONTIG
2664 	};
2665 	struct device *dev;
2666 	int i, v_idx;
2667 	int status;
2668 
2669 	dev = ice_pf_to_dev(pf);
2670 	vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq,
2671 				      sizeof(*vsi->xdp_rings), GFP_KERNEL);
2672 	if (!vsi->xdp_rings)
2673 		return -ENOMEM;
2674 
2675 	vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode;
2676 	if (__ice_vsi_get_qs(&xdp_qs_cfg))
2677 		goto err_map_xdp;
2678 
2679 	if (static_key_enabled(&ice_xdp_locking_key))
2680 		netdev_warn(vsi->netdev,
2681 			    "Could not allocate one XDP Tx ring per CPU, XDP_TX/XDP_REDIRECT actions will be slower\n");
2682 
2683 	if (ice_xdp_alloc_setup_rings(vsi))
2684 		goto clear_xdp_rings;
2685 
2686 	/* follow the logic from ice_vsi_map_rings_to_vectors */
2687 	ice_for_each_q_vector(vsi, v_idx) {
2688 		struct ice_q_vector *q_vector = vsi->q_vectors[v_idx];
2689 		int xdp_rings_per_v, q_id, q_base;
2690 
2691 		xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem,
2692 					       vsi->num_q_vectors - v_idx);
2693 		q_base = vsi->num_xdp_txq - xdp_rings_rem;
2694 
2695 		for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) {
2696 			struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_id];
2697 
2698 			xdp_ring->q_vector = q_vector;
2699 			xdp_ring->next = q_vector->tx.tx_ring;
2700 			q_vector->tx.tx_ring = xdp_ring;
2701 		}
2702 		xdp_rings_rem -= xdp_rings_per_v;
2703 	}
2704 
2705 	ice_for_each_rxq(vsi, i) {
2706 		if (static_key_enabled(&ice_xdp_locking_key)) {
2707 			vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i % vsi->num_xdp_txq];
2708 		} else {
2709 			struct ice_q_vector *q_vector = vsi->rx_rings[i]->q_vector;
2710 			struct ice_tx_ring *ring;
2711 
2712 			ice_for_each_tx_ring(ring, q_vector->tx) {
2713 				if (ice_ring_is_xdp(ring)) {
2714 					vsi->rx_rings[i]->xdp_ring = ring;
2715 					break;
2716 				}
2717 			}
2718 		}
2719 		ice_tx_xsk_pool(vsi, i);
2720 	}
2721 
2722 	/* omit the scheduler update if in reset path; XDP queues will be
2723 	 * taken into account at the end of ice_vsi_rebuild, where
2724 	 * ice_cfg_vsi_lan is being called
2725 	 */
2726 	if (ice_is_reset_in_progress(pf->state))
2727 		return 0;
2728 
2729 	/* tell the Tx scheduler that right now we have
2730 	 * additional queues
2731 	 */
2732 	for (i = 0; i < vsi->tc_cfg.numtc; i++)
2733 		max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq;
2734 
2735 	status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc,
2736 				 max_txqs);
2737 	if (status) {
2738 		dev_err(dev, "Failed VSI LAN queue config for XDP, error: %d\n",
2739 			status);
2740 		goto clear_xdp_rings;
2741 	}
2742 
2743 	/* assign the prog only when it's not already present on VSI;
2744 	 * this flow is a subject of both ethtool -L and ndo_bpf flows;
2745 	 * VSI rebuild that happens under ethtool -L can expose us to
2746 	 * the bpf_prog refcount issues as we would be swapping same
2747 	 * bpf_prog pointers from vsi->xdp_prog and calling bpf_prog_put
2748 	 * on it as it would be treated as an 'old_prog'; for ndo_bpf
2749 	 * this is not harmful as dev_xdp_install bumps the refcount
2750 	 * before calling the op exposed by the driver;
2751 	 */
2752 	if (!ice_is_xdp_ena_vsi(vsi))
2753 		ice_vsi_assign_bpf_prog(vsi, prog);
2754 
2755 	return 0;
2756 clear_xdp_rings:
2757 	ice_for_each_xdp_txq(vsi, i)
2758 		if (vsi->xdp_rings[i]) {
2759 			kfree_rcu(vsi->xdp_rings[i], rcu);
2760 			vsi->xdp_rings[i] = NULL;
2761 		}
2762 
2763 err_map_xdp:
2764 	mutex_lock(&pf->avail_q_mutex);
2765 	ice_for_each_xdp_txq(vsi, i) {
2766 		clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs);
2767 		vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX;
2768 	}
2769 	mutex_unlock(&pf->avail_q_mutex);
2770 
2771 	devm_kfree(dev, vsi->xdp_rings);
2772 	return -ENOMEM;
2773 }
2774 
2775 /**
2776  * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings
2777  * @vsi: VSI to remove XDP rings
2778  *
2779  * Detach XDP rings from irq vectors, clean up the PF bitmap and free
2780  * resources
2781  */
2782 int ice_destroy_xdp_rings(struct ice_vsi *vsi)
2783 {
2784 	u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2785 	struct ice_pf *pf = vsi->back;
2786 	int i, v_idx;
2787 
2788 	/* q_vectors are freed in reset path so there's no point in detaching
2789 	 * rings; in case of rebuild being triggered not from reset bits
2790 	 * in pf->state won't be set, so additionally check first q_vector
2791 	 * against NULL
2792 	 */
2793 	if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0])
2794 		goto free_qmap;
2795 
2796 	ice_for_each_q_vector(vsi, v_idx) {
2797 		struct ice_q_vector *q_vector = vsi->q_vectors[v_idx];
2798 		struct ice_tx_ring *ring;
2799 
2800 		ice_for_each_tx_ring(ring, q_vector->tx)
2801 			if (!ring->tx_buf || !ice_ring_is_xdp(ring))
2802 				break;
2803 
2804 		/* restore the value of last node prior to XDP setup */
2805 		q_vector->tx.tx_ring = ring;
2806 	}
2807 
2808 free_qmap:
2809 	mutex_lock(&pf->avail_q_mutex);
2810 	ice_for_each_xdp_txq(vsi, i) {
2811 		clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs);
2812 		vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX;
2813 	}
2814 	mutex_unlock(&pf->avail_q_mutex);
2815 
2816 	ice_for_each_xdp_txq(vsi, i)
2817 		if (vsi->xdp_rings[i]) {
2818 			if (vsi->xdp_rings[i]->desc) {
2819 				synchronize_rcu();
2820 				ice_free_tx_ring(vsi->xdp_rings[i]);
2821 			}
2822 			kfree_rcu(vsi->xdp_rings[i]->ring_stats, rcu);
2823 			vsi->xdp_rings[i]->ring_stats = NULL;
2824 			kfree_rcu(vsi->xdp_rings[i], rcu);
2825 			vsi->xdp_rings[i] = NULL;
2826 		}
2827 
2828 	devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings);
2829 	vsi->xdp_rings = NULL;
2830 
2831 	if (static_key_enabled(&ice_xdp_locking_key))
2832 		static_branch_dec(&ice_xdp_locking_key);
2833 
2834 	if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0])
2835 		return 0;
2836 
2837 	ice_vsi_assign_bpf_prog(vsi, NULL);
2838 
2839 	/* notify Tx scheduler that we destroyed XDP queues and bring
2840 	 * back the old number of child nodes
2841 	 */
2842 	for (i = 0; i < vsi->tc_cfg.numtc; i++)
2843 		max_txqs[i] = vsi->num_txq;
2844 
2845 	/* change number of XDP Tx queues to 0 */
2846 	vsi->num_xdp_txq = 0;
2847 
2848 	return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc,
2849 			       max_txqs);
2850 }
2851 
2852 /**
2853  * ice_vsi_rx_napi_schedule - Schedule napi on RX queues from VSI
2854  * @vsi: VSI to schedule napi on
2855  */
2856 static void ice_vsi_rx_napi_schedule(struct ice_vsi *vsi)
2857 {
2858 	int i;
2859 
2860 	ice_for_each_rxq(vsi, i) {
2861 		struct ice_rx_ring *rx_ring = vsi->rx_rings[i];
2862 
2863 		if (rx_ring->xsk_pool)
2864 			napi_schedule(&rx_ring->q_vector->napi);
2865 	}
2866 }
2867 
2868 /**
2869  * ice_vsi_determine_xdp_res - figure out how many Tx qs can XDP have
2870  * @vsi: VSI to determine the count of XDP Tx qs
2871  *
2872  * returns 0 if Tx qs count is higher than at least half of CPU count,
2873  * -ENOMEM otherwise
2874  */
2875 int ice_vsi_determine_xdp_res(struct ice_vsi *vsi)
2876 {
2877 	u16 avail = ice_get_avail_txq_count(vsi->back);
2878 	u16 cpus = num_possible_cpus();
2879 
2880 	if (avail < cpus / 2)
2881 		return -ENOMEM;
2882 
2883 	vsi->num_xdp_txq = min_t(u16, avail, cpus);
2884 
2885 	if (vsi->num_xdp_txq < cpus)
2886 		static_branch_inc(&ice_xdp_locking_key);
2887 
2888 	return 0;
2889 }
2890 
2891 /**
2892  * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP
2893  * @vsi: Pointer to VSI structure
2894  */
2895 static int ice_max_xdp_frame_size(struct ice_vsi *vsi)
2896 {
2897 	if (test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags))
2898 		return ICE_RXBUF_1664;
2899 	else
2900 		return ICE_RXBUF_3072;
2901 }
2902 
2903 /**
2904  * ice_xdp_setup_prog - Add or remove XDP eBPF program
2905  * @vsi: VSI to setup XDP for
2906  * @prog: XDP program
2907  * @extack: netlink extended ack
2908  */
2909 static int
2910 ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog,
2911 		   struct netlink_ext_ack *extack)
2912 {
2913 	unsigned int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD;
2914 	bool if_running = netif_running(vsi->netdev);
2915 	int ret = 0, xdp_ring_err = 0;
2916 
2917 	if (prog && !prog->aux->xdp_has_frags) {
2918 		if (frame_size > ice_max_xdp_frame_size(vsi)) {
2919 			NL_SET_ERR_MSG_MOD(extack,
2920 					   "MTU is too large for linear frames and XDP prog does not support frags");
2921 			return -EOPNOTSUPP;
2922 		}
2923 	}
2924 
2925 	/* hot swap progs and avoid toggling link */
2926 	if (ice_is_xdp_ena_vsi(vsi) == !!prog) {
2927 		ice_vsi_assign_bpf_prog(vsi, prog);
2928 		return 0;
2929 	}
2930 
2931 	/* need to stop netdev while setting up the program for Rx rings */
2932 	if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) {
2933 		ret = ice_down(vsi);
2934 		if (ret) {
2935 			NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed");
2936 			return ret;
2937 		}
2938 	}
2939 
2940 	if (!ice_is_xdp_ena_vsi(vsi) && prog) {
2941 		xdp_ring_err = ice_vsi_determine_xdp_res(vsi);
2942 		if (xdp_ring_err) {
2943 			NL_SET_ERR_MSG_MOD(extack, "Not enough Tx resources for XDP");
2944 		} else {
2945 			xdp_ring_err = ice_prepare_xdp_rings(vsi, prog);
2946 			if (xdp_ring_err)
2947 				NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed");
2948 		}
2949 		xdp_features_set_redirect_target(vsi->netdev, true);
2950 		/* reallocate Rx queues that are used for zero-copy */
2951 		xdp_ring_err = ice_realloc_zc_buf(vsi, true);
2952 		if (xdp_ring_err)
2953 			NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Rx resources failed");
2954 	} else if (ice_is_xdp_ena_vsi(vsi) && !prog) {
2955 		xdp_features_clear_redirect_target(vsi->netdev);
2956 		xdp_ring_err = ice_destroy_xdp_rings(vsi);
2957 		if (xdp_ring_err)
2958 			NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed");
2959 		/* reallocate Rx queues that were used for zero-copy */
2960 		xdp_ring_err = ice_realloc_zc_buf(vsi, false);
2961 		if (xdp_ring_err)
2962 			NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Rx resources failed");
2963 	}
2964 
2965 	if (if_running)
2966 		ret = ice_up(vsi);
2967 
2968 	if (!ret && prog)
2969 		ice_vsi_rx_napi_schedule(vsi);
2970 
2971 	return (ret || xdp_ring_err) ? -ENOMEM : 0;
2972 }
2973 
2974 /**
2975  * ice_xdp_safe_mode - XDP handler for safe mode
2976  * @dev: netdevice
2977  * @xdp: XDP command
2978  */
2979 static int ice_xdp_safe_mode(struct net_device __always_unused *dev,
2980 			     struct netdev_bpf *xdp)
2981 {
2982 	NL_SET_ERR_MSG_MOD(xdp->extack,
2983 			   "Please provide working DDP firmware package in order to use XDP\n"
2984 			   "Refer to Documentation/networking/device_drivers/ethernet/intel/ice.rst");
2985 	return -EOPNOTSUPP;
2986 }
2987 
2988 /**
2989  * ice_xdp - implements XDP handler
2990  * @dev: netdevice
2991  * @xdp: XDP command
2992  */
2993 static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2994 {
2995 	struct ice_netdev_priv *np = netdev_priv(dev);
2996 	struct ice_vsi *vsi = np->vsi;
2997 
2998 	if (vsi->type != ICE_VSI_PF) {
2999 		NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI");
3000 		return -EINVAL;
3001 	}
3002 
3003 	switch (xdp->command) {
3004 	case XDP_SETUP_PROG:
3005 		return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack);
3006 	case XDP_SETUP_XSK_POOL:
3007 		return ice_xsk_pool_setup(vsi, xdp->xsk.pool,
3008 					  xdp->xsk.queue_id);
3009 	default:
3010 		return -EINVAL;
3011 	}
3012 }
3013 
3014 /**
3015  * ice_ena_misc_vector - enable the non-queue interrupts
3016  * @pf: board private structure
3017  */
3018 static void ice_ena_misc_vector(struct ice_pf *pf)
3019 {
3020 	struct ice_hw *hw = &pf->hw;
3021 	u32 val;
3022 
3023 	/* Disable anti-spoof detection interrupt to prevent spurious event
3024 	 * interrupts during a function reset. Anti-spoof functionally is
3025 	 * still supported.
3026 	 */
3027 	val = rd32(hw, GL_MDCK_TX_TDPU);
3028 	val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M;
3029 	wr32(hw, GL_MDCK_TX_TDPU, val);
3030 
3031 	/* clear things first */
3032 	wr32(hw, PFINT_OICR_ENA, 0);	/* disable all */
3033 	rd32(hw, PFINT_OICR);		/* read to clear */
3034 
3035 	val = (PFINT_OICR_ECC_ERR_M |
3036 	       PFINT_OICR_MAL_DETECT_M |
3037 	       PFINT_OICR_GRST_M |
3038 	       PFINT_OICR_PCI_EXCEPTION_M |
3039 	       PFINT_OICR_VFLR_M |
3040 	       PFINT_OICR_HMC_ERR_M |
3041 	       PFINT_OICR_PE_PUSH_M |
3042 	       PFINT_OICR_PE_CRITERR_M);
3043 
3044 	wr32(hw, PFINT_OICR_ENA, val);
3045 
3046 	/* SW_ITR_IDX = 0, but don't change INTENA */
3047 	wr32(hw, GLINT_DYN_CTL(pf->oicr_irq.index),
3048 	     GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M);
3049 }
3050 
3051 /**
3052  * ice_misc_intr - misc interrupt handler
3053  * @irq: interrupt number
3054  * @data: pointer to a q_vector
3055  */
3056 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
3057 {
3058 	struct ice_pf *pf = (struct ice_pf *)data;
3059 	struct ice_hw *hw = &pf->hw;
3060 	struct device *dev;
3061 	u32 oicr, ena_mask;
3062 
3063 	dev = ice_pf_to_dev(pf);
3064 	set_bit(ICE_ADMINQ_EVENT_PENDING, pf->state);
3065 	set_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state);
3066 	set_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state);
3067 
3068 	oicr = rd32(hw, PFINT_OICR);
3069 	ena_mask = rd32(hw, PFINT_OICR_ENA);
3070 
3071 	if (oicr & PFINT_OICR_SWINT_M) {
3072 		ena_mask &= ~PFINT_OICR_SWINT_M;
3073 		pf->sw_int_count++;
3074 	}
3075 
3076 	if (oicr & PFINT_OICR_MAL_DETECT_M) {
3077 		ena_mask &= ~PFINT_OICR_MAL_DETECT_M;
3078 		set_bit(ICE_MDD_EVENT_PENDING, pf->state);
3079 	}
3080 	if (oicr & PFINT_OICR_VFLR_M) {
3081 		/* disable any further VFLR event notifications */
3082 		if (test_bit(ICE_VF_RESETS_DISABLED, pf->state)) {
3083 			u32 reg = rd32(hw, PFINT_OICR_ENA);
3084 
3085 			reg &= ~PFINT_OICR_VFLR_M;
3086 			wr32(hw, PFINT_OICR_ENA, reg);
3087 		} else {
3088 			ena_mask &= ~PFINT_OICR_VFLR_M;
3089 			set_bit(ICE_VFLR_EVENT_PENDING, pf->state);
3090 		}
3091 	}
3092 
3093 	if (oicr & PFINT_OICR_GRST_M) {
3094 		u32 reset;
3095 
3096 		/* we have a reset warning */
3097 		ena_mask &= ~PFINT_OICR_GRST_M;
3098 		reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
3099 			GLGEN_RSTAT_RESET_TYPE_S;
3100 
3101 		if (reset == ICE_RESET_CORER)
3102 			pf->corer_count++;
3103 		else if (reset == ICE_RESET_GLOBR)
3104 			pf->globr_count++;
3105 		else if (reset == ICE_RESET_EMPR)
3106 			pf->empr_count++;
3107 		else
3108 			dev_dbg(dev, "Invalid reset type %d\n", reset);
3109 
3110 		/* If a reset cycle isn't already in progress, we set a bit in
3111 		 * pf->state so that the service task can start a reset/rebuild.
3112 		 */
3113 		if (!test_and_set_bit(ICE_RESET_OICR_RECV, pf->state)) {
3114 			if (reset == ICE_RESET_CORER)
3115 				set_bit(ICE_CORER_RECV, pf->state);
3116 			else if (reset == ICE_RESET_GLOBR)
3117 				set_bit(ICE_GLOBR_RECV, pf->state);
3118 			else
3119 				set_bit(ICE_EMPR_RECV, pf->state);
3120 
3121 			/* There are couple of different bits at play here.
3122 			 * hw->reset_ongoing indicates whether the hardware is
3123 			 * in reset. This is set to true when a reset interrupt
3124 			 * is received and set back to false after the driver
3125 			 * has determined that the hardware is out of reset.
3126 			 *
3127 			 * ICE_RESET_OICR_RECV in pf->state indicates
3128 			 * that a post reset rebuild is required before the
3129 			 * driver is operational again. This is set above.
3130 			 *
3131 			 * As this is the start of the reset/rebuild cycle, set
3132 			 * both to indicate that.
3133 			 */
3134 			hw->reset_ongoing = true;
3135 		}
3136 	}
3137 
3138 	if (oicr & PFINT_OICR_TSYN_TX_M) {
3139 		ena_mask &= ~PFINT_OICR_TSYN_TX_M;
3140 		if (!hw->reset_ongoing)
3141 			set_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread);
3142 	}
3143 
3144 	if (oicr & PFINT_OICR_TSYN_EVNT_M) {
3145 		u8 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
3146 		u32 gltsyn_stat = rd32(hw, GLTSYN_STAT(tmr_idx));
3147 
3148 		ena_mask &= ~PFINT_OICR_TSYN_EVNT_M;
3149 
3150 		if (hw->func_caps.ts_func_info.src_tmr_owned) {
3151 			/* Save EVENTs from GLTSYN register */
3152 			pf->ptp.ext_ts_irq |= gltsyn_stat &
3153 					      (GLTSYN_STAT_EVENT0_M |
3154 					       GLTSYN_STAT_EVENT1_M |
3155 					       GLTSYN_STAT_EVENT2_M);
3156 
3157 			set_bit(ICE_MISC_THREAD_EXTTS_EVENT, pf->misc_thread);
3158 		}
3159 	}
3160 
3161 #define ICE_AUX_CRIT_ERR (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M | PFINT_OICR_PE_PUSH_M)
3162 	if (oicr & ICE_AUX_CRIT_ERR) {
3163 		pf->oicr_err_reg |= oicr;
3164 		set_bit(ICE_AUX_ERR_PENDING, pf->state);
3165 		ena_mask &= ~ICE_AUX_CRIT_ERR;
3166 	}
3167 
3168 	/* Report any remaining unexpected interrupts */
3169 	oicr &= ena_mask;
3170 	if (oicr) {
3171 		dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr);
3172 		/* If a critical error is pending there is no choice but to
3173 		 * reset the device.
3174 		 */
3175 		if (oicr & (PFINT_OICR_PCI_EXCEPTION_M |
3176 			    PFINT_OICR_ECC_ERR_M)) {
3177 			set_bit(ICE_PFR_REQ, pf->state);
3178 		}
3179 	}
3180 
3181 	return IRQ_WAKE_THREAD;
3182 }
3183 
3184 /**
3185  * ice_misc_intr_thread_fn - misc interrupt thread function
3186  * @irq: interrupt number
3187  * @data: pointer to a q_vector
3188  */
3189 static irqreturn_t ice_misc_intr_thread_fn(int __always_unused irq, void *data)
3190 {
3191 	struct ice_pf *pf = data;
3192 	struct ice_hw *hw;
3193 
3194 	hw = &pf->hw;
3195 
3196 	if (ice_is_reset_in_progress(pf->state))
3197 		return IRQ_HANDLED;
3198 
3199 	ice_service_task_schedule(pf);
3200 
3201 	if (test_and_clear_bit(ICE_MISC_THREAD_EXTTS_EVENT, pf->misc_thread))
3202 		ice_ptp_extts_event(pf);
3203 
3204 	if (test_and_clear_bit(ICE_MISC_THREAD_TX_TSTAMP, pf->misc_thread)) {
3205 		/* Process outstanding Tx timestamps. If there is more work,
3206 		 * re-arm the interrupt to trigger again.
3207 		 */
3208 		if (ice_ptp_process_ts(pf) == ICE_TX_TSTAMP_WORK_PENDING) {
3209 			wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M);
3210 			ice_flush(hw);
3211 		}
3212 	}
3213 
3214 	ice_irq_dynamic_ena(hw, NULL, NULL);
3215 
3216 	return IRQ_HANDLED;
3217 }
3218 
3219 /**
3220  * ice_dis_ctrlq_interrupts - disable control queue interrupts
3221  * @hw: pointer to HW structure
3222  */
3223 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw)
3224 {
3225 	/* disable Admin queue Interrupt causes */
3226 	wr32(hw, PFINT_FW_CTL,
3227 	     rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M);
3228 
3229 	/* disable Mailbox queue Interrupt causes */
3230 	wr32(hw, PFINT_MBX_CTL,
3231 	     rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M);
3232 
3233 	wr32(hw, PFINT_SB_CTL,
3234 	     rd32(hw, PFINT_SB_CTL) & ~PFINT_SB_CTL_CAUSE_ENA_M);
3235 
3236 	/* disable Control queue Interrupt causes */
3237 	wr32(hw, PFINT_OICR_CTL,
3238 	     rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M);
3239 
3240 	ice_flush(hw);
3241 }
3242 
3243 /**
3244  * ice_free_irq_msix_misc - Unroll misc vector setup
3245  * @pf: board private structure
3246  */
3247 static void ice_free_irq_msix_misc(struct ice_pf *pf)
3248 {
3249 	int misc_irq_num = pf->oicr_irq.virq;
3250 	struct ice_hw *hw = &pf->hw;
3251 
3252 	ice_dis_ctrlq_interrupts(hw);
3253 
3254 	/* disable OICR interrupt */
3255 	wr32(hw, PFINT_OICR_ENA, 0);
3256 	ice_flush(hw);
3257 
3258 	synchronize_irq(misc_irq_num);
3259 	devm_free_irq(ice_pf_to_dev(pf), misc_irq_num, pf);
3260 
3261 	ice_free_irq(pf, pf->oicr_irq);
3262 }
3263 
3264 /**
3265  * ice_ena_ctrlq_interrupts - enable control queue interrupts
3266  * @hw: pointer to HW structure
3267  * @reg_idx: HW vector index to associate the control queue interrupts with
3268  */
3269 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx)
3270 {
3271 	u32 val;
3272 
3273 	val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) |
3274 	       PFINT_OICR_CTL_CAUSE_ENA_M);
3275 	wr32(hw, PFINT_OICR_CTL, val);
3276 
3277 	/* enable Admin queue Interrupt causes */
3278 	val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) |
3279 	       PFINT_FW_CTL_CAUSE_ENA_M);
3280 	wr32(hw, PFINT_FW_CTL, val);
3281 
3282 	/* enable Mailbox queue Interrupt causes */
3283 	val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) |
3284 	       PFINT_MBX_CTL_CAUSE_ENA_M);
3285 	wr32(hw, PFINT_MBX_CTL, val);
3286 
3287 	/* This enables Sideband queue Interrupt causes */
3288 	val = ((reg_idx & PFINT_SB_CTL_MSIX_INDX_M) |
3289 	       PFINT_SB_CTL_CAUSE_ENA_M);
3290 	wr32(hw, PFINT_SB_CTL, val);
3291 
3292 	ice_flush(hw);
3293 }
3294 
3295 /**
3296  * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events
3297  * @pf: board private structure
3298  *
3299  * This sets up the handler for MSIX 0, which is used to manage the
3300  * non-queue interrupts, e.g. AdminQ and errors. This is not used
3301  * when in MSI or Legacy interrupt mode.
3302  */
3303 static int ice_req_irq_msix_misc(struct ice_pf *pf)
3304 {
3305 	struct device *dev = ice_pf_to_dev(pf);
3306 	struct ice_hw *hw = &pf->hw;
3307 	struct msi_map oicr_irq;
3308 	int err = 0;
3309 
3310 	if (!pf->int_name[0])
3311 		snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc",
3312 			 dev_driver_string(dev), dev_name(dev));
3313 
3314 	/* Do not request IRQ but do enable OICR interrupt since settings are
3315 	 * lost during reset. Note that this function is called only during
3316 	 * rebuild path and not while reset is in progress.
3317 	 */
3318 	if (ice_is_reset_in_progress(pf->state))
3319 		goto skip_req_irq;
3320 
3321 	/* reserve one vector in irq_tracker for misc interrupts */
3322 	oicr_irq = ice_alloc_irq(pf, false);
3323 	if (oicr_irq.index < 0)
3324 		return oicr_irq.index;
3325 
3326 	pf->oicr_irq = oicr_irq;
3327 	err = devm_request_threaded_irq(dev, pf->oicr_irq.virq, ice_misc_intr,
3328 					ice_misc_intr_thread_fn, 0,
3329 					pf->int_name, pf);
3330 	if (err) {
3331 		dev_err(dev, "devm_request_threaded_irq for %s failed: %d\n",
3332 			pf->int_name, err);
3333 		ice_free_irq(pf, pf->oicr_irq);
3334 		return err;
3335 	}
3336 
3337 skip_req_irq:
3338 	ice_ena_misc_vector(pf);
3339 
3340 	ice_ena_ctrlq_interrupts(hw, pf->oicr_irq.index);
3341 	wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_irq.index),
3342 	     ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S);
3343 
3344 	ice_flush(hw);
3345 	ice_irq_dynamic_ena(hw, NULL, NULL);
3346 
3347 	return 0;
3348 }
3349 
3350 /**
3351  * ice_napi_add - register NAPI handler for the VSI
3352  * @vsi: VSI for which NAPI handler is to be registered
3353  *
3354  * This function is only called in the driver's load path. Registering the NAPI
3355  * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume,
3356  * reset/rebuild, etc.)
3357  */
3358 static void ice_napi_add(struct ice_vsi *vsi)
3359 {
3360 	int v_idx;
3361 
3362 	if (!vsi->netdev)
3363 		return;
3364 
3365 	ice_for_each_q_vector(vsi, v_idx)
3366 		netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi,
3367 			       ice_napi_poll);
3368 }
3369 
3370 /**
3371  * ice_set_ops - set netdev and ethtools ops for the given netdev
3372  * @vsi: the VSI associated with the new netdev
3373  */
3374 static void ice_set_ops(struct ice_vsi *vsi)
3375 {
3376 	struct net_device *netdev = vsi->netdev;
3377 	struct ice_pf *pf = ice_netdev_to_pf(netdev);
3378 
3379 	if (ice_is_safe_mode(pf)) {
3380 		netdev->netdev_ops = &ice_netdev_safe_mode_ops;
3381 		ice_set_ethtool_safe_mode_ops(netdev);
3382 		return;
3383 	}
3384 
3385 	netdev->netdev_ops = &ice_netdev_ops;
3386 	netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic;
3387 	ice_set_ethtool_ops(netdev);
3388 
3389 	if (vsi->type != ICE_VSI_PF)
3390 		return;
3391 
3392 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
3393 			       NETDEV_XDP_ACT_XSK_ZEROCOPY |
3394 			       NETDEV_XDP_ACT_RX_SG;
3395 }
3396 
3397 /**
3398  * ice_set_netdev_features - set features for the given netdev
3399  * @netdev: netdev instance
3400  */
3401 static void ice_set_netdev_features(struct net_device *netdev)
3402 {
3403 	struct ice_pf *pf = ice_netdev_to_pf(netdev);
3404 	bool is_dvm_ena = ice_is_dvm_ena(&pf->hw);
3405 	netdev_features_t csumo_features;
3406 	netdev_features_t vlano_features;
3407 	netdev_features_t dflt_features;
3408 	netdev_features_t tso_features;
3409 
3410 	if (ice_is_safe_mode(pf)) {
3411 		/* safe mode */
3412 		netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA;
3413 		netdev->hw_features = netdev->features;
3414 		return;
3415 	}
3416 
3417 	dflt_features = NETIF_F_SG	|
3418 			NETIF_F_HIGHDMA	|
3419 			NETIF_F_NTUPLE	|
3420 			NETIF_F_RXHASH;
3421 
3422 	csumo_features = NETIF_F_RXCSUM	  |
3423 			 NETIF_F_IP_CSUM  |
3424 			 NETIF_F_SCTP_CRC |
3425 			 NETIF_F_IPV6_CSUM;
3426 
3427 	vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER |
3428 			 NETIF_F_HW_VLAN_CTAG_TX     |
3429 			 NETIF_F_HW_VLAN_CTAG_RX;
3430 
3431 	/* Enable CTAG/STAG filtering by default in Double VLAN Mode (DVM) */
3432 	if (is_dvm_ena)
3433 		vlano_features |= NETIF_F_HW_VLAN_STAG_FILTER;
3434 
3435 	tso_features = NETIF_F_TSO			|
3436 		       NETIF_F_TSO_ECN			|
3437 		       NETIF_F_TSO6			|
3438 		       NETIF_F_GSO_GRE			|
3439 		       NETIF_F_GSO_UDP_TUNNEL		|
3440 		       NETIF_F_GSO_GRE_CSUM		|
3441 		       NETIF_F_GSO_UDP_TUNNEL_CSUM	|
3442 		       NETIF_F_GSO_PARTIAL		|
3443 		       NETIF_F_GSO_IPXIP4		|
3444 		       NETIF_F_GSO_IPXIP6		|
3445 		       NETIF_F_GSO_UDP_L4;
3446 
3447 	netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM |
3448 					NETIF_F_GSO_GRE_CSUM;
3449 	/* set features that user can change */
3450 	netdev->hw_features = dflt_features | csumo_features |
3451 			      vlano_features | tso_features;
3452 
3453 	/* add support for HW_CSUM on packets with MPLS header */
3454 	netdev->mpls_features =  NETIF_F_HW_CSUM |
3455 				 NETIF_F_TSO     |
3456 				 NETIF_F_TSO6;
3457 
3458 	/* enable features */
3459 	netdev->features |= netdev->hw_features;
3460 
3461 	netdev->hw_features |= NETIF_F_HW_TC;
3462 	netdev->hw_features |= NETIF_F_LOOPBACK;
3463 
3464 	/* encap and VLAN devices inherit default, csumo and tso features */
3465 	netdev->hw_enc_features |= dflt_features | csumo_features |
3466 				   tso_features;
3467 	netdev->vlan_features |= dflt_features | csumo_features |
3468 				 tso_features;
3469 
3470 	/* advertise support but don't enable by default since only one type of
3471 	 * VLAN offload can be enabled at a time (i.e. CTAG or STAG). When one
3472 	 * type turns on the other has to be turned off. This is enforced by the
3473 	 * ice_fix_features() ndo callback.
3474 	 */
3475 	if (is_dvm_ena)
3476 		netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX |
3477 			NETIF_F_HW_VLAN_STAG_TX;
3478 
3479 	/* Leave CRC / FCS stripping enabled by default, but allow the value to
3480 	 * be changed at runtime
3481 	 */
3482 	netdev->hw_features |= NETIF_F_RXFCS;
3483 
3484 	netif_set_tso_max_size(netdev, ICE_MAX_TSO_SIZE);
3485 }
3486 
3487 /**
3488  * ice_fill_rss_lut - Fill the RSS lookup table with default values
3489  * @lut: Lookup table
3490  * @rss_table_size: Lookup table size
3491  * @rss_size: Range of queue number for hashing
3492  */
3493 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size)
3494 {
3495 	u16 i;
3496 
3497 	for (i = 0; i < rss_table_size; i++)
3498 		lut[i] = i % rss_size;
3499 }
3500 
3501 /**
3502  * ice_pf_vsi_setup - Set up a PF VSI
3503  * @pf: board private structure
3504  * @pi: pointer to the port_info instance
3505  *
3506  * Returns pointer to the successfully allocated VSI software struct
3507  * on success, otherwise returns NULL on failure.
3508  */
3509 static struct ice_vsi *
3510 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3511 {
3512 	struct ice_vsi_cfg_params params = {};
3513 
3514 	params.type = ICE_VSI_PF;
3515 	params.pi = pi;
3516 	params.flags = ICE_VSI_FLAG_INIT;
3517 
3518 	return ice_vsi_setup(pf, &params);
3519 }
3520 
3521 static struct ice_vsi *
3522 ice_chnl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi,
3523 		   struct ice_channel *ch)
3524 {
3525 	struct ice_vsi_cfg_params params = {};
3526 
3527 	params.type = ICE_VSI_CHNL;
3528 	params.pi = pi;
3529 	params.ch = ch;
3530 	params.flags = ICE_VSI_FLAG_INIT;
3531 
3532 	return ice_vsi_setup(pf, &params);
3533 }
3534 
3535 /**
3536  * ice_ctrl_vsi_setup - Set up a control VSI
3537  * @pf: board private structure
3538  * @pi: pointer to the port_info instance
3539  *
3540  * Returns pointer to the successfully allocated VSI software struct
3541  * on success, otherwise returns NULL on failure.
3542  */
3543 static struct ice_vsi *
3544 ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3545 {
3546 	struct ice_vsi_cfg_params params = {};
3547 
3548 	params.type = ICE_VSI_CTRL;
3549 	params.pi = pi;
3550 	params.flags = ICE_VSI_FLAG_INIT;
3551 
3552 	return ice_vsi_setup(pf, &params);
3553 }
3554 
3555 /**
3556  * ice_lb_vsi_setup - Set up a loopback VSI
3557  * @pf: board private structure
3558  * @pi: pointer to the port_info instance
3559  *
3560  * Returns pointer to the successfully allocated VSI software struct
3561  * on success, otherwise returns NULL on failure.
3562  */
3563 struct ice_vsi *
3564 ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi)
3565 {
3566 	struct ice_vsi_cfg_params params = {};
3567 
3568 	params.type = ICE_VSI_LB;
3569 	params.pi = pi;
3570 	params.flags = ICE_VSI_FLAG_INIT;
3571 
3572 	return ice_vsi_setup(pf, &params);
3573 }
3574 
3575 /**
3576  * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload
3577  * @netdev: network interface to be adjusted
3578  * @proto: VLAN TPID
3579  * @vid: VLAN ID to be added
3580  *
3581  * net_device_ops implementation for adding VLAN IDs
3582  */
3583 static int
3584 ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
3585 {
3586 	struct ice_netdev_priv *np = netdev_priv(netdev);
3587 	struct ice_vsi_vlan_ops *vlan_ops;
3588 	struct ice_vsi *vsi = np->vsi;
3589 	struct ice_vlan vlan;
3590 	int ret;
3591 
3592 	/* VLAN 0 is added by default during load/reset */
3593 	if (!vid)
3594 		return 0;
3595 
3596 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
3597 		usleep_range(1000, 2000);
3598 
3599 	/* Add multicast promisc rule for the VLAN ID to be added if
3600 	 * all-multicast is currently enabled.
3601 	 */
3602 	if (vsi->current_netdev_flags & IFF_ALLMULTI) {
3603 		ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3604 					       ICE_MCAST_VLAN_PROMISC_BITS,
3605 					       vid);
3606 		if (ret)
3607 			goto finish;
3608 	}
3609 
3610 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
3611 
3612 	/* Add a switch rule for this VLAN ID so its corresponding VLAN tagged
3613 	 * packets aren't pruned by the device's internal switch on Rx
3614 	 */
3615 	vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0);
3616 	ret = vlan_ops->add_vlan(vsi, &vlan);
3617 	if (ret)
3618 		goto finish;
3619 
3620 	/* If all-multicast is currently enabled and this VLAN ID is only one
3621 	 * besides VLAN-0 we have to update look-up type of multicast promisc
3622 	 * rule for VLAN-0 from ICE_SW_LKUP_PROMISC to ICE_SW_LKUP_PROMISC_VLAN.
3623 	 */
3624 	if ((vsi->current_netdev_flags & IFF_ALLMULTI) &&
3625 	    ice_vsi_num_non_zero_vlans(vsi) == 1) {
3626 		ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3627 					   ICE_MCAST_PROMISC_BITS, 0);
3628 		ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3629 					 ICE_MCAST_VLAN_PROMISC_BITS, 0);
3630 	}
3631 
3632 finish:
3633 	clear_bit(ICE_CFG_BUSY, vsi->state);
3634 
3635 	return ret;
3636 }
3637 
3638 /**
3639  * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload
3640  * @netdev: network interface to be adjusted
3641  * @proto: VLAN TPID
3642  * @vid: VLAN ID to be removed
3643  *
3644  * net_device_ops implementation for removing VLAN IDs
3645  */
3646 static int
3647 ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
3648 {
3649 	struct ice_netdev_priv *np = netdev_priv(netdev);
3650 	struct ice_vsi_vlan_ops *vlan_ops;
3651 	struct ice_vsi *vsi = np->vsi;
3652 	struct ice_vlan vlan;
3653 	int ret;
3654 
3655 	/* don't allow removal of VLAN 0 */
3656 	if (!vid)
3657 		return 0;
3658 
3659 	while (test_and_set_bit(ICE_CFG_BUSY, vsi->state))
3660 		usleep_range(1000, 2000);
3661 
3662 	ret = ice_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3663 				    ICE_MCAST_VLAN_PROMISC_BITS, vid);
3664 	if (ret) {
3665 		netdev_err(netdev, "Error clearing multicast promiscuous mode on VSI %i\n",
3666 			   vsi->vsi_num);
3667 		vsi->current_netdev_flags |= IFF_ALLMULTI;
3668 	}
3669 
3670 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
3671 
3672 	/* Make sure VLAN delete is successful before updating VLAN
3673 	 * information
3674 	 */
3675 	vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0);
3676 	ret = vlan_ops->del_vlan(vsi, &vlan);
3677 	if (ret)
3678 		goto finish;
3679 
3680 	/* Remove multicast promisc rule for the removed VLAN ID if
3681 	 * all-multicast is enabled.
3682 	 */
3683 	if (vsi->current_netdev_flags & IFF_ALLMULTI)
3684 		ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3685 					   ICE_MCAST_VLAN_PROMISC_BITS, vid);
3686 
3687 	if (!ice_vsi_has_non_zero_vlans(vsi)) {
3688 		/* Update look-up type of multicast promisc rule for VLAN 0
3689 		 * from ICE_SW_LKUP_PROMISC_VLAN to ICE_SW_LKUP_PROMISC when
3690 		 * all-multicast is enabled and VLAN 0 is the only VLAN rule.
3691 		 */
3692 		if (vsi->current_netdev_flags & IFF_ALLMULTI) {
3693 			ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx,
3694 						   ICE_MCAST_VLAN_PROMISC_BITS,
3695 						   0);
3696 			ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
3697 						 ICE_MCAST_PROMISC_BITS, 0);
3698 		}
3699 	}
3700 
3701 finish:
3702 	clear_bit(ICE_CFG_BUSY, vsi->state);
3703 
3704 	return ret;
3705 }
3706 
3707 /**
3708  * ice_rep_indr_tc_block_unbind
3709  * @cb_priv: indirection block private data
3710  */
3711 static void ice_rep_indr_tc_block_unbind(void *cb_priv)
3712 {
3713 	struct ice_indr_block_priv *indr_priv = cb_priv;
3714 
3715 	list_del(&indr_priv->list);
3716 	kfree(indr_priv);
3717 }
3718 
3719 /**
3720  * ice_tc_indir_block_unregister - Unregister TC indirect block notifications
3721  * @vsi: VSI struct which has the netdev
3722  */
3723 static void ice_tc_indir_block_unregister(struct ice_vsi *vsi)
3724 {
3725 	struct ice_netdev_priv *np = netdev_priv(vsi->netdev);
3726 
3727 	flow_indr_dev_unregister(ice_indr_setup_tc_cb, np,
3728 				 ice_rep_indr_tc_block_unbind);
3729 }
3730 
3731 /**
3732  * ice_tc_indir_block_register - Register TC indirect block notifications
3733  * @vsi: VSI struct which has the netdev
3734  *
3735  * Returns 0 on success, negative value on failure
3736  */
3737 static int ice_tc_indir_block_register(struct ice_vsi *vsi)
3738 {
3739 	struct ice_netdev_priv *np;
3740 
3741 	if (!vsi || !vsi->netdev)
3742 		return -EINVAL;
3743 
3744 	np = netdev_priv(vsi->netdev);
3745 
3746 	INIT_LIST_HEAD(&np->tc_indr_block_priv_list);
3747 	return flow_indr_dev_register(ice_indr_setup_tc_cb, np);
3748 }
3749 
3750 /**
3751  * ice_get_avail_q_count - Get count of queues in use
3752  * @pf_qmap: bitmap to get queue use count from
3753  * @lock: pointer to a mutex that protects access to pf_qmap
3754  * @size: size of the bitmap
3755  */
3756 static u16
3757 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size)
3758 {
3759 	unsigned long bit;
3760 	u16 count = 0;
3761 
3762 	mutex_lock(lock);
3763 	for_each_clear_bit(bit, pf_qmap, size)
3764 		count++;
3765 	mutex_unlock(lock);
3766 
3767 	return count;
3768 }
3769 
3770 /**
3771  * ice_get_avail_txq_count - Get count of Tx queues in use
3772  * @pf: pointer to an ice_pf instance
3773  */
3774 u16 ice_get_avail_txq_count(struct ice_pf *pf)
3775 {
3776 	return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex,
3777 				     pf->max_pf_txqs);
3778 }
3779 
3780 /**
3781  * ice_get_avail_rxq_count - Get count of Rx queues in use
3782  * @pf: pointer to an ice_pf instance
3783  */
3784 u16 ice_get_avail_rxq_count(struct ice_pf *pf)
3785 {
3786 	return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex,
3787 				     pf->max_pf_rxqs);
3788 }
3789 
3790 /**
3791  * ice_deinit_pf - Unrolls initialziations done by ice_init_pf
3792  * @pf: board private structure to initialize
3793  */
3794 static void ice_deinit_pf(struct ice_pf *pf)
3795 {
3796 	ice_service_task_stop(pf);
3797 	mutex_destroy(&pf->adev_mutex);
3798 	mutex_destroy(&pf->sw_mutex);
3799 	mutex_destroy(&pf->tc_mutex);
3800 	mutex_destroy(&pf->avail_q_mutex);
3801 	mutex_destroy(&pf->vfs.table_lock);
3802 
3803 	if (pf->avail_txqs) {
3804 		bitmap_free(pf->avail_txqs);
3805 		pf->avail_txqs = NULL;
3806 	}
3807 
3808 	if (pf->avail_rxqs) {
3809 		bitmap_free(pf->avail_rxqs);
3810 		pf->avail_rxqs = NULL;
3811 	}
3812 
3813 	if (pf->ptp.clock)
3814 		ptp_clock_unregister(pf->ptp.clock);
3815 }
3816 
3817 /**
3818  * ice_set_pf_caps - set PFs capability flags
3819  * @pf: pointer to the PF instance
3820  */
3821 static void ice_set_pf_caps(struct ice_pf *pf)
3822 {
3823 	struct ice_hw_func_caps *func_caps = &pf->hw.func_caps;
3824 
3825 	clear_bit(ICE_FLAG_RDMA_ENA, pf->flags);
3826 	if (func_caps->common_cap.rdma)
3827 		set_bit(ICE_FLAG_RDMA_ENA, pf->flags);
3828 	clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
3829 	if (func_caps->common_cap.dcb)
3830 		set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
3831 	clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags);
3832 	if (func_caps->common_cap.sr_iov_1_1) {
3833 		set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags);
3834 		pf->vfs.num_supported = min_t(int, func_caps->num_allocd_vfs,
3835 					      ICE_MAX_SRIOV_VFS);
3836 	}
3837 	clear_bit(ICE_FLAG_RSS_ENA, pf->flags);
3838 	if (func_caps->common_cap.rss_table_size)
3839 		set_bit(ICE_FLAG_RSS_ENA, pf->flags);
3840 
3841 	clear_bit(ICE_FLAG_FD_ENA, pf->flags);
3842 	if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) {
3843 		u16 unused;
3844 
3845 		/* ctrl_vsi_idx will be set to a valid value when flow director
3846 		 * is setup by ice_init_fdir
3847 		 */
3848 		pf->ctrl_vsi_idx = ICE_NO_VSI;
3849 		set_bit(ICE_FLAG_FD_ENA, pf->flags);
3850 		/* force guaranteed filter pool for PF */
3851 		ice_alloc_fd_guar_item(&pf->hw, &unused,
3852 				       func_caps->fd_fltr_guar);
3853 		/* force shared filter pool for PF */
3854 		ice_alloc_fd_shrd_item(&pf->hw, &unused,
3855 				       func_caps->fd_fltr_best_effort);
3856 	}
3857 
3858 	clear_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags);
3859 	if (func_caps->common_cap.ieee_1588)
3860 		set_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags);
3861 
3862 	pf->max_pf_txqs = func_caps->common_cap.num_txq;
3863 	pf->max_pf_rxqs = func_caps->common_cap.num_rxq;
3864 }
3865 
3866 /**
3867  * ice_init_pf - Initialize general software structures (struct ice_pf)
3868  * @pf: board private structure to initialize
3869  */
3870 static int ice_init_pf(struct ice_pf *pf)
3871 {
3872 	ice_set_pf_caps(pf);
3873 
3874 	mutex_init(&pf->sw_mutex);
3875 	mutex_init(&pf->tc_mutex);
3876 	mutex_init(&pf->adev_mutex);
3877 
3878 	INIT_HLIST_HEAD(&pf->aq_wait_list);
3879 	spin_lock_init(&pf->aq_wait_lock);
3880 	init_waitqueue_head(&pf->aq_wait_queue);
3881 
3882 	init_waitqueue_head(&pf->reset_wait_queue);
3883 
3884 	/* setup service timer and periodic service task */
3885 	timer_setup(&pf->serv_tmr, ice_service_timer, 0);
3886 	pf->serv_tmr_period = HZ;
3887 	INIT_WORK(&pf->serv_task, ice_service_task);
3888 	clear_bit(ICE_SERVICE_SCHED, pf->state);
3889 
3890 	mutex_init(&pf->avail_q_mutex);
3891 	pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL);
3892 	if (!pf->avail_txqs)
3893 		return -ENOMEM;
3894 
3895 	pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL);
3896 	if (!pf->avail_rxqs) {
3897 		bitmap_free(pf->avail_txqs);
3898 		pf->avail_txqs = NULL;
3899 		return -ENOMEM;
3900 	}
3901 
3902 	mutex_init(&pf->vfs.table_lock);
3903 	hash_init(pf->vfs.table);
3904 	ice_mbx_init_snapshot(&pf->hw);
3905 
3906 	return 0;
3907 }
3908 
3909 /**
3910  * ice_is_wol_supported - check if WoL is supported
3911  * @hw: pointer to hardware info
3912  *
3913  * Check if WoL is supported based on the HW configuration.
3914  * Returns true if NVM supports and enables WoL for this port, false otherwise
3915  */
3916 bool ice_is_wol_supported(struct ice_hw *hw)
3917 {
3918 	u16 wol_ctrl;
3919 
3920 	/* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control
3921 	 * word) indicates WoL is not supported on the corresponding PF ID.
3922 	 */
3923 	if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl))
3924 		return false;
3925 
3926 	return !(BIT(hw->port_info->lport) & wol_ctrl);
3927 }
3928 
3929 /**
3930  * ice_vsi_recfg_qs - Change the number of queues on a VSI
3931  * @vsi: VSI being changed
3932  * @new_rx: new number of Rx queues
3933  * @new_tx: new number of Tx queues
3934  * @locked: is adev device_lock held
3935  *
3936  * Only change the number of queues if new_tx, or new_rx is non-0.
3937  *
3938  * Returns 0 on success.
3939  */
3940 int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx, bool locked)
3941 {
3942 	struct ice_pf *pf = vsi->back;
3943 	int err = 0, timeout = 50;
3944 
3945 	if (!new_rx && !new_tx)
3946 		return -EINVAL;
3947 
3948 	while (test_and_set_bit(ICE_CFG_BUSY, pf->state)) {
3949 		timeout--;
3950 		if (!timeout)
3951 			return -EBUSY;
3952 		usleep_range(1000, 2000);
3953 	}
3954 
3955 	if (new_tx)
3956 		vsi->req_txq = (u16)new_tx;
3957 	if (new_rx)
3958 		vsi->req_rxq = (u16)new_rx;
3959 
3960 	/* set for the next time the netdev is started */
3961 	if (!netif_running(vsi->netdev)) {
3962 		ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
3963 		dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n");
3964 		goto done;
3965 	}
3966 
3967 	ice_vsi_close(vsi);
3968 	ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
3969 	ice_pf_dcb_recfg(pf, locked);
3970 	ice_vsi_open(vsi);
3971 done:
3972 	clear_bit(ICE_CFG_BUSY, pf->state);
3973 	return err;
3974 }
3975 
3976 /**
3977  * ice_set_safe_mode_vlan_cfg - configure PF VSI to allow all VLANs in safe mode
3978  * @pf: PF to configure
3979  *
3980  * No VLAN offloads/filtering are advertised in safe mode so make sure the PF
3981  * VSI can still Tx/Rx VLAN tagged packets.
3982  */
3983 static void ice_set_safe_mode_vlan_cfg(struct ice_pf *pf)
3984 {
3985 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
3986 	struct ice_vsi_ctx *ctxt;
3987 	struct ice_hw *hw;
3988 	int status;
3989 
3990 	if (!vsi)
3991 		return;
3992 
3993 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
3994 	if (!ctxt)
3995 		return;
3996 
3997 	hw = &pf->hw;
3998 	ctxt->info = vsi->info;
3999 
4000 	ctxt->info.valid_sections =
4001 		cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID |
4002 			    ICE_AQ_VSI_PROP_SECURITY_VALID |
4003 			    ICE_AQ_VSI_PROP_SW_VALID);
4004 
4005 	/* disable VLAN anti-spoof */
4006 	ctxt->info.sec_flags &= ~(ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA <<
4007 				  ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S);
4008 
4009 	/* disable VLAN pruning and keep all other settings */
4010 	ctxt->info.sw_flags2 &= ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA;
4011 
4012 	/* allow all VLANs on Tx and don't strip on Rx */
4013 	ctxt->info.inner_vlan_flags = ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL |
4014 		ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING;
4015 
4016 	status = ice_update_vsi(hw, vsi->idx, ctxt, NULL);
4017 	if (status) {
4018 		dev_err(ice_pf_to_dev(vsi->back), "Failed to update VSI for safe mode VLANs, err %d aq_err %s\n",
4019 			status, ice_aq_str(hw->adminq.sq_last_status));
4020 	} else {
4021 		vsi->info.sec_flags = ctxt->info.sec_flags;
4022 		vsi->info.sw_flags2 = ctxt->info.sw_flags2;
4023 		vsi->info.inner_vlan_flags = ctxt->info.inner_vlan_flags;
4024 	}
4025 
4026 	kfree(ctxt);
4027 }
4028 
4029 /**
4030  * ice_log_pkg_init - log result of DDP package load
4031  * @hw: pointer to hardware info
4032  * @state: state of package load
4033  */
4034 static void ice_log_pkg_init(struct ice_hw *hw, enum ice_ddp_state state)
4035 {
4036 	struct ice_pf *pf = hw->back;
4037 	struct device *dev;
4038 
4039 	dev = ice_pf_to_dev(pf);
4040 
4041 	switch (state) {
4042 	case ICE_DDP_PKG_SUCCESS:
4043 		dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n",
4044 			 hw->active_pkg_name,
4045 			 hw->active_pkg_ver.major,
4046 			 hw->active_pkg_ver.minor,
4047 			 hw->active_pkg_ver.update,
4048 			 hw->active_pkg_ver.draft);
4049 		break;
4050 	case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED:
4051 		dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n",
4052 			 hw->active_pkg_name,
4053 			 hw->active_pkg_ver.major,
4054 			 hw->active_pkg_ver.minor,
4055 			 hw->active_pkg_ver.update,
4056 			 hw->active_pkg_ver.draft);
4057 		break;
4058 	case ICE_DDP_PKG_ALREADY_LOADED_NOT_SUPPORTED:
4059 		dev_err(dev, "The device has a DDP package that is not supported by the driver.  The device has package '%s' version %d.%d.x.x.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
4060 			hw->active_pkg_name,
4061 			hw->active_pkg_ver.major,
4062 			hw->active_pkg_ver.minor,
4063 			ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
4064 		break;
4065 	case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED:
4066 		dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device.  The device has package '%s' version %d.%d.%d.%d.  The package file found by the driver: '%s' version %d.%d.%d.%d.\n",
4067 			 hw->active_pkg_name,
4068 			 hw->active_pkg_ver.major,
4069 			 hw->active_pkg_ver.minor,
4070 			 hw->active_pkg_ver.update,
4071 			 hw->active_pkg_ver.draft,
4072 			 hw->pkg_name,
4073 			 hw->pkg_ver.major,
4074 			 hw->pkg_ver.minor,
4075 			 hw->pkg_ver.update,
4076 			 hw->pkg_ver.draft);
4077 		break;
4078 	case ICE_DDP_PKG_FW_MISMATCH:
4079 		dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package.  Please update the device's NVM.  Entering safe mode.\n");
4080 		break;
4081 	case ICE_DDP_PKG_INVALID_FILE:
4082 		dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n");
4083 		break;
4084 	case ICE_DDP_PKG_FILE_VERSION_TOO_HIGH:
4085 		dev_err(dev, "The DDP package file version is higher than the driver supports.  Please use an updated driver.  Entering Safe Mode.\n");
4086 		break;
4087 	case ICE_DDP_PKG_FILE_VERSION_TOO_LOW:
4088 		dev_err(dev, "The DDP package file version is lower than the driver supports.  The driver requires version %d.%d.x.x.  Please use an updated DDP Package file.  Entering Safe Mode.\n",
4089 			ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
4090 		break;
4091 	case ICE_DDP_PKG_FILE_SIGNATURE_INVALID:
4092 		dev_err(dev, "The DDP package could not be loaded because its signature is not valid.  Please use a valid DDP Package.  Entering Safe Mode.\n");
4093 		break;
4094 	case ICE_DDP_PKG_FILE_REVISION_TOO_LOW:
4095 		dev_err(dev, "The DDP Package could not be loaded because its security revision is too low.  Please use an updated DDP Package.  Entering Safe Mode.\n");
4096 		break;
4097 	case ICE_DDP_PKG_LOAD_ERROR:
4098 		dev_err(dev, "An error occurred on the device while loading the DDP package.  The device will be reset.\n");
4099 		/* poll for reset to complete */
4100 		if (ice_check_reset(hw))
4101 			dev_err(dev, "Error resetting device. Please reload the driver\n");
4102 		break;
4103 	case ICE_DDP_PKG_ERR:
4104 	default:
4105 		dev_err(dev, "An unknown error occurred when loading the DDP package.  Entering Safe Mode.\n");
4106 		break;
4107 	}
4108 }
4109 
4110 /**
4111  * ice_load_pkg - load/reload the DDP Package file
4112  * @firmware: firmware structure when firmware requested or NULL for reload
4113  * @pf: pointer to the PF instance
4114  *
4115  * Called on probe and post CORER/GLOBR rebuild to load DDP Package and
4116  * initialize HW tables.
4117  */
4118 static void
4119 ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf)
4120 {
4121 	enum ice_ddp_state state = ICE_DDP_PKG_ERR;
4122 	struct device *dev = ice_pf_to_dev(pf);
4123 	struct ice_hw *hw = &pf->hw;
4124 
4125 	/* Load DDP Package */
4126 	if (firmware && !hw->pkg_copy) {
4127 		state = ice_copy_and_init_pkg(hw, firmware->data,
4128 					      firmware->size);
4129 		ice_log_pkg_init(hw, state);
4130 	} else if (!firmware && hw->pkg_copy) {
4131 		/* Reload package during rebuild after CORER/GLOBR reset */
4132 		state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size);
4133 		ice_log_pkg_init(hw, state);
4134 	} else {
4135 		dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n");
4136 	}
4137 
4138 	if (!ice_is_init_pkg_successful(state)) {
4139 		/* Safe Mode */
4140 		clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags);
4141 		return;
4142 	}
4143 
4144 	/* Successful download package is the precondition for advanced
4145 	 * features, hence setting the ICE_FLAG_ADV_FEATURES flag
4146 	 */
4147 	set_bit(ICE_FLAG_ADV_FEATURES, pf->flags);
4148 }
4149 
4150 /**
4151  * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines
4152  * @pf: pointer to the PF structure
4153  *
4154  * There is no error returned here because the driver should be able to handle
4155  * 128 Byte cache lines, so we only print a warning in case issues are seen,
4156  * specifically with Tx.
4157  */
4158 static void ice_verify_cacheline_size(struct ice_pf *pf)
4159 {
4160 	if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M)
4161 		dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n",
4162 			 ICE_CACHE_LINE_BYTES);
4163 }
4164 
4165 /**
4166  * ice_send_version - update firmware with driver version
4167  * @pf: PF struct
4168  *
4169  * Returns 0 on success, else error code
4170  */
4171 static int ice_send_version(struct ice_pf *pf)
4172 {
4173 	struct ice_driver_ver dv;
4174 
4175 	dv.major_ver = 0xff;
4176 	dv.minor_ver = 0xff;
4177 	dv.build_ver = 0xff;
4178 	dv.subbuild_ver = 0;
4179 	strscpy((char *)dv.driver_string, UTS_RELEASE,
4180 		sizeof(dv.driver_string));
4181 	return ice_aq_send_driver_ver(&pf->hw, &dv, NULL);
4182 }
4183 
4184 /**
4185  * ice_init_fdir - Initialize flow director VSI and configuration
4186  * @pf: pointer to the PF instance
4187  *
4188  * returns 0 on success, negative on error
4189  */
4190 static int ice_init_fdir(struct ice_pf *pf)
4191 {
4192 	struct device *dev = ice_pf_to_dev(pf);
4193 	struct ice_vsi *ctrl_vsi;
4194 	int err;
4195 
4196 	/* Side Band Flow Director needs to have a control VSI.
4197 	 * Allocate it and store it in the PF.
4198 	 */
4199 	ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info);
4200 	if (!ctrl_vsi) {
4201 		dev_dbg(dev, "could not create control VSI\n");
4202 		return -ENOMEM;
4203 	}
4204 
4205 	err = ice_vsi_open_ctrl(ctrl_vsi);
4206 	if (err) {
4207 		dev_dbg(dev, "could not open control VSI\n");
4208 		goto err_vsi_open;
4209 	}
4210 
4211 	mutex_init(&pf->hw.fdir_fltr_lock);
4212 
4213 	err = ice_fdir_create_dflt_rules(pf);
4214 	if (err)
4215 		goto err_fdir_rule;
4216 
4217 	return 0;
4218 
4219 err_fdir_rule:
4220 	ice_fdir_release_flows(&pf->hw);
4221 	ice_vsi_close(ctrl_vsi);
4222 err_vsi_open:
4223 	ice_vsi_release(ctrl_vsi);
4224 	if (pf->ctrl_vsi_idx != ICE_NO_VSI) {
4225 		pf->vsi[pf->ctrl_vsi_idx] = NULL;
4226 		pf->ctrl_vsi_idx = ICE_NO_VSI;
4227 	}
4228 	return err;
4229 }
4230 
4231 static void ice_deinit_fdir(struct ice_pf *pf)
4232 {
4233 	struct ice_vsi *vsi = ice_get_ctrl_vsi(pf);
4234 
4235 	if (!vsi)
4236 		return;
4237 
4238 	ice_vsi_manage_fdir(vsi, false);
4239 	ice_vsi_release(vsi);
4240 	if (pf->ctrl_vsi_idx != ICE_NO_VSI) {
4241 		pf->vsi[pf->ctrl_vsi_idx] = NULL;
4242 		pf->ctrl_vsi_idx = ICE_NO_VSI;
4243 	}
4244 
4245 	mutex_destroy(&(&pf->hw)->fdir_fltr_lock);
4246 }
4247 
4248 /**
4249  * ice_get_opt_fw_name - return optional firmware file name or NULL
4250  * @pf: pointer to the PF instance
4251  */
4252 static char *ice_get_opt_fw_name(struct ice_pf *pf)
4253 {
4254 	/* Optional firmware name same as default with additional dash
4255 	 * followed by a EUI-64 identifier (PCIe Device Serial Number)
4256 	 */
4257 	struct pci_dev *pdev = pf->pdev;
4258 	char *opt_fw_filename;
4259 	u64 dsn;
4260 
4261 	/* Determine the name of the optional file using the DSN (two
4262 	 * dwords following the start of the DSN Capability).
4263 	 */
4264 	dsn = pci_get_dsn(pdev);
4265 	if (!dsn)
4266 		return NULL;
4267 
4268 	opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL);
4269 	if (!opt_fw_filename)
4270 		return NULL;
4271 
4272 	snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg",
4273 		 ICE_DDP_PKG_PATH, dsn);
4274 
4275 	return opt_fw_filename;
4276 }
4277 
4278 /**
4279  * ice_request_fw - Device initialization routine
4280  * @pf: pointer to the PF instance
4281  */
4282 static void ice_request_fw(struct ice_pf *pf)
4283 {
4284 	char *opt_fw_filename = ice_get_opt_fw_name(pf);
4285 	const struct firmware *firmware = NULL;
4286 	struct device *dev = ice_pf_to_dev(pf);
4287 	int err = 0;
4288 
4289 	/* optional device-specific DDP (if present) overrides the default DDP
4290 	 * package file. kernel logs a debug message if the file doesn't exist,
4291 	 * and warning messages for other errors.
4292 	 */
4293 	if (opt_fw_filename) {
4294 		err = firmware_request_nowarn(&firmware, opt_fw_filename, dev);
4295 		if (err) {
4296 			kfree(opt_fw_filename);
4297 			goto dflt_pkg_load;
4298 		}
4299 
4300 		/* request for firmware was successful. Download to device */
4301 		ice_load_pkg(firmware, pf);
4302 		kfree(opt_fw_filename);
4303 		release_firmware(firmware);
4304 		return;
4305 	}
4306 
4307 dflt_pkg_load:
4308 	err = request_firmware(&firmware, ICE_DDP_PKG_FILE, dev);
4309 	if (err) {
4310 		dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n");
4311 		return;
4312 	}
4313 
4314 	/* request for firmware was successful. Download to device */
4315 	ice_load_pkg(firmware, pf);
4316 	release_firmware(firmware);
4317 }
4318 
4319 /**
4320  * ice_print_wake_reason - show the wake up cause in the log
4321  * @pf: pointer to the PF struct
4322  */
4323 static void ice_print_wake_reason(struct ice_pf *pf)
4324 {
4325 	u32 wus = pf->wakeup_reason;
4326 	const char *wake_str;
4327 
4328 	/* if no wake event, nothing to print */
4329 	if (!wus)
4330 		return;
4331 
4332 	if (wus & PFPM_WUS_LNKC_M)
4333 		wake_str = "Link\n";
4334 	else if (wus & PFPM_WUS_MAG_M)
4335 		wake_str = "Magic Packet\n";
4336 	else if (wus & PFPM_WUS_MNG_M)
4337 		wake_str = "Management\n";
4338 	else if (wus & PFPM_WUS_FW_RST_WK_M)
4339 		wake_str = "Firmware Reset\n";
4340 	else
4341 		wake_str = "Unknown\n";
4342 
4343 	dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str);
4344 }
4345 
4346 /**
4347  * ice_register_netdev - register netdev
4348  * @vsi: pointer to the VSI struct
4349  */
4350 static int ice_register_netdev(struct ice_vsi *vsi)
4351 {
4352 	int err;
4353 
4354 	if (!vsi || !vsi->netdev)
4355 		return -EIO;
4356 
4357 	err = register_netdev(vsi->netdev);
4358 	if (err)
4359 		return err;
4360 
4361 	set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
4362 	netif_carrier_off(vsi->netdev);
4363 	netif_tx_stop_all_queues(vsi->netdev);
4364 
4365 	return 0;
4366 }
4367 
4368 static void ice_unregister_netdev(struct ice_vsi *vsi)
4369 {
4370 	if (!vsi || !vsi->netdev)
4371 		return;
4372 
4373 	unregister_netdev(vsi->netdev);
4374 	clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state);
4375 }
4376 
4377 /**
4378  * ice_cfg_netdev - Allocate, configure and register a netdev
4379  * @vsi: the VSI associated with the new netdev
4380  *
4381  * Returns 0 on success, negative value on failure
4382  */
4383 static int ice_cfg_netdev(struct ice_vsi *vsi)
4384 {
4385 	struct ice_netdev_priv *np;
4386 	struct net_device *netdev;
4387 	u8 mac_addr[ETH_ALEN];
4388 
4389 	netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq,
4390 				    vsi->alloc_rxq);
4391 	if (!netdev)
4392 		return -ENOMEM;
4393 
4394 	set_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state);
4395 	vsi->netdev = netdev;
4396 	np = netdev_priv(netdev);
4397 	np->vsi = vsi;
4398 
4399 	ice_set_netdev_features(netdev);
4400 	ice_set_ops(vsi);
4401 
4402 	if (vsi->type == ICE_VSI_PF) {
4403 		SET_NETDEV_DEV(netdev, ice_pf_to_dev(vsi->back));
4404 		ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
4405 		eth_hw_addr_set(netdev, mac_addr);
4406 	}
4407 
4408 	netdev->priv_flags |= IFF_UNICAST_FLT;
4409 
4410 	/* Setup netdev TC information */
4411 	ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc);
4412 
4413 	netdev->max_mtu = ICE_MAX_MTU;
4414 
4415 	return 0;
4416 }
4417 
4418 static void ice_decfg_netdev(struct ice_vsi *vsi)
4419 {
4420 	clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state);
4421 	free_netdev(vsi->netdev);
4422 	vsi->netdev = NULL;
4423 }
4424 
4425 static int ice_start_eth(struct ice_vsi *vsi)
4426 {
4427 	int err;
4428 
4429 	err = ice_init_mac_fltr(vsi->back);
4430 	if (err)
4431 		return err;
4432 
4433 	rtnl_lock();
4434 	err = ice_vsi_open(vsi);
4435 	rtnl_unlock();
4436 
4437 	return err;
4438 }
4439 
4440 static void ice_stop_eth(struct ice_vsi *vsi)
4441 {
4442 	ice_fltr_remove_all(vsi);
4443 	ice_vsi_close(vsi);
4444 }
4445 
4446 static int ice_init_eth(struct ice_pf *pf)
4447 {
4448 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4449 	int err;
4450 
4451 	if (!vsi)
4452 		return -EINVAL;
4453 
4454 	/* init channel list */
4455 	INIT_LIST_HEAD(&vsi->ch_list);
4456 
4457 	err = ice_cfg_netdev(vsi);
4458 	if (err)
4459 		return err;
4460 	/* Setup DCB netlink interface */
4461 	ice_dcbnl_setup(vsi);
4462 
4463 	err = ice_init_mac_fltr(pf);
4464 	if (err)
4465 		goto err_init_mac_fltr;
4466 
4467 	err = ice_devlink_create_pf_port(pf);
4468 	if (err)
4469 		goto err_devlink_create_pf_port;
4470 
4471 	SET_NETDEV_DEVLINK_PORT(vsi->netdev, &pf->devlink_port);
4472 
4473 	err = ice_register_netdev(vsi);
4474 	if (err)
4475 		goto err_register_netdev;
4476 
4477 	err = ice_tc_indir_block_register(vsi);
4478 	if (err)
4479 		goto err_tc_indir_block_register;
4480 
4481 	ice_napi_add(vsi);
4482 
4483 	return 0;
4484 
4485 err_tc_indir_block_register:
4486 	ice_unregister_netdev(vsi);
4487 err_register_netdev:
4488 	ice_devlink_destroy_pf_port(pf);
4489 err_devlink_create_pf_port:
4490 err_init_mac_fltr:
4491 	ice_decfg_netdev(vsi);
4492 	return err;
4493 }
4494 
4495 static void ice_deinit_eth(struct ice_pf *pf)
4496 {
4497 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4498 
4499 	if (!vsi)
4500 		return;
4501 
4502 	ice_vsi_close(vsi);
4503 	ice_unregister_netdev(vsi);
4504 	ice_devlink_destroy_pf_port(pf);
4505 	ice_tc_indir_block_unregister(vsi);
4506 	ice_decfg_netdev(vsi);
4507 }
4508 
4509 static int ice_init_dev(struct ice_pf *pf)
4510 {
4511 	struct device *dev = ice_pf_to_dev(pf);
4512 	struct ice_hw *hw = &pf->hw;
4513 	int err;
4514 
4515 	err = ice_init_hw(hw);
4516 	if (err) {
4517 		dev_err(dev, "ice_init_hw failed: %d\n", err);
4518 		return err;
4519 	}
4520 
4521 	ice_init_feature_support(pf);
4522 
4523 	ice_request_fw(pf);
4524 
4525 	/* if ice_request_fw fails, ICE_FLAG_ADV_FEATURES bit won't be
4526 	 * set in pf->state, which will cause ice_is_safe_mode to return
4527 	 * true
4528 	 */
4529 	if (ice_is_safe_mode(pf)) {
4530 		/* we already got function/device capabilities but these don't
4531 		 * reflect what the driver needs to do in safe mode. Instead of
4532 		 * adding conditional logic everywhere to ignore these
4533 		 * device/function capabilities, override them.
4534 		 */
4535 		ice_set_safe_mode_caps(hw);
4536 	}
4537 
4538 	err = ice_init_pf(pf);
4539 	if (err) {
4540 		dev_err(dev, "ice_init_pf failed: %d\n", err);
4541 		goto err_init_pf;
4542 	}
4543 
4544 	pf->hw.udp_tunnel_nic.set_port = ice_udp_tunnel_set_port;
4545 	pf->hw.udp_tunnel_nic.unset_port = ice_udp_tunnel_unset_port;
4546 	pf->hw.udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP;
4547 	pf->hw.udp_tunnel_nic.shared = &pf->hw.udp_tunnel_shared;
4548 	if (pf->hw.tnl.valid_count[TNL_VXLAN]) {
4549 		pf->hw.udp_tunnel_nic.tables[0].n_entries =
4550 			pf->hw.tnl.valid_count[TNL_VXLAN];
4551 		pf->hw.udp_tunnel_nic.tables[0].tunnel_types =
4552 			UDP_TUNNEL_TYPE_VXLAN;
4553 	}
4554 	if (pf->hw.tnl.valid_count[TNL_GENEVE]) {
4555 		pf->hw.udp_tunnel_nic.tables[1].n_entries =
4556 			pf->hw.tnl.valid_count[TNL_GENEVE];
4557 		pf->hw.udp_tunnel_nic.tables[1].tunnel_types =
4558 			UDP_TUNNEL_TYPE_GENEVE;
4559 	}
4560 
4561 	err = ice_init_interrupt_scheme(pf);
4562 	if (err) {
4563 		dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err);
4564 		err = -EIO;
4565 		goto err_init_interrupt_scheme;
4566 	}
4567 
4568 	/* In case of MSIX we are going to setup the misc vector right here
4569 	 * to handle admin queue events etc. In case of legacy and MSI
4570 	 * the misc functionality and queue processing is combined in
4571 	 * the same vector and that gets setup at open.
4572 	 */
4573 	err = ice_req_irq_msix_misc(pf);
4574 	if (err) {
4575 		dev_err(dev, "setup of misc vector failed: %d\n", err);
4576 		goto err_req_irq_msix_misc;
4577 	}
4578 
4579 	return 0;
4580 
4581 err_req_irq_msix_misc:
4582 	ice_clear_interrupt_scheme(pf);
4583 err_init_interrupt_scheme:
4584 	ice_deinit_pf(pf);
4585 err_init_pf:
4586 	ice_deinit_hw(hw);
4587 	return err;
4588 }
4589 
4590 static void ice_deinit_dev(struct ice_pf *pf)
4591 {
4592 	ice_free_irq_msix_misc(pf);
4593 	ice_deinit_pf(pf);
4594 	ice_deinit_hw(&pf->hw);
4595 
4596 	/* Service task is already stopped, so call reset directly. */
4597 	ice_reset(&pf->hw, ICE_RESET_PFR);
4598 	pci_wait_for_pending_transaction(pf->pdev);
4599 	ice_clear_interrupt_scheme(pf);
4600 }
4601 
4602 static void ice_init_features(struct ice_pf *pf)
4603 {
4604 	struct device *dev = ice_pf_to_dev(pf);
4605 
4606 	if (ice_is_safe_mode(pf))
4607 		return;
4608 
4609 	/* initialize DDP driven features */
4610 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
4611 		ice_ptp_init(pf);
4612 
4613 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
4614 		ice_gnss_init(pf);
4615 
4616 	/* Note: Flow director init failure is non-fatal to load */
4617 	if (ice_init_fdir(pf))
4618 		dev_err(dev, "could not initialize flow director\n");
4619 
4620 	/* Note: DCB init failure is non-fatal to load */
4621 	if (ice_init_pf_dcb(pf, false)) {
4622 		clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags);
4623 		clear_bit(ICE_FLAG_DCB_ENA, pf->flags);
4624 	} else {
4625 		ice_cfg_lldp_mib_change(&pf->hw, true);
4626 	}
4627 
4628 	if (ice_init_lag(pf))
4629 		dev_warn(dev, "Failed to init link aggregation support\n");
4630 }
4631 
4632 static void ice_deinit_features(struct ice_pf *pf)
4633 {
4634 	ice_deinit_lag(pf);
4635 	if (test_bit(ICE_FLAG_DCB_CAPABLE, pf->flags))
4636 		ice_cfg_lldp_mib_change(&pf->hw, false);
4637 	ice_deinit_fdir(pf);
4638 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
4639 		ice_gnss_exit(pf);
4640 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
4641 		ice_ptp_release(pf);
4642 }
4643 
4644 static void ice_init_wakeup(struct ice_pf *pf)
4645 {
4646 	/* Save wakeup reason register for later use */
4647 	pf->wakeup_reason = rd32(&pf->hw, PFPM_WUS);
4648 
4649 	/* check for a power management event */
4650 	ice_print_wake_reason(pf);
4651 
4652 	/* clear wake status, all bits */
4653 	wr32(&pf->hw, PFPM_WUS, U32_MAX);
4654 
4655 	/* Disable WoL at init, wait for user to enable */
4656 	device_set_wakeup_enable(ice_pf_to_dev(pf), false);
4657 }
4658 
4659 static int ice_init_link(struct ice_pf *pf)
4660 {
4661 	struct device *dev = ice_pf_to_dev(pf);
4662 	int err;
4663 
4664 	err = ice_init_link_events(pf->hw.port_info);
4665 	if (err) {
4666 		dev_err(dev, "ice_init_link_events failed: %d\n", err);
4667 		return err;
4668 	}
4669 
4670 	/* not a fatal error if this fails */
4671 	err = ice_init_nvm_phy_type(pf->hw.port_info);
4672 	if (err)
4673 		dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err);
4674 
4675 	/* not a fatal error if this fails */
4676 	err = ice_update_link_info(pf->hw.port_info);
4677 	if (err)
4678 		dev_err(dev, "ice_update_link_info failed: %d\n", err);
4679 
4680 	ice_init_link_dflt_override(pf->hw.port_info);
4681 
4682 	ice_check_link_cfg_err(pf,
4683 			       pf->hw.port_info->phy.link_info.link_cfg_err);
4684 
4685 	/* if media available, initialize PHY settings */
4686 	if (pf->hw.port_info->phy.link_info.link_info &
4687 	    ICE_AQ_MEDIA_AVAILABLE) {
4688 		/* not a fatal error if this fails */
4689 		err = ice_init_phy_user_cfg(pf->hw.port_info);
4690 		if (err)
4691 			dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err);
4692 
4693 		if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) {
4694 			struct ice_vsi *vsi = ice_get_main_vsi(pf);
4695 
4696 			if (vsi)
4697 				ice_configure_phy(vsi);
4698 		}
4699 	} else {
4700 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
4701 	}
4702 
4703 	return err;
4704 }
4705 
4706 static int ice_init_pf_sw(struct ice_pf *pf)
4707 {
4708 	bool dvm = ice_is_dvm_ena(&pf->hw);
4709 	struct ice_vsi *vsi;
4710 	int err;
4711 
4712 	/* create switch struct for the switch element created by FW on boot */
4713 	pf->first_sw = kzalloc(sizeof(*pf->first_sw), GFP_KERNEL);
4714 	if (!pf->first_sw)
4715 		return -ENOMEM;
4716 
4717 	if (pf->hw.evb_veb)
4718 		pf->first_sw->bridge_mode = BRIDGE_MODE_VEB;
4719 	else
4720 		pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA;
4721 
4722 	pf->first_sw->pf = pf;
4723 
4724 	/* record the sw_id available for later use */
4725 	pf->first_sw->sw_id = pf->hw.port_info->sw_id;
4726 
4727 	err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL);
4728 	if (err)
4729 		goto err_aq_set_port_params;
4730 
4731 	vsi = ice_pf_vsi_setup(pf, pf->hw.port_info);
4732 	if (!vsi) {
4733 		err = -ENOMEM;
4734 		goto err_pf_vsi_setup;
4735 	}
4736 
4737 	return 0;
4738 
4739 err_pf_vsi_setup:
4740 err_aq_set_port_params:
4741 	kfree(pf->first_sw);
4742 	return err;
4743 }
4744 
4745 static void ice_deinit_pf_sw(struct ice_pf *pf)
4746 {
4747 	struct ice_vsi *vsi = ice_get_main_vsi(pf);
4748 
4749 	if (!vsi)
4750 		return;
4751 
4752 	ice_vsi_release(vsi);
4753 	kfree(pf->first_sw);
4754 }
4755 
4756 static int ice_alloc_vsis(struct ice_pf *pf)
4757 {
4758 	struct device *dev = ice_pf_to_dev(pf);
4759 
4760 	pf->num_alloc_vsi = pf->hw.func_caps.guar_num_vsi;
4761 	if (!pf->num_alloc_vsi)
4762 		return -EIO;
4763 
4764 	if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) {
4765 		dev_warn(dev,
4766 			 "limiting the VSI count due to UDP tunnel limitation %d > %d\n",
4767 			 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES);
4768 		pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES;
4769 	}
4770 
4771 	pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi),
4772 			       GFP_KERNEL);
4773 	if (!pf->vsi)
4774 		return -ENOMEM;
4775 
4776 	pf->vsi_stats = devm_kcalloc(dev, pf->num_alloc_vsi,
4777 				     sizeof(*pf->vsi_stats), GFP_KERNEL);
4778 	if (!pf->vsi_stats) {
4779 		devm_kfree(dev, pf->vsi);
4780 		return -ENOMEM;
4781 	}
4782 
4783 	return 0;
4784 }
4785 
4786 static void ice_dealloc_vsis(struct ice_pf *pf)
4787 {
4788 	devm_kfree(ice_pf_to_dev(pf), pf->vsi_stats);
4789 	pf->vsi_stats = NULL;
4790 
4791 	pf->num_alloc_vsi = 0;
4792 	devm_kfree(ice_pf_to_dev(pf), pf->vsi);
4793 	pf->vsi = NULL;
4794 }
4795 
4796 static int ice_init_devlink(struct ice_pf *pf)
4797 {
4798 	int err;
4799 
4800 	err = ice_devlink_register_params(pf);
4801 	if (err)
4802 		return err;
4803 
4804 	ice_devlink_init_regions(pf);
4805 	ice_devlink_register(pf);
4806 
4807 	return 0;
4808 }
4809 
4810 static void ice_deinit_devlink(struct ice_pf *pf)
4811 {
4812 	ice_devlink_unregister(pf);
4813 	ice_devlink_destroy_regions(pf);
4814 	ice_devlink_unregister_params(pf);
4815 }
4816 
4817 static int ice_init(struct ice_pf *pf)
4818 {
4819 	int err;
4820 
4821 	err = ice_init_dev(pf);
4822 	if (err)
4823 		return err;
4824 
4825 	err = ice_alloc_vsis(pf);
4826 	if (err)
4827 		goto err_alloc_vsis;
4828 
4829 	err = ice_init_pf_sw(pf);
4830 	if (err)
4831 		goto err_init_pf_sw;
4832 
4833 	ice_init_wakeup(pf);
4834 
4835 	err = ice_init_link(pf);
4836 	if (err)
4837 		goto err_init_link;
4838 
4839 	err = ice_send_version(pf);
4840 	if (err)
4841 		goto err_init_link;
4842 
4843 	ice_verify_cacheline_size(pf);
4844 
4845 	if (ice_is_safe_mode(pf))
4846 		ice_set_safe_mode_vlan_cfg(pf);
4847 	else
4848 		/* print PCI link speed and width */
4849 		pcie_print_link_status(pf->pdev);
4850 
4851 	/* ready to go, so clear down state bit */
4852 	clear_bit(ICE_DOWN, pf->state);
4853 	clear_bit(ICE_SERVICE_DIS, pf->state);
4854 
4855 	/* since everything is good, start the service timer */
4856 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
4857 
4858 	return 0;
4859 
4860 err_init_link:
4861 	ice_deinit_pf_sw(pf);
4862 err_init_pf_sw:
4863 	ice_dealloc_vsis(pf);
4864 err_alloc_vsis:
4865 	ice_deinit_dev(pf);
4866 	return err;
4867 }
4868 
4869 static void ice_deinit(struct ice_pf *pf)
4870 {
4871 	set_bit(ICE_SERVICE_DIS, pf->state);
4872 	set_bit(ICE_DOWN, pf->state);
4873 
4874 	ice_deinit_pf_sw(pf);
4875 	ice_dealloc_vsis(pf);
4876 	ice_deinit_dev(pf);
4877 }
4878 
4879 /**
4880  * ice_load - load pf by init hw and starting VSI
4881  * @pf: pointer to the pf instance
4882  */
4883 int ice_load(struct ice_pf *pf)
4884 {
4885 	struct ice_vsi_cfg_params params = {};
4886 	struct ice_vsi *vsi;
4887 	int err;
4888 
4889 	err = ice_init_dev(pf);
4890 	if (err)
4891 		return err;
4892 
4893 	vsi = ice_get_main_vsi(pf);
4894 
4895 	params = ice_vsi_to_params(vsi);
4896 	params.flags = ICE_VSI_FLAG_INIT;
4897 
4898 	err = ice_vsi_cfg(vsi, &params);
4899 	if (err)
4900 		goto err_vsi_cfg;
4901 
4902 	err = ice_start_eth(ice_get_main_vsi(pf));
4903 	if (err)
4904 		goto err_start_eth;
4905 
4906 	err = ice_init_rdma(pf);
4907 	if (err)
4908 		goto err_init_rdma;
4909 
4910 	ice_init_features(pf);
4911 	ice_service_task_restart(pf);
4912 
4913 	clear_bit(ICE_DOWN, pf->state);
4914 
4915 	return 0;
4916 
4917 err_init_rdma:
4918 	ice_vsi_close(ice_get_main_vsi(pf));
4919 err_start_eth:
4920 	ice_vsi_decfg(ice_get_main_vsi(pf));
4921 err_vsi_cfg:
4922 	ice_deinit_dev(pf);
4923 	return err;
4924 }
4925 
4926 /**
4927  * ice_unload - unload pf by stopping VSI and deinit hw
4928  * @pf: pointer to the pf instance
4929  */
4930 void ice_unload(struct ice_pf *pf)
4931 {
4932 	ice_deinit_features(pf);
4933 	ice_deinit_rdma(pf);
4934 	ice_stop_eth(ice_get_main_vsi(pf));
4935 	ice_vsi_decfg(ice_get_main_vsi(pf));
4936 	ice_deinit_dev(pf);
4937 }
4938 
4939 /**
4940  * ice_probe - Device initialization routine
4941  * @pdev: PCI device information struct
4942  * @ent: entry in ice_pci_tbl
4943  *
4944  * Returns 0 on success, negative on failure
4945  */
4946 static int
4947 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent)
4948 {
4949 	struct device *dev = &pdev->dev;
4950 	struct ice_pf *pf;
4951 	struct ice_hw *hw;
4952 	int err;
4953 
4954 	if (pdev->is_virtfn) {
4955 		dev_err(dev, "can't probe a virtual function\n");
4956 		return -EINVAL;
4957 	}
4958 
4959 	/* this driver uses devres, see
4960 	 * Documentation/driver-api/driver-model/devres.rst
4961 	 */
4962 	err = pcim_enable_device(pdev);
4963 	if (err)
4964 		return err;
4965 
4966 	err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), dev_driver_string(dev));
4967 	if (err) {
4968 		dev_err(dev, "BAR0 I/O map error %d\n", err);
4969 		return err;
4970 	}
4971 
4972 	pf = ice_allocate_pf(dev);
4973 	if (!pf)
4974 		return -ENOMEM;
4975 
4976 	/* initialize Auxiliary index to invalid value */
4977 	pf->aux_idx = -1;
4978 
4979 	/* set up for high or low DMA */
4980 	err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
4981 	if (err) {
4982 		dev_err(dev, "DMA configuration failed: 0x%x\n", err);
4983 		return err;
4984 	}
4985 
4986 	pci_set_master(pdev);
4987 
4988 	pf->pdev = pdev;
4989 	pci_set_drvdata(pdev, pf);
4990 	set_bit(ICE_DOWN, pf->state);
4991 	/* Disable service task until DOWN bit is cleared */
4992 	set_bit(ICE_SERVICE_DIS, pf->state);
4993 
4994 	hw = &pf->hw;
4995 	hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0];
4996 	pci_save_state(pdev);
4997 
4998 	hw->back = pf;
4999 	hw->port_info = NULL;
5000 	hw->vendor_id = pdev->vendor;
5001 	hw->device_id = pdev->device;
5002 	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
5003 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
5004 	hw->subsystem_device_id = pdev->subsystem_device;
5005 	hw->bus.device = PCI_SLOT(pdev->devfn);
5006 	hw->bus.func = PCI_FUNC(pdev->devfn);
5007 	ice_set_ctrlq_len(hw);
5008 
5009 	pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M);
5010 
5011 #ifndef CONFIG_DYNAMIC_DEBUG
5012 	if (debug < -1)
5013 		hw->debug_mask = debug;
5014 #endif
5015 
5016 	err = ice_init(pf);
5017 	if (err)
5018 		goto err_init;
5019 
5020 	err = ice_init_eth(pf);
5021 	if (err)
5022 		goto err_init_eth;
5023 
5024 	err = ice_init_rdma(pf);
5025 	if (err)
5026 		goto err_init_rdma;
5027 
5028 	err = ice_init_devlink(pf);
5029 	if (err)
5030 		goto err_init_devlink;
5031 
5032 	ice_init_features(pf);
5033 
5034 	return 0;
5035 
5036 err_init_devlink:
5037 	ice_deinit_rdma(pf);
5038 err_init_rdma:
5039 	ice_deinit_eth(pf);
5040 err_init_eth:
5041 	ice_deinit(pf);
5042 err_init:
5043 	pci_disable_device(pdev);
5044 	return err;
5045 }
5046 
5047 /**
5048  * ice_set_wake - enable or disable Wake on LAN
5049  * @pf: pointer to the PF struct
5050  *
5051  * Simple helper for WoL control
5052  */
5053 static void ice_set_wake(struct ice_pf *pf)
5054 {
5055 	struct ice_hw *hw = &pf->hw;
5056 	bool wol = pf->wol_ena;
5057 
5058 	/* clear wake state, otherwise new wake events won't fire */
5059 	wr32(hw, PFPM_WUS, U32_MAX);
5060 
5061 	/* enable / disable APM wake up, no RMW needed */
5062 	wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0);
5063 
5064 	/* set magic packet filter enabled */
5065 	wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0);
5066 }
5067 
5068 /**
5069  * ice_setup_mc_magic_wake - setup device to wake on multicast magic packet
5070  * @pf: pointer to the PF struct
5071  *
5072  * Issue firmware command to enable multicast magic wake, making
5073  * sure that any locally administered address (LAA) is used for
5074  * wake, and that PF reset doesn't undo the LAA.
5075  */
5076 static void ice_setup_mc_magic_wake(struct ice_pf *pf)
5077 {
5078 	struct device *dev = ice_pf_to_dev(pf);
5079 	struct ice_hw *hw = &pf->hw;
5080 	u8 mac_addr[ETH_ALEN];
5081 	struct ice_vsi *vsi;
5082 	int status;
5083 	u8 flags;
5084 
5085 	if (!pf->wol_ena)
5086 		return;
5087 
5088 	vsi = ice_get_main_vsi(pf);
5089 	if (!vsi)
5090 		return;
5091 
5092 	/* Get current MAC address in case it's an LAA */
5093 	if (vsi->netdev)
5094 		ether_addr_copy(mac_addr, vsi->netdev->dev_addr);
5095 	else
5096 		ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
5097 
5098 	flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN |
5099 		ICE_AQC_MAN_MAC_UPDATE_LAA_WOL |
5100 		ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP;
5101 
5102 	status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL);
5103 	if (status)
5104 		dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %d aq_err %s\n",
5105 			status, ice_aq_str(hw->adminq.sq_last_status));
5106 }
5107 
5108 /**
5109  * ice_remove - Device removal routine
5110  * @pdev: PCI device information struct
5111  */
5112 static void ice_remove(struct pci_dev *pdev)
5113 {
5114 	struct ice_pf *pf = pci_get_drvdata(pdev);
5115 	int i;
5116 
5117 	for (i = 0; i < ICE_MAX_RESET_WAIT; i++) {
5118 		if (!ice_is_reset_in_progress(pf->state))
5119 			break;
5120 		msleep(100);
5121 	}
5122 
5123 	if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) {
5124 		set_bit(ICE_VF_RESETS_DISABLED, pf->state);
5125 		ice_free_vfs(pf);
5126 	}
5127 
5128 	ice_service_task_stop(pf);
5129 	ice_aq_cancel_waiting_tasks(pf);
5130 	set_bit(ICE_DOWN, pf->state);
5131 
5132 	if (!ice_is_safe_mode(pf))
5133 		ice_remove_arfs(pf);
5134 	ice_deinit_features(pf);
5135 	ice_deinit_devlink(pf);
5136 	ice_deinit_rdma(pf);
5137 	ice_deinit_eth(pf);
5138 	ice_deinit(pf);
5139 
5140 	ice_vsi_release_all(pf);
5141 
5142 	ice_setup_mc_magic_wake(pf);
5143 	ice_set_wake(pf);
5144 
5145 	pci_disable_device(pdev);
5146 }
5147 
5148 /**
5149  * ice_shutdown - PCI callback for shutting down device
5150  * @pdev: PCI device information struct
5151  */
5152 static void ice_shutdown(struct pci_dev *pdev)
5153 {
5154 	struct ice_pf *pf = pci_get_drvdata(pdev);
5155 
5156 	ice_remove(pdev);
5157 
5158 	if (system_state == SYSTEM_POWER_OFF) {
5159 		pci_wake_from_d3(pdev, pf->wol_ena);
5160 		pci_set_power_state(pdev, PCI_D3hot);
5161 	}
5162 }
5163 
5164 #ifdef CONFIG_PM
5165 /**
5166  * ice_prepare_for_shutdown - prep for PCI shutdown
5167  * @pf: board private structure
5168  *
5169  * Inform or close all dependent features in prep for PCI device shutdown
5170  */
5171 static void ice_prepare_for_shutdown(struct ice_pf *pf)
5172 {
5173 	struct ice_hw *hw = &pf->hw;
5174 	u32 v;
5175 
5176 	/* Notify VFs of impending reset */
5177 	if (ice_check_sq_alive(hw, &hw->mailboxq))
5178 		ice_vc_notify_reset(pf);
5179 
5180 	dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n");
5181 
5182 	/* disable the VSIs and their queues that are not already DOWN */
5183 	ice_pf_dis_all_vsi(pf, false);
5184 
5185 	ice_for_each_vsi(pf, v)
5186 		if (pf->vsi[v])
5187 			pf->vsi[v]->vsi_num = 0;
5188 
5189 	ice_shutdown_all_ctrlq(hw);
5190 }
5191 
5192 /**
5193  * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme
5194  * @pf: board private structure to reinitialize
5195  *
5196  * This routine reinitialize interrupt scheme that was cleared during
5197  * power management suspend callback.
5198  *
5199  * This should be called during resume routine to re-allocate the q_vectors
5200  * and reacquire interrupts.
5201  */
5202 static int ice_reinit_interrupt_scheme(struct ice_pf *pf)
5203 {
5204 	struct device *dev = ice_pf_to_dev(pf);
5205 	int ret, v;
5206 
5207 	/* Since we clear MSIX flag during suspend, we need to
5208 	 * set it back during resume...
5209 	 */
5210 
5211 	ret = ice_init_interrupt_scheme(pf);
5212 	if (ret) {
5213 		dev_err(dev, "Failed to re-initialize interrupt %d\n", ret);
5214 		return ret;
5215 	}
5216 
5217 	/* Remap vectors and rings, after successful re-init interrupts */
5218 	ice_for_each_vsi(pf, v) {
5219 		if (!pf->vsi[v])
5220 			continue;
5221 
5222 		ret = ice_vsi_alloc_q_vectors(pf->vsi[v]);
5223 		if (ret)
5224 			goto err_reinit;
5225 		ice_vsi_map_rings_to_vectors(pf->vsi[v]);
5226 	}
5227 
5228 	ret = ice_req_irq_msix_misc(pf);
5229 	if (ret) {
5230 		dev_err(dev, "Setting up misc vector failed after device suspend %d\n",
5231 			ret);
5232 		goto err_reinit;
5233 	}
5234 
5235 	return 0;
5236 
5237 err_reinit:
5238 	while (v--)
5239 		if (pf->vsi[v])
5240 			ice_vsi_free_q_vectors(pf->vsi[v]);
5241 
5242 	return ret;
5243 }
5244 
5245 /**
5246  * ice_suspend
5247  * @dev: generic device information structure
5248  *
5249  * Power Management callback to quiesce the device and prepare
5250  * for D3 transition.
5251  */
5252 static int __maybe_unused ice_suspend(struct device *dev)
5253 {
5254 	struct pci_dev *pdev = to_pci_dev(dev);
5255 	struct ice_pf *pf;
5256 	int disabled, v;
5257 
5258 	pf = pci_get_drvdata(pdev);
5259 
5260 	if (!ice_pf_state_is_nominal(pf)) {
5261 		dev_err(dev, "Device is not ready, no need to suspend it\n");
5262 		return -EBUSY;
5263 	}
5264 
5265 	/* Stop watchdog tasks until resume completion.
5266 	 * Even though it is most likely that the service task is
5267 	 * disabled if the device is suspended or down, the service task's
5268 	 * state is controlled by a different state bit, and we should
5269 	 * store and honor whatever state that bit is in at this point.
5270 	 */
5271 	disabled = ice_service_task_stop(pf);
5272 
5273 	ice_unplug_aux_dev(pf);
5274 
5275 	/* Already suspended?, then there is nothing to do */
5276 	if (test_and_set_bit(ICE_SUSPENDED, pf->state)) {
5277 		if (!disabled)
5278 			ice_service_task_restart(pf);
5279 		return 0;
5280 	}
5281 
5282 	if (test_bit(ICE_DOWN, pf->state) ||
5283 	    ice_is_reset_in_progress(pf->state)) {
5284 		dev_err(dev, "can't suspend device in reset or already down\n");
5285 		if (!disabled)
5286 			ice_service_task_restart(pf);
5287 		return 0;
5288 	}
5289 
5290 	ice_setup_mc_magic_wake(pf);
5291 
5292 	ice_prepare_for_shutdown(pf);
5293 
5294 	ice_set_wake(pf);
5295 
5296 	/* Free vectors, clear the interrupt scheme and release IRQs
5297 	 * for proper hibernation, especially with large number of CPUs.
5298 	 * Otherwise hibernation might fail when mapping all the vectors back
5299 	 * to CPU0.
5300 	 */
5301 	ice_free_irq_msix_misc(pf);
5302 	ice_for_each_vsi(pf, v) {
5303 		if (!pf->vsi[v])
5304 			continue;
5305 		ice_vsi_free_q_vectors(pf->vsi[v]);
5306 	}
5307 	ice_clear_interrupt_scheme(pf);
5308 
5309 	pci_save_state(pdev);
5310 	pci_wake_from_d3(pdev, pf->wol_ena);
5311 	pci_set_power_state(pdev, PCI_D3hot);
5312 	return 0;
5313 }
5314 
5315 /**
5316  * ice_resume - PM callback for waking up from D3
5317  * @dev: generic device information structure
5318  */
5319 static int __maybe_unused ice_resume(struct device *dev)
5320 {
5321 	struct pci_dev *pdev = to_pci_dev(dev);
5322 	enum ice_reset_req reset_type;
5323 	struct ice_pf *pf;
5324 	struct ice_hw *hw;
5325 	int ret;
5326 
5327 	pci_set_power_state(pdev, PCI_D0);
5328 	pci_restore_state(pdev);
5329 	pci_save_state(pdev);
5330 
5331 	if (!pci_device_is_present(pdev))
5332 		return -ENODEV;
5333 
5334 	ret = pci_enable_device_mem(pdev);
5335 	if (ret) {
5336 		dev_err(dev, "Cannot enable device after suspend\n");
5337 		return ret;
5338 	}
5339 
5340 	pf = pci_get_drvdata(pdev);
5341 	hw = &pf->hw;
5342 
5343 	pf->wakeup_reason = rd32(hw, PFPM_WUS);
5344 	ice_print_wake_reason(pf);
5345 
5346 	/* We cleared the interrupt scheme when we suspended, so we need to
5347 	 * restore it now to resume device functionality.
5348 	 */
5349 	ret = ice_reinit_interrupt_scheme(pf);
5350 	if (ret)
5351 		dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret);
5352 
5353 	clear_bit(ICE_DOWN, pf->state);
5354 	/* Now perform PF reset and rebuild */
5355 	reset_type = ICE_RESET_PFR;
5356 	/* re-enable service task for reset, but allow reset to schedule it */
5357 	clear_bit(ICE_SERVICE_DIS, pf->state);
5358 
5359 	if (ice_schedule_reset(pf, reset_type))
5360 		dev_err(dev, "Reset during resume failed.\n");
5361 
5362 	clear_bit(ICE_SUSPENDED, pf->state);
5363 	ice_service_task_restart(pf);
5364 
5365 	/* Restart the service task */
5366 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
5367 
5368 	return 0;
5369 }
5370 #endif /* CONFIG_PM */
5371 
5372 /**
5373  * ice_pci_err_detected - warning that PCI error has been detected
5374  * @pdev: PCI device information struct
5375  * @err: the type of PCI error
5376  *
5377  * Called to warn that something happened on the PCI bus and the error handling
5378  * is in progress.  Allows the driver to gracefully prepare/handle PCI errors.
5379  */
5380 static pci_ers_result_t
5381 ice_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t err)
5382 {
5383 	struct ice_pf *pf = pci_get_drvdata(pdev);
5384 
5385 	if (!pf) {
5386 		dev_err(&pdev->dev, "%s: unrecoverable device error %d\n",
5387 			__func__, err);
5388 		return PCI_ERS_RESULT_DISCONNECT;
5389 	}
5390 
5391 	if (!test_bit(ICE_SUSPENDED, pf->state)) {
5392 		ice_service_task_stop(pf);
5393 
5394 		if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) {
5395 			set_bit(ICE_PFR_REQ, pf->state);
5396 			ice_prepare_for_reset(pf, ICE_RESET_PFR);
5397 		}
5398 	}
5399 
5400 	return PCI_ERS_RESULT_NEED_RESET;
5401 }
5402 
5403 /**
5404  * ice_pci_err_slot_reset - a PCI slot reset has just happened
5405  * @pdev: PCI device information struct
5406  *
5407  * Called to determine if the driver can recover from the PCI slot reset by
5408  * using a register read to determine if the device is recoverable.
5409  */
5410 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev)
5411 {
5412 	struct ice_pf *pf = pci_get_drvdata(pdev);
5413 	pci_ers_result_t result;
5414 	int err;
5415 	u32 reg;
5416 
5417 	err = pci_enable_device_mem(pdev);
5418 	if (err) {
5419 		dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n",
5420 			err);
5421 		result = PCI_ERS_RESULT_DISCONNECT;
5422 	} else {
5423 		pci_set_master(pdev);
5424 		pci_restore_state(pdev);
5425 		pci_save_state(pdev);
5426 		pci_wake_from_d3(pdev, false);
5427 
5428 		/* Check for life */
5429 		reg = rd32(&pf->hw, GLGEN_RTRIG);
5430 		if (!reg)
5431 			result = PCI_ERS_RESULT_RECOVERED;
5432 		else
5433 			result = PCI_ERS_RESULT_DISCONNECT;
5434 	}
5435 
5436 	return result;
5437 }
5438 
5439 /**
5440  * ice_pci_err_resume - restart operations after PCI error recovery
5441  * @pdev: PCI device information struct
5442  *
5443  * Called to allow the driver to bring things back up after PCI error and/or
5444  * reset recovery have finished
5445  */
5446 static void ice_pci_err_resume(struct pci_dev *pdev)
5447 {
5448 	struct ice_pf *pf = pci_get_drvdata(pdev);
5449 
5450 	if (!pf) {
5451 		dev_err(&pdev->dev, "%s failed, device is unrecoverable\n",
5452 			__func__);
5453 		return;
5454 	}
5455 
5456 	if (test_bit(ICE_SUSPENDED, pf->state)) {
5457 		dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n",
5458 			__func__);
5459 		return;
5460 	}
5461 
5462 	ice_restore_all_vfs_msi_state(pdev);
5463 
5464 	ice_do_reset(pf, ICE_RESET_PFR);
5465 	ice_service_task_restart(pf);
5466 	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
5467 }
5468 
5469 /**
5470  * ice_pci_err_reset_prepare - prepare device driver for PCI reset
5471  * @pdev: PCI device information struct
5472  */
5473 static void ice_pci_err_reset_prepare(struct pci_dev *pdev)
5474 {
5475 	struct ice_pf *pf = pci_get_drvdata(pdev);
5476 
5477 	if (!test_bit(ICE_SUSPENDED, pf->state)) {
5478 		ice_service_task_stop(pf);
5479 
5480 		if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) {
5481 			set_bit(ICE_PFR_REQ, pf->state);
5482 			ice_prepare_for_reset(pf, ICE_RESET_PFR);
5483 		}
5484 	}
5485 }
5486 
5487 /**
5488  * ice_pci_err_reset_done - PCI reset done, device driver reset can begin
5489  * @pdev: PCI device information struct
5490  */
5491 static void ice_pci_err_reset_done(struct pci_dev *pdev)
5492 {
5493 	ice_pci_err_resume(pdev);
5494 }
5495 
5496 /* ice_pci_tbl - PCI Device ID Table
5497  *
5498  * Wildcard entries (PCI_ANY_ID) should come last
5499  * Last entry must be all 0s
5500  *
5501  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
5502  *   Class, Class Mask, private data (not used) }
5503  */
5504 static const struct pci_device_id ice_pci_tbl[] = {
5505 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE), 0 },
5506 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP), 0 },
5507 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP), 0 },
5508 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_BACKPLANE), 0 },
5509 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_QSFP), 0 },
5510 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP), 0 },
5511 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE), 0 },
5512 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP), 0 },
5513 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP), 0 },
5514 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T), 0 },
5515 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII), 0 },
5516 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE), 0 },
5517 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP), 0 },
5518 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP), 0 },
5519 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T), 0 },
5520 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII), 0 },
5521 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE), 0 },
5522 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP), 0 },
5523 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T), 0 },
5524 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII), 0 },
5525 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE), 0 },
5526 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP), 0 },
5527 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T), 0 },
5528 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE), 0 },
5529 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP), 0 },
5530 	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_E822_SI_DFLT), 0 },
5531 	/* required last entry */
5532 	{ 0, }
5533 };
5534 MODULE_DEVICE_TABLE(pci, ice_pci_tbl);
5535 
5536 static __maybe_unused SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume);
5537 
5538 static const struct pci_error_handlers ice_pci_err_handler = {
5539 	.error_detected = ice_pci_err_detected,
5540 	.slot_reset = ice_pci_err_slot_reset,
5541 	.reset_prepare = ice_pci_err_reset_prepare,
5542 	.reset_done = ice_pci_err_reset_done,
5543 	.resume = ice_pci_err_resume
5544 };
5545 
5546 static struct pci_driver ice_driver = {
5547 	.name = KBUILD_MODNAME,
5548 	.id_table = ice_pci_tbl,
5549 	.probe = ice_probe,
5550 	.remove = ice_remove,
5551 #ifdef CONFIG_PM
5552 	.driver.pm = &ice_pm_ops,
5553 #endif /* CONFIG_PM */
5554 	.shutdown = ice_shutdown,
5555 	.sriov_configure = ice_sriov_configure,
5556 	.err_handler = &ice_pci_err_handler
5557 };
5558 
5559 /**
5560  * ice_module_init - Driver registration routine
5561  *
5562  * ice_module_init is the first routine called when the driver is
5563  * loaded. All it does is register with the PCI subsystem.
5564  */
5565 static int __init ice_module_init(void)
5566 {
5567 	int status;
5568 
5569 	pr_info("%s\n", ice_driver_string);
5570 	pr_info("%s\n", ice_copyright);
5571 
5572 	ice_wq = alloc_workqueue("%s", 0, 0, KBUILD_MODNAME);
5573 	if (!ice_wq) {
5574 		pr_err("Failed to create workqueue\n");
5575 		return -ENOMEM;
5576 	}
5577 
5578 	status = pci_register_driver(&ice_driver);
5579 	if (status) {
5580 		pr_err("failed to register PCI driver, err %d\n", status);
5581 		destroy_workqueue(ice_wq);
5582 	}
5583 
5584 	return status;
5585 }
5586 module_init(ice_module_init);
5587 
5588 /**
5589  * ice_module_exit - Driver exit cleanup routine
5590  *
5591  * ice_module_exit is called just before the driver is removed
5592  * from memory.
5593  */
5594 static void __exit ice_module_exit(void)
5595 {
5596 	pci_unregister_driver(&ice_driver);
5597 	destroy_workqueue(ice_wq);
5598 	pr_info("module unloaded\n");
5599 }
5600 module_exit(ice_module_exit);
5601 
5602 /**
5603  * ice_set_mac_address - NDO callback to set MAC address
5604  * @netdev: network interface device structure
5605  * @pi: pointer to an address structure
5606  *
5607  * Returns 0 on success, negative on failure
5608  */
5609 static int ice_set_mac_address(struct net_device *netdev, void *pi)
5610 {
5611 	struct ice_netdev_priv *np = netdev_priv(netdev);
5612 	struct ice_vsi *vsi = np->vsi;
5613 	struct ice_pf *pf = vsi->back;
5614 	struct ice_hw *hw = &pf->hw;
5615 	struct sockaddr *addr = pi;
5616 	u8 old_mac[ETH_ALEN];
5617 	u8 flags = 0;
5618 	u8 *mac;
5619 	int err;
5620 
5621 	mac = (u8 *)addr->sa_data;
5622 
5623 	if (!is_valid_ether_addr(mac))
5624 		return -EADDRNOTAVAIL;
5625 
5626 	if (test_bit(ICE_DOWN, pf->state) ||
5627 	    ice_is_reset_in_progress(pf->state)) {
5628 		netdev_err(netdev, "can't set mac %pM. device not ready\n",
5629 			   mac);
5630 		return -EBUSY;
5631 	}
5632 
5633 	if (ice_chnl_dmac_fltr_cnt(pf)) {
5634 		netdev_err(netdev, "can't set mac %pM. Device has tc-flower filters, delete all of them and try again\n",
5635 			   mac);
5636 		return -EAGAIN;
5637 	}
5638 
5639 	netif_addr_lock_bh(netdev);
5640 	ether_addr_copy(old_mac, netdev->dev_addr);
5641 	/* change the netdev's MAC address */
5642 	eth_hw_addr_set(netdev, mac);
5643 	netif_addr_unlock_bh(netdev);
5644 
5645 	/* Clean up old MAC filter. Not an error if old filter doesn't exist */
5646 	err = ice_fltr_remove_mac(vsi, old_mac, ICE_FWD_TO_VSI);
5647 	if (err && err != -ENOENT) {
5648 		err = -EADDRNOTAVAIL;
5649 		goto err_update_filters;
5650 	}
5651 
5652 	/* Add filter for new MAC. If filter exists, return success */
5653 	err = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI);
5654 	if (err == -EEXIST) {
5655 		/* Although this MAC filter is already present in hardware it's
5656 		 * possible in some cases (e.g. bonding) that dev_addr was
5657 		 * modified outside of the driver and needs to be restored back
5658 		 * to this value.
5659 		 */
5660 		netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac);
5661 
5662 		return 0;
5663 	} else if (err) {
5664 		/* error if the new filter addition failed */
5665 		err = -EADDRNOTAVAIL;
5666 	}
5667 
5668 err_update_filters:
5669 	if (err) {
5670 		netdev_err(netdev, "can't set MAC %pM. filter update failed\n",
5671 			   mac);
5672 		netif_addr_lock_bh(netdev);
5673 		eth_hw_addr_set(netdev, old_mac);
5674 		netif_addr_unlock_bh(netdev);
5675 		return err;
5676 	}
5677 
5678 	netdev_dbg(vsi->netdev, "updated MAC address to %pM\n",
5679 		   netdev->dev_addr);
5680 
5681 	/* write new MAC address to the firmware */
5682 	flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
5683 	err = ice_aq_manage_mac_write(hw, mac, flags, NULL);
5684 	if (err) {
5685 		netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %d\n",
5686 			   mac, err);
5687 	}
5688 	return 0;
5689 }
5690 
5691 /**
5692  * ice_set_rx_mode - NDO callback to set the netdev filters
5693  * @netdev: network interface device structure
5694  */
5695 static void ice_set_rx_mode(struct net_device *netdev)
5696 {
5697 	struct ice_netdev_priv *np = netdev_priv(netdev);
5698 	struct ice_vsi *vsi = np->vsi;
5699 
5700 	if (!vsi)
5701 		return;
5702 
5703 	/* Set the flags to synchronize filters
5704 	 * ndo_set_rx_mode may be triggered even without a change in netdev
5705 	 * flags
5706 	 */
5707 	set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state);
5708 	set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state);
5709 	set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags);
5710 
5711 	/* schedule our worker thread which will take care of
5712 	 * applying the new filter changes
5713 	 */
5714 	ice_service_task_schedule(vsi->back);
5715 }
5716 
5717 /**
5718  * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate
5719  * @netdev: network interface device structure
5720  * @queue_index: Queue ID
5721  * @maxrate: maximum bandwidth in Mbps
5722  */
5723 static int
5724 ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate)
5725 {
5726 	struct ice_netdev_priv *np = netdev_priv(netdev);
5727 	struct ice_vsi *vsi = np->vsi;
5728 	u16 q_handle;
5729 	int status;
5730 	u8 tc;
5731 
5732 	/* Validate maxrate requested is within permitted range */
5733 	if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) {
5734 		netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n",
5735 			   maxrate, queue_index);
5736 		return -EINVAL;
5737 	}
5738 
5739 	q_handle = vsi->tx_rings[queue_index]->q_handle;
5740 	tc = ice_dcb_get_tc(vsi, queue_index);
5741 
5742 	vsi = ice_locate_vsi_using_queue(vsi, queue_index);
5743 	if (!vsi) {
5744 		netdev_err(netdev, "Invalid VSI for given queue %d\n",
5745 			   queue_index);
5746 		return -EINVAL;
5747 	}
5748 
5749 	/* Set BW back to default, when user set maxrate to 0 */
5750 	if (!maxrate)
5751 		status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc,
5752 					       q_handle, ICE_MAX_BW);
5753 	else
5754 		status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc,
5755 					  q_handle, ICE_MAX_BW, maxrate * 1000);
5756 	if (status)
5757 		netdev_err(netdev, "Unable to set Tx max rate, error %d\n",
5758 			   status);
5759 
5760 	return status;
5761 }
5762 
5763 /**
5764  * ice_fdb_add - add an entry to the hardware database
5765  * @ndm: the input from the stack
5766  * @tb: pointer to array of nladdr (unused)
5767  * @dev: the net device pointer
5768  * @addr: the MAC address entry being added
5769  * @vid: VLAN ID
5770  * @flags: instructions from stack about fdb operation
5771  * @extack: netlink extended ack
5772  */
5773 static int
5774 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[],
5775 	    struct net_device *dev, const unsigned char *addr, u16 vid,
5776 	    u16 flags, struct netlink_ext_ack __always_unused *extack)
5777 {
5778 	int err;
5779 
5780 	if (vid) {
5781 		netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n");
5782 		return -EINVAL;
5783 	}
5784 	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
5785 		netdev_err(dev, "FDB only supports static addresses\n");
5786 		return -EINVAL;
5787 	}
5788 
5789 	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
5790 		err = dev_uc_add_excl(dev, addr);
5791 	else if (is_multicast_ether_addr(addr))
5792 		err = dev_mc_add_excl(dev, addr);
5793 	else
5794 		err = -EINVAL;
5795 
5796 	/* Only return duplicate errors if NLM_F_EXCL is set */
5797 	if (err == -EEXIST && !(flags & NLM_F_EXCL))
5798 		err = 0;
5799 
5800 	return err;
5801 }
5802 
5803 /**
5804  * ice_fdb_del - delete an entry from the hardware database
5805  * @ndm: the input from the stack
5806  * @tb: pointer to array of nladdr (unused)
5807  * @dev: the net device pointer
5808  * @addr: the MAC address entry being added
5809  * @vid: VLAN ID
5810  * @extack: netlink extended ack
5811  */
5812 static int
5813 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
5814 	    struct net_device *dev, const unsigned char *addr,
5815 	    __always_unused u16 vid, struct netlink_ext_ack *extack)
5816 {
5817 	int err;
5818 
5819 	if (ndm->ndm_state & NUD_PERMANENT) {
5820 		netdev_err(dev, "FDB only supports static addresses\n");
5821 		return -EINVAL;
5822 	}
5823 
5824 	if (is_unicast_ether_addr(addr))
5825 		err = dev_uc_del(dev, addr);
5826 	else if (is_multicast_ether_addr(addr))
5827 		err = dev_mc_del(dev, addr);
5828 	else
5829 		err = -EINVAL;
5830 
5831 	return err;
5832 }
5833 
5834 #define NETIF_VLAN_OFFLOAD_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
5835 					 NETIF_F_HW_VLAN_CTAG_TX | \
5836 					 NETIF_F_HW_VLAN_STAG_RX | \
5837 					 NETIF_F_HW_VLAN_STAG_TX)
5838 
5839 #define NETIF_VLAN_STRIPPING_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
5840 					 NETIF_F_HW_VLAN_STAG_RX)
5841 
5842 #define NETIF_VLAN_FILTERING_FEATURES	(NETIF_F_HW_VLAN_CTAG_FILTER | \
5843 					 NETIF_F_HW_VLAN_STAG_FILTER)
5844 
5845 /**
5846  * ice_fix_features - fix the netdev features flags based on device limitations
5847  * @netdev: ptr to the netdev that flags are being fixed on
5848  * @features: features that need to be checked and possibly fixed
5849  *
5850  * Make sure any fixups are made to features in this callback. This enables the
5851  * driver to not have to check unsupported configurations throughout the driver
5852  * because that's the responsiblity of this callback.
5853  *
5854  * Single VLAN Mode (SVM) Supported Features:
5855  *	NETIF_F_HW_VLAN_CTAG_FILTER
5856  *	NETIF_F_HW_VLAN_CTAG_RX
5857  *	NETIF_F_HW_VLAN_CTAG_TX
5858  *
5859  * Double VLAN Mode (DVM) Supported Features:
5860  *	NETIF_F_HW_VLAN_CTAG_FILTER
5861  *	NETIF_F_HW_VLAN_CTAG_RX
5862  *	NETIF_F_HW_VLAN_CTAG_TX
5863  *
5864  *	NETIF_F_HW_VLAN_STAG_FILTER
5865  *	NETIF_HW_VLAN_STAG_RX
5866  *	NETIF_HW_VLAN_STAG_TX
5867  *
5868  * Features that need fixing:
5869  *	Cannot simultaneously enable CTAG and STAG stripping and/or insertion.
5870  *	These are mutually exlusive as the VSI context cannot support multiple
5871  *	VLAN ethertypes simultaneously for stripping and/or insertion. If this
5872  *	is not done, then default to clearing the requested STAG offload
5873  *	settings.
5874  *
5875  *	All supported filtering has to be enabled or disabled together. For
5876  *	example, in DVM, CTAG and STAG filtering have to be enabled and disabled
5877  *	together. If this is not done, then default to VLAN filtering disabled.
5878  *	These are mutually exclusive as there is currently no way to
5879  *	enable/disable VLAN filtering based on VLAN ethertype when using VLAN
5880  *	prune rules.
5881  */
5882 static netdev_features_t
5883 ice_fix_features(struct net_device *netdev, netdev_features_t features)
5884 {
5885 	struct ice_netdev_priv *np = netdev_priv(netdev);
5886 	netdev_features_t req_vlan_fltr, cur_vlan_fltr;
5887 	bool cur_ctag, cur_stag, req_ctag, req_stag;
5888 
5889 	cur_vlan_fltr = netdev->features & NETIF_VLAN_FILTERING_FEATURES;
5890 	cur_ctag = cur_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER;
5891 	cur_stag = cur_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER;
5892 
5893 	req_vlan_fltr = features & NETIF_VLAN_FILTERING_FEATURES;
5894 	req_ctag = req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER;
5895 	req_stag = req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER;
5896 
5897 	if (req_vlan_fltr != cur_vlan_fltr) {
5898 		if (ice_is_dvm_ena(&np->vsi->back->hw)) {
5899 			if (req_ctag && req_stag) {
5900 				features |= NETIF_VLAN_FILTERING_FEATURES;
5901 			} else if (!req_ctag && !req_stag) {
5902 				features &= ~NETIF_VLAN_FILTERING_FEATURES;
5903 			} else if ((!cur_ctag && req_ctag && !cur_stag) ||
5904 				   (!cur_stag && req_stag && !cur_ctag)) {
5905 				features |= NETIF_VLAN_FILTERING_FEATURES;
5906 				netdev_warn(netdev,  "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been enabled for both types.\n");
5907 			} else if ((cur_ctag && !req_ctag && cur_stag) ||
5908 				   (cur_stag && !req_stag && cur_ctag)) {
5909 				features &= ~NETIF_VLAN_FILTERING_FEATURES;
5910 				netdev_warn(netdev,  "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been disabled for both types.\n");
5911 			}
5912 		} else {
5913 			if (req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER)
5914 				netdev_warn(netdev, "cannot support requested 802.1ad filtering setting in SVM mode\n");
5915 
5916 			if (req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER)
5917 				features |= NETIF_F_HW_VLAN_CTAG_FILTER;
5918 		}
5919 	}
5920 
5921 	if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
5922 	    (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))) {
5923 		netdev_warn(netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
5924 		features &= ~(NETIF_F_HW_VLAN_STAG_RX |
5925 			      NETIF_F_HW_VLAN_STAG_TX);
5926 	}
5927 
5928 	if (!(netdev->features & NETIF_F_RXFCS) &&
5929 	    (features & NETIF_F_RXFCS) &&
5930 	    (features & NETIF_VLAN_STRIPPING_FEATURES) &&
5931 	    !ice_vsi_has_non_zero_vlans(np->vsi)) {
5932 		netdev_warn(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n");
5933 		features &= ~NETIF_VLAN_STRIPPING_FEATURES;
5934 	}
5935 
5936 	return features;
5937 }
5938 
5939 /**
5940  * ice_set_vlan_offload_features - set VLAN offload features for the PF VSI
5941  * @vsi: PF's VSI
5942  * @features: features used to determine VLAN offload settings
5943  *
5944  * First, determine the vlan_ethertype based on the VLAN offload bits in
5945  * features. Then determine if stripping and insertion should be enabled or
5946  * disabled. Finally enable or disable VLAN stripping and insertion.
5947  */
5948 static int
5949 ice_set_vlan_offload_features(struct ice_vsi *vsi, netdev_features_t features)
5950 {
5951 	bool enable_stripping = true, enable_insertion = true;
5952 	struct ice_vsi_vlan_ops *vlan_ops;
5953 	int strip_err = 0, insert_err = 0;
5954 	u16 vlan_ethertype = 0;
5955 
5956 	vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
5957 
5958 	if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
5959 		vlan_ethertype = ETH_P_8021AD;
5960 	else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
5961 		vlan_ethertype = ETH_P_8021Q;
5962 
5963 	if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
5964 		enable_stripping = false;
5965 	if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
5966 		enable_insertion = false;
5967 
5968 	if (enable_stripping)
5969 		strip_err = vlan_ops->ena_stripping(vsi, vlan_ethertype);
5970 	else
5971 		strip_err = vlan_ops->dis_stripping(vsi);
5972 
5973 	if (enable_insertion)
5974 		insert_err = vlan_ops->ena_insertion(vsi, vlan_ethertype);
5975 	else
5976 		insert_err = vlan_ops->dis_insertion(vsi);
5977 
5978 	if (strip_err || insert_err)
5979 		return -EIO;
5980 
5981 	return 0;
5982 }
5983 
5984 /**
5985  * ice_set_vlan_filtering_features - set VLAN filtering features for the PF VSI
5986  * @vsi: PF's VSI
5987  * @features: features used to determine VLAN filtering settings
5988  *
5989  * Enable or disable Rx VLAN filtering based on the VLAN filtering bits in the
5990  * features.
5991  */
5992 static int
5993 ice_set_vlan_filtering_features(struct ice_vsi *vsi, netdev_features_t features)
5994 {
5995 	struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi);
5996 	int err = 0;
5997 
5998 	/* support Single VLAN Mode (SVM) and Double VLAN Mode (DVM) by checking
5999 	 * if either bit is set
6000 	 */
6001 	if (features &
6002 	    (NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER))
6003 		err = vlan_ops->ena_rx_filtering(vsi);
6004 	else
6005 		err = vlan_ops->dis_rx_filtering(vsi);
6006 
6007 	return err;
6008 }
6009 
6010 /**
6011  * ice_set_vlan_features - set VLAN settings based on suggested feature set
6012  * @netdev: ptr to the netdev being adjusted
6013  * @features: the feature set that the stack is suggesting
6014  *
6015  * Only update VLAN settings if the requested_vlan_features are different than
6016  * the current_vlan_features.
6017  */
6018 static int
6019 ice_set_vlan_features(struct net_device *netdev, netdev_features_t features)
6020 {
6021 	netdev_features_t current_vlan_features, requested_vlan_features;
6022 	struct ice_netdev_priv *np = netdev_priv(netdev);
6023 	struct ice_vsi *vsi = np->vsi;
6024 	int err;
6025 
6026 	current_vlan_features = netdev->features & NETIF_VLAN_OFFLOAD_FEATURES;
6027 	requested_vlan_features = features & NETIF_VLAN_OFFLOAD_FEATURES;
6028 	if (current_vlan_features ^ requested_vlan_features) {
6029 		if ((features & NETIF_F_RXFCS) &&
6030 		    (features & NETIF_VLAN_STRIPPING_FEATURES)) {
6031 			dev_err(ice_pf_to_dev(vsi->back),
6032 				"To enable VLAN stripping, you must first enable FCS/CRC stripping\n");
6033 			return -EIO;
6034 		}
6035 
6036 		err = ice_set_vlan_offload_features(vsi, features);
6037 		if (err)
6038 			return err;
6039 	}
6040 
6041 	current_vlan_features = netdev->features &
6042 		NETIF_VLAN_FILTERING_FEATURES;
6043 	requested_vlan_features = features & NETIF_VLAN_FILTERING_FEATURES;
6044 	if (current_vlan_features ^ requested_vlan_features) {
6045 		err = ice_set_vlan_filtering_features(vsi, features);
6046 		if (err)
6047 			return err;
6048 	}
6049 
6050 	return 0;
6051 }
6052 
6053 /**
6054  * ice_set_loopback - turn on/off loopback mode on underlying PF
6055  * @vsi: ptr to VSI
6056  * @ena: flag to indicate the on/off setting
6057  */
6058 static int ice_set_loopback(struct ice_vsi *vsi, bool ena)
6059 {
6060 	bool if_running = netif_running(vsi->netdev);
6061 	int ret;
6062 
6063 	if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) {
6064 		ret = ice_down(vsi);
6065 		if (ret) {
6066 			netdev_err(vsi->netdev, "Preparing device to toggle loopback failed\n");
6067 			return ret;
6068 		}
6069 	}
6070 	ret = ice_aq_set_mac_loopback(&vsi->back->hw, ena, NULL);
6071 	if (ret)
6072 		netdev_err(vsi->netdev, "Failed to toggle loopback state\n");
6073 	if (if_running)
6074 		ret = ice_up(vsi);
6075 
6076 	return ret;
6077 }
6078 
6079 /**
6080  * ice_set_features - set the netdev feature flags
6081  * @netdev: ptr to the netdev being adjusted
6082  * @features: the feature set that the stack is suggesting
6083  */
6084 static int
6085 ice_set_features(struct net_device *netdev, netdev_features_t features)
6086 {
6087 	netdev_features_t changed = netdev->features ^ features;
6088 	struct ice_netdev_priv *np = netdev_priv(netdev);
6089 	struct ice_vsi *vsi = np->vsi;
6090 	struct ice_pf *pf = vsi->back;
6091 	int ret = 0;
6092 
6093 	/* Don't set any netdev advanced features with device in Safe Mode */
6094 	if (ice_is_safe_mode(pf)) {
6095 		dev_err(ice_pf_to_dev(pf),
6096 			"Device is in Safe Mode - not enabling advanced netdev features\n");
6097 		return ret;
6098 	}
6099 
6100 	/* Do not change setting during reset */
6101 	if (ice_is_reset_in_progress(pf->state)) {
6102 		dev_err(ice_pf_to_dev(pf),
6103 			"Device is resetting, changing advanced netdev features temporarily unavailable.\n");
6104 		return -EBUSY;
6105 	}
6106 
6107 	/* Multiple features can be changed in one call so keep features in
6108 	 * separate if/else statements to guarantee each feature is checked
6109 	 */
6110 	if (changed & NETIF_F_RXHASH)
6111 		ice_vsi_manage_rss_lut(vsi, !!(features & NETIF_F_RXHASH));
6112 
6113 	ret = ice_set_vlan_features(netdev, features);
6114 	if (ret)
6115 		return ret;
6116 
6117 	/* Turn on receive of FCS aka CRC, and after setting this
6118 	 * flag the packet data will have the 4 byte CRC appended
6119 	 */
6120 	if (changed & NETIF_F_RXFCS) {
6121 		if ((features & NETIF_F_RXFCS) &&
6122 		    (features & NETIF_VLAN_STRIPPING_FEATURES)) {
6123 			dev_err(ice_pf_to_dev(vsi->back),
6124 				"To disable FCS/CRC stripping, you must first disable VLAN stripping\n");
6125 			return -EIO;
6126 		}
6127 
6128 		ice_vsi_cfg_crc_strip(vsi, !!(features & NETIF_F_RXFCS));
6129 		ret = ice_down_up(vsi);
6130 		if (ret)
6131 			return ret;
6132 	}
6133 
6134 	if (changed & NETIF_F_NTUPLE) {
6135 		bool ena = !!(features & NETIF_F_NTUPLE);
6136 
6137 		ice_vsi_manage_fdir(vsi, ena);
6138 		ena ? ice_init_arfs(vsi) : ice_clear_arfs(vsi);
6139 	}
6140 
6141 	/* don't turn off hw_tc_offload when ADQ is already enabled */
6142 	if (!(features & NETIF_F_HW_TC) && ice_is_adq_active(pf)) {
6143 		dev_err(ice_pf_to_dev(pf), "ADQ is active, can't turn hw_tc_offload off\n");
6144 		return -EACCES;
6145 	}
6146 
6147 	if (changed & NETIF_F_HW_TC) {
6148 		bool ena = !!(features & NETIF_F_HW_TC);
6149 
6150 		ena ? set_bit(ICE_FLAG_CLS_FLOWER, pf->flags) :
6151 		      clear_bit(ICE_FLAG_CLS_FLOWER, pf->flags);
6152 	}
6153 
6154 	if (changed & NETIF_F_LOOPBACK)
6155 		ret = ice_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK));
6156 
6157 	return ret;
6158 }
6159 
6160 /**
6161  * ice_vsi_vlan_setup - Setup VLAN offload properties on a PF VSI
6162  * @vsi: VSI to setup VLAN properties for
6163  */
6164 static int ice_vsi_vlan_setup(struct ice_vsi *vsi)
6165 {
6166 	int err;
6167 
6168 	err = ice_set_vlan_offload_features(vsi, vsi->netdev->features);
6169 	if (err)
6170 		return err;
6171 
6172 	err = ice_set_vlan_filtering_features(vsi, vsi->netdev->features);
6173 	if (err)
6174 		return err;
6175 
6176 	return ice_vsi_add_vlan_zero(vsi);
6177 }
6178 
6179 /**
6180  * ice_vsi_cfg_lan - Setup the VSI lan related config
6181  * @vsi: the VSI being configured
6182  *
6183  * Return 0 on success and negative value on error
6184  */
6185 int ice_vsi_cfg_lan(struct ice_vsi *vsi)
6186 {
6187 	int err;
6188 
6189 	if (vsi->netdev && vsi->type == ICE_VSI_PF) {
6190 		ice_set_rx_mode(vsi->netdev);
6191 
6192 		err = ice_vsi_vlan_setup(vsi);
6193 		if (err)
6194 			return err;
6195 	}
6196 	ice_vsi_cfg_dcb_rings(vsi);
6197 
6198 	err = ice_vsi_cfg_lan_txqs(vsi);
6199 	if (!err && ice_is_xdp_ena_vsi(vsi))
6200 		err = ice_vsi_cfg_xdp_txqs(vsi);
6201 	if (!err)
6202 		err = ice_vsi_cfg_rxqs(vsi);
6203 
6204 	return err;
6205 }
6206 
6207 /* THEORY OF MODERATION:
6208  * The ice driver hardware works differently than the hardware that DIMLIB was
6209  * originally made for. ice hardware doesn't have packet count limits that
6210  * can trigger an interrupt, but it *does* have interrupt rate limit support,
6211  * which is hard-coded to a limit of 250,000 ints/second.
6212  * If not using dynamic moderation, the INTRL value can be modified
6213  * by ethtool rx-usecs-high.
6214  */
6215 struct ice_dim {
6216 	/* the throttle rate for interrupts, basically worst case delay before
6217 	 * an initial interrupt fires, value is stored in microseconds.
6218 	 */
6219 	u16 itr;
6220 };
6221 
6222 /* Make a different profile for Rx that doesn't allow quite so aggressive
6223  * moderation at the high end (it maxes out at 126us or about 8k interrupts a
6224  * second.
6225  */
6226 static const struct ice_dim rx_profile[] = {
6227 	{2},    /* 500,000 ints/s, capped at 250K by INTRL */
6228 	{8},    /* 125,000 ints/s */
6229 	{16},   /*  62,500 ints/s */
6230 	{62},   /*  16,129 ints/s */
6231 	{126}   /*   7,936 ints/s */
6232 };
6233 
6234 /* The transmit profile, which has the same sorts of values
6235  * as the previous struct
6236  */
6237 static const struct ice_dim tx_profile[] = {
6238 	{2},    /* 500,000 ints/s, capped at 250K by INTRL */
6239 	{8},    /* 125,000 ints/s */
6240 	{40},   /*  16,125 ints/s */
6241 	{128},  /*   7,812 ints/s */
6242 	{256}   /*   3,906 ints/s */
6243 };
6244 
6245 static void ice_tx_dim_work(struct work_struct *work)
6246 {
6247 	struct ice_ring_container *rc;
6248 	struct dim *dim;
6249 	u16 itr;
6250 
6251 	dim = container_of(work, struct dim, work);
6252 	rc = (struct ice_ring_container *)dim->priv;
6253 
6254 	WARN_ON(dim->profile_ix >= ARRAY_SIZE(tx_profile));
6255 
6256 	/* look up the values in our local table */
6257 	itr = tx_profile[dim->profile_ix].itr;
6258 
6259 	ice_trace(tx_dim_work, container_of(rc, struct ice_q_vector, tx), dim);
6260 	ice_write_itr(rc, itr);
6261 
6262 	dim->state = DIM_START_MEASURE;
6263 }
6264 
6265 static void ice_rx_dim_work(struct work_struct *work)
6266 {
6267 	struct ice_ring_container *rc;
6268 	struct dim *dim;
6269 	u16 itr;
6270 
6271 	dim = container_of(work, struct dim, work);
6272 	rc = (struct ice_ring_container *)dim->priv;
6273 
6274 	WARN_ON(dim->profile_ix >= ARRAY_SIZE(rx_profile));
6275 
6276 	/* look up the values in our local table */
6277 	itr = rx_profile[dim->profile_ix].itr;
6278 
6279 	ice_trace(rx_dim_work, container_of(rc, struct ice_q_vector, rx), dim);
6280 	ice_write_itr(rc, itr);
6281 
6282 	dim->state = DIM_START_MEASURE;
6283 }
6284 
6285 #define ICE_DIM_DEFAULT_PROFILE_IX 1
6286 
6287 /**
6288  * ice_init_moderation - set up interrupt moderation
6289  * @q_vector: the vector containing rings to be configured
6290  *
6291  * Set up interrupt moderation registers, with the intent to do the right thing
6292  * when called from reset or from probe, and whether or not dynamic moderation
6293  * is enabled or not. Take special care to write all the registers in both
6294  * dynamic moderation mode or not in order to make sure hardware is in a known
6295  * state.
6296  */
6297 static void ice_init_moderation(struct ice_q_vector *q_vector)
6298 {
6299 	struct ice_ring_container *rc;
6300 	bool tx_dynamic, rx_dynamic;
6301 
6302 	rc = &q_vector->tx;
6303 	INIT_WORK(&rc->dim.work, ice_tx_dim_work);
6304 	rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6305 	rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX;
6306 	rc->dim.priv = rc;
6307 	tx_dynamic = ITR_IS_DYNAMIC(rc);
6308 
6309 	/* set the initial TX ITR to match the above */
6310 	ice_write_itr(rc, tx_dynamic ?
6311 		      tx_profile[rc->dim.profile_ix].itr : rc->itr_setting);
6312 
6313 	rc = &q_vector->rx;
6314 	INIT_WORK(&rc->dim.work, ice_rx_dim_work);
6315 	rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
6316 	rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX;
6317 	rc->dim.priv = rc;
6318 	rx_dynamic = ITR_IS_DYNAMIC(rc);
6319 
6320 	/* set the initial RX ITR to match the above */
6321 	ice_write_itr(rc, rx_dynamic ? rx_profile[rc->dim.profile_ix].itr :
6322 				       rc->itr_setting);
6323 
6324 	ice_set_q_vector_intrl(q_vector);
6325 }
6326 
6327 /**
6328  * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI
6329  * @vsi: the VSI being configured
6330  */
6331 static void ice_napi_enable_all(struct ice_vsi *vsi)
6332 {
6333 	int q_idx;
6334 
6335 	if (!vsi->netdev)
6336 		return;
6337 
6338 	ice_for_each_q_vector(vsi, q_idx) {
6339 		struct ice_q_vector *q_vector = vsi->q_vectors[q_idx];
6340 
6341 		ice_init_moderation(q_vector);
6342 
6343 		if (q_vector->rx.rx_ring || q_vector->tx.tx_ring)
6344 			napi_enable(&q_vector->napi);
6345 	}
6346 }
6347 
6348 /**
6349  * ice_up_complete - Finish the last steps of bringing up a connection
6350  * @vsi: The VSI being configured
6351  *
6352  * Return 0 on success and negative value on error
6353  */
6354 static int ice_up_complete(struct ice_vsi *vsi)
6355 {
6356 	struct ice_pf *pf = vsi->back;
6357 	int err;
6358 
6359 	ice_vsi_cfg_msix(vsi);
6360 
6361 	/* Enable only Rx rings, Tx rings were enabled by the FW when the
6362 	 * Tx queue group list was configured and the context bits were
6363 	 * programmed using ice_vsi_cfg_txqs
6364 	 */
6365 	err = ice_vsi_start_all_rx_rings(vsi);
6366 	if (err)
6367 		return err;
6368 
6369 	clear_bit(ICE_VSI_DOWN, vsi->state);
6370 	ice_napi_enable_all(vsi);
6371 	ice_vsi_ena_irq(vsi);
6372 
6373 	if (vsi->port_info &&
6374 	    (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) &&
6375 	    vsi->netdev && vsi->type == ICE_VSI_PF) {
6376 		ice_print_link_msg(vsi, true);
6377 		netif_tx_start_all_queues(vsi->netdev);
6378 		netif_carrier_on(vsi->netdev);
6379 		ice_ptp_link_change(pf, pf->hw.pf_id, true);
6380 	}
6381 
6382 	/* Perform an initial read of the statistics registers now to
6383 	 * set the baseline so counters are ready when interface is up
6384 	 */
6385 	ice_update_eth_stats(vsi);
6386 
6387 	if (vsi->type == ICE_VSI_PF)
6388 		ice_service_task_schedule(pf);
6389 
6390 	return 0;
6391 }
6392 
6393 /**
6394  * ice_up - Bring the connection back up after being down
6395  * @vsi: VSI being configured
6396  */
6397 int ice_up(struct ice_vsi *vsi)
6398 {
6399 	int err;
6400 
6401 	err = ice_vsi_cfg_lan(vsi);
6402 	if (!err)
6403 		err = ice_up_complete(vsi);
6404 
6405 	return err;
6406 }
6407 
6408 /**
6409  * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring
6410  * @syncp: pointer to u64_stats_sync
6411  * @stats: stats that pkts and bytes count will be taken from
6412  * @pkts: packets stats counter
6413  * @bytes: bytes stats counter
6414  *
6415  * This function fetches stats from the ring considering the atomic operations
6416  * that needs to be performed to read u64 values in 32 bit machine.
6417  */
6418 void
6419 ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp,
6420 			     struct ice_q_stats stats, u64 *pkts, u64 *bytes)
6421 {
6422 	unsigned int start;
6423 
6424 	do {
6425 		start = u64_stats_fetch_begin(syncp);
6426 		*pkts = stats.pkts;
6427 		*bytes = stats.bytes;
6428 	} while (u64_stats_fetch_retry(syncp, start));
6429 }
6430 
6431 /**
6432  * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters
6433  * @vsi: the VSI to be updated
6434  * @vsi_stats: the stats struct to be updated
6435  * @rings: rings to work on
6436  * @count: number of rings
6437  */
6438 static void
6439 ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi,
6440 			     struct rtnl_link_stats64 *vsi_stats,
6441 			     struct ice_tx_ring **rings, u16 count)
6442 {
6443 	u16 i;
6444 
6445 	for (i = 0; i < count; i++) {
6446 		struct ice_tx_ring *ring;
6447 		u64 pkts = 0, bytes = 0;
6448 
6449 		ring = READ_ONCE(rings[i]);
6450 		if (!ring || !ring->ring_stats)
6451 			continue;
6452 		ice_fetch_u64_stats_per_ring(&ring->ring_stats->syncp,
6453 					     ring->ring_stats->stats, &pkts,
6454 					     &bytes);
6455 		vsi_stats->tx_packets += pkts;
6456 		vsi_stats->tx_bytes += bytes;
6457 		vsi->tx_restart += ring->ring_stats->tx_stats.restart_q;
6458 		vsi->tx_busy += ring->ring_stats->tx_stats.tx_busy;
6459 		vsi->tx_linearize += ring->ring_stats->tx_stats.tx_linearize;
6460 	}
6461 }
6462 
6463 /**
6464  * ice_update_vsi_ring_stats - Update VSI stats counters
6465  * @vsi: the VSI to be updated
6466  */
6467 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi)
6468 {
6469 	struct rtnl_link_stats64 *net_stats, *stats_prev;
6470 	struct rtnl_link_stats64 *vsi_stats;
6471 	u64 pkts, bytes;
6472 	int i;
6473 
6474 	vsi_stats = kzalloc(sizeof(*vsi_stats), GFP_ATOMIC);
6475 	if (!vsi_stats)
6476 		return;
6477 
6478 	/* reset non-netdev (extended) stats */
6479 	vsi->tx_restart = 0;
6480 	vsi->tx_busy = 0;
6481 	vsi->tx_linearize = 0;
6482 	vsi->rx_buf_failed = 0;
6483 	vsi->rx_page_failed = 0;
6484 
6485 	rcu_read_lock();
6486 
6487 	/* update Tx rings counters */
6488 	ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->tx_rings,
6489 				     vsi->num_txq);
6490 
6491 	/* update Rx rings counters */
6492 	ice_for_each_rxq(vsi, i) {
6493 		struct ice_rx_ring *ring = READ_ONCE(vsi->rx_rings[i]);
6494 		struct ice_ring_stats *ring_stats;
6495 
6496 		ring_stats = ring->ring_stats;
6497 		ice_fetch_u64_stats_per_ring(&ring_stats->syncp,
6498 					     ring_stats->stats, &pkts,
6499 					     &bytes);
6500 		vsi_stats->rx_packets += pkts;
6501 		vsi_stats->rx_bytes += bytes;
6502 		vsi->rx_buf_failed += ring_stats->rx_stats.alloc_buf_failed;
6503 		vsi->rx_page_failed += ring_stats->rx_stats.alloc_page_failed;
6504 	}
6505 
6506 	/* update XDP Tx rings counters */
6507 	if (ice_is_xdp_ena_vsi(vsi))
6508 		ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->xdp_rings,
6509 					     vsi->num_xdp_txq);
6510 
6511 	rcu_read_unlock();
6512 
6513 	net_stats = &vsi->net_stats;
6514 	stats_prev = &vsi->net_stats_prev;
6515 
6516 	/* clear prev counters after reset */
6517 	if (vsi_stats->tx_packets < stats_prev->tx_packets ||
6518 	    vsi_stats->rx_packets < stats_prev->rx_packets) {
6519 		stats_prev->tx_packets = 0;
6520 		stats_prev->tx_bytes = 0;
6521 		stats_prev->rx_packets = 0;
6522 		stats_prev->rx_bytes = 0;
6523 	}
6524 
6525 	/* update netdev counters */
6526 	net_stats->tx_packets += vsi_stats->tx_packets - stats_prev->tx_packets;
6527 	net_stats->tx_bytes += vsi_stats->tx_bytes - stats_prev->tx_bytes;
6528 	net_stats->rx_packets += vsi_stats->rx_packets - stats_prev->rx_packets;
6529 	net_stats->rx_bytes += vsi_stats->rx_bytes - stats_prev->rx_bytes;
6530 
6531 	stats_prev->tx_packets = vsi_stats->tx_packets;
6532 	stats_prev->tx_bytes = vsi_stats->tx_bytes;
6533 	stats_prev->rx_packets = vsi_stats->rx_packets;
6534 	stats_prev->rx_bytes = vsi_stats->rx_bytes;
6535 
6536 	kfree(vsi_stats);
6537 }
6538 
6539 /**
6540  * ice_update_vsi_stats - Update VSI stats counters
6541  * @vsi: the VSI to be updated
6542  */
6543 void ice_update_vsi_stats(struct ice_vsi *vsi)
6544 {
6545 	struct rtnl_link_stats64 *cur_ns = &vsi->net_stats;
6546 	struct ice_eth_stats *cur_es = &vsi->eth_stats;
6547 	struct ice_pf *pf = vsi->back;
6548 
6549 	if (test_bit(ICE_VSI_DOWN, vsi->state) ||
6550 	    test_bit(ICE_CFG_BUSY, pf->state))
6551 		return;
6552 
6553 	/* get stats as recorded by Tx/Rx rings */
6554 	ice_update_vsi_ring_stats(vsi);
6555 
6556 	/* get VSI stats as recorded by the hardware */
6557 	ice_update_eth_stats(vsi);
6558 
6559 	cur_ns->tx_errors = cur_es->tx_errors;
6560 	cur_ns->rx_dropped = cur_es->rx_discards;
6561 	cur_ns->tx_dropped = cur_es->tx_discards;
6562 	cur_ns->multicast = cur_es->rx_multicast;
6563 
6564 	/* update some more netdev stats if this is main VSI */
6565 	if (vsi->type == ICE_VSI_PF) {
6566 		cur_ns->rx_crc_errors = pf->stats.crc_errors;
6567 		cur_ns->rx_errors = pf->stats.crc_errors +
6568 				    pf->stats.illegal_bytes +
6569 				    pf->stats.rx_len_errors +
6570 				    pf->stats.rx_undersize +
6571 				    pf->hw_csum_rx_error +
6572 				    pf->stats.rx_jabber +
6573 				    pf->stats.rx_fragments +
6574 				    pf->stats.rx_oversize;
6575 		cur_ns->rx_length_errors = pf->stats.rx_len_errors;
6576 		/* record drops from the port level */
6577 		cur_ns->rx_missed_errors = pf->stats.eth.rx_discards;
6578 	}
6579 }
6580 
6581 /**
6582  * ice_update_pf_stats - Update PF port stats counters
6583  * @pf: PF whose stats needs to be updated
6584  */
6585 void ice_update_pf_stats(struct ice_pf *pf)
6586 {
6587 	struct ice_hw_port_stats *prev_ps, *cur_ps;
6588 	struct ice_hw *hw = &pf->hw;
6589 	u16 fd_ctr_base;
6590 	u8 port;
6591 
6592 	port = hw->port_info->lport;
6593 	prev_ps = &pf->stats_prev;
6594 	cur_ps = &pf->stats;
6595 
6596 	if (ice_is_reset_in_progress(pf->state))
6597 		pf->stat_prev_loaded = false;
6598 
6599 	ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded,
6600 			  &prev_ps->eth.rx_bytes,
6601 			  &cur_ps->eth.rx_bytes);
6602 
6603 	ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded,
6604 			  &prev_ps->eth.rx_unicast,
6605 			  &cur_ps->eth.rx_unicast);
6606 
6607 	ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded,
6608 			  &prev_ps->eth.rx_multicast,
6609 			  &cur_ps->eth.rx_multicast);
6610 
6611 	ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded,
6612 			  &prev_ps->eth.rx_broadcast,
6613 			  &cur_ps->eth.rx_broadcast);
6614 
6615 	ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded,
6616 			  &prev_ps->eth.rx_discards,
6617 			  &cur_ps->eth.rx_discards);
6618 
6619 	ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded,
6620 			  &prev_ps->eth.tx_bytes,
6621 			  &cur_ps->eth.tx_bytes);
6622 
6623 	ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded,
6624 			  &prev_ps->eth.tx_unicast,
6625 			  &cur_ps->eth.tx_unicast);
6626 
6627 	ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded,
6628 			  &prev_ps->eth.tx_multicast,
6629 			  &cur_ps->eth.tx_multicast);
6630 
6631 	ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded,
6632 			  &prev_ps->eth.tx_broadcast,
6633 			  &cur_ps->eth.tx_broadcast);
6634 
6635 	ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded,
6636 			  &prev_ps->tx_dropped_link_down,
6637 			  &cur_ps->tx_dropped_link_down);
6638 
6639 	ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded,
6640 			  &prev_ps->rx_size_64, &cur_ps->rx_size_64);
6641 
6642 	ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded,
6643 			  &prev_ps->rx_size_127, &cur_ps->rx_size_127);
6644 
6645 	ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded,
6646 			  &prev_ps->rx_size_255, &cur_ps->rx_size_255);
6647 
6648 	ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded,
6649 			  &prev_ps->rx_size_511, &cur_ps->rx_size_511);
6650 
6651 	ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded,
6652 			  &prev_ps->rx_size_1023, &cur_ps->rx_size_1023);
6653 
6654 	ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded,
6655 			  &prev_ps->rx_size_1522, &cur_ps->rx_size_1522);
6656 
6657 	ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded,
6658 			  &prev_ps->rx_size_big, &cur_ps->rx_size_big);
6659 
6660 	ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded,
6661 			  &prev_ps->tx_size_64, &cur_ps->tx_size_64);
6662 
6663 	ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded,
6664 			  &prev_ps->tx_size_127, &cur_ps->tx_size_127);
6665 
6666 	ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded,
6667 			  &prev_ps->tx_size_255, &cur_ps->tx_size_255);
6668 
6669 	ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded,
6670 			  &prev_ps->tx_size_511, &cur_ps->tx_size_511);
6671 
6672 	ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded,
6673 			  &prev_ps->tx_size_1023, &cur_ps->tx_size_1023);
6674 
6675 	ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded,
6676 			  &prev_ps->tx_size_1522, &cur_ps->tx_size_1522);
6677 
6678 	ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded,
6679 			  &prev_ps->tx_size_big, &cur_ps->tx_size_big);
6680 
6681 	fd_ctr_base = hw->fd_ctr_base;
6682 
6683 	ice_stat_update40(hw,
6684 			  GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)),
6685 			  pf->stat_prev_loaded, &prev_ps->fd_sb_match,
6686 			  &cur_ps->fd_sb_match);
6687 	ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded,
6688 			  &prev_ps->link_xon_rx, &cur_ps->link_xon_rx);
6689 
6690 	ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded,
6691 			  &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx);
6692 
6693 	ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded,
6694 			  &prev_ps->link_xon_tx, &cur_ps->link_xon_tx);
6695 
6696 	ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded,
6697 			  &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx);
6698 
6699 	ice_update_dcb_stats(pf);
6700 
6701 	ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded,
6702 			  &prev_ps->crc_errors, &cur_ps->crc_errors);
6703 
6704 	ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded,
6705 			  &prev_ps->illegal_bytes, &cur_ps->illegal_bytes);
6706 
6707 	ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded,
6708 			  &prev_ps->mac_local_faults,
6709 			  &cur_ps->mac_local_faults);
6710 
6711 	ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded,
6712 			  &prev_ps->mac_remote_faults,
6713 			  &cur_ps->mac_remote_faults);
6714 
6715 	ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded,
6716 			  &prev_ps->rx_len_errors, &cur_ps->rx_len_errors);
6717 
6718 	ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded,
6719 			  &prev_ps->rx_undersize, &cur_ps->rx_undersize);
6720 
6721 	ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded,
6722 			  &prev_ps->rx_fragments, &cur_ps->rx_fragments);
6723 
6724 	ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded,
6725 			  &prev_ps->rx_oversize, &cur_ps->rx_oversize);
6726 
6727 	ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded,
6728 			  &prev_ps->rx_jabber, &cur_ps->rx_jabber);
6729 
6730 	cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0;
6731 
6732 	pf->stat_prev_loaded = true;
6733 }
6734 
6735 /**
6736  * ice_get_stats64 - get statistics for network device structure
6737  * @netdev: network interface device structure
6738  * @stats: main device statistics structure
6739  */
6740 static
6741 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
6742 {
6743 	struct ice_netdev_priv *np = netdev_priv(netdev);
6744 	struct rtnl_link_stats64 *vsi_stats;
6745 	struct ice_vsi *vsi = np->vsi;
6746 
6747 	vsi_stats = &vsi->net_stats;
6748 
6749 	if (!vsi->num_txq || !vsi->num_rxq)
6750 		return;
6751 
6752 	/* netdev packet/byte stats come from ring counter. These are obtained
6753 	 * by summing up ring counters (done by ice_update_vsi_ring_stats).
6754 	 * But, only call the update routine and read the registers if VSI is
6755 	 * not down.
6756 	 */
6757 	if (!test_bit(ICE_VSI_DOWN, vsi->state))
6758 		ice_update_vsi_ring_stats(vsi);
6759 	stats->tx_packets = vsi_stats->tx_packets;
6760 	stats->tx_bytes = vsi_stats->tx_bytes;
6761 	stats->rx_packets = vsi_stats->rx_packets;
6762 	stats->rx_bytes = vsi_stats->rx_bytes;
6763 
6764 	/* The rest of the stats can be read from the hardware but instead we
6765 	 * just return values that the watchdog task has already obtained from
6766 	 * the hardware.
6767 	 */
6768 	stats->multicast = vsi_stats->multicast;
6769 	stats->tx_errors = vsi_stats->tx_errors;
6770 	stats->tx_dropped = vsi_stats->tx_dropped;
6771 	stats->rx_errors = vsi_stats->rx_errors;
6772 	stats->rx_dropped = vsi_stats->rx_dropped;
6773 	stats->rx_crc_errors = vsi_stats->rx_crc_errors;
6774 	stats->rx_length_errors = vsi_stats->rx_length_errors;
6775 }
6776 
6777 /**
6778  * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI
6779  * @vsi: VSI having NAPI disabled
6780  */
6781 static void ice_napi_disable_all(struct ice_vsi *vsi)
6782 {
6783 	int q_idx;
6784 
6785 	if (!vsi->netdev)
6786 		return;
6787 
6788 	ice_for_each_q_vector(vsi, q_idx) {
6789 		struct ice_q_vector *q_vector = vsi->q_vectors[q_idx];
6790 
6791 		if (q_vector->rx.rx_ring || q_vector->tx.tx_ring)
6792 			napi_disable(&q_vector->napi);
6793 
6794 		cancel_work_sync(&q_vector->tx.dim.work);
6795 		cancel_work_sync(&q_vector->rx.dim.work);
6796 	}
6797 }
6798 
6799 /**
6800  * ice_down - Shutdown the connection
6801  * @vsi: The VSI being stopped
6802  *
6803  * Caller of this function is expected to set the vsi->state ICE_DOWN bit
6804  */
6805 int ice_down(struct ice_vsi *vsi)
6806 {
6807 	int i, tx_err, rx_err, vlan_err = 0;
6808 
6809 	WARN_ON(!test_bit(ICE_VSI_DOWN, vsi->state));
6810 
6811 	if (vsi->netdev && vsi->type == ICE_VSI_PF) {
6812 		vlan_err = ice_vsi_del_vlan_zero(vsi);
6813 		ice_ptp_link_change(vsi->back, vsi->back->hw.pf_id, false);
6814 		netif_carrier_off(vsi->netdev);
6815 		netif_tx_disable(vsi->netdev);
6816 	} else if (vsi->type == ICE_VSI_SWITCHDEV_CTRL) {
6817 		ice_eswitch_stop_all_tx_queues(vsi->back);
6818 	}
6819 
6820 	ice_vsi_dis_irq(vsi);
6821 
6822 	tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0);
6823 	if (tx_err)
6824 		netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n",
6825 			   vsi->vsi_num, tx_err);
6826 	if (!tx_err && ice_is_xdp_ena_vsi(vsi)) {
6827 		tx_err = ice_vsi_stop_xdp_tx_rings(vsi);
6828 		if (tx_err)
6829 			netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n",
6830 				   vsi->vsi_num, tx_err);
6831 	}
6832 
6833 	rx_err = ice_vsi_stop_all_rx_rings(vsi);
6834 	if (rx_err)
6835 		netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n",
6836 			   vsi->vsi_num, rx_err);
6837 
6838 	ice_napi_disable_all(vsi);
6839 
6840 	ice_for_each_txq(vsi, i)
6841 		ice_clean_tx_ring(vsi->tx_rings[i]);
6842 
6843 	if (ice_is_xdp_ena_vsi(vsi))
6844 		ice_for_each_xdp_txq(vsi, i)
6845 			ice_clean_tx_ring(vsi->xdp_rings[i]);
6846 
6847 	ice_for_each_rxq(vsi, i)
6848 		ice_clean_rx_ring(vsi->rx_rings[i]);
6849 
6850 	if (tx_err || rx_err || vlan_err) {
6851 		netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n",
6852 			   vsi->vsi_num, vsi->vsw->sw_id);
6853 		return -EIO;
6854 	}
6855 
6856 	return 0;
6857 }
6858 
6859 /**
6860  * ice_down_up - shutdown the VSI connection and bring it up
6861  * @vsi: the VSI to be reconnected
6862  */
6863 int ice_down_up(struct ice_vsi *vsi)
6864 {
6865 	int ret;
6866 
6867 	/* if DOWN already set, nothing to do */
6868 	if (test_and_set_bit(ICE_VSI_DOWN, vsi->state))
6869 		return 0;
6870 
6871 	ret = ice_down(vsi);
6872 	if (ret)
6873 		return ret;
6874 
6875 	ret = ice_up(vsi);
6876 	if (ret) {
6877 		netdev_err(vsi->netdev, "reallocating resources failed during netdev features change, may need to reload driver\n");
6878 		return ret;
6879 	}
6880 
6881 	return 0;
6882 }
6883 
6884 /**
6885  * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources
6886  * @vsi: VSI having resources allocated
6887  *
6888  * Return 0 on success, negative on failure
6889  */
6890 int ice_vsi_setup_tx_rings(struct ice_vsi *vsi)
6891 {
6892 	int i, err = 0;
6893 
6894 	if (!vsi->num_txq) {
6895 		dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n",
6896 			vsi->vsi_num);
6897 		return -EINVAL;
6898 	}
6899 
6900 	ice_for_each_txq(vsi, i) {
6901 		struct ice_tx_ring *ring = vsi->tx_rings[i];
6902 
6903 		if (!ring)
6904 			return -EINVAL;
6905 
6906 		if (vsi->netdev)
6907 			ring->netdev = vsi->netdev;
6908 		err = ice_setup_tx_ring(ring);
6909 		if (err)
6910 			break;
6911 	}
6912 
6913 	return err;
6914 }
6915 
6916 /**
6917  * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources
6918  * @vsi: VSI having resources allocated
6919  *
6920  * Return 0 on success, negative on failure
6921  */
6922 int ice_vsi_setup_rx_rings(struct ice_vsi *vsi)
6923 {
6924 	int i, err = 0;
6925 
6926 	if (!vsi->num_rxq) {
6927 		dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n",
6928 			vsi->vsi_num);
6929 		return -EINVAL;
6930 	}
6931 
6932 	ice_for_each_rxq(vsi, i) {
6933 		struct ice_rx_ring *ring = vsi->rx_rings[i];
6934 
6935 		if (!ring)
6936 			return -EINVAL;
6937 
6938 		if (vsi->netdev)
6939 			ring->netdev = vsi->netdev;
6940 		err = ice_setup_rx_ring(ring);
6941 		if (err)
6942 			break;
6943 	}
6944 
6945 	return err;
6946 }
6947 
6948 /**
6949  * ice_vsi_open_ctrl - open control VSI for use
6950  * @vsi: the VSI to open
6951  *
6952  * Initialization of the Control VSI
6953  *
6954  * Returns 0 on success, negative value on error
6955  */
6956 int ice_vsi_open_ctrl(struct ice_vsi *vsi)
6957 {
6958 	char int_name[ICE_INT_NAME_STR_LEN];
6959 	struct ice_pf *pf = vsi->back;
6960 	struct device *dev;
6961 	int err;
6962 
6963 	dev = ice_pf_to_dev(pf);
6964 	/* allocate descriptors */
6965 	err = ice_vsi_setup_tx_rings(vsi);
6966 	if (err)
6967 		goto err_setup_tx;
6968 
6969 	err = ice_vsi_setup_rx_rings(vsi);
6970 	if (err)
6971 		goto err_setup_rx;
6972 
6973 	err = ice_vsi_cfg_lan(vsi);
6974 	if (err)
6975 		goto err_setup_rx;
6976 
6977 	snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl",
6978 		 dev_driver_string(dev), dev_name(dev));
6979 	err = ice_vsi_req_irq_msix(vsi, int_name);
6980 	if (err)
6981 		goto err_setup_rx;
6982 
6983 	ice_vsi_cfg_msix(vsi);
6984 
6985 	err = ice_vsi_start_all_rx_rings(vsi);
6986 	if (err)
6987 		goto err_up_complete;
6988 
6989 	clear_bit(ICE_VSI_DOWN, vsi->state);
6990 	ice_vsi_ena_irq(vsi);
6991 
6992 	return 0;
6993 
6994 err_up_complete:
6995 	ice_down(vsi);
6996 err_setup_rx:
6997 	ice_vsi_free_rx_rings(vsi);
6998 err_setup_tx:
6999 	ice_vsi_free_tx_rings(vsi);
7000 
7001 	return err;
7002 }
7003 
7004 /**
7005  * ice_vsi_open - Called when a network interface is made active
7006  * @vsi: the VSI to open
7007  *
7008  * Initialization of the VSI
7009  *
7010  * Returns 0 on success, negative value on error
7011  */
7012 int ice_vsi_open(struct ice_vsi *vsi)
7013 {
7014 	char int_name[ICE_INT_NAME_STR_LEN];
7015 	struct ice_pf *pf = vsi->back;
7016 	int err;
7017 
7018 	/* allocate descriptors */
7019 	err = ice_vsi_setup_tx_rings(vsi);
7020 	if (err)
7021 		goto err_setup_tx;
7022 
7023 	err = ice_vsi_setup_rx_rings(vsi);
7024 	if (err)
7025 		goto err_setup_rx;
7026 
7027 	err = ice_vsi_cfg_lan(vsi);
7028 	if (err)
7029 		goto err_setup_rx;
7030 
7031 	snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
7032 		 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name);
7033 	err = ice_vsi_req_irq_msix(vsi, int_name);
7034 	if (err)
7035 		goto err_setup_rx;
7036 
7037 	ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc);
7038 
7039 	if (vsi->type == ICE_VSI_PF) {
7040 		/* Notify the stack of the actual queue counts. */
7041 		err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq);
7042 		if (err)
7043 			goto err_set_qs;
7044 
7045 		err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq);
7046 		if (err)
7047 			goto err_set_qs;
7048 	}
7049 
7050 	err = ice_up_complete(vsi);
7051 	if (err)
7052 		goto err_up_complete;
7053 
7054 	return 0;
7055 
7056 err_up_complete:
7057 	ice_down(vsi);
7058 err_set_qs:
7059 	ice_vsi_free_irq(vsi);
7060 err_setup_rx:
7061 	ice_vsi_free_rx_rings(vsi);
7062 err_setup_tx:
7063 	ice_vsi_free_tx_rings(vsi);
7064 
7065 	return err;
7066 }
7067 
7068 /**
7069  * ice_vsi_release_all - Delete all VSIs
7070  * @pf: PF from which all VSIs are being removed
7071  */
7072 static void ice_vsi_release_all(struct ice_pf *pf)
7073 {
7074 	int err, i;
7075 
7076 	if (!pf->vsi)
7077 		return;
7078 
7079 	ice_for_each_vsi(pf, i) {
7080 		if (!pf->vsi[i])
7081 			continue;
7082 
7083 		if (pf->vsi[i]->type == ICE_VSI_CHNL)
7084 			continue;
7085 
7086 		err = ice_vsi_release(pf->vsi[i]);
7087 		if (err)
7088 			dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n",
7089 				i, err, pf->vsi[i]->vsi_num);
7090 	}
7091 }
7092 
7093 /**
7094  * ice_vsi_rebuild_by_type - Rebuild VSI of a given type
7095  * @pf: pointer to the PF instance
7096  * @type: VSI type to rebuild
7097  *
7098  * Iterates through the pf->vsi array and rebuilds VSIs of the requested type
7099  */
7100 static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type)
7101 {
7102 	struct device *dev = ice_pf_to_dev(pf);
7103 	int i, err;
7104 
7105 	ice_for_each_vsi(pf, i) {
7106 		struct ice_vsi *vsi = pf->vsi[i];
7107 
7108 		if (!vsi || vsi->type != type)
7109 			continue;
7110 
7111 		/* rebuild the VSI */
7112 		err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT);
7113 		if (err) {
7114 			dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n",
7115 				err, vsi->idx, ice_vsi_type_str(type));
7116 			return err;
7117 		}
7118 
7119 		/* replay filters for the VSI */
7120 		err = ice_replay_vsi(&pf->hw, vsi->idx);
7121 		if (err) {
7122 			dev_err(dev, "replay VSI failed, error %d, VSI index %d, type %s\n",
7123 				err, vsi->idx, ice_vsi_type_str(type));
7124 			return err;
7125 		}
7126 
7127 		/* Re-map HW VSI number, using VSI handle that has been
7128 		 * previously validated in ice_replay_vsi() call above
7129 		 */
7130 		vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx);
7131 
7132 		/* enable the VSI */
7133 		err = ice_ena_vsi(vsi, false);
7134 		if (err) {
7135 			dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n",
7136 				err, vsi->idx, ice_vsi_type_str(type));
7137 			return err;
7138 		}
7139 
7140 		dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx,
7141 			 ice_vsi_type_str(type));
7142 	}
7143 
7144 	return 0;
7145 }
7146 
7147 /**
7148  * ice_update_pf_netdev_link - Update PF netdev link status
7149  * @pf: pointer to the PF instance
7150  */
7151 static void ice_update_pf_netdev_link(struct ice_pf *pf)
7152 {
7153 	bool link_up;
7154 	int i;
7155 
7156 	ice_for_each_vsi(pf, i) {
7157 		struct ice_vsi *vsi = pf->vsi[i];
7158 
7159 		if (!vsi || vsi->type != ICE_VSI_PF)
7160 			return;
7161 
7162 		ice_get_link_status(pf->vsi[i]->port_info, &link_up);
7163 		if (link_up) {
7164 			netif_carrier_on(pf->vsi[i]->netdev);
7165 			netif_tx_wake_all_queues(pf->vsi[i]->netdev);
7166 		} else {
7167 			netif_carrier_off(pf->vsi[i]->netdev);
7168 			netif_tx_stop_all_queues(pf->vsi[i]->netdev);
7169 		}
7170 	}
7171 }
7172 
7173 /**
7174  * ice_rebuild - rebuild after reset
7175  * @pf: PF to rebuild
7176  * @reset_type: type of reset
7177  *
7178  * Do not rebuild VF VSI in this flow because that is already handled via
7179  * ice_reset_all_vfs(). This is because requirements for resetting a VF after a
7180  * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want
7181  * to reset/rebuild all the VF VSI twice.
7182  */
7183 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
7184 {
7185 	struct device *dev = ice_pf_to_dev(pf);
7186 	struct ice_hw *hw = &pf->hw;
7187 	bool dvm;
7188 	int err;
7189 
7190 	if (test_bit(ICE_DOWN, pf->state))
7191 		goto clear_recovery;
7192 
7193 	dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type);
7194 
7195 #define ICE_EMP_RESET_SLEEP_MS 5000
7196 	if (reset_type == ICE_RESET_EMPR) {
7197 		/* If an EMP reset has occurred, any previously pending flash
7198 		 * update will have completed. We no longer know whether or
7199 		 * not the NVM update EMP reset is restricted.
7200 		 */
7201 		pf->fw_emp_reset_disabled = false;
7202 
7203 		msleep(ICE_EMP_RESET_SLEEP_MS);
7204 	}
7205 
7206 	err = ice_init_all_ctrlq(hw);
7207 	if (err) {
7208 		dev_err(dev, "control queues init failed %d\n", err);
7209 		goto err_init_ctrlq;
7210 	}
7211 
7212 	/* if DDP was previously loaded successfully */
7213 	if (!ice_is_safe_mode(pf)) {
7214 		/* reload the SW DB of filter tables */
7215 		if (reset_type == ICE_RESET_PFR)
7216 			ice_fill_blk_tbls(hw);
7217 		else
7218 			/* Reload DDP Package after CORER/GLOBR reset */
7219 			ice_load_pkg(NULL, pf);
7220 	}
7221 
7222 	err = ice_clear_pf_cfg(hw);
7223 	if (err) {
7224 		dev_err(dev, "clear PF configuration failed %d\n", err);
7225 		goto err_init_ctrlq;
7226 	}
7227 
7228 	ice_clear_pxe_mode(hw);
7229 
7230 	err = ice_init_nvm(hw);
7231 	if (err) {
7232 		dev_err(dev, "ice_init_nvm failed %d\n", err);
7233 		goto err_init_ctrlq;
7234 	}
7235 
7236 	err = ice_get_caps(hw);
7237 	if (err) {
7238 		dev_err(dev, "ice_get_caps failed %d\n", err);
7239 		goto err_init_ctrlq;
7240 	}
7241 
7242 	err = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL);
7243 	if (err) {
7244 		dev_err(dev, "set_mac_cfg failed %d\n", err);
7245 		goto err_init_ctrlq;
7246 	}
7247 
7248 	dvm = ice_is_dvm_ena(hw);
7249 
7250 	err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL);
7251 	if (err)
7252 		goto err_init_ctrlq;
7253 
7254 	err = ice_sched_init_port(hw->port_info);
7255 	if (err)
7256 		goto err_sched_init_port;
7257 
7258 	/* start misc vector */
7259 	err = ice_req_irq_msix_misc(pf);
7260 	if (err) {
7261 		dev_err(dev, "misc vector setup failed: %d\n", err);
7262 		goto err_sched_init_port;
7263 	}
7264 
7265 	if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) {
7266 		wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M);
7267 		if (!rd32(hw, PFQF_FD_SIZE)) {
7268 			u16 unused, guar, b_effort;
7269 
7270 			guar = hw->func_caps.fd_fltr_guar;
7271 			b_effort = hw->func_caps.fd_fltr_best_effort;
7272 
7273 			/* force guaranteed filter pool for PF */
7274 			ice_alloc_fd_guar_item(hw, &unused, guar);
7275 			/* force shared filter pool for PF */
7276 			ice_alloc_fd_shrd_item(hw, &unused, b_effort);
7277 		}
7278 	}
7279 
7280 	if (test_bit(ICE_FLAG_DCB_ENA, pf->flags))
7281 		ice_dcb_rebuild(pf);
7282 
7283 	/* If the PF previously had enabled PTP, PTP init needs to happen before
7284 	 * the VSI rebuild. If not, this causes the PTP link status events to
7285 	 * fail.
7286 	 */
7287 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
7288 		ice_ptp_reset(pf);
7289 
7290 	if (ice_is_feature_supported(pf, ICE_F_GNSS))
7291 		ice_gnss_init(pf);
7292 
7293 	/* rebuild PF VSI */
7294 	err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF);
7295 	if (err) {
7296 		dev_err(dev, "PF VSI rebuild failed: %d\n", err);
7297 		goto err_vsi_rebuild;
7298 	}
7299 
7300 	/* configure PTP timestamping after VSI rebuild */
7301 	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
7302 		ice_ptp_cfg_timestamp(pf, false);
7303 
7304 	err = ice_vsi_rebuild_by_type(pf, ICE_VSI_SWITCHDEV_CTRL);
7305 	if (err) {
7306 		dev_err(dev, "Switchdev CTRL VSI rebuild failed: %d\n", err);
7307 		goto err_vsi_rebuild;
7308 	}
7309 
7310 	if (reset_type == ICE_RESET_PFR) {
7311 		err = ice_rebuild_channels(pf);
7312 		if (err) {
7313 			dev_err(dev, "failed to rebuild and replay ADQ VSIs, err %d\n",
7314 				err);
7315 			goto err_vsi_rebuild;
7316 		}
7317 	}
7318 
7319 	/* If Flow Director is active */
7320 	if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) {
7321 		err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL);
7322 		if (err) {
7323 			dev_err(dev, "control VSI rebuild failed: %d\n", err);
7324 			goto err_vsi_rebuild;
7325 		}
7326 
7327 		/* replay HW Flow Director recipes */
7328 		if (hw->fdir_prof)
7329 			ice_fdir_replay_flows(hw);
7330 
7331 		/* replay Flow Director filters */
7332 		ice_fdir_replay_fltrs(pf);
7333 
7334 		ice_rebuild_arfs(pf);
7335 	}
7336 
7337 	ice_update_pf_netdev_link(pf);
7338 
7339 	/* tell the firmware we are up */
7340 	err = ice_send_version(pf);
7341 	if (err) {
7342 		dev_err(dev, "Rebuild failed due to error sending driver version: %d\n",
7343 			err);
7344 		goto err_vsi_rebuild;
7345 	}
7346 
7347 	ice_replay_post(hw);
7348 
7349 	/* if we get here, reset flow is successful */
7350 	clear_bit(ICE_RESET_FAILED, pf->state);
7351 
7352 	ice_plug_aux_dev(pf);
7353 	return;
7354 
7355 err_vsi_rebuild:
7356 err_sched_init_port:
7357 	ice_sched_cleanup_all(hw);
7358 err_init_ctrlq:
7359 	ice_shutdown_all_ctrlq(hw);
7360 	set_bit(ICE_RESET_FAILED, pf->state);
7361 clear_recovery:
7362 	/* set this bit in PF state to control service task scheduling */
7363 	set_bit(ICE_NEEDS_RESTART, pf->state);
7364 	dev_err(dev, "Rebuild failed, unload and reload driver\n");
7365 }
7366 
7367 /**
7368  * ice_change_mtu - NDO callback to change the MTU
7369  * @netdev: network interface device structure
7370  * @new_mtu: new value for maximum frame size
7371  *
7372  * Returns 0 on success, negative on failure
7373  */
7374 static int ice_change_mtu(struct net_device *netdev, int new_mtu)
7375 {
7376 	struct ice_netdev_priv *np = netdev_priv(netdev);
7377 	struct ice_vsi *vsi = np->vsi;
7378 	struct ice_pf *pf = vsi->back;
7379 	struct bpf_prog *prog;
7380 	u8 count = 0;
7381 	int err = 0;
7382 
7383 	if (new_mtu == (int)netdev->mtu) {
7384 		netdev_warn(netdev, "MTU is already %u\n", netdev->mtu);
7385 		return 0;
7386 	}
7387 
7388 	prog = vsi->xdp_prog;
7389 	if (prog && !prog->aux->xdp_has_frags) {
7390 		int frame_size = ice_max_xdp_frame_size(vsi);
7391 
7392 		if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) {
7393 			netdev_err(netdev, "max MTU for XDP usage is %d\n",
7394 				   frame_size - ICE_ETH_PKT_HDR_PAD);
7395 			return -EINVAL;
7396 		}
7397 	} else if (test_bit(ICE_FLAG_LEGACY_RX, pf->flags)) {
7398 		if (new_mtu + ICE_ETH_PKT_HDR_PAD > ICE_MAX_FRAME_LEGACY_RX) {
7399 			netdev_err(netdev, "Too big MTU for legacy-rx; Max is %d\n",
7400 				   ICE_MAX_FRAME_LEGACY_RX - ICE_ETH_PKT_HDR_PAD);
7401 			return -EINVAL;
7402 		}
7403 	}
7404 
7405 	/* if a reset is in progress, wait for some time for it to complete */
7406 	do {
7407 		if (ice_is_reset_in_progress(pf->state)) {
7408 			count++;
7409 			usleep_range(1000, 2000);
7410 		} else {
7411 			break;
7412 		}
7413 
7414 	} while (count < 100);
7415 
7416 	if (count == 100) {
7417 		netdev_err(netdev, "can't change MTU. Device is busy\n");
7418 		return -EBUSY;
7419 	}
7420 
7421 	netdev->mtu = (unsigned int)new_mtu;
7422 	err = ice_down_up(vsi);
7423 	if (err)
7424 		return err;
7425 
7426 	netdev_dbg(netdev, "changed MTU to %d\n", new_mtu);
7427 	set_bit(ICE_FLAG_MTU_CHANGED, pf->flags);
7428 
7429 	return err;
7430 }
7431 
7432 /**
7433  * ice_eth_ioctl - Access the hwtstamp interface
7434  * @netdev: network interface device structure
7435  * @ifr: interface request data
7436  * @cmd: ioctl command
7437  */
7438 static int ice_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
7439 {
7440 	struct ice_netdev_priv *np = netdev_priv(netdev);
7441 	struct ice_pf *pf = np->vsi->back;
7442 
7443 	switch (cmd) {
7444 	case SIOCGHWTSTAMP:
7445 		return ice_ptp_get_ts_config(pf, ifr);
7446 	case SIOCSHWTSTAMP:
7447 		return ice_ptp_set_ts_config(pf, ifr);
7448 	default:
7449 		return -EOPNOTSUPP;
7450 	}
7451 }
7452 
7453 /**
7454  * ice_aq_str - convert AQ err code to a string
7455  * @aq_err: the AQ error code to convert
7456  */
7457 const char *ice_aq_str(enum ice_aq_err aq_err)
7458 {
7459 	switch (aq_err) {
7460 	case ICE_AQ_RC_OK:
7461 		return "OK";
7462 	case ICE_AQ_RC_EPERM:
7463 		return "ICE_AQ_RC_EPERM";
7464 	case ICE_AQ_RC_ENOENT:
7465 		return "ICE_AQ_RC_ENOENT";
7466 	case ICE_AQ_RC_ENOMEM:
7467 		return "ICE_AQ_RC_ENOMEM";
7468 	case ICE_AQ_RC_EBUSY:
7469 		return "ICE_AQ_RC_EBUSY";
7470 	case ICE_AQ_RC_EEXIST:
7471 		return "ICE_AQ_RC_EEXIST";
7472 	case ICE_AQ_RC_EINVAL:
7473 		return "ICE_AQ_RC_EINVAL";
7474 	case ICE_AQ_RC_ENOSPC:
7475 		return "ICE_AQ_RC_ENOSPC";
7476 	case ICE_AQ_RC_ENOSYS:
7477 		return "ICE_AQ_RC_ENOSYS";
7478 	case ICE_AQ_RC_EMODE:
7479 		return "ICE_AQ_RC_EMODE";
7480 	case ICE_AQ_RC_ENOSEC:
7481 		return "ICE_AQ_RC_ENOSEC";
7482 	case ICE_AQ_RC_EBADSIG:
7483 		return "ICE_AQ_RC_EBADSIG";
7484 	case ICE_AQ_RC_ESVN:
7485 		return "ICE_AQ_RC_ESVN";
7486 	case ICE_AQ_RC_EBADMAN:
7487 		return "ICE_AQ_RC_EBADMAN";
7488 	case ICE_AQ_RC_EBADBUF:
7489 		return "ICE_AQ_RC_EBADBUF";
7490 	}
7491 
7492 	return "ICE_AQ_RC_UNKNOWN";
7493 }
7494 
7495 /**
7496  * ice_set_rss_lut - Set RSS LUT
7497  * @vsi: Pointer to VSI structure
7498  * @lut: Lookup table
7499  * @lut_size: Lookup table size
7500  *
7501  * Returns 0 on success, negative on failure
7502  */
7503 int ice_set_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size)
7504 {
7505 	struct ice_aq_get_set_rss_lut_params params = {};
7506 	struct ice_hw *hw = &vsi->back->hw;
7507 	int status;
7508 
7509 	if (!lut)
7510 		return -EINVAL;
7511 
7512 	params.vsi_handle = vsi->idx;
7513 	params.lut_size = lut_size;
7514 	params.lut_type = vsi->rss_lut_type;
7515 	params.lut = lut;
7516 
7517 	status = ice_aq_set_rss_lut(hw, &params);
7518 	if (status)
7519 		dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS lut, err %d aq_err %s\n",
7520 			status, ice_aq_str(hw->adminq.sq_last_status));
7521 
7522 	return status;
7523 }
7524 
7525 /**
7526  * ice_set_rss_key - Set RSS key
7527  * @vsi: Pointer to the VSI structure
7528  * @seed: RSS hash seed
7529  *
7530  * Returns 0 on success, negative on failure
7531  */
7532 int ice_set_rss_key(struct ice_vsi *vsi, u8 *seed)
7533 {
7534 	struct ice_hw *hw = &vsi->back->hw;
7535 	int status;
7536 
7537 	if (!seed)
7538 		return -EINVAL;
7539 
7540 	status = ice_aq_set_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed);
7541 	if (status)
7542 		dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS key, err %d aq_err %s\n",
7543 			status, ice_aq_str(hw->adminq.sq_last_status));
7544 
7545 	return status;
7546 }
7547 
7548 /**
7549  * ice_get_rss_lut - Get RSS LUT
7550  * @vsi: Pointer to VSI structure
7551  * @lut: Buffer to store the lookup table entries
7552  * @lut_size: Size of buffer to store the lookup table entries
7553  *
7554  * Returns 0 on success, negative on failure
7555  */
7556 int ice_get_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size)
7557 {
7558 	struct ice_aq_get_set_rss_lut_params params = {};
7559 	struct ice_hw *hw = &vsi->back->hw;
7560 	int status;
7561 
7562 	if (!lut)
7563 		return -EINVAL;
7564 
7565 	params.vsi_handle = vsi->idx;
7566 	params.lut_size = lut_size;
7567 	params.lut_type = vsi->rss_lut_type;
7568 	params.lut = lut;
7569 
7570 	status = ice_aq_get_rss_lut(hw, &params);
7571 	if (status)
7572 		dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS lut, err %d aq_err %s\n",
7573 			status, ice_aq_str(hw->adminq.sq_last_status));
7574 
7575 	return status;
7576 }
7577 
7578 /**
7579  * ice_get_rss_key - Get RSS key
7580  * @vsi: Pointer to VSI structure
7581  * @seed: Buffer to store the key in
7582  *
7583  * Returns 0 on success, negative on failure
7584  */
7585 int ice_get_rss_key(struct ice_vsi *vsi, u8 *seed)
7586 {
7587 	struct ice_hw *hw = &vsi->back->hw;
7588 	int status;
7589 
7590 	if (!seed)
7591 		return -EINVAL;
7592 
7593 	status = ice_aq_get_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed);
7594 	if (status)
7595 		dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS key, err %d aq_err %s\n",
7596 			status, ice_aq_str(hw->adminq.sq_last_status));
7597 
7598 	return status;
7599 }
7600 
7601 /**
7602  * ice_bridge_getlink - Get the hardware bridge mode
7603  * @skb: skb buff
7604  * @pid: process ID
7605  * @seq: RTNL message seq
7606  * @dev: the netdev being configured
7607  * @filter_mask: filter mask passed in
7608  * @nlflags: netlink flags passed in
7609  *
7610  * Return the bridge mode (VEB/VEPA)
7611  */
7612 static int
7613 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
7614 		   struct net_device *dev, u32 filter_mask, int nlflags)
7615 {
7616 	struct ice_netdev_priv *np = netdev_priv(dev);
7617 	struct ice_vsi *vsi = np->vsi;
7618 	struct ice_pf *pf = vsi->back;
7619 	u16 bmode;
7620 
7621 	bmode = pf->first_sw->bridge_mode;
7622 
7623 	return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags,
7624 				       filter_mask, NULL);
7625 }
7626 
7627 /**
7628  * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA)
7629  * @vsi: Pointer to VSI structure
7630  * @bmode: Hardware bridge mode (VEB/VEPA)
7631  *
7632  * Returns 0 on success, negative on failure
7633  */
7634 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode)
7635 {
7636 	struct ice_aqc_vsi_props *vsi_props;
7637 	struct ice_hw *hw = &vsi->back->hw;
7638 	struct ice_vsi_ctx *ctxt;
7639 	int ret;
7640 
7641 	vsi_props = &vsi->info;
7642 
7643 	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
7644 	if (!ctxt)
7645 		return -ENOMEM;
7646 
7647 	ctxt->info = vsi->info;
7648 
7649 	if (bmode == BRIDGE_MODE_VEB)
7650 		/* change from VEPA to VEB mode */
7651 		ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
7652 	else
7653 		/* change from VEB to VEPA mode */
7654 		ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
7655 	ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID);
7656 
7657 	ret = ice_update_vsi(hw, vsi->idx, ctxt, NULL);
7658 	if (ret) {
7659 		dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %d aq_err %s\n",
7660 			bmode, ret, ice_aq_str(hw->adminq.sq_last_status));
7661 		goto out;
7662 	}
7663 	/* Update sw flags for book keeping */
7664 	vsi_props->sw_flags = ctxt->info.sw_flags;
7665 
7666 out:
7667 	kfree(ctxt);
7668 	return ret;
7669 }
7670 
7671 /**
7672  * ice_bridge_setlink - Set the hardware bridge mode
7673  * @dev: the netdev being configured
7674  * @nlh: RTNL message
7675  * @flags: bridge setlink flags
7676  * @extack: netlink extended ack
7677  *
7678  * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is
7679  * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if
7680  * not already set for all VSIs connected to this switch. And also update the
7681  * unicast switch filter rules for the corresponding switch of the netdev.
7682  */
7683 static int
7684 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
7685 		   u16 __always_unused flags,
7686 		   struct netlink_ext_ack __always_unused *extack)
7687 {
7688 	struct ice_netdev_priv *np = netdev_priv(dev);
7689 	struct ice_pf *pf = np->vsi->back;
7690 	struct nlattr *attr, *br_spec;
7691 	struct ice_hw *hw = &pf->hw;
7692 	struct ice_sw *pf_sw;
7693 	int rem, v, err = 0;
7694 
7695 	pf_sw = pf->first_sw;
7696 	/* find the attribute in the netlink message */
7697 	br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
7698 
7699 	nla_for_each_nested(attr, br_spec, rem) {
7700 		__u16 mode;
7701 
7702 		if (nla_type(attr) != IFLA_BRIDGE_MODE)
7703 			continue;
7704 		mode = nla_get_u16(attr);
7705 		if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
7706 			return -EINVAL;
7707 		/* Continue  if bridge mode is not being flipped */
7708 		if (mode == pf_sw->bridge_mode)
7709 			continue;
7710 		/* Iterates through the PF VSI list and update the loopback
7711 		 * mode of the VSI
7712 		 */
7713 		ice_for_each_vsi(pf, v) {
7714 			if (!pf->vsi[v])
7715 				continue;
7716 			err = ice_vsi_update_bridge_mode(pf->vsi[v], mode);
7717 			if (err)
7718 				return err;
7719 		}
7720 
7721 		hw->evb_veb = (mode == BRIDGE_MODE_VEB);
7722 		/* Update the unicast switch filter rules for the corresponding
7723 		 * switch of the netdev
7724 		 */
7725 		err = ice_update_sw_rule_bridge_mode(hw);
7726 		if (err) {
7727 			netdev_err(dev, "switch rule update failed, mode = %d err %d aq_err %s\n",
7728 				   mode, err,
7729 				   ice_aq_str(hw->adminq.sq_last_status));
7730 			/* revert hw->evb_veb */
7731 			hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB);
7732 			return err;
7733 		}
7734 
7735 		pf_sw->bridge_mode = mode;
7736 	}
7737 
7738 	return 0;
7739 }
7740 
7741 /**
7742  * ice_tx_timeout - Respond to a Tx Hang
7743  * @netdev: network interface device structure
7744  * @txqueue: Tx queue
7745  */
7746 static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue)
7747 {
7748 	struct ice_netdev_priv *np = netdev_priv(netdev);
7749 	struct ice_tx_ring *tx_ring = NULL;
7750 	struct ice_vsi *vsi = np->vsi;
7751 	struct ice_pf *pf = vsi->back;
7752 	u32 i;
7753 
7754 	pf->tx_timeout_count++;
7755 
7756 	/* Check if PFC is enabled for the TC to which the queue belongs
7757 	 * to. If yes then Tx timeout is not caused by a hung queue, no
7758 	 * need to reset and rebuild
7759 	 */
7760 	if (ice_is_pfc_causing_hung_q(pf, txqueue)) {
7761 		dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n",
7762 			 txqueue);
7763 		return;
7764 	}
7765 
7766 	/* now that we have an index, find the tx_ring struct */
7767 	ice_for_each_txq(vsi, i)
7768 		if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
7769 			if (txqueue == vsi->tx_rings[i]->q_index) {
7770 				tx_ring = vsi->tx_rings[i];
7771 				break;
7772 			}
7773 
7774 	/* Reset recovery level if enough time has elapsed after last timeout.
7775 	 * Also ensure no new reset action happens before next timeout period.
7776 	 */
7777 	if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20)))
7778 		pf->tx_timeout_recovery_level = 1;
7779 	else if (time_before(jiffies, (pf->tx_timeout_last_recovery +
7780 				       netdev->watchdog_timeo)))
7781 		return;
7782 
7783 	if (tx_ring) {
7784 		struct ice_hw *hw = &pf->hw;
7785 		u32 head, val = 0;
7786 
7787 		head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])) &
7788 			QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S;
7789 		/* Read interrupt register */
7790 		val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx));
7791 
7792 		netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n",
7793 			    vsi->vsi_num, txqueue, tx_ring->next_to_clean,
7794 			    head, tx_ring->next_to_use, val);
7795 	}
7796 
7797 	pf->tx_timeout_last_recovery = jiffies;
7798 	netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n",
7799 		    pf->tx_timeout_recovery_level, txqueue);
7800 
7801 	switch (pf->tx_timeout_recovery_level) {
7802 	case 1:
7803 		set_bit(ICE_PFR_REQ, pf->state);
7804 		break;
7805 	case 2:
7806 		set_bit(ICE_CORER_REQ, pf->state);
7807 		break;
7808 	case 3:
7809 		set_bit(ICE_GLOBR_REQ, pf->state);
7810 		break;
7811 	default:
7812 		netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n");
7813 		set_bit(ICE_DOWN, pf->state);
7814 		set_bit(ICE_VSI_NEEDS_RESTART, vsi->state);
7815 		set_bit(ICE_SERVICE_DIS, pf->state);
7816 		break;
7817 	}
7818 
7819 	ice_service_task_schedule(pf);
7820 	pf->tx_timeout_recovery_level++;
7821 }
7822 
7823 /**
7824  * ice_setup_tc_cls_flower - flower classifier offloads
7825  * @np: net device to configure
7826  * @filter_dev: device on which filter is added
7827  * @cls_flower: offload data
7828  */
7829 static int
7830 ice_setup_tc_cls_flower(struct ice_netdev_priv *np,
7831 			struct net_device *filter_dev,
7832 			struct flow_cls_offload *cls_flower)
7833 {
7834 	struct ice_vsi *vsi = np->vsi;
7835 
7836 	if (cls_flower->common.chain_index)
7837 		return -EOPNOTSUPP;
7838 
7839 	switch (cls_flower->command) {
7840 	case FLOW_CLS_REPLACE:
7841 		return ice_add_cls_flower(filter_dev, vsi, cls_flower);
7842 	case FLOW_CLS_DESTROY:
7843 		return ice_del_cls_flower(vsi, cls_flower);
7844 	default:
7845 		return -EINVAL;
7846 	}
7847 }
7848 
7849 /**
7850  * ice_setup_tc_block_cb - callback handler registered for TC block
7851  * @type: TC SETUP type
7852  * @type_data: TC flower offload data that contains user input
7853  * @cb_priv: netdev private data
7854  */
7855 static int
7856 ice_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv)
7857 {
7858 	struct ice_netdev_priv *np = cb_priv;
7859 
7860 	switch (type) {
7861 	case TC_SETUP_CLSFLOWER:
7862 		return ice_setup_tc_cls_flower(np, np->vsi->netdev,
7863 					       type_data);
7864 	default:
7865 		return -EOPNOTSUPP;
7866 	}
7867 }
7868 
7869 /**
7870  * ice_validate_mqprio_qopt - Validate TCF input parameters
7871  * @vsi: Pointer to VSI
7872  * @mqprio_qopt: input parameters for mqprio queue configuration
7873  *
7874  * This function validates MQPRIO params, such as qcount (power of 2 wherever
7875  * needed), and make sure user doesn't specify qcount and BW rate limit
7876  * for TCs, which are more than "num_tc"
7877  */
7878 static int
7879 ice_validate_mqprio_qopt(struct ice_vsi *vsi,
7880 			 struct tc_mqprio_qopt_offload *mqprio_qopt)
7881 {
7882 	int non_power_of_2_qcount = 0;
7883 	struct ice_pf *pf = vsi->back;
7884 	int max_rss_q_cnt = 0;
7885 	u64 sum_min_rate = 0;
7886 	struct device *dev;
7887 	int i, speed;
7888 	u8 num_tc;
7889 
7890 	if (vsi->type != ICE_VSI_PF)
7891 		return -EINVAL;
7892 
7893 	if (mqprio_qopt->qopt.offset[0] != 0 ||
7894 	    mqprio_qopt->qopt.num_tc < 1 ||
7895 	    mqprio_qopt->qopt.num_tc > ICE_CHNL_MAX_TC)
7896 		return -EINVAL;
7897 
7898 	dev = ice_pf_to_dev(pf);
7899 	vsi->ch_rss_size = 0;
7900 	num_tc = mqprio_qopt->qopt.num_tc;
7901 	speed = ice_get_link_speed_kbps(vsi);
7902 
7903 	for (i = 0; num_tc; i++) {
7904 		int qcount = mqprio_qopt->qopt.count[i];
7905 		u64 max_rate, min_rate, rem;
7906 
7907 		if (!qcount)
7908 			return -EINVAL;
7909 
7910 		if (is_power_of_2(qcount)) {
7911 			if (non_power_of_2_qcount &&
7912 			    qcount > non_power_of_2_qcount) {
7913 				dev_err(dev, "qcount[%d] cannot be greater than non power of 2 qcount[%d]\n",
7914 					qcount, non_power_of_2_qcount);
7915 				return -EINVAL;
7916 			}
7917 			if (qcount > max_rss_q_cnt)
7918 				max_rss_q_cnt = qcount;
7919 		} else {
7920 			if (non_power_of_2_qcount &&
7921 			    qcount != non_power_of_2_qcount) {
7922 				dev_err(dev, "Only one non power of 2 qcount allowed[%d,%d]\n",
7923 					qcount, non_power_of_2_qcount);
7924 				return -EINVAL;
7925 			}
7926 			if (qcount < max_rss_q_cnt) {
7927 				dev_err(dev, "non power of 2 qcount[%d] cannot be less than other qcount[%d]\n",
7928 					qcount, max_rss_q_cnt);
7929 				return -EINVAL;
7930 			}
7931 			max_rss_q_cnt = qcount;
7932 			non_power_of_2_qcount = qcount;
7933 		}
7934 
7935 		/* TC command takes input in K/N/Gbps or K/M/Gbit etc but
7936 		 * converts the bandwidth rate limit into Bytes/s when
7937 		 * passing it down to the driver. So convert input bandwidth
7938 		 * from Bytes/s to Kbps
7939 		 */
7940 		max_rate = mqprio_qopt->max_rate[i];
7941 		max_rate = div_u64(max_rate, ICE_BW_KBPS_DIVISOR);
7942 
7943 		/* min_rate is minimum guaranteed rate and it can't be zero */
7944 		min_rate = mqprio_qopt->min_rate[i];
7945 		min_rate = div_u64(min_rate, ICE_BW_KBPS_DIVISOR);
7946 		sum_min_rate += min_rate;
7947 
7948 		if (min_rate && min_rate < ICE_MIN_BW_LIMIT) {
7949 			dev_err(dev, "TC%d: min_rate(%llu Kbps) < %u Kbps\n", i,
7950 				min_rate, ICE_MIN_BW_LIMIT);
7951 			return -EINVAL;
7952 		}
7953 
7954 		if (max_rate && max_rate > speed) {
7955 			dev_err(dev, "TC%d: max_rate(%llu Kbps) > link speed of %u Kbps\n",
7956 				i, max_rate, speed);
7957 			return -EINVAL;
7958 		}
7959 
7960 		iter_div_u64_rem(min_rate, ICE_MIN_BW_LIMIT, &rem);
7961 		if (rem) {
7962 			dev_err(dev, "TC%d: Min Rate not multiple of %u Kbps",
7963 				i, ICE_MIN_BW_LIMIT);
7964 			return -EINVAL;
7965 		}
7966 
7967 		iter_div_u64_rem(max_rate, ICE_MIN_BW_LIMIT, &rem);
7968 		if (rem) {
7969 			dev_err(dev, "TC%d: Max Rate not multiple of %u Kbps",
7970 				i, ICE_MIN_BW_LIMIT);
7971 			return -EINVAL;
7972 		}
7973 
7974 		/* min_rate can't be more than max_rate, except when max_rate
7975 		 * is zero (implies max_rate sought is max line rate). In such
7976 		 * a case min_rate can be more than max.
7977 		 */
7978 		if (max_rate && min_rate > max_rate) {
7979 			dev_err(dev, "min_rate %llu Kbps can't be more than max_rate %llu Kbps\n",
7980 				min_rate, max_rate);
7981 			return -EINVAL;
7982 		}
7983 
7984 		if (i >= mqprio_qopt->qopt.num_tc - 1)
7985 			break;
7986 		if (mqprio_qopt->qopt.offset[i + 1] !=
7987 		    (mqprio_qopt->qopt.offset[i] + qcount))
7988 			return -EINVAL;
7989 	}
7990 	if (vsi->num_rxq <
7991 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7992 		return -EINVAL;
7993 	if (vsi->num_txq <
7994 	    (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
7995 		return -EINVAL;
7996 
7997 	if (sum_min_rate && sum_min_rate > (u64)speed) {
7998 		dev_err(dev, "Invalid min Tx rate(%llu) Kbps > speed (%u) Kbps specified\n",
7999 			sum_min_rate, speed);
8000 		return -EINVAL;
8001 	}
8002 
8003 	/* make sure vsi->ch_rss_size is set correctly based on TC's qcount */
8004 	vsi->ch_rss_size = max_rss_q_cnt;
8005 
8006 	return 0;
8007 }
8008 
8009 /**
8010  * ice_add_vsi_to_fdir - add a VSI to the flow director group for PF
8011  * @pf: ptr to PF device
8012  * @vsi: ptr to VSI
8013  */
8014 static int ice_add_vsi_to_fdir(struct ice_pf *pf, struct ice_vsi *vsi)
8015 {
8016 	struct device *dev = ice_pf_to_dev(pf);
8017 	bool added = false;
8018 	struct ice_hw *hw;
8019 	int flow;
8020 
8021 	if (!(vsi->num_gfltr || vsi->num_bfltr))
8022 		return -EINVAL;
8023 
8024 	hw = &pf->hw;
8025 	for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) {
8026 		struct ice_fd_hw_prof *prof;
8027 		int tun, status;
8028 		u64 entry_h;
8029 
8030 		if (!(hw->fdir_prof && hw->fdir_prof[flow] &&
8031 		      hw->fdir_prof[flow]->cnt))
8032 			continue;
8033 
8034 		for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) {
8035 			enum ice_flow_priority prio;
8036 			u64 prof_id;
8037 
8038 			/* add this VSI to FDir profile for this flow */
8039 			prio = ICE_FLOW_PRIO_NORMAL;
8040 			prof = hw->fdir_prof[flow];
8041 			prof_id = flow + tun * ICE_FLTR_PTYPE_MAX;
8042 			status = ice_flow_add_entry(hw, ICE_BLK_FD, prof_id,
8043 						    prof->vsi_h[0], vsi->idx,
8044 						    prio, prof->fdir_seg[tun],
8045 						    &entry_h);
8046 			if (status) {
8047 				dev_err(dev, "channel VSI idx %d, not able to add to group %d\n",
8048 					vsi->idx, flow);
8049 				continue;
8050 			}
8051 
8052 			prof->entry_h[prof->cnt][tun] = entry_h;
8053 		}
8054 
8055 		/* store VSI for filter replay and delete */
8056 		prof->vsi_h[prof->cnt] = vsi->idx;
8057 		prof->cnt++;
8058 
8059 		added = true;
8060 		dev_dbg(dev, "VSI idx %d added to fdir group %d\n", vsi->idx,
8061 			flow);
8062 	}
8063 
8064 	if (!added)
8065 		dev_dbg(dev, "VSI idx %d not added to fdir groups\n", vsi->idx);
8066 
8067 	return 0;
8068 }
8069 
8070 /**
8071  * ice_add_channel - add a channel by adding VSI
8072  * @pf: ptr to PF device
8073  * @sw_id: underlying HW switching element ID
8074  * @ch: ptr to channel structure
8075  *
8076  * Add a channel (VSI) using add_vsi and queue_map
8077  */
8078 static int ice_add_channel(struct ice_pf *pf, u16 sw_id, struct ice_channel *ch)
8079 {
8080 	struct device *dev = ice_pf_to_dev(pf);
8081 	struct ice_vsi *vsi;
8082 
8083 	if (ch->type != ICE_VSI_CHNL) {
8084 		dev_err(dev, "add new VSI failed, ch->type %d\n", ch->type);
8085 		return -EINVAL;
8086 	}
8087 
8088 	vsi = ice_chnl_vsi_setup(pf, pf->hw.port_info, ch);
8089 	if (!vsi || vsi->type != ICE_VSI_CHNL) {
8090 		dev_err(dev, "create chnl VSI failure\n");
8091 		return -EINVAL;
8092 	}
8093 
8094 	ice_add_vsi_to_fdir(pf, vsi);
8095 
8096 	ch->sw_id = sw_id;
8097 	ch->vsi_num = vsi->vsi_num;
8098 	ch->info.mapping_flags = vsi->info.mapping_flags;
8099 	ch->ch_vsi = vsi;
8100 	/* set the back pointer of channel for newly created VSI */
8101 	vsi->ch = ch;
8102 
8103 	memcpy(&ch->info.q_mapping, &vsi->info.q_mapping,
8104 	       sizeof(vsi->info.q_mapping));
8105 	memcpy(&ch->info.tc_mapping, vsi->info.tc_mapping,
8106 	       sizeof(vsi->info.tc_mapping));
8107 
8108 	return 0;
8109 }
8110 
8111 /**
8112  * ice_chnl_cfg_res
8113  * @vsi: the VSI being setup
8114  * @ch: ptr to channel structure
8115  *
8116  * Configure channel specific resources such as rings, vector.
8117  */
8118 static void ice_chnl_cfg_res(struct ice_vsi *vsi, struct ice_channel *ch)
8119 {
8120 	int i;
8121 
8122 	for (i = 0; i < ch->num_txq; i++) {
8123 		struct ice_q_vector *tx_q_vector, *rx_q_vector;
8124 		struct ice_ring_container *rc;
8125 		struct ice_tx_ring *tx_ring;
8126 		struct ice_rx_ring *rx_ring;
8127 
8128 		tx_ring = vsi->tx_rings[ch->base_q + i];
8129 		rx_ring = vsi->rx_rings[ch->base_q + i];
8130 		if (!tx_ring || !rx_ring)
8131 			continue;
8132 
8133 		/* setup ring being channel enabled */
8134 		tx_ring->ch = ch;
8135 		rx_ring->ch = ch;
8136 
8137 		/* following code block sets up vector specific attributes */
8138 		tx_q_vector = tx_ring->q_vector;
8139 		rx_q_vector = rx_ring->q_vector;
8140 		if (!tx_q_vector && !rx_q_vector)
8141 			continue;
8142 
8143 		if (tx_q_vector) {
8144 			tx_q_vector->ch = ch;
8145 			/* setup Tx and Rx ITR setting if DIM is off */
8146 			rc = &tx_q_vector->tx;
8147 			if (!ITR_IS_DYNAMIC(rc))
8148 				ice_write_itr(rc, rc->itr_setting);
8149 		}
8150 		if (rx_q_vector) {
8151 			rx_q_vector->ch = ch;
8152 			/* setup Tx and Rx ITR setting if DIM is off */
8153 			rc = &rx_q_vector->rx;
8154 			if (!ITR_IS_DYNAMIC(rc))
8155 				ice_write_itr(rc, rc->itr_setting);
8156 		}
8157 	}
8158 
8159 	/* it is safe to assume that, if channel has non-zero num_t[r]xq, then
8160 	 * GLINT_ITR register would have written to perform in-context
8161 	 * update, hence perform flush
8162 	 */
8163 	if (ch->num_txq || ch->num_rxq)
8164 		ice_flush(&vsi->back->hw);
8165 }
8166 
8167 /**
8168  * ice_cfg_chnl_all_res - configure channel resources
8169  * @vsi: pte to main_vsi
8170  * @ch: ptr to channel structure
8171  *
8172  * This function configures channel specific resources such as flow-director
8173  * counter index, and other resources such as queues, vectors, ITR settings
8174  */
8175 static void
8176 ice_cfg_chnl_all_res(struct ice_vsi *vsi, struct ice_channel *ch)
8177 {
8178 	/* configure channel (aka ADQ) resources such as queues, vectors,
8179 	 * ITR settings for channel specific vectors and anything else
8180 	 */
8181 	ice_chnl_cfg_res(vsi, ch);
8182 }
8183 
8184 /**
8185  * ice_setup_hw_channel - setup new channel
8186  * @pf: ptr to PF device
8187  * @vsi: the VSI being setup
8188  * @ch: ptr to channel structure
8189  * @sw_id: underlying HW switching element ID
8190  * @type: type of channel to be created (VMDq2/VF)
8191  *
8192  * Setup new channel (VSI) based on specified type (VMDq2/VF)
8193  * and configures Tx rings accordingly
8194  */
8195 static int
8196 ice_setup_hw_channel(struct ice_pf *pf, struct ice_vsi *vsi,
8197 		     struct ice_channel *ch, u16 sw_id, u8 type)
8198 {
8199 	struct device *dev = ice_pf_to_dev(pf);
8200 	int ret;
8201 
8202 	ch->base_q = vsi->next_base_q;
8203 	ch->type = type;
8204 
8205 	ret = ice_add_channel(pf, sw_id, ch);
8206 	if (ret) {
8207 		dev_err(dev, "failed to add_channel using sw_id %u\n", sw_id);
8208 		return ret;
8209 	}
8210 
8211 	/* configure/setup ADQ specific resources */
8212 	ice_cfg_chnl_all_res(vsi, ch);
8213 
8214 	/* make sure to update the next_base_q so that subsequent channel's
8215 	 * (aka ADQ) VSI queue map is correct
8216 	 */
8217 	vsi->next_base_q = vsi->next_base_q + ch->num_rxq;
8218 	dev_dbg(dev, "added channel: vsi_num %u, num_rxq %u\n", ch->vsi_num,
8219 		ch->num_rxq);
8220 
8221 	return 0;
8222 }
8223 
8224 /**
8225  * ice_setup_channel - setup new channel using uplink element
8226  * @pf: ptr to PF device
8227  * @vsi: the VSI being setup
8228  * @ch: ptr to channel structure
8229  *
8230  * Setup new channel (VSI) based on specified type (VMDq2/VF)
8231  * and uplink switching element
8232  */
8233 static bool
8234 ice_setup_channel(struct ice_pf *pf, struct ice_vsi *vsi,
8235 		  struct ice_channel *ch)
8236 {
8237 	struct device *dev = ice_pf_to_dev(pf);
8238 	u16 sw_id;
8239 	int ret;
8240 
8241 	if (vsi->type != ICE_VSI_PF) {
8242 		dev_err(dev, "unsupported parent VSI type(%d)\n", vsi->type);
8243 		return false;
8244 	}
8245 
8246 	sw_id = pf->first_sw->sw_id;
8247 
8248 	/* create channel (VSI) */
8249 	ret = ice_setup_hw_channel(pf, vsi, ch, sw_id, ICE_VSI_CHNL);
8250 	if (ret) {
8251 		dev_err(dev, "failed to setup hw_channel\n");
8252 		return false;
8253 	}
8254 	dev_dbg(dev, "successfully created channel()\n");
8255 
8256 	return ch->ch_vsi ? true : false;
8257 }
8258 
8259 /**
8260  * ice_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate
8261  * @vsi: VSI to be configured
8262  * @max_tx_rate: max Tx rate in Kbps to be configured as maximum BW limit
8263  * @min_tx_rate: min Tx rate in Kbps to be configured as minimum BW limit
8264  */
8265 static int
8266 ice_set_bw_limit(struct ice_vsi *vsi, u64 max_tx_rate, u64 min_tx_rate)
8267 {
8268 	int err;
8269 
8270 	err = ice_set_min_bw_limit(vsi, min_tx_rate);
8271 	if (err)
8272 		return err;
8273 
8274 	return ice_set_max_bw_limit(vsi, max_tx_rate);
8275 }
8276 
8277 /**
8278  * ice_create_q_channel - function to create channel
8279  * @vsi: VSI to be configured
8280  * @ch: ptr to channel (it contains channel specific params)
8281  *
8282  * This function creates channel (VSI) using num_queues specified by user,
8283  * reconfigs RSS if needed.
8284  */
8285 static int ice_create_q_channel(struct ice_vsi *vsi, struct ice_channel *ch)
8286 {
8287 	struct ice_pf *pf = vsi->back;
8288 	struct device *dev;
8289 
8290 	if (!ch)
8291 		return -EINVAL;
8292 
8293 	dev = ice_pf_to_dev(pf);
8294 	if (!ch->num_txq || !ch->num_rxq) {
8295 		dev_err(dev, "Invalid num_queues requested: %d\n", ch->num_rxq);
8296 		return -EINVAL;
8297 	}
8298 
8299 	if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_txq) {
8300 		dev_err(dev, "cnt_q_avail (%u) less than num_queues %d\n",
8301 			vsi->cnt_q_avail, ch->num_txq);
8302 		return -EINVAL;
8303 	}
8304 
8305 	if (!ice_setup_channel(pf, vsi, ch)) {
8306 		dev_info(dev, "Failed to setup channel\n");
8307 		return -EINVAL;
8308 	}
8309 	/* configure BW rate limit */
8310 	if (ch->ch_vsi && (ch->max_tx_rate || ch->min_tx_rate)) {
8311 		int ret;
8312 
8313 		ret = ice_set_bw_limit(ch->ch_vsi, ch->max_tx_rate,
8314 				       ch->min_tx_rate);
8315 		if (ret)
8316 			dev_err(dev, "failed to set Tx rate of %llu Kbps for VSI(%u)\n",
8317 				ch->max_tx_rate, ch->ch_vsi->vsi_num);
8318 		else
8319 			dev_dbg(dev, "set Tx rate of %llu Kbps for VSI(%u)\n",
8320 				ch->max_tx_rate, ch->ch_vsi->vsi_num);
8321 	}
8322 
8323 	vsi->cnt_q_avail -= ch->num_txq;
8324 
8325 	return 0;
8326 }
8327 
8328 /**
8329  * ice_rem_all_chnl_fltrs - removes all channel filters
8330  * @pf: ptr to PF, TC-flower based filter are tracked at PF level
8331  *
8332  * Remove all advanced switch filters only if they are channel specific
8333  * tc-flower based filter
8334  */
8335 static void ice_rem_all_chnl_fltrs(struct ice_pf *pf)
8336 {
8337 	struct ice_tc_flower_fltr *fltr;
8338 	struct hlist_node *node;
8339 
8340 	/* to remove all channel filters, iterate an ordered list of filters */
8341 	hlist_for_each_entry_safe(fltr, node,
8342 				  &pf->tc_flower_fltr_list,
8343 				  tc_flower_node) {
8344 		struct ice_rule_query_data rule;
8345 		int status;
8346 
8347 		/* for now process only channel specific filters */
8348 		if (!ice_is_chnl_fltr(fltr))
8349 			continue;
8350 
8351 		rule.rid = fltr->rid;
8352 		rule.rule_id = fltr->rule_id;
8353 		rule.vsi_handle = fltr->dest_vsi_handle;
8354 		status = ice_rem_adv_rule_by_id(&pf->hw, &rule);
8355 		if (status) {
8356 			if (status == -ENOENT)
8357 				dev_dbg(ice_pf_to_dev(pf), "TC flower filter (rule_id %u) does not exist\n",
8358 					rule.rule_id);
8359 			else
8360 				dev_err(ice_pf_to_dev(pf), "failed to delete TC flower filter, status %d\n",
8361 					status);
8362 		} else if (fltr->dest_vsi) {
8363 			/* update advanced switch filter count */
8364 			if (fltr->dest_vsi->type == ICE_VSI_CHNL) {
8365 				u32 flags = fltr->flags;
8366 
8367 				fltr->dest_vsi->num_chnl_fltr--;
8368 				if (flags & (ICE_TC_FLWR_FIELD_DST_MAC |
8369 					     ICE_TC_FLWR_FIELD_ENC_DST_MAC))
8370 					pf->num_dmac_chnl_fltrs--;
8371 			}
8372 		}
8373 
8374 		hlist_del(&fltr->tc_flower_node);
8375 		kfree(fltr);
8376 	}
8377 }
8378 
8379 /**
8380  * ice_remove_q_channels - Remove queue channels for the TCs
8381  * @vsi: VSI to be configured
8382  * @rem_fltr: delete advanced switch filter or not
8383  *
8384  * Remove queue channels for the TCs
8385  */
8386 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_fltr)
8387 {
8388 	struct ice_channel *ch, *ch_tmp;
8389 	struct ice_pf *pf = vsi->back;
8390 	int i;
8391 
8392 	/* remove all tc-flower based filter if they are channel filters only */
8393 	if (rem_fltr)
8394 		ice_rem_all_chnl_fltrs(pf);
8395 
8396 	/* remove ntuple filters since queue configuration is being changed */
8397 	if  (vsi->netdev->features & NETIF_F_NTUPLE) {
8398 		struct ice_hw *hw = &pf->hw;
8399 
8400 		mutex_lock(&hw->fdir_fltr_lock);
8401 		ice_fdir_del_all_fltrs(vsi);
8402 		mutex_unlock(&hw->fdir_fltr_lock);
8403 	}
8404 
8405 	/* perform cleanup for channels if they exist */
8406 	list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
8407 		struct ice_vsi *ch_vsi;
8408 
8409 		list_del(&ch->list);
8410 		ch_vsi = ch->ch_vsi;
8411 		if (!ch_vsi) {
8412 			kfree(ch);
8413 			continue;
8414 		}
8415 
8416 		/* Reset queue contexts */
8417 		for (i = 0; i < ch->num_rxq; i++) {
8418 			struct ice_tx_ring *tx_ring;
8419 			struct ice_rx_ring *rx_ring;
8420 
8421 			tx_ring = vsi->tx_rings[ch->base_q + i];
8422 			rx_ring = vsi->rx_rings[ch->base_q + i];
8423 			if (tx_ring) {
8424 				tx_ring->ch = NULL;
8425 				if (tx_ring->q_vector)
8426 					tx_ring->q_vector->ch = NULL;
8427 			}
8428 			if (rx_ring) {
8429 				rx_ring->ch = NULL;
8430 				if (rx_ring->q_vector)
8431 					rx_ring->q_vector->ch = NULL;
8432 			}
8433 		}
8434 
8435 		/* Release FD resources for the channel VSI */
8436 		ice_fdir_rem_adq_chnl(&pf->hw, ch->ch_vsi->idx);
8437 
8438 		/* clear the VSI from scheduler tree */
8439 		ice_rm_vsi_lan_cfg(ch->ch_vsi->port_info, ch->ch_vsi->idx);
8440 
8441 		/* Delete VSI from FW, PF and HW VSI arrays */
8442 		ice_vsi_delete(ch->ch_vsi);
8443 
8444 		/* free the channel */
8445 		kfree(ch);
8446 	}
8447 
8448 	/* clear the channel VSI map which is stored in main VSI */
8449 	ice_for_each_chnl_tc(i)
8450 		vsi->tc_map_vsi[i] = NULL;
8451 
8452 	/* reset main VSI's all TC information */
8453 	vsi->all_enatc = 0;
8454 	vsi->all_numtc = 0;
8455 }
8456 
8457 /**
8458  * ice_rebuild_channels - rebuild channel
8459  * @pf: ptr to PF
8460  *
8461  * Recreate channel VSIs and replay filters
8462  */
8463 static int ice_rebuild_channels(struct ice_pf *pf)
8464 {
8465 	struct device *dev = ice_pf_to_dev(pf);
8466 	struct ice_vsi *main_vsi;
8467 	bool rem_adv_fltr = true;
8468 	struct ice_channel *ch;
8469 	struct ice_vsi *vsi;
8470 	int tc_idx = 1;
8471 	int i, err;
8472 
8473 	main_vsi = ice_get_main_vsi(pf);
8474 	if (!main_vsi)
8475 		return 0;
8476 
8477 	if (!test_bit(ICE_FLAG_TC_MQPRIO, pf->flags) ||
8478 	    main_vsi->old_numtc == 1)
8479 		return 0; /* nothing to be done */
8480 
8481 	/* reconfigure main VSI based on old value of TC and cached values
8482 	 * for MQPRIO opts
8483 	 */
8484 	err = ice_vsi_cfg_tc(main_vsi, main_vsi->old_ena_tc);
8485 	if (err) {
8486 		dev_err(dev, "failed configuring TC(ena_tc:0x%02x) for HW VSI=%u\n",
8487 			main_vsi->old_ena_tc, main_vsi->vsi_num);
8488 		return err;
8489 	}
8490 
8491 	/* rebuild ADQ VSIs */
8492 	ice_for_each_vsi(pf, i) {
8493 		enum ice_vsi_type type;
8494 
8495 		vsi = pf->vsi[i];
8496 		if (!vsi || vsi->type != ICE_VSI_CHNL)
8497 			continue;
8498 
8499 		type = vsi->type;
8500 
8501 		/* rebuild ADQ VSI */
8502 		err = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_INIT);
8503 		if (err) {
8504 			dev_err(dev, "VSI (type:%s) at index %d rebuild failed, err %d\n",
8505 				ice_vsi_type_str(type), vsi->idx, err);
8506 			goto cleanup;
8507 		}
8508 
8509 		/* Re-map HW VSI number, using VSI handle that has been
8510 		 * previously validated in ice_replay_vsi() call above
8511 		 */
8512 		vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx);
8513 
8514 		/* replay filters for the VSI */
8515 		err = ice_replay_vsi(&pf->hw, vsi->idx);
8516 		if (err) {
8517 			dev_err(dev, "VSI (type:%s) replay failed, err %d, VSI index %d\n",
8518 				ice_vsi_type_str(type), err, vsi->idx);
8519 			rem_adv_fltr = false;
8520 			goto cleanup;
8521 		}
8522 		dev_info(dev, "VSI (type:%s) at index %d rebuilt successfully\n",
8523 			 ice_vsi_type_str(type), vsi->idx);
8524 
8525 		/* store ADQ VSI at correct TC index in main VSI's
8526 		 * map of TC to VSI
8527 		 */
8528 		main_vsi->tc_map_vsi[tc_idx++] = vsi;
8529 	}
8530 
8531 	/* ADQ VSI(s) has been rebuilt successfully, so setup
8532 	 * channel for main VSI's Tx and Rx rings
8533 	 */
8534 	list_for_each_entry(ch, &main_vsi->ch_list, list) {
8535 		struct ice_vsi *ch_vsi;
8536 
8537 		ch_vsi = ch->ch_vsi;
8538 		if (!ch_vsi)
8539 			continue;
8540 
8541 		/* reconfig channel resources */
8542 		ice_cfg_chnl_all_res(main_vsi, ch);
8543 
8544 		/* replay BW rate limit if it is non-zero */
8545 		if (!ch->max_tx_rate && !ch->min_tx_rate)
8546 			continue;
8547 
8548 		err = ice_set_bw_limit(ch_vsi, ch->max_tx_rate,
8549 				       ch->min_tx_rate);
8550 		if (err)
8551 			dev_err(dev, "failed (err:%d) to rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n",
8552 				err, ch->max_tx_rate, ch->min_tx_rate,
8553 				ch_vsi->vsi_num);
8554 		else
8555 			dev_dbg(dev, "successfully rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n",
8556 				ch->max_tx_rate, ch->min_tx_rate,
8557 				ch_vsi->vsi_num);
8558 	}
8559 
8560 	/* reconfig RSS for main VSI */
8561 	if (main_vsi->ch_rss_size)
8562 		ice_vsi_cfg_rss_lut_key(main_vsi);
8563 
8564 	return 0;
8565 
8566 cleanup:
8567 	ice_remove_q_channels(main_vsi, rem_adv_fltr);
8568 	return err;
8569 }
8570 
8571 /**
8572  * ice_create_q_channels - Add queue channel for the given TCs
8573  * @vsi: VSI to be configured
8574  *
8575  * Configures queue channel mapping to the given TCs
8576  */
8577 static int ice_create_q_channels(struct ice_vsi *vsi)
8578 {
8579 	struct ice_pf *pf = vsi->back;
8580 	struct ice_channel *ch;
8581 	int ret = 0, i;
8582 
8583 	ice_for_each_chnl_tc(i) {
8584 		if (!(vsi->all_enatc & BIT(i)))
8585 			continue;
8586 
8587 		ch = kzalloc(sizeof(*ch), GFP_KERNEL);
8588 		if (!ch) {
8589 			ret = -ENOMEM;
8590 			goto err_free;
8591 		}
8592 		INIT_LIST_HEAD(&ch->list);
8593 		ch->num_rxq = vsi->mqprio_qopt.qopt.count[i];
8594 		ch->num_txq = vsi->mqprio_qopt.qopt.count[i];
8595 		ch->base_q = vsi->mqprio_qopt.qopt.offset[i];
8596 		ch->max_tx_rate = vsi->mqprio_qopt.max_rate[i];
8597 		ch->min_tx_rate = vsi->mqprio_qopt.min_rate[i];
8598 
8599 		/* convert to Kbits/s */
8600 		if (ch->max_tx_rate)
8601 			ch->max_tx_rate = div_u64(ch->max_tx_rate,
8602 						  ICE_BW_KBPS_DIVISOR);
8603 		if (ch->min_tx_rate)
8604 			ch->min_tx_rate = div_u64(ch->min_tx_rate,
8605 						  ICE_BW_KBPS_DIVISOR);
8606 
8607 		ret = ice_create_q_channel(vsi, ch);
8608 		if (ret) {
8609 			dev_err(ice_pf_to_dev(pf),
8610 				"failed creating channel TC:%d\n", i);
8611 			kfree(ch);
8612 			goto err_free;
8613 		}
8614 		list_add_tail(&ch->list, &vsi->ch_list);
8615 		vsi->tc_map_vsi[i] = ch->ch_vsi;
8616 		dev_dbg(ice_pf_to_dev(pf),
8617 			"successfully created channel: VSI %pK\n", ch->ch_vsi);
8618 	}
8619 	return 0;
8620 
8621 err_free:
8622 	ice_remove_q_channels(vsi, false);
8623 
8624 	return ret;
8625 }
8626 
8627 /**
8628  * ice_setup_tc_mqprio_qdisc - configure multiple traffic classes
8629  * @netdev: net device to configure
8630  * @type_data: TC offload data
8631  */
8632 static int ice_setup_tc_mqprio_qdisc(struct net_device *netdev, void *type_data)
8633 {
8634 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
8635 	struct ice_netdev_priv *np = netdev_priv(netdev);
8636 	struct ice_vsi *vsi = np->vsi;
8637 	struct ice_pf *pf = vsi->back;
8638 	u16 mode, ena_tc_qdisc = 0;
8639 	int cur_txq, cur_rxq;
8640 	u8 hw = 0, num_tcf;
8641 	struct device *dev;
8642 	int ret, i;
8643 
8644 	dev = ice_pf_to_dev(pf);
8645 	num_tcf = mqprio_qopt->qopt.num_tc;
8646 	hw = mqprio_qopt->qopt.hw;
8647 	mode = mqprio_qopt->mode;
8648 	if (!hw) {
8649 		clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
8650 		vsi->ch_rss_size = 0;
8651 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8652 		goto config_tcf;
8653 	}
8654 
8655 	/* Generate queue region map for number of TCF requested */
8656 	for (i = 0; i < num_tcf; i++)
8657 		ena_tc_qdisc |= BIT(i);
8658 
8659 	switch (mode) {
8660 	case TC_MQPRIO_MODE_CHANNEL:
8661 
8662 		if (pf->hw.port_info->is_custom_tx_enabled) {
8663 			dev_err(dev, "Custom Tx scheduler feature enabled, can't configure ADQ\n");
8664 			return -EBUSY;
8665 		}
8666 		ice_tear_down_devlink_rate_tree(pf);
8667 
8668 		ret = ice_validate_mqprio_qopt(vsi, mqprio_qopt);
8669 		if (ret) {
8670 			netdev_err(netdev, "failed to validate_mqprio_qopt(), ret %d\n",
8671 				   ret);
8672 			return ret;
8673 		}
8674 		memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
8675 		set_bit(ICE_FLAG_TC_MQPRIO, pf->flags);
8676 		/* don't assume state of hw_tc_offload during driver load
8677 		 * and set the flag for TC flower filter if hw_tc_offload
8678 		 * already ON
8679 		 */
8680 		if (vsi->netdev->features & NETIF_F_HW_TC)
8681 			set_bit(ICE_FLAG_CLS_FLOWER, pf->flags);
8682 		break;
8683 	default:
8684 		return -EINVAL;
8685 	}
8686 
8687 config_tcf:
8688 
8689 	/* Requesting same TCF configuration as already enabled */
8690 	if (ena_tc_qdisc == vsi->tc_cfg.ena_tc &&
8691 	    mode != TC_MQPRIO_MODE_CHANNEL)
8692 		return 0;
8693 
8694 	/* Pause VSI queues */
8695 	ice_dis_vsi(vsi, true);
8696 
8697 	if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags))
8698 		ice_remove_q_channels(vsi, true);
8699 
8700 	if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
8701 		vsi->req_txq = min_t(int, ice_get_avail_txq_count(pf),
8702 				     num_online_cpus());
8703 		vsi->req_rxq = min_t(int, ice_get_avail_rxq_count(pf),
8704 				     num_online_cpus());
8705 	} else {
8706 		/* logic to rebuild VSI, same like ethtool -L */
8707 		u16 offset = 0, qcount_tx = 0, qcount_rx = 0;
8708 
8709 		for (i = 0; i < num_tcf; i++) {
8710 			if (!(ena_tc_qdisc & BIT(i)))
8711 				continue;
8712 
8713 			offset = vsi->mqprio_qopt.qopt.offset[i];
8714 			qcount_rx = vsi->mqprio_qopt.qopt.count[i];
8715 			qcount_tx = vsi->mqprio_qopt.qopt.count[i];
8716 		}
8717 		vsi->req_txq = offset + qcount_tx;
8718 		vsi->req_rxq = offset + qcount_rx;
8719 
8720 		/* store away original rss_size info, so that it gets reused
8721 		 * form ice_vsi_rebuild during tc-qdisc delete stage - to
8722 		 * determine, what should be the rss_sizefor main VSI
8723 		 */
8724 		vsi->orig_rss_size = vsi->rss_size;
8725 	}
8726 
8727 	/* save current values of Tx and Rx queues before calling VSI rebuild
8728 	 * for fallback option
8729 	 */
8730 	cur_txq = vsi->num_txq;
8731 	cur_rxq = vsi->num_rxq;
8732 
8733 	/* proceed with rebuild main VSI using correct number of queues */
8734 	ret = ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT);
8735 	if (ret) {
8736 		/* fallback to current number of queues */
8737 		dev_info(dev, "Rebuild failed with new queues, try with current number of queues\n");
8738 		vsi->req_txq = cur_txq;
8739 		vsi->req_rxq = cur_rxq;
8740 		clear_bit(ICE_RESET_FAILED, pf->state);
8741 		if (ice_vsi_rebuild(vsi, ICE_VSI_FLAG_NO_INIT)) {
8742 			dev_err(dev, "Rebuild of main VSI failed again\n");
8743 			return ret;
8744 		}
8745 	}
8746 
8747 	vsi->all_numtc = num_tcf;
8748 	vsi->all_enatc = ena_tc_qdisc;
8749 	ret = ice_vsi_cfg_tc(vsi, ena_tc_qdisc);
8750 	if (ret) {
8751 		netdev_err(netdev, "failed configuring TC for VSI id=%d\n",
8752 			   vsi->vsi_num);
8753 		goto exit;
8754 	}
8755 
8756 	if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) {
8757 		u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
8758 		u64 min_tx_rate = vsi->mqprio_qopt.min_rate[0];
8759 
8760 		/* set TC0 rate limit if specified */
8761 		if (max_tx_rate || min_tx_rate) {
8762 			/* convert to Kbits/s */
8763 			if (max_tx_rate)
8764 				max_tx_rate = div_u64(max_tx_rate, ICE_BW_KBPS_DIVISOR);
8765 			if (min_tx_rate)
8766 				min_tx_rate = div_u64(min_tx_rate, ICE_BW_KBPS_DIVISOR);
8767 
8768 			ret = ice_set_bw_limit(vsi, max_tx_rate, min_tx_rate);
8769 			if (!ret) {
8770 				dev_dbg(dev, "set Tx rate max %llu min %llu for VSI(%u)\n",
8771 					max_tx_rate, min_tx_rate, vsi->vsi_num);
8772 			} else {
8773 				dev_err(dev, "failed to set Tx rate max %llu min %llu for VSI(%u)\n",
8774 					max_tx_rate, min_tx_rate, vsi->vsi_num);
8775 				goto exit;
8776 			}
8777 		}
8778 		ret = ice_create_q_channels(vsi);
8779 		if (ret) {
8780 			netdev_err(netdev, "failed configuring queue channels\n");
8781 			goto exit;
8782 		} else {
8783 			netdev_dbg(netdev, "successfully configured channels\n");
8784 		}
8785 	}
8786 
8787 	if (vsi->ch_rss_size)
8788 		ice_vsi_cfg_rss_lut_key(vsi);
8789 
8790 exit:
8791 	/* if error, reset the all_numtc and all_enatc */
8792 	if (ret) {
8793 		vsi->all_numtc = 0;
8794 		vsi->all_enatc = 0;
8795 	}
8796 	/* resume VSI */
8797 	ice_ena_vsi(vsi, true);
8798 
8799 	return ret;
8800 }
8801 
8802 static LIST_HEAD(ice_block_cb_list);
8803 
8804 static int
8805 ice_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8806 	     void *type_data)
8807 {
8808 	struct ice_netdev_priv *np = netdev_priv(netdev);
8809 	struct ice_pf *pf = np->vsi->back;
8810 	int err;
8811 
8812 	switch (type) {
8813 	case TC_SETUP_BLOCK:
8814 		return flow_block_cb_setup_simple(type_data,
8815 						  &ice_block_cb_list,
8816 						  ice_setup_tc_block_cb,
8817 						  np, np, true);
8818 	case TC_SETUP_QDISC_MQPRIO:
8819 		/* setup traffic classifier for receive side */
8820 		mutex_lock(&pf->tc_mutex);
8821 		err = ice_setup_tc_mqprio_qdisc(netdev, type_data);
8822 		mutex_unlock(&pf->tc_mutex);
8823 		return err;
8824 	default:
8825 		return -EOPNOTSUPP;
8826 	}
8827 	return -EOPNOTSUPP;
8828 }
8829 
8830 static struct ice_indr_block_priv *
8831 ice_indr_block_priv_lookup(struct ice_netdev_priv *np,
8832 			   struct net_device *netdev)
8833 {
8834 	struct ice_indr_block_priv *cb_priv;
8835 
8836 	list_for_each_entry(cb_priv, &np->tc_indr_block_priv_list, list) {
8837 		if (!cb_priv->netdev)
8838 			return NULL;
8839 		if (cb_priv->netdev == netdev)
8840 			return cb_priv;
8841 	}
8842 	return NULL;
8843 }
8844 
8845 static int
8846 ice_indr_setup_block_cb(enum tc_setup_type type, void *type_data,
8847 			void *indr_priv)
8848 {
8849 	struct ice_indr_block_priv *priv = indr_priv;
8850 	struct ice_netdev_priv *np = priv->np;
8851 
8852 	switch (type) {
8853 	case TC_SETUP_CLSFLOWER:
8854 		return ice_setup_tc_cls_flower(np, priv->netdev,
8855 					       (struct flow_cls_offload *)
8856 					       type_data);
8857 	default:
8858 		return -EOPNOTSUPP;
8859 	}
8860 }
8861 
8862 static int
8863 ice_indr_setup_tc_block(struct net_device *netdev, struct Qdisc *sch,
8864 			struct ice_netdev_priv *np,
8865 			struct flow_block_offload *f, void *data,
8866 			void (*cleanup)(struct flow_block_cb *block_cb))
8867 {
8868 	struct ice_indr_block_priv *indr_priv;
8869 	struct flow_block_cb *block_cb;
8870 
8871 	if (!ice_is_tunnel_supported(netdev) &&
8872 	    !(is_vlan_dev(netdev) &&
8873 	      vlan_dev_real_dev(netdev) == np->vsi->netdev))
8874 		return -EOPNOTSUPP;
8875 
8876 	if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
8877 		return -EOPNOTSUPP;
8878 
8879 	switch (f->command) {
8880 	case FLOW_BLOCK_BIND:
8881 		indr_priv = ice_indr_block_priv_lookup(np, netdev);
8882 		if (indr_priv)
8883 			return -EEXIST;
8884 
8885 		indr_priv = kzalloc(sizeof(*indr_priv), GFP_KERNEL);
8886 		if (!indr_priv)
8887 			return -ENOMEM;
8888 
8889 		indr_priv->netdev = netdev;
8890 		indr_priv->np = np;
8891 		list_add(&indr_priv->list, &np->tc_indr_block_priv_list);
8892 
8893 		block_cb =
8894 			flow_indr_block_cb_alloc(ice_indr_setup_block_cb,
8895 						 indr_priv, indr_priv,
8896 						 ice_rep_indr_tc_block_unbind,
8897 						 f, netdev, sch, data, np,
8898 						 cleanup);
8899 
8900 		if (IS_ERR(block_cb)) {
8901 			list_del(&indr_priv->list);
8902 			kfree(indr_priv);
8903 			return PTR_ERR(block_cb);
8904 		}
8905 		flow_block_cb_add(block_cb, f);
8906 		list_add_tail(&block_cb->driver_list, &ice_block_cb_list);
8907 		break;
8908 	case FLOW_BLOCK_UNBIND:
8909 		indr_priv = ice_indr_block_priv_lookup(np, netdev);
8910 		if (!indr_priv)
8911 			return -ENOENT;
8912 
8913 		block_cb = flow_block_cb_lookup(f->block,
8914 						ice_indr_setup_block_cb,
8915 						indr_priv);
8916 		if (!block_cb)
8917 			return -ENOENT;
8918 
8919 		flow_indr_block_cb_remove(block_cb, f);
8920 
8921 		list_del(&block_cb->driver_list);
8922 		break;
8923 	default:
8924 		return -EOPNOTSUPP;
8925 	}
8926 	return 0;
8927 }
8928 
8929 static int
8930 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch,
8931 		     void *cb_priv, enum tc_setup_type type, void *type_data,
8932 		     void *data,
8933 		     void (*cleanup)(struct flow_block_cb *block_cb))
8934 {
8935 	switch (type) {
8936 	case TC_SETUP_BLOCK:
8937 		return ice_indr_setup_tc_block(netdev, sch, cb_priv, type_data,
8938 					       data, cleanup);
8939 
8940 	default:
8941 		return -EOPNOTSUPP;
8942 	}
8943 }
8944 
8945 /**
8946  * ice_open - Called when a network interface becomes active
8947  * @netdev: network interface device structure
8948  *
8949  * The open entry point is called when a network interface is made
8950  * active by the system (IFF_UP). At this point all resources needed
8951  * for transmit and receive operations are allocated, the interrupt
8952  * handler is registered with the OS, the netdev watchdog is enabled,
8953  * and the stack is notified that the interface is ready.
8954  *
8955  * Returns 0 on success, negative value on failure
8956  */
8957 int ice_open(struct net_device *netdev)
8958 {
8959 	struct ice_netdev_priv *np = netdev_priv(netdev);
8960 	struct ice_pf *pf = np->vsi->back;
8961 
8962 	if (ice_is_reset_in_progress(pf->state)) {
8963 		netdev_err(netdev, "can't open net device while reset is in progress");
8964 		return -EBUSY;
8965 	}
8966 
8967 	return ice_open_internal(netdev);
8968 }
8969 
8970 /**
8971  * ice_open_internal - Called when a network interface becomes active
8972  * @netdev: network interface device structure
8973  *
8974  * Internal ice_open implementation. Should not be used directly except for ice_open and reset
8975  * handling routine
8976  *
8977  * Returns 0 on success, negative value on failure
8978  */
8979 int ice_open_internal(struct net_device *netdev)
8980 {
8981 	struct ice_netdev_priv *np = netdev_priv(netdev);
8982 	struct ice_vsi *vsi = np->vsi;
8983 	struct ice_pf *pf = vsi->back;
8984 	struct ice_port_info *pi;
8985 	int err;
8986 
8987 	if (test_bit(ICE_NEEDS_RESTART, pf->state)) {
8988 		netdev_err(netdev, "driver needs to be unloaded and reloaded\n");
8989 		return -EIO;
8990 	}
8991 
8992 	netif_carrier_off(netdev);
8993 
8994 	pi = vsi->port_info;
8995 	err = ice_update_link_info(pi);
8996 	if (err) {
8997 		netdev_err(netdev, "Failed to get link info, error %d\n", err);
8998 		return err;
8999 	}
9000 
9001 	ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err);
9002 
9003 	/* Set PHY if there is media, otherwise, turn off PHY */
9004 	if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
9005 		clear_bit(ICE_FLAG_NO_MEDIA, pf->flags);
9006 		if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) {
9007 			err = ice_init_phy_user_cfg(pi);
9008 			if (err) {
9009 				netdev_err(netdev, "Failed to initialize PHY settings, error %d\n",
9010 					   err);
9011 				return err;
9012 			}
9013 		}
9014 
9015 		err = ice_configure_phy(vsi);
9016 		if (err) {
9017 			netdev_err(netdev, "Failed to set physical link up, error %d\n",
9018 				   err);
9019 			return err;
9020 		}
9021 	} else {
9022 		set_bit(ICE_FLAG_NO_MEDIA, pf->flags);
9023 		ice_set_link(vsi, false);
9024 	}
9025 
9026 	err = ice_vsi_open(vsi);
9027 	if (err)
9028 		netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n",
9029 			   vsi->vsi_num, vsi->vsw->sw_id);
9030 
9031 	/* Update existing tunnels information */
9032 	udp_tunnel_get_rx_info(netdev);
9033 
9034 	return err;
9035 }
9036 
9037 /**
9038  * ice_stop - Disables a network interface
9039  * @netdev: network interface device structure
9040  *
9041  * The stop entry point is called when an interface is de-activated by the OS,
9042  * and the netdevice enters the DOWN state. The hardware is still under the
9043  * driver's control, but the netdev interface is disabled.
9044  *
9045  * Returns success only - not allowed to fail
9046  */
9047 int ice_stop(struct net_device *netdev)
9048 {
9049 	struct ice_netdev_priv *np = netdev_priv(netdev);
9050 	struct ice_vsi *vsi = np->vsi;
9051 	struct ice_pf *pf = vsi->back;
9052 
9053 	if (ice_is_reset_in_progress(pf->state)) {
9054 		netdev_err(netdev, "can't stop net device while reset is in progress");
9055 		return -EBUSY;
9056 	}
9057 
9058 	if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) {
9059 		int link_err = ice_force_phys_link_state(vsi, false);
9060 
9061 		if (link_err) {
9062 			netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n",
9063 				   vsi->vsi_num, link_err);
9064 			return -EIO;
9065 		}
9066 	}
9067 
9068 	ice_vsi_close(vsi);
9069 
9070 	return 0;
9071 }
9072 
9073 /**
9074  * ice_features_check - Validate encapsulated packet conforms to limits
9075  * @skb: skb buffer
9076  * @netdev: This port's netdev
9077  * @features: Offload features that the stack believes apply
9078  */
9079 static netdev_features_t
9080 ice_features_check(struct sk_buff *skb,
9081 		   struct net_device __always_unused *netdev,
9082 		   netdev_features_t features)
9083 {
9084 	bool gso = skb_is_gso(skb);
9085 	size_t len;
9086 
9087 	/* No point in doing any of this if neither checksum nor GSO are
9088 	 * being requested for this frame. We can rule out both by just
9089 	 * checking for CHECKSUM_PARTIAL
9090 	 */
9091 	if (skb->ip_summed != CHECKSUM_PARTIAL)
9092 		return features;
9093 
9094 	/* We cannot support GSO if the MSS is going to be less than
9095 	 * 64 bytes. If it is then we need to drop support for GSO.
9096 	 */
9097 	if (gso && (skb_shinfo(skb)->gso_size < ICE_TXD_CTX_MIN_MSS))
9098 		features &= ~NETIF_F_GSO_MASK;
9099 
9100 	len = skb_network_offset(skb);
9101 	if (len > ICE_TXD_MACLEN_MAX || len & 0x1)
9102 		goto out_rm_features;
9103 
9104 	len = skb_network_header_len(skb);
9105 	if (len > ICE_TXD_IPLEN_MAX || len & 0x1)
9106 		goto out_rm_features;
9107 
9108 	if (skb->encapsulation) {
9109 		/* this must work for VXLAN frames AND IPIP/SIT frames, and in
9110 		 * the case of IPIP frames, the transport header pointer is
9111 		 * after the inner header! So check to make sure that this
9112 		 * is a GRE or UDP_TUNNEL frame before doing that math.
9113 		 */
9114 		if (gso && (skb_shinfo(skb)->gso_type &
9115 			    (SKB_GSO_GRE | SKB_GSO_UDP_TUNNEL))) {
9116 			len = skb_inner_network_header(skb) -
9117 			      skb_transport_header(skb);
9118 			if (len > ICE_TXD_L4LEN_MAX || len & 0x1)
9119 				goto out_rm_features;
9120 		}
9121 
9122 		len = skb_inner_network_header_len(skb);
9123 		if (len > ICE_TXD_IPLEN_MAX || len & 0x1)
9124 			goto out_rm_features;
9125 	}
9126 
9127 	return features;
9128 out_rm_features:
9129 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
9130 }
9131 
9132 static const struct net_device_ops ice_netdev_safe_mode_ops = {
9133 	.ndo_open = ice_open,
9134 	.ndo_stop = ice_stop,
9135 	.ndo_start_xmit = ice_start_xmit,
9136 	.ndo_set_mac_address = ice_set_mac_address,
9137 	.ndo_validate_addr = eth_validate_addr,
9138 	.ndo_change_mtu = ice_change_mtu,
9139 	.ndo_get_stats64 = ice_get_stats64,
9140 	.ndo_tx_timeout = ice_tx_timeout,
9141 	.ndo_bpf = ice_xdp_safe_mode,
9142 };
9143 
9144 static const struct net_device_ops ice_netdev_ops = {
9145 	.ndo_open = ice_open,
9146 	.ndo_stop = ice_stop,
9147 	.ndo_start_xmit = ice_start_xmit,
9148 	.ndo_select_queue = ice_select_queue,
9149 	.ndo_features_check = ice_features_check,
9150 	.ndo_fix_features = ice_fix_features,
9151 	.ndo_set_rx_mode = ice_set_rx_mode,
9152 	.ndo_set_mac_address = ice_set_mac_address,
9153 	.ndo_validate_addr = eth_validate_addr,
9154 	.ndo_change_mtu = ice_change_mtu,
9155 	.ndo_get_stats64 = ice_get_stats64,
9156 	.ndo_set_tx_maxrate = ice_set_tx_maxrate,
9157 	.ndo_eth_ioctl = ice_eth_ioctl,
9158 	.ndo_set_vf_spoofchk = ice_set_vf_spoofchk,
9159 	.ndo_set_vf_mac = ice_set_vf_mac,
9160 	.ndo_get_vf_config = ice_get_vf_cfg,
9161 	.ndo_set_vf_trust = ice_set_vf_trust,
9162 	.ndo_set_vf_vlan = ice_set_vf_port_vlan,
9163 	.ndo_set_vf_link_state = ice_set_vf_link_state,
9164 	.ndo_get_vf_stats = ice_get_vf_stats,
9165 	.ndo_set_vf_rate = ice_set_vf_bw,
9166 	.ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid,
9167 	.ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid,
9168 	.ndo_setup_tc = ice_setup_tc,
9169 	.ndo_set_features = ice_set_features,
9170 	.ndo_bridge_getlink = ice_bridge_getlink,
9171 	.ndo_bridge_setlink = ice_bridge_setlink,
9172 	.ndo_fdb_add = ice_fdb_add,
9173 	.ndo_fdb_del = ice_fdb_del,
9174 #ifdef CONFIG_RFS_ACCEL
9175 	.ndo_rx_flow_steer = ice_rx_flow_steer,
9176 #endif
9177 	.ndo_tx_timeout = ice_tx_timeout,
9178 	.ndo_bpf = ice_xdp,
9179 	.ndo_xdp_xmit = ice_xdp_xmit,
9180 	.ndo_xsk_wakeup = ice_xsk_wakeup,
9181 };
9182